diff --git a/.gitignore b/.gitignore index e0081f4..7a996d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .bin/ +.grite/ diff --git a/README.md b/README.md index 0b8fd6b..f377903 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,48 @@ lockfile content without running package-manager commands. It prints direct dependencies by default; use `--depth 0` for the full graph available from the local files. +### `cache-warm` + +Prime the local package caches for dependencies this machine has not checked out. + +```bash +# Download a module and its full transitive closure into GOMODCACHE +repomap cache-warm go github.com/flanksource/clicky@v1.21.14 + +# Take the current version, compile every package, and prove it works offline +repomap cache-warm go github.com/flanksource/commons --build --verify + +# A Go dependency may also be a GitHub slug or a repository URL +repomap cache-warm go flanksource/commons +repomap cache-warm go https://github.com/flanksource/commons + +# Warm several npm packages into the pnpm store +repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0 +``` + +For each `name@version` spec, repomap creates a throwaway single-dependency +project in a temporary directory, drives the real package manager to download the +closure into the machine's shared cache, then deletes the project. Nothing in the +working tree is touched; what persists is the warmed cache (`GOMODCACHE`, the pnpm +store, or the npm cache). Omit the version to take whatever the manager considers +current — the concrete resolved version is reported back, including Go +pseudo-versions. A Go dependency can be named as a module path, a GitHub +`owner/repo` slug, or a repository URL; all three are canonicalised to the module +path before the go toolchain sees them. + +`--build` goes further than downloading. For Go it compiles every package in the +module so `GOCACHE` holds build artifacts and not just source. For npm and pnpm it +lets dependency lifecycle scripts run so native addons are compiled; it does not +run the package's own build script. + +`--verify` proves the result rather than assuming it, replaying the work with the +network disabled (`GOPROXY=off`, or an `--offline` install against a frozen +lockfile), so a cache that could not actually build offline fails loudly. + +Supported managers are `go`, `npm`, and `pnpm`. This is aimed at CI images, +sandboxes, and air-gapped builds, where a later build must succeed with no network +access. + ### `version` Print version, commit hash, build date, and Go version. diff --git a/cmd/repomap/cache_warm.go b/cmd/repomap/cache_warm.go new file mode 100644 index 0000000..5c5e72d --- /dev/null +++ b/cmd/repomap/cache_warm.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + depgraph "github.com/flanksource/repomap/deps" +) + +type CacheWarmOptions struct { + // No default: tag — a positional field carrying one has its positional values + // silently discarded. Emptiness is validated in runCacheWarm instead. + Args []string `json:"args" args:"true" help:"Package manager (go, npm, pnpm) followed by one or more name@version specs"` + Build bool `json:"build,omitempty" flag:"build" help:"Compile every package after downloading (Go) or run dependency lifecycle and native builds (npm, pnpm)"` + Verify bool `json:"verify,omitempty" flag:"verify" help:"Replay the warm with the network disabled to prove the cache is complete"` +} + +func (opts CacheWarmOptions) GetName() string { return "cache-warm ..." } + +func (opts CacheWarmOptions) Help() api.Text { + return clicky.Text(`Prime the local package caches for dependencies this machine has not checked out. + +For each name@version spec, repomap creates a throwaway single-dependency +project in a temporary directory, drives the real package manager to download +the dependency's full transitive closure into the machine's shared cache, then +deletes the project. Nothing in the working tree is touched; what persists is +the warmed cache (GOMODCACHE, the pnpm store, or the npm cache). + +Omit the version to take whatever the manager considers current. The concrete +resolved version is reported back, including Go pseudo-versions. + +A Go dependency can be given as a module path, as a GitHub owner/repo slug, or as +a repository URL — all three are canonicalised to the module path before the go +toolchain sees them. + +Use --build to go further than downloading. For Go it compiles every package in +the module so GOCACHE holds the build artifacts, not just the source. For npm and +pnpm it lets dependency lifecycle scripts run so native addons are compiled. It +does not run the package's own build script. + +Use --verify to prove the result rather than assume it: the work is replayed with +the network disabled (GOPROXY=off, or an --offline install against a frozen +lockfile), so a cache that could not actually build offline fails loudly. + +This is aimed at CI images, sandboxes, and air-gapped builds, where a later build +must succeed with no network access. + +EXAMPLES: + repomap cache-warm go github.com/flanksource/clicky@v1.21.14 + repomap cache-warm go github.com/flanksource/commons --build --verify + repomap cache-warm go flanksource/commons + repomap cache-warm go https://github.com/flanksource/commons + repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0 + repomap cache-warm npm @flanksource/icons@1.0.0 --verify + repomap cache-warm go github.com/flanksource/clicky@v1.21.14 --json`) +} + +func init() { + cmd := clicky.AddNamedCommandWithContext("cache-warm", rootCmd, CacheWarmOptions{}, runCacheWarm) + cmd.Short = "Warm the Go, npm, or pnpm cache for a dependency and optionally build it" +} + +func runCacheWarm(ctx context.Context, opts CacheWarmOptions) (any, error) { + manager, specs, err := parseCacheWarmArgs(opts.Args) + if err != nil { + return nil, err + } + results, err := depgraph.WarmCache(ctx, depgraph.WarmOptions{ + Manager: manager, + Specs: specs, + Build: opts.Build, + Verify: opts.Verify, + }) + // Returned as a slice rather than api.NewTableFrom so --json keeps the full + // WarmResult — per-step commands, durations, and errors, which is what a CI + // debugging session needs. Pretty output still renders as a table via + // WarmResult's Columns/Row. The error is returned alongside the results so a + // partial run still reports which specs succeeded. + return results, err +} + +// parseCacheWarmArgs splits the positional arguments into the manager and its +// specs. Only managers repomap can actually warm are accepted; maven and gradle +// are scan-only, and image/helm are not package caches. +func parseCacheWarmArgs(args []string) (depgraph.Manager, []string, error) { + if len(args) == 0 { + return "", nil, fmt.Errorf("expected a package manager (go, npm, or pnpm) followed by one or more name@version specs") + } + manager := depgraph.Manager(strings.ToLower(strings.TrimSpace(args[0]))) + switch manager { + case depgraph.ManagerGo, depgraph.ManagerNPM, depgraph.ManagerPNPM: + default: + return "", nil, fmt.Errorf("cache warming does not support %q (expected go, npm, or pnpm)", args[0]) + } + specs := splitCommaArgs(args[1:]) + if len(specs) == 0 { + return "", nil, fmt.Errorf("expected at least one name@version spec to warm for %s", manager) + } + return manager, specs, nil +} diff --git a/cmd/repomap/cache_warm_test.go b/cmd/repomap/cache_warm_test.go new file mode 100644 index 0000000..170f03a --- /dev/null +++ b/cmd/repomap/cache_warm_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/flanksource/clicky" + depgraph "github.com/flanksource/repomap/deps" + "github.com/spf13/cobra" +) + +// runCacheWarmArgs parses argv through a real cobra tree with a stub handler, so +// the assertions cover clicky's struct-tag binding rather than a hand-built +// options struct. +func runCacheWarmArgs(t *testing.T, argv ...string) (CacheWarmOptions, error) { + t.Helper() + var got CacheWarmOptions + root := &cobra.Command{Use: "test"} + clicky.BindAllFlags(root.PersistentFlags(), "tasks", "format") + clicky.AddNamedCommandWithContext("cache-warm", root, CacheWarmOptions{}, func(_ context.Context, opts CacheWarmOptions) (any, error) { + got = opts + return nil, nil + }) + root.SetArgs(argv) + return got, root.Execute() +} + +// The positional args field must carry no default: tag, or clicky silently drops +// the positional values. +func TestCacheWarmBindsPositionalArgsAndFlags(t *testing.T) { + got, err := runCacheWarmArgs(t, "cache-warm", "go", "github.com/acme/lib@v1.2.3", "left-pad@1.3.0", "--build", "--verify") + if err != nil { + t.Fatal(err) + } + want := []string{"go", "github.com/acme/lib@v1.2.3", "left-pad@1.3.0"} + if strings.Join(got.Args, ",") != strings.Join(want, ",") { + t.Fatalf("Args = %v, want %v", got.Args, want) + } + if !got.Build || !got.Verify { + t.Fatalf("Build = %v, Verify = %v, want both true", got.Build, got.Verify) + } +} + +func TestCacheWarmFlagsDefaultOff(t *testing.T) { + got, err := runCacheWarmArgs(t, "cache-warm", "go", "github.com/acme/lib@v1.2.3") + if err != nil { + t.Fatal(err) + } + if got.Build || got.Verify { + t.Fatalf("Build = %v, Verify = %v, want both false", got.Build, got.Verify) + } +} + +func TestParseCacheWarmArgs(t *testing.T) { + cases := []struct { + name string + args []string + wantManager depgraph.Manager + wantSpecs []string + wantErr string + }{ + { + name: "single spec", args: []string{"go", "github.com/acme/lib@v1.2.3"}, + wantManager: depgraph.ManagerGo, wantSpecs: []string{"github.com/acme/lib@v1.2.3"}, + }, + { + name: "several specs", args: []string{"pnpm", "left-pad@1.3.0", "@scope/pkg@2.0.0"}, + wantManager: depgraph.ManagerPNPM, wantSpecs: []string{"left-pad@1.3.0", "@scope/pkg@2.0.0"}, + }, + { + name: "manager casing is normalised", args: []string{"NPM", "left-pad@1.3.0"}, + wantManager: depgraph.ManagerNPM, wantSpecs: []string{"left-pad@1.3.0"}, + }, + {name: "no args", args: nil, wantErr: "manager"}, + {name: "manager but no spec", args: []string{"go"}, wantErr: "spec"}, + // Managers repomap can scan but cannot warm must be rejected by name. + {name: "unwarmable manager", args: []string{"maven", "org.acme:lib@1.0.0"}, wantErr: "maven"}, + {name: "unknown manager", args: []string{"cargo", "serde@1.0.0"}, wantErr: "cargo"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + manager, specs, err := parseCacheWarmArgs(tc.args) + if tc.wantErr != "" { + if err == nil { + t.Fatalf("expected an error mentioning %q", tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error %q should mention %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if manager != tc.wantManager { + t.Errorf("manager = %q, want %q", manager, tc.wantManager) + } + if strings.Join(specs, ",") != strings.Join(tc.wantSpecs, ",") { + t.Errorf("specs = %v, want %v", specs, tc.wantSpecs) + } + }) + } +} + +// defaultToScan rewrites anything it does not recognise into `scan ...`, so a +// misregistered name would turn this command into a silent repo scan. +func TestCacheWarmIsNotRewrittenToScan(t *testing.T) { + argv := []string{"cache-warm", "go", "github.com/acme/lib@v1.2.3"} + got := defaultToScan(argv) + if strings.Join(got, " ") != strings.Join(argv, " ") { + t.Fatalf("defaultToScan(%v) = %v, want it unchanged", argv, got) + } +} diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index 51a4a74..6cd524d 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "strings" "github.com/flanksource/clicky" @@ -11,24 +12,38 @@ import ( ) type DepsOptions struct { - Path string `json:"path" args:"true" help:"Path to scan" default:"."` - Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` - Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` - Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` - Flat bool `json:"flat,omitempty" flag:"flat" help:"Export a flat node list with edges instead of the dependency tree"` - IncludeIndirect bool `json:"include_indirect,omitempty" flag:"include-indirect" help:"Include Go indirect requirements in --depth 1 listings (ignored at other depths)"` + Path string `json:"path" args:"true" help:"Path to scan (defaults to current directory)"` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` + Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` + Kind []string `json:"kind,omitempty" flag:"kind,k" help:"Filter image/helm dependencies by Kubernetes kind, e.g. HelmRelease,Deployment (MatchItem syntax)"` + Namespace []string `json:"namespace,omitempty" flag:"namespace,n" help:"Filter image/helm dependencies by namespace (MatchItem syntax)"` + Name []string `json:"name,omitempty" flag:"name" help:"Filter image/helm dependencies by resource name (MatchItem syntax)"` + Selector []string `json:"selector,omitempty" flag:"selector,l" help:"Filter image/helm dependencies by label selector, e.g. app=nginx"` + Flat bool `json:"flat,omitempty" flag:"flat" help:"Export a flat node list with edges instead of the dependency tree"` + IncludeIndirect bool `json:"include_indirect,omitempty" flag:"include-indirect" help:"Include Go indirect requirements in --depth 1 listings (ignored at other depths)"` + ShowDuplicates bool `json:"show_duplicates,omitempty" flag:"show-duplicates" help:"Render every occurrence of duplicated dependencies and report duplicates/conflicts (default: collapse to a single resolved node with no duplicate reporting)"` } type DepsUpdateOptions struct { - Args []string `json:"args" args:"true" required:"true" help:"Dependency MatchItem expression followed by optional path"` - Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to update: go, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` - Check bool `json:"check" flag:"check" help:"Resolve and list available updates without prompting or writing"` - DryRun bool `json:"dry_run" flag:"dry-run" help:"Show planned dependency updates without running package-manager commands"` + Args []string `json:"args" args:"true" help:"Optional dependency MatchItem expression followed by optional path"` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to update: go, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns (MatchItem syntax) matched against name, manager, manager:name, manager:name@version, scope, and version; use path:/file: for manifest paths; supports comma-separated values and !exclusions"` + Kind []string `json:"kind,omitempty" flag:"kind,k" help:"Filter image/helm targets by kind, e.g. HelmRelease,Deployment (MatchItem syntax)"` + Namespace []string `json:"namespace,omitempty" flag:"namespace,n" help:"Filter image/helm targets by namespace (MatchItem syntax)"` + Name []string `json:"name,omitempty" flag:"name" help:"Filter image/helm targets by resource name (MatchItem syntax)"` + Selector []string `json:"selector,omitempty" flag:"selector,l" help:"Filter image/helm targets by label selector, e.g. app=nginx"` + Image []string `json:"image,omitempty" flag:"image" help:"Only container images matching this repo pattern (MatchItem syntax)"` + Chart []string `json:"chart,omitempty" flag:"chart" help:"Only Helm charts matching this name (MatchItem syntax)"` + Latest bool `json:"latest,omitempty" flag:"latest" help:"Resolve each matched dependency to its highest stable version"` + Version string `json:"version,omitempty" flag:"version" help:"Apply this concrete version to all matched dependencies"` + Check bool `json:"check" flag:"check" help:"Resolve and list available updates without prompting or writing"` + DryRun bool `json:"dry_run" flag:"dry-run" help:"Show planned dependency updates without running package-manager commands"` } func (opts DepsOptions) GetName() string { return "deps" } -func (opts DepsUpdateOptions) GetName() string { return "update [path]" } +func (opts DepsUpdateOptions) GetName() string { return "update [expr] [path]" } func (opts DepsOptions) Help() api.Text { return clicky.Text(`Generate dependency graphs for Go, Maven, Gradle, npm, pnpm, image, and Helm dependencies. @@ -43,6 +58,12 @@ by shelling out to the package manager (go mod graph, mvn dependency:tree, gradle dependencies). The tool must be installed; rerun with --depth 1 for offline direct-only output. +For Helm charts and container images, --depth other than 1 additionally recurses +into remote dependencies: subcharts are fetched from their Helm repositories and +image base images are resolved from registry labels and Dockerfile FROM +directives (cloning source repos). Fetches are cached under the user cache dir; +failures degrade to warnings. Use --depth 1 to stay fully offline. + The command uses the normal Clicky output flow. Use --json to write structured JSON to stdout, for example: @@ -51,33 +72,60 @@ JSON to stdout, for example: By default the JSON export contains the dependency tree under "roots". Use --flat to export a flat "nodes" list plus "edges" instead of the tree. +Shared dependencies are collapsed to their resolved (shallowest) occurrence: the +resolved node is tagged with the number of other parents and each parent that +hid a duplicate shows a trailing count. Use --show-duplicates to render every +occurrence instead. + +Image and Helm dependencies discovered from Kubernetes manifests can be narrowed +by resource metadata with --kind/--namespace/--name/--selector (MatchItem +syntax). These filters only match manifest-sourced targets; Chart.yaml chart +directories have no Kubernetes metadata and are excluded whenever any of them is +set. + EXAMPLES: repomap deps repomap deps ./service --manager go repomap deps --manager npm,pnpm --depth 0 repomap deps --manager go --depth 0 --flat --json repomap deps --manager go --include-indirect + repomap deps --depth 0 --manager go --show-duplicates repomap deps --manager image,helm ./clusters/prod + repomap deps --manager helm -k HelmRelease -n default repomap deps --filter 'github.com/flanksource/*,!*test*'`) } func (opts DepsUpdateOptions) Help() api.Text { return clicky.Text(`Update direct package, image, and Helm chart dependencies. -The required expr argument uses commons MatchItem syntax and is matched against -dependency names, manager-qualified names, versions, and scopes. Manifest path -matching is explicit with path: or file:. Matched direct -dependencies are resolved to published versions, then repomap prompts for which -dependencies and versions to apply. Applied updates are staged with git add +Dependencies are narrowed with --filter and/or the optional positional expr; +both use commons MatchItem syntax, accept comma-separated values and +!exclusions, and are combined into a single pattern set. Patterns are matched +against dependency names, manager-qualified names, versions, and scopes. +Manifest path matching is explicit with path: or file:. With +no patterns, every matched dependency is considered. Image and Helm targets (from git-tracked +Kubernetes/Flux manifests, including HelmRelease spec.chartRef OCIRepository and +HelmChart sources) can be further narrowed with --kind/--namespace/--name/ +--selector and the --image/--chart name patterns. + +By default repomap prompts for which dependencies and versions to apply. A +dependency declared in several manifests at the same version is confirmed once +and the chosen version written to every occurrence; occurrences sitting at +different current versions are still confirmed separately. Use --latest to +resolve each to its highest stable version, or --version to apply a concrete +version, both non-interactively. Applied updates are staged with git add (manifests plus lockfiles); --dry-run and --check never stage. Use --check to list updateable dependencies without prompting or writing. EXAMPLES: + repomap deps update --filter 'github.com/flanksource/*' + repomap deps update --filter '*flanksource*,!*test*' --check repomap deps update 'github.com/flanksource/*' repomap deps update '*' --check + repomap deps update --manager helm -k HelmRelease --latest + repomap deps update --manager image -n default --version 1.27.0 repomap deps update 'path:apps/*/package.json' - repomap deps update 'image:ghcr.io/flanksource/*' --manager image repomap deps update 'helm:mission-control' --manager helm repomap deps update 'npm:@flanksource/*' ./web --manager npm repomap deps update 'left-pad,!*beta*' --dry-run`) @@ -109,23 +157,22 @@ func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { Managers: managers, MaxDepth: opts.Depth, Filters: splitCommaArgs(opts.Filter), + Kind: opts.Kind, + Namespace: opts.Namespace, + Name: opts.Name, + Selector: opts.Selector, Flat: opts.Flat, IncludeIndirect: opts.IncludeIndirect, + ShowDuplicates: opts.ShowDuplicates, }) } func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { - if len(opts.Args) == 0 { - return nil, fmt.Errorf("dependency update expression is required") - } - if len(opts.Args) > 2 { - return nil, fmt.Errorf("expected [path], got %d arguments", len(opts.Args)) - } - path := "." - if len(opts.Args) == 2 { - path = opts.Args[1] + expr, rawPath, err := parseDepsUpdateArgs(opts.Args) + if err != nil { + return nil, err } - path, err := resolvePath(path) + path, err := resolvePath(rawPath) if err != nil { return nil, err } @@ -134,10 +181,18 @@ func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { return nil, err } plans, err := depgraph.Update(ctx, path, depgraph.UpdateOptions{ - Managers: managers, - Expression: []string{opts.Args[0]}, - Check: opts.Check, - DryRun: opts.DryRun, + Managers: managers, + Filters: updateFilters(opts.Filter, expr), + Kind: opts.Kind, + Namespace: opts.Namespace, + Name: opts.Name, + Selector: opts.Selector, + Image: opts.Image, + Chart: opts.Chart, + Latest: opts.Latest, + Version: opts.Version, + Check: opts.Check, + DryRun: opts.DryRun, }) if err != nil { return nil, err @@ -145,6 +200,41 @@ func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { return api.NewTableFrom(plans), nil } +// updateFilters combines --filter values with the optional positional expr into +// one MatchItem pattern set. Patterns are forwarded raw because depgraph.Update +// already comma-splits and trims them. +func updateFilters(filter []string, expr string) []string { + out := append([]string{}, filter...) + if expr != "" { + out = append(out, expr) + } + return out +} + +// parseDepsUpdateArgs interprets the optional positional [expr] [path]. With one +// argument, an existing directory is treated as the path and anything else as the +// expression, so `deps update ./clusters` and `deps update 'left-pad'` both work. +func parseDepsUpdateArgs(args []string) (expr, path string, err error) { + switch len(args) { + case 0: + return "", ".", nil + case 1: + if isExistingDir(args[0]) { + return "", args[0], nil + } + return args[0], ".", nil + case 2: + return args[0], args[1], nil + default: + return "", "", fmt.Errorf("expected [expr] [path], got %d arguments", len(args)) + } +} + +func isExistingDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + func parseManagers(values []string) ([]depgraph.Manager, error) { parts := splitCommaArgs(values) if len(parts) == 0 { diff --git a/cmd/repomap/deps_test.go b/cmd/repomap/deps_test.go index 65f7b8d..647ca00 100644 --- a/cmd/repomap/deps_test.go +++ b/cmd/repomap/deps_test.go @@ -1,12 +1,33 @@ package main import ( + "context" "strings" "testing" + "github.com/flanksource/clicky" depgraph "github.com/flanksource/repomap/deps" + "github.com/spf13/cobra" ) +func TestDepsPositionalPathHonored(t *testing.T) { + var gotPath string + root := &cobra.Command{Use: "test"} + clicky.BindAllFlags(root.PersistentFlags(), "tasks", "format") + clicky.AddNamedCommandWithContext("deps", root, DepsOptions{}, func(_ context.Context, opts DepsOptions) (*depgraph.Export, error) { + gotPath = opts.Path + return &depgraph.Export{}, nil + }) + + root.SetArgs([]string{"deps", "/tmp/some/scan/path", "--json"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + if gotPath != "/tmp/some/scan/path" { + t.Fatalf("positional path not bound: opts.Path = %q, want /tmp/some/scan/path", gotPath) + } +} + func TestParseManagers(t *testing.T) { got, err := parseManagers([]string{"go,npm", "pnpm", "image", "docker", "helm"}) if err != nil { @@ -116,3 +137,118 @@ func TestDepsUpdateCommandRegistered(t *testing.T) { t.Fatalf("manager help should document update-supported managers, got %q", manager.Usage) } } + +// deps update must declare its own --filter, otherwise the flag silently binds +// to clicky's persistent format flag (`--filter string`, a CEL output filter) +// and the MatchItem patterns never reach depgraph.Update. +func TestDepsUpdateFilterFlagShadowsGlobalCELFilter(t *testing.T) { + cmd, _, err := rootCmd.Find([]string{"deps", "update"}) + if err != nil { + t.Fatal(err) + } + flag := cmd.Flags().Lookup("filter") + if flag == nil { + t.Fatal("filter flag not registered on deps update") + } + if got := flag.Value.Type(); got != "stringSlice" { + t.Fatalf("filter flag type = %q, want stringSlice (clicky's global CEL filter is a string)", got) + } + if !strings.Contains(flag.Usage, "MatchItem syntax") { + t.Fatalf("filter help should document MatchItem syntax, got %q", flag.Usage) + } +} + +func TestDepsUpdateFilterFlagBindsToOptions(t *testing.T) { + var got DepsUpdateOptions + root := &cobra.Command{Use: "test"} + clicky.BindAllFlags(root.PersistentFlags(), "tasks", "format") + deps := clicky.AddNamedCommandWithContext("deps", root, DepsOptions{}, func(_ context.Context, _ DepsOptions) (*depgraph.Export, error) { + return &depgraph.Export{}, nil + }) + clicky.AddNamedCommandWithContext("update", deps, DepsUpdateOptions{}, func(_ context.Context, opts DepsUpdateOptions) (any, error) { + got = opts + return nil, nil + }) + + root.SetArgs([]string{"deps", "update", "--filter", "*flanksource*,!*test*", "--filter", "left-pad", "--json"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + // cobra's stringSlice comma-splits at parse time; depgraph.Update splits + // again for the positional expr, which arrives as one unsplit string. + want := []string{"*flanksource*", "!*test*", "left-pad"} + if len(got.Filter) != len(want) { + t.Fatalf("Filter = %#v, want %#v", got.Filter, want) + } + for i := range want { + if got.Filter[i] != want[i] { + t.Fatalf("Filter[%d] = %q, want %q", i, got.Filter[i], want[i]) + } + } +} + +func TestUpdateFiltersCombinesFlagAndPositionalExpr(t *testing.T) { + cases := []struct { + name string + filter []string + expr string + want []string + }{ + {name: "neither", want: nil}, + {name: "flag only", filter: []string{"npm:@scope/*"}, want: []string{"npm:@scope/*"}}, + {name: "expr only", expr: "left-pad", want: []string{"left-pad"}}, + { + name: "flag and expr combined", + filter: []string{"*flanksource*", "!*test*"}, + expr: "path:apps/*/package.json", + want: []string{"*flanksource*", "!*test*", "path:apps/*/package.json"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := updateFilters(tc.filter, tc.expr) + if len(got) != len(tc.want) { + t.Fatalf("updateFilters(%#v, %q) = %#v, want %#v", tc.filter, tc.expr, got, tc.want) + } + for i := range tc.want { + if got[i] != tc.want[i] { + t.Fatalf("pattern[%d] = %q, want %q", i, got[i], tc.want[i]) + } + } + }) + } +} + +func TestParseDepsUpdateArgs(t *testing.T) { + dir := t.TempDir() + cases := []struct { + name string + args []string + wantExpr string + wantPath string + wantErr bool + }{ + {name: "no args defaults to cwd", wantPath: "."}, + {name: "existing dir is the path", args: []string{dir}, wantPath: dir}, + {name: "non-dir is the expression", args: []string{"left-pad"}, wantExpr: "left-pad", wantPath: "."}, + {name: "expr then path", args: []string{"left-pad", dir}, wantExpr: "left-pad", wantPath: dir}, + {name: "too many args", args: []string{"a", "b", "c"}, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + expr, path, err := parseDepsUpdateArgs(tc.args) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %#v", tc.args) + } + return + } + if err != nil { + t.Fatal(err) + } + if expr != tc.wantExpr || path != tc.wantPath { + t.Fatalf("parseDepsUpdateArgs(%#v) = (%q, %q), want (%q, %q)", tc.args, expr, path, tc.wantExpr, tc.wantPath) + } + }) + } +} diff --git a/cmd/repomap/images.go b/cmd/repomap/images.go index 57b23b6..b595b7f 100644 --- a/cmd/repomap/images.go +++ b/cmd/repomap/images.go @@ -2,17 +2,27 @@ package main import ( "fmt" - "path/filepath" "strings" - "github.com/flanksource/commons/collections" "github.com/spf13/cobra" "github.com/flanksource/repomap" "github.com/flanksource/repomap/imageupdate" - "github.com/flanksource/repomap/kubernetes" ) +// versionOnly strips an image/chart current value down to its tag/version +// (dropping any registry/repo prefix and digest suffix). +func versionOnly(currentValue string) string { + if i := strings.LastIndex(currentValue, ":"); i >= 0 { + v := currentValue[i+1:] + if at := strings.Index(v, "@"); at >= 0 { + v = v[:at] + } + return v + } + return currentValue +} + // resolveConcurrency bounds how many registry/Helm version lookups run at once. const resolveConcurrency = 8 @@ -31,7 +41,7 @@ func init() { // `images update`. The first four mirror `scan`'s resource filters; --image and // --chart further narrow by image repo / chart name. type imageFilterOptions struct { - Path string `json:"path" args:"true" help:"Path to scan" default:"."` + Path string `json:"path" args:"true" help:"Path to scan (defaults to current directory)"` Kind []string `json:"kind" flag:"kind,k" help:"Filter by kind, e.g. HelmRelease,Deployment (MatchItem syntax)"` Namespace []string `json:"namespace" flag:"namespace,n" help:"Filter by namespace (MatchItem syntax)"` Name []string `json:"name" flag:"name" help:"Filter by resource name (MatchItem syntax)"` @@ -41,8 +51,9 @@ type imageFilterOptions struct { } // discoverAndFilter resolves the scan path, discovers every image/chart target -// in the repo, and applies the shared resource + image/chart filters. It returns -// the matching targets and the HelmRepository source index for chart resolution. +// in the repo (via the shared imageupdate discovery, which also resolves chart +// sources), and applies the shared resource + image/chart filters. It returns the +// matching targets and the source index for chart resolution. func discoverAndFilter(opts imageFilterOptions) ([]imageupdate.UpdateTarget, *imageupdate.SourceIndex, *repomap.ArchConf, error) { path, err := resolvePath(opts.Path) if err != nil { @@ -53,82 +64,12 @@ func discoverAndFilter(opts imageFilterOptions) ([]imageupdate.UpdateTarget, *im return nil, nil, nil, fmt.Errorf("failed to load config: %w", err) } - targets, sourceIndex, err := discoverTargets(conf, path) + res, err := imageupdate.DiscoverRepoTargets(conf, path) if err != nil { return nil, nil, nil, err } matcher := repomap.NewResourceMatcher(opts.Kind, opts.Namespace, opts.Name, opts.Selector) - targets = filterTargets(targets, matcher, opts) - return targets, sourceIndex, conf, nil -} - -// discoverTargets reads every tracked YAML file, builds the kustomize/Flux tree -// so HelmRepository sources can be resolved through Kustomization namespace -// transformers, indexes the HelmRepositories, and extracts update targets from -// files under the scan prefix. -func discoverTargets(conf *repomap.ArchConf, scanPath string) ([]imageupdate.UpdateTarget, *imageupdate.SourceIndex, error) { - files, err := gitListFiles(conf.RepoPath()) - if err != nil { - return nil, nil, fmt.Errorf("failed to list files: %w", err) - } - - var prefix string - if rel, err := filepath.Rel(conf.RepoPath(), scanPath); err == nil && rel != "." { - prefix = rel + string(filepath.Separator) - } - - // Pass 0: read all tracked YAML (paths are repo-relative POSIX from git). - contents := map[string]string{} - for _, f := range files { - if !kubernetes.IsYaml(f) { - continue - } - content, err := conf.ReadFileWithFallback(f, "") - if err != nil { - continue - } - contents[f] = content - } - - // Pass 1: build the kustomize/Flux tree and the HelmRepository index. - tree := imageupdate.BuildKustomizeTree(contents) - sourceIndex := imageupdate.NewSourceIndex(tree) - - // Pass 2: index sources repo-wide; extract targets only under the scan prefix. - var targets []imageupdate.UpdateTarget - for f, content := range contents { - _ = sourceIndex.IndexHelmRepositories(f, content) - - if prefix != "" && !strings.HasPrefix(f, prefix) { - continue - } - fileTargets, err := imageupdate.ExtractTargets(f, content) - if err != nil { - continue - } - targets = append(targets, fileTargets...) - } - return targets, sourceIndex, nil -} - -func filterTargets(targets []imageupdate.UpdateTarget, matcher repomap.ResourceMatcher, opts imageFilterOptions) []imageupdate.UpdateTarget { - var out []imageupdate.UpdateTarget - for _, t := range targets { - if !matcher.MatchesRef(t.Ref) { - continue - } - if t.Kind == imageupdate.TargetImage && len(opts.Image) > 0 { - if matched, _ := collections.MatchItem(t.Image.GetFullNameWithoutTag(), opts.Image...); !matched { - continue - } - } - if t.Kind == imageupdate.TargetChart && len(opts.Chart) > 0 { - if matched, _ := collections.MatchItem(t.ChartName, opts.Chart...); !matched { - continue - } - } - out = append(out, t) - } - return out + targets := imageupdate.Filter(res.Targets, matcher, opts.Image, opts.Chart) + return targets, res.Index, conf, nil } diff --git a/cmd/repomap/images_test_helpers.go b/cmd/repomap/images_test_helpers.go new file mode 100644 index 0000000..1648cc9 --- /dev/null +++ b/cmd/repomap/images_test_helpers.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/flanksource/repomap" + "github.com/flanksource/repomap/imageupdate" +) + +const deploymentManifest = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: default +spec: + template: + spec: + containers: + - name: web + image: nginx:1.25.3 # keep me +` + +type fakeRegistryClient struct{ tags []string } + +func (f fakeRegistryClient) Tags(context.Context) ([]string, error) { + return f.tags, nil +} + +func (f fakeRegistryClient) Digest(context.Context, string) (string, error) { + return "", nil +} + +func fakeImageResolver(tags []string) *imageupdate.Resolver { + return &imageupdate.Resolver{ + NewRegistryClient: func(context.Context, *imageupdate.ContainerImage) (imageupdate.RegistryClient, error) { + return fakeRegistryClient{tags: tags}, nil + }, + } +} + +// writeRepo creates a temp git repo with one manifest and a repomap conf rooted +// there. The manifest is committed so git ls-files discovers it. +func writeRepo(t *testing.T) (*repomap.ArchConf, string) { + t.Helper() + dir := t.TempDir() + if out, err := exec.Command("git", "-C", dir, "init").CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, out) + } + rel := "deploy.yaml" + if err := os.WriteFile(filepath.Join(dir, rel), []byte(deploymentManifest), 0o644); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("git", "-C", dir, "add", rel).CombinedOutput(); err != nil { + t.Fatalf("git add: %v: %s", err, out) + } + conf, err := repomap.GetConf(dir) + if err != nil { + t.Fatal(err) + } + return conf, rel +} diff --git a/cmd/repomap/images_update.go b/cmd/repomap/images_update.go index 96ab8a8..a31b432 100644 --- a/cmd/repomap/images_update.go +++ b/cmd/repomap/images_update.go @@ -3,19 +3,15 @@ package main import ( "context" "fmt" - "path/filepath" - "strings" + "os" "github.com/flanksource/clicky" "github.com/flanksource/clicky/api" - "github.com/flanksource/clicky/task" - flanksourceContext "github.com/flanksource/commons/context" - - "github.com/flanksource/repomap" - "github.com/flanksource/repomap/imageupdate" - "github.com/flanksource/repomap/kubernetes" + depgraph "github.com/flanksource/repomap/deps" ) +// UpdateImageOptions are the flags for the deprecated `images update` command, +// which now delegates to `deps update --manager image,helm`. type UpdateImageOptions struct { imageFilterOptions Latest bool `json:"latest" flag:"latest" help:"Resolve each target to the highest stable semver"` @@ -26,276 +22,45 @@ type UpdateImageOptions struct { func (opts UpdateImageOptions) GetName() string { return "update" } func (opts UpdateImageOptions) Help() api.Text { - return clicky.Text(`Update container image tags and Helm chart versions in tracked manifests. - -Discovers apps/v1 workload images and Flux HelmRelease chart versions in -git-tracked YAML, resolves the target version from the container registry or -Helm repository, and edits the manifest in place (preserving comments). + return clicky.Text(`DEPRECATED: use 'repomap deps update --manager image,helm' instead. -With --latest the highest stable version is chosen. With --version a specific -version is applied. With neither, an interactive picker lists available versions. +Update container image tags and Helm chart versions in tracked manifests. This +command now delegates to 'deps update', which additionally stages applied edits +with git and resolves HelmRelease spec.chartRef (OCIRepository/HelmChart) charts. EXAMPLES: - repomap images update -n default -k HelmRelease --latest - repomap images update -k Deployment --image nginx --version 1.27.0 - repomap images update -k HelmRelease --dry-run`) + repomap deps update --manager helm -n default -k HelmRelease --latest + repomap deps update --manager image -k Deployment --image nginx --version 1.27.0`) } func init() { - cmd := clicky.AddCommand(imagesCmd, UpdateImageOptions{}, runUpdateImage) - cmd.Short = "Update image tags and Helm chart versions in tracked manifests" + cmd := clicky.AddNamedCommandWithContext("update", imagesCmd, UpdateImageOptions{}, runUpdateImage) + cmd.Short = "(deprecated) Update image tags and Helm chart versions; use 'deps update'" } -// UpdatePlan is one resolved (and possibly applied) version change. -type UpdatePlan struct { - Ref kubernetes.KubernetesRef `json:"ref"` - Kind imageupdate.TargetKind `json:"kind"` - File string `json:"file"` - Field string `json:"field"` - OldValue string `json:"old_value"` - NewValue string `json:"new_value"` - Written bool `json:"written"` - DryRun bool `json:"dry_run"` - Skipped string `json:"skipped,omitempty"` -} +// runUpdateImage maps the legacy image-update flags onto deps.Update with the +// image and helm managers, so the two commands share one resolution/apply path. +func runUpdateImage(ctx context.Context, opts UpdateImageOptions) (any, error) { + fmt.Fprintln(os.Stderr, "warning: 'images update' is deprecated; use 'repomap deps update --manager image,helm'") -func (p UpdatePlan) Pretty() api.Text { - t := p.Ref.Pretty().Space() - if p.Skipped != "" { - return t.Append("skipped: "+p.Skipped, "text-muted") - } - t = t.Append(kubernetes.VersionChange{OldVersion: p.OldValue, NewVersion: p.NewValue}.Pretty()) - switch { - case p.DryRun: - t = t.Space().Append("(dry-run)", "text-yellow-600") - case p.Written: - t = t.Space().Append("written", "text-green-600") - } - return t -} - -func (UpdatePlan) Columns() []api.ColumnDef { - return []api.ColumnDef{ - api.Column("resource").Label("Resource").Build(), - api.Column("file").Label("File").Build(), - api.Column("change").Label("Change").Build(), - api.Column("status").Label("Status").Build(), - } -} - -func (p UpdatePlan) Row() map[string]any { - row := map[string]any{ - "resource": p.Ref.Pretty(), - "file": clicky.Text(p.File, "font-mono"), - "change": kubernetes.VersionChange{OldVersion: p.OldValue, NewVersion: p.NewValue}.Pretty(), - } - switch { - case p.Skipped != "": - row["status"] = clicky.Text(p.Skipped, "text-muted") - case p.DryRun: - row["status"] = clicky.Text("dry-run", "text-yellow-600") - case p.Written: - row["status"] = clicky.Text("written", "text-green-600") - } - return row -} - -func runUpdateImage(opts UpdateImageOptions) (any, error) { - if opts.Latest && opts.Version != "" { - return nil, fmt.Errorf("--latest and --version are mutually exclusive") - } - - targets, sourceIndex, conf, err := discoverAndFilter(opts.imageFilterOptions) + path, err := resolvePath(opts.Path) if err != nil { return nil, err } - if len(targets) == 0 { - return nil, fmt.Errorf("no matching image or chart targets found") - } - - resolver := imageupdate.NewResolver() - ctx := context.Background() - - // Resolve chart sources up front (cheap, no network) so the concurrent - // version lookups have a repo URL to query. - for i := range targets { - if targets[i].Kind == imageupdate.TargetChart { - if err := sourceIndex.Resolve(&targets[i]); err != nil { - return nil, err - } - } - } - - var plans []UpdatePlan - if opts.Latest || opts.Version != "" { - plans = resolveConcurrently(ctx, resolver, conf, targets, opts) - } else { - // Interactive picker must run serially (it prompts the user per target). - for _, t := range targets { - plans = append(plans, planTarget(ctx, resolver, conf, t, opts, nil)) - } - } - return api.NewTableFrom(plans), nil -} - -// resolveConcurrently runs each target's version lookup as its own clicky task, -// then applies the resulting edits serially (concurrent writes to the same file -// would race; resolution is the slow, parallelisable part). -func resolveConcurrently(ctx context.Context, resolver *imageupdate.Resolver, conf *repomap.ArchConf, targets []imageupdate.UpdateTarget, opts UpdateImageOptions) []UpdatePlan { - type resolved struct { - newValue string - skipped string - err error - } - results := make([]resolved, len(targets)) - group := task.StartGroup[int]("Resolving image versions", task.WithConcurrency(resolveConcurrency)) - for i, t := range targets { - idx, target := i, t - displayFile := displayPathForRepoFile(conf, target.File) - group.Add(taskName(target, displayFile), func(ctx flanksourceContext.Context, tk *task.Task) (int, error) { - newValue, skipped, err := resolveNewValue(ctx, resolver, target, opts, tk) - results[idx] = resolved{newValue, skipped, err} - return idx, nil - }) - } - _, _ = group.GetResults() - - plans := make([]UpdatePlan, len(targets)) - for i, t := range targets { - plans[i] = applyResolved(conf, t, results[i].newValue, results[i].skipped, results[i].err, opts) - } - return plans -} - -// planTarget resolves the new version for a target and applies the edit. tk may -// be nil when not running inside a task. -func planTarget(ctx context.Context, resolver *imageupdate.Resolver, conf *repomap.ArchConf, t imageupdate.UpdateTarget, opts UpdateImageOptions, tk *task.Task) UpdatePlan { - newValue, skipped, err := resolveNewValue(ctx, resolver, t, opts, tk) - return applyResolved(conf, t, newValue, skipped, err, opts) -} - -// resolveNewValue determines the replacement value for a target without writing. -// It returns a skip reason instead of a value when no update applies. -func resolveNewValue(ctx context.Context, resolver *imageupdate.Resolver, t imageupdate.UpdateTarget, opts UpdateImageOptions, tk *task.Task) (newValue, skipped string, err error) { - logf(tk, "resolving version") - newVersion, err := chooseVersion(ctx, resolver, t, opts) - if err != nil { - return "", "", err - } - if newVersion == "" { - return "", "no version selected", nil - } - newValue = newVersion - if t.Kind == imageupdate.TargetImage { - newValue, err = resolver.NewImageValue(ctx, t, newVersion) - if err != nil { - return "", "", err - } - } - if newValue == t.CurrentValue { - return "", "already up to date", nil - } - return newValue, "", nil -} - -// applyResolved builds the plan and applies the edit (unless dry-run, skipped, -// or errored). -func applyResolved(conf *repomap.ArchConf, t imageupdate.UpdateTarget, newValue, skipped string, resErr error, opts UpdateImageOptions) UpdatePlan { - plan := UpdatePlan{ - Ref: t.Ref, - Kind: t.Kind, - File: displayPathForRepoFile(conf, t.File), - Field: t.FieldJSONPath, - OldValue: t.CurrentValue, - DryRun: opts.DryRun, - } - if resErr != nil { - plan.Skipped = resErr.Error() - return plan - } - if skipped != "" { - plan.Skipped = skipped - return plan - } - plan.NewValue = newValue - - absFile := filepath.Join(conf.RepoPath(), t.File) - if _, err := imageupdate.ApplyEdit(absFile, t, newValue, opts.DryRun); err != nil { - plan.Skipped = err.Error() - return plan - } - plan.Written = !opts.DryRun - return plan -} - -func logf(tk *task.Task, format string, args ...any) { - if tk != nil { - tk.Infof(format, args...) - } -} - -// chooseVersion returns the target version per the CLI mode: explicit --version, -// resolved --latest, or an interactive pick from available candidates. -func chooseVersion(ctx context.Context, resolver *imageupdate.Resolver, t imageupdate.UpdateTarget, opts UpdateImageOptions) (string, error) { - if opts.Version != "" { - available, err := resolver.Available(ctx, t) - if err != nil { - return "", err - } - if !contains(available, opts.Version) { - return "", fmt.Errorf("%s: version %q is not available (have: %s)", - t.CurrentValue, opts.Version, strings.Join(available, ", ")) - } - return opts.Version, nil - } - if opts.Latest { - return resolver.ResolveLatest(ctx, t) - } - - available, err := resolver.Available(ctx, t) - if err != nil { - return "", err - } - if len(available) == 0 { - return "", fmt.Errorf("no available versions for %s", t.CurrentValue) - } - return pickVersion(t, available), nil -} - -func pickVersion(t imageupdate.UpdateTarget, available []string) string { - title := fmt.Sprintf("Select version for %s/%s (current %s)", t.Ref.Kind, t.Ref.Name, t.CurrentValue) - choice, ok := clicky.PromptSelect(available, clicky.PromptSelectOptions[string]{ - Title: title, - Render: func(v string) api.Textable { - text := clicky.Text(v) - if v == versionOnly(t.CurrentValue) { - text = text.Space().Append("(current)", "text-muted") - } - return text - }, + plans, err := depgraph.Update(ctx, path, depgraph.UpdateOptions{ + Managers: []depgraph.Manager{depgraph.ManagerImage, depgraph.ManagerHelm}, + Kind: opts.Kind, + Namespace: opts.Namespace, + Name: opts.Name, + Selector: opts.Selector, + Image: opts.Image, + Chart: opts.Chart, + Latest: opts.Latest, + Version: opts.Version, + DryRun: opts.DryRun, }) - if !ok { - return "" - } - return choice -} - -func versionOnly(currentValue string) string { - if i := strings.LastIndex(currentValue, ":"); i >= 0 { - v := currentValue[i+1:] - if at := strings.Index(v, "@"); at >= 0 { - v = v[:at] - } - return v - } - return currentValue -} - -func contains(list []string, v string) bool { - for _, item := range list { - if item == v { - return true - } + if err != nil { + return nil, err } - return false + return api.NewTableFrom(plans), nil } diff --git a/cmd/repomap/images_update_test.go b/cmd/repomap/images_update_test.go deleted file mode 100644 index 18184b2..0000000 --- a/cmd/repomap/images_update_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package main - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/flanksource/repomap" - "github.com/flanksource/repomap/imageupdate" -) - -const deploymentManifest = `apiVersion: apps/v1 -kind: Deployment -metadata: - name: web - namespace: default -spec: - template: - spec: - containers: - - name: web - image: nginx:1.25.3 # keep me -` - -type fakeRegistryClient struct{ tags []string } - -func (f fakeRegistryClient) Tags(context.Context) ([]string, error) { - return f.tags, nil -} - -func (f fakeRegistryClient) Digest(context.Context, string) (string, error) { - return "", nil -} - -func fakeImageResolver(tags []string) *imageupdate.Resolver { - return &imageupdate.Resolver{ - NewRegistryClient: func(context.Context, *imageupdate.ContainerImage) (imageupdate.RegistryClient, error) { - return fakeRegistryClient{tags: tags}, nil - }, - } -} - -// writeRepo creates a temp dir with one manifest and a repomap conf rooted there. -func writeRepo(t *testing.T) (*repomap.ArchConf, string) { - t.Helper() - dir := t.TempDir() - if out, err := exec.Command("git", "-C", dir, "init").CombinedOutput(); err != nil { - t.Fatalf("git init: %v: %s", err, out) - } - rel := "deploy.yaml" - if err := os.WriteFile(filepath.Join(dir, rel), []byte(deploymentManifest), 0o644); err != nil { - t.Fatal(err) - } - conf, err := repomap.GetConf(dir) - if err != nil { - t.Fatal(err) - } - return conf, rel -} - -func TestPlanTarget_LatestDryRun(t *testing.T) { - conf, rel := writeRepo(t) - content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - targets, err := imageupdate.ExtractTargets(rel, string(content)) - if err != nil || len(targets) != 1 { - t.Fatalf("extract: %v (%d targets)", err, len(targets)) - } - - resolver := fakeImageResolver([]string{"1.25.3", "1.27.0", "1.28.0-rc.1"}) - plan := planTarget(context.Background(), resolver, conf, targets[0], - UpdateImageOptions{Latest: true, DryRun: true}, nil) - if plan.Skipped != "" { - t.Fatalf("unexpected skip: %s", plan.Skipped) - } - if plan.NewValue != "nginx:1.27.0" { - t.Errorf("new value = %q, want nginx:1.27.0", plan.NewValue) - } - if plan.Written { - t.Error("dry-run must not write") - } - // file must be untouched - after, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - if string(after) != deploymentManifest { - t.Error("dry-run modified the manifest") - } -} - -func TestPlanTarget_VersionWritesAndPreservesComment(t *testing.T) { - conf, rel := writeRepo(t) - content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - targets, _ := imageupdate.ExtractTargets(rel, string(content)) - - resolver := fakeImageResolver([]string{"1.25.3", "1.27.0"}) - plan := planTarget(context.Background(), resolver, conf, targets[0], - UpdateImageOptions{Version: "1.27.0"}, nil) - if !plan.Written { - t.Fatalf("expected written, skipped=%q", plan.Skipped) - } - after, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - got := splitLine(string(after), 11) - want := " image: nginx:1.27.0 # keep me" - if got != want { - t.Errorf("line 11 = %q, want %q", got, want) - } -} - -func TestPlanTarget_RejectsUnavailableVersion(t *testing.T) { - conf, rel := writeRepo(t) - content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) - targets, _ := imageupdate.ExtractTargets(rel, string(content)) - - resolver := fakeImageResolver([]string{"1.25.3", "1.27.0"}) - plan := planTarget(context.Background(), resolver, conf, targets[0], - UpdateImageOptions{Version: "9.9.9"}, nil) - if plan.Skipped == "" || !strings.Contains(plan.Skipped, "not available") { - t.Fatalf("expected skip for unavailable version, got skipped=%q written=%v", plan.Skipped, plan.Written) - } -} - -func splitLine(content string, n int) string { - lines := splitLines(content) - if n < 1 || n > len(lines) { - return "" - } - return lines[n-1] -} - -func splitLines(s string) []string { - var out []string - cur := "" - for _, r := range s { - if r == '\n' { - out = append(out, cur) - cur = "" - continue - } - cur += string(r) - } - if cur != "" { - out = append(out, cur) - } - return out -} diff --git a/cmd/repomap/main.go b/cmd/repomap/main.go index 5deaf0d..0940a18 100644 --- a/cmd/repomap/main.go +++ b/cmd/repomap/main.go @@ -49,17 +49,31 @@ func init() { func main() { defer shutdown.RecoverAndShutdown() - // Default to scan when no subcommand is given - if args := os.Args[1:]; len(args) == 0 || args[0] == "" || args[0][0] == '-' { - rootCmd.SetArgs(append([]string{"scan"}, args...)) - } else if args[0] != "help" && args[0] != "completion" { - if cmd, _, _ := rootCmd.Find(args); cmd == rootCmd { - rootCmd.SetArgs(append([]string{"scan"}, args...)) - } - } + rootCmd.SetArgs(defaultToScan(os.Args[1:])) if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } + +// defaultToScan rewrites CLI args so that `scan` is the implicit subcommand: a +// bare invocation, or one that leads with scan flags or a bare path, becomes +// `scan ...`. Help and completion are left untouched so `repomap --help` (and +// `-h`) shows the root command with its full subcommand list rather than scan's +// help. +func defaultToScan(args []string) []string { + if len(args) == 0 { + return []string{"scan"} + } + if args[0] == "-h" || args[0] == "--help" || args[0] == "help" || args[0] == "completion" { + return args + } + if args[0] == "" || args[0][0] == '-' { + return append([]string{"scan"}, args...) + } + if cmd, _, _ := rootCmd.Find(args); cmd == rootCmd { + return append([]string{"scan"}, args...) + } + return args +} diff --git a/cmd/repomap/main_test.go b/cmd/repomap/main_test.go new file mode 100644 index 0000000..57a3a08 --- /dev/null +++ b/cmd/repomap/main_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "reflect" + "testing" +) + +func TestDefaultToScan(t *testing.T) { + cases := []struct { + name string + in []string + want []string + }{ + {"bare invocation runs scan", nil, []string{"scan"}}, + {"leading scan flag prepends scan", []string{"-n", "ns"}, []string{"scan", "-n", "ns"}}, + {"bare path prepends scan", []string{"some/path"}, []string{"scan", "some/path"}}, + {"long help reaches root", []string{"--help"}, []string{"--help"}}, + {"short help reaches root", []string{"-h"}, []string{"-h"}}, + {"help command untouched", []string{"help"}, []string{"help"}}, + {"completion untouched", []string{"completion", "zsh"}, []string{"completion", "zsh"}}, + {"deps subcommand untouched", []string{"deps", "."}, []string{"deps", "."}}, + {"deps help untouched", []string{"deps", "-h"}, []string{"deps", "-h"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := defaultToScan(tc.in); !reflect.DeepEqual(got, tc.want) { + t.Errorf("defaultToScan(%v) = %v, want %v", tc.in, got, tc.want) + } + }) + } +} diff --git a/deps/cachewarm.go b/deps/cachewarm.go new file mode 100644 index 0000000..f940bbb --- /dev/null +++ b/deps/cachewarm.go @@ -0,0 +1,318 @@ +package deps + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/flanksource/clicky/task" + flanksourceContext "github.com/flanksource/commons/context" + "github.com/flanksource/repomap/deps/manager/gomod" + "github.com/flanksource/repomap/deps/manager/npm" + "github.com/flanksource/repomap/deps/manager/pnpm" + "github.com/flanksource/repomap/deps/manifest" +) + +// cacheWarmConcurrency bounds how many specs warm at once. GOMODCACHE and the +// pnpm store are both lock-protected against concurrent writers, so the limit is +// about not saturating the network rather than correctness. +const cacheWarmConcurrency = 4 + +// warmers is the single place an ecosystem is wired into cache warming. +var warmers = map[Manager]manifest.Warmer{ + ManagerGo: gomod.Warmer{}, + ManagerNPM: npm.Warmer{}, + ManagerPNPM: pnpm.Warmer{}, +} + +type WarmOptions struct { + Manager Manager + Specs []string + // Build compiles every package of the target (Go) or lets dependency + // lifecycle scripts run so native addons are built (npm, pnpm). + Build bool + // Verify replays the work with the network disabled, turning "a download ran" + // into "this cache can build offline". + Verify bool + Runner CommandRunner +} + +type WarmStep struct { + Name string `json:"name"` + Command string `json:"command"` + Duration time.Duration `json:"duration"` + Error string `json:"error,omitempty"` +} + +type WarmResult struct { + Manager Manager `json:"manager"` + // Spec is the requested name@version, verbatim; Name is the manager's + // canonical form of it, and Version is what the manager resolved. + Spec string `json:"spec"` + Name string `json:"name"` + // RequestedVersion is the version the spec asked for ("latest" when it was + // omitted), kept so output has something to show when the read-back could not + // determine a concrete one. + RequestedVersion string `json:"requested_version,omitempty"` + Version string `json:"version,omitempty"` + Packages int `json:"packages,omitempty"` + Cache string `json:"cache,omitempty"` + Built bool `json:"built,omitempty"` + Verified bool `json:"verified,omitempty"` + Steps []WarmStep `json:"steps,omitempty"` + Error string `json:"error,omitempty"` + // SummaryError records a failure to read back what was warmed. The cache is + // still warm when this is set, so it does not fail the spec — but it is + // reported rather than swallowed. + SummaryError string `json:"summary_error,omitempty"` +} + +func (r WarmResult) failed() bool { return r.Error != "" } + +// WarmCache downloads each spec's full dependency closure into the machine's +// shared package cache, using a throwaway project per spec so nothing in the +// user's working tree is touched. +func WarmCache(ctx context.Context, opts WarmOptions) ([]WarmResult, error) { + warmer, ok := warmers[opts.Manager] + if !ok { + return nil, fmt.Errorf("cache warming does not support manager %q (expected go, npm, or pnpm)", opts.Manager) + } + if len(opts.Specs) == 0 { + return nil, fmt.Errorf("cache warming needs at least one name@version spec") + } + runner := opts.Runner + if runner == nil { + runner = ExecRunner{} + // Only preflight when we are really going to exec: an injected runner is a + // test double or a recorder, and need not correspond to a binary on PATH. + if _, err := exec.LookPath(warmer.Binary()); err != nil { + return nil, fmt.Errorf("%s not found on PATH, which is required to warm %s caches: %w", warmer.Binary(), opts.Manager, err) + } + } + + results := make([]WarmResult, len(opts.Specs)) + group := task.StartGroup[int]("Warming package caches", task.WithConcurrency(cacheWarmConcurrency)) + for i, spec := range opts.Specs { + idx, spec := i, spec + group.Add(fmt.Sprintf("%s %s", opts.Manager, spec), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { + results[idx] = warmSpec(ctx, warmer, runner, spec, opts, tk) + switch { + case results[idx].failed(): + tk.Errorf("%s", results[idx].Error) + tk.Failed() + case results[idx].SummaryError != "": + tk.Warnf("%s", results[idx].SummaryError) + tk.Warning() + default: + tk.Success() + } + return idx, nil + }) + } + _, _ = group.GetResults() + + // Each failure carries its own command and the tool's stderr. They are folded + // into the returned error rather than left on the results, because the scratch + // directory is already gone and a caller that only prints the error would + // otherwise have nothing to reproduce from. + var failed []string + for _, result := range results { + if result.failed() { + failed = append(failed, fmt.Sprintf("%s: %s", result.Spec, result.Error)) + } + } + if len(failed) > 0 { + return results, fmt.Errorf("failed to warm %d of %d specs:\n %s", len(failed), len(results), strings.Join(failed, "\n ")) + } + return results, nil +} + +func warmSpec(ctx context.Context, warmer manifest.Warmer, runner CommandRunner, spec string, opts WarmOptions, tk *task.Task) WarmResult { + result := WarmResult{Manager: warmer.Manager(), Spec: spec} + normalized, err := warmer.NormalizeSpec(spec) + if err != nil { + result.Error = err.Error() + return result + } + name, version, err := manifest.SplitSpec(normalized) + if err != nil { + result.Error = err.Error() + return result + } + result.Name = name + result.RequestedVersion = version + + dir, err := os.MkdirTemp("", "repomap-cache-warm-*") + if err != nil { + result.Error = err.Error() + return result + } + // The scratch project is disposable: the durable result is what landed in the + // module cache or package store. + defer func() { _ = os.RemoveAll(dir) }() + + probe, err := runProbe(ctx, warmer, runner, dir) + if err != nil { + result.Error = err.Error() + return result + } + steps, err := warmer.Steps(manifest.WarmRequest{ + Dir: dir, + Name: name, + Version: version, + Build: opts.Build, + Verify: opts.Verify, + }, probe) + if err != nil { + result.Error = err.Error() + return result + } + + tk.SetProgress(0, len(steps)) + for i, step := range steps { + tk.Infof("%s", step.Name) + started := time.Now() + err := runWarmStep(ctx, runner, dir, step) + record := WarmStep{Name: step.Name, Command: step.Detail(), Duration: time.Since(started)} + if err != nil { + record.Error = err.Error() + result.Steps = append(result.Steps, record) + result.Error = err.Error() + return result + } + result.Steps = append(result.Steps, record) + tk.SetProgress(i+1, len(steps)) + } + + result.Built = opts.Build + result.Verified = opts.Verify + summarizeWarm(ctx, &result, dir, runner) + return result +} + +// runProbe executes a manager's optional version probe inside the scratch dir and +// returns its trimmed stdout, which Steps consumes as an ordinary input. +func runProbe(ctx context.Context, warmer manifest.Warmer, runner CommandRunner, dir string) (string, error) { + cmd := warmer.Probe() + if cmd == nil { + return "", nil + } + cmd.Dir = dir + result, err := runner.Run(ctx, *cmd) + if err != nil { + return "", commandError(*cmd, result, err) + } + return strings.TrimSpace(result.Stdout), nil +} + +func runWarmStep(ctx context.Context, runner CommandRunner, dir string, step manifest.Step) error { + switch step.Kind { + case manifest.StepWrite: + return os.WriteFile(filepath.Join(dir, step.Path), step.Content, 0o600) + case manifest.StepRemove: + return os.RemoveAll(filepath.Join(dir, step.Path)) + case manifest.StepExec: + result, err := runner.Run(ctx, step.Command) + if err != nil { + return commandError(step.Command, result, err) + } + return nil + default: + return fmt.Errorf("unknown warm step kind %q for step %q", step.Kind, step.Name) + } +} + +// commandError names the exact invocation and the tool's own diagnostics. The +// scratch dir is gone by the time a caller sees this, so the message is the only +// reproduction handle there is. +func commandError(cmd manifest.Command, result CommandResult, err error) error { + if stderr := strings.TrimSpace(result.Stderr); stderr != "" { + return fmt.Errorf("%s: %w: %s", cmd.String(), err, stderr) + } + return fmt.Errorf("%s: %w", cmd.String(), err) +} + +// summarizeWarm reports what actually landed by reading the warmed scratch +// project back through repomap's own manifest resolvers, rather than re-parsing +// go.mod or a lockfile here. The manager did the resolving, so this is also where +// the concrete version (including a Go pseudo-version) is discovered. +// +// It calls discoverOffline/resolveManifest rather than Scan deliberately: Scan +// renders its own task group, which would surface read-back warnings about a +// temporary directory the user never asked about. +func summarizeWarm(ctx context.Context, result *WarmResult, dir string, runner CommandRunner) { + result.Cache = warmCachePath(ctx, result.Manager, runner, dir) + + projects, _, err := discoverOffline(dir, []Manager{result.Manager}) + if err != nil { + result.SummaryError = fmt.Sprintf("could not read back the warmed project: %s", err) + return + } + if len(projects) == 0 { + result.SummaryError = fmt.Sprintf("no %s manifest was produced in the scratch project", result.Manager) + return + } + // MaxDepth 1 keeps the read-back offline — it parses the manifest and lockfile + // the warm just wrote instead of shelling out again. + root, _, err := resolveManifest(ctx, projects[0], Options{MaxDepth: 1, IncludeIndirect: true}) + if err != nil { + result.SummaryError = fmt.Sprintf("could not read back the warmed project: %s", err) + return + } + result.Packages, result.Version = walkWarmed(root, result.Name) +} + +// walkWarmed counts every resolved dependency below root and finds the target's +// version. It walks rather than reading root.Children directly because pnpm nests +// dependencies under a synthetic importer node while go.mod lists them flat. +// +// Only nodes carrying a version are counted, which is what distinguishes a real +// package from pnpm's importer (and from the project root itself). +func walkWarmed(root *Node, name string) (count int, version string) { + seen := map[*Node]bool{} + stack := []*Node{root} + for len(stack) > 0 { + node := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if node == nil || seen[node] { + continue + } + seen[node] = true + for _, child := range node.Children { + if child.Version != "" { + count++ + if version == "" && child.Name == name { + version = child.Version + } + } + stack = append(stack, child) + } + } + return count, version +} + +// warmCachePath asks the manager where its cache lives, so the output names the +// directory that grew. A failure here is not worth reporting: it costs only the +// display of a path. +func warmCachePath(ctx context.Context, manager Manager, runner CommandRunner, dir string) string { + var cmd manifest.Command + switch manager { + case ManagerGo: + cmd = manifest.Command{Dir: dir, Name: "go", Args: []string{"env", "GOMODCACHE"}} + case ManagerPNPM: + cmd = manifest.Command{Dir: dir, Name: "pnpm", Args: []string{"store", "path"}} + case ManagerNPM: + cmd = manifest.Command{Dir: dir, Name: "npm", Args: []string{"config", "get", "cache"}} + default: + return "" + } + result, err := runner.Run(ctx, cmd) + if err != nil { + return "" + } + return strings.TrimSpace(result.Stdout) +} diff --git a/deps/cachewarm_plan.go b/deps/cachewarm_plan.go new file mode 100644 index 0000000..43db1a3 --- /dev/null +++ b/deps/cachewarm_plan.go @@ -0,0 +1,87 @@ +package deps + +import ( + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (r WarmResult) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("[%s] %s", r.Manager, r.Name), managerStyle(r.Manager)) + if version := r.displayVersion(); version != "" { + t = t.Space().Append(version, "font-mono text-muted") + } + if r.Error != "" { + return t.Space().Append("failed: "+r.Error, "text-red-600") + } + if r.Packages > 0 { + t = t.Space().Append(fmt.Sprintf("%d packages", r.Packages), "text-muted") + } + t = t.Space().Append(strings.Join(r.badges(), " "), "text-green-600") + if r.Cache != "" { + t = t.Space().Append(r.Cache, "font-mono text-muted") + } + if r.SummaryError != "" { + t = t.Space().Append("("+r.SummaryError+")", "text-yellow-600") + } + return t +} + +func (WarmResult) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("manager").Label("Manager").Build(), + api.Column("dependency").Label("Dependency").Build(), + api.Column("version").Label("Version").Build(), + api.Column("packages").Label("Packages").Build(), + api.Column("status").Label("Status").Build(), + api.Column("cache").Label("Cache").Build(), + } +} + +func (r WarmResult) Row() map[string]any { + row := map[string]any{ + "manager": clicky.Text(string(r.Manager), managerStyle(r.Manager)), + "dependency": clicky.Text(r.Name, "font-bold text-cyan-600"), + "version": clicky.Text(r.displayVersion(), "font-mono"), + "cache": clicky.Text(r.Cache, "font-mono text-muted"), + } + if r.Packages > 0 { + row["packages"] = clicky.Text(fmt.Sprintf("%d", r.Packages), "text-muted") + } else { + row["packages"] = clicky.Text("") + } + if r.Error != "" { + row["status"] = clicky.Text("failed: "+r.Error, "text-red-600") + return row + } + status := clicky.Text(strings.Join(r.badges(), " "), "text-green-600") + if r.SummaryError != "" { + status = status.Append(" ("+r.SummaryError+")", "text-yellow-600") + } + row["status"] = status + return row +} + +// badges names what the warm actually did, so "warmed" is never confused with +// "compiled" or with "proven to work offline". +func (r WarmResult) badges() []string { + badges := []string{"warmed"} + if r.Built { + badges = append(badges, "built") + } + if r.Verified { + badges = append(badges, "verified offline") + } + return badges +} + +// displayVersion prefers the version the manager resolved, falling back to what +// was requested when the read-back could not determine it. +func (r WarmResult) displayVersion() string { + if r.Version != "" { + return r.Version + } + return r.RequestedVersion +} diff --git a/deps/cachewarm_test.go b/deps/cachewarm_test.go new file mode 100644 index 0000000..623b97a --- /dev/null +++ b/deps/cachewarm_test.go @@ -0,0 +1,303 @@ +package deps + +import ( + "context" + "errors" + "strings" + "testing" +) + +// pnpm nests dependencies under a synthetic importer node, so counting +// root.Children would report the importer instead of the packages, and the +// version lookup would miss the target entirely. +func TestWalkWarmedSkipsSyntheticNodes(t *testing.T) { + leftPad := NewNode(ManagerPNPM, "left-pad", "1.3.0") + nested := NewNode(ManagerPNPM, "big-int", "1.0.2") + leftPad.Children = []*Node{nested} + importer := NewNode(ManagerPNPM, ".", "") + importer.Source = "importer" + importer.Children = []*Node{leftPad} + root := NewNode(ManagerPNPM, "scratch", "") + root.Children = []*Node{importer} + + count, version := walkWarmed(root, "left-pad") + if count != 2 { + t.Errorf("count = %d, want 2: the importer carries no version and is not a package", count) + } + if version != "1.3.0" { + t.Errorf("version = %q, want 1.3.0: the target sits below the importer", version) + } +} + +func TestWalkWarmedFindsFlatGoRequires(t *testing.T) { + root := NewNode(ManagerGo, "repomap.local/cachewarm", "") + root.Children = []*Node{ + NewNode(ManagerGo, "rsc.io/quote", "v1.5.2"), + NewNode(ManagerGo, "rsc.io/sampler", "v1.3.0"), + } + count, version := walkWarmed(root, "rsc.io/quote") + if count != 2 || version != "v1.5.2" { + t.Fatalf("walkWarmed = (%d, %q), want (2, v1.5.2)", count, version) + } +} + +func TestWarmCacheRejectsUnsupportedManager(t *testing.T) { + for _, manager := range []Manager{ManagerMaven, ManagerGradle, ManagerHelm, ManagerImage, Manager("cargo")} { + _, err := WarmCache(context.Background(), WarmOptions{ + Manager: manager, + Specs: []string{"something@1.0.0"}, + Runner: &updateFakeRunner{}, + }) + if err == nil { + t.Errorf("manager %q: expected an error", manager) + continue + } + if !strings.Contains(err.Error(), "go, npm, or pnpm") { + t.Errorf("manager %q: error should list the supported managers, got %v", manager, err) + } + } +} + +func TestWarmCacheRequiresSpecs(t *testing.T) { + if _, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Runner: &updateFakeRunner{}, + }); err == nil { + t.Fatal("expected an error when no specs are given") + } +} + +func TestWarmCacheRunsTheGoStepsInOrder(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"github.com/acme/lib@v1.2.3"}, + Build: true, + Verify: true, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 { + t.Fatalf("want 1 result, got %d", len(results)) + } + result := results[0] + if result.Error != "" { + t.Fatalf("unexpected warm error: %s", result.Error) + } + if result.Name != "github.com/acme/lib" || result.Spec != "github.com/acme/lib@v1.2.3" { + t.Errorf("result identity = %q / %q", result.Name, result.Spec) + } + if !result.Built || !result.Verified { + t.Errorf("Built = %v, Verified = %v, want both true", result.Built, result.Verified) + } + steps := []string{ + "go mod init repomap.local/cachewarm", + "go get github.com/acme/lib/...@v1.2.3", + "go mod download all", + "go build github.com/acme/lib/...", + // The verify replay is the same build with GOPROXY=off, which the argv + // alone does not show; deps/manager/gomod pins the env. + "go build github.com/acme/lib/...", + } + // Reporting where the cache lives runs after the steps, not as one of them. + want := append(append([]string{}, steps...), "go env GOMODCACHE") + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } + if len(result.Steps) != len(steps) { + t.Errorf("recorded %d steps, want %d", len(result.Steps), len(steps)) + } +} + +// The scratch directory is deleted before the error surfaces, so the message is +// the only handle the user has for reproducing the failure. +func TestWarmCacheFailureAbortsRemainingStepsAndNamesTheCommand(t *testing.T) { + runner := &updateFakeRunner{ + succeedByDefault: true, + errors: map[string]error{ + "go get github.com/acme/lib/...@v1.2.3": errors.New("exit status 1"), + }, + responses: map[string]CommandResult{ + "go get github.com/acme/lib/...@v1.2.3": {Stderr: "module github.com/acme/lib: not found"}, + }, + } + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"github.com/acme/lib@v1.2.3"}, + Build: true, + Runner: runner, + }) + if err == nil { + t.Fatal("WarmCache should report an error when a spec fails, so the CLI exits non-zero") + } + if len(results) != 1 { + t.Fatalf("want 1 result, got %d", len(results)) + } + for _, want := range []string{"go get github.com/acme/lib/...@v1.2.3", "not found"} { + if !strings.Contains(results[0].Error, want) { + t.Errorf("result error %q should contain %q", results[0].Error, want) + } + // The returned error is what a caller that only prints err sees, and the + // scratch dir is gone by then, so the detail has to survive into it too. + if !strings.Contains(err.Error(), want) { + t.Errorf("returned error %q should contain %q", err, want) + } + } + if results[0].Built { + t.Error("Built should stay false when the warm failed") + } + // init ran, get failed, and download/build must not have been attempted. + want := []string{"go mod init repomap.local/cachewarm", "go get github.com/acme/lib/...@v1.2.3"} + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } +} + +// pnpm's probe output decides the build flags, so the orchestrator has to run it +// and feed the result into Steps. +func TestWarmCacheFeedsTheProbeIntoSteps(t *testing.T) { + runner := &updateFakeRunner{ + succeedByDefault: true, + responses: map[string]CommandResult{ + "pnpm --version": {Stdout: "10.7.0\n"}, + }, + } + if _, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerPNPM, + Specs: []string{"left-pad@1.3.0"}, + Build: true, + Runner: runner, + }); err != nil { + t.Fatal(err) + } + want := []string{ + "pnpm --version", + "pnpm install --config.dangerouslyAllowAllBuilds=true", + "pnpm store path", + } + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestWarmCacheWarmsEverySpec(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"github.com/acme/one@v1.0.0", "github.com/acme/two@v2.0.0"}, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if len(results) != 2 { + t.Fatalf("want 2 results, got %d", len(results)) + } + // Specs are warmed concurrently, so assert identity by position rather than + // relying on command ordering. + if results[0].Spec != "github.com/acme/one@v1.0.0" || results[1].Spec != "github.com/acme/two@v2.0.0" { + t.Fatalf("results should stay in spec order, got %q and %q", results[0].Spec, results[1].Spec) + } + for _, result := range results { + if result.Error != "" { + t.Errorf("spec %q failed: %s", result.Spec, result.Error) + } + } +} + +// A GitHub slug and a repository URL are what a user has to hand, and both used +// to reach `go get` verbatim and fail with the toolchain's "malformed module +// path". +func TestWarmCacheNormalisesGoRepoShorthand(t *testing.T) { + for _, spec := range []string{ + "flanksource/commons", + "https://github.com/flanksource/commons", + "git@github.com:flanksource/commons.git", + } { + t.Run(spec, func(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{spec}, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if results[0].Name != "github.com/flanksource/commons" { + t.Errorf("Name = %q, want the canonical module path", results[0].Name) + } + if results[0].Spec != spec { + t.Errorf("Spec = %q, should stay the spec the user typed", results[0].Spec) + } + want := []string{ + "go mod init repomap.local/cachewarm", + "go get github.com/flanksource/commons/...@latest", + "go mod download all", + "go env GOMODCACHE", + } + if got := runner.ran(); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("commands mismatch\n got: %v\nwant: %v", got, want) + } + }) + } +} + +// A spec with no host to infer is repomap's to reject, before a scratch project +// is created or the go toolchain is asked anything. +func TestWarmCacheRejectsAGoSpecThatIsNotAModulePath(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"commons"}, + Runner: runner, + }) + if err == nil { + t.Fatal("expected an error: \"commons\" names no host and cannot become a module path") + } + if !strings.Contains(results[0].Error, "github.com/owner/repo") { + t.Errorf("error %q should say what a module path looks like", results[0].Error) + } + if got := runner.ran(); len(got) != 0 { + t.Errorf("no command should run for an unusable spec, got %v", got) + } +} + +// npm names are already registry names, and a scoped one must not be mistaken +// for a repo shorthand. +func TestWarmCacheLeavesScopedNPMNamesAlone(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerNPM, + Specs: []string{"@scope/pkg@1.0.0"}, + Runner: runner, + }) + if err != nil { + t.Fatal(err) + } + if results[0].Name != "@scope/pkg" || results[0].RequestedVersion != "1.0.0" { + t.Fatalf("result identity = %q@%q, want @scope/pkg@1.0.0", results[0].Name, results[0].RequestedVersion) + } +} + +// A bad spec must be reported against that spec rather than aborting the run +// before the others are warmed. +func TestWarmCacheReportsABadSpecWithoutSkippingTheRest(t *testing.T) { + runner := &updateFakeRunner{succeedByDefault: true} + results, err := WarmCache(context.Background(), WarmOptions{ + Manager: ManagerGo, + Specs: []string{"left-pad@", "github.com/acme/lib@v1.0.0"}, + Runner: runner, + }) + if err == nil { + t.Fatal("expected a non-nil error because one spec is invalid") + } + if results[0].Error == "" { + t.Error("the invalid spec should carry an error") + } + if results[1].Error != "" { + t.Errorf("the valid spec should still have been warmed, got %q", results[1].Error) + } +} diff --git a/deps/chart_remote.go b/deps/chart_remote.go new file mode 100644 index 0000000..cfbafb0 --- /dev/null +++ b/deps/chart_remote.go @@ -0,0 +1,43 @@ +package deps + +import ( + "context" + "fmt" +) + +// chartResolver fetches a Helm chart dependency and produces its direct children: +// nested subchart dependency nodes and image nodes harvested from the fetched +// chart's values.yaml and templates. +type chartResolver struct { + helm *helmClient +} + +func (c *chartResolver) expand(ctx context.Context, dep chartDepEntry, parentDepth int) (children []*Node, resolvedVersion string, warnings []string) { + fc, cleanup, err := c.helm.fetchChart(ctx, dep) + if err != nil { + return nil, "", []string{fmt.Sprintf("chart %s@%s: %s", dep.Name, dep.Version, err)} + } + defer cleanup() + + for _, sub := range fc.Dependencies { + if sub.Name == "" { + continue + } + n := NewNode(ManagerHelm, sub.Name, sub.Version) + n.Depth = parentDepth + 1 + n.Scope = "dependencies" + n.Source = sub.Repository + children = append(children, n) + } + + imgNodes, imgWarns := chartImageNodes(fc.Dir, fc.Dir) + for _, img := range imgNodes { + img.Depth = parentDepth + 1 + img.Direct = false + children = append(children, img) + } + for _, w := range imgWarns { + warnings = append(warnings, w.Message) + } + return children, fc.Version, warnings +} diff --git a/deps/collapse.go b/deps/collapse.go new file mode 100644 index 0000000..1fb8620 --- /dev/null +++ b/deps/collapse.go @@ -0,0 +1,55 @@ +package deps + +// collapseDuplicates rewrites package-manager roots in place so each dependency +// renders once at its resolved (shallowest) location instead of repeating under +// every parent. Later occurrences are dropped silently with no marker. Image/Helm +// roots are left untouched: repeated images are real distinct deployments. +func collapseDuplicates(roots []*Node) { + for _, root := range roots { + if root == nil || !isPackageManager(root.Manager) { + continue + } + collapseRoot(root) + } +} + +func isPackageManager(manager Manager) bool { + switch manager { + case ManagerGo, ManagerMaven, ManagerGradle, ManagerNPM, ManagerPNPM: + return true + } + return false +} + +// collapseRoot keeps the first BFS sighting (shallowest, then sorted) of each +// node ID within a single root and silently drops later sightings. +func collapseRoot(root *Node) { + sortTree(root) + + seen := map[string]bool{} + queue := []*Node{root} + for len(queue) > 0 { + parent := queue[0] + queue = queue[1:] + kept := parent.Children[:0] + for _, child := range parent.Children { + if seen[child.ID] { + continue + } + seen[child.ID] = true + kept = append(kept, child) + queue = append(queue, child) + } + parent.Children = kept + } +} + +func sortTree(node *Node) { + if node == nil { + return + } + sortChildren(node) + for _, child := range node.Children { + sortTree(child) + } +} diff --git a/deps/collapse_test.go b/deps/collapse_test.go new file mode 100644 index 0000000..a9793ec --- /dev/null +++ b/deps/collapse_test.go @@ -0,0 +1,111 @@ +package deps + +import "testing" + +// diamondRoot builds a go project where a shared dependency is reachable under +// three direct parents, the classic diamond that motivates collapsing. +// +// app +// ├── a ── shared@v1 +// ├── b ── shared@v1 +// └── c ── shared@v1 +func diamondRoot() *Node { + root := NewNode(ManagerGo, "github.com/acme/app", "") + root.Source = "go.mod" + shared := func() *Node { + n := NewNode(ManagerGo, "github.com/acme/shared", "v1.0.0") + n.Depth = 2 + return n + } + for _, name := range []string{"a", "b", "c"} { + parent := NewNode(ManagerGo, "github.com/acme/"+name, "v1.0.0") + parent.Direct = true + parent.Depth = 1 + parent.Children = []*Node{shared()} + root.Children = append(root.Children, parent) + } + return root +} + +func TestCollapseKeepsSharedDependencyOnce(t *testing.T) { + root := diamondRoot() + collapseDuplicates([]*Node{root}) + + var kept int + for _, parent := range root.Children { + if findChild(parent, "github.com/acme/shared") != nil { + kept++ + } + } + if kept != 1 { + t.Fatalf("expected shared dependency to survive under exactly one parent, got %d", kept) + } +} + +func TestCollapseDropsLaterSightingsSilently(t *testing.T) { + root := diamondRoot() + collapseDuplicates([]*Node{root}) + + a := findChild(root, "github.com/acme/a") + if findChild(a, "github.com/acme/shared") == nil { + t.Fatalf("shared should be retained under the first sorted parent 'a'") + } + for _, name := range []string{"github.com/acme/b", "github.com/acme/c"} { + parent := findChild(root, name) + if findChild(parent, "github.com/acme/shared") != nil { + t.Fatalf("parent %s should no longer carry the shared dependency", name) + } + if len(parent.Children) != 0 { + t.Fatalf("parent %s should have no children after collapse, got %d", name, len(parent.Children)) + } + } +} + +func TestCollapseLeavesImageRootsUntouched(t *testing.T) { + root := NewNode(ManagerImage, "container images", "") + dup := func() *Node { + n := NewNode(ManagerImage, "nginx", "1.25") + n.Depth = 1 + return n + } + root.Children = []*Node{dup(), dup()} + collapseDuplicates([]*Node{root}) + + if len(root.Children) != 2 { + t.Fatalf("image roots must keep every occurrence, got %d children", len(root.Children)) + } +} + +func TestCollapseIsolatesPerRoot(t *testing.T) { + first := diamondRoot() + second := diamondRoot() + collapseDuplicates([]*Node{first, second}) + + for _, root := range []*Node{first, second} { + a := findChild(root, "github.com/acme/a") + if findChild(a, "github.com/acme/shared") == nil { + t.Fatalf("each root should retain shared under its first sorted parent") + } + for _, name := range []string{"github.com/acme/b", "github.com/acme/c"} { + if findChild(findChild(root, name), "github.com/acme/shared") != nil { + t.Fatalf("each root should dedup independently; %s should not keep shared", name) + } + } + } +} + +func TestCollapseKeepsConflictingVersions(t *testing.T) { + root := NewNode(ManagerGo, "github.com/acme/app", "") + v1 := NewNode(ManagerGo, "github.com/acme/lib", "v1.0.0") + v1.Direct = true + v1.Depth = 1 + v2 := NewNode(ManagerGo, "github.com/acme/lib", "v2.0.0") + v2.Direct = true + v2.Depth = 1 + root.Children = []*Node{v1, v2} + collapseDuplicates([]*Node{root}) + + if len(root.Children) != 2 { + t.Fatalf("conflicting versions must each survive, got %d", len(root.Children)) + } +} diff --git a/deps/compare_scan.go b/deps/compare_scan.go index 27776f3..4169583 100644 --- a/deps/compare_scan.go +++ b/deps/compare_scan.go @@ -41,6 +41,10 @@ func CompareScan(ctx context.Context, path string, opts CompareOptions) (*Compar return nil, err } + // Diffs always compare the full graph: indexExport walks Roots, so collapsing + // duplicate subtrees would hide real dependency changes. + opts.ShowDuplicates = true + baseSha, err := resolveRef(ctx, repoRoot, opts.BaseRef) if err != nil { return nil, err diff --git a/deps/dockerfile.go b/deps/dockerfile.go new file mode 100644 index 0000000..74fab73 --- /dev/null +++ b/deps/dockerfile.go @@ -0,0 +1,149 @@ +package deps + +import ( + "fmt" + "regexp" + "strings" +) + +// imageRef is a parsed container image reference split into name, tag, and digest. +type imageRef struct { + Name string + Version string + Digest string +} + +var argRefPattern = regexp.MustCompile(`\$\{(\w+)\}|\$(\w+)`) + +// parseDockerfileFrom extracts the external base images referenced by FROM +// directives. It substitutes ARG/ENV defaults declared earlier in the file, +// excludes references to internal build stages (FROM ... AS ) and the +// scratch terminal, and de-duplicates. Unresolved ARG references are warned. +func parseDockerfileFrom(content string) (bases []imageRef, warnings []string) { + args := map[string]string{} + stages := map[string]bool{} + seen := map[string]bool{} + + for _, line := range dockerfileLines(content) { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + switch strings.ToUpper(fields[0]) { + case "ARG", "ENV": + if name, val, ok := parseArgAssign(fields[1:]); ok { + args[name] = val + } + case "FROM": + ref, stage, ok, warn := parseFromDirective(fields[1:], args, stages) + if warn != "" { + warnings = append(warnings, warn) + } + if stage != "" { + stages[strings.ToLower(stage)] = true + } + if !ok { + continue + } + key := ref.Name + ":" + ref.Version + "@" + ref.Digest + if !seen[key] { + seen[key] = true + bases = append(bases, ref) + } + } + } + return bases, warnings +} + +// parseFromDirective interprets the tokens after FROM: it strips flags +// (--platform=...), resolves the image token, and records an AS stage name. +// ok is false for internal stage references, scratch, and unresolved ARGs. +func parseFromDirective(tokens []string, args map[string]string, stages map[string]bool) (ref imageRef, stage string, ok bool, warn string) { + var image string + for i := 0; i < len(tokens); i++ { + tok := tokens[i] + switch { + case strings.HasPrefix(tok, "--"): + continue + case image == "": + image = tok + case strings.EqualFold(tok, "AS") && i+1 < len(tokens): + stage = tokens[i+1] + i++ + } + } + if image == "" { + return imageRef{}, stage, false, "" + } + resolved, complete := substituteArgs(image, args) + if !complete { + return imageRef{}, stage, false, fmt.Sprintf("unresolved build arg in FROM %q", image) + } + if strings.EqualFold(resolved, "scratch") || stages[strings.ToLower(resolved)] { + return imageRef{}, stage, false, "" + } + return parseImageRef(resolved), stage, true, "" +} + +// substituteArgs replaces ${VAR}/$VAR with known ARG/ENV defaults; complete is +// false when any reference cannot be resolved. +func substituteArgs(s string, args map[string]string) (out string, complete bool) { + complete = true + out = argRefPattern.ReplaceAllStringFunc(s, func(m string) string { + name := strings.Trim(m, "${}") + if v, ok := args[name]; ok { + return v + } + complete = false + return m + }) + return out, complete +} + +// parseImageRef splits an image reference into name, tag, and digest, leaving a +// registry host:port intact. +func parseImageRef(s string) imageRef { + var digest string + if at := strings.Index(s, "@"); at >= 0 { + digest = s[at+1:] + s = s[:at] + } + name, version := s, "" + if i := imageTagSeparator(s); i >= 0 { + name, version = s[:i], s[i+1:] + } + return imageRef{Name: name, Version: version, Digest: digest} +} + +func parseArgAssign(tokens []string) (name, value string, ok bool) { + if len(tokens) == 0 { + return "", "", false + } + eq := strings.SplitN(tokens[0], "=", 2) + if len(eq) != 2 { + return "", "", false + } + return eq[0], strings.Trim(eq[1], `"'`), true +} + +// dockerfileLines joins backslash line continuations and trims carriage returns. +func dockerfileLines(content string) []string { + var lines []string + var buf strings.Builder + for _, ln := range strings.Split(content, "\n") { + ln = strings.TrimRight(ln, "\r") + trimmedRight := strings.TrimRight(ln, " \t") + if strings.HasSuffix(trimmedRight, "\\") { + buf.WriteString(strings.TrimSuffix(trimmedRight, "\\")) + buf.WriteString(" ") + continue + } + buf.WriteString(ln) + lines = append(lines, strings.TrimSpace(buf.String())) + buf.Reset() + } + if buf.Len() > 0 { + lines = append(lines, strings.TrimSpace(buf.String())) + } + return lines +} diff --git a/deps/dockerfile_test.go b/deps/dockerfile_test.go new file mode 100644 index 0000000..de6b770 --- /dev/null +++ b/deps/dockerfile_test.go @@ -0,0 +1,121 @@ +package deps + +import "testing" + +func refsToStrings(refs []imageRef) []string { + out := make([]string, 0, len(refs)) + for _, r := range refs { + s := r.Name + if r.Version != "" { + s += ":" + r.Version + } + if r.Digest != "" { + s += "@" + r.Digest + } + out = append(out, s) + } + return out +} + +func TestParseDockerfileFrom(t *testing.T) { + cases := []struct { + name string + content string + want []string + }{ + { + name: "single FROM with tag", + content: "FROM nginx:1.25.3\nRUN echo hi\n", + want: []string{"nginx:1.25.3"}, + }, + { + name: "bare image no tag", + content: "FROM alpine\n", + want: []string{"alpine"}, + }, + { + name: "multi-stage excludes internal stage reference", + content: "FROM golang:1.22 AS build\n" + + "RUN go build\n" + + "FROM gcr.io/distroless/static:nonroot\n" + + "COPY --from=build /app /app\n", + want: []string{"golang:1.22", "gcr.io/distroless/static:nonroot"}, + }, + { + name: "final FROM referencing a prior stage is internal", + content: "FROM golang:1.22 AS build\n" + + "FROM build\n", + want: []string{"golang:1.22"}, + }, + { + name: "platform flag is stripped", + content: "FROM --platform=linux/amd64 ubuntu:22.04\n", + want: []string{"ubuntu:22.04"}, + }, + { + name: "ARG substitution in tag", + content: "ARG GO_VERSION=1.22\nFROM golang:${GO_VERSION}\n", + want: []string{"golang:1.22"}, + }, + { + name: "ARG substitution in registry and unbraced var", + content: "ARG REG=docker.io\nFROM $REG/library/busybox:1.36\n", + want: []string{"docker.io/library/busybox:1.36"}, + }, + { + name: "scratch is terminal and excluded", + content: "FROM scratch\nCOPY x /\n", + want: nil, + }, + { + name: "digest pinned", + content: "FROM nginx@sha256:abc123\n", + want: []string{"nginx@sha256:abc123"}, + }, + { + name: "tag and digest", + content: "FROM nginx:1.25@sha256:abc123\n", + want: []string{"nginx:1.25@sha256:abc123"}, + }, + { + name: "case-insensitive directives and comments", + content: "# base\nfrom Ubuntu:22.04 as Base\n", + want: []string{"Ubuntu:22.04"}, + }, + { + name: "registry with port keeps host colon", + content: "FROM localhost:5000/team/app:2.0\n", + want: []string{"localhost:5000/team/app:2.0"}, + }, + { + name: "duplicate external bases collapse", + content: "FROM alpine:3.20 AS a\nFROM alpine:3.20 AS b\n", + want: []string{"alpine:3.20"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, _ := parseDockerfileFrom(tc.content) + gotStrings := refsToStrings(got) + if len(gotStrings) != len(tc.want) { + t.Fatalf("bases = %v, want %v", gotStrings, tc.want) + } + for i := range tc.want { + if gotStrings[i] != tc.want[i] { + t.Fatalf("base[%d] = %q, want %q (all: %v)", i, gotStrings[i], tc.want[i], gotStrings) + } + } + }) + } +} + +func TestParseDockerfileUnresolvedArgWarns(t *testing.T) { + bases, warnings := parseDockerfileFrom("FROM $UNDEFINED_BASE\n") + if len(bases) != 0 { + t.Fatalf("unresolved ARG FROM should yield no base, got %v", refsToStrings(bases)) + } + if len(warnings) == 0 { + t.Fatalf("expected a warning for the unresolved ARG reference") + } +} diff --git a/deps/go_graph_test.go b/deps/go_graph_test.go index 6ec3051..4bf9948 100644 --- a/deps/go_graph_test.go +++ b/deps/go_graph_test.go @@ -116,6 +116,94 @@ require github.com/acme/lib v1.2.3 } } +// goDiamondScan resolves a go module whose two direct dependencies both require +// the same transitive dep, exercising the collapse pass end-to-end. +func goDiamondScan(t *testing.T, opts Options) *Export { + t.Helper() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require ( + github.com/acme/lib v1.0.0 + github.com/acme/other v1.0.0 +) +`) + opts.Managers = []Manager{ManagerGo} + opts.MaxDepth = 0 + opts.Runner = &fakeRunner{result: CommandResult{Stdout: strings.Join([]string{ + "github.com/acme/app github.com/acme/lib@v1.0.0", + "github.com/acme/app github.com/acme/other@v1.0.0", + "github.com/acme/lib@v1.0.0 github.com/acme/dep@v0.1.0", + "github.com/acme/other@v1.0.0 github.com/acme/dep@v0.1.0", + "", + }, "\n")}} + opts.Now = func() time.Time { return time.Unix(1, 0).UTC() } + got, err := Scan(context.Background(), dir, opts) + if err != nil { + t.Fatal(err) + } + return got +} + +func TestScanCollapsesDuplicatesSilentlyByDefault(t *testing.T) { + got := goDiamondScan(t, Options{}) + root := got.Roots[0] + lib := findChild(root, "github.com/acme/lib") + other := findChild(root, "github.com/acme/other") + + depUnderLib := findChild(lib, "github.com/acme/dep") + depUnderOther := findChild(other, "github.com/acme/dep") + if (depUnderLib == nil) == (depUnderOther == nil) { + t.Fatalf("dep should render under exactly one parent, lib=%v other=%v", depUnderLib != nil, depUnderOther != nil) + } + if got.Statistics.Duplicates != 0 { + t.Fatalf("default scan should not count duplicates, got %d", got.Statistics.Duplicates) + } + if len(got.Duplicates) != 0 { + t.Fatalf("default scan should not report a duplicates section, got %d", len(got.Duplicates)) + } +} + +func TestScanShowDuplicatesRendersEveryOccurrence(t *testing.T) { + got := goDiamondScan(t, Options{ShowDuplicates: true}) + root := got.Roots[0] + lib := findChild(root, "github.com/acme/lib") + other := findChild(root, "github.com/acme/other") + if findChild(lib, "github.com/acme/dep") == nil || findChild(other, "github.com/acme/dep") == nil { + t.Fatalf("--show-duplicates should keep dep under both parents") + } + if !got.Metadata.ShowDuplicates { + t.Fatalf("metadata should record show_duplicates") + } + if got.Statistics.Duplicates == 0 { + t.Fatalf("--show-duplicates should count the shared dep as a duplicate") + } + if len(got.Duplicates) == 0 { + t.Fatalf("--show-duplicates should populate the duplicates section") + } +} + +func TestScanFlatStaysFullGraph(t *testing.T) { + collapsed := goDiamondScan(t, Options{}) + flat := goDiamondScan(t, Options{Flat: true}) + // The flat export counts every node once (dedup by id), independent of the + // tree collapse, so dep is present exactly once and edges cover both parents. + var depEdges int + for _, e := range flat.Edges { + if strings.HasSuffix(e.To, "acme/dep@v0.1.0") { + depEdges++ + } + } + if depEdges != 2 { + t.Fatalf("flat edges should retain both parent→dep edges, got %d", depEdges) + } + if len(collapsed.Roots) == 0 || len(flat.Nodes) == 0 { + t.Fatalf("expected collapsed roots and flat nodes to be populated") + } +} + func TestScanGoDepthOneStaysOffline(t *testing.T) { dir := t.TempDir() writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app diff --git a/deps/helm_credentials.go b/deps/helm_credentials.go new file mode 100644 index 0000000..c8a89a6 --- /dev/null +++ b/deps/helm_credentials.go @@ -0,0 +1,112 @@ +package deps + +import ( + "crypto/tls" + "crypto/x509" + "net/http" + "os" + "strings" + + "github.com/goccy/go-yaml" + "helm.sh/helm/v3/pkg/helmpath" +) + +// helmRepoCreds mirrors one entry of Helm's repositories.yaml, carrying the +// authentication a private chart repository needs. +type helmRepoCreds struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + Username string `yaml:"username"` + Password string `yaml:"password"` + CertFile string `yaml:"certFile"` + KeyFile string `yaml:"keyFile"` + CAFile string `yaml:"caFile"` + InsecureSkipTLSVerify bool `yaml:"insecure_skip_tls_verify"` +} + +type helmRepoFile struct { + Repositories []helmRepoCreds `yaml:"repositories"` +} + +// helmCredentials reuses the user's `helm repo add` logins so chart discovery +// authenticates to private repositories exactly like the helm CLI. +type helmCredentials struct { + repos []helmRepoCreds +} + +// loadHelmCredentials reads repositories.yaml from $HELM_REPOSITORY_CONFIG or +// Helm's default config path. A missing or unreadable file yields no +// credentials (public repositories keep working). +func loadHelmCredentials() *helmCredentials { + path := os.Getenv("HELM_REPOSITORY_CONFIG") + if path == "" { + path = helmpath.ConfigPath("repositories.yaml") + } + return newHelmCredentials(path) +} + +func newHelmCredentials(path string) *helmCredentials { + data, err := os.ReadFile(path) + if err != nil { + return &helmCredentials{} + } + var f helmRepoFile + if err := yaml.Unmarshal(data, &f); err != nil { + return &helmCredentials{} + } + return &helmCredentials{repos: f.Repositories} +} + +// match returns the credentials for the repository whose URL is the longest +// prefix of url (so index.yaml/.tgz fetches under a repo inherit its login). +func (h *helmCredentials) match(url string) (helmRepoCreds, bool) { + best := -1 + var found helmRepoCreds + for _, r := range h.repos { + base := strings.TrimSuffix(r.URL, "/") + if base == "" || (url != base && !strings.HasPrefix(url, base+"/")) { + continue + } + if len(base) > best { + best, found = len(base), r + } + } + return found, best >= 0 +} + +// authorize applies the matched repository's basic-auth and TLS settings to req, +// returning the http client to use (a custom one only when TLS options apply). +func (h *helmCredentials) authorize(req *http.Request) *http.Client { + if h == nil { + return http.DefaultClient + } + cred, ok := h.match(req.URL.String()) + if !ok { + return http.DefaultClient + } + if cred.Username != "" { + req.SetBasicAuth(cred.Username, cred.Password) + } + return httpClientForCreds(cred) +} + +func httpClientForCreds(cred helmRepoCreds) *http.Client { + if cred.CAFile == "" && cred.CertFile == "" && !cred.InsecureSkipTLSVerify { + return http.DefaultClient + } + tlsCfg := &tls.Config{InsecureSkipVerify: cred.InsecureSkipTLSVerify} //nolint:gosec // honors the user's helm repo setting + if cred.CAFile != "" { + if pem, err := os.ReadFile(cred.CAFile); err == nil { + pool := x509.NewCertPool() + if pool.AppendCertsFromPEM(pem) { + tlsCfg.RootCAs = pool + } + } + } + if cred.CertFile != "" && cred.KeyFile != "" { + if cert, err := tls.LoadX509KeyPair(cred.CertFile, cred.KeyFile); err == nil { + tlsCfg.Certificates = []tls.Certificate{cert} + } + } + return &http.Client{Transport: &http.Transport{TLSClientConfig: tlsCfg}} +} diff --git a/deps/helm_credentials_test.go b/deps/helm_credentials_test.go new file mode 100644 index 0000000..af00f45 --- /dev/null +++ b/deps/helm_credentials_test.go @@ -0,0 +1,78 @@ +package deps + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestHelmCredentialsMatchLongestPrefix(t *testing.T) { + creds := &helmCredentials{repos: []helmRepoCreds{ + {Name: "broad", URL: "https://charts.example.com"}, + {Name: "narrow", URL: "https://charts.example.com/team/", Username: "u"}, + }} + got, ok := creds.match("https://charts.example.com/team/index.yaml") + if !ok || got.Name != "narrow" { + t.Fatalf("expected longest-prefix match 'narrow', got %q (ok=%v)", got.Name, ok) + } + if other, ok := creds.match("https://unrelated.example.com/index.yaml"); ok { + t.Fatalf("unrelated URL should not match, got %q", other.Name) + } +} + +func TestCacheFetchSendsHelmBasicAuth(t *testing.T) { + var gotUser, gotPass string + var hadAuth bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUser, gotPass, hadAuth = r.BasicAuth() + if !hadAuth { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte("index-body")) + })) + defer srv.Close() + + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + c.get = c.httpGet // exercise the real HTTP path + c.helmAuth = &helmCredentials{repos: []helmRepoCreds{ + {Name: "private", URL: srv.URL, Username: "alice", Password: "s3cret"}, + }} + + data, err := c.Fetch(context.Background(), srv.URL+"/index.yaml", ttlIndex) + if err != nil { + t.Fatalf("fetch with credentials failed: %v", err) + } + if string(data) != "index-body" { + t.Fatalf("body = %q, want index-body", data) + } + if !hadAuth || gotUser != "alice" || gotPass != "s3cret" { + t.Fatalf("server did not receive expected basic auth (user=%q pass set=%v hadAuth=%v)", gotUser, gotPass != "", hadAuth) + } +} + +func TestCacheFetchNoAuthForUnmatchedRepo(t *testing.T) { + var hadAuth bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _, hadAuth = r.BasicAuth() + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + c.get = c.httpGet + c.helmAuth = &helmCredentials{repos: []helmRepoCreds{ + {Name: "other", URL: "https://charts.other.com", Username: "x", Password: "y"}, + }} + + if _, err := c.Fetch(context.Background(), srv.URL+"/index.yaml", ttlIndex); err != nil { + t.Fatal(err) + } + if hadAuth { + t.Fatalf("credentials must not be sent to a non-matching repository") + } +} diff --git a/deps/image_base.go b/deps/image_base.go new file mode 100644 index 0000000..fba8d0e --- /dev/null +++ b/deps/image_base.go @@ -0,0 +1,167 @@ +package deps + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +const ( + labelBaseName = "org.opencontainers.image.base.name" + labelBaseDigest = "org.opencontainers.image.base.digest" + labelSource = "org.opencontainers.image.source" +) + +// imageResolver resolves the base image(s) of a container image by combining the +// OCI base-image label with the Dockerfile FROM directives in the image's source +// repository. +type imageResolver struct { + cache RemoteCache + parser func(content string) ([]imageRef, []string) +} + +func newImageResolver(cache RemoteCache) *imageResolver { + return &imageResolver{cache: cache, parser: parseDockerfileFrom} +} + +// baseImages returns the distinct base images of ref. It never returns a hard +// error: remote failures are reported as warnings so recursion degrades per +// branch. +func (r *imageResolver) baseImages(ctx context.Context, ref string) (bases []imageRef, warnings []string) { + cfg, err := r.cache.ImageConfig(ctx, ref) + if err != nil { + return nil, []string{fmt.Sprintf("image %s: %s", ref, err)} + } + + seen := map[string]bool{} + add := func(b imageRef) { + key := b.Name + ":" + b.Version + "@" + b.Digest + if b.Name == "" || seen[key] { + return + } + seen[key] = true + bases = append(bases, b) + } + + // Signal A: explicit OCI base-image label. + if name := cfg.Labels[labelBaseName]; name != "" { + b := parseImageRef(name) + if b.Digest == "" { + b.Digest = cfg.Labels[labelBaseDigest] + } + add(b) + } + + // Signal B: source repo Dockerfile FROM. + source := cfg.Labels[labelSource] + if source == "" { + source = sourceRepoHeuristic(ref) + } + if source != "" { + dbases, dwarn := r.dockerfileBases(ctx, ref, source) + warnings = append(warnings, dwarn...) + for _, b := range dbases { + add(b) + } + } + + if len(bases) == 0 && len(warnings) == 0 { + warnings = append(warnings, fmt.Sprintf("image %s: no base image resolvable (no base/source label, no known registry heuristic)", ref)) + } + return bases, warnings +} + +func (r *imageResolver) dockerfileBases(ctx context.Context, ref, source string) ([]imageRef, []string) { + url := normalizeGitURL(source) + if url == "" { + return nil, []string{fmt.Sprintf("image %s: unrecognized source %q", ref, source)} + } + dir, err := r.cache.GitRepo(ctx, url, parseImageRef(ref).Version) + if err != nil { + return nil, []string{fmt.Sprintf("image %s: clone %s: %s", ref, url, err)} + } + path, warnings := findDockerfile(dir) + if path == "" { + return nil, []string{fmt.Sprintf("image %s: no Dockerfile in %s", ref, source)} + } + data, err := os.ReadFile(path) + if err != nil { + return nil, append(warnings, fmt.Sprintf("image %s: reading Dockerfile: %s", ref, err)) + } + bases, pwarn := r.parser(string(data)) + return bases, append(warnings, pwarn...) +} + +// sourceRepoHeuristic maps a registry path to a probable git source URL for +// registries that mirror their org/repo layout from GitHub. +func sourceRepoHeuristic(ref string) string { + name := stripImageVersion(ref) + for _, host := range []string{"ghcr.io/", "quay.io/"} { + if strings.HasPrefix(name, host) { + parts := strings.Split(strings.TrimPrefix(name, host), "/") + if len(parts) >= 2 { + return "https://github.com/" + parts[0] + "/" + parts[1] + } + } + } + return "" +} + +// normalizeGitURL turns an OCI source label into a cloneable URL. +func normalizeGitURL(source string) string { + source = strings.TrimSpace(source) + source = strings.TrimPrefix(source, "git+") + switch { + case source == "": + return "" + case strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"), strings.HasPrefix(source, "git@"): + return strings.TrimSuffix(source, ".git") + case strings.HasPrefix(source, "github.com/"), strings.HasPrefix(source, "gitlab.com/"): + return "https://" + strings.TrimSuffix(source, ".git") + default: + return "" + } +} + +// findDockerfile returns the repo-root Dockerfile if present, else the first +// Dockerfile found in a shallow walk, warning when several exist. +func findDockerfile(root string) (string, []string) { + if p := filepath.Join(root, "Dockerfile"); fileExists(p) { + return p, nil + } + var found []string + _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() { + if d.Name() == ".git" || ignoredDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + if d.Name() == "Dockerfile" || strings.HasPrefix(d.Name(), "Dockerfile.") { + found = append(found, path) + } + return nil + }) + if len(found) == 0 { + return "", nil + } + if len(found) > 1 { + rels := make([]string, len(found)) + for i, f := range found { + rels[i], _ = filepath.Rel(root, f) + } + return found[0], []string{fmt.Sprintf("multiple Dockerfiles found, using %s (others: %s)", rels[0], strings.Join(rels[1:], ", "))} + } + return found[0], nil +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} diff --git a/deps/manager/gomod/warm.go b/deps/manager/gomod/warm.go new file mode 100644 index 0000000..4a4d92d --- /dev/null +++ b/deps/manager/gomod/warm.go @@ -0,0 +1,156 @@ +// Package gomod warms the Go module and build caches for a single module. +// +// It is named gomod rather than go because a directory named "go" reads badly at +// import sites. +package gomod + +import ( + "fmt" + "strings" + + "golang.org/x/mod/module" + + "github.com/flanksource/repomap/deps/manifest" +) + +// ModulePath is the synthetic module the scratch project declares. It is never +// published, but it must be a valid module path for `go mod init` to accept it. +const ModulePath = "repomap.local/cachewarm" + +// DefaultHost is prepended to a spec whose first path element carries no dot, +// which is what a bare owner/repo shorthand looks like. +const DefaultHost = "github.com" + +// schemes are stripped from a spec typed as a URL. Only the scheme goes: the +// rest of a repository URL already reads as a module path. +var schemes = []string{"https://", "http://", "ssh://", "git://"} + +// Warmer drives the go toolchain against a throwaway module that requires the +// target, populating GOMODCACHE and — with Build — GOCACHE. +type Warmer struct{} + +func (Warmer) Manager() manifest.Manager { return manifest.ManagerGo } + +func (Warmer) Binary() string { return "go" } + +// Probe returns nil: the go commands used here have been stable for many +// releases, so no runtime version check is needed. +func (Warmer) Probe() *manifest.Command { return nil } + +// NormalizeSpec turns what a user actually has to hand — a GitHub slug, a +// browser URL, a clone URL — into the canonical module path `go get` demands, +// and rejects anything that still is not one. Without it the only diagnostic is +// the go toolchain's own "malformed module path", which never says what repomap +// wanted. +// +// A repository URL is not always a module path (vanity domains, modules nested +// in a monorepo), so this canonicalises the common GitHub shape rather than +// claiming to resolve every repository. +func (Warmer) NormalizeSpec(spec string) (string, error) { + original := strings.TrimSpace(spec) + trimmed := original + for _, scheme := range schemes { + trimmed = strings.TrimPrefix(trimmed, scheme) + } + + name, version, err := manifest.SplitSpec(trimGitURL(trimmed)) + if err != nil { + return "", err + } + name = strings.TrimSuffix(strings.TrimSuffix(name, "/"), ".git") + + // A dotless first element is a bare owner/repo shorthand: no module path can + // start without a host, and every host has a dot. A single element is left + // alone so it fails below rather than becoming github.com/. + if head, _, ok := strings.Cut(name, "/"); ok && !strings.Contains(head, ".") { + name = DefaultHost + "/" + name + } + if err := module.CheckPath(name); err != nil { + return "", fmt.Errorf("%q is not a Go module path (expected github.com/owner/repo or owner/repo): %w", original, err) + } + return name + "@" + version, nil +} + +// trimGitURL rewrites the two clone-URL shapes a module path cannot express: +// scp syntax (git@host:owner/repo) and a leftover userinfo prefix +// (git@host/owner/repo, what stripping ssh:// leaves behind). Both must go +// before the name@version split, which would otherwise read git@ as the version +// separator. A spec with no slash is never a clone URL, so name@version is left +// intact. +func trimGitURL(spec string) string { + head, path, ok := strings.Cut(spec, "/") + at := strings.Index(head, "@") + if !ok || at < 0 { + return spec + } + // In scp syntax the colon plays the role of the first slash. + return strings.Replace(head[at+1:], ":", "/", 1) + "/" + path +} + +func (Warmer) Steps(req manifest.WarmRequest, _ string) ([]manifest.Step, error) { + switch { + case req.Dir == "": + return nil, fmt.Errorf("go warming needs a scratch directory") + case req.Name == "": + return nil, fmt.Errorf("go warming needs a module path") + case req.Version == "": + return nil, fmt.Errorf("go warming needs a version for %s", req.Name) + } + + // The /... package pattern, rather than the bare module, is what records the + // go.sum entries needed to *build* every package in the module. Resolving the + // module alone records only enough to reference it, which leaves a later + // offline build short of its dependencies. + packages := req.Name + "/..." + + steps := []manifest.Step{ + goStep("init", req, false, "mod", "init", ModulePath), + goStep("resolve", req, false, "get", packages+"@"+req.Version), + // The synthetic module imports nothing, so a bare `go mod download` would + // have no packages to work from. The `all` pattern materialises zips for + // the whole resolved graph, which is what makes a later build offline-able. + goStep("download", req, false, "mod", "download", "all"), + } + if req.Build { + steps = append(steps, goStep("build", req, false, "build", packages)) + } + if req.Verify { + steps = append(steps, verifyStep(req, packages)) + } + return steps, nil +} + +// verifyStep replays the most demanding work already done, with the proxy +// disabled so any cache miss is a hard failure rather than a silent refetch. +// With Build there are compiled packages to reproduce; without it, the strongest +// available claim is that every module zip is already resident. +func verifyStep(req manifest.WarmRequest, packages string) manifest.Step { + if req.Build { + return goStep("verify", req, true, "build", packages) + } + return goStep("verify", req, true, "mod", "download", "all") +} + +func goStep(name string, req manifest.WarmRequest, offline bool, args ...string) manifest.Step { + return manifest.Step{ + Kind: manifest.StepExec, + Name: name, + Command: manifest.Command{ + Dir: req.Dir, + Name: "go", + Args: args, + Env: goEnv(offline), + }, + } +} + +// goEnv pins the module mode explicitly. GOWORK=off matters most: without it a +// scratch dir that happens to sit inside a go.work tree silently joins that +// workspace and warms the wrong module graph. +func goEnv(offline bool) []string { + env := []string{"GOWORK=off", "GOFLAGS=-mod=mod"} + if offline { + env = append(env, "GOPROXY=off") + } + return env +} diff --git a/deps/manager/gomod/warm_test.go b/deps/manager/gomod/warm_test.go new file mode 100644 index 0000000..2e31ea2 --- /dev/null +++ b/deps/manager/gomod/warm_test.go @@ -0,0 +1,222 @@ +package gomod + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +// formatSteps renders each step as "name: argv [env]" so a single assertion pins +// the command sequence, the exact arguments, and the environment together. +func formatSteps(t *testing.T, steps []manifest.Step) []string { + t.Helper() + out := make([]string, 0, len(steps)) + for _, step := range steps { + if step.Kind != manifest.StepExec { + t.Fatalf("go warming should only produce exec steps, got %q for %q", step.Kind, step.Name) + } + out = append(out, step.Name+": "+step.Command.String()+" ["+strings.Join(step.Command.Env, " ")+"]") + } + return out +} + +func TestStepsPerFlagCombination(t *testing.T) { + const ( + online = "[GOWORK=off GOFLAGS=-mod=mod]" + offline = "[GOWORK=off GOFLAGS=-mod=mod GOPROXY=off]" + module = "github.com/acme/lib" + version = "v1.2.3" + ) + cases := []struct { + name string + build bool + verif bool + want []string + }{ + { + name: "download only", + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + }, + }, + { + name: "build compiles every package", + build: true, + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + "build: go build github.com/acme/lib/... " + online, + }, + }, + { + // Without --build there is nothing compiled to replay, so the offline + // proof is that every module zip is already resident. + name: "verify without build replays the download", + verif: true, + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + "verify: go mod download all " + offline, + }, + }, + { + name: "verify with build replays the build", + build: true, + verif: true, + want: []string{ + "init: go mod init " + ModulePath + " " + online, + "resolve: go get github.com/acme/lib/...@v1.2.3 " + online, + "download: go mod download all " + online, + "build: go build github.com/acme/lib/... " + online, + "verify: go build github.com/acme/lib/... " + offline, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", + Name: module, + Version: version, + Build: tc.build, + Verify: tc.verif, + }, "") + if err != nil { + t.Fatal(err) + } + got := formatSteps(t, steps) + if strings.Join(got, "\n") != strings.Join(tc.want, "\n") { + t.Fatalf("steps mismatch\n got: %s\nwant: %s", strings.Join(got, "\n "), strings.Join(tc.want, "\n ")) + } + }) + } +} + +// GOPROXY=off must never leak onto a warming step, or the warm would fail on a +// cold cache instead of populating it. +func TestOnlyVerifyStepDisablesTheProxy(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "github.com/acme/lib", Version: "v1.0.0", Build: true, Verify: true, + }, "") + if err != nil { + t.Fatal(err) + } + for _, step := range steps { + offline := strings.Contains(strings.Join(step.Command.Env, " "), "GOPROXY=off") + if offline != (step.Name == "verify") { + t.Errorf("step %q: GOPROXY=off present = %v, want %v", step.Name, offline, step.Name == "verify") + } + } +} + +// Every step must run inside the scratch project, otherwise go would resolve +// against whatever module happens to contain the process working directory. +func TestStepsRunInTheScratchDir(t *testing.T) { + const dir = "/tmp/repomap-cache-warm-xyz" + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: dir, Name: "github.com/acme/lib", Version: "v1.0.0", Build: true, Verify: true, + }, "") + if err != nil { + t.Fatal(err) + } + for _, step := range steps { + if step.Command.Dir != dir { + t.Errorf("step %q dir = %q, want %q", step.Name, step.Command.Dir, dir) + } + } +} + +func TestStepsRejectsIncompleteRequest(t *testing.T) { + cases := []struct { + name string + request manifest.WarmRequest + }{ + {name: "no module path", request: manifest.WarmRequest{Dir: "/scratch", Version: "v1.0.0"}}, + {name: "no version", request: manifest.WarmRequest{Dir: "/scratch", Name: "github.com/acme/lib"}}, + {name: "no dir", request: manifest.WarmRequest{Name: "github.com/acme/lib", Version: "v1.0.0"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := (Warmer{}).Steps(tc.request, ""); err == nil { + t.Fatal("expected an error for an incomplete request") + } + }) + } +} + +func TestNormalizeSpec(t *testing.T) { + const module = "github.com/flanksource/commons" + cases := []struct { + spec string + want string + }{ + // A canonical module path is already what go get wants. + {spec: "github.com/acme/lib@v1.2.3", want: "github.com/acme/lib@v1.2.3"}, + {spec: "github.com/acme/lib", want: "github.com/acme/lib@latest"}, + // A dot in the first element marks it as a host, so a vanity domain and a + // major-version suffix both survive untouched. + {spec: "gopkg.in/yaml.v3", want: "gopkg.in/yaml.v3@latest"}, + {spec: "github.com/acme/lib/v2@v2.1.0", want: "github.com/acme/lib/v2@v2.1.0"}, + // A bare owner/repo slug, the shape copied out of a GitHub page. + {spec: "flanksource/commons", want: module + "@latest"}, + {spec: "flanksource/commons@v1.2.3", want: module + "@v1.2.3"}, + {spec: " flanksource/commons ", want: module + "@latest"}, + // The three URL shapes: browser, https clone, ssh clone. + {spec: "https://github.com/flanksource/commons", want: module + "@latest"}, + {spec: "http://github.com/flanksource/commons/", want: module + "@latest"}, + {spec: "https://github.com/flanksource/commons.git@v1.2.3", want: module + "@v1.2.3"}, + {spec: "git@github.com:flanksource/commons.git", want: module + "@latest"}, + {spec: "ssh://git@github.com/flanksource/commons.git", want: module + "@latest"}, + {spec: "git://github.com/flanksource/commons.git", want: module + "@latest"}, + // A slug on another forge keeps its host rather than gaining github.com. + {spec: "https://gitlab.com/acme/lib@v1.0.0", want: "gitlab.com/acme/lib@v1.0.0"}, + } + for _, tc := range cases { + t.Run(tc.spec, func(t *testing.T) { + got, err := (Warmer{}).NormalizeSpec(tc.spec) + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Fatalf("NormalizeSpec(%q) = %q, want %q", tc.spec, got, tc.want) + } + }) + } +} + +func TestNormalizeSpecRejectsWhatCannotBeAModulePath(t *testing.T) { + cases := []struct { + name string + spec string + }{ + // One element names no host, and inventing github.com/commons would warm + // something the user never asked for. + {name: "single element", spec: "commons"}, + {name: "single element with version", spec: "commons@v1.2.3"}, + {name: "port is not a module path", spec: "github.com/acme/lib:8080"}, + {name: "empty", spec: " "}, + {name: "no version after the separator", spec: "flanksource/commons@"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := (Warmer{}).NormalizeSpec(tc.spec) + if err == nil { + t.Fatalf("NormalizeSpec(%q) = %q, want an error", tc.spec, got) + } + }) + } +} + +func TestWarmerIdentity(t *testing.T) { + if got := (Warmer{}).Manager(); got != manifest.ManagerGo { + t.Errorf("Manager() = %q, want %q", got, manifest.ManagerGo) + } + if probe := (Warmer{}).Probe(); probe != nil { + t.Errorf("Probe() = %v, want nil: go needs no runtime version check", probe) + } +} diff --git a/deps/manager/node/warm.go b/deps/manager/node/warm.go new file mode 100644 index 0000000..05b56e2 --- /dev/null +++ b/deps/manager/node/warm.go @@ -0,0 +1,83 @@ +// Package node holds the warming logic npm and pnpm share: generating the +// single-dependency package.json and assembling the install/verify step +// sequence. The npm and pnpm packages supply only the argv that differs. +package node + +import ( + "encoding/json" + "fmt" + + "github.com/flanksource/repomap/deps/manifest" +) + +// ProjectName is the synthetic package the scratch project declares. It is +// marked private so the registry client never treats it as publishable. +const ProjectName = "repomap-cache-warm" + +// Commands is how one node package manager spells the operations warming needs. +type Commands struct { + Binary string + // Install is the base argv that downloads and links the dependency. + Install []string + // IgnoreScripts suppresses dependency lifecycle scripts. It is applied unless + // the caller asked to build, since compiling native addons is the point of + // building. + IgnoreScripts string + // BuildArgs are appended when building. pnpm uses this for its version-gated + // lifecycle-script allowlist; npm needs nothing. + BuildArgs []string + // Offline is the full argv for the replay that proves the cache is complete. + Offline []string +} + +// Manifest renders the scratch package.json. This is the one place repomap +// generates a package manifest rather than editing an existing one. +func Manifest(name, version string) ([]byte, error) { + return json.MarshalIndent(struct { + Name string `json:"name"` + Version string `json:"version"` + Private bool `json:"private"` + Dependencies map[string]string `json:"dependencies"` + }{ + Name: ProjectName, + Version: "0.0.0", + Private: true, + Dependencies: map[string]string{name: version}, + }, "", " ") +} + +func Steps(req manifest.WarmRequest, cmds Commands) ([]manifest.Step, error) { + switch { + case req.Dir == "": + return nil, fmt.Errorf("%s warming needs a scratch directory", cmds.Binary) + case req.Name == "": + return nil, fmt.Errorf("%s warming needs a package name", cmds.Binary) + case req.Version == "": + return nil, fmt.Errorf("%s warming needs a version for %s", cmds.Binary, req.Name) + } + content, err := Manifest(req.Name, req.Version) + if err != nil { + return nil, err + } + + install := append([]string{}, cmds.Install...) + if req.Build { + install = append(install, cmds.BuildArgs...) + } else if cmds.IgnoreScripts != "" { + install = append(install, cmds.IgnoreScripts) + } + + steps := []manifest.Step{ + {Kind: manifest.StepWrite, Name: "manifest", Path: "package.json", Content: content}, + {Kind: manifest.StepExec, Name: "download", Command: manifest.Command{Dir: req.Dir, Name: cmds.Binary, Args: install}}, + } + if req.Verify { + // Installing over a populated node_modules is a no-op, so the tree has to + // go before the offline replay can prove the cache holds the packages. + steps = append(steps, + manifest.Step{Kind: manifest.StepRemove, Name: "clean", Path: "node_modules"}, + manifest.Step{Kind: manifest.StepExec, Name: "verify", Command: manifest.Command{Dir: req.Dir, Name: cmds.Binary, Args: cmds.Offline}}, + ) + } + return steps, nil +} diff --git a/deps/manager/node/warm_test.go b/deps/manager/node/warm_test.go new file mode 100644 index 0000000..542026a --- /dev/null +++ b/deps/manager/node/warm_test.go @@ -0,0 +1,127 @@ +package node + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +func TestManifestDeclaresExactlyOneDependency(t *testing.T) { + cases := []struct { + name string + pkg string + version string + want string + }{ + { + name: "plain package", pkg: "left-pad", version: "1.3.0", + want: `{ + "name": "repomap-cache-warm", + "version": "0.0.0", + "private": true, + "dependencies": { + "left-pad": "1.3.0" + } +}`, + }, + { + // A scoped name must survive verbatim as the dependency key. + name: "scoped package", pkg: "@flanksource/clicky-ui", version: "^2.1.0", + want: `{ + "name": "repomap-cache-warm", + "version": "0.0.0", + "private": true, + "dependencies": { + "@flanksource/clicky-ui": "^2.1.0" + } +}`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := Manifest(tc.pkg, tc.version) + if err != nil { + t.Fatal(err) + } + if string(got) != tc.want { + t.Fatalf("manifest mismatch\n got: %s\nwant: %s", got, tc.want) + } + }) + } +} + +func testCommands() Commands { + return Commands{ + Binary: "fakepm", + Install: []string{"install"}, + IgnoreScripts: "--ignore-scripts", + Offline: []string{"install", "--offline"}, + } +} + +func TestStepsWriteManifestBeforeInstalling(t *testing.T) { + steps, err := Steps(manifest.WarmRequest{Dir: "/scratch", Name: "left-pad", Version: "1.3.0"}, testCommands()) + if err != nil { + t.Fatal(err) + } + want := []string{ + "manifest: write package.json", + "download: exec fakepm install --ignore-scripts", + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestBuildDropsIgnoreScriptsAndAppendsBuildArgs(t *testing.T) { + cmds := testCommands() + cmds.BuildArgs = []string{"--allow-builds"} + steps, err := Steps(manifest.WarmRequest{Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: true}, cmds) + if err != nil { + t.Fatal(err) + } + want := []string{ + "manifest: write package.json", + "download: exec fakepm install --allow-builds", + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +// Reinstalling over a populated node_modules is a no-op, so without the removal +// the offline replay would pass without proving the cache holds anything. +func TestVerifyRemovesNodeModulesBeforeReplaying(t *testing.T) { + steps, err := Steps(manifest.WarmRequest{Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Verify: true}, testCommands()) + if err != nil { + t.Fatal(err) + } + want := []string{ + "manifest: write package.json", + "download: exec fakepm install --ignore-scripts", + "clean: remove node_modules", + "verify: exec fakepm install --offline", + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestStepsRejectsIncompleteRequest(t *testing.T) { + cases := []struct { + name string + request manifest.WarmRequest + }{ + {name: "no package name", request: manifest.WarmRequest{Dir: "/scratch", Version: "1.3.0"}}, + {name: "no version", request: manifest.WarmRequest{Dir: "/scratch", Name: "left-pad"}}, + {name: "no dir", request: manifest.WarmRequest{Name: "left-pad", Version: "1.3.0"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := Steps(tc.request, testCommands()); err == nil { + t.Fatal("expected an error for an incomplete request") + } + }) + } +} diff --git a/deps/manager/npm/warm.go b/deps/manager/npm/warm.go new file mode 100644 index 0000000..94857e1 --- /dev/null +++ b/deps/manager/npm/warm.go @@ -0,0 +1,32 @@ +// Package npm warms the npm cache for a single package. +package npm + +import ( + "github.com/flanksource/repomap/deps/manager/node" + "github.com/flanksource/repomap/deps/manifest" +) + +type Warmer struct{} + +func (Warmer) Manager() manifest.Manager { return manifest.ManagerNPM } + +func (Warmer) Binary() string { return "npm" } + +// Probe returns nil: npm has kept its lifecycle-script default, so no runtime +// version check is needed. +func (Warmer) Probe() *manifest.Command { return nil } + +// NormalizeSpec returns the spec verbatim: an npm name is already the name the +// registry knows, and rewriting would only risk mangling a scoped @scope/pkg. +func (Warmer) NormalizeSpec(spec string) (string, error) { return spec, nil } + +func (Warmer) Steps(req manifest.WarmRequest, _ string) ([]manifest.Step, error) { + return node.Steps(req, node.Commands{ + Binary: "npm", + Install: []string{"install"}, + IgnoreScripts: "--ignore-scripts", + // ci rather than install: the lockfile the download step wrote makes it the + // stricter replay, and it refuses to reach the network for anything missing. + Offline: []string{"ci", "--offline"}, + }) +} diff --git a/deps/manager/npm/warm_test.go b/deps/manager/npm/warm_test.go new file mode 100644 index 0000000..7fc5f5f --- /dev/null +++ b/deps/manager/npm/warm_test.go @@ -0,0 +1,69 @@ +package npm + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +func TestSteps(t *testing.T) { + cases := []struct { + name string + build bool + verif bool + want []string + }{ + { + name: "download suppresses lifecycle scripts", + want: []string{ + "manifest: write package.json", + "download: exec npm install --ignore-scripts", + }, + }, + { + // Compiling native addons is the point of building, so the suppression + // has to come off. + name: "build allows lifecycle scripts", + build: true, + want: []string{ + "manifest: write package.json", + "download: exec npm install", + }, + }, + { + // npm ci over npm install: the lockfile written by the download step + // makes it the stricter replay. + name: "verify replays from the lockfile offline", + verif: true, + want: []string{ + "manifest: write package.json", + "download: exec npm install --ignore-scripts", + "clean: remove node_modules", + "verify: exec npm ci --offline", + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: tc.build, Verify: tc.verif, + }, "") + if err != nil { + t.Fatal(err) + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(tc.want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, tc.want) + } + }) + } +} + +func TestWarmerIdentity(t *testing.T) { + if got := (Warmer{}).Manager(); got != manifest.ManagerNPM { + t.Errorf("Manager() = %q, want %q", got, manifest.ManagerNPM) + } + if probe := (Warmer{}).Probe(); probe != nil { + t.Errorf("Probe() = %v, want nil: npm needs no runtime version check", probe) + } +} diff --git a/deps/manager/pnpm/warm.go b/deps/manager/pnpm/warm.go new file mode 100644 index 0000000..4664642 --- /dev/null +++ b/deps/manager/pnpm/warm.go @@ -0,0 +1,70 @@ +// Package pnpm warms the pnpm content-addressable store for a single package. +package pnpm + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" + "github.com/flanksource/repomap/deps/manager/node" + "github.com/flanksource/repomap/deps/manifest" +) + +// allowAllBuilds opts every dependency into running its lifecycle scripts. pnpm +// 10.6 stopped honouring them by default — dropping --ignore-scripts is no longer +// enough — and older pnpm does not recognise the flag, hence the version gate. +const allowAllBuilds = "--config.dangerouslyAllowAllBuilds=true" + +const ( + allowAllBuildsMajor = 10 + allowAllBuildsMinor = 6 +) + +type Warmer struct{} + +func (Warmer) Manager() manifest.Manager { return manifest.ManagerPNPM } + +func (Warmer) Binary() string { return "pnpm" } + +// Probe reports the pnpm version, which decides how --build has to ask for +// lifecycle scripts. Dir is left for the orchestrator to fill in. +func (Warmer) Probe() *manifest.Command { + return &manifest.Command{Name: "pnpm", Args: []string{"--version"}} +} + +// NormalizeSpec returns the spec verbatim: an npm name is already the name the +// registry knows, and rewriting would only risk mangling a scoped @scope/pkg. +func (Warmer) NormalizeSpec(spec string) (string, error) { return spec, nil } + +func (Warmer) Steps(req manifest.WarmRequest, probe string) ([]manifest.Step, error) { + cmds := node.Commands{ + Binary: "pnpm", + Install: []string{"install"}, + IgnoreScripts: "--ignore-scripts", + Offline: []string{"install", "--offline", "--frozen-lockfile"}, + } + // The probe only matters for building; a plain warm must not depend on it. + if req.Build { + buildArgs, err := buildArgsFor(probe) + if err != nil { + return nil, err + } + cmds.BuildArgs = buildArgs + } + return node.Steps(req, cmds) +} + +func buildArgsFor(probe string) ([]string, error) { + if probe == "" { + return nil, fmt.Errorf("--build needs the pnpm version to decide how to enable dependency builds, but `pnpm --version` reported nothing") + } + version, err := semver.NewVersion(probe) + if err != nil { + return nil, fmt.Errorf("--build needs the pnpm version to decide how to enable dependency builds, but `pnpm --version` reported %q: %w", probe, err) + } + if version.Major() > allowAllBuildsMajor || + (version.Major() == allowAllBuildsMajor && version.Minor() >= allowAllBuildsMinor) { + return []string{allowAllBuilds}, nil + } + // Before 10.6, dropping --ignore-scripts is enough on its own. + return nil, nil +} diff --git a/deps/manager/pnpm/warm_test.go b/deps/manager/pnpm/warm_test.go new file mode 100644 index 0000000..2f95c20 --- /dev/null +++ b/deps/manager/pnpm/warm_test.go @@ -0,0 +1,118 @@ +package pnpm + +import ( + "strings" + "testing" + + "github.com/flanksource/repomap/deps/manifest" +) + +func TestSteps(t *testing.T) { + cases := []struct { + name string + build bool + verif bool + probe string + want []string + }{ + { + name: "download suppresses lifecycle scripts", probe: "10.7.0", + want: []string{ + "manifest: write package.json", + "download: exec pnpm install --ignore-scripts", + }, + }, + { + name: "verify replays offline against the frozen lockfile", verif: true, probe: "10.7.0", + want: []string{ + "manifest: write package.json", + "download: exec pnpm install --ignore-scripts", + "clean: remove node_modules", + "verify: exec pnpm install --offline --frozen-lockfile", + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: tc.build, Verify: tc.verif, + }, tc.probe) + if err != nil { + t.Fatal(err) + } + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(tc.want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, tc.want) + } + }) + } +} + +// pnpm 10.6 stopped running dependency lifecycle scripts even without +// --ignore-scripts, so on those versions building needs an explicit allowlist +// flag that older pnpm does not recognise. +func TestBuildFlagIsGatedOnTheProbedVersion(t *testing.T) { + cases := []struct { + version string + want string + }{ + {version: "9.12.0", want: "download: exec pnpm install"}, + {version: "10.5.9", want: "download: exec pnpm install"}, + {version: "10.6.0", want: "download: exec pnpm install --config.dangerouslyAllowAllBuilds=true"}, + {version: "10.7.1", want: "download: exec pnpm install --config.dangerouslyAllowAllBuilds=true"}, + {version: "11.0.0", want: "download: exec pnpm install --config.dangerouslyAllowAllBuilds=true"}, + } + for _, tc := range cases { + t.Run(tc.version, func(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: true, + }, tc.version) + if err != nil { + t.Fatal(err) + } + got := manifest.FormatSteps(steps) + if len(got) != 2 || got[1] != tc.want { + t.Fatalf("pnpm %s download step = %q, want %q", tc.version, got[len(got)-1], tc.want) + } + }) + } +} + +// Guessing the flag set would either skip the builds the user asked for or pass +// an argument older pnpm rejects, so an unreadable probe is a hard failure. +func TestBuildRejectsAnUnreadableProbe(t *testing.T) { + for _, probe := range []string{"", "not-a-version"} { + if _, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", Build: true, + }, probe); err == nil { + t.Errorf("probe %q: expected an error when --build cannot determine the pnpm version", probe) + } + } +} + +// Without --build the version is irrelevant, so a missing probe must not block a +// plain warm. +func TestDownloadIgnoresTheProbe(t *testing.T) { + steps, err := (Warmer{}).Steps(manifest.WarmRequest{ + Dir: "/scratch", Name: "left-pad", Version: "1.3.0", + }, "") + if err != nil { + t.Fatal(err) + } + want := []string{"manifest: write package.json", "download: exec pnpm install --ignore-scripts"} + if got := manifest.FormatSteps(steps); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("steps mismatch\n got: %v\nwant: %v", got, want) + } +} + +func TestWarmerIdentity(t *testing.T) { + if got := (Warmer{}).Manager(); got != manifest.ManagerPNPM { + t.Errorf("Manager() = %q, want %q", got, manifest.ManagerPNPM) + } + probe := (Warmer{}).Probe() + if probe == nil { + t.Fatal("Probe() = nil, want pnpm --version: the build flag depends on it") + } + if got := probe.String(); got != "pnpm --version" { + t.Errorf("Probe() = %q, want %q", got, "pnpm --version") + } +} diff --git a/deps/manifest/command.go b/deps/manifest/command.go new file mode 100644 index 0000000..19f460c --- /dev/null +++ b/deps/manifest/command.go @@ -0,0 +1,46 @@ +package manifest + +import ( + "context" + "os" + "os/exec" + "strings" +) + +type Command struct { + Dir string + Name string + Args []string + Env []string +} + +// String renders the command as it would be typed, so failures can name the +// exact invocation to rerun. +func (c Command) String() string { + return strings.Join(append([]string{c.Name}, c.Args...), " ") +} + +type CommandResult struct { + Stdout string + Stderr string +} + +type CommandRunner interface { + Run(ctx context.Context, cmd Command) (CommandResult, error) +} + +type ExecRunner struct{} + +func (ExecRunner) Run(ctx context.Context, spec Command) (CommandResult, error) { + cmd := exec.CommandContext(ctx, spec.Name, spec.Args...) + cmd.Dir = spec.Dir + if len(spec.Env) > 0 { + cmd.Env = append(os.Environ(), spec.Env...) + } + out, err := cmd.Output() + result := CommandResult{Stdout: string(out)} + if exit, ok := err.(*exec.ExitError); ok { + result.Stderr = string(exit.Stderr) + } + return result, err +} diff --git a/deps/manifest/manager.go b/deps/manifest/manager.go new file mode 100644 index 0000000..edf4716 --- /dev/null +++ b/deps/manifest/manager.go @@ -0,0 +1,17 @@ +// Package manifest holds the dependency-manager taxonomy and the process +// execution seam shared between the deps orchestrator and the per-manager +// packages under deps/manager. It is a leaf: nothing here imports deps, so the +// manager packages can depend on it without creating an import cycle. +package manifest + +type Manager string + +const ( + ManagerGo Manager = "go" + ManagerMaven Manager = "maven" + ManagerGradle Manager = "gradle" + ManagerNPM Manager = "npm" + ManagerPNPM Manager = "pnpm" + ManagerImage Manager = "image" + ManagerHelm Manager = "helm" +) diff --git a/deps/manifest/warm.go b/deps/manifest/warm.go new file mode 100644 index 0000000..6e6df52 --- /dev/null +++ b/deps/manifest/warm.go @@ -0,0 +1,115 @@ +package manifest + +import ( + "fmt" + "strings" +) + +// SplitSpec splits "name@version". The split uses the last @ at a non-zero index +// so a scoped npm name such as @scope/pkg keeps its leading @. An omitted version +// becomes "latest" and the manager decides what that means; the concrete version +// is read back after warming. +func SplitSpec(spec string) (name, version string, err error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return "", "", fmt.Errorf("empty dependency spec: expected name@version") + } + at := strings.LastIndex(spec, "@") + if at <= 0 { + if spec == "@" { + return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) + } + return spec, "latest", nil + } + name, version = spec[:at], spec[at+1:] + if name == "" || version == "" { + return "", "", fmt.Errorf("invalid dependency spec %q: expected name@version", spec) + } + return name, version, nil +} + +// WarmRequest describes one dependency to warm into the machine's shared +// package cache. Dir is a scratch project the orchestrator has already created; +// a Warmer only decides what to run inside it. +type WarmRequest struct { + Dir string + Name string + Version string + Build bool + Verify bool +} + +type StepKind string + +const ( + StepExec StepKind = "exec" + StepWrite StepKind = "write" + StepRemove StepKind = "remove" +) + +// Step is one unit of warming work. Warming is not purely exec: node ecosystems +// need a package.json written before installing, and need node_modules removed +// before an offline replay can prove anything. +type Step struct { + Kind StepKind + Name string + Command Command // StepExec + Path string // StepWrite, StepRemove — relative to WarmRequest.Dir + Content []byte // StepWrite +} + +// Detail renders just the action, with no step name, for callers that already +// report the name separately. +func (s Step) Detail() string { + switch s.Kind { + case StepExec: + return s.Command.String() + case StepWrite: + return "write " + s.Path + case StepRemove: + return "remove " + s.Path + default: + return "unknown step kind " + string(s.Kind) + } +} + +// String renders the step as "name: kind detail", so a failure can name what was +// being attempted and tests can assert a whole sequence in one comparison. +func (s Step) String() string { + if s.Kind == StepExec { + return s.Name + ": exec " + s.Detail() + } + return s.Name + ": " + s.Detail() +} + +// FormatSteps renders a sequence one line per step. +func FormatSteps(steps []Step) []string { + out := make([]string, 0, len(steps)) + for _, step := range steps { + out = append(out, step.String()) + } + return out +} + +// Warmer decides which commands warm one ecosystem. Implementations do no I/O: +// Steps is a pure function of its arguments, which is what lets the per-manager +// packages be tested by comparing argv without a toolchain, a network, or a +// process. +// +// Probe is the escape hatch for a manager that genuinely needs runtime +// information (pnpm's lifecycle-script policy changed in 10.6). The orchestrator +// runs it and feeds the trimmed stdout back into Steps as an ordinary input, +// keeping Steps pure. +type Warmer interface { + Manager() Manager + // Binary is the executable that must be on PATH, so the orchestrator can + // fail before it creates a scratch directory. + Binary() string + Probe() *Command + // NormalizeSpec canonicalises a user-supplied spec before it is split into + // name and version, so a manager can accept the shorthand its ecosystem's + // users actually type. Managers that take the spec verbatim return it + // unchanged. + NormalizeSpec(spec string) (string, error) + Steps(req WarmRequest, probe string) ([]Step, error) +} diff --git a/deps/manifest/warm_test.go b/deps/manifest/warm_test.go new file mode 100644 index 0000000..21bb10b --- /dev/null +++ b/deps/manifest/warm_test.go @@ -0,0 +1,47 @@ +package manifest + +import "testing" + +func TestSplitSpec(t *testing.T) { + cases := []struct { + spec string + wantName string + wantVersion string + wantErr bool + }{ + {spec: "github.com/acme/lib@v1.2.3", wantName: "github.com/acme/lib", wantVersion: "v1.2.3"}, + // No version means "whatever the manager considers current"; the concrete + // version is read back after warming. + {spec: "github.com/acme/lib", wantName: "github.com/acme/lib", wantVersion: "latest"}, + {spec: "left-pad@1.3.0", wantName: "left-pad", wantVersion: "1.3.0"}, + {spec: "left-pad@^1.3.0", wantName: "left-pad", wantVersion: "^1.3.0"}, + {spec: "left-pad@latest", wantName: "left-pad", wantVersion: "latest"}, + // A scoped npm name leads with @, so splitting must use the last @ and + // ignore one at index 0. + {spec: "@scope/pkg@1.0.0", wantName: "@scope/pkg", wantVersion: "1.0.0"}, + {spec: "@scope/pkg", wantName: "@scope/pkg", wantVersion: "latest"}, + // A Go branch or commit reference must survive untouched. + {spec: "github.com/acme/lib@main", wantName: "github.com/acme/lib", wantVersion: "main"}, + {spec: "", wantErr: true}, + {spec: " ", wantErr: true}, + {spec: "left-pad@", wantErr: true}, + {spec: "@", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.spec, func(t *testing.T) { + name, version, err := SplitSpec(tc.spec) + if tc.wantErr { + if err == nil { + t.Fatalf("SplitSpec(%q) = (%q, %q), want an error", tc.spec, name, version) + } + return + } + if err != nil { + t.Fatal(err) + } + if name != tc.wantName || version != tc.wantVersion { + t.Fatalf("SplitSpec(%q) = (%q, %q), want (%q, %q)", tc.spec, name, version, tc.wantName, tc.wantVersion) + } + }) + } +} diff --git a/deps/model.go b/deps/model.go index 02f1a20..87d8e50 100644 --- a/deps/model.go +++ b/deps/model.go @@ -1,17 +1,25 @@ package deps -import "time" +import ( + "time" -type Manager string + "github.com/flanksource/repomap/deps/manifest" +) + +// Manager is defined in deps/manifest so the per-manager packages under +// deps/manager can name it without importing deps. It stays an alias here, and +// the constants stay re-declared, so every existing switch and caller is +// unaffected; the underlying type is still string, so JSON is unchanged. +type Manager = manifest.Manager const ( - ManagerGo Manager = "go" - ManagerMaven Manager = "maven" - ManagerGradle Manager = "gradle" - ManagerNPM Manager = "npm" - ManagerPNPM Manager = "pnpm" - ManagerImage Manager = "image" - ManagerHelm Manager = "helm" + ManagerGo = manifest.ManagerGo + ManagerMaven = manifest.ManagerMaven + ManagerGradle = manifest.ManagerGradle + ManagerNPM = manifest.ManagerNPM + ManagerPNPM = manifest.ManagerPNPM + ManagerImage = manifest.ManagerImage + ManagerHelm = manifest.ManagerHelm ) type Mode string @@ -27,8 +35,21 @@ type Options struct { Filters []string Flat bool IncludeIndirect bool - Runner CommandRunner - Now func() time.Time + ShowDuplicates bool + + // Resource filters for image/helm targets discovered from Kubernetes + // manifests (ignored by package managers and chart-directory scanning, which + // have no Kubernetes resource metadata to match against). + Kind []string + Namespace []string + Name []string + Selector []string + + Runner CommandRunner + Now func() time.Time + // remote injects a pre-built remote resolver (cache + chart/image resolvers) + // for tests; nil in production, where Scan builds a disk-backed one. + remote *remoteDeps } type Project struct { @@ -57,6 +78,7 @@ type Metadata struct { Filter []string `json:"filter,omitempty"` MaxDepth int `json:"max_depth,omitempty"` Flat bool `json:"flat,omitempty"` + ShowDuplicates bool `json:"show_duplicates,omitempty"` ProjectsScanned int `json:"projects_scanned"` } diff --git a/deps/pretty_tree.go b/deps/pretty_tree.go index 2a346a9..698d149 100644 --- a/deps/pretty_tree.go +++ b/deps/pretty_tree.go @@ -35,8 +35,8 @@ func dependencyDisplayTree(roots []*Node, scanPath string) (api.TextTree, bool) if root == nil { continue } - switch root.Manager { - case ManagerImage, ManagerHelm: + switch { + case (root.Manager == ManagerImage || root.Manager == ManagerHelm) && root.Source == "kubernetes manifests": k8sLeaves = append(k8sLeaves, root.Children...) default: top = append(top, rootDisplayNode(root, scanPath)) @@ -57,18 +57,21 @@ func rootDisplayNode(root *Node, scanPath string) *displayNode { if rel := relDisplayPath(scanPath, root.Path); rel != "" { label = label.Space().Append(rel, "font-mono text-muted") } - return &displayNode{text: label, children: packageDisplayNodes(root.Children)} + // Chart roots mix helm subcharts with image dependencies, so their children + // keep a manager prefix; single-manager package roots omit it. + showChildManager := root.Source == "Chart.yaml" + return &displayNode{text: label, children: packageChildNodes(root, showChildManager)} } -// packageDisplayNodes renders dependency children without repeating the manager -// prefix (the root already carries it). -func packageDisplayNodes(nodes []*Node) []api.TreeNode { - sorted := sortedNodes(nodes) +// packageChildNodes renders dependency children. showManager keeps the manager +// prefix on children of roots that mix managers. +func packageChildNodes(parent *Node, showManager bool) []api.TreeNode { + sorted := sortedNodes(parent.Children) out := make([]api.TreeNode, 0, len(sorted)) for _, n := range sorted { out = append(out, &displayNode{ - text: nodeText(n, false), - children: packageDisplayNodes(n.Children), + text: nodeText(n, showManager), + children: packageChildNodes(n, showManager), }) } return out diff --git a/deps/pretty_tree_test.go b/deps/pretty_tree_test.go index f7f2f09..1925d99 100644 --- a/deps/pretty_tree_test.go +++ b/deps/pretty_tree_test.go @@ -87,10 +87,61 @@ func TestTreeDropsDefaultScopeAndDirectTags(t *testing.T) { } } +func TestTreeCollapsesSilently(t *testing.T) { + root := diamondRoot() + root.Path = "/abs/proj/go.mod" + collapseDuplicates([]*Node{root}) + out := (&Export{Metadata: Metadata{Path: "/abs/proj"}, Roots: []*Node{root}}).Pretty().String() + + var occurrences int + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "github.com/acme/shared@v1.0.0") { + occurrences++ + } + } + if occurrences != 1 { + t.Fatalf("collapsed tree should render shared exactly once, got %d:\n%s", occurrences, out) + } + if strings.Contains(out, "parent") { + t.Fatalf("collapsed tree must not carry parent-count markers:\n%s", out) + } + if strings.Contains(out, "other dependenc") { + t.Fatalf("collapsed tree must not carry hidden-duplicate markers:\n%s", out) + } +} + +func TestTreeShowDuplicatesKeepsEveryOccurrence(t *testing.T) { + root := diamondRoot() + root.Path = "/abs/proj/go.mod" + dups := analyzeDuplicates([]*Node{root}) + applyDuplicateRefs([]*Node{root}, dups) + out := (&Export{Metadata: Metadata{Path: "/abs/proj"}, Roots: []*Node{root}}).Pretty().String() + + var occurrences int + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "github.com/acme/shared@v1.0.0") { + occurrences++ + } + } + if occurrences != 3 { + t.Fatalf("--show-duplicates equivalent should render shared 3 times, got %d:\n%s", occurrences, out) + } + if strings.Contains(out, "+2 parents") { + t.Fatalf("uncollapsed tree must not carry parent-count markers:\n%s", out) + } + if strings.Contains(out, "other dependenc") { + t.Fatalf("uncollapsed tree must not carry hidden-duplicate markers:\n%s", out) + } + if !strings.Contains(out, "dup:3") { + t.Fatalf("uncollapsed tree should keep dup:N tags:\n%s", out) + } +} + func imageTreeExport() *Export { root := NewNode(ManagerImage, "container images", "") root.Depth = 0 root.Path = "/abs/proj" + root.Source = "kubernetes manifests" nginx := NewNode(ManagerImage, "nginx", "1.25.3") nginx.Depth = 1 diff --git a/deps/remote_cache.go b/deps/remote_cache.go new file mode 100644 index 0000000..d097394 --- /dev/null +++ b/deps/remote_cache.go @@ -0,0 +1,267 @@ +package deps + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sync/singleflight" +) + +// Differentiated TTLs (decision 3). Immutable-per-ref artifacts (git clones at a +// ref, chart .tgz, image config by digest) are kept long; mutable lookups +// (helm index.yaml, image tag→config) refresh daily; negatives retry soon. +const ( + ttlImmutable = 365 * 24 * time.Hour + ttlIndex = 24 * time.Hour + ttlNegative = 1 * time.Hour +) + +// ImageConfig is the resolved OCI config metadata for an image reference. +type ImageConfig struct { + Digest string + Labels map[string]string +} + +// notFoundError marks a definitively-absent remote resource (404, missing repo) +// so it can be negatively cached and degraded to a warning, distinct from a +// transient network failure which is never cached. +type notFoundError struct{ what string } + +func (e notFoundError) Error() string { return e.what + ": not found" } + +func isNotFound(err error) bool { + var nf notFoundError + return errors.As(err, &nf) +} + +// labelResolver reads an image's config digest and OCI labels from a registry. +// Production is backed by imageupdate; tests inject a fake. +type labelResolver interface { + Labels(ctx context.Context, ref string) (digest string, labels map[string]string, err error) +} + +// RemoteCache provides cached access to the remote artifacts recursion needs: +// arbitrary HTTP blobs (helm index/chart), git checkouts, and image configs. +type RemoteCache interface { + Fetch(ctx context.Context, url string, ttl time.Duration) ([]byte, error) + GitRepo(ctx context.Context, url, ref string) (dir string, err error) + ImageConfig(ctx context.Context, ref string) (ImageConfig, error) +} + +type diskCache struct { + root string + now func() time.Time + get func(ctx context.Context, url string) ([]byte, error) + runner CommandRunner + labels labelResolver + helmAuth *helmCredentials + group singleflight.Group +} + +// newDiskCache builds the production cache rooted at os.UserCacheDir()/repomap. +func newDiskCache(now func() time.Time, runner CommandRunner, labels labelResolver) (*diskCache, error) { + base, err := os.UserCacheDir() + if err != nil { + return nil, fmt.Errorf("locating user cache dir: %w", err) + } + root := filepath.Join(base, "repomap", "deps") + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, fmt.Errorf("creating cache dir %s: %w", root, err) + } + c := &diskCache{root: root, now: now, runner: runner, labels: labels, helmAuth: loadHelmCredentials()} + c.get = c.httpGet + return c, nil +} + +type cacheEntry[T any] struct { + FetchedAt time.Time + NotFound bool + Value T +} + +func (c *diskCache) Fetch(ctx context.Context, url string, ttl time.Duration) ([]byte, error) { + v, err, _ := c.group.Do("blob:"+url, func() (any, error) { + path := c.entryPath("blobs", url) + if e, ok := readEntry[[]byte](path); ok && !c.expiredAt(e.FetchedAt, e.NotFound, ttl) { + if e.NotFound { + return nil, notFoundError{url} + } + return e.Value, nil + } + data, err := c.get(ctx, url) + if isNotFound(err) { + writeEntry(path, cacheEntry[[]byte]{FetchedAt: c.now(), NotFound: true}) + return nil, notFoundError{url} + } + if err != nil { + return nil, err // transient: do not cache + } + writeEntry(path, cacheEntry[[]byte]{FetchedAt: c.now(), Value: data}) + return data, nil + }) + if err != nil { + return nil, err + } + return v.([]byte), nil +} + +func (c *diskCache) ImageConfig(ctx context.Context, ref string) (ImageConfig, error) { + v, err, _ := c.group.Do("img:"+ref, func() (any, error) { + path := c.entryPath("imageconfig", ref) + if e, ok := readEntry[ImageConfig](path); ok && !c.expiredAt(e.FetchedAt, e.NotFound, ttlIndex) { + if e.NotFound { + return ImageConfig{}, notFoundError{ref} + } + return e.Value, nil + } + digest, labels, err := c.labels.Labels(ctx, ref) + if isNotFound(err) { + writeEntry(path, cacheEntry[ImageConfig]{FetchedAt: c.now(), NotFound: true}) + return ImageConfig{}, notFoundError{ref} + } + if err != nil { + return ImageConfig{}, err + } + cfg := ImageConfig{Digest: digest, Labels: labels} + writeEntry(path, cacheEntry[ImageConfig]{FetchedAt: c.now(), Value: cfg}) + return cfg, nil + }) + if err != nil { + return ImageConfig{}, err + } + return v.(ImageConfig), nil +} + +// GitRepo clones url at ref into the cache (kept for ttlImmutable) and returns +// the checkout directory. A ref that is not a branch/tag falls back to the +// default branch. +func (c *diskCache) GitRepo(ctx context.Context, url, ref string) (string, error) { + v, err, _ := c.group.Do("git:"+url+"@"+ref, func() (any, error) { + dir := filepath.Join(c.root, "git", hashKey(url), sanitizeRef(ref)) + marker := filepath.Join(dir, ".repomap-fetched") + if data, err := os.ReadFile(marker); err == nil { + if at, perr := time.Parse(time.RFC3339, strings.TrimSpace(string(data))); perr == nil && + c.now().Sub(at) < ttlImmutable { + return dir, nil + } + } + if err := c.cloneInto(ctx, url, ref, dir); err != nil { + return "", err + } + _ = os.WriteFile(marker, []byte(c.now().Format(time.RFC3339)), 0o644) + return dir, nil + }) + if err != nil { + return "", err + } + return v.(string), nil +} + +func (c *diskCache) cloneInto(ctx context.Context, url, ref, dir string) error { + tmp, err := os.MkdirTemp(c.root, "clone-*") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(tmp) }() + + args := []string{"clone", "--depth", "1", "--single-branch", url, tmp} + if ref != "" { + args = []string{"clone", "--depth", "1", "--single-branch", "--branch", ref, url, tmp} + } + if _, err := c.runner.Run(ctx, Command{Name: "git", Args: args}); err != nil { + // Ref may not be a branch/tag; fall back to the default branch. + if ref == "" { + return notFoundError{url} + } + if _, err2 := c.runner.Run(ctx, Command{Name: "git", Args: []string{"clone", "--depth", "1", url, tmp}}); err2 != nil { + return notFoundError{url} + } + } + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return err + } + _ = os.RemoveAll(dir) + return os.Rename(tmp, dir) +} + +func (c *diskCache) httpGet(ctx context.Context, url string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + // Reuse the user's `helm repo add` login (basic auth / TLS) for matching repos. + client := c.helmAuth.authorize(req) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusNotFound { + return nil, notFoundError{url} + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("GET %s: status %d", url, resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +func (c *diskCache) expiredAt(at time.Time, notFound bool, ttl time.Duration) bool { + if notFound { + ttl = ttlNegative + } + return c.now().Sub(at) >= ttl +} + +func (c *diskCache) entryPath(kind, key string) string { + return filepath.Join(c.root, kind, hashKey(key)+".gob") +} + +func hashKey(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func sanitizeRef(ref string) string { + if ref == "" { + return "_default" + } + return strings.NewReplacer("/", "_", ":", "_", " ", "_").Replace(ref) +} + +func writeEntry[T any](path string, e cacheEntry[T]) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return + } + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(e); err != nil { + return + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, buf.Bytes(), 0o644); err != nil { + return + } + _ = os.Rename(tmp, path) +} + +func readEntry[T any](path string) (cacheEntry[T], bool) { + data, err := os.ReadFile(path) + if err != nil { + return cacheEntry[T]{}, false + } + var e cacheEntry[T] + if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&e); err != nil { + return cacheEntry[T]{}, false + } + return e, true +} diff --git a/deps/remote_cache_test.go b/deps/remote_cache_test.go new file mode 100644 index 0000000..e468d07 --- /dev/null +++ b/deps/remote_cache_test.go @@ -0,0 +1,151 @@ +package deps + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +func testClock(t time.Time) (func() time.Time, *time.Time) { + cur := t + return func() time.Time { return cur }, &cur +} + +func newTestCache(t *testing.T, now func() time.Time) *diskCache { + t.Helper() + return &diskCache{root: t.TempDir(), now: now} +} + +func TestCacheFetchHitAndTTL(t *testing.T) { + now, clock := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + return []byte("payload"), nil + } + + for i := 0; i < 3; i++ { + if _, err := c.Fetch(context.Background(), "http://x/index.yaml", ttlIndex); err != nil { + t.Fatal(err) + } + } + if calls != 1 { + t.Fatalf("expected 1 remote fetch within TTL, got %d", calls) + } + + *clock = clock.Add(ttlIndex + time.Minute) // expire + if _, err := c.Fetch(context.Background(), "http://x/index.yaml", ttlIndex); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("expected re-fetch after TTL expiry, got %d calls", calls) + } +} + +func TestCacheNegativeCaching(t *testing.T) { + now, clock := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + return nil, notFoundError{url} + } + + if _, err := c.Fetch(context.Background(), "http://x/missing", ttlImmutable); !isNotFound(err) { + t.Fatalf("want notFound, got %v", err) + } + if _, err := c.Fetch(context.Background(), "http://x/missing", ttlImmutable); !isNotFound(err) { + t.Fatalf("want cached notFound, got %v", err) + } + if calls != 1 { + t.Fatalf("negative result should be cached (1 call), got %d", calls) + } + + *clock = clock.Add(ttlNegative + time.Minute) // negatives expire on the short TTL + if _, err := c.Fetch(context.Background(), "http://x/missing", ttlImmutable); !isNotFound(err) { + t.Fatalf("want notFound after negative expiry, got %v", err) + } + if calls != 2 { + t.Fatalf("negative entry should retry after ttlNegative, got %d calls", calls) + } +} + +func TestCacheTransientErrorNotCached(t *testing.T) { + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + return nil, context.DeadlineExceeded + } + for i := 0; i < 2; i++ { + if _, err := c.Fetch(context.Background(), "http://x/flaky", ttlIndex); err == nil { + t.Fatal("expected transient error") + } + } + if calls != 2 { + t.Fatalf("transient errors must not be cached, want 2 calls got %d", calls) + } +} + +func TestCacheSingleflightCollapsesConcurrentFetches(t *testing.T) { + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + release := make(chan struct{}) + c.get = func(_ context.Context, url string) ([]byte, error) { + atomic.AddInt64(&calls, 1) + <-release // hold so all goroutines pile onto the same in-flight call + return []byte("v"), nil + } + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = c.Fetch(context.Background(), "http://x/same", ttlIndex) + }() + } + time.Sleep(20 * time.Millisecond) + close(release) + wg.Wait() + if calls != 1 { + t.Fatalf("singleflight should collapse concurrent fetches to 1, got %d", calls) + } +} + +func TestCacheImageConfig(t *testing.T) { + now, _ := testClock(time.Unix(1000, 0)) + c := newTestCache(t, now) + var calls int64 + c.labels = fakeLabels{fn: func(ref string) (string, map[string]string, error) { + atomic.AddInt64(&calls, 1) + return "sha256:deadbeef", map[string]string{"org.opencontainers.image.source": "https://github.com/acme/app"}, nil + }} + + cfg, err := c.ImageConfig(context.Background(), "ghcr.io/acme/app:1.0") + if err != nil { + t.Fatal(err) + } + if cfg.Digest != "sha256:deadbeef" || cfg.Labels["org.opencontainers.image.source"] != "https://github.com/acme/app" { + t.Fatalf("unexpected config %#v", cfg) + } + if _, err := c.ImageConfig(context.Background(), "ghcr.io/acme/app:1.0"); err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("image config should be cached, got %d calls", calls) + } +} + +type fakeLabels struct { + fn func(ref string) (string, map[string]string, error) +} + +func (f fakeLabels) Labels(_ context.Context, ref string) (string, map[string]string, error) { + return f.fn(ref) +} diff --git a/deps/remote_helm_client.go b/deps/remote_helm_client.go new file mode 100644 index 0000000..c2f26bf --- /dev/null +++ b/deps/remote_helm_client.go @@ -0,0 +1,208 @@ +package deps + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/goccy/go-yaml" + "helm.sh/helm/v3/pkg/chart/loader" +) + +// helmClient resolves Helm chart dependencies remotely. It uses the Helm SDK's +// chart loader (helm.sh/helm/v3/pkg/chart/loader) to parse fetched charts, and +// parses the repository index.yaml directly over HTTP — the SDK's pkg/repo is +// avoided because it transitively requires a k8s API version removed in the +// k8s release this module pins, and pulls oras/containerd. +// +// Pin note: helm.sh/helm/v3 is held at v3.17.x; v3.18+ bumps +// distribution/distribution/v3 past the version registry-scanner requires. +type helmClient struct { + cache RemoteCache +} + +type helmIndex struct { + Entries map[string][]helmIndexEntry `yaml:"entries"` +} + +type helmIndexEntry struct { + Version string `yaml:"version"` + URLs []string `yaml:"urls"` +} + +type fetchedChart struct { + Dir string + Dependencies []chartDepEntry + Version string +} + +// fetchChart resolves a Chart.yaml dependency to an extracted chart directory +// plus its own declared dependencies. The returned cleanup removes the +// extraction dir and must be called by the caller. +func (h *helmClient) fetchChart(ctx context.Context, dep chartDepEntry) (*fetchedChart, func(), error) { + repoURL := strings.TrimSuffix(dep.Repository, "/") + if repoURL == "" { + return nil, nil, notFoundError{dep.Name + " (no repository)"} + } + if strings.HasPrefix(repoURL, "oci://") { + return nil, nil, fmt.Errorf("oci helm repositories are not yet supported") + } + + version, tgzURL, err := h.resolveChart(ctx, repoURL, dep.Name, dep.Version) + if err != nil { + return nil, nil, err + } + tgz, err := h.cache.Fetch(ctx, tgzURL, ttlImmutable) + if err != nil { + return nil, nil, err + } + + dir, err := os.MkdirTemp("", "repomap-chart-*") + if err != nil { + return nil, nil, err + } + cleanup := func() { _ = os.RemoveAll(dir) } + if err := extractTarGz(tgz, dir); err != nil { + cleanup() + return nil, nil, err + } + chartDir, err := findChartDir(dir) + if err != nil { + cleanup() + return nil, nil, err + } + ch, err := loader.Load(chartDir) + if err != nil { + cleanup() + return nil, nil, fmt.Errorf("loading chart %s: %w", dep.Name, err) + } + + deps := make([]chartDepEntry, 0, len(ch.Metadata.Dependencies)) + for _, d := range ch.Metadata.Dependencies { + deps = append(deps, chartDepEntry{Name: d.Name, Version: d.Version, Repository: d.Repository}) + } + return &fetchedChart{Dir: chartDir, Dependencies: deps, Version: version}, cleanup, nil +} + +// resolveChart picks the newest index version satisfying the dependency's +// version constraint and returns its absolute .tgz URL. +func (h *helmClient) resolveChart(ctx context.Context, repoURL, name, constraint string) (version, tgzURL string, err error) { + data, err := h.cache.Fetch(ctx, repoURL+"/index.yaml", ttlIndex) + if err != nil { + return "", "", err + } + var idx helmIndex + if err := yaml.Unmarshal(data, &idx); err != nil { + return "", "", fmt.Errorf("parsing index.yaml for %s: %w", repoURL, err) + } + entries := idx.Entries[name] + if len(entries) == 0 { + return "", "", notFoundError{name + " in " + repoURL} + } + version, url := pickChartVersion(entries, constraint) + if version == "" { + return "", "", notFoundError{name + "@" + constraint} + } + if strings.HasPrefix(url, "oci://") { + return "", "", fmt.Errorf("chart %s@%s is published as an OCI artifact (%s), which is not yet supported", name, version, url) + } + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + url = repoURL + "/" + strings.TrimPrefix(url, "/") + } + return version, url, nil +} + +// pickChartVersion selects the highest entry satisfying constraint (a SemVer +// range). If constraint is not a valid range it falls back to an exact match. +func pickChartVersion(entries []helmIndexEntry, constraint string) (version, url string) { + cons, consErr := semver.NewConstraint(constraint) + var best *semver.Version + for _, e := range entries { + if consErr != nil { + if e.Version == constraint && len(e.URLs) > 0 { + return e.Version, e.URLs[0] + } + continue + } + v, err := semver.NewVersion(e.Version) + if err != nil || !cons.Check(v) || len(e.URLs) == 0 { + continue + } + if best == nil || v.GreaterThan(best) { + best, version, url = v, e.Version, e.URLs[0] + } + } + return version, url +} + +// extractTarGz unpacks a gzipped tar archive into dest, guarding against path +// traversal. +func extractTarGz(data []byte, dest string) error { + gz, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return err + } + defer func() { _ = gz.Close() }() + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + target := filepath.Join(dest, filepath.Clean(hdr.Name)) + if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("tar entry escapes destination: %s", hdr.Name) + } + if hdr.FileInfo().IsDir() { + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.Create(target) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { //nolint:gosec // chart archives are size-bounded + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + } +} + +// findChartDir returns the directory containing Chart.yaml within an extracted +// chart archive (charts unpack to a single top-level dir named for the chart). +func findChartDir(root string) (string, error) { + if _, err := os.Stat(filepath.Join(root, "Chart.yaml")); err == nil { + return root, nil + } + entries, err := os.ReadDir(root) + if err != nil { + return "", err + } + for _, e := range entries { + if e.IsDir() { + candidate := filepath.Join(root, e.Name()) + if _, err := os.Stat(filepath.Join(candidate, "Chart.yaml")); err == nil { + return candidate, nil + } + } + } + return "", fmt.Errorf("no Chart.yaml found under %s", root) +} diff --git a/deps/remote_helm_client_test.go b/deps/remote_helm_client_test.go new file mode 100644 index 0000000..009e689 --- /dev/null +++ b/deps/remote_helm_client_test.go @@ -0,0 +1,42 @@ +package deps + +import ( + "context" + "strings" + "testing" +) + +func TestPickChartVersion(t *testing.T) { + entries := []helmIndexEntry{ + {Version: "16.1.0", URLs: []string{"u-16.1.0"}}, + {Version: "16.7.27", URLs: []string{"u-16.7.27"}}, + {Version: "17.0.0", URLs: []string{"u-17.0.0"}}, + } + cases := []struct{ constraint, wantVer, wantURL string }{ + {"16.x.x", "16.7.27", "u-16.7.27"}, // newest satisfying the range + {"16.7.27", "16.7.27", "u-16.7.27"}, // exact + {">=17.0.0", "17.0.0", "u-17.0.0"}, + {"99.x", "", ""}, // nothing satisfies + } + for _, tc := range cases { + ver, url := pickChartVersion(entries, tc.constraint) + if ver != tc.wantVer || url != tc.wantURL { + t.Fatalf("pickChartVersion(%q) = (%q,%q), want (%q,%q)", tc.constraint, ver, url, tc.wantVer, tc.wantURL) + } + } +} + +func TestFetchChartOCIDegrades(t *testing.T) { + const repo = "https://charts.bitnami.com/bitnami" + cache := &memCache{blobs: map[string][]byte{ + repo + "/index.yaml": []byte("entries:\n postgresql:\n - version: 16.7.27\n urls:\n - oci://registry-1.docker.io/bitnamicharts/postgresql:16.7.27\n"), + }} + h := &helmClient{cache: cache} + _, _, err := h.fetchChart(context.Background(), chartDepEntry{Name: "postgresql", Version: "16.x.x", Repository: repo}) + if err == nil { + t.Fatal("expected OCI chart to degrade with an error") + } + if got := err.Error(); !strings.Contains(got, "OCI artifact") { + t.Fatalf("error should explain the OCI limitation, got %q", got) + } +} diff --git a/deps/resolve_remote.go b/deps/resolve_remote.go new file mode 100644 index 0000000..4808272 --- /dev/null +++ b/deps/resolve_remote.go @@ -0,0 +1,168 @@ +package deps + +import ( + "context" + "time" + + "github.com/flanksource/repomap/imageupdate" +) + +// defaultRemoteDepth bounds recursive fetching when MaxDepth is 0 (unlimited), +// so the network walk always terminates. +const defaultRemoteDepth = 10 + +// remoteDeps bundles the remote cache-backed resolvers used to recurse into +// chart subcharts and image base images. +type remoteDeps struct { + charts *chartResolver + images *imageResolver +} + +func remoteDepsFromCache(cache RemoteCache) *remoteDeps { + return &remoteDeps{ + charts: &chartResolver{helm: &helmClient{cache: cache}}, + images: newImageResolver(cache), + } +} + +func newRemoteDeps(opts Options) (*remoteDeps, error) { + if opts.remote != nil { + return opts.remote, nil + } + now := time.Now + if opts.Now != nil { + now = opts.Now + } + runner := opts.Runner + if runner == nil { + runner = ExecRunner{} + } + cache, err := newDiskCache(now, runner, imageupdate.NewLabelResolver()) + if err != nil { + return nil, err + } + return remoteDepsFromCache(cache), nil +} + +// resolveRemote recursively expands chart-subchart and image-base children in +// place. It is gated by the caller to MaxDepth != 1. Remote failures degrade to +// warnings; only cache initialization can fail hard. +func resolveRemote(ctx context.Context, roots []*Node, opts Options) ([]Warning, error) { + rd, err := newRemoteDeps(opts) + if err != nil { + return nil, err + } + limit := opts.MaxDepth + if limit == 0 { + limit = defaultRemoteDepth + } + visited := map[string]bool{} + var warnings []Warning + for _, root := range roots { + warnings = append(warnings, rd.expandNode(ctx, root, limit, visited)...) + } + return warnings, nil +} + +// expandNode fetches a node's remote children (if it is a fetchable chart or +// image and within depth/visited bounds), then recurses into all children +// (existing offline ones plus the newly fetched). +func (rd *remoteDeps) expandNode(ctx context.Context, node *Node, limit int, visited map[string]bool) []Warning { + if node == nil { + return nil + } + var warnings []Warning + if node.Depth < limit { + if key := remoteKey(node); key != "" && !visited[key] { + visited[key] = true + warnings = append(warnings, rd.fetchChildren(ctx, node)...) + } + } + for _, child := range node.Children { + warnings = append(warnings, rd.expandNode(ctx, child, limit, visited)...) + } + return warnings +} + +func (rd *remoteDeps) fetchChildren(ctx context.Context, node *Node) []Warning { + switch { + case isFetchableChart(node): + children, version, warns := rd.charts.expand(ctx, chartDepEntry{ + Name: node.Name, + Version: node.Version, + Repository: node.Source, + }, node.Depth) + if version != "" { + node.Version = version + node.ID = NodeID(ManagerHelm, node.Name, version) + } + node.Children = append(node.Children, children...) + return nodeWarnings(node, warns) + case isFetchableImage(node): + bases, warns := rd.images.baseImages(ctx, imageRefOf(node)) + for _, b := range bases { + node.Children = append(node.Children, baseImageNode(b, node.Depth+1)) + } + return nodeWarnings(node, warns) + } + return nil +} + +// isFetchableChart matches a Helm dependency that can be resolved from a remote +// repository — excludes the local Chart.yaml root and the synthetic k8s wrapper. +func isFetchableChart(n *Node) bool { + return n.Manager == ManagerHelm && n.Source != "" && + n.Source != "Chart.yaml" && n.Source != "kubernetes manifests" +} + +// isFetchableImage matches a real image node — excludes the synthetic +// "container images" k8s wrapper root. +func isFetchableImage(n *Node) bool { + return n.Manager == ManagerImage && n.Source != "" && n.Source != "kubernetes manifests" +} + +func remoteKey(n *Node) string { + switch { + case isFetchableChart(n): + return "helm:" + n.Name + "@" + n.Version + case isFetchableImage(n): + return "image:" + imageRefOf(n) + } + return "" +} + +// imageRefOf returns the original image reference to resolve; Source holds the +// full ref captured at discovery (e.g. "ghcr.io/acme/app:1.2.3"). +func imageRefOf(n *Node) string { + if n.Source != "" && n.Source != "kubernetes manifests" { + return n.Source + } + if n.Version != "" { + return n.Name + ":" + n.Version + } + return n.Name +} + +func baseImageNode(b imageRef, depth int) *Node { + ref := b.Name + if b.Version != "" { + ref += ":" + b.Version + } + if b.Digest != "" { + ref += "@" + b.Digest + } + n := NewNode(ManagerImage, b.Name, b.Version) + n.Depth = depth + n.Direct = false + n.Scope = "base" + n.Source = ref // makes the base itself fetchable (base-of-base recursion) + return n +} + +func nodeWarnings(node *Node, msgs []string) []Warning { + out := make([]Warning, 0, len(msgs)) + for _, m := range msgs { + out = append(out, Warning{Manager: node.Manager, Project: node.Name, Message: m}) + } + return out +} diff --git a/deps/resolve_remote_test.go b/deps/resolve_remote_test.go new file mode 100644 index 0000000..9eb47e8 --- /dev/null +++ b/deps/resolve_remote_test.go @@ -0,0 +1,196 @@ +package deps + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + "time" +) + +// memCache is an in-memory RemoteCache for recursion tests: no network, no disk +// reads beyond the git fixture dirs the test writes. +type memCache struct { + blobs map[string][]byte + images map[string]ImageConfig + gitDirs map[string]string // keyed by url, any ref +} + +func (m *memCache) Fetch(_ context.Context, url string, _ time.Duration) ([]byte, error) { + if b, ok := m.blobs[url]; ok { + return b, nil + } + return nil, notFoundError{url} +} + +func (m *memCache) GitRepo(_ context.Context, url, _ string) (string, error) { + if d, ok := m.gitDirs[url]; ok { + return d, nil + } + return "", notFoundError{url} +} + +func (m *memCache) ImageConfig(_ context.Context, ref string) (ImageConfig, error) { + if c, ok := m.images[ref]; ok { + return c, nil + } + return ImageConfig{}, notFoundError{ref} +} + +func buildChartTgz(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, content := range files { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// chartRootWithSubchart builds an offline chart root carrying a single subchart +// leaf, as the offline scan would produce it. +func chartRootWithSubchart(name, version, repo string) *Node { + root := NewNode(ManagerHelm, "parent", "1.0.0") + root.Source = "Chart.yaml" + root.Depth = 0 + sub := NewNode(ManagerHelm, name, version) + sub.Source = repo + sub.Scope = "dependencies" + sub.Direct = true + sub.Depth = 1 + root.Children = []*Node{sub} + return root +} + +func TestResolveRemoteChartRecursion(t *testing.T) { + const repo = "https://charts.example.com" + tgz := buildChartTgz(t, map[string]string{ + "flanksource-ui/Chart.yaml": "apiVersion: v2\nname: flanksource-ui\nversion: 1.4.212\n" + + "dependencies:\n - name: common\n version: 2.0.0\n repository: https://other.example.com\n", + "flanksource-ui/values.yaml": "image:\n repository: flanksource/ui\n tag: \"3.2.1\"\n", + }) + cache := &memCache{ + blobs: map[string][]byte{ + repo + "/index.yaml": []byte("entries:\n flanksource-ui:\n - version: 1.4.212\n urls:\n - " + repo + "/flanksource-ui-1.4.212.tgz\n"), + repo + "/flanksource-ui-1.4.212.tgz": tgz, + }, + } + + root := chartRootWithSubchart("flanksource-ui", "1.4.212", repo) + opts := Options{MaxDepth: 0, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + + ui := findChild(root, "flanksource-ui") + if ui == nil { + t.Fatalf("subchart node missing") + } + if common := findChild(ui, "common"); common == nil || common.Manager != ManagerHelm || common.Depth != 2 { + t.Fatalf("nested subchart 'common' not resolved: %#v (children: %v)", common, childNames(ui)) + } + img := findChild(ui, "flanksource/ui") + if img == nil || img.Manager != ManagerImage || img.Version != "3.2.1" || img.Depth != 2 { + t.Fatalf("subchart values image not harvested: %#v (children: %v)", img, childNames(ui)) + } +} + +func TestResolveRemoteImageBaseViaLabel(t *testing.T) { + cache := &memCache{images: map[string]ImageConfig{ + "app:1.0": {Labels: map[string]string{labelBaseName: "debian:12"}}, + "debian:12": {Labels: map[string]string{}}, + }} + root := imageRootWithImage("app:1.0") + opts := Options{MaxDepth: 0, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + app := findChild(root, "app") + base := findChild(app, "debian") + if base == nil || base.Version != "12" || base.Depth != 2 { + t.Fatalf("base image from OCI label not attached: %#v (children: %v)", base, childNames(app)) + } +} + +func TestResolveRemoteImageBaseViaDockerfile(t *testing.T) { + repoDir := t.TempDir() + if err := os.WriteFile(filepath.Join(repoDir, "Dockerfile"), + []byte("FROM golang:1.22 AS build\nFROM gcr.io/distroless/static:nonroot\n"), 0o644); err != nil { + t.Fatal(err) + } + cache := &memCache{ + images: map[string]ImageConfig{"ghcr.io/acme/app:1.0": {Labels: map[string]string{labelSource: "https://github.com/acme/app"}}}, + gitDirs: map[string]string{"https://github.com/acme/app": repoDir}, + } + root := imageRootWithImage("ghcr.io/acme/app:1.0") + opts := Options{MaxDepth: 2, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + app := findChild(root, "ghcr.io/acme/app") + if app == nil { + t.Fatalf("app node missing (children: %v)", childNames(root)) + } + // Both FROM lines reference external images (golang is the builder's base); + // only a bare `FROM ` reference would be excluded as internal. + if base := findChild(app, "gcr.io/distroless/static"); base == nil || base.Version != "nonroot" { + t.Fatalf("runtime FROM base not attached: %#v (children: %v)", base, childNames(app)) + } + if builder := findChild(app, "golang"); builder == nil || builder.Version != "1.22" { + t.Fatalf("builder-stage FROM base not attached: %#v (children: %v)", builder, childNames(app)) + } +} + +func TestResolveRemoteDepthLimit(t *testing.T) { + cache := &memCache{images: map[string]ImageConfig{ + "a:1": {Labels: map[string]string{labelBaseName: "b:1"}}, + "b:1": {Labels: map[string]string{labelBaseName: "c:1"}}, + "c:1": {Labels: map[string]string{labelBaseName: "d:1"}}, + }} + root := imageRootWithImage("a:1") // a is Depth 1 + opts := Options{MaxDepth: 2, remote: remoteDepsFromCache(cache)} + if _, err := resolveRemote(context.Background(), []*Node{root}, opts); err != nil { + t.Fatal(err) + } + a := findChild(root, "a") + b := findChild(a, "b") + if b == nil || b.Depth != 2 { + t.Fatalf("expected b at depth 2, got %#v", b) + } + if c := findChild(b, "c"); c != nil { + t.Fatalf("depth 2 limit should stop fetching past b, but found c: %#v", c) + } +} + +func imageRootWithImage(ref string) *Node { + root := NewNode(ManagerImage, "container images", "") + root.Source = "kubernetes manifests" + root.Depth = 0 + img := imageNodeFromRef(ref) + img.Depth = 1 + img.Direct = true + root.Children = []*Node{img} + return root +} + +func imageNodeFromRef(ref string) *Node { + r := parseImageRef(ref) + n := NewNode(ManagerImage, r.Name, r.Version) + n.Source = ref + return n +} diff --git a/deps/runner.go b/deps/runner.go index f24f607..dc3f26f 100644 --- a/deps/runner.go +++ b/deps/runner.go @@ -1,39 +1,13 @@ package deps -import ( - "context" - "os" - "os/exec" +import "github.com/flanksource/repomap/deps/manifest" + +// The process execution seam lives in deps/manifest so the per-manager packages +// under deps/manager can declare commands without importing deps. These aliases +// keep it spelled deps.Command / deps.CommandRunner for every existing caller. +type ( + Command = manifest.Command + CommandResult = manifest.CommandResult + CommandRunner = manifest.CommandRunner + ExecRunner = manifest.ExecRunner ) - -type Command struct { - Dir string - Name string - Args []string - Env []string -} - -type CommandResult struct { - Stdout string - Stderr string -} - -type CommandRunner interface { - Run(ctx context.Context, cmd Command) (CommandResult, error) -} - -type ExecRunner struct{} - -func (ExecRunner) Run(ctx context.Context, spec Command) (CommandResult, error) { - cmd := exec.CommandContext(ctx, spec.Name, spec.Args...) - cmd.Dir = spec.Dir - if len(spec.Env) > 0 { - cmd.Env = append(os.Environ(), spec.Env...) - } - out, err := cmd.Output() - result := CommandResult{Stdout: string(out)} - if exit, ok := err.(*exec.ExitError); ok { - result.Stderr = string(exit.Stderr) - } - return result, err -} diff --git a/deps/scan.go b/deps/scan.go index b8e73d8..496b34b 100644 --- a/deps/scan.go +++ b/deps/scan.go @@ -10,6 +10,8 @@ import ( "github.com/flanksource/clicky/task" flanksourceContext "github.com/flanksource/commons/context" + + "github.com/flanksource/repomap" ) func Scan(ctx context.Context, path string, opts Options) (*Export, error) { @@ -55,28 +57,62 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { roots = append(roots, projectRoots...) } + matcher := repomap.NewResourceMatcher(opts.Kind, opts.Namespace, opts.Name, opts.Selector) + + var imageErr error if scanImages { - imageRoots, imageWarnings, err := discoverImageDependencyRoots(absPath, imageScanManagers(opts.Managers)) - warnings = append(warnings, imageWarnings...) - if err != nil { - if len(roots) == 0 && packageErr != nil { - return nil, packageErr - } - if len(roots) == 0 { - return nil, err + // Chart-directory scanning is a filesystem walk and runs independently of + // the git-backed k8s-manifest discovery, so a chart dir outside a git repo + // still resolves its own subcharts and images. Chart.yaml roots carry no + // Kubernetes resource metadata, so a resource filter (kind/namespace/name/ + // selector) can only be satisfied by manifest targets; skip chart dirs when + // one is active. + if matcher.IsEmpty() { + chartRoots, chartWarnings, chartErr := discoverChartDependencyRoots(absPath, opts.Managers) + warnings = append(warnings, chartWarnings...) + if chartErr != nil { + return nil, chartErr } - warnings = append(warnings, Warning{Message: err.Error()}) + roots = append(roots, chartRoots...) } + + var imageRoots []*Node + var imageWarnings []Warning + imageRoots, imageWarnings, imageErr = discoverImageDependencyRoots(absPath, imageScanManagers(opts.Managers), matcher) + warnings = append(warnings, imageWarnings...) roots = append(roots, imageRoots...) } + // Image discovery errors (e.g. not a git repository) only matter when nothing + // else resolved; otherwise they degrade to a warning. + if imageErr != nil && len(roots) == 0 { + if packageErr != nil { + return nil, packageErr + } + return nil, imageErr + } + if imageErr != nil { + warnings = append(warnings, Warning{Message: imageErr.Error()}) + } + if packageErr != nil && len(roots) == 0 { return nil, packageErr } if len(roots) == 0 { return nil, fmt.Errorf("no dependency graphs resolved") } - projectsScanned := len(projects) + imageRootCount(roots) + projectsScanned := len(projects) + imageRootCount(roots) + chartRootCount(roots) + + // Remote recursion mirrors the transitive gate used by go/maven/gradle: at + // any depth other than 1, expand chart subcharts and image base images by + // fetching remote content. Failures degrade to warnings inside resolveRemote. + if opts.MaxDepth != 1 { + remoteWarnings, err := resolveRemote(ctx, roots, opts) + if err != nil { + return nil, err + } + warnings = append(warnings, remoteWarnings...) + } filteredRoots := make([]*Node, 0, len(roots)) for _, root := range roots { @@ -84,8 +120,11 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { filteredRoots = append(filteredRoots, filtered) } } - dups := analyzeDuplicates(filteredRoots) - applyDuplicateRefs(filteredRoots, dups) + var dups map[string]*Duplicate + if opts.ShowDuplicates { + dups = analyzeDuplicates(filteredRoots) + applyDuplicateRefs(filteredRoots, dups) + } nodes, edges, stats := flatten(filteredRoots, dups) stats.Projects = projectsScanned @@ -100,6 +139,7 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { Filter: opts.Filters, MaxDepth: opts.MaxDepth, Flat: opts.Flat, + ShowDuplicates: opts.ShowDuplicates, ProjectsScanned: projectsScanned, }, Statistics: stats, @@ -110,6 +150,9 @@ func Scan(ctx context.Context, path string, opts Options) (*Export, error) { export.Nodes = nodes export.Edges = edges } else { + if !opts.ShowDuplicates { + collapseDuplicates(filteredRoots) + } export.Roots = filteredRoots } return export, nil diff --git a/deps/scan_chart.go b/deps/scan_chart.go new file mode 100644 index 0000000..53fbcfd --- /dev/null +++ b/deps/scan_chart.go @@ -0,0 +1,383 @@ +package deps + +import ( + "fmt" + "io/fs" + "os" + osexec "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/goccy/go-yaml" +) + +// chartFile is the subset of Chart.yaml repomap reads: the chart's own identity +// and its declared subchart dependencies. +type chartFile struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Dependencies []chartDepEntry `yaml:"dependencies"` +} + +type chartDepEntry struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Repository string `yaml:"repository"` +} + +// discoverChartDependencyRoots walks for Helm Chart.yaml files and returns one +// dependency root per chart. Each root carries the subchart dependencies +// declared in Chart.yaml (helm) and the container images referenced in the +// chart's values.yaml and templates/ (image). Versions are recorded verbatim +// from the declarations; nothing is resolved over the network. +func discoverChartDependencyRoots(root string, managers []Manager) ([]*Node, []Warning, error) { + selected := managerSet(managers) + wantHelm := len(selected) == 0 || selected[ManagerHelm] + wantImage := len(selected) == 0 || selected[ManagerImage] + if !wantHelm && !wantImage { + return nil, nil, nil + } + + charts, err := discoverChartFiles(root) + if err != nil { + return nil, nil, err + } + + var roots []*Node + var warnings []Warning + for _, chartPath := range charts { + node, chartWarnings := chartDependencyRoot(root, chartPath, wantHelm, wantImage) + warnings = append(warnings, chartWarnings...) + if node != nil { + roots = append(roots, node) + } + } + sort.SliceStable(roots, func(i, j int) bool { return roots[i].Path < roots[j].Path }) + return roots, warnings, nil +} + +func chartDependencyRoot(scanRoot, chartPath string, wantHelm, wantImage bool) (*Node, []Warning) { + rel := chartRelPath(scanRoot, chartPath) + data, err := os.ReadFile(chartPath) + if err != nil { + return nil, []Warning{{Manager: ManagerHelm, Project: rel, Message: err.Error()}} + } + var chart chartFile + if err := yaml.Unmarshal(data, &chart); err != nil { + return nil, []Warning{{Manager: ManagerHelm, Project: rel, Message: fmt.Sprintf("parse Chart.yaml: %s", err)}} + } + + chartDir := filepath.Dir(chartPath) + name := chart.Name + if name == "" { + name = filepath.Base(chartDir) + } + root := NewNode(ManagerHelm, name, chart.Version) + root.Source = "Chart.yaml" + root.Path = filepath.ToSlash(chartPath) + + var children []*Node + var warnings []Warning + if wantHelm { + children = append(children, chartSubchartNodes(chart, rel)...) + } + if wantImage { + imageNodes, imgWarnings := chartImageNodes(chartDir, scanRoot) + children = append(children, imageNodes...) + warnings = append(warnings, imgWarnings...) + } + if len(children) == 0 { + return nil, warnings + } + root.Children = children + return root, warnings +} + +func chartSubchartNodes(chart chartFile, chartRel string) []*Node { + var nodes []*Node + for _, dep := range chart.Dependencies { + if dep.Name == "" { + continue + } + node := NewNode(ManagerHelm, dep.Name, dep.Version) + node.Direct = true + node.Depth = 1 + node.Scope = "dependencies" + node.Source = dep.Repository + node.Path = chartRel + nodes = append(nodes, node) + } + return nodes +} + +// chartImageNodes collects container images referenced in a chart's values.yaml +// and templates/, deduplicated by image ref. +func chartImageNodes(chartDir, scanRoot string) ([]*Node, []Warning) { + seen := map[string]*Node{} + var order []string + add := func(ref, source string) { + ref = strings.TrimSpace(strings.Trim(ref, `"'`)) + if !looksLikeImage(ref) { + return + } + node := chartImageNode(ref, source) + if _, ok := seen[node.ID]; ok { + return + } + seen[node.ID] = node + order = append(order, node.ID) + } + + var warnings []Warning + valuesPath := filepath.Join(chartDir, "values.yaml") + if data, err := os.ReadFile(valuesPath); err == nil { + var values map[string]interface{} + if err := yaml.Unmarshal(data, &values); err != nil { + warnings = append(warnings, Warning{Manager: ManagerImage, Project: chartRelPath(scanRoot, valuesPath), Message: fmt.Sprintf("parse values.yaml: %s", err)}) + } else { + source := chartRelPath(scanRoot, valuesPath) + collectValuesImages(values, func(ref string) { add(ref, source) }) + } + } + collectTemplateImages(chartDir, scanRoot, add) + + nodes := make([]*Node, 0, len(order)) + for _, id := range order { + nodes = append(nodes, seen[id]) + } + return nodes, warnings +} + +func chartImageNode(ref, source string) *Node { + name, version := splitImageRef(ref) + node := NewNode(ManagerImage, name, version) + node.Direct = true + node.Depth = 1 + node.Source = ref + node.Path = source + return node +} + +// collectValuesImages walks a parsed values.yaml tree and reports every image +// reference it finds, covering both the `image: "repo:tag"` string form and the +// structured `image: {registry, repository, tag}` map form at any nesting depth. +func collectValuesImages(node interface{}, report func(string)) { + switch v := node.(type) { + case map[string]interface{}: + if s, ok := v["image"].(string); ok { + report(s) + } + if img := imageFromMap(v); img != "" { + report(img) + } + for _, val := range v { + collectValuesImages(val, report) + } + case []interface{}: + for _, item := range v { + collectValuesImages(item, report) + } + } +} + +func imageFromMap(m map[string]interface{}) string { + if spec, ok := m["image"].(map[string]interface{}); ok { + if img := repoTagImage(spec); img != "" { + return img + } + } + return repoTagImage(m) +} + +func repoTagImage(m map[string]interface{}) string { + repo := scalarString(m["repository"]) + if repo == "" { + // `name` is a less common alias for `repository`; only trust it when a tag + // sits alongside, so non-image maps that merely have a name are ignored. + if _, hasTag := m["tag"]; hasTag { + repo = scalarString(m["name"]) + } + } + if repo == "" { + return "" + } + if registry := scalarString(m["registry"]); registry != "" { + repo = registry + "/" + repo + } + if tag := scalarString(m["tag"]); tag != "" { + return repo + ":" + tag + } + return repo +} + +func scalarString(v interface{}) string { + switch t := v.(type) { + case nil: + return "" + case string: + return t + default: + return fmt.Sprintf("%v", t) + } +} + +// collectTemplateImages scans a chart's templates/ for literal `image:` values, +// skipping Go-templated values that cannot be resolved offline. +func collectTemplateImages(chartDir, scanRoot string, add func(ref, source string)) { + templatesDir := filepath.Join(chartDir, "templates") + _ = filepath.WalkDir(templatesDir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !isTemplateFile(d.Name()) { + return nil + } + data, readErr := os.ReadFile(path) + if readErr != nil { + return nil + } + source := chartRelPath(scanRoot, path) + for _, ref := range imageValuesInTemplate(string(data)) { + add(ref, source) + } + return nil + }) +} + +func imageValuesInTemplate(content string) []string { + var refs []string + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + // Accept both `image:` and the list-item form `- image:`. + trimmed = strings.TrimSpace(strings.TrimPrefix(trimmed, "-")) + const key = "image:" + if !strings.HasPrefix(trimmed, key) { + continue + } + value := strings.TrimSpace(trimmed[len(key):]) + if value == "" || strings.Contains(value, "{{") { + continue + } + refs = append(refs, value) + } + return refs +} + +func isTemplateFile(name string) bool { + switch strings.ToLower(filepath.Ext(name)) { + case ".yaml", ".yml", ".tpl": + return true + } + return false +} + +// looksLikeImage rejects values that are clearly not container image references: +// empty, whitespace, Go-templated, or not starting with an image-ref character. +// Bare official images (e.g. busybox) are accepted since the value came from an +// explicit image field. +func looksLikeImage(ref string) bool { + if ref == "" || strings.ContainsAny(ref, " \t{}|<>\"'`") || strings.Contains(ref, "{{") { + return false + } + c := ref[0] + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') +} + +func splitImageRef(ref string) (name, version string) { + name = stripImageVersion(ref) + if i := imageTagSeparator(ref); i >= 0 { + version = ref[i+1:] + if at := strings.Index(version, "@"); at >= 0 { + version = version[:at] + } + } + return name, version +} + +func discoverChartFiles(root string) ([]string, error) { + files, ok := gitChartFiles(root) + if !ok { + var err error + files, err = walkChartFiles(root) + if err != nil { + return nil, err + } + } + return filterVendoredCharts(files), nil +} + +func gitChartFiles(root string) ([]string, bool) { + cmd := osexec.Command("git", "-C", root, "ls-files", "--cached", "--others", "--exclude-standard", "-z", "--", ".") + out, err := cmd.Output() + if err != nil { + return nil, false + } + var files []string + for _, rel := range strings.Split(string(out), "\x00") { + if rel == "" || !strings.EqualFold(filepath.Base(filepath.FromSlash(rel)), "chart.yaml") { + continue + } + files = append(files, filepath.Join(root, filepath.FromSlash(rel))) + } + sort.Strings(files) + return files, true +} + +func walkChartFiles(root string) ([]string, error) { + var files []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if path != root && ignoredDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + if strings.EqualFold(d.Name(), "Chart.yaml") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(files) + return files, nil +} + +// filterVendoredCharts drops Chart.yaml files that live in a parent chart's +// charts/ directory (downloaded dependencies), keeping standalone charts and +// monorepo charts/ layouts. +func filterVendoredCharts(files []string) []string { + present := make(map[string]bool, len(files)) + for _, f := range files { + present[filepath.Dir(f)] = true + } + out := files[:0] + for _, f := range files { + dir := filepath.Dir(f) + parent := filepath.Dir(dir) + if filepath.Base(parent) == "charts" && present[filepath.Dir(parent)] { + continue + } + out = append(out, f) + } + return out +} + +func chartRelPath(scanRoot, path string) string { + if rel, err := filepath.Rel(scanRoot, path); err == nil { + return filepath.ToSlash(rel) + } + return filepath.ToSlash(path) +} + +func chartRootCount(roots []*Node) int { + count := 0 + for _, root := range roots { + if root != nil && root.Manager == ManagerHelm && root.Source == "Chart.yaml" { + count++ + } + } + return count +} diff --git a/deps/scan_chart_test.go b/deps/scan_chart_test.go new file mode 100644 index 0000000..a66a7eb --- /dev/null +++ b/deps/scan_chart_test.go @@ -0,0 +1,192 @@ +package deps + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +const chartYAMLFixture = `apiVersion: v2 +name: web +version: 1.4.0 +dependencies: + - name: postgresql + version: "12.x.x" + repository: https://charts.bitnami.com/bitnami + - name: redis + version: 18.1.2 + repository: oci://registry-1.docker.io/bitnamicharts +` + +const chartValuesFixture = `image: + registry: docker.io + repository: bitnami/nginx + tag: "1.25.3" +sidecar: + image: busybox:1.36 +postgresql: + image: + repository: postgres + tag: 15.5 +ui: + image: + name: ghcr.io/acme/ui + tag: "3.1.0" +` + +const chartTemplateFixture = `apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + containers: + - name: app + image: ghcr.io/acme/app:2.0.1 + - name: tools + image: alpine + - image: quay.io/prometheus/node-exporter:1.7.0 + name: dash-first + - name: templated + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" +` + +func writeChartFixture(t *testing.T, dir string) { + t.Helper() + writeFile(t, filepath.Join(dir, "charts", "web", "Chart.yaml"), chartYAMLFixture) + writeFile(t, filepath.Join(dir, "charts", "web", "values.yaml"), chartValuesFixture) + writeFile(t, filepath.Join(dir, "charts", "web", "templates", "deployment.yaml"), chartTemplateFixture) +} + +func TestScanChartSubchartDependencies(t *testing.T) { + dir := t.TempDir() + writeChartFixture(t, dir) + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if len(got.Roots) != 1 { + t.Fatalf("expected one chart root, got %d: %#v", len(got.Roots), got.Roots) + } + root := got.Roots[0] + if root.Manager != ManagerHelm || root.Name != "web" || root.Version != "1.4.0" { + t.Fatalf("chart root metadata wrong: %#v", root) + } + if root.Source != "Chart.yaml" { + t.Fatalf("chart root should be sourced from Chart.yaml, got %q", root.Source) + } + + pg := findChild(root, "postgresql") + if pg == nil || pg.Version != "12.x.x" || pg.Manager != ManagerHelm { + t.Fatalf("postgresql subchart not resolved with declared range: %#v", pg) + } + if pg.Source != "https://charts.bitnami.com/bitnami" { + t.Fatalf("subchart should carry its repository as source: %#v", pg) + } + redis := findChild(root, "redis") + if redis == nil || redis.Version != "18.1.2" { + t.Fatalf("redis subchart not resolved: %#v", redis) + } +} + +func TestScanChartHelmOnlyExcludesImages(t *testing.T) { + dir := t.TempDir() + writeChartFixture(t, dir) + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + for _, child := range got.Roots[0].Children { + if child.Manager == ManagerImage { + t.Fatalf("helm-only scan should not include image children: %#v", child) + } + } +} + +func TestScanChartExtractsImagesFromValuesAndTemplates(t *testing.T) { + dir := t.TempDir() + writeChartFixture(t, dir) + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerImage}, + MaxDepth: 1, // offline: do not trigger remote recursion + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if len(got.Roots) != 1 { + t.Fatalf("expected one chart root, got %d", len(got.Roots)) + } + root := got.Roots[0] + + want := map[string]string{ + "docker.io/bitnami/nginx": "1.25.3", + "busybox": "1.36", + "postgres": "15.5", + "ghcr.io/acme/ui": "3.1.0", + "ghcr.io/acme/app": "2.0.1", + "alpine": "", // bare image from a template, no tag + "quay.io/prometheus/node-exporter": "1.7.0", // list-item `- image:` form + } + for name, version := range want { + img := findChild(root, name) + if img == nil { + t.Fatalf("expected image %q in chart, got children %#v", name, childNames(root)) + } + if img.Version != version { + t.Fatalf("image %q version = %q, want %q", name, img.Version, version) + } + if img.Manager != ManagerImage { + t.Fatalf("image %q should use the image manager: %#v", name, img) + } + } + if templated := findChild(root, "{{ .Values.image.repository }}"); templated != nil { + t.Fatalf("templated image refs must be skipped, got %#v", templated) + } +} + +func TestScanChartSkipsVendoredSubcharts(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "Chart.yaml"), "apiVersion: v2\nname: parent\nversion: 1.0.0\ndependencies:\n - name: child\n version: 1.0.0\n") + // A vendored copy of the dependency under charts/ must not become its own root. + writeFile(t, filepath.Join(dir, "charts", "child", "Chart.yaml"), "apiVersion: v2\nname: child\nversion: 1.0.0\n") + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if len(got.Roots) != 1 || got.Roots[0].Name != "parent" { + t.Fatalf("vendored subchart should not be a separate root, got %#v", childRootNames(got.Roots)) + } +} + +func childNames(node *Node) []string { + var out []string + for _, c := range node.Children { + out = append(out, c.Name) + } + return out +} + +func childRootNames(roots []*Node) []string { + var out []string + for _, r := range roots { + out = append(out, r.Name) + } + return out +} diff --git a/deps/scan_image.go b/deps/scan_image.go index 19c7f58..1905851 100644 --- a/deps/scan_image.go +++ b/deps/scan_image.go @@ -1,6 +1,7 @@ package deps import ( + "fmt" "path/filepath" "sort" @@ -8,30 +9,32 @@ import ( "github.com/flanksource/repomap/imageupdate" ) -func discoverImageDependencyRoots(root string, managers []Manager) ([]*Node, []Warning, error) { +func discoverImageDependencyRoots(root string, managers []Manager, matcher repomap.ResourceMatcher) ([]*Node, []Warning, error) { selected := managerSet(managers) conf, err := repomap.GetConf(root) if err != nil { return nil, nil, err } - targets, sourceIndex, err := discoverImageTargets(conf, root) + res, err := imageupdate.DiscoverRepoTargets(conf, root) if err != nil { return nil, nil, err } + if res.UntrackedTarget != "" { + return nil, nil, fmt.Errorf("%s is not tracked by git; deps only scans git-tracked manifests — run 'git add %s' first", res.UntrackedTarget, res.UntrackedTarget) + } + targets := imageupdate.Filter(res.Targets, matcher, nil, nil) var imageChildren []*Node var helmChildren []*Node var warnings []Warning + for _, w := range res.Warnings { + warnings = append(warnings, Warning{Manager: ManagerHelm, Project: w.File, Message: w.Message}) + } for _, target := range targets { manager := managerForUpdateTarget(target) if manager == "" || (len(selected) > 0 && !selected[manager]) { continue } - if target.Kind == imageupdate.TargetChart { - if err := sourceIndex.Resolve(&target); err != nil { - warnings = append(warnings, Warning{Manager: ManagerHelm, Project: target.File, Message: err.Error()}) - } - } node := nodeForImageTarget(target, manager) switch manager { case ManagerImage: diff --git a/deps/scan_test.go b/deps/scan_test.go index 5efbfac..c06df6d 100644 --- a/deps/scan_test.go +++ b/deps/scan_test.go @@ -160,6 +160,7 @@ func TestScanImageAndHelmManifestTargets(t *testing.T) { got, err := Scan(context.Background(), ".", Options{ Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion Now: func() time.Time { return time.Unix(1, 0).UTC() }, }) if err != nil { @@ -192,6 +193,118 @@ func TestScanImageAndHelmManifestTargets(t *testing.T) { } } +func TestScanResourceFilterByKind(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "workloads.yaml"), deploymentUpdateFixture) + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + runGit(t, dir, "add", ".") + + cases := []struct { + name string + kind string + keep Manager + drop Manager + wantChild string + }{ + {name: "HelmRelease selects the chart", kind: "HelmRelease", keep: ManagerHelm, drop: ManagerImage, wantChild: "podinfo"}, + {name: "Deployment selects container images", kind: "Deployment", keep: ManagerImage, drop: ManagerHelm, wantChild: "nginx"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := Scan(context.Background(), ".", Options{ + Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Kind: []string{tc.kind}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if dropped := findRoot(got.Roots, tc.drop); dropped != nil { + t.Fatalf("kind=%s should exclude the %s root, got %#v", tc.kind, tc.drop, dropped) + } + kept := findRoot(got.Roots, tc.keep) + if kept == nil || findChild(kept, tc.wantChild) == nil { + t.Fatalf("kind=%s should keep %s/%s, got %#v", tc.kind, tc.keep, tc.wantChild, kept) + } + }) + } +} + +func TestScanResourceFilterExcludesChartDirectories(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + writeChartFixture(t, dir) // a Chart.yaml-sourced helm root with no Kubernetes resource metadata + runGit(t, dir, "add", ".") + + got, err := Scan(context.Background(), ".", Options{ + Managers: []Manager{ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Kind: []string{"HelmRelease"}, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + for _, root := range got.Roots { + if root.Source == "Chart.yaml" { + t.Fatalf("a resource filter must exclude Chart.yaml roots (no k8s metadata), got %#v", root) + } + } + if helmRoot := findRoot(got.Roots, ManagerHelm); helmRoot == nil || findChild(helmRoot, "podinfo") == nil { + t.Fatalf("kind=HelmRelease should keep the manifest-sourced podinfo chart, got %#v", got.Roots) + } +} + +func TestScanSingleFileTarget(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "workloads.yaml"), deploymentUpdateFixture) + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + runGit(t, dir, "add", ".") + + got, err := Scan(context.Background(), filepath.Join("apps", "helmrelease.yaml"), Options{ + Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, // offline: do not trigger remote recursion + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if imageRoot := findRoot(got.Roots, ManagerImage); imageRoot != nil { + t.Fatalf("scanning a single HelmRelease file must not surface images from another file: %#v", imageRoot) + } + helmRoot := findRoot(got.Roots, ManagerHelm) + if helmRoot == nil || findChild(helmRoot, "podinfo") == nil { + t.Fatalf("single-file scan should resolve the file's own chart, got %#v", got.Roots) + } +} + +func TestScanUntrackedFileErrors(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + // Deliberately do NOT `git add` — deps only scans git-tracked manifests. + + _, err := Scan(context.Background(), filepath.Join("apps", "helmrelease.yaml"), Options{ + Managers: []Manager{ManagerImage, ManagerHelm}, + MaxDepth: 1, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err == nil { + t.Fatal("scanning an untracked file should error") + } + if !strings.Contains(err.Error(), "not tracked") { + t.Fatalf("error should explain the file is untracked, got %q", err.Error()) + } +} + func TestNodeImplementsClickyTreeNode(t *testing.T) { var _ api.TreeNode = (*Node)(nil) diff --git a/deps/update.go b/deps/update.go index 04dd058..ce5a087 100644 --- a/deps/update.go +++ b/deps/update.go @@ -9,12 +9,7 @@ import ( "sort" "strings" - "github.com/Masterminds/semver/v3" - "github.com/flanksource/clicky" - "github.com/flanksource/clicky/api" - "github.com/flanksource/clicky/task" - "github.com/flanksource/commons/collections" - flanksourceContext "github.com/flanksource/commons/context" + "github.com/flanksource/repomap" "github.com/flanksource/repomap/imageupdate" "golang.org/x/mod/modfile" ) @@ -30,7 +25,7 @@ var supportedUpdateManagers = map[Manager]bool{ } type CandidateSelector func([]UpdateChoice) ([]UpdateChoice, bool) -type VersionSelector func(UpdateChoice) (string, bool) +type VersionSelector func(UpdateVersionPrompt) (string, bool) type ImageVersionResolver interface { Available(context.Context, imageupdate.UpdateTarget) ([]string, error) @@ -39,8 +34,23 @@ type ImageVersionResolver interface { } type UpdateOptions struct { - Managers []Manager - Expression []string + Managers []Manager + Filters []string + + // Resource filters for image/helm targets (ignored by package managers). + Kind []string + Namespace []string + Name []string + Selector []string + Image []string + Chart []string + + // Latest resolves each matched dependency to its highest stable version; + // Version applies an explicit version to all matched dependencies. They are + // mutually exclusive and both bypass the interactive pickers. + Latest bool + Version string + Check bool DryRun bool Runner CommandRunner @@ -49,6 +59,15 @@ type UpdateOptions struct { SelectVersion VersionSelector } +// DiscoverFilter narrows image/helm targets during discovery by Kubernetes +// resource metadata and image/chart name patterns. It has no effect on package +// managers, whose candidates come from manifest/lockfile parsing. +type DiscoverFilter struct { + Matcher repomap.ResourceMatcher + ImagePatterns []string + ChartPatterns []string +} + type UpdateCandidate struct { Manager Manager `json:"manager"` Name string `json:"name"` @@ -66,6 +85,14 @@ type UpdateChoice struct { LatestPrerelease string `json:"latest_prerelease,omitempty"` } +// UpdateVersionPrompt is a single version question standing in for every +// selected occurrence of the same dependency. Candidate is the first +// occurrence; Files lists each distinct manifest the answer will be written to. +type UpdateVersionPrompt struct { + UpdateChoice + Files []string `json:"files"` +} + type UpdatePlan struct { Manager Manager `json:"manager"` Name string `json:"name"` @@ -86,38 +113,63 @@ func Update(ctx context.Context, path string, opts UpdateOptions) ([]UpdatePlan, if path == "" { path = "." } + if opts.Latest && opts.Version != "" { + return nil, fmt.Errorf("--latest and --version are mutually exclusive") + } managers, err := updateManagers(opts.Managers) if err != nil { return nil, err } - patterns := splitUpdatePatterns(opts.Expression) - if len(patterns) == 0 { - return nil, fmt.Errorf("dependency update expression is required") - } + patterns := splitUpdatePatterns(opts.Filters) if opts.Runner == nil { opts.Runner = ExecRunner{} } - candidates, err := DiscoverUpdateCandidates(path, managers) + filter := DiscoverFilter{ + Matcher: repomap.NewResourceMatcher(opts.Kind, opts.Namespace, opts.Name, opts.Selector), + ImagePatterns: splitUpdatePatterns(opts.Image), + ChartPatterns: splitUpdatePatterns(opts.Chart), + } + candidates, err := DiscoverUpdateCandidates(path, managers, filter) if err != nil { return nil, err } candidates = filterUpdateCandidates(candidates, patterns) if len(candidates) == 0 { - return nil, fmt.Errorf("no direct dependencies matched %q", strings.Join(patterns, ",")) + if len(patterns) > 0 { + return nil, fmt.Errorf("no direct dependencies matched %q", strings.Join(patterns, ",")) + } + return nil, fmt.Errorf("no updatable dependencies found") } - choices, plansByKey := resolveUpdateChoices(ctx, candidates, opts) - if len(choices) == 0 { - return orderedUpdatePlans(candidates, plansByKey), nil - } + // --check lists available updates without writing; it takes precedence over + // the write modes so `--check --version`/`--check --latest` never mutate files. if opts.Check { + choices, plansByKey := resolveUpdateChoices(ctx, candidates, opts) for _, choice := range choices { plansByKey[choice.Candidate.key()] = checkUpdatePlan(choice) } return orderedUpdatePlans(candidates, plansByKey), nil } + // --version applies an explicit version to every matched dependency without + // listing available versions (it only validates image/helm availability). + if opts.Version != "" { + return applyExplicitVersionUpdates(ctx, candidates, opts), nil + } + + choices, plansByKey := resolveUpdateChoices(ctx, candidates, opts) + + // --latest resolves each candidate to its highest stable version. + if opts.Latest { + applyLatestUpdates(ctx, candidates, choices, plansByKey, opts) + return orderedUpdatePlans(candidates, plansByKey), nil + } + + if len(choices) == 0 { + return orderedUpdatePlans(candidates, plansByKey), nil + } + selectCandidates := opts.SelectCandidates if selectCandidates == nil { selectCandidates = promptUpdateCandidates @@ -143,18 +195,22 @@ func Update(ctx context.Context, path string, opts UpdateOptions) ([]UpdatePlan, if selectVersion == nil { selectVersion = promptUpdateVersion } - for _, choice := range sortSelectedUpdateChoicesByFile(selected) { - version, ok := selectVersion(choice) - if !ok || strings.TrimSpace(version) == "" { - plansByKey[choice.Candidate.key()] = skippedUpdatePlan(choice.Candidate, "no version selected") - continue + // A dependency repeated across manifests poses one question, so occurrences + // sharing a version prompt are confirmed once and the answer applied to all. + for _, group := range groupChoicesForVersionPrompt(sortSelectedUpdateChoicesByFile(selected)) { + version, ok := selectVersion(newUpdateVersionPrompt(group)) + for _, choice := range group { + if !ok || strings.TrimSpace(version) == "" { + plansByKey[choice.Candidate.key()] = skippedUpdatePlan(choice.Candidate, "no version selected") + continue + } + plansByKey[choice.Candidate.key()] = applyDependencyUpdate(ctx, choice.Candidate, version, opts) } - plansByKey[choice.Candidate.key()] = applyDependencyUpdate(ctx, choice.Candidate, version, opts) } return orderedUpdatePlans(candidates, plansByKey), nil } -func DiscoverUpdateCandidates(path string, managers []Manager) ([]UpdateCandidate, error) { +func DiscoverUpdateCandidates(path string, managers []Manager, filter DiscoverFilter) ([]UpdateCandidate, error) { defaultManagers := len(managers) == 0 managers, err := updateManagers(managers) if err != nil { @@ -190,7 +246,7 @@ func DiscoverUpdateCandidates(path string, managers []Manager) ([]UpdateCandidat } } if imageManagers := imageUpdateManagers(managers); len(imageManagers) > 0 { - candidates, err := discoverImageUpdateCandidates(absPath, imageManagers) + candidates, err := discoverImageUpdateCandidates(absPath, imageManagers, filter) if err != nil { if !defaultManagers || len(out) == 0 { if packageErr != nil && defaultManagers { @@ -326,664 +382,6 @@ func filterUpdateCandidates(candidates []UpdateCandidate, patterns []string) []U return out } -func resolveUpdateChoices(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) ([]UpdateChoice, map[string]UpdatePlan) { - type result struct { - versions []string - latestStable string - latestPrerelease string - err error - } - results := make([]result, len(candidates)) - group := task.StartGroup[int]("Resolving dependency versions", task.WithConcurrency(updateResolveConcurrency)) - for i, candidate := range candidates { - idx, c := i, candidate - group.Add(updateTaskName(c), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { - tk.Infof("looking up published versions") - versions, latestStable, latestPrerelease, err := resolveCandidateVersions(ctx, opts, c) - results[idx] = result{versions: versions, latestStable: latestStable, latestPrerelease: latestPrerelease, err: err} - if err != nil { - tk.Warnf("%s", err.Error()) - tk.Warning() - } else if len(versions) == 0 { - tk.Infof("already up to date") - tk.Success() - } else { - tk.Success() - } - return idx, nil - }) - } - _, _ = group.GetResults() - - plansByKey := map[string]UpdatePlan{} - choices := make([]UpdateChoice, 0, len(candidates)) - for i, candidate := range candidates { - result := results[i] - if result.err != nil { - plansByKey[candidate.key()] = skippedUpdatePlan(candidate, result.err.Error()) - continue - } - if len(result.versions) == 0 { - continue - } - choices = append(choices, UpdateChoice{ - Candidate: candidate, - Versions: result.versions, - LatestStable: result.latestStable, - LatestPrerelease: result.latestPrerelease, - }) - } - return choices, plansByKey -} - -func resolveCandidateVersions(ctx context.Context, opts UpdateOptions, candidate UpdateCandidate) ([]string, string, string, error) { - var ( - versions []string - err error - ) - switch candidate.Manager { - case ManagerImage, ManagerHelm: - versions, _, _, err = availableImageTargetVersions(ctx, opts.ImageResolver, candidate) - default: - versions, err = AvailableDependencyVersions(ctx, opts.Runner, candidate) - } - if err != nil { - return nil, "", "", err - } - versions = updateableVersions(candidate.Current, versions) - return versions, latestStableVersion(versions), latestPrereleaseVersion(versions), nil -} - -func AvailableDependencyVersions(ctx context.Context, runner CommandRunner, candidate UpdateCandidate) ([]string, error) { - if runner == nil { - runner = ExecRunner{} - } - var ( - result CommandResult - err error - ) - switch candidate.Manager { - case ManagerGo: - result, err = runner.Run(ctx, Command{ - Dir: candidate.Dir, - Name: "go", - Args: []string{"list", "-m", "-versions", "-json", candidate.Name}, - Env: []string{"GOFLAGS=-mod=readonly"}, - }) - case ManagerNPM: - result, err = runner.Run(ctx, Command{ - Dir: candidate.Dir, - Name: "npm", - Args: []string{"view", candidate.Name, "versions", "--json"}, - }) - case ManagerPNPM: - result, err = runner.Run(ctx, Command{ - Dir: candidate.Dir, - Name: "pnpm", - Args: []string{"view", candidate.Name, "versions", "--json"}, - }) - default: - return nil, fmt.Errorf("dependency version lookup does not support manager %q", candidate.Manager) - } - if err != nil { - if result.Stderr != "" { - return nil, fmt.Errorf("%s: %w", strings.TrimSpace(result.Stderr), err) - } - return nil, err - } - versions, err := parseAvailableVersions(candidate.Manager, result.Stdout) - if err != nil { - return nil, err - } - return sortDependencyVersions(versions), nil -} - -func parseAvailableVersions(manager Manager, stdout string) ([]string, error) { - stdout = strings.TrimSpace(stdout) - if stdout == "" { - return nil, nil - } - if manager == ManagerGo { - var payload struct { - Versions []string `json:"Versions"` - } - if err := json.Unmarshal([]byte(stdout), &payload); err != nil { - return nil, err - } - return payload.Versions, nil - } - var list []string - if err := json.Unmarshal([]byte(stdout), &list); err == nil { - return list, nil - } - var single string - if err := json.Unmarshal([]byte(stdout), &single); err == nil && single != "" { - return []string{single}, nil - } - return nil, fmt.Errorf("expected JSON version array") -} - -func applyDependencyUpdate(ctx context.Context, candidate UpdateCandidate, version string, opts UpdateOptions) UpdatePlan { - if candidate.Manager == ManagerImage || candidate.Manager == ManagerHelm { - return applyImageTargetUpdate(ctx, candidate, version, opts) - } - plan := planFromCandidate(candidate) - plan.NewVersion = version - if selectedVersionIsCurrent(candidate.Current, version) { - plan.Skipped = "already at selected version" - return plan - } - cmd, err := updateCommand(candidate, version) - if err != nil { - plan.Skipped = err.Error() - return plan - } - plan.Command = append([]string{cmd.Name}, cmd.Args...) - if opts.DryRun { - plan.DryRun = true - return plan - } - runner := opts.Runner - if runner == nil { - runner = ExecRunner{} - } - if _, err := runner.Run(ctx, cmd); err != nil { - plan.Skipped = err.Error() - return plan - } - plan.Written = true - stageUpdatePlan(ctx, &plan, candidate, runner) - return plan -} - -func stageUpdatePlan(ctx context.Context, plan *UpdatePlan, candidate UpdateCandidate, runner CommandRunner) { - staged, err := stageUpdatedFiles(ctx, runner, candidate) - plan.Staged = staged - if err != nil { - plan.StageError = err.Error() - } -} - -func updateCommand(candidate UpdateCandidate, version string) (Command, error) { - target := candidate.Name + "@" + version - switch candidate.Manager { - case ManagerGo: - return Command{Dir: candidate.Dir, Name: "go", Args: []string{"get", target}}, nil - case ManagerNPM: - args := []string{"install", "--package-lock-only", "--ignore-scripts"} - if flag := packageSaveFlag(candidate.Scope); flag != "" { - args = append(args, flag) - } - args = append(args, target) - return Command{Dir: candidate.Dir, Name: "npm", Args: args}, nil - case ManagerPNPM: - args := []string{"add", "--lockfile-only", "--ignore-scripts"} - if flag := packageSaveFlag(candidate.Scope); flag != "" { - args = append(args, flag) - } - args = append(args, target) - return Command{Dir: candidate.Dir, Name: "pnpm", Args: args}, nil - default: - return Command{}, fmt.Errorf("package-manager updates do not support manager %q", candidate.Manager) - } -} - -func packageSaveFlag(scope string) string { - switch scope { - case "dependencies": - return "--save-prod" - case "devDependencies": - return "--save-dev" - case "optionalDependencies": - return "--save-optional" - case "peerDependencies": - return "--save-peer" - default: - return "" - } -} - -func promptUpdateCandidates(choices []UpdateChoice) ([]UpdateChoice, bool) { - return runUpdateChoiceTreePicker(choices) -} - -func promptUpdateVersion(choice UpdateChoice) (string, bool) { - candidate := choice.Candidate - title := fmt.Sprintf("Select version for %s in %s (current %s)", candidate.Name, candidate.File, candidate.Current) - return clicky.PromptSelect(choice.Versions, clicky.PromptSelectOptions[string]{ - Title: title, - PageSize: 12, - Render: func(version string) api.Textable { - text := clicky.Text(version, "font-mono") - var tags []string - if selectedVersionIsCurrent(candidate.Current, version) { - tags = append(tags, "current") - } - if version == choice.LatestStable { - tags = append(tags, "latest stable") - } - if version == choice.LatestPrerelease { - tags = append(tags, "latest pre-release") - } - if isPrerelease(version) { - tags = append(tags, "pre-release") - } - if len(tags) > 0 { - text = text.Space().Append("("+strings.Join(tags, ", ")+")", "text-muted") - } - return text - }, - }) -} - -type updateChoiceFileGroup struct { - File string - Choices []UpdateChoice -} - -func groupUpdateChoicesByFile(choices []UpdateChoice) []updateChoiceFileGroup { - byFile := map[string][]UpdateChoice{} - for _, choice := range choices { - file := choice.Candidate.File - byFile[file] = append(byFile[file], choice) - } - files := make([]string, 0, len(byFile)) - for file := range byFile { - files = append(files, file) - } - sort.Strings(files) - groups := make([]updateChoiceFileGroup, 0, len(files)) - for _, file := range files { - groupChoices := append([]UpdateChoice(nil), byFile[file]...) - sort.SliceStable(groupChoices, func(i, j int) bool { - return groupChoices[i].Candidate.less(groupChoices[j].Candidate) - }) - groups = append(groups, updateChoiceFileGroup{File: file, Choices: groupChoices}) - } - return groups -} - -func sortSelectedUpdateChoicesByFile(choices []UpdateChoice) []UpdateChoice { - var out []UpdateChoice - for _, group := range groupUpdateChoicesByFile(choices) { - out = append(out, group.Choices...) - } - return out -} - -func (p UpdatePlan) Pretty() api.Text { - t := clicky.Text(fmt.Sprintf("[%s] %s", p.Manager, p.Name), managerStyle(p.Manager)) - if p.OldVersion != "" || p.NewVersion != "" { - t = t.Space().Append(p.OldVersion, "font-mono text-muted") - if p.NewVersion != "" { - t = t.Append(" -> ", "text-muted").Append(p.NewVersion, "font-mono text-green-600") - } - } - switch { - case p.Skipped != "": - t = t.Space().Append("skipped: "+p.Skipped, "text-muted") - case p.Checked: - t = t.Space().Append("update available", "text-green-600") - case p.DryRun: - t = t.Space().Append("(dry-run)", "text-yellow-600") - case p.Written: - t = t.Space().Append("written", "text-green-600") - } - if len(p.Staged) > 0 { - t = t.Space().Append("staged "+strings.Join(p.Staged, ", "), "text-muted") - } - if p.StageError != "" { - t = t.Space().Append("stage failed: "+p.StageError, "text-yellow-600") - } - return t -} - -func (UpdatePlan) Columns() []api.ColumnDef { - return []api.ColumnDef{ - api.Column("manager").Label("Manager").Build(), - api.Column("dependency").Label("Dependency").Build(), - api.Column("file").Label("File").Build(), - api.Column("scope").Label("Scope").Build(), - api.Column("change").Label("Change").Build(), - api.Column("status").Label("Status").Build(), - } -} - -func (p UpdatePlan) Row() map[string]any { - row := map[string]any{ - "manager": clicky.Text(string(p.Manager), managerStyle(p.Manager)), - "dependency": clicky.Text(p.Name, "font-bold text-cyan-600"), - "file": clicky.Text(p.File, "font-mono"), - "scope": clicky.Text(p.Scope, "text-muted"), - "change": updateChangeText(p.OldVersion, p.NewVersion), - } - switch { - case p.Skipped != "": - row["status"] = clicky.Text(p.Skipped, "text-muted") - case p.Checked: - row["status"] = clicky.Text("update available", "text-green-600") - case p.DryRun: - row["status"] = clicky.Text("dry-run", "text-yellow-600") - case p.Written: - status := clicky.Text("written", "text-green-600") - if len(p.Staged) > 0 { - status = status.Append(" + staged "+strings.Join(p.Staged, ", "), "text-muted") - } - if p.StageError != "" { - status = status.Append(" (stage failed: "+p.StageError+")", "text-yellow-600") - } - row["status"] = status - default: - row["status"] = clicky.Text("") - } - return row -} - -func updateChangeText(oldVersion, newVersion string) api.Text { - text := clicky.Text(oldVersion, "font-mono text-muted") - if newVersion != "" { - text = text.Append(" -> ", "text-muted").Append(newVersion, "font-mono text-green-600") - } - return text -} - -func planFromCandidate(candidate UpdateCandidate) UpdatePlan { - return UpdatePlan{ - Manager: candidate.Manager, - Name: candidate.Name, - File: candidate.File, - Scope: candidate.Scope, - OldVersion: candidate.Current, - } -} - -func skippedUpdatePlan(candidate UpdateCandidate, reason string) UpdatePlan { - plan := planFromCandidate(candidate) - plan.Skipped = reason - return plan -} - -func checkUpdatePlan(choice UpdateChoice) UpdatePlan { - plan := planFromCandidate(choice.Candidate) - plan.NewVersion = checkUpdateVersion(choice) - plan.Checked = true - return plan -} - -func checkUpdateVersion(choice UpdateChoice) string { - if choice.LatestStable != "" { - return choice.LatestStable - } - if choice.LatestPrerelease != "" { - return choice.LatestPrerelease - } - if len(choice.Versions) > 0 { - return choice.Versions[0] - } - return "" -} - -func orderedUpdatePlans(candidates []UpdateCandidate, plansByKey map[string]UpdatePlan) []UpdatePlan { - plans := make([]UpdatePlan, 0, len(plansByKey)) - for _, candidate := range candidates { - if plan, ok := plansByKey[candidate.key()]; ok { - plans = append(plans, plan) - } - } - return plans -} - -func updateManagers(managers []Manager) ([]Manager, error) { - if len(managers) == 0 { - return []Manager{ManagerGo, ManagerNPM, ManagerPNPM, ManagerImage, ManagerHelm}, nil - } - out := make([]Manager, 0, len(managers)) - var unsupported []string - for _, manager := range managers { - if !supportedUpdateManagers[manager] { - unsupported = append(unsupported, string(manager)) - continue - } - out = append(out, manager) - } - if len(unsupported) > 0 { - sort.Strings(unsupported) - return nil, fmt.Errorf("dependency updates currently support go, npm, pnpm, image, and helm; unsupported manager(s): %s", strings.Join(unsupported, ", ")) - } - return out, nil -} - -func packageUpdateManagers(managers []Manager) []Manager { - var out []Manager - for _, manager := range managers { - switch manager { - case ManagerGo, ManagerNPM, ManagerPNPM: - out = append(out, manager) - } - } - return out -} - -func imageUpdateManagers(managers []Manager) []Manager { - var out []Manager - for _, manager := range managers { - switch manager { - case ManagerImage, ManagerHelm: - out = append(out, manager) - } - } - return out -} - -func splitUpdatePatterns(values []string) []string { - var out []string - for _, value := range values { - for _, part := range strings.Split(value, ",") { - part = strings.TrimSpace(part) - if part != "" { - out = append(out, part) - } - } - } - return out -} - -func (c UpdateCandidate) matches(patterns []string) bool { - identityPatterns, pathPatterns := splitExplicitPathPatterns(patterns) - hasPositivePattern := identityPatterns.hasPositive || pathPatterns.hasPositive - matchedPositive := false - - for _, value := range c.matchValues() { - if value == "" { - continue - } - ok, negated := collections.MatchItem(value, identityPatterns.values...) - if negated { - return false - } - if ok && identityPatterns.hasPositive { - matchedPositive = true - } - } - if len(pathPatterns.values) > 0 { - ok, negated := collections.MatchItem(c.File, pathPatterns.values...) - if negated { - return false - } - if ok && pathPatterns.hasPositive { - matchedPositive = true - } - } - if hasPositivePattern { - return matchedPositive - } - return true -} - -func (c UpdateCandidate) matchValues() []string { - return []string{ - c.Name, - string(c.Manager), - fmt.Sprintf("%s:%s", c.Manager, c.Name), - fmt.Sprintf("%s:%s@%s", c.Manager, c.Name, c.Current), - c.Scope, - c.Current, - } -} - -type updatePatternSet struct { - values []string - hasPositive bool -} - -func splitExplicitPathPatterns(patterns []string) (identity updatePatternSet, path updatePatternSet) { - for _, pattern := range patterns { - pattern = strings.TrimSpace(pattern) - if pattern == "" { - continue - } - negated := strings.HasPrefix(pattern, "!") - body := strings.TrimPrefix(pattern, "!") - field, value, explicit := strings.Cut(body, ":") - if explicit && (field == "path" || field == "file") { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if strings.HasPrefix(value, "!") { - negated = true - value = strings.TrimPrefix(value, "!") - } - if negated { - value = "!" + value - } else { - path.hasPositive = true - } - path.values = append(path.values, value) - continue - } - if !negated { - identity.hasPositive = true - } - identity.values = append(identity.values, pattern) - } - return identity, path -} - -func (c UpdateCandidate) key() string { - return strings.Join([]string{string(c.Manager), c.Dir, c.File, c.Scope, c.Name}, "\x00") -} - -func (c UpdateCandidate) less(other UpdateCandidate) bool { - if c.Manager != other.Manager { - return c.Manager < other.Manager - } - if c.File != other.File { - return c.File < other.File - } - if c.Scope != other.Scope { - return c.Scope < other.Scope - } - return c.Name < other.Name -} - -func updateTaskName(candidate UpdateCandidate) string { - return fmt.Sprintf("%s %s", candidate.Manager, candidate.Name) -} - -func isLocalUpdateSpec(spec string) bool { - spec = strings.TrimSpace(spec) - if spec == "" { - return true - } - if strings.HasPrefix(spec, "workspace:") { - return true - } - return isLocalRef(spec) -} - -func sortDependencyVersions(versions []string) []string { - type parsed struct { - orig string - ver *semver.Version - } - seen := map[string]bool{} - var parsedVersions []parsed - for _, version := range versions { - version = strings.TrimSpace(version) - if version == "" || seen[version] { - continue - } - seen[version] = true - sv, err := semver.NewVersion(version) - if err != nil { - continue - } - parsedVersions = append(parsedVersions, parsed{orig: version, ver: sv}) - } - sort.Slice(parsedVersions, func(i, j int) bool { - return parsedVersions[i].ver.GreaterThan(parsedVersions[j].ver) - }) - out := make([]string, len(parsedVersions)) - for i, item := range parsedVersions { - out[i] = item.orig - } - return out -} - -func latestStableVersion(versions []string) string { - for _, version := range versions { - if !isPrerelease(version) { - return version - } - } - return "" -} - -func latestPrereleaseVersion(versions []string) string { - for _, version := range versions { - if isPrerelease(version) { - return version - } - } - return "" -} - -func isPrerelease(version string) bool { - sv, err := semver.NewVersion(version) - return err == nil && strings.TrimSpace(sv.Prerelease()) != "" -} - -func updateableVersions(current string, versions []string) []string { - current = normalizeCurrentVersion(current) - currentSemver, currentErr := semver.NewVersion(current) - out := make([]string, 0, len(versions)) - for _, version := range versions { - if version == "" || version == current { - continue - } - versionSemver, versionErr := semver.NewVersion(version) - if currentErr != nil || versionErr != nil { - out = append(out, version) - continue - } - if versionSemver.GreaterThan(currentSemver) { - out = append(out, version) - } - } - return out -} - -func selectedVersionIsCurrent(current, selected string) bool { - return normalizeCurrentVersion(current) == selected -} - -func normalizeCurrentVersion(current string) string { - current = strings.TrimSpace(current) - current = strings.TrimPrefix(current, "npm:") - current = strings.TrimLeft(current, "^~<>= ") - if idx := strings.IndexAny(current, " |,"); idx >= 0 { - current = current[:idx] - } - return strings.TrimSpace(current) -} +// version math, candidate matching, manager helpers, plan rendering, version +// resolution, apply, and prompts live in update_version.go, update_match.go, +// update_plan.go, update_resolve.go, update_apply.go, and update_prompt.go. diff --git a/deps/update_apply.go b/deps/update_apply.go new file mode 100644 index 0000000..ed674f3 --- /dev/null +++ b/deps/update_apply.go @@ -0,0 +1,86 @@ +package deps + +import ( + "context" + "fmt" +) + +func applyDependencyUpdate(ctx context.Context, candidate UpdateCandidate, version string, opts UpdateOptions) UpdatePlan { + if candidate.Manager == ManagerImage || candidate.Manager == ManagerHelm { + return applyImageTargetUpdate(ctx, candidate, version, opts) + } + plan := planFromCandidate(candidate) + plan.NewVersion = version + if selectedVersionIsCurrent(candidate.Current, version) { + plan.Skipped = "already at selected version" + return plan + } + cmd, err := updateCommand(candidate, version) + if err != nil { + plan.Skipped = err.Error() + return plan + } + plan.Command = append([]string{cmd.Name}, cmd.Args...) + if opts.DryRun { + plan.DryRun = true + return plan + } + runner := opts.Runner + if runner == nil { + runner = ExecRunner{} + } + if _, err := runner.Run(ctx, cmd); err != nil { + plan.Skipped = err.Error() + return plan + } + plan.Written = true + stageUpdatePlan(ctx, &plan, candidate, runner) + return plan +} + +func stageUpdatePlan(ctx context.Context, plan *UpdatePlan, candidate UpdateCandidate, runner CommandRunner) { + staged, err := stageUpdatedFiles(ctx, runner, candidate) + plan.Staged = staged + if err != nil { + plan.StageError = err.Error() + } +} + +func updateCommand(candidate UpdateCandidate, version string) (Command, error) { + target := candidate.Name + "@" + version + switch candidate.Manager { + case ManagerGo: + return Command{Dir: candidate.Dir, Name: "go", Args: []string{"get", target}}, nil + case ManagerNPM: + args := []string{"install", "--package-lock-only", "--ignore-scripts"} + if flag := packageSaveFlag(candidate.Scope); flag != "" { + args = append(args, flag) + } + args = append(args, target) + return Command{Dir: candidate.Dir, Name: "npm", Args: args}, nil + case ManagerPNPM: + args := []string{"add", "--lockfile-only", "--ignore-scripts"} + if flag := packageSaveFlag(candidate.Scope); flag != "" { + args = append(args, flag) + } + args = append(args, target) + return Command{Dir: candidate.Dir, Name: "pnpm", Args: args}, nil + default: + return Command{}, fmt.Errorf("package-manager updates do not support manager %q", candidate.Manager) + } +} + +func packageSaveFlag(scope string) string { + switch scope { + case "dependencies": + return "--save-prod" + case "devDependencies": + return "--save-dev" + case "optionalDependencies": + return "--save-optional" + case "peerDependencies": + return "--save-peer" + default: + return "" + } +} diff --git a/deps/update_image.go b/deps/update_image.go index 4e8bf26..5845dcc 100644 --- a/deps/update_image.go +++ b/deps/update_image.go @@ -2,27 +2,26 @@ package deps import ( "context" + "errors" "fmt" - "os/exec" "path/filepath" - "sort" "strings" "github.com/flanksource/repomap" "github.com/flanksource/repomap/imageupdate" - "github.com/flanksource/repomap/kubernetes" ) -func discoverImageUpdateCandidates(root string, managers []Manager) ([]UpdateCandidate, error) { +func discoverImageUpdateCandidates(root string, managers []Manager, filter DiscoverFilter) ([]UpdateCandidate, error) { selected := managerSet(managers) conf, err := repomap.GetConf(root) if err != nil { return nil, err } - targets, sourceIndex, err := discoverImageTargets(conf, root) + res, err := imageupdate.DiscoverRepoTargets(conf, root) if err != nil { return nil, err } + targets := imageupdate.Filter(res.Targets, filter.Matcher, filter.ImagePatterns, filter.ChartPatterns) var out []UpdateCandidate for _, target := range targets { @@ -30,11 +29,6 @@ func discoverImageUpdateCandidates(root string, managers []Manager) ([]UpdateCan if manager == "" || !selected[manager] { continue } - if target.Kind == imageupdate.TargetChart { - if err := sourceIndex.Resolve(&target); err != nil { - return nil, err - } - } targetCopy := target out = append(out, UpdateCandidate{ Manager: manager, @@ -49,80 +43,6 @@ func discoverImageUpdateCandidates(root string, managers []Manager) ([]UpdateCan return out, nil } -func discoverImageTargets(conf *repomap.ArchConf, scanPath string) ([]imageupdate.UpdateTarget, *imageupdate.SourceIndex, error) { - files, err := gitTrackedFiles(conf.RepoPath()) - if err != nil { - return nil, nil, fmt.Errorf("failed to list files: %w", err) - } - - var prefix string - if rel, err := filepath.Rel(conf.RepoPath(), scanPath); err == nil && rel != "." { - prefix = filepath.ToSlash(rel) + "/" - } - - contents := map[string]string{} - for _, file := range files { - if !kubernetes.IsYaml(file) { - continue - } - content, err := conf.ReadFileWithFallback(file, "") - if err != nil || content == "" { - continue - } - contents[file] = content - } - - tree := imageupdate.BuildKustomizeTree(contents) - sourceIndex := imageupdate.NewSourceIndex(tree) - - keys := make([]string, 0, len(contents)) - for file := range contents { - keys = append(keys, file) - } - sort.Strings(keys) - - var targets []imageupdate.UpdateTarget - for _, file := range keys { - content := contents[file] - _ = sourceIndex.IndexHelmRepositories(file, content) - if prefix != "" && !strings.HasPrefix(file, prefix) { - continue - } - fileTargets, err := imageupdate.ExtractTargets(file, content) - if err != nil { - continue - } - targets = append(targets, fileTargets...) - } - sort.SliceStable(targets, func(i, j int) bool { - if targets[i].File != targets[j].File { - return targets[i].File < targets[j].File - } - if targets[i].FieldLine != targets[j].FieldLine { - return targets[i].FieldLine < targets[j].FieldLine - } - return updateTargetName(targets[i]) < updateTargetName(targets[j]) - }) - return targets, sourceIndex, nil -} - -func gitTrackedFiles(repoPath string) ([]string, error) { - cmd := exec.Command("git", "-C", repoPath, "ls-files", "-z") - out, err := cmd.Output() - if err != nil { - return nil, err - } - var files []string - for _, file := range strings.Split(string(out), "\x00") { - file = strings.TrimSpace(file) - if file != "" { - files = append(files, filepath.ToSlash(file)) - } - } - sort.Strings(files) - return files, nil -} - func managerForUpdateTarget(target imageupdate.UpdateTarget) Manager { switch target.Kind { case imageupdate.TargetImage: @@ -142,7 +62,12 @@ func updateTargetName(target imageupdate.UpdateTarget) string { } return stripImageVersion(target.CurrentValue) case imageupdate.TargetChart: - return target.ChartName + if target.ChartName != "" { + return target.ChartName + } + // Unresolved chartRef: fall back to the referenced source name so the + // candidate still has an identity and its lookup fails loudly by name. + return target.ChartRefName default: return target.CurrentValue } @@ -210,6 +135,11 @@ func availableImageTargetVersions(ctx context.Context, resolver ImageVersionReso if candidate.Target == nil { return nil, "", "", fmt.Errorf("%s has no image or Helm target metadata", candidate.Name) } + // An unresolved Flux source leaves RepoURL empty; surface the actionable + // resolution error rather than querying an empty registry/Helm URL. + if candidate.Target.SourceErr != "" { + return nil, "", "", errors.New(candidate.Target.SourceErr) + } if resolver == nil { resolver = imageupdate.NewResolver() } diff --git a/deps/update_match.go b/deps/update_match.go new file mode 100644 index 0000000..fd564e0 --- /dev/null +++ b/deps/update_match.go @@ -0,0 +1,212 @@ +package deps + +import ( + "fmt" + "sort" + "strings" + + "github.com/flanksource/commons/collections" + + "github.com/flanksource/repomap/imageupdate" +) + +func updateManagers(managers []Manager) ([]Manager, error) { + if len(managers) == 0 { + return []Manager{ManagerGo, ManagerNPM, ManagerPNPM, ManagerImage, ManagerHelm}, nil + } + out := make([]Manager, 0, len(managers)) + var unsupported []string + for _, manager := range managers { + if !supportedUpdateManagers[manager] { + unsupported = append(unsupported, string(manager)) + continue + } + out = append(out, manager) + } + if len(unsupported) > 0 { + sort.Strings(unsupported) + return nil, fmt.Errorf("dependency updates currently support go, npm, pnpm, image, and helm; unsupported manager(s): %s", strings.Join(unsupported, ", ")) + } + return out, nil +} + +func packageUpdateManagers(managers []Manager) []Manager { + var out []Manager + for _, manager := range managers { + switch manager { + case ManagerGo, ManagerNPM, ManagerPNPM: + out = append(out, manager) + } + } + return out +} + +func imageUpdateManagers(managers []Manager) []Manager { + var out []Manager + for _, manager := range managers { + switch manager { + case ManagerImage, ManagerHelm: + out = append(out, manager) + } + } + return out +} + +func splitUpdatePatterns(values []string) []string { + var out []string + for _, value := range values { + for _, part := range strings.Split(value, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + } + return out +} + +func (c UpdateCandidate) matches(patterns []string) bool { + identityPatterns, pathPatterns := splitExplicitPathPatterns(patterns) + hasPositivePattern := identityPatterns.hasPositive || pathPatterns.hasPositive + matchedPositive := false + + for _, value := range c.matchValues() { + if value == "" { + continue + } + ok, negated := collections.MatchItem(value, identityPatterns.values...) + if negated { + return false + } + if ok && identityPatterns.hasPositive { + matchedPositive = true + } + } + if len(pathPatterns.values) > 0 { + ok, negated := collections.MatchItem(c.File, pathPatterns.values...) + if negated { + return false + } + if ok && pathPatterns.hasPositive { + matchedPositive = true + } + } + if hasPositivePattern { + return matchedPositive + } + return true +} + +func (c UpdateCandidate) matchValues() []string { + return []string{ + c.Name, + string(c.Manager), + fmt.Sprintf("%s:%s", c.Manager, c.Name), + fmt.Sprintf("%s:%s@%s", c.Manager, c.Name, c.Current), + c.Scope, + c.Current, + } +} + +type updatePatternSet struct { + values []string + hasPositive bool +} + +func splitExplicitPathPatterns(patterns []string) (identity updatePatternSet, path updatePatternSet) { + for _, pattern := range patterns { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + negated := strings.HasPrefix(pattern, "!") + body := strings.TrimPrefix(pattern, "!") + field, value, explicit := strings.Cut(body, ":") + if explicit && (field == "path" || field == "file") { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if strings.HasPrefix(value, "!") { + negated = true + value = strings.TrimPrefix(value, "!") + } + if negated { + value = "!" + value + } else { + path.hasPositive = true + } + path.values = append(path.values, value) + continue + } + if !negated { + identity.hasPositive = true + } + identity.values = append(identity.values, pattern) + } + return identity, path +} + +func (c UpdateCandidate) key() string { + return strings.Join([]string{string(c.Manager), c.Dir, c.File, c.Scope, c.Name}, "\x00") +} + +// resolutionKey groups candidates whose published-version lookup yields the same +// result, so one network round-trip can serve every occurrence. Image tags come +// from the repository, chart versions from the chart+repo source, and package +// versions from the registry resolved in the project dir. +func (c UpdateCandidate) resolutionKey() string { + switch c.Manager { + case ManagerImage: + return strings.Join([]string{string(ManagerImage), c.Name}, "\x00") + case ManagerHelm: + return strings.Join([]string{string(ManagerHelm), c.Name, helmSourceKey(c.Target)}, "\x00") + default: + return strings.Join([]string{string(c.Manager), c.Name, c.Dir}, "\x00") + } +} + +// helmSourceKey identifies the chart repository a Helm candidate resolves +// against; charts sharing a name but a different repo (or an unresolved source) +// must not collapse onto the same lookup. +func helmSourceKey(target *imageupdate.UpdateTarget) string { + if target == nil { + return "" + } + if target.SourceErr != "" { + return "err:" + target.SourceErr + } + key := target.RepoURL + if target.IsOCI { + key += "|oci" + } + return key +} + +func (c UpdateCandidate) less(other UpdateCandidate) bool { + if c.Manager != other.Manager { + return c.Manager < other.Manager + } + if c.File != other.File { + return c.File < other.File + } + if c.Scope != other.Scope { + return c.Scope < other.Scope + } + return c.Name < other.Name +} + +func updateTaskName(candidate UpdateCandidate) string { + return fmt.Sprintf("%s %s", candidate.Manager, candidate.Name) +} + +func isLocalUpdateSpec(spec string) bool { + spec = strings.TrimSpace(spec) + if spec == "" { + return true + } + if strings.HasPrefix(spec, "workspace:") { + return true + } + return isLocalRef(spec) +} diff --git a/deps/update_modes.go b/deps/update_modes.go new file mode 100644 index 0000000..8b3b437 --- /dev/null +++ b/deps/update_modes.go @@ -0,0 +1,68 @@ +package deps + +import ( + "context" + "fmt" +) + +// applyExplicitVersionUpdates applies opts.Version to every candidate. For +// image/helm targets it first validates the version is published; package-manager +// updates pass the version straight to the manager, which validates it. +func applyExplicitVersionUpdates(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) []UpdatePlan { + plansByKey := make(map[string]UpdatePlan, len(candidates)) + for _, candidate := range candidates { + plansByKey[candidate.key()] = applyExplicitVersion(ctx, candidate, opts) + } + return orderedUpdatePlans(candidates, plansByKey) +} + +func applyExplicitVersion(ctx context.Context, candidate UpdateCandidate, opts UpdateOptions) UpdatePlan { + if selectedVersionIsCurrent(candidate.Current, opts.Version) { + return skippedUpdatePlan(candidate, "already at selected version") + } + if candidate.Manager == ManagerImage || candidate.Manager == ManagerHelm { + available, _, _, err := availableImageTargetVersions(ctx, opts.ImageResolver, candidate) + if err != nil { + return skippedUpdatePlan(candidate, err.Error()) + } + if !containsString(available, opts.Version) { + return skippedUpdatePlan(candidate, fmt.Sprintf("version %q is not available", opts.Version)) + } + } + return applyDependencyUpdate(ctx, candidate, opts.Version, opts) +} + +// applyLatestUpdates fills plansByKey with the highest-stable update for each +// candidate that has one, recording "already up to date" for the rest. Candidates +// whose version lookup already failed (a skipped plan exists) are left untouched. +func applyLatestUpdates(ctx context.Context, candidates []UpdateCandidate, choices []UpdateChoice, plansByKey map[string]UpdatePlan, opts UpdateOptions) { + choiceByKey := make(map[string]UpdateChoice, len(choices)) + for _, choice := range choices { + choiceByKey[choice.Candidate.key()] = choice + } + for _, candidate := range candidates { + key := candidate.key() + if _, done := plansByKey[key]; done { + continue + } + choice, ok := choiceByKey[key] + if !ok { + plansByKey[key] = skippedUpdatePlan(candidate, "already up to date") + continue + } + if choice.LatestStable == "" { + plansByKey[key] = skippedUpdatePlan(candidate, "no stable version available") + continue + } + plansByKey[key] = applyDependencyUpdate(ctx, candidate, choice.LatestStable, opts) + } +} + +func containsString(values []string, target string) bool { + for _, v := range values { + if v == target { + return true + } + } + return false +} diff --git a/deps/update_modes_test.go b/deps/update_modes_test.go new file mode 100644 index 0000000..5cf1851 --- /dev/null +++ b/deps/update_modes_test.go @@ -0,0 +1,258 @@ +package deps + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +const chartRefOCIFixture = `apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: podinfo + namespace: apps +spec: + chartRef: + kind: OCIRepository + name: podinfo +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: podinfo + namespace: apps +spec: + url: oci://ghcr.io/stefanprodan/charts/podinfo + ref: + tag: 6.5.0 +` + +func setupImageRepo(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + for rel, content := range files { + writeFile(t, filepath.Join(dir, rel), content) + } + runGit(t, dir, "add", ".") + return dir +} + +func TestUpdate_LatestSelectsHighestStable(t *testing.T) { + setupImageRepo(t, map[string]string{"apps/workloads.yaml": deploymentUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Latest: true, + DryRun: true, + ImageResolver: fakeImageVersionResolver{ + "nginx": {"1.28.0-rc.1", "1.27.0", "1.25.3"}, + "ghcr.io/flanksource/proxy": {"v0.4.1"}, + }, + }) + if err != nil { + t.Fatal(err) + } + + nginx := findPlan(plans, "nginx") + if nginx == nil || nginx.NewVersion != "1.27.0" || !nginx.DryRun || nginx.Written { + t.Fatalf("nginx plan = %#v, want 1.27.0 dry-run (skipping the rc prerelease)", nginx) + } + proxy := findPlan(plans, "ghcr.io/flanksource/proxy") + if proxy == nil || proxy.Skipped != "already up to date" { + t.Fatalf("proxy plan = %#v, want skipped already up to date", proxy) + } +} + +func TestUpdate_ExplicitVersionWritesEdit(t *testing.T) { + dir := setupImageRepo(t, map[string]string{"apps/workloads.yaml": deploymentUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Image: []string{"nginx"}, + Version: "1.27.0", + ImageResolver: fakeImageVersionResolver{"nginx": {"1.27.0", "1.25.3"}}, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 { + t.Fatalf("plans = %#v, want only the nginx candidate (image filter)", plans) + } + if !plans[0].Written || plans[0].NewVersion != "1.27.0" { + t.Fatalf("plan = %#v, want written 1.27.0", plans[0]) + } + got := readFileString(t, filepath.Join(dir, "apps", "workloads.yaml")) + if !strings.Contains(got, "image: nginx:1.27.0") { + t.Fatalf("manifest not updated:\n%s", got) + } + if !strings.Contains(got, "image: ghcr.io/flanksource/proxy:v0.4.1") { + t.Fatalf("image filter must leave the sidecar untouched:\n%s", got) + } +} + +func TestUpdate_ExplicitVersionRejectsUnavailable(t *testing.T) { + setupImageRepo(t, map[string]string{"apps/workloads.yaml": deploymentUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Image: []string{"nginx"}, + Version: "9.9.9", + ImageResolver: fakeImageVersionResolver{"nginx": {"1.27.0", "1.25.3"}}, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 || plans[0].Written { + t.Fatalf("plans = %#v, want one unwritten plan", plans) + } + if !strings.Contains(plans[0].Skipped, "not available") { + t.Fatalf("skip = %q, want 'not available'", plans[0].Skipped) + } +} + +func TestUpdate_ResourceFilterKindNarrowsToHelm(t *testing.T) { + setupImageRepo(t, map[string]string{ + "apps/workloads.yaml": deploymentUpdateFixture, + "apps/helmrelease.yaml": helmReleaseUpdateFixture, + }) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage, ManagerHelm}, + Kind: []string{"HelmRelease"}, + Check: true, + ImageResolver: fakeImageVersionResolver{"podinfo": {"6.6.0", "6.5.0"}}, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 || plans[0].Manager != ManagerHelm || plans[0].Name != "podinfo" { + t.Fatalf("plans = %#v, want only the HelmRelease chart", plans) + } + if plans[0].NewVersion != "6.6.0" || !plans[0].Checked { + t.Fatalf("plan = %#v, want checked 6.6.0", plans[0]) + } +} + +func TestUpdate_DedupesVersionLookupsAcrossDuplicates(t *testing.T) { + setupImageRepo(t, map[string]string{ + "apps/a.yaml": deploymentUpdateFixture, + "apps/b.yaml": deploymentUpdateFixture, // the same nginx + proxy in a second file + }) + resolver := newCountingImageVersionResolver(map[string][]string{ + "nginx": {"1.27.0", "1.25.3"}, + "ghcr.io/flanksource/proxy": {"v0.5.0", "v0.4.1"}, + }) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Check: true, + ImageResolver: resolver, + }) + if err != nil { + t.Fatal(err) + } + // Two images each appear in two files, so four updates are planned but each + // image's published versions must be looked up exactly once. + if len(plans) != 4 { + t.Fatalf("plans = %d, want 4 (two images × two files): %#v", len(plans), plans) + } + if got := resolver.callCount("nginx"); got != 1 { + t.Fatalf("nginx version lookups = %d, want 1 deduped lookup", got) + } + if got := resolver.callCount("ghcr.io/flanksource/proxy"); got != 1 { + t.Fatalf("proxy version lookups = %d, want 1 deduped lookup", got) + } +} + +func TestUpdate_FilterThenChecksOnlyFilteredSet(t *testing.T) { + setupImageRepo(t, map[string]string{ + "apps/workloads.yaml": deploymentUpdateFixture, + "apps/helmrelease.yaml": helmReleaseUpdateFixture, + }) + resolver := newCountingImageVersionResolver(map[string][]string{ + "nginx": {"1.27.0", "1.25.3"}, + "ghcr.io/flanksource/proxy": {"v0.5.0", "v0.4.1"}, + "podinfo": {"6.6.0", "6.5.0"}, + }) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage, ManagerHelm}, + Kind: []string{"HelmRelease"}, + Check: true, + ImageResolver: resolver, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 || plans[0].Manager != ManagerHelm || plans[0].Name != "podinfo" { + t.Fatalf("plans = %#v, want only the filtered HelmRelease chart", plans) + } + // The kind filter must restrict the lookups: Deployment images are never queried. + if got := resolver.callCount("nginx") + resolver.callCount("ghcr.io/flanksource/proxy"); got != 0 { + t.Fatalf("filtered-out image lookups = %d, want 0 (filter before check)", got) + } + if got := resolver.callCount("podinfo"); got != 1 { + t.Fatalf("podinfo lookups = %d, want 1", got) + } +} + +func TestUpdate_NoFiltersMatchesAll(t *testing.T) { + setupImageRepo(t, map[string]string{"apps/helmrelease.yaml": helmReleaseUpdateFixture}) + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerHelm}, + Check: true, + ImageResolver: fakeImageVersionResolver{"podinfo": {"6.6.0", "6.5.0"}}, + }) + if err != nil { + t.Fatalf("empty expression should match all, got %v", err) + } + if len(plans) != 1 || plans[0].Name != "podinfo" { + t.Fatalf("plans = %#v, want podinfo", plans) + } +} + +func TestDiscoverUpdateCandidates_ChartRefOCI(t *testing.T) { + setupImageRepo(t, map[string]string{"clusters/app.yaml": chartRefOCIFixture}) + + got, err := DiscoverUpdateCandidates(".", []Manager{ManagerHelm}, DiscoverFilter{}) + if err != nil { + t.Fatal(err) + } + helm := findUpdateCandidate(got, ManagerHelm, "podinfo") + if helm == nil { + t.Fatalf("chartRef OCIRepository not discovered: %#v", got) + } + if helm.Current != "6.5.0" { + t.Errorf("current = %q, want 6.5.0 (from OCIRepository spec.ref.tag)", helm.Current) + } + if helm.Target == nil || helm.Target.RepoURL != "oci://ghcr.io/stefanprodan/charts" || !helm.Target.IsOCI { + t.Errorf("target repo not resolved: %#v", helm.Target) + } + // The edit anchor must be the OCIRepository file, not the HelmRelease. + if helm.Target.File != "clusters/app.yaml" || helm.Target.FieldJSONPath != "$.spec.ref.tag" { + t.Errorf("anchor = %s %s, want clusters/app.yaml $.spec.ref.tag", helm.Target.File, helm.Target.FieldJSONPath) + } +} + +func findPlan(plans []UpdatePlan, name string) *UpdatePlan { + for i := range plans { + if plans[i].Name == name { + return &plans[i] + } + } + return nil +} + +func readFileString(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} diff --git a/deps/update_plan.go b/deps/update_plan.go new file mode 100644 index 0000000..c78c4ce --- /dev/null +++ b/deps/update_plan.go @@ -0,0 +1,131 @@ +package deps + +import ( + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (p UpdatePlan) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("[%s] %s", p.Manager, p.Name), managerStyle(p.Manager)) + if p.OldVersion != "" || p.NewVersion != "" { + t = t.Space().Append(p.OldVersion, "font-mono text-muted") + if p.NewVersion != "" { + t = t.Append(" -> ", "text-muted").Append(p.NewVersion, "font-mono text-green-600") + } + } + switch { + case p.Skipped != "": + t = t.Space().Append("skipped: "+p.Skipped, "text-muted") + case p.Checked: + t = t.Space().Append("update available", "text-green-600") + case p.DryRun: + t = t.Space().Append("(dry-run)", "text-yellow-600") + case p.Written: + t = t.Space().Append("written", "text-green-600") + } + if len(p.Staged) > 0 { + t = t.Space().Append("staged "+strings.Join(p.Staged, ", "), "text-muted") + } + if p.StageError != "" { + t = t.Space().Append("stage failed: "+p.StageError, "text-yellow-600") + } + return t +} + +func (UpdatePlan) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("manager").Label("Manager").Build(), + api.Column("dependency").Label("Dependency").Build(), + api.Column("file").Label("File").Build(), + api.Column("scope").Label("Scope").Build(), + api.Column("change").Label("Change").Build(), + api.Column("status").Label("Status").Build(), + } +} + +func (p UpdatePlan) Row() map[string]any { + row := map[string]any{ + "manager": clicky.Text(string(p.Manager), managerStyle(p.Manager)), + "dependency": clicky.Text(p.Name, "font-bold text-cyan-600"), + "file": clicky.Text(p.File, "font-mono"), + "scope": clicky.Text(p.Scope, "text-muted"), + "change": updateChangeText(p.OldVersion, p.NewVersion), + } + switch { + case p.Skipped != "": + row["status"] = clicky.Text(p.Skipped, "text-muted") + case p.Checked: + row["status"] = clicky.Text("update available", "text-green-600") + case p.DryRun: + row["status"] = clicky.Text("dry-run", "text-yellow-600") + case p.Written: + status := clicky.Text("written", "text-green-600") + if len(p.Staged) > 0 { + status = status.Append(" + staged "+strings.Join(p.Staged, ", "), "text-muted") + } + if p.StageError != "" { + status = status.Append(" (stage failed: "+p.StageError+")", "text-yellow-600") + } + row["status"] = status + default: + row["status"] = clicky.Text("") + } + return row +} + +func updateChangeText(oldVersion, newVersion string) api.Text { + text := clicky.Text(oldVersion, "font-mono text-muted") + if newVersion != "" { + text = text.Append(" -> ", "text-muted").Append(newVersion, "font-mono text-green-600") + } + return text +} + +func planFromCandidate(candidate UpdateCandidate) UpdatePlan { + return UpdatePlan{ + Manager: candidate.Manager, + Name: candidate.Name, + File: candidate.File, + Scope: candidate.Scope, + OldVersion: candidate.Current, + } +} + +func skippedUpdatePlan(candidate UpdateCandidate, reason string) UpdatePlan { + plan := planFromCandidate(candidate) + plan.Skipped = reason + return plan +} + +func checkUpdatePlan(choice UpdateChoice) UpdatePlan { + plan := planFromCandidate(choice.Candidate) + plan.NewVersion = checkUpdateVersion(choice) + plan.Checked = true + return plan +} + +func checkUpdateVersion(choice UpdateChoice) string { + if choice.LatestStable != "" { + return choice.LatestStable + } + if choice.LatestPrerelease != "" { + return choice.LatestPrerelease + } + if len(choice.Versions) > 0 { + return choice.Versions[0] + } + return "" +} + +func orderedUpdatePlans(candidates []UpdateCandidate, plansByKey map[string]UpdatePlan) []UpdatePlan { + plans := make([]UpdatePlan, 0, len(plansByKey)) + for _, candidate := range candidates { + if plan, ok := plansByKey[candidate.key()]; ok { + plans = append(plans, plan) + } + } + return plans +} diff --git a/deps/update_prompt.go b/deps/update_prompt.go new file mode 100644 index 0000000..a054934 --- /dev/null +++ b/deps/update_prompt.go @@ -0,0 +1,120 @@ +package deps + +import ( + "fmt" + "sort" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func promptUpdateCandidates(choices []UpdateChoice) ([]UpdateChoice, bool) { + return runUpdateChoiceTreePicker(choices) +} + +// versionPromptKey identifies occurrences that pose the identical version +// question: the same dependency, at the same current version, with the same +// available versions. Occurrences that differ in any of those are genuinely +// different decisions and stay separate prompts. +func (c UpdateChoice) versionPromptKey() string { + parts := append([]string{string(c.Candidate.Manager), c.Candidate.Name, c.Candidate.Current}, c.Versions...) + return strings.Join(parts, "\x00") +} + +// groupChoicesForVersionPrompt collapses occurrences sharing a version question +// into one group, keeping the input order of each group's first occurrence. +func groupChoicesForVersionPrompt(choices []UpdateChoice) [][]UpdateChoice { + indexByKey := map[string]int{} + var groups [][]UpdateChoice + for _, choice := range choices { + key := choice.versionPromptKey() + if i, ok := indexByKey[key]; ok { + groups[i] = append(groups[i], choice) + continue + } + indexByKey[key] = len(groups) + groups = append(groups, []UpdateChoice{choice}) + } + return groups +} + +func newUpdateVersionPrompt(group []UpdateChoice) UpdateVersionPrompt { + files := make([]string, 0, len(group)) + seen := map[string]bool{} + for _, choice := range group { + if file := choice.Candidate.File; !seen[file] { + seen[file] = true + files = append(files, file) + } + } + return UpdateVersionPrompt{UpdateChoice: group[0], Files: files} +} + +func promptUpdateVersion(prompt UpdateVersionPrompt) (string, bool) { + candidate := prompt.Candidate + location := candidate.File + if len(prompt.Files) > 1 { + location = fmt.Sprintf("%d files", len(prompt.Files)) + } + title := fmt.Sprintf("Select version for %s in %s (current %s)", candidate.Name, location, candidate.Current) + return clicky.PromptSelect(prompt.Versions, clicky.PromptSelectOptions[string]{ + Title: title, + PageSize: 12, + Render: func(version string) api.Textable { + text := clicky.Text(version, "font-mono") + var tags []string + if selectedVersionIsCurrent(candidate.Current, version) { + tags = append(tags, "current") + } + if version == prompt.LatestStable { + tags = append(tags, "latest stable") + } + if version == prompt.LatestPrerelease { + tags = append(tags, "latest pre-release") + } + if isPrerelease(version) { + tags = append(tags, "pre-release") + } + if len(tags) > 0 { + text = text.Space().Append("("+strings.Join(tags, ", ")+")", "text-muted") + } + return text + }, + }) +} + +type updateChoiceFileGroup struct { + File string + Choices []UpdateChoice +} + +func groupUpdateChoicesByFile(choices []UpdateChoice) []updateChoiceFileGroup { + byFile := map[string][]UpdateChoice{} + for _, choice := range choices { + file := choice.Candidate.File + byFile[file] = append(byFile[file], choice) + } + files := make([]string, 0, len(byFile)) + for file := range byFile { + files = append(files, file) + } + sort.Strings(files) + groups := make([]updateChoiceFileGroup, 0, len(files)) + for _, file := range files { + groupChoices := append([]UpdateChoice(nil), byFile[file]...) + sort.SliceStable(groupChoices, func(i, j int) bool { + return groupChoices[i].Candidate.less(groupChoices[j].Candidate) + }) + groups = append(groups, updateChoiceFileGroup{File: file, Choices: groupChoices}) + } + return groups +} + +func sortSelectedUpdateChoicesByFile(choices []UpdateChoice) []UpdateChoice { + var out []UpdateChoice + for _, group := range groupUpdateChoicesByFile(choices) { + out = append(out, group.Choices...) + } + return out +} diff --git a/deps/update_prompt_test.go b/deps/update_prompt_test.go new file mode 100644 index 0000000..6d05815 --- /dev/null +++ b/deps/update_prompt_test.go @@ -0,0 +1,185 @@ +package deps + +import ( + "context" + "path/filepath" + "strings" + "sync" + "testing" +) + +// A dependency declared in several manifests is one decision, so the version is +// confirmed once and the answer written to every occurrence. +func TestUpdatePromptsOnceForDependencyRepeatedAcrossFiles(t *testing.T) { + dir := t.TempDir() + for _, project := range []string{"api", "web"} { + writeFile(t, filepath.Join(dir, project, "package.json"), `{ + "name": "`+project+`", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, project, "package-lock.json"), `{"lockfileVersion": 3}`) + } + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "npm view left-pad versions --json": {Stdout: `["1.3.0","1.4.0"]`}, + }, + } + + var ( + mu sync.Mutex + prompts []UpdateVersionPrompt + ) + plans, err := Update(context.Background(), dir, UpdateOptions{ + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, + SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { + if len(choices) != 2 { + t.Fatalf("both occurrences should be selectable, got %d", len(choices)) + } + return choices, true + }, + SelectVersion: func(prompt UpdateVersionPrompt) (string, bool) { + mu.Lock() + defer mu.Unlock() + prompts = append(prompts, prompt) + return "1.4.0", true + }, + }) + if err != nil { + t.Fatal(err) + } + if len(prompts) != 1 { + t.Fatalf("version prompts = %d, want 1: %#v", len(prompts), prompts) + } + // UpdateCandidate.File is cwd-relative, so assert on the manifest suffixes. + gotFiles := prompts[0].Files + wantSuffixes := []string{"api/package.json", "web/package.json"} + if len(gotFiles) != len(wantSuffixes) { + t.Fatalf("prompt files = %#v, want one entry per manifest %#v", gotFiles, wantSuffixes) + } + for i, suffix := range wantSuffixes { + if !strings.HasSuffix(filepath.ToSlash(gotFiles[i]), suffix) { + t.Fatalf("prompt file[%d] = %q, want a path ending in %q", i, gotFiles[i], suffix) + } + } + if len(plans) != 2 { + t.Fatalf("plans = %d, want 2: %#v", len(plans), plans) + } + for _, plan := range plans { + if plan.NewVersion != "1.4.0" || plan.Skipped != "" { + t.Fatalf("single answer should apply to every occurrence, got %#v", plan) + } + } +} + +// Declining the shared prompt skips every occurrence it covered. +func TestUpdateSharedPromptCancellationSkipsAllOccurrences(t *testing.T) { + dir := t.TempDir() + for _, project := range []string{"api", "web"} { + writeFile(t, filepath.Join(dir, project, "package.json"), `{ + "name": "`+project+`", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, project, "package-lock.json"), `{"lockfileVersion": 3}`) + } + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "npm view left-pad versions --json": {Stdout: `["1.3.0","1.4.0"]`}, + }, + } + + plans, err := Update(context.Background(), dir, UpdateOptions{ + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, + SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { + return choices, true + }, + SelectVersion: func(UpdateVersionPrompt) (string, bool) { + return "", false + }, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 2 { + t.Fatalf("plans = %d, want 2: %#v", len(plans), plans) + } + for _, plan := range plans { + if plan.Skipped != "no version selected" || plan.Written { + t.Fatalf("declined prompt should skip every occurrence, got %#v", plan) + } + } +} + +func TestGroupChoicesForVersionPrompt(t *testing.T) { + choice := func(name, current, file string, versions ...string) UpdateChoice { + return UpdateChoice{ + Candidate: UpdateCandidate{Manager: ManagerNPM, Name: name, Current: current, File: file}, + Versions: versions, + } + } + cases := []struct { + name string + choices []UpdateChoice + want [][]string + }{ + { + name: "same dependency in different files shares one prompt", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.3.0", "web/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json", "web/package.json"}}, + }, + { + name: "different current versions are different decisions", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.2.0", "web/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json"}, {"web/package.json"}}, + }, + { + name: "different available versions are different decisions", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.3.0", "web/package.json", "1.4.0", "1.5.0"), + }, + want: [][]string{{"api/package.json"}, {"web/package.json"}}, + }, + { + name: "different dependencies never merge", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("right-pad", "^1.3.0", "api/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json"}, {"api/package.json"}}, + }, + { + name: "two occurrences in one file collapse to a single file entry", + choices: []UpdateChoice{ + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + choice("left-pad", "^1.3.0", "api/package.json", "1.4.0"), + }, + want: [][]string{{"api/package.json"}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + groups := groupChoicesForVersionPrompt(tc.choices) + if len(groups) != len(tc.want) { + t.Fatalf("groups = %d, want %d", len(groups), len(tc.want)) + } + for i, group := range groups { + got := newUpdateVersionPrompt(group).Files + if strings.Join(got, ",") != strings.Join(tc.want[i], ",") { + t.Fatalf("group[%d] files = %#v, want %#v", i, got, tc.want[i]) + } + } + }) + } +} diff --git a/deps/update_resolve.go b/deps/update_resolve.go new file mode 100644 index 0000000..4995dbd --- /dev/null +++ b/deps/update_resolve.go @@ -0,0 +1,169 @@ +package deps + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/flanksource/clicky/task" + flanksourceContext "github.com/flanksource/commons/context" +) + +func resolveUpdateChoices(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) ([]UpdateChoice, map[string]UpdatePlan) { + rawByKey := resolveRawVersionsByKey(ctx, candidates, opts) + + plansByKey := map[string]UpdatePlan{} + choices := make([]UpdateChoice, 0, len(candidates)) + for _, candidate := range candidates { + raw := rawByKey[candidate.resolutionKey()] + if raw.err != nil { + plansByKey[candidate.key()] = skippedUpdatePlan(candidate, raw.err.Error()) + continue + } + // The "newer than current" filtering is per occurrence: duplicates of the + // same dependency may sit at different current versions even though they + // share one published-version list. + versions := updateableVersions(candidate.Current, raw.versions) + if len(versions) == 0 { + continue + } + choices = append(choices, UpdateChoice{ + Candidate: candidate, + Versions: versions, + LatestStable: latestStableVersion(versions), + LatestPrerelease: latestPrereleaseVersion(versions), + }) + } + return choices, plansByKey +} + +type rawVersionResult struct { + versions []string + err error +} + +// resolveRawVersionsByKey looks up the published versions for every distinct +// dependency source among candidates, running one lookup per source and sharing +// it across duplicate occurrences. The same image repository, chart+repo, or +// package resolves identically wherever it is referenced, so collapsing them +// avoids redundant registry/proxy round-trips. The result is keyed by +// UpdateCandidate.resolutionKey. +func resolveRawVersionsByKey(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) map[string]rawVersionResult { + keys := make([]string, 0, len(candidates)) + rep := map[string]UpdateCandidate{} + for _, candidate := range candidates { + key := candidate.resolutionKey() + if _, ok := rep[key]; !ok { + rep[key] = candidate + keys = append(keys, key) + } + } + + results := make([]rawVersionResult, len(keys)) + group := task.StartGroup[int]("Resolving dependency versions", task.WithConcurrency(updateResolveConcurrency)) + for i, key := range keys { + idx, candidate := i, rep[key] + group.Add(updateTaskName(candidate), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { + tk.Infof("looking up published versions") + versions, err := resolveCandidateRawVersions(ctx, opts, candidate) + results[idx] = rawVersionResult{versions: versions, err: err} + if err != nil { + tk.Warnf("%s", err.Error()) + tk.Warning() + } else { + tk.Success() + } + return idx, nil + }) + } + _, _ = group.GetResults() + + byKey := make(map[string]rawVersionResult, len(keys)) + for i, key := range keys { + byKey[key] = results[i] + } + return byKey +} + +// resolveCandidateRawVersions returns a candidate source's published versions +// without the per-occurrence "newer than current" filtering, so the result can +// be cached and reused across duplicate occurrences. +func resolveCandidateRawVersions(ctx context.Context, opts UpdateOptions, candidate UpdateCandidate) ([]string, error) { + switch candidate.Manager { + case ManagerImage, ManagerHelm: + versions, _, _, err := availableImageTargetVersions(ctx, opts.ImageResolver, candidate) + return versions, err + default: + return AvailableDependencyVersions(ctx, opts.Runner, candidate) + } +} + +func AvailableDependencyVersions(ctx context.Context, runner CommandRunner, candidate UpdateCandidate) ([]string, error) { + if runner == nil { + runner = ExecRunner{} + } + var ( + result CommandResult + err error + ) + switch candidate.Manager { + case ManagerGo: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "go", + Args: []string{"list", "-m", "-versions", "-json", candidate.Name}, + Env: []string{"GOFLAGS=-mod=readonly"}, + }) + case ManagerNPM: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "npm", + Args: []string{"view", candidate.Name, "versions", "--json"}, + }) + case ManagerPNPM: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "pnpm", + Args: []string{"view", candidate.Name, "versions", "--json"}, + }) + default: + return nil, fmt.Errorf("dependency version lookup does not support manager %q", candidate.Manager) + } + if err != nil { + if result.Stderr != "" { + return nil, fmt.Errorf("%s: %w", strings.TrimSpace(result.Stderr), err) + } + return nil, err + } + versions, err := parseAvailableVersions(candidate.Manager, result.Stdout) + if err != nil { + return nil, err + } + return sortDependencyVersions(versions), nil +} + +func parseAvailableVersions(manager Manager, stdout string) ([]string, error) { + stdout = strings.TrimSpace(stdout) + if stdout == "" { + return nil, nil + } + if manager == ManagerGo { + var payload struct { + Versions []string `json:"Versions"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + return nil, err + } + return payload.Versions, nil + } + var list []string + if err := json.Unmarshal([]byte(stdout), &list); err == nil { + return list, nil + } + var single string + if err := json.Unmarshal([]byte(stdout), &single); err == nil && single != "" { + return []string{single}, nil + } + return nil, fmt.Errorf("expected JSON version array") +} diff --git a/deps/update_test.go b/deps/update_test.go index ef85456..e1f7598 100644 --- a/deps/update_test.go +++ b/deps/update_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "testing" "github.com/flanksource/repomap/imageupdate" @@ -38,7 +39,7 @@ replace github.com/acme/local => ../local }`) writeFile(t, filepath.Join(dir, "pnpm-app", "pnpm-lock.yaml"), `lockfileVersion: '9.0'`) - got, err := DiscoverUpdateCandidates(dir, nil) + got, err := DiscoverUpdateCandidates(dir, nil, DiscoverFilter{}) if err != nil { t.Fatal(err) } @@ -69,7 +70,7 @@ require github.com/acme/direct v1.2.3 }`) writeFile(t, filepath.Join(dir, "web", "package-lock.json"), `{"lockfileVersion": 3}`) - got, err := DiscoverUpdateCandidates(".", nil) + got, err := DiscoverUpdateCandidates(".", nil, DiscoverFilter{}) if err != nil { t.Fatal(err) } @@ -85,7 +86,7 @@ require github.com/acme/direct v1.2.3 } } -func TestUpdateCandidateMatchesMatchItemExpression(t *testing.T) { +func TestUpdateCandidateMatchesMatchItemFilter(t *testing.T) { candidates := []UpdateCandidate{ {Manager: ManagerGo, Name: "github.com/acme/lib", Current: "v1.0.0", File: "/work/flanksource/app/go.mod"}, {Manager: ManagerGo, Name: "github.com/flanksource/lib", Current: "v1.0.0", File: "/work/other/go.mod"}, @@ -159,14 +160,14 @@ func TestUpdateDryRunBuildsPackageManagerCommand(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - DryRun: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(choice UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.4.0", true }, }) @@ -202,10 +203,10 @@ func TestUpdateSkipsCandidatesWithoutChangesBeforePrompt(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - DryRun: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, SelectCandidates: func([]UpdateChoice) ([]UpdateChoice, bool) { t.Fatal("no-change candidates must not be shown for selection") return nil, false @@ -232,15 +233,15 @@ func TestUpdateCheckListsUpdatesWithoutPrompting(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - Check: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + Check: true, + Runner: runner, SelectCandidates: func([]UpdateChoice) ([]UpdateChoice, bool) { t.Fatal("--check must not prompt for dependency selection") return nil, false }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { t.Fatal("--check must not prompt for a version") return "", false }, @@ -271,7 +272,7 @@ func TestDiscoverUpdateCandidates_ImageAndHelmTargets(t *testing.T) { writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) runGit(t, dir, "add", ".") - got, err := DiscoverUpdateCandidates(".", []Manager{ManagerImage, ManagerHelm}) + got, err := DiscoverUpdateCandidates(".", []Manager{ManagerImage, ManagerHelm}, DiscoverFilter{}) if err != nil { t.Fatal(err) } @@ -297,6 +298,52 @@ func TestDiscoverUpdateCandidates_ImageAndHelmTargets(t *testing.T) { } } +func TestUpdateHelmReleaseWithMissingSourceReportsClearError(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseMissingSourceFixture) + runGit(t, dir, "add", ".") + + resolver := &recordingImageVersionResolver{} + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerHelm}, + Check: true, + ImageResolver: resolver, + }) + if err != nil { + t.Fatalf("Update returned fatal error, want graceful per-chart skip: %v", err) + } + if len(plans) != 1 { + t.Fatalf("plans = %#v, want 1", plans) + } + if !strings.Contains(plans[0].Skipped, "HelmRepository flux-system/mission-control-oipa which was not found in the scanned manifests") { + t.Fatalf("skipped reason = %q, want the actionable HelmRepository-not-found error", plans[0].Skipped) + } + if resolver.calls.Load() != 0 { + t.Fatalf("resolver was queried %d times; an unresolved chart source must short-circuit before any lookup", resolver.calls.Load()) + } +} + +// recordingImageVersionResolver counts version lookups so a test can assert an +// unresolved chart source short-circuits before any network query. +type recordingImageVersionResolver struct{ calls atomic.Int64 } + +func (r *recordingImageVersionResolver) Available(context.Context, imageupdate.UpdateTarget) ([]string, error) { + r.calls.Add(1) + return nil, nil +} + +func (r *recordingImageVersionResolver) ResolveLatestVersions(context.Context, imageupdate.UpdateTarget) (imageupdate.LatestVersions, error) { + r.calls.Add(1) + return imageupdate.LatestVersions{}, nil +} + +func (r *recordingImageVersionResolver) NewImageValue(context.Context, imageupdate.UpdateTarget, string) (string, error) { + r.calls.Add(1) + return "", nil +} + func TestUpdateImageDryRunUsesImageVersionResolver(t *testing.T) { dir := t.TempDir() t.Chdir(dir) @@ -306,7 +353,7 @@ func TestUpdateImageDryRunUsesImageVersionResolver(t *testing.T) { plans, err := Update(context.Background(), ".", UpdateOptions{ Managers: []Manager{ManagerImage}, - Expression: []string{"*nginx*"}, + Filters: []string{"*nginx*"}, DryRun: true, ImageResolver: fakeImageVersionResolver{"nginx": []string{"1.27.0", "1.25.3"}}, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { @@ -315,7 +362,7 @@ func TestUpdateImageDryRunUsesImageVersionResolver(t *testing.T) { } return choices, true }, - SelectVersion: func(choice UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.27.0", true }, }) @@ -350,13 +397,13 @@ func TestUpdateStagesChangedFilesAfterWrite(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + Runner: runner, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.4.0", true }, }) @@ -402,14 +449,14 @@ func TestUpdateDryRunDoesNotStage(t *testing.T) { }, } plans, err := Update(context.Background(), dir, UpdateOptions{ - Managers: []Manager{ManagerNPM}, - Expression: []string{"left-pad"}, - DryRun: true, - Runner: runner, + Managers: []Manager{ManagerNPM}, + Filters: []string{"left-pad"}, + DryRun: true, + Runner: runner, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.4.0", true }, }) @@ -454,12 +501,12 @@ func TestUpdateImageStagesEditedManifest(t *testing.T) { plans, err := Update(context.Background(), ".", UpdateOptions{ Managers: []Manager{ManagerImage}, - Expression: []string{"*nginx*"}, + Filters: []string{"*nginx*"}, ImageResolver: fakeImageVersionResolver{"nginx": []string{"1.27.0", "1.25.3"}}, SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { return choices, true }, - SelectVersion: func(UpdateChoice) (string, bool) { + SelectVersion: func(UpdateVersionPrompt) (string, bool) { return "1.27.0", true }, }) @@ -535,8 +582,8 @@ func TestUpdateCommandForManagers(t *testing.T) { func TestUpdateRejectsUnsupportedManagers(t *testing.T) { _, err := Update(context.Background(), t.TempDir(), UpdateOptions{ - Managers: []Manager{ManagerMaven}, - Expression: []string{"*"}, + Managers: []Manager{ManagerMaven}, + Filters: []string{"*"}, }) if err == nil || !strings.Contains(err.Error(), "unsupported manager") { t.Fatalf("expected unsupported manager error, got %v", err) @@ -548,6 +595,10 @@ type updateFakeRunner struct { responses map[string]CommandResult errors map[string]error commands []Command + // succeedByDefault makes an unlisted command succeed with empty output, + // for callers that assert on which commands ran rather than what they + // printed. Left false, an unlisted command is an error. + succeedByDefault bool } func (r *updateFakeRunner) Run(_ context.Context, cmd Command) (CommandResult, error) { @@ -563,9 +614,23 @@ func (r *updateFakeRunner) Run(_ context.Context, cmd Command) (CommandResult, e return result, nil } } + if r.succeedByDefault { + return CommandResult{}, nil + } return CommandResult{}, errors.New("unexpected command: " + key) } +// ran returns each command as it would have been typed, in the order received. +func (r *updateFakeRunner) ran() []string { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]string, 0, len(r.commands)) + for _, cmd := range r.commands { + out = append(out, cmd.String()) + } + return out +} + func updateCandidateLabels(candidates []UpdateCandidate) []string { labels := make([]string, 0, len(candidates)) for _, candidate := range candidates { @@ -614,6 +679,44 @@ func (fakeImageVersionResolver) NewImageValue(_ context.Context, target imageupd return stripImageVersion(target.CurrentValue) + ":" + version, nil } +// countingImageVersionResolver records how many times each target's published +// versions are looked up, so tests can assert duplicate occurrences share a +// single network round-trip. +type countingImageVersionResolver struct { + versions map[string][]string + mu sync.Mutex + calls map[string]int +} + +func newCountingImageVersionResolver(versions map[string][]string) *countingImageVersionResolver { + return &countingImageVersionResolver{versions: versions, calls: map[string]int{}} +} + +func (r *countingImageVersionResolver) Available(_ context.Context, target imageupdate.UpdateTarget) ([]string, error) { + r.mu.Lock() + r.calls[updateTargetName(target)]++ + r.mu.Unlock() + return sortDependencyVersions(r.versions[updateTargetName(target)]), nil +} + +func (r *countingImageVersionResolver) ResolveLatestVersions(_ context.Context, target imageupdate.UpdateTarget) (imageupdate.LatestVersions, error) { + versions := sortDependencyVersions(r.versions[updateTargetName(target)]) + return imageupdate.LatestVersions{ + Stable: latestStableVersion(versions), + Prerelease: latestPrereleaseVersion(versions), + }, nil +} + +func (*countingImageVersionResolver) NewImageValue(_ context.Context, target imageupdate.UpdateTarget, version string) (string, error) { + return stripImageVersion(target.CurrentValue) + ":" + version, nil +} + +func (r *countingImageVersionResolver) callCount(name string) int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[name] +} + const deploymentUpdateFixture = `apiVersion: apps/v1 kind: Deployment metadata: @@ -652,3 +755,20 @@ metadata: spec: url: https://stefanprodan.github.io/podinfo ` + +// helmReleaseMissingSourceFixture references a HelmRepository that is absent from +// the scanned manifests, mirroring the real mission-control-oipa breakage. +const helmReleaseMissingSourceFixture = `apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: mission-control-oipa +spec: + chart: + spec: + chart: mission-control-oipa-chart + version: 0.0.9 + sourceRef: + kind: HelmRepository + name: mission-control-oipa + namespace: flux-system +` diff --git a/deps/update_version.go b/deps/update_version.go new file mode 100644 index 0000000..d675906 --- /dev/null +++ b/deps/update_version.go @@ -0,0 +1,94 @@ +package deps + +import ( + "sort" + "strings" + + "github.com/Masterminds/semver/v3" +) + +func sortDependencyVersions(versions []string) []string { + type parsed struct { + orig string + ver *semver.Version + } + seen := map[string]bool{} + var parsedVersions []parsed + for _, version := range versions { + version = strings.TrimSpace(version) + if version == "" || seen[version] { + continue + } + seen[version] = true + sv, err := semver.NewVersion(version) + if err != nil { + continue + } + parsedVersions = append(parsedVersions, parsed{orig: version, ver: sv}) + } + sort.Slice(parsedVersions, func(i, j int) bool { + return parsedVersions[i].ver.GreaterThan(parsedVersions[j].ver) + }) + out := make([]string, len(parsedVersions)) + for i, item := range parsedVersions { + out[i] = item.orig + } + return out +} + +func latestStableVersion(versions []string) string { + for _, version := range versions { + if !isPrerelease(version) { + return version + } + } + return "" +} + +func latestPrereleaseVersion(versions []string) string { + for _, version := range versions { + if isPrerelease(version) { + return version + } + } + return "" +} + +func isPrerelease(version string) bool { + sv, err := semver.NewVersion(version) + return err == nil && strings.TrimSpace(sv.Prerelease()) != "" +} + +func updateableVersions(current string, versions []string) []string { + current = normalizeCurrentVersion(current) + currentSemver, currentErr := semver.NewVersion(current) + out := make([]string, 0, len(versions)) + for _, version := range versions { + if version == "" || version == current { + continue + } + versionSemver, versionErr := semver.NewVersion(version) + if currentErr != nil || versionErr != nil { + out = append(out, version) + continue + } + if versionSemver.GreaterThan(currentSemver) { + out = append(out, version) + } + } + return out +} + +func selectedVersionIsCurrent(current, selected string) bool { + return normalizeCurrentVersion(current) == selected +} + +func normalizeCurrentVersion(current string) string { + current = strings.TrimSpace(current) + current = strings.TrimPrefix(current, "npm:") + current = strings.TrimLeft(current, "^~<>= ") + if idx := strings.IndexAny(current, " |,"); idx >= 0 { + current = current[:idx] + } + return strings.TrimSpace(current) +} diff --git a/go.mod b/go.mod index a67a082..be5c7b2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/flanksource/repomap go 1.26.1 require ( - github.com/Masterminds/semver/v3 v3.4.0 + github.com/Masterminds/semver/v3 v3.5.0 github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/flanksource/clicky v1.21.48 @@ -17,6 +17,7 @@ require ( github.com/spf13/cobra v1.10.2 golang.org/x/mod v0.36.0 golang.org/x/sync v0.21.0 + helm.sh/helm/v3 v3.21.3 ) require ( @@ -161,14 +162,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect - golang.org/x/image v0.41.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sys v0.46.0 // indirect @@ -181,10 +179,10 @@ require ( gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.36.1 // indirect - k8s.io/apiextensions-apiserver v0.36.1 // indirect - k8s.io/apimachinery v0.36.1 // indirect - k8s.io/client-go v0.36.1 // indirect + k8s.io/api v0.36.2 // indirect + k8s.io/apiextensions-apiserver v0.36.2 // indirect + k8s.io/apimachinery v0.36.2 // indirect + k8s.io/client-go v0.36.2 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect diff --git a/go.sum b/go.sum index 6de3cb7..324f698 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,14 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw= @@ -127,24 +129,10 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= -github.com/flanksource/clicky v1.21.14 h1:72Ax9EAzeu/SEQiR9pgvUmpfj54W/hM+0VXTv5Qdno4= -github.com/flanksource/clicky v1.21.14/go.mod h1:+JLpose/DqOYsXWdjF+J2/rAZ7PbThzAGXWm2GUPA1s= -github.com/flanksource/clicky v1.21.40 h1:96JqmjBSNe3nTRM43KBLrAXLE2e5OIK7X4uy9Dub8Co= -github.com/flanksource/clicky v1.21.40/go.mod h1:wyMaDJleu6wHw6jgg3/BfHFXY5JSsyGG6X1vnFRLfHc= -github.com/flanksource/clicky v1.21.42 h1:MC0uklJWq4KDg2Inq9OxT4IE3SJVqVA3Zio8DftSvK0= -github.com/flanksource/clicky v1.21.42/go.mod h1:wxgvRVic4B8cWCxQ7EO57Q6zBnV49VUoGJ0i9SMdVt4= -github.com/flanksource/clicky v1.21.43 h1:xQxHUDyKQdRFTQfhGAk5G6j2r9Du2Qq1Gfn+T7dGs98= -github.com/flanksource/clicky v1.21.43/go.mod h1:wxgvRVic4B8cWCxQ7EO57Q6zBnV49VUoGJ0i9SMdVt4= -github.com/flanksource/clicky v1.21.44 h1:EwDaWXOPF7Tu5UeL/TE168n89LZqjRMUptcg1kw6884= -github.com/flanksource/clicky v1.21.44/go.mod h1:wxgvRVic4B8cWCxQ7EO57Q6zBnV49VUoGJ0i9SMdVt4= github.com/flanksource/clicky v1.21.48 h1:yCbCk6rDEgtxdknuQV6uZLsvVqCPGYNO7dHuLgbtOp4= github.com/flanksource/clicky v1.21.48/go.mod h1:2gAph3Uy90AYirGmkEP/D4okiFZlrH2XdcoXf69oZHY= -github.com/flanksource/commons v1.51.3 h1:sgQZ2s0XJTub4qmIlzRyH+eYXJP6UXmreCataP9mE7E= -github.com/flanksource/commons v1.51.3/go.mod h1:BxXJzAsRxsw0la7Y/ShEABa8ZbtGIdRi7PCRjiHDCJE= github.com/flanksource/commons v1.53.1 h1:WiMvY9XGG//L4ndYKTcmp0NjWXg4B7Wbn4hYWkITUmY= github.com/flanksource/commons v1.53.1/go.mod h1:ZII22jIDJ3fd/Mz7l0SzLFEv8aoSUg+xwfJAAVf8up4= -github.com/flanksource/gomplate/v3 v3.24.82 h1:22HOZYeNRMX40G8OF3qRAqRoR5Lkf8Vm4x3WDqugZkE= -github.com/flanksource/gomplate/v3 v3.24.82/go.mod h1:NMMZkFsjbLy/8iY8Fip5N86Y0PP6lZeq+kmPwpVVIL0= github.com/flanksource/gomplate/v3 v3.24.84 h1:UOE0yCJsczTIKRaHUvhD6tjCYrbNvOugAizuy0FVlhE= github.com/flanksource/gomplate/v3 v3.24.84/go.mod h1:NMMZkFsjbLy/8iY8Fip5N86Y0PP6lZeq+kmPwpVVIL0= github.com/flanksource/is-healthy v1.0.88 h1:ATQuKoNdp8Qfzf41/eMFajmT0qzOmZlZNG5eLK41RFo= @@ -392,8 +380,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 h1:s/1iRkCKDfhlh1JF26knRneorus8aOwVIDhvYx9WoDw= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0/go.mod h1:UI3wi0FXg1Pofb8ZBiBLhtMzgoTm1TYkMvn71fAqDzs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0 h1:mS47AX77OtFfKG4vtp+84kuGSFZHTyxtXIN269vChY0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0/go.mod h1:PJnsC41lAGncJlPUniSwM81gc80GkgWJWr3cu2nKEtU= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= @@ -530,15 +518,17 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +helm.sh/helm/v3 v3.21.3 h1:wkamdwI3liEkW6wI1l9aGqQZGxcTKyt8kx0qJLPcmCg= +helm.sh/helm/v3 v3.21.3/go.mod h1:iaJ0iNsPoTZl++7h6vzQFyT0VEVtLYJiyRBDkPOOBTs= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= -k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= -k8s.io/apiextensions-apiserver v0.36.1 h1:6JfYmPUsuUIHuN+3QxutXYWj492RqF5fBSx67GYK5Ks= -k8s.io/apiextensions-apiserver v0.36.1/go.mod h1:pLzZin90riwisdzKwv/GoTwENooytoIx5zWJb4Hkby8= -k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= -k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= -k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= -k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= +k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= +k8s.io/apiextensions-apiserver v0.36.2 h1:3O5gqOj/dt2XWWbpMe+TXWpE9yU6pjM/tXxtHHJT/K4= +k8s.io/apiextensions-apiserver v0.36.2/go.mod h1:cL1tBWe8XSaP1H30iWKGo7hf6iAUUUJPEU70dskmAnA= +k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= +k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= +k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= +k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= diff --git a/imageupdate/chartref.go b/imageupdate/chartref.go new file mode 100644 index 0000000..5e8a83e --- /dev/null +++ b/imageupdate/chartref.go @@ -0,0 +1,215 @@ +package imageupdate + +import ( + "fmt" + "strings" + + "github.com/goccy/go-yaml/ast" + + "github.com/flanksource/repomap/kubernetes" +) + +// chartSource is a Flux source object that pins a concrete chart version and can +// therefore be the edit target of a chartRef HelmRelease: an OCIRepository +// (version in spec.ref.tag) or a HelmChart (version in spec.version). Unlike a +// HelmRepository, which is a bare URL, it carries the file and line of the +// version literal so Resolve can redirect the HelmRelease's target onto it. +type chartSource struct { + kind string // OCIRepository | HelmChart + file string + versionLine int + versionPath string + version string + repoURL string + isOCI bool + chartName string + + // srcRef chains a HelmChart to its HelmRepository for the repo URL. + srcRefName string + srcRefNamespace string +} + +type chartSourceCandidate struct { + rawNamespace string + effNamespace string + src chartSource +} + +// extractChartRef builds a deferred chart target from a HelmRelease's +// spec.chartRef. Only OCIRepository/HelmChart references resolve to a queryable +// version source; GitRepository/Bucket references carry no chart version. +func extractChartRef(file string, ref kubernetes.KubernetesRef, chartRef map[string]interface{}) (UpdateTarget, bool) { + kind, _ := chartRef["kind"].(string) + name, _ := chartRef["name"].(string) + namespace, _ := chartRef["namespace"].(string) + if name == "" || (kind != "OCIRepository" && kind != "HelmChart") { + return UpdateTarget{}, false + } + return UpdateTarget{ + Ref: ref, + Kind: TargetChart, + File: file, + ChartRefKind: kind, + ChartRefName: name, + ChartRefNamespace: namespace, + }, true +} + +func (idx *SourceIndex) indexOCIRepository(file string, ref kubernetes.KubernetesRef, m map[string]interface{}, doc *ast.DocumentNode, offset int) { + spec, _ := m["spec"].(map[string]interface{}) + url, _ := spec["url"].(string) + if url == "" { + return + } + // The OCIRepository url already points at the chart artifact; split off the + // last path segment so RepoURL+ChartName recompose to it (matching the + // inline OCI HelmRepository convention the resolver expects). + repoURL, chartName := splitOCIChartURL(url) + src := chartSource{ + kind: "OCIRepository", + file: file, + versionPath: "$.spec.ref.tag", + repoURL: repoURL, + isOCI: true, + chartName: chartName, + } + if refSpec, ok := spec["ref"].(map[string]interface{}); ok { + src.version, _ = refSpec["tag"].(string) + } + src.versionLine = valueLine(doc, src.versionPath, offset) + idx.addChartSource(file, ref, src) +} + +func (idx *SourceIndex) indexHelmChart(file string, ref kubernetes.KubernetesRef, m map[string]interface{}, doc *ast.DocumentNode, offset int) { + spec, _ := m["spec"].(map[string]interface{}) + if spec == nil { + return + } + src := chartSource{ + kind: "HelmChart", + file: file, + versionPath: "$.spec.version", + } + src.chartName, _ = spec["chart"].(string) + src.version, _ = spec["version"].(string) + if sr, ok := spec["sourceRef"].(map[string]interface{}); ok { + src.srcRefName, _ = sr["name"].(string) + src.srcRefNamespace, _ = sr["namespace"].(string) + } + src.versionLine = valueLine(doc, src.versionPath, offset) + idx.addChartSource(file, ref, src) +} + +func (idx *SourceIndex) addChartSource(file string, ref kubernetes.KubernetesRef, src chartSource) { + effNS := idx.effectiveNamespace(file, ref.Namespace) + idx.chartByKey[chartKey(src.kind, ref.Namespace, ref.Name)] = src + idx.chartByKey[chartKey(src.kind, effNS, ref.Name)] = src + nameKey := src.kind + "|" + ref.Name + idx.chartByName[nameKey] = append(idx.chartByName[nameKey], chartSourceCandidate{ + rawNamespace: ref.Namespace, + effNamespace: effNS, + src: src, + }) +} + +// resolveChartRef redirects a chartRef target onto the OCIRepository/HelmChart it +// references: File/FieldLine/CurrentValue become that object's version literal and +// RepoURL/IsOCI are resolved (chaining a HelmChart through its HelmRepository). +func (idx *SourceIndex) resolveChartRef(t *UpdateTarget) error { + wantNS := t.ChartRefNamespace + if wantNS == "" && idx.kt != nil { + wantNS = idx.kt.EffectiveNamespace(t.File) + } + src, err := idx.lookupChartSource(t.ChartRefKind, t.ChartRefName, wantNS, t.Ref) + if err != nil { + return err + } + + switch src.kind { + case "OCIRepository": + t.RepoURL = src.repoURL + t.IsOCI = src.isOCI + case "HelmChart": + repo, err := idx.lookupHelmRepository(src.srcRefName, chartSourceNamespace(src, wantNS), t.Ref) + if err != nil { + return fmt.Errorf("HelmChart %s/%s: %w", src.srcRefNamespace, src.chartName, err) + } + t.RepoURL = repo.URL + t.IsOCI = repo.IsOCI + } + + t.File = src.file + t.FieldLine = src.versionLine + t.FieldJSONPath = src.versionPath + t.CurrentValue = src.version + t.ChartName = src.chartName + if t.CurrentValue == "" { + return fmt.Errorf("HelmRelease %s/%s references %s %q which has no editable version (uses semver/digest, not a tag)", + t.Ref.Namespace, t.Ref.Name, src.kind, t.ChartRefName) + } + return nil +} + +func chartSourceNamespace(src chartSource, fallback string) string { + if src.srcRefNamespace != "" { + return src.srcRefNamespace + } + return fallback +} + +func (idx *SourceIndex) lookupChartSource(kind, name, wantNS string, requester kubernetes.KubernetesRef) (chartSource, error) { + if src, ok := idx.chartByKey[chartKey(kind, wantNS, name)]; ok { + return src, nil + } + candidates := idx.chartByName[kind+"|"+name] + switch { + case len(candidates) == 0: + return chartSource{}, fmt.Errorf("HelmRelease %s/%s references %s %s which was not found in the scanned manifests", + requester.Namespace, requester.Name, kind, sourceKey(wantNS, name)) + case len(candidates) == 1: + return candidates[0].src, nil + default: + if c, ok := pickByNamespace(candidates, wantNS, + func(c chartSourceCandidate) string { return c.rawNamespace }, + func(c chartSourceCandidate) string { return c.effNamespace }); ok { + return c.src, nil + } + return chartSource{}, fmt.Errorf("HelmRelease %s/%s references %s %q which is ambiguous across namespaces %s", + requester.Namespace, requester.Name, kind, name, chartCandidateNamespaces(candidates)) + } +} + +func chartKey(kind, namespace, name string) string { + return kind + "|" + namespace + "/" + name +} + +// splitOCIChartURL splits an OCIRepository url (oci://host/path/chart) into the +// parent repo url and chart name, mirroring how an inline OCI HelmRepository +// (parent url) plus chart name recompose to the full artifact path. +func splitOCIChartURL(url string) (repoURL, chartName string) { + trimmed := strings.TrimSuffix(strings.TrimPrefix(url, "oci://"), "/") + i := strings.LastIndex(trimmed, "/") + if i < 0 { + return "oci://" + trimmed, "" + } + return "oci://" + trimmed[:i], trimmed[i+1:] +} + +func chartCandidateNamespaces(candidates []chartSourceCandidate) string { + seen := map[string]bool{} + var out []string + for _, c := range candidates { + ns := c.effNamespace + if ns == "" { + ns = c.rawNamespace + } + if ns == "" { + ns = "(none)" + } + if !seen[ns] { + seen[ns] = true + out = append(out, ns) + } + } + return strings.Join(out, ", ") +} diff --git a/imageupdate/chartref_test.go b/imageupdate/chartref_test.go new file mode 100644 index 0000000..ff82a99 --- /dev/null +++ b/imageupdate/chartref_test.go @@ -0,0 +1,108 @@ +package imageupdate + +import ( + "strings" + "testing" +) + +// resolveSingleTarget indexes sources and extracts the one chart target from a +// fixture, resolving it through the index. It is the chartRef equivalent of the +// inline helpers in sourceref_test.go. +func resolveSingleTarget(t *testing.T, name string) UpdateTarget { + t.Helper() + content := readManifest(t, name) + idx := NewSourceIndex(nil) + if err := idx.IndexSources(name, content); err != nil { + t.Fatalf("index: %v", err) + } + targets, err := ExtractTargets(name, content) + if err != nil { + t.Fatalf("extract: %v", err) + } + if len(targets) != 1 { + t.Fatalf("want 1 target, got %d: %#v", len(targets), targets) + } + tg := targets[0] + if err := idx.Resolve(&tg); err != nil { + t.Fatalf("resolve: %v", err) + } + return tg +} + +func TestExtractTargets_ChartRefDeferred(t *testing.T) { + content := readManifest(t, "helmrelease-chartref-oci.yaml") + targets, err := ExtractTargets("helmrelease-chartref-oci.yaml", content) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 { + t.Fatalf("want 1 chart target (OCIRepository is a source, not a target), got %d", len(targets)) + } + tg := targets[0] + // Before Resolve the target is deferred: it names the chartRef but has no + // version or edit anchor yet. + if tg.ChartRefKind != "OCIRepository" || tg.ChartRefName != "podinfo" { + t.Errorf("chartRef = %q/%q, want OCIRepository/podinfo", tg.ChartRefKind, tg.ChartRefName) + } + if tg.CurrentValue != "" || tg.FieldLine != 0 { + t.Errorf("deferred target should have no version/line yet: %+v", tg) + } + if tg.Ref.Kind != "HelmRelease" || tg.Ref.Name != "podinfo" { + t.Errorf("ref should stay the HelmRelease: %+v", tg.Ref) + } +} + +func TestResolveChartRef_OCIRepository(t *testing.T) { + tg := resolveSingleTarget(t, "helmrelease-chartref-oci.yaml") + if !tg.IsOCI || tg.RepoURL != "oci://ghcr.io/stefanprodan/charts" { + t.Errorf("repo = %q oci=%v, want oci://ghcr.io/stefanprodan/charts true", tg.RepoURL, tg.IsOCI) + } + if tg.ChartName != "podinfo" { + t.Errorf("chart = %q, want podinfo", tg.ChartName) + } + if tg.CurrentValue != "6.5.0" { + t.Errorf("current = %q, want 6.5.0", tg.CurrentValue) + } + // The edit anchor must move onto the OCIRepository's spec.ref.tag (line 19). + if tg.FieldJSONPath != "$.spec.ref.tag" || tg.FieldLine != 19 { + t.Errorf("anchor = %s:%d, want $.spec.ref.tag:19", tg.FieldJSONPath, tg.FieldLine) + } + if tg.File != "helmrelease-chartref-oci.yaml" { + t.Errorf("file = %q, want the OCIRepository file", tg.File) + } +} + +func TestResolveChartRef_HelmChart(t *testing.T) { + tg := resolveSingleTarget(t, "helmrelease-chartref-helmchart.yaml") + if tg.IsOCI || tg.RepoURL != "https://charts.example.com/app" { + t.Errorf("repo = %q oci=%v, want https://charts.example.com/app false", tg.RepoURL, tg.IsOCI) + } + if tg.ChartName != "app" { + t.Errorf("chart = %q, want app", tg.ChartName) + } + if tg.CurrentValue != "1.2.0" { + t.Errorf("current = %q, want 1.2.0", tg.CurrentValue) + } + // Anchor moves onto the HelmChart's spec.version (line 18). + if tg.FieldJSONPath != "$.spec.version" || tg.FieldLine != 18 { + t.Errorf("anchor = %s:%d, want $.spec.version:18", tg.FieldJSONPath, tg.FieldLine) + } +} + +func TestResolveChartRef_MissingSourceFailsLoud(t *testing.T) { + idx := NewSourceIndex(nil) + tg := UpdateTarget{Kind: TargetChart, ChartRefKind: "OCIRepository", ChartRefName: "ghost"} + tg.Ref.Namespace = "apps" + tg.Ref.Name = "podinfo" + err := idx.Resolve(&tg) + if err == nil || !strings.Contains(err.Error(), "OCIRepository") { + t.Fatalf("expected loud error naming OCIRepository, got %v", err) + } +} + +func TestSplitOCIChartURL(t *testing.T) { + repo, chart := splitOCIChartURL("oci://ghcr.io/stefanprodan/charts/podinfo") + if repo != "oci://ghcr.io/stefanprodan/charts" || chart != "podinfo" { + t.Fatalf("split = %q/%q", repo, chart) + } +} diff --git a/imageupdate/discover.go b/imageupdate/discover.go new file mode 100644 index 0000000..0438da1 --- /dev/null +++ b/imageupdate/discover.go @@ -0,0 +1,189 @@ +package imageupdate + +import ( + "os" + "path/filepath" + "sort" + "strings" + + "github.com/flanksource/commons/collections" + + "github.com/flanksource/repomap" + "github.com/flanksource/repomap/kubernetes" +) + +// TargetWarning is a non-fatal problem encountered while discovering a target — +// typically a chart whose Flux source (HelmRepository/OCIRepository/HelmChart) +// could not be resolved. Discovery keeps the target so it still surfaces. +type TargetWarning struct { + File string + Message string +} + +// DiscoverResult bundles the discovered targets, the source index used to resolve +// them, and any per-target resolution warnings. UntrackedTarget names a single +// scanned file that is a YAML manifest but is not git-tracked (so discovery could +// not see it); it is empty when the scan target is a directory or a tracked file. +type DiscoverResult struct { + Targets []UpdateTarget + Index *SourceIndex + Warnings []TargetWarning + UntrackedTarget string +} + +// DiscoverTargets runs the full image/chart discovery over a set of repo-relative +// YAML file contents: it builds the kustomize/Flux tree, indexes every chart +// source, extracts image/chart targets under scope, and resolves each chart +// target onto its source (URL + current version + edit anchor). Source-resolution +// failures become warnings (the target is kept), so a single bad HelmRelease +// never aborts discovery. scope is a repo-relative POSIX match scope ("" scans +// everything, a "/"-terminated value is a directory prefix, anything else is an +// exact file path); sources are indexed repo-wide regardless of scope so a +// HelmRelease under the scope can resolve a source defined elsewhere. +func DiscoverTargets(contents map[string]string, scope string) DiscoverResult { + tree := BuildKustomizeTree(contents) + idx := NewSourceIndex(tree) + + files := make([]string, 0, len(contents)) + for f := range contents { + files = append(files, f) + } + sort.Strings(files) + + for _, f := range files { + _ = idx.IndexSources(f, contents[f]) + } + + var targets []UpdateTarget + var warnings []TargetWarning + for _, f := range files { + if !inScanScope(f, scope) { + continue + } + fileTargets, err := ExtractTargets(f, contents[f]) + if err != nil { + continue + } + for i := range fileTargets { + t := fileTargets[i] + if effNS := tree.EffectiveNamespace(t.File); effNS != "" { + t.Ref.Namespace = effNS + } + if t.Kind == TargetChart { + if err := idx.Resolve(&t); err != nil { + t.SourceErr = err.Error() + warnings = append(warnings, TargetWarning{File: f, Message: err.Error()}) + } + } + targets = append(targets, t) + } + } + sortTargets(targets) + return DiscoverResult{Targets: targets, Index: idx, Warnings: warnings} +} + +// DiscoverRepoTargets is the convenience entry point used by the CLI: it reads +// every tracked YAML file under conf and discovers targets, scoping extraction to +// scanPath (which must live under the repo). When scanPath is a single YAML file +// that is not git-tracked, the result's UntrackedTarget is set so the caller can +// report the reason instead of silently finding nothing. +func DiscoverRepoTargets(conf *repomap.ArchConf, scanPath string) (DiscoverResult, error) { + contents, err := conf.TrackedYAMLContents() + if err != nil { + return DiscoverResult{}, err + } + scope := scanScope(conf.RepoPath(), scanPath) + result := DiscoverTargets(contents, scope) + if untracked := untrackedFileTarget(scanPath, scope, contents); untracked != "" { + result.UntrackedTarget = untracked + } + return result, nil +} + +// untrackedFileTarget returns the repo-relative path of scanPath when it is a +// single YAML file (an exact-file scope) that is absent from the git-tracked +// contents, otherwise "". Directory scopes and tracked files return "". +func untrackedFileTarget(scanPath, scope string, contents map[string]string) string { + if scope == "" || strings.HasSuffix(scope, "/") || !kubernetes.IsYaml(scanPath) { + return "" + } + if _, tracked := contents[scope]; tracked { + return "" + } + return scope +} + +// scanScope returns the repo-relative POSIX match scope for scanPath: "" for the +// repo root (match everything), a directory prefix ending in "/" for a directory, +// or the exact repo-relative file path for a single file. +func scanScope(repoPath, scanPath string) string { + rel, err := filepath.Rel(repoPath, scanPath) + if err != nil || rel == "." || rel == "" { + return "" + } + rel = filepath.ToSlash(rel) + if info, err := os.Stat(scanPath); err == nil && !info.IsDir() { + return rel + } + return rel + "/" +} + +// inScanScope reports whether repo-relative file f falls under scope: an empty +// scope matches everything, a "/"-terminated scope is a directory prefix, and any +// other scope matches that exact file. +func inScanScope(f, scope string) bool { + switch { + case scope == "": + return true + case strings.HasSuffix(scope, "/"): + return strings.HasPrefix(f, scope) + default: + return f == scope + } +} + +func sortTargets(targets []UpdateTarget) { + sort.SliceStable(targets, func(i, j int) bool { + if targets[i].File != targets[j].File { + return targets[i].File < targets[j].File + } + if targets[i].FieldLine != targets[j].FieldLine { + return targets[i].FieldLine < targets[j].FieldLine + } + return targetName(targets[i]) < targetName(targets[j]) + }) +} + +func targetName(t UpdateTarget) string { + if t.Kind == TargetImage && t.Image != nil { + return t.Image.GetFullNameWithoutTag() + } + return t.ChartName +} + +// Filter applies a resource matcher plus image/chart name patterns to a target +// set. Empty matcher and empty patterns return the targets unchanged. +func Filter(targets []UpdateTarget, matcher repomap.ResourceMatcher, imagePatterns, chartPatterns []string) []UpdateTarget { + var out []UpdateTarget + for _, t := range targets { + if !matcher.MatchesRef(t.Ref) { + continue + } + if t.Kind == TargetImage && len(imagePatterns) > 0 { + name := "" + if t.Image != nil { + name = t.Image.GetFullNameWithoutTag() + } + if matched, _ := collections.MatchItem(name, imagePatterns...); !matched { + continue + } + } + if t.Kind == TargetChart && len(chartPatterns) > 0 { + if matched, _ := collections.MatchItem(t.ChartName, chartPatterns...); !matched { + continue + } + } + out = append(out, t) + } + return out +} diff --git a/imageupdate/discover_test.go b/imageupdate/discover_test.go new file mode 100644 index 0000000..8763cbe --- /dev/null +++ b/imageupdate/discover_test.go @@ -0,0 +1,66 @@ +package imageupdate + +import "testing" + +const deploymentNoNS = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: app +spec: + template: + spec: + containers: + - name: app + image: nginx:1.0.0 +` + +const deploymentPinnedNS = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: other +spec: + template: + spec: + containers: + - name: app + image: nginx:1.0.0 +` + +// imageTarget returns the single TargetImage in targets, failing if there is not +// exactly one. +func imageTarget(t *testing.T, targets []UpdateTarget) UpdateTarget { + t.Helper() + var found []UpdateTarget + for _, tg := range targets { + if tg.Kind == TargetImage { + found = append(found, tg) + } + } + if len(found) != 1 { + t.Fatalf("want exactly one image target, got %d: %+v", len(found), targets) + } + return found[0] +} + +func TestDiscoverTargets_KustomizationNamespaceApplied(t *testing.T) { + contents := map[string]string{ + "kenya/kustomization.yaml": "namespace: hf-qa-kenya\nresources:\n - app.yaml\n", + "kenya/app.yaml": deploymentNoNS, + } + res := DiscoverTargets(contents, "kenya/") + if ns := imageTarget(t, res.Targets).Ref.Namespace; ns != "hf-qa-kenya" { + t.Errorf("target namespace = %q, want hf-qa-kenya (imposed by kustomization)", ns) + } +} + +func TestDiscoverTargets_KustomizationNamespaceOverridesPinned(t *testing.T) { + contents := map[string]string{ + "kenya/kustomization.yaml": "namespace: hf-qa-kenya\nresources:\n - app.yaml\n", + "kenya/app.yaml": deploymentPinnedNS, + } + res := DiscoverTargets(contents, "kenya/") + if ns := imageTarget(t, res.Targets).Ref.Namespace; ns != "hf-qa-kenya" { + t.Errorf("target namespace = %q, want hf-qa-kenya (kustomization wins over metadata.namespace)", ns) + } +} diff --git a/imageupdate/extract.go b/imageupdate/extract.go index 4a79af9..0fe9bb1 100644 --- a/imageupdate/extract.go +++ b/imageupdate/extract.go @@ -28,38 +28,56 @@ var appsV1Kinds = map[string]bool{ // comment-only document, which would hide trailing HelmReleases/workloads. The // per-document line offset is added back so FieldLine stays absolute in the file. func ExtractTargets(file, content string) ([]UpdateTarget, error) { - docs, err := kubernetes.ParseYAMLDocuments(content) + var targets []UpdateTarget + err := forEachResourceDoc(content, func(d resourceDoc) { + switch { + case appsV1Kinds[d.ref.Kind]: + targets = append(targets, extractImages(file, d.ast, d.ref, d.m, d.offset)...) + case d.ref.Kind == "HelmRelease": + if t, ok := extractChart(file, d.ast, d.ref, d.m, d.offset); ok { + targets = append(targets, t) + } + } + }) if err != nil { return nil, fmt.Errorf("parse %s: %w", file, err) } + return targets, nil +} - var targets []UpdateTarget +// resourceDoc is a single Kubernetes resource document with both its decoded map +// content and its single-document AST, plus the line offset that converts AST +// line positions back to absolute file lines. +type resourceDoc struct { + ref kubernetes.KubernetesRef + m map[string]interface{} + ast *ast.DocumentNode + offset int +} + +// forEachResourceDoc splits content into documents (line-based, so trailing docs +// after a comment-only document survive), and yields each Kubernetes resource +// with its decoded map and single-doc AST. Documents whose AST cannot be parsed +// are skipped. Shared by target extraction and source indexing so both agree on +// document boundaries and line numbers. +func forEachResourceDoc(content string, fn func(resourceDoc)) error { + docs, err := kubernetes.ParseYAMLDocuments(content) + if err != nil { + return err + } for _, d := range docs { m := d.Content if !kubernetes.IsKubernetesResource(m) { continue } ref := kubernetes.ExtractKubernetesRef(kubernetes.YAMLDocument{StartLine: d.StartLine, Content: m}) - - // Parse just this document for accurate field line positions, then offset - // by the document's start line to get absolute file lines. - offset := d.StartLine - 1 astFile, err := parser.ParseBytes([]byte(docText(content, d)), 0) if err != nil || len(astFile.Docs) == 0 { continue } - doc := astFile.Docs[0] - - switch { - case appsV1Kinds[ref.Kind]: - targets = append(targets, extractImages(file, doc, ref, m, offset)...) - case ref.Kind == "HelmRelease": - if t, ok := extractChart(file, doc, ref, m, offset); ok { - targets = append(targets, t) - } - } + fn(resourceDoc{ref: ref, m: m, ast: astFile.Docs[0], offset: d.StartLine - 1}) } - return targets, nil + return nil } // docText returns the raw source lines of a single document (1-based inclusive @@ -145,6 +163,14 @@ func containerList(m map[string]interface{}) []interface{} { func extractChart(file string, doc *ast.DocumentNode, ref kubernetes.KubernetesRef, m map[string]interface{}, offset int) (UpdateTarget, bool) { spec, _ := m["spec"].(map[string]interface{}) + if spec == nil { + return UpdateTarget{}, false + } + // Flux v2 spec.chartRef points at an OCIRepository/HelmChart that holds the + // version literal; emit a deferred target that Resolve redirects onto it. + if chartRef, ok := spec["chartRef"].(map[string]interface{}); ok { + return extractChartRef(file, ref, chartRef) + } chart, _ := spec["chart"].(map[string]interface{}) chartSpec, _ := chart["spec"].(map[string]interface{}) if chartSpec == nil { diff --git a/imageupdate/labels.go b/imageupdate/labels.go new file mode 100644 index 0000000..1b6a3ea --- /dev/null +++ b/imageupdate/labels.go @@ -0,0 +1,61 @@ +package imageupdate + +import ( + "context" + "fmt" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// LabelResolver reads an image's manifest digest and OCI labels from its +// registry, authenticating via the local Docker credential store. It is used by +// the deps recursion to discover an image's source repository +// (org.opencontainers.image.source) and declared base image +// (org.opencontainers.image.base.name/base.digest). +type LabelResolver struct { + creds CredentialResolver +} + +// NewLabelResolver returns a LabelResolver wired to the local Docker keychain. +func NewLabelResolver() *LabelResolver { + return &LabelResolver{creds: NewKeychainResolver()} +} + +// Labels resolves the manifest digest and OCI labels for an image reference of +// the form registry/repo:tag[@digest]. +func (l *LabelResolver) Labels(ctx context.Context, ref string) (digest string, labels map[string]string, err error) { + img := NewContainerImage(ref) + tagName := "latest" + if img.ImageTag != nil && img.ImageTag.TagName != "" { + tagName = img.ImageTag.TagName + } + tag, err := name.NewTag(img.GetFullNameWithoutTag() + ":" + tagName) + if err != nil { + return "", nil, fmt.Errorf("parse reference %s:%s: %w", img.GetFullNameWithoutTag(), tagName, err) + } + + user, pass, err := l.creds.Resolve(ctx, tag.RegistryStr()) + if err != nil { + return "", nil, err + } + var auth authn.Authenticator = authn.Anonymous + if user != "" || pass != "" { + auth = authn.FromConfig(authn.AuthConfig{Username: user, Password: pass}) + } + + remoteImage, err := remote.Image(tag, remote.WithContext(ctx), remote.WithAuth(auth)) + if err != nil { + return "", nil, fmt.Errorf("manifest for %s: %w", tag, err) + } + manifestDigest, err := remoteImage.Digest() + if err != nil { + return "", nil, fmt.Errorf("digest for %s: %w", tag, err) + } + config, err := remoteImage.ConfigFile() + if err != nil { + return "", nil, fmt.Errorf("config for %s: %w", tag, err) + } + return manifestDigest.String(), config.Config.Labels, nil +} diff --git a/imageupdate/sourceref.go b/imageupdate/sourceref.go index 0e95ef8..ececa96 100644 --- a/imageupdate/sourceref.go +++ b/imageupdate/sourceref.go @@ -27,20 +27,27 @@ type sourceCandidate struct { // SourceIndex resolves a HelmRelease sourceRef to its HelmRepository, accounting // for namespaces imposed by the Flux/kustomize tree rather than written into the // manifests. HelmRepositories are indexed under both their raw and effective -// namespace, with a name bucket for cross-namespace fallback. +// namespace, with a name bucket for cross-namespace fallback. OCIRepository and +// HelmChart objects (chartRef sources) are indexed the same way in chartBy*. type SourceIndex struct { byKey map[string]HelmRepositorySource byName map[string][]sourceCandidate - kt *KustomizeTree + + chartByKey map[string]chartSource + chartByName map[string][]chartSourceCandidate + + kt *KustomizeTree } // NewSourceIndex returns an empty index bound to a kustomize tree. The tree may // be nil, in which case effective namespaces equal raw namespaces. func NewSourceIndex(kt *KustomizeTree) *SourceIndex { return &SourceIndex{ - byKey: map[string]HelmRepositorySource{}, - byName: map[string][]sourceCandidate{}, - kt: kt, + byKey: map[string]HelmRepositorySource{}, + byName: map[string][]sourceCandidate{}, + chartByKey: map[string]chartSource{}, + chartByName: map[string][]chartSourceCandidate{}, + kt: kt, } } @@ -57,48 +64,54 @@ func (idx *SourceIndex) effectiveNamespace(file, rawNS string) string { return rawNS } -// IndexHelmRepositories parses one file and indexes every HelmRepository doc it -// contains under both its raw and effective (tree-derived) namespace. It uses -// the line-based document splitter rather than the YAML AST parser, which drops -// trailing documents in files that contain a comment-only document. -func (idx *SourceIndex) IndexHelmRepositories(file, content string) error { - docs, err := kubernetes.ParseYAMLDocuments(content) - if err != nil { - return err - } - for _, doc := range docs { - m := doc.Content - if kind, _ := m["kind"].(string); kind != "HelmRepository" { - continue - } - ref := kubernetes.ExtractKubernetesRef(kubernetes.YAMLDocument{Content: m}) - spec, _ := m["spec"].(map[string]interface{}) - url, _ := spec["url"].(string) - if url == "" { - continue +// IndexSources parses one file and indexes every Flux chart source it contains — +// HelmRepository (a bare URL) plus OCIRepository/HelmChart (chartRef sources that +// also pin a version) — under both their raw and effective (tree-derived) +// namespace, with a name bucket for cross-namespace fallback. +func (idx *SourceIndex) IndexSources(file, content string) error { + return forEachResourceDoc(content, func(d resourceDoc) { + switch d.ref.Kind { + case "HelmRepository": + idx.indexHelmRepository(file, d.ref, d.m) + case "OCIRepository": + idx.indexOCIRepository(file, d.ref, d.m, d.ast, d.offset) + case "HelmChart": + idx.indexHelmChart(file, d.ref, d.m, d.ast, d.offset) } - src := HelmRepositorySource{URL: url, IsOCI: strings.HasPrefix(url, "oci://")} - effNS := idx.effectiveNamespace(file, ref.Namespace) - - idx.byKey[sourceKey(ref.Namespace, ref.Name)] = src - idx.byKey[sourceKey(effNS, ref.Name)] = src - idx.byName[ref.Name] = append(idx.byName[ref.Name], sourceCandidate{ - rawNamespace: ref.Namespace, - effNamespace: effNS, - src: src, - }) + }) +} + +func (idx *SourceIndex) indexHelmRepository(file string, ref kubernetes.KubernetesRef, m map[string]interface{}) { + spec, _ := m["spec"].(map[string]interface{}) + url, _ := spec["url"].(string) + if url == "" { + return } - return nil + src := HelmRepositorySource{URL: url, IsOCI: strings.HasPrefix(url, "oci://")} + effNS := idx.effectiveNamespace(file, ref.Namespace) + + idx.byKey[sourceKey(ref.Namespace, ref.Name)] = src + idx.byKey[sourceKey(effNS, ref.Name)] = src + idx.byName[ref.Name] = append(idx.byName[ref.Name], sourceCandidate{ + rawNamespace: ref.Namespace, + effNamespace: effNS, + src: src, + }) } -// Resolve sets RepoURL and IsOCI on a chart target. It first tries an exact -// effective-namespace match, then falls back to matching by name in any -// namespace (a source often sits in the controller namespace, not the -// HelmRelease's). It fails loud only when the source is genuinely unresolvable. +// Resolve sets RepoURL/IsOCI (and, for chartRef targets, the edit anchor) on a +// chart target. For inline sourceRef targets it matches the HelmRepository by +// effective namespace then by name across namespaces; for chartRef targets it +// redirects onto the referenced OCIRepository/HelmChart. It is idempotent — an +// already-resolved target (RepoURL set) returns immediately — and fails loud only +// when the source is genuinely unresolvable. func (idx *SourceIndex) Resolve(t *UpdateTarget) error { - if t.Kind != TargetChart { + if t.Kind != TargetChart || t.RepoURL != "" { return nil } + if t.ChartRefName != "" { + return idx.resolveChartRef(t) + } if t.SourceRefName == "" { return fmt.Errorf("HelmRelease %s/%s has no sourceRef name", t.Ref.Namespace, t.Ref.Name) } @@ -107,49 +120,60 @@ func (idx *SourceIndex) Resolve(t *UpdateTarget) error { if wantNS == "" && idx.kt != nil { wantNS = idx.kt.EffectiveNamespace(t.File) } - - if src, ok := idx.byKey[sourceKey(wantNS, t.SourceRefName)]; ok { - return idx.apply(t, src) + src, err := idx.lookupHelmRepository(t.SourceRefName, wantNS, t.Ref) + if err != nil { + return err } + t.RepoURL = src.URL + t.IsOCI = src.IsOCI + return nil +} - candidates := idx.byName[t.SourceRefName] +// lookupHelmRepository finds a HelmRepository by effective namespace, then by +// name across namespaces (a source often sits in the controller namespace, not +// the consumer's). requester names the HelmRelease for error messages. +func (idx *SourceIndex) lookupHelmRepository(name, wantNS string, requester kubernetes.KubernetesRef) (HelmRepositorySource, error) { + if name == "" { + return HelmRepositorySource{}, fmt.Errorf("HelmRelease %s/%s has no HelmRepository sourceRef name", requester.Namespace, requester.Name) + } + if src, ok := idx.byKey[sourceKey(wantNS, name)]; ok { + return src, nil + } + candidates := idx.byName[name] switch { case len(candidates) == 0: - return fmt.Errorf("HelmRelease %s/%s references HelmRepository %s which was not found in the scanned manifests", - t.Ref.Namespace, t.Ref.Name, sourceKey(wantNS, t.SourceRefName)) + return HelmRepositorySource{}, fmt.Errorf("HelmRelease %s/%s references HelmRepository %s which was not found in the scanned manifests", + requester.Namespace, requester.Name, sourceKey(wantNS, name)) case len(candidates) == 1: - return idx.apply(t, candidates[0].src) + return candidates[0].src, nil default: - if c, ok := pickCandidate(candidates, wantNS); ok { - return idx.apply(t, c.src) + if c, ok := pickByNamespace(candidates, wantNS, + func(c sourceCandidate) string { return c.rawNamespace }, + func(c sourceCandidate) string { return c.effNamespace }); ok { + return c.src, nil } - return fmt.Errorf("HelmRelease %s/%s references HelmRepository %q which is ambiguous across namespaces %s", - t.Ref.Namespace, t.Ref.Name, t.SourceRefName, candidateNamespaces(candidates)) + return HelmRepositorySource{}, fmt.Errorf("HelmRelease %s/%s references HelmRepository %q which is ambiguous across namespaces %s", + requester.Namespace, requester.Name, name, candidateNamespaces(candidates)) } } -func (idx *SourceIndex) apply(t *UpdateTarget, src HelmRepositorySource) error { - t.RepoURL = src.URL - t.IsOCI = src.IsOCI - return nil -} - -// pickCandidate disambiguates multiple same-named sources: prefer an exact -// effective-namespace match, then a known controller namespace. -func pickCandidate(candidates []sourceCandidate, wantNS string) (sourceCandidate, bool) { +// pickByNamespace disambiguates multiple same-named candidates: prefer an exact +// raw/effective-namespace match, then a known controller namespace. +func pickByNamespace[C any](candidates []C, wantNS string, raw, eff func(C) string) (C, bool) { + var zero C for _, c := range candidates { - if c.effNamespace == wantNS || c.rawNamespace == wantNS { + if eff(c) == wantNS || raw(c) == wantNS { return c, true } } for _, ns := range controllerNamespaces { for _, c := range candidates { - if c.effNamespace == ns || c.rawNamespace == ns { + if eff(c) == ns || raw(c) == ns { return c, true } } } - return sourceCandidate{}, false + return zero, false } func candidateNamespaces(candidates []sourceCandidate) string { diff --git a/imageupdate/sourceref_test.go b/imageupdate/sourceref_test.go index 274c6f9..0140d92 100644 --- a/imageupdate/sourceref_test.go +++ b/imageupdate/sourceref_test.go @@ -8,7 +8,7 @@ import ( func TestSourceIndex_ResolveHTTP(t *testing.T) { content := readManifest(t, "helmrelease.yaml") idx := NewSourceIndex(nil) - if err := idx.IndexHelmRepositories("helmrelease.yaml", content); err != nil { + if err := idx.IndexSources("helmrelease.yaml", content); err != nil { t.Fatal(err) } targets, err := ExtractTargets("helmrelease.yaml", content) @@ -30,7 +30,7 @@ func TestSourceIndex_ResolveHTTP(t *testing.T) { func TestSourceIndex_ResolveOCI(t *testing.T) { content := readManifest(t, "helmrelease-oci.yaml") idx := NewSourceIndex(nil) - if err := idx.IndexHelmRepositories("helmrelease-oci.yaml", content); err != nil { + if err := idx.IndexSources("helmrelease-oci.yaml", content); err != nil { t.Fatal(err) } targets, _ := ExtractTargets("helmrelease-oci.yaml", content) @@ -73,7 +73,7 @@ func TestSourceIndex_TreeDerivedNamespace(t *testing.T) { kt := BuildKustomizeTree(files) idx := NewSourceIndex(kt) for f, content := range files { - if err := idx.IndexHelmRepositories(f, content); err != nil { + if err := idx.IndexSources(f, content); err != nil { t.Fatal(err) } } @@ -103,7 +103,7 @@ func TestSourceIndex_ControllerNamespaceFallback(t *testing.T) { // Two sources with the SAME name in different namespaces. repo := "apiVersion: source.toolkit.fluxcd.io/v1\nkind: HelmRepository\n" + "metadata:\n name: shared\n namespace: flux-system\nspec:\n url: oci://ghcr.io/acme/shared\n" - if err := idx.IndexHelmRepositories("sources.yaml", repo); err != nil { + if err := idx.IndexSources("sources.yaml", repo); err != nil { t.Fatal(err) } tg := UpdateTarget{Kind: TargetChart, SourceRefName: "shared", SourceRefNamespace: "my-app"} @@ -124,7 +124,7 @@ func TestSourceIndex_ControllerNamespaceFallback(t *testing.T) { func TestSourceIndex_CommentOnlyDocDoesNotDropTrailing(t *testing.T) { content := readManifest(t, "flux-multidoc.yaml") idx := NewSourceIndex(nil) - if err := idx.IndexHelmRepositories("flux.yaml", content); err != nil { + if err := idx.IndexSources("flux.yaml", content); err != nil { t.Fatal(err) } tg := UpdateTarget{Kind: TargetChart, SourceRefName: "flanksource-ecr", SourceRefNamespace: "flux-system"} @@ -144,8 +144,8 @@ func TestSourceIndex_AmbiguousNameFailsLoud(t *testing.T) { "metadata:\n name: dup\n namespace: team-a\nspec:\n url: https://a.example.com\n" b := "apiVersion: source.toolkit.fluxcd.io/v1\nkind: HelmRepository\n" + "metadata:\n name: dup\n namespace: team-b\nspec:\n url: https://b.example.com\n" - _ = idx.IndexHelmRepositories("a.yaml", a) - _ = idx.IndexHelmRepositories("b.yaml", b) + _ = idx.IndexSources("a.yaml", a) + _ = idx.IndexSources("b.yaml", b) tg := UpdateTarget{Kind: TargetChart, SourceRefName: "dup", SourceRefNamespace: "team-c"} tg.Ref.Namespace = "team-c" diff --git a/imageupdate/target.go b/imageupdate/target.go index 96b0968..4e94d74 100644 --- a/imageupdate/target.go +++ b/imageupdate/target.go @@ -37,4 +37,19 @@ type UpdateTarget struct { SourceRefNamespace string `json:"source_ref_namespace,omitempty"` RepoURL string `json:"repo_url,omitempty"` IsOCI bool `json:"oci,omitempty"` + + // ChartRef* are set when a HelmRelease selects its chart via spec.chartRef + // (Flux v2) instead of the inline spec.chart.spec template. They name the + // referenced OCIRepository/HelmChart source; SourceIndex.Resolve redirects + // File/FieldLine/CurrentValue/RepoURL onto that object (where the version + // literal actually lives) so the edit lands in the right place. + ChartRefKind string `json:"chart_ref_kind,omitempty"` + ChartRefName string `json:"chart_ref_name,omitempty"` + ChartRefNamespace string `json:"chart_ref_namespace,omitempty"` + + // SourceErr records why a chart target's Flux source could not be resolved + // (for example, the referenced HelmRepository is absent from the scanned + // manifests). When set, RepoURL is empty and version resolution must surface + // this actionable message instead of querying an empty URL. + SourceErr string `json:"source_error,omitempty"` } diff --git a/imageupdate/testdata/manifests/helmrelease-chartref-helmchart.yaml b/imageupdate/testdata/manifests/helmrelease-chartref-helmchart.yaml new file mode 100644 index 0000000..db178dc --- /dev/null +++ b/imageupdate/testdata/manifests/helmrelease-chartref-helmchart.yaml @@ -0,0 +1,29 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: app + namespace: apps +spec: + chartRef: + kind: HelmChart + name: app-chart +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmChart +metadata: + name: app-chart + namespace: apps +spec: + chart: app + version: 1.2.0 + sourceRef: + kind: HelmRepository + name: app-repo +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: app-repo + namespace: apps +spec: + url: https://charts.example.com/app diff --git a/imageupdate/testdata/manifests/helmrelease-chartref-oci.yaml b/imageupdate/testdata/manifests/helmrelease-chartref-oci.yaml new file mode 100644 index 0000000..8fbfd2a --- /dev/null +++ b/imageupdate/testdata/manifests/helmrelease-chartref-oci.yaml @@ -0,0 +1,19 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: podinfo + namespace: apps +spec: + chartRef: + kind: OCIRepository + name: podinfo +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: OCIRepository +metadata: + name: podinfo + namespace: apps +spec: + url: oci://ghcr.io/stefanprodan/charts/podinfo + ref: + tag: 6.5.0 diff --git a/tracked_yaml.go b/tracked_yaml.go new file mode 100644 index 0000000..40f1baf --- /dev/null +++ b/tracked_yaml.go @@ -0,0 +1,33 @@ +package repomap + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/flanksource/repomap/kubernetes" +) + +// TrackedYAMLContents returns every git-tracked YAML file in the repo keyed by +// its repo-relative POSIX path. Empty or unreadable files are skipped. It is the +// shared first pass for image/chart discovery (imageupdate.DiscoverTargets), +// used by both `repomap deps update` and `repomap images list`. +func (conf *ArchConf) TrackedYAMLContents() (map[string]string, error) { + result, err := conf.Exec()("ls-files") + if err != nil { + return nil, fmt.Errorf("git ls-files failed: %w", err) + } + contents := map[string]string{} + for _, line := range strings.Split(result.Stdout, "\n") { + file := filepath.ToSlash(strings.TrimSpace(line)) + if file == "" || !kubernetes.IsYaml(file) { + continue + } + content, err := conf.ReadFileWithFallback(file, "") + if err != nil || content == "" { + continue + } + contents[file] = content + } + return contents, nil +}