From 860eca316b5138325007ec727a867a9d42e9fae1 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 9 Jun 2026 07:21:49 +0300 Subject: [PATCH 01/17] feat(imageupdate): resolve both stable and pre-release versions with semantic versioning Add support for tracking both latest stable and pre-release versions separately using semantic versioning. This enables users to see available pre-release updates alongside stable updates. Key changes: - New LatestVersions struct to hold both stable and prerelease versions - ResolveLatestVersions() method returns both version classes in a single lookup - ImageInfo struct extended with LatestPrerelease and PrereleaseUpdateAvailable fields - UI columns expanded to show separate stable and pre-release update indicators - New semverUpdateAvailable() function for proper semantic version comparison - Display paths now relative to working directory for better readability - Added comprehensive tests for pre-release version handling BREAKING CHANGE: ResolveLatest() behavior unchanged but ResolveLatestVersions() is the new recommended API for getting version information. --- cmd/repomap/images_list.go | 146 ++++++++++++++++++++++---------- cmd/repomap/images_list_test.go | 59 +++++++++++-- cmd/repomap/images_update.go | 5 +- cmd/repomap/paths.go | 39 +++++++++ cmd/repomap/scan.go | 1 + imageupdate/resolver.go | 78 +++++++++-------- imageupdate/resolver_test.go | 15 ++++ 7 files changed, 253 insertions(+), 90 deletions(-) create mode 100644 cmd/repomap/paths.go diff --git a/cmd/repomap/images_list.go b/cmd/repomap/images_list.go index 73df896..4b81a6a 100644 --- a/cmd/repomap/images_list.go +++ b/cmd/repomap/images_list.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/Masterminds/semver/v3" "github.com/flanksource/clicky" "github.com/flanksource/clicky/api" "github.com/flanksource/clicky/task" @@ -15,7 +16,7 @@ import ( type ListImagesOptions struct { imageFilterOptions - Check bool `json:"check" flag:"check" help:"Resolve the latest available version from the registry/Helm repo"` + Check bool `json:"check" flag:"check" help:"Resolve the latest stable and pre-release versions from the registry/Helm repo"` } func (opts ListImagesOptions) GetName() string { return "list" } @@ -25,13 +26,13 @@ func (opts ListImagesOptions) Help() api.Text { Discovers apps/v1 workload images and Flux HelmRelease chart versions in git-tracked YAML and lists each with its current version, file, and resource. -Offline by default. With --check, the latest available version is resolved from -the container registry / Helm repository and an update-available column is shown. +Offline by default. With --check, the latest stable and pre-release versions are +resolved from the container registry / Helm repository using semantic versioning. EXAMPLES: repomap images list # all images and charts (offline) repomap images list -k Deployment # only Deployment images - repomap images list -k HelmRelease --check # show latest available chart versions`) + repomap images list -k HelmRelease --check # show latest stable and pre-release chart versions`) } func init() { @@ -39,17 +40,19 @@ func init() { cmd.Short = "List image tags and Helm chart versions in tracked manifests" } -// ImageInfo is one discovered image/chart row. When Checked, Latest and -// UpdateAvailable are populated from the registry/Helm repo. +// ImageInfo is one discovered image/chart row. When Checked, Latest, +// LatestPrerelease, and their update flags are populated from the registry/Helm repo. type ImageInfo struct { - Ref kubernetes.KubernetesRef `json:"ref"` - Kind imageupdate.TargetKind `json:"kind"` - File string `json:"file"` - Current string `json:"current"` - Latest string `json:"latest,omitempty"` - UpdateAvailable bool `json:"update_available"` - Checked bool `json:"-"` - Error string `json:"error,omitempty"` + Ref kubernetes.KubernetesRef `json:"ref"` + Kind imageupdate.TargetKind `json:"kind"` + File string `json:"file"` + Current string `json:"current"` + Latest string `json:"latest,omitempty"` + LatestPrerelease string `json:"latest_prerelease,omitempty"` + UpdateAvailable bool `json:"update_available"` + PrereleaseUpdateAvailable bool `json:"prerelease_update_available"` + Checked bool `json:"-"` + Error string `json:"error,omitempty"` } func (i ImageInfo) Pretty() api.Text { @@ -69,8 +72,10 @@ func (i ImageInfo) Columns() []api.ColumnDef { } if i.Checked { cols = append(cols, - api.Column("latest").Label("Latest").Build(), - api.Column("update").Label("Update").Build(), + api.Column("latest").Label("Latest Stable").Build(), + api.Column("stable_update").Label("Stable Update").Build(), + api.Column("latest_prerelease").Label("Latest Pre-release").Build(), + api.Column("prerelease_update").Label("Pre-release Update").Build(), ) } return cols @@ -84,23 +89,23 @@ func (i ImageInfo) Row() map[string]any { "current": clicky.Text(i.Current, "font-mono"), } if i.Checked { - switch { - case i.Error != "": + if i.Error != "" { row["latest"] = clicky.Text(i.Error, "text-red-600") - row["update"] = clicky.Text("?", "text-muted") - case i.UpdateAvailable: - row["latest"] = clicky.Text(i.Latest, "font-mono text-green-600") - row["update"] = clicky.Text("✓", "text-green-600") - default: - row["latest"] = clicky.Text(i.Latest, "font-mono text-muted") - row["update"] = clicky.Text("—", "text-muted") + row["stable_update"] = clicky.Text("?", "text-muted") + row["latest_prerelease"] = clicky.Text(i.Error, "text-red-600") + row["prerelease_update"] = clicky.Text("?", "text-muted") + } else { + row["latest"] = versionCell(i.Latest, i.UpdateAvailable) + row["stable_update"] = updateCell(i.UpdateAvailable) + row["latest_prerelease"] = versionCell(i.LatestPrerelease, i.PrereleaseUpdateAvailable) + row["prerelease_update"] = updateCell(i.PrereleaseUpdateAvailable) } } return row } func runListImages(opts ListImagesOptions) (any, error) { - targets, sourceIndex, _, err := discoverAndFilter(opts.imageFilterOptions) + targets, sourceIndex, conf, err := discoverAndFilter(opts.imageFilterOptions) if err != nil { return nil, err } @@ -112,7 +117,7 @@ func runListImages(opts ListImagesOptions) (any, error) { if opts.Check { resolver = imageupdate.NewResolver() } - infos, err := buildImageInfos(context.Background(), resolver, targets, sourceIndex, opts.Check) + infos, err := buildImageInfos(context.Background(), resolver, targets, sourceIndex, opts.Check, displayPathFuncForConf(conf)) if err != nil { return nil, err } @@ -123,11 +128,14 @@ func runListImages(opts ListImagesOptions) (any, error) { // offline mapping. With --check each target's version lookup runs as its own // clicky task (concurrently, with live progress); a per-target resolution error // is recorded on the row rather than aborting the whole listing. -func buildImageInfos(ctx context.Context, resolver *imageupdate.Resolver, targets []imageupdate.UpdateTarget, sourceIndex *imageupdate.SourceIndex, check bool) ([]ImageInfo, error) { +func buildImageInfos(ctx context.Context, resolver *imageupdate.Resolver, targets []imageupdate.UpdateTarget, sourceIndex *imageupdate.SourceIndex, check bool, displayPath displayPathFunc) ([]ImageInfo, error) { + if displayPath == nil { + displayPath = func(path string) string { return path } + } if !check { infos := make([]ImageInfo, len(targets)) for i, t := range targets { - infos[i] = baseInfo(t, false) + infos[i] = baseInfo(t, false, displayPath(t.File)) } return infos, nil } @@ -136,8 +144,9 @@ func buildImageInfos(ctx context.Context, resolver *imageupdate.Resolver, target group := task.StartGroup[int]("Resolving image versions", task.WithConcurrency(resolveConcurrency)) for i, t := range targets { idx, target := i, t - group.Add(taskName(target), func(ctx flanksourceContext.Context, tk *task.Task) (int, error) { - infos[idx] = checkInfo(ctx, resolver, sourceIndex, target, tk) + displayFile := displayPath(target.File) + group.Add(taskName(target, displayFile), func(ctx flanksourceContext.Context, tk *task.Task) (int, error) { + infos[idx] = checkInfo(ctx, resolver, sourceIndex, target, displayFile, tk) return idx, nil }) } @@ -147,14 +156,15 @@ func buildImageInfos(ctx context.Context, resolver *imageupdate.Resolver, target return infos, nil } -func baseInfo(t imageupdate.UpdateTarget, checked bool) ImageInfo { - return ImageInfo{Ref: t.Ref, Kind: t.Kind, File: t.File, Current: t.CurrentValue, Checked: checked} +func baseInfo(t imageupdate.UpdateTarget, checked bool, displayFile string) ImageInfo { + return ImageInfo{Ref: t.Ref, Kind: t.Kind, File: displayFile, Current: t.CurrentValue, Checked: checked} } -// checkInfo resolves a single target's latest version, recording any failure on -// the row. Resolution errors do not fail the task — the listing reports them. -func checkInfo(ctx context.Context, resolver *imageupdate.Resolver, sourceIndex *imageupdate.SourceIndex, t imageupdate.UpdateTarget, tk *task.Task) ImageInfo { - info := baseInfo(t, true) +// checkInfo resolves a single target's latest stable and pre-release versions, +// recording any failure on the row. Resolution errors do not fail the task; the +// listing reports them. +func checkInfo(ctx context.Context, resolver *imageupdate.Resolver, sourceIndex *imageupdate.SourceIndex, t imageupdate.UpdateTarget, displayFile string, tk *task.Task) ImageInfo { + info := baseInfo(t, true, displayFile) if t.Kind == imageupdate.TargetChart { if err := sourceIndex.Resolve(&t); err != nil { tk.Warnf("source unresolved: %v", err) @@ -162,22 +172,69 @@ func checkInfo(ctx context.Context, resolver *imageupdate.Resolver, sourceIndex return info } } - tk.Infof("looking up latest version") - latest, err := resolver.ResolveLatest(ctx, t) + tk.Infof("looking up latest stable and pre-release versions") + latest, err := resolver.ResolveLatestVersions(ctx, t) if err != nil { tk.Errorf("%v", err) info.Error = err.Error() return info } - info.Latest = latest - info.UpdateAvailable = latest != "" && latest != versionOnly(t.CurrentValue) + if latest.Stable == "" && latest.Prerelease == "" { + info.Error = fmt.Sprintf("no semver-matching version found for %s", versionSourceLabel(t)) + tk.Errorf("%s", info.Error) + return info + } + info.Latest = latest.Stable + info.LatestPrerelease = latest.Prerelease + info.UpdateAvailable = semverUpdateAvailable(t.CurrentValue, info.Latest) + info.PrereleaseUpdateAvailable = semverUpdateAvailable(t.CurrentValue, info.LatestPrerelease) return info } +func versionCell(version string, update bool) api.Text { + if version == "" { + return clicky.Text("-", "text-muted") + } + if update { + return clicky.Text(version, "font-mono text-green-600") + } + return clicky.Text(version, "font-mono text-muted") +} + +func updateCell(update bool) api.Text { + if update { + return clicky.Text("✓", "text-green-600") + } + return clicky.Text("—", "text-muted") +} + +func semverUpdateAvailable(currentValue, latest string) bool { + if latest == "" { + return false + } + current := versionOnly(currentValue) + currentSemver, currentErr := semver.NewVersion(current) + latestSemver, latestErr := semver.NewVersion(latest) + if currentErr != nil || latestErr != nil { + return latest != current + } + return latestSemver.GreaterThan(currentSemver) +} + +func versionSourceLabel(t imageupdate.UpdateTarget) string { + if t.Kind == imageupdate.TargetChart && !t.IsOCI { + return fmt.Sprintf("chart %q in %s", t.ChartName, t.RepoURL) + } + if t.Kind == imageupdate.TargetImage && t.Image != nil { + return fmt.Sprintf("image %s", t.Image.GetFullNameWithoutTag()) + } + return fmt.Sprintf("%s %q", t.Kind, t.CurrentValue) +} + // taskName builds a descriptive, unique label for a target's resolution task: // the chart/image, the file it lives in, and the namespace when one is known // (the namespace is often imposed by Flux/kustomize and left empty in the file). -func taskName(t imageupdate.UpdateTarget) string { +func taskName(t imageupdate.UpdateTarget, displayFile string) string { var subject string if t.Kind == imageupdate.TargetChart { subject = "chart " + t.ChartName @@ -187,7 +244,10 @@ func taskName(t imageupdate.UpdateTarget) string { subject += " [" + t.ContainerName + "]" } } - subject += " in " + t.File + if displayFile == "" { + displayFile = t.File + } + subject += " in " + displayFile if ns := targetNamespace(t); ns != "" { subject += " (ns " + ns + ")" } diff --git a/cmd/repomap/images_list_test.go b/cmd/repomap/images_list_test.go index 519c10f..bcf38d5 100644 --- a/cmd/repomap/images_list_test.go +++ b/cmd/repomap/images_list_test.go @@ -14,7 +14,7 @@ func TestBuildImageInfos_OfflineHasNoLatest(t *testing.T) { content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) targets, _ := imageupdate.ExtractTargets(rel, string(content)) - infos, err := buildImageInfos(context.Background(), nil, targets, imageupdate.NewSourceIndex(nil), false) + infos, err := buildImageInfos(context.Background(), nil, targets, imageupdate.NewSourceIndex(nil), false, nil) if err != nil { t.Fatal(err) } @@ -25,8 +25,8 @@ func TestBuildImageInfos_OfflineHasNoLatest(t *testing.T) { if info.Current != "nginx:1.25.3" { t.Errorf("current = %q, want nginx:1.25.3", info.Current) } - if info.Checked || info.Latest != "" { - t.Errorf("offline row should not be checked or have latest: %+v", info) + if info.Checked || info.Latest != "" || info.LatestPrerelease != "" { + t.Errorf("offline row should not be checked or have latest versions: %+v", info) } // offline Columns omit latest/update if len(info.Columns()) != 4 { @@ -39,8 +39,8 @@ func TestBuildImageInfos_CheckFlagsUpdateAvailable(t *testing.T) { content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) targets, _ := imageupdate.ExtractTargets(rel, string(content)) - resolver := fakeImageResolver([]string{"1.25.3", "1.27.0"}) - infos, err := buildImageInfos(context.Background(), resolver, targets, imageupdate.NewSourceIndex(nil), true) + resolver := fakeImageResolver([]string{"1.25.3", "1.27.0", "1.28.0-beta.1"}) + infos, err := buildImageInfos(context.Background(), resolver, targets, imageupdate.NewSourceIndex(nil), true, nil) if err != nil { t.Fatal(err) } @@ -51,11 +51,17 @@ func TestBuildImageInfos_CheckFlagsUpdateAvailable(t *testing.T) { if info.Latest != "1.27.0" { t.Errorf("latest = %q, want 1.27.0", info.Latest) } + if info.LatestPrerelease != "1.28.0-beta.1" { + t.Errorf("latest prerelease = %q, want 1.28.0-beta.1", info.LatestPrerelease) + } if !info.UpdateAvailable { t.Error("expected update available (1.25.3 -> 1.27.0)") } - if len(info.Columns()) != 6 { - t.Errorf("checked columns = %d, want 6", len(info.Columns())) + if !info.PrereleaseUpdateAvailable { + t.Error("expected prerelease update available (1.25.3 -> 1.28.0-beta.1)") + } + if len(info.Columns()) != 8 { + t.Errorf("checked columns = %d, want 8", len(info.Columns())) } } @@ -65,8 +71,45 @@ func TestBuildImageInfos_CheckNoUpdateWhenCurrentIsLatest(t *testing.T) { targets, _ := imageupdate.ExtractTargets(rel, string(content)) resolver := fakeImageResolver([]string{"1.25.3", "1.24.0"}) - infos, _ := buildImageInfos(context.Background(), resolver, targets, imageupdate.NewSourceIndex(nil), true) + infos, _ := buildImageInfos(context.Background(), resolver, targets, imageupdate.NewSourceIndex(nil), true, nil) if infos[0].UpdateAvailable { t.Errorf("no update expected when 1.25.3 is already newest; got latest=%q", infos[0].Latest) } } + +func TestBuildImageInfos_CheckDoesNotFlagOlderPrerelease(t *testing.T) { + conf, rel := writeRepo(t) + content, _ := os.ReadFile(filepath.Join(conf.RepoPath(), rel)) + targets, _ := imageupdate.ExtractTargets(rel, string(content)) + + resolver := fakeImageResolver([]string{"1.25.3", "1.24.0-rc.1"}) + infos, err := buildImageInfos(context.Background(), resolver, targets, imageupdate.NewSourceIndex(nil), true, nil) + if err != nil { + t.Fatal(err) + } + info := infos[0] + if info.LatestPrerelease != "1.24.0-rc.1" { + t.Errorf("latest prerelease = %q, want 1.24.0-rc.1", info.LatestPrerelease) + } + if info.PrereleaseUpdateAvailable { + t.Errorf("older prerelease should not be an available update: %+v", info) + } +} + +func TestDisplayPathForRepoFileRelativeToWorkingDir(t *testing.T) { + conf, _ := writeRepo(t) + oldWorkingDir := workingDir + t.Cleanup(func() { workingDir = oldWorkingDir }) + + workingDir = filepath.Join(conf.RepoPath(), "apps") + if err := os.MkdirAll(workingDir, 0o755); err != nil { + t.Fatal(err) + } + + if got := displayPathForRepoFile(conf, "apps/api/deploy.yaml"); got != "api/deploy.yaml" { + t.Errorf("inside cwd = %q, want api/deploy.yaml", got) + } + if got := displayPathForRepoFile(conf, "clusters/prod/deploy.yaml"); got != "../clusters/prod/deploy.yaml" { + t.Errorf("outside cwd = %q, want ../clusters/prod/deploy.yaml", got) + } +} diff --git a/cmd/repomap/images_update.go b/cmd/repomap/images_update.go index 7452ddf..96ab8a8 100644 --- a/cmd/repomap/images_update.go +++ b/cmd/repomap/images_update.go @@ -151,7 +151,8 @@ func resolveConcurrently(ctx context.Context, resolver *imageupdate.Resolver, co group := task.StartGroup[int]("Resolving image versions", task.WithConcurrency(resolveConcurrency)) for i, t := range targets { idx, target := i, t - group.Add(taskName(target), func(ctx flanksourceContext.Context, tk *task.Task) (int, error) { + displayFile := displayPathForRepoFile(conf, target.File) + group.Add(taskName(target, displayFile), func(ctx flanksourceContext.Context, tk *task.Task) (int, error) { newValue, skipped, err := resolveNewValue(ctx, resolver, target, opts, tk) results[idx] = resolved{newValue, skipped, err} return idx, nil @@ -203,7 +204,7 @@ func applyResolved(conf *repomap.ArchConf, t imageupdate.UpdateTarget, newValue, plan := UpdatePlan{ Ref: t.Ref, Kind: t.Kind, - File: t.File, + File: displayPathForRepoFile(conf, t.File), Field: t.FieldJSONPath, OldValue: t.CurrentValue, DryRun: opts.DryRun, diff --git a/cmd/repomap/paths.go b/cmd/repomap/paths.go new file mode 100644 index 0000000..d2bdfd0 --- /dev/null +++ b/cmd/repomap/paths.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/flanksource/repomap" +) + +type displayPathFunc func(string) string + +func displayPathForRepoFile(conf *repomap.ArchConf, repoRelPath string) string { + if conf == nil || repoRelPath == "" { + return repoRelPath + } + absPath := filepath.Join(conf.RepoPath(), filepath.FromSlash(repoRelPath)) + cwd, err := displayCwd() + if err != nil { + return filepath.ToSlash(filepath.Clean(repoRelPath)) + } + rel, err := filepath.Rel(cwd, absPath) + if err != nil { + return filepath.ToSlash(filepath.Clean(repoRelPath)) + } + return filepath.ToSlash(rel) +} + +func displayPathFuncForConf(conf *repomap.ArchConf) displayPathFunc { + return func(repoRelPath string) string { + return displayPathForRepoFile(conf, repoRelPath) + } +} + +func displayCwd() (string, error) { + if workingDir != "" { + return filepath.Abs(workingDir) + } + return os.Getwd() +} diff --git a/cmd/repomap/scan.go b/cmd/repomap/scan.go index b499615..0d8d032 100644 --- a/cmd/repomap/scan.go +++ b/cmd/repomap/scan.go @@ -102,6 +102,7 @@ func runScan(opts ScanOptions) (any, error) { if !opts.Verbose { fm.ScopeMatches = nil } + fm.Path = displayPathForRepoFile(conf, relPath) results = append(results, *fm) } diff --git a/imageupdate/resolver.go b/imageupdate/resolver.go index f53b7fb..f703ae8 100644 --- a/imageupdate/resolver.go +++ b/imageupdate/resolver.go @@ -40,6 +40,13 @@ type versionsResult struct { err error } +// LatestVersions is the newest stable and pre-release semver version published +// by a target source. Empty fields mean that class was not present. +type LatestVersions struct { + Stable string + Prerelease string +} + // NewResolver returns a Resolver wired to live registry-scanner and HTTP Helm // index clients, authenticating against the local Docker credential store. func NewResolver() *Resolver { @@ -94,18 +101,25 @@ func (r *Resolver) Available(ctx context.Context, t UpdateTarget) ([]string, err // ResolveLatest returns the highest stable (non-prerelease) version for a target. func (r *Resolver) ResolveLatest(ctx context.Context, t UpdateTarget) (string, error) { - if t.Kind == TargetImage || (t.Kind == TargetChart && t.IsOCI) { - return r.latestImageTag(ctx, t) - } - versions, err := r.sourceVersions(ctx, t) + latest, err := r.ResolveLatestVersions(ctx, t) if err != nil { return "", err } - sorted := sortSemverDesc(filterStable(versions)) - if len(sorted) == 0 { - return "", fmt.Errorf("no stable version found for chart %q in %s", t.ChartName, t.RepoURL) + if latest.Stable == "" { + return "", fmt.Errorf("no stable version found for %s", targetVersionSource(t)) + } + return latest.Stable, nil +} + +// ResolveLatestVersions returns both the highest stable and highest pre-release +// semver versions for a target. It uses the shared source version cache, so a +// caller can ask for both classes without a second registry or Helm lookup. +func (r *Resolver) ResolveLatestVersions(ctx context.Context, t UpdateTarget) (LatestVersions, error) { + versions, err := r.sourceVersions(ctx, t) + if err != nil { + return LatestVersions{}, err } - return sorted[0], nil + return latestSemverVersions(versions), nil } // sourceVersionKey identifies the upstream a target's versions come from, so two @@ -179,30 +193,6 @@ func registryImage(t UpdateTarget) *image.ContainerImage { return image.NewFromIdentifier(t.CurrentValue) } -func (r *Resolver) latestImageTag(ctx context.Context, t UpdateTarget) (string, error) { - tags, err := r.sourceVersions(ctx, t) - if err != nil { - return "", err - } - tagList := tag.NewImageTagList() - for _, name := range filterStable(tags) { - tagList.Add(tag.NewImageTag(name, time.Time{}, "")) - } - vc := image.NewVersionConstraint() - vc.Strategy = image.StrategySemVer - vc.Options = options.NewManifestOptions() - - img := registryImage(t) - newest, err := img.GetNewestVersionFromTags(ctx, vc, tagList) - if err != nil { - return "", err - } - if newest == nil { - return "", fmt.Errorf("no semver-matching tag found for image %s", img.GetFullNameWithoutTag()) - } - return newest.TagName, nil -} - // NewImageValue composes the replacement string for an image target updated to // newTag. When the current image is digest-pinned, the re-resolve policy fetches // newTag's digest and writes repo:newtag@sha256:; otherwise it writes @@ -269,16 +259,30 @@ func sortSemverDesc(versions []string) []string { return out } -func filterStable(versions []string) []string { - var out []string - for _, v := range versions { +func latestSemverVersions(versions []string) LatestVersions { + var latest LatestVersions + for _, v := range sortSemverDesc(versions) { sv, err := semver.NewVersion(v) if err != nil { continue } if strings.TrimSpace(sv.Prerelease()) == "" { - out = append(out, v) + if latest.Stable == "" { + latest.Stable = v + } + } else if latest.Prerelease == "" { + latest.Prerelease = v + } + if latest.Stable != "" && latest.Prerelease != "" { + return latest } } - return out + return latest +} + +func targetVersionSource(t UpdateTarget) string { + if t.Kind == TargetChart && !t.IsOCI { + return fmt.Sprintf("chart %q in %s", t.ChartName, t.RepoURL) + } + return fmt.Sprintf("image %s", registryImage(t).GetFullNameWithoutTag()) } diff --git a/imageupdate/resolver_test.go b/imageupdate/resolver_test.go index e05afdb..1d829c7 100644 --- a/imageupdate/resolver_test.go +++ b/imageupdate/resolver_test.go @@ -139,6 +139,21 @@ func TestResolver_LatestImage_ExcludesPrerelease(t *testing.T) { } } +func TestResolver_LatestVersions_ReturnsStableAndPrerelease(t *testing.T) { + r := mockRegistryResolver([]string{"1.25.3", "1.28.0-beta.1", "1.27.0", "latest", "1.28.0-alpha.1"}) + tg := UpdateTarget{Kind: TargetImage, CurrentValue: "nginx:1.25.3", Image: image.NewFromIdentifier("nginx:1.25.3")} + latest, err := r.ResolveLatestVersions(context.Background(), tg) + if err != nil { + t.Fatal(err) + } + if latest.Stable != "1.27.0" { + t.Errorf("stable = %q, want 1.27.0", latest.Stable) + } + if latest.Prerelease != "1.28.0-beta.1" { + t.Errorf("prerelease = %q, want 1.28.0-beta.1", latest.Prerelease) + } +} + func TestResolver_LatestChart_HTTP(t *testing.T) { r := &Resolver{HelmIndex: fakeHelmIndex{versions: []string{"6.5.0", "6.6.0", "6.7.0-rc.1", "6.5.4"}}} tg := UpdateTarget{Kind: TargetChart, ChartName: "podinfo", RepoURL: "https://example.com"} From 26b85c0d9959a50adecdbbc1c1cb3e5861ff4bd8 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 9 Jun 2026 08:38:36 +0300 Subject: [PATCH 02/17] feat(deps): add dependency graph generation for Go, Maven, Gradle, npm, and pnpm Implement a new `deps` command that generates dependency graphs for multiple package managers. Supports auto-detection of manifests, native package manager resolution with fallback to manifest parsing, filtering, and depth control. Includes comprehensive support for Go modules, Maven POMs, Gradle builds, npm/pnpm lockfiles with circular dependency detection and duplicate analysis. Features: - Auto-detect and resolve dependencies for go, maven, gradle, npm, pnpm - Native resolution via package manager tools with manifest fallback - Configurable depth limiting and dependency filtering - Duplicate and conflict detection - JSON export for programmatic consumption - Pretty-printed tree output with styled dependency metadata Refs: dependency graph analysis --- README.md | 25 +++- cmd/repomap/deps.go | 118 +++++++++++++++ cmd/repomap/deps_test.go | 55 +++++++ cmd/repomap/main.go | 2 +- deps/common.go | 76 ++++++++++ deps/discover.go | 150 ++++++++++++++++++++ deps/filter.go | 251 ++++++++++++++++++++++++++++++++ deps/go.go | 226 +++++++++++++++++++++++++++++ deps/gradle.go | 217 ++++++++++++++++++++++++++++ deps/maven.go | 197 ++++++++++++++++++++++++++ deps/model.go | 167 ++++++++++++++++++++++ deps/npm.go | 289 +++++++++++++++++++++++++++++++++++++ deps/pnpm.go | 299 +++++++++++++++++++++++++++++++++++++++ deps/pretty.go | 231 ++++++++++++++++++++++++++++++ deps/runner.go | 39 +++++ deps/scan.go | 251 ++++++++++++++++++++++++++++++++ deps/scan_test.go | 259 +++++++++++++++++++++++++++++++++ go.mod | 1 + 18 files changed, 2851 insertions(+), 2 deletions(-) create mode 100644 cmd/repomap/deps.go create mode 100644 cmd/repomap/deps_test.go create mode 100644 deps/common.go create mode 100644 deps/discover.go create mode 100644 deps/filter.go create mode 100644 deps/go.go create mode 100644 deps/gradle.go create mode 100644 deps/maven.go create mode 100644 deps/model.go create mode 100644 deps/npm.go create mode 100644 deps/pnpm.go create mode 100644 deps/pretty.go create mode 100644 deps/runner.go create mode 100644 deps/scan.go create mode 100644 deps/scan_test.go diff --git a/README.md b/README.md index 17f79c4..9e29d4c 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,29 @@ repomap eval --file rules.yaml repomap eval --expr 'kubernetes.kind == "Secret"' ``` +### `deps` + +Generate dependency graphs for Go, Maven, Gradle, npm, and pnpm projects. + +```bash +# Pretty tree output +repomap deps + +# JSON export via stdout +repomap deps --json > out.json + +# Only scan selected managers +repomap deps --manager go,pnpm + +# Include transitive dependencies +repomap deps --depth 0 +``` + +By default, `deps` auto-detects supported manifests, tries native package +manager resolution, and falls back to manifest or lockfile parsing with warnings. +It prints direct dependencies by default; use `--depth 0` for the full +transitive graph. + ### `version` Print version, commit hash, build date, and Go version. @@ -81,7 +104,7 @@ Print version, commit hash, build date, and Go version. | Flag | Description | Default | |------|-------------|---------| -| `--format` / `-o` | Output format: `pretty`, `json`, `yaml`, `csv`, `table` | `pretty` | +| `--format` / `--json` / `--yaml` / `--csv` | Output format: `pretty`, `json`, `yaml`, `csv`, `markdown`, `html` | `pretty` | ## Configuration diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go new file mode 100644 index 0000000..21b1324 --- /dev/null +++ b/cmd/repomap/deps.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + depgraph "github.com/flanksource/repomap/deps" +) + +type DepsOptions struct { + Path string `json:"path" args:"true" help:"Path to scan" default:"."` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm (repeatable or comma-separated)"` + Mode string `json:"mode,omitempty" flag:"mode" default:"auto" help:"Resolution mode: auto, native, or manifest"` + Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` + Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` + Configuration []string `json:"configuration,omitempty" flag:"configuration" help:"Gradle configuration to resolve (repeatable or comma-separated); default is all resolvable configurations"` + Strict bool `json:"strict,omitempty" flag:"strict" help:"Fail if native resolution is unavailable or fallback resolution is degraded"` +} + +func (opts DepsOptions) GetName() string { return "deps" } + +func (opts DepsOptions) Help() api.Text { + return clicky.Text(`Generate dependency graphs for Go, Maven, Gradle, npm, and pnpm projects. + +Repomap auto-detects supported manifests below the selected path, resolves +transitive dependency graphs with native tools when available, and falls back to +manifest or lockfile parsing with warnings when native resolution is unavailable. + +The command uses the normal Clicky output flow. Use --json to write structured +JSON to stdout, for example: + + repomap deps --json > out.json + +EXAMPLES: + repomap deps + repomap deps ./service --manager go + repomap deps --manager npm,pnpm --depth 0 + repomap deps --filter 'github.com/flanksource/*,!*test*' + repomap deps --mode manifest --json > deps.json`) +} + +func init() { + cmd := clicky.AddNamedCommandWithContext("deps", rootCmd, DepsOptions{}, runDeps) + cmd.Short = "Generate dependency graphs for Go, Maven, Gradle, npm, and pnpm projects" +} + +func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { + if opts.Path == "" { + opts.Path = "." + } + path, err := resolvePath(opts.Path) + if err != nil { + return nil, err + } + managers, err := parseManagers(opts.Manager) + if err != nil { + return nil, err + } + mode, err := parseDepsMode(opts.Mode) + if err != nil { + return nil, err + } + return depgraph.Scan(ctx, path, depgraph.Options{ + Managers: managers, + Mode: mode, + MaxDepth: opts.Depth, + Filters: splitCommaArgs(opts.Filter), + Configurations: splitCommaArgs(opts.Configuration), + Strict: opts.Strict, + }) +} + +func parseDepsMode(value string) (depgraph.Mode, error) { + switch depgraph.Mode(strings.TrimSpace(value)) { + case "", depgraph.ModeAuto: + return depgraph.ModeAuto, nil + case depgraph.ModeNative: + return depgraph.ModeNative, nil + case depgraph.ModeManifest: + return depgraph.ModeManifest, nil + default: + return "", fmt.Errorf("unsupported deps mode %q (expected auto, native, or manifest)", value) + } +} + +func parseManagers(values []string) ([]depgraph.Manager, error) { + parts := splitCommaArgs(values) + if len(parts) == 0 { + return nil, nil + } + out := make([]depgraph.Manager, 0, len(parts)) + for _, part := range parts { + manager := depgraph.Manager(strings.ToLower(part)) + switch manager { + case depgraph.ManagerGo, depgraph.ManagerMaven, depgraph.ManagerGradle, depgraph.ManagerNPM, depgraph.ManagerPNPM: + out = append(out, manager) + default: + return nil, fmt.Errorf("unsupported dependency manager %q (expected go, maven, gradle, npm, or pnpm)", part) + } + } + return out, nil +} + +func splitCommaArgs(values []string) []string { + var out []string + for _, value := range values { + for _, part := range strings.Split(value, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + } + return out +} diff --git a/cmd/repomap/deps_test.go b/cmd/repomap/deps_test.go new file mode 100644 index 0000000..02bb536 --- /dev/null +++ b/cmd/repomap/deps_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "strings" + "testing" + + depgraph "github.com/flanksource/repomap/deps" +) + +func TestParseManagers(t *testing.T) { + got, err := parseManagers([]string{"go,npm", "pnpm"}) + if err != nil { + t.Fatal(err) + } + want := []depgraph.Manager{depgraph.ManagerGo, depgraph.ManagerNPM, depgraph.ManagerPNPM} + if len(got) != len(want) { + t.Fatalf("len = %d, want %d: %#v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("manager[%d] = %q, want %q", i, got[i], want[i]) + } + } + if _, err := parseManagers([]string{"ruby"}); err == nil { + t.Fatal("expected unsupported manager error") + } +} + +func TestParseDepsMode(t *testing.T) { + for _, mode := range []string{"", "auto", "native", "manifest"} { + if _, err := parseDepsMode(mode); err != nil { + t.Fatalf("parseDepsMode(%q): %v", mode, err) + } + } + if _, err := parseDepsMode("lockfile"); err == nil { + t.Fatal("expected unsupported mode error") + } +} + +func TestDepsDepthDefault(t *testing.T) { + cmd, _, err := rootCmd.Find([]string{"deps"}) + if err != nil { + t.Fatal(err) + } + flag := cmd.Flags().Lookup("depth") + if flag == nil { + t.Fatal("depth flag not registered") + } + if flag.DefValue != "1" { + t.Fatalf("depth default = %q, want 1", flag.DefValue) + } + if !strings.Contains(flag.Usage, "0 = unlimited") { + t.Fatalf("depth help should document unlimited mode, got %q", flag.Usage) + } +} diff --git a/cmd/repomap/main.go b/cmd/repomap/main.go index 4e689ed..5deaf0d 100644 --- a/cmd/repomap/main.go +++ b/cmd/repomap/main.go @@ -32,7 +32,7 @@ When run without a subcommand, defaults to 'scan'.`, } func init() { - clicky.BindAllFlags(rootCmd.PersistentFlags(), "format") + clicky.BindAllFlags(rootCmd.PersistentFlags(), "tasks", "format") logger.Configure(logger.Flags{LogToStderr: true, Color: true}) rootCmd.PersistentFlags().StringVar(&workingDir, "cwd", "", "Working directory") diff --git a/deps/common.go b/deps/common.go new file mode 100644 index 0000000..2ea02e9 --- /dev/null +++ b/deps/common.go @@ -0,0 +1,76 @@ +package deps + +import ( + "path/filepath" + "sort" + "strings" +) + +func sortChildren(node *Node) { + if node == nil { + return + } + sort.SliceStable(node.Children, func(i, j int) bool { + return dependencyLess(node.Children[i], node.Children[j]) + }) +} + +func dependencyLess(a, b *Node) bool { + if a == nil { + return b != nil + } + if b == nil { + return false + } + if ar, br := dependencySortRank(a), dependencySortRank(b); ar != br { + return ar < br + } + if an, bn := strings.ToLower(a.Name), strings.ToLower(b.Name); an != bn { + return an < bn + } + if a.Name != b.Name { + return a.Name < b.Name + } + if a.Version != b.Version { + return a.Version < b.Version + } + if a.Manager != b.Manager { + return a.Manager < b.Manager + } + return a.ID < b.ID +} + +func dependencySortRank(node *Node) int { + if isReplacementDependency(node) { + return 0 + } + if node != nil && node.Direct { + return 1 + } + return 2 +} + +func isReplacementDependency(node *Node) bool { + if node == nil { + return false + } + if node.Local { + return true + } + return node.Manager == ManagerGo && node.Source != "" && node.Source != "go.mod" && node.Source != "go mod graph" +} + +func sortStrings(values []string) { + sort.Strings(values) +} + +func isLocalRef(ref string) bool { + ref = strings.TrimSpace(ref) + if ref == "" { + return false + } + if strings.HasPrefix(ref, "file:") || strings.HasPrefix(ref, "link:") || strings.HasPrefix(ref, "portal:") { + return true + } + return strings.HasPrefix(ref, ".") || strings.HasPrefix(ref, "/") || strings.HasPrefix(ref, ".."+string(filepath.Separator)) || strings.HasPrefix(ref, "../") +} diff --git a/deps/discover.go b/deps/discover.go new file mode 100644 index 0000000..2b505aa --- /dev/null +++ b/deps/discover.go @@ -0,0 +1,150 @@ +package deps + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +var ignoredDirs = map[string]bool{ + ".git": true, + ".gradle": true, + "build": true, + "dist": true, + "node_modules": true, + "target": true, + "vendor": true, +} + +func Discover(root string, managers []Manager) ([]Project, []Warning, error) { + selected := managerSet(managers) + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, nil, err + } + info, err := os.Stat(absRoot) + if err != nil { + return nil, nil, err + } + if !info.IsDir() { + absRoot = filepath.Dir(absRoot) + } + + byDir := map[string]map[string]string{} + err = filepath.WalkDir(absRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if path != absRoot && ignoredDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + name := d.Name() + manager := managerForManifest(name) + if manager == "" { + return nil + } + if len(selected) > 0 && !selected[manager] { + return nil + } + dir := filepath.Dir(path) + if byDir[dir] == nil { + byDir[dir] = map[string]string{} + } + byDir[dir][name] = path + return nil + }) + if err != nil { + return nil, nil, err + } + + var projects []Project + var warnings []Warning + for dir, files := range byDir { + if path := files["go.work"]; path != "" { + projects = append(projects, Project{Manager: ManagerGo, Dir: dir, File: path, Name: filepath.Base(dir)}) + } else if path := files["go.mod"]; path != "" { + projects = append(projects, Project{Manager: ManagerGo, Dir: dir, File: path, Name: filepath.Base(dir)}) + } + if path := files["pom.xml"]; path != "" { + projects = append(projects, Project{Manager: ManagerMaven, Dir: dir, File: path, Name: filepath.Base(dir)}) + } + if path := files["build.gradle"]; path != "" { + projects = append(projects, Project{Manager: ManagerGradle, Dir: dir, File: path, Name: filepath.Base(dir)}) + } else if path := files["build.gradle.kts"]; path != "" { + projects = append(projects, Project{Manager: ManagerGradle, Dir: dir, File: path, Name: filepath.Base(dir)}) + } + pnpmLock := files["pnpm-lock.yaml"] + npmLock := files["package-lock.json"] + shrinkwrap := files["npm-shrinkwrap.json"] + if pnpmLock != "" { + projects = append(projects, Project{Manager: ManagerPNPM, Dir: dir, File: pnpmLock, Name: filepath.Base(dir)}) + if npmLock != "" || shrinkwrap != "" { + warnings = append(warnings, Warning{ + Manager: ManagerPNPM, + Project: dir, + Message: "pnpm-lock.yaml and npm lockfile both found; using pnpm unless --manager npm is selected", + }) + if selected[ManagerNPM] { + if npmLock != "" { + projects = append(projects, Project{Manager: ManagerNPM, Dir: dir, File: npmLock, Name: filepath.Base(dir)}) + } else { + projects = append(projects, Project{Manager: ManagerNPM, Dir: dir, File: shrinkwrap, Name: filepath.Base(dir)}) + } + } + } + } else if npmLock != "" { + projects = append(projects, Project{Manager: ManagerNPM, Dir: dir, File: npmLock, Name: filepath.Base(dir)}) + } else if shrinkwrap != "" { + projects = append(projects, Project{Manager: ManagerNPM, Dir: dir, File: shrinkwrap, Name: filepath.Base(dir)}) + } else if path := files["package.json"]; path != "" && (len(selected) == 0 || selected[ManagerNPM]) { + projects = append(projects, Project{Manager: ManagerNPM, Dir: dir, File: path, Name: filepath.Base(dir)}) + } + } + + sort.Slice(projects, func(i, j int) bool { + if projects[i].Dir != projects[j].Dir { + return projects[i].Dir < projects[j].Dir + } + return projects[i].Manager < projects[j].Manager + }) + if len(projects) == 0 { + return nil, warnings, fmt.Errorf("no supported dependency manifests found under %s", absRoot) + } + return projects, warnings, nil +} + +func managerForManifest(name string) Manager { + switch strings.ToLower(name) { + case "go.mod", "go.work": + return ManagerGo + case "pom.xml": + return ManagerMaven + case "build.gradle", "build.gradle.kts": + return ManagerGradle + case "package.json", "package-lock.json", "npm-shrinkwrap.json": + return ManagerNPM + case "pnpm-lock.yaml": + return ManagerPNPM + default: + return "" + } +} + +func managerSet(managers []Manager) map[Manager]bool { + if len(managers) == 0 { + return nil + } + selected := make(map[Manager]bool, len(managers)) + for _, m := range managers { + if m != "" { + selected[m] = true + } + } + return selected +} diff --git a/deps/filter.go b/deps/filter.go new file mode 100644 index 0000000..cadd3fa --- /dev/null +++ b/deps/filter.go @@ -0,0 +1,251 @@ +package deps + +import ( + "fmt" + "sort" + "strings" + + "github.com/flanksource/commons/collections" +) + +func filterAndPrune(root *Node, filters []string, maxDepth int) *Node { + return filterAndPruneAt(root, filters, maxDepth, nil) +} + +func filterAndPruneAt(node *Node, filters []string, maxDepth int, path map[string]bool) *Node { + if node == nil { + return nil + } + if maxDepth > 0 && node.Depth > maxDepth { + return nil + } + if path == nil { + path = map[string]bool{} + } + cp := node.cloneShallow() + key := node.ID + if path[key] { + cp.Circular = true + return cp + } + path[key] = true + for _, child := range node.Children { + if filtered := filterAndPruneAt(child, filters, maxDepth, cloneBoolMap(path)); filtered != nil { + cp.Children = append(cp.Children, filtered) + } + } + if len(filters) == 0 || nodeMatches(cp, filters) || len(cp.Children) > 0 || cp.Depth == 0 { + return cp + } + return nil +} + +func nodeMatches(node *Node, filters []string) bool { + if len(filters) == 0 { + return true + } + candidates := []string{ + node.ID, + node.Name, + node.Version, + string(node.Manager), + node.Scope, + node.Source, + node.Path, + fmt.Sprintf("%s:%s", node.Manager, node.Name), + fmt.Sprintf("%s:%s@%s", node.Manager, node.Name, node.Version), + } + for _, candidate := range candidates { + if candidate != "" && collections.MatchItems(candidate, filters...) { + return true + } + } + return false +} + +func flatten(roots []*Node, duplicates map[string]*Duplicate) ([]FlatNode, []Edge, Statistics) { + nodeMap := map[string]FlatNode{} + edgeMap := map[string]Edge{} + stats := Statistics{ByManager: map[Manager]int{}} + var circular int + var visit func(*Node) + visit = func(node *Node) { + if node == nil { + return + } + if _, exists := nodeMap[node.ID]; !exists { + nodeMap[node.ID] = FlatNode{ + ID: node.ID, + Name: node.Name, + Version: node.Version, + Manager: node.Manager, + Scope: node.Scope, + Source: node.Source, + Path: node.Path, + Direct: node.Direct, + Dev: node.Dev, + Optional: node.Optional, + Local: node.Local, + Depth: node.Depth, + } + stats.ByManager[node.Manager]++ + if node.Depth > stats.MaxDepth { + stats.MaxDepth = node.Depth + } + if node.Circular { + circular++ + } + } + for _, child := range node.Children { + edgeKey := node.ID + ">" + child.ID + ">" + child.Scope + edgeMap[edgeKey] = Edge{ + From: node.ID, + To: child.ID, + Manager: child.Manager, + Scope: child.Scope, + Dev: child.Dev, + Optional: child.Optional, + } + visit(child) + } + } + for _, root := range roots { + visit(root) + } + nodes := make([]FlatNode, 0, len(nodeMap)) + for _, node := range nodeMap { + nodes = append(nodes, node) + } + sort.Slice(nodes, func(i, j int) bool { + if nodes[i].Manager != nodes[j].Manager { + return nodes[i].Manager < nodes[j].Manager + } + if nodes[i].Depth != nodes[j].Depth { + return nodes[i].Depth < nodes[j].Depth + } + return nodes[i].ID < nodes[j].ID + }) + edges := make([]Edge, 0, len(edgeMap)) + for _, edge := range edgeMap { + edges = append(edges, edge) + } + sort.Slice(edges, func(i, j int) bool { + if edges[i].From != edges[j].From { + return edges[i].From < edges[j].From + } + if edges[i].To != edges[j].To { + return edges[i].To < edges[j].To + } + return edges[i].Scope < edges[j].Scope + }) + stats.Total = len(nodes) + stats.Edges = len(edges) + stats.Circular = circular + for _, dup := range duplicates { + if dup.Count > 1 { + stats.Duplicates++ + if dup.Conflicts { + stats.Conflicts++ + } + } + } + return nodes, edges, stats +} + +func analyzeDuplicates(roots []*Node) map[string]*Duplicate { + out := map[string]*Duplicate{} + var walk func(*Node, string) + walk = func(node *Node, parentPath string) { + if node == nil { + return + } + currentPath := node.Name + if parentPath != "" { + currentPath = parentPath + " > " + node.Name + } + key := string(node.Manager) + ":" + node.Name + dup := out[key] + if dup == nil { + dup = &Duplicate{ + Name: node.Name, + Manager: node.Manager, + Versions: map[string][]string{}, + } + out[key] = dup + } + dup.Count++ + version := node.Version + if version == "" { + version = "(none)" + } + dup.Versions[version] = append(dup.Versions[version], currentPath) + for _, child := range node.Children { + walk(child, currentPath) + } + } + for _, root := range roots { + walk(root, "") + } + for _, dup := range out { + if len(dup.Versions) > 1 { + dup.Conflicts = true + } + } + return out +} + +func applyDuplicateRefs(roots []*Node, duplicates map[string]*Duplicate) { + var walk func(*Node) + walk = func(node *Node) { + if node == nil { + return + } + key := string(node.Manager) + ":" + node.Name + if dup := duplicates[key]; dup != nil && dup.Count > 1 { + node.Duplicate = &DupRef{Count: dup.Count, Conflicts: dup.Conflicts} + } + for _, child := range node.Children { + walk(child) + } + } + for _, root := range roots { + walk(root) + } +} + +func duplicatesList(duplicates map[string]*Duplicate) []Duplicate { + out := make([]Duplicate, 0) + for _, dup := range duplicates { + if dup.Count > 1 { + out = append(out, *dup) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].Manager != out[j].Manager { + return out[i].Manager < out[j].Manager + } + return out[i].Name < out[j].Name + }) + return out +} + +func cloneBoolMap(in map[string]bool) map[string]bool { + out := make(map[string]bool, len(in)) + for k, v := range in { + out[k] = v + } + return out +} + +func splitPatterns(values []string) []string { + var out []string + for _, value := range values { + for _, part := range strings.Split(value, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + } + return out +} diff --git a/deps/go.go b/deps/go.go new file mode 100644 index 0000000..a4cfb75 --- /dev/null +++ b/deps/go.go @@ -0,0 +1,226 @@ +package deps + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/mod/modfile" +) + +type goModuleInfo struct { + Path string `json:"Path"` + Version string `json:"Version"` + Main bool `json:"Main"` + Replace *goModuleInfo `json:"Replace"` + Dir string `json:"Dir"` +} + +func resolveGoNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + graphResult, err := opts.Runner.Run(ctx, Command{ + Dir: project.Dir, + Name: "go", + Args: []string{"mod", "graph"}, + Env: []string{"GOFLAGS=-mod=readonly"}, + }) + if err != nil { + return nil, nil, err + } + listResult, listErr := opts.Runner.Run(ctx, Command{ + Dir: project.Dir, + Name: "go", + Args: []string{"list", "-m", "-json", "all"}, + Env: []string{"GOFLAGS=-mod=readonly"}, + }) + infos := map[string]goModuleInfo{} + if listErr == nil { + infos = parseGoModuleInfos([]byte(listResult.Stdout)) + } + + rootToken, err := goRootToken(project, infos) + if err != nil { + return nil, nil, err + } + direct := goDirectRequires(project.File) + root := buildGoGraph(rootToken, parseGoGraph(graphResult.Stdout), infos, direct) + root.Path = project.File + root.Source = "go mod graph" + if listErr != nil { + return root, []Warning{{Manager: ManagerGo, Project: project.Dir, Message: "go list -m -json all failed; replacement metadata may be incomplete: " + listErr.Error()}}, nil + } + return root, nil, nil +} + +func resolveGoManifest(project Project) (*Node, []Warning, error) { + data, err := os.ReadFile(filepath.Join(project.Dir, "go.mod")) + if err != nil { + return nil, nil, err + } + file, err := modfile.Parse("go.mod", data, nil) + if err != nil { + return nil, nil, err + } + root := NewNode(ManagerGo, file.Module.Mod.Path, "") + root.Path = filepath.Join(project.Dir, "go.mod") + root.Source = "go.mod" + for _, req := range file.Require { + child := NewNode(ManagerGo, req.Mod.Path, req.Mod.Version) + child.Depth = 1 + child.Direct = !req.Indirect + child.Scope = "require" + if req.Indirect { + child.Scope = "indirect" + } + if rep := goReplaceFor(file, req.Mod.Path, req.Mod.Version); rep != nil { + child.Source = goReplaceSource(rep.New.Path, rep.New.Version) + child.Local = isLocalRef(rep.New.Path) + } + root.Children = append(root.Children, child) + } + sortChildren(root) + return root, []Warning{{Manager: ManagerGo, Project: project.Dir, Message: "manifest fallback includes go.mod requirements only; transitive edges are unavailable"}}, nil +} + +func parseGoModuleInfos(data []byte) map[string]goModuleInfo { + out := map[string]goModuleInfo{} + dec := json.NewDecoder(bytes.NewReader(data)) + for { + var info goModuleInfo + if err := dec.Decode(&info); err != nil { + break + } + if info.Path == "" { + continue + } + out[goToken(info.Path, info.Version)] = info + if info.Main { + out[info.Path] = info + } + } + return out +} + +func parseGoGraph(stdout string) map[string][]string { + out := map[string][]string{} + scanner := bufio.NewScanner(strings.NewReader(stdout)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + continue + } + out[fields[0]] = append(out[fields[0]], fields[1]) + } + for key := range out { + sortStrings(out[key]) + } + return out +} + +func goRootToken(project Project, infos map[string]goModuleInfo) (string, error) { + for token, info := range infos { + if info.Main { + return token, nil + } + } + data, err := os.ReadFile(filepath.Join(project.Dir, "go.mod")) + if err != nil { + return "", err + } + file, err := modfile.Parse("go.mod", data, nil) + if err != nil { + return "", err + } + return file.Module.Mod.Path, nil +} + +func goDirectRequires(path string) map[string]bool { + out := map[string]bool{} + data, err := os.ReadFile(path) + if err != nil { + return out + } + file, err := modfile.Parse("go.mod", data, nil) + if err != nil { + return out + } + for _, req := range file.Require { + out[goToken(req.Mod.Path, req.Mod.Version)] = !req.Indirect + } + return out +} + +func buildGoGraph(rootToken string, edges map[string][]string, infos map[string]goModuleInfo, direct map[string]bool) *Node { + var build func(token string, depth int, path map[string]bool) *Node + build = func(token string, depth int, path map[string]bool) *Node { + name, version := splitGoToken(token) + node := NewNode(ManagerGo, name, version) + node.Depth = depth + node.Source = "go mod graph" + if depth == 1 { + if isDirect, ok := direct[token]; ok { + node.Direct = isDirect + if isDirect { + node.Scope = "require" + } else { + node.Scope = "indirect" + } + } + } + if info, ok := infos[token]; ok && info.Replace != nil { + node.Source = goReplaceSource(info.Replace.Path, info.Replace.Version) + node.Local = isLocalRef(info.Replace.Path) || info.Replace.Dir != "" + } + if path[token] { + node.Circular = true + return node + } + path[token] = true + for _, childToken := range edges[token] { + child := build(childToken, depth+1, cloneBoolMap(path)) + node.Children = append(node.Children, child) + } + sortChildren(node) + return node + } + return build(rootToken, 0, map[string]bool{}) +} + +func splitGoToken(token string) (string, string) { + name, version, ok := strings.Cut(token, "@") + if !ok { + return token, "" + } + return name, version +} + +func goToken(path, version string) string { + if version == "" { + return path + } + return path + "@" + version +} + +func goReplaceFor(file *modfile.File, path, version string) *modfile.Replace { + for _, rep := range file.Replace { + if rep.Old.Path == path && (rep.Old.Version == "" || rep.Old.Version == version) { + return rep + } + } + return nil +} + +func goReplaceSource(path, version string) string { + if version == "" { + return path + } + return fmt.Sprintf("%s@%s", path, version) +} diff --git a/deps/gradle.go b/deps/gradle.go new file mode 100644 index 0000000..2ab8cf1 --- /dev/null +++ b/deps/gradle.go @@ -0,0 +1,217 @@ +package deps + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +type gradleExport struct { + Projects []gradleProject `json:"projects"` +} + +type gradleProject struct { + Name string `json:"name"` + Path string `json:"path"` + Configurations []gradleConfiguration `json:"configurations"` +} + +type gradleConfiguration struct { + Name string `json:"name"` + Dependencies []gradleNode `json:"dependencies"` +} + +type gradleNode struct { + Group string `json:"group"` + Module string `json:"module"` + Version string `json:"version"` + Selected string `json:"selected"` + Children []gradleNode `json:"children"` +} + +func resolveGradleNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + tmp, err := os.CreateTemp("", "repomap-gradle-*.json") + if err != nil { + return nil, nil, err + } + tmpPath := tmp.Name() + _ = tmp.Close() + defer os.Remove(tmpPath) + + initFile, err := os.CreateTemp("", "repomap-gradle-*.gradle") + if err != nil { + return nil, nil, err + } + initPath := initFile.Name() + if _, err := initFile.WriteString(gradleInitScript(tmpPath, opts.Configurations)); err != nil { + _ = initFile.Close() + return nil, nil, err + } + _ = initFile.Close() + defer os.Remove(initPath) + + bin := "gradle" + args := []string{"-I", initPath, "-q", "repomapDeps"} + if _, err := os.Stat(filepath.Join(project.Dir, "gradlew")); err == nil { + bin = "./gradlew" + } + _, err = opts.Runner.Run(ctx, Command{Dir: project.Dir, Name: bin, Args: args}) + if err != nil { + return nil, nil, err + } + data, err := os.ReadFile(tmpPath) + if err != nil { + return nil, nil, err + } + root, err := parseGradleJSON(data, project) + if err != nil { + return nil, nil, err + } + root.Path = project.File + root.Source = "gradle ResolutionResult" + return root, nil, nil +} + +func resolveGradleManifest(project Project) (*Node, []Warning, error) { + root, err := parseGradleBuildFile(project.File) + if err != nil { + return nil, nil, err + } + return root, []Warning{{Manager: ManagerGradle, Project: project.Dir, Message: "manifest fallback includes direct Gradle dependency declarations only; resolved transitive edges are unavailable"}}, nil +} + +func gradleInitScript(outputPath string, configurations []string) string { + quotedOutput := strings.ReplaceAll(outputPath, "\\", "\\\\") + quotedOutput = strings.ReplaceAll(quotedOutput, "'", "\\'") + var configSet string + if len(configurations) > 0 { + var quoted []string + for _, cfg := range configurations { + cfg = strings.TrimSpace(cfg) + if cfg != "" { + quoted = append(quoted, "'"+strings.ReplaceAll(cfg, "'", "\\'")+"'") + } + } + configSet = "[" + strings.Join(quoted, ",") + "] as Set" + } else { + configSet = "[] as Set" + } + return fmt.Sprintf(` +import groovy.json.JsonOutput +gradle.projectsEvaluated { + rootProject.tasks.register('repomapDeps') { + doLast { + def selectedConfigurations = %s + def seen = [] as Set + def convert + convert = { dep, depth -> + def id = dep.selected.id + def group = id.hasProperty('group') ? id.group : '' + def module = id.hasProperty('module') ? id.module : id.displayName + def version = id.hasProperty('version') ? id.version : '' + def key = group + ':' + module + ':' + version + if (seen.contains(key + ':' + depth)) { + return [group: group, module: module, version: version, children: []] + } + seen.add(key + ':' + depth) + return [group: group, module: module, version: version, selected: id.displayName, + children: dep.selected.dependencies.findAll { it instanceof org.gradle.api.artifacts.result.ResolvedDependencyResult }.collect { convert(it, depth + 1) }] + } + def projects = [] + allprojects.each { prj -> + def configs = [] + prj.configurations.findAll { it.canBeResolved && (selectedConfigurations.isEmpty() || selectedConfigurations.contains(it.name)) }.each { cfg -> + try { + configs << [name: cfg.name, dependencies: cfg.incoming.resolutionResult.root.dependencies.findAll { it instanceof org.gradle.api.artifacts.result.ResolvedDependencyResult }.collect { convert(it, 1) }] + } catch (Throwable ignored) {} + } + if (!configs.isEmpty()) { + projects << [name: prj.name, path: prj.path, configurations: configs] + } + } + new File('%s').text = JsonOutput.toJson([projects: projects]) + } + } +} +`, configSet, quotedOutput) +} + +func parseGradleJSON(data []byte, project Project) (*Node, error) { + var payload gradleExport + if err := json.Unmarshal(data, &payload); err != nil { + return nil, err + } + root := NewNode(ManagerGradle, filepath.Base(project.Dir), "") + root.Source = "gradle" + for _, p := range payload.Projects { + projectNode := NewNode(ManagerGradle, firstNonEmpty(p.Path, p.Name), "") + projectNode.Depth = 1 + projectNode.Direct = true + projectNode.Source = "project" + for _, cfg := range p.Configurations { + cfgNode := NewNode(ManagerGradle, cfg.Name, "") + cfgNode.Depth = 2 + cfgNode.Scope = cfg.Name + cfgNode.Source = "configuration" + for _, dep := range cfg.Dependencies { + child := convertGradleNode(dep, 3, cfg.Name) + child.Direct = true + cfgNode.Children = append(cfgNode.Children, child) + } + sortChildren(cfgNode) + projectNode.Children = append(projectNode.Children, cfgNode) + } + sortChildren(projectNode) + root.Children = append(root.Children, projectNode) + } + sortChildren(root) + return root, nil +} + +func convertGradleNode(dep gradleNode, depth int, scope string) *Node { + name := dep.Module + if dep.Group != "" { + name = dep.Group + ":" + dep.Module + } + if name == "" { + name = dep.Selected + } + node := NewNode(ManagerGradle, name, dep.Version) + node.Depth = depth + node.Scope = scope + node.Source = "gradle" + for _, childDep := range dep.Children { + node.Children = append(node.Children, convertGradleNode(childDep, depth+1, scope)) + } + sortChildren(node) + return node +} + +var gradleDepRe = regexp.MustCompile(`(?m)^\s*([A-Za-z][A-Za-z0-9_]*(?:Implementation|CompileOnly|RuntimeOnly|Api|TestImplementation|testImplementation|implementation|api|compileOnly|runtimeOnly|testRuntimeOnly|annotationProcessor|kapt)?)\s*(?:\(?\s*)["']([^:"']+):([^:"']+):([^"']+)["']`) + +func parseGradleBuildFile(path string) (*Node, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + root := NewNode(ManagerGradle, filepath.Base(filepath.Dir(path)), "") + root.Path = path + root.Source = filepath.Base(path) + for _, match := range gradleDepRe.FindAllStringSubmatch(string(data), -1) { + scope := match[1] + name := match[2] + ":" + match[3] + version := match[4] + child := NewNode(ManagerGradle, name, version) + child.Depth = 1 + child.Direct = true + child.Scope = scope + child.Dev = strings.Contains(strings.ToLower(scope), "test") + root.Children = append(root.Children, child) + } + sortChildren(root) + return root, nil +} diff --git a/deps/maven.go b/deps/maven.go new file mode 100644 index 0000000..8412811 --- /dev/null +++ b/deps/maven.go @@ -0,0 +1,197 @@ +package deps + +import ( + "context" + "encoding/json" + "encoding/xml" + "fmt" + "os" + "path/filepath" + "strings" +) + +type mavenTreeNode struct { + GroupID string `json:"groupId"` + ArtifactID string `json:"artifactId"` + Version string `json:"version"` + Type string `json:"type"` + Scope string `json:"scope"` + Optional any `json:"optional"` + Children []mavenTreeNode `json:"children"` +} + +func resolveMavenNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + tmp, err := os.CreateTemp("", "repomap-maven-*.json") + if err != nil { + return nil, nil, err + } + tmpPath := tmp.Name() + _ = tmp.Close() + defer os.Remove(tmpPath) + + _, err = opts.Runner.Run(ctx, Command{ + Dir: project.Dir, + Name: "mvn", + Args: []string{ + "-q", + "org.apache.maven.plugins:maven-dependency-plugin:3.11.0:tree", + "-DoutputType=json", + "-DoutputFile=" + tmpPath, + }, + }) + if err != nil { + return nil, nil, err + } + data, err := os.ReadFile(tmpPath) + if err != nil { + return nil, nil, err + } + if len(strings.TrimSpace(string(data))) == 0 { + return nil, nil, fmt.Errorf("maven dependency plugin produced empty JSON") + } + root, err := parseMavenJSON(data) + if err != nil { + return nil, nil, err + } + root.Path = project.File + root.Source = "mvn dependency:tree" + return root, nil, nil +} + +func resolveMavenManifest(project Project) (*Node, []Warning, error) { + root, err := parseMavenPOM(project.File) + if err != nil { + return nil, nil, err + } + return root, []Warning{{Manager: ManagerMaven, Project: project.Dir, Message: "manifest fallback includes pom.xml direct dependencies only; resolved transitive edges are unavailable"}}, nil +} + +func parseMavenJSON(data []byte) (*Node, error) { + var tree mavenTreeNode + if err := json.Unmarshal(data, &tree); err != nil { + return nil, err + } + return convertMavenTree(tree, 0), nil +} + +func convertMavenTree(tree mavenTreeNode, depth int) *Node { + name := tree.ArtifactID + if tree.GroupID != "" { + name = tree.GroupID + ":" + tree.ArtifactID + } + node := NewNode(ManagerMaven, name, tree.Version) + node.Depth = depth + node.Scope = tree.Scope + node.Optional = boolish(tree.Optional) + if tree.Type != "" { + node.Source = tree.Type + } + for _, childTree := range tree.Children { + child := convertMavenTree(childTree, depth+1) + child.Direct = depth == 0 + node.Children = append(node.Children, child) + } + sortChildren(node) + return node +} + +type pomProject struct { + XMLName xml.Name `xml:"project"` + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Parent pomParent `xml:"parent"` + Properties []xmlProperty `xml:"properties>*"` + Dependencies []pomDependency `xml:"dependencies>dependency"` +} + +type pomParent struct { + GroupID string `xml:"groupId"` + Version string `xml:"version"` +} + +type xmlProperty struct { + XMLName xml.Name + Value string `xml:",chardata"` +} + +type pomDependency struct { + GroupID string `xml:"groupId"` + ArtifactID string `xml:"artifactId"` + Version string `xml:"version"` + Scope string `xml:"scope"` + Optional string `xml:"optional"` +} + +func parseMavenPOM(path string) (*Node, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var pom pomProject + if err := xml.Unmarshal(data, &pom); err != nil { + return nil, err + } + group := firstNonEmpty(pom.GroupID, pom.Parent.GroupID) + version := firstNonEmpty(pom.Version, pom.Parent.Version) + props := map[string]string{ + "project.groupId": group, + "project.version": version, + "pom.groupId": group, + "pom.version": version, + "project.artifactId": pom.ArtifactID, + "pom.artifactId": pom.ArtifactID, + } + for _, prop := range pom.Properties { + props[prop.XMLName.Local] = strings.TrimSpace(prop.Value) + } + root := NewNode(ManagerMaven, group+":"+pom.ArtifactID, resolveProperty(version, props)) + root.Path = path + root.Source = "pom.xml" + for _, dep := range pom.Dependencies { + name := resolveProperty(dep.GroupID, props) + ":" + resolveProperty(dep.ArtifactID, props) + child := NewNode(ManagerMaven, name, resolveProperty(dep.Version, props)) + child.Depth = 1 + child.Direct = true + child.Scope = firstNonEmpty(dep.Scope, "compile") + child.Optional = strings.EqualFold(strings.TrimSpace(dep.Optional), "true") + root.Children = append(root.Children, child) + } + sortChildren(root) + return root, nil +} + +func resolveProperty(value string, props map[string]string) string { + value = strings.TrimSpace(value) + if strings.HasPrefix(value, "${") && strings.HasSuffix(value, "}") { + key := strings.TrimSuffix(strings.TrimPrefix(value, "${"), "}") + if props[key] != "" { + return props[key] + } + } + return value +} + +func boolish(value any) bool { + switch v := value.(type) { + case bool: + return v + case string: + return strings.EqualFold(v, "true") + default: + return false + } +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func mavenProjectFile(dir string) string { + return filepath.Join(dir, "pom.xml") +} diff --git a/deps/model.go b/deps/model.go new file mode 100644 index 0000000..f55aeda --- /dev/null +++ b/deps/model.go @@ -0,0 +1,167 @@ +package deps + +import "time" + +type Manager string + +const ( + ManagerGo Manager = "go" + ManagerMaven Manager = "maven" + ManagerGradle Manager = "gradle" + ManagerNPM Manager = "npm" + ManagerPNPM Manager = "pnpm" +) + +type Mode string + +const ( + ModeAuto Mode = "auto" + ModeNative Mode = "native" + ModeManifest Mode = "manifest" +) + +type Options struct { + Managers []Manager + Mode Mode + MaxDepth int + Filters []string + Configurations []string + Strict bool + Runner CommandRunner + Now func() time.Time +} + +type Project struct { + Manager Manager `json:"manager"` + Dir string `json:"dir"` + File string `json:"file"` + Name string `json:"name,omitempty"` +} + +type Export struct { + Metadata Metadata `json:"metadata"` + Roots []*Node `json:"roots"` + Nodes []FlatNode `json:"nodes"` + Edges []Edge `json:"edges"` + Statistics Statistics `json:"statistics"` + Duplicates []Duplicate `json:"duplicates,omitempty"` + Warnings []Warning `json:"warnings,omitempty"` +} + +type Metadata struct { + ExportedAt time.Time `json:"exported_at"` + Version string `json:"version"` + Path string `json:"path"` + Managers []Manager `json:"managers,omitempty"` + Mode Mode `json:"mode"` + Filter []string `json:"filter,omitempty"` + MaxDepth int `json:"max_depth,omitempty"` + Configurations []string `json:"configurations,omitempty"` + ProjectsScanned int `json:"projects_scanned"` +} + +type Node struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Manager Manager `json:"manager"` + Scope string `json:"scope,omitempty"` + Source string `json:"source,omitempty"` + Path string `json:"path,omitempty"` + Direct bool `json:"direct,omitempty"` + Dev bool `json:"dev,omitempty"` + Optional bool `json:"optional,omitempty"` + Local bool `json:"local,omitempty"` + Depth int `json:"depth"` + Circular bool `json:"circular,omitempty"` + Duplicate *DupRef `json:"duplicate,omitempty"` + Children []*Node `json:"children,omitempty"` + properties map[string]string +} + +type FlatNode struct { + ID string `json:"id"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + Manager Manager `json:"manager"` + Scope string `json:"scope,omitempty"` + Source string `json:"source,omitempty"` + Path string `json:"path,omitempty"` + Direct bool `json:"direct,omitempty"` + Dev bool `json:"dev,omitempty"` + Optional bool `json:"optional,omitempty"` + Local bool `json:"local,omitempty"` + Depth int `json:"depth"` +} + +type Edge struct { + From string `json:"from"` + To string `json:"to"` + Manager Manager `json:"manager"` + Scope string `json:"scope,omitempty"` + Dev bool `json:"dev,omitempty"` + Optional bool `json:"optional,omitempty"` +} + +type Statistics struct { + Projects int `json:"projects"` + Total int `json:"total"` + Edges int `json:"edges"` + ByManager map[Manager]int `json:"by_manager"` + MaxDepth int `json:"max_depth"` + Duplicates int `json:"duplicates"` + Conflicts int `json:"conflicts"` + Circular int `json:"circular_references"` +} + +type Duplicate struct { + Name string `json:"name"` + Manager Manager `json:"manager"` + Count int `json:"count"` + Conflicts bool `json:"conflicts"` + Versions map[string][]string `json:"versions"` +} + +type DupRef struct { + Count int `json:"count"` + Conflicts bool `json:"conflicts"` +} + +type Warning struct { + Manager Manager `json:"manager,omitempty"` + Project string `json:"project,omitempty"` + Message string `json:"message"` +} + +func NewNode(manager Manager, name, version string) *Node { + return &Node{ + ID: NodeID(manager, name, version), + Name: name, + Version: version, + Manager: manager, + } +} + +func NodeID(manager Manager, name, version string) string { + if version == "" { + return string(manager) + ":" + name + } + return string(manager) + ":" + name + "@" + version +} + +func (n *Node) cloneShallow() *Node { + if n == nil { + return nil + } + cp := *n + cp.Children = nil + cp.Duplicate = nil + cp.properties = nil + if n.properties != nil { + cp.properties = make(map[string]string, len(n.properties)) + for k, v := range n.properties { + cp.properties[k] = v + } + } + return &cp +} diff --git a/deps/npm.go b/deps/npm.go new file mode 100644 index 0000000..a23db1f --- /dev/null +++ b/deps/npm.go @@ -0,0 +1,289 @@ +package deps + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +type npmTree struct { + Name string `json:"name"` + Version string `json:"version"` + Dependencies map[string]*npmTree `json:"dependencies"` + Dev bool `json:"dev"` + Optional bool `json:"optional"` + Problems []string `json:"problems"` + PackageLockOnly bool `json:"packageLockOnly"` +} + +type packageJSON struct { + Name string `json:"name"` + Version string `json:"version"` + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + OptionalDependencies map[string]string `json:"optionalDependencies"` + PeerDependencies map[string]string `json:"peerDependencies"` +} + +type packageLock struct { + Name string `json:"name"` + Version string `json:"version"` + Lockfile int `json:"lockfileVersion"` + Packages map[string]packageLockItem `json:"packages"` + Dependencies map[string]json.RawMessage `json:"dependencies"` +} + +type packageLockItem struct { + Name string `json:"name"` + Version string `json:"version"` + Resolved string `json:"resolved"` + Link bool `json:"link"` + Dev bool `json:"dev"` + Optional bool `json:"optional"` + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + OptionalDependencies map[string]string `json:"optionalDependencies"` + PeerDependencies map[string]string `json:"peerDependencies"` +} + +type packageLockV1Item struct { + Version string `json:"version"` + Resolved string `json:"resolved"` + Dev bool `json:"dev"` + Optional bool `json:"optional"` + Dependencies map[string]packageLockV1Item `json:"dependencies"` +} + +func resolveNPMNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + args := []string{"ls", "--json", "--all"} + result, err := opts.Runner.Run(ctx, Command{Dir: project.Dir, Name: "npm", Args: args}) + if strings.TrimSpace(result.Stdout) == "" && err != nil { + return nil, nil, err + } + root, parseErr := parseNPMTree([]byte(result.Stdout), project) + if parseErr != nil { + if err != nil { + return nil, nil, fmt.Errorf("%v; also failed to parse stdout: %w", err, parseErr) + } + return nil, nil, parseErr + } + root.Path = project.File + root.Source = "npm ls" + if err != nil { + return root, []Warning{{Manager: ManagerNPM, Project: project.Dir, Message: "npm ls exited non-zero but produced parseable JSON: " + err.Error()}}, nil + } + return root, nil, nil +} + +func resolveNPMManifest(project Project) (*Node, []Warning, error) { + if filepath.Base(project.File) == "package-lock.json" || filepath.Base(project.File) == "npm-shrinkwrap.json" { + root, err := parsePackageLock(project.File) + if err == nil { + return root, nil, nil + } + } + root, err := parsePackageJSON(filepath.Join(project.Dir, "package.json")) + if err != nil { + return nil, nil, err + } + return root, []Warning{{Manager: ManagerNPM, Project: project.Dir, Message: "manifest fallback includes package.json direct dependencies only; install tree may be unavailable"}}, nil +} + +func parseNPMTree(data []byte, project Project) (*Node, error) { + var tree npmTree + if err := json.Unmarshal(data, &tree); err != nil { + return nil, err + } + if tree.Name == "" { + tree.Name = filepath.Base(project.Dir) + } + return convertNPMTree(&tree, 0), nil +} + +func convertNPMTree(tree *npmTree, depth int) *Node { + node := NewNode(ManagerNPM, tree.Name, tree.Version) + node.Depth = depth + node.Dev = tree.Dev + node.Optional = tree.Optional + node.Source = "npm" + keys := make([]string, 0, len(tree.Dependencies)) + for key := range tree.Dependencies { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + child := convertNPMTree(tree.Dependencies[key], depth+1) + child.Direct = depth == 0 + node.Children = append(node.Children, child) + } + return node +} + +func parsePackageJSON(path string) (*Node, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var pkg packageJSON + if err := json.Unmarshal(data, &pkg); err != nil { + return nil, err + } + if pkg.Name == "" { + pkg.Name = filepath.Base(filepath.Dir(path)) + } + root := NewNode(ManagerNPM, pkg.Name, pkg.Version) + root.Path = path + root.Source = "package.json" + addPackageJSONDeps(root, pkg.Dependencies, "dependencies", false, false) + addPackageJSONDeps(root, pkg.DevDependencies, "devDependencies", true, false) + addPackageJSONDeps(root, pkg.OptionalDependencies, "optionalDependencies", false, true) + addPackageJSONDeps(root, pkg.PeerDependencies, "peerDependencies", false, false) + sortChildren(root) + return root, nil +} + +func addPackageJSONDeps(root *Node, deps map[string]string, scope string, dev, optional bool) { + keys := make([]string, 0, len(deps)) + for key := range deps { + keys = append(keys, key) + } + sort.Strings(keys) + for _, name := range keys { + child := NewNode(ManagerNPM, name, deps[name]) + child.Depth = 1 + child.Direct = true + child.Scope = scope + child.Dev = dev + child.Optional = optional + child.Local = isLocalRef(deps[name]) + root.Children = append(root.Children, child) + } +} + +func parsePackageLock(path string) (*Node, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var lock packageLock + if err := json.Unmarshal(data, &lock); err != nil { + return nil, err + } + if len(lock.Packages) > 0 { + return packageLockV2Tree(path, lock), nil + } + return packageLockV1Tree(path, lock), nil +} + +func packageLockV2Tree(path string, lock packageLock) *Node { + rootPkg := lock.Packages[""] + name := firstNonEmpty(lock.Name, rootPkg.Name, filepath.Base(filepath.Dir(path))) + version := firstNonEmpty(lock.Version, rootPkg.Version) + root := NewNode(ManagerNPM, name, version) + root.Path = path + root.Source = filepath.Base(path) + root.Children = packageLockChildren("", rootPkg, lock.Packages, 1, map[string]bool{}) + sortChildren(root) + return root +} + +func packageLockChildren(parentPath string, item packageLockItem, packages map[string]packageLockItem, depth int, seen map[string]bool) []*Node { + depScopes := []struct { + scope string + deps map[string]string + dev bool + optional bool + }{ + {"dependencies", item.Dependencies, item.Dev, item.Optional}, + {"devDependencies", item.DevDependencies, true, item.Optional}, + {"optionalDependencies", item.OptionalDependencies, item.Dev, true}, + {"peerDependencies", item.PeerDependencies, item.Dev, item.Optional}, + } + var children []*Node + for _, depScope := range depScopes { + keys := make([]string, 0, len(depScope.deps)) + for key := range depScope.deps { + keys = append(keys, key) + } + sort.Strings(keys) + for _, depName := range keys { + childPath, childItem, ok := findPackageLockChild(parentPath, depName, packages) + version := depScope.deps[depName] + if ok && childItem.Version != "" { + version = childItem.Version + } + child := NewNode(ManagerNPM, depName, version) + child.Depth = depth + child.Direct = depth == 1 + child.Scope = depScope.scope + child.Dev = depScope.dev || childItem.Dev + child.Optional = depScope.optional || childItem.Optional + child.Local = childItem.Link || isLocalRef(childItem.Resolved) + child.Source = childItem.Resolved + if ok && !seen[childPath] { + nextSeen := cloneBoolMap(seen) + nextSeen[childPath] = true + child.Children = packageLockChildren(childPath, childItem, packages, depth+1, nextSeen) + } else if ok { + child.Circular = true + } + children = append(children, child) + } + } + return children +} + +func findPackageLockChild(parentPath, name string, packages map[string]packageLockItem) (string, packageLockItem, bool) { + candidates := []string{} + if parentPath != "" { + candidates = append(candidates, parentPath+"/node_modules/"+name) + } + candidates = append(candidates, "node_modules/"+name) + for _, candidate := range candidates { + if item, ok := packages[candidate]; ok { + return candidate, item, true + } + } + return "", packageLockItem{}, false +} + +func packageLockV1Tree(path string, lock packageLock) *Node { + name := firstNonEmpty(lock.Name, filepath.Base(filepath.Dir(path))) + root := NewNode(ManagerNPM, name, lock.Version) + root.Path = path + root.Source = filepath.Base(path) + keys := make([]string, 0, len(lock.Dependencies)) + for key := range lock.Dependencies { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + var item packageLockV1Item + if err := json.Unmarshal(lock.Dependencies[key], &item); err == nil { + root.Children = append(root.Children, convertPackageLockV1(key, item, 1)) + } + } + return root +} + +func convertPackageLockV1(name string, item packageLockV1Item, depth int) *Node { + node := NewNode(ManagerNPM, name, item.Version) + node.Depth = depth + node.Direct = depth == 1 + node.Dev = item.Dev + node.Optional = item.Optional + node.Source = item.Resolved + keys := make([]string, 0, len(item.Dependencies)) + for key := range item.Dependencies { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + node.Children = append(node.Children, convertPackageLockV1(key, item.Dependencies[key], depth+1)) + } + return node +} diff --git a/deps/pnpm.go b/deps/pnpm.go new file mode 100644 index 0000000..27a9e6b --- /dev/null +++ b/deps/pnpm.go @@ -0,0 +1,299 @@ +package deps + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/goccy/go-yaml" +) + +type pnpmNativeNode struct { + Name string `json:"name"` + Version string `json:"version"` + Path string `json:"path"` + Private bool `json:"private"` + Dependencies []pnpmNativeNode `json:"dependencies"` +} + +func resolvePNPMNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + args := []string{"list", "--json", "--depth", "Infinity", "--lockfile-only"} + result, err := opts.Runner.Run(ctx, Command{Dir: project.Dir, Name: "pnpm", Args: args}) + if err != nil || strings.TrimSpace(result.Stdout) == "" { + args = []string{"list", "--json", "--depth", "Infinity"} + result, err = opts.Runner.Run(ctx, Command{Dir: project.Dir, Name: "pnpm", Args: args}) + } + if err != nil && strings.TrimSpace(result.Stdout) == "" { + return nil, nil, err + } + root, parseErr := parsePNPMNative([]byte(result.Stdout), project) + if parseErr != nil { + if err != nil { + return nil, nil, fmt.Errorf("%v; also failed to parse stdout: %w", err, parseErr) + } + return nil, nil, parseErr + } + root.Path = project.File + root.Source = "pnpm list" + if err != nil { + return root, []Warning{{Manager: ManagerPNPM, Project: project.Dir, Message: "pnpm list exited non-zero but produced parseable JSON: " + err.Error()}}, nil + } + return root, nil, nil +} + +func resolvePNPMManifest(project Project) (*Node, []Warning, error) { + root, err := parsePNPMLock(project.File) + if err != nil { + return nil, nil, err + } + return root, []Warning{{Manager: ManagerPNPM, Project: project.Dir, Message: "manifest fallback parsed pnpm-lock.yaml; peer and hoisting semantics may be approximate"}}, nil +} + +func parsePNPMNative(data []byte, project Project) (*Node, error) { + var roots []pnpmNativeNode + if err := json.Unmarshal(data, &roots); err != nil { + var root pnpmNativeNode + if err2 := json.Unmarshal(data, &root); err2 != nil { + return nil, err + } + roots = []pnpmNativeNode{root} + } + root := NewNode(ManagerPNPM, filepath.Base(project.Dir), "") + root.Source = "pnpm list" + for _, nativeRoot := range roots { + child := convertPNPMNative(nativeRoot, 1) + child.Direct = true + root.Children = append(root.Children, child) + } + sortChildren(root) + return root, nil +} + +func convertPNPMNative(input pnpmNativeNode, depth int) *Node { + name := input.Name + if name == "" { + name = input.Path + } + node := NewNode(ManagerPNPM, name, input.Version) + node.Depth = depth + node.Source = input.Path + for _, dep := range input.Dependencies { + node.Children = append(node.Children, convertPNPMNative(dep, depth+1)) + } + sortChildren(node) + return node +} + +func parsePNPMLock(path string) (*Node, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var raw map[string]any + if err := yaml.Unmarshal(data, &raw); err != nil { + return nil, err + } + root := NewNode(ManagerPNPM, filepath.Base(filepath.Dir(path)), "") + root.Path = path + root.Source = filepath.Base(path) + packages := pnpmPackageIndex(raw) + importers := asMap(raw["importers"]) + if len(importers) == 0 { + importers = map[string]any{".": raw} + } + importerKeys := make([]string, 0, len(importers)) + for key := range importers { + importerKeys = append(importerKeys, key) + } + sort.Strings(importerKeys) + for _, importerKey := range importerKeys { + importerNode := NewNode(ManagerPNPM, importerKey, "") + importerNode.Depth = 1 + importerNode.Direct = true + importerNode.Source = "importer" + importerData := asMap(importers[importerKey]) + addPNPMImporterDeps(importerNode, importerData, packages) + sortChildren(importerNode) + root.Children = append(root.Children, importerNode) + } + sortChildren(root) + return root, nil +} + +type pnpmPackage struct { + Name string + Version string + Key string + Data map[string]any + Children map[string]string +} + +func pnpmPackageIndex(raw map[string]any) map[string]pnpmPackage { + out := map[string]pnpmPackage{} + for _, section := range []string{"packages", "snapshots"} { + for key, value := range asMap(raw[section]) { + name, version := splitPNPMPackageKey(key) + if name == "" { + continue + } + data := asMap(value) + pkg := pnpmPackage{Name: name, Version: version, Key: key, Data: data, Children: pnpmDeps(data)} + out[pnpmPackageID(name, version)] = pkg + if _, ok := out[name]; !ok { + out[name] = pkg + } + } + } + return out +} + +func addPNPMImporterDeps(parent *Node, importer map[string]any, packages map[string]pnpmPackage) { + sections := []struct { + name string + dev bool + optional bool + }{ + {"dependencies", false, false}, + {"devDependencies", true, false}, + {"optionalDependencies", false, true}, + } + for _, section := range sections { + deps := asMap(importer[section.name]) + keys := make([]string, 0, len(deps)) + for key := range deps { + keys = append(keys, key) + } + sort.Strings(keys) + for _, depName := range keys { + version := pnpmDependencyVersion(deps[depName]) + child := buildPNPMNode(depName, version, section.name, 2, section.dev, section.optional, packages, map[string]bool{}) + child.Direct = true + parent.Children = append(parent.Children, child) + } + } +} + +func buildPNPMNode(name, version, scope string, depth int, dev, optional bool, packages map[string]pnpmPackage, seen map[string]bool) *Node { + version = stripPNPMPeerSuffix(version) + node := NewNode(ManagerPNPM, name, version) + node.Depth = depth + node.Scope = scope + node.Dev = dev + node.Optional = optional + node.Local = isLocalRef(version) + pkg := packages[pnpmPackageID(name, version)] + if pkg.Name == "" { + pkg = packages[name] + } + if pkg.Key != "" { + node.Source = pkg.Key + } + seenKey := pnpmPackageID(name, version) + if seen[seenKey] { + node.Circular = true + return node + } + seen[seenKey] = true + keys := make([]string, 0, len(pkg.Children)) + for key := range pkg.Children { + keys = append(keys, key) + } + sort.Strings(keys) + for _, childName := range keys { + childVersion := pkg.Children[childName] + node.Children = append(node.Children, buildPNPMNode(childName, childVersion, "dependencies", depth+1, dev, optional, packages, cloneBoolMap(seen))) + } + sortChildren(node) + return node +} + +func pnpmDeps(data map[string]any) map[string]string { + out := map[string]string{} + for _, section := range []string{"dependencies", "optionalDependencies", "peerDependencies"} { + for key, value := range asMap(data[section]) { + out[key] = pnpmDependencyVersion(value) + } + } + return out +} + +func pnpmDependencyVersion(value any) string { + switch v := value.(type) { + case string: + return stripPNPMPeerSuffix(v) + case map[string]any: + if version := stringValue(v["version"]); version != "" { + return stripPNPMPeerSuffix(version) + } + if specifier := stringValue(v["specifier"]); specifier != "" { + return specifier + } + case map[any]any: + if version := stringValue(v["version"]); version != "" { + return stripPNPMPeerSuffix(version) + } + } + return "" +} + +func splitPNPMPackageKey(key string) (string, string) { + key = strings.TrimPrefix(key, "/") + key = stripPNPMPeerSuffix(key) + if idx := strings.Index(key, "("); idx >= 0 { + key = key[:idx] + } + if strings.HasPrefix(key, "@") { + idx := strings.LastIndex(key, "@") + if idx > 0 { + return key[:idx], key[idx+1:] + } + return key, "" + } + name, version, ok := strings.Cut(key, "@") + if !ok { + return key, "" + } + return name, version +} + +func stripPNPMPeerSuffix(value string) string { + if idx := strings.Index(value, "("); idx >= 0 { + return value[:idx] + } + return value +} + +func pnpmPackageID(name, version string) string { + if version == "" { + return name + } + return name + "@" + stripPNPMPeerSuffix(version) +} + +func asMap(value any) map[string]any { + out := map[string]any{} + switch v := value.(type) { + case map[string]any: + return v + case map[any]any: + for key, item := range v { + out[fmt.Sprint(key)] = item + } + } + return out +} + +func stringValue(value any) string { + switch v := value.(type) { + case nil: + return "" + case string: + return v + default: + return fmt.Sprint(v) + } +} diff --git a/deps/pretty.go b/deps/pretty.go new file mode 100644 index 0000000..17bbe60 --- /dev/null +++ b/deps/pretty.go @@ -0,0 +1,231 @@ +package deps + +import ( + "fmt" + "sort" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +func (e *Export) Pretty() api.Text { + if e == nil { + return clicky.Text("") + } + t := clicky.Text("Dependency Graph", "font-bold") + t = t.Append(fmt.Sprintf(" projects=%d nodes=%d edges=%d", e.Statistics.Projects, e.Statistics.Total, e.Statistics.Edges), "text-muted") + if tree, ok := dependencyTree(e.Roots); ok { + t = t.NewLine().Add(tree) + } + if len(e.Warnings) > 0 { + t = t.NewLine().Append("Warnings", "font-bold text-yellow-600") + for _, w := range e.Warnings { + msg := w.Message + if w.Project != "" { + msg = w.Project + ": " + msg + } + if w.Manager != "" { + msg = "[" + string(w.Manager) + "] " + msg + } + t = t.NewLine().Append("- "+msg, "text-yellow-600") + } + } + if len(e.Duplicates) > 0 { + t = t.NewLine().Append("Duplicates", "font-bold") + for _, d := range e.Duplicates { + label := fmt.Sprintf("- %s [%s] count=%d", d.Name, d.Manager, d.Count) + if d.Conflicts { + label += " conflicts" + } + t = t.NewLine().Append(label, "text-muted") + } + } + return t +} + +var _ api.TreeNode = (*Node)(nil) + +func (n *Node) Pretty() api.Text { + if n == nil { + return clicky.Text("") + } + return nodeText(n) +} + +func (n *Node) GetChildren() []api.TreeNode { + if n == nil || len(n.Children) == 0 { + return nil + } + nodeChildren := make([]*Node, 0, len(n.Children)) + for _, child := range n.Children { + if child != nil { + nodeChildren = append(nodeChildren, child) + } + } + sort.SliceStable(nodeChildren, func(i, j int) bool { + return dependencyLess(nodeChildren[i], nodeChildren[j]) + }) + children := make([]api.TreeNode, 0, len(nodeChildren)) + for _, child := range nodeChildren { + children = append(children, child) + } + return children +} + +func dependencyTree(roots []*Node) (api.TextTree, bool) { + if len(roots) == 0 { + return api.TextTree{}, false + } + nodes := make([]api.TreeNode, 0, len(roots)) + for _, root := range roots { + if root != nil { + nodes = append(nodes, root) + } + } + if len(nodes) == 0 { + return api.TextTree{}, false + } + return api.NewTree(nodes...), true +} + +type dependencyTag struct { + label string + style string +} + +func nodeText(node *Node) api.Text { + t := clicky.Text("") + t = t.Append("[", "text-muted"). + Append(string(node.Manager), managerStyle(node.Manager)). + Append("] ", "text-muted"). + Append(node.Name, "font-bold text-cyan-600") + if node.Version != "" { + t = t.Append("@"+node.Version, "font-mono text-muted") + } + tags := nodeTags(node) + if len(tags) > 0 { + t = t.Space().Append("(", "text-muted") + for i, tag := range tags { + if i > 0 { + t = t.Append(", ", "text-muted") + } + t = t.Append(tag.label, tag.style) + } + t = t.Append(")", "text-muted") + } + if node.Path != "" { + t = t.Space().Append(node.Path, "font-mono text-muted") + } + return t +} + +func nodeLabel(node *Node) string { + parts := []string{fmt.Sprintf("[%s] %s", node.Manager, node.Name)} + if node.Version != "" { + parts[0] += "@" + node.Version + } + var tags []string + if node.Scope != "" { + tags = append(tags, node.Scope) + } + if node.Direct { + tags = append(tags, "direct") + } + if node.Dev { + tags = append(tags, "dev") + } + if node.Optional { + tags = append(tags, "optional") + } + if node.Local { + tags = append(tags, "local") + } + if node.Circular { + tags = append(tags, "circular") + } + if node.Duplicate != nil { + tag := fmt.Sprintf("dup:%d", node.Duplicate.Count) + if node.Duplicate.Conflicts { + tag += ":conflict" + } + tags = append(tags, tag) + } + if len(tags) > 0 { + sort.Strings(tags) + parts = append(parts, "("+strings.Join(tags, ", ")+")") + } + if node.Path != "" { + parts = append(parts, node.Path) + } + return strings.Join(parts, " ") +} + +func nodeTags(node *Node) []dependencyTag { + var tags []dependencyTag + if node.Scope != "" { + tags = append(tags, dependencyTag{label: node.Scope, style: scopeStyle(node.Scope)}) + } + if node.Direct { + tags = append(tags, dependencyTag{label: "direct", style: "text-green-600"}) + } + if node.Dev { + tags = append(tags, dependencyTag{label: "dev", style: "text-yellow-600"}) + } + if node.Optional { + tags = append(tags, dependencyTag{label: "optional", style: "text-purple-600"}) + } + if node.Local { + tags = append(tags, dependencyTag{label: "local", style: "text-cyan-600"}) + } + if node.Circular { + tags = append(tags, dependencyTag{label: "circular", style: "font-bold text-red-600"}) + } + if node.Duplicate != nil { + tag := fmt.Sprintf("dup:%d", node.Duplicate.Count) + style := "text-orange-500" + if node.Duplicate.Conflicts { + tag += ":conflict" + style = "font-bold text-red-600" + } + tags = append(tags, dependencyTag{label: tag, style: style}) + } + sort.Slice(tags, func(i, j int) bool { + return tags[i].label < tags[j].label + }) + return tags +} + +func managerStyle(manager Manager) string { + switch manager { + case ManagerPNPM: + return "font-bold text-orange-500" + case ManagerNPM: + return "font-bold text-red-500" + case ManagerGo: + return "font-bold text-cyan-600" + case ManagerMaven: + return "font-bold text-purple-600" + case ManagerGradle: + return "font-bold text-green-600" + default: + return "font-bold text-muted" + } +} + +func scopeStyle(scope string) string { + switch strings.ToLower(scope) { + case "dependencies", "dependency", "require", "compile": + return "text-green-600" + case "devdependencies", "dev", "test", "testdependencies": + return "text-yellow-600" + case "optionaldependencies", "optional": + return "text-purple-600" + case "peerdependencies", "peer": + return "text-blue-600" + case "runtime", "runtimeclasspath", "implementation": + return "text-cyan-600" + default: + return "text-muted" + } +} diff --git a/deps/runner.go b/deps/runner.go new file mode 100644 index 0000000..f24f607 --- /dev/null +++ b/deps/runner.go @@ -0,0 +1,39 @@ +package deps + +import ( + "context" + "os" + "os/exec" +) + +type Command struct { + Dir string + Name string + Args []string + Env []string +} + +type CommandResult struct { + Stdout string + Stderr string +} + +type CommandRunner interface { + Run(ctx context.Context, cmd Command) (CommandResult, error) +} + +type ExecRunner struct{} + +func (ExecRunner) Run(ctx context.Context, spec Command) (CommandResult, error) { + cmd := exec.CommandContext(ctx, spec.Name, spec.Args...) + cmd.Dir = spec.Dir + if len(spec.Env) > 0 { + cmd.Env = append(os.Environ(), spec.Env...) + } + out, err := cmd.Output() + result := CommandResult{Stdout: string(out)} + if exit, ok := err.(*exec.ExitError); ok { + result.Stderr = string(exit.Stderr) + } + return result, err +} diff --git a/deps/scan.go b/deps/scan.go new file mode 100644 index 0000000..96471d7 --- /dev/null +++ b/deps/scan.go @@ -0,0 +1,251 @@ +package deps + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/flanksource/clicky/task" + flanksourceContext "github.com/flanksource/commons/context" +) + +func Scan(ctx context.Context, path string, opts Options) (*Export, error) { + if opts.Mode == "" { + opts.Mode = ModeAuto + } + if opts.Runner == nil { + opts.Runner = ExecRunner{} + } + now := time.Now + if opts.Now != nil { + now = opts.Now + } + absPath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + projects, warnings, err := Discover(absPath, opts.Managers) + if err != nil { + return nil, err + } + + roots, projectWarnings, err := resolveProjectsWithTasks(ctx, projects, opts) + warnings = append(warnings, projectWarnings...) + if err != nil { + return nil, err + } + if len(roots) == 0 { + return nil, fmt.Errorf("no dependency graphs resolved") + } + + filteredRoots := make([]*Node, 0, len(roots)) + for _, root := range roots { + if filtered := filterAndPrune(root, opts.Filters, opts.MaxDepth); filtered != nil { + filteredRoots = append(filteredRoots, filtered) + } + } + dups := analyzeDuplicates(filteredRoots) + applyDuplicateRefs(filteredRoots, dups) + nodes, edges, stats := flatten(filteredRoots, dups) + stats.Projects = len(projects) + + sortWarnings(warnings) + return &Export{ + Metadata: Metadata{ + ExportedAt: now(), + Version: "1.0", + Path: absPath, + Managers: opts.Managers, + Mode: opts.Mode, + Filter: opts.Filters, + MaxDepth: opts.MaxDepth, + Configurations: opts.Configurations, + ProjectsScanned: len(projects), + }, + Roots: filteredRoots, + Nodes: nodes, + Edges: edges, + Statistics: stats, + Duplicates: duplicatesList(dups), + Warnings: warnings, + }, nil +} + +type projectResolution struct { + Index int + Project Project + Root *Node + Warnings []Warning + Err error +} + +func resolveProjectsWithTasks(ctx context.Context, projects []Project, opts Options) ([]*Node, []Warning, error) { + group := task.StartGroup[projectResolution]( + "Resolving dependency graphs", + task.WithKind("repomap-deps"), + task.WithLabels(map[string]string{ + "mode": string(opts.Mode), + "projects": fmt.Sprintf("%d", len(projects)), + }), + ) + for i, project := range projects { + index := i + project := project + group.Add(projectTaskName(project), func(_ flanksourceContext.Context, tk *task.Task) (projectResolution, error) { + tk.SetProgress(0, 1) + tk.Infof("resolving %s dependencies in %s", project.Manager, project.Dir) + taskOpts := opts + taskOpts.Runner = taskCommandRunner{base: opts.Runner, task: tk} + root, warnings, err := resolveProject(ctx, project, taskOpts) + result := projectResolution{ + Index: index, + Project: project, + Root: root, + Warnings: warnings, + Err: err, + } + for _, warning := range warnings { + tk.Warnf("%s", warning.Message) + } + if err != nil { + if opts.Strict || opts.Mode == ModeNative { + tk.FailedWithError(err) + return result, err + } + tk.Warnf("%s", err.Error()) + tk.Warning() + return result, nil + } + tk.SetProgress(1, 1) + if len(warnings) > 0 { + tk.Warning() + } else { + tk.Success() + } + return result, nil + }) + } + + results, err := group.GetResults() + if err != nil { + return nil, nil, err + } + ordered := make([]projectResolution, 0, len(results)) + for _, result := range results { + ordered = append(ordered, result) + } + sort.Slice(ordered, func(i, j int) bool { + return ordered[i].Index < ordered[j].Index + }) + roots := make([]*Node, 0, len(ordered)) + var warnings []Warning + for _, result := range ordered { + warnings = append(warnings, result.Warnings...) + if result.Err != nil { + if opts.Strict || opts.Mode == ModeNative { + return nil, warnings, result.Err + } + warnings = append(warnings, Warning{Manager: result.Project.Manager, Project: result.Project.Dir, Message: result.Err.Error()}) + continue + } + if result.Root != nil { + roots = append(roots, result.Root) + } + } + return roots, warnings, nil +} + +func projectTaskName(project Project) string { + return fmt.Sprintf("%s %s", project.Manager, project.Dir) +} + +type taskCommandRunner struct { + base CommandRunner + task *task.Task +} + +func (r taskCommandRunner) Run(ctx context.Context, cmd Command) (CommandResult, error) { + if r.task != nil { + r.task.Infof("running %s %s", cmd.Name, strings.Join(cmd.Args, " ")) + } + result, err := r.base.Run(ctx, cmd) + if err != nil && r.task != nil { + r.task.Warnf("%s failed: %v", cmd.Name, err) + } + return result, err +} + +func resolveProject(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + var root *Node + var warnings []Warning + var err error + if opts.Mode != ModeManifest { + root, warnings, err = resolveNative(ctx, project, opts) + if err == nil && root != nil { + return root, warnings, nil + } + if opts.Mode == ModeNative { + return nil, warnings, fmt.Errorf("%s native resolver failed for %s: %w", project.Manager, project.Dir, err) + } + warnings = append(warnings, Warning{ + Manager: project.Manager, + Project: project.Dir, + Message: fmt.Sprintf("native resolver failed; using manifest fallback: %v", err), + }) + } + root, fallbackWarnings, err := resolveManifest(project, opts) + warnings = append(warnings, fallbackWarnings...) + if err != nil { + return nil, warnings, fmt.Errorf("%s manifest resolver failed for %s: %w", project.Manager, project.Dir, err) + } + return root, warnings, nil +} + +func resolveNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + switch project.Manager { + case ManagerGo: + return resolveGoNative(ctx, project, opts) + case ManagerMaven: + return resolveMavenNative(ctx, project, opts) + case ManagerGradle: + return resolveGradleNative(ctx, project, opts) + case ManagerNPM: + return resolveNPMNative(ctx, project, opts) + case ManagerPNPM: + return resolvePNPMNative(ctx, project, opts) + default: + return nil, nil, fmt.Errorf("unsupported manager %q", project.Manager) + } +} + +func resolveManifest(project Project, opts Options) (*Node, []Warning, error) { + switch project.Manager { + case ManagerGo: + return resolveGoManifest(project) + case ManagerMaven: + return resolveMavenManifest(project) + case ManagerGradle: + return resolveGradleManifest(project) + case ManagerNPM: + return resolveNPMManifest(project) + case ManagerPNPM: + return resolvePNPMManifest(project) + default: + return nil, nil, fmt.Errorf("unsupported manager %q", project.Manager) + } +} + +func sortWarnings(warnings []Warning) { + sort.Slice(warnings, func(i, j int) bool { + if warnings[i].Manager != warnings[j].Manager { + return warnings[i].Manager < warnings[j].Manager + } + if warnings[i].Project != warnings[j].Project { + return warnings[i].Project < warnings[j].Project + } + return warnings[i].Message < warnings[j].Message + }) +} diff --git a/deps/scan_test.go b/deps/scan_test.go new file mode 100644 index 0000000..b8dfd24 --- /dev/null +++ b/deps/scan_test.go @@ -0,0 +1,259 @@ +package deps + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/flanksource/clicky/api" +) + +func TestScanGoManifestFallback(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require ( + github.com/acme/lib v1.2.3 + github.com/acme/indirect v0.1.0 // indirect +) + +replace github.com/acme/lib => ../lib +`) + + got, err := Scan(context.Background(), dir, Options{ + Mode: ModeManifest, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if got.Metadata.ProjectsScanned != 1 { + t.Fatalf("projects scanned = %d, want 1", got.Metadata.ProjectsScanned) + } + if len(got.Roots) != 1 || got.Roots[0].Name != "github.com/acme/app" { + t.Fatalf("unexpected roots: %#v", got.Roots) + } + if len(got.Roots[0].Children) != 2 { + t.Fatalf("children = %d, want 2", len(got.Roots[0].Children)) + } + lib := findChild(got.Roots[0], "github.com/acme/lib") + if lib == nil || lib.Name != "github.com/acme/lib" || !lib.Local || lib.Source != "../lib" { + t.Fatalf("replace/local metadata not captured: %#v", lib) + } + if got.Statistics.Total != 3 || got.Statistics.Edges != 2 { + t.Fatalf("stats = %+v, want total=3 edges=2", got.Statistics) + } + data, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"nodes"`) || !strings.Contains(string(data), `"edges"`) { + t.Fatalf("json export missing graph fields: %s", data) + } + if strings.Contains(string(data), `"tree"`) { + t.Fatalf("json export should not include presentation-only tree fields: %s", data) + } +} + +func TestNodeImplementsClickyTreeNode(t *testing.T) { + var _ api.TreeNode = (*Node)(nil) + + root := NewNode(ManagerGo, "root", "") + child := NewNode(ManagerGo, "github.com/acme/lib", "v1.0.0") + root.Children = []*Node{child, nil} + + children := root.GetChildren() + if len(children) != 1 { + t.Fatalf("children = %d, want 1", len(children)) + } + if got := children[0].Pretty().String(); !strings.Contains(got, "github.com/acme/lib@v1.0.0") { + t.Fatalf("unexpected child label: %q", got) + } + if ansi := children[0].Pretty().ANSI(); !strings.Contains(ansi, "\x1b[") { + t.Fatalf("expected styled dependency label to emit ANSI color, got %q", ansi) + } +} + +func TestTreeChildrenSortByTypeThenName(t *testing.T) { + root := NewNode(ManagerGo, "root", "") + replacement := NewNode(ManagerGo, "z-replacement", "v1.0.0") + replacement.Local = true + replacement.Direct = true + directB := NewNode(ManagerGo, "b-direct", "v1.0.0") + directB.Direct = true + directA := NewNode(ManagerGo, "a-direct", "v1.0.0") + directA.Direct = true + indirectB := NewNode(ManagerGo, "b-indirect", "v1.0.0") + indirectA := NewNode(ManagerGo, "a-indirect", "v1.0.0") + + root.Children = []*Node{indirectB, directB, replacement, indirectA, directA} + got := treeChildNames(root.GetChildren()) + want := []string{"z-replacement", "a-direct", "b-direct", "a-indirect", "b-indirect"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("tree child order = %#v, want %#v", got, want) + } + + sortChildren(root) + got = nodeChildNames(root.Children) + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("stored child order = %#v, want %#v", got, want) + } +} + +func TestFilterAndDepthPreserveAncestors(t *testing.T) { + root := NewNode(ManagerGo, "root", "") + child := NewNode(ManagerGo, "github.com/acme/lib", "v1.0.0") + child.Depth = 1 + grandchild := NewNode(ManagerGo, "github.com/acme/target", "v2.0.0") + grandchild.Depth = 2 + other := NewNode(ManagerGo, "github.com/acme/other", "v1.0.0") + other.Depth = 1 + child.Children = []*Node{grandchild} + root.Children = []*Node{child, other} + + filtered := filterAndPrune(root, []string{"*target*"}, 0) + if filtered == nil || len(filtered.Children) != 1 { + t.Fatalf("expected only matching branch, got %#v", filtered) + } + if filtered.Children[0].Name != "github.com/acme/lib" || len(filtered.Children[0].Children) != 1 { + t.Fatalf("expected ancestor plus target child, got %#v", filtered.Children[0]) + } + directOnly := filterAndPrune(root, nil, 1) + if directOnly == nil || len(directOnly.Children) != 2 || len(directOnly.Children[0].Children) != 0 { + t.Fatalf("depth=1 should keep direct deps only, got %#v", directOnly) + } +} + +func TestParsePackageLockV3(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "package-lock.json") + writeFile(t, path, `{ + "name": "app", + "version": "1.0.0", + "lockfileVersion": 3, + "packages": { + "": { + "name": "app", + "version": "1.0.0", + "dependencies": { + "left-pad": "^1.3.0", + "@scope/pkg": "2.0.0" + } + }, + "node_modules/left-pad": { + "version": "1.3.0", + "dependencies": { + "repeat-string": "1.6.1" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1" + }, + "node_modules/@scope/pkg": { + "version": "2.0.0", + "dev": true + } + } +}`) + + root, err := parsePackageLock(path) + if err != nil { + t.Fatal(err) + } + if root.Name != "app" || len(root.Children) != 2 { + t.Fatalf("unexpected root: %#v", root) + } + leftPad := findChild(root, "left-pad") + if leftPad == nil || leftPad.Version != "1.3.0" || len(leftPad.Children) != 1 { + t.Fatalf("left-pad tree not resolved: %#v", leftPad) + } + scoped := findChild(root, "@scope/pkg") + if scoped == nil || !scoped.Dev { + t.Fatalf("scoped dev package metadata missing: %#v", scoped) + } +} + +func TestParsePNPMLock(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "pnpm-lock.yaml") + writeFile(t, path, `lockfileVersion: '9.0' +importers: + .: + dependencies: + left-pad: + specifier: ^1.3.0 + version: 1.3.0 + devDependencies: + local-tool: + specifier: file:../tool + version: file:../tool +packages: + left-pad@1.3.0: + dependencies: + repeat-string: 1.6.1 + repeat-string@1.6.1: {} + "local-tool@file:../tool": {} +`) + + root, err := parsePNPMLock(path) + if err != nil { + t.Fatal(err) + } + importer := findChild(root, ".") + if importer == nil { + t.Fatalf("importer not found: %#v", root) + } + leftPad := findChild(importer, "left-pad") + if leftPad == nil || len(leftPad.Children) != 1 || leftPad.Children[0].Name != "repeat-string" { + t.Fatalf("pnpm dependency tree not resolved: %#v", leftPad) + } + local := findChild(importer, "local-tool") + if local == nil || !local.Local || !local.Dev { + t.Fatalf("pnpm local dev dependency metadata missing: %#v", local) + } +} + +func findChild(root *Node, name string) *Node { + for _, child := range root.Children { + if child.Name == name { + return child + } + } + return nil +} + +func treeChildNames(children []api.TreeNode) []string { + names := make([]string, 0, len(children)) + for _, child := range children { + if node, ok := child.(*Node); ok { + names = append(names, node.Name) + } + } + return names +} + +func nodeChildNames(children []*Node) []string { + names := make([]string, 0, len(children)) + for _, child := range children { + if child != nil { + names = append(names, child.Name) + } + } + return names +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/go.mod b/go.mod index e2463e0..ba9f2ad 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/samber/oops v1.21.0 github.com/spf13/cobra v1.10.2 github.com/stretchr/testify v1.11.1 + golang.org/x/mod v0.36.0 golang.org/x/sync v0.20.0 ) From afb4154c46f6a2366ae8a0edf030cbc3443597da Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 9 Jun 2026 10:23:59 +0300 Subject: [PATCH 03/17] feat(deps): add dependency update command with interactive selection Implement a new `deps update` subcommand that enables users to discover, resolve, and apply updates to direct dependencies across Go, npm, pnpm, container images, and Helm charts. Key features: - Interactive tree-based UI for selecting dependencies to update with filtering support - Version resolution and availability checking via package managers and image registries - Dry-run mode to preview changes without applying them - Check mode to list available updates without prompting - Support for MatchItem expressions to filter dependencies by name, manager, scope, and file path - Automatic git-aware manifest discovery that respects .gitignore - Enhanced pnpm support with dependency map parsing Changes include: - New UpdateOptions, UpdateCandidate, UpdateChoice, and UpdatePlan types - Image and Helm chart update discovery and application via imageupdate package - Interactive tree picker UI using bubbletea for dependency selection - Version sorting and filtering logic with semantic versioning support - Comprehensive test coverage for all update scenarios - Refactored manifest discovery to use git ls-files for better performance --- cmd/repomap/deps.go | 84 ++++ cmd/repomap/deps_test.go | 42 ++ deps/discover.go | 82 +++- deps/model.go | 2 + deps/pnpm.go | 49 +- deps/pnpm_test.go | 45 ++ deps/pretty.go | 4 + deps/scan_test.go | 49 ++ deps/update.go | 981 +++++++++++++++++++++++++++++++++++++++ deps/update_image.go | 265 +++++++++++ deps/update_test.go | 507 ++++++++++++++++++++ deps/update_tree.go | 602 ++++++++++++++++++++++++ deps/update_tree_test.go | 131 ++++++ go.mod | 2 +- 14 files changed, 2822 insertions(+), 23 deletions(-) create mode 100644 deps/pnpm_test.go create mode 100644 deps/update.go create mode 100644 deps/update_image.go create mode 100644 deps/update_test.go create mode 100644 deps/update_tree.go create mode 100644 deps/update_tree_test.go diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index 21b1324..3541563 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -20,8 +20,17 @@ type DepsOptions struct { Strict bool `json:"strict,omitempty" flag:"strict" help:"Fail if native resolution is unavailable or fallback resolution is degraded"` } +type DepsUpdateOptions struct { + Args []string `json:"args" args:"true" required:"true" help:"Dependency MatchItem expression followed by optional path"` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to update: go, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Check bool `json:"check" flag:"check" help:"Resolve and list available updates without prompting or writing"` + DryRun bool `json:"dry_run" flag:"dry-run" help:"Show planned dependency updates without running package-manager commands"` +} + func (opts DepsOptions) GetName() string { return "deps" } +func (opts DepsUpdateOptions) GetName() string { return "update [path]" } + func (opts DepsOptions) Help() api.Text { return clicky.Text(`Generate dependency graphs for Go, Maven, Gradle, npm, and pnpm projects. @@ -42,9 +51,33 @@ EXAMPLES: repomap deps --mode manifest --json > deps.json`) } +func (opts DepsUpdateOptions) Help() api.Text { + return clicky.Text(`Update direct package, image, and Helm chart dependencies. + +The required expr argument uses commons MatchItem syntax and is matched against +dependency names, manager-qualified names, versions, and scopes. Manifest path +matching is explicit with path: or file:. Matched direct +dependencies are resolved to published versions, then repomap prompts for which +dependencies and versions to apply. + +Use --check to list updateable dependencies without prompting or writing. + +EXAMPLES: + repomap deps update 'github.com/flanksource/*' + repomap deps update '*' --check + repomap deps update 'path:apps/*/package.json' + repomap deps update 'image:ghcr.io/flanksource/*' --manager image + repomap deps update 'helm:mission-control' --manager helm + repomap deps update 'npm:@flanksource/*' ./web --manager npm + repomap deps update 'left-pad,!*beta*' --dry-run`) +} + func init() { cmd := clicky.AddNamedCommandWithContext("deps", rootCmd, DepsOptions{}, runDeps) cmd.Short = "Generate dependency graphs for Go, Maven, Gradle, npm, and pnpm projects" + + updateCmd := clicky.AddNamedCommandWithContext("update", cmd, DepsUpdateOptions{}, runDepsUpdate) + updateCmd.Short = "Update direct package, image, and Helm chart dependencies" } func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { @@ -73,6 +106,37 @@ func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { }) } +func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { + if len(opts.Args) == 0 { + return nil, fmt.Errorf("dependency update expression is required") + } + if len(opts.Args) > 2 { + return nil, fmt.Errorf("expected [path], got %d arguments", len(opts.Args)) + } + path := "." + if len(opts.Args) == 2 { + path = opts.Args[1] + } + path, err := resolvePath(path) + if err != nil { + return nil, err + } + managers, err := parseUpdateManagers(opts.Manager) + if err != nil { + return nil, err + } + plans, err := depgraph.Update(ctx, path, depgraph.UpdateOptions{ + Managers: managers, + Expression: []string{opts.Args[0]}, + Check: opts.Check, + DryRun: opts.DryRun, + }) + if err != nil { + return nil, err + } + return api.NewTableFrom(plans), nil +} + func parseDepsMode(value string) (depgraph.Mode, error) { switch depgraph.Mode(strings.TrimSpace(value)) { case "", depgraph.ModeAuto: @@ -104,6 +168,26 @@ func parseManagers(values []string) ([]depgraph.Manager, error) { return out, nil } +func parseUpdateManagers(values []string) ([]depgraph.Manager, error) { + parts := splitCommaArgs(values) + if len(parts) == 0 { + return nil, nil + } + out := make([]depgraph.Manager, 0, len(parts)) + for _, part := range parts { + manager := depgraph.Manager(strings.ToLower(part)) + switch manager { + case "docker": + out = append(out, depgraph.ManagerImage) + case depgraph.ManagerGo, depgraph.ManagerNPM, depgraph.ManagerPNPM, depgraph.ManagerImage, depgraph.ManagerHelm: + out = append(out, manager) + default: + return nil, fmt.Errorf("unsupported dependency update manager %q (expected go, npm, pnpm, image/docker, or helm)", part) + } + } + return out, nil +} + func splitCommaArgs(values []string) []string { var out []string for _, value := range values { diff --git a/cmd/repomap/deps_test.go b/cmd/repomap/deps_test.go index 02bb536..7766f73 100644 --- a/cmd/repomap/deps_test.go +++ b/cmd/repomap/deps_test.go @@ -26,6 +26,25 @@ func TestParseManagers(t *testing.T) { } } +func TestParseUpdateManagers(t *testing.T) { + got, err := parseUpdateManagers([]string{"go,image", "docker", "helm"}) + if err != nil { + t.Fatal(err) + } + want := []depgraph.Manager{depgraph.ManagerGo, depgraph.ManagerImage, depgraph.ManagerImage, depgraph.ManagerHelm} + if len(got) != len(want) { + t.Fatalf("len = %d, want %d: %#v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("manager[%d] = %q, want %q", i, got[i], want[i]) + } + } + if _, err := parseUpdateManagers([]string{"maven"}); err == nil { + t.Fatal("expected unsupported update manager error") + } +} + func TestParseDepsMode(t *testing.T) { for _, mode := range []string{"", "auto", "native", "manifest"} { if _, err := parseDepsMode(mode); err != nil { @@ -53,3 +72,26 @@ func TestDepsDepthDefault(t *testing.T) { t.Fatalf("depth help should document unlimited mode, got %q", flag.Usage) } } + +func TestDepsUpdateCommandRegistered(t *testing.T) { + cmd, _, err := rootCmd.Find([]string{"deps", "update", "github.com/flanksource/*"}) + if err != nil { + t.Fatal(err) + } + if cmd == nil || !strings.HasPrefix(cmd.Use, "update") { + t.Fatalf("expected deps update command, got %#v", cmd) + } + if flag := cmd.Flags().Lookup("dry-run"); flag == nil { + t.Fatal("dry-run flag not registered") + } + if flag := cmd.Flags().Lookup("check"); flag == nil { + t.Fatal("check flag not registered") + } + manager := cmd.Flags().Lookup("manager") + if manager == nil { + t.Fatal("manager flag not registered") + } + if !strings.Contains(manager.Usage, "go, npm, pnpm, image/docker, helm") { + t.Fatalf("manager help should document update-supported managers, got %q", manager.Usage) + } +} diff --git a/deps/discover.go b/deps/discover.go index 2b505aa..3780d41 100644 --- a/deps/discover.go +++ b/deps/discover.go @@ -4,6 +4,7 @@ import ( "fmt" "io/fs" "os" + osexec "os/exec" "path/filepath" "sort" "strings" @@ -33,34 +34,26 @@ func Discover(root string, managers []Manager) ([]Project, []Warning, error) { absRoot = filepath.Dir(absRoot) } + files, err := discoverManifestFiles(absRoot) + if err != nil { + return nil, nil, err + } + byDir := map[string]map[string]string{} - err = filepath.WalkDir(absRoot, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - if path != absRoot && ignoredDirs[d.Name()] { - return filepath.SkipDir - } - return nil - } - name := d.Name() + for _, path := range files { + name := filepath.Base(path) manager := managerForManifest(name) if manager == "" { - return nil + continue } if len(selected) > 0 && !selected[manager] { - return nil + continue } dir := filepath.Dir(path) if byDir[dir] == nil { byDir[dir] = map[string]string{} } byDir[dir][name] = path - return nil - }) - if err != nil { - return nil, nil, err } var projects []Project @@ -119,6 +112,61 @@ func Discover(root string, managers []Manager) ([]Project, []Warning, error) { return projects, warnings, nil } +func discoverManifestFiles(root string) ([]string, error) { + if files, ok := gitManifestFiles(root); ok { + return files, nil + } + return walkManifestFiles(root) +} + +func gitManifestFiles(root string) ([]string, bool) { + cmd := osexec.Command("git", "-C", root, "ls-files", "--cached", "--others", "--exclude-standard", "-z", "--", ".") + out, err := cmd.Output() + if err != nil { + return nil, false + } + var files []string + for _, rel := range strings.Split(string(out), "\x00") { + if rel == "" { + continue + } + name := filepath.Base(filepath.FromSlash(rel)) + if managerForManifest(name) == "" { + continue + } + files = append(files, filepath.Join(root, filepath.FromSlash(rel))) + } + sort.Strings(files) + return files, true +} + +func walkManifestFiles(root string) ([]string, error) { + var files []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if path != root && ignoredDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + name := d.Name() + manager := managerForManifest(name) + if manager == "" { + return nil + } + files = append(files, path) + return nil + }) + if err != nil { + return nil, err + } + sort.Strings(files) + return files, nil +} + func managerForManifest(name string) Manager { switch strings.ToLower(name) { case "go.mod", "go.work": diff --git a/deps/model.go b/deps/model.go index f55aeda..6652c7d 100644 --- a/deps/model.go +++ b/deps/model.go @@ -10,6 +10,8 @@ const ( ManagerGradle Manager = "gradle" ManagerNPM Manager = "npm" ManagerPNPM Manager = "pnpm" + ManagerImage Manager = "image" + ManagerHelm Manager = "helm" ) type Mode string diff --git a/deps/pnpm.go b/deps/pnpm.go index 27a9e6b..d035814 100644 --- a/deps/pnpm.go +++ b/deps/pnpm.go @@ -13,11 +13,50 @@ import ( ) type pnpmNativeNode struct { - Name string `json:"name"` - Version string `json:"version"` - Path string `json:"path"` - Private bool `json:"private"` - Dependencies []pnpmNativeNode `json:"dependencies"` + Name string `json:"name"` + From string `json:"from"` + Version string `json:"version"` + Path string `json:"path"` + Private bool `json:"private"` + Dependencies pnpmNativeDeps `json:"dependencies"` +} + +type pnpmNativeDeps []pnpmNativeNode + +func (d *pnpmNativeDeps) UnmarshalJSON(data []byte) error { + if strings.TrimSpace(string(data)) == "null" { + *d = nil + return nil + } + var list []pnpmNativeNode + if err := json.Unmarshal(data, &list); err == nil { + *d = list + return nil + } + var object map[string]json.RawMessage + if err := json.Unmarshal(data, &object); err != nil { + return err + } + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]pnpmNativeNode, 0, len(keys)) + for _, key := range keys { + var node pnpmNativeNode + if err := json.Unmarshal(object[key], &node); err != nil { + var version string + if err2 := json.Unmarshal(object[key], &version); err2 != nil { + return fmt.Errorf("dependency %s: %w", key, err) + } + node.Version = version + } + node.Name = firstNonEmpty(node.Name, node.From, key) + out = append(out, node) + } + *d = out + return nil } func resolvePNPMNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { diff --git a/deps/pnpm_test.go b/deps/pnpm_test.go new file mode 100644 index 0000000..9a27b4c --- /dev/null +++ b/deps/pnpm_test.go @@ -0,0 +1,45 @@ +package deps + +import ( + "path/filepath" + "testing" +) + +func TestParsePNPMNativeDependencyMap(t *testing.T) { + dir := t.TempDir() + root, err := parsePNPMNative([]byte(`[ + { + "name": "app", + "version": "1.0.0", + "path": "/workspace/app", + "dependencies": { + "left-pad": { + "version": "1.3.0", + "path": "/workspace/app/node_modules/left-pad", + "dependencies": { + "repeat-string": { + "from": "repeat-string", + "version": "1.6.1", + "path": "/workspace/app/node_modules/repeat-string" + } + } + } + } + } +]`), Project{Manager: ManagerPNPM, Dir: dir, File: filepath.Join(dir, "pnpm-lock.yaml")}) + if err != nil { + t.Fatal(err) + } + app := findChild(root, "app") + if app == nil || app.Version != "1.0.0" || !app.Direct { + t.Fatalf("app node not parsed: %#v", app) + } + leftPad := findChild(app, "left-pad") + if leftPad == nil || leftPad.Version != "1.3.0" { + t.Fatalf("map dependency not parsed: %#v", leftPad) + } + repeatString := findChild(leftPad, "repeat-string") + if repeatString == nil || repeatString.Version != "1.6.1" { + t.Fatalf("nested map dependency not parsed: %#v", repeatString) + } +} diff --git a/deps/pretty.go b/deps/pretty.go index 17bbe60..38ddfa1 100644 --- a/deps/pretty.go +++ b/deps/pretty.go @@ -208,6 +208,10 @@ func managerStyle(manager Manager) string { return "font-bold text-purple-600" case ManagerGradle: return "font-bold text-green-600" + case ManagerImage: + return "font-bold text-blue-600" + case ManagerHelm: + return "font-bold text-indigo-600" default: return "font-bold text-muted" } diff --git a/deps/scan_test.go b/deps/scan_test.go index b8dfd24..96d4f9e 100644 --- a/deps/scan_test.go +++ b/deps/scan_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -219,6 +220,37 @@ packages: } } +func TestDiscoverRespectsGitignore(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + dir := t.TempDir() + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, ".gitignore"), `ignored-dir/ +ignored-package.json +`) + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 +`) + writeFile(t, filepath.Join(dir, "visible", "package.json"), `{"name":"visible"}`) + writeFile(t, filepath.Join(dir, "ignored-dir", "package.json"), `{"name":"ignored-dir"}`) + writeFile(t, filepath.Join(dir, "ignored-package.json"), `{"name":"ignored-file"}`) + + projects, _, err := Discover(dir, nil) + if err != nil { + t.Fatal(err) + } + got := projectFiles(projects) + want := []string{ + filepath.Join(dir, "go.mod"), + filepath.Join(dir, "visible", "package.json"), + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("projects:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + func findChild(root *Node, name string) *Node { for _, child := range root.Children { if child.Name == name { @@ -228,6 +260,23 @@ func findChild(root *Node, name string) *Node { return nil } +func projectFiles(projects []Project) []string { + out := make([]string, 0, len(projects)) + for _, project := range projects { + out = append(out, project.File) + } + return out +} + +func runGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, out) + } +} + func treeChildNames(children []api.TreeNode) []string { names := make([]string, 0, len(children)) for _, child := range children { diff --git a/deps/update.go b/deps/update.go new file mode 100644 index 0000000..c004006 --- /dev/null +++ b/deps/update.go @@ -0,0 +1,981 @@ +package deps + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/flanksource/clicky/task" + "github.com/flanksource/commons/collections" + flanksourceContext "github.com/flanksource/commons/context" + "github.com/flanksource/repomap/imageupdate" + "golang.org/x/mod/modfile" +) + +const updateResolveConcurrency = 8 + +var supportedUpdateManagers = map[Manager]bool{ + ManagerGo: true, + ManagerNPM: true, + ManagerPNPM: true, + ManagerImage: true, + ManagerHelm: true, +} + +type CandidateSelector func([]UpdateChoice) ([]UpdateChoice, bool) +type VersionSelector func(UpdateChoice) (string, bool) + +type ImageVersionResolver interface { + Available(context.Context, imageupdate.UpdateTarget) ([]string, error) + ResolveLatestVersions(context.Context, imageupdate.UpdateTarget) (imageupdate.LatestVersions, error) + NewImageValue(context.Context, imageupdate.UpdateTarget, string) (string, error) +} + +type UpdateOptions struct { + Managers []Manager + Expression []string + Check bool + DryRun bool + Runner CommandRunner + ImageResolver ImageVersionResolver + SelectCandidates CandidateSelector + SelectVersion VersionSelector +} + +type UpdateCandidate struct { + Manager Manager `json:"manager"` + Name string `json:"name"` + Current string `json:"current"` + Scope string `json:"scope,omitempty"` + File string `json:"file"` + Dir string `json:"dir"` + Target *imageupdate.UpdateTarget `json:"-"` +} + +type UpdateChoice struct { + Candidate UpdateCandidate `json:"candidate"` + Versions []string `json:"versions"` + LatestStable string `json:"latest_stable,omitempty"` + LatestPrerelease string `json:"latest_prerelease,omitempty"` +} + +type UpdatePlan struct { + Manager Manager `json:"manager"` + Name string `json:"name"` + File string `json:"file"` + Scope string `json:"scope,omitempty"` + OldVersion string `json:"old_version"` + NewVersion string `json:"new_version,omitempty"` + Command []string `json:"command,omitempty"` + Written bool `json:"written"` + DryRun bool `json:"dry_run"` + Checked bool `json:"checked,omitempty"` + Skipped string `json:"skipped,omitempty"` +} + +func Update(ctx context.Context, path string, opts UpdateOptions) ([]UpdatePlan, error) { + if path == "" { + path = "." + } + managers, err := updateManagers(opts.Managers) + if err != nil { + return nil, err + } + patterns := splitUpdatePatterns(opts.Expression) + if len(patterns) == 0 { + return nil, fmt.Errorf("dependency update expression is required") + } + if opts.Runner == nil { + opts.Runner = ExecRunner{} + } + + candidates, err := DiscoverUpdateCandidates(path, managers) + if err != nil { + return nil, err + } + candidates = filterUpdateCandidates(candidates, patterns) + if len(candidates) == 0 { + return nil, fmt.Errorf("no direct dependencies matched %q", strings.Join(patterns, ",")) + } + + choices, plansByKey := resolveUpdateChoices(ctx, candidates, opts) + if len(choices) == 0 { + return orderedUpdatePlans(candidates, plansByKey), nil + } + if opts.Check { + for _, choice := range choices { + plansByKey[choice.Candidate.key()] = checkUpdatePlan(choice) + } + return orderedUpdatePlans(candidates, plansByKey), nil + } + + selectCandidates := opts.SelectCandidates + if selectCandidates == nil { + selectCandidates = promptUpdateCandidates + } + selected, ok := selectCandidates(choices) + if !ok { + for _, choice := range choices { + plansByKey[choice.Candidate.key()] = skippedUpdatePlan(choice.Candidate, "selection cancelled") + } + return orderedUpdatePlans(candidates, plansByKey), nil + } + selectedKeys := map[string]UpdateChoice{} + for _, choice := range selected { + selectedKeys[choice.Candidate.key()] = choice + } + for _, choice := range choices { + if _, ok := selectedKeys[choice.Candidate.key()]; !ok { + plansByKey[choice.Candidate.key()] = skippedUpdatePlan(choice.Candidate, "not selected") + } + } + + selectVersion := opts.SelectVersion + if selectVersion == nil { + selectVersion = promptUpdateVersion + } + for _, choice := range sortSelectedUpdateChoicesByFile(selected) { + version, ok := selectVersion(choice) + if !ok || strings.TrimSpace(version) == "" { + plansByKey[choice.Candidate.key()] = skippedUpdatePlan(choice.Candidate, "no version selected") + continue + } + plansByKey[choice.Candidate.key()] = applyDependencyUpdate(ctx, choice.Candidate, version, opts) + } + return orderedUpdatePlans(candidates, plansByKey), nil +} + +func DiscoverUpdateCandidates(path string, managers []Manager) ([]UpdateCandidate, error) { + defaultManagers := len(managers) == 0 + managers, err := updateManagers(managers) + if err != nil { + return nil, err + } + absPath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + var out []UpdateCandidate + var packageErr error + if packageManagers := packageUpdateManagers(managers); len(packageManagers) > 0 { + projects, _, err := Discover(absPath, packageManagers) + if err != nil { + packageErr = err + } else { + for _, project := range projects { + switch project.Manager { + case ManagerGo: + candidates, err := discoverGoUpdateCandidates(project) + if err != nil { + return nil, err + } + out = append(out, candidates...) + case ManagerNPM, ManagerPNPM: + candidates, err := discoverPackageJSONUpdateCandidates(project) + if err != nil { + return nil, err + } + out = append(out, candidates...) + } + } + } + } + if imageManagers := imageUpdateManagers(managers); len(imageManagers) > 0 { + candidates, err := discoverImageUpdateCandidates(absPath, imageManagers) + if err != nil { + if !defaultManagers || len(out) == 0 { + if packageErr != nil && defaultManagers { + return nil, packageErr + } + return nil, err + } + } else { + out = append(out, candidates...) + } + } + if len(out) == 0 && packageErr != nil { + return nil, packageErr + } + if len(out) == 0 { + return nil, fmt.Errorf("no supported dependency manifests or image/chart targets found under %s", absPath) + } + relativizeUpdateCandidateFiles(out) + sort.SliceStable(out, func(i, j int) bool { + return out[i].less(out[j]) + }) + return out, nil +} + +func relativizeUpdateCandidateFiles(candidates []UpdateCandidate) { + for i := range candidates { + candidates[i].File = cwdRelativePath(candidates[i].File) + } +} + +func cwdRelativePath(path string) string { + if strings.TrimSpace(path) == "" { + return path + } + absPath, err := filepath.Abs(path) + if err != nil { + return filepath.ToSlash(path) + } + cwd, err := os.Getwd() + if err != nil { + return filepath.ToSlash(absPath) + } + rel, err := filepath.Rel(cwd, absPath) + if err != nil { + return filepath.ToSlash(absPath) + } + if rel == "." { + return filepath.ToSlash(filepath.Base(absPath)) + } + return filepath.ToSlash(rel) +} + +func discoverGoUpdateCandidates(project Project) ([]UpdateCandidate, error) { + data, err := os.ReadFile(filepath.Join(project.Dir, "go.mod")) + if err != nil { + return nil, err + } + file, err := modfile.Parse("go.mod", data, nil) + if err != nil { + return nil, err + } + var out []UpdateCandidate + for _, req := range file.Require { + if req.Indirect { + continue + } + if rep := goReplaceFor(file, req.Mod.Path, req.Mod.Version); rep != nil && isLocalRef(rep.New.Path) { + continue + } + out = append(out, UpdateCandidate{ + Manager: ManagerGo, + Name: req.Mod.Path, + Current: req.Mod.Version, + Scope: "require", + File: filepath.Join(project.Dir, "go.mod"), + Dir: project.Dir, + }) + } + return out, nil +} + +func discoverPackageJSONUpdateCandidates(project Project) ([]UpdateCandidate, error) { + path := filepath.Join(project.Dir, "package.json") + data, err := os.ReadFile(path) + if err != nil { + return nil, nil + } + var pkg packageJSON + if err := json.Unmarshal(data, &pkg); err != nil { + return nil, err + } + sections := []struct { + scope string + deps map[string]string + }{ + {"dependencies", pkg.Dependencies}, + {"devDependencies", pkg.DevDependencies}, + {"optionalDependencies", pkg.OptionalDependencies}, + {"peerDependencies", pkg.PeerDependencies}, + } + var out []UpdateCandidate + for _, section := range sections { + keys := make([]string, 0, len(section.deps)) + for name := range section.deps { + keys = append(keys, name) + } + sort.Strings(keys) + for _, name := range keys { + current := strings.TrimSpace(section.deps[name]) + if isLocalUpdateSpec(current) { + continue + } + out = append(out, UpdateCandidate{ + Manager: project.Manager, + Name: name, + Current: current, + Scope: section.scope, + File: path, + Dir: project.Dir, + }) + } + } + return out, nil +} + +func filterUpdateCandidates(candidates []UpdateCandidate, patterns []string) []UpdateCandidate { + out := make([]UpdateCandidate, 0, len(candidates)) + for _, candidate := range candidates { + if candidate.matches(patterns) { + out = append(out, candidate) + } + } + return out +} + +func resolveUpdateChoices(ctx context.Context, candidates []UpdateCandidate, opts UpdateOptions) ([]UpdateChoice, map[string]UpdatePlan) { + type result struct { + versions []string + latestStable string + latestPrerelease string + err error + } + results := make([]result, len(candidates)) + group := task.StartGroup[int]("Resolving dependency versions", task.WithConcurrency(updateResolveConcurrency)) + for i, candidate := range candidates { + idx, c := i, candidate + group.Add(updateTaskName(c), func(_ flanksourceContext.Context, tk *task.Task) (int, error) { + tk.Infof("looking up published versions") + versions, latestStable, latestPrerelease, err := resolveCandidateVersions(ctx, opts, c) + results[idx] = result{versions: versions, latestStable: latestStable, latestPrerelease: latestPrerelease, err: err} + if err != nil { + tk.Warnf("%s", err.Error()) + tk.Warning() + } else if len(versions) == 0 { + tk.Infof("already up to date") + tk.Success() + } else { + tk.Success() + } + return idx, nil + }) + } + _, _ = group.GetResults() + + plansByKey := map[string]UpdatePlan{} + choices := make([]UpdateChoice, 0, len(candidates)) + for i, candidate := range candidates { + result := results[i] + if result.err != nil { + plansByKey[candidate.key()] = skippedUpdatePlan(candidate, result.err.Error()) + continue + } + if len(result.versions) == 0 { + continue + } + choices = append(choices, UpdateChoice{ + Candidate: candidate, + Versions: result.versions, + LatestStable: result.latestStable, + LatestPrerelease: result.latestPrerelease, + }) + } + return choices, plansByKey +} + +func resolveCandidateVersions(ctx context.Context, opts UpdateOptions, candidate UpdateCandidate) ([]string, string, string, error) { + var ( + versions []string + err error + ) + switch candidate.Manager { + case ManagerImage, ManagerHelm: + versions, _, _, err = availableImageTargetVersions(ctx, opts.ImageResolver, candidate) + default: + versions, err = AvailableDependencyVersions(ctx, opts.Runner, candidate) + } + if err != nil { + return nil, "", "", err + } + versions = updateableVersions(candidate.Current, versions) + return versions, latestStableVersion(versions), latestPrereleaseVersion(versions), nil +} + +func AvailableDependencyVersions(ctx context.Context, runner CommandRunner, candidate UpdateCandidate) ([]string, error) { + if runner == nil { + runner = ExecRunner{} + } + var ( + result CommandResult + err error + ) + switch candidate.Manager { + case ManagerGo: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "go", + Args: []string{"list", "-m", "-versions", "-json", candidate.Name}, + Env: []string{"GOFLAGS=-mod=readonly"}, + }) + case ManagerNPM: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "npm", + Args: []string{"view", candidate.Name, "versions", "--json"}, + }) + case ManagerPNPM: + result, err = runner.Run(ctx, Command{ + Dir: candidate.Dir, + Name: "pnpm", + Args: []string{"view", candidate.Name, "versions", "--json"}, + }) + default: + return nil, fmt.Errorf("dependency version lookup does not support manager %q", candidate.Manager) + } + if err != nil { + if result.Stderr != "" { + return nil, fmt.Errorf("%s: %w", strings.TrimSpace(result.Stderr), err) + } + return nil, err + } + versions, err := parseAvailableVersions(candidate.Manager, result.Stdout) + if err != nil { + return nil, err + } + return sortDependencyVersions(versions), nil +} + +func parseAvailableVersions(manager Manager, stdout string) ([]string, error) { + stdout = strings.TrimSpace(stdout) + if stdout == "" { + return nil, nil + } + if manager == ManagerGo { + var payload struct { + Versions []string `json:"Versions"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + return nil, err + } + return payload.Versions, nil + } + var list []string + if err := json.Unmarshal([]byte(stdout), &list); err == nil { + return list, nil + } + var single string + if err := json.Unmarshal([]byte(stdout), &single); err == nil && single != "" { + return []string{single}, nil + } + return nil, fmt.Errorf("expected JSON version array") +} + +func applyDependencyUpdate(ctx context.Context, candidate UpdateCandidate, version string, opts UpdateOptions) UpdatePlan { + if candidate.Manager == ManagerImage || candidate.Manager == ManagerHelm { + return applyImageTargetUpdate(ctx, candidate, version, opts) + } + plan := planFromCandidate(candidate) + plan.NewVersion = version + if selectedVersionIsCurrent(candidate.Current, version) { + plan.Skipped = "already at selected version" + return plan + } + cmd, err := updateCommand(candidate, version) + if err != nil { + plan.Skipped = err.Error() + return plan + } + plan.Command = append([]string{cmd.Name}, cmd.Args...) + if opts.DryRun { + plan.DryRun = true + return plan + } + runner := opts.Runner + if runner == nil { + runner = ExecRunner{} + } + if _, err := runner.Run(ctx, cmd); err != nil { + plan.Skipped = err.Error() + return plan + } + plan.Written = true + return plan +} + +func updateCommand(candidate UpdateCandidate, version string) (Command, error) { + target := candidate.Name + "@" + version + switch candidate.Manager { + case ManagerGo: + return Command{Dir: candidate.Dir, Name: "go", Args: []string{"get", target}}, nil + case ManagerNPM: + args := []string{"install", "--package-lock-only", "--ignore-scripts"} + if flag := packageSaveFlag(candidate.Scope); flag != "" { + args = append(args, flag) + } + args = append(args, target) + return Command{Dir: candidate.Dir, Name: "npm", Args: args}, nil + case ManagerPNPM: + args := []string{"add", "--lockfile-only", "--ignore-scripts"} + if flag := packageSaveFlag(candidate.Scope); flag != "" { + args = append(args, flag) + } + args = append(args, target) + return Command{Dir: candidate.Dir, Name: "pnpm", Args: args}, nil + default: + return Command{}, fmt.Errorf("package-manager updates do not support manager %q", candidate.Manager) + } +} + +func packageSaveFlag(scope string) string { + switch scope { + case "dependencies": + return "--save-prod" + case "devDependencies": + return "--save-dev" + case "optionalDependencies": + return "--save-optional" + case "peerDependencies": + return "--save-peer" + default: + return "" + } +} + +func promptUpdateCandidates(choices []UpdateChoice) ([]UpdateChoice, bool) { + return runUpdateChoiceTreePicker(choices) +} + +func promptUpdateVersion(choice UpdateChoice) (string, bool) { + candidate := choice.Candidate + title := fmt.Sprintf("Select version for %s in %s (current %s)", candidate.Name, candidate.File, candidate.Current) + return clicky.PromptSelect(choice.Versions, clicky.PromptSelectOptions[string]{ + Title: title, + PageSize: 12, + Render: func(version string) api.Textable { + text := clicky.Text(version, "font-mono") + var tags []string + if selectedVersionIsCurrent(candidate.Current, version) { + tags = append(tags, "current") + } + if version == choice.LatestStable { + tags = append(tags, "latest stable") + } + if version == choice.LatestPrerelease { + tags = append(tags, "latest pre-release") + } + if isPrerelease(version) { + tags = append(tags, "pre-release") + } + if len(tags) > 0 { + text = text.Space().Append("("+strings.Join(tags, ", ")+")", "text-muted") + } + return text + }, + }) +} + +type updateChoiceFileGroup struct { + File string + Choices []UpdateChoice +} + +func groupUpdateChoicesByFile(choices []UpdateChoice) []updateChoiceFileGroup { + byFile := map[string][]UpdateChoice{} + for _, choice := range choices { + file := choice.Candidate.File + byFile[file] = append(byFile[file], choice) + } + files := make([]string, 0, len(byFile)) + for file := range byFile { + files = append(files, file) + } + sort.Strings(files) + groups := make([]updateChoiceFileGroup, 0, len(files)) + for _, file := range files { + groupChoices := append([]UpdateChoice(nil), byFile[file]...) + sort.SliceStable(groupChoices, func(i, j int) bool { + return groupChoices[i].Candidate.less(groupChoices[j].Candidate) + }) + groups = append(groups, updateChoiceFileGroup{File: file, Choices: groupChoices}) + } + return groups +} + +func sortSelectedUpdateChoicesByFile(choices []UpdateChoice) []UpdateChoice { + var out []UpdateChoice + for _, group := range groupUpdateChoicesByFile(choices) { + out = append(out, group.Choices...) + } + return out +} + +func renderUpdateChoice(choice UpdateChoice) api.Textable { + c := choice.Candidate + text := clicky.Text(fmt.Sprintf("[%s] %s", c.Manager, c.Name), managerStyle(c.Manager)). + Append("@"+c.Current, "font-mono text-muted") + if choice.LatestStable != "" { + text = text.Space().Append("latest "+choice.LatestStable, "text-green-600") + } + if choice.LatestPrerelease != "" { + text = text.Space().Append("pre "+choice.LatestPrerelease, "text-yellow-600") + } + if c.Scope != "" { + text = text.Space().Append(c.Scope, "text-muted") + } + return text +} + +func (p UpdatePlan) Pretty() api.Text { + t := clicky.Text(fmt.Sprintf("[%s] %s", p.Manager, p.Name), managerStyle(p.Manager)) + if p.OldVersion != "" || p.NewVersion != "" { + t = t.Space().Append(p.OldVersion, "font-mono text-muted") + if p.NewVersion != "" { + t = t.Append(" -> ", "text-muted").Append(p.NewVersion, "font-mono text-green-600") + } + } + switch { + case p.Skipped != "": + t = t.Space().Append("skipped: "+p.Skipped, "text-muted") + case p.Checked: + t = t.Space().Append("update available", "text-green-600") + case p.DryRun: + t = t.Space().Append("(dry-run)", "text-yellow-600") + case p.Written: + t = t.Space().Append("written", "text-green-600") + } + return t +} + +func (UpdatePlan) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("manager").Label("Manager").Build(), + api.Column("dependency").Label("Dependency").Build(), + api.Column("file").Label("File").Build(), + api.Column("scope").Label("Scope").Build(), + api.Column("change").Label("Change").Build(), + api.Column("status").Label("Status").Build(), + } +} + +func (p UpdatePlan) Row() map[string]any { + row := map[string]any{ + "manager": clicky.Text(string(p.Manager), managerStyle(p.Manager)), + "dependency": clicky.Text(p.Name, "font-bold text-cyan-600"), + "file": clicky.Text(p.File, "font-mono"), + "scope": clicky.Text(p.Scope, "text-muted"), + "change": updateChangeText(p.OldVersion, p.NewVersion), + } + switch { + case p.Skipped != "": + row["status"] = clicky.Text(p.Skipped, "text-muted") + case p.Checked: + row["status"] = clicky.Text("update available", "text-green-600") + case p.DryRun: + row["status"] = clicky.Text("dry-run", "text-yellow-600") + case p.Written: + row["status"] = clicky.Text("written", "text-green-600") + default: + row["status"] = clicky.Text("") + } + return row +} + +func updateChangeText(oldVersion, newVersion string) api.Text { + text := clicky.Text(oldVersion, "font-mono text-muted") + if newVersion != "" { + text = text.Append(" -> ", "text-muted").Append(newVersion, "font-mono text-green-600") + } + return text +} + +func planFromCandidate(candidate UpdateCandidate) UpdatePlan { + return UpdatePlan{ + Manager: candidate.Manager, + Name: candidate.Name, + File: candidate.File, + Scope: candidate.Scope, + OldVersion: candidate.Current, + } +} + +func skippedUpdatePlan(candidate UpdateCandidate, reason string) UpdatePlan { + plan := planFromCandidate(candidate) + plan.Skipped = reason + return plan +} + +func checkUpdatePlan(choice UpdateChoice) UpdatePlan { + plan := planFromCandidate(choice.Candidate) + plan.NewVersion = checkUpdateVersion(choice) + plan.Checked = true + return plan +} + +func checkUpdateVersion(choice UpdateChoice) string { + if choice.LatestStable != "" { + return choice.LatestStable + } + if choice.LatestPrerelease != "" { + return choice.LatestPrerelease + } + if len(choice.Versions) > 0 { + return choice.Versions[0] + } + return "" +} + +func orderedUpdatePlans(candidates []UpdateCandidate, plansByKey map[string]UpdatePlan) []UpdatePlan { + plans := make([]UpdatePlan, 0, len(plansByKey)) + for _, candidate := range candidates { + if plan, ok := plansByKey[candidate.key()]; ok { + plans = append(plans, plan) + } + } + return plans +} + +func updateManagers(managers []Manager) ([]Manager, error) { + if len(managers) == 0 { + return []Manager{ManagerGo, ManagerNPM, ManagerPNPM, ManagerImage, ManagerHelm}, nil + } + out := make([]Manager, 0, len(managers)) + var unsupported []string + for _, manager := range managers { + if !supportedUpdateManagers[manager] { + unsupported = append(unsupported, string(manager)) + continue + } + out = append(out, manager) + } + if len(unsupported) > 0 { + sort.Strings(unsupported) + return nil, fmt.Errorf("dependency updates currently support go, npm, pnpm, image, and helm; unsupported manager(s): %s", strings.Join(unsupported, ", ")) + } + return out, nil +} + +func packageUpdateManagers(managers []Manager) []Manager { + var out []Manager + for _, manager := range managers { + switch manager { + case ManagerGo, ManagerNPM, ManagerPNPM: + out = append(out, manager) + } + } + return out +} + +func imageUpdateManagers(managers []Manager) []Manager { + var out []Manager + for _, manager := range managers { + switch manager { + case ManagerImage, ManagerHelm: + out = append(out, manager) + } + } + return out +} + +func splitUpdatePatterns(values []string) []string { + var out []string + for _, value := range values { + for _, part := range strings.Split(value, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + } + return out +} + +func (c UpdateCandidate) matches(patterns []string) bool { + identityPatterns, pathPatterns := splitExplicitPathPatterns(patterns) + hasPositivePattern := identityPatterns.hasPositive || pathPatterns.hasPositive + matchedPositive := false + + for _, value := range c.matchValues() { + if value == "" { + continue + } + ok, negated := collections.MatchItem(value, identityPatterns.values...) + if negated { + return false + } + if ok && identityPatterns.hasPositive { + matchedPositive = true + } + } + if len(pathPatterns.values) > 0 { + ok, negated := collections.MatchItem(c.File, pathPatterns.values...) + if negated { + return false + } + if ok && pathPatterns.hasPositive { + matchedPositive = true + } + } + if hasPositivePattern { + return matchedPositive + } + return true +} + +func (c UpdateCandidate) matchValues() []string { + return []string{ + c.Name, + string(c.Manager), + fmt.Sprintf("%s:%s", c.Manager, c.Name), + fmt.Sprintf("%s:%s@%s", c.Manager, c.Name, c.Current), + c.Scope, + c.Current, + } +} + +type updatePatternSet struct { + values []string + hasPositive bool +} + +func splitExplicitPathPatterns(patterns []string) (identity updatePatternSet, path updatePatternSet) { + for _, pattern := range patterns { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + continue + } + negated := strings.HasPrefix(pattern, "!") + body := strings.TrimPrefix(pattern, "!") + field, value, explicit := strings.Cut(body, ":") + if explicit && (field == "path" || field == "file") { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if strings.HasPrefix(value, "!") { + negated = true + value = strings.TrimPrefix(value, "!") + } + if negated { + value = "!" + value + } else { + path.hasPositive = true + } + path.values = append(path.values, value) + continue + } + if !negated { + identity.hasPositive = true + } + identity.values = append(identity.values, pattern) + } + return identity, path +} + +func (c UpdateCandidate) key() string { + return strings.Join([]string{string(c.Manager), c.Dir, c.File, c.Scope, c.Name}, "\x00") +} + +func (c UpdateCandidate) less(other UpdateCandidate) bool { + if c.Manager != other.Manager { + return c.Manager < other.Manager + } + if c.File != other.File { + return c.File < other.File + } + if c.Scope != other.Scope { + return c.Scope < other.Scope + } + return c.Name < other.Name +} + +func updateTaskName(candidate UpdateCandidate) string { + return fmt.Sprintf("%s %s", candidate.Manager, candidate.Name) +} + +func isLocalUpdateSpec(spec string) bool { + spec = strings.TrimSpace(spec) + if spec == "" { + return true + } + if strings.HasPrefix(spec, "workspace:") { + return true + } + return isLocalRef(spec) +} + +func sortDependencyVersions(versions []string) []string { + type parsed struct { + orig string + ver *semver.Version + } + seen := map[string]bool{} + var parsedVersions []parsed + for _, version := range versions { + version = strings.TrimSpace(version) + if version == "" || seen[version] { + continue + } + seen[version] = true + sv, err := semver.NewVersion(version) + if err != nil { + continue + } + parsedVersions = append(parsedVersions, parsed{orig: version, ver: sv}) + } + sort.Slice(parsedVersions, func(i, j int) bool { + return parsedVersions[i].ver.GreaterThan(parsedVersions[j].ver) + }) + out := make([]string, len(parsedVersions)) + for i, item := range parsedVersions { + out[i] = item.orig + } + return out +} + +func latestStableVersion(versions []string) string { + for _, version := range versions { + if !isPrerelease(version) { + return version + } + } + return "" +} + +func latestPrereleaseVersion(versions []string) string { + for _, version := range versions { + if isPrerelease(version) { + return version + } + } + return "" +} + +func isPrerelease(version string) bool { + sv, err := semver.NewVersion(version) + return err == nil && strings.TrimSpace(sv.Prerelease()) != "" +} + +func updateableVersions(current string, versions []string) []string { + current = normalizeCurrentVersion(current) + currentSemver, currentErr := semver.NewVersion(current) + out := make([]string, 0, len(versions)) + for _, version := range versions { + if version == "" || version == current { + continue + } + versionSemver, versionErr := semver.NewVersion(version) + if currentErr != nil || versionErr != nil { + out = append(out, version) + continue + } + if versionSemver.GreaterThan(currentSemver) { + out = append(out, version) + } + } + return out +} + +func selectedVersionIsCurrent(current, selected string) bool { + return normalizeCurrentVersion(current) == selected +} + +func normalizeCurrentVersion(current string) string { + current = strings.TrimSpace(current) + current = strings.TrimPrefix(current, "npm:") + current = strings.TrimLeft(current, "^~<>= ") + if idx := strings.IndexAny(current, " |,"); idx >= 0 { + current = current[:idx] + } + return strings.TrimSpace(current) +} diff --git a/deps/update_image.go b/deps/update_image.go new file mode 100644 index 0000000..834ef16 --- /dev/null +++ b/deps/update_image.go @@ -0,0 +1,265 @@ +package deps + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/flanksource/repomap" + "github.com/flanksource/repomap/imageupdate" + "github.com/flanksource/repomap/kubernetes" +) + +func discoverImageUpdateCandidates(root string, managers []Manager) ([]UpdateCandidate, error) { + selected := managerSet(managers) + conf, err := repomap.GetConf(root) + if err != nil { + return nil, err + } + targets, sourceIndex, err := discoverImageTargets(conf, root) + if err != nil { + return nil, err + } + + var out []UpdateCandidate + for _, target := range targets { + manager := managerForUpdateTarget(target) + if manager == "" || !selected[manager] { + continue + } + if target.Kind == imageupdate.TargetChart { + if err := sourceIndex.Resolve(&target); err != nil { + return nil, err + } + } + targetCopy := target + out = append(out, UpdateCandidate{ + Manager: manager, + Name: updateTargetName(target), + Current: updateTargetCurrentVersion(target), + Scope: updateTargetScope(target), + File: filepath.Join(conf.RepoPath(), filepath.FromSlash(target.File)), + Dir: conf.RepoPath(), + Target: &targetCopy, + }) + } + return out, nil +} + +func discoverImageTargets(conf *repomap.ArchConf, scanPath string) ([]imageupdate.UpdateTarget, *imageupdate.SourceIndex, error) { + files, err := gitTrackedFiles(conf.RepoPath()) + if err != nil { + return nil, nil, fmt.Errorf("failed to list files: %w", err) + } + + var prefix string + if rel, err := filepath.Rel(conf.RepoPath(), scanPath); err == nil && rel != "." { + prefix = filepath.ToSlash(rel) + "/" + } + + contents := map[string]string{} + for _, file := range files { + if !kubernetes.IsYaml(file) { + continue + } + content, err := conf.ReadFileWithFallback(file, "") + if err != nil || content == "" { + continue + } + contents[file] = content + } + + tree := imageupdate.BuildKustomizeTree(contents) + sourceIndex := imageupdate.NewSourceIndex(tree) + + keys := make([]string, 0, len(contents)) + for file := range contents { + keys = append(keys, file) + } + sort.Strings(keys) + + var targets []imageupdate.UpdateTarget + for _, file := range keys { + content := contents[file] + _ = sourceIndex.IndexHelmRepositories(file, content) + if prefix != "" && !strings.HasPrefix(file, prefix) { + continue + } + fileTargets, err := imageupdate.ExtractTargets(file, content) + if err != nil { + continue + } + targets = append(targets, fileTargets...) + } + sort.SliceStable(targets, func(i, j int) bool { + if targets[i].File != targets[j].File { + return targets[i].File < targets[j].File + } + if targets[i].FieldLine != targets[j].FieldLine { + return targets[i].FieldLine < targets[j].FieldLine + } + return updateTargetName(targets[i]) < updateTargetName(targets[j]) + }) + return targets, sourceIndex, nil +} + +func gitTrackedFiles(repoPath string) ([]string, error) { + cmd := exec.Command("git", "-C", repoPath, "ls-files", "-z") + out, err := cmd.Output() + if err != nil { + return nil, err + } + var files []string + for _, file := range strings.Split(string(out), "\x00") { + file = strings.TrimSpace(file) + if file != "" { + files = append(files, filepath.ToSlash(file)) + } + } + sort.Strings(files) + return files, nil +} + +func managerForUpdateTarget(target imageupdate.UpdateTarget) Manager { + switch target.Kind { + case imageupdate.TargetImage: + return ManagerImage + case imageupdate.TargetChart: + return ManagerHelm + default: + return "" + } +} + +func updateTargetName(target imageupdate.UpdateTarget) string { + switch target.Kind { + case imageupdate.TargetImage: + if target.Image != nil { + return target.Image.GetFullNameWithoutTag() + } + return stripImageVersion(target.CurrentValue) + case imageupdate.TargetChart: + return target.ChartName + default: + return target.CurrentValue + } +} + +func updateTargetCurrentVersion(target imageupdate.UpdateTarget) string { + if target.Kind == imageupdate.TargetImage { + return imageVersionOnly(target.CurrentValue) + } + return target.CurrentValue +} + +func updateTargetScope(target imageupdate.UpdateTarget) string { + ref := target.Ref.Kind + if target.Ref.Namespace != "" { + ref += "/" + target.Ref.Namespace + } + if target.Ref.Name != "" { + ref += "/" + target.Ref.Name + } + switch target.Kind { + case imageupdate.TargetImage: + if target.ContainerName != "" { + return ref + " container/" + target.ContainerName + } + case imageupdate.TargetChart: + return ref + " chart" + } + return ref +} + +func stripImageVersion(value string) string { + if at := strings.Index(value, "@"); at >= 0 { + value = value[:at] + } + if i := imageTagSeparator(value); i >= 0 { + return value[:i] + } + return value +} + +func imageVersionOnly(value string) string { + if i := imageTagSeparator(value); i >= 0 { + version := value[i+1:] + if at := strings.Index(version, "@"); at >= 0 { + version = version[:at] + } + return version + } + return value +} + +func imageTagSeparator(value string) int { + colon := strings.LastIndex(value, ":") + if colon < 0 { + return -1 + } + if slash := strings.LastIndex(value, "/"); slash > colon { + return -1 + } + return colon +} + +func availableImageTargetVersions(ctx context.Context, resolver ImageVersionResolver, candidate UpdateCandidate) ([]string, string, string, error) { + if candidate.Target == nil { + return nil, "", "", fmt.Errorf("%s has no image or Helm target metadata", candidate.Name) + } + if resolver == nil { + resolver = imageupdate.NewResolver() + } + target := *candidate.Target + latest, err := resolver.ResolveLatestVersions(ctx, target) + if err != nil { + return nil, "", "", err + } + versions, err := resolver.Available(ctx, target) + if err != nil { + return nil, "", "", err + } + return versions, latest.Stable, latest.Prerelease, nil +} + +func applyImageTargetUpdate(ctx context.Context, candidate UpdateCandidate, version string, opts UpdateOptions) UpdatePlan { + plan := planFromCandidate(candidate) + plan.NewVersion = version + plan.DryRun = opts.DryRun + if selectedVersionIsCurrent(candidate.Current, version) { + plan.Skipped = "already at selected version" + return plan + } + if candidate.Target == nil { + plan.Skipped = "missing image or Helm target metadata" + return plan + } + resolver := opts.ImageResolver + if resolver == nil { + resolver = imageupdate.NewResolver() + } + target := *candidate.Target + newValue := version + if target.Kind == imageupdate.TargetImage { + resolved, err := resolver.NewImageValue(ctx, target, version) + if err != nil { + plan.Skipped = err.Error() + return plan + } + newValue = resolved + } + if newValue == target.CurrentValue { + plan.Skipped = "already at selected version" + return plan + } + absFile := filepath.Join(candidate.Dir, filepath.FromSlash(target.File)) + if _, err := imageupdate.ApplyEdit(absFile, target, newValue, opts.DryRun); err != nil { + plan.Skipped = err.Error() + return plan + } + plan.Written = !opts.DryRun + return plan +} diff --git a/deps/update_test.go b/deps/update_test.go new file mode 100644 index 0000000..d5a30ba --- /dev/null +++ b/deps/update_test.go @@ -0,0 +1,507 @@ +package deps + +import ( + "context" + "errors" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/flanksource/repomap/imageupdate" +) + +func TestDiscoverUpdateCandidates_DirectOnly(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require ( + github.com/acme/direct v1.2.3 + github.com/acme/indirect v0.1.0 // indirect + github.com/acme/local v0.2.0 +) + +replace github.com/acme/local => ../local +`) + writeFile(t, filepath.Join(dir, "web", "package.json"), `{ + "name": "web", + "dependencies": {"left-pad": "^1.3.0", "local-tool": "file:../tool"}, + "devDependencies": {"typescript": "~5.5.0"} +}`) + writeFile(t, filepath.Join(dir, "web", "package-lock.json"), `{"lockfileVersion": 3}`) + writeFile(t, filepath.Join(dir, "pnpm-app", "package.json"), `{ + "name": "pnpm-app", + "dependencies": {"@scope/pkg": "2.0.0"}, + "devDependencies": {"workspace-tool": "workspace:*"} +}`) + writeFile(t, filepath.Join(dir, "pnpm-app", "pnpm-lock.yaml"), `lockfileVersion: '9.0'`) + + got, err := DiscoverUpdateCandidates(dir, nil) + if err != nil { + t.Fatal(err) + } + names := updateCandidateLabels(got) + want := []string{ + "go:github.com/acme/direct:require:v1.2.3", + "npm:left-pad:dependencies:^1.3.0", + "npm:typescript:devDependencies:~5.5.0", + "pnpm:@scope/pkg:dependencies:2.0.0", + } + if strings.Join(names, "\n") != strings.Join(want, "\n") { + t.Fatalf("candidates:\n%s\nwant:\n%s", strings.Join(names, "\n"), strings.Join(want, "\n")) + } +} + +func TestDiscoverUpdateCandidates_FilePathsRelativeToCWD(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require github.com/acme/direct v1.2.3 +`) + writeFile(t, filepath.Join(dir, "web", "package.json"), `{ + "name": "web", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, "web", "package-lock.json"), `{"lockfileVersion": 3}`) + + got, err := DiscoverUpdateCandidates(".", nil) + if err != nil { + t.Fatal(err) + } + files := updateCandidateFiles(got) + want := []string{"go.mod", "web/package.json"} + if strings.Join(files, "\n") != strings.Join(want, "\n") { + t.Fatalf("candidate files:\n%s\nwant:\n%s", strings.Join(files, "\n"), strings.Join(want, "\n")) + } + for _, file := range files { + if filepath.IsAbs(file) { + t.Fatalf("candidate file should be relative to cwd, got %q", file) + } + } +} + +func TestUpdateCandidateMatchesMatchItemExpression(t *testing.T) { + candidates := []UpdateCandidate{ + {Manager: ManagerGo, Name: "github.com/acme/lib", Current: "v1.0.0", File: "/work/flanksource/app/go.mod"}, + {Manager: ManagerGo, Name: "github.com/flanksource/lib", Current: "v1.0.0", File: "/work/other/go.mod"}, + {Manager: ManagerNPM, Name: "@scope/pkg", Current: "^2.0.0", File: "/work/flanksource/app/package.json"}, + } + got := filterUpdateCandidates(candidates, []string{"go:github.com/acme/*"}) + if len(got) != 1 || got[0].Name != "github.com/acme/lib" { + t.Fatalf("unexpected go match: %#v", got) + } + got = filterUpdateCandidates(candidates, []string{"*", "!npm:*"}) + if len(got) != 2 || got[0].Manager != ManagerGo || got[1].Manager != ManagerGo { + t.Fatalf("negated manager match failed: %#v", got) + } + got = filterUpdateCandidates(candidates, []string{"*flanksource*"}) + if len(got) != 1 || got[0].Name != "github.com/flanksource/lib" { + t.Fatalf("unqualified match should not match file paths: %#v", got) + } + got = filterUpdateCandidates(candidates, []string{"path:*flanksource*"}) + if len(got) != 2 || got[0].Name != "github.com/acme/lib" || got[1].Name != "@scope/pkg" { + t.Fatalf("explicit path match failed: %#v", got) + } + got = filterUpdateCandidates(candidates, []string{"file:*package.json"}) + if len(got) != 1 || got[0].Name != "@scope/pkg" { + t.Fatalf("explicit file match failed: %#v", got) + } +} + +func TestAvailableDependencyVersionsParsesAndSorts(t *testing.T) { + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "go list -m -versions -json github.com/acme/lib": { + Stdout: `{"Path":"github.com/acme/lib","Versions":["v1.0.0","v1.2.0-beta.1","v1.1.0","not-semver"]}`, + }, + "npm view left-pad versions --json": { + Stdout: `["1.0.0","1.3.0","1.3.0-beta.1","latest"]`, + }, + }, + } + goVersions, err := AvailableDependencyVersions(context.Background(), runner, UpdateCandidate{ + Manager: ManagerGo, + Name: "github.com/acme/lib", + }) + if err != nil { + t.Fatal(err) + } + if strings.Join(goVersions, ",") != "v1.2.0-beta.1,v1.1.0,v1.0.0" { + t.Fatalf("go versions = %#v", goVersions) + } + npmVersions, err := AvailableDependencyVersions(context.Background(), runner, UpdateCandidate{ + Manager: ManagerNPM, + Name: "left-pad", + }) + if err != nil { + t.Fatal(err) + } + if strings.Join(npmVersions, ",") != "1.3.0,1.3.0-beta.1,1.0.0" { + t.Fatalf("npm versions = %#v", npmVersions) + } +} + +func TestUpdateDryRunBuildsPackageManagerCommand(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "package.json"), `{ + "name": "web", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, "package-lock.json"), `{"lockfileVersion": 3}`) + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "npm view left-pad versions --json": {Stdout: `["1.3.0","1.4.0"]`}, + }, + } + plans, err := Update(context.Background(), dir, UpdateOptions{ + Managers: []Manager{ManagerNPM}, + Expression: []string{"left-pad"}, + DryRun: true, + Runner: runner, + SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { + return choices, true + }, + SelectVersion: func(choice UpdateChoice) (string, bool) { + return "1.4.0", true + }, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 { + t.Fatalf("plans = %d, want 1: %#v", len(plans), plans) + } + plan := plans[0] + if !plan.DryRun || plan.Written || plan.Skipped != "" { + t.Fatalf("unexpected plan status: %#v", plan) + } + wantCommand := "npm install --package-lock-only --ignore-scripts --save-prod left-pad@1.4.0" + if strings.Join(plan.Command, " ") != wantCommand { + t.Fatalf("command = %q, want %q", strings.Join(plan.Command, " "), wantCommand) + } + if len(runner.commands) != 1 || runner.commands[0].Name != "npm" || runner.commands[0].Args[0] != "view" { + t.Fatalf("dry-run should only run version lookup, got %#v", runner.commands) + } +} + +func TestUpdateSkipsCandidatesWithoutChangesBeforePrompt(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "package.json"), `{ + "name": "web", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, "package-lock.json"), `{"lockfileVersion": 3}`) + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "npm view left-pad versions --json": {Stdout: `["1.3.0"]`}, + }, + } + plans, err := Update(context.Background(), dir, UpdateOptions{ + Managers: []Manager{ManagerNPM}, + Expression: []string{"left-pad"}, + DryRun: true, + Runner: runner, + SelectCandidates: func([]UpdateChoice) ([]UpdateChoice, bool) { + t.Fatal("no-change candidates must not be shown for selection") + return nil, false + }, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 0 { + t.Fatalf("plans = %#v, want no rows for already-current dependencies", plans) + } +} + +func TestUpdateCheckListsUpdatesWithoutPrompting(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "package.json"), `{ + "name": "web", + "dependencies": {"left-pad": "^1.3.0"} +}`) + writeFile(t, filepath.Join(dir, "package-lock.json"), `{"lockfileVersion": 3}`) + runner := &updateFakeRunner{ + responses: map[string]CommandResult{ + "npm view left-pad versions --json": {Stdout: `["1.4.0","1.3.0"]`}, + }, + } + plans, err := Update(context.Background(), dir, UpdateOptions{ + Managers: []Manager{ManagerNPM}, + Expression: []string{"left-pad"}, + Check: true, + Runner: runner, + SelectCandidates: func([]UpdateChoice) ([]UpdateChoice, bool) { + t.Fatal("--check must not prompt for dependency selection") + return nil, false + }, + SelectVersion: func(UpdateChoice) (string, bool) { + t.Fatal("--check must not prompt for a version") + return "", false + }, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 { + t.Fatalf("plans = %#v, want 1 checked update", plans) + } + plan := plans[0] + if !plan.Checked || plan.DryRun || plan.Written || plan.Skipped != "" { + t.Fatalf("unexpected check plan status: %#v", plan) + } + if plan.OldVersion != "^1.3.0" || plan.NewVersion != "1.4.0" { + t.Fatalf("unexpected check version change: %#v", plan) + } + if len(runner.commands) != 1 || runner.commands[0].Name != "npm" || runner.commands[0].Args[0] != "view" { + t.Fatalf("--check should only run version lookup, got %#v", runner.commands) + } +} + +func TestDiscoverUpdateCandidates_ImageAndHelmTargets(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "workloads.yaml"), deploymentUpdateFixture) + writeFile(t, filepath.Join(dir, "apps", "helmrelease.yaml"), helmReleaseUpdateFixture) + runGit(t, dir, "add", ".") + + got, err := DiscoverUpdateCandidates(".", []Manager{ManagerImage, ManagerHelm}) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("candidates = %d, want 3: %#v", len(got), got) + } + byManager := map[Manager]int{} + for _, candidate := range got { + byManager[candidate.Manager]++ + if filepath.IsAbs(candidate.File) { + t.Fatalf("candidate file should be relative to cwd, got %q", candidate.File) + } + if candidate.Target == nil { + t.Fatalf("candidate missing target metadata: %#v", candidate) + } + } + if byManager[ManagerImage] != 2 || byManager[ManagerHelm] != 1 { + t.Fatalf("manager counts = %#v, want image=2 helm=1", byManager) + } + helm := findUpdateCandidate(got, ManagerHelm, "podinfo") + if helm == nil || helm.Current != "6.5.0" || helm.Target.RepoURL == "" { + t.Fatalf("helm candidate not resolved correctly: %#v", helm) + } +} + +func TestUpdateImageDryRunUsesImageVersionResolver(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + runGit(t, dir, "init") + writeFile(t, filepath.Join(dir, "apps", "workloads.yaml"), deploymentUpdateFixture) + runGit(t, dir, "add", ".") + + plans, err := Update(context.Background(), ".", UpdateOptions{ + Managers: []Manager{ManagerImage}, + Expression: []string{"*nginx*"}, + DryRun: true, + ImageResolver: fakeImageVersionResolver{"nginx": []string{"1.27.0", "1.25.3"}}, + SelectCandidates: func(choices []UpdateChoice) ([]UpdateChoice, bool) { + if len(choices) != 1 { + t.Fatalf("choices = %#v, want exactly nginx", choices) + } + return choices, true + }, + SelectVersion: func(choice UpdateChoice) (string, bool) { + return "1.27.0", true + }, + }) + if err != nil { + t.Fatal(err) + } + if len(plans) != 1 { + t.Fatalf("plans = %#v, want 1", plans) + } + plan := plans[0] + if plan.Manager != ManagerImage || !plan.DryRun || plan.Written || plan.Skipped != "" { + t.Fatalf("unexpected image plan: %#v", plan) + } + if plan.File != "apps/workloads.yaml" || plan.OldVersion != "1.25.3" || plan.NewVersion != "1.27.0" { + t.Fatalf("unexpected image change: %#v", plan) + } +} + +func TestGroupUpdateChoicesByFile(t *testing.T) { + choices := []UpdateChoice{ + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "zeta", File: "/repo/web/package.json", Scope: "dependencies"}}, + {Candidate: UpdateCandidate{Manager: ManagerGo, Name: "github.com/acme/lib", File: "/repo/go.mod", Scope: "require"}}, + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "alpha", File: "/repo/web/package.json", Scope: "dependencies"}}, + } + groups := groupUpdateChoicesByFile(choices) + if len(groups) != 2 { + t.Fatalf("groups = %d, want 2: %#v", len(groups), groups) + } + if groups[0].File != "/repo/go.mod" || groups[1].File != "/repo/web/package.json" { + t.Fatalf("unexpected file order: %#v", groups) + } + webChoices := groups[1].Choices + if len(webChoices) != 2 || webChoices[0].Candidate.Name != "alpha" || webChoices[1].Candidate.Name != "zeta" { + t.Fatalf("web choices not sorted within file: %#v", webChoices) + } +} + +func TestUpdateCommandForManagers(t *testing.T) { + cases := []struct { + candidate UpdateCandidate + version string + want string + }{ + { + candidate: UpdateCandidate{Manager: ManagerGo, Name: "github.com/acme/lib"}, + version: "v1.2.0", + want: "go get github.com/acme/lib@v1.2.0", + }, + { + candidate: UpdateCandidate{Manager: ManagerNPM, Name: "typescript", Scope: "devDependencies"}, + version: "5.6.0", + want: "npm install --package-lock-only --ignore-scripts --save-dev typescript@5.6.0", + }, + { + candidate: UpdateCandidate{Manager: ManagerPNPM, Name: "@scope/pkg", Scope: "peerDependencies"}, + version: "2.1.0", + want: "pnpm add --lockfile-only --ignore-scripts --save-peer @scope/pkg@2.1.0", + }, + } + for _, tc := range cases { + cmd, err := updateCommand(tc.candidate, tc.version) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(append([]string{cmd.Name}, cmd.Args...), " "); got != tc.want { + t.Fatalf("command = %q, want %q", got, tc.want) + } + } +} + +func TestUpdateRejectsUnsupportedManagers(t *testing.T) { + _, err := Update(context.Background(), t.TempDir(), UpdateOptions{ + Managers: []Manager{ManagerMaven}, + Expression: []string{"*"}, + }) + if err == nil || !strings.Contains(err.Error(), "unsupported manager") { + t.Fatalf("expected unsupported manager error, got %v", err) + } +} + +type updateFakeRunner struct { + mu sync.Mutex + responses map[string]CommandResult + errors map[string]error + commands []Command +} + +func (r *updateFakeRunner) Run(_ context.Context, cmd Command) (CommandResult, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.commands = append(r.commands, cmd) + key := strings.Join(append([]string{cmd.Name}, cmd.Args...), " ") + if r.errors != nil && r.errors[key] != nil { + return r.responses[key], r.errors[key] + } + if r.responses != nil { + if result, ok := r.responses[key]; ok { + return result, nil + } + } + return CommandResult{}, errors.New("unexpected command: " + key) +} + +func updateCandidateLabels(candidates []UpdateCandidate) []string { + labels := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + labels = append(labels, strings.Join([]string{ + string(candidate.Manager), + candidate.Name, + candidate.Scope, + candidate.Current, + }, ":")) + } + return labels +} + +func updateCandidateFiles(candidates []UpdateCandidate) []string { + files := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + files = append(files, candidate.File) + } + return files +} + +func findUpdateCandidate(candidates []UpdateCandidate, manager Manager, name string) *UpdateCandidate { + for i := range candidates { + if candidates[i].Manager == manager && candidates[i].Name == name { + return &candidates[i] + } + } + return nil +} + +type fakeImageVersionResolver map[string][]string + +func (r fakeImageVersionResolver) Available(_ context.Context, target imageupdate.UpdateTarget) ([]string, error) { + return sortDependencyVersions(r[updateTargetName(target)]), nil +} + +func (r fakeImageVersionResolver) ResolveLatestVersions(_ context.Context, target imageupdate.UpdateTarget) (imageupdate.LatestVersions, error) { + versions := sortDependencyVersions(r[updateTargetName(target)]) + return imageupdate.LatestVersions{ + Stable: latestStableVersion(versions), + Prerelease: latestPrereleaseVersion(versions), + }, nil +} + +func (fakeImageVersionResolver) NewImageValue(_ context.Context, target imageupdate.UpdateTarget, version string) (string, error) { + return stripImageVersion(target.CurrentValue) + ":" + version, nil +} + +const deploymentUpdateFixture = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: default +spec: + template: + spec: + containers: + - name: web + image: nginx:1.25.3 + - name: sidecar + image: ghcr.io/flanksource/proxy:v0.4.1 +` + +const helmReleaseUpdateFixture = `apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: podinfo + namespace: default +spec: + chart: + spec: + chart: podinfo + version: 6.5.0 + sourceRef: + kind: HelmRepository + name: podinfo + namespace: flux-system +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: podinfo + namespace: flux-system +spec: + url: https://stefanprodan.github.io/podinfo +` diff --git a/deps/update_tree.go b/deps/update_tree.go new file mode 100644 index 0000000..3574c15 --- /dev/null +++ b/deps/update_tree.go @@ -0,0 +1,602 @@ +package deps + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/task" +) + +const ( + updateTreeStyleHeader = "font-bold" + updateTreeStyleHelp = "text-muted" + updateTreeStyleMuted = "text-muted" + updateTreeStyleCursor = "text-cyan-600 font-bold" + updateTreeStyleCheckOn = "text-green-500 font-bold" + updateTreeStyleCheckPartial = "text-muted" +) + +func updateTreeStyled(s, style string) string { + return clicky.Text(s, style).ANSI() +} + +type updateChoiceTreeNode struct { + Name string + Path string + Depth int + IsDir bool + FileGroup bool + Expanded bool + Selected bool + Children []*updateChoiceTreeNode + Choice *UpdateChoice + Parent *updateChoiceTreeNode +} + +type updateChoiceTreeModel struct { + root *updateChoiceTreeNode + visible []*updateChoiceTreeNode + cursor int + width int + height int + filtering bool + filterQuery string + cancelled bool + submitted bool +} + +func newUpdateChoiceTreeModel(choices []UpdateChoice) updateChoiceTreeModel { + root := &updateChoiceTreeNode{Name: "", Path: "", IsDir: true, Expanded: true} + for _, choice := range choices { + insertUpdateChoice(root, choice) + } + sortUpdateChoiceTree(root) + m := updateChoiceTreeModel{root: root, height: 20, width: 80} + m.rebuildVisible() + return m +} + +func insertUpdateChoice(root *updateChoiceTreeNode, choice UpdateChoice) { + filePath := cleanUpdateTreePath(choice.Candidate.File) + parts := strings.Split(filePath, "/") + current := root + for i, segment := range parts { + if segment == "" { + continue + } + isLast := i == len(parts)-1 + child := findUpdateTreeChild(current, segment, isLast) + if child == nil { + path := strings.Join(parts[:i+1], "/") + child = &updateChoiceTreeNode{ + Name: segment, + Path: path, + Depth: current.Depth + 1, + IsDir: true, + FileGroup: isLast, + Expanded: true, + Parent: current, + } + current.Children = append(current.Children, child) + } + current = child + } + if current == root { + current = &updateChoiceTreeNode{ + Name: "(unknown file)", + Path: "(unknown file)", + Depth: 1, + IsDir: true, + FileGroup: true, + Expanded: true, + Parent: root, + } + root.Children = append(root.Children, current) + } + + choiceCopy := choice + leaf := &updateChoiceTreeNode{ + Name: choice.Candidate.Name, + Path: choice.Candidate.key(), + Depth: current.Depth + 1, + IsDir: false, + Choice: &choiceCopy, + Parent: current, + } + current.Children = append(current.Children, leaf) +} + +func cleanUpdateTreePath(file string) string { + file = filepath.ToSlash(strings.TrimSpace(file)) + file = strings.TrimPrefix(file, "./") + file = strings.TrimPrefix(file, "/") + if file == "" { + return "(unknown file)" + } + return file +} + +func findUpdateTreeChild(n *updateChoiceTreeNode, name string, fileGroup bool) *updateChoiceTreeNode { + for _, child := range n.Children { + if child.Name == name && child.IsDir && child.FileGroup == fileGroup { + return child + } + } + return nil +} + +func sortUpdateChoiceTree(n *updateChoiceTreeNode) { + sort.SliceStable(n.Children, func(i, j int) bool { + a, b := n.Children[i], n.Children[j] + if a.IsDir != b.IsDir { + return a.IsDir && !b.IsDir + } + if a.FileGroup != b.FileGroup { + return !a.FileGroup && b.FileGroup + } + if a.Name != b.Name { + return a.Name < b.Name + } + if a.Choice != nil && b.Choice != nil { + return a.Choice.Candidate.less(b.Choice.Candidate) + } + return a.Path < b.Path + }) + for _, child := range n.Children { + if child.IsDir { + sortUpdateChoiceTree(child) + } + } +} + +func (m *updateChoiceTreeModel) rebuildVisible() { + m.visible = m.visible[:0] + query := normalizedUpdateTreeFilter(m.filterQuery) + if query != "" { + for _, child := range m.root.Children { + appendUpdateTreeVisibleFiltered(&m.visible, child, query) + } + if len(m.visible) == 0 { + m.cursor = 0 + return + } + if m.cursor >= len(m.visible) { + m.cursor = len(m.visible) - 1 + } + return + } + for _, child := range m.root.Children { + appendUpdateTreeVisible(&m.visible, child) + } + if m.cursor >= len(m.visible) { + m.cursor = max(0, len(m.visible)-1) + } +} + +func appendUpdateTreeVisible(out *[]*updateChoiceTreeNode, n *updateChoiceTreeNode) { + *out = append(*out, n) + if n.IsDir && n.Expanded { + for _, child := range n.Children { + appendUpdateTreeVisible(out, child) + } + } +} + +func appendUpdateTreeVisibleFiltered(out *[]*updateChoiceTreeNode, n *updateChoiceTreeNode, query string) bool { + if updateTreeNodeMatchesFilter(n, query) { + *out = append(*out, n) + if n.IsDir { + for _, child := range n.Children { + appendUpdateTreeVisibleAll(out, child) + } + } + return true + } + if !n.IsDir { + return false + } + + var childMatches []*updateChoiceTreeNode + for _, child := range n.Children { + var visibleChild []*updateChoiceTreeNode + if appendUpdateTreeVisibleFiltered(&visibleChild, child, query) { + childMatches = append(childMatches, visibleChild...) + } + } + if len(childMatches) == 0 { + return false + } + *out = append(*out, n) + *out = append(*out, childMatches...) + return true +} + +func appendUpdateTreeVisibleAll(out *[]*updateChoiceTreeNode, n *updateChoiceTreeNode) { + *out = append(*out, n) + if n.IsDir { + for _, child := range n.Children { + appendUpdateTreeVisibleAll(out, child) + } + } +} + +func normalizedUpdateTreeFilter(query string) string { + return strings.ToLower(strings.TrimSpace(query)) +} + +func updateTreeNodeMatchesFilter(n *updateChoiceTreeNode, query string) bool { + if query == "" { + return true + } + haystack := strings.ToLower(n.Path + " " + n.Name) + if n.Choice != nil { + choice := *n.Choice + candidate := choice.Candidate + haystack += " " + strings.ToLower(strings.Join([]string{ + candidate.Name, + string(candidate.Manager), + string(candidate.Manager) + ":" + candidate.Name, + candidate.Current, + candidate.Scope, + candidate.File, + choice.LatestStable, + strings.Join(choice.Versions, " "), + }, " ")) + } + return strings.Contains(haystack, query) +} + +func (m updateChoiceTreeModel) Init() tea.Cmd { return nil } + +func (m updateChoiceTreeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width = msg.Width + m.height = msg.Height + return m, nil + case tea.KeyMsg: + return m.handleKey(msg) + } + return m, nil +} + +func (m updateChoiceTreeModel) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + if m.filtering { + return m.handleFilterKey(msg) + } + switch msg.String() { + case "ctrl+c", "esc", "q": + m.cancelled = true + return m, tea.Quit + case "/": + m.filtering = true + case "enter": + m.submitted = true + return m, tea.Quit + case "down", "j": + if m.cursor < len(m.visible)-1 { + m.cursor++ + } + case "up", "k": + if m.cursor > 0 { + m.cursor-- + } + case "right", "l": + if n := m.currentNode(); n != nil && n.IsDir && !n.Expanded { + n.Expanded = true + m.rebuildVisible() + } + case "left", "h": + if n := m.currentNode(); n != nil && n.IsDir && n.Expanded { + n.Expanded = false + m.rebuildVisible() + } else if n != nil && n.Parent != nil && n.Parent != m.root { + m.moveCursorTo(n.Parent) + } + case " ": + if n := m.currentNode(); n != nil { + toggleUpdateTreeNode(n) + } + case "a": + if n := m.currentNode(); n != nil { + toggleUpdateTreeNode(n.containerOrSelf()) + } + case "ctrl+a": + toggleUpdateTreeNode(m.root) + } + return m, nil +} + +func (m updateChoiceTreeModel) handleFilterKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + switch msg.Type { + case tea.KeyCtrlC: + m.cancelled = true + return m, tea.Quit + case tea.KeyEsc: + m.filtering = false + m.filterQuery = "" + m.rebuildVisible() + return m, nil + case tea.KeyEnter: + m.filtering = false + return m, nil + case tea.KeyBackspace: + m.filterQuery = trimLastUpdateTreeRune(m.filterQuery) + m.rebuildVisible() + return m, nil + case tea.KeyCtrlU: + m.filterQuery = "" + m.rebuildVisible() + return m, nil + } + if len(msg.Runes) > 0 { + m.filterQuery += string(msg.Runes) + m.rebuildVisible() + } + return m, nil +} + +func trimLastUpdateTreeRune(s string) string { + if s == "" { + return "" + } + runes := []rune(s) + return string(runes[:len(runes)-1]) +} + +func (m updateChoiceTreeModel) currentNode() *updateChoiceTreeNode { + if m.cursor < 0 || m.cursor >= len(m.visible) { + return nil + } + return m.visible[m.cursor] +} + +func (m *updateChoiceTreeModel) moveCursorTo(target *updateChoiceTreeNode) { + for i, node := range m.visible { + if node == target { + m.cursor = i + return + } + } +} + +func (n *updateChoiceTreeNode) containerOrSelf() *updateChoiceTreeNode { + if n.IsDir { + return n + } + if n.Parent != nil { + return n.Parent + } + return n +} + +func toggleUpdateTreeNode(n *updateChoiceTreeNode) { + target := !allUpdateTreeLeavesSelected(n) + setUpdateTreeLeavesSelected(n, target) +} + +func allUpdateTreeLeavesSelected(n *updateChoiceTreeNode) bool { + if !n.IsDir { + return n.Selected + } + if len(n.Children) == 0 { + return false + } + for _, child := range n.Children { + if !allUpdateTreeLeavesSelected(child) { + return false + } + } + return true +} + +func anyUpdateTreeLeafSelected(n *updateChoiceTreeNode) bool { + if !n.IsDir { + return n.Selected + } + for _, child := range n.Children { + if anyUpdateTreeLeafSelected(child) { + return true + } + } + return false +} + +func setUpdateTreeLeavesSelected(n *updateChoiceTreeNode, selected bool) { + if !n.IsDir { + n.Selected = selected + return + } + for _, child := range n.Children { + setUpdateTreeLeavesSelected(child, selected) + } +} + +func (m updateChoiceTreeModel) selectedChoices() []UpdateChoice { + var out []UpdateChoice + var walk func(n *updateChoiceTreeNode) + walk = func(n *updateChoiceTreeNode) { + if !n.IsDir { + if n.Selected && n.Choice != nil { + out = append(out, *n.Choice) + } + return + } + for _, child := range n.Children { + walk(child) + } + } + walk(m.root) + return out +} + +func (m updateChoiceTreeModel) View() string { + var b strings.Builder + selectedCount := len(m.selectedChoices()) + totalLeaves := countUpdateTreeLeaves(m.root) + visibleLeaves := countVisibleUpdateTreeLeaves(m.visible) + filterActive := normalizedUpdateTreeFilter(m.filterQuery) != "" + + b.WriteString(updateTreeStyled("Select dependencies to update", updateTreeStyleHeader)) + b.WriteString(" ") + b.WriteString(updateTreeStyled(fmt.Sprintf("(%d / %d selected)", selectedCount, totalLeaves), updateTreeStyleMuted)) + if filterActive { + b.WriteString(" ") + b.WriteString(updateTreeStyled(fmt.Sprintf("filter=%q (%d dependencies)", m.filterQuery, visibleLeaves), updateTreeStyleMuted)) + } + b.WriteByte('\n') + if m.filtering { + b.WriteString(updateTreeStyled( + fmt.Sprintf(" filter: %s type=search backspace=delete ctrl+u=clear enter=keep esc=clear", m.filterQuery), + updateTreeStyleHelp, + )) + } else { + b.WriteString(updateTreeStyled( + " /=filter space=toggle a=toggle file ctrl+a=all enter=confirm esc=cancel", + updateTreeStyleHelp, + )) + } + b.WriteString("\n\n") + + pageSize := max(m.height-5, 5) + start := 0 + if m.cursor >= pageSize { + start = m.cursor - pageSize + 1 + } + end := min(start+pageSize, len(m.visible)) + for i := start; i < end; i++ { + b.WriteString(renderUpdateTreeRow(m.visible[i], i == m.cursor, filterActive)) + b.WriteByte('\n') + } + if len(m.visible) == 0 { + b.WriteString(updateTreeStyled(" No dependencies match the current filter", updateTreeStyleHelp)) + b.WriteByte('\n') + } + return b.String() +} + +func renderUpdateTreeRow(n *updateChoiceTreeNode, isCursor bool, forceExpanded bool) string { + cursor := " " + if isCursor { + cursor = updateTreeStyled("> ", updateTreeStyleCursor) + } + indent := strings.Repeat(" ", max(0, n.Depth-1)) + check := updateTreeCheckbox(n) + name := n.Name + if n.IsDir { + expand := "v " + if !forceExpanded && !n.Expanded { + expand = "> " + } + suffix := "/" + if n.FileGroup { + suffix = "" + } + name = expand + name + suffix + } + if isCursor { + name = updateTreeStyled(name, updateTreeStyleHeader) + } + row := fmt.Sprintf("%s%s%s %s", cursor, indent, check, name) + if !n.IsDir && n.Choice != nil { + row += " " + updateTreeChoiceChips(*n.Choice) + } + return row +} + +func updateTreeCheckbox(n *updateChoiceTreeNode) string { + if !n.IsDir { + if n.Selected { + return updateTreeStyled("[x]", updateTreeStyleCheckOn) + } + return "[ ]" + } + switch { + case allUpdateTreeLeavesSelected(n): + return updateTreeStyled("[x]", updateTreeStyleCheckOn) + case anyUpdateTreeLeafSelected(n): + return updateTreeStyled("[~]", updateTreeStyleCheckPartial) + default: + return "[ ]" + } +} + +func updateTreeChoiceChips(choice UpdateChoice) string { + candidate := choice.Candidate + parts := []string{ + updateTreeStyled(string(candidate.Manager), managerStyle(candidate.Manager)), + updateTreeStyled("@"+candidate.Current, "font-mono text-muted"), + } + if choice.LatestStable != "" { + parts = append(parts, updateTreeStyled("latest "+choice.LatestStable, "text-green-600")) + } + if choice.LatestPrerelease != "" { + parts = append(parts, updateTreeStyled("pre "+choice.LatestPrerelease, "text-yellow-600")) + } + if candidate.Scope != "" { + parts = append(parts, updateTreeStyled(candidate.Scope, "text-muted")) + } + return strings.Join(parts, updateTreeStyled(" * ", updateTreeStyleMuted)) +} + +func countUpdateTreeLeaves(n *updateChoiceTreeNode) int { + if !n.IsDir { + return 1 + } + total := 0 + for _, child := range n.Children { + total += countUpdateTreeLeaves(child) + } + return total +} + +func countVisibleUpdateTreeLeaves(nodes []*updateChoiceTreeNode) int { + total := 0 + for _, node := range nodes { + if !node.IsDir { + total++ + } + } + return total +} + +func runUpdateChoiceTreePicker(choices []UpdateChoice) ([]UpdateChoice, bool) { + if len(choices) == 0 { + return nil, false + } + model := newUpdateChoiceTreeModel(choices) + if len(model.visible) == 0 { + return nil, false + } + + tty, err := os.OpenFile("/dev/tty", os.O_RDWR, 0) + if err != nil { + return nil, false + } + defer tty.Close() + + releaseTerminal, _ := task.AcquirePromptTerminal() + if releaseTerminal != nil { + defer releaseTerminal() + } + + program := tea.NewProgram(model, tea.WithInput(tty), tea.WithOutput(tty), tea.WithAltScreen()) + final, err := program.Run() + if err != nil { + if errors.Is(err, tea.ErrInterrupted) { + return nil, false + } + return nil, false + } + finished, ok := final.(updateChoiceTreeModel) + if !ok || finished.cancelled || !finished.submitted { + return nil, false + } + return finished.selectedChoices(), true +} diff --git a/deps/update_tree_test.go b/deps/update_tree_test.go new file mode 100644 index 0000000..8e12a1e --- /dev/null +++ b/deps/update_tree_test.go @@ -0,0 +1,131 @@ +package deps + +import ( + "sort" + "strings" + "testing" + + tea "github.com/charmbracelet/bubbletea" +) + +func TestUpdateChoiceTreeGroupsDependenciesUnderFiles(t *testing.T) { + m := newUpdateChoiceTreeModel([]UpdateChoice{ + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "zeta", Current: "1.0.0", Scope: "dependencies", File: "web/package.json"}}, + {Candidate: UpdateCandidate{Manager: ManagerGo, Name: "github.com/acme/lib", Current: "v1.0.0", Scope: "require", File: "go.mod"}}, + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "alpha", Current: "1.0.0", Scope: "dependencies", File: "web/package.json"}}, + }) + + goMod := updateTreeNodeAtPath(m, "go.mod") + if goMod == nil || !goMod.FileGroup { + t.Fatalf("go.mod file group not found: %#v", updateTreeVisiblePaths(m)) + } + webPackage := updateTreeNodeAtPath(m, "web/package.json") + if webPackage == nil || !webPackage.FileGroup { + t.Fatalf("web/package.json file group not found: %#v", updateTreeVisiblePaths(m)) + } + got := updateTreeChildNames(webPackage) + if strings.Join(got, ",") != "alpha,zeta" { + t.Fatalf("web package choices = %#v, want alpha,zeta", got) + } +} + +func TestUpdateChoiceTreeCtrlASelectsAndClearsAll(t *testing.T) { + m := newUpdateChoiceTreeModel([]UpdateChoice{ + {Candidate: UpdateCandidate{Manager: ManagerGo, Name: "github.com/acme/lib", File: "go.mod"}}, + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "left-pad", File: "web/package.json"}}, + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "typescript", File: "web/package.json"}}, + }) + + m = updateTreeTestKey(m, tea.KeyMsg{Type: tea.KeyCtrlA}) + got := selectedUpdateTreeNames(m) + if strings.Join(got, ",") != "github.com/acme/lib,left-pad,typescript" { + t.Fatalf("selected after ctrl+a = %#v", got) + } + + m = updateTreeTestKey(m, tea.KeyMsg{Type: tea.KeyCtrlA}) + if got := selectedUpdateTreeNames(m); len(got) != 0 { + t.Fatalf("selected after second ctrl+a = %#v, want none", got) + } +} + +func TestUpdateChoiceTreeFilterMatchesDependencyAndFile(t *testing.T) { + m := newUpdateChoiceTreeModel([]UpdateChoice{ + {Candidate: UpdateCandidate{Manager: ManagerGo, Name: "github.com/acme/lib", File: "go.mod"}}, + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "@flanksource/ui", File: "web/package.json"}}, + {Candidate: UpdateCandidate{Manager: ManagerNPM, Name: "typescript", File: "web/package.json"}}, + }) + + m = updateTreeTestKey(m, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("/")}) + m = updateTreeTestKey(m, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("flanksource")}) + if got := updateTreeVisibleLeafNames(m); strings.Join(got, ",") != "@flanksource/ui" { + t.Fatalf("visible leaves for dependency filter = %#v", got) + } + + m = updateTreeTestKey(m, tea.KeyMsg{Type: tea.KeyCtrlU}) + m = updateTreeTestKey(m, tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("go.mod")}) + if got := updateTreeVisibleLeafNames(m); strings.Join(got, ",") != "github.com/acme/lib" { + t.Fatalf("visible leaves for file filter = %#v", got) + } +} + +func updateTreeTestKey(m updateChoiceTreeModel, msg tea.KeyMsg) updateChoiceTreeModel { + updated, _ := m.handleKey(msg) + return updated.(updateChoiceTreeModel) +} + +func updateTreeNodeAtPath(m updateChoiceTreeModel, path string) *updateChoiceTreeNode { + var found *updateChoiceTreeNode + var walk func(*updateChoiceTreeNode) + walk = func(node *updateChoiceTreeNode) { + if found != nil { + return + } + if node.Path == path { + found = node + return + } + for _, child := range node.Children { + walk(child) + } + } + walk(m.root) + return found +} + +func updateTreeChildNames(node *updateChoiceTreeNode) []string { + out := make([]string, 0, len(node.Children)) + for _, child := range node.Children { + out = append(out, child.Name) + } + return out +} + +func selectedUpdateTreeNames(m updateChoiceTreeModel) []string { + choices := m.selectedChoices() + out := make([]string, 0, len(choices)) + for _, choice := range choices { + out = append(out, choice.Candidate.Name) + } + sort.Strings(out) + return out +} + +func updateTreeVisibleLeafNames(m updateChoiceTreeModel) []string { + var out []string + for _, node := range m.visible { + if node.IsDir || node.Choice == nil { + continue + } + out = append(out, node.Choice.Candidate.Name) + } + sort.Strings(out) + return out +} + +func updateTreeVisiblePaths(m updateChoiceTreeModel) []string { + out := make([]string, 0, len(m.visible)) + for _, node := range m.visible { + out = append(out, node.Path) + } + return out +} diff --git a/go.mod b/go.mod index ba9f2ad..c6357dd 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 github.com/argoproj-labs/argocd-image-updater/registry-scanner v1.2.1 github.com/bmatcuk/doublestar/v4 v4.10.0 + github.com/charmbracelet/bubbletea v1.3.10 github.com/flanksource/clicky v1.21.14 github.com/flanksource/commons v1.51.3 github.com/ghodss/yaml v1.0.0 @@ -51,7 +52,6 @@ require ( github.com/cert-manager/cert-manager v1.20.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect - github.com/charmbracelet/bubbletea v1.3.10 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/huh v1.0.0 // indirect github.com/charmbracelet/lipgloss v1.1.0 // indirect From 4de4cec4b0304913291bda20c219c8984b6b34ae Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Thu, 11 Jun 2026 08:11:46 +0300 Subject: [PATCH 04/17] feat(deps): refactor dependency resolution to offline-first with optional transitive graph support Replace native/manifest/auto resolution modes with a single offline-first approach that reads local manifests and lockfiles without running package-manager commands. Add --depth 0 support for transitive graphs via tool-specific resolvers (go mod graph, mvn dependency:tree, gradle dependencies) that fail fast with toolError when tools are unavailable, suggesting --depth 1 for offline output. Add new features: - deps diff subcommand to compare dependency graphs across git revisions - --flat flag to export flat node/edge lists instead of tree structure - --include-indirect flag for Go to include indirect requirements at --depth 1 - Support for image and Helm chart discovery from Kubernetes manifests - Comparison analysis with added/removed/updated change tracking Remove deprecated options: - --mode (native/manifest/auto) - --configuration (Gradle-specific) - --strict flag Breaking changes: - Default behavior now offline-only; use --depth 0 for transitive resolution - JSON export structure changes: roots/nodes/edges now conditional on --flat flag - Metadata.configurations replaced with Metadata.flat Refs: refactor to simplify resolution logic and improve offline-first user experience --- README.md | 8 +- cmd/repomap/deps.go | 72 +++++----- cmd/repomap/deps_diff.go | 92 +++++++++++++ cmd/repomap/deps_diff_test.go | 53 ++++++++ cmd/repomap/deps_test.go | 39 ++++-- deps/common.go | 6 +- deps/compare.go | 243 ++++++++++++++++++++++++++++++++++ deps/compare_pretty.go | 168 +++++++++++++++++++++++ deps/compare_pretty_test.go | 60 +++++++++ deps/compare_scan.go | 125 +++++++++++++++++ deps/compare_scan_test.go | 123 +++++++++++++++++ deps/compare_test.go | 139 +++++++++++++++++++ deps/discover.go | 18 ++- deps/edgegraph.go | 89 +++++++++++++ deps/edgegraph_test.go | 99 ++++++++++++++ deps/filter.go | 14 -- deps/go.go | 184 ++----------------------- deps/go_graph.go | 134 +++++++++++++++++++ deps/go_graph_test.go | 144 ++++++++++++++++++++ deps/go_test.go | 49 +++++++ deps/gradle.go | 178 +------------------------ deps/gradle_tree.go | 202 ++++++++++++++++++++++++++++ deps/gradle_tree_test.go | 84 ++++++++++++ deps/maven.go | 98 +------------- deps/maven_tree.go | 123 +++++++++++++++++ deps/maven_tree_test.go | 99 ++++++++++++++ deps/model.go | 26 ++-- deps/npm.go | 70 +--------- deps/npm_test.go | 22 +++ deps/pnpm.go | 111 +--------------- deps/pnpm_test.go | 45 ------- deps/pretty.go | 95 +++++++------ deps/scan.go | 220 ++++++++++++++++++------------ deps/scan_image.go | 92 +++++++++++++ deps/scan_test.go | 154 +++++++++++++++++++-- deps/update.go | 16 --- deps/update_tree.go | 2 +- 37 files changed, 2589 insertions(+), 907 deletions(-) create mode 100644 cmd/repomap/deps_diff.go create mode 100644 cmd/repomap/deps_diff_test.go create mode 100644 deps/compare.go create mode 100644 deps/compare_pretty.go create mode 100644 deps/compare_pretty_test.go create mode 100644 deps/compare_scan.go create mode 100644 deps/compare_scan_test.go create mode 100644 deps/compare_test.go create mode 100644 deps/edgegraph.go create mode 100644 deps/edgegraph_test.go create mode 100644 deps/go_graph.go create mode 100644 deps/go_graph_test.go create mode 100644 deps/go_test.go create mode 100644 deps/gradle_tree.go create mode 100644 deps/gradle_tree_test.go create mode 100644 deps/maven_tree.go create mode 100644 deps/maven_tree_test.go create mode 100644 deps/npm_test.go delete mode 100644 deps/pnpm_test.go create mode 100644 deps/scan_image.go diff --git a/README.md b/README.md index 9e29d4c..0b8fd6b 100644 --- a/README.md +++ b/README.md @@ -91,10 +91,10 @@ repomap deps --manager go,pnpm repomap deps --depth 0 ``` -By default, `deps` auto-detects supported manifests, tries native package -manager resolution, and falls back to manifest or lockfile parsing with warnings. -It prints direct dependencies by default; use `--depth 0` for the full -transitive graph. +By default, `deps` auto-detects supported manifests and reads local manifest or +lockfile content without running package-manager commands. It prints direct +dependencies by default; use `--depth 0` for the full graph available from the +local files. ### `version` diff --git a/cmd/repomap/deps.go b/cmd/repomap/deps.go index 3541563..1f21cf8 100644 --- a/cmd/repomap/deps.go +++ b/cmd/repomap/deps.go @@ -11,13 +11,12 @@ import ( ) type DepsOptions struct { - Path string `json:"path" args:"true" help:"Path to scan" default:"."` - Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm (repeatable or comma-separated)"` - Mode string `json:"mode,omitempty" flag:"mode" default:"auto" help:"Resolution mode: auto, native, or manifest"` - Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` - Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` - Configuration []string `json:"configuration,omitempty" flag:"configuration" help:"Gradle configuration to resolve (repeatable or comma-separated); default is all resolvable configurations"` - Strict bool `json:"strict,omitempty" flag:"strict" help:"Fail if native resolution is unavailable or fallback resolution is degraded"` + Path string `json:"path" args:"true" help:"Path to scan" default:"."` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth (1 = direct only, 0 = unlimited)"` + Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns matched against id, name, version, manager, source, or path; supports comma-separated values and !exclusions"` + Flat bool `json:"flat,omitempty" flag:"flat" help:"Export a flat node list with edges instead of the dependency tree"` + IncludeIndirect bool `json:"include_indirect,omitempty" flag:"include-indirect" help:"Include Go indirect requirements in --depth 1 listings (ignored at other depths)"` } type DepsUpdateOptions struct { @@ -32,23 +31,34 @@ func (opts DepsOptions) GetName() string { return "deps" } func (opts DepsUpdateOptions) GetName() string { return "update [path]" } func (opts DepsOptions) Help() api.Text { - return clicky.Text(`Generate dependency graphs for Go, Maven, Gradle, npm, and pnpm projects. + return clicky.Text(`Generate dependency graphs for Go, Maven, Gradle, npm, pnpm, image, and Helm dependencies. -Repomap auto-detects supported manifests below the selected path, resolves -transitive dependency graphs with native tools when available, and falls back to -manifest or lockfile parsing with warnings when native resolution is unavailable. +Repomap auto-detects supported manifests below the selected path and resolves +dependency graphs from local manifest and lockfile content without running +package-manager commands. Image and Helm dependencies are discovered from +git-tracked Kubernetes manifests. + +For Go, Maven, and Gradle, --depth other than 1 resolves transitive dependencies +by shelling out to the package manager (go mod graph, mvn dependency:tree, +gradle dependencies). The tool must be installed; rerun with --depth 1 for +offline direct-only output. The command uses the normal Clicky output flow. Use --json to write structured JSON to stdout, for example: repomap deps --json > out.json +By default the JSON export contains the dependency tree under "roots". Use --flat +to export a flat "nodes" list plus "edges" instead of the tree. + EXAMPLES: repomap deps repomap deps ./service --manager go repomap deps --manager npm,pnpm --depth 0 - repomap deps --filter 'github.com/flanksource/*,!*test*' - repomap deps --mode manifest --json > deps.json`) + repomap deps --manager go --depth 0 --flat --json + repomap deps --manager go --include-indirect + repomap deps --manager image,helm ./clusters/prod + repomap deps --filter 'github.com/flanksource/*,!*test*'`) } func (opts DepsUpdateOptions) Help() api.Text { @@ -78,6 +88,8 @@ func init() { updateCmd := clicky.AddNamedCommandWithContext("update", cmd, DepsUpdateOptions{}, runDepsUpdate) updateCmd.Short = "Update direct package, image, and Helm chart dependencies" + + registerDepsDiff(cmd) } func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { @@ -92,17 +104,12 @@ func runDeps(ctx context.Context, opts DepsOptions) (*depgraph.Export, error) { if err != nil { return nil, err } - mode, err := parseDepsMode(opts.Mode) - if err != nil { - return nil, err - } return depgraph.Scan(ctx, path, depgraph.Options{ - Managers: managers, - Mode: mode, - MaxDepth: opts.Depth, - Filters: splitCommaArgs(opts.Filter), - Configurations: splitCommaArgs(opts.Configuration), - Strict: opts.Strict, + Managers: managers, + MaxDepth: opts.Depth, + Filters: splitCommaArgs(opts.Filter), + Flat: opts.Flat, + IncludeIndirect: opts.IncludeIndirect, }) } @@ -137,19 +144,6 @@ func runDepsUpdate(ctx context.Context, opts DepsUpdateOptions) (any, error) { return api.NewTableFrom(plans), nil } -func parseDepsMode(value string) (depgraph.Mode, error) { - switch depgraph.Mode(strings.TrimSpace(value)) { - case "", depgraph.ModeAuto: - return depgraph.ModeAuto, nil - case depgraph.ModeNative: - return depgraph.ModeNative, nil - case depgraph.ModeManifest: - return depgraph.ModeManifest, nil - default: - return "", fmt.Errorf("unsupported deps mode %q (expected auto, native, or manifest)", value) - } -} - func parseManagers(values []string) ([]depgraph.Manager, error) { parts := splitCommaArgs(values) if len(parts) == 0 { @@ -159,10 +153,12 @@ func parseManagers(values []string) ([]depgraph.Manager, error) { for _, part := range parts { manager := depgraph.Manager(strings.ToLower(part)) switch manager { - case depgraph.ManagerGo, depgraph.ManagerMaven, depgraph.ManagerGradle, depgraph.ManagerNPM, depgraph.ManagerPNPM: + case "docker": + out = append(out, depgraph.ManagerImage) + case depgraph.ManagerGo, depgraph.ManagerMaven, depgraph.ManagerGradle, depgraph.ManagerNPM, depgraph.ManagerPNPM, depgraph.ManagerImage, depgraph.ManagerHelm: out = append(out, manager) default: - return nil, fmt.Errorf("unsupported dependency manager %q (expected go, maven, gradle, npm, or pnpm)", part) + return nil, fmt.Errorf("unsupported dependency manager %q (expected go, maven, gradle, npm, pnpm, image/docker, or helm)", part) } } return out, nil diff --git a/cmd/repomap/deps_diff.go b/cmd/repomap/deps_diff.go new file mode 100644 index 0000000..6a29fef --- /dev/null +++ b/cmd/repomap/deps_diff.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" + "github.com/spf13/cobra" + depgraph "github.com/flanksource/repomap/deps" +) + +type DepsDiffOptions struct { + Args []string `json:"args" args:"true" required:"true" help:" or .., optionally followed by a path"` + Manager []string `json:"manager,omitempty" flag:"manager" help:"Dependency manager to include: go, maven, gradle, npm, pnpm, image/docker, helm (repeatable or comma-separated)"` + Depth int `json:"depth,omitempty" flag:"depth" default:"1" help:"Maximum dependency depth to compare (1 = direct only, 0 = unlimited)"` + Filter []string `json:"filter,omitempty" flag:"filter" help:"Dependency filter patterns applied to both sides; supports comma-separated values and !exclusions"` +} + +func (opts DepsDiffOptions) GetName() string { return "diff [path]" } + +func (opts DepsDiffOptions) Help() api.Text { + return clicky.Text(`Compare dependency graphs across git revisions. + +Resolves dependencies at a base revision and compares them against the working +tree (default) or a second revision with ref1..ref2. Each non-working-tree side +is checked out into a temporary git worktree so image and Helm discovery work +unchanged. A dirty working tree is compared as-is. + +Pass --depth 0 to compare full transitive graphs (requires the package manager +tools, e.g. go, mvn, gradle); the default --depth 1 compares direct dependencies +offline. + +EXAMPLES: + repomap deps diff HEAD~3 + repomap deps diff main + repomap deps diff v1.0.0..v1.1.0 ./service + repomap deps diff HEAD~5 --manager go --depth 0`) +} + +func registerDepsDiff(parent *cobra.Command) { + diffCmd := clicky.AddNamedCommandWithContext("diff", parent, DepsDiffOptions{}, runDepsDiff) + diffCmd.Short = "Compare dependency graphs across git revisions" +} + +func runDepsDiff(ctx context.Context, opts DepsDiffOptions) (*depgraph.Comparison, error) { + if len(opts.Args) == 0 { + return nil, fmt.Errorf("a git ref or ref1..ref2 range is required") + } + if len(opts.Args) > 2 { + return nil, fmt.Errorf("expected [path], got %d arguments", len(opts.Args)) + } + baseRef, headRef, err := parseRefRange(opts.Args[0]) + if err != nil { + return nil, err + } + path := "." + if len(opts.Args) == 2 { + path = opts.Args[1] + } + path, err = resolvePath(path) + if err != nil { + return nil, err + } + managers, err := parseManagers(opts.Manager) + if err != nil { + return nil, err + } + return depgraph.CompareScan(ctx, path, depgraph.CompareOptions{ + Options: depgraph.Options{ + Managers: managers, + MaxDepth: opts.Depth, + Filters: splitCommaArgs(opts.Filter), + }, + BaseRef: baseRef, + HeadRef: headRef, + }) +} + +// parseRefRange splits a "ref1..ref2" range or returns a single ref with an +// empty head (meaning the working tree). Both sides of a range are required. +func parseRefRange(arg string) (baseRef, headRef string, err error) { + if !strings.Contains(arg, "..") { + return arg, "", nil + } + left, right, _ := strings.Cut(arg, "..") + if left == "" || right == "" { + return "", "", fmt.Errorf("invalid ref range %q: both sides of .. are required", arg) + } + return left, right, nil +} diff --git a/cmd/repomap/deps_diff_test.go b/cmd/repomap/deps_diff_test.go new file mode 100644 index 0000000..b024071 --- /dev/null +++ b/cmd/repomap/deps_diff_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "strings" + "testing" +) + +func TestDepsDiffCommandRegistered(t *testing.T) { + cmd, _, err := rootCmd.Find([]string{"deps", "diff", "HEAD~1"}) + if err != nil { + t.Fatal(err) + } + if cmd == nil || !strings.HasPrefix(cmd.Use, "diff") { + t.Fatalf("expected deps diff command, got %#v", cmd) + } + for _, name := range []string{"depth", "manager", "filter"} { + if flag := cmd.Flags().Lookup(name); flag == nil { + t.Fatalf("%s flag not registered on deps diff", name) + } + } + if depth := cmd.Flags().Lookup("depth"); depth.DefValue != "1" { + t.Fatalf("depth default = %q, want 1", depth.DefValue) + } +} + +func TestParseRefRange(t *testing.T) { + cases := []struct { + arg string + wantBase string + wantHead string + wantErr bool + }{ + {"HEAD", "HEAD", "", false}, + {"v1.0.0..v1.1.0", "v1.0.0", "v1.1.0", false}, + {"..head", "", "", true}, + {"base..", "", "", true}, + } + for _, tc := range cases { + base, head, err := parseRefRange(tc.arg) + if tc.wantErr { + if err == nil { + t.Fatalf("parseRefRange(%q) expected error", tc.arg) + } + continue + } + if err != nil { + t.Fatalf("parseRefRange(%q) unexpected error: %v", tc.arg, err) + } + if base != tc.wantBase || head != tc.wantHead { + t.Fatalf("parseRefRange(%q) = (%q, %q), want (%q, %q)", tc.arg, base, head, tc.wantBase, tc.wantHead) + } + } +} diff --git a/cmd/repomap/deps_test.go b/cmd/repomap/deps_test.go index 7766f73..65f7b8d 100644 --- a/cmd/repomap/deps_test.go +++ b/cmd/repomap/deps_test.go @@ -8,11 +8,18 @@ import ( ) func TestParseManagers(t *testing.T) { - got, err := parseManagers([]string{"go,npm", "pnpm"}) + got, err := parseManagers([]string{"go,npm", "pnpm", "image", "docker", "helm"}) if err != nil { t.Fatal(err) } - want := []depgraph.Manager{depgraph.ManagerGo, depgraph.ManagerNPM, depgraph.ManagerPNPM} + want := []depgraph.Manager{ + depgraph.ManagerGo, + depgraph.ManagerNPM, + depgraph.ManagerPNPM, + depgraph.ManagerImage, + depgraph.ManagerImage, + depgraph.ManagerHelm, + } if len(got) != len(want) { t.Fatalf("len = %d, want %d: %#v", len(got), len(want), got) } @@ -45,14 +52,15 @@ func TestParseUpdateManagers(t *testing.T) { } } -func TestParseDepsMode(t *testing.T) { - for _, mode := range []string{"", "auto", "native", "manifest"} { - if _, err := parseDepsMode(mode); err != nil { - t.Fatalf("parseDepsMode(%q): %v", mode, err) - } +func TestDepsNativeResolutionFlagsRemoved(t *testing.T) { + cmd, _, err := rootCmd.Find([]string{"deps"}) + if err != nil { + t.Fatal(err) } - if _, err := parseDepsMode("lockfile"); err == nil { - t.Fatal("expected unsupported mode error") + for _, name := range []string{"mode", "configuration", "strict"} { + if flag := cmd.Flags().Lookup(name); flag != nil { + t.Fatalf("%s flag should be removed from deps listing", name) + } } } @@ -73,6 +81,19 @@ func TestDepsDepthDefault(t *testing.T) { } } +func TestDepsFlatAndIncludeIndirectFlags(t *testing.T) { + cmd, _, err := rootCmd.Find([]string{"deps"}) + if err != nil { + t.Fatal(err) + } + if flag := cmd.Flags().Lookup("flat"); flag == nil { + t.Fatal("flat flag not registered") + } + if flag := cmd.Flags().Lookup("include-indirect"); flag == nil { + t.Fatal("include-indirect flag not registered") + } +} + func TestDepsUpdateCommandRegistered(t *testing.T) { cmd, _, err := rootCmd.Find([]string{"deps", "update", "github.com/flanksource/*"}) if err != nil { diff --git a/deps/common.go b/deps/common.go index 2ea02e9..1412e25 100644 --- a/deps/common.go +++ b/deps/common.go @@ -57,11 +57,7 @@ func isReplacementDependency(node *Node) bool { if node.Local { return true } - return node.Manager == ManagerGo && node.Source != "" && node.Source != "go.mod" && node.Source != "go mod graph" -} - -func sortStrings(values []string) { - sort.Strings(values) + return node.Manager == ManagerGo && node.Source != "" && node.Source != "go.mod" } func isLocalRef(ref string) bool { diff --git a/deps/compare.go b/deps/compare.go new file mode 100644 index 0000000..8dcfa55 --- /dev/null +++ b/deps/compare.go @@ -0,0 +1,243 @@ +package deps + +import ( + "path/filepath" + "sort" + "strings" +) + +type ChangeType string + +const ( + ChangeAdded ChangeType = "added" + ChangeRemoved ChangeType = "removed" + ChangeUpdated ChangeType = "updated" +) + +// Change describes a single dependency difference between two scans, scoped to a +// project (manifest) within the scanned tree. +type Change struct { + Type ChangeType `json:"type"` + Manager Manager `json:"manager"` + Name string `json:"name"` + Project string `json:"project"` + OldVersion string `json:"old_version,omitempty"` + NewVersion string `json:"new_version,omitempty"` + OldScope string `json:"old_scope,omitempty"` + NewScope string `json:"new_scope,omitempty"` + Direct bool `json:"direct,omitempty"` + Depth int `json:"depth,omitempty"` +} + +type ComparisonMetadata struct { + Path string `json:"path"` + BaseRef string `json:"base_ref"` + HeadRef string `json:"head_ref"` +} + +type ComparisonStatistics struct { + Added int `json:"added"` + Removed int `json:"removed"` + Updated int `json:"updated"` +} + +// Comparison is the result of diffing two dependency graphs. +type Comparison struct { + Metadata ComparisonMetadata `json:"metadata"` + Added []Change `json:"added,omitempty"` + Removed []Change `json:"removed,omitempty"` + Updated []Change `json:"updated,omitempty"` + Statistics ComparisonStatistics `json:"statistics"` + Warnings []Warning `json:"warnings,omitempty"` +} + +type depEntry struct { + manager Manager + name string + versions map[string]bool + scope string + direct bool + depth int +} + +// Compare diffs the dependency graphs of two exports. Dependencies are keyed by +// manager+name within each project (manifest), so the same package in different +// projects is compared independently. Duplicate occurrences of a package within +// a project are collapsed into a joined version set. +func Compare(base, head *Export) *Comparison { + baseIndex := indexExport(base) + headIndex := indexExport(head) + + comparison := &Comparison{} + for _, project := range sortedKeys(unionKeys(baseIndex, headIndex)) { + baseDeps := baseIndex[project] + headDeps := headIndex[project] + for _, key := range sortedKeys(unionKeys(baseDeps, headDeps)) { + before, hasBefore := baseDeps[key] + after, hasAfter := headDeps[key] + switch { + case hasBefore && !hasAfter: + comparison.Removed = append(comparison.Removed, removedChange(project, before)) + case !hasBefore && hasAfter: + comparison.Added = append(comparison.Added, addedChange(project, after)) + default: + if change, ok := updatedChange(project, before, after); ok { + comparison.Updated = append(comparison.Updated, change) + } + } + } + } + comparison.Statistics = ComparisonStatistics{ + Added: len(comparison.Added), + Removed: len(comparison.Removed), + Updated: len(comparison.Updated), + } + return comparison +} + +func addedChange(project string, e *depEntry) Change { + return Change{ + Type: ChangeAdded, + Manager: e.manager, + Name: e.name, + Project: project, + NewVersion: versionSetString(e.versions), + NewScope: e.scope, + Direct: e.direct, + Depth: e.depth, + } +} + +func removedChange(project string, e *depEntry) Change { + return Change{ + Type: ChangeRemoved, + Manager: e.manager, + Name: e.name, + Project: project, + OldVersion: versionSetString(e.versions), + OldScope: e.scope, + Direct: e.direct, + Depth: e.depth, + } +} + +func updatedChange(project string, before, after *depEntry) (Change, bool) { + oldVersion := versionSetString(before.versions) + newVersion := versionSetString(after.versions) + if oldVersion == newVersion && before.scope == after.scope { + return Change{}, false + } + return Change{ + Type: ChangeUpdated, + Manager: after.manager, + Name: after.name, + Project: project, + OldVersion: oldVersion, + NewVersion: newVersion, + OldScope: before.scope, + NewScope: after.scope, + Direct: after.direct, + Depth: after.depth, + }, true +} + +func indexExport(export *Export) map[string]map[string]*depEntry { + out := map[string]map[string]*depEntry{} + if export == nil { + return out + } + for _, root := range export.Roots { + project := normalizeProject(export.Metadata.Path, root) + deps := out[project] + if deps == nil { + deps = map[string]*depEntry{} + out[project] = deps + } + for _, child := range root.Children { + collectEntries(child, deps) + } + } + return out +} + +func collectEntries(node *Node, deps map[string]*depEntry) { + if node == nil { + return + } + key := string(node.Manager) + ":" + node.Name + entry := deps[key] + if entry == nil { + entry = &depEntry{ + manager: node.Manager, + name: node.Name, + versions: map[string]bool{}, + scope: node.Scope, + direct: node.Direct, + depth: node.Depth, + } + deps[key] = entry + } else { + if node.Depth < entry.depth { + entry.depth = node.Depth + } + if node.Direct { + entry.direct = true + } + if entry.scope == "" { + entry.scope = node.Scope + } + } + if node.Version != "" { + entry.versions[node.Version] = true + } + for _, child := range node.Children { + collectEntries(child, deps) + } +} + +func normalizeProject(basePath string, root *Node) string { + target := root.Path + if target == "" { + return root.Name + } + if basePath == "" { + return filepath.ToSlash(target) + } + rel, err := filepath.Rel(basePath, target) + if err != nil { + return filepath.ToSlash(target) + } + return filepath.ToSlash(rel) +} + +func versionSetString(versions map[string]bool) string { + if len(versions) == 0 { + return "" + } + list := make([]string, 0, len(versions)) + for version := range versions { + list = append(list, version) + } + sort.Strings(list) + return strings.Join(list, ", ") +} + +func unionKeys[V any](a, b map[string]V) map[string]bool { + keys := map[string]bool{} + for key := range a { + keys[key] = true + } + for key := range b { + keys[key] = true + } + return keys +} + +func sortedKeys(set map[string]bool) []string { + keys := make([]string, 0, len(set)) + for key := range set { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/deps/compare_pretty.go b/deps/compare_pretty.go new file mode 100644 index 0000000..fd6cc27 --- /dev/null +++ b/deps/compare_pretty.go @@ -0,0 +1,168 @@ +package deps + +import ( + "fmt" + "sort" + + "github.com/flanksource/clicky" + "github.com/flanksource/clicky/api" +) + +// diffNode adapts a comparison group (project header or change leaf) to the +// clicky tree renderer. +type diffNode struct { + text api.Text + children []api.TreeNode +} + +var _ api.TreeNode = (*diffNode)(nil) + +func (d *diffNode) Pretty() api.Text { return d.text } + +func (d *diffNode) GetChildren() []api.TreeNode { return d.children } + +func (c *Comparison) Pretty() api.Text { + if c == nil { + return clicky.Text("") + } + t := clicky.Text("Dependency Diff", "font-bold") + t = t.Append(summaryText(c.Statistics)) + if c.Metadata.BaseRef != "" { + t = t.Append(fmt.Sprintf(" %s..%s", c.Metadata.BaseRef, headRefLabel(c.Metadata.HeadRef)), "text-muted") + } + + nodes := diffProjectNodes(c) + if len(nodes) == 0 { + return t.NewLine().Append("No dependency changes", "text-muted") + } + t = t.NewLine().Add(api.NewTree(nodes...)) + + if len(c.Warnings) > 0 { + t = t.NewLine().Append("Warnings", "font-bold text-yellow-600") + for _, w := range c.Warnings { + t = t.NewLine().Append("- "+w.Message, "text-yellow-600") + } + } + return t +} + +func summaryText(stats ComparisonStatistics) api.Text { + return clicky.Text(" +", "text-muted"). + Append(fmt.Sprintf("%d", stats.Added), "text-green-600"). + Append(" -", "text-muted"). + Append(fmt.Sprintf("%d", stats.Removed), "text-red-600"). + Append(" ~", "text-muted"). + Append(fmt.Sprintf("%d", stats.Updated), "text-yellow-600") +} + +func headRefLabel(headRef string) string { + if headRef == "" { + return "working tree" + } + return headRef +} + +func diffProjectNodes(c *Comparison) []api.TreeNode { + byProject := map[string][]Change{} + for _, change := range allChanges(c) { + byProject[change.Project] = append(byProject[change.Project], change) + } + projects := make([]string, 0, len(byProject)) + for project := range byProject { + projects = append(projects, project) + } + sort.Strings(projects) + + nodes := make([]api.TreeNode, 0, len(projects)) + for _, project := range projects { + changes := byProject[project] + sort.SliceStable(changes, func(i, j int) bool { + if changes[i].Manager != changes[j].Manager { + return changes[i].Manager < changes[j].Manager + } + return changes[i].Name < changes[j].Name + }) + projectNode := &diffNode{text: clicky.Text(project, "font-bold")} + for _, change := range changes { + projectNode.children = append(projectNode.children, &diffNode{text: changeText(change)}) + } + nodes = append(nodes, projectNode) + } + return nodes +} + +func allChanges(c *Comparison) []Change { + out := make([]Change, 0, len(c.Added)+len(c.Removed)+len(c.Updated)) + out = append(out, c.Added...) + out = append(out, c.Removed...) + out = append(out, c.Updated...) + return out +} + +func changeText(change Change) api.Text { + t := clicky.Text("[", "text-muted"). + Append(string(change.Manager), managerStyle(change.Manager)). + Append("] ", "text-muted") + switch change.Type { + case ChangeAdded: + t = t.Append(change.Name, "font-bold text-green-600") + if change.NewVersion != "" { + t = t.Append("@"+change.NewVersion, "font-mono text-green-600") + } + t = t.Space().Append("(new)", "text-green-600") + case ChangeRemoved: + t = t.Append(change.Name, "font-bold line-through text-red-600") + if change.OldVersion != "" { + t = t.Append("@"+change.OldVersion, "font-mono line-through text-red-600") + } + t = t.Space().Append("(removed)", "text-red-600") + case ChangeUpdated: + t = t.Append(change.Name, "font-bold text-cyan-600"). + Space().Append(versionOrNone(change.OldVersion), "font-mono text-muted"). + Append(" → ", "text-yellow-600"). + Append(versionOrNone(change.NewVersion), "font-mono text-yellow-600") + if change.OldScope != change.NewScope { + t = t.Space().Append(fmt.Sprintf("(%s → %s)", change.OldScope, change.NewScope), "text-muted") + } + } + return t +} + +func versionOrNone(version string) string { + if version == "" { + return "(none)" + } + return version +} + +// Columns implements the clicky TableProvider interface for flat change tables. +func (Change) Columns() []api.ColumnDef { + return []api.ColumnDef{ + api.Column("type").Label("Type").Build(), + api.Column("manager").Label("Manager").Build(), + api.Column("dependency").Label("Dependency").Build(), + api.Column("project").Label("Project").Build(), + api.Column("change").Label("Change").Build(), + } +} + +func (change Change) Row() map[string]any { + return map[string]any{ + "type": clicky.Text(string(change.Type), changeTypeStyle(change.Type)), + "manager": clicky.Text(string(change.Manager), managerStyle(change.Manager)), + "dependency": clicky.Text(change.Name, "font-bold text-cyan-600"), + "project": clicky.Text(change.Project, "font-mono"), + "change": changeText(change), + } +} + +func changeTypeStyle(changeType ChangeType) string { + switch changeType { + case ChangeAdded: + return "text-green-600" + case ChangeRemoved: + return "text-red-600" + default: + return "text-yellow-600" + } +} diff --git a/deps/compare_pretty_test.go b/deps/compare_pretty_test.go new file mode 100644 index 0000000..c90692d --- /dev/null +++ b/deps/compare_pretty_test.go @@ -0,0 +1,60 @@ +package deps + +import ( + "encoding/json" + "strings" + "testing" +) + +func sampleComparison() *Comparison { + return &Comparison{ + Metadata: ComparisonMetadata{Path: "/repo", BaseRef: "HEAD~1", HeadRef: ""}, + Added: []Change{{Type: ChangeAdded, Manager: ManagerGo, Name: "fresh", Project: "svc/go.mod", NewVersion: "0.1"}}, + Removed: []Change{{Type: ChangeRemoved, Manager: ManagerGo, Name: "gone", Project: "svc/go.mod", OldVersion: "1.0"}}, + Updated: []Change{{Type: ChangeUpdated, Manager: ManagerGo, Name: "bump", Project: "svc/go.mod", OldVersion: "1.0", NewVersion: "2.0"}}, + Statistics: ComparisonStatistics{Added: 1, Removed: 1, Updated: 1}, + } +} + +func TestComparisonPrettyMarkers(t *testing.T) { + out := sampleComparison().Pretty().String() + for _, want := range []string{"+1", "-1", "~1", "svc/go.mod", "fresh", "(new)", "gone", "(removed)", "bump", "→", "2.0", "working tree"} { + if !strings.Contains(out, want) { + t.Fatalf("pretty output missing %q:\n%s", want, out) + } + } +} + +func TestComparisonPrettyEmpty(t *testing.T) { + out := (&Comparison{}).Pretty().String() + if !strings.Contains(out, "No dependency changes") { + t.Fatalf("empty comparison should report no changes, got %q", out) + } +} + +func TestComparisonPrettyPrunesToChangedProjects(t *testing.T) { + c := sampleComparison() + c.Added = append(c.Added, Change{Type: ChangeAdded, Manager: ManagerNPM, Name: "left-pad", Project: "web/package.json", NewVersion: "1.3.0"}) + out := c.Pretty().String() + if !strings.Contains(out, "web/package.json") || !strings.Contains(out, "svc/go.mod") { + t.Fatalf("both changed projects should appear:\n%s", out) + } +} + +func TestComparisonJSONShape(t *testing.T) { + data, err := json.Marshal(sampleComparison()) + if err != nil { + t.Fatal(err) + } + body := string(data) + for _, want := range []string{`"added"`, `"removed"`, `"updated"`, `"statistics"`} { + if !strings.Contains(body, want) { + t.Fatalf("json missing %q: %s", want, body) + } + } + for _, unwanted := range []string{`"roots"`, `"nodes"`, `"children"`} { + if strings.Contains(body, unwanted) { + t.Fatalf("json should not leak tree field %q: %s", unwanted, body) + } + } +} diff --git a/deps/compare_scan.go b/deps/compare_scan.go new file mode 100644 index 0000000..27776f3 --- /dev/null +++ b/deps/compare_scan.go @@ -0,0 +1,125 @@ +package deps + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/flanksource/repomap" +) + +// CompareOptions configures a dependency diff. BaseRef is required; HeadRef may +// be empty to compare against the current working tree. +type CompareOptions struct { + Options + BaseRef string + HeadRef string +} + +// CompareScan resolves dependency graphs at two git revisions (or a revision and +// the working tree) and diffs them. Each non-working-tree side is materialized +// with `git worktree add --detach` so image/Helm discovery, which requires a +// real checkout, works unchanged. A dirty working tree is not an error. +func CompareScan(ctx context.Context, path string, opts CompareOptions) (*Comparison, error) { + if opts.BaseRef == "" { + return nil, fmt.Errorf("base git ref is required") + } + absPath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + repoRoot := repomap.FindGitRoot(absPath) + if repoRoot == "" { + return nil, fmt.Errorf("not a git repository: %s", absPath) + } + relPath, err := filepath.Rel(repoRoot, absPath) + if err != nil { + return nil, err + } + + baseSha, err := resolveRef(ctx, repoRoot, opts.BaseRef) + if err != nil { + return nil, err + } + baseExport, err := scanAtRef(ctx, repoRoot, baseSha, relPath, opts.Options) + if err != nil { + return nil, err + } + + var headExport *Export + if opts.HeadRef == "" { + headExport, err = Scan(ctx, absPath, opts.Options) + if err != nil { + return nil, err + } + } else { + headSha, refErr := resolveRef(ctx, repoRoot, opts.HeadRef) + if refErr != nil { + return nil, refErr + } + headExport, err = scanAtRef(ctx, repoRoot, headSha, relPath, opts.Options) + if err != nil { + return nil, err + } + } + + comparison := Compare(baseExport, headExport) + comparison.Metadata = ComparisonMetadata{Path: absPath, BaseRef: opts.BaseRef, HeadRef: opts.HeadRef} + return comparison, nil +} + +func scanAtRef(ctx context.Context, repoRoot, sha, relPath string, opts Options) (*Export, error) { + worktree, cleanup, err := addWorktree(ctx, repoRoot, sha) + if err != nil { + return nil, err + } + defer cleanup() + scanDir := filepath.Join(worktree, relPath) + if _, statErr := os.Stat(scanDir); statErr != nil { + return nil, fmt.Errorf("path %q does not exist at ref %s: %w", relPath, sha, statErr) + } + return Scan(ctx, scanDir, opts) +} + +func addWorktree(ctx context.Context, repoRoot, sha string) (string, func(), error) { + parent, err := os.MkdirTemp("", "repomap-diff-*") + if err != nil { + return "", nil, err + } + worktree := filepath.Join(parent, "worktree") + if _, err := runGitCmd(ctx, repoRoot, "worktree", "add", "--detach", worktree, sha); err != nil { + _ = os.RemoveAll(parent) + return "", nil, err + } + cleanup := func() { + _, _ = runGitCmd(ctx, repoRoot, "worktree", "remove", "--force", worktree) + _ = os.RemoveAll(parent) + } + return worktree, cleanup, nil +} + +func resolveRef(ctx context.Context, repoRoot, ref string) (string, error) { + out, err := runGitCmd(ctx, repoRoot, "rev-parse", "--verify", ref+"^{commit}") + if err != nil { + return "", fmt.Errorf("invalid git ref %q: %w", ref, err) + } + return strings.TrimSpace(out), nil +} + +func runGitCmd(ctx context.Context, dir string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if msg := strings.TrimSpace(stderr.String()); msg != "" { + return "", fmt.Errorf("git %s: %s: %w", strings.Join(args, " "), msg, err) + } + return "", fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + } + return stdout.String(), nil +} diff --git a/deps/compare_scan_test.go b/deps/compare_scan_test.go new file mode 100644 index 0000000..f33bd1c --- /dev/null +++ b/deps/compare_scan_test.go @@ -0,0 +1,123 @@ +package deps + +import ( + "context" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func initRepo(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + dir := t.TempDir() + runGit(t, dir, "init") + runGit(t, dir, "config", "user.email", "test@example.com") + runGit(t, dir, "config", "user.name", "test") + return dir +} + +func gitOutput(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.Output() + if err != nil { + t.Fatalf("git %s failed: %v", strings.Join(args, " "), err) + } + return string(out) +} + +func TestCompareScanHeadDiffAgainstWorkingTree(t *testing.T) { + dir := initRepo(t) + gomod := filepath.Join(dir, "go.mod") + writeFile(t, gomod, "module github.com/acme/app\n\ngo 1.22\n\nrequire github.com/acme/a v1.0.0\n") + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-m", "base") + writeFile(t, gomod, "module github.com/acme/app\n\ngo 1.22\n\nrequire github.com/acme/a v1.5.0\n") + + cmp, err := CompareScan(context.Background(), dir, CompareOptions{Options: Options{MaxDepth: 1}, BaseRef: "HEAD"}) + if err != nil { + t.Fatal(err) + } + if len(cmp.Updated) != 1 || cmp.Updated[0].Name != "github.com/acme/a" || cmp.Updated[0].OldVersion != "v1.0.0" || cmp.Updated[0].NewVersion != "v1.5.0" { + t.Fatalf("expected a 1.0.0 -> 2.0.0, got %+v", cmp.Updated) + } + if lines := strings.TrimSpace(gitOutput(t, dir, "worktree", "list")); strings.Count(lines, "\n") != 0 { + t.Fatalf("worktree not cleaned up:\n%s", lines) + } +} + +func TestCompareScanRefRange(t *testing.T) { + dir := initRepo(t) + gomod := filepath.Join(dir, "go.mod") + writeFile(t, gomod, "module github.com/acme/app\n\ngo 1.22\n\nrequire github.com/acme/a v1.0.0\n") + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-m", "v1") + writeFile(t, gomod, "module github.com/acme/app\n\ngo 1.22\n\nrequire github.com/acme/a v1.5.0\n") + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-m", "v2") + + cmp, err := CompareScan(context.Background(), dir, CompareOptions{Options: Options{MaxDepth: 1}, BaseRef: "HEAD~1", HeadRef: "HEAD"}) + if err != nil { + t.Fatal(err) + } + if len(cmp.Updated) != 1 || cmp.Updated[0].NewVersion != "v1.5.0" { + t.Fatalf("ref1..ref2 diff failed: %+v", cmp.Updated) + } +} + +func TestCompareScanFilterAppliedBothSides(t *testing.T) { + dir := initRepo(t) + gomod := filepath.Join(dir, "go.mod") + writeFile(t, gomod, "module github.com/acme/app\n\ngo 1.22\n\nrequire (\n\tgithub.com/acme/alpha v1.0.0\n\tgithub.com/acme/beta v1.0.0\n)\n") + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-m", "base") + writeFile(t, gomod, "module github.com/acme/app\n\ngo 1.22\n\nrequire (\n\tgithub.com/acme/alpha v1.5.0\n\tgithub.com/acme/beta v1.5.0\n)\n") + + cmp, err := CompareScan(context.Background(), dir, CompareOptions{Options: Options{MaxDepth: 1, Filters: []string{"*alpha*"}}, BaseRef: "HEAD"}) + if err != nil { + t.Fatal(err) + } + if len(cmp.Updated) != 1 || !strings.Contains(cmp.Updated[0].Name, "alpha") { + t.Fatalf("filter should restrict both sides to alpha, got %+v", cmp.Updated) + } +} + +func TestCompareScanInvalidRef(t *testing.T) { + dir := initRepo(t) + writeFile(t, filepath.Join(dir, "go.mod"), "module github.com/acme/app\n\ngo 1.22\n") + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-m", "base") + + _, err := CompareScan(context.Background(), dir, CompareOptions{Options: Options{MaxDepth: 1}, BaseRef: "does-not-exist"}) + if err == nil || !strings.Contains(err.Error(), "invalid git ref") { + t.Fatalf("expected invalid ref error, got %v", err) + } +} + +func TestCompareScanNonGitDir(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), "module github.com/acme/app\n\ngo 1.22\n") + _, err := CompareScan(context.Background(), dir, CompareOptions{Options: Options{MaxDepth: 1}, BaseRef: "HEAD"}) + if err == nil || !strings.Contains(err.Error(), "not a git repository") { + t.Fatalf("expected non-git error, got %v", err) + } +} + +func TestCompareScanPathMissingAtRef(t *testing.T) { + dir := initRepo(t) + writeFile(t, filepath.Join(dir, "go.mod"), "module github.com/acme/app\n\ngo 1.22\n") + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-m", "base") + // svc only exists in the working tree, not at HEAD. + writeFile(t, filepath.Join(dir, "svc", "go.mod"), "module github.com/acme/svc\n\ngo 1.22\n") + + _, err := CompareScan(context.Background(), filepath.Join(dir, "svc"), CompareOptions{Options: Options{MaxDepth: 1}, BaseRef: "HEAD"}) + if err == nil || !strings.Contains(err.Error(), "does not exist at ref") { + t.Fatalf("expected path-missing-at-ref error, got %v", err) + } +} diff --git a/deps/compare_test.go b/deps/compare_test.go new file mode 100644 index 0000000..b2324cd --- /dev/null +++ b/deps/compare_test.go @@ -0,0 +1,139 @@ +package deps + +import ( + "path/filepath" + "testing" +) + +func goDep(name, version, scope string) *Node { + n := NewNode(ManagerGo, name, version) + n.Scope = scope + n.Depth = 1 + n.Direct = true + return n +} + +func goProject(metaPath, modRel string, children ...*Node) *Node { + root := NewNode(ManagerGo, "github.com/acme/"+modRel, "") + root.Path = filepath.Join(metaPath, modRel, "go.mod") + root.Children = children + return root +} + +func exportWith(metaPath string, roots ...*Node) *Export { + return &Export{Metadata: Metadata{Path: metaPath}, Roots: roots} +} + +func totalChanges(c *Comparison) int { + return len(c.Added) + len(c.Removed) + len(c.Updated) +} + +func TestCompareIdenticalIsEmpty(t *testing.T) { + base := exportWith("/repo", goProject("/repo", "svc", goDep("a", "1.0", "require"))) + head := exportWith("/repo", goProject("/repo", "svc", goDep("a", "1.0", "require"))) + got := Compare(base, head) + if totalChanges(got) != 0 { + t.Fatalf("identical graphs should produce no changes, got %+v", got) + } +} + +func TestCompareAddRemoveUpdate(t *testing.T) { + base := exportWith("/repo", goProject("/repo", "svc", + goDep("keep", "1.0", "require"), + goDep("bump", "1.0", "require"), + goDep("gone", "1.0", "require"), + )) + head := exportWith("/repo", goProject("/repo", "svc", + goDep("keep", "1.0", "require"), + goDep("bump", "2.0", "require"), + goDep("fresh", "0.1", "require"), + )) + got := Compare(base, head) + if len(got.Added) != 1 || got.Added[0].Name != "fresh" || got.Added[0].NewVersion != "0.1" { + t.Fatalf("added = %+v", got.Added) + } + if len(got.Removed) != 1 || got.Removed[0].Name != "gone" { + t.Fatalf("removed = %+v", got.Removed) + } + if len(got.Updated) != 1 || got.Updated[0].OldVersion != "1.0" || got.Updated[0].NewVersion != "2.0" { + t.Fatalf("updated = %+v", got.Updated) + } + if got.Updated[0].Project != "svc/go.mod" { + t.Fatalf("project key = %q, want svc/go.mod", got.Updated[0].Project) + } +} + +func TestCompareScopeOnlyChangeIsUpdate(t *testing.T) { + base := exportWith("/repo", goProject("/repo", "svc", goDep("a", "1.0", "require"))) + head := exportWith("/repo", goProject("/repo", "svc", goDep("a", "1.0", "indirect"))) + got := Compare(base, head) + if len(got.Updated) != 1 || got.Updated[0].OldScope != "require" || got.Updated[0].NewScope != "indirect" { + t.Fatalf("scope-only update not detected: %+v", got.Updated) + } +} + +func TestComparePerProjectKeying(t *testing.T) { + base := exportWith("/repo", + goProject("/repo", "a", goDep("lib", "1.0", "require")), + goProject("/repo", "b", goDep("lib", "1.0", "require")), + ) + head := exportWith("/repo", + goProject("/repo", "a", goDep("lib", "2.0", "require")), + goProject("/repo", "b", goDep("lib", "1.0", "require")), + ) + got := Compare(base, head) + if len(got.Updated) != 1 || got.Updated[0].Project != "a/go.mod" { + t.Fatalf("expected only project a to change, got %+v", got.Updated) + } +} + +func TestCompareDuplicateVersionSets(t *testing.T) { + dupA := goDep("lib", "1.0", "require") + dupB := goDep("lib", "2.0", "require") + base := exportWith("/repo", goProject("/repo", "svc", dupA, dupB)) + head := exportWith("/repo", goProject("/repo", "svc", goDep("lib", "2.0", "require"))) + got := Compare(base, head) + if len(got.Updated) != 1 || got.Updated[0].OldVersion != "1.0, 2.0" || got.Updated[0].NewVersion != "2.0" { + t.Fatalf("duplicate version set not joined: %+v", got.Updated) + } +} + +func TestComparePathNormalization(t *testing.T) { + base := exportWith("/repo/base", goProject("/repo/base", "svc", goDep("a", "1.0", "require"))) + head := exportWith("/repo/head", goProject("/repo/head", "svc", goDep("a", "2.0", "require"))) + got := Compare(base, head) + if len(got.Updated) != 1 || got.Updated[0].Project != "svc/go.mod" { + t.Fatalf("absolute worktree paths not normalized to relative project key: %+v", got.Updated) + } +} + +func TestCompareWholeProjectAddRemove(t *testing.T) { + base := exportWith("/repo", goProject("/repo", "old", goDep("a", "1.0", "require"))) + head := exportWith("/repo", goProject("/repo", "new", goDep("b", "1.0", "require"))) + got := Compare(base, head) + if len(got.Removed) != 1 || got.Removed[0].Project != "old/go.mod" { + t.Fatalf("removed project deps = %+v", got.Removed) + } + if len(got.Added) != 1 || got.Added[0].Project != "new/go.mod" { + t.Fatalf("added project deps = %+v", got.Added) + } +} + +func TestCompareDeterministicOrdering(t *testing.T) { + base := exportWith("/repo", goProject("/repo", "svc")) + head := exportWith("/repo", goProject("/repo", "svc", + goDep("zebra", "1.0", "require"), + goDep("alpha", "1.0", "require"), + goDep("mango", "1.0", "require"), + )) + got := Compare(base, head) + want := []string{"alpha", "mango", "zebra"} + if len(got.Added) != 3 { + t.Fatalf("added = %d, want 3", len(got.Added)) + } + for i, name := range want { + if got.Added[i].Name != name { + t.Fatalf("added[%d] = %q, want %q (sorted)", i, got.Added[i].Name, name) + } + } +} diff --git a/deps/discover.go b/deps/discover.go index 3780d41..0286635 100644 --- a/deps/discover.go +++ b/deps/discover.go @@ -21,6 +21,14 @@ var ignoredDirs = map[string]bool{ } func Discover(root string, managers []Manager) ([]Project, []Warning, error) { + return discover(root, managers, true) +} + +func discoverOffline(root string, managers []Manager) ([]Project, []Warning, error) { + return discover(root, managers, false) +} + +func discover(root string, managers []Manager, useGit bool) ([]Project, []Warning, error) { selected := managerSet(managers) absRoot, err := filepath.Abs(root) if err != nil { @@ -34,7 +42,7 @@ func Discover(root string, managers []Manager) ([]Project, []Warning, error) { absRoot = filepath.Dir(absRoot) } - files, err := discoverManifestFiles(absRoot) + files, err := discoverManifestFiles(absRoot, useGit) if err != nil { return nil, nil, err } @@ -112,9 +120,11 @@ func Discover(root string, managers []Manager) ([]Project, []Warning, error) { return projects, warnings, nil } -func discoverManifestFiles(root string) ([]string, error) { - if files, ok := gitManifestFiles(root); ok { - return files, nil +func discoverManifestFiles(root string, useGit bool) ([]string, error) { + if useGit { + if files, ok := gitManifestFiles(root); ok { + return files, nil + } } return walkManifestFiles(root) } diff --git a/deps/edgegraph.go b/deps/edgegraph.go new file mode 100644 index 0000000..5699c7e --- /dev/null +++ b/deps/edgegraph.go @@ -0,0 +1,89 @@ +package deps + +// edgeGraph is a manager-agnostic adjacency representation of a resolved +// dependency graph. Keys identify nodes (typically Node.ID); Nodes holds a +// childless template per key and Edges lists each node's direct children in a +// stable order. Transitive resolvers (go mod graph, maven dependency:tree, +// gradle dependencies) build an edgeGraph and hand it to buildTreeFromEdgeGraph. +type edgeGraph struct { + RootKey string + Nodes map[string]*Node + Edges map[string][]string +} + +type edgeTreeOptions struct { + Graph edgeGraph + MaxDepth int +} + +// buildTreeFromEdgeGraph expands an edgeGraph into a Node tree. Each node is +// expanded (its children materialized) only at its shallowest BFS occurrence; +// deeper re-occurrences become childless leaves so existing analyzeDuplicates +// can tag them and filterAndPruneAt can mark ancestor-path repeats Circular. +// Children beyond MaxDepth (>0) are pruned during construction. +func buildTreeFromEdgeGraph(opts edgeTreeOptions) *Node { + graph := opts.Graph + if graph.RootKey == "" || graph.Nodes[graph.RootKey] == nil { + return nil + } + depths := edgeGraphDepths(graph) + expanded := map[string]bool{} + + var build func(key string, depth int) *Node + build = func(key string, depth int) *Node { + template := graph.Nodes[key] + if template == nil { + return nil + } + node := template.cloneShallow() + node.Depth = depth + if depth == depths[key] && !expanded[key] { + expanded[key] = true + for _, childKey := range graph.Edges[key] { + if opts.MaxDepth > 0 && depth+1 > opts.MaxDepth { + break + } + if child := build(childKey, depth+1); child != nil { + node.Children = append(node.Children, child) + } + } + } + return node + } + + root := build(graph.RootKey, 0) + sortChildren(root) + return root +} + +// markDirectByDepth flags depth-1 nodes as direct dependencies, matching the +// offline manifest resolvers that mark declared dependencies as direct. +func markDirectByDepth(root *Node) { + if root == nil { + return + } + for _, child := range root.Children { + if child != nil { + child.Direct = child.Depth == 1 + } + } +} + +// edgeGraphDepths returns the shortest-path depth of every reachable node from +// RootKey via breadth-first traversal. +func edgeGraphDepths(graph edgeGraph) map[string]int { + depths := map[string]int{graph.RootKey: 0} + queue := []string{graph.RootKey} + for len(queue) > 0 { + key := queue[0] + queue = queue[1:] + for _, child := range graph.Edges[key] { + if _, seen := depths[child]; seen { + continue + } + depths[child] = depths[key] + 1 + queue = append(queue, child) + } + } + return depths +} diff --git a/deps/edgegraph_test.go b/deps/edgegraph_test.go new file mode 100644 index 0000000..a608f84 --- /dev/null +++ b/deps/edgegraph_test.go @@ -0,0 +1,99 @@ +package deps + +import "testing" + +// diamondGraph: root -> a, b; a -> shared; b -> shared. +func diamondGraph() edgeGraph { + mk := func(name string) *Node { return NewNode(ManagerGo, name, "v1") } + keyA := mk("a").ID + keyB := mk("b").ID + keyShared := mk("shared").ID + root := mk("root") + return edgeGraph{ + RootKey: root.ID, + Nodes: map[string]*Node{ + root.ID: root, + keyA: mk("a"), + keyB: mk("b"), + keyShared: mk("shared"), + }, + Edges: map[string][]string{ + root.ID: {keyA, keyB}, + keyA: {keyShared}, + keyB: {keyShared}, + }, + } +} + +func TestEdgeGraphDepthsDiamond(t *testing.T) { + graph := diamondGraph() + depths := edgeGraphDepths(graph) + if depths[NewNode(ManagerGo, "shared", "v1").ID] != 2 { + t.Fatalf("shared depth = %d, want 2", depths[NewNode(ManagerGo, "shared", "v1").ID]) + } + if depths[NewNode(ManagerGo, "a", "v1").ID] != 1 { + t.Fatalf("a depth = %d, want 1", depths[NewNode(ManagerGo, "a", "v1").ID]) + } +} + +func TestBuildTreeFirstOccurrenceExpandsOnceDuplicateLeaf(t *testing.T) { + root := buildTreeFromEdgeGraph(edgeTreeOptions{Graph: diamondGraph()}) + if root == nil || len(root.Children) != 2 { + t.Fatalf("root children = %#v", root) + } + a := findChild(root, "a") + b := findChild(root, "b") + occurrences := 0 + for _, branch := range []*Node{a, b} { + shared := findChild(branch, "shared") + if shared == nil { + continue + } + occurrences++ + if len(shared.Children) != 0 { + t.Fatalf("shared should be a childless leaf at re-occurrence, got %#v", shared) + } + } + if occurrences != 2 { + t.Fatalf("shared should appear under both branches, got %d", occurrences) + } +} + +func TestBuildTreeTerminatesOnCycle(t *testing.T) { + mk := func(name string) *Node { return NewNode(ManagerGo, name, "v1") } + rootKey := mk("root").ID + aKey := mk("a").ID + graph := edgeGraph{ + RootKey: rootKey, + Nodes: map[string]*Node{rootKey: mk("root"), aKey: mk("a")}, + Edges: map[string][]string{ + rootKey: {aKey}, + aKey: {rootKey}, // cycle a -> root + }, + } + root := buildTreeFromEdgeGraph(edgeTreeOptions{Graph: graph}) + a := findChild(root, "a") + if a == nil { + t.Fatalf("expected child a, got %#v", root) + } + // root re-occurs as a leaf under a (deeper than its BFS depth 0), not expanded. + rootLeaf := findChild(a, "root") + if rootLeaf == nil || len(rootLeaf.Children) != 0 { + t.Fatalf("cycle back-edge should be a leaf, got %#v", rootLeaf) + } + // filterAndPrune marks the ancestor-path repeat as circular. + pruned := filterAndPrune(root, nil, 0) + if leaf := findChild(findChild(pruned, "a"), "root"); leaf == nil || !leaf.Circular { + t.Fatalf("expected circular marker on back-edge, got %#v", leaf) + } +} + +func TestBuildTreeMaxDepthPruning(t *testing.T) { + root := buildTreeFromEdgeGraph(edgeTreeOptions{Graph: diamondGraph(), MaxDepth: 1}) + if root == nil || len(root.Children) != 2 { + t.Fatalf("root children = %#v", root) + } + if a := findChild(root, "a"); a == nil || len(a.Children) != 0 { + t.Fatalf("depth-1 child should have no children at MaxDepth 1, got %#v", a) + } +} diff --git a/deps/filter.go b/deps/filter.go index cadd3fa..ddb3216 100644 --- a/deps/filter.go +++ b/deps/filter.go @@ -3,7 +3,6 @@ package deps import ( "fmt" "sort" - "strings" "github.com/flanksource/commons/collections" ) @@ -236,16 +235,3 @@ func cloneBoolMap(in map[string]bool) map[string]bool { } return out } - -func splitPatterns(values []string) []string { - var out []string - for _, value := range values { - for _, part := range strings.Split(value, ",") { - part = strings.TrimSpace(part) - if part != "" { - out = append(out, part) - } - } - } - return out -} diff --git a/deps/go.go b/deps/go.go index a4cfb75..7ca88a9 100644 --- a/deps/go.go +++ b/deps/go.go @@ -1,67 +1,15 @@ package deps import ( - "bufio" - "bytes" - "context" - "encoding/json" "fmt" "os" "path/filepath" - "strings" "golang.org/x/mod/modfile" ) -type goModuleInfo struct { - Path string `json:"Path"` - Version string `json:"Version"` - Main bool `json:"Main"` - Replace *goModuleInfo `json:"Replace"` - Dir string `json:"Dir"` -} - -func resolveGoNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { - graphResult, err := opts.Runner.Run(ctx, Command{ - Dir: project.Dir, - Name: "go", - Args: []string{"mod", "graph"}, - Env: []string{"GOFLAGS=-mod=readonly"}, - }) - if err != nil { - return nil, nil, err - } - listResult, listErr := opts.Runner.Run(ctx, Command{ - Dir: project.Dir, - Name: "go", - Args: []string{"list", "-m", "-json", "all"}, - Env: []string{"GOFLAGS=-mod=readonly"}, - }) - infos := map[string]goModuleInfo{} - if listErr == nil { - infos = parseGoModuleInfos([]byte(listResult.Stdout)) - } - - rootToken, err := goRootToken(project, infos) - if err != nil { - return nil, nil, err - } - direct := goDirectRequires(project.File) - root := buildGoGraph(rootToken, parseGoGraph(graphResult.Stdout), infos, direct) - root.Path = project.File - root.Source = "go mod graph" - if listErr != nil { - return root, []Warning{{Manager: ManagerGo, Project: project.Dir, Message: "go list -m -json all failed; replacement metadata may be incomplete: " + listErr.Error()}}, nil - } - return root, nil, nil -} - -func resolveGoManifest(project Project) (*Node, []Warning, error) { - data, err := os.ReadFile(filepath.Join(project.Dir, "go.mod")) - if err != nil { - return nil, nil, err - } - file, err := modfile.Parse("go.mod", data, nil) +func resolveGoManifest(project Project, opts Options) (*Node, []Warning, error) { + file, err := loadGoModFile(project.Dir) if err != nil { return nil, nil, err } @@ -69,6 +17,9 @@ func resolveGoManifest(project Project) (*Node, []Warning, error) { root.Path = filepath.Join(project.Dir, "go.mod") root.Source = "go.mod" for _, req := range file.Require { + if req.Indirect && !opts.IncludeIndirect { + continue + } child := NewNode(ManagerGo, req.Mod.Path, req.Mod.Version) child.Depth = 1 child.Direct = !req.Indirect @@ -83,130 +34,15 @@ func resolveGoManifest(project Project) (*Node, []Warning, error) { root.Children = append(root.Children, child) } sortChildren(root) - return root, []Warning{{Manager: ManagerGo, Project: project.Dir, Message: "manifest fallback includes go.mod requirements only; transitive edges are unavailable"}}, nil -} - -func parseGoModuleInfos(data []byte) map[string]goModuleInfo { - out := map[string]goModuleInfo{} - dec := json.NewDecoder(bytes.NewReader(data)) - for { - var info goModuleInfo - if err := dec.Decode(&info); err != nil { - break - } - if info.Path == "" { - continue - } - out[goToken(info.Path, info.Version)] = info - if info.Main { - out[info.Path] = info - } - } - return out -} - -func parseGoGraph(stdout string) map[string][]string { - out := map[string][]string{} - scanner := bufio.NewScanner(strings.NewReader(stdout)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - fields := strings.Fields(line) - if len(fields) != 2 { - continue - } - out[fields[0]] = append(out[fields[0]], fields[1]) - } - for key := range out { - sortStrings(out[key]) - } - return out + return root, []Warning{{Manager: ManagerGo, Project: project.Dir, Message: "offline go.mod parsing includes declared requirements only; transitive edges are unavailable"}}, nil } -func goRootToken(project Project, infos map[string]goModuleInfo) (string, error) { - for token, info := range infos { - if info.Main { - return token, nil - } - } - data, err := os.ReadFile(filepath.Join(project.Dir, "go.mod")) - if err != nil { - return "", err - } - file, err := modfile.Parse("go.mod", data, nil) +func loadGoModFile(dir string) (*modfile.File, error) { + data, err := os.ReadFile(filepath.Join(dir, "go.mod")) if err != nil { - return "", err - } - return file.Module.Mod.Path, nil -} - -func goDirectRequires(path string) map[string]bool { - out := map[string]bool{} - data, err := os.ReadFile(path) - if err != nil { - return out - } - file, err := modfile.Parse("go.mod", data, nil) - if err != nil { - return out - } - for _, req := range file.Require { - out[goToken(req.Mod.Path, req.Mod.Version)] = !req.Indirect - } - return out -} - -func buildGoGraph(rootToken string, edges map[string][]string, infos map[string]goModuleInfo, direct map[string]bool) *Node { - var build func(token string, depth int, path map[string]bool) *Node - build = func(token string, depth int, path map[string]bool) *Node { - name, version := splitGoToken(token) - node := NewNode(ManagerGo, name, version) - node.Depth = depth - node.Source = "go mod graph" - if depth == 1 { - if isDirect, ok := direct[token]; ok { - node.Direct = isDirect - if isDirect { - node.Scope = "require" - } else { - node.Scope = "indirect" - } - } - } - if info, ok := infos[token]; ok && info.Replace != nil { - node.Source = goReplaceSource(info.Replace.Path, info.Replace.Version) - node.Local = isLocalRef(info.Replace.Path) || info.Replace.Dir != "" - } - if path[token] { - node.Circular = true - return node - } - path[token] = true - for _, childToken := range edges[token] { - child := build(childToken, depth+1, cloneBoolMap(path)) - node.Children = append(node.Children, child) - } - sortChildren(node) - return node - } - return build(rootToken, 0, map[string]bool{}) -} - -func splitGoToken(token string) (string, string) { - name, version, ok := strings.Cut(token, "@") - if !ok { - return token, "" - } - return name, version -} - -func goToken(path, version string) string { - if version == "" { - return path + return nil, err } - return path + "@" + version + return modfile.Parse("go.mod", data, nil) } func goReplaceFor(file *modfile.File, path, version string) *modfile.Replace { diff --git a/deps/go_graph.go b/deps/go_graph.go new file mode 100644 index 0000000..70bd74b --- /dev/null +++ b/deps/go_graph.go @@ -0,0 +1,134 @@ +package deps + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "strings" + + "golang.org/x/mod/modfile" +) + +// resolveGoGraph resolves the transitive Go dependency graph by shelling out to +// `go mod graph`. It fails fast with a toolError when the go binary is missing +// or the command fails, suggesting --depth 1 for offline direct-only output. +func resolveGoGraph(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + if _, err := exec.LookPath("go"); err != nil { + return nil, nil, toolError{fmt.Errorf("go binary not found on PATH; rerun with --depth 1 for offline direct-only output: %w", err)} + } + file, err := loadGoModFile(project.Dir) + if err != nil { + return nil, nil, err + } + mainModule := file.Module.Mod.Path + + result, err := opts.Runner.Run(ctx, Command{ + Dir: project.Dir, + Name: "go", + Args: []string{"mod", "graph"}, + Env: []string{"GOFLAGS=-mod=mod"}, + }) + if err != nil { + detail := strings.TrimSpace(result.Stderr) + if detail == "" { + detail = err.Error() + } + return nil, nil, toolError{fmt.Errorf("go mod graph failed in %s (rerun with --depth 1 for offline direct-only output): %s", project.Dir, detail)} + } + + graph, err := parseGoModGraph(result.Stdout, mainModule) + if err != nil { + return nil, nil, toolError{fmt.Errorf("go mod graph parse failed in %s: %w", project.Dir, err)} + } + + root := buildTreeFromEdgeGraph(edgeTreeOptions{Graph: graph, MaxDepth: opts.MaxDepth}) + if root == nil { + return nil, nil, toolError{fmt.Errorf("go mod graph produced no nodes for %s", mainModule)} + } + root.Path = filepath.Join(project.Dir, "go.mod") + root.Source = "go.mod" + applyGoModMetadata(root, file) + + warning := Warning{ + Manager: ManagerGo, + Project: project.Dir, + Message: "go mod graph reports the module requirement graph (MVS inputs); it may list module versions that are not selected in the final build list", + } + return root, []Warning{warning}, nil +} + +// parseGoModGraph converts `go mod graph` output into an edgeGraph. Each line is +// " " where a node is "path@version" (the main module appears without +// a version). The root key is the main module. +func parseGoModGraph(output, mainModule string) (edgeGraph, error) { + graph := edgeGraph{ + RootKey: NodeID(ManagerGo, mainModule, ""), + Nodes: map[string]*Node{}, + Edges: map[string][]string{}, + } + graph.Nodes[graph.RootKey] = NewNode(ManagerGo, mainModule, "") + + seenEdge := map[string]bool{} + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != 2 { + return edgeGraph{}, fmt.Errorf("malformed go mod graph line: %q", line) + } + fromKey := goModNodeKey(&graph, fields[0]) + toKey := goModNodeKey(&graph, fields[1]) + edgeKey := fromKey + ">" + toKey + if seenEdge[edgeKey] { + continue + } + seenEdge[edgeKey] = true + graph.Edges[fromKey] = append(graph.Edges[fromKey], toKey) + } + return graph, nil +} + +func goModNodeKey(graph *edgeGraph, field string) string { + path, version, _ := strings.Cut(field, "@") + key := NodeID(ManagerGo, path, version) + if _, ok := graph.Nodes[key]; !ok { + graph.Nodes[key] = NewNode(ManagerGo, path, version) + } + return key +} + +// applyGoModMetadata overlays go.mod metadata onto the resolved graph: direct vs +// indirect requirement scope and replace directives (Source/Local). +func applyGoModMetadata(root *Node, file *modfile.File) { + requires := map[string]*modfile.Require{} + for _, req := range file.Require { + requires[req.Mod.Path] = req + } + var walk func(n *Node) + walk = func(n *Node) { + if n == nil { + return + } + if req, ok := requires[n.Name]; ok { + n.Direct = !req.Indirect + if req.Indirect { + n.Scope = "indirect" + } else { + n.Scope = "require" + } + } + if rep := goReplaceFor(file, n.Name, n.Version); rep != nil { + n.Source = goReplaceSource(rep.New.Path, rep.New.Version) + n.Local = isLocalRef(rep.New.Path) + } + for _, child := range n.Children { + walk(child) + } + } + for _, child := range root.Children { + walk(child) + } +} diff --git a/deps/go_graph_test.go b/deps/go_graph_test.go new file mode 100644 index 0000000..6ec3051 --- /dev/null +++ b/deps/go_graph_test.go @@ -0,0 +1,144 @@ +package deps + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + "time" +) + +type fakeRunner struct { + result CommandResult + err error + calls []Command + onRun func(Command) (CommandResult, error) +} + +func (f *fakeRunner) Run(_ context.Context, cmd Command) (CommandResult, error) { + f.calls = append(f.calls, cmd) + if f.onRun != nil { + return f.onRun(cmd) + } + return f.result, f.err +} + +func TestParseGoModGraph(t *testing.T) { + output := strings.Join([]string{ + "github.com/acme/app github.com/acme/lib@v1.2.3", + "github.com/acme/app github.com/acme/other@v0.5.0", + "github.com/acme/lib@v1.2.3 github.com/acme/dep@v0.1.0", + "github.com/acme/other@v0.5.0 github.com/acme/dep@v0.1.0", + "", + }, "\n") + + graph, err := parseGoModGraph(output, "github.com/acme/app") + if err != nil { + t.Fatal(err) + } + rootKey := NodeID(ManagerGo, "github.com/acme/app", "") + if graph.RootKey != rootKey { + t.Fatalf("root key = %q, want %q", graph.RootKey, rootKey) + } + if len(graph.Edges[rootKey]) != 2 { + t.Fatalf("root edges = %d, want 2", len(graph.Edges[rootKey])) + } + depKey := NodeID(ManagerGo, "github.com/acme/dep", "v0.1.0") + depths := edgeGraphDepths(graph) + if depths[depKey] != 2 { + t.Fatalf("dep depth = %d, want 2 (diamond shortest path)", depths[depKey]) + } +} + +func TestResolveGoGraphMergesGoModMetadata(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require ( + github.com/acme/lib v1.2.3 + github.com/acme/indirect v0.1.0 // indirect +) + +replace github.com/acme/lib => ../lib +`) + runner := &fakeRunner{result: CommandResult{Stdout: strings.Join([]string{ + "github.com/acme/app github.com/acme/lib@v1.2.3", + "github.com/acme/app github.com/acme/indirect@v0.1.0", + "github.com/acme/lib@v1.2.3 github.com/acme/dep@v0.3.0", + "", + }, "\n")}} + + root, warnings, err := resolveGoGraph(context.Background(), Project{Manager: ManagerGo, Dir: dir}, Options{Runner: runner, MaxDepth: 0}) + if err != nil { + t.Fatal(err) + } + if len(warnings) != 1 || !strings.Contains(warnings[0].Message, "MVS") { + t.Fatalf("expected MVS warning, got %#v", warnings) + } + lib := findChild(root, "github.com/acme/lib") + if lib == nil || !lib.Direct || lib.Scope != "require" || !lib.Local || lib.Source != "../lib" { + t.Fatalf("lib metadata not merged: %#v", lib) + } + indirect := findChild(root, "github.com/acme/indirect") + if indirect == nil || indirect.Direct || indirect.Scope != "indirect" { + t.Fatalf("indirect metadata not merged: %#v", indirect) + } + dep := findChild(lib, "github.com/acme/dep") + if dep == nil || dep.Depth != 2 { + t.Fatalf("transitive dep not resolved at depth 2: %#v", dep) + } +} + +func TestResolveGoGraphToolFailureIsFatal(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require github.com/acme/lib v1.2.3 +`) + runner := &fakeRunner{err: fmt.Errorf("exit status 1"), result: CommandResult{Stderr: "go: updates to go.mod needed"}} + + _, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerGo}, + MaxDepth: 0, + Runner: runner, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err == nil { + t.Fatal("expected fatal tool error to abort scan") + } + if !strings.Contains(err.Error(), "--depth 1") { + t.Fatalf("error should suggest --depth 1, got %v", err) + } +} + +func TestScanGoDepthOneStaysOffline(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), `module github.com/acme/app + +go 1.22 + +require github.com/acme/lib v1.2.3 +`) + runner := &fakeRunner{} + + got, err := Scan(context.Background(), dir, Options{ + Managers: []Manager{ManagerGo}, + MaxDepth: 1, + Runner: runner, + Now: func() time.Time { return time.Unix(1, 0).UTC() }, + }) + if err != nil { + t.Fatal(err) + } + if len(runner.calls) != 0 { + t.Fatalf("runner should not be called at depth 1, got %d calls", len(runner.calls)) + } + if len(got.Roots) != 1 { + t.Fatalf("roots = %d, want 1", len(got.Roots)) + } +} diff --git a/deps/go_test.go b/deps/go_test.go new file mode 100644 index 0000000..3d2b7c8 --- /dev/null +++ b/deps/go_test.go @@ -0,0 +1,49 @@ +package deps + +import ( + "path/filepath" + "testing" +) + +const goModWithIndirect = `module github.com/acme/app + +go 1.22 + +require ( + github.com/acme/lib v1.2.3 + github.com/acme/indirect v0.1.0 // indirect +) +` + +func TestGoManifestExcludesIndirectByDefault(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), goModWithIndirect) + + root, _, err := resolveGoManifest(Project{Manager: ManagerGo, Dir: dir}, Options{}) + if err != nil { + t.Fatal(err) + } + if len(root.Children) != 1 { + t.Fatalf("children = %d, want 1 (indirect excluded)", len(root.Children)) + } + if root.Children[0].Name != "github.com/acme/lib" { + t.Fatalf("unexpected direct child: %#v", root.Children[0]) + } +} + +func TestGoManifestIncludeIndirect(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "go.mod"), goModWithIndirect) + + root, _, err := resolveGoManifest(Project{Manager: ManagerGo, Dir: dir}, Options{IncludeIndirect: true}) + if err != nil { + t.Fatal(err) + } + if len(root.Children) != 2 { + t.Fatalf("children = %d, want 2 (indirect included)", len(root.Children)) + } + indirect := findChild(root, "github.com/acme/indirect") + if indirect == nil || indirect.Direct || indirect.Scope != "indirect" { + t.Fatalf("indirect metadata not captured: %#v", indirect) + } +} diff --git a/deps/gradle.go b/deps/gradle.go index 2ab8cf1..488420a 100644 --- a/deps/gradle.go +++ b/deps/gradle.go @@ -1,194 +1,18 @@ package deps import ( - "context" - "encoding/json" - "fmt" "os" "path/filepath" "regexp" "strings" ) -type gradleExport struct { - Projects []gradleProject `json:"projects"` -} - -type gradleProject struct { - Name string `json:"name"` - Path string `json:"path"` - Configurations []gradleConfiguration `json:"configurations"` -} - -type gradleConfiguration struct { - Name string `json:"name"` - Dependencies []gradleNode `json:"dependencies"` -} - -type gradleNode struct { - Group string `json:"group"` - Module string `json:"module"` - Version string `json:"version"` - Selected string `json:"selected"` - Children []gradleNode `json:"children"` -} - -func resolveGradleNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { - tmp, err := os.CreateTemp("", "repomap-gradle-*.json") - if err != nil { - return nil, nil, err - } - tmpPath := tmp.Name() - _ = tmp.Close() - defer os.Remove(tmpPath) - - initFile, err := os.CreateTemp("", "repomap-gradle-*.gradle") - if err != nil { - return nil, nil, err - } - initPath := initFile.Name() - if _, err := initFile.WriteString(gradleInitScript(tmpPath, opts.Configurations)); err != nil { - _ = initFile.Close() - return nil, nil, err - } - _ = initFile.Close() - defer os.Remove(initPath) - - bin := "gradle" - args := []string{"-I", initPath, "-q", "repomapDeps"} - if _, err := os.Stat(filepath.Join(project.Dir, "gradlew")); err == nil { - bin = "./gradlew" - } - _, err = opts.Runner.Run(ctx, Command{Dir: project.Dir, Name: bin, Args: args}) - if err != nil { - return nil, nil, err - } - data, err := os.ReadFile(tmpPath) - if err != nil { - return nil, nil, err - } - root, err := parseGradleJSON(data, project) - if err != nil { - return nil, nil, err - } - root.Path = project.File - root.Source = "gradle ResolutionResult" - return root, nil, nil -} - func resolveGradleManifest(project Project) (*Node, []Warning, error) { root, err := parseGradleBuildFile(project.File) if err != nil { return nil, nil, err } - return root, []Warning{{Manager: ManagerGradle, Project: project.Dir, Message: "manifest fallback includes direct Gradle dependency declarations only; resolved transitive edges are unavailable"}}, nil -} - -func gradleInitScript(outputPath string, configurations []string) string { - quotedOutput := strings.ReplaceAll(outputPath, "\\", "\\\\") - quotedOutput = strings.ReplaceAll(quotedOutput, "'", "\\'") - var configSet string - if len(configurations) > 0 { - var quoted []string - for _, cfg := range configurations { - cfg = strings.TrimSpace(cfg) - if cfg != "" { - quoted = append(quoted, "'"+strings.ReplaceAll(cfg, "'", "\\'")+"'") - } - } - configSet = "[" + strings.Join(quoted, ",") + "] as Set" - } else { - configSet = "[] as Set" - } - return fmt.Sprintf(` -import groovy.json.JsonOutput -gradle.projectsEvaluated { - rootProject.tasks.register('repomapDeps') { - doLast { - def selectedConfigurations = %s - def seen = [] as Set - def convert - convert = { dep, depth -> - def id = dep.selected.id - def group = id.hasProperty('group') ? id.group : '' - def module = id.hasProperty('module') ? id.module : id.displayName - def version = id.hasProperty('version') ? id.version : '' - def key = group + ':' + module + ':' + version - if (seen.contains(key + ':' + depth)) { - return [group: group, module: module, version: version, children: []] - } - seen.add(key + ':' + depth) - return [group: group, module: module, version: version, selected: id.displayName, - children: dep.selected.dependencies.findAll { it instanceof org.gradle.api.artifacts.result.ResolvedDependencyResult }.collect { convert(it, depth + 1) }] - } - def projects = [] - allprojects.each { prj -> - def configs = [] - prj.configurations.findAll { it.canBeResolved && (selectedConfigurations.isEmpty() || selectedConfigurations.contains(it.name)) }.each { cfg -> - try { - configs << [name: cfg.name, dependencies: cfg.incoming.resolutionResult.root.dependencies.findAll { it instanceof org.gradle.api.artifacts.result.ResolvedDependencyResult }.collect { convert(it, 1) }] - } catch (Throwable ignored) {} - } - if (!configs.isEmpty()) { - projects << [name: prj.name, path: prj.path, configurations: configs] - } - } - new File('%s').text = JsonOutput.toJson([projects: projects]) - } - } -} -`, configSet, quotedOutput) -} - -func parseGradleJSON(data []byte, project Project) (*Node, error) { - var payload gradleExport - if err := json.Unmarshal(data, &payload); err != nil { - return nil, err - } - root := NewNode(ManagerGradle, filepath.Base(project.Dir), "") - root.Source = "gradle" - for _, p := range payload.Projects { - projectNode := NewNode(ManagerGradle, firstNonEmpty(p.Path, p.Name), "") - projectNode.Depth = 1 - projectNode.Direct = true - projectNode.Source = "project" - for _, cfg := range p.Configurations { - cfgNode := NewNode(ManagerGradle, cfg.Name, "") - cfgNode.Depth = 2 - cfgNode.Scope = cfg.Name - cfgNode.Source = "configuration" - for _, dep := range cfg.Dependencies { - child := convertGradleNode(dep, 3, cfg.Name) - child.Direct = true - cfgNode.Children = append(cfgNode.Children, child) - } - sortChildren(cfgNode) - projectNode.Children = append(projectNode.Children, cfgNode) - } - sortChildren(projectNode) - root.Children = append(root.Children, projectNode) - } - sortChildren(root) - return root, nil -} - -func convertGradleNode(dep gradleNode, depth int, scope string) *Node { - name := dep.Module - if dep.Group != "" { - name = dep.Group + ":" + dep.Module - } - if name == "" { - name = dep.Selected - } - node := NewNode(ManagerGradle, name, dep.Version) - node.Depth = depth - node.Scope = scope - node.Source = "gradle" - for _, childDep := range dep.Children { - node.Children = append(node.Children, convertGradleNode(childDep, depth+1, scope)) - } - sortChildren(node) - return node + return root, []Warning{{Manager: ManagerGradle, Project: project.Dir, Message: "offline Gradle build-file parsing includes direct dependency declarations only; resolved transitive edges are unavailable"}}, nil } var gradleDepRe = regexp.MustCompile(`(?m)^\s*([A-Za-z][A-Za-z0-9_]*(?:Implementation|CompileOnly|RuntimeOnly|Api|TestImplementation|testImplementation|implementation|api|compileOnly|runtimeOnly|testRuntimeOnly|annotationProcessor|kapt)?)\s*(?:\(?\s*)["']([^:"']+):([^:"']+):([^"']+)["']`) diff --git a/deps/gradle_tree.go b/deps/gradle_tree.go new file mode 100644 index 0000000..7f2a3c5 --- /dev/null +++ b/deps/gradle_tree.go @@ -0,0 +1,202 @@ +package deps + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +const gradleDefaultConfiguration = "runtimeClasspath" + +// resolveGradleTree resolves the transitive Gradle dependency graph by running +// the `dependencies` task for the runtimeClasspath configuration. It fails fast +// with a toolError when gradle is missing, the command fails, or the +// configuration is absent, suggesting --depth 1 for offline output. +func resolveGradleTree(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + command := gradleCommand(project.Dir) + result, err := opts.Runner.Run(ctx, Command{ + Dir: project.Dir, + Name: command, + Args: []string{"-q", "dependencies", "--configuration", gradleDefaultConfiguration}, + }) + if err != nil { + detail := strings.TrimSpace(result.Stderr) + if detail == "" { + detail = err.Error() + } + return nil, nil, toolError{fmt.Errorf("gradle dependencies --configuration %s failed in %s (rerun with --depth 1 for offline direct-only output): %s", gradleDefaultConfiguration, project.Dir, detail)} + } + + graph, err := parseGradleDependencyTree(result.Stdout, filepath.Base(filepath.Dir(project.File))) + if err != nil { + return nil, nil, toolError{fmt.Errorf("gradle dependencies parse failed in %s: %w", project.Dir, err)} + } + root := buildTreeFromEdgeGraph(edgeTreeOptions{Graph: graph, MaxDepth: opts.MaxDepth}) + if root == nil { + return nil, nil, toolError{fmt.Errorf("gradle dependencies produced no nodes for %s", project.Dir)} + } + root.Path = project.File + root.Source = filepath.Base(project.File) + applyGradleScope(root) + markDirectByDepth(root) + return root, nil, nil +} + +// gradleCommand prefers a ./gradlew wrapper found by walking up from dir, and +// falls back to a gradle binary on PATH. +func gradleCommand(dir string) string { + current := dir + for { + wrapper := filepath.Join(current, "gradlew") + if info, err := os.Stat(wrapper); err == nil && !info.IsDir() { + return wrapper + } + parent := filepath.Dir(current) + if parent == current { + break + } + current = parent + } + return "gradle" +} + +// parseGradleDependencyTree parses the indented ASCII tree printed by the gradle +// `dependencies` task into an edgeGraph rooted at a synthetic project node. +// It resolves "-> version" overrides, treats "(*)" repeated subtrees as leaves, +// skips "(c)" constraint and "(n)" unresolved entries, and marks "project :x" +// nodes as local. +func parseGradleDependencyTree(output, rootName string) (edgeGraph, error) { + rootKey := NodeID(ManagerGradle, rootName, "") + graph := edgeGraph{ + RootKey: rootKey, + Nodes: map[string]*Node{rootKey: NewNode(ManagerGradle, rootName, "")}, + Edges: map[string][]string{}, + } + type frame struct { + depth int + key string + } + stack := []frame{} + seenEdge := map[string]bool{} + + for _, raw := range strings.Split(output, "\n") { + col, ok := gradleMarkerColumn(raw) + if !ok { + continue + } + depth := col/gradleIndentWidth + 1 + entry := strings.TrimSpace(raw[col+gradleMarkerWidth:]) + node, skip := gradleNodeFromEntry(entry) + if skip { + continue + } + key := node.ID + if _, exists := graph.Nodes[key]; !exists { + graph.Nodes[key] = node + } + for len(stack) > 0 && stack[len(stack)-1].depth >= depth { + stack = stack[:len(stack)-1] + } + parentKey := rootKey + if len(stack) > 0 { + parentKey = stack[len(stack)-1].key + } + edgeKey := parentKey + ">" + key + if !seenEdge[edgeKey] { + seenEdge[edgeKey] = true + graph.Edges[parentKey] = append(graph.Edges[parentKey], key) + } + stack = append(stack, frame{depth: depth, key: key}) + } + return graph, nil +} + +const ( + gradleIndentWidth = 5 + gradleMarkerWidth = 5 // "+--- " or "\--- " +) + +func gradleMarkerColumn(line string) (int, bool) { + for _, marker := range []string{"+--- ", "\\--- "} { + idx := strings.Index(line, marker) + if idx < 0 { + continue + } + if gradleIsIndent(line[:idx]) { + return idx, true + } + } + return 0, false +} + +func gradleIsIndent(prefix string) bool { + for _, r := range prefix { + if r != ' ' && r != '|' { + return false + } + } + return true +} + +func gradleNodeFromEntry(entry string) (*Node, bool) { + entry = strings.TrimSpace(entry) + if entry == "" { + return nil, true + } + if strings.HasSuffix(entry, "(c)") || strings.HasSuffix(entry, "(n)") { + return nil, true + } + entry = strings.TrimSpace(strings.TrimSuffix(entry, "(*)")) + if strings.HasPrefix(entry, "project ") { + name := strings.TrimSpace(strings.TrimPrefix(entry, "project")) + node := NewNode(ManagerGradle, name, "") + node.Local = true + return node, false + } + left, right, hasArrow := strings.Cut(entry, " -> ") + name, version := gradleCoordinate(left) + if hasArrow { + version = gradleResolvedVersion(strings.TrimSpace(right)) + } + return NewNode(ManagerGradle, name, version), false +} + +func gradleCoordinate(text string) (name, version string) { + parts := strings.Split(strings.TrimSpace(text), ":") + switch len(parts) { + case 3: + return parts[0] + ":" + parts[1], parts[2] + case 2: + return parts[0] + ":" + parts[1], "" + default: + return strings.TrimSpace(text), "" + } +} + +func gradleResolvedVersion(right string) string { + if strings.Contains(right, ":") { + parts := strings.Split(right, ":") + return parts[len(parts)-1] + } + return right +} + +func applyGradleScope(root *Node) { + var walk func(n *Node) + walk = func(n *Node) { + if n == nil { + return + } + if n.Scope == "" { + n.Scope = gradleDefaultConfiguration + } + for _, child := range n.Children { + walk(child) + } + } + for _, child := range root.Children { + walk(child) + } +} diff --git a/deps/gradle_tree_test.go b/deps/gradle_tree_test.go new file mode 100644 index 0000000..4c981be --- /dev/null +++ b/deps/gradle_tree_test.go @@ -0,0 +1,84 @@ +package deps + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" +) + +const gradleTreeFixture = ` +runtimeClasspath - Runtime classpath of source set 'main'. ++--- org.foo:bar:1.0 +| \--- org.baz:qux:2.0 ++--- org.abc:def:1.0 -> 1.2 +\--- project :submodule + \--- org.x:y:3.0 (*) + +(*) - Indicates repeated occurrences of a transitive dependency subtree. +` + +func TestParseGradleDependencyTree(t *testing.T) { + graph, err := parseGradleDependencyTree(gradleTreeFixture, "app") + if err != nil { + t.Fatal(err) + } + root := buildTreeFromEdgeGraph(edgeTreeOptions{Graph: graph}) + applyGradleScope(root) + + bar := findChild(root, "org.foo:bar") + if bar == nil || bar.Version != "1.0" { + t.Fatalf("bar not parsed: %#v", bar) + } + qux := findChild(bar, "org.baz:qux") + if qux == nil || qux.Depth != 2 { + t.Fatalf("transitive qux not at depth 2: %#v", qux) + } + def := findChild(root, "org.abc:def") + if def == nil || def.Version != "1.2" { + t.Fatalf("resolved-version override not applied: %#v", def) + } + sub := findChild(root, ":submodule") + if sub == nil || !sub.Local { + t.Fatalf("project dependency not marked local: %#v", sub) + } + if root.Children[0].Scope != gradleDefaultConfiguration { + t.Fatalf("scope not applied: %#v", root.Children[0]) + } +} + +func TestGradleCommandPrefersWrapper(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "gradlew"), "#!/bin/sh\n") + sub := filepath.Join(root, "service") + writeFile(t, filepath.Join(sub, "build.gradle"), "") + + got := gradleCommand(sub) + if got != filepath.Join(root, "gradlew") { + t.Fatalf("gradleCommand = %q, want wrapper at repo root", got) + } + + bare := t.TempDir() + if gradleCommand(bare) != "gradle" { + t.Fatalf("expected fallback to gradle binary when no wrapper found") + } +} + +func TestResolveGradleTreeFailureIsFatal(t *testing.T) { + dir := t.TempDir() + runner := &fakeRunner{err: fmt.Errorf("exit status 1"), result: CommandResult{Stderr: "Configuration 'runtimeClasspath' not found"}} + + _, _, err := resolveGradleTree(context.Background(), Project{Manager: ManagerGradle, Dir: dir, File: filepath.Join(dir, "build.gradle")}, Options{Runner: runner, MaxDepth: 0}) + if err == nil { + t.Fatal("expected fatal tool error") + } + var te toolError + if !errors.As(err, &te) { + t.Fatalf("expected toolError, got %T: %v", err, err) + } + if !strings.Contains(err.Error(), gradleDefaultConfiguration) { + t.Fatalf("error should name the configuration, got %v", err) + } +} diff --git a/deps/maven.go b/deps/maven.go index 8412811..9c86fb2 100644 --- a/deps/maven.go +++ b/deps/maven.go @@ -1,98 +1,17 @@ package deps import ( - "context" - "encoding/json" "encoding/xml" - "fmt" "os" - "path/filepath" "strings" ) -type mavenTreeNode struct { - GroupID string `json:"groupId"` - ArtifactID string `json:"artifactId"` - Version string `json:"version"` - Type string `json:"type"` - Scope string `json:"scope"` - Optional any `json:"optional"` - Children []mavenTreeNode `json:"children"` -} - -func resolveMavenNative(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { - tmp, err := os.CreateTemp("", "repomap-maven-*.json") - if err != nil { - return nil, nil, err - } - tmpPath := tmp.Name() - _ = tmp.Close() - defer os.Remove(tmpPath) - - _, err = opts.Runner.Run(ctx, Command{ - Dir: project.Dir, - Name: "mvn", - Args: []string{ - "-q", - "org.apache.maven.plugins:maven-dependency-plugin:3.11.0:tree", - "-DoutputType=json", - "-DoutputFile=" + tmpPath, - }, - }) - if err != nil { - return nil, nil, err - } - data, err := os.ReadFile(tmpPath) - if err != nil { - return nil, nil, err - } - if len(strings.TrimSpace(string(data))) == 0 { - return nil, nil, fmt.Errorf("maven dependency plugin produced empty JSON") - } - root, err := parseMavenJSON(data) - if err != nil { - return nil, nil, err - } - root.Path = project.File - root.Source = "mvn dependency:tree" - return root, nil, nil -} - func resolveMavenManifest(project Project) (*Node, []Warning, error) { root, err := parseMavenPOM(project.File) if err != nil { return nil, nil, err } - return root, []Warning{{Manager: ManagerMaven, Project: project.Dir, Message: "manifest fallback includes pom.xml direct dependencies only; resolved transitive edges are unavailable"}}, nil -} - -func parseMavenJSON(data []byte) (*Node, error) { - var tree mavenTreeNode - if err := json.Unmarshal(data, &tree); err != nil { - return nil, err - } - return convertMavenTree(tree, 0), nil -} - -func convertMavenTree(tree mavenTreeNode, depth int) *Node { - name := tree.ArtifactID - if tree.GroupID != "" { - name = tree.GroupID + ":" + tree.ArtifactID - } - node := NewNode(ManagerMaven, name, tree.Version) - node.Depth = depth - node.Scope = tree.Scope - node.Optional = boolish(tree.Optional) - if tree.Type != "" { - node.Source = tree.Type - } - for _, childTree := range tree.Children { - child := convertMavenTree(childTree, depth+1) - child.Direct = depth == 0 - node.Children = append(node.Children, child) - } - sortChildren(node) - return node + return root, []Warning{{Manager: ManagerMaven, Project: project.Dir, Message: "offline pom.xml parsing includes direct dependencies only; resolved transitive edges are unavailable"}}, nil } type pomProject struct { @@ -172,17 +91,6 @@ func resolveProperty(value string, props map[string]string) string { return value } -func boolish(value any) bool { - switch v := value.(type) { - case bool: - return v - case string: - return strings.EqualFold(v, "true") - default: - return false - } -} - func firstNonEmpty(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { @@ -191,7 +99,3 @@ func firstNonEmpty(values ...string) string { } return "" } - -func mavenProjectFile(dir string) string { - return filepath.Join(dir, "pom.xml") -} diff --git a/deps/maven_tree.go b/deps/maven_tree.go new file mode 100644 index 0000000..2423a5e --- /dev/null +++ b/deps/maven_tree.go @@ -0,0 +1,123 @@ +package deps + +import ( + "context" + "fmt" + "os" + "strings" +) + +// resolveMavenTree resolves the transitive Maven dependency graph by running +// `mvn dependency:tree` with TGF output. It fails fast with a toolError when mvn +// is missing or the command fails, suggesting --depth 1 for offline output. +func resolveMavenTree(ctx context.Context, project Project, opts Options) (*Node, []Warning, error) { + tmp, err := os.CreateTemp("", "repomap-mvn-tree-*.tgf") + if err != nil { + return nil, nil, err + } + outFile := tmp.Name() + _ = tmp.Close() + defer func() { _ = os.Remove(outFile) }() + + result, err := opts.Runner.Run(ctx, Command{ + Dir: project.Dir, + Name: "mvn", + Args: []string{"-B", "-N", "dependency:tree", "-DoutputType=tgf", "-DoutputFile=" + outFile}, + }) + if err != nil { + detail := strings.TrimSpace(result.Stderr) + if detail == "" { + detail = err.Error() + } + return nil, nil, toolError{fmt.Errorf("mvn dependency:tree failed in %s (rerun with --depth 1 for offline direct-only output): %s", project.Dir, detail)} + } + + data, err := os.ReadFile(outFile) + if err != nil { + return nil, nil, toolError{fmt.Errorf("mvn dependency:tree produced no output file for %s: %w", project.Dir, err)} + } + graph, err := parseMavenTGF(string(data)) + if err != nil { + return nil, nil, toolError{fmt.Errorf("mvn dependency:tree parse failed in %s: %w", project.Dir, err)} + } + + root := buildTreeFromEdgeGraph(edgeTreeOptions{Graph: graph, MaxDepth: opts.MaxDepth}) + if root == nil { + return nil, nil, toolError{fmt.Errorf("mvn dependency:tree produced no nodes for %s", project.Dir)} + } + root.Path = project.File + root.Source = "pom.xml" + markDirectByDepth(root) + return root, nil, nil +} + +// parseMavenTGF parses Trivial Graph Format produced by dependency:tree. The +// node section lists "