Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
.bin/
.grite/
42 changes: 42 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,48 @@ lockfile content without running package-manager commands. It prints direct
dependencies by default; use `--depth 0` for the full graph available from the
local files.

### `cache-warm`

Prime the local package caches for dependencies this machine has not checked out.

```bash
# Download a module and its full transitive closure into GOMODCACHE
repomap cache-warm go github.com/flanksource/clicky@v1.21.14

# Take the current version, compile every package, and prove it works offline
repomap cache-warm go github.com/flanksource/commons --build --verify

# A Go dependency may also be a GitHub slug or a repository URL
repomap cache-warm go flanksource/commons
repomap cache-warm go https://github.com/flanksource/commons

# Warm several npm packages into the pnpm store
repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0
```

For each `name@version` spec, repomap creates a throwaway single-dependency
project in a temporary directory, drives the real package manager to download the
closure into the machine's shared cache, then deletes the project. Nothing in the
working tree is touched; what persists is the warmed cache (`GOMODCACHE`, the pnpm
store, or the npm cache). Omit the version to take whatever the manager considers
current — the concrete resolved version is reported back, including Go
pseudo-versions. A Go dependency can be named as a module path, a GitHub
`owner/repo` slug, or a repository URL; all three are canonicalised to the module
path before the go toolchain sees them.

`--build` goes further than downloading. For Go it compiles every package in the
module so `GOCACHE` holds build artifacts and not just source. For npm and pnpm it
lets dependency lifecycle scripts run so native addons are compiled; it does not
run the package's own build script.

`--verify` proves the result rather than assuming it, replaying the work with the
network disabled (`GOPROXY=off`, or an `--offline` install against a frozen
lockfile), so a cache that could not actually build offline fails loudly.

Supported managers are `go`, `npm`, and `pnpm`. This is aimed at CI images,
sandboxes, and air-gapped builds, where a later build must succeed with no network
access.

### `version`

Print version, commit hash, build date, and Go version.
Expand Down
103 changes: 103 additions & 0 deletions cmd/repomap/cache_warm.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package main

import (
"context"
"fmt"
"strings"

"github.com/flanksource/clicky"
"github.com/flanksource/clicky/api"
depgraph "github.com/flanksource/repomap/deps"
)

type CacheWarmOptions struct {
// No default: tag — a positional field carrying one has its positional values
// silently discarded. Emptiness is validated in runCacheWarm instead.
Args []string `json:"args" args:"true" help:"Package manager (go, npm, pnpm) followed by one or more name@version specs"`
Build bool `json:"build,omitempty" flag:"build" help:"Compile every package after downloading (Go) or run dependency lifecycle and native builds (npm, pnpm)"`
Verify bool `json:"verify,omitempty" flag:"verify" help:"Replay the warm with the network disabled to prove the cache is complete"`
}

func (opts CacheWarmOptions) GetName() string { return "cache-warm <manager> <name@version>..." }

func (opts CacheWarmOptions) Help() api.Text {
return clicky.Text(`Prime the local package caches for dependencies this machine has not checked out.

For each name@version spec, repomap creates a throwaway single-dependency
project in a temporary directory, drives the real package manager to download
the dependency's full transitive closure into the machine's shared cache, then
deletes the project. Nothing in the working tree is touched; what persists is
the warmed cache (GOMODCACHE, the pnpm store, or the npm cache).

Omit the version to take whatever the manager considers current. The concrete
resolved version is reported back, including Go pseudo-versions.

A Go dependency can be given as a module path, as a GitHub owner/repo slug, or as
a repository URL — all three are canonicalised to the module path before the go
toolchain sees them.

Use --build to go further than downloading. For Go it compiles every package in
the module so GOCACHE holds the build artifacts, not just the source. For npm and
pnpm it lets dependency lifecycle scripts run so native addons are compiled. It
does not run the package's own build script.

Use --verify to prove the result rather than assume it: the work is replayed with
the network disabled (GOPROXY=off, or an --offline install against a frozen
lockfile), so a cache that could not actually build offline fails loudly.

This is aimed at CI images, sandboxes, and air-gapped builds, where a later build
must succeed with no network access.

EXAMPLES:
repomap cache-warm go github.com/flanksource/clicky@v1.21.14
repomap cache-warm go github.com/flanksource/commons --build --verify
repomap cache-warm go flanksource/commons
repomap cache-warm go https://github.com/flanksource/commons
repomap cache-warm pnpm react@18.2.0 react-dom@18.2.0
repomap cache-warm npm @flanksource/icons@1.0.0 --verify
repomap cache-warm go github.com/flanksource/clicky@v1.21.14 --json`)
}

func init() {
cmd := clicky.AddNamedCommandWithContext("cache-warm", rootCmd, CacheWarmOptions{}, runCacheWarm)
cmd.Short = "Warm the Go, npm, or pnpm cache for a dependency and optionally build it"
}

func runCacheWarm(ctx context.Context, opts CacheWarmOptions) (any, error) {
manager, specs, err := parseCacheWarmArgs(opts.Args)
if err != nil {
return nil, err
}
results, err := depgraph.WarmCache(ctx, depgraph.WarmOptions{
Manager: manager,
Specs: specs,
Build: opts.Build,
Verify: opts.Verify,
})
// Returned as a slice rather than api.NewTableFrom so --json keeps the full
// WarmResult — per-step commands, durations, and errors, which is what a CI
// debugging session needs. Pretty output still renders as a table via
// WarmResult's Columns/Row. The error is returned alongside the results so a
// partial run still reports which specs succeeded.
return results, err
}

// parseCacheWarmArgs splits the positional arguments into the manager and its
// specs. Only managers repomap can actually warm are accepted; maven and gradle
// are scan-only, and image/helm are not package caches.
func parseCacheWarmArgs(args []string) (depgraph.Manager, []string, error) {
if len(args) == 0 {
return "", nil, fmt.Errorf("expected a package manager (go, npm, or pnpm) followed by one or more name@version specs")
}
manager := depgraph.Manager(strings.ToLower(strings.TrimSpace(args[0])))
switch manager {
case depgraph.ManagerGo, depgraph.ManagerNPM, depgraph.ManagerPNPM:
default:
return "", nil, fmt.Errorf("cache warming does not support %q (expected go, npm, or pnpm)", args[0])
}
specs := splitCommaArgs(args[1:])
if len(specs) == 0 {
return "", nil, fmt.Errorf("expected at least one name@version spec to warm for %s", manager)
}
return manager, specs, nil
}
114 changes: 114 additions & 0 deletions cmd/repomap/cache_warm_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package main

import (
"context"
"strings"
"testing"

"github.com/flanksource/clicky"
depgraph "github.com/flanksource/repomap/deps"
"github.com/spf13/cobra"
)

// runCacheWarmArgs parses argv through a real cobra tree with a stub handler, so
// the assertions cover clicky's struct-tag binding rather than a hand-built
// options struct.
func runCacheWarmArgs(t *testing.T, argv ...string) (CacheWarmOptions, error) {
t.Helper()
var got CacheWarmOptions
root := &cobra.Command{Use: "test"}
clicky.BindAllFlags(root.PersistentFlags(), "tasks", "format")
clicky.AddNamedCommandWithContext("cache-warm", root, CacheWarmOptions{}, func(_ context.Context, opts CacheWarmOptions) (any, error) {
got = opts
return nil, nil
})
root.SetArgs(argv)
return got, root.Execute()
}

// The positional args field must carry no default: tag, or clicky silently drops
// the positional values.
func TestCacheWarmBindsPositionalArgsAndFlags(t *testing.T) {
got, err := runCacheWarmArgs(t, "cache-warm", "go", "github.com/acme/lib@v1.2.3", "left-pad@1.3.0", "--build", "--verify")
if err != nil {
t.Fatal(err)
}
want := []string{"go", "github.com/acme/lib@v1.2.3", "left-pad@1.3.0"}
if strings.Join(got.Args, ",") != strings.Join(want, ",") {
t.Fatalf("Args = %v, want %v", got.Args, want)
}
if !got.Build || !got.Verify {
t.Fatalf("Build = %v, Verify = %v, want both true", got.Build, got.Verify)
}
}

func TestCacheWarmFlagsDefaultOff(t *testing.T) {
got, err := runCacheWarmArgs(t, "cache-warm", "go", "github.com/acme/lib@v1.2.3")
if err != nil {
t.Fatal(err)
}
if got.Build || got.Verify {
t.Fatalf("Build = %v, Verify = %v, want both false", got.Build, got.Verify)
}
}

func TestParseCacheWarmArgs(t *testing.T) {
cases := []struct {
name string
args []string
wantManager depgraph.Manager
wantSpecs []string
wantErr string
}{
{
name: "single spec", args: []string{"go", "github.com/acme/lib@v1.2.3"},
wantManager: depgraph.ManagerGo, wantSpecs: []string{"github.com/acme/lib@v1.2.3"},
},
{
name: "several specs", args: []string{"pnpm", "left-pad@1.3.0", "@scope/pkg@2.0.0"},
wantManager: depgraph.ManagerPNPM, wantSpecs: []string{"left-pad@1.3.0", "@scope/pkg@2.0.0"},
},
{
name: "manager casing is normalised", args: []string{"NPM", "left-pad@1.3.0"},
wantManager: depgraph.ManagerNPM, wantSpecs: []string{"left-pad@1.3.0"},
},
{name: "no args", args: nil, wantErr: "manager"},
{name: "manager but no spec", args: []string{"go"}, wantErr: "spec"},
// Managers repomap can scan but cannot warm must be rejected by name.
{name: "unwarmable manager", args: []string{"maven", "org.acme:lib@1.0.0"}, wantErr: "maven"},
{name: "unknown manager", args: []string{"cargo", "serde@1.0.0"}, wantErr: "cargo"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
manager, specs, err := parseCacheWarmArgs(tc.args)
if tc.wantErr != "" {
if err == nil {
t.Fatalf("expected an error mentioning %q", tc.wantErr)
}
if !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("error %q should mention %q", err, tc.wantErr)
}
return
}
if err != nil {
t.Fatal(err)
}
if manager != tc.wantManager {
t.Errorf("manager = %q, want %q", manager, tc.wantManager)
}
if strings.Join(specs, ",") != strings.Join(tc.wantSpecs, ",") {
t.Errorf("specs = %v, want %v", specs, tc.wantSpecs)
}
})
}
}

// defaultToScan rewrites anything it does not recognise into `scan ...`, so a
// misregistered name would turn this command into a silent repo scan.
func TestCacheWarmIsNotRewrittenToScan(t *testing.T) {
argv := []string{"cache-warm", "go", "github.com/acme/lib@v1.2.3"}
got := defaultToScan(argv)
if strings.Join(got, " ") != strings.Join(argv, " ") {
t.Fatalf("defaultToScan(%v) = %v, want it unchanged", argv, got)
}
}
Loading