diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go index 95182bb..b51d3c6 100644 --- a/hash-persister/hash-persister.go +++ b/hash-persister/hash-persister.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/bazel-contrib/target-determinator/cli" @@ -272,6 +273,9 @@ func runSeeded(cfg *config) (seededOutcome, error) { seedData.TargetEdges, seedData.SeedCompatibilityFingerprint, ) + if cfg.SeedableOutput { + persistedData.DependencyHashes = seedData.DependencyHashes + } if err := pkg.WritePersistedData(cfg.OutputFile, persistedData); err != nil { return seededOutcome{}, fmt.Errorf("failed to persist hashes: %w", err) } @@ -281,6 +285,37 @@ func runSeeded(cfg *config) (seededOutcome, error) { return outcome, nil } + commitRev, err := pkg.NewLabelledGitRev(cfg.Context.WorkspacePath, cfg.CommitSha, "commit") + if err != nil { + return seededOutcome{}, fmt.Errorf("failed to resolve commit %s: %w", cfg.CommitSha, err) + } + + // Phase: probe dirty packages to prune false-positive rdeps. + // + // When a BUILD.bazel changes, all targets in the package are marked + // dirty and their rdeps cascade through the graph. But most existing + // targets are often unchanged (e.g. a new sibling was added). A small + // probe query on just the dirty packages lets us compare hashes against + // the seed and only propagate rdeps from targets that actually changed. + unprunedDirtyStarCount := len(dirtyResult.DirtyStarLabels) + if unprunedDirtyStarCount > len(dirtyResult.DirtyLabels) { + // Kept in a temporary: probePruneDirtySet returns a nil result + // alongside its error, so assigning straight to dirtyResult would + // discard the set the fallback is supposed to carry on with. + pruned, probeErr := probePruneDirtySet(cfg, commitRev, dirtyResult, seedData, changedFiles) + if probeErr != nil { + log.Printf("Probe pruning failed, continuing with unpruned dirty set: %v", probeErr) + } else { + dirtyResult = pruned + // The report drives CI metrics, so it has to describe the set + // actually used rather than the one before pruning. + outcome.DirtyTargetCount = len(dirtyResult.DirtyStarLabels) + log.Printf("Probe pruning: %d dirty* -> %d dirty* (%d eliminated)", + unprunedDirtyStarCount, len(dirtyResult.DirtyStarLabels), + unprunedDirtyStarCount-len(dirtyResult.DirtyStarLabels)) + } + } + estimatedRecomputedTargets := countDirtySeedTargets(seedData.TargetHashes, dirtyResult.DirtyStarLabels) seedTargetCount := len(seedData.TargetHashes) if shouldFallbackForRecomputation( @@ -320,11 +355,6 @@ func runSeeded(cfg *config) (seededOutcome, error) { return fallback("unscopable_target_pattern", fmt.Sprintf("cannot scope targets pattern: %v", err)) } - commitRev, err := pkg.NewLabelledGitRev(cfg.Context.WorkspacePath, cfg.CommitSha, "commit") - if err != nil { - return seededOutcome{}, fmt.Errorf("failed to resolve commit %s: %w", cfg.CommitSha, err) - } - phaseStart = time.Now() scopedTargets, err := pkg.ParseTargetsList(scopedPattern) if err != nil { @@ -384,6 +414,124 @@ func runSeeded(cfg *config) (seededOutcome, error) { return outcome, nil } +// probePruneDirtySet runs a small probe query on just the dirty packages, +// hashes those targets with seed hashes for their dependencies, and returns +// a pruned DirtySetResult where rdeps are only propagated from targets +// whose hash actually changed. +func probePruneDirtySet( + cfg *config, + commitRev pkg.LabelledGitRev, + dirtyResult *pkg.DirtySetResult, + seedData *pkg.PersistedHashData, + changedFiles map[string]string, +) (*pkg.DirtySetResult, error) { + phaseStart := time.Now() + + probe, err := probePackages(cfg, commitRev, dirtyResult.DirtyPackages, seedData) + if err != nil { + return nil, err + } + log.Printf("Phase probe completed in %v (%d hashed, %d source files, in %d packages)", + time.Since(phaseStart), len(probe.Hashes), len(probe.SourceFiles), + len(dirtyResult.DirtyPackages)) + + return pkg.PruneDirtySet(dirtyResult, seedData, probe, changedFiles), nil +} + +// probePackages queries and hashes targets in the given packages, seeding +// dependency hashes from the seed file so that only the dirty packages need +// a Bazel query. +func probePackages( + cfg *config, + commitRev pkg.LabelledGitRev, + dirtyPackages []string, + seedData *pkg.PersistedHashData, +) (pkg.ProbeResult, error) { + // Query the dirty packages raw, deliberately bypassing the targets + // pattern. That pattern excludes manual-tagged targets, which covers + // most labels in a dirty package (npm links, platform() rules, JS build + // internals). Those labels still appear in the seed's edge map, so the + // probe must hash them to prove they are unchanged. ":*" rather than + // ":all" so source files are included too. + probePattern := buildProbePattern(dirtyPackages) + probeTargets, err := pkg.ParseTargetsList(probePattern) + if err != nil { + return pkg.ProbeResult{}, fmt.Errorf("failed to parse probe targets: %w", err) + } + + probeResults, probeCleanup, err := pkg.LoadIncompleteMetadata(cfg.Context, commitRev, probeTargets) + if err != nil { + probeCleanup() + return pkg.ProbeResult{}, fmt.Errorf("probe query failed: %w", err) + } + defer probeCleanup() + + probeSeedHashes, err := buildExternalSeedHashes(seedData, dirtyPackages) + if err != nil { + return pkg.ProbeResult{}, err + } + if err := probeResults.TargetHashCache.SeedHashes(probeSeedHashes); err != nil { + return pkg.ProbeResult{}, fmt.Errorf("cannot seed probe hashes: %w", err) + } + + if err := probeResults.PrefillCache(); err != nil { + return pkg.ProbeResult{}, fmt.Errorf("probe hashing failed: %w", err) + } + + // Read hashes straight from the cache rather than from MatchingTargets: + // it additionally holds the transitively-computed hashes of dependencies + // outside the probed packages, which cost nothing extra to include. + return pkg.ProbeResult{ + Hashes: probeResults.TargetHashCache.ExtractHexHashes(), + SourceFiles: probeResults.TargetHashCache.SourceFileLabels(), + }, nil +} + +// buildProbePattern returns a bazel query expression covering every target +// in the given packages, including manual-tagged rules and source files. +func buildProbePattern(dirtyPackages []string) string { + if len(dirtyPackages) == 0 { + return "set()" + } + terms := make([]string, 0, len(dirtyPackages)) + for _, p := range dirtyPackages { + if p == "//" { + terms = append(terms, "//:*") + } else { + terms = append(terms, p+":*") + } + } + return "(" + strings.Join(terms, " + ") + ")" +} + +// buildExternalSeedHashes collects seed hashes for all targets NOT in the +// given packages, so that dependency hashes resolve without a full query. +func buildExternalSeedHashes( + seedData *pkg.PersistedHashData, + dirtyPackages []string, +) (map[string][]byte, error) { + dirtyPkgSet := make(map[string]bool, len(dirtyPackages)) + for _, p := range dirtyPackages { + dirtyPkgSet[p] = true + } + hashes := make(map[string][]byte) + for _, source := range []map[string]map[string]string{seedData.TargetHashes, seedData.DependencyHashes} { + for label, configMap := range source { + if dirtyPkgSet[pkg.LabelPackage(label)] { + continue + } + for configStr, hashHex := range configMap { + hashBytes, err := hex.DecodeString(hashHex) + if err != nil { + return nil, fmt.Errorf("invalid seed hash for %s: %w", label, err) + } + hashes[label+"\x00"+configStr] = hashBytes + } + } + } + return hashes, nil +} + func countDirtySeedTargets(targetHashes map[string]map[string]string, dirtyLabels map[string]bool) int { count := 0 for label := range targetHashes { @@ -425,19 +573,24 @@ func validateSeed(seedData *pkg.PersistedHashData, expectedSha, expectedFingerpr func reusableSeedHashes(seedData *pkg.PersistedHashData, dirtyLabels map[string]bool) (map[string][]byte, error) { hashes := make(map[string][]byte) - for label, configMap := range seedData.TargetHashes { - if dirtyLabels[label] { - continue - } - for configStr, hashHex := range configMap { - hashBytes, err := hex.DecodeString(hashHex) - if err != nil { - return nil, fmt.Errorf("invalid hash hex for %s: %w", label, err) + // Dependency hashes are as reusable as target hashes for seeding the + // cache; they are only kept in a separate bucket so that diffing sees + // the target set alone. + for _, source := range []map[string]map[string]string{seedData.TargetHashes, seedData.DependencyHashes} { + for label, configMap := range source { + if dirtyLabels[label] { + continue } - if len(hashBytes) != sha256.Size { - return nil, fmt.Errorf("invalid hash length for %s: got %d bytes, want %d", label, len(hashBytes), sha256.Size) + for configStr, hashHex := range configMap { + hashBytes, err := hex.DecodeString(hashHex) + if err != nil { + return nil, fmt.Errorf("invalid hash hex for %s: %w", label, err) + } + if len(hashBytes) != sha256.Size { + return nil, fmt.Errorf("invalid hash length for %s: got %d bytes, want %d", label, len(hashBytes), sha256.Size) + } + hashes[label+"\x00"+configStr] = hashBytes } - hashes[label+"\x00"+configStr] = hashBytes } } return hashes, nil @@ -482,6 +635,7 @@ func mergePersistedData( } } mergedEdges := mergePersistedEntries(seedData.TargetEdges, dirtyLabels, freshEdges) + persistedData.DependencyHashes = pkg.ExtractDependencyHashes(persistedData.TargetHashes, mergedEdges, queryResults.TargetHashCache) persistedData.FormatVersion = pkg.CurrentPersistedHashFormatVersion persistedData.SeedCompatibilityFingerprint = compatibilityFingerprint persistedData.TargetEdges = mergedEdges diff --git a/hash-persister/hash-persister_test.go b/hash-persister/hash-persister_test.go index 4619e4e..14e43ce 100644 --- a/hash-persister/hash-persister_test.go +++ b/hash-persister/hash-persister_test.go @@ -280,3 +280,23 @@ func TestMergePersistedEntriesReplacesDirtyState(t *testing.T) { t.Fatalf("merged edges = %#v, want %#v", got, wantEdges) } } + +func TestBuildProbePattern(t *testing.T) { + // ":*" rather than ":all" so source files are covered, and no targets + // pattern wrapper so manual-tagged targets are not filtered out. + for _, tc := range []struct { + name string + packages []string + want string + }{ + {"empty", nil, "set()"}, + {"root", []string{"//"}, "(//:*)"}, + {"several", []string{"//", "//ci", "//tools/binaries"}, "(//:* + //ci:* + //tools/binaries:*)"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := buildProbePattern(tc.packages); got != tc.want { + t.Errorf("buildProbePattern(%v) = %q, want %q", tc.packages, got, tc.want) + } + }) + } +} diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go index 32ca836..e545f3a 100644 --- a/pkg/dirty_set.go +++ b/pkg/dirty_set.go @@ -170,6 +170,137 @@ func ComputeDirtySet( return result } +// ProbeResult is what a probe of the dirty packages learned about them. +type ProbeResult struct { + // Hashes maps "label\x00configuration" to the hex hash computed at the + // destination revision. + Hashes map[string]string + // SourceFiles is the set of probed labels that are source files. Their + // dirtiness comes from the git diff rather than from Hashes. + SourceFiles map[string]bool +} + +// PruneDirtySet narrows a DirtySetResult by propagating reverse +// dependencies only from targets that actually changed, rather than from +// every target in a dirty package. A BUILD.bazel edit marks its whole +// package dirty, so in a high-fanout package the unpruned rdeps closure can +// cover most of the repository even when only one target really changed. +// +// A source file is unchanged exactly when the git diff does not mention it. +// Every other label is unchanged when its probe hash matches the seed; +// labels the probe or the seed omits are conservatively treated as changed. +// +// DirtyLabels and DirtyPackages are preserved — those packages still need +// re-listing via wildcards — and only DirtyStarLabels is recomputed. +func PruneDirtySet( + original *DirtySetResult, + seed *PersistedHashData, + probe ProbeResult, + changedFiles map[string]string, +) *DirtySetResult { + actuallyChanged := findChangedTargets(original.DirtyLabels, seed, probe, changedFiles) + newDirtyStar := propagateFrom(original.DirtyLabels, actuallyChanged, seed.TargetEdges) + + return &DirtySetResult{ + DirtyLabels: original.DirtyLabels, + DirtyStarLabels: newDirtyStar, + DirtyPackages: original.DirtyPackages, + } +} + +// findChangedTargets returns the subset of dirtyLabels that changed between +// the seed and the destination revision. +func findChangedTargets( + dirtyLabels map[string]bool, + seed *PersistedHashData, + probe ProbeResult, + changedFiles map[string]string, +) map[string]bool { + changed := make(map[string]bool) + for label := range dirtyLabels { + if probe.SourceFiles[label] { + if _, ok := changedFiles[labelToPath(label)]; ok { + changed[label] = true + } + continue + } + if targetHashChanged(label, seed.SeedHashes(label), probe.Hashes) { + changed[label] = true + } + } + return changed +} + +// labelToPath returns the workspace-relative path a main-repo label refers +// to, e.g. "//pkg:sub/f.java" becomes "pkg/sub/f.java". +func labelToPath(label string) string { + if idx := strings.Index(label, "//"); idx >= 0 { + label = label[idx+len("//"):] + } + pkg, name, found := strings.Cut(label, ":") + switch { + case !found: + return pkg + case pkg == "": + return name + default: + return pkg + "/" + name + } +} + +// targetHashChanged reports whether a single target's probe hash differs +// from its seed hash. Returns true for new targets (nil seedConfigs) and +// targets missing from probe results. +func targetHashChanged(label string, seedConfigs map[string]string, probeHashes map[string]string) bool { + if seedConfigs == nil { + return true + } + for config, seedHex := range seedConfigs { + probeHex, ok := probeHashes[label+"\x00"+config] + if !ok || probeHex != seedHex { + return true + } + } + return false +} + +// propagateFrom builds DirtyStarLabels by including all directly dirty +// labels and BFS-propagating rdeps only from the actuallyChanged subset. +func propagateFrom(dirtyLabels, actuallyChanged map[string]bool, edges map[string][]string) map[string]bool { + rdeps := BuildRdeps(edges) + + // Every directly dirty label is dirty* regardless of whether it + // changed, but membership of the result must not be mistaken for + // having been traversed: an unchanged dirty label still has to be + // walked through to reach what lies behind it. Hence a separate + // visited set from the result set. + result := make(map[string]bool, len(dirtyLabels)) + for label := range dirtyLabels { + result[label] = true + } + + visited := make(map[string]bool, len(actuallyChanged)) + queue := make([]string, 0, len(actuallyChanged)) + for label := range actuallyChanged { + if !visited[label] { + visited[label] = true + queue = append(queue, label) + } + } + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, rdep := range rdeps[current] { + result[rdep] = true + if !visited[rdep] { + visited[rdep] = true + queue = append(queue, rdep) + } + } + } + return result +} + func isFallbackTrigger(basename string) bool { if strings.HasSuffix(basename, ".bzl") { return true diff --git a/pkg/dirty_set_test.go b/pkg/dirty_set_test.go index 2aaee6f..6d14df3 100644 --- a/pkg/dirty_set_test.go +++ b/pkg/dirty_set_test.go @@ -462,3 +462,265 @@ func TestLabelToPackage(t *testing.T) { } } } + +func TestPruneDirtySetEliminatesUnchangedRdeps(t *testing.T) { + // Setup: package //tools/binaries has two aliases. //app:binary depends + // on //tools/binaries:existing_tool. A BUILD.bazel change makes both + // aliases directly dirty, propagating to //app:binary. + edges := map[string][]string{ + "//tools/binaries:existing_tool": {}, + "//tools/binaries:new_tool": {}, + "//app:binary": {"//tools/binaries:existing_tool"}, + "//other:lib": {"//app:binary"}, + } + allLabels := CollectAllLabels(edges, nil) + + changedFiles := map[string]string{ + "tools/binaries/BUILD.bazel": "M", + } + + original := ComputeDirtySet(changedFiles, edges, allLabels, nil) + + // Verify the unpruned dirty set cascades broadly. + if !original.DirtyStarLabels["//app:binary"] { + t.Fatal("expected //app:binary in unpruned DirtyStarLabels") + } + if !original.DirtyStarLabels["//other:lib"] { + t.Fatal("expected //other:lib in unpruned DirtyStarLabels") + } + + // Seed hashes: existing_tool has hash "aaaa...", new_tool is absent (new target). + seedHashes := map[string]map[string]string{ + "//tools/binaries:existing_tool": {"": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + "//app:binary": {"": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, + "//other:lib": {"": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}, + } + + // Probe hashes: existing_tool is UNCHANGED, new_tool is new. + probeHashes := map[string]string{ + "//tools/binaries:existing_tool\x00": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "//tools/binaries:new_tool\x00": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + } + + pruned := PruneDirtySet(original, &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, ProbeResult{Hashes: probeHashes}, nil) + + // Directly dirty labels should be preserved. + if !pruned.DirtyStarLabels["//tools/binaries:existing_tool"] { + t.Error("expected //tools/binaries:existing_tool in pruned DirtyStarLabels (still directly dirty)") + } + if !pruned.DirtyStarLabels["//tools/binaries:new_tool"] { + t.Error("expected //tools/binaries:new_tool in pruned DirtyStarLabels (new target, actually changed)") + } + + // The key assertion: //app:binary and //other:lib should NOT be in the + // pruned dirty set because existing_tool's hash didn't change. + // new_tool has no rdeps, so its change doesn't propagate. + if pruned.DirtyStarLabels["//app:binary"] { + t.Error("//app:binary should have been pruned — its dep //tools/binaries:existing_tool is unchanged") + } + if pruned.DirtyStarLabels["//other:lib"] { + t.Error("//other:lib should have been pruned — transitive dep unchanged") + } + + // DirtyPackages should be preserved. + if len(pruned.DirtyPackages) != 1 || pruned.DirtyPackages[0] != "//tools/binaries" { + t.Errorf("expected DirtyPackages=[//tools/binaries], got %v", pruned.DirtyPackages) + } +} + +func TestPruneDirtySetPreservesChangedRdeps(t *testing.T) { + // When a target's hash actually changes, its rdeps must remain dirty. + edges := map[string][]string{ + "//lib:changed": {"//lib:src.java"}, + "//lib:same": {}, + "//app:consumer": {"//lib:changed"}, + } + allLabels := CollectAllLabels(edges, nil) + + original := ComputeDirtySet( + map[string]string{"lib/BUILD.bazel": "M"}, edges, allLabels, nil, + ) + + seedHashes := map[string]map[string]string{ + "//lib:changed": {"": "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000"}, + "//lib:same": {"": "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000"}, + "//lib:src.java": {"": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000"}, + "//app:consumer": {"": "dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000"}, + } + + // Probe: //lib:changed has a DIFFERENT hash, //lib:same is unchanged. + probeHashes := map[string]string{ + "//lib:changed\x00": "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000", + "//lib:same\x00": "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000", + "//lib:src.java\x00": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000", + } + + pruned := PruneDirtySet(original, &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, ProbeResult{Hashes: probeHashes}, nil) + + if !pruned.DirtyStarLabels["//app:consumer"] { + t.Error("//app:consumer must remain dirty — its dep //lib:changed has a different hash") + } +} + +func TestPruneDirtySetNoOpWhenAllChanged(t *testing.T) { + edges := map[string][]string{ + "//pkg:a": {}, + "//app:dep": {"//pkg:a"}, + } + allLabels := CollectAllLabels(edges, nil) + + original := ComputeDirtySet( + map[string]string{"pkg/BUILD.bazel": "M"}, edges, allLabels, nil, + ) + + seedHashes := map[string]map[string]string{ + "//pkg:a": {"": "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000"}, + "//app:dep": {"": "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000"}, + } + + // Probe: //pkg:a hash changed. + probeHashes := map[string]string{ + "//pkg:a\x00": "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000", + } + + pruned := PruneDirtySet(original, &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, ProbeResult{Hashes: probeHashes}, nil) + + // Everything should remain dirty — same as unpruned. + if !pruned.DirtyStarLabels["//app:dep"] { + t.Error("//app:dep must remain dirty when //pkg:a changed") + } +} + +func TestPruneDirtySetJudgesSourceFilesByGitNotHash(t *testing.T) { + // A source file carries no seed hash; the git diff decides. //pkg:kept.java + // is untouched so its consumer must not propagate, while //pkg:edited.java + // is in the diff so its consumer must. + edges := map[string][]string{ + "//pkg:uses_kept": {"//pkg:kept.java"}, + "//pkg:uses_edited": {"//pkg:edited.java"}, + "//app:via_kept": {"//pkg:uses_kept"}, + "//app:via_edited": {"//pkg:uses_edited"}, + } + allLabels := CollectAllLabels(edges, nil) + original := ComputeDirtySet( + map[string]string{"pkg/edited.java": "M"}, edges, allLabels, nil, + ) + + // Both source files are directly dirty before pruning, and both + // consumers are reachable from them. + for _, label := range []string{"//app:via_kept", "//app:via_edited"} { + if !original.DirtyStarLabels[label] { + t.Fatalf("expected %s in unpruned DirtyStarLabels", label) + } + } + + // A consumer of a changed source file hashes differently, as it would + // in reality; the consumer of the untouched one does not. + unchanged := "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000" + differs := "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000" + seedHashes := map[string]map[string]string{ + "//pkg:uses_kept": {"": unchanged}, + "//pkg:uses_edited": {"": unchanged}, + } + probe := ProbeResult{ + Hashes: map[string]string{ + "//pkg:uses_kept\x00": unchanged, + "//pkg:uses_edited\x00": differs, + }, + SourceFiles: map[string]bool{ + "//pkg:kept.java": true, + "//pkg:edited.java": true, + }, + } + + pruned := PruneDirtySet(original, + &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, + probe, map[string]string{"pkg/edited.java": "M"}) + + if pruned.DirtyStarLabels["//app:via_kept"] { + t.Error("//app:via_kept should be pruned: pkg/kept.java is not in the git diff") + } + if !pruned.DirtyStarLabels["//app:via_edited"] { + t.Error("//app:via_edited must stay dirty: pkg/edited.java is in the git diff") + } +} + +func TestLabelToPath(t *testing.T) { + for _, tt := range []struct{ label, want string }{ + {"//pkg:file.java", "pkg/file.java"}, + {"//pkg:src/main/java/App.java", "pkg/src/main/java/App.java"}, + {"//:root.txt", "root.txt"}, + {"//a/b/c:d.txt", "a/b/c/d.txt"}, + {"//pkg", "pkg"}, + } { + if got := labelToPath(tt.label); got != tt.want { + t.Errorf("labelToPath(%q) = %q, want %q", tt.label, got, tt.want) + } + } +} + +func TestPropagateTraversesThroughUnchangedDirtyLabels(t *testing.T) { + // //pkg:middle is dirty (its package changed) but its own hash did not + // change. It still has to be walked through, or //app:behind is lost. + // Membership of the result set must not double as "already traversed". + edges := map[string][]string{ + "//pkg:changed": {"//pkg:src.java"}, + "//pkg:middle": {"//pkg:changed"}, + "//app:behind": {"//pkg:middle"}, + } + allLabels := CollectAllLabels(edges, nil) + original := ComputeDirtySet( + map[string]string{"pkg/src.java": "M"}, edges, allLabels, nil, + ) + + unchanged := "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000" + seedHashes := map[string]map[string]string{ + "//pkg:changed": {"": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000"}, + "//pkg:middle": {"": unchanged}, + } + probe := ProbeResult{ + Hashes: map[string]string{ + "//pkg:changed\x00": "dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000", + "//pkg:middle\x00": unchanged, + }, + SourceFiles: map[string]bool{"//pkg:src.java": true}, + } + + pruned := PruneDirtySet(original, + &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, + probe, map[string]string{"pkg/src.java": "M"}) + + if !pruned.DirtyStarLabels["//app:behind"] { + t.Error("//app:behind must be reached by traversing through the unchanged dirty label //pkg:middle") + } +} + +func TestPruneDirtySetReadsDependencyHashes(t *testing.T) { + // A manual-tagged dep lives in DependencyHashes, not TargetHashes, yet + // must still be comparable or its reverse dependencies all propagate. + edges := map[string][]string{ + "//pkg:tool": {}, + "//app:consumer": {"//pkg:tool"}, + } + allLabels := CollectAllLabels(edges, nil) + original := ComputeDirtySet( + map[string]string{"pkg/BUILD.bazel": "M"}, edges, allLabels, nil, + ) + if !original.DirtyStarLabels["//app:consumer"] { + t.Fatal("expected //app:consumer in unpruned DirtyStarLabels") + } + + unchanged := "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000" + seed := &PersistedHashData{ + TargetHashes: map[string]map[string]string{}, + DependencyHashes: map[string]map[string]string{"//pkg:tool": {"": unchanged}}, + TargetEdges: edges, + } + probe := ProbeResult{Hashes: map[string]string{"//pkg:tool\x00": unchanged}} + + pruned := PruneDirtySet(original, seed, probe, nil) + + if pruned.DirtyStarLabels["//app:consumer"] { + t.Error("//app:consumer should be pruned: //pkg:tool is unchanged per DependencyHashes") + } +} diff --git a/pkg/hash_cache.go b/pkg/hash_cache.go index 60d8e6f..95b9e65 100644 --- a/pkg/hash_cache.go +++ b/pkg/hash_cache.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "encoding/binary" + "encoding/hex" "errors" "fmt" "io" @@ -202,6 +203,42 @@ func (thc *TargetHashCache) ExtractHashes() map[string][]byte { return result } +// ExtractHexHashes is ExtractHashes with the hashes hex-encoded, matching +// the representation used in persisted hash files. +func (thc *TargetHashCache) ExtractHexHashes() map[string]string { + raw := thc.ExtractHashes() + hashes := make(map[string]string, len(raw)) + for key, hash := range raw { + hashes[key] = hex.EncodeToString(hash) + } + return hashes +} + +// SourceFileLabels returns the labels in the cache's query context that are +// source files. Whether a source file changed is answered by the git diff, +// so these labels need neither a persisted hash nor a hash comparison. +func (thc *TargetHashCache) SourceFileLabels() map[string]bool { + sourceFiles := make(map[string]bool) + for label, configurations := range thc.context { + for _, configuredTarget := range configurations { + if configuredTarget.GetTarget().GetType() == build.Target_SOURCE_FILE { + sourceFiles[label.String()] = true + break + } + } + } + return sourceFiles +} + +// splitHashKey splits a "