From 69a0ab4d228919a82b2d2b92f060538d51c38811 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Tue, 22 Sep 2026 20:09:12 +0200
Subject: [PATCH 01/15] perf(hash-persister): probe dirty packages before rdeps
 propagation

When a BUILD.bazel changes, all targets in the package are marked dirty
and their reverse dependencies cascade through the graph. For high-fanout
packages like tools/binaries (which contains widely-used aliases), adding
a single new target inflates the dirty set to hundreds of thousands of
targets even though existing targets are unchanged.

Add a probe phase to runSeeded that queries only the dirty packages,
hashes those targets against seed dependency hashes, and compares with
the seed. Only targets whose hash actually changed propagate to reverse
dependencies. This dramatically reduces the scoped query size when
BUILD.bazel changes don't affect most existing targets.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go | 118 ++++++++++++++++++++++++++--
 pkg/dirty_set.go                 | 113 +++++++++++++++++++++++++++
 pkg/dirty_set_test.go            | 128 +++++++++++++++++++++++++++++++
 3 files changed, 354 insertions(+), 5 deletions(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index 95182bb..c83b304 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -281,6 +281,30 @@ func runSeeded(cfg *config) (seededOutcome, error) {
 		return outcome, nil
 	}
 
+	commitRev, err := pkg.NewLabelledGitRev(cfg.Context.WorkspacePath, cfg.CommitSha, "commit")
+	if err != nil {
+		return seededOutcome{}, fmt.Errorf("failed to resolve commit %s: %w", cfg.CommitSha, err)
+	}
+
+	// Phase: probe dirty packages to prune false-positive rdeps.
+	//
+	// When a BUILD.bazel changes, all targets in the package are marked
+	// dirty and their rdeps cascade through the graph. But most existing
+	// targets are often unchanged (e.g. a new sibling was added). A small
+	// probe query on just the dirty packages lets us compare hashes against
+	// the seed and only propagate rdeps from targets that actually changed.
+	unprunedDirtyStarCount := len(dirtyResult.DirtyStarLabels)
+	if unprunedDirtyStarCount > len(dirtyResult.DirtyLabels) {
+		dirtyResult, err = probePruneDirtySet(cfg, commitRev, dirtyResult, seedData)
+		if err != nil {
+			log.Printf("Probe pruning failed, continuing with unpruned dirty set: %v", err)
+		} else {
+			log.Printf("Probe pruning: %d dirty* -> %d dirty* (%d eliminated)",
+				unprunedDirtyStarCount, len(dirtyResult.DirtyStarLabels),
+				unprunedDirtyStarCount-len(dirtyResult.DirtyStarLabels))
+		}
+	}
+
 	estimatedRecomputedTargets := countDirtySeedTargets(seedData.TargetHashes, dirtyResult.DirtyStarLabels)
 	seedTargetCount := len(seedData.TargetHashes)
 	if shouldFallbackForRecomputation(
@@ -320,11 +344,6 @@ func runSeeded(cfg *config) (seededOutcome, error) {
 		return fallback("unscopable_target_pattern", fmt.Sprintf("cannot scope targets pattern: %v", err))
 	}
 
-	commitRev, err := pkg.NewLabelledGitRev(cfg.Context.WorkspacePath, cfg.CommitSha, "commit")
-	if err != nil {
-		return seededOutcome{}, fmt.Errorf("failed to resolve commit %s: %w", cfg.CommitSha, err)
-	}
-
 	phaseStart = time.Now()
 	scopedTargets, err := pkg.ParseTargetsList(scopedPattern)
 	if err != nil {
@@ -384,6 +403,95 @@ func runSeeded(cfg *config) (seededOutcome, error) {
 	return outcome, nil
 }
 
+// probePruneDirtySet runs a small probe query on just the dirty packages,
+// hashes those targets with seed hashes for their dependencies, and returns
+// a pruned DirtySetResult where rdeps are only propagated from targets
+// whose hash actually changed.
+func probePruneDirtySet(
+	cfg *config,
+	commitRev pkg.LabelledGitRev,
+	dirtyResult *pkg.DirtySetResult,
+	seedData *pkg.PersistedHashData,
+) (*pkg.DirtySetResult, error) {
+	phaseStart := time.Now()
+
+	probeHashes, err := probePackageHashes(cfg, commitRev, dirtyResult.DirtyPackages, seedData)
+	if err != nil {
+		return nil, err
+	}
+	log.Printf("Phase probe completed in %v (%d targets in %d packages)",
+		time.Since(phaseStart), len(probeHashes), len(dirtyResult.DirtyPackages))
+
+	return pkg.PruneDirtySet(dirtyResult, seedData.TargetHashes, seedData.TargetEdges, probeHashes), nil
+}
+
+// probePackageHashes queries and hashes targets in the given packages,
+// seeding dependency hashes from the seed file so that only the dirty
+// packages need a Bazel query.
+func probePackageHashes(
+	cfg *config,
+	commitRev pkg.LabelledGitRev,
+	dirtyPackages []string,
+	seedData *pkg.PersistedHashData,
+) (map[string]string, error) {
+	probeUniverse := pkg.BuildScopedUniverse(dirtyPackages, nil)
+	probePattern, err := pkg.ScopeTargetsPattern(cfg.Targets.String(), probeUniverse)
+	if err != nil {
+		return nil, fmt.Errorf("cannot scope probe pattern: %w", err)
+	}
+	probeTargets, err := pkg.ParseTargetsList(probePattern)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse probe targets: %w", err)
+	}
+
+	probeResults, probeCleanup, err := pkg.LoadIncompleteMetadata(cfg.Context, commitRev, probeTargets)
+	if err != nil {
+		probeCleanup()
+		return nil, fmt.Errorf("probe query failed: %w", err)
+	}
+	defer probeCleanup()
+
+	probeSeedHashes, err := buildExternalSeedHashes(seedData, dirtyPackages)
+	if err != nil {
+		return nil, err
+	}
+	if err := probeResults.TargetHashCache.SeedHashes(probeSeedHashes); err != nil {
+		return nil, fmt.Errorf("cannot seed probe hashes: %w", err)
+	}
+
+	if err := probeResults.PrefillCache(); err != nil {
+		return nil, fmt.Errorf("probe hashing failed: %w", err)
+	}
+
+	return pkg.ProbeHashesFromQueryResults(probeResults)
+}
+
+// buildExternalSeedHashes collects seed hashes for all targets NOT in the
+// given packages, so that dependency hashes resolve without a full query.
+func buildExternalSeedHashes(
+	seedData *pkg.PersistedHashData,
+	dirtyPackages []string,
+) (map[string][]byte, error) {
+	dirtyPkgSet := make(map[string]bool, len(dirtyPackages))
+	for _, p := range dirtyPackages {
+		dirtyPkgSet[p] = true
+	}
+	hashes := make(map[string][]byte)
+	for label, configMap := range seedData.TargetHashes {
+		if dirtyPkgSet[pkg.LabelPackage(label)] {
+			continue
+		}
+		for configStr, hashHex := range configMap {
+			hashBytes, err := hex.DecodeString(hashHex)
+			if err != nil {
+				return nil, fmt.Errorf("invalid seed hash for %s: %w", label, err)
+			}
+			hashes[label+"\x00"+configStr] = hashBytes
+		}
+	}
+	return hashes, nil
+}
+
 func countDirtySeedTargets(targetHashes map[string]map[string]string, dirtyLabels map[string]bool) int {
 	count := 0
 	for label := range targetHashes {
diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go
index 32ca836..fee71bf 100644
--- a/pkg/dirty_set.go
+++ b/pkg/dirty_set.go
@@ -1,6 +1,7 @@
 package pkg
 
 import (
+	"encoding/hex"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -170,6 +171,118 @@ func ComputeDirtySet(
 	return result
 }
 
+// PruneDirtySet narrows a DirtySetResult by re-propagating reverse
+// dependencies only from targets whose hashes actually changed. The caller
+// must supply probeHashes — a map from "label\x00configuration" to the hex
+// hash computed at the destination revision — for every target in the
+// directly dirty packages. Targets whose probe hash matches the seed are
+// excluded from rdeps propagation, dramatically reducing the dirty set when
+// a BUILD.bazel change doesn't affect most existing targets (e.g. adding a
+// new target to a high-fanout package).
+//
+// seedHashes is the TargetHashes map from the seed file (label → config → hex hash).
+// edges is the TargetEdges map from the seed file.
+// original is the DirtySetResult from ComputeDirtySet.
+// probeHashes maps "label\x00config" → hex hash for targets in dirty packages.
+//
+// PruneDirtySet preserves the original DirtyLabels and DirtyPackages
+// (packages still need re-listing via wildcards) but recomputes
+// DirtyStarLabels from scratch.
+func PruneDirtySet(
+	original *DirtySetResult,
+	seedHashes map[string]map[string]string,
+	edges map[string][]string,
+	probeHashes map[string]string,
+) *DirtySetResult {
+	actuallyChanged := findChangedTargets(original.DirtyLabels, seedHashes, probeHashes)
+	newDirtyStar := propagateFrom(original.DirtyLabels, actuallyChanged, edges)
+
+	return &DirtySetResult{
+		DirtyLabels:     original.DirtyLabels,
+		DirtyStarLabels: newDirtyStar,
+		DirtyPackages:   original.DirtyPackages,
+	}
+}
+
+// findChangedTargets returns the subset of dirtyLabels whose probe hash
+// differs from the seed. New targets (absent from seed) and targets absent
+// from probe results are treated as changed.
+func findChangedTargets(
+	dirtyLabels map[string]bool,
+	seedHashes map[string]map[string]string,
+	probeHashes map[string]string,
+) map[string]bool {
+	changed := make(map[string]bool)
+	for label := range dirtyLabels {
+		if targetHashChanged(label, seedHashes[label], probeHashes) {
+			changed[label] = true
+		}
+	}
+	return changed
+}
+
+// targetHashChanged reports whether a single target's probe hash differs
+// from its seed hash. Returns true for new targets (nil seedConfigs) and
+// targets missing from probe results.
+func targetHashChanged(label string, seedConfigs map[string]string, probeHashes map[string]string) bool {
+	if seedConfigs == nil {
+		return true
+	}
+	for config, seedHex := range seedConfigs {
+		probeHex, ok := probeHashes[label+"\x00"+config]
+		if !ok || probeHex != seedHex {
+			return true
+		}
+	}
+	return false
+}
+
+// propagateFrom builds DirtyStarLabels by including all directly dirty
+// labels and BFS-propagating rdeps only from the actuallyChanged subset.
+func propagateFrom(dirtyLabels, actuallyChanged map[string]bool, edges map[string][]string) map[string]bool {
+	rdeps := BuildRdeps(edges)
+	result := make(map[string]bool, len(actuallyChanged))
+
+	for label := range dirtyLabels {
+		result[label] = true
+	}
+
+	queue := make([]string, 0, len(actuallyChanged))
+	for label := range actuallyChanged {
+		queue = append(queue, label)
+	}
+	for len(queue) > 0 {
+		current := queue[0]
+		queue = queue[1:]
+		for _, rdep := range rdeps[current] {
+			if !result[rdep] {
+				result[rdep] = true
+				queue = append(queue, rdep)
+			}
+		}
+	}
+	return result
+}
+
+// ProbeHashesFromQueryResults extracts hex-encoded hashes for all matching
+// targets in a QueryResults, keyed as "label\x00configuration". This is
+// used by the probe phase to compare against seed hashes.
+func ProbeHashesFromQueryResults(queryResults *QueryResults) (map[string]string, error) {
+	hashes := make(map[string]string)
+	for _, label := range queryResults.MatchingTargets.Labels() {
+		for _, cfg := range queryResults.MatchingTargets.ConfigurationsFor(label) {
+			hash, err := queryResults.TargetHashCache.Hash(LabelAndConfiguration{
+				Label: label, Configuration: cfg,
+			})
+			if err != nil {
+				return nil, err
+			}
+			hashes[label.String()+"\x00"+cfg.String()] = hex.EncodeToString(hash)
+		}
+	}
+	return hashes, nil
+}
+
 func isFallbackTrigger(basename string) bool {
 	if strings.HasSuffix(basename, ".bzl") {
 		return true
diff --git a/pkg/dirty_set_test.go b/pkg/dirty_set_test.go
index 2aaee6f..8894f83 100644
--- a/pkg/dirty_set_test.go
+++ b/pkg/dirty_set_test.go
@@ -462,3 +462,131 @@ func TestLabelToPackage(t *testing.T) {
 		}
 	}
 }
+
+func TestPruneDirtySetEliminatesUnchangedRdeps(t *testing.T) {
+	// Setup: package //tools/binaries has two aliases. //app:binary depends
+	// on //tools/binaries:existing_tool. A BUILD.bazel change makes both
+	// aliases directly dirty, propagating to //app:binary.
+	edges := map[string][]string{
+		"//tools/binaries:existing_tool": {},
+		"//tools/binaries:new_tool":      {},
+		"//app:binary":                   {"//tools/binaries:existing_tool"},
+		"//other:lib":                    {"//app:binary"},
+	}
+	allLabels := CollectAllLabels(edges, nil)
+
+	changedFiles := map[string]string{
+		"tools/binaries/BUILD.bazel": "M",
+	}
+
+	original := ComputeDirtySet(changedFiles, edges, allLabels, nil)
+
+	// Verify the unpruned dirty set cascades broadly.
+	if !original.DirtyStarLabels["//app:binary"] {
+		t.Fatal("expected //app:binary in unpruned DirtyStarLabels")
+	}
+	if !original.DirtyStarLabels["//other:lib"] {
+		t.Fatal("expected //other:lib in unpruned DirtyStarLabels")
+	}
+
+	// Seed hashes: existing_tool has hash "aaaa...", new_tool is absent (new target).
+	seedHashes := map[string]map[string]string{
+		"//tools/binaries:existing_tool": {"": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
+		"//app:binary":                   {"": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},
+		"//other:lib":                    {"": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"},
+	}
+
+	// Probe hashes: existing_tool is UNCHANGED, new_tool is new.
+	probeHashes := map[string]string{
+		"//tools/binaries:existing_tool\x00": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		"//tools/binaries:new_tool\x00":      "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
+	}
+
+	pruned := PruneDirtySet(original, seedHashes, edges, probeHashes)
+
+	// Directly dirty labels should be preserved.
+	if !pruned.DirtyStarLabels["//tools/binaries:existing_tool"] {
+		t.Error("expected //tools/binaries:existing_tool in pruned DirtyStarLabels (still directly dirty)")
+	}
+	if !pruned.DirtyStarLabels["//tools/binaries:new_tool"] {
+		t.Error("expected //tools/binaries:new_tool in pruned DirtyStarLabels (new target, actually changed)")
+	}
+
+	// The key assertion: //app:binary and //other:lib should NOT be in the
+	// pruned dirty set because existing_tool's hash didn't change.
+	// new_tool has no rdeps, so its change doesn't propagate.
+	if pruned.DirtyStarLabels["//app:binary"] {
+		t.Error("//app:binary should have been pruned — its dep //tools/binaries:existing_tool is unchanged")
+	}
+	if pruned.DirtyStarLabels["//other:lib"] {
+		t.Error("//other:lib should have been pruned — transitive dep unchanged")
+	}
+
+	// DirtyPackages should be preserved.
+	if len(pruned.DirtyPackages) != 1 || pruned.DirtyPackages[0] != "//tools/binaries" {
+		t.Errorf("expected DirtyPackages=[//tools/binaries], got %v", pruned.DirtyPackages)
+	}
+}
+
+func TestPruneDirtySetPreservesChangedRdeps(t *testing.T) {
+	// When a target's hash actually changes, its rdeps must remain dirty.
+	edges := map[string][]string{
+		"//lib:changed":  {"//lib:src.java"},
+		"//lib:same":     {},
+		"//app:consumer": {"//lib:changed"},
+	}
+	allLabels := CollectAllLabels(edges, nil)
+
+	original := ComputeDirtySet(
+		map[string]string{"lib/BUILD.bazel": "M"}, edges, allLabels, nil,
+	)
+
+	seedHashes := map[string]map[string]string{
+		"//lib:changed":  {"": "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000"},
+		"//lib:same":     {"": "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000"},
+		"//lib:src.java": {"": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000"},
+		"//app:consumer": {"": "dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000"},
+	}
+
+	// Probe: //lib:changed has a DIFFERENT hash, //lib:same is unchanged.
+	probeHashes := map[string]string{
+		"//lib:changed\x00":  "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000",
+		"//lib:same\x00":     "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000",
+		"//lib:src.java\x00": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000",
+	}
+
+	pruned := PruneDirtySet(original, seedHashes, edges, probeHashes)
+
+	if !pruned.DirtyStarLabels["//app:consumer"] {
+		t.Error("//app:consumer must remain dirty — its dep //lib:changed has a different hash")
+	}
+}
+
+func TestPruneDirtySetNoOpWhenAllChanged(t *testing.T) {
+	edges := map[string][]string{
+		"//pkg:a":   {},
+		"//app:dep": {"//pkg:a"},
+	}
+	allLabels := CollectAllLabels(edges, nil)
+
+	original := ComputeDirtySet(
+		map[string]string{"pkg/BUILD.bazel": "M"}, edges, allLabels, nil,
+	)
+
+	seedHashes := map[string]map[string]string{
+		"//pkg:a":   {"": "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000"},
+		"//app:dep": {"": "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000"},
+	}
+
+	// Probe: //pkg:a hash changed.
+	probeHashes := map[string]string{
+		"//pkg:a\x00": "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000",
+	}
+
+	pruned := PruneDirtySet(original, seedHashes, edges, probeHashes)
+
+	// Everything should remain dirty — same as unpruned.
+	if !pruned.DirtyStarLabels["//app:dep"] {
+		t.Error("//app:dep must remain dirty when //pkg:a changed")
+	}
+}

From 8a2355ce37ae11a9bce6a7478bd265cb6e24946d Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 13:45:34 +0200
Subject: [PATCH 02/15] debug(hash-persister): log which probe targets changed
 vs unchanged

Temporary diagnostic to understand why 191K targets remain dirty after
probe pruning. Logs each changed target with its reason (new_target,
hash_differs, missing_from_probe).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 pkg/dirty_set.go | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go
index fee71bf..827491d 100644
--- a/pkg/dirty_set.go
+++ b/pkg/dirty_set.go
@@ -2,6 +2,7 @@ package pkg
 
 import (
 	"encoding/hex"
+	"log"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -213,9 +214,35 @@ func findChangedTargets(
 	probeHashes map[string]string,
 ) map[string]bool {
 	changed := make(map[string]bool)
+	unchanged := make(map[string]bool)
 	for label := range dirtyLabels {
 		if targetHashChanged(label, seedHashes[label], probeHashes) {
 			changed[label] = true
+		} else {
+			unchanged[label] = true
+		}
+	}
+	if len(changed) > 0 || len(unchanged) > 0 {
+		log.Printf("Probe hash comparison: %d changed, %d unchanged out of %d dirty labels",
+			len(changed), len(unchanged), len(dirtyLabels))
+		for label := range changed {
+			reason := "hash_differs"
+			if seedHashes[label] == nil {
+				reason = "new_target"
+			} else {
+				for config, seedHex := range seedHashes[label] {
+					probeHex, ok := probeHashes[label+"\x00"+config]
+					if !ok {
+						reason = "missing_from_probe"
+						break
+					}
+					if probeHex != seedHex {
+						reason = "hash_differs"
+						break
+					}
+				}
+			}
+			log.Printf("  changed: %s (%s)", label, reason)
 		}
 	}
 	return changed

From 4cf16848808aca7d607177dfd727803197a0f319 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 14:49:29 +0200
Subject: [PATCH 03/15] perf(hash-persister): persist dependency hashes in
 seedable output

Include hashes for edge-map-only targets (manual-tagged deps, platform()
rules, generated file outputs) in the seedable target_hashes. These
targets are already computed in the hash cache via recursive Hash() calls
but were previously not persisted, causing the probe to treat them as
"new" and over-propagate rdeps.

Also removes the temporary diagnostic logging from the previous commit.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go |  1 +
 pkg/dirty_set.go                 | 27 ------------------------
 pkg/hash_persistence.go          | 35 ++++++++++++++++++++++++++++++++
 pkg/hash_persistence_test.go     | 17 ++++++++++++++--
 4 files changed, 51 insertions(+), 29 deletions(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index c83b304..466d133 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -590,6 +590,7 @@ func mergePersistedData(
 		}
 	}
 	mergedEdges := mergePersistedEntries(seedData.TargetEdges, dirtyLabels, freshEdges)
+	pkg.AddDependencyHashes(persistedData.TargetHashes, mergedEdges, queryResults.TargetHashCache)
 	persistedData.FormatVersion = pkg.CurrentPersistedHashFormatVersion
 	persistedData.SeedCompatibilityFingerprint = compatibilityFingerprint
 	persistedData.TargetEdges = mergedEdges
diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go
index 827491d..fee71bf 100644
--- a/pkg/dirty_set.go
+++ b/pkg/dirty_set.go
@@ -2,7 +2,6 @@ package pkg
 
 import (
 	"encoding/hex"
-	"log"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -214,35 +213,9 @@ func findChangedTargets(
 	probeHashes map[string]string,
 ) map[string]bool {
 	changed := make(map[string]bool)
-	unchanged := make(map[string]bool)
 	for label := range dirtyLabels {
 		if targetHashChanged(label, seedHashes[label], probeHashes) {
 			changed[label] = true
-		} else {
-			unchanged[label] = true
-		}
-	}
-	if len(changed) > 0 || len(unchanged) > 0 {
-		log.Printf("Probe hash comparison: %d changed, %d unchanged out of %d dirty labels",
-			len(changed), len(unchanged), len(dirtyLabels))
-		for label := range changed {
-			reason := "hash_differs"
-			if seedHashes[label] == nil {
-				reason = "new_target"
-			} else {
-				for config, seedHex := range seedHashes[label] {
-					probeHex, ok := probeHashes[label+"\x00"+config]
-					if !ok {
-						reason = "missing_from_probe"
-						break
-					}
-					if probeHex != seedHex {
-						reason = "hash_differs"
-						break
-					}
-				}
-			}
-			log.Printf("  changed: %s (%s)", label, reason)
 		}
 	}
 	return changed
diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index d95fb63..ee43654 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/bazel-contrib/target-determinator/third_party/protobuf/bazel/build"
@@ -183,6 +184,7 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 		if err != nil {
 			return fmt.Errorf("failed to extract target edges: %w", err)
 		}
+		AddDependencyHashes(targetHashes, targetEdges, queryResults.TargetHashCache)
 		compatibilityFingerprint, err := ComputeSeedCompatibilityFingerprint(context, targetsPattern, queryResults.BazelRelease)
 		if err != nil {
 			return err
@@ -195,6 +197,39 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 	return writePersistedData(filePath, &persistedData, !seedable)
 }
 
+// addDependencyHashes supplements targetHashes with hashes for labels that
+// appear in the edge map but are not matching targets (e.g. manual-tagged
+// deps, platform() rules, generated file outputs). These hashes are already
+// computed in the cache via recursive Hash() calls. Including them in the
+// seed lets the probe compare them and avoid false-positive rdeps
+// propagation.
+func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[string][]string, cache *TargetHashCache) {
+	allCachedHashes := cache.ExtractHashes()
+
+	// Index cached hashes by label for efficient lookup.
+	cachedByLabel := make(map[string]map[string]string)
+	for key, hashBytes := range allCachedHashes {
+		idx := strings.IndexByte(key, '\x00')
+		if idx < 0 {
+			continue
+		}
+		label, config := key[:idx], key[idx+1:]
+		if cachedByLabel[label] == nil {
+			cachedByLabel[label] = make(map[string]string)
+		}
+		cachedByLabel[label][config] = hex.EncodeToString(hashBytes)
+	}
+
+	for label := range edges {
+		if _, ok := targetHashes[label]; ok {
+			continue
+		}
+		if configs, ok := cachedByLabel[label]; ok {
+			targetHashes[label] = configs
+		}
+	}
+}
+
 // WritePersistedData writes a PersistedHashData struct directly to a JSON file.
 // Used by the seeded path which builds PersistedHashData by merging seed and
 // freshly computed data instead of extracting from QueryResults.
diff --git a/pkg/hash_persistence_test.go b/pkg/hash_persistence_test.go
index 260cc00..a12926d 100644
--- a/pkg/hash_persistence_test.go
+++ b/pkg/hash_persistence_test.go
@@ -360,7 +360,20 @@ func TestPersistenceModesPreserveHashesAndGeneratedFileEdges(t *testing.T) {
 	if got := seedable.TargetEdges[outputLabel.String()]; len(got) != 1 || got[0] != generatorLabel.String() {
 		t.Fatalf("seedable generated-file edge = %v, want [%s]", got, generatorLabel)
 	}
-	if !reflect.DeepEqual(seedable.TargetHashes, legacy.TargetHashes) {
-		t.Fatalf("seedable hashes differ from legacy hashes:\nseedable: %v\nlegacy: %v", seedable.TargetHashes, legacy.TargetHashes)
+	// Seedable hashes are a superset of legacy hashes — they include
+	// dependency-only targets (e.g. generated files) that appear in the
+	// edge map but not in MatchingTargets.
+	for label, legacyConfigs := range legacy.TargetHashes {
+		seedableConfigs, ok := seedable.TargetHashes[label]
+		if !ok {
+			t.Fatalf("seedable hashes missing legacy label %s", label)
+		}
+		if !reflect.DeepEqual(seedableConfigs, legacyConfigs) {
+			t.Fatalf("seedable hashes for %s differ from legacy: seedable=%v legacy=%v", label, seedableConfigs, legacyConfigs)
+		}
+	}
+	// The generated file output should now have a hash in seedable mode.
+	if _, ok := seedable.TargetHashes[outputLabel.String()]; !ok {
+		t.Fatal("seedable hashes should include dependency-only target //gen:output")
 	}
 }

From 1beee13838d45f7453190c9ed9f22ebc36edf152 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 15:15:00 +0200
Subject: [PATCH 04/15] fix(hash-persister): extract probe hashes from full
 cache, not just matching targets

The probe query uses the same targets pattern that excludes manual-tagged
targets. ProbeHashesFromQueryResults only extracted hashes for matching
targets, missing dependency-only labels (platform, npm, generated files)
even though their hashes were computed transitively during PrefillCache.

Use ProbeHashesFromCache to extract all cached hashes, so the probe can
compare every label in the seed against its current hash.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go |  6 +++++-
 pkg/dirty_set.go                 | 13 +++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index 466d133..4c304ff 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -463,7 +463,11 @@ func probePackageHashes(
 		return nil, fmt.Errorf("probe hashing failed: %w", err)
 	}
 
-	return pkg.ProbeHashesFromQueryResults(probeResults)
+	// Extract hashes for matching targets plus all transitively-computed
+	// dependency hashes (manual-tagged, platform, generated file targets).
+	// ProbeHashesFromQueryResults only returns matching targets; the cache
+	// also has hashes for their deps computed during PrefillCache.
+	return pkg.ProbeHashesFromCache(probeResults)
 }
 
 // buildExternalSeedHashes collects seed hashes for all targets NOT in the
diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go
index fee71bf..77b8fe4 100644
--- a/pkg/dirty_set.go
+++ b/pkg/dirty_set.go
@@ -283,6 +283,19 @@ func ProbeHashesFromQueryResults(queryResults *QueryResults) (map[string]string,
 	return hashes, nil
 }
 
+// ProbeHashesFromCache extracts hex-encoded hashes for ALL targets in the
+// hash cache, not just matching targets. This includes dependency-only
+// targets (manual-tagged, platform rules, generated files) whose hashes
+// were computed transitively during PrefillCache.
+func ProbeHashesFromCache(queryResults *QueryResults) (map[string]string, error) {
+	allCachedHashes := queryResults.TargetHashCache.ExtractHashes()
+	hashes := make(map[string]string, len(allCachedHashes))
+	for key, hashBytes := range allCachedHashes {
+		hashes[key] = hex.EncodeToString(hashBytes)
+	}
+	return hashes, nil
+}
+
 func isFallbackTrigger(basename string) bool {
 	if strings.HasSuffix(basename, ".bzl") {
 		return true

From 4ec2be1e251e822b80febc9b579e2a50baa24f01 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 15:43:10 +0200
Subject: [PATCH 05/15] fix(hash-persister): cover all dirty labels in seed and
 probe

Two gaps left the probe unable to compare most dirty-package labels, so
they were conservatively marked changed and their reverse dependencies
propagated anyway.

Seed side: AddDependencyHashes iterated only edge keys, missing leaf
labels that appear solely as dependency values (source files, npm /ref
targets). Measured on a real seed, 4417 of 13081 dirty labels were
missed this way, including //:service-info.yaml whose rdeps closure is
95k targets.

Probe side: the probe query applied the targets pattern, which excludes
manual-tagged targets. platform() rules, npm link targets and JS build
internals are all manual-tagged, so the probe hashed only 115 of 13081
dirty labels. Query the dirty packages raw with ":*" instead, which
covers manual-tagged rules and source files alike.

With both gaps closed every dirty label has a seed hash and a probe hash
to compare. On the measured case only 7 targets genuinely change, and
their combined rdeps closure is 7, so dirty* should fall from ~280k to
roughly the 13081 directly dirty labels.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go | 30 +++++++++++++++++++++++++-----
 pkg/hash_persistence.go          | 13 +++++++++++--
 2 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index 4c304ff..5f9364f 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -14,6 +14,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/bazel-contrib/target-determinator/cli"
@@ -434,11 +435,13 @@ func probePackageHashes(
 	dirtyPackages []string,
 	seedData *pkg.PersistedHashData,
 ) (map[string]string, error) {
-	probeUniverse := pkg.BuildScopedUniverse(dirtyPackages, nil)
-	probePattern, err := pkg.ScopeTargetsPattern(cfg.Targets.String(), probeUniverse)
-	if err != nil {
-		return nil, fmt.Errorf("cannot scope probe pattern: %w", err)
-	}
+	// Query the dirty packages raw, deliberately bypassing the targets
+	// pattern. That pattern excludes manual-tagged targets, which covers
+	// most labels in a dirty package (npm links, platform() rules, JS build
+	// internals). Those labels still appear in the seed's edge map, so the
+	// probe must hash them to prove they are unchanged. ":*" rather than
+	// ":all" so source files are included too.
+	probePattern := buildProbePattern(dirtyPackages)
 	probeTargets, err := pkg.ParseTargetsList(probePattern)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse probe targets: %w", err)
@@ -470,6 +473,23 @@ func probePackageHashes(
 	return pkg.ProbeHashesFromCache(probeResults)
 }
 
+// buildProbePattern returns a bazel query expression covering every target
+// in the given packages, including manual-tagged rules and source files.
+func buildProbePattern(dirtyPackages []string) string {
+	if len(dirtyPackages) == 0 {
+		return "set()"
+	}
+	terms := make([]string, 0, len(dirtyPackages))
+	for _, p := range dirtyPackages {
+		if p == "//" {
+			terms = append(terms, "//:*")
+		} else {
+			terms = append(terms, p+":*")
+		}
+	}
+	return "(" + strings.Join(terms, " + ") + ")"
+}
+
 // buildExternalSeedHashes collects seed hashes for all targets NOT in the
 // given packages, so that dependency hashes resolve without a full query.
 func buildExternalSeedHashes(
diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index ee43654..3febf57 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -220,14 +220,23 @@ func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[st
 		cachedByLabel[label][config] = hex.EncodeToString(hashBytes)
 	}
 
-	for label := range edges {
+	add := func(label string) {
 		if _, ok := targetHashes[label]; ok {
-			continue
+			return
 		}
 		if configs, ok := cachedByLabel[label]; ok {
 			targetHashes[label] = configs
 		}
 	}
+	// Cover both edge keys and dependency values. Leaf labels (source files,
+	// npm /ref targets) never appear as keys, so iterating keys alone would
+	// miss them and leave the probe unable to compare their hashes.
+	for label, deps := range edges {
+		add(label)
+		for _, dep := range deps {
+			add(dep)
+		}
+	}
 }
 
 // WritePersistedData writes a PersistedHashData struct directly to a JSON file.

From 1272f9459f920c38b9eebef77397bf4ef2fff5cf Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 15:55:54 +0200
Subject: [PATCH 06/15] refactor(hash-persister): tidy up incremental probe
 after iteration

Consolidation pass over the probe work, no behaviour change:

- Drop ProbeHashesFromQueryResults, dead since the probe switched to
  reading the cache, and replace the package-level ProbeHashesFromCache
  with a TargetHashCache.ExtractHexHashes method next to ExtractHashes.
  It only ever touched the cache and could not fail, so the QueryResults
  parameter and error return were both noise.
- Index only the labels AddDependencyHashes actually needs instead of
  every cached hash, avoiding a transient map-of-maps over ~530k entries.
- Fix its doc comment to name the exported identifier, size the
  propagateFrom result map by the set it is actually filled from, and
  trim doc comments that restated their signatures.
- Cover the two behaviours that iteration left untested: dependency
  values reached only as edge values, and the probe pattern shape.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go      |  9 ++--
 hash-persister/hash-persister_test.go | 20 +++++++++
 pkg/dirty_set.go                      | 61 +++++----------------------
 pkg/hash_cache.go                     | 21 +++++++++
 pkg/hash_persistence.go               | 60 ++++++++++++--------------
 pkg/hash_persistence_test.go          | 38 +++++++++++++++++
 6 files changed, 122 insertions(+), 87 deletions(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index 5f9364f..d0ee043 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -466,11 +466,10 @@ func probePackageHashes(
 		return nil, fmt.Errorf("probe hashing failed: %w", err)
 	}
 
-	// Extract hashes for matching targets plus all transitively-computed
-	// dependency hashes (manual-tagged, platform, generated file targets).
-	// ProbeHashesFromQueryResults only returns matching targets; the cache
-	// also has hashes for their deps computed during PrefillCache.
-	return pkg.ProbeHashesFromCache(probeResults)
+	// Read straight from the cache rather than from MatchingTargets: it
+	// additionally holds the transitively-computed hashes of dependencies
+	// outside the probed packages, which cost nothing extra to include.
+	return probeResults.TargetHashCache.ExtractHexHashes(), nil
 }
 
 // buildProbePattern returns a bazel query expression covering every target
diff --git a/hash-persister/hash-persister_test.go b/hash-persister/hash-persister_test.go
index 4619e4e..14e43ce 100644
--- a/hash-persister/hash-persister_test.go
+++ b/hash-persister/hash-persister_test.go
@@ -280,3 +280,23 @@ func TestMergePersistedEntriesReplacesDirtyState(t *testing.T) {
 		t.Fatalf("merged edges = %#v, want %#v", got, wantEdges)
 	}
 }
+
+func TestBuildProbePattern(t *testing.T) {
+	// ":*" rather than ":all" so source files are covered, and no targets
+	// pattern wrapper so manual-tagged targets are not filtered out.
+	for _, tc := range []struct {
+		name     string
+		packages []string
+		want     string
+	}{
+		{"empty", nil, "set()"},
+		{"root", []string{"//"}, "(//:*)"},
+		{"several", []string{"//", "//ci", "//tools/binaries"}, "(//:* + //ci:* + //tools/binaries:*)"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := buildProbePattern(tc.packages); got != tc.want {
+				t.Errorf("buildProbePattern(%v) = %q, want %q", tc.packages, got, tc.want)
+			}
+		})
+	}
+}
diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go
index 77b8fe4..f863ebc 100644
--- a/pkg/dirty_set.go
+++ b/pkg/dirty_set.go
@@ -1,7 +1,6 @@
 package pkg
 
 import (
-	"encoding/hex"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -171,23 +170,17 @@ func ComputeDirtySet(
 	return result
 }
 
-// PruneDirtySet narrows a DirtySetResult by re-propagating reverse
-// dependencies only from targets whose hashes actually changed. The caller
-// must supply probeHashes — a map from "label\x00configuration" to the hex
-// hash computed at the destination revision — for every target in the
-// directly dirty packages. Targets whose probe hash matches the seed are
-// excluded from rdeps propagation, dramatically reducing the dirty set when
-// a BUILD.bazel change doesn't affect most existing targets (e.g. adding a
-// new target to a high-fanout package).
+// PruneDirtySet narrows a DirtySetResult by propagating reverse
+// dependencies only from targets whose hash actually changed, rather than
+// from every target in a dirty package. A BUILD.bazel edit marks its whole
+// package dirty, so in a high-fanout package the unpruned rdeps closure can
+// cover most of the repository even when only one target really changed.
 //
-// seedHashes is the TargetHashes map from the seed file (label → config → hex hash).
-// edges is the TargetEdges map from the seed file.
-// original is the DirtySetResult from ComputeDirtySet.
-// probeHashes maps "label\x00config" → hex hash for targets in dirty packages.
-//
-// PruneDirtySet preserves the original DirtyLabels and DirtyPackages
-// (packages still need re-listing via wildcards) but recomputes
-// DirtyStarLabels from scratch.
+// probeHashes must cover every label in the dirty packages, keyed as
+// "label\x00configuration"; labels it omits are conservatively treated as
+// changed. DirtyLabels and DirtyPackages are preserved — those packages
+// still need re-listing via wildcards — and only DirtyStarLabels is
+// recomputed.
 func PruneDirtySet(
 	original *DirtySetResult,
 	seedHashes map[string]map[string]string,
@@ -241,7 +234,7 @@ func targetHashChanged(label string, seedConfigs map[string]string, probeHashes
 // labels and BFS-propagating rdeps only from the actuallyChanged subset.
 func propagateFrom(dirtyLabels, actuallyChanged map[string]bool, edges map[string][]string) map[string]bool {
 	rdeps := BuildRdeps(edges)
-	result := make(map[string]bool, len(actuallyChanged))
+	result := make(map[string]bool, len(dirtyLabels))
 
 	for label := range dirtyLabels {
 		result[label] = true
@@ -264,38 +257,6 @@ func propagateFrom(dirtyLabels, actuallyChanged map[string]bool, edges map[strin
 	return result
 }
 
-// ProbeHashesFromQueryResults extracts hex-encoded hashes for all matching
-// targets in a QueryResults, keyed as "label\x00configuration". This is
-// used by the probe phase to compare against seed hashes.
-func ProbeHashesFromQueryResults(queryResults *QueryResults) (map[string]string, error) {
-	hashes := make(map[string]string)
-	for _, label := range queryResults.MatchingTargets.Labels() {
-		for _, cfg := range queryResults.MatchingTargets.ConfigurationsFor(label) {
-			hash, err := queryResults.TargetHashCache.Hash(LabelAndConfiguration{
-				Label: label, Configuration: cfg,
-			})
-			if err != nil {
-				return nil, err
-			}
-			hashes[label.String()+"\x00"+cfg.String()] = hex.EncodeToString(hash)
-		}
-	}
-	return hashes, nil
-}
-
-// ProbeHashesFromCache extracts hex-encoded hashes for ALL targets in the
-// hash cache, not just matching targets. This includes dependency-only
-// targets (manual-tagged, platform rules, generated files) whose hashes
-// were computed transitively during PrefillCache.
-func ProbeHashesFromCache(queryResults *QueryResults) (map[string]string, error) {
-	allCachedHashes := queryResults.TargetHashCache.ExtractHashes()
-	hashes := make(map[string]string, len(allCachedHashes))
-	for key, hashBytes := range allCachedHashes {
-		hashes[key] = hex.EncodeToString(hashBytes)
-	}
-	return hashes, nil
-}
-
 func isFallbackTrigger(basename string) bool {
 	if strings.HasSuffix(basename, ".bzl") {
 		return true
diff --git a/pkg/hash_cache.go b/pkg/hash_cache.go
index 60d8e6f..d18d65c 100644
--- a/pkg/hash_cache.go
+++ b/pkg/hash_cache.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/binary"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -202,6 +203,26 @@ func (thc *TargetHashCache) ExtractHashes() map[string][]byte {
 	return result
 }
 
+// ExtractHexHashes is ExtractHashes with the hashes hex-encoded, matching
+// the representation used in persisted hash files.
+func (thc *TargetHashCache) ExtractHexHashes() map[string]string {
+	raw := thc.ExtractHashes()
+	hashes := make(map[string]string, len(raw))
+	for key, hash := range raw {
+		hashes[key] = hex.EncodeToString(hash)
+	}
+	return hashes
+}
+
+// splitHashKey splits a "<label>\x00<configuration>" cache key.
+func splitHashKey(key string) (label, configuration string, ok bool) {
+	idx := strings.IndexByte(key, '\x00')
+	if idx < 0 {
+		return "", "", false
+	}
+	return key[:idx], key[idx+1:], true
+}
+
 // RestoreHashes populates the cache with pre-computed hashes and freezes the cache.
 // Keys must be formatted as "<label>\x00<configuration>".
 func (thc *TargetHashCache) RestoreHashes(hashes map[string][]byte) error {
diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index 3febf57..4f05077 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"os"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/bazel-contrib/target-determinator/third_party/protobuf/bazel/build"
@@ -197,45 +196,42 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 	return writePersistedData(filePath, &persistedData, !seedable)
 }
 
-// addDependencyHashes supplements targetHashes with hashes for labels that
-// appear in the edge map but are not matching targets (e.g. manual-tagged
-// deps, platform() rules, generated file outputs). These hashes are already
-// computed in the cache via recursive Hash() calls. Including them in the
-// seed lets the probe compare them and avoid false-positive rdeps
-// propagation.
+// AddDependencyHashes supplements targetHashes with hashes for labels that
+// appear in the edge map but are not matching targets — manual-tagged deps,
+// platform() rules, generated file outputs. Their hashes were already
+// computed in the cache via recursive Hash() calls, so persisting them is
+// free, and without them incremental hashing cannot tell whether such a
+// label changed and must conservatively propagate its reverse dependencies.
 func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[string][]string, cache *TargetHashCache) {
-	allCachedHashes := cache.ExtractHashes()
-
-	// Index cached hashes by label for efficient lookup.
-	cachedByLabel := make(map[string]map[string]string)
-	for key, hashBytes := range allCachedHashes {
-		idx := strings.IndexByte(key, '\x00')
-		if idx < 0 {
-			continue
+	// Both edge keys and dependency values: leaf labels (source files, npm
+	// /ref targets) never appear as keys, so iterating keys alone misses them.
+	wanted := make(map[string]bool)
+	consider := func(label string) {
+		if _, ok := targetHashes[label]; !ok {
+			wanted[label] = true
 		}
-		label, config := key[:idx], key[idx+1:]
-		if cachedByLabel[label] == nil {
-			cachedByLabel[label] = make(map[string]string)
+	}
+	for label, deps := range edges {
+		consider(label)
+		for _, dep := range deps {
+			consider(dep)
 		}
-		cachedByLabel[label][config] = hex.EncodeToString(hashBytes)
+	}
+	if len(wanted) == 0 {
+		return
 	}
 
-	add := func(label string) {
-		if _, ok := targetHashes[label]; ok {
-			return
+	for key, hashHex := range cache.ExtractHexHashes() {
+		label, configuration, ok := splitHashKey(key)
+		if !ok || !wanted[label] {
+			continue
 		}
-		if configs, ok := cachedByLabel[label]; ok {
+		configs := targetHashes[label]
+		if configs == nil {
+			configs = make(map[string]string)
 			targetHashes[label] = configs
 		}
-	}
-	// Cover both edge keys and dependency values. Leaf labels (source files,
-	// npm /ref targets) never appear as keys, so iterating keys alone would
-	// miss them and leave the probe unable to compare their hashes.
-	for label, deps := range edges {
-		add(label)
-		for _, dep := range deps {
-			add(dep)
-		}
+		configs[configuration] = hashHex
 	}
 }
 
diff --git a/pkg/hash_persistence_test.go b/pkg/hash_persistence_test.go
index a12926d..ac58030 100644
--- a/pkg/hash_persistence_test.go
+++ b/pkg/hash_persistence_test.go
@@ -1,6 +1,9 @@
 package pkg
 
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -377,3 +380,38 @@ func TestPersistenceModesPreserveHashesAndGeneratedFileEdges(t *testing.T) {
 		t.Fatal("seedable hashes should include dependency-only target //gen:output")
 	}
 }
+
+func TestAddDependencyHashesCoversEdgeKeysAndDependencyValues(t *testing.T) {
+	filled := func(b byte) []byte { return bytes.Repeat([]byte{b}, sha256.Size) }
+
+	cache := NewTargetHashCache(nil, &Normalizer{}, "release 8.0.0", true, nil)
+	if err := cache.SeedHashes(map[string][]byte{
+		"//pkg:rule\x00":     filled(0x11),
+		"//pkg:leaf.txt\x00": filled(0x22),
+		"//pkg:already\x00":  filled(0x33),
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// leaf.txt appears only as a dependency value, never as a key — the
+	// shape that leaves a label unhashable by the probe if missed.
+	edges := map[string][]string{
+		"//pkg:rule":    {"//pkg:leaf.txt"},
+		"//pkg:already": {},
+	}
+	targetHashes := map[string]map[string]string{
+		"//pkg:already": {"": "preexisting"},
+	}
+
+	AddDependencyHashes(targetHashes, edges, cache)
+
+	if got := targetHashes["//pkg:rule"][""]; got != hex.EncodeToString(filled(0x11)) {
+		t.Errorf("edge key //pkg:rule not added, got %q", got)
+	}
+	if got := targetHashes["//pkg:leaf.txt"][""]; got != hex.EncodeToString(filled(0x22)) {
+		t.Errorf("dependency-value-only //pkg:leaf.txt not added, got %q", got)
+	}
+	if got := targetHashes["//pkg:already"][""]; got != "preexisting" {
+		t.Errorf("existing hash overwritten with %q", got)
+	}
+}

From de2f10b9d9725c117b63a088372825ee6182ddfe Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 16:02:34 +0200
Subject: [PATCH 07/15] fix(hash-persister): do not persist the empty-hash
 sentinel

TargetHashCache returns a zero-length slice rather than a hash for a
source-file label whose file does not exist, or which is a directory
spuriously listed in srcs (bazelbuild/bazel#14611, #14678). ExtractHashes
only filters nil, so the sentinel passed through.

Persisting dependency values started surfacing those labels in the seed,
and seed validation requires every hash to be sha256-sized, so a single
one rejected the entire seed and fell the run back to full hashing.

Omit them instead. A label without a seed hash is treated as changed,
which is the safe direction.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 pkg/hash_persistence.go      |  9 +++++++++
 pkg/hash_persistence_test.go | 25 +++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index 4f05077..0d559f8 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -226,6 +226,15 @@ func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[st
 		if !ok || !wanted[label] {
 			continue
 		}
+		// The cache yields a zero-length sentinel rather than a hash for
+		// labels whose file does not exist or is a directory. Persisting
+		// one would fail seed validation, which requires every hash to be
+		// sha256-sized, and cause the whole seed to be rejected. Omitting
+		// the label instead just makes incremental hashing treat it as
+		// changed, which is the safe direction.
+		if len(hashHex) != hex.EncodedLen(sha256.Size) {
+			continue
+		}
 		configs := targetHashes[label]
 		if configs == nil {
 			configs = make(map[string]string)
diff --git a/pkg/hash_persistence_test.go b/pkg/hash_persistence_test.go
index ac58030..d20b176 100644
--- a/pkg/hash_persistence_test.go
+++ b/pkg/hash_persistence_test.go
@@ -415,3 +415,28 @@ func TestAddDependencyHashesCoversEdgeKeysAndDependencyValues(t *testing.T) {
 		t.Errorf("existing hash overwritten with %q", got)
 	}
 }
+
+func TestAddDependencyHashesSkipsEmptyHashSentinel(t *testing.T) {
+	// The cache returns a zero-length sentinel instead of a hash when a
+	// label's file is missing or is a directory. Persisting it would make
+	// the whole seed fail validation.
+	cache := NewTargetHashCache(nil, &Normalizer{}, "release 8.0.0", true, nil)
+	if err := cache.SeedHashes(map[string][]byte{
+		"//pkg:real\x00":    bytes.Repeat([]byte{0x44}, sha256.Size),
+		"//pkg:missing\x00": {},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	targetHashes := map[string]map[string]string{}
+	AddDependencyHashes(targetHashes, map[string][]string{
+		"//pkg:real": {"//pkg:missing"},
+	}, cache)
+
+	if _, ok := targetHashes["//pkg:real"]; !ok {
+		t.Error("//pkg:real should have been added")
+	}
+	if got, ok := targetHashes["//pkg:missing"]; ok {
+		t.Errorf("empty-hash sentinel should not be persisted, got %v", got)
+	}
+}

From 178db4c548a2a4b38aa6a77bff6a0f2a0019a62a Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 16:16:42 +0200
Subject: [PATCH 08/15] fix(hash-persister): decide source files from git, and
 fix rdeps traversal

Source files were being persisted into the seed so the probe could
compare their hashes. That was the wrong instrument: runSeeded already
computes a git diff, and a source-file label maps one to one onto a
path, so git answers the question outright. Hashing to rediscover it
added 485k of the 505k labels the previous commit put in the seed, more
than twenty times the 20k rule and generated labels that genuinely need
one, and dangling source-file labels are also the only source of the
empty-hash sentinel that rejected whole seeds.

The probe already queries every label in the dirty packages, so it
reports which of them are source files; those are judged by the git
diff and the rest by hash, and AddDependencyHashes skips them entirely.

Fixes a separate bug this exposed: propagateFrom pre-seeded the result
set with every dirty label and then used that same set as the BFS
visited set, so propagation stopped at the first dirty label instead of
passing through it and dropped everything behind. Usually masked,
because a changed dependency normally changes its consumer's hash and
makes it a root in its own right, but the failure mode is a missing
impacted target, so the sets are now kept apart.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go |  41 ++++++------
 pkg/dirty_set.go                 |  81 +++++++++++++++++------
 pkg/dirty_set_test.go            | 108 ++++++++++++++++++++++++++++++-
 pkg/hash_cache.go                |  16 +++++
 pkg/hash_persistence.go          |  18 ++++--
 5 files changed, 221 insertions(+), 43 deletions(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index d0ee043..27bbbee 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -296,7 +296,7 @@ func runSeeded(cfg *config) (seededOutcome, error) {
 	// the seed and only propagate rdeps from targets that actually changed.
 	unprunedDirtyStarCount := len(dirtyResult.DirtyStarLabels)
 	if unprunedDirtyStarCount > len(dirtyResult.DirtyLabels) {
-		dirtyResult, err = probePruneDirtySet(cfg, commitRev, dirtyResult, seedData)
+		dirtyResult, err = probePruneDirtySet(cfg, commitRev, dirtyResult, seedData, changedFiles)
 		if err != nil {
 			log.Printf("Probe pruning failed, continuing with unpruned dirty set: %v", err)
 		} else {
@@ -413,28 +413,30 @@ func probePruneDirtySet(
 	commitRev pkg.LabelledGitRev,
 	dirtyResult *pkg.DirtySetResult,
 	seedData *pkg.PersistedHashData,
+	changedFiles map[string]string,
 ) (*pkg.DirtySetResult, error) {
 	phaseStart := time.Now()
 
-	probeHashes, err := probePackageHashes(cfg, commitRev, dirtyResult.DirtyPackages, seedData)
+	probe, err := probePackages(cfg, commitRev, dirtyResult.DirtyPackages, seedData)
 	if err != nil {
 		return nil, err
 	}
-	log.Printf("Phase probe completed in %v (%d targets in %d packages)",
-		time.Since(phaseStart), len(probeHashes), len(dirtyResult.DirtyPackages))
+	log.Printf("Phase probe completed in %v (%d hashed, %d source files, in %d packages)",
+		time.Since(phaseStart), len(probe.Hashes), len(probe.SourceFiles),
+		len(dirtyResult.DirtyPackages))
 
-	return pkg.PruneDirtySet(dirtyResult, seedData.TargetHashes, seedData.TargetEdges, probeHashes), nil
+	return pkg.PruneDirtySet(dirtyResult, seedData.TargetHashes, seedData.TargetEdges, probe, changedFiles), nil
 }
 
-// probePackageHashes queries and hashes targets in the given packages,
-// seeding dependency hashes from the seed file so that only the dirty
-// packages need a Bazel query.
-func probePackageHashes(
+// probePackages queries and hashes targets in the given packages, seeding
+// dependency hashes from the seed file so that only the dirty packages need
+// a Bazel query.
+func probePackages(
 	cfg *config,
 	commitRev pkg.LabelledGitRev,
 	dirtyPackages []string,
 	seedData *pkg.PersistedHashData,
-) (map[string]string, error) {
+) (pkg.ProbeResult, error) {
 	// Query the dirty packages raw, deliberately bypassing the targets
 	// pattern. That pattern excludes manual-tagged targets, which covers
 	// most labels in a dirty package (npm links, platform() rules, JS build
@@ -444,32 +446,35 @@ func probePackageHashes(
 	probePattern := buildProbePattern(dirtyPackages)
 	probeTargets, err := pkg.ParseTargetsList(probePattern)
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse probe targets: %w", err)
+		return pkg.ProbeResult{}, fmt.Errorf("failed to parse probe targets: %w", err)
 	}
 
 	probeResults, probeCleanup, err := pkg.LoadIncompleteMetadata(cfg.Context, commitRev, probeTargets)
 	if err != nil {
 		probeCleanup()
-		return nil, fmt.Errorf("probe query failed: %w", err)
+		return pkg.ProbeResult{}, fmt.Errorf("probe query failed: %w", err)
 	}
 	defer probeCleanup()
 
 	probeSeedHashes, err := buildExternalSeedHashes(seedData, dirtyPackages)
 	if err != nil {
-		return nil, err
+		return pkg.ProbeResult{}, err
 	}
 	if err := probeResults.TargetHashCache.SeedHashes(probeSeedHashes); err != nil {
-		return nil, fmt.Errorf("cannot seed probe hashes: %w", err)
+		return pkg.ProbeResult{}, fmt.Errorf("cannot seed probe hashes: %w", err)
 	}
 
 	if err := probeResults.PrefillCache(); err != nil {
-		return nil, fmt.Errorf("probe hashing failed: %w", err)
+		return pkg.ProbeResult{}, fmt.Errorf("probe hashing failed: %w", err)
 	}
 
-	// Read straight from the cache rather than from MatchingTargets: it
-	// additionally holds the transitively-computed hashes of dependencies
+	// Read hashes straight from the cache rather than from MatchingTargets:
+	// it additionally holds the transitively-computed hashes of dependencies
 	// outside the probed packages, which cost nothing extra to include.
-	return probeResults.TargetHashCache.ExtractHexHashes(), nil
+	return pkg.ProbeResult{
+		Hashes:      probeResults.TargetHashCache.ExtractHexHashes(),
+		SourceFiles: probeResults.TargetHashCache.SourceFileLabels(),
+	}, nil
 }
 
 // buildProbePattern returns a bazel query expression covering every target
diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go
index f863ebc..c5c50c4 100644
--- a/pkg/dirty_set.go
+++ b/pkg/dirty_set.go
@@ -170,24 +170,36 @@ func ComputeDirtySet(
 	return result
 }
 
+// ProbeResult is what a probe of the dirty packages learned about them.
+type ProbeResult struct {
+	// Hashes maps "label\x00configuration" to the hex hash computed at the
+	// destination revision.
+	Hashes map[string]string
+	// SourceFiles is the set of probed labels that are source files. Their
+	// dirtiness comes from the git diff rather than from Hashes.
+	SourceFiles map[string]bool
+}
+
 // PruneDirtySet narrows a DirtySetResult by propagating reverse
-// dependencies only from targets whose hash actually changed, rather than
-// from every target in a dirty package. A BUILD.bazel edit marks its whole
+// dependencies only from targets that actually changed, rather than from
+// every target in a dirty package. A BUILD.bazel edit marks its whole
 // package dirty, so in a high-fanout package the unpruned rdeps closure can
 // cover most of the repository even when only one target really changed.
 //
-// probeHashes must cover every label in the dirty packages, keyed as
-// "label\x00configuration"; labels it omits are conservatively treated as
-// changed. DirtyLabels and DirtyPackages are preserved — those packages
-// still need re-listing via wildcards — and only DirtyStarLabels is
-// recomputed.
+// A source file is unchanged exactly when the git diff does not mention it.
+// Every other label is unchanged when its probe hash matches the seed;
+// labels the probe or the seed omits are conservatively treated as changed.
+//
+// DirtyLabels and DirtyPackages are preserved — those packages still need
+// re-listing via wildcards — and only DirtyStarLabels is recomputed.
 func PruneDirtySet(
 	original *DirtySetResult,
 	seedHashes map[string]map[string]string,
 	edges map[string][]string,
-	probeHashes map[string]string,
+	probe ProbeResult,
+	changedFiles map[string]string,
 ) *DirtySetResult {
-	actuallyChanged := findChangedTargets(original.DirtyLabels, seedHashes, probeHashes)
+	actuallyChanged := findChangedTargets(original.DirtyLabels, seedHashes, probe, changedFiles)
 	newDirtyStar := propagateFrom(original.DirtyLabels, actuallyChanged, edges)
 
 	return &DirtySetResult{
@@ -197,23 +209,46 @@ func PruneDirtySet(
 	}
 }
 
-// findChangedTargets returns the subset of dirtyLabels whose probe hash
-// differs from the seed. New targets (absent from seed) and targets absent
-// from probe results are treated as changed.
+// findChangedTargets returns the subset of dirtyLabels that changed between
+// the seed and the destination revision.
 func findChangedTargets(
 	dirtyLabels map[string]bool,
 	seedHashes map[string]map[string]string,
-	probeHashes map[string]string,
+	probe ProbeResult,
+	changedFiles map[string]string,
 ) map[string]bool {
 	changed := make(map[string]bool)
 	for label := range dirtyLabels {
-		if targetHashChanged(label, seedHashes[label], probeHashes) {
+		if probe.SourceFiles[label] {
+			if _, ok := changedFiles[labelToPath(label)]; ok {
+				changed[label] = true
+			}
+			continue
+		}
+		if targetHashChanged(label, seedHashes[label], probe.Hashes) {
 			changed[label] = true
 		}
 	}
 	return changed
 }
 
+// labelToPath returns the workspace-relative path a main-repo label refers
+// to, e.g. "//pkg:sub/f.java" becomes "pkg/sub/f.java".
+func labelToPath(label string) string {
+	if idx := strings.Index(label, "//"); idx >= 0 {
+		label = label[idx+len("//"):]
+	}
+	pkg, name, found := strings.Cut(label, ":")
+	switch {
+	case !found:
+		return pkg
+	case pkg == "":
+		return name
+	default:
+		return pkg + "/" + name
+	}
+}
+
 // targetHashChanged reports whether a single target's probe hash differs
 // from its seed hash. Returns true for new targets (nil seedConfigs) and
 // targets missing from probe results.
@@ -234,22 +269,32 @@ func targetHashChanged(label string, seedConfigs map[string]string, probeHashes
 // labels and BFS-propagating rdeps only from the actuallyChanged subset.
 func propagateFrom(dirtyLabels, actuallyChanged map[string]bool, edges map[string][]string) map[string]bool {
 	rdeps := BuildRdeps(edges)
-	result := make(map[string]bool, len(dirtyLabels))
 
+	// Every directly dirty label is dirty* regardless of whether it
+	// changed, but membership of the result must not be mistaken for
+	// having been traversed: an unchanged dirty label still has to be
+	// walked through to reach what lies behind it. Hence a separate
+	// visited set from the result set.
+	result := make(map[string]bool, len(dirtyLabels))
 	for label := range dirtyLabels {
 		result[label] = true
 	}
 
+	visited := make(map[string]bool, len(actuallyChanged))
 	queue := make([]string, 0, len(actuallyChanged))
 	for label := range actuallyChanged {
-		queue = append(queue, label)
+		if !visited[label] {
+			visited[label] = true
+			queue = append(queue, label)
+		}
 	}
 	for len(queue) > 0 {
 		current := queue[0]
 		queue = queue[1:]
 		for _, rdep := range rdeps[current] {
-			if !result[rdep] {
-				result[rdep] = true
+			result[rdep] = true
+			if !visited[rdep] {
+				visited[rdep] = true
 				queue = append(queue, rdep)
 			}
 		}
diff --git a/pkg/dirty_set_test.go b/pkg/dirty_set_test.go
index 8894f83..3637ab3 100644
--- a/pkg/dirty_set_test.go
+++ b/pkg/dirty_set_test.go
@@ -502,7 +502,7 @@ func TestPruneDirtySetEliminatesUnchangedRdeps(t *testing.T) {
 		"//tools/binaries:new_tool\x00":      "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges, probeHashes)
+	pruned := PruneDirtySet(original, seedHashes, edges, ProbeResult{Hashes: probeHashes}, nil)
 
 	// Directly dirty labels should be preserved.
 	if !pruned.DirtyStarLabels["//tools/binaries:existing_tool"] {
@@ -555,7 +555,7 @@ func TestPruneDirtySetPreservesChangedRdeps(t *testing.T) {
 		"//lib:src.java\x00": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000",
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges, probeHashes)
+	pruned := PruneDirtySet(original, seedHashes, edges, ProbeResult{Hashes: probeHashes}, nil)
 
 	if !pruned.DirtyStarLabels["//app:consumer"] {
 		t.Error("//app:consumer must remain dirty — its dep //lib:changed has a different hash")
@@ -583,10 +583,112 @@ func TestPruneDirtySetNoOpWhenAllChanged(t *testing.T) {
 		"//pkg:a\x00": "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000",
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges, probeHashes)
+	pruned := PruneDirtySet(original, seedHashes, edges, ProbeResult{Hashes: probeHashes}, nil)
 
 	// Everything should remain dirty — same as unpruned.
 	if !pruned.DirtyStarLabels["//app:dep"] {
 		t.Error("//app:dep must remain dirty when //pkg:a changed")
 	}
 }
+
+func TestPruneDirtySetJudgesSourceFilesByGitNotHash(t *testing.T) {
+	// A source file carries no seed hash; the git diff decides. //pkg:kept.java
+	// is untouched so its consumer must not propagate, while //pkg:edited.java
+	// is in the diff so its consumer must.
+	edges := map[string][]string{
+		"//pkg:uses_kept":   {"//pkg:kept.java"},
+		"//pkg:uses_edited": {"//pkg:edited.java"},
+		"//app:via_kept":    {"//pkg:uses_kept"},
+		"//app:via_edited":  {"//pkg:uses_edited"},
+	}
+	allLabels := CollectAllLabels(edges, nil)
+	original := ComputeDirtySet(
+		map[string]string{"pkg/edited.java": "M"}, edges, allLabels, nil,
+	)
+
+	// Both source files are directly dirty before pruning, and both
+	// consumers are reachable from them.
+	for _, label := range []string{"//app:via_kept", "//app:via_edited"} {
+		if !original.DirtyStarLabels[label] {
+			t.Fatalf("expected %s in unpruned DirtyStarLabels", label)
+		}
+	}
+
+	// A consumer of a changed source file hashes differently, as it would
+	// in reality; the consumer of the untouched one does not.
+	unchanged := "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000"
+	differs := "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000"
+	seedHashes := map[string]map[string]string{
+		"//pkg:uses_kept":   {"": unchanged},
+		"//pkg:uses_edited": {"": unchanged},
+	}
+	probe := ProbeResult{
+		Hashes: map[string]string{
+			"//pkg:uses_kept\x00":   unchanged,
+			"//pkg:uses_edited\x00": differs,
+		},
+		SourceFiles: map[string]bool{
+			"//pkg:kept.java":   true,
+			"//pkg:edited.java": true,
+		},
+	}
+
+	pruned := PruneDirtySet(original, seedHashes, edges,
+		probe, map[string]string{"pkg/edited.java": "M"})
+
+	if pruned.DirtyStarLabels["//app:via_kept"] {
+		t.Error("//app:via_kept should be pruned: pkg/kept.java is not in the git diff")
+	}
+	if !pruned.DirtyStarLabels["//app:via_edited"] {
+		t.Error("//app:via_edited must stay dirty: pkg/edited.java is in the git diff")
+	}
+}
+
+func TestLabelToPath(t *testing.T) {
+	for _, tt := range []struct{ label, want string }{
+		{"//pkg:file.java", "pkg/file.java"},
+		{"//pkg:src/main/java/App.java", "pkg/src/main/java/App.java"},
+		{"//:root.txt", "root.txt"},
+		{"//a/b/c:d.txt", "a/b/c/d.txt"},
+		{"//pkg", "pkg"},
+	} {
+		if got := labelToPath(tt.label); got != tt.want {
+			t.Errorf("labelToPath(%q) = %q, want %q", tt.label, got, tt.want)
+		}
+	}
+}
+
+func TestPropagateTraversesThroughUnchangedDirtyLabels(t *testing.T) {
+	// //pkg:middle is dirty (its package changed) but its own hash did not
+	// change. It still has to be walked through, or //app:behind is lost.
+	// Membership of the result set must not double as "already traversed".
+	edges := map[string][]string{
+		"//pkg:changed": {"//pkg:src.java"},
+		"//pkg:middle":  {"//pkg:changed"},
+		"//app:behind":  {"//pkg:middle"},
+	}
+	allLabels := CollectAllLabels(edges, nil)
+	original := ComputeDirtySet(
+		map[string]string{"pkg/src.java": "M"}, edges, allLabels, nil,
+	)
+
+	unchanged := "bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000bbbb0000"
+	seedHashes := map[string]map[string]string{
+		"//pkg:changed": {"": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000"},
+		"//pkg:middle":  {"": unchanged},
+	}
+	probe := ProbeResult{
+		Hashes: map[string]string{
+			"//pkg:changed\x00": "dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000dddd0000",
+			"//pkg:middle\x00":  unchanged,
+		},
+		SourceFiles: map[string]bool{"//pkg:src.java": true},
+	}
+
+	pruned := PruneDirtySet(original, seedHashes, edges, probe,
+		map[string]string{"pkg/src.java": "M"})
+
+	if !pruned.DirtyStarLabels["//app:behind"] {
+		t.Error("//app:behind must be reached by traversing through the unchanged dirty label //pkg:middle")
+	}
+}
diff --git a/pkg/hash_cache.go b/pkg/hash_cache.go
index d18d65c..95b9e65 100644
--- a/pkg/hash_cache.go
+++ b/pkg/hash_cache.go
@@ -214,6 +214,22 @@ func (thc *TargetHashCache) ExtractHexHashes() map[string]string {
 	return hashes
 }
 
+// SourceFileLabels returns the labels in the cache's query context that are
+// source files. Whether a source file changed is answered by the git diff,
+// so these labels need neither a persisted hash nor a hash comparison.
+func (thc *TargetHashCache) SourceFileLabels() map[string]bool {
+	sourceFiles := make(map[string]bool)
+	for label, configurations := range thc.context {
+		for _, configuredTarget := range configurations {
+			if configuredTarget.GetTarget().GetType() == build.Target_SOURCE_FILE {
+				sourceFiles[label.String()] = true
+				break
+			}
+		}
+	}
+	return sourceFiles
+}
+
 // splitHashKey splits a "<label>\x00<configuration>" cache key.
 func splitHashKey(key string) (label, configuration string, ok bool) {
 	idx := strings.IndexByte(key, '\x00')
diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index 0d559f8..c393f3e 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -202,14 +202,24 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 // computed in the cache via recursive Hash() calls, so persisting them is
 // free, and without them incremental hashing cannot tell whether such a
 // label changed and must conservatively propagate its reverse dependencies.
+//
+// Source files are deliberately excluded. The git diff already says whether
+// a file changed, so hashing one to rediscover that is redundant, and they
+// outnumber the labels that do need a hash by more than twenty to one.
 func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[string][]string, cache *TargetHashCache) {
-	// Both edge keys and dependency values: leaf labels (source files, npm
-	// /ref targets) never appear as keys, so iterating keys alone misses them.
+	sourceFiles := cache.SourceFileLabels()
+
+	// Both edge keys and dependency values: leaf labels (npm /ref targets)
+	// never appear as keys, so iterating keys alone misses them.
 	wanted := make(map[string]bool)
 	consider := func(label string) {
-		if _, ok := targetHashes[label]; !ok {
-			wanted[label] = true
+		if _, ok := targetHashes[label]; ok {
+			return
+		}
+		if sourceFiles[label] {
+			return
 		}
+		wanted[label] = true
 	}
 	for label, deps := range edges {
 		consider(label)

From de7cd7f4ff90ccec1eab3edee7784fe462761bb0 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 16:31:46 +0200
Subject: [PATCH 09/15] fix(hash-persister): keep dependency hashes out of the
 target set

Dependency hashes were being written into TargetHashes. That map is the
target set diffing compares, so the seeded run merged them into its
output and the shadow comparison reported 81509 added targets, every one
of them a dependency label rather than a real target. Once the
incremental output feeds target selection directly the same labels would
become phantom impacted targets.

Persist them under a separate dependency_hashes key instead. Seeding
reads both, since either is equally reusable as a pre-computed hash, and
diffing keeps seeing TargetHashes alone.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go | 54 +++++++++++++++++++-------------
 pkg/dirty_set.go                 | 11 +++----
 pkg/dirty_set_test.go            | 44 ++++++++++++++++++++++----
 pkg/hash_persistence.go          | 43 ++++++++++++++++++-------
 pkg/hash_persistence_test.go     | 27 ++++++++++------
 5 files changed, 124 insertions(+), 55 deletions(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index 27bbbee..45d54a0 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -273,6 +273,9 @@ func runSeeded(cfg *config) (seededOutcome, error) {
 			seedData.TargetEdges,
 			seedData.SeedCompatibilityFingerprint,
 		)
+		if cfg.SeedableOutput {
+			persistedData.DependencyHashes = seedData.DependencyHashes
+		}
 		if err := pkg.WritePersistedData(cfg.OutputFile, persistedData); err != nil {
 			return seededOutcome{}, fmt.Errorf("failed to persist hashes: %w", err)
 		}
@@ -425,7 +428,7 @@ func probePruneDirtySet(
 		time.Since(phaseStart), len(probe.Hashes), len(probe.SourceFiles),
 		len(dirtyResult.DirtyPackages))
 
-	return pkg.PruneDirtySet(dirtyResult, seedData.TargetHashes, seedData.TargetEdges, probe, changedFiles), nil
+	return pkg.PruneDirtySet(dirtyResult, seedData, probe, changedFiles), nil
 }
 
 // probePackages queries and hashes targets in the given packages, seeding
@@ -505,16 +508,18 @@ func buildExternalSeedHashes(
 		dirtyPkgSet[p] = true
 	}
 	hashes := make(map[string][]byte)
-	for label, configMap := range seedData.TargetHashes {
-		if dirtyPkgSet[pkg.LabelPackage(label)] {
-			continue
-		}
-		for configStr, hashHex := range configMap {
-			hashBytes, err := hex.DecodeString(hashHex)
-			if err != nil {
-				return nil, fmt.Errorf("invalid seed hash for %s: %w", label, err)
+	for _, source := range []map[string]map[string]string{seedData.TargetHashes, seedData.DependencyHashes} {
+		for label, configMap := range source {
+			if dirtyPkgSet[pkg.LabelPackage(label)] {
+				continue
+			}
+			for configStr, hashHex := range configMap {
+				hashBytes, err := hex.DecodeString(hashHex)
+				if err != nil {
+					return nil, fmt.Errorf("invalid seed hash for %s: %w", label, err)
+				}
+				hashes[label+"\x00"+configStr] = hashBytes
 			}
-			hashes[label+"\x00"+configStr] = hashBytes
 		}
 	}
 	return hashes, nil
@@ -561,19 +566,24 @@ func validateSeed(seedData *pkg.PersistedHashData, expectedSha, expectedFingerpr
 
 func reusableSeedHashes(seedData *pkg.PersistedHashData, dirtyLabels map[string]bool) (map[string][]byte, error) {
 	hashes := make(map[string][]byte)
-	for label, configMap := range seedData.TargetHashes {
-		if dirtyLabels[label] {
-			continue
-		}
-		for configStr, hashHex := range configMap {
-			hashBytes, err := hex.DecodeString(hashHex)
-			if err != nil {
-				return nil, fmt.Errorf("invalid hash hex for %s: %w", label, err)
+	// Dependency hashes are as reusable as target hashes for seeding the
+	// cache; they are only kept in a separate bucket so that diffing sees
+	// the target set alone.
+	for _, source := range []map[string]map[string]string{seedData.TargetHashes, seedData.DependencyHashes} {
+		for label, configMap := range source {
+			if dirtyLabels[label] {
+				continue
 			}
-			if len(hashBytes) != sha256.Size {
-				return nil, fmt.Errorf("invalid hash length for %s: got %d bytes, want %d", label, len(hashBytes), sha256.Size)
+			for configStr, hashHex := range configMap {
+				hashBytes, err := hex.DecodeString(hashHex)
+				if err != nil {
+					return nil, fmt.Errorf("invalid hash hex for %s: %w", label, err)
+				}
+				if len(hashBytes) != sha256.Size {
+					return nil, fmt.Errorf("invalid hash length for %s: got %d bytes, want %d", label, len(hashBytes), sha256.Size)
+				}
+				hashes[label+"\x00"+configStr] = hashBytes
 			}
-			hashes[label+"\x00"+configStr] = hashBytes
 		}
 	}
 	return hashes, nil
@@ -618,7 +628,7 @@ func mergePersistedData(
 		}
 	}
 	mergedEdges := mergePersistedEntries(seedData.TargetEdges, dirtyLabels, freshEdges)
-	pkg.AddDependencyHashes(persistedData.TargetHashes, mergedEdges, queryResults.TargetHashCache)
+	persistedData.DependencyHashes = pkg.ExtractDependencyHashes(persistedData.TargetHashes, mergedEdges, queryResults.TargetHashCache)
 	persistedData.FormatVersion = pkg.CurrentPersistedHashFormatVersion
 	persistedData.SeedCompatibilityFingerprint = compatibilityFingerprint
 	persistedData.TargetEdges = mergedEdges
diff --git a/pkg/dirty_set.go b/pkg/dirty_set.go
index c5c50c4..e545f3a 100644
--- a/pkg/dirty_set.go
+++ b/pkg/dirty_set.go
@@ -194,13 +194,12 @@ type ProbeResult struct {
 // re-listing via wildcards — and only DirtyStarLabels is recomputed.
 func PruneDirtySet(
 	original *DirtySetResult,
-	seedHashes map[string]map[string]string,
-	edges map[string][]string,
+	seed *PersistedHashData,
 	probe ProbeResult,
 	changedFiles map[string]string,
 ) *DirtySetResult {
-	actuallyChanged := findChangedTargets(original.DirtyLabels, seedHashes, probe, changedFiles)
-	newDirtyStar := propagateFrom(original.DirtyLabels, actuallyChanged, edges)
+	actuallyChanged := findChangedTargets(original.DirtyLabels, seed, probe, changedFiles)
+	newDirtyStar := propagateFrom(original.DirtyLabels, actuallyChanged, seed.TargetEdges)
 
 	return &DirtySetResult{
 		DirtyLabels:     original.DirtyLabels,
@@ -213,7 +212,7 @@ func PruneDirtySet(
 // the seed and the destination revision.
 func findChangedTargets(
 	dirtyLabels map[string]bool,
-	seedHashes map[string]map[string]string,
+	seed *PersistedHashData,
 	probe ProbeResult,
 	changedFiles map[string]string,
 ) map[string]bool {
@@ -225,7 +224,7 @@ func findChangedTargets(
 			}
 			continue
 		}
-		if targetHashChanged(label, seedHashes[label], probe.Hashes) {
+		if targetHashChanged(label, seed.SeedHashes(label), probe.Hashes) {
 			changed[label] = true
 		}
 	}
diff --git a/pkg/dirty_set_test.go b/pkg/dirty_set_test.go
index 3637ab3..6d14df3 100644
--- a/pkg/dirty_set_test.go
+++ b/pkg/dirty_set_test.go
@@ -502,7 +502,7 @@ func TestPruneDirtySetEliminatesUnchangedRdeps(t *testing.T) {
 		"//tools/binaries:new_tool\x00":      "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges, ProbeResult{Hashes: probeHashes}, nil)
+	pruned := PruneDirtySet(original, &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, ProbeResult{Hashes: probeHashes}, nil)
 
 	// Directly dirty labels should be preserved.
 	if !pruned.DirtyStarLabels["//tools/binaries:existing_tool"] {
@@ -555,7 +555,7 @@ func TestPruneDirtySetPreservesChangedRdeps(t *testing.T) {
 		"//lib:src.java\x00": "cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000cccc0000",
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges, ProbeResult{Hashes: probeHashes}, nil)
+	pruned := PruneDirtySet(original, &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, ProbeResult{Hashes: probeHashes}, nil)
 
 	if !pruned.DirtyStarLabels["//app:consumer"] {
 		t.Error("//app:consumer must remain dirty — its dep //lib:changed has a different hash")
@@ -583,7 +583,7 @@ func TestPruneDirtySetNoOpWhenAllChanged(t *testing.T) {
 		"//pkg:a\x00": "ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000ffff0000",
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges, ProbeResult{Hashes: probeHashes}, nil)
+	pruned := PruneDirtySet(original, &PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges}, ProbeResult{Hashes: probeHashes}, nil)
 
 	// Everything should remain dirty — same as unpruned.
 	if !pruned.DirtyStarLabels["//app:dep"] {
@@ -633,7 +633,8 @@ func TestPruneDirtySetJudgesSourceFilesByGitNotHash(t *testing.T) {
 		},
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges,
+	pruned := PruneDirtySet(original,
+		&PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges},
 		probe, map[string]string{"pkg/edited.java": "M"})
 
 	if pruned.DirtyStarLabels["//app:via_kept"] {
@@ -685,10 +686,41 @@ func TestPropagateTraversesThroughUnchangedDirtyLabels(t *testing.T) {
 		SourceFiles: map[string]bool{"//pkg:src.java": true},
 	}
 
-	pruned := PruneDirtySet(original, seedHashes, edges, probe,
-		map[string]string{"pkg/src.java": "M"})
+	pruned := PruneDirtySet(original,
+		&PersistedHashData{TargetHashes: seedHashes, TargetEdges: edges},
+		probe, map[string]string{"pkg/src.java": "M"})
 
 	if !pruned.DirtyStarLabels["//app:behind"] {
 		t.Error("//app:behind must be reached by traversing through the unchanged dirty label //pkg:middle")
 	}
 }
+
+func TestPruneDirtySetReadsDependencyHashes(t *testing.T) {
+	// A manual-tagged dep lives in DependencyHashes, not TargetHashes, yet
+	// must still be comparable or its reverse dependencies all propagate.
+	edges := map[string][]string{
+		"//pkg:tool":     {},
+		"//app:consumer": {"//pkg:tool"},
+	}
+	allLabels := CollectAllLabels(edges, nil)
+	original := ComputeDirtySet(
+		map[string]string{"pkg/BUILD.bazel": "M"}, edges, allLabels, nil,
+	)
+	if !original.DirtyStarLabels["//app:consumer"] {
+		t.Fatal("expected //app:consumer in unpruned DirtyStarLabels")
+	}
+
+	unchanged := "aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000aaaa0000"
+	seed := &PersistedHashData{
+		TargetHashes:     map[string]map[string]string{},
+		DependencyHashes: map[string]map[string]string{"//pkg:tool": {"": unchanged}},
+		TargetEdges:      edges,
+	}
+	probe := ProbeResult{Hashes: map[string]string{"//pkg:tool\x00": unchanged}}
+
+	pruned := PruneDirtySet(original, seed, probe, nil)
+
+	if pruned.DirtyStarLabels["//app:consumer"] {
+		t.Error("//app:consumer should be pruned: //pkg:tool is unchanged per DependencyHashes")
+	}
+}
diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index c393f3e..e3fc7aa 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -43,10 +43,25 @@ type PersistedHashData struct {
 	// independent) because CI uses --query-backend=query with a single null
 	// configuration.
 	TargetEdges map[string][]string `json:"target_edges,omitempty"`
+	// DependencyHashes holds hashes for labels that appear in TargetEdges
+	// without being matching targets, such as manual-tagged deps and
+	// platform() rules. Incremental hashing needs them to tell whether such
+	// a label changed, but they are not part of the target set and must
+	// stay out of TargetHashes, which is what diffing compares.
+	DependencyHashes map[string]map[string]string `json:"dependency_hashes,omitempty"`
 	// Metadata contains additional information about the computation
 	Metadata HashMetadata `json:"metadata"`
 }
 
+// SeedHashes returns the recorded per-configuration hashes for a label,
+// whether it was persisted as a matching target or as a dependency.
+func (d *PersistedHashData) SeedHashes(label string) map[string]string {
+	if hashes, ok := d.TargetHashes[label]; ok {
+		return hashes
+	}
+	return d.DependencyHashes[label]
+}
+
 // SeedCompatibility describes every invocation-level input that must remain
 // identical for persisted hashes to be safely reused. Git revision, timestamp,
 // workspace path, and output path are deliberately excluded.
@@ -183,7 +198,7 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 		if err != nil {
 			return fmt.Errorf("failed to extract target edges: %w", err)
 		}
-		AddDependencyHashes(targetHashes, targetEdges, queryResults.TargetHashCache)
+		persistedData.DependencyHashes = ExtractDependencyHashes(targetHashes, targetEdges, queryResults.TargetHashCache)
 		compatibilityFingerprint, err := ComputeSeedCompatibilityFingerprint(context, targetsPattern, queryResults.BazelRelease)
 		if err != nil {
 			return err
@@ -196,17 +211,21 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 	return writePersistedData(filePath, &persistedData, !seedable)
 }
 
-// AddDependencyHashes supplements targetHashes with hashes for labels that
-// appear in the edge map but are not matching targets — manual-tagged deps,
-// platform() rules, generated file outputs. Their hashes were already
-// computed in the cache via recursive Hash() calls, so persisting them is
-// free, and without them incremental hashing cannot tell whether such a
-// label changed and must conservatively propagate its reverse dependencies.
+// ExtractDependencyHashes returns hashes for labels that appear in the edge
+// map but are not matching targets — manual-tagged deps, platform() rules,
+// generated file outputs. Their hashes were already computed in the cache
+// via recursive Hash() calls, so recording them is free, and without them
+// incremental hashing cannot tell whether such a label changed and must
+// conservatively propagate its reverse dependencies.
+//
+// They are returned separately rather than merged into targetHashes: only
+// matching targets belong in the target set that diffing compares, and
+// mixing these in makes them surface as spurious added targets.
 //
 // Source files are deliberately excluded. The git diff already says whether
 // a file changed, so hashing one to rediscover that is redundant, and they
 // outnumber the labels that do need a hash by more than twenty to one.
-func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[string][]string, cache *TargetHashCache) {
+func ExtractDependencyHashes(targetHashes map[string]map[string]string, edges map[string][]string, cache *TargetHashCache) map[string]map[string]string {
 	sourceFiles := cache.SourceFileLabels()
 
 	// Both edge keys and dependency values: leaf labels (npm /ref targets)
@@ -228,9 +247,10 @@ func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[st
 		}
 	}
 	if len(wanted) == 0 {
-		return
+		return nil
 	}
 
+	dependencyHashes := make(map[string]map[string]string)
 	for key, hashHex := range cache.ExtractHexHashes() {
 		label, configuration, ok := splitHashKey(key)
 		if !ok || !wanted[label] {
@@ -245,13 +265,14 @@ func AddDependencyHashes(targetHashes map[string]map[string]string, edges map[st
 		if len(hashHex) != hex.EncodedLen(sha256.Size) {
 			continue
 		}
-		configs := targetHashes[label]
+		configs := dependencyHashes[label]
 		if configs == nil {
 			configs = make(map[string]string)
-			targetHashes[label] = configs
+			dependencyHashes[label] = configs
 		}
 		configs[configuration] = hashHex
 	}
+	return dependencyHashes
 }
 
 // WritePersistedData writes a PersistedHashData struct directly to a JSON file.
diff --git a/pkg/hash_persistence_test.go b/pkg/hash_persistence_test.go
index d20b176..769fa6a 100644
--- a/pkg/hash_persistence_test.go
+++ b/pkg/hash_persistence_test.go
@@ -375,9 +375,14 @@ func TestPersistenceModesPreserveHashesAndGeneratedFileEdges(t *testing.T) {
 			t.Fatalf("seedable hashes for %s differ from legacy: seedable=%v legacy=%v", label, seedableConfigs, legacyConfigs)
 		}
 	}
-	// The generated file output should now have a hash in seedable mode.
-	if _, ok := seedable.TargetHashes[outputLabel.String()]; !ok {
-		t.Fatal("seedable hashes should include dependency-only target //gen:output")
+	// The generated file's hash belongs in DependencyHashes, never in
+	// TargetHashes: diffing compares the target set, so a dependency label
+	// appearing there surfaces as a spurious added target.
+	if _, ok := seedable.DependencyHashes[outputLabel.String()]; !ok {
+		t.Fatal("seedable output should record //gen:output in DependencyHashes")
+	}
+	if _, ok := seedable.TargetHashes[outputLabel.String()]; ok {
+		t.Fatal("//gen:output must not leak into TargetHashes")
 	}
 }
 
@@ -403,17 +408,20 @@ func TestAddDependencyHashesCoversEdgeKeysAndDependencyValues(t *testing.T) {
 		"//pkg:already": {"": "preexisting"},
 	}
 
-	AddDependencyHashes(targetHashes, edges, cache)
+	dependencyHashes := ExtractDependencyHashes(targetHashes, edges, cache)
 
-	if got := targetHashes["//pkg:rule"][""]; got != hex.EncodeToString(filled(0x11)) {
+	if got := dependencyHashes["//pkg:rule"][""]; got != hex.EncodeToString(filled(0x11)) {
 		t.Errorf("edge key //pkg:rule not added, got %q", got)
 	}
-	if got := targetHashes["//pkg:leaf.txt"][""]; got != hex.EncodeToString(filled(0x22)) {
+	if got := dependencyHashes["//pkg:leaf.txt"][""]; got != hex.EncodeToString(filled(0x22)) {
 		t.Errorf("dependency-value-only //pkg:leaf.txt not added, got %q", got)
 	}
 	if got := targetHashes["//pkg:already"][""]; got != "preexisting" {
 		t.Errorf("existing hash overwritten with %q", got)
 	}
+	if _, ok := dependencyHashes["//pkg:already"]; ok {
+		t.Error("a label already in targetHashes must not be duplicated into dependencyHashes")
+	}
 }
 
 func TestAddDependencyHashesSkipsEmptyHashSentinel(t *testing.T) {
@@ -428,15 +436,14 @@ func TestAddDependencyHashesSkipsEmptyHashSentinel(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	targetHashes := map[string]map[string]string{}
-	AddDependencyHashes(targetHashes, map[string][]string{
+	dependencyHashes := ExtractDependencyHashes(map[string]map[string]string{}, map[string][]string{
 		"//pkg:real": {"//pkg:missing"},
 	}, cache)
 
-	if _, ok := targetHashes["//pkg:real"]; !ok {
+	if _, ok := dependencyHashes["//pkg:real"]; !ok {
 		t.Error("//pkg:real should have been added")
 	}
-	if got, ok := targetHashes["//pkg:missing"]; ok {
+	if got, ok := dependencyHashes["//pkg:missing"]; ok {
 		t.Errorf("empty-hash sentinel should not be persisted, got %v", got)
 	}
 }

From 8af1231a66a5a45ef9473036f5dbefb6d5a2a291 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 17:43:01 +0200
Subject: [PATCH 10/15] perf(hash-persister): stream the seedable artifact
 instead of buffering it

encoding/json marshals a whole value into one in-memory buffer before
writing any of it, growing that buffer by doubling. A seedable artifact
is over a gigabyte, so persisting one meant gigabytes of allocation and
copying and a peak resident size around twice the output. That phase
measured 41.5s against 3s for the compact artifact, tracking output size
rather than the work involved.

Emit the large maps entry by entry into a buffered writer so memory
stays flat. Labels and hex hashes never need escaping, so strings take a
straight copy and anything else defers to the stdlib rather than
reimplementing its escaping rules. Keys are sorted as encoding/json
sorts them, keeping the artifact reproducible.

The compact artifact keeps using the stdlib: it is two orders of
magnitude smaller, so the buffering costs little and hand-indenting
would not earn its keep.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 pkg/hash_persistence.go      | 185 ++++++++++++++++++++++++++++++++++-
 pkg/hash_persistence_test.go |  99 +++++++++++++++++++
 2 files changed, 282 insertions(+), 2 deletions(-)

diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index e3fc7aa..c18de8b 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -1,12 +1,14 @@
 package pkg
 
 import (
+	"bufio"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"os"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/bazel-contrib/target-determinator/third_party/protobuf/bazel/build"
@@ -289,16 +291,195 @@ func writePersistedData(filePath string, data *PersistedHashData, pretty bool) e
 	}
 	defer file.Close()
 
-	encoder := json.NewEncoder(file)
 	if pretty {
+		// The legacy artifact is two orders of magnitude smaller than a
+		// seedable one, so the encoder's buffering costs little here and
+		// indenting by hand would not earn its keep.
+		encoder := json.NewEncoder(file)
 		encoder.SetIndent("", "  ")
+		if err := encoder.Encode(data); err != nil {
+			return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
+		}
+		return nil
 	}
-	if err := encoder.Encode(data); err != nil {
+
+	writer := bufio.NewWriterSize(file, 1<<20)
+	if err := streamPersistedData(writer, data); err != nil {
 		return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
 	}
+	if err := writer.Flush(); err != nil {
+		return fmt.Errorf("failed to flush hash data to %s: %w", filePath, err)
+	}
 	return nil
 }
 
+// streamPersistedData writes data as JSON incrementally.
+//
+// encoding/json marshals a whole value into one in-memory buffer before
+// writing a byte of it, and grows that buffer by doubling. A seedable
+// artifact is over a gigabyte, so that means gigabytes of allocation and
+// copying, and a peak resident size around twice the output. Emitting the
+// large maps entry by entry keeps memory flat.
+//
+// Map keys are sorted, matching encoding/json, so the artifact stays
+// reproducible.
+func streamPersistedData(writer *bufio.Writer, data *PersistedHashData) error {
+	out := &jsonWriter{w: writer}
+	out.raw("{")
+	// Fields are emitted in struct order, and the omitempty ones are
+	// skipped on the same conditions encoding/json would apply, so the
+	// legacy shape is unchanged.
+	if data.FormatVersion != 0 {
+		out.key("format_version")
+		out.raw(strconv.Itoa(data.FormatVersion))
+		out.raw(",")
+	}
+	if data.SeedCompatibilityFingerprint != "" {
+		out.key("seed_compatibility_fingerprint")
+		out.str(data.SeedCompatibilityFingerprint)
+		out.raw(",")
+	}
+	out.key("git_commit_sha")
+	out.str(data.GitCommitSha)
+	out.raw(",")
+	out.key("timestamp")
+	out.marshal(data.Timestamp)
+	out.raw(",")
+	out.key("bazel_release")
+	out.str(data.BazelRelease)
+	out.raw(",")
+	out.key("target_hashes")
+	out.hashes(data.TargetHashes)
+	if len(data.TargetEdges) > 0 {
+		out.raw(",")
+		out.key("target_edges")
+		out.edges(data.TargetEdges)
+	}
+	if len(data.DependencyHashes) > 0 {
+		out.raw(",")
+		out.key("dependency_hashes")
+		out.hashes(data.DependencyHashes)
+	}
+	out.raw(",")
+	out.key("metadata")
+	out.marshal(data.Metadata)
+	out.raw("}\n")
+	return out.err
+}
+
+// jsonWriter emits JSON to a buffered writer, holding the first error so
+// that callers need not check after every token.
+type jsonWriter struct {
+	w   *bufio.Writer
+	err error
+}
+
+func (j *jsonWriter) raw(s string) {
+	if j.err != nil {
+		return
+	}
+	_, j.err = j.w.WriteString(s)
+}
+
+func (j *jsonWriter) key(name string) {
+	j.str(name)
+	j.raw(":")
+}
+
+// str writes a JSON string. Labels and hex hashes never need escaping, so
+// the common path is a straight copy; anything else defers to the stdlib
+// rather than reimplementing its escaping rules.
+func (j *jsonWriter) str(s string) {
+	if j.err != nil {
+		return
+	}
+	if !needsJSONEscape(s) {
+		j.raw(`"`)
+		j.raw(s)
+		j.raw(`"`)
+		return
+	}
+	j.marshal(s)
+}
+
+func (j *jsonWriter) marshal(v any) {
+	if j.err != nil {
+		return
+	}
+	encoded, err := json.Marshal(v)
+	if err != nil {
+		j.err = err
+		return
+	}
+	_, j.err = j.w.Write(encoded)
+}
+
+// hashes writes a label to configuration to hash mapping. A nil map is
+// written as null, as encoding/json would, since TargetHashes has no
+// omitempty.
+func (j *jsonWriter) hashes(m map[string]map[string]string) {
+	if m == nil {
+		j.raw("null")
+		return
+	}
+	j.raw("{")
+	for i, label := range sortedKeys(m) {
+		if i > 0 {
+			j.raw(",")
+		}
+		j.key(label)
+		j.raw("{")
+		for k, configuration := range sortedKeys(m[label]) {
+			if k > 0 {
+				j.raw(",")
+			}
+			j.key(configuration)
+			j.str(m[label][configuration])
+		}
+		j.raw("}")
+	}
+	j.raw("}")
+}
+
+// edges writes a label to dependency labels mapping. Dependency lists are
+// already sorted by ExtractEdges and are left in that order.
+func (j *jsonWriter) edges(m map[string][]string) {
+	j.raw("{")
+	for i, label := range sortedKeys(m) {
+		if i > 0 {
+			j.raw(",")
+		}
+		j.key(label)
+		j.raw("[")
+		for k, dep := range m[label] {
+			if k > 0 {
+				j.raw(",")
+			}
+			j.str(dep)
+		}
+		j.raw("]")
+	}
+	j.raw("}")
+}
+
+func sortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for key := range m {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func needsJSONEscape(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if c := s[i]; c < 0x20 || c == '"' || c == '\\' {
+			return true
+		}
+	}
+	return false
+}
+
 // LoadPersistedHashes loads persisted hash data from a JSON file
 func LoadPersistedHashes(filePath string) (*PersistedHashData, error) {
 	file, err := os.Open(filePath)
diff --git a/pkg/hash_persistence_test.go b/pkg/hash_persistence_test.go
index 769fa6a..e0da5d6 100644
--- a/pkg/hash_persistence_test.go
+++ b/pkg/hash_persistence_test.go
@@ -447,3 +447,102 @@ func TestAddDependencyHashesSkipsEmptyHashSentinel(t *testing.T) {
 		t.Errorf("empty-hash sentinel should not be persisted, got %v", got)
 	}
 }
+
+// A realistic payload must stream byte-for-byte what encoding/json would
+// produce, which pins field order, omitempty behaviour and key sorting.
+func TestStreamedSeedableOutputMatchesStdlibBytes(t *testing.T) {
+	data := &PersistedHashData{
+		FormatVersion:                CurrentPersistedHashFormatVersion,
+		SeedCompatibilityFingerprint: "fingerprint",
+		GitCommitSha:                 "abc123",
+		Timestamp:                    time.Unix(1700000000, 0).UTC(),
+		BazelRelease:                 "release 9.2.0",
+		TargetHashes: map[string]map[string]string{
+			"//z:last":  {"": "bb"},
+			"//a:first": {"": "aa", "cfg": "cc"},
+		},
+		TargetEdges: map[string][]string{
+			"//a:first": {"//z:last", "@@repo+ext//p:q"},
+		},
+		DependencyHashes: map[string]map[string]string{
+			"//tools/binaries:platform": {"": "dd"},
+		},
+		Metadata: HashMetadata{TargetsPattern: "//...", TotalTargets: 3},
+	}
+
+	path := filepath.Join(t.TempDir(), "seedable.json")
+	if err := writePersistedData(path, data, false); err != nil {
+		t.Fatal(err)
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want, err := json.Marshal(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want = append(want, '\n')
+
+	if !bytes.Equal(got, want) {
+		t.Fatalf("streamed output differs from encoding/json:\n got: %s\nwant: %s", got, want)
+	}
+}
+
+func TestStreamedOutputRoundTripsAwkwardStrings(t *testing.T) {
+	// The fast path copies strings verbatim, so anything needing escaping
+	// has to fall through to the stdlib correctly.
+	awkward := "//pkg:a\"quote\\backslash\u0001ctrl\tünïcode"
+	data := &PersistedHashData{
+		FormatVersion: CurrentPersistedHashFormatVersion,
+		GitCommitSha:  "abc123",
+		Timestamp:     time.Unix(1700000000, 0).UTC(),
+		BazelRelease:  "release 9.2.0",
+		TargetHashes:  map[string]map[string]string{awkward: {awkward: awkward}},
+		TargetEdges:   map[string][]string{awkward: {awkward}},
+		Metadata:      HashMetadata{TargetsPattern: awkward, TotalTargets: 1},
+	}
+
+	path := filepath.Join(t.TempDir(), "awkward.json")
+	if err := writePersistedData(path, data, false); err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := LoadPersistedHashes(path)
+	if err != nil {
+		t.Fatalf("streamed output is not parseable: %v", err)
+	}
+	if !reflect.DeepEqual(loaded.TargetHashes, data.TargetHashes) {
+		t.Errorf("target hashes did not round-trip: %v", loaded.TargetHashes)
+	}
+	if !reflect.DeepEqual(loaded.TargetEdges, data.TargetEdges) {
+		t.Errorf("target edges did not round-trip: %v", loaded.TargetEdges)
+	}
+}
+
+func TestStreamedOutputOmitsEmptyFieldsLikeStdlib(t *testing.T) {
+	// A v8-shaped artifact must not gain incremental keys.
+	data := &PersistedHashData{
+		GitCommitSha: "abc123",
+		Timestamp:    time.Unix(1700000000, 0).UTC(),
+		BazelRelease: "release 9.2.0",
+		TargetHashes: map[string]map[string]string{"//a:b": {"": "aa"}},
+		Metadata:     HashMetadata{TargetsPattern: "//...", TotalTargets: 1},
+	}
+	path := filepath.Join(t.TempDir(), "v8.json")
+	if err := writePersistedData(path, data, false); err != nil {
+		t.Fatal(err)
+	}
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, key := range []string{
+		"format_version", "seed_compatibility_fingerprint",
+		"target_edges", "dependency_hashes",
+	} {
+		if strings.Contains(string(got), key) {
+			t.Errorf("empty field %q should have been omitted: %s", key, got)
+		}
+	}
+}

From 841fef7bdcf8e3b30258800696d55acbddbb7fb5 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 18:53:29 +0200
Subject: [PATCH 11/15] Revert "perf(hash-persister): stream the seedable
 artifact instead of buffering it"

This reverts commit 8af1231a66a5a45ef9473036f5dbefb6d5a2a291.
---
 pkg/hash_persistence.go      | 185 +----------------------------------
 pkg/hash_persistence_test.go |  99 -------------------
 2 files changed, 2 insertions(+), 282 deletions(-)

diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index c18de8b..e3fc7aa 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -1,14 +1,12 @@
 package pkg
 
 import (
-	"bufio"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"os"
 	"sort"
-	"strconv"
 	"time"
 
 	"github.com/bazel-contrib/target-determinator/third_party/protobuf/bazel/build"
@@ -291,195 +289,16 @@ func writePersistedData(filePath string, data *PersistedHashData, pretty bool) e
 	}
 	defer file.Close()
 
+	encoder := json.NewEncoder(file)
 	if pretty {
-		// The legacy artifact is two orders of magnitude smaller than a
-		// seedable one, so the encoder's buffering costs little here and
-		// indenting by hand would not earn its keep.
-		encoder := json.NewEncoder(file)
 		encoder.SetIndent("", "  ")
-		if err := encoder.Encode(data); err != nil {
-			return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
-		}
-		return nil
 	}
-
-	writer := bufio.NewWriterSize(file, 1<<20)
-	if err := streamPersistedData(writer, data); err != nil {
+	if err := encoder.Encode(data); err != nil {
 		return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
 	}
-	if err := writer.Flush(); err != nil {
-		return fmt.Errorf("failed to flush hash data to %s: %w", filePath, err)
-	}
 	return nil
 }
 
-// streamPersistedData writes data as JSON incrementally.
-//
-// encoding/json marshals a whole value into one in-memory buffer before
-// writing a byte of it, and grows that buffer by doubling. A seedable
-// artifact is over a gigabyte, so that means gigabytes of allocation and
-// copying, and a peak resident size around twice the output. Emitting the
-// large maps entry by entry keeps memory flat.
-//
-// Map keys are sorted, matching encoding/json, so the artifact stays
-// reproducible.
-func streamPersistedData(writer *bufio.Writer, data *PersistedHashData) error {
-	out := &jsonWriter{w: writer}
-	out.raw("{")
-	// Fields are emitted in struct order, and the omitempty ones are
-	// skipped on the same conditions encoding/json would apply, so the
-	// legacy shape is unchanged.
-	if data.FormatVersion != 0 {
-		out.key("format_version")
-		out.raw(strconv.Itoa(data.FormatVersion))
-		out.raw(",")
-	}
-	if data.SeedCompatibilityFingerprint != "" {
-		out.key("seed_compatibility_fingerprint")
-		out.str(data.SeedCompatibilityFingerprint)
-		out.raw(",")
-	}
-	out.key("git_commit_sha")
-	out.str(data.GitCommitSha)
-	out.raw(",")
-	out.key("timestamp")
-	out.marshal(data.Timestamp)
-	out.raw(",")
-	out.key("bazel_release")
-	out.str(data.BazelRelease)
-	out.raw(",")
-	out.key("target_hashes")
-	out.hashes(data.TargetHashes)
-	if len(data.TargetEdges) > 0 {
-		out.raw(",")
-		out.key("target_edges")
-		out.edges(data.TargetEdges)
-	}
-	if len(data.DependencyHashes) > 0 {
-		out.raw(",")
-		out.key("dependency_hashes")
-		out.hashes(data.DependencyHashes)
-	}
-	out.raw(",")
-	out.key("metadata")
-	out.marshal(data.Metadata)
-	out.raw("}\n")
-	return out.err
-}
-
-// jsonWriter emits JSON to a buffered writer, holding the first error so
-// that callers need not check after every token.
-type jsonWriter struct {
-	w   *bufio.Writer
-	err error
-}
-
-func (j *jsonWriter) raw(s string) {
-	if j.err != nil {
-		return
-	}
-	_, j.err = j.w.WriteString(s)
-}
-
-func (j *jsonWriter) key(name string) {
-	j.str(name)
-	j.raw(":")
-}
-
-// str writes a JSON string. Labels and hex hashes never need escaping, so
-// the common path is a straight copy; anything else defers to the stdlib
-// rather than reimplementing its escaping rules.
-func (j *jsonWriter) str(s string) {
-	if j.err != nil {
-		return
-	}
-	if !needsJSONEscape(s) {
-		j.raw(`"`)
-		j.raw(s)
-		j.raw(`"`)
-		return
-	}
-	j.marshal(s)
-}
-
-func (j *jsonWriter) marshal(v any) {
-	if j.err != nil {
-		return
-	}
-	encoded, err := json.Marshal(v)
-	if err != nil {
-		j.err = err
-		return
-	}
-	_, j.err = j.w.Write(encoded)
-}
-
-// hashes writes a label to configuration to hash mapping. A nil map is
-// written as null, as encoding/json would, since TargetHashes has no
-// omitempty.
-func (j *jsonWriter) hashes(m map[string]map[string]string) {
-	if m == nil {
-		j.raw("null")
-		return
-	}
-	j.raw("{")
-	for i, label := range sortedKeys(m) {
-		if i > 0 {
-			j.raw(",")
-		}
-		j.key(label)
-		j.raw("{")
-		for k, configuration := range sortedKeys(m[label]) {
-			if k > 0 {
-				j.raw(",")
-			}
-			j.key(configuration)
-			j.str(m[label][configuration])
-		}
-		j.raw("}")
-	}
-	j.raw("}")
-}
-
-// edges writes a label to dependency labels mapping. Dependency lists are
-// already sorted by ExtractEdges and are left in that order.
-func (j *jsonWriter) edges(m map[string][]string) {
-	j.raw("{")
-	for i, label := range sortedKeys(m) {
-		if i > 0 {
-			j.raw(",")
-		}
-		j.key(label)
-		j.raw("[")
-		for k, dep := range m[label] {
-			if k > 0 {
-				j.raw(",")
-			}
-			j.str(dep)
-		}
-		j.raw("]")
-	}
-	j.raw("}")
-}
-
-func sortedKeys[V any](m map[string]V) []string {
-	keys := make([]string, 0, len(m))
-	for key := range m {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-	return keys
-}
-
-func needsJSONEscape(s string) bool {
-	for i := 0; i < len(s); i++ {
-		if c := s[i]; c < 0x20 || c == '"' || c == '\\' {
-			return true
-		}
-	}
-	return false
-}
-
 // LoadPersistedHashes loads persisted hash data from a JSON file
 func LoadPersistedHashes(filePath string) (*PersistedHashData, error) {
 	file, err := os.Open(filePath)
diff --git a/pkg/hash_persistence_test.go b/pkg/hash_persistence_test.go
index e0da5d6..769fa6a 100644
--- a/pkg/hash_persistence_test.go
+++ b/pkg/hash_persistence_test.go
@@ -447,102 +447,3 @@ func TestAddDependencyHashesSkipsEmptyHashSentinel(t *testing.T) {
 		t.Errorf("empty-hash sentinel should not be persisted, got %v", got)
 	}
 }
-
-// A realistic payload must stream byte-for-byte what encoding/json would
-// produce, which pins field order, omitempty behaviour and key sorting.
-func TestStreamedSeedableOutputMatchesStdlibBytes(t *testing.T) {
-	data := &PersistedHashData{
-		FormatVersion:                CurrentPersistedHashFormatVersion,
-		SeedCompatibilityFingerprint: "fingerprint",
-		GitCommitSha:                 "abc123",
-		Timestamp:                    time.Unix(1700000000, 0).UTC(),
-		BazelRelease:                 "release 9.2.0",
-		TargetHashes: map[string]map[string]string{
-			"//z:last":  {"": "bb"},
-			"//a:first": {"": "aa", "cfg": "cc"},
-		},
-		TargetEdges: map[string][]string{
-			"//a:first": {"//z:last", "@@repo+ext//p:q"},
-		},
-		DependencyHashes: map[string]map[string]string{
-			"//tools/binaries:platform": {"": "dd"},
-		},
-		Metadata: HashMetadata{TargetsPattern: "//...", TotalTargets: 3},
-	}
-
-	path := filepath.Join(t.TempDir(), "seedable.json")
-	if err := writePersistedData(path, data, false); err != nil {
-		t.Fatal(err)
-	}
-	got, err := os.ReadFile(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	want, err := json.Marshal(data)
-	if err != nil {
-		t.Fatal(err)
-	}
-	want = append(want, '\n')
-
-	if !bytes.Equal(got, want) {
-		t.Fatalf("streamed output differs from encoding/json:\n got: %s\nwant: %s", got, want)
-	}
-}
-
-func TestStreamedOutputRoundTripsAwkwardStrings(t *testing.T) {
-	// The fast path copies strings verbatim, so anything needing escaping
-	// has to fall through to the stdlib correctly.
-	awkward := "//pkg:a\"quote\\backslash\u0001ctrl\tünïcode"
-	data := &PersistedHashData{
-		FormatVersion: CurrentPersistedHashFormatVersion,
-		GitCommitSha:  "abc123",
-		Timestamp:     time.Unix(1700000000, 0).UTC(),
-		BazelRelease:  "release 9.2.0",
-		TargetHashes:  map[string]map[string]string{awkward: {awkward: awkward}},
-		TargetEdges:   map[string][]string{awkward: {awkward}},
-		Metadata:      HashMetadata{TargetsPattern: awkward, TotalTargets: 1},
-	}
-
-	path := filepath.Join(t.TempDir(), "awkward.json")
-	if err := writePersistedData(path, data, false); err != nil {
-		t.Fatal(err)
-	}
-	loaded, err := LoadPersistedHashes(path)
-	if err != nil {
-		t.Fatalf("streamed output is not parseable: %v", err)
-	}
-	if !reflect.DeepEqual(loaded.TargetHashes, data.TargetHashes) {
-		t.Errorf("target hashes did not round-trip: %v", loaded.TargetHashes)
-	}
-	if !reflect.DeepEqual(loaded.TargetEdges, data.TargetEdges) {
-		t.Errorf("target edges did not round-trip: %v", loaded.TargetEdges)
-	}
-}
-
-func TestStreamedOutputOmitsEmptyFieldsLikeStdlib(t *testing.T) {
-	// A v8-shaped artifact must not gain incremental keys.
-	data := &PersistedHashData{
-		GitCommitSha: "abc123",
-		Timestamp:    time.Unix(1700000000, 0).UTC(),
-		BazelRelease: "release 9.2.0",
-		TargetHashes: map[string]map[string]string{"//a:b": {"": "aa"}},
-		Metadata:     HashMetadata{TargetsPattern: "//...", TotalTargets: 1},
-	}
-	path := filepath.Join(t.TempDir(), "v8.json")
-	if err := writePersistedData(path, data, false); err != nil {
-		t.Fatal(err)
-	}
-	got, err := os.ReadFile(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, key := range []string{
-		"format_version", "seed_compatibility_fingerprint",
-		"target_edges", "dependency_hashes",
-	} {
-		if strings.Contains(string(got), key) {
-			t.Errorf("empty field %q should have been omitted: %s", key, got)
-		}
-	}
-}

From 80b504a5981497ff0ce1f873d42a1160d19106e2 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 19:08:01 +0200
Subject: [PATCH 12/15] chore(hash-persister): time the persist phase's
 sub-steps

The persist phase is 39s for a seedable artifact against 3s for a
compact one, and an attempt to cut it by streaming the encode changed
nothing measurable, so the cost is not where it was assumed to be.

Time each sub-step separately, and split marshalling from the file
write, so the phase can be attributed to hash collection, edge
extraction, dependency hashes, formatting or disk rather than guessed
at. The split mirrors what json.Encoder already does internally, so the
output is unchanged.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 pkg/hash_persistence.go | 40 +++++++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index e3fc7aa..a8a14b6 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"log"
 	"os"
 	"sort"
 	"time"
@@ -159,6 +160,7 @@ func PersistSeedableHashes(filePath string, gitCommitSha string, queryResults *Q
 }
 
 func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResults, context *Context, targetsPattern string, seedable bool) error {
+	stepStart := time.Now()
 	targetHashes := make(map[string]map[string]string)
 	totalTargets := 0
 
@@ -181,6 +183,8 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 		}
 	}
 
+	log.Printf("Persist step collect-hashes completed in %v (%d hashes)", time.Since(stepStart), totalTargets)
+
 	persistedData := PersistedHashData{
 		GitCommitSha: gitCommitSha,
 		Timestamp:    time.Now(),
@@ -194,11 +198,17 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 	}
 
 	if seedable {
+		stepStart = time.Now()
 		targetEdges, err := ExtractEdges(queryResults)
 		if err != nil {
 			return fmt.Errorf("failed to extract target edges: %w", err)
 		}
+		log.Printf("Persist step extract-edges completed in %v (%d edges)", time.Since(stepStart), len(targetEdges))
+
+		stepStart = time.Now()
 		persistedData.DependencyHashes = ExtractDependencyHashes(targetHashes, targetEdges, queryResults.TargetHashCache)
+		log.Printf("Persist step dependency-hashes completed in %v (%d hashes)", time.Since(stepStart), len(persistedData.DependencyHashes))
+
 		compatibilityFingerprint, err := ComputeSeedCompatibilityFingerprint(context, targetsPattern, queryResults.BazelRelease)
 		if err != nil {
 			return err
@@ -283,19 +293,35 @@ func WritePersistedData(filePath string, data *PersistedHashData) error {
 }
 
 func writePersistedData(filePath string, data *PersistedHashData, pretty bool) error {
+	// Marshalling and writing are timed apart so that a slow persist phase
+	// can be attributed to formatting or to disk rather than guessed at.
+	// This mirrors what json.Encoder does internally: marshal the whole
+	// document into one buffer, then issue a single write.
+	marshalStart := time.Now()
+	var encoded []byte
+	var err error
+	if pretty {
+		encoded, err = json.MarshalIndent(data, "", "  ")
+	} else {
+		encoded, err = json.Marshal(data)
+	}
+	if err != nil {
+		return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
+	}
+	encoded = append(encoded, '\n')
+	log.Printf("Persist step marshal completed in %v (%.1f MB)",
+		time.Since(marshalStart), float64(len(encoded))/1e6)
+
+	writeStart := time.Now()
 	file, err := os.Create(filePath)
 	if err != nil {
 		return fmt.Errorf("failed to create hash file %s: %w", filePath, err)
 	}
 	defer file.Close()
-
-	encoder := json.NewEncoder(file)
-	if pretty {
-		encoder.SetIndent("", "  ")
-	}
-	if err := encoder.Encode(data); err != nil {
-		return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
+	if _, err := file.Write(encoded); err != nil {
+		return fmt.Errorf("failed to write hash data to %s: %w", filePath, err)
 	}
+	log.Printf("Persist step write completed in %v", time.Since(writeStart))
 	return nil
 }
 

From 98243c222ce8bc2bb4fec2f1fb771eed7a19a1ae Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 19:35:45 +0200
Subject: [PATCH 13/15] perf(hash-persister): stop re-canonicalising every
 repeated dependency

Instrumenting the persist phase showed it is dominated by ExtractEdges
at 27.7s of 42.6s, not by formatting (5.2s) or the disk write (0.7s).

Rule inputs arrive as strings and leave as strings, but each of the 13.6
million occurrences was parsed into a Label and formatted back, then
inserted into a set keyed on the full ~100-byte string. Those
occurrences are only 1.21 million distinct labels, so canonicalise each
distinct string once and reuse it, and deduplicate by sorting and
compacting rather than by hashing every dependency into a set.

Shortcutting on the label's shape instead would be wrong: "//pkg:pkg"
canonicalises to "//pkg", so skipping the parse would record labels
inconsistent with how matching targets are written and make seed lookups
silently miss. Memoising still canonicalises, just once per string.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 pkg/hash_persistence.go      | 59 ++++++++++++++++++---------
 pkg/hash_persistence_test.go | 79 ++++++++++++++++++++++++++++++++++++
 2 files changed, 119 insertions(+), 19 deletions(-)

diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index a8a14b6..909252b 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"slices"
 	"sort"
 	"time"
 
@@ -105,43 +106,63 @@ type HashMetadata struct {
 // making the graph conservative. Because seeded mode is query-only in normal
 // operation, each label ordinarily has a single null configuration.
 func ExtractEdges(queryResults *QueryResults) (map[string][]string, error) {
-	edgeSets := make(map[string]map[string]struct{})
 	edges := make(map[string][]string)
 
+	// Rule inputs arrive as strings and leave as strings, but each one has
+	// to be canonicalised in between, which parses it into a Label and
+	// formats it back. There are an order of magnitude more occurrences
+	// than distinct labels, so canonicalise each distinct string once.
+	//
+	// Shortcutting on the label's shape instead would be wrong: "//pkg:pkg"
+	// canonicalises to "//pkg", and skipping that would produce labels
+	// inconsistent with how matching targets are recorded.
+	canonical := make(map[string]string)
+	canonicalise := func(input string) (string, error) {
+		if cached, ok := canonical[input]; ok {
+			return cached, nil
+		}
+		parsed, err := queryResults.TargetHashCache.ParseCanonicalLabel(input)
+		if err != nil {
+			return "", err
+		}
+		result := parsed.String()
+		canonical[input] = result
+		return result, nil
+	}
+
 	for lbl, configMap := range queryResults.TransitiveConfiguredTargets {
 		lblStr := lbl.String()
 		for _, ct := range configMap {
 			target := ct.GetTarget()
-			var dependencyLabels []gazelle_label.Label
+			var dependencies []string
 			switch target.GetType() {
 			case build.Target_RULE:
-				var err error
-				dependencyLabels, err = canonicalRuleInputLabels(queryResults.TargetHashCache, target.GetRule())
-				if err != nil {
-					return nil, fmt.Errorf("failed to extract dependencies of %s: %w", lblStr, err)
+				for _, input := range target.GetRule().RuleInput {
+					dep, err := canonicalise(input)
+					if err != nil {
+						return nil, fmt.Errorf("failed to extract dependencies of %s: %w", lblStr, err)
+					}
+					dependencies = append(dependencies, dep)
 				}
 			case build.Target_GENERATED_FILE:
 				generatingRule, err := canonicalGeneratingRuleLabel(queryResults.TargetHashCache, target.GetGeneratedFile())
 				if err != nil {
 					return nil, fmt.Errorf("failed to extract dependencies of %s: %w", lblStr, err)
 				}
-				dependencyLabels = append(dependencyLabels, generatingRule)
+				dependencies = append(dependencies, generatingRule.String())
 			}
-			if len(dependencyLabels) > 0 {
-				if edgeSets[lblStr] == nil {
-					edgeSets[lblStr] = make(map[string]struct{})
-				}
-				for _, dep := range dependencyLabels {
-					edgeSets[lblStr][dep.String()] = struct{}{}
-				}
+			if len(dependencies) > 0 {
+				edges[lblStr] = append(edges[lblStr], dependencies...)
 			}
 		}
 	}
-	for lbl, deps := range edgeSets {
-		for dep := range deps {
-			edges[lbl] = append(edges[lbl], dep)
-		}
-		sort.Strings(edges[lbl])
+
+	// Deduplicating by sorting and compacting, rather than through a set,
+	// avoids hashing every dependency string. The result is sorted either
+	// way, and a label may repeat across configurations.
+	for lbl, deps := range edges {
+		sort.Strings(deps)
+		edges[lbl] = slices.Compact(deps)
 	}
 	return edges, nil
 }
diff --git a/pkg/hash_persistence_test.go b/pkg/hash_persistence_test.go
index 769fa6a..1324782 100644
--- a/pkg/hash_persistence_test.go
+++ b/pkg/hash_persistence_test.go
@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -447,3 +448,81 @@ func TestAddDependencyHashesSkipsEmptyHashSentinel(t *testing.T) {
 		t.Errorf("empty-hash sentinel should not be persisted, got %v", got)
 	}
 }
+
+func TestExtractEdgesOmitsTargetsWithoutDependencies(t *testing.T) {
+	// A dependency-less target must not appear in the edge map at all.
+	// Appending an empty slice would create the key with a nil value,
+	// which serialises as null and inflates the artifact.
+	configuration := NormalizeConfiguration("")
+	leaf := mustParseLabel("//pkg:leaf")
+	consumer := mustParseLabel("//pkg:consumer")
+	transitive := map[gazelle_label.Label]map[Configuration]*analysis.ConfiguredTarget{
+		leaf: {configuration: {Target: &build.Target{
+			Type: build.Target_RULE.Enum(),
+			Rule: &build.Rule{Name: proto.String(leaf.String()), RuleClass: proto.String("filegroup")},
+		}}},
+		consumer: {configuration: {Target: &build.Target{
+			Type: build.Target_RULE.Enum(),
+			Rule: &build.Rule{
+				Name: proto.String(consumer.String()), RuleClass: proto.String("java_library"),
+				// The same input twice: deduplication must survive the
+				// switch from a set to sort-and-compact.
+				RuleInput: []string{leaf.String(), leaf.String()},
+			},
+		}}},
+	}
+	queryResults := &QueryResults{
+		TransitiveConfiguredTargets: transitive,
+		TargetHashCache:             NewTargetHashCache(transitive, &Normalizer{}, "release 8.0.0", true, nil),
+	}
+
+	edges, err := ExtractEdges(queryResults)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := edges[leaf.String()]; ok {
+		t.Errorf("dependency-less %s should be absent, got %v", leaf, edges[leaf.String()])
+	}
+	if got := edges[consumer.String()]; len(got) != 1 || got[0] != leaf.String() {
+		t.Errorf("consumer edges = %v, want exactly one %s", got, leaf)
+	}
+}
+
+func TestExtractEdgesCanonicalisesRepeatedLabelsConsistently(t *testing.T) {
+	// The canonicalisation memo must return what parsing would, including
+	// for "//pkg:pkg", which canonicalises to "//pkg". Shortcutting on the
+	// label's shape would break this.
+	configuration := NormalizeConfiguration("")
+	consumer := mustParseLabel("//app:consumer")
+	transitive := map[gazelle_label.Label]map[Configuration]*analysis.ConfiguredTarget{
+		consumer: {configuration: {Target: &build.Target{
+			Type: build.Target_RULE.Enum(),
+			Rule: &build.Rule{
+				Name: proto.String(consumer.String()), RuleClass: proto.String("java_library"),
+				RuleInput: []string{"//pkg:pkg", "@//other:thing", "//pkg:pkg"},
+			},
+		}}},
+	}
+	queryResults := &QueryResults{
+		TransitiveConfiguredTargets: transitive,
+		TargetHashCache:             NewTargetHashCache(transitive, &Normalizer{}, "release 8.0.0", true, nil),
+	}
+
+	edges, err := ExtractEdges(queryResults)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	normalizer := &Normalizer{}
+	for _, input := range []string{"//pkg:pkg", "@//other:thing"} {
+		parsed, err := normalizer.ParseCanonicalLabel(input)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !slices.Contains(edges[consumer.String()], parsed.String()) {
+			t.Errorf("%q should appear canonicalised as %q, got %v",
+				input, parsed.String(), edges[consumer.String()])
+		}
+	}
+}

From 22131ba27ff092093a550610d64fe3cf1b820102 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 19:54:27 +0200
Subject: [PATCH 14/15] Revert "chore(hash-persister): time the persist phase's
 sub-steps"

This reverts commit 80b504a5981497ff0ce1f873d42a1160d19106e2.
---
 pkg/hash_persistence.go | 40 +++++++---------------------------------
 1 file changed, 7 insertions(+), 33 deletions(-)

diff --git a/pkg/hash_persistence.go b/pkg/hash_persistence.go
index 909252b..db4e72c 100644
--- a/pkg/hash_persistence.go
+++ b/pkg/hash_persistence.go
@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"log"
 	"os"
 	"slices"
 	"sort"
@@ -181,7 +180,6 @@ func PersistSeedableHashes(filePath string, gitCommitSha string, queryResults *Q
 }
 
 func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResults, context *Context, targetsPattern string, seedable bool) error {
-	stepStart := time.Now()
 	targetHashes := make(map[string]map[string]string)
 	totalTargets := 0
 
@@ -204,8 +202,6 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 		}
 	}
 
-	log.Printf("Persist step collect-hashes completed in %v (%d hashes)", time.Since(stepStart), totalTargets)
-
 	persistedData := PersistedHashData{
 		GitCommitSha: gitCommitSha,
 		Timestamp:    time.Now(),
@@ -219,17 +215,11 @@ func persistHashes(filePath string, gitCommitSha string, queryResults *QueryResu
 	}
 
 	if seedable {
-		stepStart = time.Now()
 		targetEdges, err := ExtractEdges(queryResults)
 		if err != nil {
 			return fmt.Errorf("failed to extract target edges: %w", err)
 		}
-		log.Printf("Persist step extract-edges completed in %v (%d edges)", time.Since(stepStart), len(targetEdges))
-
-		stepStart = time.Now()
 		persistedData.DependencyHashes = ExtractDependencyHashes(targetHashes, targetEdges, queryResults.TargetHashCache)
-		log.Printf("Persist step dependency-hashes completed in %v (%d hashes)", time.Since(stepStart), len(persistedData.DependencyHashes))
-
 		compatibilityFingerprint, err := ComputeSeedCompatibilityFingerprint(context, targetsPattern, queryResults.BazelRelease)
 		if err != nil {
 			return err
@@ -314,35 +304,19 @@ func WritePersistedData(filePath string, data *PersistedHashData) error {
 }
 
 func writePersistedData(filePath string, data *PersistedHashData, pretty bool) error {
-	// Marshalling and writing are timed apart so that a slow persist phase
-	// can be attributed to formatting or to disk rather than guessed at.
-	// This mirrors what json.Encoder does internally: marshal the whole
-	// document into one buffer, then issue a single write.
-	marshalStart := time.Now()
-	var encoded []byte
-	var err error
-	if pretty {
-		encoded, err = json.MarshalIndent(data, "", "  ")
-	} else {
-		encoded, err = json.Marshal(data)
-	}
-	if err != nil {
-		return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
-	}
-	encoded = append(encoded, '\n')
-	log.Printf("Persist step marshal completed in %v (%.1f MB)",
-		time.Since(marshalStart), float64(len(encoded))/1e6)
-
-	writeStart := time.Now()
 	file, err := os.Create(filePath)
 	if err != nil {
 		return fmt.Errorf("failed to create hash file %s: %w", filePath, err)
 	}
 	defer file.Close()
-	if _, err := file.Write(encoded); err != nil {
-		return fmt.Errorf("failed to write hash data to %s: %w", filePath, err)
+
+	encoder := json.NewEncoder(file)
+	if pretty {
+		encoder.SetIndent("", "  ")
+	}
+	if err := encoder.Encode(data); err != nil {
+		return fmt.Errorf("failed to encode hash data to %s: %w", filePath, err)
 	}
-	log.Printf("Persist step write completed in %v", time.Since(writeStart))
 	return nil
 }
 

From 527ab4250f538dfc5bda3adb49055064e20fc907 Mon Sep 17 00:00:00 2001
From: Hongxin Liang <honnix@users.noreply.github.com>
Date: Wed, 23 Sep 2026 20:02:39 +0200
Subject: [PATCH 15/15] fix(hash-persister): do not lose the dirty set when the
 probe fails

probePruneDirtySet returns a nil result alongside its error, so
assigning it straight to dirtyResult set it to nil on the error path.
The code then logged that it was continuing with the unpruned set and
dereferenced nil on the next line, turning a recoverable probe failure
into a panic. Hold the result in a temporary and only adopt it on
success.

Also update the execution report's dirty target count after a successful
prune. It drives CI metrics, and reporting the pre-probe figure would
have shown 280k dirty targets for a run that actually used 13k.

Both found in review by mattbrown.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 hash-persister/hash-persister.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/hash-persister/hash-persister.go b/hash-persister/hash-persister.go
index 45d54a0..b51d3c6 100644
--- a/hash-persister/hash-persister.go
+++ b/hash-persister/hash-persister.go
@@ -299,10 +299,17 @@ func runSeeded(cfg *config) (seededOutcome, error) {
 	// the seed and only propagate rdeps from targets that actually changed.
 	unprunedDirtyStarCount := len(dirtyResult.DirtyStarLabels)
 	if unprunedDirtyStarCount > len(dirtyResult.DirtyLabels) {
-		dirtyResult, err = probePruneDirtySet(cfg, commitRev, dirtyResult, seedData, changedFiles)
-		if err != nil {
-			log.Printf("Probe pruning failed, continuing with unpruned dirty set: %v", err)
+		// Kept in a temporary: probePruneDirtySet returns a nil result
+		// alongside its error, so assigning straight to dirtyResult would
+		// discard the set the fallback is supposed to carry on with.
+		pruned, probeErr := probePruneDirtySet(cfg, commitRev, dirtyResult, seedData, changedFiles)
+		if probeErr != nil {
+			log.Printf("Probe pruning failed, continuing with unpruned dirty set: %v", probeErr)
 		} else {
+			dirtyResult = pruned
+			// The report drives CI metrics, so it has to describe the set
+			// actually used rather than the one before pruning.
+			outcome.DirtyTargetCount = len(dirtyResult.DirtyStarLabels)
 			log.Printf("Probe pruning: %d dirty* -> %d dirty* (%d eliminated)",
 				unprunedDirtyStarCount, len(dirtyResult.DirtyStarLabels),
 				unprunedDirtyStarCount-len(dirtyResult.DirtyStarLabels))