⚡ Bolt: lock-free map-reduce regional listing

.jules/bolt.md

@@ -1,5 +1,9 @@
 # Bolt's Journal
 
+## 2026-03-09 - Lock-Free Map-Reduce Slice Pattern for Regional Listings
+**Learning:** Performing multi-regional operations (such as listing services or domain mappings across all Cloud Run locations) concurrently can lead to mutex lock contention and excessive heap allocation/copy overhead when multiple goroutines write to a shared slice protected by `sync.Mutex`. Utilizing a pre-allocated slice-of-slices pattern indexed by region eliminates the lock entirely. Furthermore, calculating the precise required final slice capacity from the sub-slices and pre-allocating the aggregated slice before flattening avoids intermediate slice growth/copying altogether, improving concurrency and reducing memory allocations.
+**Action:** When querying multiple data sources concurrently, use a lock-free slice-of-slices pattern and pre-calculate exact capacity before merging into a single pre-allocated flat slice.
+
 ## 2026-03-08 - GCP Logging Client Caching & Connection Longevity
 **Learning:** Establishing the GCP Stackdriver Logging client requires repeated Google credential discovery and connection establishment, causing high latency (~300ms) inside a reactive TUI interface. Caching `logadmin.Client` instances via a project-aware map with thread-safe `sync.Mutex` ensures subsequent streaming and log extraction operations are instantaneous. Crucially, calling `Close()` on individual stream terminations must be a no-op to prevent premature teardown of connection pools shared across other active streaming views.
 **Action:** Keep GCP Logging clients cached globally by project and handle connection termination via a no-op `Close` method, while adding test-isolation resets in unit tests.
@@ -12,10 +16,6 @@
 **Learning:** Failing to cache the GCP Revisions client meant that navigating and reloading service revisions in the TUI invoked credential discovery and gRPC setup on every revision fetch. Standardizing the stateful `sync.Mutex` lazy-initialization pattern ensures that once a client is cached, subsequent service revision lists are lightning fast.
 **Action:** Consistently inspect all subcommand and subpackage API wrappers to ensure that client connections are pooled and not destroyed on request boundaries.
 
-## 2026-03-05 - GCP Services Client Caching
-**Learning:** Initializing Google Cloud Platform API clients and discovering credentials (ADC) on every single request in a terminal user interface (TUI) introduces a major latency bottleneck (~300ms per request) due to repetitive file system lookups and TCP/gRPC handshakes. Caching the client wrapper using a thread-safe pattern (`sync.Mutex`) avoids this completely. Importantly, client construction must use `context.Background()` rather than request-scoped contexts to prevent cancellation of the shared client connection pool when a single request context is canceled.
-**Action:** Always verify if external API or cloud clients are lazily initialized and cached as singletons/long-lived clients across successive TUI interactions, rather than being created and closed on every API call.
-
 ## 2025-05-15 - [GCP Client Creation Overhead]
 **Learning:** Creating a new GCP client for every API call (especially in a TUI that frequently refreshes and can list all regions) introduces significant latency due to repeated credential discovery, TLS handshakes, and gRPC connection establishment. When listing "All" regions, this results in 24 simultaneous client creations and connection setups.
 **Action:** Reuse a single, thread-safe GCP client per package (service, job, etc.) to leverage connection pooling and reduce overhead.

internal/run/api/domainmapping/domainmapping.go

@@ -50,25 +50,34 @@ func List(project, region string) ([]model.DomainMapping, error) {
 }
 
 func listAllRegions(project string) ([]model.DomainMapping, error) {
-	var (
-		mu             sync.Mutex
-		domainMappings []model.DomainMapping
-		wg             sync.WaitGroup
-	)
+	regions := api_region.List()
+	results := make([][]model.DomainMapping, len(regions))
+	var wg sync.WaitGroup
 
-	for _, region := range api_region.List() {
+	for i, region := range regions {
 		wg.Add(1)
-		go func(r string) {
+		go func(idx int, r string) {
 			defer wg.Done()
 			if dms, err := List(project, r); err == nil {
-				mu.Lock()
-				domainMappings = append(domainMappings, dms...)
-				mu.Unlock()
+				results[idx] = dms
 			}
-		}(region)
+		}(i, region)
 	}
 
 	wg.Wait()
+
+	// Pre-calculate exact capacity needed to eliminate dynamic slice growth
+	var totalSize int
+	for _, dms := range results {
+		totalSize += len(dms)
+	}
+
+	// Lock-free flat pre-allocated slice
+	domainMappings := make([]model.DomainMapping, 0, totalSize)
+	for _, dms := range results {
+		domainMappings = append(domainMappings, dms...)
+	}
+
 	return domainMappings, nil
 }
 

internal/run/api/job/job.go

@@ -79,27 +79,36 @@ func mapJob(resp *runpb.Job, region string) model.Job {
 }
 
 func listAllRegions(project string) ([]model.Job, error) {
-	var (
-		mu   sync.Mutex
-		jobs []model.Job
-		wg   sync.WaitGroup
-	)
+	regions := api_region.List()
+	results := make([][]model.Job, len(regions))
+	var wg sync.WaitGroup
 
-	for _, region := range api_region.List() {
+	for i, region := range regions {
 		wg.Add(1)
-		go func(r string) {
+		go func(idx int, r string) {
 			defer wg.Done()
 			// Call List recursively for each region
 			// We ignore errors here to allow partial success (e.g. if one region is down or disabled)
 			if j, err := List(project, r); err == nil {
-				mu.Lock()
-				jobs = append(jobs, j...)
-				mu.Unlock()
+				results[idx] = j
 			}
-		}(region)
+		}(i, region)
 	}
 
 	wg.Wait()
+
+	// Pre-calculate exact capacity needed to eliminate dynamic slice growth
+	var totalSize int
+	for _, j := range results {
+		totalSize += len(j)
+	}
+
+	// Lock-free flat pre-allocated slice
+	jobs := make([]model.Job, 0, totalSize)
+	for _, j := range results {
+		jobs = append(jobs, j...)
+	}
+
 	return jobs, nil
 }
 

internal/run/api/service/service.go

@@ -299,26 +299,35 @@ func UpdateTraffic(ctx context.Context, project, region, serviceName string, tar
 }
 
 func listAllRegions(project string) ([]model.Service, error) {
-	var (
-		mu       sync.Mutex
-		services []model.Service
-		wg       sync.WaitGroup
-	)
+	regions := api_region.List()
+	results := make([][]model.Service, len(regions))
+	var wg sync.WaitGroup
 
-	for _, region := range api_region.List() {
+	for i, region := range regions {
 		wg.Add(1)
-		go func(r string) {
+		go func(idx int, r string) {
 			defer wg.Done()
 			// Call List recursively for each region
 			// We ignore errors here to allow partial success (e.g. if one region is down or disabled)
 			if s, err := List(project, r); err == nil {
-				mu.Lock()
-				services = append(services, s...)
-				mu.Unlock()
+				results[idx] = s
 			}
-		}(region)
+		}(i, region)
 	}
 
 	wg.Wait()
+
+	// Pre-calculate exact capacity needed to eliminate dynamic slice growth
+	var totalSize int
+	for _, s := range results {
+		totalSize += len(s)
+	}
+
+	// Lock-free flat pre-allocated slice
+	services := make([]model.Service, 0, totalSize)
+	for _, s := range results {
+		services = append(services, s...)
+	}
+
 	return services, nil
 }

internal/run/api/workerpool/workerpool.go

@@ -111,27 +111,36 @@ func UpdateScaling(ctx context.Context, project, region, workerPoolName string,
 }
 
 func listAllRegions(project string) ([]model.WorkerPool, error) {
-	var (
-		mu          sync.Mutex
-		workerPools []model.WorkerPool
-		wg          sync.WaitGroup
-	)
+	regions := api_region.List()
+	results := make([][]model.WorkerPool, len(regions))
+	var wg sync.WaitGroup
 
-	for _, region := range api_region.List() {
+	for i, region := range regions {
 		wg.Add(1)
-		go func(r string) {
+		go func(idx int, r string) {
 			defer wg.Done()
 			// Call List recursively for each region
 			// We ignore errors here to allow partial success (e.g. if one region is down or disabled)
 			if wp, err := List(project, r); err == nil {
-				mu.Lock()
-				workerPools = append(workerPools, wp...)
-				mu.Unlock()
+				results[idx] = wp
 			}
-		}(region)
+		}(i, region)
 	}
 
 	wg.Wait()
+
+	// Pre-calculate exact capacity needed to eliminate dynamic slice growth
+	var totalSize int
+	for _, wp := range results {
+		totalSize += len(wp)
+	}
+
+	// Lock-free flat pre-allocated slice
+	workerPools := make([]model.WorkerPool, 0, totalSize)
+	for _, wp := range results {
+		workerPools = append(workerPools, wp...)
+	}
+
 	return workerPools, nil
 }