diff --git a/apps/teleop-native/Package.swift b/apps/teleop-native/Package.swift new file mode 100644 index 00000000..2d7151e2 --- /dev/null +++ b/apps/teleop-native/Package.swift @@ -0,0 +1,22 @@ +// swift-tools-version: 6.0 + +import PackageDescription + +let package = Package( + name: "TeleopNative", + platforms: [ + .iOS(.v17), + .macOS(.v14), + ], + products: [ + .library(name: "TeleopUI", targets: ["TeleopUI"]), + .executable(name: "TeleopMac", targets: ["TeleopMac"]), + ], + targets: [ + .target(name: "TeleopUI"), + .executableTarget( + name: "TeleopMac", + dependencies: ["TeleopUI"] + ), + ] +) diff --git a/apps/teleop-native/README.md b/apps/teleop-native/README.md new file mode 100644 index 00000000..2c675712 --- /dev/null +++ b/apps/teleop-native/README.md @@ -0,0 +1,29 @@ +# Teleop Native + +Native SwiftUI prototype for low-latency voice teleoperation on macOS and iOS. + +## OpenUI Result Summary + +- First screen is the control surface, not a landing page. +- One primary animated voice button drives the interaction. +- Visible text is limited to a tiny transient agent cue. +- Telemetry is icon-first: signal, power, and motion are visual gauges. +- Voice/chat owns intent. The button starts, interrupts, resumes, or reconnects. +- WebRTC/media plumbing is intentionally a future integration layer behind the current `TeleopSession` state object. + +## Architecture Target + +`Mobile or macOS app -> WebRTC session -> global relay -> transceiver -> realtime agent -> teleop gateway -> device` + +The app should keep hard safety controls local to the device gateway. The model can interpret intent, but stop limits and degraded-network behavior should not depend on a cloud round trip. + +## Run Mac Prototype + +```bash +cd apps/teleop-native +swift run TeleopMac +``` + +## iOS + +Open `apps/teleop-native/Package.swift` in Xcode and use `TeleopUI` from a new iOS app target. The shared SwiftUI view is `TeleopHomeView`. diff --git a/apps/teleop-native/Sources/TeleopMac/TeleopMacApp.swift b/apps/teleop-native/Sources/TeleopMac/TeleopMacApp.swift new file mode 100644 index 00000000..4014f43c --- /dev/null +++ b/apps/teleop-native/Sources/TeleopMac/TeleopMacApp.swift @@ -0,0 +1,13 @@ +import SwiftUI +import TeleopUI + +@main +struct TeleopMacApp: App { + var body: some Scene { + WindowGroup { + TeleopHomeView() + .frame(minWidth: 360, minHeight: 620) + } + .windowStyle(.hiddenTitleBar) + } +} diff --git a/apps/teleop-native/Sources/TeleopUI/TeleopHomeView.swift b/apps/teleop-native/Sources/TeleopUI/TeleopHomeView.swift new file mode 100644 index 00000000..a246b50f --- /dev/null +++ b/apps/teleop-native/Sources/TeleopUI/TeleopHomeView.swift @@ -0,0 +1,135 @@ +import SwiftUI + +public struct TeleopHomeView: View { + @StateObject private var session = TeleopSession() + + public init() {} + + public var body: some View { + ZStack { + TeleopBackground(state: session.state) + + VStack(spacing: 26) { + TelemetryStrip(sample: session.telemetry, state: session.state) + .padding(.top, 26) + + Spacer(minLength: 20) + + TeleopPulseButton(state: session.state) { + session.pressPrimary() + } + + if !session.agentSummary.isEmpty { + AgentCue(text: session.agentSummary) + .transition(.opacity.combined(with: .scale(scale: 0.96))) + } + + Spacer(minLength: 32) + } + .padding(.horizontal, 24) + .padding(.bottom, 20) + } + .accessibilityElement(children: .contain) + .onDisappear { + session.stop() + } + } +} + +private struct TeleopBackground: View { + let state: TeleopConnectionState + + var body: some View { + LinearGradient( + colors: backgroundColors, + startPoint: .topLeading, + endPoint: .bottomTrailing + ) + .ignoresSafeArea() + .animation(.easeInOut(duration: 0.35), value: state) + } + + private var backgroundColors: [Color] { + switch state { + case .idle: + return [Color(red: 0.05, green: 0.06, blue: 0.07), Color(red: 0.0, green: 0.09, blue: 0.08)] + case .connecting: + return [Color(red: 0.03, green: 0.07, blue: 0.10), Color(red: 0.0, green: 0.18, blue: 0.16)] + case .live: + return [Color(red: 0.02, green: 0.11, blue: 0.08), Color(red: 0.10, green: 0.16, blue: 0.07)] + case .thinking: + return [Color(red: 0.10, green: 0.09, blue: 0.02), Color(red: 0.16, green: 0.11, blue: 0.02)] + case .fault: + return [Color(red: 0.12, green: 0.03, blue: 0.04), Color(red: 0.04, green: 0.02, blue: 0.03)] + } + } +} + +private struct TelemetryStrip: View { + let sample: TelemetrySample + let state: TeleopConnectionState + + var body: some View { + HStack(spacing: 18) { + GaugeGlyph( + systemName: "point.3.connected.trianglepath.dotted", + value: signalValue, + tint: .mint + ) + GaugeGlyph(systemName: "bolt.fill", value: sample.battery, tint: .yellow) + GaugeGlyph(systemName: "waveform.path.ecg", value: motionValue, tint: .cyan) + } + .opacity(state == .idle ? 0.38 : 1) + .animation(.easeInOut(duration: 0.25), value: state) + .accessibilityLabel("Telemetry") + } + + private var signalValue: Double { + state == .idle ? 0.12 : sample.signal + } + + private var motionValue: Double { + state == .idle ? 0.08 : (sample.motion + 1.0) / 2.0 + } +} + +private struct GaugeGlyph: View { + let systemName: String + let value: Double + let tint: Color + + var body: some View { + ZStack { + Circle() + .stroke(.white.opacity(0.12), lineWidth: 4) + Circle() + .trim(from: 0, to: max(0.05, min(1.0, value))) + .stroke(tint, style: StrokeStyle(lineWidth: 4, lineCap: .round)) + .rotationEffect(.degrees(-90)) + Image(systemName: systemName) + .font(.system(size: 17, weight: .semibold)) + .foregroundStyle(.white.opacity(0.86)) + } + .frame(width: 48, height: 48) + .animation(.smooth(duration: 0.18), value: value) + } +} + +private struct AgentCue: View { + let text: String + + var body: some View { + Text(text) + .font(.system(size: 13, weight: .medium, design: .rounded)) + .foregroundStyle(.white.opacity(0.82)) + .padding(.horizontal, 14) + .padding(.vertical, 8) + .background(.white.opacity(0.10), in: Capsule()) + .accessibilityLabel(text) + } +} + +#Preview { + TeleopHomeView() + .frame(width: 390, height: 740) +} diff --git a/apps/teleop-native/Sources/TeleopUI/TeleopPulseButton.swift b/apps/teleop-native/Sources/TeleopUI/TeleopPulseButton.swift new file mode 100644 index 00000000..9d7a2d8a --- /dev/null +++ b/apps/teleop-native/Sources/TeleopUI/TeleopPulseButton.swift @@ -0,0 +1,104 @@ +import SwiftUI + +struct TeleopPulseButton: View { + let state: TeleopConnectionState + let action: () -> Void + + @State private var pulse = false + + var body: some View { + Button(action: action) { + ZStack { + ForEach(0..<3, id: \.self) { index in + Circle() + .stroke(ringColor.opacity(0.30), lineWidth: 2) + .frame(width: 170 + CGFloat(index * 42), height: 170 + CGFloat(index * 42)) + .scaleEffect(pulse ? 1.12 : 0.86) + .opacity(pulse ? 0.0 : 0.55) + .animation( + .easeOut(duration: 1.45) + .repeatForever(autoreverses: false) + .delay(Double(index) * 0.18), + value: pulse + ) + } + + Circle() + .fill(buttonFill) + .frame(width: 168, height: 168) + .shadow(color: ringColor.opacity(0.34), radius: 30, y: 16) + + Image(systemName: iconName) + .font(.system(size: 58, weight: .semibold)) + .foregroundStyle(.white) + .symbolEffect(.pulse, options: .repeating, value: isAnimated) + } + .frame(width: 280, height: 280) + } + .buttonStyle(.plain) + .accessibilityLabel(accessibilityLabel) + .onAppear { + pulse = true + } + .onChange(of: state) { _, newState in + pulse = newState != .idle + } + } + + private var iconName: String { + switch state { + case .idle: + return "mic.fill" + case .connecting: + return "antenna.radiowaves.left.and.right" + case .live: + return "waveform" + case .thinking: + return "hand.raised.fill" + case .fault: + return "exclamationmark.triangle.fill" + } + } + + private var accessibilityLabel: String { + switch state { + case .idle: + return "Start voice teleoperation" + case .connecting: + return "Cancel connection" + case .live: + return "Interrupt or stop motion" + case .thinking: + return "Resume live control" + case .fault: + return "Reconnect" + } + } + + private var buttonFill: LinearGradient { + LinearGradient( + colors: [ringColor.opacity(0.95), ringColor.opacity(0.48)], + startPoint: .topLeading, + endPoint: .bottomTrailing + ) + } + + private var ringColor: Color { + switch state { + case .idle: + return .teal + case .connecting: + return .cyan + case .live: + return .green + case .thinking: + return .orange + case .fault: + return .red + } + } + + private var isAnimated: Bool { + state == .connecting || state == .live || state == .thinking + } +} diff --git a/apps/teleop-native/Sources/TeleopUI/TeleopSession.swift b/apps/teleop-native/Sources/TeleopUI/TeleopSession.swift new file mode 100644 index 00000000..7627f727 --- /dev/null +++ b/apps/teleop-native/Sources/TeleopUI/TeleopSession.swift @@ -0,0 +1,107 @@ +import Foundation +import SwiftUI + +public enum TeleopConnectionState: Equatable, Sendable { + case idle + case connecting + case live + case thinking + case fault +} + +public struct TelemetrySample: Equatable, Sendable { + public var latencyMS: Int + public var signal: Double + public var battery: Double + public var motion: Double + + public static let empty = TelemetrySample( + latencyMS: 0, + signal: 0, + battery: 1, + motion: 0 + ) +} + +@MainActor +public final class TeleopSession: ObservableObject { + @Published public private(set) var state: TeleopConnectionState = .idle + @Published public private(set) var telemetry: TelemetrySample = .empty + @Published public private(set) var agentSummary: String = "" + + private var telemetryTask: Task? + + public init() {} + + public func pressPrimary() { + switch state { + case .idle, .fault: + start() + case .connecting: + stop() + case .live: + interrupt() + case .thinking: + state = .live + agentSummary = "" + } + } + + public func stop() { + telemetryTask?.cancel() + telemetryTask = nil + telemetry = .empty + agentSummary = "" + state = .idle + } + + private func start() { + telemetryTask?.cancel() + telemetry = .empty + agentSummary = "" + state = .connecting + + telemetryTask = Task { [weak self] in + try? await Task.sleep(for: .milliseconds(320)) + guard !Task.isCancelled else { return } + self?.becomeLive() + } + } + + private func becomeLive() { + state = .live + telemetryTask = Task { [weak self] in + var tick = 0 + while !Task.isCancelled { + let latency = 22 + Int.random(in: 0...18) + let signal = Double.random(in: 0.72...0.98) + let battery = Double.random(in: 0.68...0.96) + let motion = sin(Double(tick) / 5.0) + await MainActor.run { + self?.telemetry = TelemetrySample( + latencyMS: latency, + signal: signal, + battery: battery, + motion: motion + ) + } + tick += 1 + try? await Task.sleep(for: .milliseconds(180)) + } + } + } + + private func interrupt() { + state = .thinking + agentSummary = "holding" + + Task { [weak self] in + try? await Task.sleep(for: .milliseconds(420)) + guard !Task.isCancelled else { return } + await MainActor.run { + self?.state = .live + self?.agentSummary = "" + } + } + } +} diff --git a/bin/hermes-sm b/bin/hermes-sm new file mode 100755 index 00000000..a0282195 --- /dev/null +++ b/bin/hermes-sm @@ -0,0 +1,6 @@ +#!/usr/bin/env node +/** + * Hermes-SM CLI Launcher (ESM) + * Delegates to built CLI in dist without requiring tsx. + */ +import('../dist/src/cli/hermes-sm.js'); diff --git a/bin/hermes-smd b/bin/hermes-smd new file mode 100755 index 00000000..b0455bd9 --- /dev/null +++ b/bin/hermes-smd @@ -0,0 +1,6 @@ +#!/usr/bin/env node +/** + * Hermes-SMD CLI Launcher (ESM) — danger mode + * Delegates to built CLI in dist without requiring tsx. + */ +import('../dist/src/cli/hermes-sm.js'); diff --git a/docs/benchmark-report.md b/docs/benchmark-report.md new file mode 100644 index 00000000..9e47b8a6 --- /dev/null +++ b/docs/benchmark-report.md @@ -0,0 +1,60 @@ +# StackMemory Hook Benchmark Report + +> Generated: 2026-05-17T00:40:52.796Z +> Data: 7589 tool calls across 181 sessions + +## Baseline (before hooks) + +| Metric | Value | % of total | +|--------|------:|----------:| +| Total tool calls | 7589 | 100% | +| Read calls | 1462 | 19.3% | +| Duplicate reads | 918 | 12.1% | +| Bash calls | 3352 | 44.2% | +| Bash → should be Glob | 422 | 5.6% | +| Bash → should be Read | 122 | 1.6% | +| Bash → should be Grep | 130 | 1.7% | +| Bash (git) | 468 | 6.2% | +| Bash (legit) | 2210 | 29.1% | +| ToolSearch calls | 108 | 1.4% | + +## Hook Effectiveness (projected) + +### 1. Dedup Reads (escalation at 3x soft / 5x STOP) +- Would warn (3-4x): **249** calls +- Would STOP (5x+): **420** calls +- Combined catch: **669** / 1462 reads = **45.8%** +- Token savings estimate: ~84K tokens (STOP prevents re-read) + +### 2. Auto-Route (Bash → dedicated tools) +- Replaceable calls caught: **674** / 3352 Bash calls = **20.1%** +- Breakdown: 422 ls/find → Glob, 122 cat/head → Read, 130 grep → Grep +- Token savings estimate: ~34K tokens (reduced overhead per call) + +### 3. Prewarm (pre-fetch deferred tool schemas) +- ToolSearch calls observed: **108** +- Unique deferred tools: **42** +- Top 8 tools cover: ~8 tools +- Estimated catches: **~108** avoided ToolSearch calls +- Token savings estimate: ~16K tokens + +### 4. Script-Suggest (pattern → script) +- Git sequences (3+ cmds): **41** → git-ops.ts +- gh run calls: **1** → build-status.ts +- WebFetch calls: **120** → web-fetch.ts +- WebSearch calls: **75** → web-search.ts +- Total suggestions would fire: **237** +- Token savings estimate: ~190K tokens (each script replaces ~4 calls) + +## Summary + +| Hook | Catches | Est. token savings | +|------|--------:|------------------:| +| Dedup STOP | 420 reads | ~84K | +| Auto-route | 674 Bash calls | ~34K | +| Prewarm | ~108 ToolSearch | ~16K | +| Script-suggest | 237 patterns | ~190K | +| **Total** | | **~324K** | + +Baseline total estimated tokens: ~1518K +Projected waste reduction: **21.3%** diff --git a/package-lock.json b/package-lock.json index 68a465c8..2992f1f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@stackmemoryai/stackmemory", - "version": "1.12.0", + "version": "1.13.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@stackmemoryai/stackmemory", - "version": "1.12.0", + "version": "1.13.0", "hasInstallScript": true, "license": "BUSL-1.1", "dependencies": { @@ -34,6 +34,7 @@ "helmet": "^8.1.0", "ignore": "^7.0.5", "inquirer": "^9.3.8", + "js-tiktoken": "^1.0.21", "msgpackr": "^1.10.1", "node-pty": "^1.1.0", "open": "^11.0.0", @@ -55,6 +56,8 @@ "codex-sm": "dist/src/cli/codex-sm.js", "codex-smd": "bin/codex-smd", "gemini-sm": "bin/gemini-sm", + "hermes-sm": "bin/hermes-sm", + "hermes-smd": "bin/hermes-smd", "opencode-sm": "bin/opencode-sm", "stackmemory": "dist/src/cli/index.js" }, @@ -8673,6 +8676,15 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", diff --git a/package.json b/package.json index 0fc63217..2b436893 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@stackmemoryai/stackmemory", - "version": "1.12.0", - "description": "Lossless, project-scoped memory for AI coding tools. Durable context across sessions with 56 MCP tools, FTS5 search, conductor orchestrator, loop/watch monitoring, snapshot capture, pre-flight overlap checks, Claude/Codex/OpenCode wrappers, Linear sync, and automatic hooks.", + "version": "1.13.0", + "description": "Lossless, project-scoped memory for AI coding tools. Durable context across sessions with 60+ MCP tools, FTS5 search, cloud sync, conductor orchestration, loop/watch monitoring, trace optimization, Claude/Codex/OpenCode/Gemini wrappers, skill packs, and automatic hooks.", "engines": { "node": ">=20.0.0", "npm": ">=10.0.0" @@ -15,7 +15,9 @@ "claude-sm": "bin/claude-sm", "claude-smd": "bin/claude-smd", "opencode-sm": "bin/opencode-sm", - "gemini-sm": "bin/gemini-sm" + "gemini-sm": "bin/gemini-sm", + "hermes-sm": "bin/hermes-sm", + "hermes-smd": "bin/hermes-smd" }, "files": [ "bin", @@ -46,6 +48,7 @@ "scripts/smoke-init-db.sh", "templates", "packs", + "docs/guides/README_INSTALL.md", "README.md", "LICENSE" ], @@ -176,6 +179,7 @@ "helmet": "^8.1.0", "ignore": "^7.0.5", "inquirer": "^9.3.8", + "js-tiktoken": "^1.0.21", "msgpackr": "^1.10.1", "node-pty": "^1.1.0", "open": "^11.0.0", diff --git a/packages/sdk/package-lock.json b/packages/sdk/package-lock.json index 62be3204..abedde8a 100644 --- a/packages/sdk/package-lock.json +++ b/packages/sdk/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "better-sqlite3": "^11.8.1", + "js-tiktoken": "^1.0.21", "js-yaml": "^4.1.0", "zod": "^3.24.2" }, @@ -19,6 +20,9 @@ "@types/node": "^22.13.10", "typescript": "^5.8.2", "vitest": "^3.0.9" + }, + "engines": { + "node": ">=20.0.0" } }, "node_modules/@esbuild/aix-ppc64": { @@ -1352,6 +1356,15 @@ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "license": "ISC" }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 4500629b..771327e4 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "better-sqlite3": "^11.8.1", + "js-tiktoken": "^1.0.21", "js-yaml": "^4.1.0", "zod": "^3.24.2" }, diff --git a/packages/sdk/src/__tests__/sdk.test.ts b/packages/sdk/src/__tests__/sdk.test.ts index 7b0d9ca0..ebd558ce 100644 --- a/packages/sdk/src/__tests__/sdk.test.ts +++ b/packages/sdk/src/__tests__/sdk.test.ts @@ -273,9 +273,13 @@ describe('StackMemory SDK', () => { }); describe('pure functions', () => { - it('estimateTokens approximates', () => { - expect(estimateTokens('hello')).toBe(2); + it('estimateTokens returns positive count for non-empty strings', () => { + expect(estimateTokens('hello')).toBeGreaterThan(0); expect(estimateTokens('')).toBe(0); + // Longer text should produce more tokens + expect(estimateTokens('hello world foo bar baz')).toBeGreaterThan( + estimateTokens('hello') + ); }); it('hashContent is deterministic', () => { diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 7bb90227..6bb61014 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -15,7 +15,11 @@ export { ProvenanceStore } from './provenance.js'; // Pure functions export { scoreConfidence } from './confidence-scorer.js'; -export { estimateTokens, hashContent } from './token-estimator.js'; +export { + estimateTokens, + isTiktokenActive, + hashContent, +} from './token-estimator.js'; // Types export type { diff --git a/packages/sdk/src/token-estimator.ts b/packages/sdk/src/token-estimator.ts index f92db70a..4baedf0c 100644 --- a/packages/sdk/src/token-estimator.ts +++ b/packages/sdk/src/token-estimator.ts @@ -1,15 +1,46 @@ /** * Token estimation and content hashing utilities. + * + * Uses js-tiktoken (cl100k_base) for accurate counts. + * Falls back to chars/4 heuristic if encoder fails to load. */ import { createHash } from 'crypto'; +import { createRequire } from 'module'; -/** Estimate token count using chars/4 approximation. */ +type Encoder = { encode: (text: string) => number[] }; + +let encoder: Encoder | null = null; +let initAttempted = false; + +function getEncoder(): Encoder | null { + if (initAttempted) return encoder; + initAttempted = true; + try { + const require = createRequire(import.meta.url); + const tiktoken = require('js-tiktoken'); + encoder = tiktoken.getEncoding('cl100k_base'); + } catch { + encoder = null; + } + return encoder; +} + +/** Estimate token count. Accurate when tiktoken loads, heuristic otherwise. */ export function estimateTokens(content: string): number { if (!content) return 0; + const enc = getEncoder(); + if (enc) { + return enc.encode(content).length; + } return Math.ceil(content.length / 4); } +/** Whether tiktoken is active (for diagnostics). */ +export function isTiktokenActive(): boolean { + return getEncoder() !== null; +} + /** SHA-256 hex digest of content. */ export function hashContent(content: string): string { return createHash('sha256').update(content).digest('hex'); diff --git a/scripts/benchmark-hooks.ts b/scripts/benchmark-hooks.ts new file mode 100644 index 00000000..30d6f99b --- /dev/null +++ b/scripts/benchmark-hooks.ts @@ -0,0 +1,246 @@ +#!/usr/bin/env bun +/** + * benchmark-hooks.ts — Replay action-stream through hooks, measure effectiveness + * + * Reads ~/.stackmemory/desire-paths/action-stream.jsonl and simulates + * what each hook would have caught/suggested, producing a before/after report. + * + * Usage: bun run scripts/benchmark-hooks.ts [--output docs/benchmark-report.md] + */ + +import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs'; +import { join } from 'path'; +import { execSync } from 'child_process'; + +const HOME = process.env.HOME || '/tmp'; +const STREAM_FILE = join(HOME, '.stackmemory/desire-paths/action-stream.jsonl'); +const OUTPUT_FLAG = process.argv.indexOf('--output'); +const OUTPUT_PATH = + OUTPUT_FLAG !== -1 + ? process.argv[OUTPUT_FLAG + 1] + : join(import.meta.dir, '..', 'docs', 'benchmark-report.md'); + +interface Entry { + ts: string; + sid: string; + tool: string; + target: string; + dur?: number; +} + +// --- Load action stream --- +const raw = readFileSync(STREAM_FILE, 'utf-8'); +const entries: Entry[] = raw + .split('\n') + .filter(Boolean) + .map((l) => { + try { + return JSON.parse(l); + } catch { + return null; + } + }) + .filter(Boolean) as Entry[]; + +console.log( + `Loaded ${entries.length} entries from ${new Set(entries.map((e) => e.sid)).size} sessions` +); + +// --- Metrics --- +const metrics = { + total: entries.length, + sessions: new Set(entries.map((e) => e.sid)).size, + + // Dedup analysis + reads: 0, + duplicateReads: 0, + wouldWarn3x: 0, + wouldStop5x: 0, + + // Auto-route analysis + bashTotal: 0, + bashAsRead: 0, + bashAsGlob: 0, + bashAsGrep: 0, + bashGit: 0, + bashLegit: 0, + + // Script-suggest analysis + gitSequences: 0, + ghRunCalls: 0, + webFetchCalls: 0, + webSearchCalls: 0, + scriptSuggestions: 0, + + // Prewarm analysis + toolSearchCalls: 0, + uniqueDeferredTools: new Set(), + prewarmWouldCatch: 0, +}; + +// --- Dedup simulation --- +const sessionReads: Record> = {}; + +for (const e of entries) { + if (e.tool === 'Read') { + metrics.reads++; + const key = e.sid; + if (!sessionReads[key]) sessionReads[key] = {}; + const count = (sessionReads[key][e.target] || 0) + 1; + sessionReads[key][e.target] = count; + if (count >= 5) metrics.wouldStop5x++; + else if (count >= 3) metrics.wouldWarn3x++; + if (count > 1) metrics.duplicateReads++; + } +} + +// --- Auto-route simulation --- +for (const e of entries) { + if (e.tool !== 'Bash') continue; + metrics.bashTotal++; + const cmd = e.target || ''; + + if (/^(cat|head|tail|sed\s+-n|nl)\s/.test(cmd)) { + metrics.bashAsRead++; + } else if (/^(ls|find)\s/.test(cmd)) { + metrics.bashAsGlob++; + } else if (/^(grep|rg|ag)\s/.test(cmd)) { + metrics.bashAsGrep++; + } else if (/^git\s/.test(cmd)) { + metrics.bashGit++; + } else { + metrics.bashLegit++; + } +} + +// --- Script-suggest simulation --- +// Count sequences of 3+ git bash calls per session +const sessionTools: Record = {}; +for (const e of entries) { + if (!sessionTools[e.sid]) sessionTools[e.sid] = []; + sessionTools[e.sid].push(e); +} + +for (const [, tools] of Object.entries(sessionTools)) { + let gitStreak = 0; + for (const t of tools) { + if (t.tool === 'Bash' && /^git\s/.test(t.target || '')) { + gitStreak++; + if (gitStreak === 3) metrics.gitSequences++; + } else { + gitStreak = 0; + } + if (t.tool === 'Bash' && /gh\s+run\s/.test(t.target || '')) + metrics.ghRunCalls++; + if (t.tool === 'WebFetch') metrics.webFetchCalls++; + if (t.tool === 'WebSearch') metrics.webSearchCalls++; + } +} +metrics.scriptSuggestions = + metrics.gitSequences + + metrics.ghRunCalls + + metrics.webFetchCalls + + metrics.webSearchCalls; + +// --- Prewarm simulation --- +const DEFERRED_PREFIXES = [ + 'mcp__', + 'TaskCreate', + 'TaskUpdate', + 'TaskGet', + 'WebFetch', + 'WebSearch', +]; +for (const e of entries) { + if (e.tool === 'ToolSearch') metrics.toolSearchCalls++; + if (DEFERRED_PREFIXES.some((p) => e.tool.startsWith(p))) { + metrics.uniqueDeferredTools.add(e.tool); + } +} +// If we pre-warm top 8, how many ToolSearch calls would we avoid? +// Estimate: each unique deferred tool needs 1 ToolSearch fetch per session +const topTools = [...metrics.uniqueDeferredTools].slice(0, 8); +metrics.prewarmWouldCatch = Math.min( + metrics.toolSearchCalls, + topTools.length * metrics.sessions * 0.3 +); // ~30% of sessions use each + +// --- Generate report --- +const replaceable = + metrics.bashAsRead + metrics.bashAsGlob + metrics.bashAsGrep; +const report = `# StackMemory Hook Benchmark Report + +> Generated: ${new Date().toISOString()} +> Data: ${metrics.total} tool calls across ${metrics.sessions} sessions + +## Baseline (before hooks) + +| Metric | Value | % of total | +|--------|------:|----------:| +| Total tool calls | ${metrics.total} | 100% | +| Read calls | ${metrics.reads} | ${((metrics.reads / metrics.total) * 100).toFixed(1)}% | +| Duplicate reads | ${metrics.duplicateReads} | ${((metrics.duplicateReads / metrics.total) * 100).toFixed(1)}% | +| Bash calls | ${metrics.bashTotal} | ${((metrics.bashTotal / metrics.total) * 100).toFixed(1)}% | +| Bash → should be Glob | ${metrics.bashAsGlob} | ${((metrics.bashAsGlob / metrics.total) * 100).toFixed(1)}% | +| Bash → should be Read | ${metrics.bashAsRead} | ${((metrics.bashAsRead / metrics.total) * 100).toFixed(1)}% | +| Bash → should be Grep | ${metrics.bashAsGrep} | ${((metrics.bashAsGrep / metrics.total) * 100).toFixed(1)}% | +| Bash (git) | ${metrics.bashGit} | ${((metrics.bashGit / metrics.total) * 100).toFixed(1)}% | +| Bash (legit) | ${metrics.bashLegit} | ${((metrics.bashLegit / metrics.total) * 100).toFixed(1)}% | +| ToolSearch calls | ${metrics.toolSearchCalls} | ${((metrics.toolSearchCalls / metrics.total) * 100).toFixed(1)}% | + +## Hook Effectiveness (projected) + +### 1. Dedup Reads (escalation at 3x soft / 5x STOP) +- Would warn (3-4x): **${metrics.wouldWarn3x}** calls +- Would STOP (5x+): **${metrics.wouldStop5x}** calls +- Combined catch: **${metrics.wouldWarn3x + metrics.wouldStop5x}** / ${metrics.reads} reads = **${(((metrics.wouldWarn3x + metrics.wouldStop5x) / metrics.reads) * 100).toFixed(1)}%** +- Token savings estimate: ~${((metrics.wouldStop5x * 200) / 1000).toFixed(0)}K tokens (STOP prevents re-read) + +### 2. Auto-Route (Bash → dedicated tools) +- Replaceable calls caught: **${replaceable}** / ${metrics.bashTotal} Bash calls = **${((replaceable / metrics.bashTotal) * 100).toFixed(1)}%** +- Breakdown: ${metrics.bashAsGlob} ls/find → Glob, ${metrics.bashAsRead} cat/head → Read, ${metrics.bashAsGrep} grep → Grep +- Token savings estimate: ~${((replaceable * 50) / 1000).toFixed(0)}K tokens (reduced overhead per call) + +### 3. Prewarm (pre-fetch deferred tool schemas) +- ToolSearch calls observed: **${metrics.toolSearchCalls}** +- Unique deferred tools: **${metrics.uniqueDeferredTools.size}** +- Top 8 tools cover: ~${topTools.length} tools +- Estimated catches: **~${Math.round(metrics.prewarmWouldCatch)}** avoided ToolSearch calls +- Token savings estimate: ~${((metrics.prewarmWouldCatch * 150) / 1000).toFixed(0)}K tokens + +### 4. Script-Suggest (pattern → script) +- Git sequences (3+ cmds): **${metrics.gitSequences}** → git-ops.ts +- gh run calls: **${metrics.ghRunCalls}** → build-status.ts +- WebFetch calls: **${metrics.webFetchCalls}** → web-fetch.ts +- WebSearch calls: **${metrics.webSearchCalls}** → web-search.ts +- Total suggestions would fire: **${metrics.scriptSuggestions}** +- Token savings estimate: ~${((metrics.scriptSuggestions * 4 * 200) / 1000).toFixed(0)}K tokens (each script replaces ~4 calls) + +## Summary + +| Hook | Catches | Est. token savings | +|------|--------:|------------------:| +| Dedup STOP | ${metrics.wouldStop5x} reads | ~${((metrics.wouldStop5x * 200) / 1000).toFixed(0)}K | +| Auto-route | ${replaceable} Bash calls | ~${((replaceable * 50) / 1000).toFixed(0)}K | +| Prewarm | ~${Math.round(metrics.prewarmWouldCatch)} ToolSearch | ~${((metrics.prewarmWouldCatch * 150) / 1000).toFixed(0)}K | +| Script-suggest | ${metrics.scriptSuggestions} patterns | ~${((metrics.scriptSuggestions * 4 * 200) / 1000).toFixed(0)}K | +| **Total** | | **~${((metrics.wouldStop5x * 200 + replaceable * 50 + metrics.prewarmWouldCatch * 150 + metrics.scriptSuggestions * 4 * 200) / 1000).toFixed(0)}K** | + +Baseline total estimated tokens: ~${((metrics.total * 200) / 1000).toFixed(0)}K +Projected waste reduction: **${(((metrics.wouldStop5x * 200 + replaceable * 50 + metrics.prewarmWouldCatch * 150 + metrics.scriptSuggestions * 4 * 200) / (metrics.total * 200)) * 100).toFixed(1)}%** +`; + +// Write report +const dir = join(import.meta.dir, '..', 'docs'); +if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); +writeFileSync(OUTPUT_PATH, report); +console.log(`\nReport written to: ${OUTPUT_PATH}`); +console.log(`\n--- Quick Summary ---`); +console.log( + `Dedup would catch: ${metrics.wouldWarn3x + metrics.wouldStop5x} reads (${metrics.wouldStop5x} hard-stopped)` +); +console.log(`Auto-route would catch: ${replaceable} Bash calls`); +console.log(`Script-suggest would fire: ${metrics.scriptSuggestions} times`); +console.log( + `Projected total savings: ~${((metrics.wouldStop5x * 200 + replaceable * 50 + metrics.prewarmWouldCatch * 150 + metrics.scriptSuggestions * 4 * 200) / 1000).toFixed(0)}K tokens` +); diff --git a/src/cli/codex-sm.ts b/src/cli/codex-sm.ts index 2718a992..d533daad 100644 --- a/src/cli/codex-sm.ts +++ b/src/cli/codex-sm.ts @@ -253,6 +253,54 @@ class CodexSM { } } + /** + * Emit context budget advice based on tool-call count from checkpoint state. + * Mirrors the Claude Code context-budget.js hook. + */ + private emitContextBudgetAdvice(): void { + const COMPACT_SUGGEST = 50; + const COMPACT_STRONG = 65; + const RESTART_RECOMMEND = 80; + + try { + const stateFile = path.join( + os.homedir(), + '.stackmemory', + `checkpoint-state-${this.config.instanceId}.json` + ); + if (!fs.existsSync(stateFile)) return; + + const state = JSON.parse(fs.readFileSync(stateFile, 'utf-8')); + const cwd = process.cwd(); + const toolCount = state.projects?.[cwd]?.toolCount || 0; + + if (toolCount >= RESTART_RECOMMEND) { + console.log( + chalk.yellow( + `[CONTEXT_BUDGET] ${toolCount} tool calls (~150K+ tokens). ` + + `Recommend: save context then start fresh session.` + ) + ); + } else if (toolCount >= COMPACT_STRONG) { + console.log( + chalk.yellow( + `[CONTEXT_BUDGET] ${toolCount} tool calls (~100-130K tokens). ` + + `Context heavy — consider compacting or restarting.` + ) + ); + } else if (toolCount >= COMPACT_SUGGEST) { + console.log( + chalk.gray( + `[CONTEXT_BUDGET] ${toolCount} tool calls (~80-100K tokens). ` + + `Context getting heavy.` + ) + ); + } + } catch { + // Silent — never block exit + } + } + private loadContext(): void { if (!this.config.contextEnabled) return; try { @@ -591,6 +639,9 @@ class CodexSM { // Non-fatal: don't block exit } + // Context budget check + this.emitContextBudgetAdvice(); + if (this.config.tracingEnabled) { const summary = trace.getExecutionSummary(); console.log(); diff --git a/src/cli/commands/orchestrator.ts b/src/cli/commands/orchestrator.ts index 030d31d5..1ecb640e 100644 --- a/src/cli/commands/orchestrator.ts +++ b/src/cli/commands/orchestrator.ts @@ -9,6 +9,7 @@ */ import { spawn, execSync, type ChildProcess } from 'child_process'; +import { estimateTokens } from '../../core/cache/token-estimator.js'; import { appendFileSync, existsSync, @@ -2295,7 +2296,7 @@ export class Conductor { } if (block.type === 'text' && block.text) { const text = block.text as string; - run.tokensUsed += Math.ceil(text.length / 4); + run.tokensUsed += estimateTokens(text); turnTextParts.push(text); } } @@ -2489,7 +2490,7 @@ export class Conductor { // Estimate tokens from message sizes if (msg.method === 'item/text' && params?.text) { - run.tokensUsed += Math.ceil((params.text as string).length / 4); + run.tokensUsed += estimateTokens(params.text as string); } // Update agent status file periodically (every 5 tool calls) diff --git a/src/cli/commands/scaffold.ts b/src/cli/commands/scaffold.ts new file mode 100644 index 00000000..3d9fbae9 --- /dev/null +++ b/src/cli/commands/scaffold.ts @@ -0,0 +1,117 @@ +#!/usr/bin/env node +/** + * stackmemory scaffold — Create a Company OS folder structure. + * + * Scaffolds company/, wiki/, skills/, clients/, raw/, .stackmemory/config.yml + * for local context management. Files are indexed by the MCP server on boot. + */ + +import { Command } from 'commander'; +import chalk from 'chalk'; +import * as fs from 'fs'; +import * as path from 'path'; + +const DIRS = ['company', 'wiki', 'skills', 'clients', 'raw', '.stackmemory']; + +const TEMPLATES: Record = { + 'company/voice.md': + '---\nname: Voice Guide\ndescription: How we write and communicate\n---\n\n# Voice Guide\n\n## Tone\n- [Your tone descriptors here]\n\n## Words we use\n- [Preferred terms]\n\n## Words we avoid\n- [Banned terms]\n', + 'company/team.md': + '---\nname: Team Directory\ndescription: Who works here and what they do\n---\n\n# Team\n\n| Name | Role | Contact |\n|------|------|---------||\n', + 'company/design.md': + '---\nname: Design System\ndescription: Logos, colors, components\n---\n\n# Design System\n\n## Colors\n- Primary:\n- Secondary:\n\n## Logos\n- [paths or URLs]\n', + 'wiki/README.md': + '# Wiki — SOPs & Playbooks\n\nAdd markdown files here. Files with skill frontmatter become Claude skills.\n', + 'skills/README.md': + '# Skills\n\nClaude skill-packs. Each file is a markdown instruction set with frontmatter.\n\n```yaml\n---\nname: skill-name\ndescription: What this skill does\nactivates_on: [keyword1, keyword2]\nversion: "1.0"\n---\n```\n', + 'clients/README.md': + '# Clients\n\nEach client gets a subfolder with icp.md, voice.md, campaigns/, context/.\n', + 'raw/README.md': + '# Raw\n\nUnstructured data: transcripts, research, scrapes.\n', + '.stackmemory/config.yml': `# StackMemory Company OS Configuration + +sources: + - path: ./company + type: reference + - path: ./wiki + type: sop + - path: ./skills + type: skill + - path: ./raw + type: raw + +tenants: {} + +freshness_threshold_hours: 24 + +skill_rot: + enabled: true + stale_days: 90 + correction_threshold: 5 +`, +}; + +export function createScaffoldCommand(): Command { + const cmd = new Command('scaffold') + .alias('os-init') + .description( + 'Scaffold a Company OS folder structure for local context management' + ) + .option('--force', 'Overwrite existing template files') + .option('--dir ', 'Target directory (default: current directory)') + .action(async (options: { force?: boolean; dir?: string }) => { + const targetDir = path.resolve(options.dir || process.cwd()); + const created: string[] = []; + const skipped: string[] = []; + + // Create directories + for (const dir of DIRS) { + const fullPath = path.join(targetDir, dir); + if (!fs.existsSync(fullPath)) { + fs.mkdirSync(fullPath, { recursive: true }); + created.push(dir + '/'); + } + } + + // Create template files + for (const [relPath, content] of Object.entries(TEMPLATES)) { + const fullPath = path.join(targetDir, relPath); + const dir = path.dirname(fullPath); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + + if (fs.existsSync(fullPath) && !options.force) { + skipped.push(relPath); + continue; + } + + fs.writeFileSync(fullPath, content, 'utf-8'); + created.push(relPath); + } + + console.log(chalk.cyan('\n Company OS scaffolded\n')); + + if (created.length) { + console.log(chalk.green(` Created: ${created.length} files/dirs`)); + for (const f of created) { + console.log(chalk.gray(` + ${f}`)); + } + } + + if (skipped.length) { + console.log(chalk.gray(` Skipped: ${skipped.length} (already exist)`)); + } + + console.log(); + console.log(chalk.gray(' Next steps:')); + console.log( + chalk.gray(' 1. Edit company/voice.md with your tone and brand') + ); + console.log(chalk.gray(' 2. Add skills to skills/ as markdown files')); + console.log( + chalk.gray(' 3. Set COMPANY_OS_ROOT=. in .env for MCP auto-indexing') + ); + console.log(); + }); + + return cmd; +} diff --git a/src/cli/commands/tasks.ts b/src/cli/commands/tasks.ts index 15a720cf..3841c1c5 100644 --- a/src/cli/commands/tasks.ts +++ b/src/cli/commands/tasks.ts @@ -6,11 +6,24 @@ import { Command } from 'commander'; import Database from 'better-sqlite3'; import { join } from 'path'; -import { existsSync } from 'fs'; +import { existsSync, mkdirSync, writeFileSync, readFileSync } from 'fs'; import { LinearTaskManager, TaskPriority, } from '../../features/tasks/linear-task-manager.js'; +import { + parseMasterTasks, + getNextTask, + addTaskToFile, + updateTaskInFile, + type TaskPriority as MdPriority, + type TaskStatus as MdStatus, + type TaskSync, +} from '../../core/tasks/md-task-parser.js'; +import { + MASTER_TASKS_TEMPLATE, + TASKS_CONFIG_TEMPLATE, +} from '../../core/tasks/master-tasks-template.js'; /** Raw task row from task_cache table */ interface TaskCacheRow { @@ -263,9 +276,193 @@ export function createTaskCommands(): Command { } }); + // ── Init: scaffold master-tasks.md ───────────────────────── + tasks + .command('init') + .description('Scaffold .stackmemory/tasks/master-tasks.md') + .action(() => { + const projectRoot = process.cwd(); + const tasksDir = join(projectRoot, '.stackmemory', 'tasks'); + const mdPath = join(tasksDir, 'master-tasks.md'); + const configPath = join(tasksDir, 'config.json'); + + if (existsSync(mdPath)) { + console.log(`Already exists: ${mdPath}`); + return; + } + + mkdirSync(tasksDir, { recursive: true }); + writeFileSync(mdPath, MASTER_TASKS_TEMPLATE, 'utf-8'); + writeFileSync( + configPath, + JSON.stringify(TASKS_CONFIG_TEMPLATE, null, 2), + 'utf-8' + ); + console.log(`Created: ${mdPath}`); + console.log(`Created: ${configPath}`); + }); + + // ── MD subcommands (local-first master-tasks.md) ────────── + const md = new Command('md').description( + 'Local-first task management via master-tasks.md' + ); + + md.command('list') + .alias('ls') + .description('List tasks from master-tasks.md') + .option('-p, --priority

', 'Filter by priority (P0, P1, P2, P3)') + .option( + '-s, --status ', + 'Filter by status (todo, active, done, blocked, cut)' + ) + .option('-o, --owner ', 'Filter by owner (@me, @agent, @defer)') + .option('--json', 'Output as JSON') + .action((options) => { + const mdPath = resolveMdPath(); + if (!mdPath) return; + + let tasks = parseMasterTasks(readFileSync(mdPath, 'utf-8')); + + if (options.priority) + tasks = tasks.filter((t) => t.priority === options.priority); + if (options.status) + tasks = tasks.filter((t) => t.status === options.status); + if (options.owner) tasks = tasks.filter((t) => t.owner === options.owner); + + if (options.json) { + console.log(JSON.stringify(tasks, null, 2)); + return; + } + + if (tasks.length === 0) { + console.log('No tasks found'); + return; + } + + console.log(`\nTasks (${tasks.length})\n`); + for (const t of tasks) { + const pColor = + t.priority === 'P0' + ? '\x1b[31m' + : t.priority === 'P1' + ? '\x1b[33m' + : '\x1b[90m'; + const sIcon = + t.status === 'done' + ? '[x]' + : t.status === 'active' + ? '[>]' + : t.status === 'blocked' + ? '[!]' + : '[ ]'; + console.log( + `${sIcon} ${pColor}${t.priority}\x1b[0m ${t.id} ${t.task} ${t.owner} ${t.branchPr ? `(${t.branchPr})` : ''}` + ); + } + console.log(''); + }); + + md.command('next') + .description('Show the next task to work on') + .option('--json', 'Output as JSON') + .action((options) => { + const mdPath = resolveMdPath(); + if (!mdPath) return; + + const tasks = parseMasterTasks(readFileSync(mdPath, 'utf-8')); + const next = getNextTask(tasks); + + if (!next) { + console.log('No actionable tasks'); + return; + } + + if (options.json) { + console.log(JSON.stringify(next)); + return; + } + + console.log(`\nNext: ${next.id} [${next.priority}] ${next.task}`); + console.log(` Owner: ${next.owner} | Sync: ${next.sync}`); + if (next.notes) console.log(` Notes: ${next.notes}`); + console.log(''); + }); + + md.command('add ') + .description('Add a task to master-tasks.md') + .option('-p, --priority

', 'Priority (P0-P3)', 'P1') + .option('-o, --owner ', 'Owner (@me, @agent, @defer)', '@me') + .option('-s, --sync ', 'Sync target (local, linear, gh)', 'local') + .option('-b, --branch ', 'Branch or PR') + .option('-n, --notes ', 'Notes') + .action((description, options) => { + const mdPath = resolveMdPath(); + if (!mdPath) return; + + const id = addTaskToFile(mdPath, { + priority: options.priority as MdPriority, + status: 'todo', + owner: options.owner, + sync: options.sync as TaskSync, + task: description, + branchPr: options.branch || '', + notes: options.notes || '', + }); + + console.log(`Added: ${id} ${description}`); + }); + + md.command('update ') + .description('Update a task in master-tasks.md') + .option( + '-s, --status ', + 'New status (todo, active, done, blocked, cut)' + ) + .option('-p, --priority

', 'New priority (P0-P3)') + .option('-o, --owner ', 'New owner') + .option('-b, --branch ', 'Branch or PR') + .option('-n, --notes ', 'Notes') + .option('--sync ', 'Sync target (local, linear, gh)') + .action((taskId, options) => { + const mdPath = resolveMdPath(); + if (!mdPath) return; + + try { + const updates: Record = {}; + if (options.status) updates.status = options.status; + if (options.priority) updates.priority = options.priority; + if (options.owner) updates.owner = options.owner; + if (options.branch) updates.branchPr = options.branch; + if (options.notes) updates.notes = options.notes; + if (options.sync) updates.sync = options.sync; + + updateTaskInFile(mdPath, taskId.toUpperCase(), updates); + console.log(`Updated: ${taskId.toUpperCase()}`); + } catch (err) { + console.error(`Error: ${(err as Error).message}`); + } + }); + + tasks.addCommand(md); + return tasks; } +/** Resolve master-tasks.md path — check .stackmemory/tasks/ then project root */ +function resolveMdPath(): string | null { + const projectRoot = process.cwd(); + const smPath = join(projectRoot, '.stackmemory', 'tasks', 'master-tasks.md'); + if (existsSync(smPath)) return smPath; + + const rootPath = join(projectRoot, 'master-tasks.md'); + if (existsSync(rootPath)) return rootPath; + + console.error( + 'No master-tasks.md found. Run "stackmemory tasks init" first.' + ); + return null; +} + function findTaskByPartialId( projectRoot: string, partialId: string diff --git a/src/cli/hermes-sm.ts b/src/cli/hermes-sm.ts new file mode 100644 index 00000000..80d92f42 --- /dev/null +++ b/src/cli/hermes-sm.ts @@ -0,0 +1,315 @@ +#!/usr/bin/env node + +/** + * hermes-sm: Hermes wrapper with StackMemory context persistence + * + * Automatically manages: + * - Context save/restore across Hermes sessions + * - Daemon health check + auto-start + * - Desire-path action stream logging + * - Determinism watcher for reproducibility tracking + * - Instance ID + tracing + */ + +import { spawn, execSync, execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { program } from 'commander'; +import { v4 as uuidv4 } from 'uuid'; +import chalk from 'chalk'; +import { initializeTracing, trace } from '../core/trace/index.js'; +import { resolveRealCliBin } from './utils/real-cli-bin.js'; +import { + type DeterminismWatcherHandle, + startDeterminismWatcher, + stopDeterminismWatcher, +} from './utils/determinism-watcher.js'; +import { + canonicalStateStore, + projectIdFromIdentifier, +} from '../core/shared-state/canonical-store.js'; + +interface HermesConfig { + instanceId: string; + contextEnabled: boolean; + task?: string; + tracingEnabled: boolean; + verboseTracing: boolean; + hermesBin?: string; + sessionStartTime: number; + model?: string; + provider?: string; + resume?: string; +} + +const SM_DIR = path.join(os.homedir(), '.stackmemory'); +const HERMES_CONFIG_PATH = path.join(SM_DIR, 'hermes-sm.json'); + +interface HermesSMConfig { + defaultTracing: boolean; + defaultContext: boolean; +} + +const DEFAULT_CONFIG: HermesSMConfig = { + defaultTracing: true, + defaultContext: true, +}; + +function loadConfig(): HermesSMConfig { + try { + if (fs.existsSync(HERMES_CONFIG_PATH)) { + return { ...DEFAULT_CONFIG, ...JSON.parse(fs.readFileSync(HERMES_CONFIG_PATH, 'utf8')) }; + } + } catch { + // Use defaults + } + return { ...DEFAULT_CONFIG }; +} + +function resolveHermesBin(): string { + // Check common locations + const candidates = [ + path.join(os.homedir(), '.local', 'bin', 'hermes'), + '/usr/local/bin/hermes', + '/opt/homebrew/bin/hermes', + ]; + + for (const bin of candidates) { + if (fs.existsSync(bin)) return bin; + } + + // Try PATH + try { + const which = execSync('which hermes', { encoding: 'utf8' }).trim(); + if (which) return which; + } catch { + // Not in PATH + } + + throw new Error( + 'hermes not found. Install: pip install hermes-agent or check ~/.local/bin/hermes' + ); +} + +function ensureDaemon(): void { + const pidFile = path.join(SM_DIR, 'daemon', 'daemon.pid'); + + try { + if (fs.existsSync(pidFile)) { + const pid = parseInt(fs.readFileSync(pidFile, 'utf8').trim(), 10); + try { + process.kill(pid, 0); // Check if alive + return; // Running + } catch { + // Dead — clean up + fs.unlinkSync(pidFile); + } + } + } catch { + // Can't check — try starting + } + + // Start daemon + try { + execSync('stackmemory daemon start', { stdio: 'ignore', timeout: 5000 }); + console.log(chalk.dim(' ↳ StackMemory daemon started')); + } catch { + // Non-fatal — daemon features degrade gracefully + } +} + +function writeSessionHeartbeat(instanceId: string): NodeJS.Timeout { + const sessionsDir = path.join(SM_DIR, 'sessions'); + if (!fs.existsSync(sessionsDir)) fs.mkdirSync(sessionsDir, { recursive: true }); + + const heartbeatFile = path.join(sessionsDir, `session-${Date.now()}.heartbeat`); + fs.writeFileSync(heartbeatFile, instanceId); + + // Update heartbeat every 60s + const interval = setInterval(() => { + try { + const now = new Date(); + fs.utimesSync(heartbeatFile, now, now); + } catch { + // Non-fatal + } + }, 60_000); + + interval.unref(); + + return interval; +} + +class HermesSM { + private config: HermesConfig; + private detWatcher?: DeterminismWatcherHandle; + private heartbeatInterval?: NodeJS.Timeout; + + constructor(config: HermesConfig) { + this.config = config; + } + + async run(): Promise { + const { instanceId, tracingEnabled, verboseTracing } = this.config; + + console.log(chalk.cyan('╭─ hermes-sm ─────────────────────────────╮')); + console.log(chalk.cyan(`│ Instance: ${instanceId.slice(0, 8)} │`)); + console.log(chalk.cyan('╰──────────────────────────────────────────╯')); + + // 1. Ensure daemon is running + ensureDaemon(); + + // 2. Initialize tracing + if (tracingEnabled) { + initializeTracing({ + serviceName: 'hermes-sm', + verbose: verboseTracing, + }); + trace('session_start', { instanceId, tool: 'hermes' }); + } + + // 3. Start heartbeat + this.heartbeatInterval = writeSessionHeartbeat(instanceId); + + // 4. Start determinism watcher + if (this.config.contextEnabled) { + try { + this.detWatcher = startDeterminismWatcher({ + projectId: projectIdFromIdentifier(process.cwd()), + sessionId: instanceId, + }); + } catch { + // Non-fatal + } + } + + // 5. Load handoff context if available + let handoffContext = ''; + if (this.config.contextEnabled) { + try { + const projectId = projectIdFromIdentifier(process.cwd()); + const store = canonicalStateStore(); + const handoff = store.getLatestHandoff(projectId); + if (handoff) { + handoffContext = handoff.content || ''; + console.log(chalk.dim(` ↳ Restored handoff: ${handoff.summary?.slice(0, 60) || 'previous session'}`)); + } + } catch { + // No handoff available + } + } + + // 6. Build hermes command + const hermesBin = this.config.hermesBin || resolveHermesBin(); + const args: string[] = []; + + if (this.config.resume) { + args.push('--resume', this.config.resume); + } else if (this.config.task) { + args.push('-z', this.config.task); + } + + if (this.config.model) { + args.push('-m', this.config.model); + } + + if (this.config.provider) { + args.push('--provider', this.config.provider); + } + + // Pass session ID for desire-path tracking + args.push('--pass-session-id'); + + // 7. Set environment for hooks + const env = { + ...process.env, + STACKMEMORY_SESSION: instanceId, + STACKMEMORY_TOOL: 'hermes', + STACKMEMORY_PROJECT: process.cwd(), + }; + + // Inject handoff context as system prompt prefix if available + if (handoffContext) { + env.HERMES_SYSTEM_PREFIX = handoffContext.slice(0, 2000); + } + + // 8. Spawn hermes + console.log(chalk.dim(` ↳ ${hermesBin} ${args.join(' ')}`)); + + const child = spawn(hermesBin, args, { + stdio: 'inherit', + env, + cwd: process.cwd(), + }); + + // 9. Handle exit + child.on('exit', (code) => { + this.cleanup(); + + if (tracingEnabled) { + trace('session_end', { + instanceId, + exitCode: code, + duration: Date.now() - this.config.sessionStartTime, + }); + } + + process.exit(code || 0); + }); + + // Handle signals + const handleSignal = (signal: NodeJS.Signals) => { + child.kill(signal); + }; + process.on('SIGINT', () => handleSignal('SIGINT')); + process.on('SIGTERM', () => handleSignal('SIGTERM')); + } + + private cleanup(): void { + if (this.detWatcher) { + stopDeterminismWatcher(this.detWatcher); + } + if (this.heartbeatInterval) { + clearInterval(this.heartbeatInterval); + } + } +} + +// ─── CLI ────────────────────────────────────────────────────── + +const smConfig = loadConfig(); + +program + .name('hermes-smd') + .description('Hermes with StackMemory context persistence, daemon auto-start, and desire-path tracking') + .argument('[prompt...]', 'Initial prompt for hermes') + .option('--resume ', 'Resume a Hermes session by ID') + .option('-m, --model ', 'Model to use') + .option('--provider ', 'Model provider') + .option('--no-context', 'Disable context persistence') + .option('--no-tracing', 'Disable tracing') + .option('--verbose-trace', 'Verbose tracing output') + .option('--hermes-bin ', 'Path to hermes binary') + .action(async (prompt: string[], options) => { + const instanceId = uuidv4(); + const task = prompt.length > 0 ? prompt.join(' ') : undefined; + + const config: HermesConfig = { + instanceId, + contextEnabled: options.context !== false && smConfig.defaultContext, + task, + tracingEnabled: options.tracing !== false && smConfig.defaultTracing, + verboseTracing: options.verboseTrace || false, + hermesBin: options.hermesBin, + sessionStartTime: Date.now(), + model: options.model, + provider: options.provider, + resume: options.resume, + }; + + const sm = new HermesSM(config); + await sm.run(); + }); + +program.parse(); diff --git a/src/cli/index.ts b/src/cli/index.ts index 09ea6e4e..05270bcb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -79,6 +79,7 @@ import { createLoopCommand } from './commands/loop.js'; import { createSkillCommand } from './commands/skill.js'; import { createPackCommand } from './commands/pack.js'; import { createCacheCommand } from './commands/cache.js'; +import { createScaffoldCommand } from './commands/scaffold.js'; import chalk from 'chalk'; import * as fs from 'fs'; import * as path from 'path'; @@ -830,6 +831,7 @@ program.addCommand(createRulesCommand()); program.addCommand(createSkillCommand()); program.addCommand(createPackCommand()); program.addCommand(createCacheCommand()); +program.addCommand(createScaffoldCommand()); // Register setup and diagnostic commands registerSetupCommands(program); diff --git a/src/core/cache/token-estimator.ts b/src/core/cache/token-estimator.ts index 4b691b6e..af42dec8 100644 --- a/src/core/cache/token-estimator.ts +++ b/src/core/cache/token-estimator.ts @@ -1,18 +1,48 @@ /** - * Token estimation and content hashing utilities + * Token estimation and content hashing utilities. + * + * Uses js-tiktoken (cl100k_base) for accurate counts. + * Falls back to chars/4 heuristic if encoder fails to load. */ import { createHash } from 'crypto'; +import { createRequire } from 'module'; + +type Encoder = { encode: (text: string) => number[] }; + +let encoder: Encoder | null = null; +let initAttempted = false; + +function getEncoder(): Encoder | null { + if (initAttempted) return encoder; + initAttempted = true; + try { + const require = createRequire(import.meta.url); + const tiktoken = require('js-tiktoken'); + encoder = tiktoken.getEncoding('cl100k_base'); + } catch { + encoder = null; + } + return encoder; +} /** - * Estimate token count using chars/4 approximation. - * Good enough for cache dedup -- no tiktoken dependency needed. + * Estimate token count. Accurate when tiktoken loads, heuristic otherwise. */ export function estimateTokens(content: string): number { if (!content) return 0; + const enc = getEncoder(); + if (enc) { + return enc.encode(content).length; + } return Math.ceil(content.length / 4); } +/** Whether tiktoken is active (for diagnostics). */ +export function isTiktokenActive(): boolean { + return getEncoder() !== null; +} + /** * SHA-256 hex digest of content for content-addressable lookup. */ diff --git a/src/core/context/recursive-context-manager.ts b/src/core/context/recursive-context-manager.ts index cb6fb781..e12d8c9e 100644 --- a/src/core/context/recursive-context-manager.ts +++ b/src/core/context/recursive-context-manager.ts @@ -8,6 +8,7 @@ import { DualStackManager } from './dual-stack-manager.js'; import { ContextRetriever } from '../retrieval/context-retriever.js'; import { logger } from '../monitoring/logger.js'; +import { estimateTokens } from '../cache/token-estimator.js'; import { ValidationError, ErrorCode } from '../errors/index.js'; import * as fs from 'fs'; import * as path from 'path'; @@ -552,9 +553,6 @@ export class RecursiveContextManager { const selected: ContextChunk[] = []; let totalTokens = 0; - // Rough token estimation (1 token ≈ 4 chars) - const estimateTokens = (text: string) => Math.ceil(text.length / 4); - for (const chunk of chunks) { const chunkTokens = estimateTokens(chunk.content); diff --git a/src/core/context/rehydration.ts b/src/core/context/rehydration.ts index 4bc7a930..e1f91471 100644 --- a/src/core/context/rehydration.ts +++ b/src/core/context/rehydration.ts @@ -9,6 +9,7 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import { logger } from '../monitoring/logger.js'; +import { estimateTokens } from '../cache/token-estimator.js'; import { FrameManager } from './index.js'; import type { _Anchor, Event } from './index.js'; import { @@ -114,8 +115,7 @@ export class CompactionHandler { * Track token usage from a message */ trackTokens(content: string): void { - // Rough estimation: 1 token ≈ 4 characters - const estimatedTokens = Math.ceil(content.length / 4); + const estimatedTokens = estimateTokens(content); this.tokenAccumulator += estimatedTokens; this.metrics.estimatedTokens += estimatedTokens; diff --git a/src/core/retrieval/llm-context-retrieval.ts b/src/core/retrieval/llm-context-retrieval.ts index d41ad1a2..ef4bf669 100644 --- a/src/core/retrieval/llm-context-retrieval.ts +++ b/src/core/retrieval/llm-context-retrieval.ts @@ -20,6 +20,7 @@ import { RetrievalMetadata, } from './types.js'; import { logger } from '../monitoring/logger.js'; +import { estimateTokens } from '../cache/token-estimator.js'; import { LazyContextLoader } from '../performance/lazy-context-loader.js'; import { ContextCache } from '../performance/context-cache.js'; import { LLMProvider, createLLMProvider } from './llm-provider.js'; @@ -185,12 +186,12 @@ class HeuristicAnalyzer { let tokens = 50; // Base frame header tokens += frame.eventCount * 30; // Estimate per event tokens += frame.anchorCount * 40; // Estimate per anchor - if (frame.digestPreview) tokens += frame.digestPreview.length / 4; + if (frame.digestPreview) tokens += estimateTokens(frame.digestPreview); return Math.floor(tokens); } private estimateSummaryTokens(summary: CompressedSummary): number { - return Math.floor(JSON.stringify(summary).length / 4); + return estimateTokens(JSON.stringify(summary)); } private assessQueryComplexity( @@ -599,8 +600,8 @@ Respond with only the JSON object, no other text.`; })), metadata: { analysisTimeMs: 0, - summaryTokens: Math.floor( - JSON.stringify(request.compressedSummary).length / 4 + summaryTokens: estimateTokens( + JSON.stringify(request.compressedSummary) ), queryComplexity: this.assessQueryComplexity(request.currentQuery), matchedPatterns: [], diff --git a/src/core/retrieval/unified-context-assembler.ts b/src/core/retrieval/unified-context-assembler.ts index 1c64a967..d10bf652 100644 --- a/src/core/retrieval/unified-context-assembler.ts +++ b/src/core/retrieval/unified-context-assembler.ts @@ -16,6 +16,7 @@ import { DiffMemStatus, } from '../../integrations/diffmem/types.js'; import { logger } from '../monitoring/logger.js'; +import { estimateTokens } from '../cache/token-estimator.js'; /** * Configuration for unified context assembly @@ -95,15 +96,6 @@ export const DEFAULT_UNIFIED_CONTEXT_CONFIG: UnifiedContextConfig = { privacyMode: 'standard', }; -/** - * Estimate token count from content - * Uses rough approximation: 1 token ≈ 4 characters - */ -function estimateTokens(content: string): number { - if (!content) return 0; - return Math.ceil(content.length / 4); -} - /** * Truncate content to fit within token budget */ diff --git a/src/core/skill-packs/__tests__/parser.test.ts b/src/core/skill-packs/__tests__/parser.test.ts index aa76ef68..c6de97d2 100644 --- a/src/core/skill-packs/__tests__/parser.test.ts +++ b/src/core/skill-packs/__tests__/parser.test.ts @@ -112,6 +112,30 @@ describe('parsePackYaml', () => { expect(() => parsePackYaml(yaml.dump(bad))).toThrow(); }); + it('should accept content licenses like CC-BY-4.0', () => { + const manifest = { + name: 'learning/opportunities', + version: '1.0.0', + description: 'Learning exercises for AI-assisted coding', + author: 'drcathicks', + license: 'CC-BY-4.0', + }; + const pack = parsePackYaml(yaml.dump(manifest)); + expect(pack.manifest.license).toBe('CC-BY-4.0'); + }); + + it('should accept custom license strings', () => { + const manifest = { + name: 'test/custom', + version: '1.0.0', + description: 'Custom license pack', + author: 'test', + license: 'Proprietary', + }; + const pack = parsePackYaml(yaml.dump(manifest)); + expect(pack.manifest.license).toBe('Proprietary'); + }); + it('should accept all valid runtime types', () => { for (const type of ['local', 'e2b', 'cua', 'modal']) { const manifest = { diff --git a/src/core/skill-packs/types.ts b/src/core/skill-packs/types.ts index 12bad85f..0f79ed2c 100644 --- a/src/core/skill-packs/types.ts +++ b/src/core/skill-packs/types.ts @@ -90,6 +90,32 @@ export const SkillPackExampleSchema = z.object({ export type SkillPackExample = z.infer; +// ============================================================ +// LICENSE +// ============================================================ + +/** + * Known licenses for skill packs. + * Code licenses (MIT, Apache-2.0, ISC) and content licenses (CC-BY-4.0, CC-BY-SA-4.0) + * are both valid — skills are often prompt text (content) rather than executable code. + */ +export const KnownLicenseSchema = z.enum([ + 'MIT', + 'Apache-2.0', + 'ISC', + 'BSD-2-Clause', + 'BSD-3-Clause', + 'CC-BY-4.0', + 'CC-BY-SA-4.0', + 'CC0-1.0', + 'UNLICENSED', +]); + +export type KnownLicense = z.infer; + +/** Accepts known SPDX identifiers or any custom string */ +const LicenseSchema = KnownLicenseSchema.or(z.string().min(1)); + // ============================================================ // PACK NAME (namespace/pack-name) // ============================================================ @@ -111,7 +137,7 @@ export const SkillPackManifestSchema = z.object({ version: SemverSchema, description: z.string().min(1), author: z.string().min(1), - license: z.string().default('MIT'), + license: LicenseSchema.default('MIT'), runtime: SkillPackRuntimeSchema.optional(), ingestion: SkillPackIngestionSchema.optional(), ontology: SkillPackOntologySchema.optional(), diff --git a/src/core/tasks/__tests__/md-task-parser.test.ts b/src/core/tasks/__tests__/md-task-parser.test.ts new file mode 100644 index 00000000..3f510ced --- /dev/null +++ b/src/core/tasks/__tests__/md-task-parser.test.ts @@ -0,0 +1,272 @@ +/** + * Tests for master-tasks.md parser + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { + parseMasterTasks, + serializeTaskRows, + updateTaskInFile, + addTaskToFile, + getNextTask, + type MasterTask, +} from '../md-task-parser.js'; + +const SAMPLE_MD = `# Master Tasks + +> Rules here + +## Active Tasks + +| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +| T01 | P0 | todo | @me | linear | Fix API key 401 | | Prod issue | +| T02 | P1 | active | @agent | local | Twitter connector | feature/twitter | Phase 1 | +| T03 | P1 | done | @agent | local | Feedback routes | feature/feedback merged | Phase 4 | +| T04 | P2 | blocked | @me | local | Entity resolution | | Blocked on T01 | +| T05 | P3 | todo | @defer | local | Reddit connector | | Low priority | + +## Done (archive monthly) +`; + +describe('parseMasterTasks', () => { + it('should parse all rows from a valid table', () => { + const tasks = parseMasterTasks(SAMPLE_MD); + expect(tasks).toHaveLength(5); + }); + + it('should parse fields correctly', () => { + const tasks = parseMasterTasks(SAMPLE_MD); + const t01 = tasks[0]; + expect(t01.id).toBe('T01'); + expect(t01.priority).toBe('P0'); + expect(t01.status).toBe('todo'); + expect(t01.owner).toBe('@me'); + expect(t01.sync).toBe('linear'); + expect(t01.task).toBe('Fix API key 401'); + expect(t01.branchPr).toBe(''); + expect(t01.notes).toBe('Prod issue'); + }); + + it('should handle empty table', () => { + const md = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +`; + expect(parseMasterTasks(md)).toEqual([]); + }); + + it('should skip malformed rows', () => { + const md = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +| T01 | P0 | todo | @me | local | Good row | | | +| bad row missing pipes +| T02 | INVALID | todo | @me | local | Bad priority | | | +`; + const tasks = parseMasterTasks(md); + expect(tasks).toHaveLength(1); + expect(tasks[0].id).toBe('T01'); + }); + + it('should default unknown status to todo', () => { + const md = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +| T01 | P0 | unknown_status | @me | local | Test | | | +`; + const tasks = parseMasterTasks(md); + expect(tasks[0].status).toBe('todo'); + }); +}); + +describe('serializeTaskRows', () => { + it('should produce valid pipe-delimited rows', () => { + const tasks: MasterTask[] = [ + { + id: 'T01', + priority: 'P0', + status: 'todo', + owner: '@me', + sync: 'local', + task: 'Do the thing', + branchPr: '', + notes: 'urgent', + }, + ]; + const result = serializeTaskRows(tasks); + expect(result).toBe( + '| T01 | P0 | todo | @me | local | Do the thing | | urgent |' + ); + }); + + it('should round-trip parse → serialize → parse', () => { + const original = parseMasterTasks(SAMPLE_MD); + const serialized = serializeTaskRows(original); + // Re-wrap with header for parsing + const rewrapped = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +${serialized} +`; + const reparsed = parseMasterTasks(rewrapped); + expect(reparsed).toEqual(original); + }); +}); + +describe('getNextTask', () => { + it('should return P0 before P1', () => { + const tasks = parseMasterTasks(SAMPLE_MD); + const next = getNextTask(tasks); + expect(next?.id).toBe('T01'); + expect(next?.priority).toBe('P0'); + }); + + it('should skip done and blocked tasks', () => { + const md = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +| T01 | P0 | done | @me | local | Done task | | | +| T02 | P0 | blocked | @me | local | Blocked task | | | +| T03 | P1 | todo | @me | local | Available task | | | +`; + const next = getNextTask(parseMasterTasks(md)); + expect(next?.id).toBe('T03'); + }); + + it('should return undefined when all tasks are done', () => { + const md = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +| T01 | P0 | done | @me | local | Done | | | +`; + expect(getNextTask(parseMasterTasks(md))).toBeUndefined(); + }); + + it('should prefer @agent over @defer at same priority', () => { + const md = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +| T01 | P1 | todo | @defer | local | Deferred | | | +| T02 | P1 | todo | @agent | local | Agent task | | | +`; + const next = getNextTask(parseMasterTasks(md)); + expect(next?.id).toBe('T02'); + }); + + it('should include active tasks (already started)', () => { + const md = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +| T01 | P0 | active | @me | local | In progress | | | +| T02 | P0 | todo | @me | local | Not started | | | +`; + const next = getNextTask(parseMasterTasks(md)); + expect(next?.id).toBe('T01'); + }); +}); + +describe('file operations', () => { + let tmpDir: string; + let filePath: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sm-tasks-')); + filePath = path.join(tmpDir, 'master-tasks.md'); + fs.writeFileSync(filePath, SAMPLE_MD); + }); + + afterEach(() => { + try { + fs.rmSync(tmpDir, { recursive: true }); + } catch { + // cleanup best-effort + } + }); + + describe('updateTaskInFile', () => { + it('should update status in place', () => { + updateTaskInFile(filePath, 'T01', { status: 'active' }); + const tasks = parseMasterTasks(fs.readFileSync(filePath, 'utf-8')); + expect(tasks.find((t) => t.id === 'T01')?.status).toBe('active'); + }); + + it('should update multiple fields', () => { + updateTaskInFile(filePath, 'T02', { + status: 'done', + branchPr: 'feature/twitter merged', + }); + const tasks = parseMasterTasks(fs.readFileSync(filePath, 'utf-8')); + const t02 = tasks.find((t) => t.id === 'T02'); + expect(t02?.status).toBe('done'); + expect(t02?.branchPr).toBe('feature/twitter merged'); + }); + + it('should throw on unknown task id', () => { + expect(() => + updateTaskInFile(filePath, 'T99', { status: 'done' }) + ).toThrow('Task T99 not found'); + }); + + it('should preserve other rows unchanged', () => { + const before = parseMasterTasks(fs.readFileSync(filePath, 'utf-8')); + updateTaskInFile(filePath, 'T01', { status: 'active' }); + const after = parseMasterTasks(fs.readFileSync(filePath, 'utf-8')); + + // T01 changed + expect(after.find((t) => t.id === 'T01')?.status).toBe('active'); + // Others unchanged + expect(after.find((t) => t.id === 'T02')).toEqual( + before.find((t) => t.id === 'T02') + ); + expect(after.find((t) => t.id === 'T04')).toEqual( + before.find((t) => t.id === 'T04') + ); + }); + }); + + describe('addTaskToFile', () => { + it('should auto-increment id', () => { + const id = addTaskToFile(filePath, { + priority: 'P1', + status: 'todo', + owner: '@me', + sync: 'local', + task: 'New task', + branchPr: '', + notes: '', + }); + expect(id).toBe('T06'); // T05 is last existing + }); + + it('should be parseable after adding', () => { + addTaskToFile(filePath, { + priority: 'P0', + status: 'todo', + owner: '@agent', + sync: 'linear', + task: 'Urgent new task', + branchPr: '', + notes: 'added programmatically', + }); + const tasks = parseMasterTasks(fs.readFileSync(filePath, 'utf-8')); + expect(tasks).toHaveLength(6); + const last = tasks[tasks.length - 1]; + expect(last.id).toBe('T06'); + expect(last.task).toBe('Urgent new task'); + expect(last.sync).toBe('linear'); + }); + + it('should work on empty table', () => { + const emptyMd = `| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| +`; + fs.writeFileSync(filePath, emptyMd); + const id = addTaskToFile(filePath, { + priority: 'P1', + status: 'todo', + owner: '@me', + sync: 'local', + task: 'First task', + branchPr: '', + notes: '', + }); + expect(id).toBe('T01'); + }); + }); +}); diff --git a/src/core/tasks/master-tasks-template.ts b/src/core/tasks/master-tasks-template.ts new file mode 100644 index 00000000..32bdfc57 --- /dev/null +++ b/src/core/tasks/master-tasks-template.ts @@ -0,0 +1,40 @@ +/** + * Default template for master-tasks.md scaffold. + */ + +export const MASTER_TASKS_TEMPLATE = `# Master Tasks + +> Single source of truth for what to build. Local-first, optionally syncs to Linear/GitHub. +> Powers \\\`/next\\\` task selection. Referenced by CLAUDE.md and AGENTS.md. + +## Rules + +1. **Local-first**: This file is canonical. Linear/GH are downstream mirrors, not sources. +2. **One owner**: Every task has exactly one owner. \\\`@me\\\` = you, \\\`@agent\\\` = dispatch to sub-agent, \\\`@defer\\\` = not assigned. +3. **Priority tiers**: \\\`P0\\\` now (blocking), \\\`P1\\\` this week, \\\`P2\\\` next sprint, \\\`P3\\\` someday. +4. **Status flow**: \\\`todo\\\` → \\\`active\\\` → \\\`done\\\` | \\\`blocked\\\` | \\\`cut\\\`. +5. **Sync targets**: \\\`local\\\` (stays here), \\\`linear\\\` (create/update Linear issue), \\\`gh\\\` (GitHub issue/PR). +6. **Agent /next reads P0 first, then P1**: Skip blocked, done, cut. Prefer @agent tasks unless @me explicitly set. +7. **Keep it scannable**: One line per task in the table. Details go in notes column or linked doc. +8. **Update on completion**: Mark done with date. Don't delete — move to Done section monthly. + +## Active Tasks + +| id | P | status | owner | sync | task | branch/PR | notes | +|----|---|--------|-------|------|------|-----------|-------| + +## Done (archive monthly) + +_Move completed tasks here at end of month._ + + +`; + +export const TASKS_CONFIG_TEMPLATE = { + linear: { team: '', project: '' }, + github: { repo: '' }, + defaultSync: 'local', + defaultOwner: '@me', +}; diff --git a/src/core/tasks/md-task-parser.ts b/src/core/tasks/md-task-parser.ts new file mode 100644 index 00000000..0bd45e91 --- /dev/null +++ b/src/core/tasks/md-task-parser.ts @@ -0,0 +1,232 @@ +/** + * Markdown Task Parser + * Parses and serializes master-tasks.md table format. + * Pure file I/O — no database dependency. + */ + +import { readFileSync, writeFileSync } from 'fs'; + +// ── Types ────────────────────────────────────────────────── + +export type TaskPriority = 'P0' | 'P1' | 'P2' | 'P3'; +export type TaskStatus = 'todo' | 'active' | 'done' | 'blocked' | 'cut'; +export type TaskSync = 'local' | 'linear' | 'gh'; + +export interface MasterTask { + id: string; + priority: TaskPriority; + status: TaskStatus; + owner: string; + sync: TaskSync; + task: string; + branchPr: string; + notes: string; +} + +// ── Constants ────────────────────────────────────────────── + +const HEADER_RE = /^\|\s*id\s*\|/i; +const SEPARATOR_RE = /^\|[\s-|]+\|$/; +const PRIORITIES: TaskPriority[] = ['P0', 'P1', 'P2', 'P3']; +const STATUSES: TaskStatus[] = ['todo', 'active', 'done', 'blocked', 'cut']; +const SYNCS: TaskSync[] = ['local', 'linear', 'gh']; + +// ── Parser ───────────────────────────────────────────────── + +/** + * Parse master-tasks.md content into typed task objects. + * Skips header row and separator row. Ignores malformed rows. + */ +export function parseMasterTasks(content: string): MasterTask[] { + const lines = content.split('\n'); + const tasks: MasterTask[] = []; + let inTable = false; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed.startsWith('|')) { + if (inTable) break; // table ended + continue; + } + + if (HEADER_RE.test(trimmed)) { + inTable = true; + continue; + } + if (SEPARATOR_RE.test(trimmed)) continue; + if (!inTable) continue; + + const cells = trimmed + .split('|') + .slice(1, -1) // drop empty first/last from leading/trailing pipes + .map((c) => c.trim()); + + if (cells.length < 8) continue; + + const [id, priority, status, owner, sync, task, branchPr, notes] = cells; + + if (!id || !PRIORITIES.includes(priority as TaskPriority)) continue; + + tasks.push({ + id, + priority: priority as TaskPriority, + status: STATUSES.includes(status as TaskStatus) + ? (status as TaskStatus) + : 'todo', + owner: owner || '@me', + sync: SYNCS.includes(sync as TaskSync) ? (sync as TaskSync) : 'local', + task: task || '', + branchPr: branchPr || '', + notes: notes || '', + }); + } + + return tasks; +} + +/** + * Serialize tasks back to markdown table rows (no header/rules — caller adds those). + */ +export function serializeTaskRows(tasks: MasterTask[]): string { + return tasks + .map( + (t) => + `| ${t.id} | ${t.priority} | ${t.status} | ${t.owner} | ${t.sync} | ${t.task} | ${t.branchPr} | ${t.notes} |` + ) + .join('\n'); +} + +// ── File Operations ──────────────────────────────────────── + +/** + * Update a task in master-tasks.md by id. Preserves all other content. + */ +export function updateTaskInFile( + filePath: string, + taskId: string, + updates: Partial> +): void { + const content = readFileSync(filePath, 'utf-8'); + const lines = content.split('\n'); + let found = false; + + const updated = lines.map((line) => { + const trimmed = line.trim(); + if ( + !trimmed.startsWith('|') || + HEADER_RE.test(trimmed) || + SEPARATOR_RE.test(trimmed) + ) { + return line; + } + + const cells = trimmed + .split('|') + .slice(1, -1) + .map((c) => c.trim()); + + if (cells.length < 8 || cells[0] !== taskId) return line; + + found = true; + const task: MasterTask = { + id: cells[0], + priority: (updates.priority ?? cells[1]) as TaskPriority, + status: (updates.status ?? cells[2]) as TaskStatus, + owner: updates.owner ?? cells[3], + sync: (updates.sync ?? cells[4]) as TaskSync, + task: updates.task ?? cells[5], + branchPr: updates.branchPr ?? cells[6], + notes: updates.notes ?? cells[7], + }; + + return `| ${task.id} | ${task.priority} | ${task.status} | ${task.owner} | ${task.sync} | ${task.task} | ${task.branchPr} | ${task.notes} |`; + }); + + if (!found) throw new Error(`Task ${taskId} not found in ${filePath}`); + writeFileSync(filePath, updated.join('\n'), 'utf-8'); +} + +/** + * Add a task to master-tasks.md. Auto-assigns next id (T01, T02...). + * Inserts before the "## Done" section or at end of active table. + */ +export function addTaskToFile( + filePath: string, + task: Omit +): string { + const content = readFileSync(filePath, 'utf-8'); + const existing = parseMasterTasks(content); + + // Auto-increment id + const maxNum = existing.reduce((max, t) => { + const n = parseInt(t.id.replace(/^T/, ''), 10); + return isNaN(n) ? max : Math.max(max, n); + }, 0); + const id = `T${String(maxNum + 1).padStart(2, '0')}`; + + const newRow = `| ${id} | ${task.priority} | ${task.status} | ${task.owner} | ${task.sync} | ${task.task} | ${task.branchPr} | ${task.notes} |`; + + // Find insertion point: after last table row in Active Tasks, before Done section + const lines = content.split('\n'); + let insertIdx = -1; + let inTable = false; + + for (let i = 0; i < lines.length; i++) { + const trimmed = lines[i].trim(); + + if (HEADER_RE.test(trimmed)) { + inTable = true; + continue; + } + if (inTable && trimmed.startsWith('|') && !SEPARATOR_RE.test(trimmed)) { + insertIdx = i; // track last data row + } + if (inTable && !trimmed.startsWith('|') && trimmed !== '') { + break; // left the table + } + } + + if (insertIdx === -1) { + // No data rows yet — insert after separator + for (let i = 0; i < lines.length; i++) { + if (SEPARATOR_RE.test(lines[i].trim())) { + insertIdx = i; + break; + } + } + } + + lines.splice(insertIdx + 1, 0, newRow); + writeFileSync(filePath, lines.join('\n'), 'utf-8'); + return id; +} + +/** + * Get the next task to work on. + * Priority: P0 > P1 > P2 > P3. Skip blocked/done/cut. Prefer @agent over @defer. + */ +export function getNextTask(tasks: MasterTask[]): MasterTask | undefined { + const actionable = tasks.filter( + (t) => t.status === 'todo' || t.status === 'active' + ); + + if (actionable.length === 0) return undefined; + + // Sort by priority (P0 first), then by owner preference (@agent > @me > @defer) + const ownerRank: Record = { + '@agent': 0, + '@me': 1, + '@defer': 2, + }; + + actionable.sort((a, b) => { + const pDiff = + PRIORITIES.indexOf(a.priority) - PRIORITIES.indexOf(b.priority); + if (pDiff !== 0) return pDiff; + const aRank = ownerRank[a.owner] ?? 1; + const bRank = ownerRank[b.owner] ?? 1; + return aRank - bRank; + }); + + return actionable[0]; +} diff --git a/src/daemon/daemon-config.ts b/src/daemon/daemon-config.ts index 1e846a6a..681f1ef8 100644 --- a/src/daemon/daemon-config.ts +++ b/src/daemon/daemon-config.ts @@ -61,6 +61,36 @@ export interface FileWatchConfig extends DaemonServiceConfig { debounceMs: number; } +export interface TelemetryServiceConfig extends DaemonServiceConfig { + maxSnapshots: number; // rolling history cap (default 90) +} + +export interface DesirePathConfig extends DaemonServiceConfig { + /** Min occurrences to be a pattern (default 3) */ + minFrequency: number; + /** Min distinct sessions for a pattern (default 2) */ + minSessions: number; + /** Max JSONL file size before rotation in bytes (default 10MB) */ + maxLogSizeBytes: number; + /** Days to retain action stream data (default 30) */ + retentionDays: number; + /** Max sequence length to detect (default 8) */ + maxSequenceLength: number; + /** Auto-promote skills above this confidence (0-1, default 0.8). Set to 1 to disable. */ + autoPromoteThreshold: number; + /** Min sessions required for auto-promotion (default 5) */ + autoPromoteMinSessions: number; + /** Directory to promote skills into */ + skillsDir?: string; +} + +export interface ResearchStreamConfig extends DaemonServiceConfig { + /** Keywords to filter signals by relevance */ + keywords: string[]; + /** Max signals to keep per scan cycle (default 50) */ + maxSignalsPerScan: number; +} + export interface DaemonConfig { version: string; context: ContextServiceConfig; @@ -69,6 +99,9 @@ export interface DaemonConfig { maintenance: MaintenanceServiceConfig; memory: MemoryServiceConfig; fileWatch: FileWatchConfig; + telemetry: TelemetryServiceConfig; + desirePaths: DesirePathConfig; + researchStream: ResearchStreamConfig; heartbeatInterval: number; // seconds inactivityTimeout: number; // minutes, 0 = disabled logLevel: 'debug' | 'info' | 'warn' | 'error'; @@ -116,6 +149,32 @@ export const DEFAULT_DAEMON_CONFIG: DaemonConfig = { ignore: ['node_modules', '.git', 'dist', 'build', '.stackmemory'], debounceMs: 2000, }, + telemetry: { + enabled: true, // opt-out via STACKMEMORY_TELEMETRY=0 + interval: 1440, // 24 hours + maxSnapshots: 90, // ~3 months of daily + }, + desirePaths: { + enabled: true, // opt-out via STACKMEMORY_DESIRE_PATHS=0 + interval: 360, // scan every 6 hours + minFrequency: 3, // 3+ occurrences to be a pattern + minSessions: 2, // across 2+ distinct sessions + maxLogSizeBytes: 10 * 1024 * 1024, // 10MB rotation + retentionDays: 30, + maxSequenceLength: 8, + autoPromoteThreshold: 0.8, + autoPromoteMinSessions: 5, + }, + researchStream: { + enabled: true, // opt-out via STACKMEMORY_RESEARCH_STREAM=0 + interval: 360, // every 6 hours + keywords: [ + 'agent', 'ai', 'llm', 'mcp', 'context', 'memory', + 'orchestration', 'skill', 'workflow', 'automation', + 'browser agent', 'coding assistant', + ], + maxSignalsPerScan: 50, + }, heartbeatInterval: 60, // 1 minute inactivityTimeout: 0, // Disabled by default logLevel: 'info', diff --git a/src/daemon/services/__tests__/desire-path-service.test.ts b/src/daemon/services/__tests__/desire-path-service.test.ts new file mode 100644 index 00000000..41557899 --- /dev/null +++ b/src/daemon/services/__tests__/desire-path-service.test.ts @@ -0,0 +1,269 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { + mkdtempSync, + rmSync, + writeFileSync, + readFileSync, + existsSync, + mkdirSync, +} from 'fs'; +import { join } from 'path'; +import { tmpdir, homedir } from 'os'; +import { + DaemonDesirePathService, + type DesirePathConfig, +} from '../desire-path-service.js'; + +// Override SM_DIR for tests by using the service's logAction method +// which writes to ~/.stackmemory/desire-paths/ — we test the public API + +describe('DaemonDesirePathService', () => { + let tmpDir: string; + let config: DesirePathConfig; + let logs: Array<{ level: string; msg: string; data?: unknown }>; + let onLog: (level: string, msg: string, data?: unknown) => void; + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'sm-dp-')); + logs = []; + onLog = (level, msg, data) => logs.push({ level, msg, data }); + config = { + enabled: true, + interval: 360, + minFrequency: 2, // lower for tests + minSessions: 2, + maxLogSizeBytes: 10 * 1024 * 1024, + retentionDays: 30, + maxSequenceLength: 6, + }; + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + describe('parseHookEvent', () => { + it('sanitizes file paths into glob patterns', () => { + const entry = DaemonDesirePathService.parseHookEvent( + 'Read', + '/src/runtime/agent-runner.js', + 'sess-1' + ); + expect(entry.tool).toBe('Read'); + expect(entry.target).toBe('/src/runtime/*.js'); + expect(entry.sid).toBe('sess-1'); + }); + + it('sanitizes bash commands to command + first arg', () => { + const entry = DaemonDesirePathService.parseHookEvent( + 'Bash', + 'npx jest src/runtime --no-coverage', + 'sess-1' + ); + expect(entry.tool).toBe('Bash'); + expect(entry.target).toBe('npx jest'); + }); + + it('handles empty args', () => { + const entry = DaemonDesirePathService.parseHookEvent( + 'Grep', + '', + 'sess-1' + ); + expect(entry.target).toBe('*'); + }); + + it('truncates long args', () => { + const longArg = 'a'.repeat(100); + const entry = DaemonDesirePathService.parseHookEvent( + 'Glob', + longArg, + 'sess-1' + ); + expect(entry.target.length).toBeLessThanOrEqual(50); + }); + }); + + describe('pattern detection', () => { + it('detects repeated sequences across sessions', () => { + const service = new DaemonDesirePathService(config, onLog); + + // Simulate action stream directly by writing JSONL + const dpDir = join(homedir(), '.stackmemory', 'desire-paths'); + mkdirSync(dpDir, { recursive: true }); + const streamFile = join(dpDir, 'action-stream.jsonl'); + + // Session 1: Read → Edit → Bash + const actions = [ + { + ts: '2026-05-09T10:00:00Z', + sid: 'sess-1', + tool: 'Read', + target: 'src/runtime/*.js', + }, + { + ts: '2026-05-09T10:00:01Z', + sid: 'sess-1', + tool: 'Edit', + target: 'src/runtime/*.js', + }, + { + ts: '2026-05-09T10:00:02Z', + sid: 'sess-1', + tool: 'Bash', + target: 'npx jest', + }, + // Session 2: same pattern + { + ts: '2026-05-09T11:00:00Z', + sid: 'sess-2', + tool: 'Read', + target: 'src/runtime/*.js', + }, + { + ts: '2026-05-09T11:00:01Z', + sid: 'sess-2', + tool: 'Edit', + target: 'src/runtime/*.js', + }, + { + ts: '2026-05-09T11:00:02Z', + sid: 'sess-2', + tool: 'Bash', + target: 'npx jest', + }, + // Session 3: same pattern again + { + ts: '2026-05-09T12:00:00Z', + sid: 'sess-3', + tool: 'Read', + target: 'src/runtime/*.js', + }, + { + ts: '2026-05-09T12:00:01Z', + sid: 'sess-3', + tool: 'Edit', + target: 'src/runtime/*.js', + }, + { + ts: '2026-05-09T12:00:02Z', + sid: 'sess-3', + tool: 'Bash', + target: 'npx jest', + }, + ]; + + writeFileSync( + streamFile, + actions.map((a) => JSON.stringify(a)).join('\n') + '\n' + ); + + const patterns = service.detectPatterns(); + + expect(patterns.length).toBeGreaterThan(0); + // Should find the Read→Edit→Bash sequence + const fullPattern = patterns.find((p) => p.sequence.length === 3); + expect(fullPattern).toBeDefined(); + expect(fullPattern!.frequency).toBeGreaterThanOrEqual(3); + expect(fullPattern!.sessions).toBeGreaterThanOrEqual(2); + + // Cleanup + rmSync(streamFile, { force: true }); + }); + + it('returns empty for insufficient data', () => { + const service = new DaemonDesirePathService(config, onLog); + const patterns = service.detectPatterns(); + // May return empty or patterns from previous test — just verify no crash + expect(Array.isArray(patterns)).toBe(true); + }); + }); + + describe('skill suggestion', () => { + it('generates skill markdown from patterns', () => { + const service = new DaemonDesirePathService(config, onLog); + + const patterns = [ + { + id: 'test-1', + sequence: [ + 'Read:src/runtime/*.js', + 'Edit:src/runtime/*.js', + 'Bash:npx jest', + ], + frequency: 5, + sessions: 3, + avg_steps: 3, + first_seen: '2026-05-09T10:00:00Z', + last_seen: '2026-05-09T12:00:00Z', + score: 15, + }, + ]; + + const suggestions = service.generateSuggestions(patterns); + + expect(suggestions.length).toBe(1); + expect(suggestions[0].name).toContain('auto-'); + expect(suggestions[0].steps.length).toBe(3); + expect(suggestions[0].confidence).toBeGreaterThan(0); + expect(suggestions[0].pattern_id).toBe('test-1'); + + // Check suggestion file was written + const suggestionsDir = join( + homedir(), + '.stackmemory', + 'desire-paths', + 'suggestions' + ); + const files = require('fs') + .readdirSync(suggestionsDir) + .filter((f: string) => f.endsWith('.skill.md')); + expect(files.length).toBeGreaterThan(0); + + // Read and verify markdown structure + const content = readFileSync(join(suggestionsDir, files[0]), 'utf-8'); + expect(content).toContain('---'); + expect(content).toContain('status: suggested'); + expect(content).toContain('Auto-Detected Workflow'); + }); + }); + + describe('opt-out', () => { + it('respects environment variable', () => { + const orig = process.env.STACKMEMORY_DESIRE_PATHS; + process.env.STACKMEMORY_DESIRE_PATHS = '0'; + + const service = new DaemonDesirePathService(config, onLog); + service.logAction({ + ts: new Date().toISOString(), + sid: 'test', + tool: 'Read', + target: 'foo', + }); + + // Should not increment counter + expect(service.getState().actionsLogged).toBe(0); + + if (orig === undefined) delete process.env.STACKMEMORY_DESIRE_PATHS; + else process.env.STACKMEMORY_DESIRE_PATHS = orig; + }); + + it('respects config.enabled = false', () => { + const disabledConfig = { ...config, enabled: false }; + const service = new DaemonDesirePathService(disabledConfig, onLog); + service.start(); + expect(logs.some((l) => l.msg.includes('disabled'))).toBe(true); + }); + }); + + describe('getState', () => { + it('returns current state', () => { + const service = new DaemonDesirePathService(config, onLog); + const state = service.getState(); + expect(state.actionsLogged).toBe(0); + expect(state.patternsDetected).toBe(0); + expect(state.suggestionsGenerated).toBe(0); + expect(state.errors).toEqual([]); + }); + }); +}); diff --git a/src/daemon/services/desire-path-service.ts b/src/daemon/services/desire-path-service.ts new file mode 100644 index 00000000..2c472c9e --- /dev/null +++ b/src/daemon/services/desire-path-service.ts @@ -0,0 +1,718 @@ +/** + * Desire-Path Service — logs tool calls, detects repeated workflows, + * and auto-suggests skills to replace manual work. + * + * Three components: + * 1. ActionStreamLogger — captures tool:target pairs from hook events + * 2. PatternDetector — finds repeated sequences across sessions + * 3. SkillSuggester — generates skill frontmatter from top patterns + * + * Storage: ~/.stackmemory/desire-paths/action-stream.jsonl (append-only) + * Patterns: ~/.stackmemory/desire-paths/patterns.json + * Suggestions: ~/.stackmemory/desire-paths/suggestions/ + * + * Opt out: STACKMEMORY_DESIRE_PATHS=0 or desirePaths.enabled: false + */ + +import { + existsSync, + readFileSync, + writeFileSync, + appendFileSync, + mkdirSync, + readdirSync, + statSync, + renameSync, +} from 'fs'; +import { join, basename, dirname, extname } from 'path'; +import { homedir } from 'os'; +import { randomUUID } from 'crypto'; +import type { DaemonServiceConfig } from '../daemon-config.js'; + +// ─── Types ──────────────────────────────────────────────────── + +export interface DesirePathConfig extends DaemonServiceConfig { + /** Min occurrences to be a pattern (default 3) */ + minFrequency: number; + /** Min distinct sessions for a pattern (default 2) */ + minSessions: number; + /** Max JSONL file size before rotation in bytes (default 10MB) */ + maxLogSizeBytes: number; + /** Days to retain action stream data (default 30) */ + retentionDays: number; + /** Max sequence length to detect (default 8) */ + maxSequenceLength: number; + /** Auto-promote skills above this confidence (0-1, default 0.8). Set to 1 to disable. */ + autoPromoteThreshold: number; + /** Min sessions required for auto-promotion (default 5) */ + autoPromoteMinSessions: number; + /** Directory to promote skills into (default: cwd/.claude/skills/knowledge or skills/) */ + skillsDir?: string; +} + +export interface ActionEntry { + ts: string; // ISO timestamp + sid: string; // session ID + tool: string; // tool name (Read, Edit, Bash, Grep, etc.) + target: string; // sanitized first arg (file path pattern, command prefix) + dur?: number; // duration ms +} + +export interface DetectedPattern { + id: string; + sequence: string[]; // e.g. ["Read:src/runtime/*.js", "Edit:src/runtime/*.js", "Bash:npx jest*"] + frequency: number; // how many times observed + sessions: number; // across how many distinct sessions + avg_steps: number; // average total steps in sessions containing this pattern + first_seen: string; // ISO + last_seen: string; // ISO + score: number; // frequency × sessions (simple ranking) +} + +export interface SkillSuggestion { + name: string; + description: string; + inputs: Array<{ name: string; type: string; required: boolean; description: string }>; + outputs: Array<{ name: string; type: string; description: string }>; + steps: string[]; + pattern_id: string; + confidence: number; // 0-1 based on pattern strength + generated_at: string; +} + +export interface DesirePathState { + lastScanTime: number; + actionsLogged: number; + patternsDetected: number; + suggestionsGenerated: number; + skillsAutoPromoted: number; + errors: string[]; +} + +// ─── Constants ──────────────────────────────────────────────── + +const SM_DIR = join(homedir(), '.stackmemory'); +const DP_DIR = join(SM_DIR, 'desire-paths'); +const STREAM_FILE = join(DP_DIR, 'action-stream.jsonl'); +const PATTERNS_FILE = join(DP_DIR, 'patterns.json'); +const SUGGESTIONS_DIR = join(DP_DIR, 'suggestions'); + +const MAX_LOG_SIZE = 10 * 1024 * 1024; // 10MB +const TOOL_TARGET_SENSITIVE = new Set(['Bash']); // tools where target may contain secrets + +// ─── Utilities ──────────────────────────────────────────────── + +/** Sanitize a file path into a glob pattern (strip specific names, keep structure). */ +function sanitizePath(filePath: string): string { + if (!filePath) return '*'; + // Keep directory structure, replace specific filenames with wildcards + const dir = dirname(filePath); + const ext = extname(filePath); + if (ext) { + return `${dir}/*${ext}`; + } + return `${dir}/*`; +} + +/** Sanitize a bash command to just the command name + first arg pattern. */ +function sanitizeCommand(cmd: string): string { + if (!cmd) return '*'; + const parts = cmd.trim().split(/\s+/); + const command = parts[0]; + // Keep first meaningful arg (skip flags) + const firstArg = parts.slice(1).find(p => !p.startsWith('-')); + if (firstArg) { + return `${command} ${firstArg.length > 30 ? firstArg.slice(0, 30) + '*' : firstArg}`; + } + return command; +} + +/** Build a tool:target key from an action entry. */ +function actionKey(entry: ActionEntry): string { + return `${entry.tool}:${entry.target}`; +} + +/** Hash a sequence for dedup. Uses pipe delimiter (safe — not in tool:target keys). */ +function sequenceHash(seq: string[]): string { + return seq.join('|'); +} + +// ─── Service ────────────────────────────────────────────────── + +export class DaemonDesirePathService { + private config: DesirePathConfig; + private state: DesirePathState; + private scanTimeout?: NodeJS.Timeout; + private isRunning = false; + private onLog: (level: string, message: string, data?: unknown) => void; + private lastActivityTime = 0; // last time an action was logged + private consecutiveIdleScans = 0; // scans with no new actions + + constructor( + config: DesirePathConfig, + onLog: (level: string, message: string, data?: unknown) => void + ) { + this.config = config; + this.onLog = onLog; + this.state = { + lastScanTime: 0, + actionsLogged: 0, + patternsDetected: 0, + suggestionsGenerated: 0, + skillsAutoPromoted: 0, + errors: [], + }; + } + + private isOptedOut(): boolean { + if ( + process.env.STACKMEMORY_DESIRE_PATHS === '0' || + process.env.STACKMEMORY_DESIRE_PATHS === 'false' + ) { + return true; + } + return !this.config.enabled; + } + + // ─── 1. Action Stream Logger ───────────────────────────── + + /** Append a tool call to the action stream. Called from hook events. */ + logAction(entry: ActionEntry): void { + if (this.isOptedOut()) return; + + try { + mkdirSync(DP_DIR, { recursive: true }); + + // Rotate if too large + if (existsSync(STREAM_FILE)) { + const stat = statSync(STREAM_FILE); + if (stat.size > (this.config.maxLogSizeBytes || MAX_LOG_SIZE)) { + const rotated = `${STREAM_FILE}.${Date.now()}.bak`; + renameSync(STREAM_FILE, rotated); + this.onLog('INFO', 'Action stream rotated', { size: stat.size }); + } + } + + appendFileSync(STREAM_FILE, JSON.stringify(entry) + '\n', 'utf-8'); + this.state.actionsLogged++; + this.lastActivityTime = Date.now(); + } catch (err) { + this.addError(String(err)); + } + } + + /** Parse a hook event into an ActionEntry. */ + static parseHookEvent(toolName: string, firstArg: string, sessionId: string, durationMs?: number): ActionEntry { + let target: string; + + if (TOOL_TARGET_SENSITIVE.has(toolName)) { + target = sanitizeCommand(firstArg); + } else if (firstArg && (firstArg.includes('/') || firstArg.includes('\\'))) { + target = sanitizePath(firstArg); + } else { + target = firstArg ? firstArg.slice(0, 50) : '*'; + } + + return { + ts: new Date().toISOString(), + sid: sessionId, + tool: toolName, + target, + dur: durationMs, + }; + } + + // ─── 2. Pattern Detector ────────────────────────────────── + + /** Scan the action stream for repeated sequences. */ + detectPatterns(): DetectedPattern[] { + if (!existsSync(STREAM_FILE)) return []; + + // Load all entries + let entries: ActionEntry[]; + try { + const lines = readFileSync(STREAM_FILE, 'utf-8').trim().split('\n'); + entries = lines + .map(line => { try { return JSON.parse(line); } catch { return null; } }) + .filter(Boolean) as ActionEntry[]; + } catch { + return []; + } + + // Cap to last 10K entries for performance + if (entries.length > 10000) entries = entries.slice(-10000); + if (entries.length < 3) return []; + + // Group by session + const sessions = new Map(); + for (const entry of entries) { + const sid = entry.sid || 'unknown'; + if (!sessions.has(sid)) sessions.set(sid, []); + sessions.get(sid)!.push(entry); + } + + // Extract subsequences from each session + const maxLen = this.config.maxSequenceLength || 8; + const minLen = 2; + const sequenceCounts = new Map; firstSeen: string; lastSeen: string }>(); + + for (const [sid, actions] of sessions) { + const keys = actions.map(actionKey); + + // Sliding window: extract all subsequences of length minLen..maxLen + for (let len = minLen; len <= Math.min(maxLen, keys.length); len++) { + for (let i = 0; i <= keys.length - len; i++) { + const subseq = keys.slice(i, i + len); + const hash = sequenceHash(subseq); + + if (!sequenceCounts.has(hash)) { + sequenceCounts.set(hash, { + count: 0, + sessions: new Set(), + firstSeen: actions[i].ts, + lastSeen: actions[i + len - 1].ts, + }); + } + + const entry = sequenceCounts.get(hash)!; + entry.count++; + entry.sessions.add(sid); + if (actions[i + len - 1].ts > entry.lastSeen) { + entry.lastSeen = actions[i + len - 1].ts; + } + } + } + } + + // Filter: min frequency and min sessions + const minFreq = this.config.minFrequency || 3; + const minSess = this.config.minSessions || 2; + const patterns: DetectedPattern[] = []; + + for (const [hash, data] of sequenceCounts) { + if (data.count >= minFreq && data.sessions.size >= minSess) { + const sequence = hash.split('|'); + patterns.push({ + id: randomUUID().slice(0, 8), + sequence, + frequency: data.count, + sessions: data.sessions.size, + avg_steps: sequence.length, + first_seen: data.firstSeen, + last_seen: data.lastSeen, + score: data.count * data.sessions.size, + }); + } + } + + // Sort by score descending, deduplicate (prefer longer sequences) + patterns.sort((a, b) => b.score - a.score); + + // Remove subsequences of higher-scored patterns + const filtered: DetectedPattern[] = []; + const seenHashes = new Set(); + + for (const pattern of patterns) { + const hash = sequenceHash(pattern.sequence); + // Check if this is a subsequence of an already-accepted pattern + let isSubseq = false; + for (const accepted of filtered) { + const acceptedHash = sequenceHash(accepted.sequence); + if (acceptedHash.includes(hash) && acceptedHash !== hash) { + isSubseq = true; + break; + } + } + if (!isSubseq && !seenHashes.has(hash)) { + filtered.push(pattern); + seenHashes.add(hash); + } + } + + // Keep top 20 + const topPatterns = filtered.slice(0, 20); + this.state.patternsDetected = topPatterns.length; + + // Persist + try { + writeFileSync(PATTERNS_FILE, JSON.stringify({ patterns: topPatterns, updated_at: new Date().toISOString() }, null, 2)); + } catch (err) { + this.addError(String(err)); + } + + return topPatterns; + } + + /** Load previously detected patterns. */ + loadPatterns(): DetectedPattern[] { + try { + if (!existsSync(PATTERNS_FILE)) return []; + const data = JSON.parse(readFileSync(PATTERNS_FILE, 'utf-8')); + return data.patterns || []; + } catch { + return []; + } + } + + // ─── 3. Skill Suggester ─────────────────────────────────── + + /** Generate skill suggestions from detected patterns. */ + generateSuggestions(patterns?: DetectedPattern[]): SkillSuggestion[] { + const pats = patterns || this.loadPatterns(); + if (pats.length === 0) return []; + + mkdirSync(SUGGESTIONS_DIR, { recursive: true }); + const suggestions: SkillSuggestion[] = []; + + for (const pattern of pats.slice(0, 10)) { + const suggestion = this.patternToSuggestion(pattern); + if (!suggestion) continue; + + suggestions.push(suggestion); + + // Write as a skill.md file + const fileName = `${suggestion.name}.skill.md`; + const content = this.renderSkillMarkdown(suggestion); + try { + writeFileSync(join(SUGGESTIONS_DIR, fileName), content, 'utf-8'); + } catch (err) { + this.addError(String(err)); + } + } + + this.state.suggestionsGenerated = suggestions.length; + + // Auto-promote high-confidence suggestions + this.autoPromote(suggestions); + + return suggestions; + } + + // ─── 4. Auto-Promotion ──────────────────────────────────── + + /** + * Auto-promote skills above confidence threshold. + * Copies from suggestions/ to the project's skills/ directory. + * Only promotes if: confidence ≥ threshold AND sessions ≥ minSessions. + */ + private autoPromote(suggestions: SkillSuggestion[]): void { + const threshold = this.config.autoPromoteThreshold ?? 0.8; + const minSessions = this.config.autoPromoteMinSessions ?? 5; + + if (threshold >= 1) return; // disabled + + // Find target skills directory + const skillsDir = this.config.skillsDir || this.findSkillsDir(); + if (!skillsDir) return; + + const patterns = this.loadPatterns(); + + for (const suggestion of suggestions) { + if (suggestion.confidence < threshold) continue; + + // Check session count from the pattern + const pattern = patterns.find(p => p.id === suggestion.pattern_id); + if (!pattern || pattern.sessions < minSessions) continue; + + // Check if already promoted + const destFile = join(skillsDir, `${suggestion.name}.skill.md`); + if (existsSync(destFile)) continue; + + // Promote + const srcFile = join(SUGGESTIONS_DIR, `${suggestion.name}.skill.md`); + if (!existsSync(srcFile)) continue; + + try { + mkdirSync(skillsDir, { recursive: true }); + let content = readFileSync(srcFile, 'utf-8'); + content = content.replace('status: suggested', 'status: auto-promoted'); + writeFileSync(destFile, content, 'utf-8'); + + this.state.skillsAutoPromoted++; + this.onLog('INFO', `Skill auto-promoted: ${suggestion.name}`, { + confidence: suggestion.confidence, + sessions: pattern.sessions, + frequency: pattern.frequency, + dest: destFile, + }); + } catch (err) { + this.addError(`Auto-promote failed for ${suggestion.name}: ${String(err)}`); + } + } + } + + /** Find the best skills directory for auto-promotion. */ + private findSkillsDir(): string | null { + const cwd = process.cwd(); + + // Priority: .claude/skills/knowledge > skills/ > null + const candidates = [ + join(cwd, '.claude', 'skills', 'knowledge'), + join(cwd, 'skills'), + ]; + + for (const dir of candidates) { + if (existsSync(dir)) return dir; + } + + // Create .claude/skills/knowledge if .claude exists + const claudeDir = join(cwd, '.claude'); + if (existsSync(claudeDir)) { + const target = join(claudeDir, 'skills', 'knowledge'); + try { + mkdirSync(target, { recursive: true }); + return target; + } catch { + return null; + } + } + + return null; + } + + private patternToSuggestion(pattern: DetectedPattern): SkillSuggestion | null { + if (pattern.sequence.length < 2) return null; + + // Extract dominant tools and targets + const tools = pattern.sequence.map(s => { + const [tool, target] = s.split(':', 2); + return { tool, target: target || '*' }; + }); + + // Derive name from tools + dominant target directory + const toolNames = [...new Set(tools.map(t => t.tool.toLowerCase()))]; + const targets = tools.map(t => t.target).filter(t => t !== '*'); + const dominantDir = targets.length > 0 + ? targets[0].split('/').slice(0, 3).join('-').replace(/[^a-zA-Z0-9-]/g, '') + : ''; + const nameSuffix = dominantDir ? `-${dominantDir}` : ''; + const name = `auto-${toolNames.join('-')}${nameSuffix}`; + + // Infer inputs from first step's target + const firstTarget = tools[0].target; + const inputs: SkillSuggestion['inputs'] = []; + if (firstTarget && firstTarget !== '*') { + inputs.push({ + name: 'target_path', + type: 'string', + required: true, + description: `Path pattern (observed: ${firstTarget})`, + }); + } + + // Infer outputs from last step + const lastTool = tools[tools.length - 1]; + const outputs: SkillSuggestion['outputs'] = [{ + name: 'result', + type: 'string', + description: `Output from ${lastTool.tool}`, + }]; + + // Build steps + const steps = tools.map((t, i) => `${i + 1}. ${t.tool}: ${t.target}`); + + const confidence = Math.min(1, (pattern.score / 20)); + + return { + name, + description: `Auto-detected workflow: ${toolNames.join(' → ')} (seen ${pattern.frequency}× across ${pattern.sessions} sessions)`, + inputs, + outputs, + steps, + pattern_id: pattern.id, + confidence, + generated_at: new Date().toISOString(), + }; + } + + private renderSkillMarkdown(suggestion: SkillSuggestion): string { + const inputsYaml = suggestion.inputs.length > 0 + ? suggestion.inputs.map(i => + ` - name: ${i.name}\n type: ${i.type}\n required: ${i.required}\n description: "${i.description}"` + ).join('\n') + : ''; + + const outputsYaml = suggestion.outputs.map(o => + ` - name: ${o.name}\n type: ${o.type}\n description: "${o.description}"` + ).join('\n'); + + return [ + '---', + `name: ${suggestion.name}`, + `description: "${suggestion.description}"`, + `status: suggested`, + `pattern_id: ${suggestion.pattern_id}`, + `confidence: ${suggestion.confidence.toFixed(2)}`, + `generated_at: ${suggestion.generated_at}`, + suggestion.inputs.length > 0 ? `inputs:\n${inputsYaml}` : '', + `outputs:\n${outputsYaml}`, + '---', + '', + `# ${suggestion.name}`, + '', + '## Auto-Detected Workflow', + '', + `> This skill was auto-generated from ${suggestion.pattern_id} detected patterns.`, + '> Review and edit before promoting to an active skill.', + '', + '## Steps', + '', + ...suggestion.steps, + '', + '## Notes', + '', + '- Edit this file to refine the workflow', + '- Move to your `skills/` directory to activate', + `- Confidence: ${(suggestion.confidence * 100).toFixed(0)}%`, + ].filter(line => line !== '').join('\n') + '\n'; + } + + // ─── Lifecycle (adaptive backoff) ────────────────────────── + // + // Active sessions: scan every 1 hour + // Idle (no actions): backoff 1h → 2h → 4h → 8h → 12h (cap) + // New activity resets to 1h immediately + + private static readonly BASE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour + private static readonly MAX_INTERVAL_MS = 12 * 60 * 60 * 1000; // 12 hours + private static readonly IDLE_THRESHOLD_MS = 30 * 60 * 1000; // 30 min = idle + + private getNextInterval(): number { + const now = Date.now(); + const timeSinceActivity = now - this.lastActivityTime; + + // If recent activity, scan hourly + if (this.lastActivityTime > 0 && timeSinceActivity < DaemonDesirePathService.IDLE_THRESHOLD_MS) { + this.consecutiveIdleScans = 0; + return DaemonDesirePathService.BASE_INTERVAL_MS; + } + + // Backoff: 1h × 2^idle_scans, capped at 12h + const backoff = DaemonDesirePathService.BASE_INTERVAL_MS * Math.pow(2, this.consecutiveIdleScans); + return Math.min(backoff, DaemonDesirePathService.MAX_INTERVAL_MS); + } + + start(): void { + if (this.isRunning || this.isOptedOut()) { + if (this.isOptedOut()) { + this.onLog('INFO', 'Desire-path detection disabled'); + } + return; + } + + this.isRunning = true; + mkdirSync(DP_DIR, { recursive: true }); + + this.onLog('INFO', 'Desire-path service started (adaptive backoff: 1h active, up to 12h idle)'); + + // First scan after 2 minutes + this.scanTimeout = setTimeout(() => { + if (!this.isRunning) return; + this.runScanAndScheduleNext(); + }, 120_000); + + if (this.scanTimeout.unref) this.scanTimeout.unref(); + } + + stop(): void { + if (this.scanTimeout) { + clearTimeout(this.scanTimeout); + this.scanTimeout = undefined; + } + this.isRunning = false; + } + + private runScanAndScheduleNext(): void { + this.runScan(); + + if (!this.isRunning) return; + + const nextMs = this.getNextInterval(); + this.onLog('DEBUG', 'Next scan scheduled', { + next_min: Math.round(nextMs / 60_000), + idle_scans: this.consecutiveIdleScans, + }); + + this.scanTimeout = setTimeout(() => { + if (!this.isRunning) return; + this.runScanAndScheduleNext(); + }, nextMs); + + if (this.scanTimeout.unref) this.scanTimeout.unref(); + } + + private runScan(): void { + const prevActionsLogged = this.state.actionsLogged; + + try { + const patterns = this.detectPatterns(); + if (patterns.length > 0) { + const suggestions = this.generateSuggestions(patterns); + this.onLog('INFO', 'Desire-path scan complete', { + patterns: patterns.length, + suggestions: suggestions.length, + topPattern: patterns[0] ? sequenceHash(patterns[0].sequence) : 'none', + interval_min: Math.round(this.getNextInterval() / 60_000), + }); + } + this.state.lastScanTime = Date.now(); + + // Track idle scans (no new actions since last scan) + if (this.state.actionsLogged === prevActionsLogged) { + this.consecutiveIdleScans++; + } else { + this.consecutiveIdleScans = 0; + } + } catch (err) { + this.addError(String(err)); + this.onLog('ERROR', 'Desire-path scan failed', { error: String(err) }); + } + } + + private addError(err: string): void { + this.state.errors.push(err); + if (this.state.errors.length > 10) { + this.state.errors = this.state.errors.slice(-10); + } + } + + getState(): DesirePathState { + return { ...this.state }; + } + + /** Get current suggestions for CLI/MCP consumption. */ + getSuggestions(): SkillSuggestion[] { + try { + if (!existsSync(SUGGESTIONS_DIR)) return []; + const files = readdirSync(SUGGESTIONS_DIR).filter(f => f.endsWith('.skill.md')); + return files.map(f => { + const content = readFileSync(join(SUGGESTIONS_DIR, f), 'utf-8'); + const match = content.match(/^---\n([\s\S]*?)\n---/); + if (!match) return null; + try { + // Parse frontmatter minimally + const lines = match[1].split('\n'); + const meta: Record = {}; + for (const line of lines) { + const kv = line.match(/^(\w[\w_-]*):\s*(.*)/); + if (kv) meta[kv[1]] = kv[2].trim().replace(/^["']|["']$/g, ''); + } + return { + name: meta.name || basename(f, '.skill.md'), + description: meta.description || '', + pattern_id: meta.pattern_id || '', + confidence: parseFloat(meta.confidence || '0'), + generated_at: meta.generated_at || '', + inputs: [], + outputs: [], + steps: [], + } as SkillSuggestion; + } catch { + return null; + } + }).filter(Boolean) as SkillSuggestion[]; + } catch { + return []; + } + } +} diff --git a/src/daemon/services/research-stream-service.ts b/src/daemon/services/research-stream-service.ts new file mode 100644 index 00000000..bdde0c48 --- /dev/null +++ b/src/daemon/services/research-stream-service.ts @@ -0,0 +1,462 @@ +/** + * Research Stream Service — scans external market signals and feeds them + * into the desire-path ecosystem for competitive awareness. + * + * Sources (no API keys required): + * 1. Hacker News front page (Firebase API) + * 2. GitHub trending repos (Search API) + * 3. Product Hunt RSS (skipped if unavailable) + * + * Storage: + * ~/.stackmemory/desire-paths/research-stream.jsonl (append-only) + * ~/.stackmemory/desire-paths/research-digest.json (weekly top-N) + * + * Opt out: STACKMEMORY_RESEARCH_STREAM=0 or researchStream.enabled: false + */ + +import { + existsSync, + readFileSync, + writeFileSync, + appendFileSync, + mkdirSync, +} from 'fs'; +import { join } from 'path'; +import { homedir } from 'os'; +import type { DaemonServiceConfig } from '../daemon-config.js'; + +// ─── Types ──────────────────────────────────────────────────── + +export interface ResearchStreamConfig extends DaemonServiceConfig { + /** Keywords to filter signals by relevance */ + keywords: string[]; + /** Max signals to keep per scan cycle */ + maxSignalsPerScan: number; +} + +export interface ResearchSignal { + ts: string; + source: 'hackernews' | 'github' | 'producthunt'; + type: 'trending' | 'new_repo' | 'launch'; + title: string; + url: string; + score: number; + keywords_matched: string[]; + relevance: number; +} + +export interface ResearchDigest { + week: string; + signals: ResearchSignal[]; + themes: string[]; + generated_at: string; +} + +export interface ResearchStreamState { + lastScanTime: number; + signalsCollected: number; + digestsGenerated: number; + errors: string[]; +} + +// ─── Constants ──────────────────────────────────────────────── + +const SM_DIR = join(homedir(), '.stackmemory'); +const DP_DIR = join(SM_DIR, 'desire-paths'); +const STREAM_FILE = join(DP_DIR, 'research-stream.jsonl'); +const DIGEST_FILE = join(DP_DIR, 'research-digest.json'); + +const HN_TOP_URL = 'https://hacker-news.firebaseio.com/v0/topstories.json'; +const HN_ITEM_URL = 'https://hacker-news.firebaseio.com/v0/item'; +const GH_SEARCH_URL = 'https://api.github.com/search/repositories'; + +const RATE_LIMIT_MS = 1100; // 1.1s between requests (safe for GitHub) + +// ─── Utilities ──────────────────────────────────────────────── + +/** Sleep for ms. */ +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +/** Calculate ISO week string (e.g. "2026-W19"). */ +function isoWeek(date: Date): string { + const d = new Date(date.getTime()); + d.setHours(0, 0, 0, 0); + d.setDate(d.getDate() + 3 - ((d.getDay() + 6) % 7)); + const week1 = new Date(d.getFullYear(), 0, 4); + const weekNum = 1 + Math.round(((d.getTime() - week1.getTime()) / 86400000 - 3 + ((week1.getDay() + 6) % 7)) / 7); + return `${d.getFullYear()}-W${String(weekNum).padStart(2, '0')}`; +} + +/** Score relevance of a title against keyword list. Returns 0-1. */ +function scoreRelevance(title: string, keywords: string[]): { score: number; matched: string[] } { + const lower = title.toLowerCase(); + const matched: string[] = []; + + for (const kw of keywords) { + if (lower.includes(kw.toLowerCase())) { + matched.push(kw); + } + } + + if (matched.length === 0) return { score: 0, matched: [] }; + + // Base score from match count, diminishing returns + const score = Math.min(1, 0.3 + (matched.length * 0.2)); + return { score, matched }; +} + +/** Safe fetch with timeout. Returns null on any error. */ +async function safeFetch(url: string, timeoutMs = 10_000): Promise { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': 'StackMemory-ResearchStream/1.0' }, + }); + clearTimeout(timer); + if (!res.ok) return null; + return await res.json(); + } catch { + return null; + } +} + +// ─── Service ────────────────────────────────────────────────── + +export class ResearchStreamService { + private config: ResearchStreamConfig; + private state: ResearchStreamState; + private intervalId?: NodeJS.Timeout; + private isRunning = false; + private onLog: (level: string, message: string, data?: unknown) => void; + + constructor( + config: ResearchStreamConfig, + onLog: (level: string, message: string, data?: unknown) => void + ) { + this.config = config; + this.onLog = onLog; + this.state = { + lastScanTime: 0, + signalsCollected: 0, + digestsGenerated: 0, + errors: [], + }; + } + + private isOptedOut(): boolean { + if ( + process.env.STACKMEMORY_RESEARCH_STREAM === '0' || + process.env.STACKMEMORY_RESEARCH_STREAM === 'false' + ) { + return true; + } + return !this.config.enabled; + } + + // ─── Source: Hacker News ────────────────────────────────── + + private async fetchHackerNews(): Promise { + const signals: ResearchSignal[] = []; + + const topIds = await safeFetch(HN_TOP_URL) as number[] | null; + if (!topIds || !Array.isArray(topIds)) { + this.onLog('WARN', 'HN top stories fetch failed'); + return signals; + } + + const ids = topIds.slice(0, 10); + + for (const id of ids) { + await sleep(200); // gentle rate limit for HN + const item = await safeFetch(`${HN_ITEM_URL}/${id}.json`) as { + title?: string; + url?: string; + score?: number; + } | null; + + if (!item || !item.title) continue; + + const { score: relevance, matched } = scoreRelevance(item.title, this.config.keywords); + if (relevance === 0) continue; + + signals.push({ + ts: new Date().toISOString(), + source: 'hackernews', + type: 'trending', + title: item.title, + url: item.url || `https://news.ycombinator.com/item?id=${id}`, + score: item.score || 0, + keywords_matched: matched, + relevance, + }); + } + + return signals; + } + + // ─── Source: GitHub Trending ─────────────────────────────── + + private async fetchGitHubTrending(): Promise { + const signals: ResearchSignal[] = []; + + const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + const dateStr = sevenDaysAgo.toISOString().split('T')[0]; + const url = `${GH_SEARCH_URL}?q=created:>${dateStr}&sort=stars&order=desc&per_page=10`; + + await sleep(RATE_LIMIT_MS); + const data = await safeFetch(url) as { + items?: Array<{ + full_name?: string; + html_url?: string; + description?: string; + stargazers_count?: number; + }>; + } | null; + + if (!data || !data.items) { + this.onLog('WARN', 'GitHub trending fetch failed'); + return signals; + } + + for (const repo of data.items) { + const text = `${repo.full_name || ''} ${repo.description || ''}`; + const { score: relevance, matched } = scoreRelevance(text, this.config.keywords); + if (relevance === 0) continue; + + signals.push({ + ts: new Date().toISOString(), + source: 'github', + type: 'new_repo', + title: `${repo.full_name}: ${(repo.description || '').slice(0, 120)}`, + url: repo.html_url || '', + score: repo.stargazers_count || 0, + keywords_matched: matched, + relevance, + }); + } + + return signals; + } + + // ─── Source: Product Hunt (placeholder) ──────────────────── + + private async fetchProductHunt(): Promise { + // No free API available without key — log and skip + this.onLog('DEBUG', 'Product Hunt source unavailable (no API key)'); + return []; + } + + // ─── Core Scan ──────────────────────────────────────────── + + private async runScan(): Promise { + try { + mkdirSync(DP_DIR, { recursive: true }); + + // Fetch from all sources + const [hnSignals, ghSignals, phSignals] = await Promise.all([ + this.fetchHackerNews(), + this.fetchGitHubTrending(), + this.fetchProductHunt(), + ]); + + const allSignals = [...hnSignals, ...ghSignals, ...phSignals]; + + // Sort by relevance descending, cap at maxSignalsPerScan + allSignals.sort((a, b) => b.relevance - a.relevance || b.score - a.score); + const capped = allSignals.slice(0, this.config.maxSignalsPerScan); + + // Deduplicate against existing stream (by URL) + const existingUrls = this.loadExistingUrls(); + const newSignals = capped.filter(s => !existingUrls.has(s.url)); + + // Append to JSONL + if (newSignals.length > 0) { + const lines = newSignals.map(s => JSON.stringify(s)).join('\n') + '\n'; + appendFileSync(STREAM_FILE, lines, 'utf-8'); + this.state.signalsCollected += newSignals.length; + } + + this.state.lastScanTime = Date.now(); + + this.onLog('INFO', 'Research scan complete', { + hn: hnSignals.length, + gh: ghSignals.length, + ph: phSignals.length, + new: newSignals.length, + total: this.state.signalsCollected, + }); + + // Update weekly digest + this.updateDigest(); + } catch (err) { + this.addError(String(err)); + this.onLog('ERROR', 'Research scan failed', { error: String(err) }); + } + } + + /** Load existing URLs from the stream file for dedup. */ + private loadExistingUrls(): Set { + const urls = new Set(); + try { + if (!existsSync(STREAM_FILE)) return urls; + const lines = readFileSync(STREAM_FILE, 'utf-8').trim().split('\n'); + for (const line of lines) { + try { + const entry = JSON.parse(line) as ResearchSignal; + if (entry.url) urls.add(entry.url); + } catch { + // skip malformed lines + } + } + } catch { + // file read error + } + return urls; + } + + // ─── Weekly Digest ──────────────────────────────────────── + + private updateDigest(): void { + try { + if (!existsSync(STREAM_FILE)) return; + + const lines = readFileSync(STREAM_FILE, 'utf-8').trim().split('\n'); + const now = new Date(); + const currentWeek = isoWeek(now); + const weekStart = Date.now() - 7 * 24 * 60 * 60 * 1000; + + // Collect this week's signals + const weekSignals: ResearchSignal[] = []; + for (const line of lines) { + try { + const entry = JSON.parse(line) as ResearchSignal; + if (new Date(entry.ts).getTime() >= weekStart) { + weekSignals.push(entry); + } + } catch { + // skip + } + } + + if (weekSignals.length === 0) return; + + // Sort by relevance, take top 20 + weekSignals.sort((a, b) => b.relevance - a.relevance || b.score - a.score); + const topSignals = weekSignals.slice(0, 20); + + // Extract themes from keyword frequency + const keywordCounts = new Map(); + for (const signal of topSignals) { + for (const kw of signal.keywords_matched) { + keywordCounts.set(kw, (keywordCounts.get(kw) || 0) + 1); + } + } + + const themes = [...keywordCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, 5) + .map(([kw, count]) => `${kw} (${count} signals)`); + + const digest: ResearchDigest = { + week: currentWeek, + signals: topSignals, + themes, + generated_at: now.toISOString(), + }; + + writeFileSync(DIGEST_FILE, JSON.stringify(digest, null, 2), 'utf-8'); + this.state.digestsGenerated++; + + this.onLog('INFO', 'Research digest updated', { + week: currentWeek, + signals: topSignals.length, + themes: themes.length, + }); + } catch (err) { + this.addError(String(err)); + } + } + + // ─── Lifecycle ──────────────────────────────────────────── + + start(): void { + if (this.isRunning || this.isOptedOut()) { + if (this.isOptedOut()) { + this.onLog('INFO', 'Research stream disabled'); + } + return; + } + + this.isRunning = true; + mkdirSync(DP_DIR, { recursive: true }); + + const intervalMs = (this.config.interval || 360) * 60 * 1000; // default 6h + + this.onLog('INFO', 'Research stream service started', { + interval_min: this.config.interval, + keywords: this.config.keywords.length, + }); + + // First scan after 60s (let other services settle) + setTimeout(() => { + if (!this.isRunning) return; + this.runScan(); + }, 60_000); + + this.intervalId = setInterval(() => { + if (!this.isRunning) return; + this.runScan(); + }, intervalMs); + + if (this.intervalId.unref) this.intervalId.unref(); + } + + stop(): void { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = undefined; + } + this.isRunning = false; + } + + getState(): ResearchStreamState { + return { ...this.state }; + } + + /** Manually trigger a scan (for CLI/MCP). */ + async triggerScan(): Promise { + const before = this.state.signalsCollected; + await this.runScan(); + // Return signals from this scan + try { + if (!existsSync(STREAM_FILE)) return []; + const lines = readFileSync(STREAM_FILE, 'utf-8').trim().split('\n'); + return lines.slice(-(this.state.signalsCollected - before)) + .map(l => { try { return JSON.parse(l); } catch { return null; } }) + .filter(Boolean) as ResearchSignal[]; + } catch { + return []; + } + } + + /** Get the latest digest. */ + getDigest(): ResearchDigest | null { + try { + if (!existsSync(DIGEST_FILE)) return null; + return JSON.parse(readFileSync(DIGEST_FILE, 'utf-8')) as ResearchDigest; + } catch { + return null; + } + } + + private addError(err: string): void { + this.state.errors.push(err); + if (this.state.errors.length > 10) { + this.state.errors = this.state.errors.slice(-10); + } + } +} diff --git a/src/daemon/services/telemetry-service.ts b/src/daemon/services/telemetry-service.ts new file mode 100644 index 00000000..36b24a29 --- /dev/null +++ b/src/daemon/services/telemetry-service.ts @@ -0,0 +1,237 @@ +/** + * Telemetry Service — opt-out anonymous usage snapshots. + * + * Collects daemon health, session counts, skill usage, and handoff + * stats. Stores rolling history in ~/.stackmemory/telemetry.json. + * No PII — instance ID is random hex, no emails/names/paths. + * + * Opt out: STACKMEMORY_TELEMETRY=0 or telemetry.enabled: false in config. + */ + +import { existsSync, readFileSync, writeFileSync, readdirSync, statSync, mkdirSync } from 'fs'; +import { join, dirname } from 'path'; +import { homedir, platform } from 'os'; +import { randomBytes } from 'crypto'; +import type { DaemonServiceConfig } from '../daemon-config.js'; + +export interface TelemetryServiceConfig extends DaemonServiceConfig { + maxSnapshots: number; // rolling history cap +} + +export interface TelemetrySnapshot { + instance_id: string; + collected_at: string; + platform: string; + node_version: string; + daemon: { + uptime_s: number; + context_saves: number; + memory_triggers: number; + ram_percent: number; + errors: number; + } | null; + sessions: { + total_heartbeats: number; + active_now: number; + }; + skills: { + audit_entries: number; + }; + handoffs: { + total: number; + }; +} + +export interface TelemetryServiceState { + lastSnapshotTime: number; + snapshotCount: number; + errors: string[]; +} + +const SM_DIR = join(homedir(), '.stackmemory'); +const INSTANCE_ID_FILE = join(SM_DIR, 'instance-id'); +const TELEMETRY_FILE = join(SM_DIR, 'telemetry.json'); +const SESSIONS_DIR = join(SM_DIR, 'sessions'); +const STALE_MS = 10 * 60 * 1000; // 10 min + +export class DaemonTelemetryService { + private config: TelemetryServiceConfig; + private state: TelemetryServiceState; + private intervalId?: NodeJS.Timeout; + private isRunning = false; + private onLog: (level: string, message: string, data?: unknown) => void; + private getDaemonState?: () => any; + + constructor( + config: TelemetryServiceConfig, + onLog: (level: string, message: string, data?: unknown) => void, + getDaemonState?: () => any + ) { + this.config = config; + this.onLog = onLog; + this.getDaemonState = getDaemonState; + this.state = { lastSnapshotTime: 0, snapshotCount: 0, errors: [] }; + } + + private isOptedOut(): boolean { + if (process.env.STACKMEMORY_TELEMETRY === '0' || process.env.STACKMEMORY_TELEMETRY === 'false') { + return true; + } + return !this.config.enabled; + } + + private getInstanceId(): string { + try { + if (existsSync(INSTANCE_ID_FILE)) { + return readFileSync(INSTANCE_ID_FILE, 'utf-8').trim(); + } + } catch { + // Regenerate + } + const id = randomBytes(16).toString('hex'); + try { + writeFileSync(INSTANCE_ID_FILE, id, 'utf-8'); + } catch { + // Ephemeral + } + return id; + } + + private countSessions(): { total_heartbeats: number; active_now: number } { + try { + if (!existsSync(SESSIONS_DIR)) return { total_heartbeats: 0, active_now: 0 }; + const files = readdirSync(SESSIONS_DIR).filter(f => f.endsWith('.heartbeat')); + const now = Date.now(); + let active = 0; + for (const file of files) { + try { + const stat = statSync(join(SESSIONS_DIR, file)); + if (now - stat.mtimeMs < STALE_MS) active++; + } catch { + // Skip + } + } + return { total_heartbeats: files.length, active_now: active }; + } catch { + return { total_heartbeats: 0, active_now: 0 }; + } + } + + private countSkillAudit(): number { + try { + const auditPath = join(SM_DIR, 'skill-audit.jsonl'); + if (!existsSync(auditPath)) return 0; + return readFileSync(auditPath, 'utf-8').trim().split('\n').length; + } catch { + return 0; + } + } + + private countHandoffs(): number { + try { + const handoffsDir = join(SM_DIR, 'handoffs'); + if (!existsSync(handoffsDir)) return 0; + return readdirSync(handoffsDir).filter(f => f.endsWith('.md')).length; + } catch { + return 0; + } + } + + collect(): TelemetrySnapshot | { opted_out: true } { + if (this.isOptedOut()) return { opted_out: true }; + + const daemonState = this.getDaemonState?.(); + const sessions = this.countSessions(); + + return { + instance_id: this.getInstanceId(), + collected_at: new Date().toISOString(), + platform: platform(), + node_version: process.version, + daemon: daemonState ? { + uptime_s: Math.round((daemonState.uptime || 0) / 1000), + context_saves: daemonState.services?.context?.saveCount || 0, + memory_triggers: daemonState.services?.memory?.triggerCount || 0, + ram_percent: Math.round((daemonState.services?.memory?.currentRamPercent || 0) * 100), + errors: (daemonState.errors || []).length, + } : null, + sessions, + skills: { audit_entries: this.countSkillAudit() }, + handoffs: { total: this.countHandoffs() }, + }; + } + + save(): TelemetrySnapshot | null { + const snapshot = this.collect(); + if ('opted_out' in snapshot) return null; + + let history: TelemetrySnapshot[] = []; + try { + if (existsSync(TELEMETRY_FILE)) { + const data = JSON.parse(readFileSync(TELEMETRY_FILE, 'utf-8')); + history = Array.isArray(data.snapshots) ? data.snapshots : []; + } + } catch { + history = []; + } + + history.push(snapshot); + const max = this.config.maxSnapshots || 90; + if (history.length > max) history = history.slice(-max); + + try { + const dir = dirname(TELEMETRY_FILE); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(TELEMETRY_FILE, JSON.stringify({ version: 1, snapshots: history }, null, 2), 'utf-8'); + } catch (err) { + this.state.errors.push(String(err)); + if (this.state.errors.length > 5) this.state.errors = this.state.errors.slice(-5); + this.onLog('ERROR', 'Failed to save telemetry', { error: String(err) }); + return null; + } + + this.state.lastSnapshotTime = Date.now(); + this.state.snapshotCount++; + return snapshot; + } + + start(): void { + if (this.isRunning || this.isOptedOut()) { + if (this.isOptedOut()) { + this.onLog('INFO', 'Telemetry disabled — opt-out active'); + } + return; + } + + this.isRunning = true; + const intervalMs = (this.config.interval || 1440) * 60 * 1000; // default 24h + + this.onLog('INFO', 'Telemetry service started', { interval_min: this.config.interval }); + + // First snapshot after 30s + setTimeout(() => { + if (!this.isRunning) return; + const snap = this.save(); + if (snap) this.onLog('INFO', 'Telemetry snapshot saved', { sessions: snap.sessions.active_now }); + }, 30_000); + + this.intervalId = setInterval(() => { + const snap = this.save(); + if (snap) this.onLog('INFO', 'Telemetry snapshot saved', { sessions: snap.sessions.active_now }); + }, intervalMs); + + if (this.intervalId.unref) this.intervalId.unref(); + } + + stop(): void { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = undefined; + } + this.isRunning = false; + } + + getState(): TelemetryServiceState { + return { ...this.state }; + } +} diff --git a/src/daemon/unified-daemon.ts b/src/daemon/unified-daemon.ts index 0bf7ffd4..fea41ef5 100644 --- a/src/daemon/unified-daemon.ts +++ b/src/daemon/unified-daemon.ts @@ -29,6 +29,9 @@ import { DaemonLinearService } from './services/linear-service.js'; import { DaemonGitHubService } from './services/github-service.js'; import { DaemonMaintenanceService } from './services/maintenance-service.js'; import { DaemonMemoryService } from './services/memory-service.js'; +import { DaemonTelemetryService } from './services/telemetry-service.js'; +import { DaemonDesirePathService } from './services/desire-path-service.js'; +import { ResearchStreamService } from './services/research-stream-service.js'; interface LogEntry { timestamp: string; @@ -46,6 +49,9 @@ export class UnifiedDaemon { private githubService: DaemonGitHubService; private maintenanceService: DaemonMaintenanceService; private memoryService: DaemonMemoryService; + private telemetryService: DaemonTelemetryService; + private desirePathService: DaemonDesirePathService; + private researchStreamService: ResearchStreamService; private heartbeatInterval?: NodeJS.Timeout; private isShuttingDown = false; private startTime: number = 0; @@ -79,6 +85,22 @@ export class UnifiedDaemon { this.config.memory, (level, msg, data) => this.log(level, 'memory', msg, data) ); + + this.telemetryService = new DaemonTelemetryService( + this.config.telemetry, + (level, msg, data) => this.log(level, 'telemetry', msg, data), + () => this.getStatus() + ); + + this.desirePathService = new DaemonDesirePathService( + this.config.desirePaths, + (level, msg, data) => this.log(level, 'desire-paths', msg, data) + ); + + this.researchStreamService = new ResearchStreamService( + this.config.researchStream, + (level, msg, data) => this.log(level, 'research-stream', msg, data) + ); } private log( @@ -290,6 +312,8 @@ export class UnifiedDaemon { githubSyncs: this.githubService.getState().syncCount, maintenanceRuns: this.maintenanceService.getState().ftsRebuilds, memoryTriggers: this.memoryService.getState().triggerCount, + telemetrySnapshots: this.telemetryService.getState().snapshotCount, + researchSignals: this.researchStreamService.getState().signalsCollected, }); // Stop heartbeat @@ -304,6 +328,9 @@ export class UnifiedDaemon { this.githubService.stop(); this.maintenanceService.stop(); this.memoryService.stop(); + this.telemetryService.stop(); + this.desirePathService.stop(); + this.researchStreamService.stop(); // Cleanup this.cleanup(); @@ -348,6 +375,9 @@ export class UnifiedDaemon { await this.githubService.start(); this.maintenanceService.start(); this.memoryService.start(); + this.telemetryService.start(); + this.desirePathService.start(); + this.researchStreamService.start(); // Start heartbeat this.heartbeatInterval = setInterval(() => { diff --git a/src/features/tasks/task-aware-context.ts b/src/features/tasks/task-aware-context.ts index 03f333e7..ba3bbab4 100644 --- a/src/features/tasks/task-aware-context.ts +++ b/src/features/tasks/task-aware-context.ts @@ -11,6 +11,7 @@ import { FrameManager, } from '../../core/context/index.js'; import { logger } from '../../core/monitoring/logger.js'; +import { estimateTokens } from '../../core/cache/token-estimator.js'; /** Raw task row from SQLite before hydration */ interface TaskRow { @@ -476,7 +477,7 @@ export class TaskAwareContextManager { activeTasks.forEach((task) => { const line = `- [${task.status.toUpperCase()}] ${task.name} (${task.priority})\n`; context += line; - totalTokens += line.length / 4; // Rough token estimate + totalTokens += estimateTokens(line); relevanceScores[task.task_id] = 1.0; }); context += '\n'; diff --git a/src/hooks/daemon-health-check.sh b/src/hooks/daemon-health-check.sh new file mode 100755 index 00000000..130202c5 --- /dev/null +++ b/src/hooks/daemon-health-check.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# daemon-health-check.sh — SessionStart hook for Claude Code / Codex +# +# Checks if the StackMemory daemon is alive. If not, restarts it. +# Install as a Claude Code SessionStart hook in settings.json: +# { "event": "SessionStart", "command": "~/.stackmemory/bin/daemon-health-check.sh" } +# +# Self-healing: runs on every new session. If daemon is down, brings it back. + +SM_DIR="${HOME}/.stackmemory" +PID_FILE="${SM_DIR}/daemon/daemon.pid" +STATUS_FILE="${SM_DIR}/daemon/daemon.status" + +# Check if PID file exists and process is alive +check_daemon() { + if [ ! -f "$PID_FILE" ]; then + return 1 + fi + + local pid + pid=$(cat "$PID_FILE" 2>/dev/null) + if [ -z "$pid" ]; then + return 1 + fi + + # Check if process is actually running + if kill -0 "$pid" 2>/dev/null; then + return 0 + else + return 1 + fi +} + +restart_daemon() { + # Clean stale PID + rm -f "$PID_FILE" 2>/dev/null + + # Update status to reflect it's down + if [ -f "$STATUS_FILE" ]; then + # Mark as not running (best-effort JSON update) + local tmp + tmp=$(mktemp) + node -e " + const fs = require('fs'); + try { + const s = JSON.parse(fs.readFileSync('${STATUS_FILE}', 'utf-8')); + s.running = false; + s.errors = (s.errors || []).concat('daemon died, restarted by health check at ' + new Date().toISOString()); + fs.writeFileSync('${tmp}', JSON.stringify(s, null, 2)); + } catch { process.exit(0); } + " 2>/dev/null && mv "$tmp" "$STATUS_FILE" 2>/dev/null + fi + + # Try stackmemory CLI first, fall back to direct daemon start + if command -v stackmemory &>/dev/null; then + stackmemory daemon start &>/dev/null & + elif [ -f "${SM_DIR}/bin/stackmemory" ]; then + "${SM_DIR}/bin/stackmemory" daemon start &>/dev/null & + else + # Direct node invocation as last resort + local daemon_script + daemon_script=$(find "${HOME}/.nvm" "/opt/homebrew/lib" "/usr/local/lib" -path "*/stackmemory/dist/src/daemon/unified-daemon.js" 2>/dev/null | head -1) + if [ -n "$daemon_script" ]; then + node "$daemon_script" &>/dev/null & + fi + fi +} + +# Main +if check_daemon; then + # Daemon alive — emit brief status for hook output + echo '{"hookSpecificOutput":{"daemonAlive":true,"pid":'$(cat "$PID_FILE")'}}' +else + restart_daemon + # Wait briefly for startup + sleep 1 + if check_daemon; then + echo '{"hookSpecificOutput":{"daemonAlive":true,"restarted":true,"pid":'$(cat "$PID_FILE" 2>/dev/null || echo 0)'}}' + else + echo '{"hookSpecificOutput":{"daemonAlive":false,"restartAttempted":true}}' + fi +fi diff --git a/src/hooks/dedup-reads.cjs b/src/hooks/dedup-reads.cjs new file mode 100755 index 00000000..3c040890 --- /dev/null +++ b/src/hooks/dedup-reads.cjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node +// dedup-reads.cjs — PostToolUse hook for Claude Code +// +// Detects duplicate file reads in a session and warns when a file is read 3+ +// times without being modified in between. Helps reduce wasted tool calls. +// +// Install in ~/.claude/settings.json (or .claude/settings.local.json per-project): +// +// { +// "hooks": { +// "PostToolUse": [ +// { +// "matcher": "Read", +// "hooks": [ +// { "type": "command", "command": "node /Users/jwu/Dev/stackmemory/src/hooks/dedup-reads.cjs" } +// ] +// } +// ] +// } +// } +// +// Opt out: STACKMEMORY_DEDUP_READS=0 + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +if (process.env.STACKMEMORY_DEDUP_READS === '0' || process.env.STACKMEMORY_DEDUP_READS === 'false') { + process.exit(0); +} + +const SM_DIR = path.join(process.env.HOME || '', '.stackmemory'); +const DP_DIR = path.join(SM_DIR, 'desire-paths'); + +function run() { + let raw = ''; + try { + raw = fs.readFileSync(0, 'utf-8'); + } catch { + return; + } + + let input; + try { + input = JSON.parse(raw); + } catch { + return; + } + + const toolName = input.tool_name || input.toolName; + const toolInput = input.tool_input || input.input || {}; + + let filePath; + + if (toolName === 'Read') { + filePath = toolInput.file_path || toolInput.filePath; + } else if (toolName === 'Bash') { + // Codex reads files via Bash (cat, sed, head, nl, etc.) — extract file path + const cmd = toolInput.command || ''; + const readMatch = cmd.match(/^(?:cat|head|tail|sed\s+-n|nl)\s+['"]?([^\s'";<>|&]+)/); + if (readMatch && readMatch[1] && !readMatch[1].startsWith('-')) { + filePath = readMatch[1]; + } + } + + if (!filePath) return; + + const sessionId = input.session_id || input.sessionId + || process.env.STACKMEMORY_SESSION || process.env.CLAUDE_SESSION_ID + || 'default'; + + // Get current mtime + let mtimeMs = 0; + try { + mtimeMs = fs.statSync(filePath).mtimeMs; + } catch { + // File may not exist (e.g., error read) — skip tracking + return; + } + + fs.mkdirSync(DP_DIR, { recursive: true }); + + const stateFile = path.join(DP_DIR, `dedup-${sessionId}.json`); + const lockFile = stateFile + '.lock'; + + // Acquire lock (spin up to 200ms) + let lockFd; + const deadline = Date.now() + 200; + while (Date.now() < deadline) { + try { + lockFd = fs.openSync(lockFile, 'wx'); + break; + } catch { + // Lock held — spin briefly + const wait = Date.now() + 5; + while (Date.now() < wait) {} // busy-wait 5ms (no setTimeout in sync hook) + } + } + if (lockFd === undefined) return; // couldn't acquire lock — skip silently + + try { + // Load state under lock + let state = {}; + try { + state = JSON.parse(fs.readFileSync(stateFile, 'utf-8')); + } catch { + // First call or corrupted — start fresh + } + + const entry = state[filePath]; + + if (!entry) { + state[filePath] = { count: 1, lastMtime: mtimeMs }; + } else if (mtimeMs !== entry.lastMtime) { + state[filePath] = { count: 1, lastMtime: mtimeMs }; + } else { + entry.count += 1; + entry.lastMtime = mtimeMs; + + if (entry.count >= 3) { + const basename = path.basename(filePath); + let msg; + if (entry.count >= 5) { + msg = `[STOP] ${basename} read ${entry.count}x (unchanged). You already have this content. Do NOT read again — use what you have.`; + } else { + msg = `[dedup] ${basename} read ${entry.count}x this session (unchanged) — use cached content`; + } + process.stdout.write(JSON.stringify({ systemMessage: msg }) + '\n'); + } + } + + fs.writeFileSync(stateFile, JSON.stringify(state), 'utf-8'); + } finally { + // Release lock + try { fs.closeSync(lockFd); } catch {} + try { fs.unlinkSync(lockFile); } catch {} + } +} + +try { + run(); +} catch { + // Non-fatal — never crash the hook pipeline +} diff --git a/src/hooks/desire-path-hook.sh b/src/hooks/desire-path-hook.sh new file mode 100755 index 00000000..ec30fc31 --- /dev/null +++ b/src/hooks/desire-path-hook.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# desire-path-hook.sh — PostToolUse hook for Claude Code +# +# Captures tool name + sanitized first arg to the action stream. +# No data/content captured — just the tool:target pair for pattern detection. +# +# Install in Claude Code settings.json: +# { "event": "PostToolUse", "command": "~/.stackmemory/bin/desire-path-hook.sh" } +# +# Or in .claude/settings.local.json per-project. +# +# Opt out: STACKMEMORY_DESIRE_PATHS=0 + +# Quick exit if opted out +[ "$STACKMEMORY_DESIRE_PATHS" = "0" ] && exit 0 +[ "$STACKMEMORY_DESIRE_PATHS" = "false" ] && exit 0 + +SM_DIR="${HOME}/.stackmemory" +DP_DIR="${SM_DIR}/desire-paths" +STREAM_FILE="${DP_DIR}/action-stream.jsonl" +MAX_SIZE=10485760 # 10MB + +# Read hook input from stdin (Claude Code passes JSON) +INPUT=$(cat) + +# Extract tool name and first arg from hook input +TOOL_NAME=$(echo "$INPUT" | node -e " + const d = JSON.parse(require('fs').readFileSync(0,'utf-8')); + console.log(d.tool_name || d.toolName || 'unknown'); +" 2>/dev/null || echo "unknown") + +FIRST_ARG=$(echo "$INPUT" | node -e " + const d = JSON.parse(require('fs').readFileSync(0,'utf-8')); + const args = d.tool_input || d.input || {}; + // Get the most meaningful arg (file_path, command, pattern, etc.) + const key = Object.keys(args).find(k => + ['file_path','command','pattern','path','query','skill_path','url'].includes(k) + ) || Object.keys(args)[0]; + const val = key ? String(args[key] || '').slice(0, 100) : ''; + console.log(val); +" 2>/dev/null || echo "") + +DURATION=$(echo "$INPUT" | node -e " + const d = JSON.parse(require('fs').readFileSync(0,'utf-8')); + console.log(d.duration_ms || d.duration || 0); +" 2>/dev/null || echo "0") + +# Session ID from env or generate +SESSION_ID="${STACKMEMORY_SESSION:-${CLAUDE_SESSION_ID:-$(date +%s)}}" + +# Ensure directory exists +mkdir -p "$DP_DIR" 2>/dev/null + +# Rotate if too large +if [ -f "$STREAM_FILE" ]; then + FILE_SIZE=$(stat -f%z "$STREAM_FILE" 2>/dev/null || stat -c%s "$STREAM_FILE" 2>/dev/null || echo 0) + if [ "$FILE_SIZE" -gt "$MAX_SIZE" ]; then + mv "$STREAM_FILE" "${STREAM_FILE}.$(date +%s).bak" 2>/dev/null + fi +fi + +# Append entry (no content/data — just tool + target pattern) +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +echo "{\"ts\":\"${TIMESTAMP}\",\"sid\":\"${SESSION_ID}\",\"tool\":\"${TOOL_NAME}\",\"target\":\"${FIRST_ARG}\",\"dur\":${DURATION}}" >> "$STREAM_FILE" + +# --- Script suggestions: detect patterns that match existing scripts --- +echo "$INPUT" | node "$(dirname "$0")/script-suggest.cjs" 2>/dev/null + +# --- Auto-route: suggest dedicated tools for replaceable Bash calls --- +if [ "$TOOL_NAME" = "Bash" ] && [ -n "$FIRST_ARG" ]; then + SUGGESTION=$(echo "$FIRST_ARG" | node -e " + const cmd = require('fs').readFileSync(0,'utf-8').trim(); + // ls/find → Glob + if (/^ls\s/.test(cmd) || /^find\s/.test(cmd)) { + const dir = cmd.replace(/^(ls|find)\s+/, '').split(/\s/)[0] || '.'; + console.log('[route] Use Glob instead of \"' + cmd.slice(0,40) + '\" — e.g. Glob(pattern=\"**/*\", path=\"' + dir + '\")'); + } + // cat/head/tail → Read + else if (/^(cat|head|tail|sed\s+-n|nl)\s/.test(cmd)) { + const file = cmd.replace(/^(cat|head|tail|sed\s+-n|nl)\s+/, '').split(/\s/)[0] || ''; + if (file && !file.startsWith('-')) { + console.log('[route] Use Read instead of \"' + cmd.slice(0,40) + '\" — Read(file_path=\"' + file + '\")'); + } + } + // grep/rg → Grep + else if (/^(grep|rg|ag)\s/.test(cmd)) { + const parts = cmd.split(/\s+/); + const pattern = parts[1] || ''; + console.log('[route] Use Grep instead of \"' + cmd.slice(0,40) + '\" — Grep(pattern=\"' + pattern + '\")'); + } + " 2>/dev/null) + + if [ -n "$SUGGESTION" ]; then + echo "{\"systemMessage\":\"$SUGGESTION\"}" + fi +fi diff --git a/src/hooks/diffmem-hooks.ts b/src/hooks/diffmem-hooks.ts index c40c0373..c0fb601a 100644 --- a/src/hooks/diffmem-hooks.ts +++ b/src/hooks/diffmem-hooks.ts @@ -4,6 +4,7 @@ */ import { logger } from '../core/monitoring/logger.js'; +import { estimateTokens } from '../core/cache/token-estimator.js'; import type { HookEventEmitter, HookEventData } from './events.js'; import type { FrameManager } from '../core/context/frame-manager.js'; import type { @@ -256,7 +257,7 @@ export class DiffMemHooks { for (const [category, contents] of sections) { const label = categoryLabels[category] || category; const categoryHeader = `\n### ${label}`; - const headerTokens = Math.ceil(categoryHeader.length / 4); + const headerTokens = estimateTokens(categoryHeader); if (estimatedTokens + headerTokens > maxTokens) { break; @@ -267,7 +268,7 @@ export class DiffMemHooks { for (const content of contents) { const contentLine = `- ${content}`; - const contentTokens = Math.ceil(contentLine.length / 4); + const contentTokens = estimateTokens(contentLine); if (estimatedTokens + contentTokens > maxTokens) { lines.push('- (additional items truncated for token budget)'); diff --git a/src/hooks/prewarm-tools.cjs b/src/hooks/prewarm-tools.cjs new file mode 100644 index 00000000..91da7a2f --- /dev/null +++ b/src/hooks/prewarm-tools.cjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node +/** + * prewarm-tools.cjs — SessionStart hook + * + * Emits a system message telling Claude to pre-fetch schemas for + * the most frequently used deferred MCP tools, avoiding repeated + * ToolSearch calls mid-conversation. + * + * Data source: ~/.stackmemory/desire-paths/action-stream.jsonl + * Learns from actual usage — top N deferred tools by frequency. + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const SM_DIR = path.join(process.env.HOME || '', '.stackmemory'); +const STREAM_FILE = path.join(SM_DIR, 'desire-paths', 'action-stream.jsonl'); +const CACHE_FILE = path.join(SM_DIR, 'desire-paths', 'prewarm-cache.json'); +const CACHE_TTL = 24 * 60 * 60 * 1000; // 24h + +// Known deferred tool prefixes (MCP tools that need ToolSearch) +const DEFERRED_PREFIXES = ['mcp__', 'TaskCreate', 'TaskUpdate', 'TaskGet', 'WebFetch', 'WebSearch']; + +function isDeferred(tool) { + return DEFERRED_PREFIXES.some(p => tool.startsWith(p)); +} + +function getTopTools() { + // Check cache first + try { + const cache = JSON.parse(fs.readFileSync(CACHE_FILE, 'utf-8')); + if (Date.now() - cache.ts < CACHE_TTL && cache.tools?.length > 0) { + return cache.tools; + } + } catch {} + + // Parse action stream + if (!fs.existsSync(STREAM_FILE)) return []; + + const counts = {}; + const lines = fs.readFileSync(STREAM_FILE, 'utf-8').split('\n'); + + for (const line of lines) { + if (!line) continue; + try { + const d = JSON.parse(line); + const tool = d.tool || ''; + if (isDeferred(tool)) { + counts[tool] = (counts[tool] || 0) + 1; + } + } catch {} + } + + // Sort by frequency, take top 8 + const sorted = Object.entries(counts) + .sort((a, b) => b[1] - a[1]) + .slice(0, 8) + .map(([tool]) => tool); + + // Cache result + try { + fs.mkdirSync(path.dirname(CACHE_FILE), { recursive: true }); + fs.writeFileSync(CACHE_FILE, JSON.stringify({ ts: Date.now(), tools: sorted })); + } catch {} + + return sorted; +} + +function main() { + const tools = getTopTools(); + if (tools.length === 0) return; + + // Group by prefix for efficient ToolSearch queries + const mcpTools = tools.filter(t => t.startsWith('mcp__')); + const builtinTools = tools.filter(t => !t.startsWith('mcp__')); + + const parts = []; + if (mcpTools.length > 0) { + parts.push(`select:${mcpTools.join(',')}`); + } + if (builtinTools.length > 0) { + parts.push(`select:${builtinTools.join(',')}`); + } + + const msg = `[prewarm] Frequently used deferred tools detected. Pre-fetch with: ToolSearch(query="${parts[0]}", max_results=${tools.length})`; + process.stdout.write(JSON.stringify({ systemMessage: msg }) + '\n'); +} + +try { + main(); +} catch {} diff --git a/src/hooks/script-suggest.cjs b/src/hooks/script-suggest.cjs new file mode 100644 index 00000000..35b5a8da --- /dev/null +++ b/src/hooks/script-suggest.cjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node +/** + * script-suggest.cjs — Suggests existing scripts when tool patterns match. + * + * Called by desire-path-hook.sh with: echo "$INPUT" | node script-suggest.cjs + * Outputs JSON systemMessage if a script match is found, empty otherwise. + * + * Pattern matching is based on recent N tool calls in the session. + * When a sequence matches a known script's purpose, suggest it. + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const SM_DIR = path.join(process.env.HOME || '', '.stackmemory'); +const DP_DIR = path.join(SM_DIR, 'desire-paths'); +const SCRIPTS_DIR = path.join(process.env.HOME || '', '.claude', 'scripts'); +const BUN = '/Users/jwu/.bun/bin/bun'; + +// Script patterns: tool sequences or single-call patterns that map to scripts +const SCRIPT_MAP = [ + { + name: 'git-ops', + // 3+ git commands in a row without Edit/Write in between + match: (recent) => { + const gitCmds = recent.filter(r => r.tool === 'Bash' && /^git\s/.test(r.target)); + return gitCmds.length >= 3; + }, + suggestion: `${BUN} run ${SCRIPTS_DIR}/git-ops.ts --status`, + label: 'git-ops --status', + }, + { + name: 'build-status', + match: (recent) => { + return recent.some(r => r.tool === 'Bash' && /gh\s+run\s+(list|view)/.test(r.target)); + }, + suggestion: `${BUN} run ${SCRIPTS_DIR}/build-status.ts`, + label: 'build-status', + }, + { + name: 'web-fetch', + match: (recent) => { + return recent.some(r => r.tool === 'WebFetch'); + }, + suggestion: (recent) => { + const wf = recent.find(r => r.tool === 'WebFetch'); + const url = wf ? wf.target : ''; + return `${BUN} run ${SCRIPTS_DIR}/web-fetch.ts ${url}`; + }, + label: 'web-fetch', + }, + { + name: 'web-search', + match: (recent) => { + return recent.some(r => r.tool === 'WebSearch'); + }, + suggestion: (recent) => { + const ws = recent.find(r => r.tool === 'WebSearch'); + const q = ws ? ws.target : ''; + return `${BUN} run ${SCRIPTS_DIR}/web-search.ts "${q}"`; + }, + label: 'web-search', + }, +]; + +function main() { + let input; + try { + input = JSON.parse(fs.readFileSync(0, 'utf-8')); + } catch { + return; + } + + const sessionId = input.session_id || input.sessionId + || process.env.STACKMEMORY_SESSION || process.env.CLAUDE_SESSION_ID || ''; + + if (!sessionId) return; + + // Read recent entries from action stream for this session (last 10) + const streamFile = path.join(DP_DIR, 'action-stream.jsonl'); + if (!fs.existsSync(streamFile)) return; + + const lines = fs.readFileSync(streamFile, 'utf-8').split('\n'); + const recent = []; + // Read backwards for efficiency + for (let i = lines.length - 1; i >= 0 && recent.length < 10; i--) { + if (!lines[i]) continue; + try { + const d = JSON.parse(lines[i]); + if (d.sid === sessionId) { + recent.unshift(d); + } + } catch {} + } + + if (recent.length < 2) return; + + // Check cooldown — don't suggest the same script twice in 5 minutes + const cooldownFile = path.join(DP_DIR, `suggest-cooldown-${sessionId}.json`); + let cooldowns = {}; + try { + cooldowns = JSON.parse(fs.readFileSync(cooldownFile, 'utf-8')); + } catch {} + + const now = Date.now(); + + for (const rule of SCRIPT_MAP) { + if (cooldowns[rule.name] && now - cooldowns[rule.name] < 300000) continue; + if (!rule.match(recent)) continue; + + // Verify script exists + const scriptPath = path.join(SCRIPTS_DIR, `${rule.name}.ts`); + if (!fs.existsSync(scriptPath)) continue; + + const cmd = typeof rule.suggestion === 'function' ? rule.suggestion(recent) : rule.suggestion; + const msg = `[script] Consider using: Bash("${cmd}") — the ${rule.label} script handles this in one call`; + process.stdout.write(JSON.stringify({ systemMessage: msg }) + '\n'); + + // Set cooldown + cooldowns[rule.name] = now; + try { + fs.writeFileSync(cooldownFile, JSON.stringify(cooldowns)); + } catch {} + + return; // One suggestion per invocation + } +} + +try { + main(); +} catch {} diff --git a/src/integrations/anthropic/client.ts b/src/integrations/anthropic/client.ts index 5fc0bf4e..c4e68c55 100644 --- a/src/integrations/anthropic/client.ts +++ b/src/integrations/anthropic/client.ts @@ -6,6 +6,7 @@ */ import { logger } from '../../core/monitoring/logger.js'; +import { estimateTokens } from '../../core/cache/token-estimator.js'; import { STRUCTURED_RESPONSE_SUFFIX } from '../../orchestrators/multimodal/constants.js'; export interface CompletionRequest { @@ -314,8 +315,8 @@ describe('validateInput', () => { stopReason: 'stop_sequence', model: request.model, usage: { - inputTokens: Math.ceil(request.prompt.length / 4), - outputTokens: Math.ceil(content.length / 4), + inputTokens: estimateTokens(request.prompt), + outputTokens: estimateTokens(content), }, }; } diff --git a/src/integrations/claude-code/subagent-client.ts b/src/integrations/claude-code/subagent-client.ts index 78190892..0e28deb1 100644 --- a/src/integrations/claude-code/subagent-client.ts +++ b/src/integrations/claude-code/subagent-client.ts @@ -6,6 +6,7 @@ */ import { logger } from '../../core/monitoring/logger.js'; +import { estimateTokens } from '../../core/cache/token-estimator.js'; import { STRUCTURED_RESPONSE_SUFFIX } from '../../orchestrators/multimodal/constants.js'; import { spawn } from 'child_process'; import * as fs from 'fs'; @@ -288,7 +289,7 @@ export class ClaudeCodeSubagentClient { output: result.text, duration: Date.now() - startTime, subagentType: request.type, - tokens: this.estimateTokens(fullPrompt + result.text), + tokens: estimateTokens(fullPrompt + result.text), }; } catch (error: any) { // Detect quota/rate limit errors and overflow to Kimi @@ -773,18 +774,10 @@ function greetUser(name: string): string { output: `Mock ${request.type} subagent completed successfully`, duration: Date.now() - startTime, subagentType: request.type, - tokens: this.estimateTokens(JSON.stringify(result)), + tokens: estimateTokens(JSON.stringify(result)), }; } - /** - * Estimate token usage - */ - private estimateTokens(text: string): number { - // Rough estimation: 1 token ≈ 4 characters - return Math.ceil(text.length / 4); - } - /** * Cleanup temporary files */ diff --git a/src/integrations/claude-code/task-coordinator.ts b/src/integrations/claude-code/task-coordinator.ts index d25bc884..7515bb5c 100644 --- a/src/integrations/claude-code/task-coordinator.ts +++ b/src/integrations/claude-code/task-coordinator.ts @@ -8,6 +8,7 @@ import { v4 as uuidv4 } from 'uuid'; import { spawn } from 'child_process'; import { logger } from '../../core/monitoring/logger.js'; +import { estimateTokens } from '../../core/cache/token-estimator.js'; import { ClaudeCodeAgent } from './agent-bridge.js'; export interface TaskExecution { @@ -508,8 +509,7 @@ export class ClaudeCodeTaskCoordinator { * Estimate token usage */ private estimateTokenUsage(prompt: string, response: string): number { - // Rough estimation: ~4 characters per token - return Math.ceil((prompt.length + response.length) / 4); + return estimateTokens(prompt + response); } /** diff --git a/src/integrations/mcp/server.ts b/src/integrations/mcp/server.ts index 75fbf487..795e3d58 100644 --- a/src/integrations/mcp/server.ts +++ b/src/integrations/mcp/server.ts @@ -28,6 +28,14 @@ import { } from 'fs'; import { homedir } from 'os'; import { compactPlan } from '../../orchestrators/multimodal/utils.js'; +import { + parseMasterTasks, + getNextTask, + addTaskToFile, + updateTaskInFile, + type TaskPriority as MdPriority, + type TaskSync, +} from '../../core/tasks/md-task-parser.js'; import { filterPending } from './pending-utils.js'; import { join, dirname } from 'path'; import { execSync } from 'child_process'; @@ -1911,6 +1919,18 @@ class LocalStackMemoryMCP { result = this.handleTraceEventAnnotate(args); break; + case 'get_next_master_task': + result = this.handleGetNextMasterTask(args); + break; + + case 'update_master_task': + result = this.handleUpdateMasterTask(args); + break; + + case 'create_master_task': + result = this.handleCreateMasterTask(args); + break; + default: throw new Error(`Unknown tool: ${name}`); } @@ -4054,6 +4074,124 @@ ${typeBreakdown}`, process.on('SIGTERM', printCacheSummary); process.on('exit', printCacheSummary); } + + // ── Master Task Handlers ─────────────────────────────────── + + private resolveMasterTasksPath(): string | null { + const smPath = join( + this.projectRoot, + '.stackmemory', + 'tasks', + 'master-tasks.md' + ); + if (existsSync(smPath)) return smPath; + const rootPath = join(this.projectRoot, 'master-tasks.md'); + if (existsSync(rootPath)) return rootPath; + return null; + } + + private handleGetNextMasterTask(args: Record) { + const mdPath = this.resolveMasterTasksPath(); + if (!mdPath) { + return { + content: [ + { + type: 'text', + text: 'No master-tasks.md found. Run "stackmemory tasks init" to create one.', + }, + ], + }; + } + + let tasks = parseMasterTasks(readFileSync(mdPath, 'utf-8')); + if (args.owner) { + tasks = tasks.filter((t) => t.owner === String(args.owner)); + } + + const next = getNextTask(tasks); + if (!next) { + return { + content: [{ type: 'text', text: 'No actionable tasks found.' }], + }; + } + + return { + content: [{ type: 'text', text: JSON.stringify(next, null, 2) }], + }; + } + + private handleUpdateMasterTask(args: Record) { + const mdPath = this.resolveMasterTasksPath(); + if (!mdPath) { + return { + content: [{ type: 'text', text: 'No master-tasks.md found.' }], + }; + } + + const taskId = String(args.task_id || '').toUpperCase(); + if (!taskId) { + return { + content: [{ type: 'text', text: 'task_id is required.' }], + isError: true, + }; + } + + const updates: Record = {}; + if (args.status) updates.status = String(args.status); + if (args.priority) updates.priority = String(args.priority); + if (args.owner) updates.owner = String(args.owner); + if (args.branch_pr) updates.branchPr = String(args.branch_pr); + if (args.notes) updates.notes = String(args.notes); + if (args.sync) updates.sync = String(args.sync); + + try { + updateTaskInFile(mdPath, taskId, updates); + return { + content: [{ type: 'text', text: `Updated ${taskId}` }], + }; + } catch (err) { + return { + content: [{ type: 'text', text: (err as Error).message }], + isError: true, + }; + } + } + + private handleCreateMasterTask(args: Record) { + const mdPath = this.resolveMasterTasksPath(); + if (!mdPath) { + return { + content: [ + { + type: 'text', + text: 'No master-tasks.md found. Run "stackmemory tasks init" to create one.', + }, + ], + }; + } + + const task = String(args.task || ''); + if (!task) { + return { + content: [{ type: 'text', text: 'task description is required.' }], + isError: true, + }; + } + + const id = addTaskToFile(mdPath, { + priority: String(args.priority || 'P1') as MdPriority, + status: 'todo', + owner: String(args.owner || '@me'), + sync: String(args.sync || 'local') as TaskSync, + task, + branchPr: '', + notes: String(args.notes || ''), + }); + + return { + content: [{ type: 'text', text: `Created ${id}: ${task}` }], + }; + } } // Export the class diff --git a/src/integrations/mcp/tool-definitions.ts b/src/integrations/mcp/tool-definitions.ts index 75870983..e28acff3 100644 --- a/src/integrations/mcp/tool-definitions.ts +++ b/src/integrations/mcp/tool-definitions.ts @@ -37,6 +37,7 @@ export class MCPToolDefinitions { ...this.getProvenantTools(), ...this.getCrossSearchTools(), ...this.getCloudSyncTools(), + ...this.getMasterTaskTools(), ]; } @@ -1589,6 +1590,8 @@ export class MCPToolDefinitions { return this.getProvenantTools(); case 'cloud_sync': return this.getCloudSyncTools(); + case 'master_tasks': + return this.getMasterTaskTools(); default: return []; } @@ -1637,4 +1640,102 @@ export class MCPToolDefinitions { }, ]; } + + private getMasterTaskTools(): MCPToolDefinition[] { + return [ + { + name: 'get_next_master_task', + description: + 'Get the highest-priority actionable task from master-tasks.md. Returns the next task to work on based on priority (P0 > P1 > P2 > P3), skipping blocked/done/cut tasks.', + inputSchema: { + type: 'object', + properties: { + owner: { + type: 'string', + description: + 'Filter by owner (@me, @agent, @defer). If omitted, returns highest priority regardless of owner.', + }, + }, + }, + }, + { + name: 'update_master_task', + description: + 'Update a task in master-tasks.md by task ID (e.g. T01). Can update status, priority, owner, branch, notes, or sync target.', + inputSchema: { + type: 'object', + properties: { + task_id: { + type: 'string', + description: 'Task ID (e.g. T01, T02)', + }, + status: { + type: 'string', + enum: ['todo', 'active', 'done', 'blocked', 'cut'], + description: 'New status', + }, + priority: { + type: 'string', + enum: ['P0', 'P1', 'P2', 'P3'], + description: 'New priority', + }, + owner: { + type: 'string', + description: 'New owner (@me, @agent, @defer)', + }, + branch_pr: { + type: 'string', + description: 'Branch name or PR link', + }, + notes: { + type: 'string', + description: 'Updated notes', + }, + sync: { + type: 'string', + enum: ['local', 'linear', 'gh'], + description: 'Sync target', + }, + }, + required: ['task_id'], + }, + }, + { + name: 'create_master_task', + description: + 'Add a new task to master-tasks.md. Auto-assigns the next available ID (T01, T02...).', + inputSchema: { + type: 'object', + properties: { + task: { + type: 'string', + description: 'Task description', + }, + priority: { + type: 'string', + enum: ['P0', 'P1', 'P2', 'P3'], + default: 'P1', + description: 'Priority level', + }, + owner: { + type: 'string', + default: '@me', + description: 'Task owner (@me, @agent, @defer)', + }, + sync: { + type: 'string', + enum: ['local', 'linear', 'gh'], + default: 'local', + description: 'Sync target', + }, + notes: { + type: 'string', + description: 'Optional notes', + }, + }, + required: ['task'], + }, + }, + ]; + } } diff --git a/src/integrations/ralph/context/context-budget-manager.ts b/src/integrations/ralph/context/context-budget-manager.ts index 47e51c07..023d4394 100644 --- a/src/integrations/ralph/context/context-budget-manager.ts +++ b/src/integrations/ralph/context/context-budget-manager.ts @@ -4,6 +4,7 @@ */ import { logger } from '../../../core/monitoring/logger.js'; +import { estimateTokens as estimateTokensCore } from '../../../core/cache/token-estimator.js'; import { IterationContext, TaskContext, @@ -19,7 +20,6 @@ export class ContextBudgetManager { private config: RalphStackMemoryConfig['contextBudget']; private tokenUsage: Map = new Map(); private readonly DEFAULT_MAX_TOKENS = 4000; - private readonly TOKEN_CHAR_RATIO = 0.25; // Rough estimate: 1 token ≈ 4 chars constructor(config?: Partial) { this.config = { @@ -41,17 +41,7 @@ export class ContextBudgetManager { */ estimateTokens(text: string): number { if (!text) return 0; - - // More accurate estimation based on common patterns - const baseTokens = text.length * this.TOKEN_CHAR_RATIO; - - // Adjust for code content (typically more dense) - const codeMultiplier = this.detectCodeContent(text) ? 1.2 : 1.0; - - // Adjust for JSON content (typically less dense) - const jsonMultiplier = this.detectJsonContent(text) ? 0.9 : 1.0; - - return Math.ceil(baseTokens * codeMultiplier * jsonMultiplier); + return estimateTokensCore(text); } /** @@ -347,32 +337,6 @@ export class ContextBudgetManager { } } - /** - * Detect if text contains code - */ - private detectCodeContent(text: string): boolean { - const codePatterns = [ - /function\s+\w+\s*\(/, - /class\s+\w+/, - /const\s+\w+\s*=/, - /import\s+.*from/, - /\{[\s\S]*\}/, - ]; - return codePatterns.some((pattern) => pattern.test(text)); - } - - /** - * Detect if text contains JSON - */ - private detectJsonContent(text: string): boolean { - try { - JSON.parse(text); - return true; - } catch { - return text.includes('"') && text.includes(':') && text.includes('{'); - } - } - /** * Truncate text with ellipsis */ diff --git a/src/integrations/ralph/patterns/oracle-worker-pattern.ts b/src/integrations/ralph/patterns/oracle-worker-pattern.ts index 71fd31c5..182f9a81 100644 --- a/src/integrations/ralph/patterns/oracle-worker-pattern.ts +++ b/src/integrations/ralph/patterns/oracle-worker-pattern.ts @@ -9,6 +9,7 @@ import { v4 as uuidv4 } from 'uuid'; import { logger } from '../../../core/monitoring/logger.js'; +import { estimateTokens } from '../../../core/cache/token-estimator.js'; import { SwarmCoordinator } from '../swarm/swarm-coordinator.js'; import { RalphStackMemoryBridge } from '../bridge/ralph-stackmemory-bridge.js'; @@ -139,7 +140,7 @@ export class OracleWorkerCoordinator extends SwarmCoordinator { const taskId = uuidv4(); const oraclePrompt = this.buildOraclePrompt(type, description, hints); - const estimatedTokens = this.estimateTokens(oraclePrompt); + const estimatedTokens = estimateTokens(oraclePrompt); logger.info('Oracle task created', { taskId, @@ -238,7 +239,7 @@ Remember: Your intelligence is expensive. Focus on high-value strategic thinking const result = await ralph.run(); // Track Oracle costs - const tokens = this.estimateTokens(result); + const tokens = estimateTokens(result); const cost = tokens * this.oracle.costPerToken; this.costTracker.oracleSpent += cost; @@ -353,7 +354,7 @@ Execute your task now. // Track worker costs const workerModel = this.selectWorkerForTask(task); - const tokens = this.estimateTokens(result); + const tokens = estimateTokens(result); const cost = tokens * workerModel.costPerToken; this.costTracker.workerSpent += cost; @@ -415,14 +416,6 @@ Execute your task now. return []; } - /** - * Estimate token usage for cost calculation - */ - private estimateTokens(text: string): number { - // Rough estimation: ~4 characters per token - return Math.ceil(text.length / 4); - } - /** * Log cost analysis and efficiency metrics */ diff --git a/src/orchestrators/multimodal/determinism.ts b/src/orchestrators/multimodal/determinism.ts index 3d4163f7..08f757c9 100644 --- a/src/orchestrators/multimodal/determinism.ts +++ b/src/orchestrators/multimodal/determinism.ts @@ -1,4 +1,5 @@ import { createHash } from 'crypto'; +import { estimateTokens } from '../../core/cache/token-estimator.js'; import { appendFileSync, existsSync, @@ -116,7 +117,7 @@ function normalizeResult(result: HarnessResult) { function estimateContextTokens(result: HarnessResult): number { const normalized = normalizeResult(result); - return Math.ceil(stableStringify(normalized).length / 4); + return estimateTokens(stableStringify(normalized)); } function toSnapshot(result: HarnessResult, index: number): DeterminismSnapshot { diff --git a/src/orchestrators/multimodal/harness.ts b/src/orchestrators/multimodal/harness.ts index 1d1920ea..0d4dbc69 100644 --- a/src/orchestrators/multimodal/harness.ts +++ b/src/orchestrators/multimodal/harness.ts @@ -1,3 +1,4 @@ +import { estimateTokens } from '../../core/cache/token-estimator.js'; import { callClaude, callCodexCLI, @@ -293,7 +294,7 @@ export async function runSpike( editAttempts: editMetrics.editAttempts, editSuccesses: editMetrics.editSuccesses, editFuzzyFallbacks: editMetrics.editFuzzyFallbacks, - contextTokens: Math.ceil(finalDiff.length / 4), + contextTokens: estimateTokens(finalDiff), }; // Persist audit + metrics unless explicitly disabled for replay/smoke runs. diff --git a/src/skills/recursive-agent-orchestrator.ts b/src/skills/recursive-agent-orchestrator.ts index 68babcb0..c25622ce 100644 --- a/src/skills/recursive-agent-orchestrator.ts +++ b/src/skills/recursive-agent-orchestrator.ts @@ -15,6 +15,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { logger } from '../core/monitoring/logger.js'; +import { estimateTokens } from '../core/cache/token-estimator.js'; import { FrameManager } from '../core/context/index.js'; import { DualStackManager } from '../core/context/dual-stack-manager.js'; import { ContextRetriever } from '../core/retrieval/context-retriever.js'; @@ -550,8 +551,7 @@ Rules: // Process agent response node.result = response.result; - node.tokens = - response.tokens || this.estimateTokens(JSON.stringify(response)); + node.tokens = response.tokens || estimateTokens(JSON.stringify(response)); node.cost = this.calculateNodeCost(node.tokens, agentConfig.model); // Share results with other agents if real-time sharing is enabled @@ -820,11 +820,6 @@ Rules: ].join('\n'); } - private estimateTokens(text: string): number { - // Rough estimation: 1 token ≈ 4 characters - return Math.ceil(text.length / 4); - } - private async shareAgentResults(_node: TaskNode): Promise { // Share results with other agents via Redis or shared context logger.debug('Sharing agent results', { nodeId: _node.id });