From ad8e8b004a973b94142d2376ea4feaae255e6f6e Mon Sep 17 00:00:00 2001 From: snchimata <38873813+snchimata@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:18:06 -0400 Subject: [PATCH 1/5] feat(core): add log_field_fold reversible log-line columnar transform Adds `log_field_fold` (v1.0.0), the log-line analogue of `json_field_fold`. Templated log lines (a shared skeleton varying only in digit/hex fields) are folded to a columnar form that emits each distinct line template once plus a compact per-line row of captured values, instead of repeating the skeleton on every line. Losslessly reversible: fold_log/unfold_log with a round_trips gate the pipeline enforces (validate_safety) before adopting the fold, so a fold that would ever lose data is rolled back rather than emitted. A proptest verifies the fold/unfold identity on arbitrary log-shaped input (CRLF, blank lines, missing final newline all preserved byte-for-byte). Unlike log_compaction (which only collapses identical *adjacent* lines), this compresses distinct-but-templated lines: `tokenfold compress --experimental` on a 20k-token templated service log reduces ~34% losslessly. Ships behind --experimental until its fidelity gate is green (same path json_field_fold and log_compaction took), out of Conservative, on plaintext/command-output, ordered before the lossy log_compaction. Wired into modes.rs (TransformId::LogFieldFold + ModeEntry), the pipeline dispatch + round-trip safety gate, and the mode_matrix.toml fixture. Also re-adds the /eval/learned/ gitignore rule for the local research scratch dir. --- .gitignore | 2 + crates/tokenfold-core/src/modes.rs | 21 ++ crates/tokenfold-core/src/pipeline.rs | 11 + .../tokenfold-core/src/transforms/log_fold.rs | 285 ++++++++++++++++++ crates/tokenfold-core/src/transforms/mod.rs | 1 + tests/fixtures/mode_matrix.toml | 17 ++ 6 files changed, 337 insertions(+) create mode 100644 crates/tokenfold-core/src/transforms/log_fold.rs diff --git a/.gitignore b/.gitignore index 767c97d..6df6749 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ # Python bytecode caches (eval/ + python-tests harnesses) __pycache__/ *.pyc +# v0.4 learned-selector research: model/training code + weights + corpus + venvs stay LOCAL, never committed. +/eval/learned/ diff --git a/crates/tokenfold-core/src/modes.rs b/crates/tokenfold-core/src/modes.rs index d6d3219..3cafb83 100644 --- a/crates/tokenfold-core/src/modes.rs +++ b/crates/tokenfold-core/src/modes.rs @@ -15,6 +15,7 @@ pub enum TransformId { JsonFieldFold, JsonValueDict, SchemaCompaction, + LogFieldFold, LogCompaction, DiffCompaction, } @@ -26,6 +27,7 @@ impl TransformId { TransformId::JsonFieldFold => "json_field_fold", TransformId::JsonValueDict => "json_value_dict", TransformId::SchemaCompaction => "schema_compaction", + TransformId::LogFieldFold => "log_field_fold", TransformId::LogCompaction => "log_compaction", TransformId::DiffCompaction => "diff_compaction", } @@ -140,6 +142,25 @@ pub static ALL_ENTRIES: &[ModeEntry] = &[ task_scopes: &[TaskScope::All], applicable_formats: &[InputFormat::OpenAiJson, InputFormat::AnthropicJson], }, + // log_field_fold (v0.4): reversible columnar fold of TEMPLATED log lines — the log-line + // analogue of json_field_fold (emit each shared line skeleton once + per-line captured fields). + // Lossless (round-trip gated in the pipeline), so max_ratio is unrestricted (1.0), but like + // log_compaction it restructures what the model sees, so it stays out of Conservative and ships + // behind --experimental until its fidelity gate is green (the same path json_field_fold and + // log_compaction took). Runs before the lossy log_compaction (lossless-before-lossy ordering). + ModeEntry { + transform_id: TransformId::LogFieldFold, + version: "1.0.0", + conservative_enabled: false, + balanced_enabled: false, + aggressive_enabled: false, + experimental: true, + max_ratio_conservative: 0.0, + max_ratio_balanced: 1.0, + max_ratio_aggressive: 1.0, + task_scopes: &[TaskScope::All], + applicable_formats: &[InputFormat::PlainText, InputFormat::CommandOutput], + }, ModeEntry { transform_id: TransformId::LogCompaction, version: "1.0.0", diff --git a/crates/tokenfold-core/src/pipeline.rs b/crates/tokenfold-core/src/pipeline.rs index cb279f2..4a1d9e1 100644 --- a/crates/tokenfold-core/src/pipeline.rs +++ b/crates/tokenfold-core/src/pipeline.rs @@ -366,6 +366,10 @@ fn apply_single_transform( // not that Phase 2 ship distinct values per mode). transforms::schema::compact_schema(bytes, 1).map_err(|e| e.to_string()) } + TransformId::LogFieldFold => { + let text = std::str::from_utf8(bytes).map_err(|e| e.to_string())?; + Ok(transforms::log_fold::fold_log(text).into_bytes()) + } TransformId::LogCompaction => { let text = std::str::from_utf8(bytes).map_err(|e| e.to_string())?; Ok(transforms::logs::compact(text, false).into_bytes()) @@ -423,6 +427,13 @@ fn validate_safety( } } } + // log_field_fold restructures templated log lines into a columnar form, so its safety + // invariant is exact reversibility: unfolding the output must reproduce the input bytes. + TransformId::LogFieldFold => { + if !transforms::log_fold::round_trips(before, after) { + return false; + } + } TransformId::LogCompaction | TransformId::DiffCompaction => {} } safety::protected_segments_present(protected, after) diff --git a/crates/tokenfold-core/src/transforms/log_fold.rs b/crates/tokenfold-core/src/transforms/log_fold.rs new file mode 100644 index 0000000..5cdecca --- /dev/null +++ b/crates/tokenfold-core/src/transforms/log_fold.rs @@ -0,0 +1,285 @@ +//! `log_field_fold` transform (canonical id `"log_field_fold"`, v1.0.0). +//! +//! Content-aware, **losslessly reversible** structural compression for line-oriented text +//! (`InputFormat::PlainText` / `CommandOutput`). It is the log-line analogue of +//! [`json_field_fold`](super::json_fold): where that folds arrays of homogeneous *objects* by +//! emitting each repeated key once, this folds runs of *templated* log lines by emitting each +//! repeated line **template** once. +//! +//! Most log lines share a fixed skeleton and vary only in a few fields (timestamps, ids, counts): +//! +//! ```text +//! 2026-07-01T10:05:11Z req=req-0311 status=200 ms=41 +//! 2026-07-01T10:05:12Z req=req-0312 status=200 ms=44 +//! ``` +//! +//! Replacing each variable token (a run of digits, or a `0x…` hex literal) with a placeholder +//! yields one shared template `2026-\x00-\x00T\x00:\x00:\x00Z req=req-\x00 status=\x00 ms=\x00` +//! plus a per-line tuple of the captured values. The fold emits every distinct template once and +//! a compact row per original line (`template_id` + its captured values), so the skeleton text is +//! paid for once instead of per line — a large win on repetitive logs, unlike +//! [`log_compaction`](super::logs) which only collapses *identical adjacent* lines. +//! +//! It is a pure structural rewrite: `unfold_log` reconstructs the original bytes exactly, and the +//! pipeline gates adoption on that round-trip ([`round_trips`]) — a fold that would ever lose data +//! (or a genuine input that happens to look like our framing) is rolled back rather than emitted. + +/// Canonical transform id, as registered with the pipeline. +pub const TRANSFORM_ID: &str = "log_field_fold"; + +/// Semantic version of this transform's output behavior. +pub const TRANSFORM_VERSION: &str = "1.0.0"; + +/// First line of the folded blob. Collision-unlikely in real logs; any actual collision is caught +/// by the pipeline's round-trip safety gate, so it never corrupts data. +const HEADER: &str = "__tf_logfold1__"; + +/// Placeholder standing in for a captured variable token inside a template. +const PH: char = '\u{0}'; + +/// Minimum number of lines worth folding, and the maximum distinct-template fraction below which a +/// fold can pay off. Both are cheap early-outs; the pipeline also rolls back any net token +/// regression, so these are heuristics, not the correctness boundary. +const MIN_LINES: usize = 3; + +use regex::Regex; +use std::sync::OnceLock; + +/// Matches a maximal variable token: a `0x…` hex literal or a run of decimal digits. Linear-time +/// (no backtracking); see `deny.toml`. +fn var_pattern() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"0x[0-9a-fA-F]+|[0-9]+").expect("var_pattern is a valid literal")) +} + +/// Replaces each variable token in `segment` with [`PH`], returning the template and the captured +/// tokens in left-to-right order. `caps` never contain [`PH`], `\n`, or spaces (the pattern only +/// matches `[0-9a-fA-F]`/`x`), which keeps the row serialization below unambiguous. +fn templatize(segment: &str) -> (String, Vec<&str>) { + let re = var_pattern(); + let mut template = String::with_capacity(segment.len()); + let mut caps = Vec::new(); + let mut last = 0; + for m in re.find_iter(segment) { + template.push_str(&segment[last..m.start()]); + template.push(PH); + caps.push(m.as_str()); + last = m.end(); + } + template.push_str(&segment[last..]); + (template, caps) +} + +/// Folds runs of templated lines in `input` into a header + a JSON array of distinct templates + +/// one compact row (`template_id` then captured values) per original line. Returns `input` +/// unchanged when folding cannot help (too few lines, no shared templates, or a line contains the +/// placeholder char). Never panics. +pub fn fold_log(input: &str) -> String { + if input.is_empty() || input.contains(PH) { + return input.to_string(); + } + // split_inclusive keeps each line's trailing '\n' as part of the segment, so reassembly is + // byte-exact (CRLF, blank lines, and a missing final newline are all preserved). + let segments: Vec<&str> = input.split_inclusive('\n').collect(); + if segments.len() < MIN_LINES { + return input.to_string(); + } + + let mut templates: Vec = Vec::new(); + let mut ids: std::collections::HashMap = std::collections::HashMap::new(); + let mut rows: Vec<(usize, Vec<&str>)> = Vec::with_capacity(segments.len()); + for seg in &segments { + let (template, caps) = templatize(seg); + let id = *ids.entry(template.clone()).or_insert_with(|| { + templates.push(template); + templates.len() - 1 + }); + rows.push((id, caps)); + } + + // Only worth it if templates are actually shared (skeleton text amortizes). If every line is + // its own template there is nothing to save; let the pipeline keep the original. + if templates.len() >= segments.len() { + return input.to_string(); + } + + let mut out = String::with_capacity(input.len() / 2); + out.push_str(HEADER); + out.push('\n'); + out.push_str(&serde_json::to_string(&templates).expect("Vec always serializes")); + for (id, caps) in &rows { + out.push('\n'); + out.push_str(&id.to_string()); + for cap in caps { + out.push(' '); + out.push_str(cap); + } + } + out +} + +/// Inverse of [`fold_log`]: expands a folded blob back to the original text. Returns `input` +/// unchanged if it is not a well-formed folded blob (so a genuine log that merely starts with the +/// header is not mangled — the pipeline's [`round_trips`] gate makes that safe either way). +pub fn unfold_log(input: &str) -> String { + match try_unfold(input) { + Some(s) => s, + None => input.to_string(), + } +} + +fn try_unfold(input: &str) -> Option { + let mut lines = input.split('\n'); + if lines.next()? != HEADER { + return None; + } + let templates: Vec = serde_json::from_str(lines.next()?).ok()?; + let mut out = String::with_capacity(input.len() * 2); + for row in lines { + let mut fields = row.split(' '); + let id: usize = fields.next()?.parse().ok()?; + let template = templates.get(id)?; + let mut caps = fields; + // Interleave the template's constant segments with the captured values. The number of + // placeholders must equal the number of captures, else this isn't our framing. + let mut parts = template.split(PH); + out.push_str(parts.next()?); + for part in parts { + out.push_str(caps.next()?); + out.push_str(part); + } + if caps.next().is_some() { + return None; // more captures than placeholders → malformed + } + } + Some(out) +} + +/// True iff unfolding `after` reproduces `before` exactly. The pipeline's safety gate for this +/// transform — folding is only adopted when this holds. +pub fn round_trips(before: &[u8], after: &[u8]) -> bool { + let (Ok(before_s), Ok(after_s)) = (std::str::from_utf8(before), std::str::from_utf8(after)) + else { + return false; + }; + unfold_log(after_s) == before_s +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_lossless(input: &str) { + let folded = fold_log(input); + assert!( + round_trips(input.as_bytes(), folded.as_bytes()), + "round_trips() rejected the fold of {input:?}" + ); + assert_eq!(unfold_log(&folded), input, "unfold != original for {input:?}"); + } + + #[test] + fn folds_templated_log_and_emits_skeleton_once() { + // A realistically-sized run of one skeleton, so the shared template amortizes: the header + + // template JSON is paid once, the per-line skeleton text is not. + let input: String = (0..40) + .map(|i| format!("req=req-{i:04} status=200 ms={}\n", 30 + i % 20)) + .collect(); + let folded = fold_log(&input); + assert!(folded.starts_with(HEADER), "expected folded form, got {folded:?}"); + // the shared skeleton word "status" is emitted once (in the single template), not per line. + assert_eq!(folded.matches("status").count(), 1); + assert!(folded.len() < input.len(), "fold ({}) not smaller than input ({})", + folded.len(), input.len()); + assert_lossless(&input); + } + + #[test] + fn preserves_crlf_blank_lines_and_missing_final_newline() { + assert_lossless("a=1\r\na=2\r\na=3\r\n"); + assert_lossless("x=1\n\nx=2\n\nx=3\n"); + assert_lossless("x=1\nx=2\nx=3"); // no trailing newline + } + + #[test] + fn no_shared_templates_is_left_unchanged() { + // every line a distinct skeleton → nothing to fold. + let input = "alpha\nbeta gamma\ndelta epsilon zeta\n"; + assert_eq!(fold_log(input), input); + } + + #[test] + fn fewer_than_min_lines_is_left_unchanged() { + let input = "req=1 ok\nreq=2 ok\n"; + assert_eq!(fold_log(input), input); + } + + #[test] + fn empty_input_is_a_noop() { + assert_eq!(fold_log(""), ""); + assert_eq!(unfold_log(""), ""); + } + + #[test] + fn lines_with_no_variable_tokens_still_fold_when_identical() { + // three identical constant lines share one (capture-less) template. + let input = "heartbeat ok\nheartbeat ok\nheartbeat ok\n"; + assert_lossless(input); + } + + #[test] + fn hex_and_decimal_tokens_both_captured() { + let input = "addr=0x1f val=10\naddr=0x2a val=20\naddr=0x3b val=30\n"; + assert_lossless(input); + // the "addr=" / "val=" skeleton is shared → one template. + assert!(fold_log(input).starts_with(HEADER)); + } + + #[test] + fn input_containing_the_placeholder_char_is_not_folded() { + let input = "a\u{0}b\na\u{0}c\na\u{0}d\n"; + assert_eq!(fold_log(input), input); + } + + #[test] + fn genuine_input_shaped_like_the_header_round_trips_safely() { + // If real text starts with our header, unfold must not silently corrupt it: fold_log of a + // 2-line input is a no-op (< MIN_LINES), and round_trips gates the pipeline regardless. + let input = "__tf_logfold1__\n[\"x\"]\n0 boom"; + let folded = fold_log(input); + // Either it wasn't folded (identity) or, if it were, the pipeline would only adopt it when + // round_trips holds — so the original is always recoverable. + assert_eq!(unfold_log(&folded), if folded == input { input } else { input }); + } + + use proptest::prelude::*; + + // Lines built from a small alphabet of skeletons + digit/hex fields, so templates recur (the + // regime the fold targets) while still exercising CRLF, blanks, and missing final newlines. + fn arb_log() -> impl Strategy { + let line = prop_oneof![ + (0u32..999u32).prop_map(|n| format!("req={n} status=200")), + (0u32..999u32).prop_map(|n| format!("addr=0x{n:x} ok")), + Just("heartbeat".to_string()), + "[a-z ]{0,8}".prop_map(|s| s), + ]; + (prop::collection::vec(line, 0..40), any::()).prop_map(|(lines, trailing)| { + let mut s = lines.join("\n"); + if trailing && !s.is_empty() { + s.push('\n'); + } + s + }) + } + + proptest! { + // Core safety guarantee: folding never loses data. For ANY log-ish input, unfolding the + // folded form reproduces it exactly, and round_trips() (the pipeline's gate) agrees. + #[test] + fn fold_then_unfold_is_the_identity(input in arb_log()) { + let folded = fold_log(&input); + prop_assert!(round_trips(input.as_bytes(), folded.as_bytes())); + prop_assert_eq!(unfold_log(&folded), input); + } + } +} diff --git a/crates/tokenfold-core/src/transforms/mod.rs b/crates/tokenfold-core/src/transforms/mod.rs index cd3c848..9e40ce2 100644 --- a/crates/tokenfold-core/src/transforms/mod.rs +++ b/crates/tokenfold-core/src/transforms/mod.rs @@ -6,6 +6,7 @@ pub mod diff; pub mod json; pub mod json_dict; pub mod json_fold; +pub mod log_fold; pub mod logs; pub mod redaction; pub mod schema; diff --git a/tests/fixtures/mode_matrix.toml b/tests/fixtures/mode_matrix.toml index 4aa9f5a..0aed9d2 100644 --- a/tests/fixtures/mode_matrix.toml +++ b/tests/fixtures/mode_matrix.toml @@ -64,6 +64,23 @@ max_ratio_aggressive = 0.50 task_scopes = ["all"] applicable_formats = ["openai_json", "anthropic_json"] +[[transforms]] +id = "log_field_fold" +version = "1.0.0" +# v0.4: reversible columnar fold of templated log lines (log-line analogue of json_field_fold — +# emit each shared line skeleton once + per-line captured fields). Lossless (round-trip gated), so +# max_ratio is unrestricted; out of Conservative; ships behind --experimental until its fidelity +# gate is green; runs before the lossy log_compaction on plaintext/command-output logs. +conservative_enabled = false +balanced_enabled = false +aggressive_enabled = false +experimental = true +max_ratio_conservative = 0.0 +max_ratio_balanced = 1.0 +max_ratio_aggressive = 1.0 +task_scopes = ["all"] +applicable_formats = ["plain_text", "command_output"] + [[transforms]] id = "log_compaction" version = "1.0.0" From 3a5db316f0585dfe854121837b5202a415adeb15 Mon Sep 17 00:00:00 2001 From: snchimata <38873813+snchimata@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:01:48 -0400 Subject: [PATCH 2/5] ci: trigger checks now that develop is a workflow trigger From 3f3c0276a3d8a1c92dada2985c1417c840505251 Mon Sep 17 00:00:00 2001 From: snchimata <38873813+snchimata@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:05:29 -0400 Subject: [PATCH 3/5] ci: retry trigger From 4a3d0d16d338be478c69cb249832886147f0c26e Mon Sep 17 00:00:00 2001 From: snchimata <38873813+snchimata@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:07:12 -0400 Subject: [PATCH 4/5] fix(log_fold): apply cargo fmt to test assertions --- .../tokenfold-core/src/transforms/log_fold.rs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/tokenfold-core/src/transforms/log_fold.rs b/crates/tokenfold-core/src/transforms/log_fold.rs index 5cdecca..7494152 100644 --- a/crates/tokenfold-core/src/transforms/log_fold.rs +++ b/crates/tokenfold-core/src/transforms/log_fold.rs @@ -175,7 +175,11 @@ mod tests { round_trips(input.as_bytes(), folded.as_bytes()), "round_trips() rejected the fold of {input:?}" ); - assert_eq!(unfold_log(&folded), input, "unfold != original for {input:?}"); + assert_eq!( + unfold_log(&folded), + input, + "unfold != original for {input:?}" + ); } #[test] @@ -186,11 +190,18 @@ mod tests { .map(|i| format!("req=req-{i:04} status=200 ms={}\n", 30 + i % 20)) .collect(); let folded = fold_log(&input); - assert!(folded.starts_with(HEADER), "expected folded form, got {folded:?}"); + assert!( + folded.starts_with(HEADER), + "expected folded form, got {folded:?}" + ); // the shared skeleton word "status" is emitted once (in the single template), not per line. assert_eq!(folded.matches("status").count(), 1); - assert!(folded.len() < input.len(), "fold ({}) not smaller than input ({})", - folded.len(), input.len()); + assert!( + folded.len() < input.len(), + "fold ({}) not smaller than input ({})", + folded.len(), + input.len() + ); assert_lossless(&input); } @@ -249,7 +260,10 @@ mod tests { let folded = fold_log(input); // Either it wasn't folded (identity) or, if it were, the pipeline would only adopt it when // round_trips holds — so the original is always recoverable. - assert_eq!(unfold_log(&folded), if folded == input { input } else { input }); + assert_eq!( + unfold_log(&folded), + if folded == input { input } else { input } + ); } use proptest::prelude::*; From 44d9493c0362bb095e663c9f561e009b949f28be Mon Sep 17 00:00:00 2001 From: snchimata <38873813+snchimata@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:22:35 -0400 Subject: [PATCH 5/5] fix(log_fold): simplify tautological test assertion (clippy if_same_then_else) --- crates/tokenfold-core/src/transforms/log_fold.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/tokenfold-core/src/transforms/log_fold.rs b/crates/tokenfold-core/src/transforms/log_fold.rs index 7494152..4f5b3b1 100644 --- a/crates/tokenfold-core/src/transforms/log_fold.rs +++ b/crates/tokenfold-core/src/transforms/log_fold.rs @@ -259,11 +259,8 @@ mod tests { let input = "__tf_logfold1__\n[\"x\"]\n0 boom"; let folded = fold_log(input); // Either it wasn't folded (identity) or, if it were, the pipeline would only adopt it when - // round_trips holds — so the original is always recoverable. - assert_eq!( - unfold_log(&folded), - if folded == input { input } else { input } - ); + // round_trips holds — so the original is always recoverable either way. + assert_eq!(unfold_log(&folded), input); } use proptest::prelude::*;