From bfcf246efa8aa4d4b6729d1bc754d5227715f95b Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Mon, 6 Jul 2026 00:21:44 +0800 Subject: [PATCH 01/11] init --- retina.html | 3918 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 3918 insertions(+) create mode 100644 retina.html diff --git a/retina.html b/retina.html new file mode 100644 index 000000000..bcfce3505 --- /dev/null +++ b/retina.html @@ -0,0 +1,3918 @@ + + + + + + Retina Animation Demo + + + +
+
+ + +
+ + +
+ +
+
+ +
+
+ + + + From 071827e308f2bc7c202f459d628401e77a92f906 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Mon, 6 Jul 2026 23:24:05 +0800 Subject: [PATCH 02/11] fix: retina write buffer --- retina.html | 431 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 264 insertions(+), 167 deletions(-) diff --git a/retina.html b/retina.html index bcfce3505..b6710342c 100644 --- a/retina.html +++ b/retina.html @@ -13,8 +13,8 @@ --surface-subtle: #f7f9fc; --text: #122033; --muted: #667386; - --border: #cbd5e1; - --border-soft: #dde5ee; + --border: #b8c4d3; + --border-soft: #c7d2df; --primary: #315fcb; --primary-hover: #254ead; --primary-soft: #eaf0ff; @@ -586,8 +586,8 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

}; const TYPE = { - moduleTitle: 16, - subTitle: 14, + moduleTitle: 18, + subTitle: 16, text: 12 }; const MODULE_TITLE_X = 16; @@ -628,14 +628,15 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const TOP_RUNTIME_GAP = 14; const RUNTIME_Y = SERVICE_Y + SERVICE_H + TOP_RUNTIME_GAP; const RUNTIME_H = MIDDLE_BOTTOM - RUNTIME_Y; + const RETINA_REPLAYER_H = 500; const LEFT_PANEL_X = 44; const LEFT_PANEL_W = 390; const SERVICE_X = LEFT_PANEL_X + LEFT_PANEL_W + 10; const SERVICE_RIGHT = 1424; const SERVICE_GAP = 18; const RUNTIME_STORAGE_GAP = 18; - const RUNTIME_W = Math.round((SERVICE_RIGHT - SERVICE_X - RUNTIME_STORAGE_GAP) * .50); - const STORAGE_X = SERVICE_X + RUNTIME_W + RUNTIME_STORAGE_GAP; + const RETINA_REPLAYER_W = 450; + const STORAGE_X = SERVICE_X + RETINA_REPLAYER_W + RUNTIME_STORAGE_GAP; const STORAGE_W = SERVICE_RIGHT - STORAGE_X; const TOP_SERVICE_W = Math.round((SERVICE_RIGHT - SERVICE_X - SERVICE_GAP * 2) / 3); const INDEX_W = SERVICE_RIGHT - (SERVICE_X + (TOP_SERVICE_W + SERVICE_GAP) * 2); @@ -661,7 +662,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

tx: { x: SERVICE_X, y: SERVICE_Y, w: TOP_SERVICE_W, h: TX_SERVICE_H, title: 'Transaction Service', sub: 'MVCC time boundary: HWM / LWM' }, catalog: { x: SERVICE_X + TOP_SERVICE_W + SERVICE_GAP, y: SERVICE_Y, w: TOP_SERVICE_W, h: SERVICE_H, title: 'Metadata Service', sub: 'schema, stats, visible file list' }, index: { x: SERVICE_X + (TOP_SERVICE_W + SERVICE_GAP) * 2, y: SERVICE_Y, w: INDEX_W, h: SERVICE_H, title: 'Index Service', sub: 'primaryKey→rowId + rowId→RowLocation' }, - runtime: { x: SERVICE_X, y: RUNTIME_Y, w: RUNTIME_W, h: RUNTIME_H, title: 'Retina replayer', sub: 'RetinaWorkerService RPC + write / visibility state' }, + runtime: { x: SERVICE_X, y: RUNTIME_Y, w: RETINA_REPLAYER_W, h: RETINA_REPLAYER_H, title: 'Retina replayer', sub: 'RetinaWorkerService RPC + write / visibility state' }, storage: { x: STORAGE_X, y: RUNTIME_Y, w: STORAGE_W, h: RUNTIME_H, title: 'Storage', sub: 'flushed immutable memtable + regular columnar files' }, query: { x: 1450, y: MIDDLE_TOP, w: 316, h: MIDDLE_H, title: 'Query Engine', sub: 'parallel splits → VFoR gate → visible batches' } @@ -728,9 +729,9 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

txState: 'Transaction Service ready' }, lanes: [ - laneModel(0, 'I:pk160', 5, 2, ['(2,1276)'], 1266), - laneModel(1, 'U:pk132', 7, 3, ['(5,1278)', '(7,1282)'], 1264), - laneModel(2, 'D:pk208', 4, 2, ['(1,1279)'], 1267) + laneModel(0, 'I:pk160', 5, 0, ['(2,1276)'], 1266), + laneModel(1, 'U:pk132', 7, 0, ['(5,1278)', '(7,1282)'], 1264), + laneModel(2, 'D:pk208', 4, 0, ['(1,1279)'], 1267) ], index: { hot: 'pk132', @@ -741,6 +742,8 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

storage: { regularFiles: ['file-041.pxl', 'file-042.pxl'], objectEntries: ['obj-77'], + pendingFlushObject: null, + flushedHot: null, staleRatio: 0.31, agingQueue: [], newFile: 'none', @@ -929,16 +932,16 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

['ingress', 'tx'], [tok('begin tx8421', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=1290', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], m => { m.sink.sinkStep = 'begin'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=1290'; }, 'mvcc', ['tx']), step('Pixels Sink creates the INSERT row in the mirror transaction and sends its replay RPC.', 3.8, - ['ingress', 'runtime'], [tok('INSERT', 'ingress.insertRight', 'runtime.left', flow.replay, .18, .86)], + ['ingress', 'runtime'], [tok('INSERT', 'ingress.right', 'runtime.left', flow.replay, .18, .86)], m => { m.sink.sinkStep = 'row1'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay INSERT'; m.sink.rpcBatch = 'INSERT pending'; m.lanes[0].currentOp = 'I:row'; }, 'dispatch', ['dispatch']), step('Pixels Sink creates the DELETE row next and sends its replay RPC after INSERT.', 3.8, - ['ingress', 'runtime'], [tok('DELETE', 'ingress.deleteRight', 'runtime.left', flow.delete, .18, .86)], + ['ingress', 'runtime'], [tok('DELETE', 'ingress.right', 'runtime.left', flow.delete, .18, .86)], m => { m.sink.sinkStep = 'row2'; m.sink.requestState = 'replay DELETE'; m.sink.rpcBatch = 'DELETE pending'; m.lanes[1].currentOp = 'D:row'; }, 'dispatch', ['dispatch']), step('Pixels Sink creates the UPDATE row last and sends its replay RPC.', 3.8, - ['ingress', 'runtime'], [tok('UPDATE', 'ingress.updateRight', 'runtime.left', flow.index, .18, .86)], + ['ingress', 'runtime'], [tok('UPDATE', 'ingress.right', 'runtime.left', flow.index, .18, .86)], m => { m.sink.sinkStep = 'row3'; m.sink.requestState = 'replay UPDATE'; m.sink.rpcBatch = 'UPDATE pending'; m.lanes[2].currentOp = 'U:row'; }, 'dispatch', ['dispatch']), step('After every replay RPC returns OK, Pixels Sink can finish the mirror transaction.', 3.6, - ['ingress', 'runtime'], [tok('all replay OK', 'runtime.left', 'ingress.replayRight', flow.replay, .16, .84)], + ['ingress', 'runtime'], [tok('all replay OK', 'runtime.left', 'ingress.right', flow.replay, .16, .84)], m => { m.sink.sinkStep = 'rowsOk'; m.sink.requestState = 'all replay OK'; m.sink.rpcBatch = 'all replay OK'; }, 'mvcc', []), step('Pixels Sink commits tx8421 and Transaction Service advances HWM internally.', 4.2, ['ingress', 'tx'], [tok('commit tx8421', 'ingress.commitRight', 'tx.left', flow.tx, .18, .58)], @@ -949,53 +952,65 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

['cdcStream', 'ingress'], [tok('tx43 rows', 'cdcStream.out', 'ingress.top', flow.cdc, .08, .78)], m => { m.sink.sinkStep = 'replay'; m.sink.coordState = 'tx43 mirror tx'; m.sink.requestState = 'replay rows'; m.sink.routingRule = 'inspect row image'; }, 'cdc', ['cdc']), step('Insert rows use the after-image primary key, so the new tuple can be staged for replay.', 4.0, - ['ingress', 'lane0'], [tok('I:pk160', 'ingress.replayRight', 'lane0.in', flow.replay, .12, .82)], + ['ingress', 'lane0'], [tok('I:pk160', 'ingress.right', 'runtime.left', flow.replay, .12, .82)], m => { m.sink.routingRule = 'insert: after key'; m.sink.writePlan = 'pk160 queued'; m.lanes[0].currentOp = 'I:pk160'; m.lanes[0].status = 'after key staged'; }, 'dispatch', ['dispatch']), step('Delete rows use the before-image primary key because the current tuple location must be resolved from the old key.', 4.0, - ['ingress', 'lane1', 'index'], [tok('D:pk208', 'ingress.replayRight', 'lane1.in', flow.delete, .08, .58), tok('old key lookup', 'lane1.in', 'index.in', flow.index, .52, .92)], + ['ingress', 'lane1', 'index'], [tok('D:pk208', 'ingress.right', 'runtime.left', flow.delete, .08, .58), tok('old key lookup', 'lane1.in', 'index.in', flow.index, .52, .92)], m => { m.sink.routingRule = 'delete: before key'; m.sink.writePlan = 'pk208 queued'; m.lanes[1].currentOp = 'D:pk208'; m.index.hot = 'pk208'; m.index.operation = 'resolve before key'; }, 'dispatch', ['dispatch', 'index']), step('If an update changes its primary key, Pixels Sink preserves the delete-old then insert-new order before replay.', 4.4, - ['ingress', 'lane1', 'lane2'], [tok('delete old pk132', 'ingress.replayRight', 'lane1.in', flow.delete, .08, .58), tok('insert new pk733', 'ingress.replayRight', 'lane2.in', flow.replay, .40, .92)], + ['ingress', 'lane1', 'lane2'], [tok('delete old pk132', 'ingress.right', 'runtime.left', flow.delete, .08, .58), tok('insert new pk733', 'ingress.right', 'runtime.left', flow.replay, .40, .92)], m => { m.sink.routingRule = 'pk update: split'; m.sink.writePlan = 'delete old before insert new'; m.sink.rpcBatch = 'tx43 rows staged'; m.lanes[1].currentOp = 'D:pk132'; m.lanes[2].currentOp = 'I:pk733'; }, 'dispatch', ['dispatch']), step('Pixels Sink begins mirror transactions with Transaction Service, then sends replay RPCs to Retina replayers.', 4.6, - ['ingress', 'tx', 'runtime'], [tok('tx43 ts1291', 'tx.left', 'ingress.beginRight', flow.tx, .04, .34), tok('tx44 ts1292', 'tx.left', 'ingress.beginRight', flow.tx, .16, .46), tok('tx45 ts1293', 'tx.left', 'ingress.beginRight', flow.tx, .28, .58), tok('replay RPCs', 'ingress.replayRight', 'runtime.left', flow.replay, .58, .96)], + ['ingress', 'tx', 'runtime'], [tok('tx43 ts1291', 'tx.left', 'ingress.beginRight', flow.tx, .04, .34), tok('tx44 ts1292', 'tx.left', 'ingress.beginRight', flow.tx, .16, .46), tok('tx45 ts1293', 'tx.left', 'ingress.beginRight', flow.tx, .28, .58), tok('replay RPCs', 'ingress.right', 'runtime.left', flow.replay, .58, .96)], m => { m.sink.sinkStep = 'replay'; m.sink.requestMode = 'replay RPC'; m.sink.requestState = '3 mirror tx'; m.sink.rpcBatch = 'tx43 + tx44 + tx45'; m.sink.txState = 'timestamps assigned'; }, 'mvcc', ['tx']), step('Inside Retina, the same RPC applies rows from different transactions while preserving their visibility timestamps.', 4.2, - ['ingress', 'lane0', 'lane1', 'lane2'], [tok('tx43 row', 'ingress.replayRight', 'lane0.in', flow.replay, .08, .58), tok('tx44 row', 'ingress.replayRight', 'lane1.in', flow.delete, .22, .74), tok('tx45 row', 'ingress.replayRight', 'lane2.in', flow.replay, .36, .90)], + ['ingress', 'lane0', 'lane1', 'lane2'], [tok('tx43 row', 'ingress.right', 'runtime.left', flow.replay, .08, .58), tok('tx44 row', 'ingress.right', 'runtime.left', flow.delete, .22, .74), tok('tx45 row', 'ingress.right', 'runtime.left', flow.replay, .36, .90)], m => { m.sink.requestState = 'apply one RPC'; m.sink.rpcBatch = '3 tx rows'; m.lanes[0].currentOp = 'tx43 I:pk160'; m.lanes[1].currentOp = 'tx44 D:pk208'; m.lanes[2].currentOp = 'tx45 I:pk733'; }, 'dispatch', ['dispatch']) ], insert: [ ...singleOpCdcPrelude('insert', 'tx1290'), - step('Pixels Sink assembles tx1290 as a mirror transaction with INSERT row changes.', 3.8, - ['ingress'], [], - m => { m.sink.sinkStep = 'build'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'mirror tx ready'; }, 'cdc', ['cdc']), - step('Transaction Service assigns timestamp=1290; Pixels Sink sends the INSERT row changes as replay RPCs.', 3.8, - ['ingress', 'tx', 'runtime'], [tok('ts=1290', 'tx.left', 'ingress.beginRight', flow.tx, .12, .58), tok('INSERT replay', 'ingress.replayRight', 'runtime.left', flow.replay, .50, .96)], - m => { m.sink.sinkStep = 'replay'; m.sink.requestState = 'replay RPC ready'; m.sink.txState = 'timestamp=1290'; }, 'mvcc', ['tx']), - step('Retina replayer processes INSERT row changes against write and visibility state.', 4.0, - ['ingress', 'lane0', 'lane1', 'lane2'], - [tok('pk160', 'ingress.replayRight', 'lane0.in', flow.replay, .08, .86), tok('pk174', 'ingress.replayRight', 'lane1.in', flow.replay, .14, .92), tok('pk181', 'ingress.replayRight', 'lane2.in', flow.replay, .20, .98)], - m => { m.lanes.forEach((l, i) => { l.currentOp = `I:pk${160 + i * 14}`; l.status = 'replay row'; }); }, 'dispatch', ['dispatch']), - step('Retina appends the after image to its write buffer. Index Service records key -> rowId and rowId -> RowLocation for the new version.', 4.3, - ['lane1', 'index'], - [tok('after image', 'lane1.in', 'lane1.active', flow.replay, .05, .55), tok('pk174 -> rowId7807', 'lane1.active', 'index.in', flow.index, .45, .95)], - m => { const l = m.lanes[1]; l.activeRows = 8; l.highlight = 'active'; m.index.primaryEntries.unshift('pk174 -> rowId7807'); m.index.mainEntries.unshift('rowId7807 -> fileId78/rg0/off7'); m.index.hot = 'pk174'; m.index.operation = 'putMainIndexEntries + putPrimaryIndexEntries'; }, 'writeBuffer', ['index']), - step('When the active memtable fills, it seals into an immutable memtable. Replay continues with a fresh active memtable instead of blocking on storage.', 3.9, - ['lane1'], [tok('seal memtable', 'lane1.active', 'lane1.immutable', flow.storage, .10, .82)], - m => { const l = m.lanes[1]; l.activeRows = 1; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', ['flush'], 'seal'), - step('A flush process writes the immutable memtable to Storage; the resulting object entry is the flushed immutable memtable representation.', 4.5, - ['lane1', 'storage', 'index'], - [tok('flushed memtable', 'lane1.immutable', 'storage.memtable', flow.storage, .06, .68), tok('object entry metadata', 'storage.memtable', 'index.in', flow.index, .50, .96)], - m => { m.storage.objectEntries.unshift('obj-78'); m.lanes[1].highlight = 'immutable'; m.lanes[1].immutable = Math.max(0, m.lanes[1].immutable - 1); }, 'writeBuffer', ['flush', 'index'], 'flush'), - ...commitSequence('tx1290', 1291, 'tx1290 committed; HWM=1291') + step('Pixels Sink begins mirror transaction tx1290 and Transaction Service assigns timestamp=1290.', 4.0, + ['ingress', 'tx'], [tok('begin tx1290', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=1290', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], + m => { m.sink.sinkStep = 'begin'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=1290'; }, 'mvcc', ['tx']), + step('Pixels Sink sends the INSERT row changes after the mirror transaction has a timestamp.', 3.6, + ['ingress'], [tok('INSERT', 'ingress.right', 'runtime.left', flow.replay, .18, .86)], + m => { m.sink.sinkStep = 'replay'; m.sink.requestState = 'replay RPC ready'; m.sink.txState = 'timestamp=1290'; }, 'dispatch', []), + step('Retina applies the INSERT to the write buffer first: the active memtable grows by one row.', 4.0, + ['lane1'], + [], + m => { const l = m.lanes[1]; l.currentOp = 'I:pk174'; l.status = 'append after image'; l.activeRows = 8; l.highlight = 'active'; m.sink.rpcBatch = 'INSERT applying'; }, 'writeBuffer', []), + step('The active memtable reaches capacity, so Retina seals it into the immutable queue and opens a fresh empty active memtable.', 3.9, + ['lane1'], [], + m => { const l = m.lanes[1]; l.activeRows = 0; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', [], 'seal'), + step('The sealed memtable now sits in the immutable queue while the active memtable is empty again.', 1.2, + ['lane1'], [], + m => { const l = m.lanes[1]; l.highlight = 'immutable'; }, 'writeBuffer', []), + step('A background flush asynchronously moves the immutable memtable to Storage.', 3.8, + ['lane1'], [], + m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; }, 'writeBuffer', [], 'flush'), + step('After the row is appended, Retina calls Index Service to insert the primary and row-location indexes.', 4.2, + ['index'], [tok('put index pk174', 'runtime.right', 'index.left', flow.index, .12, .82)], + m => { const l = m.lanes[1]; l.highlight = null; if (m.storage.pendingFlushObject && !m.storage.objectEntries.includes(m.storage.pendingFlushObject)) m.storage.objectEntries.unshift(m.storage.pendingFlushObject); m.storage.pendingFlushObject = null; m.index.primaryEntries.unshift('pk174 -> rowId7807'); m.index.mainEntries.unshift('rowId7807 -> fileId78/rg0/off7'); m.index.hot = 'pk174'; m.index.operation = 'putMainIndexEntries + putPrimaryIndexEntries'; }, 'index', ['index']), + step('Index Service finishes the insert index update and returns to Retina.', 3.2, + ['index'], [tok('index OK', 'index.left', 'runtime.right', flow.index, .18, .82)], + m => { m.index.operation = 'index insert OK'; m.sink.rpcBatch = 'INSERT replay OK'; }, 'index', ['index']), + step('Retina returns replay OK to Pixels Sink; the INSERT operation is complete for this mirror transaction.', 3.4, + ['ingress'], [tok('replay OK', 'runtime.left', 'ingress.right', flow.replay, .12, .82)], + m => { m.sink.sinkStep = 'rowsOk'; m.sink.requestState = 'INSERT replay OK'; m.sink.rpcBatch = 'INSERT replay OK'; }, 'mvcc', []), + step(`Pixels Sink asks Transaction Service to commit tx1290.`, 3.2, + ['ingress', 'tx'], [tok(`commit tx1290`, 'ingress.commitRight', 'tx.left', flow.tx, .12, .82)], + m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'commit requested'; m.sink.txState = `tx1290 commit in flight`; }, 'mvcc', ['tx']), + step(`Transaction Service advances HWM internally for later snapshots.`, 2.8, + ['tx'], [], + m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'HWM advanced'; m.sink.txState = 'tx1290 committed; HWM=1291'; m.tx.hwm = 1291; }, 'mvcc', ['tx']) ], delete: [ ...singleOpCdcPrelude('delete', 'tx1291'), step('Delete replay starts after Pixels Sink begins mirror transaction tx1291 with timestamp=1291.', 3.7, - ['ingress', 'tx', 'lane1'], [tok('timestamp=1291', 'tx.left', 'ingress.beginRight', flow.tx, .10, .52), tok('DELETE replay', 'ingress.replayRight', 'lane1.in', flow.delete, .50, .96)], + ['ingress', 'tx', 'lane1'], [tok('timestamp=1291', 'tx.left', 'ingress.beginRight', flow.tx, .10, .52), tok('DELETE', 'ingress.right', 'runtime.left', flow.delete, .50, .96)], m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=1291'; }, 'mvcc', ['cdc', 'tx']), step('Retina receives D:pk208 and calls Index Service deletePrimaryIndexEntries to remove the primary entry and return the current RowLocation.', 4.0, - ['ingress', 'lane1', 'index'], [tok('D:pk208', 'ingress.replayRight', 'lane1.in', flow.delete, .05, .48), tok('deletePrimary pk208', 'lane1.in', 'index.in', flow.index, .42, .88)], + ['ingress', 'lane1', 'index'], [tok('D:pk208', 'ingress.right', 'runtime.left', flow.delete, .05, .48), tok('deletePrimary pk208', 'lane1.in', 'index.in', flow.index, .42, .88)], m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'D:pk208'; l.status = 'delete primary entry'; m.index.hot = 'pk208'; m.index.operation = 'deletePrimaryIndexEntries pk208'; }, 'index', ['index']), step('Index Service returns RowLocation(fileId, rgId, rowOffset). The old tuple stays in place until visibility says otherwise.', 3.7, ['index', 'lane1'], [tok('file-042 rg0 off7', 'index.out', 'lane1.visibility', flow.index, .08, .80)], @@ -1008,7 +1023,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

update: [ ...singleOpCdcPrelude('update', 'tx1292'), step('Update replay arrives after Pixels Sink begins mirror transaction tx1292 with timestamp=1292.', 3.8, - ['ingress', 'tx', 'lane1'], [tok('timestamp=1292', 'tx.left', 'ingress.beginRight', flow.tx, .10, .52), tok('UPDATE replay', 'ingress.replayRight', 'lane1.in', flow.index, .50, .96)], + ['ingress', 'tx', 'lane1'], [tok('timestamp=1292', 'tx.left', 'ingress.beginRight', flow.tx, .10, .52), tok('UPDATE', 'ingress.right', 'runtime.left', flow.index, .50, .96)], m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay UPDATE'; m.sink.txState = 'timestamp=1292'; }, 'mvcc', ['tx']), step('Retina first inserts the after image into the write buffer and allocates a new rowId plus RowLocation for the new tuple version.', 4.2, ['lane1', 'index'], [tok('after image', 'lane1.in', 'lane1.active', flow.index, .06, .52), tok('new RowLocation', 'lane1.active', 'index.in', flow.index, .48, .94)], @@ -1052,7 +1067,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

['lane1', 'runtime'], [tok('fold delete items', 'lane1.deleteArray', 'lane1.baseBitmap', flow.gc, .08, .78)], m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.safeTs = DEFAULT_GC_SAFE_TS; l.deleteTs = ['(9,1291)']; l.baseBitmap = '0010100000000000'; l.baseTs = DEFAULT_GC_SAFE_TS; l.highlight = 'baseBitmap'; }, 'gc', ['visibility']), step('After bitmap folding, foreground replay can continue while Retina prepares Storage GC file groups; no running query is holding the GC safe timestamp back.', 4.0, - ['lane1', 'lane2', 'runtime'], [tok('folded bitmap', 'lane1.baseBitmap', 'storage.aging', flow.gc, .05, .50), tok('replay continues', 'ingress.replayRight', 'lane2.in', flow.replay, .34, .82)], + ['lane1', 'lane2', 'runtime'], [tok('folded bitmap', 'lane1.baseBitmap', 'storage.aging', flow.gc, .05, .50), tok('replay continues', 'ingress.right', 'runtime.left', flow.replay, .34, .82)], m => { m.lanes[DETAIL_REPLAY_ID].safeTs = DEFAULT_GC_SAFE_TS; m.lanes[2].status = 'replay continues'; m.query.active = false; }, 'gc', ['dispatch', 'query']), step('Storage GC selects low-valid-ratio regular files. They remain regular columnar files, just highlighted as rewrite candidates.', 4.0, ['runtime', 'storage'], [tok('candidate files', 'runtime.right', 'storage.aging', flow.gc, .08, .76)], @@ -1170,8 +1185,8 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

subtle: dark ? '#141b24' : '#f7f9fc', text: dark ? '#edf2f7' : '#122033', muted: dark ? '#9aa8b8' : '#667386', - border: dark ? '#344256' : '#cbd5e1', - borderSoft: dark ? '#2a3546' : '#dde5ee', + border: dark ? '#344256' : '#b8c4d3', + borderSoft: dark ? '#2a3546' : '#c7d2df', grid: dark ? '#263244' : '#dce5ef', shadow: dark ? 'rgba(0,0,0,.28)' : 'rgba(15,23,42,.10)', accent: dark ? '#6ea8ff' : '#315fcb' @@ -1390,6 +1405,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

drawIndex(c); drawReplayers(c); drawStorage(c); + drawAsyncFlushMemtable(c); drawTokens(c); drawCommitWait(c); if (state.flash > 0) { @@ -1463,8 +1479,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const st = currentStep(); const tokenParts = (st.tokens || []).flatMap(t => t.path || []); if (key === 'runtime') { - return ['lane0', 'lane1', 'lane2'].some(isHighlighted) || - tokenParts.some(part => typeof part === 'string' && /^lane[0-2]\b/.test(part)); + return false; } if (key === 'ingress') { return tokenParts.some(part => typeof part === 'string' && part.startsWith('ingress.')); @@ -1683,8 +1698,9 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

if (i > progress.reachedIndex) return; const x = map(cdcStreamRailP(i + 1, records.length), 0, 1, startX, endX); const cardW = 62; + const cardTextSize = TYPE.text; drawCdcDot(x, railY, true, record.color, c); - drawRecordTypeCard(x - cardW / 2, railY + 14, cardW, 24, record.label, record.color, c, TYPE.text); + drawRecordTypeCard(x - cardW / 2, railY + 14, cardW, 24, record.label, record.color, c, cardTextSize); }); if (progress.done) return; @@ -1711,7 +1727,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

roundRect(txn.x, txn.y, txn.w, txn.h, 12, true, true); ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; fitText(`mirror transaction`, txn.x + PANEL_TITLE_X, txn.y + PANEL_TITLE_Y, txn.w - PANEL_TITLE_X * 2, 'left'); if (activeId === 'empty') return; @@ -1768,7 +1784,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = 1; roundRect(row.x + 32, row.y + 17, 24, 24, 8, true, true); ctx.fillStyle = row.color; - ctx.font = typeFont(TYPE.micro, 900); + ctx.font = typeFont(TYPE.text, 900); ctx.textAlign = 'center'; ctx.fillText(String(row.order), row.x + 44, row.y + 33); ctx.fillStyle = row.color; @@ -1823,7 +1839,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = 1; roundRect(cx - 9, y - 11, 18, 22, 5, true, true); ctx.fillStyle = flow.replay; - ctx.font = typeFont(TYPE.micro, 900); + ctx.font = typeFont(TYPE.text, 900); ctx.textAlign = 'center'; ctx.fillText('OK', cx, y + 3); } @@ -2239,7 +2255,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = 1; roundRect(x - w / 2, y - h / 2, w, h, 5, false, true); ctx.fillStyle = '#ffffff'; - ctx.font = typeFont(TYPE.micro, 900); + ctx.font = typeFont(TYPE.text, 900); ctx.textAlign = 'center'; fitText(shortRowLabel(label), x - w / 2 + 4, y + 3, w - 8, 'center'); ctx.restore(); @@ -2272,7 +2288,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = 1; roundRect(x - 26, y - 11, 52, 22, 8, false, true); ctx.fillStyle = '#ffffff'; - ctx.font = typeFont(TYPE.micro, 900); + ctx.font = typeFont(TYPE.text, 900); ctx.textAlign = 'center'; fitText(label, x - 22, y + 4, 44, 'center'); ctx.restore(); @@ -2447,7 +2463,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

roundRect(x, y, w, h, 9, true, true); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; fitText('PRIMARY INDEX MAP', x + 10, y + 19, w - 20, 'left'); @@ -2478,7 +2494,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

roundRect(x, y, w, h, 9, true, true); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; ctx.fillText('MAIN INDEX', x + 9, y + 19); @@ -2597,7 +2613,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const y = r.y + MODULE_TITLE_Y + MODULE_CONTENT_GAP; const frame = { x: r.x + pad, y, w: r.w - pad * 2, h: r.y + r.h - y - MODULE_CONTENT_PAD }; const gap = 20; - const writeH = Math.round((frame.h - gap) * .43); + const writeH = Math.round((frame.h - gap) * .45); const visibilityH = frame.h - gap - writeH; return { frame, @@ -2606,6 +2622,47 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

}; } + function replayWriteBufferLayout(x, y, w, h) { + const top = y + 50; + const bottom = y + h - MODULE_CONTENT_PAD; + const panelH = Math.max(72, bottom - top); + const gap = 28; + const innerPad = MODULE_CONTENT_PAD; + const activePanelW = 84; + const activePanel = { x: x + innerPad, y: top, w: activePanelW, h: panelH }; + const queueX = activePanel.x + activePanel.w + gap; + const queue = { x: queueX, y: top, w: x + w - innerPad - queueX, h: panelH }; + const maxVisible = 3; + const itemGap = 12; + const memW = Math.max(42, Math.min(78, (queue.w - 18 - itemGap * (maxVisible - 1)) / maxVisible)); + const memY = top + 28; + const memH = Math.max(36, panelH - 36); + const activeMem = { x: activePanel.x + (activePanel.w - memW) / 2, y: memY, w: memW, h: memH }; + const fillPadX = 10; + const fillPadTop = 4; + const fillPadBottom = 6; + const fullFill = { + x: activeMem.x + fillPadX, + y: activeMem.y + fillPadTop, + w: activeMem.w - fillPadX * 2, + h: activeMem.h - fillPadTop - fillPadBottom + }; + return { activePanel, queue, activeMem, fullFill, memW: fullFill.w, memY: fullFill.y, memH: fullFill.h, itemGap }; + } + + function replayVisibilityLayout(x, y, w, h) { + const rowX = x + 20; + const rowY = y + 46; + const rowW = w - 32; + const labelW = Math.floor(rowW * 0.31); + const bitmapX = rowX + labelW; + const bitmapW = rowW - labelW; + const chainY = y + 88; + const maxChainH = Math.max(40, y + h - MODULE_CONTENT_PAD - chainY); + const chainH = Math.max(44, maxChainH); + return { rowX, rowY, rowW, labelW, bitmapX, bitmapW, chainY, chainH }; + } + function activeReplayLane() { const st = currentStep(); const parts = []; @@ -2627,108 +2684,127 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

drawReplayVisibility(visibility.x, visibility.y, visibility.w, visibility.h, lane, c); } + function drawNeutralPanelBox(x, y, w, h, title, c, radius = 9, active = false) { + ctx.fillStyle = active ? alpha(c.panel3, .88) : alpha(c.panel2, .78); + ctx.strokeStyle = active ? c.border : c.borderSoft; + ctx.lineWidth = active ? 1.4 : 1; + roundRect(x, y, w, h, radius, true, true); + if (!title) return; + ctx.fillStyle = c.text; + ctx.font = typeFont(TYPE.subTitle, 900); + ctx.textAlign = 'left'; + fitText(title, x + PANEL_TITLE_X, y + PANEL_TITLE_Y, w - PANEL_TITLE_X * 2, 'left'); + } + function drawReplayWriteBuffer(x, y, w, h, lane, c) { - smallBox(x, y, w, h, 'write buffer', flow.storage, c, state.detailFocus === 'writeBuffer' || lane.highlight === 'active' || lane.highlight === 'immutable'); + const writeActive = state.detailFocus === 'writeBuffer' || lane.highlight === 'active' || lane.highlight === 'immutable'; + drawNeutralPanelBox(x, y, w, h, 'write buffer', c, 9, writeActive); const st = currentStep(); const p = stepProgress(true); - const activeX = x + 16, activeY = y + 54; - const blockW = 78, blockH = Math.max(72, Math.min(98, h - 72)); - const immX = activeX + blockW + 24; - const immW = x + w - 14 - immX; + const layout = replayWriteBufferLayout(x, y, w, h); + const { activePanel, queue, activeMem } = layout; const prevRows = lane.prevActiveRows ?? lane.activeRows; const displayRows = lerp(prevRows, lane.activeRows, p); - const fillH = (blockH - 10) * clamp(displayRows / lane.activeCap, 0, 1); + const fillH = layout.fullFill.h * clamp(displayRows / lane.activeCap, 0, 1); + + drawNeutralPanelBox(activePanel.x, activePanel.y, activePanel.w, activePanel.h, '', c, 6, lane.highlight === 'active'); + ctx.fillStyle = c.text; + ctx.font = typeFont(TYPE.text, 800); + ctx.textAlign = 'left'; + ctx.fillText('active', activePanel.x + 8, activePanel.y + 18); - ctx.fillStyle = alpha(flow.replay, lane.highlight === 'active' ? .24 : .11); - ctx.strokeStyle = alpha(flow.replay, lane.highlight === 'active' ? .62 : .34); - roundRect(activeX, activeY, blockW, blockH, 8, true, true); ctx.fillStyle = alpha(flow.replay, .34); - roundRect(activeX + 10, activeY + blockH - 6 - fillH, blockW - 20, Math.max(5, fillH), 5, true, false); + roundRect(layout.fullFill.x, layout.fullFill.y + layout.fullFill.h - fillH, layout.fullFill.w, Math.max(5, fillH), 5, true, false); ctx.fillStyle = c.text; ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'center'; - ctx.fillText('active', activeX + blockW / 2, activeY + 14); - ctx.fillText(`${Math.round(displayRows)}/${lane.activeCap}`, activeX + blockW / 2, activeY + blockH - 8); - - if (st.anim !== 'seal') { - const a = { x: activeX + blockW + 5, y: activeY + blockH / 2 }; - const b = { x: immX - 7, y: activeY + blockH / 2 }; - ctx.strokeStyle = alpha(flow.storage, .50); - ctx.lineWidth = 1.5; - drawPolyline([a, b]); - arrowHead(a, b, alpha(flow.storage, .68)); - } + ctx.fillText(`${Math.round(displayRows)}/${lane.activeCap}`, activeMem.x + activeMem.w / 2, activeMem.y + activeMem.h - 14); - drawCompactImmQueue(immX, activeY, immW, blockH, lane.immutable, lane.highlight === 'immutable', c); + const sealMoving = st.anim === 'seal' && p < 1; + const flushMoving = st.anim === 'flush' && p < 1 && lane.prevImmutable > 0; + const visibleImmutable = sealMoving || flushMoving ? lane.prevImmutable : lane.immutable; + const queueCount = flushMoving ? 0 : visibleImmutable; + drawCompactImmQueue(layout, queueCount, lane.highlight === 'immutable', c, { suppressEmpty: sealMoving || flushMoving }); if (st.anim === 'seal' && p < 1) { - const ghostX = lerp(activeX, immX + 8, p); + const targetX = firstImmutableMemX(layout, lane.prevImmutable + 1); + const ghostX = lerp(layout.fullFill.x, targetX, p); ctx.save(); ctx.globalAlpha = Math.min(1, p * 3) * .85; ctx.fillStyle = alpha(flow.gc, .26); ctx.strokeStyle = alpha(flow.gc, .62); - roundRect(ghostX, activeY, blockW, blockH, 6, true, true); - ctx.fillStyle = flow.gc; - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'center'; - ctx.fillText('sealing', ghostX + blockW / 2, activeY + blockH / 2 + 3); + roundRect(ghostX, layout.fullFill.y, layout.fullFill.w, layout.fullFill.h, 6, true, true); ctx.restore(); } - if (st.anim === 'flush' && p < 1 && lane.prevImmutable > 0) { - const ghostX = immX + immW - 28 + p * 46; - ctx.save(); - ctx.globalAlpha = (1 - p) * .85; - ctx.fillStyle = alpha(flow.gc, .20); - ctx.strokeStyle = alpha(flow.gc, .50); - roundRect(ghostX, activeY + 7, 24, blockH - 14, 4, true, true); - ctx.restore(); - } } - function drawCompactImmQueue(x, y, w, h, count, active, c) { + function firstImmutableMemX(layout, count) { + const { queue, memW, itemGap } = layout; + const n = Math.max(1, Math.min(count, 3)); + const totalW = memW * n + itemGap * (n - 1); + return queue.x + Math.max(9, (queue.w - totalW) / 2); + } + + function drawCompactImmQueue(layout, count, active, c, opts = {}) { + const { queue, memW, memY, memH, itemGap } = layout; const color = flow.gc; - ctx.fillStyle = alpha(color, active ? .14 : .06); - ctx.strokeStyle = alpha(color, active ? .52 : .24); - ctx.lineWidth = active ? 1.5 : 1; - roundRect(x, y, w, h, 6, true, true); - ctx.fillStyle = color; + drawNeutralPanelBox(queue.x, queue.y, queue.w, queue.h, '', c, 6, active); + ctx.fillStyle = c.text; ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'left'; - ctx.fillText('imm FIFO', x + 8, y + 18); + ctx.fillText('immut queue', queue.x + 8, queue.y + 18); const n = Math.min(count, 3); - if (!n) { + if (count <= 0) { + if (opts.suppressEmpty) return; ctx.fillStyle = c.muted; - ctx.font = typeFont(TYPE.text, 0); + ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'center'; - ctx.fillText('empty', x + w / 2, y + h / 2 + 6); + ctx.fillText('empty', queue.x + queue.w / 2, queue.y + queue.h / 2 + 6); return; } - const gap = 4; - const cellW = Math.max(24, Math.min(38, (w - 18 - gap * (n - 1)) / n)); - const startX = x + Math.max(9, (w - (cellW * n + gap * (n - 1))) / 2); + const startX = firstImmutableMemX(layout, count); for (let i = 0; i < n; i++) { - const bx = startX + i * (cellW + gap); + const bx = startX + i * (memW + itemGap); const label = count > 3 && i === n - 1 ? `+${count - 2}` : `b${count - i}`; ctx.fillStyle = alpha(color, active ? .22 : .12); ctx.strokeStyle = alpha(color, active ? .56 : .28); - roundRect(bx, y + 28, cellW, h - 36, 5, true, true); + roundRect(bx, memY, memW, memH, 5, true, true); ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.micro, 800); + ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'center'; - fitText(label, bx + 3, y + h / 2 + 5, cellW - 6, 'center'); + fitText(label, bx + 3, memY + memH / 2 + 5, memW - 6, 'center'); } } + function drawAsyncFlushMemtable(c) { + const st = currentStep(); + if (st.anim !== 'flush') return; + const p = stepProgress(true); + if (p >= 1) return; + const lane = activeReplayLane(); + if ((lane.prevImmutable || 0) <= 0) return; + const { write } = replayerPanelLayout(); + const layout = replayWriteBufferLayout(write.x, write.y, write.w, write.h); + const startX = firstImmutableMemX(layout, lane.prevImmutable); + const start = { x: startX + layout.memW / 2, y: layout.memY + layout.memH / 2 }; + const end = point('storage.memtable'); + const x = lerp(start.x, end.x, p); + const y = lerp(start.y, end.y, p); + + ctx.save(); + ctx.fillStyle = alpha(flow.storage, .22); + ctx.strokeStyle = alpha(flow.storage, .62); + ctx.lineWidth = 1.4; + roundRect(x - layout.memW / 2, y - layout.memH / 2, layout.memW, layout.memH, 5, true, true); + ctx.restore(); + } + function drawReplayVisibility(x, y, w, h, lane, c) { - smallBox(x, y, w, h, 'visibility structure', flow.visibility, c, state.detailFocus === 'visibility' || state.detailFocus === 'gc' || lane.highlight === 'visibility' || lane.highlight === 'baseBitmap' || lane.highlight === 'foldEligible'); + drawNeutralPanelBox(x, y, w, h, 'visibility structure', c); - const rowX = x + 12; - const rowY = y + 46; - const rowW = w - 24; - const labelW = Math.floor(rowW * 0.35); - const bitmapX = rowX + labelW; - const bitmapW = rowW - labelW; + const layout = replayVisibilityLayout(x, y, w, h); + const { rowX, rowY, labelW, bitmapX, bitmapW, chainY, chainH } = layout; ctx.fillStyle = c.text; ctx.font = typeFont(TYPE.text, 900); @@ -2736,24 +2812,27 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

fitText(`baseTs=${lane.baseTs}`, rowX, rowY + 14, labelW - 4, 'left'); drawFixedDeleteBitmap(lane, bitmapX, rowY, bitmapW, c); - const chainY = y + 92; - const chainH = Math.max(64, h - 108); - ctx.fillStyle = alpha(flow.delete, lane.highlight === 'visibility' || lane.highlight === 'foldEligible' ? .18 : .08); - ctx.strokeStyle = alpha(flow.delete, lane.highlight === 'visibility' || lane.highlight === 'foldEligible' ? .46 : .24); - roundRect(x + 10, chainY, w - 20, chainH, 7, true, true); - ctx.fillStyle = flow.delete; + const chainX = x + MODULE_CONTENT_PAD; + const chainW = w - MODULE_CONTENT_PAD * 2; + const chainPad = 8; + const itemPad = 20; + ctx.fillStyle = alpha(c.panel2, .78); + ctx.strokeStyle = c.borderSoft; + ctx.lineWidth = 1; + roundRect(chainX, chainY, chainW, chainH, 7, true, true); + ctx.fillStyle = c.text; ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'left'; - ctx.fillText('delete ts chain', x + 18, chainY + 22); - drawCompactDeleteTsItems(x + 18, chainY + 34, w - 36, lane, c); + ctx.fillText('delete ts chain', chainX + chainPad, chainY + 22); + drawCompactDeleteTsItems(chainX + itemPad, chainY + 34, chainW - itemPad * 2, lane, c); } function drawFixedDeleteBitmap(lane, x, y, w, c) { const len = 16; const bits = (lane.baseBitmap || '').padEnd(len, '0').slice(0, len); - const gap = 2; - const cellW = Math.max(4, Math.min(6, Math.floor((w - gap * (len - 1)) / len))); - const cellH = 14; + const gap = 3; + const cellW = Math.max(7, Math.min(12, Math.floor((w - gap * (len - 1)) / len))); + const cellH = 18; const totalW = len * cellW + (len - 1) * gap; const startX = x + Math.max(0, (w - totalW) / 2); const pendingDeletes = new Set(); @@ -2772,8 +2851,8 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const folded = bits[i] === '1'; const eligible = eligibleDeletes.has(i); const pending = pendingDeletes.has(i); - let fill = alpha(c.border, .20); - let stroke = alpha(c.border, .42); + let fill = alpha(c.border, .22); + let stroke = alpha(c.border, .56); if (folded) { const color = lane.highlight === 'baseBitmap' ? flow.gc : flow.visibility; @@ -2795,21 +2874,27 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

} function drawCompactDeleteTsItems(x, y, w, lane, c) { - const visible = lane.deleteTs.slice(0, 3); - const gap = 4; - const itemW = Math.min(72, Math.floor((w - gap * 2) / 3)); + const columns = 4; + const rows = 2; + const visible = lane.deleteTs.slice(0, columns * rows); + const itemW = Math.min(72, Math.floor((w - 18 * (columns - 1)) / columns)); + const gapX = columns > 1 ? (w - itemW * columns) / (columns - 1) : 0; + const itemH = 24; + const rowGap = 10; visible.forEach((item, i) => { + const col = i % columns; + const row = Math.floor(i / columns); const ts = deleteTsValue(item); const eligible = lane.safeTs != null && ts <= lane.safeTs; const active = lane.highlight === 'foldEligible' && eligible; const color = active ? flow.gc : flow.delete; - drawPill(x + i * (itemW + gap), y, itemW, 24, item, color, c, active, 8); + drawPill(x + col * (itemW + gapX), y + row * (itemH + rowGap), itemW, itemH, item, color, c, active, 8); }); - if (lane.deleteTs.length > 3) { + if (lane.deleteTs.length > columns * rows) { ctx.fillStyle = c.muted; - ctx.font = typeFont(TYPE.micro, 800); + ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'right'; - ctx.fillText(`+${lane.deleteTs.length - 3}`, x + w, y + 17); + ctx.fillText(`+${lane.deleteTs.length - columns * rows}`, x + w, y + rows * itemH + rowGap); } } @@ -2828,12 +2913,23 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

drawComponent(r, c, { color: flow.storage, key: 'storage' }); const layout = storagePanelsLayout(r); const panels = [ - { ...layout.memtable, title: 'flushed immutable memtable', color: flow.storage, kind: 'OBJ', labels: model.storage.objectEntries.slice(0, 4), active: state.detailFocus === 'writeBuffer' || state.activeChannels.includes('flush'), maxItems: 4 }, + { ...layout.memtable, title: 'flushed immutable memtable', color: flow.storage, kind: 'OBJ', labels: storageObjectLabels().slice(0, 4), active: false, hotLabel: model.storage.flushedHot, maxItems: 4 }, { ...layout.files, title: 'regular columnar files', color: flow.replay, kind: 'FILE', labels: regularStorageFileLabels(), active: ['query', 'gc', 'storageGc'].includes(state.detailFocus), regular: true, maxItems: 5 } ]; panels.forEach(panel => drawStoragePanel(panel.x, panel.y, panel.w, panel.h, panel, c)); } + function storageObjectLabels() { + const labels = [...model.storage.objectEntries]; + const pending = model.storage.pendingFlushObject; + if (pending) { + const st = currentStep(); + const showPending = st.anim !== 'flush' || stepProgress(true) > .86; + if (showPending && !labels.includes(pending)) labels.unshift(pending); + } + return labels; + } + function storagePanelsLayout(r) { const x = r.x + MODULE_CONTENT_PAD; const y = r.y + MODULE_TITLE_Y + MODULE_CONTENT_GAP; @@ -2869,7 +2965,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = panel.active ? 1.5 : 1; roundRect(x, y, w, h, 10, true, true); ctx.fillStyle = panel.color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; fitText(panel.title, x + PANEL_TITLE_X, y + PANEL_TITLE_Y, w - PANEL_TITLE_X * 2, 'left'); const labels = panel.labels.length ? panel.labels : ['empty']; @@ -2891,7 +2987,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const iconKind = retired ? 'RETIRED' : temporary ? 'TEMP' : candidate ? 'CAND' : panel.kind; const iconX = startX + i * (iconW + iconGap); drawTabularStorageIcon(iconX, iconY, iconW, iconH, String(label), iconKind, panel.color, c, { - active: panel.active, + active: panel.active || String(label) === panel.hotLabel, candidate, retired, newFile, @@ -2964,7 +3060,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = active ? 1.4 : 1; roundRect(x, y, w, h, 7, true, true); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; fitText('SCHEMA', x + 8, y + 14, w - 16, 'left'); ctx.strokeStyle = alpha(color, active ? .66 : .34); @@ -2992,7 +3088,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = active ? 1.4 : 1; roundRect(x, y, w, h, 7, true, true); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; ctx.fillText('STATS', x + 8, y + 14); const bars = [7, 12, 9, 16]; @@ -3025,7 +3121,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.arc(knobX, railY, 5, 0, Math.PI * 2); ctx.fill(); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'center'; fitText(switching ? 'ATOMIC SWITCH' : 'LOCKED CATALOG', x + 6, y + 14, w - 12, 'center'); } @@ -3037,7 +3133,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = queryActive ? 1.5 : 1; roundRect(x, y, w, h, 8, true, true); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; fitText('VISIBLE MANIFEST', x + 8, y + 14, w - 16, 'left'); const files = model.catalog.visible.slice(0, 3); @@ -3072,7 +3168,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = active ? 1.4 : 1; roundRect(x, y, w, h, 8, true, true); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'center'; fitText('RETIRED', x + 5, y + 14, w - 10, 'center'); ctx.fillStyle = label === 'empty' ? c.muted : color; @@ -3089,7 +3185,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.lineWidth = hasPending ? 1.4 : 1; roundRect(x, y, w, h, 8, true, true); ctx.fillStyle = hasPending ? color : c.muted; - ctx.font = typeFont(TYPE.text, 900); + ctx.font = typeFont(TYPE.subTitle, 900); ctx.textAlign = 'left'; fitText(`STAGING ${pending}`, x + 8, y + 15, w * .44, 'left'); @@ -3281,7 +3377,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

if (kind === 'regular') { for (let i = 0; i < 3; i++) roundRect(x + i * 5, y + i * 4, 24, 20, 4, true, true); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.micro, 800); + ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'center'; ctx.fillText('RG', x + 17, y + 17); } else if (kind === 'active') { @@ -3294,7 +3390,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.fillStyle = alpha(color, .28); roundRect(x + 7, y, 20, 8, 4, true, false); ctx.fillStyle = color; - ctx.font = typeFont(TYPE.micro, 900); + ctx.font = typeFont(TYPE.text, 900); ctx.textAlign = 'center'; ctx.fillText('OBJ', x + 17, y + 20); } @@ -3500,7 +3596,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

roundRect(x, y, w, h, Math.min(10, h / 2), true, true); ctx.fillStyle = active ? color : c.text; const requested = pillTypeSize(fs); - const size = Math.max(TYPE.micro, Math.min(requested, h - 6)); + const size = Math.max(TYPE.text, Math.min(requested, h - 6)); ctx.font = typeFont(size, 900); ctx.textAlign = 'center'; fitText(label, x + 6, y + h / 2 + size * .36, w - 12, 'center'); @@ -3598,19 +3694,10 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function replayAnchorSet(slot) { const layout = replayerPanelLayout(); const { frame, write, visibility } = layout; - const activeX = write.x + 16; - const activeY = write.y + 54; - const blockW = 78; - const blockH = Math.max(72, Math.min(98, write.h - 72)); - const immX = activeX + blockW + 24; - const immW = write.x + write.w - 14 - immX; - const rowX = visibility.x + 12; - const rowY = visibility.y + 46; - const rowW = visibility.w - 24; - const labelW = Math.floor(rowW * 0.35); - const bitmapX = rowX + labelW; - const bitmapW = rowW - labelW; - const chainY = visibility.y + 92; + const writeLayout = replayWriteBufferLayout(write.x, write.y, write.w, write.h); + const { queue, fullFill } = writeLayout; + const visibilityLayout = replayVisibilityLayout(visibility.x, visibility.y, visibility.w, visibility.h); + const { rowY, bitmapX, bitmapW, chainY, chainH } = visibilityLayout; const offset = (slot - 1) * 18; return { center: [frame.x + frame.w / 2, frame.y + frame.h / 2], @@ -3620,10 +3707,10 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

right: [frame.x + frame.w, frame.y + frame.h / 2 + offset], top: [frame.x + frame.w / 2, frame.y], bottom: [frame.x + frame.w / 2, frame.y + frame.h], - active: [activeX + blockW / 2, activeY + blockH / 2], - immutable: [immX + immW / 2, activeY + blockH / 2], + active: [fullFill.x + fullFill.w / 2, fullFill.y + fullFill.h / 2], + immutable: [queue.x + queue.w / 2, queue.y + queue.h / 2], visibility: [visibility.x + visibility.w / 2, visibility.y + visibility.h / 2], - deleteArray: [visibility.x + visibility.w / 2, chainY + 38], + deleteArray: [visibility.x + visibility.w / 2, chainY + chainH / 2], baseBitmap: [bitmapX + bitmapW / 2, rowY + 7], write: [write.x + write.w / 2, write.y + write.h / 2] }; @@ -3662,6 +3749,16 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return arrPoint(anchors[pos] || anchors.center); } if (parts[0] === 'runtime') { + const borderAnchors = { + center: [r.x + r.w / 2, r.y + r.h / 2], + left: [r.x, r.y + r.h / 2], + right: [r.x + r.w, r.y + r.h / 2], + top: [r.x + r.w / 2, r.y], + bottom: [r.x + r.w / 2, r.y + r.h], + in: [r.x, r.y + r.h / 2], + out: [r.x + r.w, r.y + r.h / 2] + }; + if (borderAnchors[pos]) return arrPoint(borderAnchors[pos]); const anchors = replayAnchorSet(1); if (anchors[pos]) return arrPoint(anchors[pos]); } From 835d0124d0db31ce7ed3c20b4589c196f8ad14cc Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Tue, 7 Jul 2026 14:03:21 +0800 Subject: [PATCH 03/11] fix: retina visibility --- retina.html | 530 +++++++++++++++++++--------------------------------- 1 file changed, 197 insertions(+), 333 deletions(-) diff --git a/retina.html b/retina.html index b6710342c..8c2e89d5e 100644 --- a/retina.html +++ b/retina.html @@ -22,7 +22,6 @@ --replay-path: #168a52; --query-path: #0e7490; --index-path: #2867c7; - --metadata-path: #6d5bd0; --gc-path: #b7791f; --danger: #c2413d; --shadow-sm: 0 2px 8px rgba(15, 23, 42, .06); @@ -66,7 +65,6 @@ --replay-path: #48c78e; --query-path: #38bdf8; --index-path: #7aa7ff; - --metadata-path: #a78bfa; --gc-path: #f0b35a; --danger: #f87171; --shadow-sm: 0 2px 10px rgba(0, 0, 0, .20); @@ -566,10 +564,10 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

cdc: { label: 'CDC Mirror Transaction', type: 'pixels-sink flow', summary: 'Pixels Sink reads the ordered CDC log, reconstructs one mirror transaction, begins it in Retina, replays row changes, and commits only after replay OK.' }, routing: { label: 'Mirror Row Replay', type: 'sink write path', summary: 'Pixels Sink sends row changes from a mirror transaction to Retina replayers; single-operation scenes keep the replay behavior easy to inspect.' }, insert: { label: 'Insert Replay', type: 'write path', summary: 'Insert row changes arrive through a mirror transaction, append to the write buffer, then sealed memtables flush to object entries.' }, - delete: { label: 'Delete Replay', type: 'visibility path', summary: 'Delete unfolds Index Service lookup and RGVisibility metadata: deletePrimaryIndexEntries returns the old RowLocation, then delete timestamp entries update visibility without rewriting columnar files.' }, + delete: { label: 'Delete Replay', type: 'visibility path', summary: 'Delete unfolds Index Service lookup and RGVisibility state: deletePrimaryIndexEntries returns the old RowLocation, then delete timestamp entries update visibility without rewriting columnar files.' }, update: { label: 'Update Replay', type: 'MVCC versioning', summary: 'Update inserts the new tuple version first, updates the primary index, then deletes the old tuple version under one timestamp.' }, query: { label: 'Query Path', type: 'parallel VFoR read path', summary: 'Query plans source splits for regular files, active memtables, and flushed objects in parallel; returned batches merge through one VFoR visibility gate before operators consume visible rows.' }, - gc: { label: 'Garbage Collection', type: 'background function', summary: 'GC is shown as Retina bitmap folding plus Storage GC rewriting regular files, retiring old files in metadata, and deleting them after cleanupAt.' } + gc: { label: 'Garbage Collection', type: 'background function', summary: 'GC is shown as Retina bitmap folding plus Storage GC rewriting regular files, retiring old files, and deleting them after cleanupAt.' } }; const flow = { @@ -577,7 +575,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

replay: '#168a52', query: '#0e7490', visibility: '#6d5bd0', - metadata: '#7c3aed', gc: '#b7791f', tx: '#315fcb', storage: '#0f766e', @@ -638,8 +635,8 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const RETINA_REPLAYER_W = 450; const STORAGE_X = SERVICE_X + RETINA_REPLAYER_W + RUNTIME_STORAGE_GAP; const STORAGE_W = SERVICE_RIGHT - STORAGE_X; - const TOP_SERVICE_W = Math.round((SERVICE_RIGHT - SERVICE_X - SERVICE_GAP * 2) / 3); - const INDEX_W = SERVICE_RIGHT - (SERVICE_X + (TOP_SERVICE_W + SERVICE_GAP) * 2); + const TOP_SERVICE_W = Math.round((SERVICE_RIGHT - SERVICE_X - SERVICE_GAP) / 2); + const INDEX_W = SERVICE_RIGHT - (SERVICE_X + TOP_SERVICE_W + SERVICE_GAP); const CDC_STREAM_H = 116; const CDC_STREAM_ROW_RECORDS = 3; const CDC_STREAM_RECORD_COUNT = CDC_STREAM_ROW_RECORDS + 2; @@ -660,8 +657,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

sink: { x: PIXELS_SINK_X + 14, y: INGRESS_Y + 292, w: PIXELS_SINK_W - 28, h: 96, title: 'Pixels Sink replay', sub: 'replay path anchors' }, tx: { x: SERVICE_X, y: SERVICE_Y, w: TOP_SERVICE_W, h: TX_SERVICE_H, title: 'Transaction Service', sub: 'MVCC time boundary: HWM / LWM' }, - catalog: { x: SERVICE_X + TOP_SERVICE_W + SERVICE_GAP, y: SERVICE_Y, w: TOP_SERVICE_W, h: SERVICE_H, title: 'Metadata Service', sub: 'schema, stats, visible file list' }, - index: { x: SERVICE_X + (TOP_SERVICE_W + SERVICE_GAP) * 2, y: SERVICE_Y, w: INDEX_W, h: SERVICE_H, title: 'Index Service', sub: 'primaryKey→rowId + rowId→RowLocation' }, + index: { x: SERVICE_X + TOP_SERVICE_W + SERVICE_GAP, y: SERVICE_Y, w: INDEX_W, h: SERVICE_H, title: 'Index Service', sub: 'primaryKey→rowId + rowId→RowLocation' }, runtime: { x: SERVICE_X, y: RUNTIME_Y, w: RETINA_REPLAYER_W, h: RETINA_REPLAYER_H, title: 'Retina replayer', sub: 'RetinaWorkerService RPC + write / visibility state' }, storage: { x: STORAGE_X, y: RUNTIME_Y, w: STORAGE_W, h: RUNTIME_H, title: 'Storage', sub: 'flushed immutable memtable + regular columnar files' }, @@ -676,9 +672,8 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

sink: ['Pixels Sink replay anchor', 'Invisible anchor region for replay paths.'], runtime: ['Retina replayer', 'RetinaServerImpl handles RetinaWorkerService RPCs and keeps write-buffer and visibility state.'], tx: ['Transaction Service', 'Exposes MVCC time boundaries. Callers get snapshot or safeTs on demand; the panel stays focused on HWM and LWM.'], - index: ['Index Service', 'Maps primary keys to rowIds and physical row locations; delete/update use this map before touching visibility metadata.'], - storage: ['Storage', 'Stores flushed immutable memtables as object entries plus regular columnar files. Storage GC rewrites visible rows into a new regular file and retires old files through metadata.'], - catalog: ['Metadata Service', 'Publishes schema, statistics, and visible-file metadata for queries and storage rewrite switches.'], + index: ['Index Service', 'Maps primary keys to rowIds and physical row locations; delete/update use this map before touching visibility state.'], + storage: ['Storage', 'Stores flushed immutable memtables as object entries plus regular columnar files. Storage GC rewrites visible rows into a new regular file and retires old files.'], query: ['Query Engine', 'Schedules regular file scans, active memtable RPC, and flushed-object reads as parallel split-like tasks; all returned batches pass through the same VFoR visibility gate.'], lane0: ['Retina replayer', 'Current operation, write buffer, and visibility structure.'], lane1: ['Retina replayer', 'Current operation, write buffer, and visibility structure.'], @@ -750,11 +745,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

newFileState: 'none', cleanup: 'idle' }, - catalog: { - visible: ['file-041.pxl', 'file-042.pxl'], - pending: 'none', - switchState: 'stable' - }, query: { id: 'q44', transTs: DEFAULT_QUERY_TS, @@ -845,7 +835,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.writePlan = 'pk160, pk174, pk181 queued'; m.sink.rpcBatch = 'tx1290 only'; m.sink.requestState = 'read CDC log'; - m.sink.txState = 'tx1290 metadata'; + m.sink.txState = 'tx1290 ready'; } if (scene === 'delete') { m.sink.mirrorTx = 'tx1291'; @@ -860,9 +850,9 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.writePlan = 'pk208 queued'; m.sink.rpcBatch = 'tx1291 only'; m.sink.requestState = 'read CDC log'; - m.sink.txState = 'tx1291 metadata'; + m.sink.txState = 'tx1291 ready'; m.index.hot = 'pk208'; - m.index.operation = 'deletePrimaryIndexEntries pk208'; + m.index.operation = 'await Retina delete'; } if (scene === 'update') { m.sink.mirrorTx = 'tx1292'; @@ -877,7 +867,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.writePlan = 'pk132 queued'; m.sink.rpcBatch = 'tx1292 only'; m.sink.requestState = 'read CDC log'; - m.sink.txState = 'tx1292 metadata'; + m.sink.txState = 'tx1292 ready'; m.index.hot = 'pk132'; m.index.operation = 'insert new + update primary'; } @@ -909,7 +899,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.storage.newFile = 'none'; m.storage.newFileState = 'none'; m.storage.cleanup = 'await memory phase'; - m.catalog.pending = 'await storage phase'; } return m; } @@ -931,21 +920,78 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

step('Pixels Sink creates the mirror transaction BEGIN and Transaction Service assigns its timestamp.', 4.0, ['ingress', 'tx'], [tok('begin tx8421', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=1290', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], m => { m.sink.sinkStep = 'begin'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=1290'; }, 'mvcc', ['tx']), - step('Pixels Sink creates the INSERT row in the mirror transaction and sends its replay RPC.', 3.8, + step('Pixels Sink creates the INSERT row in the mirror transaction and sends its replay RPC.', 3.4, ['ingress', 'runtime'], [tok('INSERT', 'ingress.right', 'runtime.left', flow.replay, .18, .86)], - m => { m.sink.sinkStep = 'row1'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay INSERT'; m.sink.rpcBatch = 'INSERT pending'; m.lanes[0].currentOp = 'I:row'; }, 'dispatch', ['dispatch']), - step('Pixels Sink creates the DELETE row next and sends its replay RPC after INSERT.', 3.8, + m => { m.sink.sinkStep = 'row1'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay INSERT'; m.sink.rpcBatch = 'INSERT pending'; m.lanes[1].currentOp = 'I:pk174'; }, 'dispatch', ['dispatch']), + step('Retina applies the INSERT to the write buffer: the active memtable grows by one row.', 4.0, + ['lane1'], [], + m => { const l = m.lanes[1]; l.currentOp = 'I:pk174'; l.status = 'append after image'; l.activeRows = 8; l.highlight = 'active'; m.sink.rpcBatch = 'INSERT applying'; }, 'writeBuffer', []), + step('The active memtable reaches capacity, so Retina seals it into the immutable queue.', 3.9, + ['lane1'], [], + m => { const l = m.lanes[1]; l.activeRows = 0; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', [], 'seal'), + step('The sealed memtable now sits in the immutable queue while replay opens a fresh active memtable.', 1.2, + ['lane1'], [], + m => { m.lanes[1].highlight = 'immutable'; }, 'writeBuffer', []), + step('A background flush asynchronously moves the immutable memtable to Storage.', 3.8, + ['lane1'], [], + m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; }, 'writeBuffer', [], 'flush'), + step('After the new row is durable, Retina calls Index Service to insert the primary and row-location indexes.', 4.2, + ['index'], [tok('put index pk174', 'runtime.right', 'index.left', flow.index, .12, .82)], + m => { const l = m.lanes[1]; l.highlight = null; if (m.storage.pendingFlushObject && !m.storage.objectEntries.includes(m.storage.pendingFlushObject)) m.storage.objectEntries.unshift(m.storage.pendingFlushObject); m.storage.pendingFlushObject = null; m.index.primaryEntries.unshift('pk174 -> rowId7807'); m.index.mainEntries.unshift('rowId7807 -> fileId78/rg0/off7'); m.index.hot = 'pk174'; m.index.operation = 'putMainIndexEntries + putPrimaryIndexEntries'; }, 'index', ['index']), + step('Index Service finishes the insert index update and returns to Retina.', 3.2, + ['index'], [tok('index OK', 'index.left', 'runtime.right', flow.index, .18, .82)], + m => { m.index.operation = 'index insert OK'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), + step('Retina returns OK for INSERT before Pixels Sink creates the next row change.', 3.2, + ['ingress'], [tok('OK', 'runtime.left', 'ingress.right', flow.replay, .12, .82)], + m => { m.sink.sinkStep = 'row1Ok'; m.sink.requestState = 'INSERT OK'; m.sink.rpcBatch = 'OK'; }, 'mvcc', []), + step('Only after INSERT returns OK, Pixels Sink creates the DELETE row and sends its replay RPC.', 3.4, ['ingress', 'runtime'], [tok('DELETE', 'ingress.right', 'runtime.left', flow.delete, .18, .86)], - m => { m.sink.sinkStep = 'row2'; m.sink.requestState = 'replay DELETE'; m.sink.rpcBatch = 'DELETE pending'; m.lanes[1].currentOp = 'D:row'; }, 'dispatch', ['dispatch']), - step('Pixels Sink creates the UPDATE row last and sends its replay RPC.', 3.8, + m => { m.sink.sinkStep = 'row2'; m.sink.requestState = 'replay DELETE'; m.sink.rpcBatch = 'DELETE pending'; m.lanes[1].currentOp = 'D:pk208'; }, 'dispatch', ['dispatch']), + step('Retina calls Index Service to delete the primary and row-location index entries for pk208.', 4.0, + ['index'], [tok('delete index pk208', 'runtime.right', 'index.left', flow.index, .12, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'DELETE'; l.status = 'delete index entries'; l.highlight = null; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk174 -> rowId7807', 'pk132 -> rowId7705']; m.index.mainEntries = ['rowId7807 -> fileId78/rg0/off7', 'rowId7705 -> fileId77/rg1/off5']; m.index.operation = 'deletePrimaryIndexEntries pk208'; m.sink.rpcBatch = 'DELETE applying'; }, 'index', ['index']), + step('Index Service finishes the delete and returns the old row location to Retina.', 3.2, + ['index'], [tok('index delete OK', 'index.left', 'runtime.right', flow.index, .18, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'index delete OK'; m.index.operation = 'RowLocation off7 returned'; }, 'index', ['index']), + step('Retina appends the delete timestamp into the visibility structure delete-ts chain.', 4.2, + ['lane1'], [tok('(off7,1291)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'append delete timestamp'; l.highlight = 'visibility'; if (!l.deleteTs.includes('(7,1291)')) l.deleteTs.push('(7,1291)'); m.sink.rpcBatch = 'DELETE applying'; m.storage.staleRatio = 0.36; m.storage.cleanup = 'file unchanged'; }, 'visibility', ['visibility'], 'appendDeleteTs'), + step('Retina returns OK for DELETE before Pixels Sink creates the UPDATE row.', 3.2, + ['ingress'], [tok('OK', 'runtime.left', 'ingress.right', flow.delete, .12, .82)], + m => { m.sink.sinkStep = 'row2Ok'; m.sink.requestState = 'DELETE OK'; m.sink.rpcBatch = 'OK'; m.lanes[DETAIL_REPLAY_ID].highlight = null; }, 'mvcc', []), + step('Only after DELETE returns OK, Pixels Sink creates the UPDATE row and sends its replay RPC.', 3.4, ['ingress', 'runtime'], [tok('UPDATE', 'ingress.right', 'runtime.left', flow.index, .18, .86)], - m => { m.sink.sinkStep = 'row3'; m.sink.requestState = 'replay UPDATE'; m.sink.rpcBatch = 'UPDATE pending'; m.lanes[2].currentOp = 'U:row'; }, 'dispatch', ['dispatch']), - step('After every replay RPC returns OK, Pixels Sink can finish the mirror transaction.', 3.6, - ['ingress', 'runtime'], [tok('all replay OK', 'runtime.left', 'ingress.right', flow.replay, .16, .84)], - m => { m.sink.sinkStep = 'rowsOk'; m.sink.requestState = 'all replay OK'; m.sink.rpcBatch = 'all replay OK'; }, 'mvcc', []), - step('Pixels Sink commits tx8421 and Transaction Service advances HWM internally.', 4.2, + m => { m.sink.sinkStep = 'row3'; m.sink.requestState = 'replay UPDATE'; m.sink.rpcBatch = 'UPDATE pending'; m.lanes[1].currentOp = 'U:pk132'; }, 'dispatch', ['dispatch']), + step('Retina inserts the UPDATE after image into the write buffer as the new tuple version.', 4.0, + ['lane1'], [], + m => { const l = m.lanes[1]; l.currentOp = 'U:pk132'; l.status = 'append after image'; l.activeRows = 8; l.highlight = 'active'; m.index.operation = 'await update index'; m.sink.rpcBatch = 'UPDATE applying'; }, 'writeBuffer', []), + step('The active memtable reaches capacity, so Retina seals the new tuple version into the immutable queue.', 3.9, + ['lane1'], [], + m => { const l = m.lanes[1]; l.activeRows = 0; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', [], 'seal'), + step('The sealed memtable now sits in the immutable queue while a fresh active memtable is open.', 1.2, + ['lane1'], [], + m => { m.lanes[1].highlight = 'immutable'; }, 'writeBuffer', []), + step('A background flush asynchronously moves the immutable memtable containing the new tuple version to Storage.', 3.8, + ['lane1'], [], + m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-79'; m.storage.flushedHot = 'obj-79'; }, 'writeBuffer', [], 'flush'), + step('Retina calls Index Service to update the primary index to the new row location.', 3.6, + ['index'], [tok('update index pk132', 'runtime.right', 'index.left', flow.index, .12, .86)], + m => { const l = m.lanes[1]; l.highlight = null; if (m.storage.pendingFlushObject && !m.storage.objectEntries.includes(m.storage.pendingFlushObject)) m.storage.objectEntries.unshift(m.storage.pendingFlushObject); m.storage.pendingFlushObject = null; m.index.primaryEntries = ['pk132 -> rowId7806', 'pk174 -> rowId7807']; m.index.mainEntries = ['rowId7806 -> fileId79/rg0/off6', 'rowId7807 -> fileId78/rg0/off7', 'rowId7705 -> fileId77/rg1/off5']; m.index.hot = 'pk132'; m.index.operation = 'updatePrimaryIndexEntries pk132'; }, 'index', ['index']), + step('Index Service returns the previous row location for the old tuple version.', 3.0, + ['index'], [tok('prevLoc off5', 'index.left', 'runtime.right', flow.index, .16, .84)], + m => { m.index.operation = 'prevLoc off5 returned'; }, 'index', ['index']), + step('Using the returned old location, Retina deletes the old tuple version by appending a delete timestamp to visibility.', 4.2, + ['lane1'], [tok('(off5,1292)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[1]; l.highlight = 'visibility'; if (!l.deleteTs.includes('(5,1292)')) l.deleteTs.push('(5,1292)'); m.index.operation = 'update index OK'; }, 'visibility', ['visibility'], 'appendDeleteTs'), + step('Retina returns OK for UPDATE. Pixels Sink can now finish the mirror transaction.', 3.2, + ['ingress'], [tok('OK', 'runtime.left', 'ingress.right', flow.index, .12, .82)], + m => { m.sink.sinkStep = 'row3Ok'; m.sink.requestState = 'UPDATE OK'; m.sink.rpcBatch = 'OK'; m.lanes[1].highlight = null; }, 'mvcc', []), + step('Pixels Sink creates the COMMIT record and asks Transaction Service to commit tx8421.', 3.2, ['ingress', 'tx'], [tok('commit tx8421', 'ingress.commitRight', 'tx.left', flow.tx, .18, .58)], - m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'HWM advanced'; m.sink.txState = 'tx8421 committed; HWM=1291'; m.sink.rpcBatch = 'tx8421 replay OK'; m.tx.hwm = 1291; }, 'mvcc', ['tx']) + m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'commit requested'; m.sink.txState = 'tx8421 commit in flight'; m.sink.rpcBatch = 'OK'; }, 'mvcc', ['tx']), + step('Transaction Service advances HWM internally for later snapshots.', 2.8, + [], [], + m => { m.sink.sinkStep = 'committed'; m.sink.requestState = 'HWM advanced'; m.sink.txState = 'tx8421 committed; HWM=1291'; m.tx.hwm = 1291; }, 'mvcc', []) ], routing: [ step('Pixels Sink has reconstructed tx43 from the CDC stream and starts replaying its row changes.', 3.8, @@ -993,50 +1039,65 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { const l = m.lanes[1]; l.highlight = null; if (m.storage.pendingFlushObject && !m.storage.objectEntries.includes(m.storage.pendingFlushObject)) m.storage.objectEntries.unshift(m.storage.pendingFlushObject); m.storage.pendingFlushObject = null; m.index.primaryEntries.unshift('pk174 -> rowId7807'); m.index.mainEntries.unshift('rowId7807 -> fileId78/rg0/off7'); m.index.hot = 'pk174'; m.index.operation = 'putMainIndexEntries + putPrimaryIndexEntries'; }, 'index', ['index']), step('Index Service finishes the insert index update and returns to Retina.', 3.2, ['index'], [tok('index OK', 'index.left', 'runtime.right', flow.index, .18, .82)], - m => { m.index.operation = 'index insert OK'; m.sink.rpcBatch = 'INSERT replay OK'; }, 'index', ['index']), - step('Retina returns replay OK to Pixels Sink; the INSERT operation is complete for this mirror transaction.', 3.4, - ['ingress'], [tok('replay OK', 'runtime.left', 'ingress.right', flow.replay, .12, .82)], - m => { m.sink.sinkStep = 'rowsOk'; m.sink.requestState = 'INSERT replay OK'; m.sink.rpcBatch = 'INSERT replay OK'; }, 'mvcc', []), + m => { m.index.operation = 'index insert OK'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), + step('Retina returns OK to Pixels Sink; the INSERT operation is complete for this mirror transaction.', 3.4, + ['ingress'], [tok('OK', 'runtime.left', 'ingress.right', flow.replay, .12, .82)], + m => { m.sink.sinkStep = 'rowsOk'; m.sink.requestState = 'OK'; m.sink.rpcBatch = 'OK'; }, 'mvcc', []), step(`Pixels Sink asks Transaction Service to commit tx1290.`, 3.2, ['ingress', 'tx'], [tok(`commit tx1290`, 'ingress.commitRight', 'tx.left', flow.tx, .12, .82)], m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'commit requested'; m.sink.txState = `tx1290 commit in flight`; }, 'mvcc', ['tx']), step(`Transaction Service advances HWM internally for later snapshots.`, 2.8, - ['tx'], [], - m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'HWM advanced'; m.sink.txState = 'tx1290 committed; HWM=1291'; m.tx.hwm = 1291; }, 'mvcc', ['tx']) + [], [], + m => { m.sink.sinkStep = 'committed'; m.sink.requestState = 'HWM advanced'; m.sink.txState = 'tx1290 committed; HWM=1291'; m.tx.hwm = 1291; }, 'mvcc', []) ], delete: [ ...singleOpCdcPrelude('delete', 'tx1291'), - step('Delete replay starts after Pixels Sink begins mirror transaction tx1291 with timestamp=1291.', 3.7, - ['ingress', 'tx', 'lane1'], [tok('timestamp=1291', 'tx.left', 'ingress.beginRight', flow.tx, .10, .52), tok('DELETE', 'ingress.right', 'runtime.left', flow.delete, .50, .96)], - m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=1291'; }, 'mvcc', ['cdc', 'tx']), - step('Retina receives D:pk208 and calls Index Service deletePrimaryIndexEntries to remove the primary entry and return the current RowLocation.', 4.0, - ['ingress', 'lane1', 'index'], [tok('D:pk208', 'ingress.right', 'runtime.left', flow.delete, .05, .48), tok('deletePrimary pk208', 'lane1.in', 'index.in', flow.index, .42, .88)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'D:pk208'; l.status = 'delete primary entry'; m.index.hot = 'pk208'; m.index.operation = 'deletePrimaryIndexEntries pk208'; }, 'index', ['index']), - step('Index Service returns RowLocation(fileId, rgId, rowOffset). The old tuple stays in place until visibility says otherwise.', 3.7, - ['index', 'lane1'], [tok('file-042 rg0 off7', 'index.out', 'lane1.visibility', flow.index, .08, .80)], - m => { m.index.primaryEntries = ['pk208 -> rowId8208', 'pk132 -> rowId7705']; m.index.mainEntries = ['rowId8208 -> file-042/rg0/off7', 'rowId7705 -> fileId77/rg1/off5']; m.index.operation = 'RowLocation returned'; m.lanes[DETAIL_REPLAY_ID].highlight = 'visibility'; }, 'index', ['index']), - step('Delete appends (offset, deleteTs) into the tile visibility delete-ts array. Queries see the change through VFoR metadata rather than a file rewrite.', 4.4, - ['lane1', 'storage'], [tok('(off7,1291)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .08, .76), tok('metadata only', 'lane1.deleteArray', 'storage.files', flow.visibility, .60, .96)], - m => { m.lanes[DETAIL_REPLAY_ID].deleteTs.push('(7,1291)'); m.storage.staleRatio = 0.36; m.storage.cleanup = 'file unchanged'; }, 'visibility', ['visibility']), + step('Pixels Sink begins mirror transaction tx1291 and Transaction Service assigns timestamp=1291.', 4.0, + ['ingress', 'tx'], [tok('begin tx1291', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=1291', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], + m => { m.sink.sinkStep = 'begin'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=1291'; }, 'mvcc', ['tx']), + step('Pixels Sink sends the DELETE row changes after the mirror transaction has a timestamp.', 3.6, + ['ingress'], [tok('DELETE', 'ingress.right', 'runtime.left', flow.delete, .18, .86)], + m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=1291'; }, 'dispatch', []), + step('Retina calls Index Service to delete the primary and row-location index entries for pk208.', 4.0, + ['index'], [tok('delete index pk208', 'runtime.right', 'index.left', flow.index, .12, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'DELETE'; l.status = 'delete index entries'; l.highlight = null; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk132 -> rowId7705']; m.index.mainEntries = ['rowId7705 -> fileId77/rg1/off5']; m.index.operation = 'deletePrimaryIndexEntries pk208'; m.sink.rpcBatch = 'DELETE applying'; }, 'index', ['index']), + step('Index Service finishes the delete and returns the old row location to Retina.', 3.2, + ['index'], [tok('index delete OK', 'index.left', 'runtime.right', flow.index, .18, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'index delete OK'; m.index.operation = 'RowLocation off7 returned'; }, 'index', ['index']), + step('Retina then appends the delete timestamp into the visibility structure delete-ts chain.', 4.2, + ['lane1'], [tok('(off7,1291)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'append delete timestamp'; l.highlight = 'visibility'; if (!l.deleteTs.includes('(7,1291)')) l.deleteTs.push('(7,1291)'); m.sink.rpcBatch = 'DELETE applying'; m.storage.staleRatio = 0.36; m.storage.cleanup = 'file unchanged'; }, 'visibility', ['visibility'], 'appendDeleteTs'), ...commitSequence('tx1291', 1292, 'delete committed; HWM=1292') ], update: [ ...singleOpCdcPrelude('update', 'tx1292'), - step('Update replay arrives after Pixels Sink begins mirror transaction tx1292 with timestamp=1292.', 3.8, - ['ingress', 'tx', 'lane1'], [tok('timestamp=1292', 'tx.left', 'ingress.beginRight', flow.tx, .10, .52), tok('UPDATE', 'ingress.right', 'runtime.left', flow.index, .50, .96)], - m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay UPDATE'; m.sink.txState = 'timestamp=1292'; }, 'mvcc', ['tx']), - step('Retina first inserts the after image into the write buffer and allocates a new rowId plus RowLocation for the new tuple version.', 4.2, - ['lane1', 'index'], [tok('after image', 'lane1.in', 'lane1.active', flow.index, .06, .52), tok('new RowLocation', 'lane1.active', 'index.in', flow.index, .48, .94)], - m => { m.lanes[1].currentOp = 'U:pk132'; m.lanes[1].activeRows = 6; m.index.mainEntries = ['rowId7806 -> fileId78/rg0/off6', 'rowId7705 -> file-041/rg1/off5', 'rowId8208 -> file-042/rg0/off7']; m.index.hot = 'pk132'; m.index.operation = 'insertRecord -> rowId7806'; m.lanes[1].highlight = 'active'; }, 'writeBuffer', ['index']), - step('Index Service updatePrimaryIndexEntries installs the new primary entry and returns prevLocs for the old tuple version.', 4.2, - ['lane1', 'index'], [tok('updatePrimary pk132', 'lane1.active', 'index.in', flow.index, .06, .54), tok('prevLoc off5', 'index.out', 'lane1.visibility', flow.index, .46, .94)], - m => { m.index.primaryEntries = ['pk132 -> rowId7806', 'pk208 -> rowId8208']; m.index.hot = 'pk132'; m.index.operation = 'updatePrimaryIndexEntries -> prevLocs'; m.lanes[1].highlight = 'visibility'; }, 'index', ['index']), - step('Retina then deletes the old data by appending a delete timestamp to RGVisibility for each returned previous RowLocation.', 4.2, - ['lane1', 'index'], [tok('delete old off5', 'index.out', 'lane1.deleteArray', flow.delete, .08, .78)], - m => { m.lanes[1].deleteTs.push('(5,1292)'); m.index.operation = 'deleteRecord(prevLocs)'; m.lanes[1].highlight = 'visibility'; }, 'visibility', ['index', 'visibility']), - step('Multiple versions of the same primary key can coexist. Query transaction timestamp decides whether the old or new version is visible.', 4.0, - ['tx', 'query', 'lane1'], [tok(`transTs ${DEFAULT_QUERY_TS} sees old`, 'query.left', 'tx.window', flow.query, .08, .55), tok('transTs 1292 sees new', 'tx.window', 'lane1.visibility', flow.visibility, .48, .94)], - m => { m.query.transTs = DEFAULT_QUERY_TS; m.query.active = true; m.tx.lwm = DEFAULT_QUERY_TS; }, 'visibility', ['query', 'visibility']), + step('Pixels Sink begins mirror transaction tx1292 and Transaction Service assigns timestamp=1292.', 4.0, + ['ingress', 'tx'], [tok('begin tx1292', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=1292', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], + m => { m.sink.sinkStep = 'begin'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=1292'; }, 'mvcc', ['tx']), + step('Pixels Sink sends the UPDATE row changes after the mirror transaction has a timestamp.', 3.6, + ['ingress'], [tok('UPDATE', 'ingress.right', 'runtime.left', flow.index, .18, .86)], + m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay UPDATE'; m.sink.txState = 'timestamp=1292'; }, 'dispatch', []), + step('Retina first inserts the after image into the write buffer as the new tuple version.', 4.0, + ['lane1'], [], + m => { const l = m.lanes[1]; l.currentOp = 'U:pk132'; l.status = 'append after image'; l.activeRows = 8; l.highlight = 'active'; m.index.operation = 'await update index'; m.sink.rpcBatch = 'UPDATE applying'; }, 'writeBuffer', []), + step('The active memtable reaches capacity, so Retina seals the new tuple version into the immutable queue.', 3.9, + ['lane1'], [], + m => { const l = m.lanes[1]; l.activeRows = 0; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', [], 'seal'), + step('The sealed memtable now sits in the immutable queue while a fresh active memtable is open.', 1.2, + ['lane1'], [], + m => { const l = m.lanes[1]; l.highlight = 'immutable'; }, 'writeBuffer', []), + step('A background flush asynchronously moves the immutable memtable containing the new tuple version to Storage.', 3.8, + ['lane1'], [], + m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; }, 'writeBuffer', [], 'flush'), + step('Retina calls Index Service to update the primary index to the new row location.', 3.6, + ['index'], [tok('update index pk132', 'runtime.right', 'index.left', flow.index, .12, .86)], + m => { const l = m.lanes[1]; l.highlight = null; if (m.storage.pendingFlushObject && !m.storage.objectEntries.includes(m.storage.pendingFlushObject)) m.storage.objectEntries.unshift(m.storage.pendingFlushObject); m.storage.pendingFlushObject = null; m.index.primaryEntries = ['pk132 -> rowId7806', 'pk208 -> rowId8208']; m.index.mainEntries = ['rowId7806 -> fileId78/rg0/off6', 'rowId7705 -> file-041/rg1/off5', 'rowId8208 -> file-042/rg0/off7']; m.index.hot = 'pk132'; m.index.operation = 'updatePrimaryIndexEntries pk132'; }, 'index', ['index']), + step('Index Service returns the previous row location for the old tuple version.', 3.0, + ['index'], [tok('prevLoc off5', 'index.left', 'runtime.right', flow.index, .16, .84)], + m => { m.index.operation = 'prevLoc off5 returned'; }, 'index', ['index']), + step('Using the returned old location, Retina deletes the old tuple version by appending a delete timestamp to visibility.', 4.2, + ['lane1'], [tok('(off5,1292)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[1]; l.highlight = 'visibility'; if (!l.deleteTs.includes('(5,1292)')) l.deleteTs.push('(5,1292)'); m.index.operation = 'update index OK'; }, 'visibility', ['visibility'], 'appendDeleteTs'), ...commitSequence('tx1292', 1293, 'update committed; HWM=1293') ], query: [ @@ -1071,19 +1132,19 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { m.lanes[DETAIL_REPLAY_ID].safeTs = DEFAULT_GC_SAFE_TS; m.lanes[2].status = 'replay continues'; m.query.active = false; }, 'gc', ['dispatch', 'query']), step('Storage GC selects low-valid-ratio regular files. They remain regular columnar files, just highlighted as rewrite candidates.', 4.0, ['runtime', 'storage'], [tok('candidate files', 'runtime.right', 'storage.aging', flow.gc, .08, .76)], - m => { m.storage.agingQueue = ['file-038.pxl', 'file-039.pxl']; m.storage.cleanup = 'rewrite candidates'; m.catalog.pending = 'select file-038.pxl'; }, 'storageGc', ['storage']), + m => { m.storage.agingQueue = ['file-038.pxl', 'file-039.pxl']; m.storage.cleanup = 'rewrite candidates'; }, 'storageGc', ['storage']), step('Storage GC rewrites still-visible tuple versions into a new TEMPORARY_GC file inside the regular-files panel.', 4.5, ['storage', 'tx'], [tok(`safeTs=${DEFAULT_GC_SAFE_TS}`, 'tx.hwm', 'storage.files', flow.tx, .08, .48), tok('visible tuples', 'storage.aging', 'storage.newFile', flow.storage, .42, .96)], - m => { m.storage.newFile = 'file-043.pxl'; m.storage.newFileState = 'TEMPORARY_GC'; m.storage.cleanup = 'new TEMPORARY_GC'; m.catalog.pending = 'stage file-043.pxl'; }, 'storageGc', ['tx']), + m => { m.storage.newFile = 'file-043.pxl'; m.storage.newFileState = 'TEMPORARY_GC'; m.storage.cleanup = 'new TEMPORARY_GC'; }, 'storageGc', ['tx']), step('Before publishing it, Retina registers dual-write, synchronizes visibility, and refreshes main and primary index locations.', 4.6, ['storage', 'index', 'lane2'], [tok('location map', 'storage.newFile', 'index.in', flow.index, .06, .56), tok('sync visibility', 'storage.newFile', 'lane2.in', flow.visibility, .48, .94)], - m => { m.index.operation = 'storage GC location map'; m.index.mainEntries = ['rowId6032 -> file-043/rg0/off2', 'rowId6033 -> file-043/rg0/off3']; m.catalog.pending = 'dual-write file-043.pxl'; m.lanes[2].deleteTs.push('(dual,1293)'); m.lanes[2].highlight = 'visibility'; }, 'index', ['index', 'visibility']), - step('Metadata Service atomicSwapFiles promotes the new file to REGULAR and marks old regular files RETIRED with cleanupAt.', 4.2, - ['catalog', 'storage'], [tok('TEMP -> REGULAR', 'storage.newFile', 'catalog.staging', flow.gc, .08, .46), tok('old -> RETIRED', 'storage.aging', 'catalog.retired', flow.delete, .44, .90)], - m => { m.storage.newFileState = 'REGULAR'; m.storage.regularFiles = ['file-041.pxl', 'file-042.pxl', 'file-043.pxl']; m.storage.cleanup = 'old files RETIRED cleanupAt'; m.catalog.pending = 'file-038 -> file-043'; m.catalog.visible = ['file-041.pxl', 'file-042.pxl', 'file-043.pxl']; m.catalog.switchState = 'switching'; }, 'storageGc', ['catalog']), + m => { m.index.operation = 'storage GC location map'; m.index.mainEntries = ['rowId6032 -> file-043/rg0/off2', 'rowId6033 -> file-043/rg0/off3']; m.lanes[2].deleteTs.push('(dual,1293)'); m.lanes[2].highlight = 'visibility'; }, 'index', ['index', 'visibility']), + step('Retina promotes the new file to REGULAR and marks old regular files RETIRED with cleanupAt.', 4.2, + ['storage'], [tok('TEMP -> REGULAR', 'storage.newFile', 'storage.files', flow.gc, .08, .46), tok('old -> RETIRED', 'storage.aging', 'storage.files', flow.delete, .44, .90)], + m => { m.storage.newFileState = 'REGULAR'; m.storage.regularFiles = ['file-041.pxl', 'file-042.pxl', 'file-043.pxl']; m.storage.cleanup = 'old files RETIRED cleanupAt'; }, 'storageGc', ['storage']), step('After retireDelayMs / cleanupAt, processRetiredFiles reclaims visibility and index state, then physically deletes retired files.', 4.0, - ['catalog', 'storage', 'lane2'], [tok('cleanupAt reached', 'catalog.retired', 'storage.aging', flow.gc, .08, .72), tok('reclaim metadata', 'lane2.out', 'runtime.right', flow.gc, .32, .92)], - m => { m.storage.staleRatio = 0.18; m.storage.agingQueue = []; m.storage.newFile = 'none'; m.storage.newFileState = 'none'; m.storage.regularFiles = ['file-041.pxl', 'file-042.pxl', 'file-043.pxl']; m.storage.cleanup = 'retired files physically deleted'; m.catalog.switchState = 'stable'; m.catalog.pending = 'none'; m.lanes[2].safeTs = DEFAULT_GC_SAFE_TS; m.lanes[2].highlight = 'baseBitmap'; }, 'gc', ['visibility']) + ['storage', 'lane2'], [tok('cleanupAt reached', 'storage.files', 'storage.aging', flow.gc, .08, .72), tok('reclaim state', 'lane2.out', 'runtime.right', flow.gc, .32, .92)], + m => { m.storage.staleRatio = 0.18; m.storage.agingQueue = []; m.storage.newFile = 'none'; m.storage.newFileState = 'none'; m.storage.regularFiles = ['file-041.pxl', 'file-042.pxl', 'file-043.pxl']; m.storage.cleanup = 'retired files physically deleted'; m.lanes[2].safeTs = DEFAULT_GC_SAFE_TS; m.lanes[2].highlight = 'baseBitmap'; }, 'gc', ['visibility']) ] }; @@ -1124,18 +1185,15 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function commitSequence(txId, hwm, committedStatus) { return [ - step(`Pixels Sink waits to commit ${txId} until every replay request for that mirror transaction has returned OK.`, 3.4, - ['ingress', 'runtime'], [], - m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'wait replay OK'; m.sink.txState = `${txId} not committed`; m.sink.rpcBatch = `${txId} replay pending`; }, 'mvcc', [], 'waitCommit'), - step(`Retina replayer returns replay OK for all row changes in ${txId}; only now is the mirror transaction eligible to commit.`, 3.4, - ['runtime', 'ingress'], [tok(`all replay OK`, 'runtime.left', 'ingress.commitRight', flow.replay, .12, .82)], - m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'all replay OK'; m.sink.txState = `${txId} commit eligible`; m.sink.rpcBatch = `${txId} replay OK`; }, 'mvcc', []), + step(`Retina replayer returns OK for the row change in ${txId}; only now is the mirror transaction eligible to commit.`, 3.4, + ['runtime', 'ingress'], [tok(`OK`, 'runtime.left', 'ingress.right', flow.replay, .12, .82)], + m => { m.sink.sinkStep = 'rowsOk'; m.sink.requestState = 'OK'; m.sink.txState = `${txId} commit eligible`; m.sink.rpcBatch = 'OK'; }, 'mvcc', []), step(`Pixels Sink asks Transaction Service to commit ${txId}.`, 3.2, ['ingress', 'tx'], [tok(`commit ${txId}`, 'ingress.commitRight', 'tx.left', flow.tx, .12, .82)], m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'commit requested'; m.sink.txState = `${txId} commit in flight`; }, 'mvcc', ['tx']), step(`Transaction Service advances HWM internally for later snapshots.`, 2.8, - ['tx'], [], - m => { m.sink.sinkStep = 'commit'; m.sink.requestState = 'HWM advanced'; m.sink.txState = committedStatus; m.tx.hwm = hwm; }, 'mvcc', ['tx']) + [], [], + m => { m.sink.sinkStep = 'committed'; m.sink.requestState = 'HWM advanced'; m.sink.txState = committedStatus; m.tx.hwm = hwm; }, 'mvcc', []) ]; } @@ -1366,7 +1424,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function hitTest(x, y) { const zones = [ ['cdcStream', rects.cdcStream], ['ingress', rects.ingress], ['runtime', rects.runtime], - ['tx', rects.tx], ['index', rects.index], ['catalog', rects.catalog], + ['tx', rects.tx], ['index', rects.index], ['storage', rects.storage], ['query', rects.query] ]; for (let i = zones.length - 1; i >= 0; i--) { @@ -1401,7 +1459,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

drawPixelsSink(c); drawQuery(c); drawTransactionService(c); - drawCatalog(c); drawIndex(c); drawReplayers(c); drawStorage(c); @@ -1592,7 +1649,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function currentPixelsSinkStep() { const explicit = model.sink.sinkStep; - if (['empty', 'read', 'build', 'begin', 'row1', 'row2', 'row3', 'rowsOk', 'replay', 'commit'].includes(explicit)) return explicit; + if (['empty', 'read', 'build', 'begin', 'row1', 'row1Ok', 'row2', 'row2Ok', 'row3', 'row3Ok', 'rowsOk', 'replay', 'commit', 'committed'].includes(explicit)) return explicit; const text = `${model.sink.requestState} ${model.sink.rpcBatch} ${model.sink.txState}`; if (/HWM|commit|OK|pending|eligible/i.test(text)) return 'commit'; if (/replay|RPC|row/i.test(text)) return 'replay'; @@ -1719,7 +1776,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function drawMirrorTransactionPanel(layout, activeId, c) { const { txn, begin, rows, replay, commit } = layout; - const active = !!activeId && activeId !== 'empty'; + const active = !!activeId && activeId !== 'empty' && activeId !== 'committed'; const color = pixelsSinkActivityColor(activeId); ctx.fillStyle = alpha(c.panel2, active ? .82 : .72); ctx.strokeStyle = active ? alpha(color, .56) : c.borderSoft; @@ -1733,26 +1790,26 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

if (activeId === 'empty') return; const visibleRows = visibleMirrorRows(activeId, rows.length); - const visibleCommit = activeId === 'commit'; + const visibleCommit = activeId === 'commit' || activeId === 'committed'; drawMirrorBeginCard(begin, activeId === 'begin' || activeId === 'build', c); drawMirrorStackArea(layout, visibleRows, ['row1', 'row2', 'row3', 'replay', 'rowsOk'].includes(activeId), c); rows.slice(0, visibleRows).forEach((row, i) => drawMirrorRowCard(row, rowStageActive(activeId, i), c)); - if (visibleCommit) drawMirrorCommitCard(commit, true, c); + if (visibleCommit) drawMirrorCommitCard(commit, activeId === 'commit', c); } function pixelsSinkActivityColor(activeId) { if (activeId === 'replay') return currentReplayOperationColor(); - if (activeId === 'row1') return flow.replay; - if (activeId === 'row2') return flow.delete; - if (activeId === 'row3') return flow.index; + if (activeId === 'row1' || activeId === 'row1Ok') return flow.replay; + if (activeId === 'row2' || activeId === 'row2Ok') return flow.delete; + if (activeId === 'row3' || activeId === 'row3Ok') return flow.index; return flow.cdc; } function visibleMirrorRows(activeId, total) { - if (['replay', 'rowsOk', 'commit'].includes(activeId)) return total; - if (activeId === 'row1') return Math.min(1, total); - if (activeId === 'row2') return Math.min(2, total); - if (activeId === 'row3') return Math.min(3, total); + if (['replay', 'rowsOk', 'commit', 'committed'].includes(activeId)) return total; + if (activeId === 'row1' || activeId === 'row1Ok') return Math.min(1, total); + if (activeId === 'row2' || activeId === 'row2Ok') return Math.min(2, total); + if (activeId === 'row3' || activeId === 'row3Ok') return Math.min(3, total); return 0; } @@ -1801,9 +1858,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.font = typeFont(TYPE.text, 900); ctx.textAlign = 'left'; ctx.fillText('COMMIT', box.x + 18, box.y + 30); - if (!active) { - drawServiceChip(box.x + box.w - 86, box.y + 13, 68, 36, 'HWM', flow.cdc, c, false); - } } function drawMiniTxCard(x, y, w, h, label, color, c, active) { @@ -2801,7 +2855,9 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

} function drawReplayVisibility(x, y, w, h, lane, c) { - drawNeutralPanelBox(x, y, w, h, 'visibility structure', c); + const visibilityActive = state.detailFocus === 'visibility' || state.detailFocus === 'gc' || + lane.highlight === 'visibility' || lane.highlight === 'baseBitmap' || lane.highlight === 'foldEligible'; + drawNeutralPanelBox(x, y, w, h, 'visibility structure', c, 9, visibilityActive); const layout = replayVisibilityLayout(x, y, w, h); const { rowX, rowY, labelW, bitmapX, bitmapW, chainY, chainH } = layout; @@ -2816,9 +2872,9 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const chainW = w - MODULE_CONTENT_PAD * 2; const chainPad = 8; const itemPad = 20; - ctx.fillStyle = alpha(c.panel2, .78); - ctx.strokeStyle = c.borderSoft; - ctx.lineWidth = 1; + ctx.fillStyle = visibilityActive ? alpha(flow.visibility, .10) : alpha(c.panel2, .78); + ctx.strokeStyle = visibilityActive ? alpha(flow.visibility, .48) : c.borderSoft; + ctx.lineWidth = visibilityActive ? 1.4 : 1; roundRect(chainX, chainY, chainW, chainH, 7, true, true); ctx.fillStyle = c.text; ctx.font = typeFont(TYPE.text, 800); @@ -2876,7 +2932,10 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function drawCompactDeleteTsItems(x, y, w, lane, c) { const columns = 4; const rows = 2; - const visible = lane.deleteTs.slice(0, columns * rows); + const st = currentStep(); + const hideAppendingItem = st.anim === 'appendDeleteTs' && stepProgress() < .82; + const items = hideAppendingItem ? lane.deleteTs.slice(0, -1) : lane.deleteTs; + const visible = items.slice(0, columns * rows); const itemW = Math.min(72, Math.floor((w - 18 * (columns - 1)) / columns)); const gapX = columns > 1 ? (w - itemW * columns) / (columns - 1) : 0; const itemH = 24; @@ -2886,18 +2945,47 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const row = Math.floor(i / columns); const ts = deleteTsValue(item); const eligible = lane.safeTs != null && ts <= lane.safeTs; - const active = lane.highlight === 'foldEligible' && eligible; - const color = active ? flow.gc : flow.delete; + const gcActive = lane.highlight === 'foldEligible' && eligible; + const appendVisible = st.anim !== 'appendDeleteTs' || !hideAppendingItem; + const active = gcActive || (lane.highlight === 'visibility' && appendVisible && i === visible.length - 1); + const color = gcActive ? flow.gc : flow.delete; drawPill(x + col * (itemW + gapX), y + row * (itemH + rowGap), itemW, itemH, item, color, c, active, 8); }); - if (lane.deleteTs.length > columns * rows) { + if (hideAppendingItem) { + drawAppendingDeleteTsItem(x, y, itemW, itemH, gapX, rowGap, columns, rows, lane, c); + } + if (items.length > columns * rows) { ctx.fillStyle = c.muted; ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'right'; - ctx.fillText(`+${lane.deleteTs.length - columns * rows}`, x + w, y + rows * itemH + rowGap); + ctx.fillText(`+${items.length - columns * rows}`, x + w, y + rows * itemH + rowGap); } } + function drawAppendingDeleteTsItem(x, y, itemW, itemH, gapX, rowGap, columns, rows, lane, c) { + const p = stepProgress(); + if (p < .10) return; + const item = lane.deleteTs[lane.deleteTs.length - 1]; + if (!item) return; + const t = ease(clamp((p - .10) / (.82 - .10), 0, 1)); + const finalIndex = Math.min(lane.deleteTs.length - 1, columns * rows - 1); + const col = finalIndex % columns; + const row = Math.floor(finalIndex / columns); + const finalX = x + col * (itemW + gapX); + const finalY = y + row * (itemH + rowGap); + const finalCenter = { x: finalX + itemW / 2, y: finalY + itemH / 2 }; + const start = point('lane1.visibility'); + const cx = lerp(start.x, finalCenter.x, t); + const cy = lerp(start.y, finalCenter.y, t); + const scale = lerp(1.75, 1, t); + + ctx.save(); + ctx.translate(cx, cy); + ctx.scale(scale, scale); + drawPill(-itemW / 2, -itemH / 2, itemW, itemH, item, flow.delete, c, true, 8); + ctx.restore(); + } + function deleteOffsetValue(item) { const m = /\((\d+),/.exec(item); return m ? Number(m[1]) : Number.NaN; @@ -3026,219 +3114,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

fitText(label, x + 8, y + h - 10, w - 16, 'left'); } - function drawCatalog(c) { - const r = rects.catalog; - const color = flow.metadata || flow.visibility; - const active = isHighlighted('catalog'); - const queryActive = active && state.detailFocus === 'query'; - const switchActive = active && (state.detailFocus === 'storageGc' || model.catalog.switchState === 'switching'); - - drawComponent(r, c, { color, key: 'catalog' }); - - const x = r.x + MODULE_CONTENT_PAD; - const topY = r.y + MODULE_TITLE_Y + MODULE_CONTENT_GAP; - const w = r.w - MODULE_CONTENT_PAD * 2; - const topH = 28; - const shelfY = topY + topH + 10; - const shelfH = 44; - const stagingY = shelfY + shelfH + 8; - const stagingH = Math.max(18, r.y + r.h - stagingY - MODULE_CONTENT_PAD); - drawMetadataBlueprint(x, topY, 86, topH, c, queryActive); - drawMetadataStats(x + 96, topY, 76, topH, c, queryActive); - drawMetadataSwitch(x + 182, topY, w - 182, topH, c, switchActive); - - drawMetadataShelf(x, shelfY, 202, shelfH, c, queryActive, switchActive); - drawMetadataRetiredBin(x + 212, shelfY, w - 212, shelfH, c, switchActive); - drawMetadataStaging(x, stagingY, w, stagingH, c, switchActive); - drawMetadataCatalogMotion(x, shelfY, 202, stagingY, w, c); - } - - function drawMetadataBlueprint(x, y, w, h, c, active) { - const color = flow.metadata || flow.visibility; - ctx.fillStyle = alpha(color, active ? .18 : .08); - ctx.strokeStyle = alpha(color, active ? .56 : .24); - ctx.lineWidth = active ? 1.4 : 1; - roundRect(x, y, w, h, 7, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'left'; - fitText('SCHEMA', x + 8, y + 14, w - 16, 'left'); - ctx.strokeStyle = alpha(color, active ? .66 : .34); - ctx.lineWidth = 1; - for (let i = 0; i < 3; i++) { - const yy = y + 15 + i * 4; - ctx.beginPath(); - ctx.moveTo(x + 8, yy); - ctx.lineTo(x + w - 8, yy); - ctx.stroke(); - } - for (let i = 1; i < 3; i++) { - const xx = x + 8 + i * ((w - 16) / 3); - ctx.beginPath(); - ctx.moveTo(xx, y + 14); - ctx.lineTo(xx, y + h - 5); - ctx.stroke(); - } - } - - function drawMetadataStats(x, y, w, h, c, active) { - const color = flow.query; - ctx.fillStyle = alpha(color, active ? .16 : .07); - ctx.strokeStyle = alpha(color, active ? .52 : .22); - ctx.lineWidth = active ? 1.4 : 1; - roundRect(x, y, w, h, 7, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'left'; - ctx.fillText('STATS', x + 8, y + 14); - const bars = [7, 12, 9, 16]; - bars.forEach((bh, i) => { - ctx.fillStyle = alpha(color, active ? .62 : .38); - roundRect(x + 34 + i * 8, y + h - 5 - bh, 5, bh, 2, true, false); - }); - } - - function drawMetadataSwitch(x, y, w, h, c, active) { - const color = active ? flow.gc : (flow.metadata || flow.visibility); - const switching = model.catalog.switchState === 'switching'; - const p = stepProgress(true); - ctx.fillStyle = alpha(color, active ? .18 : .07); - ctx.strokeStyle = alpha(color, active ? .56 : .24); - ctx.lineWidth = active ? 1.5 : 1; - roundRect(x, y, w, h, 7, true, true); - const railX = x + 12; - const railY = y + h / 2 + 3; - const railW = w - 24; - ctx.strokeStyle = alpha(color, .44); - ctx.lineWidth = 3; - ctx.beginPath(); - ctx.moveTo(railX, railY); - ctx.lineTo(railX + railW, railY); - ctx.stroke(); - const knobX = switching ? lerp(railX + 6, railX + railW - 6, p) : railX + 8; - ctx.fillStyle = color; - ctx.beginPath(); - ctx.arc(knobX, railY, 5, 0, Math.PI * 2); - ctx.fill(); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'center'; - fitText(switching ? 'ATOMIC SWITCH' : 'LOCKED CATALOG', x + 6, y + 14, w - 12, 'center'); - } - - function drawMetadataShelf(x, y, w, h, c, queryActive, switchActive) { - const color = flow.metadata || flow.visibility; - ctx.fillStyle = alpha(color, queryActive ? .16 : .07); - ctx.strokeStyle = alpha(color, queryActive ? .54 : .24); - ctx.lineWidth = queryActive ? 1.5 : 1; - roundRect(x, y, w, h, 8, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'left'; - fitText('VISIBLE MANIFEST', x + 8, y + 14, w - 16, 'left'); - const files = model.catalog.visible.slice(0, 3); - const gap = 5; - const cardW = Math.min(58, (w - 16 - gap * Math.max(files.length - 1, 0)) / Math.max(files.length, 1)); - const cardY = y + 18; - files.forEach((file, i) => { - const cardX = x + 8 + i * (cardW + gap); - const isNew = file === model.storage.newFile || /file-043/.test(file); - drawMetadataFileCard(cardX, cardY, cardW, Math.max(22, h - 24), file, isNew ? flow.storage : color, c, queryActive || (switchActive && isNew)); - }); - } - - function drawMetadataFileCard(x, y, w, h, file, color, c, active) { - ctx.fillStyle = alpha(color, active ? .24 : .11); - ctx.strokeStyle = alpha(color, active ? .62 : .30); - ctx.lineWidth = active ? 1.3 : 1; - roundRect(x, y, w, h, 4, true, true); - ctx.fillStyle = alpha(color, .34); - roundRect(x + 4, y + 4, w - 8, 2.5, 2, true, false); - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText(shortFileLabel(file), x + 3, y + h - 7, w - 6, 'center'); - } - - function drawMetadataRetiredBin(x, y, w, h, c, active) { - const label = catalogRetiredLabel(); - const color = label === 'clean' ? flow.replay : flow.delete; - ctx.fillStyle = alpha(color, active ? .15 : .06); - ctx.strokeStyle = alpha(color, active ? .48 : .22); - ctx.lineWidth = active ? 1.4 : 1; - roundRect(x, y, w, h, 8, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'center'; - fitText('RETIRED', x + 5, y + 14, w - 10, 'center'); - ctx.fillStyle = label === 'empty' ? c.muted : color; - ctx.font = typeFont(TYPE.text, 900); - fitText(label, x + 5, y + h - 10, w - 10, 'center'); - } - - function drawMetadataStaging(x, y, w, h, c, active) { - const color = flow.gc; - const pending = catalogPendingLabel(); - const hasPending = pending !== 'empty'; - ctx.fillStyle = alpha(color, hasPending ? .18 : .05); - ctx.strokeStyle = alpha(color, hasPending ? .52 : .18); - ctx.lineWidth = hasPending ? 1.4 : 1; - roundRect(x, y, w, h, 8, true, true); - ctx.fillStyle = hasPending ? color : c.muted; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'left'; - fitText(`STAGING ${pending}`, x + 8, y + 15, w * .44, 'left'); - - const start = { x: x + w * .50, y: y + h / 2 }; - const gate = { x: x + w * .70, y: y + h / 2 }; - const end = { x: x + w - 10, y: y + h / 2 }; - ctx.strokeStyle = alpha(active ? flow.gc : c.border, hasPending ? .62 : .30); - ctx.lineWidth = 1.5; - drawPolyline([start, gate, end]); - arrowHead(gate, end, alpha(active ? flow.gc : c.border, hasPending ? .72 : .36)); - ctx.fillStyle = active ? flow.gc : c.muted; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText(hasPending ? 'publish gate' : 'no pending switch', x + w * .50, y + 15, w * .44, 'center'); - } - - function drawMetadataCatalogMotion(x, shelfY, shelfW, stagingY, totalW, c) { - if (model.catalog.switchState !== 'switching') return; - const pending = catalogPendingLabel(); - if (pending === 'empty') return; - const p = stepProgress(true); - const start = { x: x + 78, y: stagingY + 8 }; - const end = { x: x + shelfW - 35, y: shelfY + 23 }; - ctx.save(); - ctx.globalAlpha = .88; - drawMetadataFileCard(lerp(start.x, end.x, p) - 32, lerp(start.y, end.y, p) - 11, 64, 24, pending, flow.storage, c, true); - ctx.restore(); - } - - function shortFileLabel(file) { - const m = /file-(\d+)/.exec(String(file)); - return m ? m[1] : String(file).replace('.pxl', '').slice(0, 6); - } - - function catalogPendingLabel() { - if (model.storage.newFile && model.storage.newFile !== 'none') return shortFileLabel(model.storage.newFile); - const pending = String(model.catalog.pending || 'none'); - if (pending === 'none') return 'empty'; - const matches = pending.match(/file-\d+/g); - if (matches && matches.length) return shortFileLabel(matches[matches.length - 1]); - if (/await/.test(pending)) return 'waiting'; - return pending.slice(0, 10); - } - - function catalogRetiredLabel() { - if (/old file removed|physically deleted/.test(model.storage.cleanup)) return 'clean'; - const pending = String(model.catalog.pending || ''); - const matches = pending.match(/file-\d+/g); - if (matches && matches.length > 1) return shortFileLabel(matches[0]); - if ((state.detailFocus === 'gc' || state.detailFocus === 'storageGc') && model.storage.agingQueue.length) return shortFileLabel(model.storage.agingQueue[0]); - return 'empty'; - } - function queryLayout(r, splits) { const x = r.x + 18; const w = r.w - 36; @@ -3461,6 +3336,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const st = currentStep(); const p = stepProgress(); st.tokens.forEach(t => { + if (st.anim === 'appendDeleteTs' && (t.path || []).includes('lane1.deleteArray')) return; if (p < t.start || p > t.end) return; const local = ease((p - t.start) / (t.end - t.start)); const pt = pathPoint(t.path.map(point), local); @@ -3764,12 +3640,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

} const txWindow = parts[0] === 'tx' ? txWindowLayout(r) : null; const txWindowY = txWindow ? txWindow.y + txWindow.h / 2 : r.y + r.h / 2; - const catalogTopY = r.y + MODULE_TITLE_Y + MODULE_CONTENT_GAP; - const catalogTopH = 28; - const catalogShelfY = catalogTopY + catalogTopH + 10; - const catalogShelfH = 44; - const catalogStagingY = catalogShelfY + catalogShelfH + 8; - const catalogStagingH = Math.max(18, r.y + r.h - catalogStagingY - MODULE_CONTENT_PAD); const query = parts[0] === 'query' ? queryLayout(r, model.query ? model.query.splits : []) : null; const pixelsSink = parts[0] === 'ingress' ? pixelsSinkLayout(r) : null; const storageLayout = parts[0] === 'storage' ? storagePanelsLayout(r) : null; @@ -3858,13 +3728,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

files: storageFilesCenter, aging: storageAgingCenter, newFile: storageNewFileCenter, - catalog: [r.x + r.w * .40, r.y + r.h], - storage: [r.x + r.w * .76, r.y + r.h], - schema: [r.x + 60, catalogTopY + catalogTopH / 2], - manifest: [r.x + 112, catalogShelfY + catalogShelfH / 2], - staging: [r.x + 96, catalogStagingY + catalogStagingH / 2], - switch: [r.x + r.w - 74, catalogTopY + catalogTopH / 2], - retired: [r.x + r.w - 54, catalogShelfY + catalogShelfH / 2] + storage: [r.x + r.w * .76, r.y + r.h] }; return arrPoint(custom[pos] || custom.center); } From a7bf70c729f5a0009a755e620a4530b74a4bbc0b Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Wed, 8 Jul 2026 19:31:38 +0800 Subject: [PATCH 04/11] fix: all --- retina.html | 782 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 537 insertions(+), 245 deletions(-) diff --git a/retina.html b/retina.html index 8c2e89d5e..78ed8a0d9 100644 --- a/retina.html +++ b/retina.html @@ -499,7 +499,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

- - + @@ -561,8 +561,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const autoSceneOrder = ['cdc']; const sceneOrder = ['insert', 'delete', 'update', 'gc']; const sceneMeta = { - cdc: { label: 'CDC Mirror Transaction', type: 'pixels-sink flow', summary: 'Pixels Sink reads the ordered CDC log, reconstructs one mirror transaction, begins it in Retina, replays row changes, and commits only after replay OK.' }, - routing: { label: 'Mirror Row Replay', type: 'sink write path', summary: 'Pixels Sink sends row changes from a mirror transaction to Retina replayers; single-operation scenes keep the replay behavior easy to inspect.' }, + cdc: { label: 'Mirror Transaction', type: 'pixels-sink flow', summary: 'Pixels Sink creates one mirror transaction, begins it in Retina, replays row changes, and commits only after replay OK.' }, insert: { label: 'Insert Replay', type: 'write path', summary: 'Insert row changes arrive through a mirror transaction, append to the write buffer, then sealed memtables flush to object entries.' }, delete: { label: 'Delete Replay', type: 'visibility path', summary: 'Delete unfolds Index Service lookup and RGVisibility state: get returns the old RowLocation, then delete timestamp entries update visibility without rewriting columnar files.' }, update: { label: 'Update Replay', type: 'MVCC versioning', summary: 'Update inserts the new tuple version first, updates the primary index, then deletes the old tuple version under one timestamp.' }, @@ -609,7 +608,9 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const DEFAULT_HWM = 290; const DEFAULT_LWM = 281; const DEFAULT_QUERY_TS = DEFAULT_HWM - 1; - const DEFAULT_GC_SAFE_TS = DEFAULT_HWM - 1; + const DEFAULT_GC_SAFE_TS = DEFAULT_LWM - 1; + const GC_BASE_BITMAP = '0000010100000000'; + const GC_FOLD_TARGET_OFFSET = 11; function typeFont(size, weight = 800, family = 'Inter, system-ui, sans-serif') { const prefix = weight ? `${weight} ` : ''; @@ -656,23 +657,16 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const STORAGE_H = MODULE_TITLE_Y + MODULE_CONTENT_GAP + STORAGE_COMPACT_PANEL_H * 2 + STORAGE_PANEL_GAP + STORAGE_EDGE_PAD; const TOP_SERVICE_W = 430; const INDEX_W = 481; - const CDC_STREAM_H = 116; - const CDC_STREAM_ROW_RECORDS = 3; - const CDC_STREAM_RECORD_COUNT = CDC_STREAM_ROW_RECORDS + 2; - const CDC_STREAM_CARD_W = 72; - const CDC_STREAM_RAIL_MARGIN_P = .08; - const CDC_STREAM_SINGLE_OP_RAIL_MARGIN_P = .20; - const CDC_STREAM_RECORD_GAP_UNITS = 1; - const CDC_TO_SINK_GAP = 100; - const INGRESS_Y = MIDDLE_TOP + CDC_STREAM_H + CDC_TO_SINK_GAP; + const INGRESS_Y = MIDDLE_TOP; const PIXELS_SINK_W = 285; const PIXELS_SINK_H = 394; const MIRROR_ROW_CARD_W = 163; const MIRROR_BEGIN_STACK_GAP = 12; const MIRROR_STACK_COMMIT_GAP = 10; + const MIRROR_BEGIN_TS_REPLY_END = .82; + const MIRROR_BEGIN_TS_FADE_END = .92; const PIXELS_SINK_X = LEFT_PANEL_X + (LEFT_PANEL_W - PIXELS_SINK_W) / 2; const rects = { - cdcStream: { x: LEFT_PANEL_X, y: MIDDLE_TOP, w: LEFT_PANEL_W, h: CDC_STREAM_H, title: 'CDC stream', sub: 'row changes + commit marker' }, ingress: { x: PIXELS_SINK_X, y: INGRESS_Y, w: PIXELS_SINK_W, h: PIXELS_SINK_H, title: 'Pixels Sink', sub: 'mirror transaction -> replay RPC' }, sink: { x: PIXELS_SINK_X + 14, y: INGRESS_Y + 292, w: PIXELS_SINK_W - 28, h: 96, title: 'Pixels Sink replay', sub: 'replay path anchors' }, @@ -687,8 +681,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const DETAIL_REPLAY_ID = 1; const info = { - cdcStream: ['CDC stream', 'External ordered row-change stream. It carries row records plus a commit marker for one source transaction.'], - ingress: ['Pixels Sink', 'Consumes the external CDC stream, reconstructs mirror transactions, sends replay RPCs, and commits after replay OK.'], + ingress: ['Pixels Sink', 'Creates mirror transactions, sends replay RPCs, and commits after replay OK.'], sink: ['Pixels Sink replay anchor', 'Invisible anchor region for replay paths.'], runtime: ['Retina replayer', 'RetinaServerImpl handles RetinaWorkerService RPCs and keeps write-buffer and visibility state.'], tx: ['Transaction Service', 'Exposes MVCC time boundaries. Callers get snapshot or safeTs on demand; the panel stays focused on HWM and LWM.'], @@ -730,10 +723,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

sinkStep: null, cdcTx: 1208, changeCount: 72410, - cdcLog: 'TXN or ROW record', - cdcRecord: 'CDC tx1287', - cdcPhase: 'idle', - coordState: 'waiting for CDC log', + coordState: 'mirror tx ready', cachedRows: '0 cached', routingRule: 'by row key', writePlan: 'waiting for row changes', @@ -755,10 +745,13 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

operation: 'idle' }, storage: { - regularFiles: ['file-06.pxl', 'file-07.pxl'], + regularFiles: ['file-0.pxl', 'file-1.pxl'], objectEntries: ['obj-77'], + prevRegularFiles: ['file-0.pxl', 'file-1.pxl'], + prevObjectEntries: ['obj-77'], pendingFlushObject: null, flushedHot: null, + compactionFile: null, staleRatio: 0.31, agingQueue: [], newFile: 'none', @@ -802,7 +795,10 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

regularFile: `file-0${id + 6}.pxl`, baseTs, baseBitmap: '0000000000000000', + prevBaseTs: baseTs, + prevBaseBitmap: '0000000000000000', deleteTs: deletes, + prevDeleteTs: [...deletes], safeTs: null, throughput: 72 + id * 11, highlight: null @@ -818,46 +814,24 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.rowChanges = ['INSERT', 'DELETE', 'UPDATE']; m.sink.mirrorOps = ['insert', 'delete', 'update']; m.sink.sinkStep = 'empty'; - m.sink.cdcPhase = 'begin'; - m.sink.cdcLog = 'ordered CDC log'; - m.sink.cdcRecord = 'BEGIN'; - m.sink.coordState = 'read CDC log'; + m.sink.coordState = 'mirror tx ready'; m.sink.cachedRows = ''; - m.sink.requestState = 'read CDC log'; + m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'await begin'; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-07/rg0/off7']; } - if (scene === 'routing') { - m.sink.mirrorTx = 'tx8503'; - m.sink.rowChanges = ['customer 1', 'orders 2']; - m.sink.mirrorOps = ['insert', 'delete', 'update']; - m.sink.sinkStep = 'replay'; - m.sink.cdcPhase = 'complete'; - m.sink.cdcLog = 'ROW records ready'; - m.sink.cdcRecord = 'ROW txn:tx8503 orders before:{pk:208} after:null'; - m.sink.coordState = 'tx8503 complete'; - m.sink.cachedRows = '0 cached'; - m.sink.routingRule = 'after / before / split'; - m.sink.writePlan = 'rows staged by key image'; - m.sink.rpcBatch = 'batch open'; - m.sink.requestState = 'routing rows'; - m.sink.txState = 'Pixels tx pending'; - } if (scene === 'insert') { m.sink.mirrorTx = 'tx290'; m.sink.rowChanges = ['orders 3']; m.sink.mirrorOps = ['insert']; m.sink.sinkStep = 'empty'; - m.sink.cdcPhase = 'begin'; - m.sink.cdcLog = 'ordered CDC log'; - m.sink.cdcRecord = 'BEGIN'; - m.sink.coordState = 'read CDC log'; + m.sink.coordState = 'mirror tx ready'; m.sink.routingRule = 'after-image key'; m.sink.writePlan = 'pk160, pk174, pk181 queued'; m.sink.rpcBatch = 'tx290 only'; - m.sink.requestState = 'read CDC log'; + m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'tx290 ready'; } if (scene === 'delete') { @@ -865,14 +839,11 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.rowChanges = ['orders 1']; m.sink.mirrorOps = ['delete']; m.sink.sinkStep = 'empty'; - m.sink.cdcPhase = 'begin'; - m.sink.cdcLog = 'ordered CDC log'; - m.sink.cdcRecord = 'BEGIN'; - m.sink.coordState = 'read CDC log'; + m.sink.coordState = 'mirror tx ready'; m.sink.routingRule = 'before-image key'; m.sink.writePlan = 'pk208 queued'; m.sink.rpcBatch = 'tx290 only'; - m.sink.requestState = 'read CDC log'; + m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'tx290 ready'; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk@212 -> rowId820']; @@ -884,14 +855,11 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.rowChanges = ['customer 1']; m.sink.mirrorOps = ['update']; m.sink.sinkStep = 'empty'; - m.sink.cdcPhase = 'begin'; - m.sink.cdcLog = 'ordered CDC log'; - m.sink.cdcRecord = 'BEGIN'; - m.sink.coordState = 'read CDC log'; + m.sink.coordState = 'mirror tx ready'; m.sink.routingRule = 'same primary key'; m.sink.writePlan = 'pk132 queued'; m.sink.rpcBatch = 'tx290 only'; - m.sink.requestState = 'read CDC log'; + m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'tx290 ready'; m.index.hot = 'pk132'; m.index.primaryEntries = ['pk@212 -> rowId820']; @@ -918,53 +886,37 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.query.operatorInput = 'waiting visible batches'; } if (scene === 'gc') { - m.tx.lwm = DEFAULT_LWM; - m.lanes[DETAIL_REPLAY_ID].deleteTs = ['(2,272)', '(4,278)', '(7,283)', '(9,291)']; - m.lanes[DETAIL_REPLAY_ID].baseTs = 268; - m.storage.staleRatio = 0.58; - m.storage.agingQueue = ['file-04.pxl', 'file-05.pxl']; - m.storage.newFile = 'none'; - m.storage.newFileState = 'none'; - m.storage.cleanup = 'await memory phase'; + const l = m.lanes[DETAIL_REPLAY_ID]; + l.baseBitmap = GC_BASE_BITMAP; + l.prevBaseBitmap = GC_BASE_BITMAP; } return m; } const scenes = { cdc: [ - step('The CDC stream starts with BEGIN on the ordered source log.', cdcStreamPhaseUnits('begin'), - ['cdcStream'], [], - m => { m.sink.sinkStep = 'empty'; m.sink.cdcPhase = 'begin'; m.sink.cdcRecord = 'BEGIN'; m.sink.requestState = 'waiting for complete CDC log'; }, 'cdc', ['cdc']), - step('Row changes arrive in source-log order: INSERT, DELETE, then UPDATE.', cdcStreamPhaseUnits('rows'), - ['cdcStream'], [], - m => { m.sink.sinkStep = 'empty'; m.sink.cdcPhase = 'rows'; m.sink.cdcRecord = 'ROWS'; m.sink.requestState = 'CDC rows streaming'; }, 'cdc', ['cdc']), - step('COMMIT closes the CDC stream.', cdcStreamPhaseUnits('commit'), - ['cdcStream'], [], - m => { m.sink.sinkStep = 'empty'; m.sink.cdcPhase = 'commit'; m.sink.cdcRecord = 'COMMIT'; m.sink.requestState = 'complete CDC log'; }, 'cdc', ['cdc']), - step('The CDC log is complete and is handed to Pixels Sink.', 1.2, - ['cdcStream', 'ingress'], [tok('complete log', 'cdcStream.out', 'ingress.top', flow.cdc, .14, .82)], - m => { m.sink.sinkStep = 'empty'; m.sink.cdcPhase = 'complete'; m.sink.cdcRecord = 'COMMIT'; m.sink.requestState = 'complete CDC log'; }, 'cdc', ['cdc']), step('Pixels Sink creates the mirror transaction BEGIN and Transaction Service assigns its timestamp.', 4.0, - ['ingress', 'tx'], [tok('begin tx8421', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], - m => { m.sink.sinkStep = 'begin'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=290'; }, 'mvcc', ['tx']), + ['ingress', 'tx'], [tok('begin tx8421', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, MIRROR_BEGIN_TS_REPLY_END)], + m => { m.sink.sinkStep = 'begin'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=290'; }, 'mvcc', ['tx']), step('Pixels Sink creates the INSERT row in the mirror transaction and sends its replay RPC.', 3.4, ['ingress', 'runtime'], [tok('INSERT', 'ingress.right', 'runtime.left', flow.replay, .18, .86)], - m => { m.sink.sinkStep = 'row1'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay INSERT'; m.sink.rpcBatch = 'INSERT pending'; m.lanes[1].currentOp = 'I:pk174'; }, 'dispatch', ['dispatch']), - step('Retina applies the INSERT to the write buffer: the active memtable grows by one row.', 4.0, + m => { m.sink.sinkStep = 'row1'; m.sink.requestState = 'replay INSERT'; m.sink.rpcBatch = 'INSERT pending'; m.lanes[1].currentOp = 'I:pk174'; }, 'dispatch', ['dispatch']), + step('Retina applies the INSERT to the write buffer: the active memtable grows by one row.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.currentOp = 'I:pk174'; l.status = 'append after image'; l.activeRows = 8; l.highlight = 'active'; m.sink.rpcBatch = 'INSERT applying'; }, 'writeBuffer', []), - step('The active memtable reaches capacity, so Retina seals it into the immutable queue.', 3.9, + step('The active memtable reaches capacity, so Retina seals it into the immutable queue.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.activeRows = 0; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', [], 'seal'), step('The sealed memtable now sits in the immutable queue while replay opens a fresh active memtable.', 1.2, ['lane1'], [], m => { m.lanes[1].highlight = 'immutable'; }, 'writeBuffer', []), - step('A background flush asynchronously moves the immutable memtable to Storage.', 3.8, + step('A background flush asynchronously moves the immutable memtable to Storage.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; }, 'writeBuffer', [], 'flush'), step('The flushed object is now fixed in Storage before Retina moves on to index maintenance.', 1.2, [], [], m => { m.lanes[1].highlight = null; settlePendingFlushObject(m); }, 'storage', []), + compactFlushedObjectsToFile(), step('After the new row is durable, Retina calls Index Service to insert the primary and row-location indexes.', 4.2, ['index'], [tok('put pk@290', 'runtime.right', 'index.left', flow.index, .12, .82)], m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.index.hot = 'pk174'; m.index.operation = 'put pk@290 in flight'; }, 'index', ['index']), @@ -979,7 +931,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { m.sink.sinkStep = 'row1Ok'; m.sink.requestState = 'INSERT OK'; m.sink.rpcBatch = 'OK'; }, 'mvcc', []), step('Pixels Sink sends the DELETE row changes after the mirror transaction has a timestamp.', 3.6, ['ingress'], [tok('DELETE', 'ingress.right', 'runtime.left', flow.delete, .18, .86)], - m => { m.sink.sinkStep = 'row2'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=290'; m.sink.rpcBatch = 'DELETE pending'; m.lanes[1].currentOp = 'D:pk208'; }, 'dispatch', []), + m => { m.sink.sinkStep = 'row2'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=290'; m.sink.rpcBatch = 'DELETE pending'; m.lanes[1].currentOp = 'D:pk208'; }, 'dispatch', []), step('Retina sends a delete request to Index Service for the second index entry.', 4.0, ['index'], [tok('delete pk@290', 'runtime.right', 'index.left', flow.index, .12, .82)], m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'DELETE'; l.status = 'delete index entry'; l.highlight = null; m.index.hot = 'pk208'; m.index.operation = 'delete pk@290 in flight'; m.sink.rpcBatch = 'DELETE applying'; }, 'index', ['index']), @@ -997,8 +949,8 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { m.sink.sinkStep = 'row2Ok'; m.sink.requestState = 'DELETE OK'; m.sink.rpcBatch = 'OK'; clearDeleteTsChainEffects(m, DETAIL_REPLAY_ID); }, 'mvcc', []), step('Pixels Sink sends the UPDATE row changes after the mirror transaction has a timestamp.', 3.6, ['ingress'], [tok('UPDATE', 'ingress.right', 'runtime.left', flow.index, .18, .86)], - m => { m.sink.sinkStep = 'row3'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay UPDATE'; m.sink.txState = 'timestamp=290'; m.sink.rpcBatch = 'UPDATE pending'; m.lanes[1].currentOp = 'U:pk132'; }, 'dispatch', []), - step('Retina inserts the UPDATE after image into the active memtable as the new tuple version.', 4.0, + m => { m.sink.sinkStep = 'row3'; m.sink.requestState = 'replay UPDATE'; m.sink.txState = 'timestamp=290'; m.sink.rpcBatch = 'UPDATE pending'; m.lanes[1].currentOp = 'U:pk132'; }, 'dispatch', []), + step('Retina inserts the UPDATE after image into the active memtable as the new tuple version.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.currentOp = 'U:pk132'; l.status = 'append after image'; l.activeRows = 1; l.highlight = 'active'; m.index.operation = 'await update index'; m.sink.rpcBatch = 'UPDATE applying'; }, 'writeBuffer', []), step('Because the active memtable is not full, Retina sends an update request to Index Service.', 3.6, @@ -1026,50 +978,30 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

[], [], m => { m.sink.sinkStep = 'committed'; m.sink.requestState = 'HWM advanced'; m.sink.txState = 'tx8421 committed; HWM=291'; m.tx.hwm = 291; }, 'mvcc', []) ], - routing: [ - step('Pixels Sink has reconstructed tx43 from the CDC stream and starts replaying its row changes.', 3.8, - ['cdcStream', 'ingress'], [tok('tx43 rows', 'cdcStream.out', 'ingress.top', flow.cdc, .08, .78)], - m => { m.sink.sinkStep = 'replay'; m.sink.coordState = 'tx43 mirror tx'; m.sink.requestState = 'replay rows'; m.sink.routingRule = 'inspect row image'; }, 'cdc', ['cdc']), - step('Insert rows use the after-image primary key, so the new tuple can be staged for replay.', 4.0, - ['ingress', 'lane0'], [tok('I:pk160', 'ingress.right', 'runtime.left', flow.replay, .12, .82)], - m => { m.sink.routingRule = 'insert: after key'; m.sink.writePlan = 'pk160 queued'; m.lanes[0].currentOp = 'I:pk160'; m.lanes[0].status = 'after key staged'; }, 'dispatch', ['dispatch']), - step('Delete rows use the before-image primary key because the current tuple location must be resolved from the old key.', 4.0, - ['ingress', 'lane1', 'index'], [tok('D:pk208', 'ingress.right', 'runtime.left', flow.delete, .08, .58), tok('old key lookup', 'lane1.in', 'index.in', flow.index, .52, .92)], - m => { m.sink.routingRule = 'delete: before key'; m.sink.writePlan = 'pk208 queued'; m.lanes[1].currentOp = 'D:pk208'; m.index.hot = 'pk208'; m.index.operation = 'resolve before key'; }, 'dispatch', ['dispatch', 'index']), - step('If an update changes its primary key, Pixels Sink preserves the delete-old then insert-new order before replay.', 4.4, - ['ingress', 'lane1', 'lane2'], [tok('delete old pk132', 'ingress.right', 'runtime.left', flow.delete, .08, .58), tok('insert new pk733', 'ingress.right', 'runtime.left', flow.replay, .40, .92)], - m => { m.sink.routingRule = 'pk update: split'; m.sink.writePlan = 'delete old before insert new'; m.sink.rpcBatch = 'tx43 rows staged'; m.lanes[1].currentOp = 'D:pk132'; m.lanes[2].currentOp = 'I:pk733'; }, 'dispatch', ['dispatch']), - step('Pixels Sink begins mirror transactions with Transaction Service, then sends replay RPCs to Retina replayers.', 4.6, - ['ingress', 'tx', 'runtime'], [tok('tx43 ts291', 'tx.left', 'ingress.beginRight', flow.tx, .04, .34), tok('tx44 ts292', 'tx.left', 'ingress.beginRight', flow.tx, .16, .46), tok('tx45 ts293', 'tx.left', 'ingress.beginRight', flow.tx, .28, .58), tok('replay RPCs', 'ingress.right', 'runtime.left', flow.replay, .58, .96)], - m => { m.sink.sinkStep = 'replay'; m.sink.requestMode = 'replay RPC'; m.sink.requestState = '3 mirror tx'; m.sink.rpcBatch = 'tx43 + tx44 + tx45'; m.sink.txState = 'timestamps assigned'; }, 'mvcc', ['tx']), - step('Inside Retina, the same RPC applies rows from different transactions while preserving their visibility timestamps.', 4.2, - ['ingress', 'lane0', 'lane1', 'lane2'], [tok('tx43 row', 'ingress.right', 'runtime.left', flow.replay, .08, .58), tok('tx44 row', 'ingress.right', 'runtime.left', flow.delete, .22, .74), tok('tx45 row', 'ingress.right', 'runtime.left', flow.replay, .36, .90)], - m => { m.sink.requestState = 'apply one RPC'; m.sink.rpcBatch = '3 tx rows'; m.lanes[0].currentOp = 'tx43 I:pk160'; m.lanes[1].currentOp = 'tx44 D:pk208'; m.lanes[2].currentOp = 'tx45 I:pk733'; }, 'dispatch', ['dispatch']) - ], insert: [ - ...singleOpCdcPrelude('insert', 'tx290'), step('Pixels Sink begins mirror transaction tx290 and Transaction Service assigns timestamp=290.', 4.0, - ['ingress', 'tx'], [tok('begin tx290', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], + ['ingress', 'tx'], [tok('begin tx290', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, MIRROR_BEGIN_TS_REPLY_END)], m => { m.sink.sinkStep = 'begin'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=290'; }, 'mvcc', ['tx']), step('Pixels Sink sends the INSERT row changes after the mirror transaction has a timestamp.', 3.6, ['ingress'], [tok('INSERT', 'ingress.right', 'runtime.left', flow.replay, .18, .86)], m => { m.sink.sinkStep = 'replay'; m.sink.requestState = 'replay RPC ready'; m.sink.txState = 'timestamp=290'; }, 'dispatch', []), - step('Retina applies the INSERT to the write buffer first: the active memtable grows by one row.', 4.0, + step('Retina applies the INSERT to the write buffer first: the active memtable grows by one row.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.currentOp = 'I:pk174'; l.status = 'append after image'; l.activeRows = 8; l.highlight = 'active'; m.sink.rpcBatch = 'INSERT applying'; }, 'writeBuffer', []), - step('The active memtable reaches capacity, so Retina seals it into the immutable queue and opens a fresh empty active memtable.', 3.9, + step('The active memtable reaches capacity, so Retina seals it into the immutable queue and opens a fresh empty active memtable.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.activeRows = 0; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', [], 'seal'), step('The sealed memtable now sits in the immutable queue while the active memtable is empty again.', 1.2, ['lane1'], [], m => { const l = m.lanes[1]; l.highlight = 'immutable'; }, 'writeBuffer', []), - step('A background flush asynchronously moves the immutable memtable to Storage.', 3.8, + step('A background flush asynchronously moves the immutable memtable to Storage.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; }, 'writeBuffer', [], 'flush'), step('The flushed object is now fixed in Storage before Retina moves on to index maintenance.', 1.2, [], [], m => { m.lanes[1].highlight = null; settlePendingFlushObject(m); }, 'storage', []), + compactFlushedObjectsToFile(), step('After the row is appended, Retina calls Index Service to insert the primary and row-location indexes.', 4.2, ['index'], [tok('put pk@290', 'runtime.right', 'index.left', flow.index, .12, .82)], m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.index.hot = 'pk174'; m.index.operation = 'put pk@290 in flight'; }, 'index', ['index']), @@ -1090,13 +1022,12 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { m.sink.sinkStep = 'committed'; m.sink.requestState = 'HWM advanced'; m.sink.txState = 'tx290 committed; HWM=291'; m.tx.hwm = 291; }, 'mvcc', []) ], delete: [ - ...singleOpCdcPrelude('delete', 'tx290'), step('Pixels Sink begins mirror transaction tx290 and Transaction Service assigns timestamp=290.', 4.0, - ['ingress', 'tx'], [tok('begin tx290', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], - m => { m.sink.sinkStep = 'begin'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=290'; }, 'mvcc', ['tx']), + ['ingress', 'tx'], [tok('begin tx290', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, MIRROR_BEGIN_TS_REPLY_END)], + m => { m.sink.sinkStep = 'begin'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=290'; }, 'mvcc', ['tx']), step('Pixels Sink sends the DELETE row changes after the mirror transaction has a timestamp.', 3.6, ['ingress'], [tok('DELETE', 'ingress.right', 'runtime.left', flow.delete, .18, .86)], - m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=290'; }, 'dispatch', []), + m => { m.sink.sinkStep = 'replay'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=290'; }, 'dispatch', []), step('Retina sends a delete request to Index Service for pk208 before deleting visibility.', 4.0, ['index'], [tok('delete pk@212', 'runtime.right', 'index.left', flow.index, .12, .82)], m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'DELETE'; l.status = 'delete index entry'; l.highlight = null; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-07/rg0/off7']; m.index.operation = 'delete pk@212 in flight'; m.sink.rpcBatch = 'DELETE applying'; }, 'index', ['index']), @@ -1112,23 +1043,22 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

...commitSequence('tx290', 291, 'delete committed; HWM=291', DETAIL_REPLAY_ID) ], update: [ - ...singleOpCdcPrelude('update', 'tx290'), step('Pixels Sink begins mirror transaction tx290 and Transaction Service assigns timestamp=290.', 4.0, - ['ingress', 'tx'], [tok('begin tx290', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, .92)], - m => { m.sink.sinkStep = 'begin'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=290'; }, 'mvcc', ['tx']), + ['ingress', 'tx'], [tok('begin tx290', 'ingress.beginRight', 'tx.left', flow.tx, .12, .52), tok('ts=290', 'tx.left', 'ingress.beginRight', flow.tx, .54, MIRROR_BEGIN_TS_REPLY_END)], + m => { m.sink.sinkStep = 'begin'; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'timestamp=290'; }, 'mvcc', ['tx']), step('Pixels Sink sends the UPDATE row changes after the mirror transaction has a timestamp.', 3.6, ['ingress'], [tok('UPDATE', 'ingress.right', 'runtime.left', flow.index, .18, .86)], - m => { m.sink.sinkStep = 'replay'; m.sink.cdcPhase = 'complete'; m.sink.requestState = 'replay UPDATE'; m.sink.txState = 'timestamp=290'; }, 'dispatch', []), - step('Retina first inserts the after image into the write buffer as the new tuple version.', 4.0, + m => { m.sink.sinkStep = 'replay'; m.sink.requestState = 'replay UPDATE'; m.sink.txState = 'timestamp=290'; }, 'dispatch', []), + step('Retina first inserts the after image into the write buffer as the new tuple version.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.currentOp = 'U:pk132'; l.status = 'append after image'; l.activeRows = 8; l.highlight = 'active'; m.index.operation = 'await update index'; m.sink.rpcBatch = 'UPDATE applying'; }, 'writeBuffer', []), - step('The active memtable reaches capacity, so Retina seals the new tuple version into the immutable queue.', 3.9, + step('The active memtable reaches capacity, so Retina seals the new tuple version into the immutable queue.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.activeRows = 0; l.immutable += 1; l.highlight = 'immutable'; }, 'writeBuffer', [], 'seal'), step('The sealed memtable now sits in the immutable queue while a fresh active memtable is open.', 1.2, ['lane1'], [], m => { const l = m.lanes[1]; l.highlight = 'immutable'; }, 'writeBuffer', []), - step('A background flush asynchronously moves the immutable memtable containing the new tuple version to Storage.', 3.8, + step('A background flush asynchronously moves the immutable memtable containing the new tuple version to Storage.', 2.0, ['lane1'], [], m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; }, 'writeBuffer', [], 'flush'), step('The flushed object is now fixed in Storage before Retina moves on to index maintenance.', 1.2, @@ -1169,33 +1099,24 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { m.query.gate.state = 'done'; m.query.queueState = '4 visible batches'; m.query.operatorInput = 'visible batches only'; m.query.returnedRows = 18804; }, 'query', ['query']) ], gc: [ - step(`RetinaResourceManager.runGC asks Transaction Service for safeTs=${DEFAULT_GC_SAFE_TS} (HWM - 1, no running query) and sends it to RGVisibility.garbageCollect(safeTs).`, 3.8, - ['tx', 'lane1', 'runtime'], [tok('Memory GC asks', 'runtime.top', 'tx.bottom', flow.gc, .08, .42), tok(`safeTs=${DEFAULT_GC_SAFE_TS}`, 'tx.hwm', 'lane1.visibility', flow.gc, .42, .92)], - m => { m.lanes[DETAIL_REPLAY_ID].safeTs = DEFAULT_GC_SAFE_TS; m.lanes[DETAIL_REPLAY_ID].highlight = 'visibility'; m.storage.cleanup = 'waiting for bitmap folding'; }, 'gc', ['visibility']), - step('Visibility structure compares safeTs with the delete ts chain and highlights items that are safe to fold.', 4.0, - ['lane1', 'runtime'], [tok('safe items', 'lane1.visibility', 'lane1.deleteArray', flow.visibility, .08, .70), tok('<= safeTs', 'lane1.deleteArray', 'lane1.baseBitmap', flow.gc, .55, .94)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.safeTs = DEFAULT_GC_SAFE_TS; l.deleteTs = ['(2,272)', '(4,278)', '(7,283)', '(9,291)']; l.highlight = 'foldEligible'; }, 'gc', ['visibility']), - step('garbageCollect(safeTs) folds the highlighted delete items into baseBitmap, advances baseTs, and leaves newer delete items in the chain.', 4.0, - ['lane1', 'runtime'], [tok('fold delete items', 'lane1.deleteArray', 'lane1.baseBitmap', flow.gc, .08, .78)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.safeTs = DEFAULT_GC_SAFE_TS; l.deleteTs = ['(9,291)']; l.baseBitmap = '0010100000000000'; l.baseTs = DEFAULT_GC_SAFE_TS; l.highlight = 'baseBitmap'; }, 'gc', ['visibility']), - step('After bitmap folding, foreground replay can continue while Retina prepares Storage GC file groups; no running query is holding the GC safe timestamp back.', 4.0, - ['lane1', 'lane2', 'runtime'], [tok('folded bitmap', 'lane1.baseBitmap', 'storage.aging', flow.gc, .05, .50), tok('replay continues', 'ingress.right', 'runtime.left', flow.replay, .34, .82)], - m => { m.lanes[DETAIL_REPLAY_ID].safeTs = DEFAULT_GC_SAFE_TS; m.lanes[2].status = 'replay continues'; m.query.active = false; }, 'gc', ['dispatch', 'query']), - step('Storage GC selects low-valid-ratio regular files. They remain regular columnar files, just highlighted as rewrite candidates.', 4.0, - ['runtime', 'storage'], [tok('candidate files', 'runtime.right', 'storage.aging', flow.gc, .08, .76)], - m => { m.storage.agingQueue = ['file-04.pxl', 'file-05.pxl']; m.storage.cleanup = 'rewrite candidates'; }, 'storageGc', ['storage']), - step('Storage GC rewrites still-visible tuple versions into a new TEMPORARY_GC file inside the regular-files panel.', 4.5, - ['storage', 'tx'], [tok(`safeTs=${DEFAULT_GC_SAFE_TS}`, 'tx.hwm', 'storage.files', flow.tx, .08, .48), tok('visible tuples', 'storage.aging', 'storage.newFile', flow.storage, .42, .96)], - m => { m.storage.newFile = 'file-08.pxl'; m.storage.newFileState = 'TEMPORARY_GC'; m.storage.cleanup = 'new TEMPORARY_GC'; }, 'storageGc', ['tx']), - step('Before publishing it, Retina registers dual-write, synchronizes visibility, and refreshes main and primary index locations.', 4.6, - ['storage', 'index', 'lane2'], [tok('location map', 'storage.newFile', 'index.in', flow.index, .06, .56), tok('sync visibility', 'storage.newFile', 'lane2.in', flow.visibility, .48, .94)], - m => { m.index.operation = 'storage GC location map'; m.index.mainEntries = ['rowId822 -> file-08/rg0/off2', 'rowId823 -> file-08/rg0/off3']; m.lanes[2].deleteTs.push('(dual,293)'); m.lanes[2].highlight = 'visibility'; }, 'index', ['index', 'visibility']), - step('Retina promotes the new file to REGULAR and marks old regular files RETIRED with cleanupAt.', 4.2, - ['storage'], [tok('TEMP -> REGULAR', 'storage.newFile', 'storage.files', flow.gc, .08, .46), tok('old -> RETIRED', 'storage.aging', 'storage.files', flow.delete, .44, .90)], - m => { m.storage.newFileState = 'REGULAR'; m.storage.regularFiles = ['file-06.pxl', 'file-07.pxl', 'file-08.pxl']; m.storage.cleanup = 'old files RETIRED cleanupAt'; }, 'storageGc', ['storage']), - step('After retireDelayMs / cleanupAt, processRetiredFiles reclaims visibility and index state, then physically deletes retired files.', 4.0, - ['storage', 'lane2'], [tok('cleanupAt reached', 'storage.files', 'storage.aging', flow.gc, .08, .72), tok('reclaim state', 'lane2.out', 'runtime.right', flow.gc, .32, .92)], - m => { m.storage.staleRatio = 0.18; m.storage.agingQueue = []; m.storage.newFile = 'none'; m.storage.newFileState = 'none'; m.storage.regularFiles = ['file-06.pxl', 'file-07.pxl', 'file-08.pxl']; m.storage.cleanup = 'retired files physically deleted'; m.lanes[2].safeTs = DEFAULT_GC_SAFE_TS; m.lanes[2].highlight = 'baseBitmap'; }, 'gc', ['visibility']) + step(`Transaction Service sends safeTs=${DEFAULT_GC_SAFE_TS} to Retina.`, 3.0, + ['tx'], [tok(`safeTs=${DEFAULT_GC_SAFE_TS}`, 'tx.bottom', 'runtime.top', flow.tx, .16, .86)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.safeTs = m.tx.lwm - 1; l.highlight = null; m.storage.cleanup = 'safeTs received'; }, 'mvcc', ['tx']), + step('Retina checks the delete ts chain and highlights entries with ts <= safeTs.', 3.2, + [], [], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.safeTs = m.tx.lwm - 1; l.highlight = 'foldEligible'; }, 'gc', ['visibility']), + step('Retina folds eligible delete timestamps into baseBitmap, advances baseTs to safeTs, and removes those entries from the chain.', 4.0, + [], [], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; foldDeleteTsIntoBaseBitmap(l, m.tx.lwm - 1); }, 'gc', ['visibility'], 'foldDeleteTs'), + step('Retina finishes the fold and clears the focus before Storage starts its file-validity check.', 1.2, + [], [], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.highlight = null; m.storage.cleanup = 'safeTs folded'; }, 'overview', []), + step('Storage checks regular files and highlights files whose valid ratio is at or below the compaction threshold.', 4.0, + [], [], + m => { m.lanes[DETAIL_REPLAY_ID].highlight = null; m.storage.cleanup = 'valid ratio check'; }, 'storageGc', ['storage'], 'validRatioCheck'), + step('Storage merges file-0 and file-1 into file-2, then removes the old files.', 4.0, + [], [], + m => { m.lanes[DETAIL_REPLAY_ID].highlight = null; m.storage.compactionFile = 'file-2.pxl'; m.storage.regularFiles = ['file-2.pxl']; m.storage.cleanup = 'compact files'; }, 'storageGc', ['storage'], 'compactRegularFiles') ] }; @@ -1224,23 +1145,17 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.storage.flushedHot = null; } - function singleOpCdcPrelude(op, txId) { - const spec = normalizeMirrorOp(op, 0); - const recordCount = cdcStreamRecordCount(1); - return [ - step(`The CDC stream starts ${txId} with BEGIN on the ordered source log.`, cdcStreamPhaseUnits('begin', recordCount), - ['cdcStream'], [], - m => { m.sink.sinkStep = 'empty'; m.sink.cdcPhase = 'begin'; m.sink.cdcRecord = 'BEGIN'; m.sink.requestState = 'waiting for CDC row'; }, 'cdc', ['cdc']), - step(`The ${spec.label} row change arrives on the CDC stream.`, cdcStreamPhaseUnits('rows', recordCount), - ['cdcStream'], [], - m => { m.sink.sinkStep = 'empty'; m.sink.cdcPhase = 'rows'; m.sink.cdcRecord = spec.label; m.sink.requestState = `${spec.label} row streaming`; }, 'cdc', ['cdc']), - step(`COMMIT closes the ${txId} CDC stream.`, cdcStreamPhaseUnits('commit', recordCount), - ['cdcStream'], [], - m => { m.sink.sinkStep = 'empty'; m.sink.cdcPhase = 'commit'; m.sink.cdcRecord = 'COMMIT'; m.sink.requestState = 'complete CDC log'; }, 'cdc', ['cdc']), - step(`The complete ${txId} CDC log is handed to Pixels Sink.`, 1.2, - ['cdcStream', 'ingress'], [tok('complete log', 'cdcStream.out', 'ingress.top', flow.cdc, .14, .82)], - m => { m.sink.sinkStep = 'build'; m.sink.cdcPhase = 'complete'; m.sink.cdcRecord = 'COMMIT'; m.sink.coordState = `${txId} complete`; m.sink.requestState = 'complete CDC log'; }, 'cdc', ['cdc']) - ]; + function compactFlushedObjectsToFile(fileLabel = 'file-2.pxl') { + return step(`Storage compacts the flushed objects into ${fileLabel}, moves it into files, then removes the old objects.`, 4.0, + ['storage'], [], + m => { + m.storage.compactionFile = fileLabel; + m.storage.objectEntries = []; + if (!m.storage.regularFiles.includes(fileLabel)) m.storage.regularFiles.push(fileLabel); + m.storage.pendingFlushObject = null; + m.storage.flushedHot = null; + m.storage.cleanup = 'compact objects to file'; + }, 'storage', ['storage'], 'compactObjectsToFile'); } function commitSequence(txId, hwm, committedStatus, clearDeleteTsLaneId = null) { @@ -1360,7 +1275,17 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const st = currentStep(); state.detailFocus = st.detail || 'overview'; state.activeChannels = st.channels || []; - model.lanes.forEach(l => { l.prevActiveRows = l.activeRows; l.prevImmutable = l.immutable; }); + model.lanes.forEach(l => { + l.prevActiveRows = l.activeRows; + l.prevImmutable = l.immutable; + l.prevDeleteTs = [...l.deleteTs]; + l.prevBaseTs = l.baseTs; + l.prevBaseBitmap = l.baseBitmap; + }); + if (model.storage) { + model.storage.prevRegularFiles = [...model.storage.regularFiles]; + model.storage.prevObjectEntries = [...model.storage.objectEntries]; + } if (st.apply) st.apply(model); updateControls(); } @@ -1494,7 +1419,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function hitTest(x, y) { const zones = [ - ['cdcStream', rects.cdcStream], ['ingress', rects.ingress], ['runtime', rects.runtime], + ['ingress', rects.ingress], ['runtime', rects.runtime], ['tx', rects.tx], ['index', rects.index], ['storage', rects.storage] ]; @@ -1526,7 +1451,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

drawCanvasHeader(c); drawBoundaries(c); drawStaticFlows(c); - drawExternalCdcStream(c); drawPixelsSink(c); drawTransactionService(c); drawIndex(c); @@ -1618,7 +1542,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

} function moduleActivityColor(key) { - if (key === 'cdcStream' || key === 'ingress') return flow.cdc; + if (key === 'ingress') return flow.cdc; if (key === 'query') return flow.query; if (key === 'runtime') return currentReplayOperationColor(); if (key === 'index') return currentIndexOperationColor(); @@ -1744,115 +1668,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

drawMirrorTransactionPanel(layout, activeId, c); } - function drawExternalCdcStream(c) { - drawCdcStreamPanel(rects.cdcStream, currentPixelsSinkStep(), c); - } - - function cdcStreamRecords() { - const rows = [{ label: 'BEGIN', color: flow.cdc, begin: true }]; - rows.push(...mirrorTxnOps().map(op => ({ label: op.label, color: op.color, row: true }))); - rows.push({ label: 'COMMIT', color: flow.cdc, commit: true }); - return rows; - } - - function cdcStreamRecordCount(rowRecords = CDC_STREAM_ROW_RECORDS) { - return rowRecords + 2; - } - - function cdcStreamProgressState(records) { - const phase = model && model.sink ? model.sink.cdcPhase : 'idle'; - const local = stepProgress(); - const maxPos = records.length; - const rowCount = Math.max(1, records.length - 2); - let cursorPos = 0; - if (phase === 'begin') cursorPos = local; - else if (phase === 'rows') cursorPos = 1 + local * rowCount; - else if (phase === 'commit') cursorPos = maxPos - 1 + local; - else if (phase === 'complete') cursorPos = maxPos; - cursorPos = clamp(cursorPos, 0, maxPos); - return { - phase, - cursorPos, - reachedIndex: Math.floor(cursorPos + 1e-6) - 1, - cursorP: cdcStreamRailP(cursorPos, maxPos), - done: phase === 'complete' - }; - } - - function cdcStreamPhaseUnits(phase, recordCount = CDC_STREAM_RECORD_COUNT) { - const maxPos = Math.max(2, recordCount); - const rowCount = Math.max(1, maxPos - 2); - const ranges = { - begin: [0, 1], - rows: [1, 1 + rowCount], - commit: [maxPos - 1, maxPos] - }; - const range = ranges[phase]; - if (!range) return CDC_STREAM_RECORD_GAP_UNITS; - const referenceGapP = Math.abs(cdcStreamRailP(2, CDC_STREAM_RECORD_COUNT) - cdcStreamRailP(1, CDC_STREAM_RECORD_COUNT)); - const phaseP = Math.abs(cdcStreamRailP(range[1], maxPos) - cdcStreamRailP(range[0], maxPos)); - return Math.max(.1, phaseP / Math.max(.001, referenceGapP) * CDC_STREAM_RECORD_GAP_UNITS); - } - - function cdcStreamRailMargin(maxPos) { - return maxPos === cdcStreamRecordCount(1) ? CDC_STREAM_SINGLE_OP_RAIL_MARGIN_P : CDC_STREAM_RAIL_MARGIN_P; - } - - function cdcStreamRailP(cursorPos, maxPos) { - const margin = cdcStreamRailMargin(maxPos); - if (maxPos <= 1) return .5; - if (cursorPos <= 1) return lerp(0, margin, clamp(cursorPos, 0, 1)); - return margin + clamp((cursorPos - 1) / (maxPos - 1), 0, 1) * (1 - margin * 2); - } - - function drawCdcStreamPanel(stream, activeId, c) { - const phase = model && model.sink ? model.sink.cdcPhase : 'idle'; - const active = isHighlighted('cdcStream') || ['begin', 'rows', 'commit'].includes(phase); - ctx.fillStyle = active ? alpha(flow.cdc, c.dark ? .16 : .10) : alpha(c.elevated, .94); - ctx.strokeStyle = active ? alpha(flow.cdc, .62) : c.borderSoft; - ctx.lineWidth = active ? 1.4 : 1; - roundRect(stream.x, stream.y, stream.w, stream.h, 10, true, true); - - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.moduleTitle, 900); - ctx.textAlign = 'left'; - fitText('CDC stream', stream.x + MODULE_TITLE_X, stream.y + MODULE_TITLE_Y, stream.w - MODULE_TITLE_X * 2, 'left'); - - const railY = stream.y + 58; - const startX = stream.x + MODULE_CONTENT_PAD; - const endX = stream.x + stream.w - MODULE_CONTENT_PAD; - ctx.strokeStyle = active ? alpha(flow.cdc, .62) : alpha(c.border, .46); - ctx.lineWidth = 2.5; - ctx.beginPath(); - ctx.moveTo(startX, railY); - ctx.lineTo(endX, railY); - ctx.stroke(); - - const records = cdcStreamRecords(); - const progress = cdcStreamProgressState(records); - records.forEach((record, i) => { - if (i > progress.reachedIndex) return; - const x = map(cdcStreamRailP(i + 1, records.length), 0, 1, startX, endX); - const cardW = CDC_STREAM_CARD_W; - const cardTextSize = TYPE.text; - drawCdcDot(x, railY, true, record.color, c); - drawRecordTypeCard(x - cardW / 2, railY + 14, cardW, 24, record.label, record.color, c, cardTextSize); - }); - - if (progress.done) return; - const p = progress.cursorP; - const cursorX = map(p, 0, 1, startX, endX); - ctx.fillStyle = flow.cdc; - ctx.beginPath(); - ctx.arc(cursorX, railY, 6, 0, Math.PI * 2); - ctx.fill(); - ctx.strokeStyle = alpha('#ffffff', .74); - ctx.lineWidth = 1.2; - ctx.beginPath(); - ctx.arc(cursorX, railY, 11, 0, Math.PI * 2); - ctx.stroke(); - } - function drawMirrorTransactionPanel(layout, activeId, c) { const { txn, begin, rows, replay, commit } = layout; const active = !!activeId && activeId !== 'empty' && activeId !== 'committed'; @@ -1906,11 +1721,15 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.textAlign = 'left'; ctx.fillText('BEGIN', box.x + 18, box.y + 29); const tsLabel = mirrorBeginTimestampLabel(); - if (tsLabel) { + const tsAlpha = mirrorBeginTimestampAlpha(); + if (tsLabel && tsAlpha > 0) { + ctx.save(); + ctx.globalAlpha *= tsAlpha; ctx.fillStyle = c.text; ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'right'; fitText(tsLabel, box.x + box.w - 82, box.y + 31, 64, 'right'); + ctx.restore(); } } @@ -1918,10 +1737,16 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const text = String(model.sink.txState || ''); const match = text.match(/\b(?:timestamp|ts)\s*=\s*(\d{1,4})\b/i); if (!match) return ''; - if (currentPixelsSinkStep() === 'begin' && stepProgress() < .54) return ''; return `ts=${match[1]}`; } + function mirrorBeginTimestampAlpha() { + if (currentPixelsSinkStep() !== 'begin') return 1; + const p = stepProgress(); + if (p < MIRROR_BEGIN_TS_REPLY_END) return 0; + return ease(clamp((p - MIRROR_BEGIN_TS_REPLY_END) / (MIRROR_BEGIN_TS_FADE_END - MIRROR_BEGIN_TS_REPLY_END), 0, 1)); + } + function drawMirrorRowCard(row, active, c) { ctx.save(); ctx.shadowColor = alpha(row.color, active ? .22 : .12); @@ -2030,13 +1855,13 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

roundRect(x, y, w, h, 8, true, true); } - function drawRecordTypeCard(x, y, w, h, label, color, c, fs = TYPE.text) { + function drawRecordTypeCard(x, y, w, h, label, color, c, fs = TYPE.text, weight = 900) { ctx.fillStyle = alpha(color, .16); ctx.strokeStyle = alpha(color, .48); ctx.lineWidth = 1; roundRect(x, y, w, h, 6, true, true); ctx.fillStyle = color; - ctx.font = typeFont(fs, 900); + ctx.font = typeFont(fs, weight); ctx.textAlign = 'center'; fitText(label, x + 4, y + h / 2 + fs * .36, w - 8, 'center'); } @@ -3023,10 +2848,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const layout = replayVisibilityLayout(x, y, w, h); const { rowX, rowY, labelW, bitmapX, bitmapW, chainY, chainH } = layout; - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'left'; - fitText(`baseTs=${lane.baseTs}`, rowX, rowY + 14, labelW - 4, 'left'); + drawBaseTsLabel(lane, rowX, rowY + 14, labelW - 4, c); drawFixedDeleteBitmap(lane, bitmapX, rowY, bitmapW, c); const chainX = x + MODULE_CONTENT_PAD; @@ -3041,19 +2863,50 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'left'; ctx.fillText('delete ts chain', chainX + chainPad, chainY + 22); - drawCompactDeleteTsItems(chainX + itemPad, chainY + 34, chainW - itemPad * 2, lane, c); + const itemsY = chainY + 34; + drawCompactDeleteTsItems(chainX + itemPad, itemsY, chainW - itemPad * 2, lane, c); + drawFoldingDeleteTsItems(chainX + itemPad, itemsY, chainW - itemPad * 2, bitmapX, rowY, bitmapW, lane, c); + drawDeleteTsConditionPill(chainX + itemPad, itemsY + 32, chainW - itemPad * 2, lane, c); + } + + function drawBaseTsLabel(lane, x, y, w, c) { + const foldActive = currentStep().anim === 'foldDeleteTs'; + const p = stepProgress(); + const showUpdatedBaseTs = !foldActive || p >= .72; + const labelTs = showUpdatedBaseTs ? lane.baseTs : lane.prevBaseTs; + ctx.save(); + ctx.fillStyle = foldActive && showUpdatedBaseTs ? flow.gc : c.text; + ctx.font = typeFont(TYPE.text, foldActive ? 900 : 900); + ctx.textAlign = 'left'; + if (foldActive && showUpdatedBaseTs) ctx.globalAlpha *= lerp(.72, 1, ease(clamp((p - .72) / .18, 0, 1))); + fitText(`baseTs=${labelTs}`, x, y, w, 'left'); + ctx.restore(); + } + + function drawDeleteTsConditionPill(x, y, w, lane, c) { + if (lane.highlight !== 'foldEligible') return; + const p = stepProgress(); + if (p < .12) return; + const a = ease(clamp((p - .12) / .28, 0, 1)); + const out = 1 - ease(clamp((p - .62) / .10, 0, 1)); + if (out <= 0) return; + const pillW = Math.min(118, Math.max(86, w * .44)); + const pillH = 24; + ctx.save(); + ctx.globalAlpha *= a * out; + drawPill(x + (w - pillW) / 2, y, pillW, pillH, 'ts <= safeTs', flow.visibility, c, true, 8); + ctx.restore(); } function drawFixedDeleteBitmap(lane, x, y, w, c) { const len = 16; - const bits = (lane.baseBitmap || '').padEnd(len, '0').slice(0, len); - const gap = 3; - const cellW = Math.max(7, Math.min(12, Math.floor((w - gap * (len - 1)) / len))); - const cellH = 18; - const totalW = len * cellW + (len - 1) * gap; - const startX = x + Math.max(0, (w - totalW) / 2); + const foldActive = currentStep().anim === 'foldDeleteTs'; + const foldP = stepProgress(); + const showFoldedBits = !foldActive || foldP >= .72; + const bits = ((showFoldedBits ? lane.baseBitmap : lane.prevBaseBitmap) || '').padEnd(len, '0').slice(0, len); const pendingDeletes = new Set(); const eligibleDeletes = new Set(); + const newlyFolded = new Set(); lane.deleteTs.forEach(item => { const offset = deleteOffsetValue(item); @@ -3063,18 +2916,28 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

eligibleDeletes.add(offset); } }); + if (foldActive) { + (lane.prevDeleteTs || []).forEach(item => { + const offset = foldBitmapOffsetValue(item, lane); + if (Number.isFinite(offset) && offset >= 0 && offset < len && lane.safeTs != null && deleteTsValue(item) <= lane.safeTs) { + newlyFolded.add(offset); + } + }); + } for (let i = 0; i < len; i++) { const folded = bits[i] === '1'; const eligible = eligibleDeletes.has(i); const pending = pendingDeletes.has(i); + const justFolded = newlyFolded.has(i); let fill = alpha(c.border, .22); let stroke = alpha(c.border, .56); if (folded) { - const color = lane.highlight === 'baseBitmap' ? flow.gc : flow.visibility; - fill = alpha(color, .62); - stroke = alpha(color, .78); + const color = justFolded ? flow.gc : flow.delete; + const foldedAlpha = justFolded ? lerp(.28, .72, ease(clamp((foldP - .72) / .18, 0, 1))) : .62; + fill = justFolded ? alpha(color, foldedAlpha) : alpha(flow.delete, .46); + stroke = justFolded ? alpha(color, .88) : alpha(flow.delete, .70); } else if (eligible) { fill = alpha(flow.gc, .50); stroke = alpha(flow.gc, .76); @@ -3085,18 +2948,34 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.fillStyle = fill; ctx.strokeStyle = stroke; - ctx.lineWidth = folded || pending ? 1.2 : 1; - roundRect(startX + i * (cellW + gap), y, cellW, cellH, 2.5, true, true); + ctx.lineWidth = folded || pending ? (justFolded ? 1.8 : 1.2) : 1; + const cell = bitmapCellRect(x, y, w, i); + roundRect(cell.x, cell.y, cell.w, cell.h, 2.5, true, true); } } + function bitmapCellMetrics(x, w, len = 16) { + const gap = 3; + const cellW = Math.max(7, Math.min(12, Math.floor((w - gap * (len - 1)) / len))); + const cellH = 18; + const totalW = len * cellW + (len - 1) * gap; + const startX = x + Math.max(0, (w - totalW) / 2); + return { gap, cellW, cellH, startX }; + } + + function bitmapCellRect(x, y, w, index) { + const m = bitmapCellMetrics(x, w); + return { x: m.startX + index * (m.cellW + m.gap), y, w: m.cellW, h: m.cellH }; + } + function drawCompactDeleteTsItems(x, y, w, lane, c) { const columns = 4; const rows = 2; const st = currentStep(); const hideAppendingItem = st.anim === 'appendDeleteTs' && stepProgress() < .82; const appendSettled = st.anim === 'appendDeleteTs' && stepProgress() >= .82; - const items = hideAppendingItem ? lane.deleteTs.slice(0, -1) : lane.deleteTs; + const foldingDeleteTs = st.anim === 'foldDeleteTs' && stepProgress() < .82; + const items = foldingDeleteTs ? (lane.prevDeleteTs || lane.deleteTs) : hideAppendingItem ? lane.deleteTs.slice(0, -1) : lane.deleteTs; const visible = items.slice(0, columns * rows); const itemW = Math.min(72, Math.floor((w - 18 * (columns - 1)) / columns)); const gapX = columns > 1 ? (w - itemW * columns) / (columns - 1) : 0; @@ -3107,10 +2986,12 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const row = Math.floor(i / columns); const ts = deleteTsValue(item); const eligible = lane.safeTs != null && ts <= lane.safeTs; - const gcActive = lane.highlight === 'foldEligible' && eligible; + const gcActive = foldEligibleHighlightReady() && lane.highlight === 'foldEligible' && eligible; + const foldingEligible = foldingDeleteTs && eligible; + if (foldingEligible) return; const appendVisible = st.anim !== 'appendDeleteTs' || !hideAppendingItem; - const active = gcActive || (lane.highlight === 'visibility' && appendVisible && !appendSettled && i === visible.length - 1); - const color = gcActive ? flow.gc : flow.delete; + const active = gcActive || foldingEligible || (lane.highlight === 'visibility' && appendVisible && !appendSettled && i === visible.length - 1); + const color = gcActive || foldingEligible ? flow.gc : active || st.anim === 'appendDeleteTs' ? flow.delete : c.border; drawPill(x + col * (itemW + gapX), y + row * (itemH + rowGap), itemW, itemH, item, color, c, active, 8); }); if (hideAppendingItem) { @@ -3124,6 +3005,62 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

} } + function foldEligibleHighlightReady() { + if (currentStep().anim === 'foldDeleteTs') return true; + if (activeReplayLane().highlight !== 'foldEligible') return true; + return stepProgress() >= .72; + } + + function drawFoldingDeleteTsItems(chainX, chainY, chainW, bitmapX, bitmapY, bitmapW, lane, c) { + if (currentStep().anim !== 'foldDeleteTs') return; + const p = stepProgress(); + if (p >= .82) return; + const t = ease(clamp((p - .08) / .70, 0, 1)); + const layout = deleteTsItemLayout(chainW); + const items = (lane.prevDeleteTs || []).slice(0, layout.columns * layout.rows); + items.forEach((item, i) => { + if (lane.safeTs == null || deleteTsValue(item) > lane.safeTs) return; + const offset = foldBitmapOffsetValue(item, lane); + if (!Number.isFinite(offset)) return; + const start = deleteTsItemCenter(chainX, chainY, layout, i); + const target = bitmapCellCenter(bitmapX, bitmapY, bitmapW, offset); + const cx = lerp(start.x, target.x, t); + const cy = lerp(start.y, target.y, t); + const alphaOut = 1 - ease(clamp((p - .54) / .28, 0, 1)); + + ctx.save(); + ctx.globalAlpha *= alphaOut; + ctx.translate(cx, cy); + ctx.scale(lerp(1, .82, t), lerp(1, .82, t)); + drawPill(-layout.itemW / 2, -layout.itemH / 2, layout.itemW, layout.itemH, item, flow.gc, c, true, 8); + ctx.restore(); + }); + } + + function deleteTsItemLayout(w) { + const columns = 4; + const rows = 2; + const itemW = Math.min(72, Math.floor((w - 18 * (columns - 1)) / columns)); + const gapX = columns > 1 ? (w - itemW * columns) / (columns - 1) : 0; + const itemH = 24; + const rowGap = 10; + return { columns, rows, itemW, gapX, itemH, rowGap }; + } + + function deleteTsItemCenter(x, y, layout, index) { + const col = index % layout.columns; + const row = Math.floor(index / layout.columns); + return { + x: x + col * (layout.itemW + layout.gapX) + layout.itemW / 2, + y: y + row * (layout.itemH + layout.rowGap) + layout.itemH / 2 + }; + } + + function bitmapCellCenter(x, y, w, index) { + const cell = bitmapCellRect(x, y, w, index); + return { x: cell.x + cell.w / 2, y: cell.y + cell.h / 2 }; + } + function drawAppendingDeleteTsItem(x, y, itemW, itemH, gapX, rowGap, columns, rows, lane, c) { const p = stepProgress(); if (p < .10) return; @@ -3148,6 +3085,33 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.restore(); } + function foldDeleteTsIntoBaseBitmap(lane, safeTs) { + const len = 16; + const bits = (lane.baseBitmap || '').padEnd(len, '0').slice(0, len).split(''); + const remaining = []; + lane.deleteTs.forEach(item => { + const ts = deleteTsValue(item); + const offset = foldBitmapOffsetValue(item, lane); + if (ts <= safeTs) { + if (Number.isFinite(offset) && offset >= 0 && offset < len) bits[offset] = '1'; + } else { + remaining.push(item); + } + }); + lane.safeTs = safeTs; + lane.baseTs = safeTs; + lane.baseBitmap = bits.join(''); + lane.deleteTs = remaining; + lane.highlight = 'baseBitmap'; + } + + function foldBitmapOffsetValue(item, lane) { + if (lane && lane.id === DETAIL_REPLAY_ID && deleteTsValue(item) <= DEFAULT_GC_SAFE_TS) { + return GC_FOLD_TARGET_OFFSET; + } + return deleteOffsetValue(item); + } + function deleteOffsetValue(item) { const m = /\((\d+),/.exec(item); return m ? Number(m[1]) : Number.NaN; @@ -3165,7 +3129,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

drawFlushedMemtablePanel(layout.memtable.x, layout.memtable.y, layout.memtable.w, layout.memtable.h, { title: 'blocks', color: flow.storage, - labels: storageObjectLabels().slice(0, STORAGE_ITEM_CAPACITY), + labels: storageObjectLabelsForDraw().slice(0, STORAGE_ITEM_CAPACITY), hotLabel: model.storage.flushedHot }, c); drawStoragePanel(layout.files.x, layout.files.y, layout.files.w, layout.files.h, { @@ -3173,17 +3137,123 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

title: 'files', color: flow.replay, kind: 'FILE', - labels: regularStorageFileLabels(), - active: ['query', 'gc', 'storageGc'].includes(state.detailFocus), + labels: regularStorageFileLabelsForDraw(), + active: ['query', 'storageGc'].includes(state.detailFocus) || ['compactObjectsToFile', 'validRatioCheck', 'compactRegularFiles'].includes(currentStep().anim), regular: true, maxItems: STORAGE_ITEM_CAPACITY }, c); + drawStorageCompactionFileOverlay(c); + drawRegularFileCompactionOverlay(c); } function storageObjectLabels() { return [...model.storage.objectEntries]; } + function storageObjectLabelsForDraw() { + if (currentStep().anim === 'compactObjectsToFile') { + const previous = model.storage.prevObjectEntries || []; + if (previous.length) return [...previous]; + } + return storageObjectLabels(); + } + + function drawStorageCompactionFileOverlay(c) { + if (currentStep().anim !== 'compactObjectsToFile') return; + const p = stepProgress(); + if (p >= .70) return; + + const fileLabel = model.storage.compactionFile || 'file-2.pxl'; + const previousObjects = model.storage.prevObjectEntries || []; + if (!previousObjects.length) return; + + const layout = storagePanelsLayout(rects.storage); + const objectLayout = flushedMemtableObjectLayout(layout.memtable, previousObjects); + const objectCenters = objectLayout.visibleLabels.map((_, i) => ({ + x: objectLayout.xForIndex(i) + objectLayout.objW / 2, + y: objectLayout.objY + objectLayout.objH / 2 + })); + const start = averagePoints(objectCenters); + + const fileLabels = regularStorageFileLabels(); + const fileLayout = flushedMemtableObjectLayout(layout.files, fileLabels, STORAGE_ITEM_CAPACITY, 'left'); + let fileIndex = fileLayout.visibleLabels.indexOf(fileLabel); + if (fileIndex < 0) fileIndex = Math.max(0, fileLayout.visibleLabels.length - 1); + const end = { + x: fileLayout.xForIndex(fileIndex) + fileLayout.objW / 2, + y: fileLayout.objY + fileLayout.objH / 2 + }; + + const move = ease(clamp((p - .06) / .50, 0, 1)); + const fadeIn = clamp(p / .12, 0, 1); + const fadeOut = p > .58 ? 1 - clamp((p - .58) / .12, 0, 1) : 1; + const x = lerp(start.x, end.x, move); + const y = lerp(start.y, end.y, move); + + ctx.save(); + ctx.globalAlpha *= fadeIn * fadeOut; + drawCompactStorageFileIcon( + x - fileLayout.objW / 2, + y - fileLayout.objH / 2, + fileLayout.objW, + fileLayout.objH, + displayStorageFileLabel(fileLabel), + flow.storage, + c, + true + ); + ctx.restore(); + } + + function drawRegularFileCompactionOverlay(c) { + if (currentStep().anim !== 'compactRegularFiles') return; + const p = stepProgress(); + const fileLabel = model.storage.compactionFile || 'file-2.pxl'; + const previousFiles = model.storage.prevRegularFiles || []; + if (!previousFiles.length) return; + + const layout = storagePanelsLayout(rects.storage); + const oldLayout = flushedMemtableObjectLayout(layout.files, previousFiles, STORAGE_ITEM_CAPACITY, 'left'); + const oldCenters = oldLayout.visibleLabels.map((_, i) => ({ + x: oldLayout.xForIndex(i) + oldLayout.objW / 2, + y: oldLayout.objY + oldLayout.objH / 2 + })); + const start = averagePoints(oldCenters); + + const finalLabels = [...previousFiles, fileLabel]; + const finalLayout = flushedMemtableObjectLayout(layout.files, finalLabels, STORAGE_ITEM_CAPACITY, 'left'); + const finalIndex = Math.min(previousFiles.length, finalLayout.visibleLabels.length - 1); + const end = { + x: finalLayout.xForIndex(finalIndex) + finalLayout.objW / 2, + y: finalLayout.objY + finalLayout.objH / 2 + }; + + const t = ease(clamp((p - .06) / .50, 0, 1)); + const fadeIn = ease(clamp(p / .14, 0, 1)); + const x = lerp(start.x, end.x, t); + const y = lerp(start.y, end.y, t); + + ctx.save(); + ctx.globalAlpha *= fadeIn; + drawCompactStorageFileIcon( + x - finalLayout.objW / 2, + y - finalLayout.objH / 2, + finalLayout.objW, + finalLayout.objH, + displayStorageFileLabel(fileLabel), + flow.storage, + c, + true + ); + ctx.restore(); + } + + function averagePoints(points) { + if (!points.length) return { x: 0, y: 0 }; + const sum = points.reduce((acc, p) => ({ x: acc.x + p.x, y: acc.y + p.y }), { x: 0, y: 0 }); + return { x: sum.x / points.length, y: sum.y / points.length }; + } + function storagePanelsLayout(r) { const x = r.x + STORAGE_EDGE_PAD; const y = r.y + MODULE_TITLE_Y + MODULE_CONTENT_GAP; @@ -3212,7 +3282,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return labels.slice(0, STORAGE_ITEM_CAPACITY); } - function flushedMemtableObjectLayout(panel, labels, maxItems = STORAGE_ITEM_CAPACITY) { + function flushedMemtableObjectLayout(panel, labels, maxItems = STORAGE_ITEM_CAPACITY, align = 'right') { const metrics = retinaMemtableMetrics(); const objW = STORAGE_COMPACT_ITEM_W; const objY = panel.y + STORAGE_PANEL_TITLE_Y + STORAGE_TITLE_ITEM_GAP; @@ -3222,6 +3292,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const innerW = Math.max(objW, panel.w - edgePad * 2); const maxFit = Math.max(1, Math.floor((innerW + gap) / (objW + gap))); const visibleLabels = labels.slice(0, Math.min(maxItems, maxFit)); + const leftX = panel.x + edgePad; const rightX = panel.x + panel.w - edgePad - objW; return { objW, @@ -3229,8 +3300,11 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

objY, gap, visibleLabels, + leftX, rightX, - xForIndex: i => rightX - (visibleLabels.length - 1 - i) * (objW + gap) + xForIndex: i => align === 'left' + ? leftX + i * (objW + gap) + : rightX - (visibleLabels.length - 1 - i) * (objW + gap) }; } @@ -3242,6 +3316,13 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return [...new Set(files)]; } + function regularStorageFileLabelsForDraw() { + if (currentStep().anim === 'compactRegularFiles') { + return [...(model.storage.prevRegularFiles || [])]; + } + return regularStorageFileLabels(); + } + function drawStoragePanel(x, y, w, h, panel, c) { ctx.fillStyle = panel.active ? alpha(c.panel3, .88) : alpha(c.panel2, .78); ctx.strokeStyle = panel.active ? c.border : c.borderSoft; @@ -3292,19 +3373,87 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return; } - const layout = flushedMemtableObjectLayout(bounds, labels, panel.maxItems || 5); + const layout = flushedMemtableObjectLayout(bounds, labels, panel.maxItems || 5, 'left'); layout.visibleLabels.forEach((label, i) => { const labelText = String(label); const oldFile = model.storage.agingQueue.includes(labelText); const retired = oldFile && /RETIRED|cleanupAt|physically/.test(model.storage.cleanup); const candidate = oldFile && !retired; const newFile = labelText === model.storage.newFile; + const compactedFile = isCompactionFile(labelText); + const regularCompactionFile = isRegularCompactionFile(labelText); + const compactingOldFile = isCompactingOldRegularFile(labelText); + const fileAlpha = compactedFile ? compactionFileSettledAlpha() : regularCompactionFile ? regularCompactionFileSettledAlpha() : compactingOldFile ? regularCompactionOldFileAlpha() : 1; + if (fileAlpha <= 0) return; const temporary = newFile && model.storage.newFileState === 'TEMPORARY_GC'; - const color = retired ? flow.delete : candidate || temporary ? flow.gc : newFile ? flow.storage : panel.color; - const active = panel.active || candidate || retired || newFile || temporary; + const ratioCandidate = isValidRatioCandidateFile(labelText); + const color = compactingOldFile ? flow.delete : retired ? flow.delete : candidate || temporary || ratioCandidate ? flow.gc : newFile || compactedFile || regularCompactionFile ? flow.storage : panel.color; + const active = panel.active || candidate || retired || newFile || temporary || compactedFile || regularCompactionFile || compactingOldFile || ratioCandidate; const fileX = layout.xForIndex(i); + ctx.save(); + ctx.globalAlpha *= fileAlpha; drawCompactStorageFileIcon(fileX, layout.objY, layout.objW, layout.objH, displayStorageFileLabel(labelText), color, c, active); + ctx.restore(); }); + drawValidRatioConditionPill(bounds, layout, c); + } + + function isCompactionFile(label) { + return currentStep().anim === 'compactObjectsToFile' && label === model.storage.compactionFile; + } + + function isRegularCompactionFile(label) { + return currentStep().anim === 'compactRegularFiles' && label === model.storage.compactionFile; + } + + function isCompactingOldRegularFile(label) { + return currentStep().anim === 'compactRegularFiles' && (model.storage.prevRegularFiles || []).includes(label); + } + + function isValidRatioCandidateFile(label) { + return currentStep().anim === 'validRatioCheck' && + stepProgress() >= .58 && + (model.storage.regularFiles || []).includes(label); + } + + function compactionFileSettledAlpha() { + const p = stepProgress(); + if (p < .58) return 0; + return ease(clamp((p - .58) / .22, 0, 1)); + } + + function regularCompactionFileSettledAlpha() { + const p = stepProgress(); + if (p < .58) return 0; + return ease(clamp((p - .58) / .24, 0, 1)); + } + + function regularCompactionOldFileAlpha() { + const p = stepProgress(); + return 1 - ease(clamp((p - .52) / .34, 0, 1)); + } + + function drawValidRatioConditionPill(bounds, layout, c) { + if (currentStep().anim !== 'validRatioCheck') return; + const p = stepProgress(); + if (p < .12) return; + const fadeIn = ease(clamp((p - .12) / .28, 0, 1)); + const fadeOut = 1 - ease(clamp((p - .48) / .10, 0, 1)); + if (fadeOut <= 0) return; + const lastIndex = Math.max(0, layout.visibleLabels.length - 1); + const lastFileRight = layout.xForIndex(lastIndex) + layout.objW; + const blankRight = bounds.x + bounds.w; + const blankW = Math.max(0, blankRight - lastFileRight); + const sideGap = 4; + const availableW = Math.max(0, blankW - sideGap * 2); + const pillW = availableW >= 110 ? Math.min(140, availableW) : availableW; + const pillH = 24; + const x = lastFileRight + (blankW - pillW) / 2; + const y = layout.objY + layout.objH / 2 - pillH / 2; + ctx.save(); + ctx.globalAlpha *= fadeIn * fadeOut; + drawPill(x, y, pillW, pillH, 'valid ratio <= 0.5', flow.gc, c, true, 8); + ctx.restore(); } function displayStorageFileLabel(label) { @@ -3361,17 +3510,55 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

layout.visibleLabels.forEach((label, i) => { const objX = layout.xForIndex(i); - ctx.fillStyle = alpha(panel.color, .13); - ctx.strokeStyle = alpha(panel.color, .30); - ctx.lineWidth = 1; + const style = storageObjectIconStyle(String(label), panel); + if (style.alpha <= 0) return; + ctx.save(); + ctx.globalAlpha *= style.alpha; + ctx.fillStyle = alpha(style.color, style.active ? .24 : .13); + ctx.strokeStyle = alpha(style.color, style.active ? .62 : .30); + ctx.lineWidth = style.active ? 1.5 : 1; roundRect(objX, layout.objY, layout.objW, layout.objH, 5, true, true); ctx.fillStyle = c.text; ctx.font = typeFont(TYPE.text, 800); ctx.textAlign = 'center'; fitText(displayStorageObjectLabel(label), objX + 3, layout.objY + layout.objH / 2 + 5, layout.objW - 6, 'center'); + if (style.deleting) drawStorageObjectDeleteMark(objX, layout.objY, layout.objW, layout.objH); + ctx.restore(); }); } + function storageObjectIconStyle(label, panel) { + const deleting = currentStep().anim === 'compactObjectsToFile' && + (model.storage.prevObjectEntries || []).includes(label) && + stepProgress() >= .50; + if (!deleting) { + return { + color: panel.color, + alpha: 1, + active: String(label) === String(panel.hotLabel || ''), + deleting: false + }; + } + const fade = ease(clamp((stepProgress() - .50) / .46, 0, 1)); + return { + color: flow.delete, + alpha: lerp(1, 0, fade), + active: true, + deleting: true + }; + } + + function drawStorageObjectDeleteMark(x, y, w, h) { + ctx.strokeStyle = alpha(flow.delete, .82); + ctx.lineWidth = 2; + ctx.beginPath(); + ctx.moveTo(x + 10, y + 10); + ctx.lineTo(x + w - 10, y + h - 10); + ctx.moveTo(x + w - 10, y + 10); + ctx.lineTo(x + 10, y + h - 10); + ctx.stroke(); + } + function displayStorageObjectLabel(label) { return String(label).replace(/obj-(\d+)$/i, (_, n) => `obj-${n.slice(-1)}`); } @@ -3866,6 +4053,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const { queue, fullFill } = writeLayout; const visibilityLayout = replayVisibilityLayout(visibility.x, visibility.y, visibility.w, visibility.h); const { rowY, bitmapX, bitmapW, chainY, chainH } = visibilityLayout; + const deleteConditionY = chainY + 68; const offset = (slot - 1) * 18; return { center: [frame.x + frame.w / 2, frame.y + frame.h / 2], @@ -3879,6 +4067,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

immutable: [queue.x + queue.w / 2, queue.y + queue.h / 2], visibility: [visibility.x + visibility.w / 2, visibility.y + visibility.h / 2], deleteArray: [visibility.x + visibility.w / 2, chainY + chainH / 2], + deleteCondition: [visibility.x + visibility.w / 2, deleteConditionY], baseBitmap: [bitmapX + bitmapW / 2, rowY + 7], write: [write.x + write.w / 2, write.y + write.h / 2] }; @@ -3902,20 +4091,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const r = rects[parts[0]]; if (!r) return { x: 0, y: 0 }; const pos = parts[1] || 'center'; - if (parts[0] === 'cdcStream') { - const railY = r.y + 58; - const anchors = { - center: [r.x + r.w / 2, r.y + r.h / 2], - left: [r.x, railY], - right: [r.x + r.w, railY], - top: [r.x + r.w / 2, r.y], - bottom: [r.x + r.w / 2, r.y + r.h], - out: [r.x + r.w / 2, r.y + r.h], - row: [r.x + r.w / 2, railY], - commit: [r.x + r.w - 42, railY] - }; - return arrPoint(anchors[pos] || anchors.center); - } if (parts[0] === 'runtime') { const borderAnchors = { center: [r.x + r.w / 2, r.y + r.h / 2], From 978a6c32ba3d767cb891f0d41b1d1bb9fc0aa273 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Thu, 9 Jul 2026 00:34:55 +0800 Subject: [PATCH 06/11] fix --- retina.html | 953 +++------------------------------------------------- 1 file changed, 42 insertions(+), 911 deletions(-) diff --git a/retina.html b/retina.html index 5b92a8c36..b81bcc906 100644 --- a/retina.html +++ b/retina.html @@ -617,10 +617,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return `${prefix}${size}px ${family}`; } - function monoFont(size, weight = 800) { - return typeFont(size, weight, 'monospace, Inter, system-ui'); - } - function pillTypeSize(fs) { return TYPE.text; } @@ -668,8 +664,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const PIXELS_SINK_X = LEFT_PANEL_X + (LEFT_PANEL_W - PIXELS_SINK_W) / 2; const rects = { ingress: { x: PIXELS_SINK_X, y: INGRESS_Y, w: PIXELS_SINK_W, h: PIXELS_SINK_H, title: 'Pixels Sink', sub: 'mirror transaction -> replay RPC' }, - sink: { x: PIXELS_SINK_X + 14, y: INGRESS_Y + 292, w: PIXELS_SINK_W - 28, h: 96, title: 'Pixels Sink replay', sub: 'replay path anchors' }, - tx: { x: SERVICE_X, y: SERVICE_Y, w: TOP_SERVICE_W, h: TX_SERVICE_H, title: 'Transaction Service', sub: 'MVCC time boundary: HWM / LWM' }, index: { x: SERVICE_X + TOP_SERVICE_W + SERVICE_GAP, y: SERVICE_Y, w: INDEX_W, h: SERVICE_H, title: 'Index Service', sub: 'primaryKey→rowId + rowId→RowLocation' }, runtime: { x: SERVICE_X, y: RUNTIME_Y, w: RETINA_REPLAYER_W, h: RETINA_REPLAYER_H, title: 'Retina replayer', sub: 'RetinaWorkerService RPC + write / visibility state' }, @@ -682,7 +676,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const info = { ingress: ['Pixels Sink', 'Creates mirror transactions, sends replay RPCs, and commits after replay OK.'], - sink: ['Pixels Sink replay anchor', 'Invisible anchor region for replay paths.'], runtime: ['Retina replayer', 'RetinaServerImpl handles RetinaWorkerService RPCs and keeps write-buffer and visibility state.'], tx: ['Transaction Service', 'Exposes MVCC time boundaries. Callers get snapshot or safeTs on demand; the panel stays focused on HWM and LWM.'], index: ['Index Service', 'Maps primary keys to rowIds and physical row locations; delete/update use this map before touching visibility state.'], @@ -721,27 +714,21 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

rowChanges: ['pk160', 'pk132', 'pk208'], mirrorOps: ['insert', 'delete', 'update'], sinkStep: null, - cdcTx: 1208, - changeCount: 72410, - coordState: 'mirror tx ready', - cachedRows: '0 cached', routingRule: 'by row key', writePlan: 'waiting for row changes', rpcBatch: '1 tx per request', - transServer: 'ready', - requestMode: 'replay RPC', requestState: 'ready', txState: 'Transaction Service ready' }, lanes: [ laneModel(0, 'I:pk160', 5, 0, ['(2,276)'], 266), - laneModel(1, 'U:pk132', 7, 0, ['(5,278)', '(7,282)'], 264), + laneModel(1, 'U:pk132', 7, 0, ['(5,278)', '(9,282)'], 264), laneModel(2, 'D:pk208', 4, 0, ['(1,279)'], 267) ], index: { hot: 'pk132', primaryEntries: ['pk@212 -> rowId820'], - mainEntries: ['rowId820 -> file-06/rg1/off5'], + mainEntries: ['rowId820 -> file-0/rg1/off02'], operation: 'idle' }, storage: { @@ -799,6 +786,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

prevBaseBitmap: '0000000000000000', deleteTs: deletes, prevDeleteTs: [...deletes], + bitmapDeleteTs: [...deletes], safeTs: null, throughput: 72 + id * 11, highlight: null @@ -814,20 +802,17 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.rowChanges = ['INSERT', 'DELETE', 'UPDATE']; m.sink.mirrorOps = ['insert', 'delete', 'update']; m.sink.sinkStep = 'empty'; - m.sink.coordState = 'mirror tx ready'; - m.sink.cachedRows = ''; m.sink.requestState = 'begin mirror tx'; m.sink.txState = 'await begin'; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk@212 -> rowId820']; - m.index.mainEntries = ['rowId820 -> file-07/rg0/off7']; + m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; } if (scene === 'insert') { m.sink.mirrorTx = 'tx290'; m.sink.rowChanges = ['orders 3']; m.sink.mirrorOps = ['insert']; m.sink.sinkStep = 'empty'; - m.sink.coordState = 'mirror tx ready'; m.sink.routingRule = 'after-image key'; m.sink.writePlan = 'pk160, pk174, pk181 queued'; m.sink.rpcBatch = 'tx290 only'; @@ -839,7 +824,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.rowChanges = ['orders 1']; m.sink.mirrorOps = ['delete']; m.sink.sinkStep = 'empty'; - m.sink.coordState = 'mirror tx ready'; m.sink.routingRule = 'before-image key'; m.sink.writePlan = 'pk208 queued'; m.sink.rpcBatch = 'tx290 only'; @@ -847,7 +831,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.txState = 'tx290 ready'; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk@212 -> rowId820']; - m.index.mainEntries = ['rowId820 -> file-07/rg0/off7']; + m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.operation = 'await Retina delete'; } if (scene === 'update') { @@ -855,7 +839,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.rowChanges = ['customer 1']; m.sink.mirrorOps = ['update']; m.sink.sinkStep = 'empty'; - m.sink.coordState = 'mirror tx ready'; m.sink.routingRule = 'same primary key'; m.sink.writePlan = 'pk132 queued'; m.sink.rpcBatch = 'tx290 only'; @@ -863,7 +846,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m.sink.txState = 'tx290 ready'; m.index.hot = 'pk132'; m.index.primaryEntries = ['pk@212 -> rowId820']; - m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; + m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.operation = 'insert new + update primary'; } if (scene === 'query') { @@ -912,17 +895,17 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { m.lanes[1].highlight = 'immutable'; }, 'writeBuffer', []), step('A background flush asynchronously moves the immutable memtable to Storage.', 2.0, ['lane1'], [], - m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; }, 'writeBuffer', [], 'flush'), + m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; }, 'writeBuffer', [], 'flush'), step('The flushed object is now fixed in Storage before Retina moves on to index maintenance.', 1.2, [], [], m => { m.lanes[1].highlight = null; settlePendingFlushObject(m); }, 'storage', []), compactFlushedObjectsToFile(), step('After the new row is durable, Retina calls Index Service to insert the primary and row-location indexes.', 4.2, ['index'], [tok('put pk@290', 'runtime.right', 'index.left', flow.index, .12, .82)], - m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.index.hot = 'pk174'; m.index.operation = 'put pk@290 in flight'; }, 'index', ['index']), + m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.hot = 'pk174'; m.index.operation = 'put pk@290 in flight'; }, 'index', ['index']), step('Index Service inserts the new primary and row-location entries locally.', 2.4, ['index'], [], - m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5', 'rowId821 -> file-07/rg0/off7']; m.index.operation = 'index insert entry applied'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), + m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02', 'rowId821 -> file-2/rg0/off15']; m.index.operation = 'index insert entry applied'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), step('Only after the index insert is complete does Index Service return OK to Retina.', 3.0, ['index'], [tok('index OK', 'index.left', 'runtime.right', flow.index, .18, .82)], m => { m.index.operation = 'index insert OK'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), @@ -939,11 +922,11 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

['index'], [], m => { m.index.operation = 'delete index entry pk@290 applied'; }, 'index', ['index']), step('Only after the entry is deleted, Index Service returns the deleted main-index location.', 3.2, - ['index'], [tok('file-07/rg0/off7', 'index.left', 'runtime.right', flow.index, .18, .82)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'deleted location returned'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.index.operation = 'deleted loc file-07/rg0/off7 returned for pk@290'; }, 'index', ['index']), + ['index'], [tok('file-2/rg0/off15', 'index.left', 'runtime.right', flow.index, .18, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'deleted location returned'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.operation = 'deleted loc file-2/rg0/off15 returned for pk@290'; }, 'index', ['index']), step('Retina then appends the delete timestamp into the visibility structure delete-ts chain.', 4.2, - ['lane1'], [tok('(off7,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'append delete timestamp'; l.highlight = 'visibility'; if (!l.deleteTs.includes('(7,290)')) l.deleteTs.push('(7,290)'); m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.sink.rpcBatch = 'DELETE applying'; m.storage.staleRatio = 0.36; m.storage.cleanup = 'file unchanged'; }, 'visibility', ['visibility'], 'appendDeleteTs'), + ['lane1'], [tok('(off15,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'append delete timestamp'; l.highlight = 'visibility'; if (!l.deleteTs.includes('(15,290)')) l.deleteTs.push('(15,290)'); m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.sink.rpcBatch = 'DELETE applying'; m.storage.staleRatio = 0.36; m.storage.cleanup = 'file unchanged'; }, 'visibility', ['visibility'], 'appendDeleteTs'), step('Retina returns OK for DELETE before Pixels Sink creates the UPDATE row.', 3.2, ['ingress'], [tok('OK', 'runtime.left', 'ingress.right', flow.delete, .12, .82)], m => { m.sink.sinkStep = 'row2Ok'; m.sink.requestState = 'DELETE OK'; m.sink.rpcBatch = 'OK'; clearDeleteTsChainEffects(m, DETAIL_REPLAY_ID); }, 'mvcc', []), @@ -955,19 +938,19 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { const l = m.lanes[1]; l.currentOp = 'U:pk132'; l.status = 'append after image'; l.activeRows = 1; l.highlight = 'active'; m.index.operation = 'await update index'; m.sink.rpcBatch = 'UPDATE applying'; }, 'writeBuffer', []), step('Because the active memtable is not full, Retina sends an update request to Index Service.', 3.6, ['index'], [tok('update pk@290', 'runtime.right', 'index.left', flow.index, .12, .86)], - m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.index.hot = 'pk132'; m.index.operation = 'update pk@290 in flight'; }, 'index', ['index']), + m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.hot = 'pk132'; m.index.operation = 'update pk@290 in flight'; }, 'index', ['index']), step('Index Service first inserts the new versioned primary-index entry locally.', 2.4, ['index'], [], - m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5', 'rowId821 -> file-07/rg0/off0']; m.index.operation = 'put version pk@290 applied'; }, 'index', ['index']), + m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02', 'rowId821 -> file-3/rg0/off00']; m.index.operation = 'put version pk@290 applied'; }, 'index', ['index']), step('Index Service then deletes the old index entry for the previous tuple version.', 2.4, ['index'], [], m => { m.index.operation = 'delete index entry pk@212 applied'; }, 'index', ['index']), step('Only after the old entry is deleted, Index Service returns the deleted main-index location.', 3.0, - ['index'], [tok('file-06/rg1/off5', 'index.left', 'runtime.right', flow.index, .16, .84)], - m => { m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-07/rg0/off0']; m.index.operation = 'deleted loc file-06/rg1/off5 returned for pk@212'; }, 'index', ['index']), + ['index'], [tok('file-0/rg1/off02', 'index.left', 'runtime.right', flow.index, .16, .84)], + m => { m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-3/rg0/off00']; m.index.operation = 'deleted loc file-0/rg1/off02 returned for pk@212'; }, 'index', ['index']), step('Using the returned old location, Retina deletes the old tuple version by appending a delete timestamp to visibility.', 4.2, - ['lane1'], [tok('(off5,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], - m => { const l = m.lanes[1]; l.highlight = 'visibility'; if (!l.deleteTs.includes('(5,290)')) l.deleteTs.push('(5,290)'); m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-07/rg0/off0']; m.index.operation = 'update index OK'; }, 'visibility', ['visibility'], 'appendDeleteTs'), + ['lane1'], [tok('(off2,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[1]; l.highlight = 'visibility'; if (!l.deleteTs.includes('(2,290)')) l.deleteTs.push('(2,290)'); m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-3/rg0/off00']; m.index.operation = 'update index OK'; }, 'visibility', ['visibility'], 'appendDeleteTs'), step('Retina returns OK for UPDATE. Pixels Sink can now finish the mirror transaction.', 3.2, ['ingress'], [tok('OK', 'runtime.left', 'ingress.right', flow.index, .12, .82)], m => { m.sink.sinkStep = 'row3Ok'; m.sink.requestState = 'UPDATE OK'; m.sink.rpcBatch = 'OK'; clearDeleteTsChainEffects(m, 1); }, 'mvcc', []), @@ -997,17 +980,17 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { const l = m.lanes[1]; l.highlight = 'immutable'; }, 'writeBuffer', []), step('A background flush asynchronously moves the immutable memtable to Storage.', 2.0, ['lane1'], [], - m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; }, 'writeBuffer', [], 'flush'), + m => { const l = m.lanes[1]; l.highlight = 'immutable'; l.immutable = Math.max(0, l.immutable - 1); m.storage.pendingFlushObject = 'obj-78'; m.storage.flushedHot = 'obj-78'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; }, 'writeBuffer', [], 'flush'), step('The flushed object is now fixed in Storage before Retina moves on to index maintenance.', 1.2, [], [], m => { m.lanes[1].highlight = null; settlePendingFlushObject(m); }, 'storage', []), compactFlushedObjectsToFile(), step('After the row is appended, Retina calls Index Service to insert the primary and row-location indexes.', 4.2, ['index'], [tok('put pk@290', 'runtime.right', 'index.left', flow.index, .12, .82)], - m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.index.hot = 'pk174'; m.index.operation = 'put pk@290 in flight'; }, 'index', ['index']), + m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.hot = 'pk174'; m.index.operation = 'put pk@290 in flight'; }, 'index', ['index']), step('Index Service inserts the new primary and row-location entries locally.', 2.4, ['index'], [], - m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5', 'rowId821 -> file-07/rg0/off7']; m.index.operation = 'index insert entry applied'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), + m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02', 'rowId821 -> file-2/rg0/off15']; m.index.operation = 'index insert entry applied'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), step('Only after the index insert is complete does Index Service return OK to Retina.', 3.0, ['index'], [tok('index OK', 'index.left', 'runtime.right', flow.index, .18, .82)], m => { m.index.operation = 'index insert OK'; m.sink.rpcBatch = 'INSERT applying'; }, 'index', ['index']), @@ -1030,16 +1013,16 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

m => { m.sink.sinkStep = 'replay'; m.sink.requestState = 'replay DELETE'; m.sink.txState = 'timestamp=290'; }, 'dispatch', []), step('Retina sends a delete request to Index Service for pk208 before deleting visibility.', 4.0, ['index'], [tok('delete pk@212', 'runtime.right', 'index.left', flow.index, .12, .82)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'DELETE'; l.status = 'delete index entry'; l.highlight = null; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-07/rg0/off7']; m.index.operation = 'delete pk@212 in flight'; m.sink.rpcBatch = 'DELETE applying'; }, 'index', ['index']), + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.currentOp = 'DELETE'; l.status = 'delete index entry'; l.highlight = null; m.index.hot = 'pk208'; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.operation = 'delete pk@212 in flight'; m.sink.rpcBatch = 'DELETE applying'; }, 'index', ['index']), step('Index Service deletes the matched primary and row-location entries locally.', 2.4, ['index'], [], m => { m.index.operation = 'delete index entry pk@212 applied'; }, 'index', ['index']), step('Only after the entry is deleted, Index Service returns the deleted main-index location.', 3.2, - ['index'], [tok('file-07/rg0/off7', 'index.left', 'runtime.right', flow.index, .18, .82)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'deleted location returned'; m.index.primaryEntries = []; m.index.mainEntries = []; m.index.operation = 'deleted loc file-07/rg0/off7 returned for pk@212'; }, 'index', ['index']), + ['index'], [tok('file-0/rg1/off02', 'index.left', 'runtime.right', flow.index, .18, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'deleted location returned'; m.index.primaryEntries = []; m.index.mainEntries = []; m.index.operation = 'deleted loc file-0/rg1/off02 returned for pk@212'; }, 'index', ['index']), step('Retina then appends the delete timestamp into the visibility structure delete-ts chain.', 4.2, - ['lane1'], [tok('(off7,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], - m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'append delete timestamp'; l.highlight = 'visibility'; if (!l.deleteTs.includes('(7,290)')) l.deleteTs.push('(7,290)'); m.index.primaryEntries = []; m.index.mainEntries = []; m.sink.rpcBatch = 'DELETE applying'; m.storage.staleRatio = 0.36; m.storage.cleanup = 'file unchanged'; }, 'visibility', ['visibility'], 'appendDeleteTs'), + ['lane1'], [tok('(off2,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[DETAIL_REPLAY_ID]; l.status = 'append delete timestamp'; l.highlight = 'visibility'; if (!l.deleteTs.includes('(2,290)')) l.deleteTs.push('(2,290)'); m.index.primaryEntries = []; m.index.mainEntries = []; m.sink.rpcBatch = 'DELETE applying'; m.storage.staleRatio = 0.36; m.storage.cleanup = 'file unchanged'; }, 'visibility', ['visibility'], 'appendDeleteTs'), ...commitSequence('tx290', 291, 'delete committed; HWM=291', DETAIL_REPLAY_ID) ], update: [ @@ -1064,21 +1047,22 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

step('The flushed object is now fixed in Storage before Retina moves on to index maintenance.', 1.2, [], [], m => { m.lanes[1].highlight = null; settlePendingFlushObject(m); }, 'storage', []), + compactFlushedObjectsToFile(), step('Retina sends an update request to Index Service for pk132 at the new timestamp.', 3.6, ['index'], [tok('update pk@290', 'runtime.right', 'index.left', flow.index, .12, .86)], - m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5']; m.index.hot = 'pk132'; m.index.operation = 'update pk@290 in flight'; }, 'index', ['index']), + m => { const l = m.lanes[1]; l.highlight = null; m.index.primaryEntries = ['pk@212 -> rowId820']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02']; m.index.hot = 'pk132'; m.index.operation = 'update pk@290 in flight'; }, 'index', ['index']), step('Index Service first inserts the new versioned primary-index entry locally.', 2.4, ['index'], [], - m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-06/rg1/off5', 'rowId821 -> file-07/rg0/off6']; m.index.operation = 'put version pk@290 applied'; }, 'index', ['index']), + m => { m.index.primaryEntries = ['pk@212 -> rowId820', 'pk@290 -> rowId821']; m.index.mainEntries = ['rowId820 -> file-0/rg1/off02', 'rowId821 -> file-2/rg0/off15']; m.index.operation = 'put version pk@290 applied'; }, 'index', ['index']), step('Index Service then deletes the old index entry for the previous tuple version.', 2.4, ['index'], [], m => { m.index.operation = 'delete index entry pk@212 applied'; }, 'index', ['index']), step('Only after the old entry is deleted, Index Service returns the deleted main-index location.', 3.0, - ['index'], [tok('file-06/rg1/off5', 'index.left', 'runtime.right', flow.index, .16, .84)], - m => { m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-07/rg0/off6']; m.index.operation = 'deleted loc file-06/rg1/off5 returned for pk@212'; }, 'index', ['index']), + ['index'], [tok('file-0/rg1/off02', 'index.left', 'runtime.right', flow.index, .16, .84)], + m => { m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-2/rg0/off15']; m.index.operation = 'deleted loc file-0/rg1/off02 returned for pk@212'; }, 'index', ['index']), step('Using the returned old location, Retina deletes the old tuple version by appending a delete timestamp to visibility.', 4.2, - ['lane1'], [tok('(off5,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], - m => { const l = m.lanes[1]; l.highlight = 'visibility'; if (!l.deleteTs.includes('(5,290)')) l.deleteTs.push('(5,290)'); m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-07/rg0/off6']; m.index.operation = 'update index OK'; }, 'visibility', ['visibility'], 'appendDeleteTs'), + ['lane1'], [tok('(off2,290)', 'lane1.visibility', 'lane1.deleteArray', flow.delete, .10, .82)], + m => { const l = m.lanes[1]; l.highlight = 'visibility'; if (!l.deleteTs.includes('(2,290)')) l.deleteTs.push('(2,290)'); m.index.primaryEntries = ['pk@290 -> rowId821']; m.index.mainEntries = ['rowId821 -> file-2/rg0/off15']; m.index.operation = 'update index OK'; }, 'visibility', ['visibility'], 'appendDeleteTs'), ...commitSequence('tx290', 291, 'update committed; HWM=291', 1) ], query: [ @@ -1686,7 +1670,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const visibleRows = visibleMirrorRows(activeId, rows.length); const visibleCommit = activeId === 'commit' || activeId === 'committed'; drawMirrorBeginCard(begin, activeId === 'begin' || activeId === 'build', c); - drawMirrorStackArea(layout, visibleRows, ['row1', 'row2', 'row3', 'replay', 'rowsOk'].includes(activeId), c); rows.slice(0, visibleRows).forEach((row, i) => drawMirrorRowCard(row, rowStageActive(activeId, i), c)); if (visibleCommit) drawMirrorCommitCard(commit, activeId === 'commit', c); } @@ -1711,9 +1694,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return activeId === 'build' || activeId === 'replay' || activeId === `row${index + 1}`; } - function drawMirrorStackArea(layout, visibleRows, active, c) { - } - function drawMirrorBeginCard(box, active, c) { drawCdcCard(box.x, box.y, box.w, box.h, flow.cdc, active, c); ctx.fillStyle = flow.cdc; @@ -1779,75 +1759,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.fillText('COMMIT', box.x + 18, box.y + 30); } - function drawMiniTxCard(x, y, w, h, label, color, c, active) { - ctx.fillStyle = alpha(color, active ? .18 : .10); - ctx.strokeStyle = alpha(color, active ? .56 : .30); - ctx.lineWidth = 1; - roundRect(x, y, w, h, 7, true, true); - drawTxnBracket(x + 8, y + 9, w - 16, h - 18, color, c); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText(label, x + 6, y + h / 2 + 4, w - 12, 'center'); - } - - function drawServiceChip(x, y, w, h, label, color, c, active) { - ctx.fillStyle = alpha(color, active ? .18 : .10); - ctx.strokeStyle = alpha(color, active ? .58 : .30); - ctx.lineWidth = 1; - roundRect(x, y, w, h, 8, true, true); - ctx.fillStyle = alpha(color, active ? .26 : .14); - roundRect(x + 7, y + 7, w - 14, Math.max(18, h - 14), 6, true, false); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText(label, x + 6, y + h / 2 + 5, w - 12, 'center'); - } - - function drawReplayOkDots(x, y, c, active) { - for (let i = 0; i < 3; i++) { - const cx = x + i * 24; - ctx.fillStyle = alpha(flow.replay, active ? .20 : .11); - ctx.strokeStyle = alpha(flow.replay, active ? .55 : .30); - ctx.lineWidth = 1; - roundRect(cx - 9, y - 11, 18, 22, 5, true, true); - ctx.fillStyle = flow.replay; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - ctx.fillText('OK', cx, y + 3); - } - } - - function drawMirrorTxChips(x, y, w, c, active) { - const labels = ['BEGIN', 'INSERT', 'DELETE', 'UPDATE', 'COMMIT']; - const gap = 6; - const chipW = Math.floor((w - gap * (labels.length - 1)) / labels.length); - labels.forEach((label, i) => { - const color = label === 'DELETE' ? flow.delete : label === 'COMMIT' || label === 'BEGIN' ? flow.tx : flow.replay; - const xx = x + i * (chipW + gap); - ctx.fillStyle = alpha(color, active ? .18 : .10); - ctx.strokeStyle = alpha(color, active ? .48 : .26); - ctx.lineWidth = 1; - roundRect(xx, y, chipW, 30, 7, true, true); - ctx.fillStyle = active ? color : c.text; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText(label, xx + 4, y + 19, chipW - 8, 'center'); - }); - } - - function drawCdcDot(cx, cy, active, color, c) { - ctx.fillStyle = active ? color : alpha(color, .30); - ctx.beginPath(); - ctx.arc(cx, cy, active ? 5 : 4, 0, Math.PI * 2); - ctx.fill(); - ctx.strokeStyle = active ? alpha(color, .78) : alpha(c.border, .45); - ctx.lineWidth = 2; - ctx.beginPath(); - ctx.arc(cx, cy, active ? 10 : 7, 0, Math.PI * 2); - ctx.stroke(); - } - function drawCdcCard(x, y, w, h, color, active, c, opts = {}) { ctx.fillStyle = opts.opaque ? c.panel : active ? alpha(color, c.dark ? .18 : .11) : alpha(c.panel2, .70); ctx.strokeStyle = active ? alpha(color, .62) : c.borderSoft; @@ -1855,56 +1766,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

roundRect(x, y, w, h, 8, true, true); } - function drawRecordTypeCard(x, y, w, h, label, color, c, fs = TYPE.text, weight = 900) { - ctx.fillStyle = alpha(color, .16); - ctx.strokeStyle = alpha(color, .48); - ctx.lineWidth = 1; - roundRect(x, y, w, h, 6, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(fs, weight); - ctx.textAlign = 'center'; - fitText(label, x + 4, y + h / 2 + fs * .36, w - 8, 'center'); - } - - function drawFoldCorner(x, y, color, c) { - ctx.fillStyle = alpha(color, .30); - ctx.beginPath(); - ctx.moveTo(x, y); - ctx.lineTo(x + 7, y); - ctx.lineTo(x + 7, y + 7); - ctx.closePath(); - ctx.fill(); - } - - function drawMiniArrow(x1, y1, x2, y2, color) { - ctx.strokeStyle = alpha(color, .62); - ctx.lineWidth = 1.5; - ctx.beginPath(); - ctx.moveTo(x1, y1); - ctx.lineTo(x2, y2); - ctx.stroke(); - arrowHead({ x: x1, y: y1 }, { x: x2, y: y2 }, alpha(color, .72)); - } - - function drawTxnBracket(x, y, w, h, color, c) { - ctx.strokeStyle = alpha(color, .70); - ctx.lineWidth = 2; - ctx.beginPath(); - ctx.moveTo(x + 7, y); - ctx.lineTo(x, y); - ctx.lineTo(x, y + h); - ctx.lineTo(x + 7, y + h); - ctx.moveTo(x + w - 7, y); - ctx.lineTo(x + w, y); - ctx.lineTo(x + w, y + h); - ctx.lineTo(x + w - 7, y + h); - ctx.stroke(); - ctx.fillStyle = alpha(color, .20); - ctx.beginPath(); - ctx.arc(x + w / 2, y + h / 2, 8, 0, Math.PI * 2); - ctx.fill(); - } - function drawLogStripes(x, y, w, h, color, c) { for (let i = 0; i < 4; i++) { ctx.fillStyle = alpha(color, .18 + i * .04); @@ -1938,213 +1799,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

} } - function drawCommitStamp(x, y, w, h, color, c) { - ctx.fillStyle = alpha(color, .13); - ctx.strokeStyle = alpha(color, .52); - ctx.lineWidth = 1.4; - ctx.beginPath(); - ctx.arc(x + w / 2, y + h / 2, Math.min(w, h) / 2, 0, Math.PI * 2); - ctx.fill(); - ctx.stroke(); - ctx.strokeStyle = color; - ctx.lineWidth = 2; - ctx.beginPath(); - ctx.moveTo(x + 8, y + 15); - ctx.lineTo(x + 13, y + 20); - ctx.lineTo(x + 21, y + 9); - ctx.stroke(); - } - - function drawSink(c) { - const r = rects.sink; - drawComponent(r, c, { color: flow.replay, key: 'sink' }); - const layout = writerLayout(); - drawRowChangeStagingPanel(layout, c); - drawReplayRpcPanel(layout, c); - drawWriterFlowAnimations(layout, c); - } - - function writerLayout() { - const r = rects.sink; - const x = r.x + 16; - const w = r.w - 32; - const top = r.y + 70; - const bottom = r.y + r.h - 22; - const gap = 20; - const stagingH = Math.min(194, Math.round((bottom - top - gap) * .52)); - const staging = { x, y: top, w, h: stagingH }; - const rpc = { x, y: staging.y + staging.h + gap, w, h: bottom - (staging.y + staging.h + gap) }; - return { r, staging, rpc }; - } - - function drawRowChangeStagingPanel(layout, c) { - const { x, y, w, h } = layout.staging; - drawCdcCard(x, y, w, h, flow.replay, /route|stage|queued|pk|row/i.test(`${model.sink.requestState} ${model.sink.writePlan}`), c); - ctx.fillStyle = flow.replay; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'left'; - fitText('row-change staging', x + PANEL_TITLE_X, y + PANEL_TITLE_Y, w - PANEL_TITLE_X * 2, 'left'); - drawRecordSheet(x + 12, y + 46, 50, 48, flow.replay, c); - drawRouteHub(x + 92, y + 70, flow.replay, c); - drawMiniArrow(x + 118, y + 70, x + w - 34, y + 70, flow.replay); - drawPill(x + 12, y + h - 68, w - 24, 28, displaySinkText(model.sink.routingRule || 'row image rule'), flow.index, c, /after|before|split|key|route/i.test(model.sink.routingRule || ''), 9); - drawPill(x + 12, y + h - 34, w - 24, 26, displaySinkText(model.sink.writePlan || 'rows queued'), flow.replay, c, /pk|row|queued|stage/i.test(model.sink.writePlan || ''), 8); - drawStagedRows(x + 130, y + 43, w - 156, 60, c); - } - - function drawStagedRows(x, y, w, h, c) { - const rows = stagedRowLabels(); - const visible = rows.slice(0, 3); - const gap = 8; - const cardW = Math.max(48, Math.min(72, (w - gap * Math.max(0, visible.length - 1)) / Math.max(1, visible.length))); - const total = cardW * visible.length + gap * Math.max(0, visible.length - 1); - const start = x + Math.max(0, (w - total) / 2); - visible.forEach((label, i) => { - const color = /^D/i.test(label) ? flow.delete : /^U/i.test(label) ? flow.index : flow.replay; - drawStagedRowCard(start + i * (cardW + gap), y + 8, cardW, Math.min(36, h - 12), label, color, c); - }); - if (!visible.length) { - drawEmptyRowSlot(x + 8, y + 18, w - 16, 14, c); - } - } - - function stagedRowLabels() { - const labels = []; - model.lanes.forEach(l => { - const short = shortRowLabel(l.currentOp); - if (short && short !== 'row' && !labels.includes(short)) labels.push(short); - }); - if (!labels.length) { - const plan = String(model.sink.writePlan || ''); - const matches = plan.match(/[IUD]?\s*pk\d+/gi) || plan.match(/pk\d+/gi) || []; - matches.forEach(m => labels.push(m.replace(/\s+/g, ' '))); - } - return labels; - } - - function drawStagedRowCard(x, y, w, h, label, color, c) { - ctx.fillStyle = alpha(color, .18); - ctx.strokeStyle = alpha(color, .50); - ctx.lineWidth = 1; - roundRect(x, y, w, h, 5, true, true); - drawRowGlyph(x + 6, y + 11, color, false, c); - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'left'; - fitText(label || 'row', x + 18, y + h / 2 + 5, w - 22, 'left'); - } - - function drawEmptyRowSlot(x, y, w, h, c) { - ctx.strokeStyle = alpha(c.border, .36); - ctx.lineWidth = 1; - ctx.setLineDash([3, 4]); - roundRect(x, y, w, h, 4, false, true); - ctx.setLineDash([]); - } - - function drawReplayRpcPanel(layout, c) { - const { x, y, w, h } = layout.rpc; - drawCdcCard(x, y, w, h, flow.tx, /RPC|batch|OK|pending|rows/i.test(model.sink.rpcBatch), c); - ctx.fillStyle = flow.tx; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'left'; - fitText('replay RPC', x + PANEL_TITLE_X, y + PANEL_TITLE_Y, w - PANEL_TITLE_X * 2, 'left'); - const sendLane = streamLane(layout, false); - const ackLane = streamLane(layout, true); - drawStreamLane(sendLane.a.x, sendLane.a.y, sendLane.b.x - sendLane.a.x, flow.replay, c, false); - drawStreamLane(ackLane.a.x, ackLane.a.y, ackLane.b.x - ackLane.a.x, flow.tx, c, true); - ctx.fillStyle = alpha(flow.replay, .78); - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'right'; - ctx.fillText('send', x + 88, sendLane.a.y + 3); - ctx.fillStyle = alpha(flow.tx, .78); - ctx.fillText('ack', x + 88, ackLane.a.y + 3); - } - - function streamLane(layout, reverse) { - const r = layout.rpc; - const y = reverse ? r.y + r.h - 56 : r.y + 48; - return { - a: { x: r.x + 92, y }, - b: { x: r.x + r.w - 18, y } - }; - } - - function drawStreamLane(x, y, w, color, c, reverse) { - ctx.strokeStyle = alpha(color, .34); - ctx.lineWidth = 2; - ctx.beginPath(); - ctx.moveTo(x, y); - ctx.lineTo(x + w, y); - ctx.stroke(); - for (let i = 0; i < 4; i++) { - const px = x + 8 + i * Math.max(16, (w - 24) / 3); - ctx.fillStyle = alpha(color, .18 + i * .025); - roundRect(px - 5, y - 4, 10, 8, 3, true, false); - } - arrowHead( - reverse ? { x: x + w - 10, y } : { x: x + w - 18, y }, - reverse ? { x: x + w - 30, y } : { x: x + w, y }, - alpha(color, .52) - ); - } - - function drawWriterFlowAnimations(layout, c) { - const incoming = activeTokenProgress(t => tokenDirection(t, 'cdc', 'sink')); - if (incoming && incoming.local > .34) { - const a = { x: layout.r.x + layout.r.w / 2, y: layout.r.y + 38 }; - const b = stagingRowCenter(layout); - drawMovingMiniRow(a, b, clamp((incoming.local - .34) / .66, 0, 1), incoming.token.label, flow.replay, c); - } - - const dispatch = activeTokenProgress(t => tokenDirection(t, 'sink', 'lane')); - if (dispatch) { - const a = stagingRowCenter(layout); - const b = { x: layout.r.x + layout.r.w - 8, y: a.y }; - drawMovingMiniRow(a, b, dispatch.local, dispatch.token.label, dispatch.token.color || flow.replay, c); - } - - const send = repeatingTokenProgress(t => tokenDirection(t, 'sink', 'runtime')) || - activeTokenProgress(t => tokenDirection(t, 'sink', 'runtime')); - if (send) drawStagedRowToStream(layout, send, c); - if (!send && writerStreamActive()) { - drawStagedRowToStream(layout, { - token: { label: stagedRowLabels()[0] || 'row', color: flow.replay }, - local: streamCycleProgress() - }, c); - } - - const recv = repeatingTokenProgress(t => tokenDirection(t, 'runtime', 'sink')) || - activeTokenProgress(t => tokenDirection(t, 'runtime', 'sink')); - if (recv) drawMovingRpcStreamChunk(layout, recv.local, true, c); - } - - function writerStreamActive() { - const text = `${model.sink.requestState} ${model.sink.rpcBatch}`; - if (/1 tx per request|ready$/i.test(text)) return false; - return /replay|dispatch|grouped|batch|tx\d+|rows|pending|OK/i.test(text); - } - - function activeTokenProgress(predicate) { - const st = currentStep(); - const p = stepProgress(); - for (const token of st.tokens) { - if (!predicate(token)) continue; - if (p < token.start || p > token.end) continue; - return { token, local: ease((p - token.start) / (token.end - token.start)) }; - } - return null; - } - - function repeatingTokenProgress(predicate) { - const st = currentStep(); - const token = st.tokens.find(predicate); - if (!token) return null; - const p = stepProgress(); - if (p < Math.max(0, token.start - .12) || p > Math.min(1, token.end + .12)) return null; - return { token, local: streamCycleProgress() }; - } - function tokenDirection(token, fromPrefix, toPrefix) { const from = tokenPrefixIndex(token, fromPrefix); const to = tokenPrefixIndex(token, toPrefix); @@ -2155,209 +1809,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return token.path.findIndex(part => typeof part === 'string' && part.startsWith(prefix)); } - function stagingRowCenter(layout) { - const s = layout.staging; - return { x: s.x + s.w * .58, y: s.y + Math.min(58, s.h - 42) }; - } - - function drawStagedRowToStream(layout, send, c) { - const rowStart = stagingRowCenter(layout); - const lane = streamLane(layout, false); - const rowEnd = { x: lane.a.x + 8, y: lane.a.y }; - drawInternalMotionPath(rowStart, rowEnd, send.token.color || flow.replay, c); - drawActiveStreamGlow(layout, false, c); - - const mergeT = clamp(send.local / .62, 0, 1); - if (send.local < .82) { - drawMovingMiniRow(rowStart, rowEnd, mergeT, send.token.label, send.token.color || flow.replay, c, 48, 18); - } - - const streamT = clamp((send.local - .16) / .84, 0, 1); - drawMovingRpcStreamChunk(layout, streamT, false, c, 'row'); - if (send.local > .36) drawMovingRpcStreamChunk(layout, (streamT + .36) % 1, false, c, 'row'); - if (send.local > .58) drawMovingRpcStreamChunk(layout, (streamT + .68) % 1, false, c, 'row'); - } - - function drawActiveStreamGlow(layout, reverse, c) { - const lane = streamLane(layout, reverse); - const color = reverse ? flow.tx : flow.replay; - ctx.save(); - ctx.strokeStyle = alpha(color, .72); - ctx.lineWidth = 4; - ctx.lineCap = 'round'; - ctx.beginPath(); - ctx.moveTo(lane.a.x, lane.a.y); - ctx.lineTo(lane.b.x, lane.b.y); - ctx.stroke(); - ctx.strokeStyle = alpha('#ffffff', .68); - ctx.lineWidth = 1.2; - ctx.beginPath(); - ctx.moveTo(lane.a.x, lane.a.y); - ctx.lineTo(lane.b.x, lane.b.y); - ctx.stroke(); - ctx.restore(); - } - - function drawInternalMotionPath(a, b, color, c) { - ctx.save(); - ctx.strokeStyle = alpha(color, .32); - ctx.lineWidth = 1.6; - ctx.setLineDash([4, 5]); - ctx.beginPath(); - ctx.moveTo(a.x, a.y + 3); - ctx.lineTo(a.x, b.y); - ctx.lineTo(b.x, b.y); - ctx.stroke(); - ctx.setLineDash([]); - arrowHead({ x: b.x - 12, y: b.y }, b, alpha(color, .50)); - ctx.restore(); - } - - function drawMovingMiniRow(a, b, t, label, color, c, ww, hh) { - const x = lerp(a.x, b.x, clamp(t, 0, 1)); - const y = lerp(a.y, b.y, clamp(t, 0, 1)); - const w = ww || 48; - const h = hh || 22; - ctx.save(); - ctx.shadowColor = alpha(color, .24); - ctx.shadowBlur = 8; - ctx.fillStyle = color; - roundRect(x - w / 2, y - h / 2, w, h, 5, true, false); - ctx.shadowColor = 'transparent'; - ctx.strokeStyle = alpha('#ffffff', .80); - ctx.lineWidth = 1; - roundRect(x - w / 2, y - h / 2, w, h, 5, false, true); - ctx.fillStyle = '#ffffff'; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText(shortRowLabel(label), x - w / 2 + 4, y + 3, w - 8, 'center'); - ctx.restore(); - } - - function shortRowLabel(label) { - const text = String(label || 'row'); - const op = text.match(/\b([IUD]):?pk(\d+)/i) || text.match(/\b([IUD]).*?(pk\d+)/i); - if (op) return /^\d+$/.test(op[2]) ? `${op[1].toUpperCase()} pk${op[2]}` : `${op[1].toUpperCase()} ${op[2]}`; - const pk = text.match(/pk\d+/i); - if (pk) return pk[0]; - return 'row'; - } - - function drawMovingRpcStreamChunk(layout, t, reverse, c, labelOverride) { - const lane = streamLane(layout, reverse); - const a = lane.a; - const b = lane.b; - const p = reverse ? 1 - clamp(t, 0, 1) : clamp(t, 0, 1); - const x = lerp(a.x, b.x, p); - const y = lerp(a.y, b.y, p); - const label = labelOverride || (reverse ? 'ack' : 'chunk'); - ctx.save(); - ctx.shadowColor = alpha(reverse ? flow.replay : flow.tx, .26); - ctx.shadowBlur = 9; - ctx.fillStyle = reverse ? flow.replay : flow.tx; - roundRect(x - 26, y - 11, 52, 22, 8, true, false); - ctx.shadowColor = 'transparent'; - ctx.strokeStyle = alpha('#ffffff', .82); - ctx.lineWidth = 1; - roundRect(x - 26, y - 11, 52, 22, 8, false, true); - ctx.fillStyle = '#ffffff'; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText(label, x - 22, y + 4, 44, 'center'); - ctx.restore(); - } - - function drawRpcEndpoint(x, y, w, h, color, c) { - ctx.fillStyle = alpha(color, .14); - ctx.strokeStyle = alpha(color, .44); - ctx.lineWidth = 1; - roundRect(x, y, w, h, 7, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - fitText('Retina OK', x + 4, y + 18, w - 8, 'center'); - } - - function drawWriterRequestCard(x, y, w, h, c) { - drawCdcCard(x, y, w, h, flow.replay, /replay/i.test(model.sink.requestMode), c); - drawRecordSheet(x + 14, y + 10, 28, 28, flow.replay, c); - drawRecordTypeCard(x + 54, y + 12, 86, 22, 'replay RPC', flow.replay, c); - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'left'; - fitText(displaySinkText(model.sink.requestState), x + 54, y + 43, w - 66, 'left'); - } - - function drawWriterRpcCard(x, y, w, h, c) { - drawCdcCard(x, y, w, h, flow.tx, /batch|RPC|OK|pending|rows/i.test(model.sink.rpcBatch), c); - drawRpcBundle(x + 12, y + 9, flow.tx, c); - const txCount = rpcTxCount(); - for (let i = 0; i < txCount; i++) { - drawTxDot(x + 64 + i * 24, y + 19, flow.tx, c); - } - drawMiniArrow(x + w - 44, y + 24, x + w - 16, y + 24, flow.replay); - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'left'; - fitText(displaySinkText(model.sink.rpcBatch), x + 12, y + 41, w - 24, 'left'); - } - - function rpcTxCount() { - const text = `${model.sink.rpcBatch} ${model.sink.requestState}`; - if (/3 tx|tx43 \+ tx44 \+ tx45|3 tx rows/i.test(text)) return 3; - if (/tx\d+.*tx\d+/i.test(text)) return 2; - return 1; - } - - function drawRecordSheet(x, y, w, h, color, c) { - ctx.fillStyle = alpha(color, .12); - ctx.strokeStyle = alpha(color, .48); - ctx.lineWidth = 1; - roundRect(x, y, w, h, 5, true, true); - drawFoldCorner(x + w - 9, y + 3, color, c); - ctx.strokeStyle = alpha(color, .38); - ctx.beginPath(); - for (let i = 0; i < 3; i++) { - ctx.moveTo(x + 7, y + 10 + i * 6); - ctx.lineTo(x + w - 8, y + 10 + i * 6); - } - ctx.stroke(); - } - - function drawRouteHub(x, y, color, c) { - ctx.fillStyle = alpha(color, .18); - ctx.strokeStyle = alpha(color, .56); - ctx.lineWidth = 1.2; - ctx.beginPath(); - ctx.arc(x, y, 12, 0, Math.PI * 2); - ctx.fill(); - ctx.stroke(); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - ctx.fillText('pk', x, y + 3); - } - - function drawRpcBundle(x, y, color, c) { - ctx.fillStyle = alpha(color, .14); - ctx.strokeStyle = alpha(color, .42); - roundRect(x, y, 40, 22, 6, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - ctx.fillText('RPC', x + 20, y + 17); - } - - function drawTxDot(cx, cy, color, c) { - ctx.fillStyle = alpha(color, .16); - ctx.strokeStyle = alpha(color, .52); - ctx.lineWidth = 1; - ctx.beginPath(); - ctx.arc(cx, cy, 8, 0, Math.PI * 2); - ctx.fill(); - ctx.stroke(); - } - function txWindowLayout(r) { const y = r.y + MODULE_TITLE_Y + MODULE_CONTENT_GAP; const h = r.y + r.h - y - MODULE_CONTENT_PAD; @@ -2622,15 +2073,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return String(key || ''); } - function compactLocation(location) { - return String(location || '') - .replace('.pxl', '') - .replace('fileId', 'f') - .replace('file-', 'f') - .replace('/rg', '/r') - .replace('/off', '/o'); - } - function drawReplayers(c) { drawReplayerInternals(c); } @@ -2908,10 +2350,15 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const eligibleDeletes = new Set(); const newlyFolded = new Set(); - lane.deleteTs.forEach(item => { + const bitmapDeleteTs = lane.bitmapDeleteTs || lane.deleteTs; + bitmapDeleteTs.forEach(item => { const offset = deleteOffsetValue(item); if (!Number.isFinite(offset) || offset < 0 || offset >= len) return; pendingDeletes.add(offset); + }); + lane.deleteTs.forEach(item => { + const offset = deleteOffsetValue(item); + if (!Number.isFinite(offset) || offset < 0 || offset >= len) return; if (lane.safeTs != null && deleteTsValue(item) <= lane.safeTs) { eligibleDeletes.add(offset); } @@ -2977,7 +2424,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

const foldingDeleteTs = st.anim === 'foldDeleteTs' && stepProgress() < .82; const items = foldingDeleteTs ? (lane.prevDeleteTs || lane.deleteTs) : hideAppendingItem ? lane.deleteTs.slice(0, -1) : lane.deleteTs; const visible = items.slice(0, columns * rows); - const itemW = Math.min(72, Math.floor((w - 18 * (columns - 1)) / columns)); + const itemW = Math.min(90, Math.floor((w - 6 * (columns - 1)) / columns)); const gapX = columns > 1 ? (w - itemW * columns) / (columns - 1) : 0; const itemH = 24; const rowGap = 10; @@ -3040,7 +2487,7 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function deleteTsItemLayout(w) { const columns = 4; const rows = 2; - const itemW = Math.min(72, Math.floor((w - 18 * (columns - 1)) / columns)); + const itemW = Math.min(90, Math.floor((w - 6 * (columns - 1)) / columns)); const gapX = columns > 1 ? (w - itemW * columns) / (columns - 1) : 0; const itemH = 24; const rowGap = 10; @@ -3627,190 +3074,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

return { x, w, scheduler, splitBoxes, gateBox, queueBox, opsBox }; } - function drawQuery(c) { - const r = rects.query; - const q = model.query; - const splits = q.splits || []; - const gate = q.gate || {}; - const { x, w, scheduler, splitBoxes, gateBox, queueBox, opsBox } = queryLayout(r, splits); - drawBoundary(r, c, 'query'); - - drawPill(x, r.y + 64, 86, 30, q.id, flow.query, c, q.active, 11); - drawPill(x + 96, r.y + 64, w - 96, 30, `transTs=${q.transTs}`, flow.tx, c, q.active, 11); - - splitBoxes.forEach(box => { - drawQueryInternalLink([ - [scheduler.x + scheduler.w / 2, scheduler.y + scheduler.h], - [box.portX, scheduler.y + scheduler.h + 12], - [box.portX, box.y] - ], querySplitColor(box.split.kind), c, .28); - drawQueryInternalLink([ - [box.portX, box.y + box.h], - [box.portX, gateBox.y - 12], - [gateBox.x + gateBox.w / 2, gateBox.y] - ], querySplitColor(box.split.kind), c, .24); - }); - drawQueryInternalLink([[gateBox.x + gateBox.w / 2, gateBox.y + gateBox.h], [queueBox.x + queueBox.w / 2, queueBox.y]], flow.visibility, c, .34); - drawQueryInternalLink([[queueBox.x + queueBox.w / 2, queueBox.y + queueBox.h], [opsBox.x + opsBox.w / 2, opsBox.y]], flow.query, c, .34); - - smallBox(scheduler.x, scheduler.y, scheduler.w, scheduler.h, 'Split Scheduler', flow.query, c, state.detailFocus === 'query'); - drawPill(scheduler.x + 12, scheduler.y + 32, scheduler.w - 24, 28, q.parallelism || 'parallel split execution', flow.query, c, q.active, 10); - drawSchedulerGlyph(scheduler.x + scheduler.w - 42, scheduler.y + 21, flow.query, c); - - splitBoxes.forEach((box, i) => drawQuerySplitCard(box, c, i)); - drawVisibilityGateBox(gateBox, gate, c); - drawVisibleQueueBox(queueBox, q, c); - drawOperatorPipelineBox(opsBox, q, c); - } - - function querySplitColor(kind) { - if (kind === 'active') return flow.replay; - if (kind === 'flushed') return flow.storage; - return flow.query; - } - - function drawQueryInternalLink(points, color, c, opacity) { - const pts = points.map(arrPoint); - ctx.save(); - ctx.strokeStyle = alpha(color, opacity); - ctx.lineWidth = 1.6; - ctx.lineCap = 'round'; - ctx.setLineDash([5, 6]); - drawPolyline(pts); - ctx.setLineDash([]); - arrowHead(pts[pts.length - 2], pts[pts.length - 1], alpha(color, opacity + .18)); - ctx.restore(); - } - - function drawSchedulerGlyph(cx, cy, color, c) { - ctx.save(); - ctx.fillStyle = alpha(color, .20); - ctx.strokeStyle = alpha(color, .55); - ctx.lineWidth = 1.2; - ctx.beginPath(); ctx.arc(cx, cy, 10, 0, Math.PI * 2); ctx.fill(); ctx.stroke(); - for (let i = 0; i < 3; i++) { - const y = cy + 18 + i * 6; - ctx.strokeStyle = alpha(color, .42); - ctx.beginPath(); ctx.moveTo(cx - 18, y); ctx.lineTo(cx + 18, y); ctx.stroke(); - ctx.fillStyle = i === 1 ? color : alpha(c.text, .55); - ctx.beginPath(); ctx.arc(cx - 18 + i * 18, y, 2.4, 0, Math.PI * 2); ctx.fill(); - } - ctx.restore(); - } - - function drawQuerySplitCard(box, c, index) { - const split = box.split; - const color = querySplitColor(split.kind); - const active = split.status !== 'waiting' || state.detailFocus === 'query' || state.activeChannels.includes('flush'); - ctx.fillStyle = active ? alpha(color, c.dark ? .16 : .10) : alpha(c.panel2, .80); - ctx.strokeStyle = active ? alpha(color, .58) : c.borderSoft; - ctx.lineWidth = active ? 1.5 : 1; - roundRect(box.x, box.y, box.w, box.h, 9, true, true); - drawQuerySplitIcon(split.kind, box.x + 12, box.y + 16, color, c); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'left'; - fitText(`${split.id} · ${split.title}`, box.x + 52, box.y + 25, box.w - 122, 'left'); - ctx.fillStyle = c.muted; - ctx.font = typeFont(TYPE.text, 0); - fitText(split.source, box.x + 52, box.y + 46, box.w - 122, 'left'); - drawPill(box.x + box.w - 76, box.y + 14, 64, 26, split.status, color, c, active, 9); - drawProgressBar(box.x + 52, box.y + 60, box.w - 122, 8, split.progress || 0, color, c); - drawBatchDots(box.x + box.w - 62, box.y + 58, Math.max(split.batches || 0, split.visible || 0), color, c); - ctx.fillStyle = alpha(c.muted, .72); - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'center'; - ctx.fillText(`task ${index + 1}`, box.x + 25, box.y + box.h - 10); - } - - function drawQuerySplitIcon(kind, x, y, color, c) { - ctx.save(); - ctx.strokeStyle = alpha(color, .55); - ctx.fillStyle = alpha(color, .18); - ctx.lineWidth = 1; - if (kind === 'regular') { - for (let i = 0; i < 3; i++) roundRect(x + i * 5, y + i * 4, 24, 20, 4, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'center'; - ctx.fillText('RG', x + 17, y + 17); - } else if (kind === 'active') { - roundRect(x, y, 34, 28, 5, true, true); - ctx.strokeStyle = alpha(color, .42); - for (let i = 1; i < 3; i++) { ctx.beginPath(); ctx.moveTo(x + i * 11, y + 4); ctx.lineTo(x + i * 11, y + 24); ctx.stroke(); } - for (let i = 1; i < 2; i++) { ctx.beginPath(); ctx.moveTo(x + 4, y + i * 14); ctx.lineTo(x + 30, y + i * 14); ctx.stroke(); } - } else { - roundRect(x + 2, y + 4, 30, 22, 6, true, true); - ctx.fillStyle = alpha(color, .28); - roundRect(x + 7, y, 20, 8, 4, true, false); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.text, 900); - ctx.textAlign = 'center'; - ctx.fillText('OBJ', x + 17, y + 20); - } - ctx.restore(); - } - - function drawProgressBar(x, y, w, h, progress, color, c) { - const p = clamp(progress, 0, 1); - ctx.fillStyle = alpha(c.border, .38); - roundRect(x, y, w, h, h / 2, true, false); - ctx.fillStyle = alpha(color, .72); - roundRect(x, y, Math.max(h, w * p), h, h / 2, true, false); - } - - function drawBatchDots(x, y, count, color, c) { - const n = Math.max(3, Math.min(5, count || 0)); - for (let i = 0; i < n; i++) { - const filled = i < (count || 0); - ctx.fillStyle = filled ? alpha(color, .62) : alpha(c.border, .42); - roundRect(x + i * 9, y, 7, 10, 2, true, false); - } - } - - function drawVisibilityGateBox(box, gate, c) { - smallBox(box.x, box.y, box.w, box.h, 'VFoR Visibility Gate', flow.visibility, c, state.detailFocus === 'visibility'); - const fx = box.x + 18, fy = box.y + 34; - ctx.fillStyle = alpha(flow.visibility, c.dark ? .22 : .14); - ctx.strokeStyle = alpha(flow.visibility, .58); - ctx.lineWidth = 1.2; - ctx.beginPath(); - ctx.moveTo(fx, fy); - ctx.lineTo(fx + 60, fy); - ctx.lineTo(fx + 42, fy + 36); - ctx.lineTo(fx + 42, fy + 50); - ctx.lineTo(fx + 18, fy + 50); - ctx.lineTo(fx + 18, fy + 36); - ctx.closePath(); - ctx.fill(); ctx.stroke(); - drawBits(gate.bitmap || '0010100110', box.x + 92, box.y + 36, c, flow.visibility); - drawPill(box.x + 88, box.y + 62, box.w - 104, 28, gate.rule || 'visibility rule', flow.visibility, c, true, 9); - metricRows(box.x + 88, box.y + 91, box.w - 104, [ - ['input', `${gate.inputBatches || 0} batches`], - ['output', `${gate.visibleBatches || 0} visible`] - ], c); - } - - function drawVisibleQueueBox(box, q, c) { - smallBox(box.x, box.y, box.w, box.h, 'Visible Batch Queue', flow.query, c, state.detailFocus === 'query'); - drawBatchDots(box.x + 14, box.y + 34, q.gate ? q.gate.visibleBatches : 0, flow.query, c); - drawPill(box.x + 72, box.y + 33, box.w - 86, 28, q.queueState || 'waiting', flow.query, c, (q.gate && q.gate.visibleBatches > 0), 10); - } - - function drawOperatorPipelineBox(box, q, c) { - smallBox(box.x, box.y, box.w, box.h, 'Downstream Operators', flow.query, c, state.detailFocus === 'query'); - const labels = ['Scan', 'Filter', 'Project', 'Agg']; - const bw = 62; - labels.forEach((label, i) => { - const bx = box.x + 12 + i * 68; - drawPill(bx, box.y + 34, bw, 28, label, flow.query, c, q.returnedRows > 0 || i === 0, 9); - }); - ctx.fillStyle = c.muted; - ctx.font = typeFont(TYPE.text, 0); - ctx.textAlign = 'left'; - fitText(`${q.operatorInput} · rows ${q.returnedRows.toLocaleString()}`, box.x + 12, box.y + 76, box.w - 24, 'left'); - } - function drawTokens(c) { const st = currentStep(); const p = stepProgress(); @@ -3897,54 +3160,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

ctx.restore(); } - function drawGroup(r, c, color, key) { - const highlighted = isHighlighted(key); - ctx.fillStyle = highlighted ? alpha(color, c.dark ? .12 : .08) : alpha(color, c.dark ? .05 : .035); - ctx.strokeStyle = highlighted ? alpha(color, .58) : alpha(color, .24); - ctx.lineWidth = highlighted ? 1.7 : 1; - roundRect(r.x, r.y, r.w, r.h, 14, true, true); - } - - function smallBox(x, y, w, h, title, color, c, active) { - ctx.fillStyle = active ? alpha(color, c.dark ? .18 : .11) : alpha(c.panel2, .78); - ctx.strokeStyle = active ? alpha(color, .62) : c.borderSoft; - ctx.lineWidth = active ? 1.5 : 1; - roundRect(x, y, w, h, 9, true, true); - ctx.fillStyle = color; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'left'; - fitText(title, x + PANEL_TITLE_X, y + PANEL_TITLE_Y, w - PANEL_TITLE_X * 2, 'left'); - } - - function drawMiniStack(x, y, w, rows, c) { - rows.forEach((row, i) => { - const yy = y + i * 40; - ctx.fillStyle = alpha(row[2], c.dark ? .13 : .08); - ctx.strokeStyle = alpha(row[2], .28); - roundRect(x, yy, w, 32, 8, true, true); - ctx.fillStyle = row[2]; - ctx.font = typeFont(TYPE.subTitle, 900); - ctx.textAlign = 'left'; - ctx.fillText(row[0], x + 10, yy + 17); - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 0); - fitText(row[1], x + 10, yy + 32, w - 20, 'left'); - }); - } - - function metricRows(x, y, w, rows, c) { - rows.forEach((row, i) => { - const yy = y + i * 22; - ctx.fillStyle = c.muted; - ctx.font = typeFont(TYPE.text, 0); - ctx.textAlign = 'left'; - fitText(row[0], x, yy, w * .42, 'left'); - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 900); - fitText(row[1], x + w * .43, yy, w * .57, 'right'); - }); - } - function drawPill(x, y, w, h, label, color, c, active, fs) { ctx.fillStyle = alpha(color, active ? .22 : .10); ctx.strokeStyle = alpha(color, active ? .60 : .26); @@ -3957,73 +3172,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

fitText(label, x + 6, y + h / 2 + size * .36, w - 12, 'center'); } - function drawBlockPile(x, y, w, h, count, active, label, color, c) { - ctx.fillStyle = alpha(color, active ? .22 : .10); - ctx.strokeStyle = alpha(color, active ? .60 : .28); - roundRect(x, y, w, h, 6, true, true); - for (let i = 0; i < Math.min(count, 3); i++) { - ctx.fillStyle = alpha(color, .22 + i * .06); - roundRect(x + 7, y + 6 + i * 5, w - 14, 4, 2, true, false); - } - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'center'; - ctx.fillText(label, x + w / 2, y + h - 8); - } - - function immCellW(queueW, n) { - return Math.min(72, Math.floor((queueW - 56 - Math.max(0, n - 1) * 5) / Math.max(n, 1))); - } - - function drawImmFifoQueue(x, y, w, h, count, active, c) { - const color = flow.gc; - ctx.fillStyle = alpha(color, active ? .14 : .06); - ctx.strokeStyle = alpha(color, active ? .52 : .24); - ctx.lineWidth = active ? 1.5 : 1; - roundRect(x, y, w, h, 6, true, true); - ctx.font = typeFont(TYPE.text, 800); - ctx.fillStyle = color; - ctx.textAlign = 'left'; - ctx.fillText('imm', x + 7, y + 12); - ctx.fillStyle = alpha(color, active ? .70 : .42); - ctx.textAlign = 'right'; - ctx.fillText('flush →', x + w - 7, y + 12); - if (count === 0) { - ctx.fillStyle = c.muted; - ctx.font = typeFont(TYPE.text, 0); - ctx.textAlign = 'center'; - ctx.fillText('empty', x + w / 2, y + h / 2 + 4); - return; - } - const maxVisible = 4; - const nVisible = Math.min(count, maxVisible); - const cw = immCellW(w, nVisible); - const cellH = h - 6; - const cellY = y + 3; - for (let i = 0; i < nVisible; i++) { - const bx = x + 28 + i * (cw + 5); - const ageAlpha = active ? (.14 + i * .05) : (.07 + i * .025); - ctx.fillStyle = alpha(color, ageAlpha); - ctx.strokeStyle = alpha(color, active ? (.46 + i * .04) : (.22 + i * .02)); - ctx.lineWidth = 1; - roundRect(bx, cellY, cw, cellH, 4, true, true); - const isOverflow = count > maxVisible && i === maxVisible - 1; - const label = isOverflow ? `+${count - maxVisible + 1}` : `blk${count - i}`; - ctx.fillStyle = c.text; - ctx.font = typeFont(TYPE.text, 800); - ctx.textAlign = 'center'; - fitText(label, bx + 2, cellY + cellH / 2 + 4, cw - 4, 'center'); - } - } - - function drawBits(bits, x, y, c, color) { - for (let i = 0; i < bits.length; i++) { - const on = bits[i] === '1'; - ctx.fillStyle = on ? alpha(color, .54) : alpha(c.border, .42); - roundRect(x + i * 11, y, 9, 14, 3, true, false); - } - } - function drawFlow(names, color, c, label, opacity, width) { const pts = names.map(point); ctx.save(); @@ -4244,23 +3392,6 @@

Retina: Transactional CDC Replay for Real-Time Lakehouse Analytics

function isHighlighted(key) { return currentStep().highlights.includes(key); } - function wrapText(text, x, y, maxWidth, lineHeight) { - const words = text.split(' '); - let line = ''; - let yy = y; - for (const word of words) { - const test = line ? line + ' ' + word : word; - if (ctx.measureText(test).width > maxWidth && line) { - ctx.fillText(line, x, yy); - line = word; - yy += lineHeight; - } else { - line = test; - } - } - if (line) ctx.fillText(line, x, yy); - } - function fitText(text, x, y, maxWidth, align) { if (!Number.isFinite(maxWidth) || maxWidth <= 0) return; let str = String(text); From abecbada2966182f98d4452f678f9691fe620499 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Thu, 9 Jul 2026 00:51:28 +0800 Subject: [PATCH 07/11] fix --- retina.html | 244 +++------------------------------------------------- 1 file changed, 10 insertions(+), 234 deletions(-) diff --git a/retina.html b/retina.html index b81bcc906..30c93aba3 100644 --- a/retina.html +++ b/retina.html @@ -112,7 +112,6 @@ } button:focus-visible { outline: 3px solid color-mix(in srgb, var(--primary) 24%, transparent); outline-offset: 2px; } - button.secondary, button.scene-btn { color: var(--text); background: color-mix(in srgb, var(--surface-elevated) 94%, transparent); @@ -120,7 +119,6 @@ box-shadow: none; } - button.secondary:hover, button.scene-btn:hover { background: var(--surface); border-color: color-mix(in srgb, var(--primary) 30%, var(--border)); @@ -173,81 +171,6 @@ gap: var(--gap); } - .hero { - display: flex; - align-items: center; - justify-content: space-between; - gap: clamp(10px, 1vw, 16px); - margin: 0; - width: 100%; - padding: clamp(7px, .7vw, 10px) clamp(9px, .9vw, 12px); - background: color-mix(in srgb, var(--surface-elevated) 42%, transparent); - border: 1px solid color-mix(in srgb, var(--border-soft) 58%, transparent); - border-radius: 14px; - backdrop-filter: blur(10px); - } - - .title { - min-width: 0; - } - - .title h1 { - margin: 0; - font-size: 18px; - line-height: 1.16; - letter-spacing: -.02em; - } - - .title p { - margin: 6px 0 0; - color: var(--muted); - font-size: 12px; - } - - .hero-badges { - display: flex; - flex-wrap: wrap; - justify-content: flex-end; - gap: 8px; - color: var(--muted); - font-size: 12px; - white-space: nowrap; - } - - .badge { - border: 1px solid var(--border); - border-radius: 999px; - background: color-mix(in srgb, var(--panel) 82%, transparent); - padding: 6px 10px; - } - - .view-actions { - display: inline-flex; - align-items: center; - justify-content: flex-end; - gap: clamp(4px, .45vw, 6px); - padding: 4px; - background: color-mix(in srgb, var(--surface) 72%, transparent); - border: 1px solid color-mix(in srgb, var(--border-soft) 88%, transparent); - border-radius: 14px; - box-shadow: none; - backdrop-filter: blur(10px); - flex: 0 0 auto; - } - - .view-actions .icon-btn { - background: transparent; - color: var(--text); - border-color: transparent; - box-shadow: none; - } - - .view-actions .icon-btn:hover { - background: var(--primary-soft); - color: var(--primary); - border-color: color-mix(in srgb, var(--primary) 26%, transparent); - } - .demo-layout { width: 100%; max-width: none; @@ -455,27 +378,11 @@ .control-rail .icon-btn, .control-rail .scene-btn { width: var(--rail-btn); min-width: var(--rail-btn); justify-content: center; padding: 0; } .control-rail button::after { display: none; } } - - @media (max-width: 700px) {} - - /* Fullscreen layout is disabled while fullscreen controls are commented out. */
- -