Replace per-save-point array snapshots with an undo trail

Sources/_StringProcessing/Engine/Backtracking.swift

@@ -11,38 +11,42 @@
 
 extension Processor {
   struct SavePoint {
+    /// The current position in the instruction list.
     var pc: InstructionAddress
+
+    /// The current position in the input, when this save point represents a single position.
+    ///
+    /// `pos` is `nil` whenever `quantifiedRange` has a value, and can also be `nil`
+    /// when the save point is only for backtracking to a previous instruction address.
     var pos: Position?
 
-    // Quantifiers may store a range of positions to restore to
+    /// The current range in the input, when this save point represents a range of positions.
     var quantifiedRange: Range<Position>?
 
     // FIXME: refactor, for now this field is only used for quantifier save
     //        points. We should try to separate out the concerns better.
+    /// Indicates whether this save point has scalar or grapheme semantics.
     var isScalarSemantics: Bool
 
-    // FIXME: Save minimal info (e.g. stack position and
-    // perhaps current start)
-    var captureEnds: [_StoredCapture]
+    // These properties store indices into the mutable `Processor.Registers`
+    // LoggingArray logs. On backtrack, each log is unwound down to the
+    // point saved here.
 
-    // The int registers store values that can be relevant to
-    // backtracking, such as the number of trips in a quantification.
-    var intRegisters: [Int]
-    // Same with position registers
-    var posRegisters: [Input.Index]
+    /// The length of the log of the `captures` register when this save point was created,
+    /// for backtracking on failure.
+    var captureLogEnd: Int
+    /// The length of the log of the `ints` register when this save point was created,
+    /// for backtracking on failure.
+    var intLogEnd: Int
+    /// The length of the log of the `positions` register when this save point was created,
+    /// for backtracking on failure.
+    var positionLogEnd: Int
+    /// The length of the log of the `values` register when this save point was created,
+    /// for backtracking on failure.
+    var valueLogEnd: Int
 
-    var destructure: (
-      pc: InstructionAddress,
-      pos: Position?,
-      captureEnds: [_StoredCapture],
-      intRegisters: [Int],
-      PositionRegister: [Input.Index]
-    ) {
-      return (pc, pos, captureEnds, intRegisters, posRegisters)
-    }
-
-    // Whether this save point is quantified, meaning it has a range of
-    // possible positions to explore.
+    /// Whether this save point is quantified, meaning it has a range of
+    /// possible positions to explore.
     var isQuantified: Bool {
       quantifiedRange != nil
     }
@@ -77,9 +81,10 @@ extension Processor {
       pos: currentPosition,
       quantifiedRange: nil,
       isScalarSemantics: false,
-      captureEnds: storedCaptures,
-      intRegisters: registers.ints,
-      posRegisters: registers.positions)
+      captureLogEnd: registers.storedCaptures.logCount,
+      intLogEnd: registers.ints.logCount,
+      positionLogEnd: registers.positions.logCount,
+      valueLogEnd: registers.values.logCount)
   }
 
   func makeAddressOnlySavePoint(
@@ -90,9 +95,10 @@ extension Processor {
       pos: nil,
       quantifiedRange: nil,
       isScalarSemantics: false,
-      captureEnds: storedCaptures,
-      intRegisters: registers.ints,
-      posRegisters: registers.positions)
+      captureLogEnd: registers.storedCaptures.logCount,
+      intLogEnd: registers.ints.logCount,
+      positionLogEnd: registers.positions.logCount,
+      valueLogEnd: registers.values.logCount)
   }
 
   func makeQuantifiedSavePoint(
@@ -104,9 +110,10 @@ extension Processor {
       pos: nil,
       quantifiedRange: range,
       isScalarSemantics: isScalarSemantics,
-      captureEnds: storedCaptures,
-      intRegisters: registers.ints,
-      posRegisters: registers.positions)
+      captureLogEnd: registers.storedCaptures.logCount,
+      intLogEnd: registers.ints.logCount,
+      positionLogEnd: registers.positions.logCount,
+      valueLogEnd: registers.values.logCount)
   }
 }
 

Sources/_StringProcessing/Engine/MEBuilder.swift

@@ -428,12 +428,10 @@ extension MEProgram.Builder {
       matcherFunctions: matcherFunctions,
       numInts: nextIntRegister.rawValue,
       numValues: nextValueRegister.rawValue,
-      numPositions: nextPositionRegister.rawValue
+      numPositions: nextPositionRegister.rawValue,
+      numCaptures: nextCaptureRegister.rawValue
     )
 
-    let storedCaps = Array(
-      repeating: Processor._StoredCapture(), count: nextCaptureRegister.rawValue)
-
     let meProgram = MEProgram(
       instructions: InstructionList(instructions),
       wholeMatchValueRegister: wholeMatchValue,
@@ -443,8 +441,7 @@ extension MEProgram.Builder {
       referencedCaptureOffsets: referencedCaptureOffsets,
       initialOptions: initialOptions,
       canOnlyMatchAtStart: canOnlyMatchAtStart,
-      registers: regs,
-      storedCaptures: storedCaps)
+      registers: regs)
     return meProgram
   }
 

Sources/_StringProcessing/Engine/MECapture.swift

@@ -11,25 +11,6 @@
 
 internal import _RegexParser
 
-/*
-
- TODO: Specialized data structure for all captures:
-
- - We want to be able to refer to COW prefixes for which
-   simple appends do not invalidate
- - We want a compact save-point representation
-
- TODO: Conjectures:
-
- - We should be able to remove the entire capture history,
-   lazily recomputing it on-request from the initial stored
-   save point
- - We should be able to keep these flat and simple, lazily
-   constructing structured types on-request
-
- */
-
-
 extension Processor {
   struct _StoredCapture {
     var range: Range<Position>? = nil

Sources/_StringProcessing/Engine/MEProgram.swift

@@ -36,15 +36,25 @@ struct MEProgram {
   // processors can be spun up quicker (useful for running same regex
   // over many, many smaller inputs).
   var registers: Processor.Registers
-  var storedCaptures: [Processor._StoredCapture]
-
 }
 
 extension MEProgram: CustomStringConvertible {
   var description: String {
     // TODO: Re-instate better pretty-printing functionality
 
     var result = """
+    Capture list: \(captureList)
+    Instructions: \(instructions.count)
+    Register counts:
+      ints: \(registers.ints.count)
+      positions: \(registers.positions.count)
+      values: \(registers.values.count)
+      elements: \(registers.elements.count)
+      bitsets: \(registers.bitsets.count)
+      consumeFunctions: \(registers.consumeFunctions.count)
+      transformFunctions: \(registers.transformFunctions.count)
+      matcherFunctions: \(registers.matcherFunctions.count)
+    canOnlyMatchAtStart: \(canOnlyMatchAtStart)
 
     """
     // TODO: Extract into formatting code

Sources/_StringProcessing/Engine/Processor.swift

@@ -82,8 +82,6 @@ struct Processor {
 
   var savePoints: [SavePoint] = []
 
-  var storedCaptures: Array<_StoredCapture>
-
   var state: State = .inProgress
 
   var failureReason: Error? = nil
@@ -125,11 +123,10 @@ extension Processor {
     // Initialize registers from stored starting state
     self.registers = program.registers
 
-    self.storedCaptures = program.storedCaptures
-
     _checkInvariants()
   }
 
+  @inline(always)
   mutating func reset(
     currentPosition: Position,
     searchBounds: Range<Position>
@@ -145,10 +142,6 @@ extension Processor {
       self.savePoints.removeAll(keepingCapacity: true)
     }
 
-    for idx in storedCaptures.indices {
-      storedCaptures[idx] = .init()
-    }
-
     self.state = .inProgress
     self.failureReason = nil
 
@@ -162,7 +155,7 @@ extension Processor {
     _checkInvariants()
     guard self.controller == Controller(pc: 0),
           self.savePoints.isEmpty,
-          self.storedCaptures.allSatisfy({ $0.range == nil }),
+          !self.registers.isDirty,
           self.state == .inProgress,
           self.failureReason == nil
     else {
@@ -377,40 +370,58 @@ extension Processor {
       state = .fail
       return
     }
-    let (pc, pos, capEnds, intRegisters, posRegisters): (
-      pc: InstructionAddress,
-      pos: Position?,
-      captureEnds: [_StoredCapture],
-      intRegisters: [Int],
-      PositionRegister: [Input.Index]
-    )
 
     let idx = savePoints.index(before: savePoints.endIndex)
 
     // If we have a quantifier save point, move the next range position into
     // pos instead of removing it
+    let sp: SavePoint
     if savePoints[idx].isQuantified {
       savePoints[idx].takePositionFromQuantifiedRange(input)
-      (pc, pos, capEnds, intRegisters, posRegisters) = savePoints[idx].destructure
+      sp = savePoints[idx]
     } else {
-      (pc, pos, capEnds, intRegisters, posRegisters) = savePoints.removeLast().destructure
+      sp = savePoints.removeLast()
     }
 
-    assert(capEnds.count == storedCaptures.count)
+    controller.pc = sp.pc
+    currentPosition = sp.pos ?? currentPosition
 
-    controller.pc = pc
-    currentPosition = pos ?? currentPosition
-    registers.ints = intRegisters
-    registers.positions = posRegisters
+    registers.ints.undo(to: sp.intLogEnd)
+    registers.positions.undo(to: sp.positionLogEnd)
+    registers.values.undo(to: sp.valueLogEnd)
 
     if !preservingCaptures {
-      // Reset all capture information
-      storedCaptures = capEnds
+      registers.storedCaptures.undo(to: sp.captureLogEnd)
     }
+    // If preserving captures, leave the capture log entries recorded since
+    // this save point untouched (rather than replaying or discarding them):
+    // `storedCaptures` keeps the values from the successful sub-match, and
+    // the log entries remain available so that an older, still-live save
+    // point can still correctly undo them on its own future backtrack.
 
     metrics.addBacktrack()
   }
 
+  // MARK: Capture mutation
+
+  mutating func setCapture(_ capNum: Int, startingAt pos: Position) {
+    updateRegister(at: CaptureRegister(capNum)) {
+      $0.startCapture(pos)
+    }
+  }
+
+  mutating func setCapture(_ capNum: Int, endingAt pos: Position) {
+    updateRegister(at: CaptureRegister(capNum)) {
+      $0.endCapture(pos)
+    }
+  }
+
+  mutating func setCaptureValue(_ capNum: Int, _ value: Any) {
+    updateRegister(at: CaptureRegister(capNum)) {
+      $0.registerValue(value)
+    }
+  }
+
   mutating func abort(_ e: Error? = nil) {
     if let e = e {
       self.failureReason = e
@@ -459,12 +470,11 @@ extension Processor {
       let (imm, reg) = payload.pairedImmediateInt
       let int = Int(asserting: imm)
       assert(int == imm)
-
-      registers[reg] = int
+      updateRegister(at: reg, to: int)
       controller.step()
     case .moveCurrentPosition:
       let reg = payload.position
-      registers[reg] = currentPosition
+      updateRegister(at: reg, to: currentPosition)
       controller.step()
     case .restorePosition:
       let reg = payload.position
@@ -478,7 +488,7 @@ extension Processor {
       if registers[int] == 0 {
         controller.pc = addr
       } else {
-        registers[int] -= 1
+        updateRegister(at: int) { $0 -= 1 }
         controller.step()
       }
     case .condBranchSamePosition:
@@ -622,7 +632,7 @@ extension Processor {
           signalFailure()
           return
         }
-        registers[valReg] = val
+        updateRegister(at: valReg, to: val)
         resume(at: nextIdx)
         controller.step()
       } catch {
@@ -634,13 +644,13 @@ extension Processor {
       let (isScalarMode, capture) = payload.captureAndMode
       let capNum = Int(
         asserting: capture.rawValue)
-      guard capNum < storedCaptures.count else {
+      guard capNum < registers.storedCaptures.count else {
         fatalError("Should this be an assert?")
       }
       // TODO:
       //   Should we assert it's not finished yet?
       //   What's the behavior there?
-      let cap = storedCaptures[capNum]
+      let cap = registers.storedCaptures[capture]
       guard let range = cap.range else {
         signalFailure()
         return
@@ -652,13 +662,13 @@ extension Processor {
     case .beginCapture:
       let capNum = Int(
         asserting: payload.capture.rawValue)
-      storedCaptures[capNum].startCapture(currentPosition)
+      setCapture(capNum, startingAt: currentPosition)
       controller.step()
 
     case .endCapture:
       let capNum = Int(
         asserting: payload.capture.rawValue)
-      storedCaptures[capNum].endCapture(currentPosition)
+      setCapture(capNum, endingAt: currentPosition)
       controller.step()
 
     case .transformCapture:
@@ -668,11 +678,11 @@ extension Processor {
 
       do {
         // FIXME: Pass input or the slice?
-        guard let value = try transform(input, storedCaptures[capNum]) else {
+        guard let value = try transform(input, registers.storedCaptures[cap]) else {
           signalFailure()
           return
         }
-        storedCaptures[capNum].registerValue(value)
+        setCaptureValue(capNum, value)
         controller.step()
       } catch {
         abort(error)
@@ -683,7 +693,7 @@ extension Processor {
       let (val, cap) = payload.pairedValueCapture
       let value = registers[val]
       let capNum = Int(asserting: cap.rawValue)
-      storedCaptures[capNum].registerValue(value)
+      setCaptureValue(capNum, value)
       controller.step()
     }
   }