diff --git a/pom.xml b/pom.xml index 46cec4ee61c4..d1b370b3f9c7 100644 --- a/pom.xml +++ b/pom.xml @@ -252,6 +252,7 @@ view-helper virtual-proxy visitor + write-ahead-log backpressure actor-model rate-limiting-pattern diff --git a/write-ahead-log/README.md b/write-ahead-log/README.md new file mode 100644 index 000000000000..45caab13cadf --- /dev/null +++ b/write-ahead-log/README.md @@ -0,0 +1,217 @@ +--- +title: "Write-Ahead Log (WAL) Pattern in Java: Ensuring Data Durability and Crash Recovery" +shortTitle: Write-Ahead Log +description: "Learn about the Write-Ahead Log (WAL) design pattern in Java. Discover how append-only logging guarantees data durability and fast crash recovery in database engines and distributed systems." +category: Data Access +language: en +tag: + - Data access + - Storage + - Fault tolerance + - Transactions + - Performance +--- + +## Also known as + +* Append-Only Log +* Redo Log +* Journaling + +## Intent of Write-Ahead Log Pattern + +The Write-Ahead Log (WAL) design pattern ensures data durability and system recoverability in database engines, distributed consensus protocols, and transactional systems. It enforces a strict order of operations where any state mutation (e.g., insert, update, delete) must be written sequentially to an append-only log file on stable storage (disk) before it is applied to the main database state or in-memory storage structures. + +## Detailed Explanation of Write-Ahead Log Pattern with Real-World Examples + +Real-world example + +> Imagine an accountant managing a company's ledger. Before modifying the main financial summary balance sheets, the accountant immediately records every incoming transaction line-by-line into a sequential physical logbook. If power cuts out mid-day or the summary balance sheets are damaged, the accountant can re-open the physical logbook, replay every recorded entry from the beginning, and perfectly recalculate the final financial state. + +In plain words + +> Write-Ahead Log guarantees that no state mutation is lost during sudden system crashes by writing changes to a fast append-only disk log file before updating the in-memory store. + +Wikipedia says + +> In computer science, write-ahead logging (WAL) is a family of techniques for providing atomicity and durability (two of the ACID properties) in database systems. In a system using WAL, all modifications are written to a log before they are applied. Usually both redo and undo information are stored in the log. + +Class Diagram + +```mermaid +classDiagram + class OperationType { + <> + SET + DELETE + CHECKPOINT + } + + class LogEntry { + -long sequenceNumber + -OperationType type + -String key + -String value + +toLogString() String + +fromLogString(String line)$ LogEntry + } + + class WriteAheadLog { + -File logFile + -AtomicLong sequenceNumberCounter + +append(OperationType type, String key, String value) LogEntry + +readAll() List~LogEntry~ + +clear() void + } + + class DatabaseStore { + -WriteAheadLog wal + -Map~String, String~ memTable + +put(String key, String value) void + +delete(String key) void + +get(String key) String + +checkpoint() void + +simulateCrash() void + +recover() void + } + + DatabaseStore --> WriteAheadLog + WriteAheadLog --> LogEntry + LogEntry --> OperationType +``` + +## Programmatic Example of Write-Ahead Log Pattern in Java + +The `WriteAheadLog` class manages append-only sequential writes to disk: + +```java +public class WriteAheadLog { + private final File logFile; + private final AtomicLong sequenceNumberCounter = new AtomicLong(0); + + public synchronized LogEntry append(OperationType type, String key, String value) throws IOException { + long nextSeq = sequenceNumberCounter.incrementAndGet(); + LogEntry entry = new LogEntry(nextSeq, type, key, value); + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(logFile, true))) { + writer.write(entry.toLogString()); + writer.newLine(); + writer.flush(); + } + return entry; + } +} +``` + +The `DatabaseStore` class coordinates writing to the log before modifying its in-memory `MemTable`: + +```java +public class DatabaseStore { + private final WriteAheadLog wal; + private final Map memTable = new HashMap<>(); + + public synchronized void put(String key, String value) throws IOException { + wal.append(OperationType.SET, key, value); + memTable.put(key, value); + } + + public synchronized void delete(String key) throws IOException { + wal.append(OperationType.DELETE, key, null); + memTable.remove(key); + } + + public synchronized void recover() { + memTable.clear(); + List entries = wal.readAll(); + for (LogEntry entry : entries) { + if (entry.getType() == OperationType.SET) { + memTable.put(entry.getKey(), entry.getValue()); + } else if (entry.getType() == OperationType.DELETE) { + memTable.remove(entry.getKey()); + } + } + } +} +``` + +The `App` class demonstrates initialization, writes, crash simulation, and WAL recovery: + +```java +@Slf4j +public class App { + public static void main(String[] args) { + try { + File logFile = File.createTempFile("wal_demo", ".log"); + WriteAheadLog wal = new WriteAheadLog(logFile); + DatabaseStore store = new DatabaseStore(wal); + + store.put("user:101", "Alice"); + store.put("user:102", "Bob"); + store.delete("user:103"); + + // Simulating system crash where in-memory state is lost + store.simulateCrash(); + + // System reboot & recovery from WAL log replay + store.recover(); + + LOGGER.info("MemTable post recovery: {}", store.getMemTableSnapshot()); + } catch (IOException e) { + LOGGER.error("Error running WAL demo", e); + } + } +} +``` + +Program output: + +```text +15:45:00.100 [main] INFO com.iluwatar.writeaheadlog.App -- === 1. Initializing Storage Engine with WAL === +15:45:00.105 [main] INFO com.iluwatar.writeaheadlog.WriteAheadLog -- WAL Entry appended & flushed to disk: LogEntry(sequenceNumber=1, type=SET, key=user:101, value=Alice) +15:45:00.106 [main] INFO com.iluwatar.writeaheadlog.DatabaseStore -- Applied SET operation to MemTable: user:101 = Alice +15:45:00.107 [main] INFO com.iluwatar.writeaheadlog.App -- === 3. Simulating Unexpected System Crash === +15:45:00.108 [main] INFO com.iluwatar.writeaheadlog.DatabaseStore -- !!! SIMULATED SYSTEM CRASH: In-memory MemTable has been wiped !!! +15:45:00.109 [main] INFO com.iluwatar.writeaheadlog.App -- === 4. System Restart & Recovery from WAL === +15:45:00.110 [main] INFO com.iluwatar.writeaheadlog.DatabaseStore -- Starting recovery process from WAL... +15:45:00.112 [main] INFO com.iluwatar.writeaheadlog.DatabaseStore -- Recovery completed. Replayed 5 log entries into MemTable. +15:45:00.113 [main] INFO com.iluwatar.writeaheadlog.App -- MemTable snapshot post recovery: {user:101=Alice, user:102=Bob Smith} +``` + +## When to Use the Write-Ahead Log Pattern in Java + +* Building storage engines or key-value data stores requiring ACID durability guarantees. +* Implementing fault-tolerant distributed consensus protocols (e.g., Raft, Paxos). +* System architectures where random disk I/O is expensive, allowing sequential append-only writes for maximum throughput. +* Message brokers or event streams requiring replayability after failure. + +## Real-World Applications of Write-Ahead Log Pattern in Java + +* **PostgreSQL / MySQL (InnoDB):** Uses WAL / Redo Log for crash recovery and replication. +* **SQLite:** Write-Ahead Logging mode for concurrency and atomic commits. +* **Apache Cassandra / RocksDB:** Appends mutations to CommitLog / WAL before MemTable updates. +* **Apache Kafka / Raft:** Log replication across distributed nodes for consensus and state machine replication. + +## Benefits and Trade-offs of Write-Ahead Log Pattern + +Benefits: + +* **High Performance:** Sequential disk writes are significantly faster than random disk updates (e.g., updating B-Trees directly). +* **Durability & Fault Tolerance:** Guarantees no committed transaction is lost during sudden system crashes. +* **Simplicity of Recovery:** Replaying ordered log records deterministically restores the exact last-known state. + +Trade-offs: + +* **Storage Overhead:** Log files grow over time, requiring periodic checkpointing and log truncation. +* **Recovery Time:** Large log files without checkpoints can lead to slow startup/recovery times. + +## Related Java Design Patterns + +* [Event Sourcing](https://java-design-patterns.com/patterns/event-sourcing/): Captures state mutations as a sequence of events, similar to log replay. +* [Command](https://java-design-patterns.com/patterns/command/): Encapsulates requests as objects, which can be serialized into WAL entries. +* [Memento](https://java-design-patterns.com/patterns/memento/): Stores state snapshots (checkpoints) to truncate logs. + +## References and Credits + +* [Designing Data-Intensive Applications (Martin Kleppmann)](https://www.oreilly.com/library/view/designing-data-intensive-applications/9781491903063/) +* [PostgreSQL Documentation: Write-Ahead Logging (WAL)](https://www.postgresql.org/docs/current/wal-intro.html) +* [Raft Consensus Algorithm Paper](https://raft.github.io/) diff --git a/write-ahead-log/pom.xml b/write-ahead-log/pom.xml new file mode 100644 index 000000000000..adb766953a47 --- /dev/null +++ b/write-ahead-log/pom.xml @@ -0,0 +1,87 @@ + + + + 4.0.0 + + com.iluwatar + java-design-patterns + 1.26.0-SNAPSHOT + + write-ahead-log + 1.26.0-SNAPSHOT + write-ahead-log + http://maven.apache.org + + UTF-8 + + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + org.projectlombok + lombok + ${lombok.version} + provided + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.mockito + mockito-core + test + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + + + + com.iluwatar.writeaheadlog.App + + + + + + + + + diff --git a/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/App.java b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/App.java new file mode 100644 index 000000000000..e8e9fae2f763 --- /dev/null +++ b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/App.java @@ -0,0 +1,82 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import java.io.File; +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; + +/** + * Main application class demonstrating the Write-Ahead Log (WAL) design pattern. + * + *

The WAL pattern guarantees durability by ensuring every mutation (SET, DELETE) is written to a + * persistent append-only log file on disk BEFORE updating in-memory state. If the system crashes + * unexpectedly, replaying log entries from the WAL file restores state. + */ +@Slf4j +public class App { + + /** + * Application entry point. + * + * @param args command line arguments + */ + public static void main(String[] args) { + try { + File logFile = File.createTempFile("wal_demo", ".log"); + logFile.deleteOnExit(); + + LOGGER.info( + "=== 1. Initializing Storage Engine with WAL at {} ===", logFile.getAbsolutePath()); + WriteAheadLog wal = new WriteAheadLog(logFile); + DatabaseStore store = new DatabaseStore(wal); + + LOGGER.info("=== 2. Performing Data Operations (Write-Ahead Logging) ==="); + store.put("user:101", "Alice"); + store.put("user:102", "Bob"); + store.put("user:103", "Charlie"); + store.put("user:102", "Bob Smith"); + store.delete("user:103"); + store.checkpoint(); + + LOGGER.info("MemTable snapshot before crash: {}", store.getMemTableSnapshot()); + + LOGGER.info("=== 3. Simulating Unexpected System Crash ==="); + store.simulateCrash(); + LOGGER.info("MemTable snapshot after crash: {}", store.getMemTableSnapshot()); + + LOGGER.info("=== 4. System Restart & Recovery from WAL ==="); + store.recover(); + LOGGER.info("MemTable snapshot post recovery: {}", store.getMemTableSnapshot()); + + if (logFile.exists()) { + logFile.delete(); + } + } catch (IOException e) { + LOGGER.error("An error occurred during WAL demonstration: {}", e.getMessage(), e); + } + } +} diff --git a/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/DatabaseStore.java b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/DatabaseStore.java new file mode 100644 index 000000000000..dd6d94fce7aa --- /dev/null +++ b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/DatabaseStore.java @@ -0,0 +1,131 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +/** + * Storage engine demonstrating Write-Ahead Log (WAL) pattern. All mutations are logged and flushed + * to persistent WAL storage before modifying the in-memory MemTable. + */ +@Slf4j +public class DatabaseStore { + + @Getter private final WriteAheadLog wal; + private final Map memTable = new HashMap<>(); + + /** + * Constructs DatabaseStore with the specified WriteAheadLog. + * + * @param wal persistent write-ahead log manager + */ + public DatabaseStore(WriteAheadLog wal) { + this.wal = wal; + } + + /** + * Stores a key-value pair. First writes to WAL on disk, then updates the in-memory MemTable. + * + * @param key target entry key + * @param value target entry value + * @throws IOException if writing to WAL fails + */ + public synchronized void put(String key, String value) throws IOException { + wal.append(OperationType.SET, key, value); + memTable.put(key, value); + LOGGER.info("Applied SET operation to MemTable: {} = {}", key, value); + } + + /** + * Removes a key-value pair. First writes DELETE operation to WAL on disk, then updates the + * in-memory MemTable. + * + * @param key target entry key to delete + * @throws IOException if writing to WAL fails + */ + public synchronized void delete(String key) throws IOException { + wal.append(OperationType.DELETE, key, null); + memTable.remove(key); + LOGGER.info("Applied DELETE operation to MemTable: {}", key); + } + + /** + * Retrieves value associated with key from the in-memory MemTable. + * + * @param key key to lookup + * @return value or null if non-existent + */ + public synchronized String get(String key) { + return memTable.get(key); + } + + /** + * Returns an unmodifiable view of current in-memory MemTable state. + * + * @return unmodifiable map of stored data + */ + public synchronized Map getMemTableSnapshot() { + return Collections.unmodifiableMap(new HashMap<>(memTable)); + } + + /** + * Writes a CHECKPOINT log record. + * + * @throws IOException if writing to WAL fails + */ + public synchronized void checkpoint() throws IOException { + wal.append(OperationType.CHECKPOINT, null, null); + LOGGER.info("Checkpoint written to WAL."); + } + + /** Simulates a system crash or power outage where in-memory state is wiped. */ + public synchronized void simulateCrash() { + memTable.clear(); + LOGGER.info("!!! SIMULATED SYSTEM CRASH: In-memory MemTable has been wiped !!!"); + } + + /** Replays log entries from persistent Write-Ahead Log to fully recover in-memory state. */ + public synchronized void recover() { + LOGGER.info("Starting recovery process from WAL..."); + memTable.clear(); + List entries = wal.readAll(); + + for (LogEntry entry : entries) { + if (entry.getType() == OperationType.SET) { + memTable.put(entry.getKey(), entry.getValue()); + } else if (entry.getType() == OperationType.DELETE) { + memTable.remove(entry.getKey()); + } + } + LOGGER.info("Recovery completed. Replayed {} log entries into MemTable.", entries.size()); + } +} diff --git a/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/LogEntry.java b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/LogEntry.java new file mode 100644 index 000000000000..7ae212c8337c --- /dev/null +++ b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/LogEntry.java @@ -0,0 +1,83 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** Represents a single record entry in the Write-Ahead Log. */ +@Getter +@AllArgsConstructor +@EqualsAndHashCode +@ToString +public class LogEntry { + + private static final String DELIMITER = "|"; + + private final long sequenceNumber; + private final OperationType type; + private final String key; + private final String value; + + /** + * Serializes the LogEntry to a delimited string format suitable for append-only logging. + * + * @return delimited log line representation + */ + public String toLogString() { + return sequenceNumber + + DELIMITER + + type + + DELIMITER + + (key != null ? key : "") + + DELIMITER + + (value != null ? value : ""); + } + + /** + * Deserializes a delimited string line into a LogEntry instance. + * + * @param line serialized log line + * @return parsed LogEntry + * @throws IllegalArgumentException if the log line format is invalid + */ + public static LogEntry fromLogString(String line) { + if (line == null || line.isBlank()) { + throw new IllegalArgumentException("Log line cannot be null or blank"); + } + String[] parts = line.split("\\|", -1); + if (parts.length < 4) { + throw new IllegalArgumentException("Invalid log line format: " + line); + } + long sequenceNumber = Long.parseLong(parts[0]); + OperationType type = OperationType.valueOf(parts[1]); + String key = parts[2].isEmpty() ? null : parts[2]; + String value = parts[3].isEmpty() ? null : parts[3]; + return new LogEntry(sequenceNumber, type, key, value); + } +} diff --git a/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/OperationType.java b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/OperationType.java new file mode 100644 index 000000000000..33e06111bdc4 --- /dev/null +++ b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/OperationType.java @@ -0,0 +1,33 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +/** Enumeration representing the type of operations logged in the Write-Ahead Log. */ +public enum OperationType { + SET, + DELETE, + CHECKPOINT +} diff --git a/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/WriteAheadLog.java b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/WriteAheadLog.java new file mode 100644 index 000000000000..4ea2044ff2fc --- /dev/null +++ b/write-ahead-log/src/main/java/com/iluwatar/writeaheadlog/WriteAheadLog.java @@ -0,0 +1,134 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +/** + * Manages sequential append-only logging to persistent disk storage. Ensures state changes are + * flushed to stable storage before in-memory updates. + */ +@Slf4j +public class WriteAheadLog { + + @Getter private final File logFile; + private final AtomicLong sequenceNumberCounter = new AtomicLong(0); + + /** + * Initializes WriteAheadLog with target log file. If log file exists, calculates the initial + * sequence number from existing entries. + * + * @param logFile file to use for append-only log entries + */ + public WriteAheadLog(File logFile) { + this.logFile = logFile; + initSequenceNumber(); + } + + private void initSequenceNumber() { + if (logFile.exists()) { + List existingEntries = readAll(); + if (!existingEntries.isEmpty()) { + long maxSeq = existingEntries.get(existingEntries.size() - 1).getSequenceNumber(); + sequenceNumberCounter.set(maxSeq); + } + } + } + + /** + * Appends an entry to the log file and flushes to ensure persistence. + * + * @param type operation type (SET, DELETE, CHECKPOINT) + * @param key operation target key + * @param value operation target value + * @return recorded LogEntry + * @throws IOException if writing to persistent storage fails + */ + public synchronized LogEntry append(OperationType type, String key, String value) + throws IOException { + long nextSeq = sequenceNumberCounter.incrementAndGet(); + LogEntry entry = new LogEntry(nextSeq, type, key, value); + + try (BufferedWriter writer = new BufferedWriter(new FileWriter(logFile, true))) { + writer.write(entry.toLogString()); + writer.newLine(); + writer.flush(); + } + LOGGER.info("WAL Entry appended & flushed to disk: {}", entry); + return entry; + } + + /** + * Reads all log entries sequentially from the persistent log file. + * + * @return list of parsed LogEntries in sequential order + */ + public synchronized List readAll() { + List entries = new ArrayList<>(); + if (!logFile.exists()) { + return entries; + } + + try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.isBlank()) { + entries.add(LogEntry.fromLogString(line)); + } + } + } catch (IOException e) { + LOGGER.error("Failed to read log entries from WAL file: {}", logFile.getAbsolutePath(), e); + } + return entries; + } + + /** + * Clears the log file and resets the sequence number counter. Typically invoked after a + * successful checkpoint. + */ + public synchronized void clear() { + if (logFile.exists()) { + try { + Files.delete(logFile.toPath()); + sequenceNumberCounter.set(0); + LOGGER.info("WAL log cleared successfully."); + } catch (IOException e) { + LOGGER.error("Failed to clear WAL file: {}", logFile.getAbsolutePath(), e); + } + } + } +} diff --git a/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/AppTest.java b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/AppTest.java new file mode 100644 index 000000000000..b3bdbe3360ba --- /dev/null +++ b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/AppTest.java @@ -0,0 +1,38 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +import org.junit.jupiter.api.Test; + +class AppTest { + + @Test + void testAppMainExecutesWithoutExceptions() { + assertDoesNotThrow(() -> App.main(new String[0])); + } +} diff --git a/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/DatabaseStoreTest.java b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/DatabaseStoreTest.java new file mode 100644 index 000000000000..08288547348b --- /dev/null +++ b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/DatabaseStoreTest.java @@ -0,0 +1,96 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class DatabaseStoreTest { + + private File tempFile; + private WriteAheadLog wal; + private DatabaseStore store; + + @BeforeEach + void setUp() throws IOException { + tempFile = File.createTempFile("db_store_test", ".log"); + wal = new WriteAheadLog(tempFile); + store = new DatabaseStore(wal); + } + + @AfterEach + void tearDown() { + if (tempFile != null && tempFile.exists()) { + tempFile.delete(); + } + } + + @Test + void testPutAndGet() throws IOException { + store.put("key1", "val1"); + assertEquals("val1", store.get("key1")); + } + + @Test + void testDelete() throws IOException { + store.put("key1", "val1"); + assertEquals("val1", store.get("key1")); + + store.delete("key1"); + assertNull(store.get("key1")); + } + + @Test + void testSimulateCrashAndRecovery() throws IOException { + store.put("key1", "val1"); + store.put("key2", "val2"); + store.put("key1", "val1_updated"); + store.delete("key2"); + store.checkpoint(); + + Map beforeCrashSnapshot = store.getMemTableSnapshot(); + assertEquals("val1_updated", beforeCrashSnapshot.get("key1")); + assertNull(beforeCrashSnapshot.get("key2")); + + store.simulateCrash(); + assertNull(store.get("key1")); + assertNull(store.get("key2")); + assertTrue(store.getMemTableSnapshot().isEmpty()); + + store.recover(); + Map recoveredSnapshot = store.getMemTableSnapshot(); + assertEquals("val1_updated", recoveredSnapshot.get("key1")); + assertNull(recoveredSnapshot.get("key2")); + } +} diff --git a/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/LogEntryTest.java b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/LogEntryTest.java new file mode 100644 index 000000000000..ea031e04c8a0 --- /dev/null +++ b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/LogEntryTest.java @@ -0,0 +1,80 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class LogEntryTest { + + @Test + void testToLogStringAndFromLogString() { + LogEntry entry = new LogEntry(1, OperationType.SET, "key1", "val1"); + String logString = entry.toLogString(); + assertEquals("1|SET|key1|val1", logString); + + LogEntry parsed = LogEntry.fromLogString(logString); + assertEquals(entry, parsed); + assertEquals(1, parsed.getSequenceNumber()); + assertEquals(OperationType.SET, parsed.getType()); + assertEquals("key1", parsed.getKey()); + assertEquals("val1", parsed.getValue()); + } + + @Test + void testDeleteLogEntrySerialization() { + LogEntry entry = new LogEntry(2, OperationType.DELETE, "key2", null); + String logString = entry.toLogString(); + assertEquals("2|DELETE|key2|", logString); + + LogEntry parsed = LogEntry.fromLogString(logString); + assertEquals(entry, parsed); + assertEquals("key2", parsed.getKey()); + assertNull(parsed.getValue()); + } + + @Test + void testInvalidLogStringThrowsException() { + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromLogString(null)); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromLogString(" ")); + assertThrows(IllegalArgumentException.class, () -> LogEntry.fromLogString("1|SET|key1")); + } + + @Test + void testEqualsAndHashCode() { + LogEntry entry1 = new LogEntry(1, OperationType.SET, "k", "v"); + LogEntry entry2 = new LogEntry(1, OperationType.SET, "k", "v"); + LogEntry entry3 = new LogEntry(2, OperationType.SET, "k", "v"); + + assertEquals(entry1, entry2); + assertEquals(entry1.hashCode(), entry2.hashCode()); + assertNotEquals(entry1, entry3); + } +} diff --git a/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/WriteAheadLogTest.java b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/WriteAheadLogTest.java new file mode 100644 index 000000000000..8f0410414e3c --- /dev/null +++ b/write-ahead-log/src/test/java/com/iluwatar/writeaheadlog/WriteAheadLogTest.java @@ -0,0 +1,134 @@ +/* + * This project is licensed under the MIT license. Module model-view-viewmodel is using ZK framework licensed under LGPL (see lgpl-3.0.txt). + * + * The MIT License + * Copyright © 2014-2022 Ilkka Seppälä + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.iluwatar.writeaheadlog; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class WriteAheadLogTest { + + private File tempFile; + private WriteAheadLog wal; + + @BeforeEach + void setUp() throws IOException { + tempFile = File.createTempFile("wal_test", ".log"); + wal = new WriteAheadLog(tempFile); + } + + @AfterEach + void tearDown() { + if (tempFile != null && tempFile.exists()) { + if (tempFile.isDirectory()) { + File[] files = tempFile.listFiles(); + if (files != null) { + for (File f : files) { + f.delete(); + } + } + } + tempFile.delete(); + } + } + + @Test + void testAppendAndReadAll() throws IOException { + LogEntry e1 = wal.append(OperationType.SET, "k1", "v1"); + LogEntry e2 = wal.append(OperationType.SET, "k2", "v2"); + LogEntry e3 = wal.append(OperationType.DELETE, "k1", null); + + assertEquals(1, e1.getSequenceNumber()); + assertEquals(2, e2.getSequenceNumber()); + assertEquals(3, e3.getSequenceNumber()); + + List entries = wal.readAll(); + assertEquals(3, entries.size()); + assertEquals(e1, entries.get(0)); + assertEquals(e2, entries.get(1)); + assertEquals(e3, entries.get(2)); + } + + @Test + void testSequenceNumberResumptionOnReopen() throws IOException { + wal.append(OperationType.SET, "k1", "v1"); + wal.append(OperationType.SET, "k2", "v2"); + + WriteAheadLog reopenedWal = new WriteAheadLog(tempFile); + LogEntry newEntry = reopenedWal.append(OperationType.SET, "k3", "v3"); + + assertEquals(3, newEntry.getSequenceNumber()); + } + + @Test + void testClear() throws IOException { + wal.append(OperationType.SET, "k1", "v1"); + assertTrue(tempFile.exists()); + + wal.clear(); + assertFalse(tempFile.exists()); + + List entries = wal.readAll(); + assertNotNull(entries); + assertTrue(entries.isEmpty()); + } + + @Test + void testReadAllIOExceptionHandling() throws IOException { + File dir = Files.createTempDirectory("wal_dir_test").toFile(); + WriteAheadLog dirWal = new WriteAheadLog(dir); + + List entries = dirWal.readAll(); + assertNotNull(entries); + assertTrue(entries.isEmpty()); + + dir.delete(); + } + + @Test + void testClearIOExceptionHandling() throws IOException { + File dir = Files.createTempDirectory("wal_nonempty_dir_test").toFile(); + File child = new File(dir, "child.txt"); + child.createNewFile(); + + WriteAheadLog dirWal = new WriteAheadLog(dir); + dirWal.clear(); + + assertTrue(dir.exists()); + + child.delete(); + dir.delete(); + } +}