Parse documents into a shared model, render Markdown once

.gitignore

@@ -11,4 +11,4 @@ xcuserdata
 .kotlin
 #So we don't accidentally commit our private keys
 *.gpg
-*.py
+*.py!scripts/*.py

README.md

@@ -20,6 +20,32 @@ Kotlin Multiplatform (JVM + Android) library that converts documents to Markdown
 | Plain text | `.txt` and others |
 | Markdown | `.md` (passthrough) |
 
+## Architecture
+
+Every format is parsed into one shared document model, and a single renderer serializes that model
+to GitHub-Flavored Markdown:
+
+```
+bytes ──► MimeDetector ──► DocumentConverter.parse ──► Document ──► MarkdownRenderer ──► Markdown
+                                (per format)          (blocks,        (one GFM
+                                                       inlines,        serializer)
+                                                       tables,
+                                                       assets)
+```
+
+Converters contain no Markdown syntax, so escaping, table shaping, list indentation and spacing are
+fixed once for all formats. The model is public: `mid.parse(path)` returns the `Document`, and
+`ConversionResult.document` exposes it alongside the rendered Markdown.
+
+```kotlin
+val document = mid.parse("/path/to/report.docx")
+document.blocks.filterIsInstance<Table>().forEach { println(it.rows.size) }
+
+// Render with different options
+val compact = MarkdownRenderer(MarkdownOptions(padTableColumns = true, imagesAsText = true))
+println(compact.render(document))
+```
+
 ## Setup
 
 ```kotlin
@@ -69,8 +95,10 @@ class MyConverter : DocumentConverter {
     override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean =
         info.extension == "xyz"
 
-    override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult =
-        ConversionResult(markdown = String(bytes))
+    override fun parse(bytes: ByteArray, info: StreamInfo): Document = document {
+        heading(1, "Custom")
+        paragraph(bytes.decodeToString())
+    }
 }
 
 val mid = MarkItDown()
@@ -99,3 +127,18 @@ mid.register(HtmlConverter())
 | `FileConversionException` | Converter threw during conversion |
 
 Both extend `MarkItDownException`.
+
+## Benchmark
+
+`scripts/benchmark.py` converts the test fixtures with MikroMarkdown, Python
+[markitdown](https://github.com/microsoft/markitdown) and Rust
+[anydoc](https://github.com/firecrawl/anydoc), then reports content recall, structure counts, table
+integrity and timings to `build/benchmark/report.md`:
+
+```bash
+./gradlew :cli:installDist
+python3 scripts/benchmark.py
+```
+
+Engines whose CLI is missing are skipped. anydoc only handles binary formats, so it sits out the
+HTML/JSON/XML fixtures.

cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt

@@ -33,4 +33,11 @@ class MarkItDownCommand : CliktCommand(name = "markitdown") {
     }
 }
 
-fun main(args: Array<String>) = MarkItDownCommand().main(args)
+fun main(args: Array<String>) {
+    // PDFBox pulls in the Log4j API; without a provider it writes a banner to stdout,
+    // which would corrupt the Markdown we print there.
+    System.setProperty("log4j2.loggerContextFactory", "org.apache.logging.log4j.simple.SimpleLoggerContextFactory")
+    System.setProperty("log4j2.simplelogLevel", "OFF")
+    System.setProperty("log4j2.statusLoggerLevel", "OFF")
+    MarkItDownCommand().main(args)
+}

gradle/libs.versions.toml

@@ -8,7 +8,6 @@ kotlinx-io = "0.9.0"
 kotlinx-resources = "0.15.0"
 
 commons-csv = "1.14.1"
-flexmark = "0.64.8"
 jackson = "2.21.3"
 jsoup = "1.22.2"
 junit = "6.1.0"
@@ -25,7 +24,6 @@ kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.re
 kotlinx-resources = { module = "com.goncalossilva:resources", version.ref = "kotlinx-resources" }
 
 commons-csv = { module = "org.apache.commons:commons-csv", version.ref = "commons-csv" }
-flexmark-html2md = { module = "com.vladsch.flexmark:flexmark-html2md-converter", version.ref = "flexmark" }
 jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" }
 jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" }
 junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }

library/build.gradle.kts

@@ -41,7 +41,6 @@ kotlin {
 
         jvmMain.dependencies {
             implementation(libs.jsoup)
-            implementation(libs.flexmark.html2md)
             implementation(libs.jackson.kotlin)
             implementation(libs.commons.csv)
             implementation(libs.poi.ooxml)
@@ -51,7 +50,6 @@ kotlin {
 
         androidMain.dependencies {
             implementation(libs.jsoup)
-            implementation(libs.flexmark.html2md)
             implementation(libs.jackson.kotlin)
             implementation(libs.commons.csv)
             implementation(libs.poi.ooxml)

library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt

@@ -1,8 +1,10 @@
 package io.github.lemcoder.mikromarkdown.converters
 
-import io.github.lemcoder.mikromarkdown.ConversionResult
 import io.github.lemcoder.mikromarkdown.DocumentConverter
 import io.github.lemcoder.mikromarkdown.StreamInfo
+import io.github.lemcoder.mikromarkdown.model.Document
+import io.github.lemcoder.mikromarkdown.model.Table
+import io.github.lemcoder.mikromarkdown.model.TableCell
 import org.apache.commons.csv.CSVFormat
 import org.apache.commons.csv.CSVParser
 import java.io.InputStreamReader
@@ -12,28 +14,19 @@ class CsvConverter : DocumentConverter {
         return info.extension == "csv" || info.mimetype in setOf("text/csv", "application/csv")
     }
 
-    override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult {
+    override fun parse(bytes: ByteArray, info: StreamInfo): Document {
         val reader = InputStreamReader(bytes.inputStream(), Charsets.UTF_8)
-        val allRecords = CSVParser(reader, CSVFormat.DEFAULT.builder().setTrim(true).build()).records
+        val records = CSVParser(reader, CSVFormat.DEFAULT.builder().setTrim(true).build()).records
+        if (records.isEmpty()) return Document()
 
-        if (allRecords.isEmpty()) return ConversionResult(markdown = "")
+        val header = records[0].toList()
+        if (header.isEmpty()) return Document()
 
-        val headers = allRecords[0].toList()
-        if (headers.isEmpty()) return ConversionResult(markdown = "")
-
-        val sb = StringBuilder()
-        sb.appendLine(headers.map { it.escapeCell() }.joinToString(" | ", "| ", " |"))
-        sb.appendLine(headers.map { "---" }.joinToString(" | ", "| ", " |"))
-
-        for (i in 1 until allRecords.size) {
-            val cells = (0 until headers.size).map { col ->
-                allRecords[i].get(col).escapeCell()
-            }
-            sb.appendLine(cells.joinToString(" | ", "| ", " |"))
+        val rows = records.drop(1).map { record ->
+            // Ragged rows are padded by the renderer; only extra columns need trimming here.
+            List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") }
         }
 
-        return ConversionResult(markdown = sb.toString().trimEnd())
+        return Document(blocks = listOf(Table(header = header.map { TableCell(it) }, rows = rows)))
     }
-
-    private fun String.escapeCell(): String = replace("|", "\\|").replace("\n", " ")
 }

library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt

@@ -1,112 +1,162 @@
 package io.github.lemcoder.mikromarkdown.converters
 
-import java.util.Base64
-import io.github.lemcoder.mikromarkdown.ConversionResult
 import io.github.lemcoder.mikromarkdown.DocumentConverter
 import io.github.lemcoder.mikromarkdown.StreamInfo
+import io.github.lemcoder.mikromarkdown.model.Asset
+import io.github.lemcoder.mikromarkdown.model.Block
+import io.github.lemcoder.mikromarkdown.model.Document
+import io.github.lemcoder.mikromarkdown.model.Heading
+import io.github.lemcoder.mikromarkdown.model.Image
+import io.github.lemcoder.mikromarkdown.model.Inline
+import io.github.lemcoder.mikromarkdown.model.ListBlock
+import io.github.lemcoder.mikromarkdown.model.ListItem
+import io.github.lemcoder.mikromarkdown.model.Paragraph
+import io.github.lemcoder.mikromarkdown.model.Table
+import io.github.lemcoder.mikromarkdown.model.TableCell
+import io.github.lemcoder.mikromarkdown.model.Text
+import io.github.lemcoder.mikromarkdown.model.plainText
+import io.github.lemcoder.mikromarkdown.model.styled
 import org.apache.poi.xwpf.usermodel.XWPFDocument
 import org.apache.poi.xwpf.usermodel.XWPFParagraph
 import org.apache.poi.xwpf.usermodel.XWPFTable
+import java.util.Base64
 
 class DocxConverter : DocumentConverter {
     override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean {
         return info.extension == "docx" ||
                info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
     }
 
-    override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult {
-        val doc = XWPFDocument(bytes.inputStream())
-        val sb = StringBuilder()
-        var title: String? = null
-
-        for (element in doc.bodyElements) {
-            when (element) {
-                is XWPFParagraph -> {
-                    val md = convertParagraph(element)
-                    if (md.isNotBlank()) {
-                        if (title == null && headingLevel(element.styleID) > 0) {
-                            title = element.text
+    override fun parse(bytes: ByteArray, info: StreamInfo): Document {
+        val docx = XWPFDocument(bytes.inputStream())
+        try {
+            val blocks = mutableListOf<Block>()
+            val assets = mutableListOf<Asset>()
+            var title: String? = docx.properties?.coreProperties?.title?.trim()?.ifBlank { null }
+            val pendingListItems = mutableListOf<Pair<Int, List<Inline>>>()
+
+            fun flushList() {
+                if (pendingListItems.isEmpty()) return
+                blocks += buildNestedList(pendingListItems)
+                pendingListItems.clear()
+            }
+
+            for (element in docx.bodyElements) {
+                when (element) {
+                    is XWPFParagraph -> {
+                        val content = paragraphInlines(element, assets)
+                        if (content.plainText().isBlank() && content.none { it is Image }) continue
+
+                        val level = headingLevel(element.styleID)
+                        when {
+                            level > 0 -> {
+                                flushList()
+                                if (title == null) title = content.plainText().trim()
+                                blocks += Heading(level, content)
+                            }
+
+                            element.numID != null ->
+                                pendingListItems += (element.numIlvl?.toInt() ?: 0).coerceAtLeast(0) to content
+
+                            else -> {
+                                flushList()
+                                blocks += Paragraph(content)
+                            }
                         }
-                        sb.appendLine(md)
-                        sb.appendLine()
                     }
-                }
-                is XWPFTable -> {
-                    val tableMd = convertTable(element)
-                    if (tableMd.isNotBlank()) {
-                        sb.appendLine(tableMd)
+
+                    is XWPFTable -> {
+                        flushList()
+                        table(element)?.let { blocks += it }
                     }
                 }
             }
-        }
+            flushList()
 
-        doc.close()
-        return ConversionResult(markdown = sb.toString(), title = title)
+            return Document(blocks = blocks, title = title, assets = assets)
+        } finally {
+            docx.close()
+        }
     }
 
-    private fun convertParagraph(para: XWPFParagraph): String {
-        val rawText = para.runs.joinToString("") { run ->
-            val pics = run.embeddedPictures
-            if (pics.isNotEmpty()) {
-                return@joinToString pics.joinToString("") { pic ->
-                    val descr = pic.description ?: ""
-                    val data = pic.pictureData
-                    if (data != null) {
-                        val mime = data.pictureTypeEnum.contentType
-                        val b64 = Base64.getEncoder().encodeToString(data.data)
-                        "![$descr](data:$mime;base64,$b64)"
-                    } else {
-                        "![$descr]()"
+    private fun paragraphInlines(para: XWPFParagraph, assets: MutableList<Asset>): List<Inline> {
+        val out = mutableListOf<Inline>()
+        for (run in para.runs) {
+            val pictures = run.embeddedPictures
+            if (pictures.isNotEmpty()) {
+                for (picture in pictures) {
+                    val data = picture.pictureData
+                    val alt = picture.description.orEmpty()
+                    if (data == null) {
+                        if (alt.isNotBlank()) out += Text(alt)
+                        continue
                     }
+                    val mime = data.pictureTypeEnum.contentType
+                    val id = data.fileName ?: "image-${assets.size + 1}"
+                    assets += Asset(id = id, mediaType = mime, bytes = data.data, name = data.fileName)
+                    out += Image(
+                        alt = alt,
+                        url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}",
+                        assetId = id,
+                    )
                 }
+                continue
             }
-            var text = run.text() ?: ""
-            if (text.isBlank()) return@joinToString text
-            when {
-                run.isBold && run.isItalic -> "***$text***"
-                run.isBold -> "**$text**"
-                run.isItalic -> "*$text*"
-                else -> text
+
+            val text = run.text() ?: continue
+            if (text.isEmpty()) continue
+            if (text.isBlank()) {
+                out += Text(text)
+                continue
             }
+            out += styled(listOf(Text(text)), bold = run.isBold, italic = run.isItalic, strike = run.isStrikeThrough)
         }
+        return out
+    }
 
-        if (rawText.isBlank()) return ""
-
-        val level = headingLevel(para.styleID)
-        val isListItem = para.numID != null
+    /** Rebuilds Word's flat numbering levels into nested list blocks. */
+    private fun buildNestedList(items: List<Pair<Int, List<Inline>>>): ListBlock {
+        var index = 0
 
-        return when {
-            level > 0 -> "${"#".repeat(level)} $rawText"
-            isListItem -> {
-                val indent = "  ".repeat((para.numIlvl?.toInt() ?: 0).coerceAtLeast(0))
-                "$indent- $rawText"
+        fun build(level: Int): List<ListItem> {
+            val result = mutableListOf<ListItem>()
+            while (index < items.size) {
+                val (itemLevel, content) = items[index]
+                when {
+                    itemLevel < level -> break
+                    itemLevel == level -> {
+                        index++
+                        val children = if (index < items.size && items[index].first > level) {
+                            listOf(ListBlock(ordered = false, items = build(items[index].first)))
+                        } else {
+                            emptyList()
+                        }
+                        result += ListItem(listOf(Paragraph(content)) + children)
+                    }
+                    // A deeper first item without a parent: promote it to this level.
+                    else -> result += ListItem(listOf(ListBlock(ordered = false, items = build(itemLevel))))
+                }
             }
-            else -> rawText
+            return result
         }
+
+        return ListBlock(ordered = false, items = build(items.minOf { it.first }))
     }
 
-    private fun convertTable(table: XWPFTable): String {
+    private fun table(table: XWPFTable): Table? {
         val rows = table.rows
-        if (rows.isEmpty()) return ""
-
-        val sb = StringBuilder()
-        val header = rows[0].tableCells.map { it.text.replace("|", "\\|") }
-        sb.appendLine(header.joinToString(" | ", "| ", " |"))
-        sb.appendLine(header.map { "---" }.joinToString(" | ", "| ", " |"))
-
-        for (i in 1 until rows.size) {
-            val cells = rows[i].tableCells.map { it.text.replace("|", "\\|") }
-            sb.appendLine(cells.joinToString(" | ", "| ", " |"))
-        }
-
-        return sb.toString().trimEnd()
+        if (rows.isEmpty()) return null
+        val header = rows[0].tableCells.map { TableCell(it.text.trim()) }
+        val body = rows.drop(1).map { row -> row.tableCells.map { TableCell(it.text.trim()) } }
+        return Table(header = header, rows = body)
     }
 
     private fun headingLevel(style: String?): Int {
         val s = style?.replace("\\s+".toRegex(), "") ?: return 0
         if (s.startsWith("Heading", ignoreCase = true)) {
             return s.drop(7).toIntOrNull()?.coerceIn(1, 6) ?: 0
         }
+        // OOXML numeric style IDs 1-6 map directly to heading levels
         return s.toIntOrNull()?.takeIf { it in 1..6 } ?: 0
     }
 }