diff --git a/.gitignore b/.gitignore index e3c4105..ded06dc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ xcuserdata .kotlin #So we don't accidentally commit our private keys *.gpg -*.py \ No newline at end of file +*.py!scripts/*.py diff --git a/README.md b/README.md index 0f8980f..403ed5a 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,32 @@ Kotlin Multiplatform (JVM + Android) library that converts documents to Markdown | Plain text | `.txt` and others | | Markdown | `.md` (passthrough) | +## Architecture + +Every format is parsed into one shared document model, and a single renderer serializes that model +to GitHub-Flavored Markdown: + +``` +bytes ──► MimeDetector ──► DocumentConverter.parse ──► Document ──► MarkdownRenderer ──► Markdown + (per format) (blocks, (one GFM + inlines, serializer) + tables, + assets) +``` + +Converters contain no Markdown syntax, so escaping, table shaping, list indentation and spacing are +fixed once for all formats. The model is public: `mid.parse(path)` returns the `Document`, and +`ConversionResult.document` exposes it alongside the rendered Markdown. + +```kotlin +val document = mid.parse("/path/to/report.docx") +document.blocks.filterIsInstance().forEach { println(it.rows.size) } + +// Render with different options +val compact = MarkdownRenderer(MarkdownOptions(padTableColumns = true, imagesAsText = true)) +println(compact.render(document)) +``` + ## Setup ```kotlin @@ -69,8 +95,10 @@ class MyConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean = info.extension == "xyz" - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult = - ConversionResult(markdown = String(bytes)) + override fun parse(bytes: ByteArray, info: StreamInfo): Document = document { + heading(1, "Custom") + paragraph(bytes.decodeToString()) + } } val mid = MarkItDown() @@ -99,3 +127,18 @@ mid.register(HtmlConverter()) | `FileConversionException` | Converter threw during conversion | Both extend `MarkItDownException`. + +## Benchmark + +`scripts/benchmark.py` converts the test fixtures with MikroMarkdown, Python +[markitdown](https://github.com/microsoft/markitdown) and Rust +[anydoc](https://github.com/firecrawl/anydoc), then reports content recall, structure counts, table +integrity and timings to `build/benchmark/report.md`: + +```bash +./gradlew :cli:installDist +python3 scripts/benchmark.py +``` + +Engines whose CLI is missing are skipped. anydoc only handles binary formats, so it sits out the +HTML/JSON/XML fixtures. diff --git a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt index fbb9632..d7ca118 100644 --- a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt +++ b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt @@ -33,4 +33,11 @@ class MarkItDownCommand : CliktCommand(name = "markitdown") { } } -fun main(args: Array) = MarkItDownCommand().main(args) +fun main(args: Array) { + // PDFBox pulls in the Log4j API; without a provider it writes a banner to stdout, + // which would corrupt the Markdown we print there. + System.setProperty("log4j2.loggerContextFactory", "org.apache.logging.log4j.simple.SimpleLoggerContextFactory") + System.setProperty("log4j2.simplelogLevel", "OFF") + System.setProperty("log4j2.statusLoggerLevel", "OFF") + MarkItDownCommand().main(args) +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9793c62..c6771a9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,7 +8,6 @@ kotlinx-io = "0.9.0" kotlinx-resources = "0.15.0" commons-csv = "1.14.1" -flexmark = "0.64.8" jackson = "2.21.3" jsoup = "1.22.2" junit = "6.1.0" @@ -25,7 +24,6 @@ kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.re kotlinx-resources = { module = "com.goncalossilva:resources", version.ref = "kotlinx-resources" } commons-csv = { module = "org.apache.commons:commons-csv", version.ref = "commons-csv" } -flexmark-html2md = { module = "com.vladsch.flexmark:flexmark-html2md-converter", version.ref = "flexmark" } jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" } diff --git a/library/build.gradle.kts b/library/build.gradle.kts index a277013..b3a51cf 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -41,7 +41,6 @@ kotlin { jvmMain.dependencies { implementation(libs.jsoup) - implementation(libs.flexmark.html2md) implementation(libs.jackson.kotlin) implementation(libs.commons.csv) implementation(libs.poi.ooxml) @@ -51,7 +50,6 @@ kotlin { androidMain.dependencies { implementation(libs.jsoup) - implementation(libs.flexmark.html2md) implementation(libs.jackson.kotlin) implementation(libs.commons.csv) implementation(libs.poi.ooxml) diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt index 71c1976..b93bda6 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt @@ -1,8 +1,10 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser import java.io.InputStreamReader @@ -12,28 +14,19 @@ class CsvConverter : DocumentConverter { return info.extension == "csv" || info.mimetype in setOf("text/csv", "application/csv") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val reader = InputStreamReader(bytes.inputStream(), Charsets.UTF_8) - val allRecords = CSVParser(reader, CSVFormat.DEFAULT.builder().setTrim(true).build()).records + val records = CSVParser(reader, CSVFormat.DEFAULT.builder().setTrim(true).build()).records + if (records.isEmpty()) return Document() - if (allRecords.isEmpty()) return ConversionResult(markdown = "") + val header = records[0].toList() + if (header.isEmpty()) return Document() - val headers = allRecords[0].toList() - if (headers.isEmpty()) return ConversionResult(markdown = "") - - val sb = StringBuilder() - sb.appendLine(headers.map { it.escapeCell() }.joinToString(" | ", "| ", " |")) - sb.appendLine(headers.map { "---" }.joinToString(" | ", "| ", " |")) - - for (i in 1 until allRecords.size) { - val cells = (0 until headers.size).map { col -> - allRecords[i].get(col).escapeCell() - } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) + val rows = records.drop(1).map { record -> + // Ragged rows are padded by the renderer; only extra columns need trimming here. + List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } } - return ConversionResult(markdown = sb.toString().trimEnd()) + return Document(blocks = listOf(Table(header = header.map { TableCell(it) }, rows = rows))) } - - private fun String.escapeCell(): String = replace("|", "\\|").replace("\n", " ") } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt index d7dce7a..b1bc8e6 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt @@ -1,12 +1,25 @@ package io.github.lemcoder.mikromarkdown.converters -import java.util.Base64 -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Asset +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.Image +import io.github.lemcoder.mikromarkdown.model.Inline +import io.github.lemcoder.mikromarkdown.model.ListBlock +import io.github.lemcoder.mikromarkdown.model.ListItem +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text +import io.github.lemcoder.mikromarkdown.model.plainText +import io.github.lemcoder.mikromarkdown.model.styled import org.apache.poi.xwpf.usermodel.XWPFDocument import org.apache.poi.xwpf.usermodel.XWPFParagraph import org.apache.poi.xwpf.usermodel.XWPFTable +import java.util.Base64 class DocxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -14,92 +27,128 @@ class DocxConverter : DocumentConverter { info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - val doc = XWPFDocument(bytes.inputStream()) - val sb = StringBuilder() - var title: String? = null - - for (element in doc.bodyElements) { - when (element) { - is XWPFParagraph -> { - val md = convertParagraph(element) - if (md.isNotBlank()) { - if (title == null && headingLevel(element.styleID) > 0) { - title = element.text + override fun parse(bytes: ByteArray, info: StreamInfo): Document { + val docx = XWPFDocument(bytes.inputStream()) + try { + val blocks = mutableListOf() + val assets = mutableListOf() + var title: String? = docx.properties?.coreProperties?.title?.trim()?.ifBlank { null } + val pendingListItems = mutableListOf>>() + + fun flushList() { + if (pendingListItems.isEmpty()) return + blocks += buildNestedList(pendingListItems) + pendingListItems.clear() + } + + for (element in docx.bodyElements) { + when (element) { + is XWPFParagraph -> { + val content = paragraphInlines(element, assets) + if (content.plainText().isBlank() && content.none { it is Image }) continue + + val level = headingLevel(element.styleID) + when { + level > 0 -> { + flushList() + if (title == null) title = content.plainText().trim() + blocks += Heading(level, content) + } + + element.numID != null -> + pendingListItems += (element.numIlvl?.toInt() ?: 0).coerceAtLeast(0) to content + + else -> { + flushList() + blocks += Paragraph(content) + } } - sb.appendLine(md) - sb.appendLine() } - } - is XWPFTable -> { - val tableMd = convertTable(element) - if (tableMd.isNotBlank()) { - sb.appendLine(tableMd) + + is XWPFTable -> { + flushList() + table(element)?.let { blocks += it } } } } - } + flushList() - doc.close() - return ConversionResult(markdown = sb.toString(), title = title) + return Document(blocks = blocks, title = title, assets = assets) + } finally { + docx.close() + } } - private fun convertParagraph(para: XWPFParagraph): String { - val rawText = para.runs.joinToString("") { run -> - val pics = run.embeddedPictures - if (pics.isNotEmpty()) { - return@joinToString pics.joinToString("") { pic -> - val descr = pic.description ?: "" - val data = pic.pictureData - if (data != null) { - val mime = data.pictureTypeEnum.contentType - val b64 = Base64.getEncoder().encodeToString(data.data) - "![$descr](data:$mime;base64,$b64)" - } else { - "![$descr]()" + private fun paragraphInlines(para: XWPFParagraph, assets: MutableList): List { + val out = mutableListOf() + for (run in para.runs) { + val pictures = run.embeddedPictures + if (pictures.isNotEmpty()) { + for (picture in pictures) { + val data = picture.pictureData + val alt = picture.description.orEmpty() + if (data == null) { + if (alt.isNotBlank()) out += Text(alt) + continue } + val mime = data.pictureTypeEnum.contentType + val id = data.fileName ?: "image-${assets.size + 1}" + assets += Asset(id = id, mediaType = mime, bytes = data.data, name = data.fileName) + out += Image( + alt = alt, + url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", + assetId = id, + ) } + continue } - var text = run.text() ?: "" - if (text.isBlank()) return@joinToString text - when { - run.isBold && run.isItalic -> "***$text***" - run.isBold -> "**$text**" - run.isItalic -> "*$text*" - else -> text + + val text = run.text() ?: continue + if (text.isEmpty()) continue + if (text.isBlank()) { + out += Text(text) + continue } + out += styled(listOf(Text(text)), bold = run.isBold, italic = run.isItalic, strike = run.isStrikeThrough) } + return out + } - if (rawText.isBlank()) return "" - - val level = headingLevel(para.styleID) - val isListItem = para.numID != null + /** Rebuilds Word's flat numbering levels into nested list blocks. */ + private fun buildNestedList(items: List>>): ListBlock { + var index = 0 - return when { - level > 0 -> "${"#".repeat(level)} $rawText" - isListItem -> { - val indent = " ".repeat((para.numIlvl?.toInt() ?: 0).coerceAtLeast(0)) - "$indent- $rawText" + fun build(level: Int): List { + val result = mutableListOf() + while (index < items.size) { + val (itemLevel, content) = items[index] + when { + itemLevel < level -> break + itemLevel == level -> { + index++ + val children = if (index < items.size && items[index].first > level) { + listOf(ListBlock(ordered = false, items = build(items[index].first))) + } else { + emptyList() + } + result += ListItem(listOf(Paragraph(content)) + children) + } + // A deeper first item without a parent: promote it to this level. + else -> result += ListItem(listOf(ListBlock(ordered = false, items = build(itemLevel)))) + } } - else -> rawText + return result } + + return ListBlock(ordered = false, items = build(items.minOf { it.first })) } - private fun convertTable(table: XWPFTable): String { + private fun table(table: XWPFTable): Table? { val rows = table.rows - if (rows.isEmpty()) return "" - - val sb = StringBuilder() - val header = rows[0].tableCells.map { it.text.replace("|", "\\|") } - sb.appendLine(header.joinToString(" | ", "| ", " |")) - sb.appendLine(header.map { "---" }.joinToString(" | ", "| ", " |")) - - for (i in 1 until rows.size) { - val cells = rows[i].tableCells.map { it.text.replace("|", "\\|") } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) - } - - return sb.toString().trimEnd() + if (rows.isEmpty()) return null + val header = rows[0].tableCells.map { TableCell(it.text.trim()) } + val body = rows.drop(1).map { row -> row.tableCells.map { TableCell(it.text.trim()) } } + return Table(header = header, rows = body) } private fun headingLevel(style: String?): Int { @@ -107,6 +156,7 @@ class DocxConverter : DocumentConverter { if (s.startsWith("Heading", ignoreCase = true)) { return s.drop(7).toIntOrNull()?.coerceIn(1, 6) ?: 0 } + // OOXML numeric style IDs 1-6 map directly to heading levels return s.toIntOrNull()?.takeIf { it in 1..6 } ?: 0 } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt index e9b0183..9c80056 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt @@ -1,9 +1,13 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo -import io.github.lemcoder.mikromarkdown.utils.HtmlToMarkdown +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Strong +import io.github.lemcoder.mikromarkdown.model.Text +import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument import org.w3c.dom.Element import org.xml.sax.InputSource import java.io.StringReader @@ -11,45 +15,46 @@ import java.util.zip.ZipInputStream import javax.xml.parsers.DocumentBuilderFactory class EpubConverter : DocumentConverter { + private val metaFields = listOf( + "title" to "Title", + "creator" to "Authors", + "language" to "Language", + "description" to "Description", + "identifier" to "Identifier", + ) + override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "epub" || info.mimetype == "application/epub+zip" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val entries = readZip(bytes) - val containerXml = entries["META-INF/container.xml"] ?: return ConversionResult(markdown = "") - val opfPath = parseOpfPath(containerXml) ?: return ConversionResult(markdown = "") - - val opfBytes = entries[opfPath] ?: entries[opfPath.removePrefix("/")] ?: return ConversionResult(markdown = "") + val containerXml = entries["META-INF/container.xml"] ?: return Document() + val opfPath = parseOpfPath(containerXml) ?: return Document() + val opfBytes = entries[opfPath] ?: entries[opfPath.removePrefix("/")] ?: return Document() val opfDir = opfPath.substringBeforeLast("/", "") val (manifest, spine, metadata) = parseOpf(opfBytes) - val sb = StringBuilder() + val blocks = mutableListOf() var title: String? = metadata["title"] - val metaFields = listOf("title" to "Title", "creator" to "Authors", "language" to "Language", - "description" to "Description", "identifier" to "Identifier") for ((key, label) in metaFields) { - metadata[key]?.let { sb.appendLine("**$label:** $it") } + val value = metadata[key] ?: continue + blocks += Paragraph(listOf(Strong(listOf(Text("$label:"))), Text(" $value"))) } - if (sb.isNotEmpty()) sb.appendLine() for (idref in spine) { val href = manifest[idref] ?: continue val fullPath = if (opfDir.isEmpty()) href else "$opfDir/$href" val htmlBytes = entries[fullPath] ?: entries[fullPath.removePrefix("/")] ?: continue - val html = htmlBytes.toString(Charsets.UTF_8) - val (markdown, chapterTitle) = HtmlToMarkdown.convert(html) - if (title == null && chapterTitle != null) title = chapterTitle - if (markdown.isNotBlank()) { - sb.appendLine(markdown) - sb.appendLine() - } + val chapter = HtmlToDocument.parse(htmlBytes.toString(Charsets.UTF_8)) + if (title == null) title = chapter.title + blocks += chapter.blocks } - return ConversionResult(markdown = sb.toString(), title = title) + return Document(blocks = blocks, title = title, metadata = metadata) } private fun readZip(bytes: ByteArray): Map { diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt index 9898f16..fb5dffd 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt @@ -1,9 +1,9 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo -import io.github.lemcoder.mikromarkdown.utils.HtmlToMarkdown +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument class HtmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -11,9 +11,6 @@ class HtmlConverter : DocumentConverter { info.mimetype in setOf("text/html", "application/xhtml+xml") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - val html = bytes.toString(Charsets.UTF_8) - val (markdown, title) = HtmlToMarkdown.convert(html) - return ConversionResult(markdown = markdown, title = title) - } + override fun parse(bytes: ByteArray, info: StreamInfo): Document = + HtmlToDocument.parse(bytes.toString(Charsets.UTF_8), info.localPath.orEmpty()) } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt index 6fd5a28..026c9d1 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt @@ -5,9 +5,10 @@ import com.fasterxml.jackson.core.util.DefaultIndenter import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.CodeBlock +import io.github.lemcoder.mikromarkdown.model.Document class JsonConverter : DocumentConverter { private val writer = ObjectMapper().apply { @@ -25,14 +26,13 @@ class JsonConverter : DocumentConverter { return info.extension == "json" || info.mimetype in setOf("application/json", "text/json") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val json = bytes.toString(Charsets.UTF_8) val pretty = try { - val node = ObjectMapper().readTree(json) - writer.writeValueAsString(node) + writer.writeValueAsString(ObjectMapper().readTree(json)) } catch (_: Exception) { json } - return ConversionResult(markdown = pretty) + return Document(blocks = listOf(CodeBlock(pretty, "json"))) } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt index 3223562..262e38b 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt @@ -1,23 +1,31 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult +import com.tom_roush.pdfbox.pdmodel.PDDocument +import com.tom_roush.pdfbox.text.PDFTextStripper import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo -import com.tom_roush.pdfbox.text.PDFTextStripper -import com.tom_roush.pdfbox.pdmodel.PDDocument +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.utils.plainTextBlocks class PdfConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "pdf" || info.mimetype == "application/pdf" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val doc = PDDocument.load(bytes) - val text = try { - PDFTextStripper().getText(doc) + try { + val title = doc.documentInformation?.title?.trim()?.ifBlank { null } + // Paragraph markers let the text blocks split on real paragraph breaks + // instead of collapsing a page into one block. + val stripper = PDFTextStripper().apply { + sortByPosition = true + setAddMoreFormatting(true) + paragraphStart = "\n" + } + return Document(blocks = plainTextBlocks(stripper.getText(doc)), title = title) } finally { doc.close() } - return ConversionResult(markdown = text) } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt index bf57bd1..4c363e5 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt @@ -1,16 +1,28 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.HtmlComment +import io.github.lemcoder.mikromarkdown.model.Image +import io.github.lemcoder.mikromarkdown.model.ListBlock +import io.github.lemcoder.mikromarkdown.model.ListItem +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text import org.apache.poi.sl.usermodel.Placeholder +import org.apache.poi.sl.usermodel.Shape +import org.apache.poi.xslf.usermodel.XMLSlideShow +import org.apache.poi.xslf.usermodel.XSLFChart import org.apache.poi.xslf.usermodel.XSLFGraphicFrame import org.apache.poi.xslf.usermodel.XSLFGroupShape import org.apache.poi.xslf.usermodel.XSLFPictureShape import org.apache.poi.xslf.usermodel.XSLFSimpleShape import org.apache.poi.xslf.usermodel.XSLFTable import org.apache.poi.xslf.usermodel.XSLFTextShape -import org.apache.poi.xslf.usermodel.XMLSlideShow import org.openxmlformats.schemas.drawingml.x2006.chart.CTAxDataSource import org.openxmlformats.schemas.drawingml.x2006.chart.CTNumDataSource import org.openxmlformats.schemas.drawingml.x2006.chart.CTSerTx @@ -22,204 +34,205 @@ class PptxConverter : DocumentConverter { info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val slideShow = XMLSlideShow(bytes.inputStream()) - val sb = StringBuilder() - var title: String? = null - - for ((index, slide) in slideShow.slides.withIndex()) { - sb.appendLine("") - processShapes(slide.shapes, sb, index == 0) { t -> if (title == null) title = t } - - // Speaker notes - val notes = slide.notes - if (notes != null) { - val notesText = notes.shapes - .filterIsInstance() - .filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } - .joinToString("\n") { it.text } - .trim() - if (notesText.isNotBlank()) { - sb.appendLine() - sb.appendLine("### Notes:") - sb.appendLine(notesText) + try { + val blocks = mutableListOf() + var title: String? = null + + for ((index, slide) in slideShow.slides.withIndex()) { + blocks += HtmlComment("Slide number: ${index + 1}") + blocks += shapeBlocks(slide.shapes) { if (index == 0 && title == null) title = it } + + val notes = slide.notes?.shapes + ?.filterIsInstance() + ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } + ?.joinToString("\n") { it.text } + ?.trim() + .orEmpty() + if (notes.isNotBlank()) { + blocks += Heading(3, listOf(Text("Notes:"))) + blocks += notes.lines().filter { it.isNotBlank() }.map { Paragraph(listOf(Text(it.trim()))) } } } - sb.appendLine() + return Document(blocks = blocks, title = title) + } finally { + slideShow.close() } - - slideShow.close() - return ConversionResult(markdown = sb.toString(), title = title) } - private fun processShapes( - shapes: Iterable>, - sb: StringBuilder, - isFirstSlide: Boolean, - onTitle: (String) -> Unit, - ) { + private fun shapeBlocks(shapes: Iterable>, onTitle: (String) -> Unit): List { + val blocks = mutableListOf() for (shape in shapes) { when { - shape is XSLFGroupShape -> processShapes(shape.shapes, sb, isFirstSlide, onTitle) + shape is XSLFGroupShape -> blocks += shapeBlocks(shape.shapes, onTitle) + shape is XSLFTextShape -> { val text = shape.text.trim() if (text.isBlank()) continue val placeholder = (shape as? XSLFSimpleShape)?.placeholder - val isTitle = placeholder == Placeholder.TITLE || - placeholder == Placeholder.CENTERED_TITLE - - if (isTitle) { - sb.appendLine("# $text") - if (isFirstSlide) onTitle(text) - } else { - for (para in shape.textParagraphs) { - val paraText = para.text.trim() - if (paraText.isBlank()) continue - if (para.isBullet) sb.appendLine("- $paraText") - else sb.appendLine(paraText) + if (placeholder == Placeholder.TITLE || placeholder == Placeholder.CENTERED_TITLE) { + blocks += Heading(1, listOf(Text(text))) + onTitle(text) + continue + } + + // Consecutive bullet paragraphs become one list; plain ones stay paragraphs. + val bullets = mutableListOf() + fun flushBullets() { + if (bullets.isEmpty()) return + blocks += ListBlock(ordered = false, items = bullets.toList()) + bullets.clear() + } + for (para in shape.textParagraphs) { + val paraText = para.text.trim() + if (paraText.isBlank()) continue + if (para.isBullet) { + bullets += ListItem(listOf(Paragraph(listOf(Text(paraText))))) + } else { + flushBullets() + blocks += Paragraph(listOf(Text(paraText))) } } + flushBullets() } + shape is XSLFPictureShape -> { - val descr = (shape.xmlObject as? CTPicture)?.nvPicPr?.cNvPr?.descr ?: "" - val altText = if (descr.isNotBlank()) descr else shape.shapeName + val description = (shape.xmlObject as? CTPicture)?.nvPicPr?.cNvPr?.descr.orEmpty() + val alt = description.ifBlank { shape.shapeName } val filename = shape.shapeName.replace(Regex("\\W"), "") + ".jpg" - sb.appendLine("![$altText]($filename)") - } - shape is XSLFGraphicFrame && shape.hasChart() -> { - sb.append(convertChart(shape.chart)) + blocks += Paragraph(listOf(Image(alt, filename))) } - shape is XSLFTable -> sb.appendLine(convertTable(shape)) + + shape is XSLFGraphicFrame && shape.hasChart() -> blocks += chartBlocks(shape.chart) + + shape is XSLFTable -> table(shape)?.let { blocks += it } } } + return blocks } - private fun convertChart(chart: org.apache.poi.xslf.usermodel.XSLFChart): String { - val sb = StringBuilder() - sb.append("\n\n### Chart") + private fun chartBlocks(chart: XSLFChart): List { + val blocks = mutableListOf() + blocks += Heading(3, listOf(Text(listOfNotNull("Chart", chartTitle(chart)).joinToString(": ")))) - try { - val ctChart = chart.ctChart - if (ctChart.isSetTitle) { - val tx = ctChart.title?.tx - val titleText = when { - tx?.isSetRich == true -> - tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t ?: "" } }.joinToString("") - tx?.isSetStrRef == true -> - tx.strRef?.strCache?.ptList?.firstOrNull()?.v ?: "" - else -> "" - } - if (titleText.isNotBlank()) sb.append(": $titleText") - } - } catch (_: Exception) {} + val series = try { + seriesOf(chart) + } catch (_: Exception) { + blocks += Paragraph(listOf(Text("[unsupported chart]"))) + return blocks + } + if (series.isEmpty()) return blocks + + val rowCount = series.maxOf { it.categories.size } + blocks += Table( + header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, + rows = (0 until rowCount).map { row -> + val category = series.first().categories.getOrElse(row) { "" } + (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } + }, + ) + return blocks + } - sb.append("\n\n") + private fun chartTitle(chart: XSLFChart): String? = try { + val ctChart = chart.ctChart + if (!ctChart.isSetTitle) { + null + } else { + val tx = ctChart.title?.tx + when { + tx?.isSetRich == true -> + tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") + tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v + else -> null + }?.ifBlank { null } + } + } catch (_: Exception) { + null + } - data class SeriesData(val name: String, val categories: List, val values: List) + private data class Series(val name: String, val categories: List, val values: List) - fun catStrings(cat: CTAxDataSource?): List { - if (cat == null) return emptyList() - return when { - cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() - cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - cat.isSetNumLit -> cat.numLit?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - cat.isSetStrLit -> cat.strLit?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() - else -> emptyList() - } + private fun seriesOf(chart: XSLFChart): List { + val plotArea = chart.ctChart.plotArea + val out = mutableListOf() + + fun categoryValues(cat: CTAxDataSource?): List = when { + cat == null -> emptyList() + cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() + cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + cat.isSetNumLit -> cat.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + cat.isSetStrLit -> cat.strLit?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() + else -> emptyList() } - fun valStrings(v: CTNumDataSource?): List { - if (v == null) return emptyList() - return when { - v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - else -> emptyList() - } + fun numericValues(v: CTNumDataSource?): List = when { + v == null -> emptyList() + v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + else -> emptyList() } - fun serTitle(tx: CTSerTx?): String = when { + fun seriesName(tx: CTSerTx?): String = when { tx == null -> "" tx.isSetV -> tx.v - tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v ?: "" + tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v.orEmpty() else -> "" } - val seriesList = mutableListOf() - try { - val plotArea = chart.ctChart.plotArea - - for (c in plotArea.barChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.bar3DChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.lineChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.line3DChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.areaChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.area3DChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.scatterChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetXVal) s.xVal else null), - valStrings(if (s.isSetYVal) s.yVal else null))) - - } catch (_: Exception) { - return sb.append("[unsupported chart]\n\n").toString() - } - - if (seriesList.isEmpty()) return sb.toString() - - val header = listOf("Category") + seriesList.map { it.name } - sb.appendLine("| " + header.joinToString(" | ") + " |") - sb.appendLine("|" + "---|".repeat(header.size)) - - val numRows = seriesList.maxOfOrNull { it.categories.size } ?: 0 - for (i in 0 until numRows) { - val cat = seriesList.first().categories.getOrElse(i) { "" } - val vals = seriesList.map { it.values.getOrElse(i) { "" } } - sb.appendLine("| " + (listOf(cat) + vals).joinToString(" | ") + " |") - } - - return sb.toString() + for (c in plotArea.barChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.bar3DChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.lineChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.line3DChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.areaChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.area3DChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.scatterChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetXVal) s.xVal else null), + numericValues(if (s.isSetYVal) s.yVal else null), + ) + for (c in plotArea.pieChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + + return out.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } } - private fun convertTable(table: XSLFTable): String { + private fun table(table: XSLFTable): Table? { val rows = table.rows - if (rows.isEmpty()) return "" - - val sb = StringBuilder() - val header = rows[0].cells.map { it.text.trim().replace("|", "\\|") } - sb.appendLine(header.joinToString(" | ", "| ", " |")) - sb.appendLine(header.map { "---" }.joinToString(" | ", "| ", " |")) - - for (i in 1 until rows.size) { - val cells = rows[i].cells.map { it.text.trim().replace("|", "\\|") } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) - } - - return sb.toString().trimEnd() + if (rows.isEmpty()) return null + return Table( + header = rows[0].cells.map { TableCell(it.text.trim()) }, + rows = rows.drop(1).map { row -> row.cells.map { TableCell(it.text.trim()) } }, + ) } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt index 8449c69..984dec0 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt @@ -1,8 +1,13 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text import org.apache.poi.ss.usermodel.Cell import org.apache.poi.ss.usermodel.CellType import org.apache.poi.ss.usermodel.DataFormatter @@ -17,37 +22,31 @@ class XlsxConverter : DocumentConverter { info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val workbook = XSSFWorkbook(bytes.inputStream()) - val sb = StringBuilder() + try { + val blocks = mutableListOf() - for (sheet in workbook) { - val rows = sheet.toList() - if (rows.isEmpty()) continue + for (sheet in workbook) { + val rows = sheet.toList() + if (rows.isEmpty()) continue - val maxCols = rows.maxOf { it.lastCellNum.toInt().coerceAtLeast(0) } - if (maxCols == 0) continue + val columns = rows.maxOf { it.lastCellNum.toInt().coerceAtLeast(0) } + if (columns == 0) continue - sb.appendLine("## ${sheet.sheetName}") - sb.appendLine() - - val headerCells = (0 until maxCols).map { col -> - cellValue(rows[0].getCell(col)).replace("|", "\\|") + blocks += Heading(2, listOf(Text(sheet.sheetName))) + blocks += Table( + header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, + rows = rows.drop(1).map { row -> + (0 until columns).map { TableCell(cellValue(row.getCell(it))) } + }, + ) } - sb.appendLine(headerCells.joinToString(" | ", "| ", " |")) - sb.appendLine(headerCells.map { "---" }.joinToString(" | ", "| ", " |")) - for (i in 1 until rows.size) { - val cells = (0 until maxCols).map { col -> - cellValue(rows[i].getCell(col)).replace("|", "\\|") - } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) - } - sb.appendLine() + return Document(blocks = blocks) + } finally { + workbook.close() } - - workbook.close() - return ConversionResult(markdown = sb.toString()) } private fun cellValue(cell: Cell?): String { diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt index e19d71a..8979d57 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt @@ -1,8 +1,9 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.CodeBlock +import io.github.lemcoder.mikromarkdown.model.Document import org.xml.sax.InputSource import java.io.StringReader import java.io.StringWriter @@ -17,10 +18,9 @@ class XmlConverter : DocumentConverter { return info.extension == "xml" || info.mimetype in setOf("text/xml", "application/xml") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - val xml = bytes.toString(Charsets.UTF_8) - val pretty = prettyPrint(xml) - return ConversionResult(markdown = "```xml\n$pretty\n```") + override fun parse(bytes: ByteArray, info: StreamInfo): Document { + val pretty = prettyPrint(bytes.toString(Charsets.UTF_8)) + return Document(blocks = listOf(CodeBlock(pretty, "xml"))) } private fun prettyPrint(xml: String): String = try { diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt new file mode 100644 index 0000000..d7c9296 --- /dev/null +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt @@ -0,0 +1,266 @@ +package io.github.lemcoder.mikromarkdown.utils + +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.BlockQuote +import io.github.lemcoder.mikromarkdown.model.CodeBlock +import io.github.lemcoder.mikromarkdown.model.CodeSpan +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Emphasis +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.Image +import io.github.lemcoder.mikromarkdown.model.Inline +import io.github.lemcoder.mikromarkdown.model.LineBreak +import io.github.lemcoder.mikromarkdown.model.Link +import io.github.lemcoder.mikromarkdown.model.ListBlock +import io.github.lemcoder.mikromarkdown.model.ListItem +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Strikethrough +import io.github.lemcoder.mikromarkdown.model.Strong +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text +import io.github.lemcoder.mikromarkdown.model.ThematicBreak +import io.github.lemcoder.mikromarkdown.model.plainText +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.nodes.Node +import org.jsoup.nodes.TextNode + +/** + * Walks an HTML DOM into the shared document model. + * + * Replaces the previous HTML → Markdown string conversion: tables, lists and inline + * emphasis become model nodes, so the Markdown renderer owns all syntax decisions. + */ +object HtmlToDocument { + + private val DROPPED_TAGS = setOf( + "script", "style", "noscript", "template", "button", "svg", "iframe", "form", "input", "select", + ) + + private val HEADINGS = mapOf("h1" to 1, "h2" to 2, "h3" to 3, "h4" to 4, "h5" to 5, "h6" to 6) + + /** Tags that only group other content; their children are lifted into the parent block flow. */ + private val CONTAINERS = setOf( + "div", "section", "article", "main", "header", "footer", "aside", "nav", "body", "html", + "figure", "details", "summary", "fieldset", "center", "hgroup", "picture", "colgroup", + ) + + fun parse(html: String, baseUri: String = ""): Document { + val doc = Jsoup.parse(html, baseUri) + doc.select(DROPPED_TAGS.joinToString(", ")).remove() + val title = doc.title().ifBlank { null } + val root = doc.body() ?: doc + val blocks = blocks(root) + return Document(blocks = blocks, title = title ?: blocks.headingTitle()) + } + + private fun List.headingTitle(): String? = + (firstOrNull { it is Heading } as? Heading)?.content?.plainText()?.trim()?.ifBlank { null } + + /** Converts an element's children into blocks, flushing runs of inline content into paragraphs. */ + private fun blocks(parent: Element): List { + val out = mutableListOf() + val pending = mutableListOf() + + fun flush() { + val trimmed = pending.trimEdges() + if (trimmed.isNotEmpty()) out += Paragraph(trimmed) + pending.clear() + } + + for (node in parent.childNodes()) { + when { + node is TextNode -> { + val text = node.normalizedText() + if (text.isNotEmpty()) pending += Text(text) + } + + node is Element && node.isBlockLevel() -> { + flush() + out += blockFor(node) + } + + node is Element && node.tagName() == "br" -> pending += LineBreak + + // Inline elements (links, emphasis, …) keep their wrapper — appendNode, not children. + node is Element -> pending.appendNode(node) + } + } + flush() + return out + } + + private fun Element.isBlockLevel(): Boolean = when (tagName()) { + in HEADINGS, "p", "ul", "ol", "table", "pre", "blockquote", "hr", "dl", "li", "figcaption" -> true + in CONTAINERS -> true + else -> false + } + + private fun blockFor(element: Element): List = when (val tag = element.tagName()) { + in HEADINGS -> listOfNotNull( + inlines(element).trimEdges() + .takeIf { it.isNotEmpty() } + ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) }, + ) + + "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) + "ul", "ol" -> listBlock(element) + "table" -> table(element) + "pre" -> listOf(codeBlock(element)) + "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() + "hr" -> listOf(ThematicBreak) + "dl" -> definitionList(element) + "figcaption" -> listOfNotNull( + inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) }, + ) + // Containers and stray
  • outside a list contribute their children directly. + else -> blocks(element) + } + + private fun listBlock(element: Element): List { + val ordered = element.tagName() == "ol" + val start = element.attr("start").toIntOrNull() ?: 1 + val items = element.children() + .filter { it.tagName() == "li" } + .map { li -> ListItem(blocks = blocks(li)) } + .filter { it.blocks.isNotEmpty() } + if (items.isEmpty()) return emptyList() + return listOf(ListBlock(ordered = ordered, items = items, start = start)) + } + + private fun codeBlock(element: Element): Block { + val code = element.selectFirst("code") ?: element + val language = code.classNames() + .firstOrNull { it.startsWith("language-") || it.startsWith("lang-") } + ?.substringAfter('-') + return CodeBlock(code.wholeText().trimEnd(), language) + } + + private fun definitionList(element: Element): List { + val out = mutableListOf() + for (child in element.children()) { + val content = inlines(child).trimEdges() + if (content.isEmpty()) continue + when (child.tagName()) { + "dt" -> out += Paragraph(listOf(Strong(content))) + "dd" -> out += Paragraph(content) + } + } + return out + } + + private fun table(element: Element): List { + val caption = element.selectFirst("caption")?.let { inlines(it).trimEdges() } ?: emptyList() + val rows = element.select("tr").filter { it.parentTable() === element } + + var header: List = emptyList() + val body = mutableListOf>() + + for ((index, tr) in rows.withIndex()) { + val cells = tr.children() + .filter { it.tagName() == "th" || it.tagName() == "td" } + .map { cell -> + TableCell( + content = inlines(cell).trimEdges(), + colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + ) + } + if (cells.isEmpty()) continue + + val isHeaderRow = index == 0 && + tr.children().all { it.tagName() == "th" } && + (tr.parent()?.tagName() == "thead" || header.isEmpty()) + + if (isHeaderRow && header.isEmpty()) header = cells else body += cells + } + + if (header.isEmpty() && body.isEmpty()) return emptyList() + return listOf(Table(header = header, rows = body, caption = caption)) + } + + /** The nearest enclosing table, so nested tables do not steal each other's rows. */ + private fun Element.parentTable(): Element? = parents().firstOrNull { it.tagName() == "table" } + + private fun inlines(element: Element): List { + val out = mutableListOf() + for (node in element.childNodes()) out.appendNode(node) + return out + } + + private fun MutableList.appendNode(node: Node) { + when (node) { + is TextNode -> { + val text = node.normalizedText() + if (text.isNotEmpty()) add(Text(text)) + } + + is Element -> when (node.tagName()) { + "br" -> add(LineBreak) + "img" -> { + val src = node.attr("abs:src").ifBlank { node.attr("src") } + val alt = node.attr("alt") + if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) + } + + "a" -> { + val href = node.attr("abs:href").ifBlank { node.attr("href") } + val content = inlines(node).trimEdges() + when { + content.isEmpty() -> Unit + href.isBlank() || href.startsWith("javascript:") -> addAll(content) + else -> add(Link(content, href, node.attr("title").ifBlank { null })) + } + } + + "strong", "b" -> wrapped(node) { Strong(it) } + "em", "i", "cite", "var" -> wrapped(node) { Emphasis(it) } + "del", "s", "strike" -> wrapped(node) { Strikethrough(it) } + "code", "kbd", "samp", "tt" -> { + val code = node.wholeText().trim() + if (code.isNotEmpty()) add(CodeSpan(code)) + } + + // Block-level content encountered inline (e.g. a
    inside a
    • ): keep its text. + else -> addAll(inlines(node)) + } + } + } + + private inline fun MutableList.wrapped(node: Element, wrap: (List) -> Inline) { + val content = inlines(node) + val trimmed = content.trimEdges() + if (trimmed.isEmpty()) return + if (content.startsWithSpace()) add(Text(" ")) + add(wrap(trimmed)) + if (content.endsWithSpace()) add(Text(" ")) + } + + /** HTML collapses runs of whitespace; do the same before the text reaches the model. */ + // Non-breaking spaces are not collapsible whitespace in HTML, so they survive verbatim. + private fun TextNode.normalizedText(): String = + wholeText.replace(Regex("\\s+"), " ") + + private fun List.startsWithSpace(): Boolean = + (firstOrNull() as? Text)?.value?.startsWith(" ") == true + + private fun List.endsWithSpace(): Boolean = + (lastOrNull() as? Text)?.value?.endsWith(" ") == true + + /** Drops leading/trailing whitespace-only text so emphasis markers hug their content. */ + private fun List.trimEdges(): List { + var start = 0 + var end = size + while (start < end && this[start].isBlankText()) start++ + while (end > start && this[end - 1].isBlankText()) end-- + if (start >= end) return emptyList() + val slice = subList(start, end).toMutableList() + (slice.first() as? Text)?.let { slice[0] = Text(it.value.trimStart()) } + (slice.last() as? Text)?.let { slice[slice.lastIndex] = Text(it.value.trimEnd()) } + return slice.filter { !(it is Text && it.value.isEmpty()) } + } + + // Kotlin's trim()/isBlank() drop Unicode spacing (thin, hair, …) but keep NBSP, which is content. + private fun Inline.isBlankText(): Boolean = this is Text && value.isBlank() +} diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToMarkdown.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToMarkdown.kt deleted file mode 100644 index b116f05..0000000 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToMarkdown.kt +++ /dev/null @@ -1,22 +0,0 @@ -package io.github.lemcoder.mikromarkdown.utils - -import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter -import com.vladsch.flexmark.util.data.MutableDataSet -import org.jsoup.Jsoup - -object HtmlToMarkdown { - private val options = MutableDataSet().apply { - set(FlexmarkHtmlConverter.SETEXT_HEADINGS, false) - } - private val converter = FlexmarkHtmlConverter.builder(options).build() - - fun convert(html: String, baseUri: String = ""): Pair { - val doc = Jsoup.parse(html, baseUri) - doc.select("script, style, button").remove() - doc.select("th[scope=row]").tagName("td") - val title = doc.title().ifEmpty { null } - val body = doc.body().html() - val markdown = converter.convert(body) - return markdown to title - } -} diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/ConversionResult.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/ConversionResult.kt index 36e6d66..1f94fff 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/ConversionResult.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/ConversionResult.kt @@ -1,6 +1,10 @@ package io.github.lemcoder.mikromarkdown +import io.github.lemcoder.mikromarkdown.model.Document + data class ConversionResult( val markdown: String, val title: String? = null, + /** The intermediate model the Markdown was rendered from. */ + val document: Document = Document(), ) diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt index 18bbe9a..07310a2 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt @@ -1,6 +1,13 @@ package io.github.lemcoder.mikromarkdown +import io.github.lemcoder.mikromarkdown.model.Document + +/** + * Parses one input format into the shared [Document] model. + * + * Converters never produce Markdown — rendering is [io.github.lemcoder.mikromarkdown.render.MarkdownRenderer]'s job. + */ interface DocumentConverter { fun accepts(bytes: ByteArray, info: StreamInfo): Boolean - fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult + fun parse(bytes: ByteArray, info: StreamInfo): Document } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt index 7b20f19..0514b9f 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt @@ -1,47 +1,62 @@ package io.github.lemcoder.mikromarkdown -import io.github.lemcoder.mikromarkdown.utils.MarkdownNormalizer +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.render.MarkdownRenderer import kotlinx.io.buffered import kotlinx.io.files.Path import kotlinx.io.files.SystemFileSystem import kotlinx.io.readByteArray -class MikroMarkdown(private val mimeDetector: MimeDetector) { +/** + * Pipeline entry point: detect format, parse to a [Document], render Markdown. + * + * Rendering happens here rather than inside converters, so every format shares one serializer. + */ +class MikroMarkdown( + private val mimeDetector: MimeDetector, + private val renderer: MarkdownRenderer = MarkdownRenderer.Default, +) { private val converters = mutableListOf>() fun register(converter: DocumentConverter, priority: Double = 0.0) { converters.add(converter to priority) } - fun convert(path: String): ConversionResult { + fun convert(path: String): ConversionResult = render(parse(path)) + + fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult = render(parse(bytes, info)) + + /** Parses without rendering, for callers that want the document model itself. */ + fun parse(path: String): Document { val info = mimeDetector.detect(path) val bytes = SystemFileSystem.source(Path(path)).buffered().use { it.readByteArray() } - return dispatch(bytes, info) - } - - fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - return dispatch(bytes, info) + return parse(bytes, info) } - private fun dispatch(bytes: ByteArray, info: StreamInfo): ConversionResult { + fun parse(bytes: ByteArray, info: StreamInfo): Document { val sorted = converters.sortedBy { it.second } for ((converter, _) in sorted) { - if (converter.accepts(bytes, info)) { - val result = try { - converter.convert(bytes, info) - } catch (e: MarkItDownException) { - throw e - } catch (e: Exception) { - throw FileConversionException( - "Conversion failed with ${converter::class.simpleName}: ${e.message}", - e, - ) - } - return result.copy(markdown = MarkdownNormalizer.normalize(result.markdown)) + if (!converter.accepts(bytes, info)) continue + return try { + converter.parse(bytes, info) + } catch (e: MarkItDownException) { + throw e + } catch (e: Exception) { + throw FileConversionException( + "Conversion failed with ${converter::class.simpleName}: ${e.message}", + e, + ) } } throw UnsupportedFormatException( "No converter found for: ${info.extension ?: info.mimetype ?: "unknown"}", ) } + + fun render(document: Document): ConversionResult = + ConversionResult( + markdown = renderer.render(document), + title = document.title, + document = document, + ) } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/MarkdownPassthroughConverter.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/MarkdownPassthroughConverter.kt index 67cb53f..98253cc 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/MarkdownPassthroughConverter.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/MarkdownPassthroughConverter.kt @@ -1,15 +1,15 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.RawBlock class MarkdownPassthroughConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension in setOf("md", "markdown") || info.mimetype == "text/markdown" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - return ConversionResult(markdown = bytes.decodeToString()) - } + override fun parse(bytes: ByteArray, info: StreamInfo): Document = + Document(blocks = listOf(RawBlock(bytes.decodeToString()))) } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt index 3e84585..cc095d5 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt @@ -1,8 +1,9 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.RawBlock class PlainTextConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -10,7 +11,6 @@ class PlainTextConverter : DocumentConverter { info.mimetype == "text/plain" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - return ConversionResult(markdown = bytes.decodeToString()) - } + override fun parse(bytes: ByteArray, info: StreamInfo): Document = + Document(blocks = listOf(RawBlock(bytes.decodeToString()))) } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt new file mode 100644 index 0000000..6b6c205 --- /dev/null +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt @@ -0,0 +1,127 @@ +package io.github.lemcoder.mikromarkdown.model + +/** + * Format-independent document model. + * + * Every converter parses its input into a [Document]; a single renderer turns + * documents into Markdown. Output quirks are therefore fixed once, in the + * renderer, rather than per format. + */ +data class Document( + val blocks: List = emptyList(), + val title: String? = null, + val metadata: Map = emptyMap(), + val assets: List = emptyList(), +) + +/** An embedded binary resource (image, thumbnail, object) referenced by [Image.assetId]. */ +data class Asset( + val id: String, + val mediaType: String, + val bytes: ByteArray, + val name: String? = null, +) { + override fun equals(other: Any?): Boolean = + this === other || (other is Asset && id == other.id && mediaType == other.mediaType && name == other.name) + + override fun hashCode(): Int = id.hashCode() +} + +sealed interface Block + +data class Heading( + val level: Int, + val content: List, + val anchor: String? = null, +) : Block + +data class Paragraph(val content: List) : Block + +data class CodeBlock(val code: String, val language: String? = null) : Block + +data class BlockQuote(val blocks: List) : Block + +data class ListBlock( + val ordered: Boolean, + val items: List, + val start: Int = 1, + /** GFM loose lists put a blank line between items. */ + val loose: Boolean = false, +) : Block + +data class ListItem( + val blocks: List, + /** Non-null makes this a GFM task-list item. */ + val checked: Boolean? = null, +) + +data class Table( + val header: List = emptyList(), + val rows: List> = emptyList(), + val alignments: List = emptyList(), + val caption: List = emptyList(), +) : Block + +data class TableCell( + val content: List, + val colSpan: Int = 1, + val rowSpan: Int = 1, +) { + constructor(text: String) : this(if (text.isEmpty()) emptyList() else listOf(Text(text))) +} + +enum class Alignment { NONE, LEFT, CENTER, RIGHT } + +data object ThematicBreak : Block + +/** Rendered verbatim as an HTML comment. Used for structural markers such as slide numbers. */ +data class HtmlComment(val text: String) : Block + +/** Escape hatch for content that is already Markdown (or must not be touched). */ +data class RawBlock(val text: String) : Block + +sealed interface Inline + +data class Text(val value: String) : Inline + +data class Strong(val content: List) : Inline + +data class Emphasis(val content: List) : Inline + +data class Strikethrough(val content: List) : Inline + +data class CodeSpan(val code: String) : Inline + +data class Link(val content: List, val url: String, val title: String? = null) : Inline + +data class Image( + val alt: String, + val url: String, + val title: String? = null, + val assetId: String? = null, +) : Inline + +/** Hard line break inside a paragraph. */ +data object LineBreak : Inline + +/** Inline content that is already Markdown/HTML and must be emitted verbatim. */ +data class RawInline(val text: String) : Inline + +/** Flattens inline content to its plain-text form (used for titles, anchors, alt text). */ +fun List.plainText(): String = buildString { appendPlain(this@plainText) } + +private fun StringBuilder.appendPlain(inlines: List) { + for (inline in inlines) { + when (inline) { + is Text -> append(inline.value) + is Strong -> appendPlain(inline.content) + is Emphasis -> appendPlain(inline.content) + is Strikethrough -> appendPlain(inline.content) + is CodeSpan -> append(inline.code) + is Link -> appendPlain(inline.content) + is Image -> append(inline.alt) + is RawInline -> append(inline.text) + LineBreak -> append(' ') + } + } +} diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt new file mode 100644 index 0000000..a5fd5fa --- /dev/null +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt @@ -0,0 +1,151 @@ +package io.github.lemcoder.mikromarkdown.model + +/** + * Small builder used by converters so parsing code stays free of Markdown syntax. + * + * ``` + * document { + * heading(1, "Report") + * paragraph("Body text") + * table(header = listOf("A", "B"), rows = listOf(listOf("1", "2"))) + * } + * ``` + */ +fun document(block: DocumentBuilder.() -> Unit): Document = DocumentBuilder().apply(block).build() + +class DocumentBuilder { + private val blocks = mutableListOf() + private val assets = mutableListOf() + private val metadata = mutableMapOf() + var title: String? = null + + fun add(block: Block) { + blocks += block + } + + fun addAll(blocks: Iterable) { + this.blocks += blocks + } + + fun asset(asset: Asset) { + assets += asset + } + + fun meta(key: String, value: String?) { + if (!value.isNullOrBlank()) metadata[key] = value + } + + /** Records [text] as the document title unless one was already found. */ + fun titleIfAbsent(text: String?) { + if (title == null && !text.isNullOrBlank()) title = text + } + + fun heading(level: Int, text: String) { + if (text.isNotBlank()) add(Heading(level, listOf(Text(text)))) + } + + fun heading(level: Int, content: List) { + if (content.isNotEmpty()) add(Heading(level, content)) + } + + fun paragraph(text: String) { + if (text.isNotBlank()) add(Paragraph(listOf(Text(text)))) + } + + fun paragraph(content: List) { + if (content.isNotEmpty()) add(Paragraph(content)) + } + + fun code(code: String, language: String? = null) { + add(CodeBlock(code, language)) + } + + fun comment(text: String) { + add(HtmlComment(text)) + } + + fun raw(text: String) { + if (text.isNotBlank()) add(RawBlock(text)) + } + + fun bulletList(items: List) { + if (items.isEmpty()) return + add(ListBlock(ordered = false, items = items.map { ListItem(listOf(Paragraph(listOf(Text(it))))) })) + } + + fun table( + header: List, + rows: List>, + alignments: List = emptyList(), + ) { + if (header.isEmpty() && rows.isEmpty()) return + add( + Table( + header = header.map { TableCell(it) }, + rows = rows.map { row -> row.map { TableCell(it) } }, + alignments = alignments, + ), + ) + } + + fun build(): Document = Document( + blocks = blocks.toList(), + title = title, + metadata = metadata.toMap(), + assets = assets.toList(), + ) +} + +/** Builds a list of inlines without repeating `listOf(...)` wrappers in parsers. */ +fun inlines(block: InlineBuilder.() -> Unit): List = InlineBuilder().apply(block).build() + +class InlineBuilder { + private val items = mutableListOf() + + fun text(value: String) { + if (value.isNotEmpty()) items += Text(value) + } + + fun strong(value: String) { + if (value.isNotEmpty()) items += Strong(listOf(Text(value))) + } + + fun emphasis(value: String) { + if (value.isNotEmpty()) items += Emphasis(listOf(Text(value))) + } + + fun code(value: String) { + if (value.isNotEmpty()) items += CodeSpan(value) + } + + fun link(text: String, url: String) { + items += Link(listOf(Text(text)), url) + } + + fun image(alt: String, url: String, assetId: String? = null) { + items += Image(alt, url, assetId = assetId) + } + + fun lineBreak() { + items += LineBreak + } + + operator fun plusAssign(inline: Inline) { + items += inline + } + + operator fun plusAssign(inlines: List) { + items += inlines + } + + fun build(): List = items.toList() +} + +/** Wraps [content] in the emphasis combination described by the flags. */ +fun styled(content: List, bold: Boolean, italic: Boolean, strike: Boolean = false): List { + var result = content + if (strike) result = listOf(Strikethrough(result)) + if (italic) result = listOf(Emphasis(result)) + if (bold) result = listOf(Strong(result)) + return result +} diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt new file mode 100644 index 0000000..d457376 --- /dev/null +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt @@ -0,0 +1,369 @@ +package io.github.lemcoder.mikromarkdown.render + +import io.github.lemcoder.mikromarkdown.model.Alignment +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.BlockQuote +import io.github.lemcoder.mikromarkdown.model.CodeBlock +import io.github.lemcoder.mikromarkdown.model.CodeSpan +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Emphasis +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.HtmlComment +import io.github.lemcoder.mikromarkdown.model.Image +import io.github.lemcoder.mikromarkdown.model.Inline +import io.github.lemcoder.mikromarkdown.model.LineBreak +import io.github.lemcoder.mikromarkdown.model.Link +import io.github.lemcoder.mikromarkdown.model.ListBlock +import io.github.lemcoder.mikromarkdown.model.ListItem +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.RawBlock +import io.github.lemcoder.mikromarkdown.model.RawInline +import io.github.lemcoder.mikromarkdown.model.Strikethrough +import io.github.lemcoder.mikromarkdown.model.Strong +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text +import io.github.lemcoder.mikromarkdown.model.ThematicBreak + +data class MarkdownOptions( + val bullet: Char = '-', + val strongMarker: String = "**", + val emphasisMarker: String = "*", + /** Escape Markdown-significant characters in [Text] nodes. */ + val escapeText: Boolean = true, + /** Newlines inside table cells are replaced with this, since GFM rows are single-line. */ + val tableCellLineBreak: String = "
    ", + /** Pad table columns so the source table lines up. Off keeps output compact. */ + val padTableColumns: Boolean = false, + /** Emit a `key: value` YAML front matter block when the document carries metadata. */ + val frontMatter: Boolean = false, + val maxHeadingLevel: Int = 6, + /** Drop images entirely, keeping only their alt text. */ + val imagesAsText: Boolean = false, + /** Longest image URL kept inline; longer ones (e.g. base64 data URIs) are still emitted. */ + val maxInlineImageUrl: Int = Int.MAX_VALUE, +) { + companion object { + val Default = MarkdownOptions() + } +} + +/** Serializes a [Document] to GitHub-Flavored Markdown. The only place Markdown syntax is produced. */ +class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.Default) { + + fun render(document: Document): String { + val body = renderBlocks(document.blocks) + if (!options.frontMatter || document.metadata.isEmpty()) return body + val front = document.metadata.entries.joinToString("\n") { (k, v) -> + "$k: ${v.replace("\n", " ")}" + } + return "---\n$front\n---\n\n$body".trimEnd() + } + + fun render(blocks: List): String = renderBlocks(blocks) + + fun renderInline(inlines: List): String = inlines(inlines, TextContext.INLINE) + + private fun renderBlocks(blocks: List): String { + val chunks = mutableListOf() + for (block in blocks) { + val rendered = renderBlock(block) + if (rendered.isNotEmpty()) chunks += rendered + } + return chunks.joinToString("\n\n").trim() + } + + private fun renderBlock(block: Block): String = when (block) { + is Heading -> { + val text = inlines(block.content, TextContext.HEADING).collapseLines() + if (text.isBlank()) "" else "${"#".repeat(block.level.coerceIn(1, options.maxHeadingLevel))} $text" + } + + is Paragraph -> inlines(block.content, TextContext.BLOCK).trimEnd() + + is CodeBlock -> { + val fence = "`".repeat(maxOf(3, longestBacktickRun(block.code) + 1)) + "$fence${block.language.orEmpty()}\n${block.code.trimEnd('\n')}\n$fence" + } + + is BlockQuote -> renderBlocks(block.blocks) + .lines() + .joinToString("\n") { if (it.isEmpty()) ">" else "> $it" } + + is ListBlock -> renderList(block, indent = "") + + is Table -> renderTable(block) + + ThematicBreak -> "---" + + is HtmlComment -> "" + + // Already-Markdown content: only whitespace is normalized, never syntax. + is RawBlock -> block.text + .replace("\r\n", "\n") + .lines() + .joinToString("\n") { it.trimEnd() } + .replace(Regex("\n{3,}"), "\n\n") + .trim() + } + + private fun renderList(list: ListBlock, indent: String): String { + val lines = mutableListOf() + list.items.forEachIndexed { index, item -> + val marker = if (list.ordered) "${list.start + index}. " else "${options.bullet} " + val checkbox = when (item.checked) { + true -> "[x] " + false -> "[ ] " + null -> "" + } + val childIndent = indent + " ".repeat(marker.length) + val body = renderItemBlocks(item, childIndent) + val firstLine = body.firstOrNull().orEmpty() + lines += "$indent$marker$checkbox$firstLine".trimEnd() + lines += body.drop(1) + if (list.loose && index != list.items.lastIndex) lines += "" + } + return lines.joinToString("\n").trimEnd() + } + + /** Renders an item's blocks as lines, with continuation lines already indented. */ + private fun renderItemBlocks(item: ListItem, childIndent: String): List { + val lines = mutableListOf() + item.blocks.forEachIndexed { index, block -> + if (index > 0) lines += "" + val rendered = when (block) { + is ListBlock -> renderList(block, childIndent) + else -> renderBlock(block).lines().joinToString("\n") { if (it.isEmpty()) it else childIndent + it } + } + if (rendered.isEmpty()) return@forEachIndexed + lines += rendered.lines() + } + // The first line's indent is supplied by the marker itself. + if (lines.isNotEmpty()) lines[0] = lines[0].removePrefix(childIndent) + return lines + } + + private fun renderTable(table: Table): String { + val bodyRows = table.rows.map { expandSpans(it) } + val headerCells = expandSpans(table.header) + val columns = maxOf(headerCells.size, bodyRows.maxOfOrNull { it.size } ?: 0) + if (columns == 0) return "" + + val header = pad(headerCells, columns) + val rows = bodyRows.map { pad(it, columns) } + val alignments = List(columns) { table.alignments.getOrElse(it) { Alignment.NONE } } + + val widths = if (options.padTableColumns) { + List(columns) { col -> + maxOf( + 3, + header[col].length, + rows.maxOfOrNull { it[col].length } ?: 0, + ) + } + } else { + null + } + + val out = StringBuilder() + out.append(row(header, widths)).append('\n') + out.append(delimiterRow(alignments, widths)).append('\n') + for ((index, r) in rows.withIndex()) { + out.append(row(r, widths)) + if (index != rows.lastIndex) out.append('\n') + } + if (table.caption.isNotEmpty()) { + out.append("\n\n").append(options.emphasisMarker) + .append(inlines(table.caption, TextContext.INLINE).collapseLines()) + .append(options.emphasisMarker) + } + return out.toString() + } + + /** GFM has no colspan: a spanning cell keeps its text and the covered columns render empty. */ + private fun expandSpans(cells: List): List { + val out = mutableListOf() + for (cell in cells) { + out += cellText(cell) + repeat((cell.colSpan - 1).coerceAtLeast(0)) { out += "" } + } + return out + } + + private fun cellText(cell: TableCell): String = + inlines(cell.content, TextContext.TABLE) + .replace("\r\n", "\n") + .lines() + .joinToString(options.tableCellLineBreak) { it.trim() } + .trim() + + private fun pad(cells: List, columns: Int): List = + if (cells.size >= columns) cells.take(columns) else cells + List(columns - cells.size) { "" } + + private fun row(cells: List, widths: List?): String = + cells.mapIndexed { index, cell -> + if (widths == null) cell else cell.padEnd(widths[index]) + }.joinToString(" | ", "| ", " |") + + private fun delimiterRow(alignments: List, widths: List?): String = + alignments.mapIndexed { index, alignment -> + val width = widths?.get(index) ?: 3 + when (alignment) { + Alignment.NONE -> "-".repeat(width) + Alignment.LEFT -> ":" + "-".repeat(width - 1) + Alignment.RIGHT -> "-".repeat(width - 1) + ":" + Alignment.CENTER -> ":" + "-".repeat(width - 2) + ":" + } + }.joinToString(" | ", "| ", " |") + + private fun inlines(inlines: List, context: TextContext): String { + val sb = StringBuilder() + for (inline in inlines) sb.appendInline(inline, context) + return sb.toString() + } + + private fun StringBuilder.appendInline(inline: Inline, context: TextContext) { + when (inline) { + is Text -> append(escape(inline.value, context, atLineStart = isAtLineStart(context))) + + is Strong -> wrap(inline.content, options.strongMarker, context) + is Emphasis -> wrap(inline.content, options.emphasisMarker, context) + is Strikethrough -> wrap(inline.content, "~~", context) + + is CodeSpan -> { + val ticks = "`".repeat(longestBacktickRun(inline.code) + 1) + val padding = if (inline.code.startsWith('`') || inline.code.endsWith('`')) " " else "" + append(ticks).append(padding).append(inline.code.replace("\n", " ")).append(padding).append(ticks) + } + + is Link -> { + val label = inlines(inline.content, TextContext.INLINE).collapseLines().ifBlank { inline.url } + append('[').append(label).append("](").append(encodeUrl(inline.url)) + inline.title?.let { append(" \"").append(it.replace("\"", "\\\"")).append('"') } + append(')') + } + + is Image -> { + val alt = inline.alt.replace("\n", " ").replace("]", "\\]") + if (options.imagesAsText || inline.url.isBlank()) { + if (alt.isNotBlank()) append(alt) + } else if (inline.url.length > options.maxInlineImageUrl) { + if (alt.isNotBlank()) append(alt) + } else { + append("![").append(alt).append("](").append(encodeUrl(inline.url)) + inline.title?.let { append(" \"").append(it.replace("\"", "\\\"")).append('"') } + append(')') + } + } + + LineBreak -> if (context == TextContext.TABLE) append(' ') else append("\\\n") + + is RawInline -> append(inline.text) + } + } + + private fun StringBuilder.wrap(content: List, marker: String, context: TextContext) { + val inner = inlines(content, context) + if (inner.isBlank()) { + append(inner) + return + } + // Markers must hug the text: "**bold** " not "** bold **". + val leading = inner.takeWhile { it.isWhitespace() } + val trailing = inner.takeLastWhile { it.isWhitespace() } + append(leading).append(marker).append(inner.trim()).append(marker).append(trailing) + } + + private fun StringBuilder.isAtLineStart(context: TextContext): Boolean = + context != TextContext.TABLE && (isEmpty() || last() == '\n') + + private fun escape(value: String, context: TextContext, atLineStart: Boolean): String { + val text = value.replace("\r\n", "\n").replace('\r', '\n') + if (!options.escapeText) { + return if (context == TextContext.TABLE) text.replace("|", "\\|") else text + } + return buildString(text.length) { + var lineStart = atLineStart + for ((index, ch) in text.withIndex()) { + val prev = text.getOrNull(index - 1) + val next = text.getOrNull(index + 1) + when { + ch == '\n' -> { + append(ch) + lineStart = true + continue + } + + ch == '\\' -> append("\\\\") + ch == '|' && context == TextContext.TABLE -> append("\\|") + ch == '*' -> append("\\*") + ch == '`' -> append("\\`") + ch == '[' || ch == ']' -> append('\\').append(ch) + // Intraword underscores (snake_case) are not emphasis in CommonMark. + ch == '_' && !(prev?.isLetterOrDigit() == true && next?.isLetterOrDigit() == true) -> + append("\\_") + + ch == '<' && next?.let { it.isLetter() || it == '/' || it == '!' } == true -> append("\\<") + ch == '&' && looksLikeEntity(text, index) -> append("\\&") + + lineStart && (ch == '#' || ch == '>' || ch == '=') -> append('\\').append(ch) + lineStart && (ch == '-' || ch == '+') && next?.isWhitespace() != false -> append('\\').append(ch) + lineStart && ch.isDigit() && startsOrderedList(text, index) -> { + append(ch) + // Escape the delimiter instead of the digits: "1\. item". + } + + lineStart && (ch == '.' || ch == ')') && prev?.isDigit() == true && + startsOrderedList(text, index - 1) -> append('\\').append(ch) + + else -> append(ch) + } + if (!ch.isWhitespace()) lineStart = false + } + } + } + + private fun startsOrderedList(text: String, digitIndex: Int): Boolean { + var start = digitIndex + while (start > 0 && text[start - 1].isDigit()) start-- + if (start > 0 && !text[start - 1].isWhitespace()) return false + var end = digitIndex + while (end + 1 < text.length && text[end + 1].isDigit()) end++ + val delimiter = text.getOrNull(end + 1) ?: return false + if (delimiter != '.' && delimiter != ')') return false + val after = text.getOrNull(end + 2) + return after == null || after == ' ' || after == '\n' + } + + private fun looksLikeEntity(text: String, index: Int): Boolean { + val semicolon = text.indexOf(';', index) + if (semicolon <= index || semicolon - index > 10) return false + return text.substring(index + 1, semicolon).all { it.isLetterOrDigit() || it == '#' } + } + + private fun encodeUrl(url: String): String = + url.replace(" ", "%20").replace("(", "%28").replace(")", "%29") + + private fun longestBacktickRun(text: String): Int { + var longest = 0 + var current = 0 + for (ch in text) { + if (ch == '`') { + current++ + if (current > longest) longest = current + } else { + current = 0 + } + } + return longest + } + + private fun String.collapseLines(): String = + replace("\r\n", "\n").lines().joinToString(" ") { it.trim() }.trim() + + private enum class TextContext { BLOCK, INLINE, HEADING, TABLE } + + companion object { + val Default = MarkdownRenderer() + } +} diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/MarkdownNormalizer.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/MarkdownNormalizer.kt deleted file mode 100644 index 682329b..0000000 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/MarkdownNormalizer.kt +++ /dev/null @@ -1,8 +0,0 @@ -package io.github.lemcoder.mikromarkdown.utils - -object MarkdownNormalizer { - fun normalize(text: String): String { - val stripped = text.lines().joinToString("\n") { it.trimEnd() } - return stripped.replace(Regex("\n{3,}"), "\n\n").trim() - } -} diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt new file mode 100644 index 0000000..87810ed --- /dev/null +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt @@ -0,0 +1,82 @@ +package io.github.lemcoder.mikromarkdown.utils + +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Text + +/** + * Turns extracted plain text (PDF pages, speaker notes, …) into paragraph blocks. + * + * Blank lines separate paragraphs; soft-wrapped lines inside a paragraph are rejoined, + * so the Markdown does not inherit the source layout's line breaks. + */ +fun plainTextBlocks(text: String, reflow: Boolean = true): List { + // Form feeds mark PDF page breaks; treat them as paragraph boundaries. + val normalized = text.replace("\r\n", "\n").replace('\r', '\n').replace('\u000C', '\n') + val vocabulary = if (reflow) wordsIn(normalized) else emptySet() + val paragraphs = mutableListOf() + + for (chunk in normalized.split(Regex("\n[ \t]*\n"))) { + val lines = chunk.lines().map { it.trim() }.filter { it.isNotEmpty() } + if (lines.isEmpty()) continue + val joined = if (reflow) joinWrappedLines(lines, vocabulary) else lines.joinToString("\n") + // A paragraph ending mid-word means the break was a layout artifact, not a real one. + val previous = paragraphs.lastOrNull() + if (reflow && previous != null && previous.endsWithWordBreak()) { + paragraphs[paragraphs.lastIndex] = joinWrappedLines(listOf(previous, joined), vocabulary) + } else { + paragraphs += joined + } + } + + return paragraphs.map { Paragraph(listOf(Text(it))) } +} + +private fun String.endsWithWordBreak(): Boolean = + endsWith("-") && length > 1 && this[length - 2].isLetter() + +private val WORD = Regex("[\\p{L}]{2,}") + +/** Words the document uses on their own; the de-hyphenation heuristic consults this. */ +private fun wordsIn(text: String): Set = + WORD.findAll(text).map { it.value.lowercase() }.toSet() + +/** + * Rejoins soft-wrapped lines. + * + * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are + * words the document uses elsewhere on their own (`conversation-` + `centric`), the hyphen is a + * real compound and stays. + */ +private fun joinWrappedLines(lines: List, vocabulary: Set): String { + val sb = StringBuilder() + for ((index, line) in lines.withIndex()) { + if (index == 0) { + sb.append(line) + continue + } + val head = sb.lastFragment() + val tail = line.takeWhile { it.isLetter() }.lowercase() + val hyphenated = sb.isNotEmpty() && sb.last() == '-' && head.isNotEmpty() && tail.isNotEmpty() + val realCompound = hyphenated && head in vocabulary && tail in vocabulary + + when { + hyphenated && !realCompound -> { + sb.setLength(sb.length - 1) + sb.append(line) + } + + hyphenated -> sb.append(line) + else -> sb.append(' ').append(line) + } + } + return sb.toString() +} + +/** The word immediately before a trailing hyphen, lowercased. */ +private fun StringBuilder.lastFragment(): String { + if (isEmpty() || last() != '-') return "" + var start = length - 1 + while (start > 0 && this[start - 1].isLetter()) start-- + return substring(start, length - 1).lowercase() +} diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt index 71c1976..b93bda6 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt @@ -1,8 +1,10 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser import java.io.InputStreamReader @@ -12,28 +14,19 @@ class CsvConverter : DocumentConverter { return info.extension == "csv" || info.mimetype in setOf("text/csv", "application/csv") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val reader = InputStreamReader(bytes.inputStream(), Charsets.UTF_8) - val allRecords = CSVParser(reader, CSVFormat.DEFAULT.builder().setTrim(true).build()).records + val records = CSVParser(reader, CSVFormat.DEFAULT.builder().setTrim(true).build()).records + if (records.isEmpty()) return Document() - if (allRecords.isEmpty()) return ConversionResult(markdown = "") + val header = records[0].toList() + if (header.isEmpty()) return Document() - val headers = allRecords[0].toList() - if (headers.isEmpty()) return ConversionResult(markdown = "") - - val sb = StringBuilder() - sb.appendLine(headers.map { it.escapeCell() }.joinToString(" | ", "| ", " |")) - sb.appendLine(headers.map { "---" }.joinToString(" | ", "| ", " |")) - - for (i in 1 until allRecords.size) { - val cells = (0 until headers.size).map { col -> - allRecords[i].get(col).escapeCell() - } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) + val rows = records.drop(1).map { record -> + // Ragged rows are padded by the renderer; only extra columns need trimming here. + List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } } - return ConversionResult(markdown = sb.toString().trimEnd()) + return Document(blocks = listOf(Table(header = header.map { TableCell(it) }, rows = rows))) } - - private fun String.escapeCell(): String = replace("|", "\\|").replace("\n", " ") } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt index bb4f272..b1bc8e6 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt @@ -1,8 +1,21 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Asset +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.Image +import io.github.lemcoder.mikromarkdown.model.Inline +import io.github.lemcoder.mikromarkdown.model.ListBlock +import io.github.lemcoder.mikromarkdown.model.ListItem +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text +import io.github.lemcoder.mikromarkdown.model.plainText +import io.github.lemcoder.mikromarkdown.model.styled import org.apache.poi.xwpf.usermodel.XWPFDocument import org.apache.poi.xwpf.usermodel.XWPFParagraph import org.apache.poi.xwpf.usermodel.XWPFTable @@ -14,92 +27,128 @@ class DocxConverter : DocumentConverter { info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - val doc = XWPFDocument(bytes.inputStream()) - val sb = StringBuilder() - var title: String? = null - - for (element in doc.bodyElements) { - when (element) { - is XWPFParagraph -> { - val md = convertParagraph(element) - if (md.isNotBlank()) { - if (title == null && headingLevel(element.styleID) > 0) { - title = element.text + override fun parse(bytes: ByteArray, info: StreamInfo): Document { + val docx = XWPFDocument(bytes.inputStream()) + try { + val blocks = mutableListOf() + val assets = mutableListOf() + var title: String? = docx.properties?.coreProperties?.title?.trim()?.ifBlank { null } + val pendingListItems = mutableListOf>>() + + fun flushList() { + if (pendingListItems.isEmpty()) return + blocks += buildNestedList(pendingListItems) + pendingListItems.clear() + } + + for (element in docx.bodyElements) { + when (element) { + is XWPFParagraph -> { + val content = paragraphInlines(element, assets) + if (content.plainText().isBlank() && content.none { it is Image }) continue + + val level = headingLevel(element.styleID) + when { + level > 0 -> { + flushList() + if (title == null) title = content.plainText().trim() + blocks += Heading(level, content) + } + + element.numID != null -> + pendingListItems += (element.numIlvl?.toInt() ?: 0).coerceAtLeast(0) to content + + else -> { + flushList() + blocks += Paragraph(content) + } } - sb.appendLine(md) - sb.appendLine() } - } - is XWPFTable -> { - val tableMd = convertTable(element) - if (tableMd.isNotBlank()) { - sb.appendLine(tableMd) + + is XWPFTable -> { + flushList() + table(element)?.let { blocks += it } } } } - } + flushList() - doc.close() - return ConversionResult(markdown = sb.toString(), title = title) + return Document(blocks = blocks, title = title, assets = assets) + } finally { + docx.close() + } } - private fun convertParagraph(para: XWPFParagraph): String { - val rawText = para.runs.joinToString("") { run -> - val pics = run.embeddedPictures - if (pics.isNotEmpty()) { - return@joinToString pics.joinToString("") { pic -> - val descr = pic.description ?: "" - val data = pic.pictureData - if (data != null) { - val mime = data.pictureTypeEnum.contentType - val b64 = Base64.getEncoder().encodeToString(data.data) - "![$descr](data:$mime;base64,$b64)" - } else { - "![$descr]()" + private fun paragraphInlines(para: XWPFParagraph, assets: MutableList): List { + val out = mutableListOf() + for (run in para.runs) { + val pictures = run.embeddedPictures + if (pictures.isNotEmpty()) { + for (picture in pictures) { + val data = picture.pictureData + val alt = picture.description.orEmpty() + if (data == null) { + if (alt.isNotBlank()) out += Text(alt) + continue } + val mime = data.pictureTypeEnum.contentType + val id = data.fileName ?: "image-${assets.size + 1}" + assets += Asset(id = id, mediaType = mime, bytes = data.data, name = data.fileName) + out += Image( + alt = alt, + url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", + assetId = id, + ) } + continue } - var text = run.text() ?: "" - if (text.isBlank()) return@joinToString text - when { - run.isBold && run.isItalic -> "***$text***" - run.isBold -> "**$text**" - run.isItalic -> "*$text*" - else -> text + + val text = run.text() ?: continue + if (text.isEmpty()) continue + if (text.isBlank()) { + out += Text(text) + continue } + out += styled(listOf(Text(text)), bold = run.isBold, italic = run.isItalic, strike = run.isStrikeThrough) } + return out + } - if (rawText.isBlank()) return "" - - val level = headingLevel(para.styleID) - val isListItem = para.numID != null + /** Rebuilds Word's flat numbering levels into nested list blocks. */ + private fun buildNestedList(items: List>>): ListBlock { + var index = 0 - return when { - level > 0 -> "${"#".repeat(level)} $rawText" - isListItem -> { - val indent = " ".repeat((para.numIlvl?.toInt() ?: 0).coerceAtLeast(0)) - "$indent- $rawText" + fun build(level: Int): List { + val result = mutableListOf() + while (index < items.size) { + val (itemLevel, content) = items[index] + when { + itemLevel < level -> break + itemLevel == level -> { + index++ + val children = if (index < items.size && items[index].first > level) { + listOf(ListBlock(ordered = false, items = build(items[index].first))) + } else { + emptyList() + } + result += ListItem(listOf(Paragraph(content)) + children) + } + // A deeper first item without a parent: promote it to this level. + else -> result += ListItem(listOf(ListBlock(ordered = false, items = build(itemLevel)))) + } } - else -> rawText + return result } + + return ListBlock(ordered = false, items = build(items.minOf { it.first })) } - private fun convertTable(table: XWPFTable): String { + private fun table(table: XWPFTable): Table? { val rows = table.rows - if (rows.isEmpty()) return "" - - val sb = StringBuilder() - val header = rows[0].tableCells.map { it.text.replace("|", "\\|") } - sb.appendLine(header.joinToString(" | ", "| ", " |")) - sb.appendLine(header.map { "---" }.joinToString(" | ", "| ", " |")) - - for (i in 1 until rows.size) { - val cells = rows[i].tableCells.map { it.text.replace("|", "\\|") } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) - } - - return sb.toString().trimEnd() + if (rows.isEmpty()) return null + val header = rows[0].tableCells.map { TableCell(it.text.trim()) } + val body = rows.drop(1).map { row -> row.tableCells.map { TableCell(it.text.trim()) } } + return Table(header = header, rows = body) } private fun headingLevel(style: String?): Int { diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt index e9b0183..9c80056 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt @@ -1,9 +1,13 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo -import io.github.lemcoder.mikromarkdown.utils.HtmlToMarkdown +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Strong +import io.github.lemcoder.mikromarkdown.model.Text +import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument import org.w3c.dom.Element import org.xml.sax.InputSource import java.io.StringReader @@ -11,45 +15,46 @@ import java.util.zip.ZipInputStream import javax.xml.parsers.DocumentBuilderFactory class EpubConverter : DocumentConverter { + private val metaFields = listOf( + "title" to "Title", + "creator" to "Authors", + "language" to "Language", + "description" to "Description", + "identifier" to "Identifier", + ) + override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "epub" || info.mimetype == "application/epub+zip" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val entries = readZip(bytes) - val containerXml = entries["META-INF/container.xml"] ?: return ConversionResult(markdown = "") - val opfPath = parseOpfPath(containerXml) ?: return ConversionResult(markdown = "") - - val opfBytes = entries[opfPath] ?: entries[opfPath.removePrefix("/")] ?: return ConversionResult(markdown = "") + val containerXml = entries["META-INF/container.xml"] ?: return Document() + val opfPath = parseOpfPath(containerXml) ?: return Document() + val opfBytes = entries[opfPath] ?: entries[opfPath.removePrefix("/")] ?: return Document() val opfDir = opfPath.substringBeforeLast("/", "") val (manifest, spine, metadata) = parseOpf(opfBytes) - val sb = StringBuilder() + val blocks = mutableListOf() var title: String? = metadata["title"] - val metaFields = listOf("title" to "Title", "creator" to "Authors", "language" to "Language", - "description" to "Description", "identifier" to "Identifier") for ((key, label) in metaFields) { - metadata[key]?.let { sb.appendLine("**$label:** $it") } + val value = metadata[key] ?: continue + blocks += Paragraph(listOf(Strong(listOf(Text("$label:"))), Text(" $value"))) } - if (sb.isNotEmpty()) sb.appendLine() for (idref in spine) { val href = manifest[idref] ?: continue val fullPath = if (opfDir.isEmpty()) href else "$opfDir/$href" val htmlBytes = entries[fullPath] ?: entries[fullPath.removePrefix("/")] ?: continue - val html = htmlBytes.toString(Charsets.UTF_8) - val (markdown, chapterTitle) = HtmlToMarkdown.convert(html) - if (title == null && chapterTitle != null) title = chapterTitle - if (markdown.isNotBlank()) { - sb.appendLine(markdown) - sb.appendLine() - } + val chapter = HtmlToDocument.parse(htmlBytes.toString(Charsets.UTF_8)) + if (title == null) title = chapter.title + blocks += chapter.blocks } - return ConversionResult(markdown = sb.toString(), title = title) + return Document(blocks = blocks, title = title, metadata = metadata) } private fun readZip(bytes: ByteArray): Map { diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt index 9898f16..fb5dffd 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt @@ -1,9 +1,9 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo -import io.github.lemcoder.mikromarkdown.utils.HtmlToMarkdown +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument class HtmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -11,9 +11,6 @@ class HtmlConverter : DocumentConverter { info.mimetype in setOf("text/html", "application/xhtml+xml") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - val html = bytes.toString(Charsets.UTF_8) - val (markdown, title) = HtmlToMarkdown.convert(html) - return ConversionResult(markdown = markdown, title = title) - } + override fun parse(bytes: ByteArray, info: StreamInfo): Document = + HtmlToDocument.parse(bytes.toString(Charsets.UTF_8), info.localPath.orEmpty()) } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt index 6fd5a28..026c9d1 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt @@ -5,9 +5,10 @@ import com.fasterxml.jackson.core.util.DefaultIndenter import com.fasterxml.jackson.core.util.DefaultPrettyPrinter import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.registerKotlinModule -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.CodeBlock +import io.github.lemcoder.mikromarkdown.model.Document class JsonConverter : DocumentConverter { private val writer = ObjectMapper().apply { @@ -25,14 +26,13 @@ class JsonConverter : DocumentConverter { return info.extension == "json" || info.mimetype in setOf("application/json", "text/json") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val json = bytes.toString(Charsets.UTF_8) val pretty = try { - val node = ObjectMapper().readTree(json) - writer.writeValueAsString(node) + writer.writeValueAsString(ObjectMapper().readTree(json)) } catch (_: Exception) { json } - return ConversionResult(markdown = pretty) + return Document(blocks = listOf(CodeBlock(pretty, "json"))) } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt index d399482..816bcf4 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt @@ -1,8 +1,9 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.utils.plainTextBlocks import org.apache.pdfbox.Loader import org.apache.pdfbox.text.PDFTextStripper @@ -11,13 +12,20 @@ class PdfConverter : DocumentConverter { return info.extension == "pdf" || info.mimetype == "application/pdf" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val doc = Loader.loadPDF(bytes) - val text = try { - PDFTextStripper().getText(doc) + try { + val title = doc.documentInformation?.title?.trim()?.ifBlank { null } + // Paragraph markers let the text blocks split on real paragraph breaks + // instead of collapsing a page into one block. + val stripper = PDFTextStripper().apply { + sortByPosition = true + setAddMoreFormatting(true) + paragraphStart = "\n" + } + return Document(blocks = plainTextBlocks(stripper.getText(doc)), title = title) } finally { doc.close() } - return ConversionResult(markdown = text) } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt index bf57bd1..4c363e5 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt @@ -1,16 +1,28 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.HtmlComment +import io.github.lemcoder.mikromarkdown.model.Image +import io.github.lemcoder.mikromarkdown.model.ListBlock +import io.github.lemcoder.mikromarkdown.model.ListItem +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text import org.apache.poi.sl.usermodel.Placeholder +import org.apache.poi.sl.usermodel.Shape +import org.apache.poi.xslf.usermodel.XMLSlideShow +import org.apache.poi.xslf.usermodel.XSLFChart import org.apache.poi.xslf.usermodel.XSLFGraphicFrame import org.apache.poi.xslf.usermodel.XSLFGroupShape import org.apache.poi.xslf.usermodel.XSLFPictureShape import org.apache.poi.xslf.usermodel.XSLFSimpleShape import org.apache.poi.xslf.usermodel.XSLFTable import org.apache.poi.xslf.usermodel.XSLFTextShape -import org.apache.poi.xslf.usermodel.XMLSlideShow import org.openxmlformats.schemas.drawingml.x2006.chart.CTAxDataSource import org.openxmlformats.schemas.drawingml.x2006.chart.CTNumDataSource import org.openxmlformats.schemas.drawingml.x2006.chart.CTSerTx @@ -22,204 +34,205 @@ class PptxConverter : DocumentConverter { info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val slideShow = XMLSlideShow(bytes.inputStream()) - val sb = StringBuilder() - var title: String? = null - - for ((index, slide) in slideShow.slides.withIndex()) { - sb.appendLine("") - processShapes(slide.shapes, sb, index == 0) { t -> if (title == null) title = t } - - // Speaker notes - val notes = slide.notes - if (notes != null) { - val notesText = notes.shapes - .filterIsInstance() - .filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } - .joinToString("\n") { it.text } - .trim() - if (notesText.isNotBlank()) { - sb.appendLine() - sb.appendLine("### Notes:") - sb.appendLine(notesText) + try { + val blocks = mutableListOf() + var title: String? = null + + for ((index, slide) in slideShow.slides.withIndex()) { + blocks += HtmlComment("Slide number: ${index + 1}") + blocks += shapeBlocks(slide.shapes) { if (index == 0 && title == null) title = it } + + val notes = slide.notes?.shapes + ?.filterIsInstance() + ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } + ?.joinToString("\n") { it.text } + ?.trim() + .orEmpty() + if (notes.isNotBlank()) { + blocks += Heading(3, listOf(Text("Notes:"))) + blocks += notes.lines().filter { it.isNotBlank() }.map { Paragraph(listOf(Text(it.trim()))) } } } - sb.appendLine() + return Document(blocks = blocks, title = title) + } finally { + slideShow.close() } - - slideShow.close() - return ConversionResult(markdown = sb.toString(), title = title) } - private fun processShapes( - shapes: Iterable>, - sb: StringBuilder, - isFirstSlide: Boolean, - onTitle: (String) -> Unit, - ) { + private fun shapeBlocks(shapes: Iterable>, onTitle: (String) -> Unit): List { + val blocks = mutableListOf() for (shape in shapes) { when { - shape is XSLFGroupShape -> processShapes(shape.shapes, sb, isFirstSlide, onTitle) + shape is XSLFGroupShape -> blocks += shapeBlocks(shape.shapes, onTitle) + shape is XSLFTextShape -> { val text = shape.text.trim() if (text.isBlank()) continue val placeholder = (shape as? XSLFSimpleShape)?.placeholder - val isTitle = placeholder == Placeholder.TITLE || - placeholder == Placeholder.CENTERED_TITLE - - if (isTitle) { - sb.appendLine("# $text") - if (isFirstSlide) onTitle(text) - } else { - for (para in shape.textParagraphs) { - val paraText = para.text.trim() - if (paraText.isBlank()) continue - if (para.isBullet) sb.appendLine("- $paraText") - else sb.appendLine(paraText) + if (placeholder == Placeholder.TITLE || placeholder == Placeholder.CENTERED_TITLE) { + blocks += Heading(1, listOf(Text(text))) + onTitle(text) + continue + } + + // Consecutive bullet paragraphs become one list; plain ones stay paragraphs. + val bullets = mutableListOf() + fun flushBullets() { + if (bullets.isEmpty()) return + blocks += ListBlock(ordered = false, items = bullets.toList()) + bullets.clear() + } + for (para in shape.textParagraphs) { + val paraText = para.text.trim() + if (paraText.isBlank()) continue + if (para.isBullet) { + bullets += ListItem(listOf(Paragraph(listOf(Text(paraText))))) + } else { + flushBullets() + blocks += Paragraph(listOf(Text(paraText))) } } + flushBullets() } + shape is XSLFPictureShape -> { - val descr = (shape.xmlObject as? CTPicture)?.nvPicPr?.cNvPr?.descr ?: "" - val altText = if (descr.isNotBlank()) descr else shape.shapeName + val description = (shape.xmlObject as? CTPicture)?.nvPicPr?.cNvPr?.descr.orEmpty() + val alt = description.ifBlank { shape.shapeName } val filename = shape.shapeName.replace(Regex("\\W"), "") + ".jpg" - sb.appendLine("![$altText]($filename)") - } - shape is XSLFGraphicFrame && shape.hasChart() -> { - sb.append(convertChart(shape.chart)) + blocks += Paragraph(listOf(Image(alt, filename))) } - shape is XSLFTable -> sb.appendLine(convertTable(shape)) + + shape is XSLFGraphicFrame && shape.hasChart() -> blocks += chartBlocks(shape.chart) + + shape is XSLFTable -> table(shape)?.let { blocks += it } } } + return blocks } - private fun convertChart(chart: org.apache.poi.xslf.usermodel.XSLFChart): String { - val sb = StringBuilder() - sb.append("\n\n### Chart") + private fun chartBlocks(chart: XSLFChart): List { + val blocks = mutableListOf() + blocks += Heading(3, listOf(Text(listOfNotNull("Chart", chartTitle(chart)).joinToString(": ")))) - try { - val ctChart = chart.ctChart - if (ctChart.isSetTitle) { - val tx = ctChart.title?.tx - val titleText = when { - tx?.isSetRich == true -> - tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t ?: "" } }.joinToString("") - tx?.isSetStrRef == true -> - tx.strRef?.strCache?.ptList?.firstOrNull()?.v ?: "" - else -> "" - } - if (titleText.isNotBlank()) sb.append(": $titleText") - } - } catch (_: Exception) {} + val series = try { + seriesOf(chart) + } catch (_: Exception) { + blocks += Paragraph(listOf(Text("[unsupported chart]"))) + return blocks + } + if (series.isEmpty()) return blocks + + val rowCount = series.maxOf { it.categories.size } + blocks += Table( + header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, + rows = (0 until rowCount).map { row -> + val category = series.first().categories.getOrElse(row) { "" } + (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } + }, + ) + return blocks + } - sb.append("\n\n") + private fun chartTitle(chart: XSLFChart): String? = try { + val ctChart = chart.ctChart + if (!ctChart.isSetTitle) { + null + } else { + val tx = ctChart.title?.tx + when { + tx?.isSetRich == true -> + tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") + tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v + else -> null + }?.ifBlank { null } + } + } catch (_: Exception) { + null + } - data class SeriesData(val name: String, val categories: List, val values: List) + private data class Series(val name: String, val categories: List, val values: List) - fun catStrings(cat: CTAxDataSource?): List { - if (cat == null) return emptyList() - return when { - cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() - cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - cat.isSetNumLit -> cat.numLit?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - cat.isSetStrLit -> cat.strLit?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() - else -> emptyList() - } + private fun seriesOf(chart: XSLFChart): List { + val plotArea = chart.ctChart.plotArea + val out = mutableListOf() + + fun categoryValues(cat: CTAxDataSource?): List = when { + cat == null -> emptyList() + cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() + cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + cat.isSetNumLit -> cat.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + cat.isSetStrLit -> cat.strLit?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() + else -> emptyList() } - fun valStrings(v: CTNumDataSource?): List { - if (v == null) return emptyList() - return when { - v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v ?: "" } ?: emptyList() - else -> emptyList() - } + fun numericValues(v: CTNumDataSource?): List = when { + v == null -> emptyList() + v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() + else -> emptyList() } - fun serTitle(tx: CTSerTx?): String = when { + fun seriesName(tx: CTSerTx?): String = when { tx == null -> "" tx.isSetV -> tx.v - tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v ?: "" + tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v.orEmpty() else -> "" } - val seriesList = mutableListOf() - try { - val plotArea = chart.ctChart.plotArea - - for (c in plotArea.barChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.bar3DChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.lineChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.line3DChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.areaChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.area3DChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetCat) s.cat else null), - valStrings(if (s.isSetVal) s.`val` else null))) - for (c in plotArea.scatterChartList) - for (s in c.serList) seriesList.add(SeriesData( - serTitle(if (s.isSetTx) s.tx else null), - catStrings(if (s.isSetXVal) s.xVal else null), - valStrings(if (s.isSetYVal) s.yVal else null))) - - } catch (_: Exception) { - return sb.append("[unsupported chart]\n\n").toString() - } - - if (seriesList.isEmpty()) return sb.toString() - - val header = listOf("Category") + seriesList.map { it.name } - sb.appendLine("| " + header.joinToString(" | ") + " |") - sb.appendLine("|" + "---|".repeat(header.size)) - - val numRows = seriesList.maxOfOrNull { it.categories.size } ?: 0 - for (i in 0 until numRows) { - val cat = seriesList.first().categories.getOrElse(i) { "" } - val vals = seriesList.map { it.values.getOrElse(i) { "" } } - sb.appendLine("| " + (listOf(cat) + vals).joinToString(" | ") + " |") - } - - return sb.toString() + for (c in plotArea.barChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.bar3DChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.lineChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.line3DChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.areaChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.area3DChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + for (c in plotArea.scatterChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetXVal) s.xVal else null), + numericValues(if (s.isSetYVal) s.yVal else null), + ) + for (c in plotArea.pieChartList) for (s in c.serList) out += Series( + seriesName(if (s.isSetTx) s.tx else null), + categoryValues(if (s.isSetCat) s.cat else null), + numericValues(if (s.isSetVal) s.`val` else null), + ) + + return out.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } } - private fun convertTable(table: XSLFTable): String { + private fun table(table: XSLFTable): Table? { val rows = table.rows - if (rows.isEmpty()) return "" - - val sb = StringBuilder() - val header = rows[0].cells.map { it.text.trim().replace("|", "\\|") } - sb.appendLine(header.joinToString(" | ", "| ", " |")) - sb.appendLine(header.map { "---" }.joinToString(" | ", "| ", " |")) - - for (i in 1 until rows.size) { - val cells = rows[i].cells.map { it.text.trim().replace("|", "\\|") } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) - } - - return sb.toString().trimEnd() + if (rows.isEmpty()) return null + return Table( + header = rows[0].cells.map { TableCell(it.text.trim()) }, + rows = rows.drop(1).map { row -> row.cells.map { TableCell(it.text.trim()) } }, + ) } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt index 8449c69..984dec0 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt @@ -1,8 +1,13 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text import org.apache.poi.ss.usermodel.Cell import org.apache.poi.ss.usermodel.CellType import org.apache.poi.ss.usermodel.DataFormatter @@ -17,37 +22,31 @@ class XlsxConverter : DocumentConverter { info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { + override fun parse(bytes: ByteArray, info: StreamInfo): Document { val workbook = XSSFWorkbook(bytes.inputStream()) - val sb = StringBuilder() + try { + val blocks = mutableListOf() - for (sheet in workbook) { - val rows = sheet.toList() - if (rows.isEmpty()) continue + for (sheet in workbook) { + val rows = sheet.toList() + if (rows.isEmpty()) continue - val maxCols = rows.maxOf { it.lastCellNum.toInt().coerceAtLeast(0) } - if (maxCols == 0) continue + val columns = rows.maxOf { it.lastCellNum.toInt().coerceAtLeast(0) } + if (columns == 0) continue - sb.appendLine("## ${sheet.sheetName}") - sb.appendLine() - - val headerCells = (0 until maxCols).map { col -> - cellValue(rows[0].getCell(col)).replace("|", "\\|") + blocks += Heading(2, listOf(Text(sheet.sheetName))) + blocks += Table( + header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, + rows = rows.drop(1).map { row -> + (0 until columns).map { TableCell(cellValue(row.getCell(it))) } + }, + ) } - sb.appendLine(headerCells.joinToString(" | ", "| ", " |")) - sb.appendLine(headerCells.map { "---" }.joinToString(" | ", "| ", " |")) - for (i in 1 until rows.size) { - val cells = (0 until maxCols).map { col -> - cellValue(rows[i].getCell(col)).replace("|", "\\|") - } - sb.appendLine(cells.joinToString(" | ", "| ", " |")) - } - sb.appendLine() + return Document(blocks = blocks) + } finally { + workbook.close() } - - workbook.close() - return ConversionResult(markdown = sb.toString()) } private fun cellValue(cell: Cell?): String { diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt index e19d71a..8979d57 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt @@ -1,8 +1,9 @@ package io.github.lemcoder.mikromarkdown.converters -import io.github.lemcoder.mikromarkdown.ConversionResult import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo +import io.github.lemcoder.mikromarkdown.model.CodeBlock +import io.github.lemcoder.mikromarkdown.model.Document import org.xml.sax.InputSource import java.io.StringReader import java.io.StringWriter @@ -17,10 +18,9 @@ class XmlConverter : DocumentConverter { return info.extension == "xml" || info.mimetype in setOf("text/xml", "application/xml") } - override fun convert(bytes: ByteArray, info: StreamInfo): ConversionResult { - val xml = bytes.toString(Charsets.UTF_8) - val pretty = prettyPrint(xml) - return ConversionResult(markdown = "```xml\n$pretty\n```") + override fun parse(bytes: ByteArray, info: StreamInfo): Document { + val pretty = prettyPrint(bytes.toString(Charsets.UTF_8)) + return Document(blocks = listOf(CodeBlock(pretty, "xml"))) } private fun prettyPrint(xml: String): String = try { diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt new file mode 100644 index 0000000..d7c9296 --- /dev/null +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt @@ -0,0 +1,266 @@ +package io.github.lemcoder.mikromarkdown.utils + +import io.github.lemcoder.mikromarkdown.model.Block +import io.github.lemcoder.mikromarkdown.model.BlockQuote +import io.github.lemcoder.mikromarkdown.model.CodeBlock +import io.github.lemcoder.mikromarkdown.model.CodeSpan +import io.github.lemcoder.mikromarkdown.model.Document +import io.github.lemcoder.mikromarkdown.model.Emphasis +import io.github.lemcoder.mikromarkdown.model.Heading +import io.github.lemcoder.mikromarkdown.model.Image +import io.github.lemcoder.mikromarkdown.model.Inline +import io.github.lemcoder.mikromarkdown.model.LineBreak +import io.github.lemcoder.mikromarkdown.model.Link +import io.github.lemcoder.mikromarkdown.model.ListBlock +import io.github.lemcoder.mikromarkdown.model.ListItem +import io.github.lemcoder.mikromarkdown.model.Paragraph +import io.github.lemcoder.mikromarkdown.model.Strikethrough +import io.github.lemcoder.mikromarkdown.model.Strong +import io.github.lemcoder.mikromarkdown.model.Table +import io.github.lemcoder.mikromarkdown.model.TableCell +import io.github.lemcoder.mikromarkdown.model.Text +import io.github.lemcoder.mikromarkdown.model.ThematicBreak +import io.github.lemcoder.mikromarkdown.model.plainText +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.nodes.Node +import org.jsoup.nodes.TextNode + +/** + * Walks an HTML DOM into the shared document model. + * + * Replaces the previous HTML → Markdown string conversion: tables, lists and inline + * emphasis become model nodes, so the Markdown renderer owns all syntax decisions. + */ +object HtmlToDocument { + + private val DROPPED_TAGS = setOf( + "script", "style", "noscript", "template", "button", "svg", "iframe", "form", "input", "select", + ) + + private val HEADINGS = mapOf("h1" to 1, "h2" to 2, "h3" to 3, "h4" to 4, "h5" to 5, "h6" to 6) + + /** Tags that only group other content; their children are lifted into the parent block flow. */ + private val CONTAINERS = setOf( + "div", "section", "article", "main", "header", "footer", "aside", "nav", "body", "html", + "figure", "details", "summary", "fieldset", "center", "hgroup", "picture", "colgroup", + ) + + fun parse(html: String, baseUri: String = ""): Document { + val doc = Jsoup.parse(html, baseUri) + doc.select(DROPPED_TAGS.joinToString(", ")).remove() + val title = doc.title().ifBlank { null } + val root = doc.body() ?: doc + val blocks = blocks(root) + return Document(blocks = blocks, title = title ?: blocks.headingTitle()) + } + + private fun List.headingTitle(): String? = + (firstOrNull { it is Heading } as? Heading)?.content?.plainText()?.trim()?.ifBlank { null } + + /** Converts an element's children into blocks, flushing runs of inline content into paragraphs. */ + private fun blocks(parent: Element): List { + val out = mutableListOf() + val pending = mutableListOf() + + fun flush() { + val trimmed = pending.trimEdges() + if (trimmed.isNotEmpty()) out += Paragraph(trimmed) + pending.clear() + } + + for (node in parent.childNodes()) { + when { + node is TextNode -> { + val text = node.normalizedText() + if (text.isNotEmpty()) pending += Text(text) + } + + node is Element && node.isBlockLevel() -> { + flush() + out += blockFor(node) + } + + node is Element && node.tagName() == "br" -> pending += LineBreak + + // Inline elements (links, emphasis, …) keep their wrapper — appendNode, not children. + node is Element -> pending.appendNode(node) + } + } + flush() + return out + } + + private fun Element.isBlockLevel(): Boolean = when (tagName()) { + in HEADINGS, "p", "ul", "ol", "table", "pre", "blockquote", "hr", "dl", "li", "figcaption" -> true + in CONTAINERS -> true + else -> false + } + + private fun blockFor(element: Element): List = when (val tag = element.tagName()) { + in HEADINGS -> listOfNotNull( + inlines(element).trimEdges() + .takeIf { it.isNotEmpty() } + ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) }, + ) + + "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) + "ul", "ol" -> listBlock(element) + "table" -> table(element) + "pre" -> listOf(codeBlock(element)) + "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() + "hr" -> listOf(ThematicBreak) + "dl" -> definitionList(element) + "figcaption" -> listOfNotNull( + inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) }, + ) + // Containers and stray
  • outside a list contribute their children directly. + else -> blocks(element) + } + + private fun listBlock(element: Element): List { + val ordered = element.tagName() == "ol" + val start = element.attr("start").toIntOrNull() ?: 1 + val items = element.children() + .filter { it.tagName() == "li" } + .map { li -> ListItem(blocks = blocks(li)) } + .filter { it.blocks.isNotEmpty() } + if (items.isEmpty()) return emptyList() + return listOf(ListBlock(ordered = ordered, items = items, start = start)) + } + + private fun codeBlock(element: Element): Block { + val code = element.selectFirst("code") ?: element + val language = code.classNames() + .firstOrNull { it.startsWith("language-") || it.startsWith("lang-") } + ?.substringAfter('-') + return CodeBlock(code.wholeText().trimEnd(), language) + } + + private fun definitionList(element: Element): List { + val out = mutableListOf() + for (child in element.children()) { + val content = inlines(child).trimEdges() + if (content.isEmpty()) continue + when (child.tagName()) { + "dt" -> out += Paragraph(listOf(Strong(content))) + "dd" -> out += Paragraph(content) + } + } + return out + } + + private fun table(element: Element): List { + val caption = element.selectFirst("caption")?.let { inlines(it).trimEdges() } ?: emptyList() + val rows = element.select("tr").filter { it.parentTable() === element } + + var header: List = emptyList() + val body = mutableListOf>() + + for ((index, tr) in rows.withIndex()) { + val cells = tr.children() + .filter { it.tagName() == "th" || it.tagName() == "td" } + .map { cell -> + TableCell( + content = inlines(cell).trimEdges(), + colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + ) + } + if (cells.isEmpty()) continue + + val isHeaderRow = index == 0 && + tr.children().all { it.tagName() == "th" } && + (tr.parent()?.tagName() == "thead" || header.isEmpty()) + + if (isHeaderRow && header.isEmpty()) header = cells else body += cells + } + + if (header.isEmpty() && body.isEmpty()) return emptyList() + return listOf(Table(header = header, rows = body, caption = caption)) + } + + /** The nearest enclosing table, so nested tables do not steal each other's rows. */ + private fun Element.parentTable(): Element? = parents().firstOrNull { it.tagName() == "table" } + + private fun inlines(element: Element): List { + val out = mutableListOf() + for (node in element.childNodes()) out.appendNode(node) + return out + } + + private fun MutableList.appendNode(node: Node) { + when (node) { + is TextNode -> { + val text = node.normalizedText() + if (text.isNotEmpty()) add(Text(text)) + } + + is Element -> when (node.tagName()) { + "br" -> add(LineBreak) + "img" -> { + val src = node.attr("abs:src").ifBlank { node.attr("src") } + val alt = node.attr("alt") + if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) + } + + "a" -> { + val href = node.attr("abs:href").ifBlank { node.attr("href") } + val content = inlines(node).trimEdges() + when { + content.isEmpty() -> Unit + href.isBlank() || href.startsWith("javascript:") -> addAll(content) + else -> add(Link(content, href, node.attr("title").ifBlank { null })) + } + } + + "strong", "b" -> wrapped(node) { Strong(it) } + "em", "i", "cite", "var" -> wrapped(node) { Emphasis(it) } + "del", "s", "strike" -> wrapped(node) { Strikethrough(it) } + "code", "kbd", "samp", "tt" -> { + val code = node.wholeText().trim() + if (code.isNotEmpty()) add(CodeSpan(code)) + } + + // Block-level content encountered inline (e.g. a
    inside a
    		• ): keep its text. + else -> addAll(inlines(node)) + } + } + } + + private inline fun MutableList.wrapped(node: Element, wrap: (List) -> Inline) { + val content = inlines(node) + val trimmed = content.trimEdges() + if (trimmed.isEmpty()) return + if (content.startsWithSpace()) add(Text(" ")) + add(wrap(trimmed)) + if (content.endsWithSpace()) add(Text(" ")) + } + + /** HTML collapses runs of whitespace; do the same before the text reaches the model. */ + // Non-breaking spaces are not collapsible whitespace in HTML, so they survive verbatim. + private fun TextNode.normalizedText(): String = + wholeText.replace(Regex("\\s+"), " ") + + private fun List.startsWithSpace(): Boolean = + (firstOrNull() as? Text)?.value?.startsWith(" ") == true + + private fun List.endsWithSpace(): Boolean = + (lastOrNull() as? Text)?.value?.endsWith(" ") == true + + /** Drops leading/trailing whitespace-only text so emphasis markers hug their content. */ + private fun List.trimEdges(): List { + var start = 0 + var end = size + while (start < end && this[start].isBlankText()) start++ + while (end > start && this[end - 1].isBlankText()) end-- + if (start >= end) return emptyList() + val slice = subList(start, end).toMutableList() + (slice.first() as? Text)?.let { slice[0] = Text(it.value.trimStart()) } + (slice.last() as? Text)?.let { slice[slice.lastIndex] = Text(it.value.trimEnd()) } + return slice.filter { !(it is Text && it.value.isEmpty()) } + } + + // Kotlin's trim()/isBlank() drop Unicode spacing (thin, hair, …) but keep NBSP, which is content. + private fun Inline.isBlankText(): Boolean = this is Text && value.isBlank() +} diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToMarkdown.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToMarkdown.kt deleted file mode 100644 index f7d612d..0000000 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToMarkdown.kt +++ /dev/null @@ -1,24 +0,0 @@ -package io.github.lemcoder.mikromarkdown.utils - -import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter -import com.vladsch.flexmark.util.data.MutableDataSet -import org.jsoup.Jsoup - -object HtmlToMarkdown { - private val options = MutableDataSet().apply { - set(FlexmarkHtmlConverter.SETEXT_HEADINGS, false) - } - private val converter = FlexmarkHtmlConverter.builder(options).build() - - fun convert(html: String, baseUri: String = ""): Pair { - val doc = Jsoup.parse(html, baseUri) - doc.select("script, style, button").remove() - // Flexmark drops rows whose header cell uses (infobox/navbox pattern). - // Converting them to preserves content while allowing Flexmark to process the rows. - doc.select("th[scope=row]").tagName("td") - val title = doc.title().ifEmpty { null } - val body = doc.body()?.html() ?: doc.html() - val markdown = converter.convert(body) - return markdown to title - } -} diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt index 2db8274..ce1faa1 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt @@ -60,6 +60,10 @@ class PythonComparisonTest { .replace(Regex("(?m)^.*move to sidebar.*$"), "") .replace(Regex("(?m)^\\s*hide\\s*$"), "") .replace(Regex("(?m)^[*\\s]*Toggle\\b.*$"), "") + // Java's \s ignores Unicode spacing (thin/hair/narrow), Python's does not. + .replace(Regex("[\\u2000-\\u200A\\u202F\\u205F\\u3000]"), " ") + // Tracking pixels live in