From ffd1a5fc9a2a709c6aefd32fa8f42333090eb48b Mon Sep 17 00:00:00 2001 From: mikolaj Date: Tue, 11 Aug 2026 01:29:16 +0200 Subject: [PATCH 1/3] Rename the MarkItDown API surface to MikroMarkdown The factory functions and exception base class still carried the name of the upstream project this was ported from, which no longer matches anything in the library. - MarkItDown() -> MikroMarkdown() (JVM), MarkItDown(context) -> MikroMarkdown(context) - MarkItDownException -> MikroMarkdownException - CLI command class and its help name now read mikromarkdown - README updated; the remaining MarkItDown mention is the upstream project link Breaking for 0.1.0 consumers, with no deprecated aliases, matching the convert -> parse change. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 12 ++++++------ cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt | 12 ++++++------ .../mikromarkdown/FileIntegrationTest.android.kt | 4 ++-- .../github/lemcoder/mikromarkdown/TestMarkItDown.kt | 3 --- .../lemcoder/mikromarkdown/TestMikroMarkdown.kt | 3 +++ .../{MarkItDown.kt => MikroMarkdownFactory.kt} | 7 ++++++- .../io/github/lemcoder/mikromarkdown/Exceptions.kt | 6 +++--- .../github/lemcoder/mikromarkdown/MikroMarkdown.kt | 2 +- .../lemcoder/mikromarkdown/FileIntegrationTest.kt | 4 ++-- .../{MarkItDown.kt => MikroMarkdownFactory.kt} | 3 ++- .../github/lemcoder/mikromarkdown/DumpOutputTest.kt | 2 +- .../mikromarkdown/FileIntegrationTest.jvm.kt | 2 +- .../lemcoder/mikromarkdown/PythonComparisonTest.kt | 2 +- 13 files changed, 34 insertions(+), 28 deletions(-) delete mode 100644 library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMarkItDown.kt create mode 100644 library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt rename library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/{MarkItDown.kt => MikroMarkdownFactory.kt} (84%) rename library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/{MarkItDown.kt => MikroMarkdownFactory.kt} (88%) diff --git a/README.md b/README.md index 403ed5a..f4b0e24 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,9 @@ dependencies { ### JVM ```kotlin -import io.github.lemcoder.mikromarkdown.MarkItDown +import io.github.lemcoder.mikromarkdown.MikroMarkdown -val mid = MarkItDown() +val mid = MikroMarkdown() // from file path val result = mid.convert("/path/to/document.docx") @@ -78,10 +78,10 @@ println(result.title) // nullable, extracted from document metadata ### Android ```kotlin -import io.github.lemcoder.mikromarkdown.MarkItDown +import io.github.lemcoder.mikromarkdown.MikroMarkdown // pass Context to enable PDF support -val mid = MarkItDown(context) +val mid = MikroMarkdown(context) val result = mid.convert(file.absolutePath) ``` @@ -101,7 +101,7 @@ class MyConverter : DocumentConverter { } } -val mid = MarkItDown() +val mid = MikroMarkdown() mid.register(MyConverter()) // default priority 0.0 mid.register(FallbackConverter(), priority = 10.0) // higher = later ``` @@ -126,7 +126,7 @@ mid.register(HtmlConverter()) | `UnsupportedFormatException` | No registered converter accepted the input | | `FileConversionException` | Converter threw during conversion | -Both extend `MarkItDownException`. +Both extend `MikroMarkdownException`. ## Benchmark diff --git a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt index d7ca118..a3a6b46 100644 --- a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt +++ b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt @@ -6,23 +6,23 @@ import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.optional import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.path -import io.github.lemcoder.mikromarkdown.MarkItDown +import io.github.lemcoder.mikromarkdown.MikroMarkdown import io.github.lemcoder.mikromarkdown.StreamInfo -class MarkItDownCommand : CliktCommand(name = "markitdown") { +class MikroMarkdownCommand : CliktCommand(name = "mikromarkdown") { private val file by argument("FILE", help = "Input file (reads stdin if omitted)").path(mustExist = true).optional() private val output by option("-o", "--output", help = "Output file (default: stdout)").path() private val extension by option("-x", "--extension", help = "File extension hint (e.g. html)") private val mimeType by option("-m", "--mime-type", help = "MIME type hint (e.g. text/html)") override fun run() { - val markItDown = MarkItDown() + val mikroMarkdown = MikroMarkdown() val result = if (file != null) { - markItDown.convert(file!!.toFile().absolutePath) + mikroMarkdown.convert(file!!.toFile().absolutePath) } else { val info = StreamInfo(extension = extension, mimetype = mimeType) - markItDown.convert(System.`in`.readBytes(), info) + mikroMarkdown.convert(System.`in`.readBytes(), info) } if (output != null) { @@ -39,5 +39,5 @@ fun main(args: Array) { System.setProperty("log4j2.loggerContextFactory", "org.apache.logging.log4j.simple.SimpleLoggerContextFactory") System.setProperty("log4j2.simplelogLevel", "OFF") System.setProperty("log4j2.statusLoggerLevel", "OFF") - MarkItDownCommand().main(args) + MikroMarkdownCommand().main(args) } diff --git a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt index 0fb6901..c48b58c 100644 --- a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt +++ b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt @@ -1,5 +1,5 @@ package io.github.lemcoder.mikromarkdown -actual fun testMarkItDown(): MikroMarkdown { - return MarkItDown(context = null) +actual fun testMikroMarkdown(): MikroMarkdown { + return MikroMarkdown(context = null) } \ No newline at end of file diff --git a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMarkItDown.kt b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMarkItDown.kt deleted file mode 100644 index b4e22bd..0000000 --- a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMarkItDown.kt +++ /dev/null @@ -1,3 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMarkItDown(): MikroMarkdown = MarkItDown(context = null) diff --git a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt new file mode 100644 index 0000000..1b9af9d --- /dev/null +++ b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt @@ -0,0 +1,3 @@ +package io.github.lemcoder.mikromarkdown + +actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown(context = null) diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt similarity index 84% rename from library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt rename to library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 77b91df..05929da 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -16,7 +16,12 @@ import io.github.lemcoder.mikromarkdown.converters.XmlConverter import io.github.lemcoder.mikromarkdown.utils.AndroidMimeDetector import java.io.File -fun MarkItDown(context: Context? = null): MikroMarkdown = MikroMarkdown(AndroidMimeDetector).apply { +/** + * A [MikroMarkdown] with every Android converter registered. + * + * PDF support needs a [Context]: pdfbox-android loads its resources from the app's assets. + */ +fun MikroMarkdown(context: Context? = null): MikroMarkdown = MikroMarkdown(AndroidMimeDetector).apply { register(MarkdownPassthroughConverter()) register(HtmlConverter()) register(CsvConverter()) diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt index 80f3b6a..9d25f11 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt @@ -1,7 +1,7 @@ package io.github.lemcoder.mikromarkdown -sealed class MarkItDownException(message: String, cause: Throwable? = null) : Exception(message, cause) +sealed class MikroMarkdownException(message: String, cause: Throwable? = null) : Exception(message, cause) -class UnsupportedFormatException(message: String) : MarkItDownException(message) +class UnsupportedFormatException(message: String) : MikroMarkdownException(message) -class FileConversionException(message: String, cause: Throwable? = null) : MarkItDownException(message, cause) +class FileConversionException(message: String, cause: Throwable? = null) : MikroMarkdownException(message, cause) diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt index 0514b9f..8b8deae 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt @@ -39,7 +39,7 @@ class MikroMarkdown( if (!converter.accepts(bytes, info)) continue return try { converter.parse(bytes, info) - } catch (e: MarkItDownException) { + } catch (e: MikroMarkdownException) { throw e } catch (e: Exception) { throw FileConversionException( diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt index bb6065e..0a850a4 100644 --- a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt @@ -5,10 +5,10 @@ import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue -expect fun testMarkItDown(): MikroMarkdown +expect fun testMikroMarkdown(): MikroMarkdown class FileIntegrationTest { - private val mid = testMarkItDown() + private val mid = testMikroMarkdown() private fun assertConversion( filename: String, diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt similarity index 88% rename from library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt rename to library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 6b72477..4827bb5 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -14,7 +14,8 @@ import io.github.lemcoder.mikromarkdown.converters.XlsxConverter import io.github.lemcoder.mikromarkdown.converters.XmlConverter import java.io.File -fun MarkItDown(): MikroMarkdown = MikroMarkdown(TikaMimeDetector).apply { +/** A [MikroMarkdown] with every JVM converter registered and Tika-based format detection. */ +fun MikroMarkdown(): MikroMarkdown = MikroMarkdown(TikaMimeDetector).apply { register(MarkdownPassthroughConverter()) register(HtmlConverter()) register(CsvConverter()) diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt index 951583d..3f242fa 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt @@ -4,7 +4,7 @@ import org.junit.jupiter.api.Test import java.io.File class DumpOutputTest { - private val mid = MarkItDown() + private val mid = MikroMarkdown() @Test fun dumpAll() { diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt index 4befd5c..4edb5d8 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt @@ -1,3 +1,3 @@ package io.github.lemcoder.mikromarkdown -actual fun testMarkItDown(): MikroMarkdown = MarkItDown() \ No newline at end of file +actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() \ No newline at end of file diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt index ce1faa1..81f6041 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt @@ -95,7 +95,7 @@ class PythonComparisonTest { markitdownCmd.isNotEmpty(), "markitdown CLI not available — install via `pip install markitdown` or `uv tool install markitdown`", ) - mid = MarkItDown() + mid = MikroMarkdown() } private fun resolveMarkitdownCmd(): List { From 152f8924b207e2264348f5363a502c5656497467 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Tue, 11 Aug 2026 00:18:40 +0200 Subject: [PATCH 2/3] Vendor anydoc as a submodule The benchmark compares our Markdown against anydoc, and its parsers are the reference for the document-model architecture. Pinning it next to the existing markitdown submodule keeps both references at a known revision. Pinned at v0.1.7. Co-Authored-By: Claude Opus 5 (1M context) --- .gitmodules | 3 +++ anydoc | 1 + 2 files changed, 4 insertions(+) create mode 160000 anydoc diff --git a/.gitmodules b/.gitmodules index e8b0843..1a7c4c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "markitdown"] path = markitdown url = https://github.com/microsoft/markitdown +[submodule "anydoc"] + path = anydoc + url = https://github.com/firecrawl/anydoc.git diff --git a/anydoc b/anydoc new file mode 160000 index 0000000..4a45add --- /dev/null +++ b/anydoc @@ -0,0 +1 @@ +Subproject commit 4a45addbd607e8b59f0c263bca26aab228e10370 From 33e716aee17ca83da97864378a063359483bcabd Mon Sep 17 00:00:00 2001 From: mikolaj Date: Tue, 11 Aug 2026 01:39:46 +0200 Subject: [PATCH 3/3] Add ktfmt, detekt and Konsist architecture rules Three guards, wired into `check` and a CI lint job: - ktfmt (kotlinlang style, 120 columns) formats every source set. ktfmt-gradle only derives tasks for the common and JVM ones, so the library registers matching tasks for androidMain/androidHostTest/androidDeviceTest, which hold half the converters and would otherwise go unchecked. - detekt on top of its default config, with overrides in config/detekt/detekt.yml limited to rules that clash with deliberate choices here (broad catches in converters, PascalCase factory functions). - Konsist encodes the pipeline's boundaries as tests: the model depends on nothing and stays platform-free, converters never import the renderer, and Markdown syntax only appears under render/. Each rule was verified to fail against a deliberate violation. Fixing detekt's findings turned up a real bug: the escaping pass tracked line-start state in a mutable flag that was already false by the time it reached an ordered-list delimiter, so both the digit and delimiter branches were dead and "1. item" in running text could still open a list. Escaping is now position-based, and honours leading indentation. Other cleanups it prompted: - PPTX chart extraction collapsed from eight near-identical loops to five toSeries() helpers, since bar/bar3D, line/line3D and area/area3D each share a generated series type - MikroMarkdown.parse selects the converter first, with failure wrapping in a single parseOrFail helper - the EPUB manifest predicate and the CLI command class each moved out into their own named unit The bulk of the diff is the one-time ktfmt sweep. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/gradle.yml | 15 ++ README.md | 15 ++ build.gradle.kts | 32 +++ cli/build.gradle.kts | 14 +- .../main/kotlin/com/mikromarkdown/cli/Main.kt | 31 --- .../mikromarkdown/cli/MikroMarkdownCommand.kt | 34 +++ config/detekt/detekt.yml | 48 ++++ gradle/libs.versions.toml | 6 + library/build.gradle.kts | 46 ++-- .../FileIntegrationTest.android.kt | 2 +- .../mikromarkdown/MikroMarkdownFactory.kt | 31 +-- .../mikromarkdown/converters/CsvConverter.kt | 11 +- .../mikromarkdown/converters/DocxConverter.kt | 26 ++- .../mikromarkdown/converters/EpubConverter.kt | 42 ++-- .../mikromarkdown/converters/HtmlConverter.kt | 3 +- .../mikromarkdown/converters/JsonConverter.kt | 36 +-- .../mikromarkdown/converters/PdfConverter.kt | 11 +- .../mikromarkdown/converters/PptxConverter.kt | 165 +++++++------ .../mikromarkdown/converters/XlsxConverter.kt | 19 +- .../mikromarkdown/converters/XmlConverter.kt | 37 +-- .../mikromarkdown/utils/HtmlToDocument.kt | 221 +++++++++++------- .../mikromarkdown/DocumentConverter.kt | 1 + .../lemcoder/mikromarkdown/MikroMarkdown.kt | 32 +-- .../converters/PlainTextConverter.kt | 3 +- .../lemcoder/mikromarkdown/model/Document.kt | 12 +- .../mikromarkdown/model/DocumentBuilder.kt | 15 +- .../mikromarkdown/render/MarkdownRenderer.kt | 216 +++++++++-------- .../mikromarkdown/utils/TextBlocks.kt | 15 +- .../mikromarkdown/FileIntegrationTest.kt | 127 +++++----- .../mikromarkdown/MikroMarkdownFactory.kt | 29 +-- .../mikromarkdown/converters/CsvConverter.kt | 11 +- .../mikromarkdown/converters/DocxConverter.kt | 26 ++- .../mikromarkdown/converters/EpubConverter.kt | 42 ++-- .../mikromarkdown/converters/HtmlConverter.kt | 3 +- .../mikromarkdown/converters/JsonConverter.kt | 36 +-- .../mikromarkdown/converters/PdfConverter.kt | 11 +- .../mikromarkdown/converters/PptxConverter.kt | 165 +++++++------ .../mikromarkdown/converters/XlsxConverter.kt | 19 +- .../mikromarkdown/converters/XmlConverter.kt | 37 +-- .../mikromarkdown/utils/HtmlToDocument.kt | 221 +++++++++++------- .../mikromarkdown/utils/TikaMimeDetector.kt | 9 +- .../mikromarkdown/ArchitectureTest.kt | 63 +++++ .../lemcoder/mikromarkdown/DumpOutputTest.kt | 13 +- .../mikromarkdown/FileIntegrationTest.jvm.kt | 2 +- .../mikromarkdown/PythonComparisonTest.kt | 75 +++--- settings.gradle.kts | 2 + 46 files changed, 1195 insertions(+), 835 deletions(-) create mode 100644 cli/src/main/kotlin/com/mikromarkdown/cli/MikroMarkdownCommand.kt create mode 100644 config/detekt/detekt.yml create mode 100644 library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index fe4d926..2c10c1d 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -33,3 +33,18 @@ jobs: - name: Run tests uses: gradle/actions/setup-gradle@v4 - run: ./gradlew :library:${{ matrix.target }} + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Validate Gradle Wrapper + uses: gradle/actions/wrapper-validation@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + - uses: gradle/actions/setup-gradle@v4 + - name: Formatting and static analysis + run: ./gradlew ktfmtCheck detekt diff --git a/README.md b/README.md index f4b0e24..74ed13d 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,21 @@ mid.register(HtmlConverter()) Both extend `MikroMarkdownException`. +## Code quality + +| tool | task | what it guards | +|---|---|---| +| [ktfmt](https://github.com/facebook/ktfmt) | `./gradlew ktfmtFormat` / `ktfmtCheck` | formatting (kotlinlang style, 120 columns) | +| [detekt](https://detekt.dev) | `./gradlew detekt` | static analysis; overrides in `config/detekt/detekt.yml` | +| [Konsist](https://docs.konsist.lemonappdev.com) | `./gradlew :library:jvmTest --tests '*ArchitectureTest*'` | pipeline boundaries | + +`./gradlew check` runs all three. The Konsist rules encode the architecture: the model depends on +nothing, converters never import the renderer, and Markdown syntax appears only under `render/` — +so a converter cannot quietly start building Markdown strings again. + +ktfmt-gradle only derives tasks for the common and JVM source sets, so `library/build.gradle.kts` +registers matching tasks for the Android ones. + ## Benchmark `scripts/benchmark.py` converts the test fixtures with MikroMarkdown, Python diff --git a/build.gradle.kts b/build.gradle.kts index 5662bb4..393cb69 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,4 +2,36 @@ plugins { alias(libs.plugins.android.kotlin.multiplatform.library) apply false alias(libs.plugins.kotlinMultiplatform) apply false alias(libs.plugins.kotlin.jvm) apply false + alias(libs.plugins.ktfmt) + alias(libs.plugins.detekt) +} + +allprojects { + apply(plugin = rootProject.libs.plugins.ktfmt.get().pluginId) + apply(plugin = rootProject.libs.plugins.detekt.get().pluginId) + + ktfmt { + // Matches the existing sources: 4-space indent, no import reordering surprises. + kotlinLangStyle() + maxWidth = 120 + } + + detekt { + buildUponDefaultConfig = true + parallel = true + config.setFrom(rootProject.file("config/detekt/detekt.yml")) + basePath = rootProject.projectDir.absolutePath + // Generated sources and the vendored reference checkouts are not ours to lint. + source.setFrom(files("src").filter { it.exists() }) + } + + tasks.withType().configureEach { + jvmTarget = "21" + reports { + html.required = true + sarif.required = true + md.required = false + txt.required = false + } + } } diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index ae93acb..9601598 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -3,21 +3,13 @@ plugins { application } -java { - toolchain { - languageVersion = JavaLanguageVersion.of(21) - } -} +java { toolchain { languageVersion = JavaLanguageVersion.of(21) } } -application { - mainClass = "com.mikromarkdown.cli.MainKt" -} +application { mainClass = "com.mikromarkdown.cli.MainKt" } dependencies { implementation(project(":library")) implementation(libs.clikt) } -tasks.test { - useJUnitPlatform() -} +tasks.test { useJUnitPlatform() } diff --git a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt index a3a6b46..11974c1 100644 --- a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt +++ b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt @@ -1,37 +1,6 @@ package com.mikromarkdown.cli -import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.main -import com.github.ajalt.clikt.parameters.arguments.argument -import com.github.ajalt.clikt.parameters.arguments.optional -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.types.path -import io.github.lemcoder.mikromarkdown.MikroMarkdown -import io.github.lemcoder.mikromarkdown.StreamInfo - -class MikroMarkdownCommand : CliktCommand(name = "mikromarkdown") { - private val file by argument("FILE", help = "Input file (reads stdin if omitted)").path(mustExist = true).optional() - private val output by option("-o", "--output", help = "Output file (default: stdout)").path() - private val extension by option("-x", "--extension", help = "File extension hint (e.g. html)") - private val mimeType by option("-m", "--mime-type", help = "MIME type hint (e.g. text/html)") - - override fun run() { - val mikroMarkdown = MikroMarkdown() - - val result = if (file != null) { - mikroMarkdown.convert(file!!.toFile().absolutePath) - } else { - val info = StreamInfo(extension = extension, mimetype = mimeType) - mikroMarkdown.convert(System.`in`.readBytes(), info) - } - - if (output != null) { - output!!.toFile().writeText(result.markdown) - } else { - print(result.markdown) - } - } -} fun main(args: Array) { // PDFBox pulls in the Log4j API; without a provider it writes a banner to stdout, diff --git a/cli/src/main/kotlin/com/mikromarkdown/cli/MikroMarkdownCommand.kt b/cli/src/main/kotlin/com/mikromarkdown/cli/MikroMarkdownCommand.kt new file mode 100644 index 0000000..03f6f5a --- /dev/null +++ b/cli/src/main/kotlin/com/mikromarkdown/cli/MikroMarkdownCommand.kt @@ -0,0 +1,34 @@ +package com.mikromarkdown.cli + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.optional +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import io.github.lemcoder.mikromarkdown.MikroMarkdown +import io.github.lemcoder.mikromarkdown.StreamInfo + +class MikroMarkdownCommand : CliktCommand(name = "mikromarkdown") { + private val file by argument("FILE", help = "Input file (reads stdin if omitted)").path(mustExist = true).optional() + private val output by option("-o", "--output", help = "Output file (default: stdout)").path() + private val extension by option("-x", "--extension", help = "File extension hint (e.g. html)") + private val mimeType by option("-m", "--mime-type", help = "MIME type hint (e.g. text/html)") + + override fun run() { + val mikroMarkdown = MikroMarkdown() + + val result = + if (file != null) { + mikroMarkdown.convert(file!!.toFile().absolutePath) + } else { + val info = StreamInfo(extension = extension, mimetype = mimeType) + mikroMarkdown.convert(System.`in`.readBytes(), info) + } + + if (output != null) { + output!!.toFile().writeText(result.markdown) + } else { + print(result.markdown) + } + } +} diff --git a/config/detekt/detekt.yml b/config/detekt/detekt.yml new file mode 100644 index 0000000..b84ea8d --- /dev/null +++ b/config/detekt/detekt.yml @@ -0,0 +1,48 @@ +# Overrides on top of detekt's default config (buildUponDefaultConfig = true). +# Only rules that clash with deliberate choices in this codebase are relaxed. + +build: + maxIssues: 0 + +complexity: + # Parsers walk large format-specific type hierarchies; splitting them further hurts readability. + CyclomaticComplexMethod: + threshold: 25 + LongMethod: + threshold: 90 + LongParameterList: + functionThreshold: 8 + constructorThreshold: 10 + NestedBlockDepth: + threshold: 6 + TooManyFunctions: + thresholdInFiles: 30 + thresholdInClasses: 30 + thresholdInObjects: 30 + ignorePrivate: true + +exceptions: + # Format libraries throw broadly; converters degrade gracefully rather than propagate. + TooGenericExceptionCaught: + active: false + SwallowedException: + active: false + +naming: + FunctionNaming: + # Factory functions mirror the type they build: MikroMarkdown(), document {}. + functionPattern: '[a-zA-Z][a-zA-Z0-9]*' + +style: + MagicNumber: + active: false + MaxLineLength: + maxLineLength: 120 + ReturnCount: + max: 6 + ForbiddenComment: + active: false + UnusedPrivateMember: + active: true + LoopWithTooManyJumpStatements: + maxJumpCount: 4 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c6771a9..44f96f3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,9 @@ poi = "5.5.1" tika = "3.3.0" clikt = "5.1.0" coreKtx = "1.7.0" +detekt = "1.23.8" +ktfmt = "0.27.0" +konsist = "0.17.3" [libraries] kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } @@ -33,6 +36,7 @@ poi-ooxml = { module = "org.apache.poi:poi-ooxml", version.ref = "poi" } tika-core = { module = "org.apache.tika:tika-core", version.ref = "tika" } clikt = { module = "com.github.ajalt.clikt:clikt", version.ref = "clikt" } core-ktx = { group = "androidx.test", name = "core-ktx", version.ref = "coreKtx" } +konsist = { module = "com.lemonappdev:konsist", version.ref = "konsist" } [plugins] android-kotlin-multiplatform-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } @@ -40,3 +44,5 @@ kotlinMultiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version.ref = "vanniktechMavenPublish" } kotlinx-resources = { id = "com.goncalossilva.resources", version.ref = "kotlinx-resources" } +detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } +ktfmt = { id = "com.ncorti.ktfmt.gradle", version.ref = "ktfmt" } diff --git a/library/build.gradle.kts b/library/build.gradle.kts index b3a51cf..ad92d04 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -7,16 +7,13 @@ plugins { } group = "io.github.lemcoder" + version = "0.1.0" kotlin { jvm { - compilerOptions { - jvmTarget = JvmTarget.JVM_21 - } - testRuns["test"].executionTask.configure { - useJUnitPlatform() - } + compilerOptions { jvmTarget = JvmTarget.JVM_21 } + testRuns["test"].executionTask.configure { useJUnitPlatform() } } androidLibrary { @@ -25,19 +22,13 @@ kotlin { minSdk = libs.versions.android.minSdk.get().toInt() withHostTestBuilder {}.configure {} - withDeviceTestBuilder { - sourceSetTreeName = "test" - } + withDeviceTestBuilder { sourceSetTreeName = "test" } - compilerOptions { - jvmTarget = JvmTarget.JVM_11 - } + compilerOptions { jvmTarget = JvmTarget.JVM_11 } } sourceSets { - commonMain.dependencies { - implementation(libs.kotlinx.io.core) - } + commonMain.dependencies { implementation(libs.kotlinx.io.core) } jvmMain.dependencies { implementation(libs.jsoup) @@ -64,6 +55,31 @@ kotlin { jvmTest.dependencies { implementation(libs.junit.jupiter) implementation(libs.kotlin.test) + implementation(libs.konsist) } } } + +// ktfmt-gradle only derives tasks for the common and JVM source sets, so the Android +// ones — where half the converters live — would go unformatted and unchecked. +run { + val androidSources = fileTree("src") { include("android*/**/*.kt") } + val template = tasks.named("ktfmtFormatKmpCommonMain") + + val formatAndroid = + tasks.register("ktfmtFormatAndroidSourceSets") { + ktfmtClasspath.from(template.map { it.ktfmtClasspath }) + formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) + setSource(androidSources) + } + val checkAndroid = + tasks.register("ktfmtCheckAndroidSourceSets") { + ktfmtClasspath.from(template.map { it.ktfmtClasspath }) + formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) + setSource(androidSources) + } + + tasks.named("ktfmtFormat") { dependsOn(formatAndroid) } + tasks.named("ktfmtCheck") { dependsOn(checkAndroid) } + tasks.named("check") { dependsOn(checkAndroid) } +} diff --git a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt index c48b58c..97d5560 100644 --- a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt +++ b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt @@ -2,4 +2,4 @@ package io.github.lemcoder.mikromarkdown actual fun testMikroMarkdown(): MikroMarkdown { return MikroMarkdown(context = null) -} \ No newline at end of file +} diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 05929da..07b15c8 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -21,21 +21,22 @@ import java.io.File * * PDF support needs a [Context]: pdfbox-android loads its resources from the app's assets. */ -fun MikroMarkdown(context: Context? = null): MikroMarkdown = MikroMarkdown(AndroidMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(DocxConverter()) - register(XlsxConverter()) - register(PptxConverter()) - register(EpubConverter()) - if (context != null) { - PDFBoxResourceLoader.init(context) - register(PdfConverter()) +fun MikroMarkdown(context: Context? = null): MikroMarkdown = + MikroMarkdown(AndroidMimeDetector).apply { + register(MarkdownPassthroughConverter()) + register(HtmlConverter()) + register(CsvConverter()) + register(JsonConverter()) + register(XmlConverter()) + register(DocxConverter()) + register(XlsxConverter()) + register(PptxConverter()) + register(EpubConverter()) + if (context != null) { + PDFBoxResourceLoader.init(context) + register(PdfConverter()) + } + register(PlainTextConverter(), priority = 10.0) } - register(PlainTextConverter(), priority = 10.0) -} fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt index b93bda6..9f11ac2 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt @@ -5,9 +5,9 @@ import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.Document import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell +import java.io.InputStreamReader import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser -import java.io.InputStreamReader class CsvConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -22,10 +22,11 @@ class CsvConverter : DocumentConverter { val header = records[0].toList() if (header.isEmpty()) return Document() - val rows = records.drop(1).map { record -> - // Ragged rows are padded by the renderer; only extra columns need trimming here. - List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } - } + val rows = + records.drop(1).map { record -> + // Ragged rows are padded by the renderer; only extra columns need trimming here. + List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } + } return Document(blocks = listOf(Table(header = header.map { TableCell(it) }, rows = rows))) } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt index b1bc8e6..79a154a 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt @@ -16,15 +16,15 @@ import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.model.plainText import io.github.lemcoder.mikromarkdown.model.styled +import java.util.Base64 import org.apache.poi.xwpf.usermodel.XWPFDocument import org.apache.poi.xwpf.usermodel.XWPFParagraph import org.apache.poi.xwpf.usermodel.XWPFTable -import java.util.Base64 class DocxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "docx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -94,11 +94,12 @@ class DocxConverter : DocumentConverter { val mime = data.pictureTypeEnum.contentType val id = data.fileName ?: "image-${assets.size + 1}" assets += Asset(id = id, mediaType = mime, bytes = data.data, name = data.fileName) - out += Image( - alt = alt, - url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", - assetId = id, - ) + out += + Image( + alt = alt, + url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", + assetId = id, + ) } continue } @@ -126,11 +127,12 @@ class DocxConverter : DocumentConverter { itemLevel < level -> break itemLevel == level -> { index++ - val children = if (index < items.size && items[index].first > level) { - listOf(ListBlock(ordered = false, items = build(items[index].first))) - } else { - emptyList() - } + val children = + if (index < items.size && items[index].first > level) { + listOf(ListBlock(ordered = false, items = build(items[index].first))) + } else { + emptyList() + } result += ListItem(listOf(Paragraph(content)) + children) } // A deeper first item without a parent: promote it to this level. diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt index 9c80056..2542a64 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt @@ -8,20 +8,21 @@ import io.github.lemcoder.mikromarkdown.model.Paragraph import io.github.lemcoder.mikromarkdown.model.Strong import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument -import org.w3c.dom.Element -import org.xml.sax.InputSource import java.io.StringReader import java.util.zip.ZipInputStream import javax.xml.parsers.DocumentBuilderFactory +import org.w3c.dom.Element +import org.xml.sax.InputSource class EpubConverter : DocumentConverter { - private val metaFields = listOf( - "title" to "Title", - "creator" to "Authors", - "language" to "Language", - "description" to "Description", - "identifier" to "Identifier", - ) + private val metaFields = + listOf( + "title" to "Title", + "creator" to "Authors", + "language" to "Language", + "description" to "Description", + "identifier" to "Identifier", + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "epub" || info.mimetype == "application/epub+zip" @@ -100,8 +101,7 @@ class EpubConverter : DocumentConverter { val id = item.getAttribute("id") val href = item.getAttribute("href") val mediaType = item.getAttribute("media-type") - if (id.isNotEmpty() && href.isNotEmpty() && - (mediaType.contains("html") || mediaType.contains("xhtml"))) { + if (id.isNotEmpty() && href.isNotEmpty() && isReadableChapter(mediaType)) { manifest[id] = href } } @@ -117,12 +117,16 @@ class EpubConverter : DocumentConverter { return Triple(manifest, spine, metadata) } - private fun parseXml(bytes: ByteArray) = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = false - factory.isExpandEntityReferences = false - factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) - } catch (_: Exception) { - null - } + /** Only the spine's (X)HTML documents carry text; images and styles are skipped. */ + private fun isReadableChapter(mediaType: String): Boolean = mediaType.contains("html") + + private fun parseXml(bytes: ByteArray) = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = false + factory.isExpandEntityReferences = false + factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) + } catch (_: Exception) { + null + } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt index fb5dffd..b716b5d 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt @@ -7,8 +7,7 @@ import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument class HtmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { - return info.extension in setOf("html", "htm") || - info.mimetype in setOf("text/html", "application/xhtml+xml") + return info.extension in setOf("html", "htm") || info.mimetype in setOf("text/html", "application/xhtml+xml") } override fun parse(bytes: ByteArray, info: StreamInfo): Document = diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt index 026c9d1..9b4dbf4 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt @@ -11,16 +11,21 @@ import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document class JsonConverter : DocumentConverter { - private val writer = ObjectMapper().apply { - registerKotlinModule() - }.writer(object : DefaultPrettyPrinter() { - init { - indentArraysWith(DefaultIndenter(" ", "\n")) - indentObjectsWith(DefaultIndenter(" ", "\n")) - } - override fun createInstance() = this - override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") - }) + private val writer = + ObjectMapper() + .apply { registerKotlinModule() } + .writer( + object : DefaultPrettyPrinter() { + init { + indentArraysWith(DefaultIndenter(" ", "\n")) + indentObjectsWith(DefaultIndenter(" ", "\n")) + } + + override fun createInstance() = this + + override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") + } + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "json" || info.mimetype in setOf("application/json", "text/json") @@ -28,11 +33,12 @@ class JsonConverter : DocumentConverter { override fun parse(bytes: ByteArray, info: StreamInfo): Document { val json = bytes.toString(Charsets.UTF_8) - val pretty = try { - writer.writeValueAsString(ObjectMapper().readTree(json)) - } catch (_: Exception) { - json - } + val pretty = + try { + writer.writeValueAsString(ObjectMapper().readTree(json)) + } catch (_: Exception) { + json + } return Document(blocks = listOf(CodeBlock(pretty, "json"))) } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt index 262e38b..9eb0647 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt @@ -18,11 +18,12 @@ class PdfConverter : DocumentConverter { val title = doc.documentInformation?.title?.trim()?.ifBlank { null } // Paragraph markers let the text blocks split on real paragraph breaks // instead of collapsing a page into one block. - val stripper = PDFTextStripper().apply { - sortByPosition = true - setAddMoreFormatting(true) - paragraphStart = "\n" - } + val stripper = + PDFTextStripper().apply { + sortByPosition = true + setAddMoreFormatting(true) + paragraphStart = "\n" + } return Document(blocks = plainTextBlocks(stripper.getText(doc)), title = title) } finally { doc.close() diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt index 4c363e5..78578a7 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt @@ -23,15 +23,20 @@ import org.apache.poi.xslf.usermodel.XSLFPictureShape import org.apache.poi.xslf.usermodel.XSLFSimpleShape import org.apache.poi.xslf.usermodel.XSLFTable import org.apache.poi.xslf.usermodel.XSLFTextShape +import org.openxmlformats.schemas.drawingml.x2006.chart.CTAreaSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTAxDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTBarSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTLineSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTNumDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTPieSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTScatterSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTSerTx import org.openxmlformats.schemas.presentationml.x2006.main.CTPicture class PptxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "pptx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" + info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -44,12 +49,14 @@ class PptxConverter : DocumentConverter { blocks += HtmlComment("Slide number: ${index + 1}") blocks += shapeBlocks(slide.shapes) { if (index == 0 && title == null) title = it } - val notes = slide.notes?.shapes - ?.filterIsInstance() - ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } - ?.joinToString("\n") { it.text } - ?.trim() - .orEmpty() + val notes = + slide.notes + ?.shapes + ?.filterIsInstance() + ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } + ?.joinToString("\n") { it.text } + ?.trim() + .orEmpty() if (notes.isNotBlank()) { blocks += Heading(3, listOf(Text("Notes:"))) blocks += notes.lines().filter { it.isNotBlank() }.map { Paragraph(listOf(Text(it.trim()))) } @@ -118,49 +125,83 @@ class PptxConverter : DocumentConverter { val blocks = mutableListOf() blocks += Heading(3, listOf(Text(listOfNotNull("Chart", chartTitle(chart)).joinToString(": ")))) - val series = try { - seriesOf(chart) - } catch (_: Exception) { - blocks += Paragraph(listOf(Text("[unsupported chart]"))) - return blocks - } + val series = + try { + seriesOf(chart) + } catch (_: Exception) { + blocks += Paragraph(listOf(Text("[unsupported chart]"))) + return blocks + } if (series.isEmpty()) return blocks val rowCount = series.maxOf { it.categories.size } - blocks += Table( - header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, - rows = (0 until rowCount).map { row -> - val category = series.first().categories.getOrElse(row) { "" } - (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } - }, - ) + blocks += + Table( + header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, + rows = + (0 until rowCount).map { row -> + val category = series.first().categories.getOrElse(row) { "" } + (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } + }, + ) return blocks } - private fun chartTitle(chart: XSLFChart): String? = try { - val ctChart = chart.ctChart - if (!ctChart.isSetTitle) { + private fun chartTitle(chart: XSLFChart): String? = + try { + val ctChart = chart.ctChart + if (!ctChart.isSetTitle) { + null + } else { + val tx = ctChart.title?.tx + when { + tx?.isSetRich == true -> + tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") + tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v + else -> null + }?.ifBlank { null } + } + } catch (_: Exception) { null - } else { - val tx = ctChart.title?.tx - when { - tx?.isSetRich == true -> - tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") - tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v - else -> null - }?.ifBlank { null } } - } catch (_: Exception) { - null - } private data class Series(val name: String, val categories: List, val values: List) private fun seriesOf(chart: XSLFChart): List { - val plotArea = chart.ctChart.plotArea - val out = mutableListOf() + val plot = chart.ctChart.plotArea + // bar/bar3D, line/line3D and area/area3D each share one generated series type. + val all = + plot.barChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.bar3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.lineChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.line3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.areaChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.area3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.scatterChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.pieChartList.flatMap { it.serList.map { s -> s.toSeries() } } + return all.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } + } + + private fun CTBarSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTLineSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTAreaSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTPieSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) - fun categoryValues(cat: CTAxDataSource?): List = when { + private fun CTScatterSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetXVal) xVal else null, if (isSetYVal) yVal else null) + + private fun series(tx: CTSerTx?, categories: CTAxDataSource?, values: CTNumDataSource?) = + Series(seriesName(tx), categoryValues(categories), numericValues(values)) + + private fun categoryValues(cat: CTAxDataSource?): List = + when { cat == null -> emptyList() cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() @@ -169,64 +210,22 @@ class PptxConverter : DocumentConverter { else -> emptyList() } - fun numericValues(v: CTNumDataSource?): List = when { + private fun numericValues(v: CTNumDataSource?): List = + when { v == null -> emptyList() v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() else -> emptyList() } - fun seriesName(tx: CTSerTx?): String = when { + private fun seriesName(tx: CTSerTx?): String = + when { tx == null -> "" tx.isSetV -> tx.v tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v.orEmpty() else -> "" } - for (c in plotArea.barChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.bar3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.lineChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.line3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.areaChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.area3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.scatterChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetXVal) s.xVal else null), - numericValues(if (s.isSetYVal) s.yVal else null), - ) - for (c in plotArea.pieChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - - return out.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } - } - private fun table(table: XSLFTable): Table? { val rows = table.rows if (rows.isEmpty()) return null diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt index 984dec0..e601f5c 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt @@ -8,18 +8,18 @@ import io.github.lemcoder.mikromarkdown.model.Heading import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text +import kotlin.math.floor import org.apache.poi.ss.usermodel.Cell import org.apache.poi.ss.usermodel.CellType import org.apache.poi.ss.usermodel.DataFormatter import org.apache.poi.xssf.usermodel.XSSFWorkbook -import kotlin.math.floor class XlsxConverter : DocumentConverter { private val formatter = DataFormatter() override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "xlsx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -35,12 +35,12 @@ class XlsxConverter : DocumentConverter { if (columns == 0) continue blocks += Heading(2, listOf(Text(sheet.sheetName))) - blocks += Table( - header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, - rows = rows.drop(1).map { row -> - (0 until columns).map { TableCell(cellValue(row.getCell(it))) } - }, - ) + blocks += + Table( + header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, + rows = + rows.drop(1).map { row -> (0 until columns).map { TableCell(cellValue(row.getCell(it))) } }, + ) } return Document(blocks = blocks) @@ -54,8 +54,7 @@ class XlsxConverter : DocumentConverter { return when (cell.cellType) { CellType.NUMERIC -> { val v = cell.numericCellValue - if (v == floor(v) && !v.isInfinite()) v.toLong().toString() - else formatter.formatCellValue(cell) + if (v == floor(v) && !v.isInfinite()) v.toLong().toString() else formatter.formatCellValue(cell) } CellType.BLANK -> "" else -> formatter.formatCellValue(cell).trim() diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt index 8979d57..7d3e6ac 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt @@ -4,7 +4,6 @@ import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document -import org.xml.sax.InputSource import java.io.StringReader import java.io.StringWriter import javax.xml.parsers.DocumentBuilderFactory @@ -12,6 +11,7 @@ import javax.xml.transform.OutputKeys import javax.xml.transform.TransformerFactory import javax.xml.transform.dom.DOMSource import javax.xml.transform.stream.StreamResult +import org.xml.sax.InputSource class XmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -23,23 +23,24 @@ class XmlConverter : DocumentConverter { return Document(blocks = listOf(CodeBlock(pretty, "xml"))) } - private fun prettyPrint(xml: String): String = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = true - factory.isIgnoringElementContentWhitespace = true - val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) + private fun prettyPrint(xml: String): String = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = true + factory.isIgnoringElementContentWhitespace = true + val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) - val tf = TransformerFactory.newInstance() - tf.setAttribute("indent-number", 2) - val transformer = tf.newTransformer() - transformer.setOutputProperty(OutputKeys.INDENT, "yes") - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") + val tf = TransformerFactory.newInstance() + tf.setAttribute("indent-number", 2) + val transformer = tf.newTransformer() + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") - val writer = StringWriter() - transformer.transform(DOMSource(doc), StreamResult(writer)) - writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") - } catch (_: Exception) { - xml - } + val writer = StringWriter() + transformer.transform(DOMSource(doc), StreamResult(writer)) + writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") + } catch (_: Exception) { + xml + } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt index d7c9296..510e036 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt @@ -29,22 +29,49 @@ import org.jsoup.nodes.TextNode /** * Walks an HTML DOM into the shared document model. * - * Replaces the previous HTML → Markdown string conversion: tables, lists and inline - * emphasis become model nodes, so the Markdown renderer owns all syntax decisions. + * Replaces the previous HTML → Markdown string conversion: tables, lists and inline emphasis become model nodes, so the + * Markdown renderer owns all syntax decisions. */ object HtmlToDocument { - private val DROPPED_TAGS = setOf( - "script", "style", "noscript", "template", "button", "svg", "iframe", "form", "input", "select", - ) + private val DROPPED_TAGS = + setOf( + "script", + "style", + "noscript", + "template", + "button", + "svg", + "iframe", + "form", + "input", + "select", + ) private val HEADINGS = mapOf("h1" to 1, "h2" to 2, "h3" to 3, "h4" to 4, "h5" to 5, "h6" to 6) /** Tags that only group other content; their children are lifted into the parent block flow. */ - private val CONTAINERS = setOf( - "div", "section", "article", "main", "header", "footer", "aside", "nav", "body", "html", - "figure", "details", "summary", "fieldset", "center", "hgroup", "picture", "colgroup", - ) + private val CONTAINERS = + setOf( + "div", + "section", + "article", + "main", + "header", + "footer", + "aside", + "nav", + "body", + "html", + "figure", + "details", + "summary", + "fieldset", + "center", + "hgroup", + "picture", + "colgroup", + ) fun parse(html: String, baseUri: String = ""): Document { val doc = Jsoup.parse(html, baseUri) @@ -91,49 +118,66 @@ object HtmlToDocument { return out } - private fun Element.isBlockLevel(): Boolean = when (tagName()) { - in HEADINGS, "p", "ul", "ol", "table", "pre", "blockquote", "hr", "dl", "li", "figcaption" -> true - in CONTAINERS -> true - else -> false - } - - private fun blockFor(element: Element): List = when (val tag = element.tagName()) { - in HEADINGS -> listOfNotNull( - inlines(element).trimEdges() - .takeIf { it.isNotEmpty() } - ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) }, - ) + private fun Element.isBlockLevel(): Boolean = + when (tagName()) { + in HEADINGS, + "p", + "ul", + "ol", + "table", + "pre", + "blockquote", + "hr", + "dl", + "li", + "figcaption" -> true + in CONTAINERS -> true + else -> false + } - "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) - "ul", "ol" -> listBlock(element) - "table" -> table(element) - "pre" -> listOf(codeBlock(element)) - "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() - "hr" -> listOf(ThematicBreak) - "dl" -> definitionList(element) - "figcaption" -> listOfNotNull( - inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) }, - ) - // Containers and stray
  • outside a list contribute their children directly. - else -> blocks(element) - } + private fun blockFor(element: Element): List = + when (val tag = element.tagName()) { + in HEADINGS -> + listOfNotNull( + inlines(element) + .trimEdges() + .takeIf { it.isNotEmpty() } + ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) } + ) + + "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) + "ul", + "ol" -> listBlock(element) + "table" -> table(element) + "pre" -> listOf(codeBlock(element)) + "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() + "hr" -> listOf(ThematicBreak) + "dl" -> definitionList(element) + "figcaption" -> + listOfNotNull( + inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) } + ) + // Containers and stray
  • outside a list contribute their children directly. + else -> blocks(element) + } private fun listBlock(element: Element): List { val ordered = element.tagName() == "ol" val start = element.attr("start").toIntOrNull() ?: 1 - val items = element.children() - .filter { it.tagName() == "li" } - .map { li -> ListItem(blocks = blocks(li)) } - .filter { it.blocks.isNotEmpty() } + val items = + element + .children() + .filter { it.tagName() == "li" } + .map { li -> ListItem(blocks = blocks(li)) } + .filter { it.blocks.isNotEmpty() } if (items.isEmpty()) return emptyList() return listOf(ListBlock(ordered = ordered, items = items, start = start)) } private fun codeBlock(element: Element): Block { val code = element.selectFirst("code") ?: element - val language = code.classNames() - .firstOrNull { it.startsWith("language-") || it.startsWith("lang-") } - ?.substringAfter('-') + val language = + code.classNames().firstOrNull { it.startsWith("language-") || it.startsWith("lang-") }?.substringAfter('-') return CodeBlock(code.wholeText().trimEnd(), language) } @@ -158,20 +202,22 @@ object HtmlToDocument { val body = mutableListOf>() for ((index, tr) in rows.withIndex()) { - val cells = tr.children() - .filter { it.tagName() == "th" || it.tagName() == "td" } - .map { cell -> - TableCell( - content = inlines(cell).trimEdges(), - colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - ) - } + val cells = + tr.children() + .filter { it.tagName() == "th" || it.tagName() == "td" } + .map { cell -> + TableCell( + content = inlines(cell).trimEdges(), + colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + ) + } if (cells.isEmpty()) continue - val isHeaderRow = index == 0 && - tr.children().all { it.tagName() == "th" } && - (tr.parent()?.tagName() == "thead" || header.isEmpty()) + val isHeaderRow = + index == 0 && + tr.children().all { it.tagName() == "th" } && + (tr.parent()?.tagName() == "thead" || header.isEmpty()) if (isHeaderRow && header.isEmpty()) header = cells else body += cells } @@ -196,35 +242,45 @@ object HtmlToDocument { if (text.isNotEmpty()) add(Text(text)) } - is Element -> when (node.tagName()) { - "br" -> add(LineBreak) - "img" -> { - val src = node.attr("abs:src").ifBlank { node.attr("src") } - val alt = node.attr("alt") - if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) - } + is Element -> + when (node.tagName()) { + "br" -> add(LineBreak) + "img" -> { + val src = node.attr("abs:src").ifBlank { node.attr("src") } + val alt = node.attr("alt") + if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) + } - "a" -> { - val href = node.attr("abs:href").ifBlank { node.attr("href") } - val content = inlines(node).trimEdges() - when { - content.isEmpty() -> Unit - href.isBlank() || href.startsWith("javascript:") -> addAll(content) - else -> add(Link(content, href, node.attr("title").ifBlank { null })) + "a" -> { + val href = node.attr("abs:href").ifBlank { node.attr("href") } + val content = inlines(node).trimEdges() + when { + content.isEmpty() -> Unit + href.isBlank() || href.startsWith("javascript:") -> addAll(content) + else -> add(Link(content, href, node.attr("title").ifBlank { null })) + } } - } - "strong", "b" -> wrapped(node) { Strong(it) } - "em", "i", "cite", "var" -> wrapped(node) { Emphasis(it) } - "del", "s", "strike" -> wrapped(node) { Strikethrough(it) } - "code", "kbd", "samp", "tt" -> { - val code = node.wholeText().trim() - if (code.isNotEmpty()) add(CodeSpan(code)) - } + "strong", + "b" -> wrapped(node) { Strong(it) } + "em", + "i", + "cite", + "var" -> wrapped(node) { Emphasis(it) } + "del", + "s", + "strike" -> wrapped(node) { Strikethrough(it) } + "code", + "kbd", + "samp", + "tt" -> { + val code = node.wholeText().trim() + if (code.isNotEmpty()) add(CodeSpan(code)) + } - // Block-level content encountered inline (e.g. a
    inside a ): keep its text. - else -> addAll(inlines(node)) - } + // Block-level content encountered inline (e.g. a
    inside a ): keep its text. + else -> addAll(inlines(node)) + } } } @@ -239,14 +295,11 @@ object HtmlToDocument { /** HTML collapses runs of whitespace; do the same before the text reaches the model. */ // Non-breaking spaces are not collapsible whitespace in HTML, so they survive verbatim. - private fun TextNode.normalizedText(): String = - wholeText.replace(Regex("\\s+"), " ") + private fun TextNode.normalizedText(): String = wholeText.replace(Regex("\\s+"), " ") - private fun List.startsWithSpace(): Boolean = - (firstOrNull() as? Text)?.value?.startsWith(" ") == true + private fun List.startsWithSpace(): Boolean = (firstOrNull() as? Text)?.value?.startsWith(" ") == true - private fun List.endsWithSpace(): Boolean = - (lastOrNull() as? Text)?.value?.endsWith(" ") == true + private fun List.endsWithSpace(): Boolean = (lastOrNull() as? Text)?.value?.endsWith(" ") == true /** Drops leading/trailing whitespace-only text so emphasis markers hug their content. */ private fun List.trimEdges(): List { diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt index 07310a2..ba8cc28 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt @@ -9,5 +9,6 @@ import io.github.lemcoder.mikromarkdown.model.Document */ interface DocumentConverter { fun accepts(bytes: ByteArray, info: StreamInfo): Boolean + fun parse(bytes: ByteArray, info: StreamInfo): Document } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt index 8b8deae..d6d2c40 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt @@ -34,25 +34,25 @@ class MikroMarkdown( } fun parse(bytes: ByteArray, info: StreamInfo): Document { - val sorted = converters.sortedBy { it.second } - for ((converter, _) in sorted) { - if (!converter.accepts(bytes, info)) continue - return try { - converter.parse(bytes, info) - } catch (e: MikroMarkdownException) { - throw e - } catch (e: Exception) { - throw FileConversionException( - "Conversion failed with ${converter::class.simpleName}: ${e.message}", - e, + val converter = + converters.sortedBy { it.second }.firstOrNull { (candidate, _) -> candidate.accepts(bytes, info) }?.first + ?: throw UnsupportedFormatException( + "No converter found for: ${info.extension ?: info.mimetype ?: "unknown"}" ) - } - } - throw UnsupportedFormatException( - "No converter found for: ${info.extension ?: info.mimetype ?: "unknown"}", - ) + + return converter.parseOrFail(bytes, info) } + /** Converter failures surface as [FileConversionException]; our own exceptions pass through. */ + private fun DocumentConverter.parseOrFail(bytes: ByteArray, info: StreamInfo): Document = + try { + parse(bytes, info) + } catch (e: MikroMarkdownException) { + throw e + } catch (e: Exception) { + throw FileConversionException("Conversion failed with ${this::class.simpleName}: ${e.message}", e) + } + fun render(document: Document): ConversionResult = ConversionResult( markdown = renderer.render(document), diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt index cc095d5..9649640 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt @@ -7,8 +7,7 @@ import io.github.lemcoder.mikromarkdown.model.RawBlock class PlainTextConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { - return info.extension in setOf("txt", "log", "text") || - info.mimetype == "text/plain" + return info.extension in setOf("txt", "log", "text") || info.mimetype == "text/plain" } override fun parse(bytes: ByteArray, info: StreamInfo): Document = diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt index 6b6c205..2f33e52 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt @@ -3,9 +3,8 @@ package io.github.lemcoder.mikromarkdown.model /** * Format-independent document model. * - * Every converter parses its input into a [Document]; a single renderer turns - * documents into Markdown. Output quirks are therefore fixed once, in the - * renderer, rather than per format. + * Every converter parses its input into a [Document]; a single renderer turns documents into Markdown. Output quirks + * are therefore fixed once, in the renderer, rather than per format. */ data class Document( val blocks: List = emptyList(), @@ -70,7 +69,12 @@ data class TableCell( constructor(text: String) : this(if (text.isEmpty()) emptyList() else listOf(Text(text))) } -enum class Alignment { NONE, LEFT, CENTER, RIGHT } +enum class Alignment { + NONE, + LEFT, + CENTER, + RIGHT, +} data object ThematicBreak : Block diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt index a5fd5fa..4a57e40 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt @@ -84,16 +84,17 @@ class DocumentBuilder { header = header.map { TableCell(it) }, rows = rows.map { row -> row.map { TableCell(it) } }, alignments = alignments, - ), + ) ) } - fun build(): Document = Document( - blocks = blocks.toList(), - title = title, - metadata = metadata.toMap(), - assets = assets.toList(), - ) + fun build(): Document = + Document( + blocks = blocks.toList(), + title = title, + metadata = metadata.toMap(), + assets = assets.toList(), + ) } /** Builds a list of inlines without repeating `listOf(...)` wrappers in parsers. */ diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt index d457376..2285c9c 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt @@ -54,9 +54,7 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De fun render(document: Document): String { val body = renderBlocks(document.blocks) if (!options.frontMatter || document.metadata.isEmpty()) return body - val front = document.metadata.entries.joinToString("\n") { (k, v) -> - "$k: ${v.replace("\n", " ")}" - } + val front = document.metadata.entries.joinToString("\n") { (k, v) -> "$k: ${v.replace("\n", " ")}" } return "---\n$front\n---\n\n$body".trimEnd() } @@ -73,49 +71,51 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return chunks.joinToString("\n\n").trim() } - private fun renderBlock(block: Block): String = when (block) { - is Heading -> { - val text = inlines(block.content, TextContext.HEADING).collapseLines() - if (text.isBlank()) "" else "${"#".repeat(block.level.coerceIn(1, options.maxHeadingLevel))} $text" - } + private fun renderBlock(block: Block): String = + when (block) { + is Heading -> { + val text = inlines(block.content, TextContext.HEADING).collapseLines() + if (text.isBlank()) "" else "${"#".repeat(block.level.coerceIn(1, options.maxHeadingLevel))} $text" + } - is Paragraph -> inlines(block.content, TextContext.BLOCK).trimEnd() + is Paragraph -> inlines(block.content, TextContext.BLOCK).trimEnd() - is CodeBlock -> { - val fence = "`".repeat(maxOf(3, longestBacktickRun(block.code) + 1)) - "$fence${block.language.orEmpty()}\n${block.code.trimEnd('\n')}\n$fence" - } + is CodeBlock -> { + val fence = "`".repeat(maxOf(3, longestBacktickRun(block.code) + 1)) + "$fence${block.language.orEmpty()}\n${block.code.trimEnd('\n')}\n$fence" + } - is BlockQuote -> renderBlocks(block.blocks) - .lines() - .joinToString("\n") { if (it.isEmpty()) ">" else "> $it" } + is BlockQuote -> + renderBlocks(block.blocks).lines().joinToString("\n") { if (it.isEmpty()) ">" else "> $it" } - is ListBlock -> renderList(block, indent = "") + is ListBlock -> renderList(block, indent = "") - is Table -> renderTable(block) + is Table -> renderTable(block) - ThematicBreak -> "---" + ThematicBreak -> "---" - is HtmlComment -> "" + is HtmlComment -> "" - // Already-Markdown content: only whitespace is normalized, never syntax. - is RawBlock -> block.text - .replace("\r\n", "\n") - .lines() - .joinToString("\n") { it.trimEnd() } - .replace(Regex("\n{3,}"), "\n\n") - .trim() - } + // Already-Markdown content: only whitespace is normalized, never syntax. + is RawBlock -> + block.text + .replace("\r\n", "\n") + .lines() + .joinToString("\n") { it.trimEnd() } + .replace(Regex("\n{3,}"), "\n\n") + .trim() + } private fun renderList(list: ListBlock, indent: String): String { val lines = mutableListOf() list.items.forEachIndexed { index, item -> val marker = if (list.ordered) "${list.start + index}. " else "${options.bullet} " - val checkbox = when (item.checked) { - true -> "[x] " - false -> "[ ] " - null -> "" - } + val checkbox = + when (item.checked) { + true -> "[x] " + false -> "[ ] " + null -> "" + } val childIndent = indent + " ".repeat(marker.length) val body = renderItemBlocks(item, childIndent) val firstLine = body.firstOrNull().orEmpty() @@ -131,10 +131,11 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De val lines = mutableListOf() item.blocks.forEachIndexed { index, block -> if (index > 0) lines += "" - val rendered = when (block) { - is ListBlock -> renderList(block, childIndent) - else -> renderBlock(block).lines().joinToString("\n") { if (it.isEmpty()) it else childIndent + it } - } + val rendered = + when (block) { + is ListBlock -> renderList(block, childIndent) + else -> renderBlock(block).lines().joinToString("\n") { if (it.isEmpty()) it else childIndent + it } + } if (rendered.isEmpty()) return@forEachIndexed lines += rendered.lines() } @@ -153,17 +154,18 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De val rows = bodyRows.map { pad(it, columns) } val alignments = List(columns) { table.alignments.getOrElse(it) { Alignment.NONE } } - val widths = if (options.padTableColumns) { - List(columns) { col -> - maxOf( - 3, - header[col].length, - rows.maxOfOrNull { it[col].length } ?: 0, - ) + val widths = + if (options.padTableColumns) { + List(columns) { col -> + maxOf( + 3, + header[col].length, + rows.maxOfOrNull { it[col].length } ?: 0, + ) + } + } else { + null } - } else { - null - } val out = StringBuilder() out.append(row(header, widths)).append('\n') @@ -173,7 +175,8 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De if (index != rows.lastIndex) out.append('\n') } if (table.caption.isNotEmpty()) { - out.append("\n\n").append(options.emphasisMarker) + out.append("\n\n") + .append(options.emphasisMarker) .append(inlines(table.caption, TextContext.INLINE).collapseLines()) .append(options.emphasisMarker) } @@ -201,20 +204,22 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De if (cells.size >= columns) cells.take(columns) else cells + List(columns - cells.size) { "" } private fun row(cells: List, widths: List?): String = - cells.mapIndexed { index, cell -> - if (widths == null) cell else cell.padEnd(widths[index]) - }.joinToString(" | ", "| ", " |") + cells + .mapIndexed { index, cell -> if (widths == null) cell else cell.padEnd(widths[index]) } + .joinToString(" | ", "| ", " |") private fun delimiterRow(alignments: List, widths: List?): String = - alignments.mapIndexed { index, alignment -> - val width = widths?.get(index) ?: 3 - when (alignment) { - Alignment.NONE -> "-".repeat(width) - Alignment.LEFT -> ":" + "-".repeat(width - 1) - Alignment.RIGHT -> "-".repeat(width - 1) + ":" - Alignment.CENTER -> ":" + "-".repeat(width - 2) + ":" + alignments + .mapIndexed { index, alignment -> + val width = widths?.get(index) ?: 3 + when (alignment) { + Alignment.NONE -> "-".repeat(width) + Alignment.LEFT -> ":" + "-".repeat(width - 1) + Alignment.RIGHT -> "-".repeat(width - 1) + ":" + Alignment.CENTER -> ":" + "-".repeat(width - 2) + ":" + } } - }.joinToString(" | ", "| ", " |") + .joinToString(" | ", "| ", " |") private fun inlines(inlines: List, context: TextContext): String { val sb = StringBuilder() @@ -283,46 +288,54 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return if (context == TextContext.TABLE) text.replace("|", "\\|") else text } return buildString(text.length) { - var lineStart = atLineStart - for ((index, ch) in text.withIndex()) { - val prev = text.getOrNull(index - 1) - val next = text.getOrNull(index + 1) - when { - ch == '\n' -> { - append(ch) - lineStart = true - continue - } - - ch == '\\' -> append("\\\\") - ch == '|' && context == TextContext.TABLE -> append("\\|") - ch == '*' -> append("\\*") - ch == '`' -> append("\\`") - ch == '[' || ch == ']' -> append('\\').append(ch) - // Intraword underscores (snake_case) are not emphasis in CommonMark. - ch == '_' && !(prev?.isLetterOrDigit() == true && next?.isLetterOrDigit() == true) -> - append("\\_") - - ch == '<' && next?.let { it.isLetter() || it == '/' || it == '!' } == true -> append("\\<") - ch == '&' && looksLikeEntity(text, index) -> append("\\&") - - lineStart && (ch == '#' || ch == '>' || ch == '=') -> append('\\').append(ch) - lineStart && (ch == '-' || ch == '+') && next?.isWhitespace() != false -> append('\\').append(ch) - lineStart && ch.isDigit() && startsOrderedList(text, index) -> { - append(ch) - // Escape the delimiter instead of the digits: "1\. item". - } - - lineStart && (ch == '.' || ch == ')') && prev?.isDigit() == true && - startsOrderedList(text, index - 1) -> append('\\').append(ch) - - else -> append(ch) - } - if (!ch.isWhitespace()) lineStart = false + for (index in text.indices) { + if (needsEscape(text, index, context, atLineStart)) append('\\') + append(text[index]) } } } + private fun needsEscape(text: String, index: Int, context: TextContext, atLineStart: Boolean): Boolean = + when (val ch = text[index]) { + '\\', + '*', + '`', + '[', + ']' -> true + '|' -> context == TextContext.TABLE + // Intraword underscores (snake_case) are not emphasis in CommonMark. + '_' -> !isIntraword(text, index) + '<' -> text.getOrNull(index + 1)?.let { it.isLetter() || it == '/' || it == '!' } == true + '&' -> looksLikeEntity(text, index) + // The rest only introduce block syntax at the start of a line. + '#', + '>', + '=' -> startsLine(text, index, atLineStart) + '-', + '+' -> startsLine(text, index, atLineStart) && text.getOrNull(index + 1)?.isWhitespace() != false + // "1. item" in running text would become a list; escape the delimiter, not the digits. + '.', + ')' -> followsOrderedListMarker(text, index, atLineStart) + else -> false + } + + private fun isIntraword(text: String, index: Int): Boolean = + text.getOrNull(index - 1)?.isLetterOrDigit() == true && text.getOrNull(index + 1)?.isLetterOrDigit() == true + + /** True when only indentation separates [index] from the start of its line. */ + private fun startsLine(text: String, index: Int, atLineStart: Boolean): Boolean { + var i = index - 1 + while (i >= 0 && (text[i] == ' ' || text[i] == '\t')) i-- + return if (i < 0) atLineStart else text[i] == '\n' + } + + private fun followsOrderedListMarker(text: String, index: Int, atLineStart: Boolean): Boolean { + var digits = index - 1 + while (digits >= 0 && text[digits].isDigit()) digits-- + if (digits == index - 1) return false + return startsLine(text, digits + 1, atLineStart) && startsOrderedList(text, index - 1) + } + private fun startsOrderedList(text: String, digitIndex: Int): Boolean { var start = digitIndex while (start > 0 && text[start - 1].isDigit()) start-- @@ -341,8 +354,7 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return text.substring(index + 1, semicolon).all { it.isLetterOrDigit() || it == '#' } } - private fun encodeUrl(url: String): String = - url.replace(" ", "%20").replace("(", "%28").replace(")", "%29") + private fun encodeUrl(url: String): String = url.replace(" ", "%20").replace("(", "%28").replace(")", "%29") private fun longestBacktickRun(text: String): Int { var longest = 0 @@ -358,10 +370,14 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return longest } - private fun String.collapseLines(): String = - replace("\r\n", "\n").lines().joinToString(" ") { it.trim() }.trim() + private fun String.collapseLines(): String = replace("\r\n", "\n").lines().joinToString(" ") { it.trim() }.trim() - private enum class TextContext { BLOCK, INLINE, HEADING, TABLE } + private enum class TextContext { + BLOCK, + INLINE, + HEADING, + TABLE, + } companion object { val Default = MarkdownRenderer() diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt index 87810ed..bdbc49d 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt @@ -7,8 +7,8 @@ import io.github.lemcoder.mikromarkdown.model.Text /** * Turns extracted plain text (PDF pages, speaker notes, …) into paragraph blocks. * - * Blank lines separate paragraphs; soft-wrapped lines inside a paragraph are rejoined, - * so the Markdown does not inherit the source layout's line breaks. + * Blank lines separate paragraphs; soft-wrapped lines inside a paragraph are rejoined, so the Markdown does not inherit + * the source layout's line breaks. */ fun plainTextBlocks(text: String, reflow: Boolean = true): List { // Form feeds mark PDF page breaks; treat them as paragraph boundaries. @@ -32,21 +32,18 @@ fun plainTextBlocks(text: String, reflow: Boolean = true): List { return paragraphs.map { Paragraph(listOf(Text(it))) } } -private fun String.endsWithWordBreak(): Boolean = - endsWith("-") && length > 1 && this[length - 2].isLetter() +private fun String.endsWithWordBreak(): Boolean = endsWith("-") && length > 1 && this[length - 2].isLetter() private val WORD = Regex("[\\p{L}]{2,}") /** Words the document uses on their own; the de-hyphenation heuristic consults this. */ -private fun wordsIn(text: String): Set = - WORD.findAll(text).map { it.value.lowercase() }.toSet() +private fun wordsIn(text: String): Set = WORD.findAll(text).map { it.value.lowercase() }.toSet() /** * Rejoins soft-wrapped lines. * - * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are - * words the document uses elsewhere on their own (`conversation-` + `centric`), the hyphen is a - * real compound and stays. + * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are words the document uses + * elsewhere on their own (`conversation-` + `centric`), the hyphen is a real compound and stays. */ private fun joinWrappedLines(lines: List, vocabulary: Set): String { val sb = StringBuilder() diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt index 0a850a4..076e5aa 100644 --- a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt @@ -28,75 +28,84 @@ class FileIntegrationTest { } @Test - fun testDocx() = assertConversion( - filename = "test.docx", - mustInclude = listOf( - "314b0a30-5b04-470b-b9f7-eed2c2bec74a", - "49e168b7-d2ae-407f-a055-2167576f39a1", - "# Abstract", - "# Introduction", - "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", - ), - ) + fun testDocx() = + assertConversion( + filename = "test.docx", + mustInclude = + listOf( + "314b0a30-5b04-470b-b9f7-eed2c2bec74a", + "49e168b7-d2ae-407f-a055-2167576f39a1", + "# Abstract", + "# Introduction", + "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", + ), + ) @Test - fun testXlsx() = assertConversion( - filename = "test.xlsx", - mustInclude = listOf( - "09060124-b5e7-4717-9d07-3c046eb", - "6ff4173b-42a5-4784-9b19-f49caff4d93d", - "affc7dad-52dc-4b98-9b5d-51e65d8a8ad0", - ), - ) + fun testXlsx() = + assertConversion( + filename = "test.xlsx", + mustInclude = + listOf( + "09060124-b5e7-4717-9d07-3c046eb", + "6ff4173b-42a5-4784-9b19-f49caff4d93d", + "affc7dad-52dc-4b98-9b5d-51e65d8a8ad0", + ), + ) @Test - fun testPptx() = assertConversion( - filename = "test.pptx", - mustInclude = listOf( - "2cdda5c8-e50e-4db4-b5f0-9722a649f455", - "04191ea8-5c73-4215-a1d3-1cfb43aaaf12", - "44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a", - "1b92870d-e3b5-4e65-8153-919f4ff45592", - "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", - ), - ) + fun testPptx() = + assertConversion( + filename = "test.pptx", + mustInclude = + listOf( + "2cdda5c8-e50e-4db4-b5f0-9722a649f455", + "04191ea8-5c73-4215-a1d3-1cfb43aaaf12", + "44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a", + "1b92870d-e3b5-4e65-8153-919f4ff45592", + "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", + ), + ) @Test - fun testBlogHtml() = assertConversion( - filename = "test_blog.html", - mustInclude = listOf( - "Large language models (LLMs) are powerful tools that can generate natural language texts for various applications", - "an example where high cost can easily prevent a generic complex", - ), - ) + fun testBlogHtml() = + assertConversion( + filename = "test_blog.html", + mustInclude = + listOf( + "Large language models (LLMs) are powerful tools that can generate natural language texts", + "an example where high cost can easily prevent a generic complex", + ), + ) @Test - fun testWikipediaHtml() = assertConversion( - filename = "test_wikipedia.html", - mustInclude = listOf( - "Microsoft entered the operating system (OS) business in 1980", - ), - mustNotInclude = listOf( - "move to sidebar", - ), - ) + fun testWikipediaHtml() = + assertConversion( + filename = "test_wikipedia.html", + mustInclude = listOf("Microsoft entered the operating system (OS) business in 1980"), + mustNotInclude = listOf("move to sidebar"), + ) @Test - fun testJson() = assertConversion( - filename = "test.json", - mustInclude = listOf( - "5b64c88c-b3c3-4510-bcb8-da0b200602d8", - "9700dc99-6685-40b4-9a3a-5e406dcb37f3", - ), - ) + fun testJson() = + assertConversion( + filename = "test.json", + mustInclude = + listOf( + "5b64c88c-b3c3-4510-bcb8-da0b200602d8", + "9700dc99-6685-40b4-9a3a-5e406dcb37f3", + ), + ) @Test - fun testEpub() = assertConversion( - filename = "test.epub", - mustInclude = listOf( - "A test EPUB document for MarkItDown testing", - "Chapter 1", - "Chapter 2", - ), - ) + fun testEpub() = + assertConversion( + filename = "test.epub", + mustInclude = + listOf( + "A test EPUB document for MarkItDown testing", + "Chapter 1", + "Chapter 2", + ), + ) } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 4827bb5..d08cb64 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -1,6 +1,5 @@ package io.github.lemcoder.mikromarkdown -import io.github.lemcoder.mikromarkdown.utils.TikaMimeDetector import io.github.lemcoder.mikromarkdown.converters.CsvConverter import io.github.lemcoder.mikromarkdown.converters.DocxConverter import io.github.lemcoder.mikromarkdown.converters.EpubConverter @@ -12,21 +11,23 @@ import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter import io.github.lemcoder.mikromarkdown.converters.PptxConverter import io.github.lemcoder.mikromarkdown.converters.XlsxConverter import io.github.lemcoder.mikromarkdown.converters.XmlConverter +import io.github.lemcoder.mikromarkdown.utils.TikaMimeDetector import java.io.File /** A [MikroMarkdown] with every JVM converter registered and Tika-based format detection. */ -fun MikroMarkdown(): MikroMarkdown = MikroMarkdown(TikaMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(DocxConverter()) - register(XlsxConverter()) - register(PptxConverter()) - register(EpubConverter()) - register(PdfConverter()) - register(PlainTextConverter(), priority = 10.0) -} +fun MikroMarkdown(): MikroMarkdown = + MikroMarkdown(TikaMimeDetector).apply { + register(MarkdownPassthroughConverter()) + register(HtmlConverter()) + register(CsvConverter()) + register(JsonConverter()) + register(XmlConverter()) + register(DocxConverter()) + register(XlsxConverter()) + register(PptxConverter()) + register(EpubConverter()) + register(PdfConverter()) + register(PlainTextConverter(), priority = 10.0) + } fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt index b93bda6..9f11ac2 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt @@ -5,9 +5,9 @@ import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.Document import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell +import java.io.InputStreamReader import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser -import java.io.InputStreamReader class CsvConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -22,10 +22,11 @@ class CsvConverter : DocumentConverter { val header = records[0].toList() if (header.isEmpty()) return Document() - val rows = records.drop(1).map { record -> - // Ragged rows are padded by the renderer; only extra columns need trimming here. - List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } - } + val rows = + records.drop(1).map { record -> + // Ragged rows are padded by the renderer; only extra columns need trimming here. + List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } + } return Document(blocks = listOf(Table(header = header.map { TableCell(it) }, rows = rows))) } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt index b1bc8e6..79a154a 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt @@ -16,15 +16,15 @@ import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.model.plainText import io.github.lemcoder.mikromarkdown.model.styled +import java.util.Base64 import org.apache.poi.xwpf.usermodel.XWPFDocument import org.apache.poi.xwpf.usermodel.XWPFParagraph import org.apache.poi.xwpf.usermodel.XWPFTable -import java.util.Base64 class DocxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "docx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -94,11 +94,12 @@ class DocxConverter : DocumentConverter { val mime = data.pictureTypeEnum.contentType val id = data.fileName ?: "image-${assets.size + 1}" assets += Asset(id = id, mediaType = mime, bytes = data.data, name = data.fileName) - out += Image( - alt = alt, - url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", - assetId = id, - ) + out += + Image( + alt = alt, + url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", + assetId = id, + ) } continue } @@ -126,11 +127,12 @@ class DocxConverter : DocumentConverter { itemLevel < level -> break itemLevel == level -> { index++ - val children = if (index < items.size && items[index].first > level) { - listOf(ListBlock(ordered = false, items = build(items[index].first))) - } else { - emptyList() - } + val children = + if (index < items.size && items[index].first > level) { + listOf(ListBlock(ordered = false, items = build(items[index].first))) + } else { + emptyList() + } result += ListItem(listOf(Paragraph(content)) + children) } // A deeper first item without a parent: promote it to this level. diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt index 9c80056..2542a64 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt @@ -8,20 +8,21 @@ import io.github.lemcoder.mikromarkdown.model.Paragraph import io.github.lemcoder.mikromarkdown.model.Strong import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument -import org.w3c.dom.Element -import org.xml.sax.InputSource import java.io.StringReader import java.util.zip.ZipInputStream import javax.xml.parsers.DocumentBuilderFactory +import org.w3c.dom.Element +import org.xml.sax.InputSource class EpubConverter : DocumentConverter { - private val metaFields = listOf( - "title" to "Title", - "creator" to "Authors", - "language" to "Language", - "description" to "Description", - "identifier" to "Identifier", - ) + private val metaFields = + listOf( + "title" to "Title", + "creator" to "Authors", + "language" to "Language", + "description" to "Description", + "identifier" to "Identifier", + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "epub" || info.mimetype == "application/epub+zip" @@ -100,8 +101,7 @@ class EpubConverter : DocumentConverter { val id = item.getAttribute("id") val href = item.getAttribute("href") val mediaType = item.getAttribute("media-type") - if (id.isNotEmpty() && href.isNotEmpty() && - (mediaType.contains("html") || mediaType.contains("xhtml"))) { + if (id.isNotEmpty() && href.isNotEmpty() && isReadableChapter(mediaType)) { manifest[id] = href } } @@ -117,12 +117,16 @@ class EpubConverter : DocumentConverter { return Triple(manifest, spine, metadata) } - private fun parseXml(bytes: ByteArray) = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = false - factory.isExpandEntityReferences = false - factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) - } catch (_: Exception) { - null - } + /** Only the spine's (X)HTML documents carry text; images and styles are skipped. */ + private fun isReadableChapter(mediaType: String): Boolean = mediaType.contains("html") + + private fun parseXml(bytes: ByteArray) = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = false + factory.isExpandEntityReferences = false + factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) + } catch (_: Exception) { + null + } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt index fb5dffd..b716b5d 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt @@ -7,8 +7,7 @@ import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument class HtmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { - return info.extension in setOf("html", "htm") || - info.mimetype in setOf("text/html", "application/xhtml+xml") + return info.extension in setOf("html", "htm") || info.mimetype in setOf("text/html", "application/xhtml+xml") } override fun parse(bytes: ByteArray, info: StreamInfo): Document = diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt index 026c9d1..9b4dbf4 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt @@ -11,16 +11,21 @@ import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document class JsonConverter : DocumentConverter { - private val writer = ObjectMapper().apply { - registerKotlinModule() - }.writer(object : DefaultPrettyPrinter() { - init { - indentArraysWith(DefaultIndenter(" ", "\n")) - indentObjectsWith(DefaultIndenter(" ", "\n")) - } - override fun createInstance() = this - override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") - }) + private val writer = + ObjectMapper() + .apply { registerKotlinModule() } + .writer( + object : DefaultPrettyPrinter() { + init { + indentArraysWith(DefaultIndenter(" ", "\n")) + indentObjectsWith(DefaultIndenter(" ", "\n")) + } + + override fun createInstance() = this + + override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") + } + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "json" || info.mimetype in setOf("application/json", "text/json") @@ -28,11 +33,12 @@ class JsonConverter : DocumentConverter { override fun parse(bytes: ByteArray, info: StreamInfo): Document { val json = bytes.toString(Charsets.UTF_8) - val pretty = try { - writer.writeValueAsString(ObjectMapper().readTree(json)) - } catch (_: Exception) { - json - } + val pretty = + try { + writer.writeValueAsString(ObjectMapper().readTree(json)) + } catch (_: Exception) { + json + } return Document(blocks = listOf(CodeBlock(pretty, "json"))) } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt index 816bcf4..94543a8 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt @@ -18,11 +18,12 @@ class PdfConverter : DocumentConverter { val title = doc.documentInformation?.title?.trim()?.ifBlank { null } // Paragraph markers let the text blocks split on real paragraph breaks // instead of collapsing a page into one block. - val stripper = PDFTextStripper().apply { - sortByPosition = true - setAddMoreFormatting(true) - paragraphStart = "\n" - } + val stripper = + PDFTextStripper().apply { + sortByPosition = true + setAddMoreFormatting(true) + paragraphStart = "\n" + } return Document(blocks = plainTextBlocks(stripper.getText(doc)), title = title) } finally { doc.close() diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt index 4c363e5..78578a7 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt @@ -23,15 +23,20 @@ import org.apache.poi.xslf.usermodel.XSLFPictureShape import org.apache.poi.xslf.usermodel.XSLFSimpleShape import org.apache.poi.xslf.usermodel.XSLFTable import org.apache.poi.xslf.usermodel.XSLFTextShape +import org.openxmlformats.schemas.drawingml.x2006.chart.CTAreaSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTAxDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTBarSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTLineSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTNumDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTPieSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTScatterSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTSerTx import org.openxmlformats.schemas.presentationml.x2006.main.CTPicture class PptxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "pptx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" + info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -44,12 +49,14 @@ class PptxConverter : DocumentConverter { blocks += HtmlComment("Slide number: ${index + 1}") blocks += shapeBlocks(slide.shapes) { if (index == 0 && title == null) title = it } - val notes = slide.notes?.shapes - ?.filterIsInstance() - ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } - ?.joinToString("\n") { it.text } - ?.trim() - .orEmpty() + val notes = + slide.notes + ?.shapes + ?.filterIsInstance() + ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } + ?.joinToString("\n") { it.text } + ?.trim() + .orEmpty() if (notes.isNotBlank()) { blocks += Heading(3, listOf(Text("Notes:"))) blocks += notes.lines().filter { it.isNotBlank() }.map { Paragraph(listOf(Text(it.trim()))) } @@ -118,49 +125,83 @@ class PptxConverter : DocumentConverter { val blocks = mutableListOf() blocks += Heading(3, listOf(Text(listOfNotNull("Chart", chartTitle(chart)).joinToString(": ")))) - val series = try { - seriesOf(chart) - } catch (_: Exception) { - blocks += Paragraph(listOf(Text("[unsupported chart]"))) - return blocks - } + val series = + try { + seriesOf(chart) + } catch (_: Exception) { + blocks += Paragraph(listOf(Text("[unsupported chart]"))) + return blocks + } if (series.isEmpty()) return blocks val rowCount = series.maxOf { it.categories.size } - blocks += Table( - header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, - rows = (0 until rowCount).map { row -> - val category = series.first().categories.getOrElse(row) { "" } - (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } - }, - ) + blocks += + Table( + header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, + rows = + (0 until rowCount).map { row -> + val category = series.first().categories.getOrElse(row) { "" } + (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } + }, + ) return blocks } - private fun chartTitle(chart: XSLFChart): String? = try { - val ctChart = chart.ctChart - if (!ctChart.isSetTitle) { + private fun chartTitle(chart: XSLFChart): String? = + try { + val ctChart = chart.ctChart + if (!ctChart.isSetTitle) { + null + } else { + val tx = ctChart.title?.tx + when { + tx?.isSetRich == true -> + tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") + tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v + else -> null + }?.ifBlank { null } + } + } catch (_: Exception) { null - } else { - val tx = ctChart.title?.tx - when { - tx?.isSetRich == true -> - tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") - tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v - else -> null - }?.ifBlank { null } } - } catch (_: Exception) { - null - } private data class Series(val name: String, val categories: List, val values: List) private fun seriesOf(chart: XSLFChart): List { - val plotArea = chart.ctChart.plotArea - val out = mutableListOf() + val plot = chart.ctChart.plotArea + // bar/bar3D, line/line3D and area/area3D each share one generated series type. + val all = + plot.barChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.bar3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.lineChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.line3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.areaChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.area3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.scatterChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.pieChartList.flatMap { it.serList.map { s -> s.toSeries() } } + return all.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } + } + + private fun CTBarSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTLineSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTAreaSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTPieSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) - fun categoryValues(cat: CTAxDataSource?): List = when { + private fun CTScatterSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetXVal) xVal else null, if (isSetYVal) yVal else null) + + private fun series(tx: CTSerTx?, categories: CTAxDataSource?, values: CTNumDataSource?) = + Series(seriesName(tx), categoryValues(categories), numericValues(values)) + + private fun categoryValues(cat: CTAxDataSource?): List = + when { cat == null -> emptyList() cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() @@ -169,64 +210,22 @@ class PptxConverter : DocumentConverter { else -> emptyList() } - fun numericValues(v: CTNumDataSource?): List = when { + private fun numericValues(v: CTNumDataSource?): List = + when { v == null -> emptyList() v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() else -> emptyList() } - fun seriesName(tx: CTSerTx?): String = when { + private fun seriesName(tx: CTSerTx?): String = + when { tx == null -> "" tx.isSetV -> tx.v tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v.orEmpty() else -> "" } - for (c in plotArea.barChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.bar3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.lineChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.line3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.areaChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.area3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.scatterChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetXVal) s.xVal else null), - numericValues(if (s.isSetYVal) s.yVal else null), - ) - for (c in plotArea.pieChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - - return out.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } - } - private fun table(table: XSLFTable): Table? { val rows = table.rows if (rows.isEmpty()) return null diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt index 984dec0..e601f5c 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt @@ -8,18 +8,18 @@ import io.github.lemcoder.mikromarkdown.model.Heading import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text +import kotlin.math.floor import org.apache.poi.ss.usermodel.Cell import org.apache.poi.ss.usermodel.CellType import org.apache.poi.ss.usermodel.DataFormatter import org.apache.poi.xssf.usermodel.XSSFWorkbook -import kotlin.math.floor class XlsxConverter : DocumentConverter { private val formatter = DataFormatter() override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "xlsx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -35,12 +35,12 @@ class XlsxConverter : DocumentConverter { if (columns == 0) continue blocks += Heading(2, listOf(Text(sheet.sheetName))) - blocks += Table( - header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, - rows = rows.drop(1).map { row -> - (0 until columns).map { TableCell(cellValue(row.getCell(it))) } - }, - ) + blocks += + Table( + header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, + rows = + rows.drop(1).map { row -> (0 until columns).map { TableCell(cellValue(row.getCell(it))) } }, + ) } return Document(blocks = blocks) @@ -54,8 +54,7 @@ class XlsxConverter : DocumentConverter { return when (cell.cellType) { CellType.NUMERIC -> { val v = cell.numericCellValue - if (v == floor(v) && !v.isInfinite()) v.toLong().toString() - else formatter.formatCellValue(cell) + if (v == floor(v) && !v.isInfinite()) v.toLong().toString() else formatter.formatCellValue(cell) } CellType.BLANK -> "" else -> formatter.formatCellValue(cell).trim() diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt index 8979d57..7d3e6ac 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt @@ -4,7 +4,6 @@ import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document -import org.xml.sax.InputSource import java.io.StringReader import java.io.StringWriter import javax.xml.parsers.DocumentBuilderFactory @@ -12,6 +11,7 @@ import javax.xml.transform.OutputKeys import javax.xml.transform.TransformerFactory import javax.xml.transform.dom.DOMSource import javax.xml.transform.stream.StreamResult +import org.xml.sax.InputSource class XmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -23,23 +23,24 @@ class XmlConverter : DocumentConverter { return Document(blocks = listOf(CodeBlock(pretty, "xml"))) } - private fun prettyPrint(xml: String): String = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = true - factory.isIgnoringElementContentWhitespace = true - val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) + private fun prettyPrint(xml: String): String = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = true + factory.isIgnoringElementContentWhitespace = true + val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) - val tf = TransformerFactory.newInstance() - tf.setAttribute("indent-number", 2) - val transformer = tf.newTransformer() - transformer.setOutputProperty(OutputKeys.INDENT, "yes") - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") + val tf = TransformerFactory.newInstance() + tf.setAttribute("indent-number", 2) + val transformer = tf.newTransformer() + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") - val writer = StringWriter() - transformer.transform(DOMSource(doc), StreamResult(writer)) - writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") - } catch (_: Exception) { - xml - } + val writer = StringWriter() + transformer.transform(DOMSource(doc), StreamResult(writer)) + writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") + } catch (_: Exception) { + xml + } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt index d7c9296..510e036 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt @@ -29,22 +29,49 @@ import org.jsoup.nodes.TextNode /** * Walks an HTML DOM into the shared document model. * - * Replaces the previous HTML → Markdown string conversion: tables, lists and inline - * emphasis become model nodes, so the Markdown renderer owns all syntax decisions. + * Replaces the previous HTML → Markdown string conversion: tables, lists and inline emphasis become model nodes, so the + * Markdown renderer owns all syntax decisions. */ object HtmlToDocument { - private val DROPPED_TAGS = setOf( - "script", "style", "noscript", "template", "button", "svg", "iframe", "form", "input", "select", - ) + private val DROPPED_TAGS = + setOf( + "script", + "style", + "noscript", + "template", + "button", + "svg", + "iframe", + "form", + "input", + "select", + ) private val HEADINGS = mapOf("h1" to 1, "h2" to 2, "h3" to 3, "h4" to 4, "h5" to 5, "h6" to 6) /** Tags that only group other content; their children are lifted into the parent block flow. */ - private val CONTAINERS = setOf( - "div", "section", "article", "main", "header", "footer", "aside", "nav", "body", "html", - "figure", "details", "summary", "fieldset", "center", "hgroup", "picture", "colgroup", - ) + private val CONTAINERS = + setOf( + "div", + "section", + "article", + "main", + "header", + "footer", + "aside", + "nav", + "body", + "html", + "figure", + "details", + "summary", + "fieldset", + "center", + "hgroup", + "picture", + "colgroup", + ) fun parse(html: String, baseUri: String = ""): Document { val doc = Jsoup.parse(html, baseUri) @@ -91,49 +118,66 @@ object HtmlToDocument { return out } - private fun Element.isBlockLevel(): Boolean = when (tagName()) { - in HEADINGS, "p", "ul", "ol", "table", "pre", "blockquote", "hr", "dl", "li", "figcaption" -> true - in CONTAINERS -> true - else -> false - } - - private fun blockFor(element: Element): List = when (val tag = element.tagName()) { - in HEADINGS -> listOfNotNull( - inlines(element).trimEdges() - .takeIf { it.isNotEmpty() } - ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) }, - ) + private fun Element.isBlockLevel(): Boolean = + when (tagName()) { + in HEADINGS, + "p", + "ul", + "ol", + "table", + "pre", + "blockquote", + "hr", + "dl", + "li", + "figcaption" -> true + in CONTAINERS -> true + else -> false + } - "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) - "ul", "ol" -> listBlock(element) - "table" -> table(element) - "pre" -> listOf(codeBlock(element)) - "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() - "hr" -> listOf(ThematicBreak) - "dl" -> definitionList(element) - "figcaption" -> listOfNotNull( - inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) }, - ) - // Containers and stray
  • outside a list contribute their children directly. - else -> blocks(element) - } + private fun blockFor(element: Element): List = + when (val tag = element.tagName()) { + in HEADINGS -> + listOfNotNull( + inlines(element) + .trimEdges() + .takeIf { it.isNotEmpty() } + ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) } + ) + + "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) + "ul", + "ol" -> listBlock(element) + "table" -> table(element) + "pre" -> listOf(codeBlock(element)) + "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() + "hr" -> listOf(ThematicBreak) + "dl" -> definitionList(element) + "figcaption" -> + listOfNotNull( + inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) } + ) + // Containers and stray
  • outside a list contribute their children directly. + else -> blocks(element) + } private fun listBlock(element: Element): List { val ordered = element.tagName() == "ol" val start = element.attr("start").toIntOrNull() ?: 1 - val items = element.children() - .filter { it.tagName() == "li" } - .map { li -> ListItem(blocks = blocks(li)) } - .filter { it.blocks.isNotEmpty() } + val items = + element + .children() + .filter { it.tagName() == "li" } + .map { li -> ListItem(blocks = blocks(li)) } + .filter { it.blocks.isNotEmpty() } if (items.isEmpty()) return emptyList() return listOf(ListBlock(ordered = ordered, items = items, start = start)) } private fun codeBlock(element: Element): Block { val code = element.selectFirst("code") ?: element - val language = code.classNames() - .firstOrNull { it.startsWith("language-") || it.startsWith("lang-") } - ?.substringAfter('-') + val language = + code.classNames().firstOrNull { it.startsWith("language-") || it.startsWith("lang-") }?.substringAfter('-') return CodeBlock(code.wholeText().trimEnd(), language) } @@ -158,20 +202,22 @@ object HtmlToDocument { val body = mutableListOf>() for ((index, tr) in rows.withIndex()) { - val cells = tr.children() - .filter { it.tagName() == "th" || it.tagName() == "td" } - .map { cell -> - TableCell( - content = inlines(cell).trimEdges(), - colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - ) - } + val cells = + tr.children() + .filter { it.tagName() == "th" || it.tagName() == "td" } + .map { cell -> + TableCell( + content = inlines(cell).trimEdges(), + colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + ) + } if (cells.isEmpty()) continue - val isHeaderRow = index == 0 && - tr.children().all { it.tagName() == "th" } && - (tr.parent()?.tagName() == "thead" || header.isEmpty()) + val isHeaderRow = + index == 0 && + tr.children().all { it.tagName() == "th" } && + (tr.parent()?.tagName() == "thead" || header.isEmpty()) if (isHeaderRow && header.isEmpty()) header = cells else body += cells } @@ -196,35 +242,45 @@ object HtmlToDocument { if (text.isNotEmpty()) add(Text(text)) } - is Element -> when (node.tagName()) { - "br" -> add(LineBreak) - "img" -> { - val src = node.attr("abs:src").ifBlank { node.attr("src") } - val alt = node.attr("alt") - if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) - } + is Element -> + when (node.tagName()) { + "br" -> add(LineBreak) + "img" -> { + val src = node.attr("abs:src").ifBlank { node.attr("src") } + val alt = node.attr("alt") + if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) + } - "a" -> { - val href = node.attr("abs:href").ifBlank { node.attr("href") } - val content = inlines(node).trimEdges() - when { - content.isEmpty() -> Unit - href.isBlank() || href.startsWith("javascript:") -> addAll(content) - else -> add(Link(content, href, node.attr("title").ifBlank { null })) + "a" -> { + val href = node.attr("abs:href").ifBlank { node.attr("href") } + val content = inlines(node).trimEdges() + when { + content.isEmpty() -> Unit + href.isBlank() || href.startsWith("javascript:") -> addAll(content) + else -> add(Link(content, href, node.attr("title").ifBlank { null })) + } } - } - "strong", "b" -> wrapped(node) { Strong(it) } - "em", "i", "cite", "var" -> wrapped(node) { Emphasis(it) } - "del", "s", "strike" -> wrapped(node) { Strikethrough(it) } - "code", "kbd", "samp", "tt" -> { - val code = node.wholeText().trim() - if (code.isNotEmpty()) add(CodeSpan(code)) - } + "strong", + "b" -> wrapped(node) { Strong(it) } + "em", + "i", + "cite", + "var" -> wrapped(node) { Emphasis(it) } + "del", + "s", + "strike" -> wrapped(node) { Strikethrough(it) } + "code", + "kbd", + "samp", + "tt" -> { + val code = node.wholeText().trim() + if (code.isNotEmpty()) add(CodeSpan(code)) + } - // Block-level content encountered inline (e.g. a
    inside a ): keep its text. - else -> addAll(inlines(node)) - } + // Block-level content encountered inline (e.g. a
    inside a ): keep its text. + else -> addAll(inlines(node)) + } } } @@ -239,14 +295,11 @@ object HtmlToDocument { /** HTML collapses runs of whitespace; do the same before the text reaches the model. */ // Non-breaking spaces are not collapsible whitespace in HTML, so they survive verbatim. - private fun TextNode.normalizedText(): String = - wholeText.replace(Regex("\\s+"), " ") + private fun TextNode.normalizedText(): String = wholeText.replace(Regex("\\s+"), " ") - private fun List.startsWithSpace(): Boolean = - (firstOrNull() as? Text)?.value?.startsWith(" ") == true + private fun List.startsWithSpace(): Boolean = (firstOrNull() as? Text)?.value?.startsWith(" ") == true - private fun List.endsWithSpace(): Boolean = - (lastOrNull() as? Text)?.value?.endsWith(" ") == true + private fun List.endsWithSpace(): Boolean = (lastOrNull() as? Text)?.value?.endsWith(" ") == true /** Drops leading/trailing whitespace-only text so emphasis markers hug their content. */ private fun List.trimEdges(): List { diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt index 40cdc01..f819614 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt @@ -2,15 +2,20 @@ package io.github.lemcoder.mikromarkdown.utils import io.github.lemcoder.mikromarkdown.MimeDetector import io.github.lemcoder.mikromarkdown.StreamInfo -import org.apache.tika.Tika import java.io.File +import org.apache.tika.Tika object TikaMimeDetector : MimeDetector { private val tika = Tika() override fun detect(path: String): StreamInfo { val file = File(path) - val mimetype = try { tika.detect(file) } catch (_: Exception) { null } + val mimetype = + try { + tika.detect(file) + } catch (_: Exception) { + null + } val extension = file.extension.lowercase().ifEmpty { null } return StreamInfo( mimetype = mimetype, diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt new file mode 100644 index 0000000..a604680 --- /dev/null +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt @@ -0,0 +1,63 @@ +package io.github.lemcoder.mikromarkdown + +import com.lemonappdev.konsist.api.Konsist +import com.lemonappdev.konsist.api.architecture.KoArchitectureCreator.assertArchitecture +import com.lemonappdev.konsist.api.architecture.Layer +import com.lemonappdev.konsist.api.verify.assertFalse +import com.lemonappdev.konsist.api.verify.assertTrue +import kotlin.test.Test + +/** + * Guards the pipeline's boundaries: converters parse into the model, the renderer serializes it, and neither reaches + * across. Without these, Markdown syntax leaks back into converters — the exact drift the document model was introduced + * to end. + */ +class ArchitectureTest { + + private val production = Konsist.scopeFromProject(sourceSetName = null).files.filterNot { it.path.contains("Test") } + + @Test + fun `layers depend in one direction only`() { + Konsist.scopeFromProject().assertArchitecture { + val model = Layer("Model", "io.github.lemcoder.mikromarkdown.model..") + val render = Layer("Render", "io.github.lemcoder.mikromarkdown.render..") + val converters = Layer("Converters", "io.github.lemcoder.mikromarkdown.converters..") + + // The model is the shared vocabulary: it may not know about its producers or consumers. + model.dependsOnNothing() + render.dependsOn(model) + converters.dependsOn(model) + } + } + + @Test + fun `converters do not reach into the renderer`() { + production + .filter { it.packagee?.name?.contains(".converters") == true } + .assertFalse { file -> file.hasImport { it.name.contains(".render.") } } + } + + @Test + fun `markdown syntax only lives in the renderer`() { + val markdownLiterals = Regex("""\"(\|[^"]*\||#{1,6} |\*\*|!\[|```)""") + + production + .filterNot { it.packagee?.name?.contains(".render") == true } + .assertFalse { file -> markdownLiterals.containsMatchIn(file.text) } + } + + @Test + fun `converters implement DocumentConverter and are named accordingly`() { + Konsist.scopeFromProject() + .classes() + .filter { it.resideInPackage("..converters..") && it.name.endsWith("Converter") } + .assertTrue { it.hasParentInterface { parent -> parent.name == "DocumentConverter" } } + } + + @Test + fun `the model stays free of platform dependencies`() { + production + .filter { it.packagee?.name?.contains(".model") == true } + .assertFalse { file -> file.hasImport { it.name.startsWith("java.") || it.name.startsWith("android.") } } + } +} diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt index 3f242fa..84c9627 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt @@ -1,14 +1,23 @@ package io.github.lemcoder.mikromarkdown -import org.junit.jupiter.api.Test import java.io.File +import org.junit.jupiter.api.Test class DumpOutputTest { private val mid = MikroMarkdown() @Test fun dumpAll() { - for (name in listOf("test.docx","test.xlsx","test.pptx","test.epub","test.json","test_blog.html","test_wikipedia.html")) { + for (name in + listOf( + "test.docx", + "test.xlsx", + "test.pptx", + "test.epub", + "test.json", + "test_blog.html", + "test_wikipedia.html", + )) { val url = javaClass.classLoader.getResource("test_files/$name") ?: continue val output = mid.convert(File(url.toURI()).absolutePath).markdown File("/tmp/kt_$name.md").writeText(output) diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt index 4edb5d8..7c1314e 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt @@ -1,3 +1,3 @@ package io.github.lemcoder.mikromarkdown -actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() \ No newline at end of file +actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt index 81f6041..b744e55 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt @@ -1,38 +1,32 @@ package io.github.lemcoder.mikromarkdown -import org.junit.jupiter.api.Assumptions.assumeTrue -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test import java.io.File import java.util.concurrent.TimeUnit import kotlin.test.assertTrue +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test class PythonComparisonTest { - @Test - fun testDocx() = compare("test.docx") + @Test fun testDocx() = compare("test.docx") - @Test - fun testXlsx() = compare("test.xlsx") + @Test fun testXlsx() = compare("test.xlsx") - @Test - fun testPptx() = compare("test.pptx") + @Test fun testPptx() = compare("test.pptx") - @Test - fun testEpub() = compare("test.epub") + @Test fun testEpub() = compare("test.epub") - @Test - fun testJson() = compare("test.json") + @Test fun testJson() = compare("test.json") - @Test - fun testBlogHtml() = compare("test_blog.html") + @Test fun testBlogHtml() = compare("test_blog.html") - @Test - fun testWikipediaHtml() = compare("test_wikipedia.html") + @Test fun testWikipediaHtml() = compare("test_wikipedia.html") private fun compare(filename: String) { - val url = javaClass.classLoader.getResource("test_files/$filename") - ?: error("Resource not found: test_files/$filename") + val url = + javaClass.classLoader.getResource("test_files/$filename") + ?: error("Resource not found: test_files/$filename") val file = File(url.toURI()) val pythonOutput = runMarkitdown(file.absolutePath) @@ -49,9 +43,9 @@ class PythonComparisonTest { // Normalize unicode characters to ASCII equivalents so both converters compare fairly private fun normalize(text: String): String = text - .replace("–", "-") // en dash - .replace("—", "-") // em dash - .replace(" ", " ") // non-breaking space + .replace("–", "-") // en dash + .replace("—", "-") // em dash + .replace(" ", " ") // non-breaking space .replace(Regex("%([0-9A-Fa-f]{2})")) { mr -> val cp = mr.groupValues[1].toInt(16) if (cp in 0x20..0x7E) cp.toChar().toString() else mr.value @@ -67,10 +61,7 @@ class PythonComparisonTest { // Tokens longer than 3 chars, lowercased, deduped private fun tokenize(text: String): Set = - normalize(text).lowercase() - .split(Regex("[\\s\\p{Punct}]+")) - .filter { it.length > 3 } - .toSet() + normalize(text).lowercase().split(Regex("[\\s\\p{Punct}]+")).filter { it.length > 3 }.toSet() private fun tokenSimilarity(reference: String, actual: String): Double { val refTokens = tokenize(reference) @@ -99,20 +90,22 @@ class PythonComparisonTest { } private fun resolveMarkitdownCmd(): List { - val candidates = listOf( - listOf("markitdown"), - listOf("python", "-m", "markitdown"), - listOf("python3", "-m", "markitdown"), - listOf("uvx", "markitdown[all]"), - listOf("uvx", "markitdown"), - ) + val candidates = + listOf( + listOf("markitdown"), + listOf("python", "-m", "markitdown"), + listOf("python3", "-m", "markitdown"), + listOf("uvx", "markitdown[all]"), + listOf("uvx", "markitdown"), + ) return candidates.firstOrNull { cmd -> try { - val process = ProcessBuilder(cmd + "--help") - .redirectErrorStream(true) - .start() + val process = ProcessBuilder(cmd + "--help").redirectErrorStream(true).start() val finished = process.waitFor(10, TimeUnit.SECONDS) - if (!finished) { process.destroyForcibly(); return@firstOrNull false } + if (!finished) { + process.destroyForcibly() + return@firstOrNull false + } process.exitValue() == 0 } catch (_: Exception) { false @@ -121,12 +114,14 @@ class PythonComparisonTest { } fun runMarkitdown(path: String): String { - val process = ProcessBuilder(markitdownCmd + path) - .start() + val process = ProcessBuilder(markitdownCmd + path).start() val stdout = process.inputStream.bufferedReader().readText() val finished = process.waitFor(30, TimeUnit.SECONDS) assumeTrue(finished, "markitdown timed out on $path") - assumeTrue(process.exitValue() == 0, "markitdown failed on $path: ${process.errorStream.bufferedReader().readText()}") + assumeTrue( + process.exitValue() == 0, + "markitdown failed on $path: ${process.errorStream.bufferedReader().readText()}", + ) return stdout } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 5fc871d..c3b33ad 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,5 +14,7 @@ dependencyResolutionManagement { } rootProject.name = "mikromarkdown" + include(":library") + include(":cli")