diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index fe4d926..2c10c1d 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -33,3 +33,18 @@ jobs: - name: Run tests uses: gradle/actions/setup-gradle@v4 - run: ./gradlew :library:${{ matrix.target }} + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Validate Gradle Wrapper + uses: gradle/actions/wrapper-validation@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + - uses: gradle/actions/setup-gradle@v4 + - name: Formatting and static analysis + run: ./gradlew ktfmtCheck detekt diff --git a/.gitmodules b/.gitmodules index e8b0843..1a7c4c3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "markitdown"] path = markitdown url = https://github.com/microsoft/markitdown +[submodule "anydoc"] + path = anydoc + url = https://github.com/firecrawl/anydoc.git diff --git a/README.md b/README.md index 403ed5a..74ed13d 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,9 @@ dependencies { ### JVM ```kotlin -import io.github.lemcoder.mikromarkdown.MarkItDown +import io.github.lemcoder.mikromarkdown.MikroMarkdown -val mid = MarkItDown() +val mid = MikroMarkdown() // from file path val result = mid.convert("/path/to/document.docx") @@ -78,10 +78,10 @@ println(result.title) // nullable, extracted from document metadata ### Android ```kotlin -import io.github.lemcoder.mikromarkdown.MarkItDown +import io.github.lemcoder.mikromarkdown.MikroMarkdown // pass Context to enable PDF support -val mid = MarkItDown(context) +val mid = MikroMarkdown(context) val result = mid.convert(file.absolutePath) ``` @@ -101,7 +101,7 @@ class MyConverter : DocumentConverter { } } -val mid = MarkItDown() +val mid = MikroMarkdown() mid.register(MyConverter()) // default priority 0.0 mid.register(FallbackConverter(), priority = 10.0) // higher = later ``` @@ -126,7 +126,22 @@ mid.register(HtmlConverter()) | `UnsupportedFormatException` | No registered converter accepted the input | | `FileConversionException` | Converter threw during conversion | -Both extend `MarkItDownException`. +Both extend `MikroMarkdownException`. + +## Code quality + +| tool | task | what it guards | +|---|---|---| +| [ktfmt](https://github.com/facebook/ktfmt) | `./gradlew ktfmtFormat` / `ktfmtCheck` | formatting (kotlinlang style, 120 columns) | +| [detekt](https://detekt.dev) | `./gradlew detekt` | static analysis; overrides in `config/detekt/detekt.yml` | +| [Konsist](https://docs.konsist.lemonappdev.com) | `./gradlew :library:jvmTest --tests '*ArchitectureTest*'` | pipeline boundaries | + +`./gradlew check` runs all three. The Konsist rules encode the architecture: the model depends on +nothing, converters never import the renderer, and Markdown syntax appears only under `render/` — +so a converter cannot quietly start building Markdown strings again. + +ktfmt-gradle only derives tasks for the common and JVM source sets, so `library/build.gradle.kts` +registers matching tasks for the Android ones. ## Benchmark diff --git a/anydoc b/anydoc new file mode 160000 index 0000000..4a45add --- /dev/null +++ b/anydoc @@ -0,0 +1 @@ +Subproject commit 4a45addbd607e8b59f0c263bca26aab228e10370 diff --git a/build.gradle.kts b/build.gradle.kts index 5662bb4..393cb69 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,4 +2,36 @@ plugins { alias(libs.plugins.android.kotlin.multiplatform.library) apply false alias(libs.plugins.kotlinMultiplatform) apply false alias(libs.plugins.kotlin.jvm) apply false + alias(libs.plugins.ktfmt) + alias(libs.plugins.detekt) +} + +allprojects { + apply(plugin = rootProject.libs.plugins.ktfmt.get().pluginId) + apply(plugin = rootProject.libs.plugins.detekt.get().pluginId) + + ktfmt { + // Matches the existing sources: 4-space indent, no import reordering surprises. + kotlinLangStyle() + maxWidth = 120 + } + + detekt { + buildUponDefaultConfig = true + parallel = true + config.setFrom(rootProject.file("config/detekt/detekt.yml")) + basePath = rootProject.projectDir.absolutePath + // Generated sources and the vendored reference checkouts are not ours to lint. + source.setFrom(files("src").filter { it.exists() }) + } + + tasks.withType().configureEach { + jvmTarget = "21" + reports { + html.required = true + sarif.required = true + md.required = false + txt.required = false + } + } } diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index ae93acb..9601598 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -3,21 +3,13 @@ plugins { application } -java { - toolchain { - languageVersion = JavaLanguageVersion.of(21) - } -} +java { toolchain { languageVersion = JavaLanguageVersion.of(21) } } -application { - mainClass = "com.mikromarkdown.cli.MainKt" -} +application { mainClass = "com.mikromarkdown.cli.MainKt" } dependencies { implementation(project(":library")) implementation(libs.clikt) } -tasks.test { - useJUnitPlatform() -} +tasks.test { useJUnitPlatform() } diff --git a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt index d7ca118..11974c1 100644 --- a/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt +++ b/cli/src/main/kotlin/com/mikromarkdown/cli/Main.kt @@ -1,37 +1,6 @@ package com.mikromarkdown.cli -import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.main -import com.github.ajalt.clikt.parameters.arguments.argument -import com.github.ajalt.clikt.parameters.arguments.optional -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.types.path -import io.github.lemcoder.mikromarkdown.MarkItDown -import io.github.lemcoder.mikromarkdown.StreamInfo - -class MarkItDownCommand : CliktCommand(name = "markitdown") { - private val file by argument("FILE", help = "Input file (reads stdin if omitted)").path(mustExist = true).optional() - private val output by option("-o", "--output", help = "Output file (default: stdout)").path() - private val extension by option("-x", "--extension", help = "File extension hint (e.g. html)") - private val mimeType by option("-m", "--mime-type", help = "MIME type hint (e.g. text/html)") - - override fun run() { - val markItDown = MarkItDown() - - val result = if (file != null) { - markItDown.convert(file!!.toFile().absolutePath) - } else { - val info = StreamInfo(extension = extension, mimetype = mimeType) - markItDown.convert(System.`in`.readBytes(), info) - } - - if (output != null) { - output!!.toFile().writeText(result.markdown) - } else { - print(result.markdown) - } - } -} fun main(args: Array) { // PDFBox pulls in the Log4j API; without a provider it writes a banner to stdout, @@ -39,5 +8,5 @@ fun main(args: Array) { System.setProperty("log4j2.loggerContextFactory", "org.apache.logging.log4j.simple.SimpleLoggerContextFactory") System.setProperty("log4j2.simplelogLevel", "OFF") System.setProperty("log4j2.statusLoggerLevel", "OFF") - MarkItDownCommand().main(args) + MikroMarkdownCommand().main(args) } diff --git a/cli/src/main/kotlin/com/mikromarkdown/cli/MikroMarkdownCommand.kt b/cli/src/main/kotlin/com/mikromarkdown/cli/MikroMarkdownCommand.kt new file mode 100644 index 0000000..03f6f5a --- /dev/null +++ b/cli/src/main/kotlin/com/mikromarkdown/cli/MikroMarkdownCommand.kt @@ -0,0 +1,34 @@ +package com.mikromarkdown.cli + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.arguments.optional +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.path +import io.github.lemcoder.mikromarkdown.MikroMarkdown +import io.github.lemcoder.mikromarkdown.StreamInfo + +class MikroMarkdownCommand : CliktCommand(name = "mikromarkdown") { + private val file by argument("FILE", help = "Input file (reads stdin if omitted)").path(mustExist = true).optional() + private val output by option("-o", "--output", help = "Output file (default: stdout)").path() + private val extension by option("-x", "--extension", help = "File extension hint (e.g. html)") + private val mimeType by option("-m", "--mime-type", help = "MIME type hint (e.g. text/html)") + + override fun run() { + val mikroMarkdown = MikroMarkdown() + + val result = + if (file != null) { + mikroMarkdown.convert(file!!.toFile().absolutePath) + } else { + val info = StreamInfo(extension = extension, mimetype = mimeType) + mikroMarkdown.convert(System.`in`.readBytes(), info) + } + + if (output != null) { + output!!.toFile().writeText(result.markdown) + } else { + print(result.markdown) + } + } +} diff --git a/config/detekt/detekt.yml b/config/detekt/detekt.yml new file mode 100644 index 0000000..b84ea8d --- /dev/null +++ b/config/detekt/detekt.yml @@ -0,0 +1,48 @@ +# Overrides on top of detekt's default config (buildUponDefaultConfig = true). +# Only rules that clash with deliberate choices in this codebase are relaxed. + +build: + maxIssues: 0 + +complexity: + # Parsers walk large format-specific type hierarchies; splitting them further hurts readability. + CyclomaticComplexMethod: + threshold: 25 + LongMethod: + threshold: 90 + LongParameterList: + functionThreshold: 8 + constructorThreshold: 10 + NestedBlockDepth: + threshold: 6 + TooManyFunctions: + thresholdInFiles: 30 + thresholdInClasses: 30 + thresholdInObjects: 30 + ignorePrivate: true + +exceptions: + # Format libraries throw broadly; converters degrade gracefully rather than propagate. + TooGenericExceptionCaught: + active: false + SwallowedException: + active: false + +naming: + FunctionNaming: + # Factory functions mirror the type they build: MikroMarkdown(), document {}. + functionPattern: '[a-zA-Z][a-zA-Z0-9]*' + +style: + MagicNumber: + active: false + MaxLineLength: + maxLineLength: 120 + ReturnCount: + max: 6 + ForbiddenComment: + active: false + UnusedPrivateMember: + active: true + LoopWithTooManyJumpStatements: + maxJumpCount: 4 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c6771a9..44f96f3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,6 +17,9 @@ poi = "5.5.1" tika = "3.3.0" clikt = "5.1.0" coreKtx = "1.7.0" +detekt = "1.23.8" +ktfmt = "0.27.0" +konsist = "0.17.3" [libraries] kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" } @@ -33,6 +36,7 @@ poi-ooxml = { module = "org.apache.poi:poi-ooxml", version.ref = "poi" } tika-core = { module = "org.apache.tika:tika-core", version.ref = "tika" } clikt = { module = "com.github.ajalt.clikt:clikt", version.ref = "clikt" } core-ktx = { group = "androidx.test", name = "core-ktx", version.ref = "coreKtx" } +konsist = { module = "com.lemonappdev:konsist", version.ref = "konsist" } [plugins] android-kotlin-multiplatform-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } @@ -40,3 +44,5 @@ kotlinMultiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version.ref = "vanniktechMavenPublish" } kotlinx-resources = { id = "com.goncalossilva.resources", version.ref = "kotlinx-resources" } +detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } +ktfmt = { id = "com.ncorti.ktfmt.gradle", version.ref = "ktfmt" } diff --git a/library/build.gradle.kts b/library/build.gradle.kts index b3a51cf..ad92d04 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -7,16 +7,13 @@ plugins { } group = "io.github.lemcoder" + version = "0.1.0" kotlin { jvm { - compilerOptions { - jvmTarget = JvmTarget.JVM_21 - } - testRuns["test"].executionTask.configure { - useJUnitPlatform() - } + compilerOptions { jvmTarget = JvmTarget.JVM_21 } + testRuns["test"].executionTask.configure { useJUnitPlatform() } } androidLibrary { @@ -25,19 +22,13 @@ kotlin { minSdk = libs.versions.android.minSdk.get().toInt() withHostTestBuilder {}.configure {} - withDeviceTestBuilder { - sourceSetTreeName = "test" - } + withDeviceTestBuilder { sourceSetTreeName = "test" } - compilerOptions { - jvmTarget = JvmTarget.JVM_11 - } + compilerOptions { jvmTarget = JvmTarget.JVM_11 } } sourceSets { - commonMain.dependencies { - implementation(libs.kotlinx.io.core) - } + commonMain.dependencies { implementation(libs.kotlinx.io.core) } jvmMain.dependencies { implementation(libs.jsoup) @@ -64,6 +55,31 @@ kotlin { jvmTest.dependencies { implementation(libs.junit.jupiter) implementation(libs.kotlin.test) + implementation(libs.konsist) } } } + +// ktfmt-gradle only derives tasks for the common and JVM source sets, so the Android +// ones — where half the converters live — would go unformatted and unchecked. +run { + val androidSources = fileTree("src") { include("android*/**/*.kt") } + val template = tasks.named("ktfmtFormatKmpCommonMain") + + val formatAndroid = + tasks.register("ktfmtFormatAndroidSourceSets") { + ktfmtClasspath.from(template.map { it.ktfmtClasspath }) + formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) + setSource(androidSources) + } + val checkAndroid = + tasks.register("ktfmtCheckAndroidSourceSets") { + ktfmtClasspath.from(template.map { it.ktfmtClasspath }) + formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) + setSource(androidSources) + } + + tasks.named("ktfmtFormat") { dependsOn(formatAndroid) } + tasks.named("ktfmtCheck") { dependsOn(checkAndroid) } + tasks.named("check") { dependsOn(checkAndroid) } +} diff --git a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt index 0fb6901..97d5560 100644 --- a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt +++ b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt @@ -1,5 +1,5 @@ package io.github.lemcoder.mikromarkdown -actual fun testMarkItDown(): MikroMarkdown { - return MarkItDown(context = null) -} \ No newline at end of file +actual fun testMikroMarkdown(): MikroMarkdown { + return MikroMarkdown(context = null) +} diff --git a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMarkItDown.kt b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMarkItDown.kt deleted file mode 100644 index b4e22bd..0000000 --- a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMarkItDown.kt +++ /dev/null @@ -1,3 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMarkItDown(): MikroMarkdown = MarkItDown(context = null) diff --git a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt new file mode 100644 index 0000000..1b9af9d --- /dev/null +++ b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt @@ -0,0 +1,3 @@ +package io.github.lemcoder.mikromarkdown + +actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown(context = null) diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt similarity index 57% rename from library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt rename to library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 77b91df..07b15c8 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -16,21 +16,27 @@ import io.github.lemcoder.mikromarkdown.converters.XmlConverter import io.github.lemcoder.mikromarkdown.utils.AndroidMimeDetector import java.io.File -fun MarkItDown(context: Context? = null): MikroMarkdown = MikroMarkdown(AndroidMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(DocxConverter()) - register(XlsxConverter()) - register(PptxConverter()) - register(EpubConverter()) - if (context != null) { - PDFBoxResourceLoader.init(context) - register(PdfConverter()) +/** + * A [MikroMarkdown] with every Android converter registered. + * + * PDF support needs a [Context]: pdfbox-android loads its resources from the app's assets. + */ +fun MikroMarkdown(context: Context? = null): MikroMarkdown = + MikroMarkdown(AndroidMimeDetector).apply { + register(MarkdownPassthroughConverter()) + register(HtmlConverter()) + register(CsvConverter()) + register(JsonConverter()) + register(XmlConverter()) + register(DocxConverter()) + register(XlsxConverter()) + register(PptxConverter()) + register(EpubConverter()) + if (context != null) { + PDFBoxResourceLoader.init(context) + register(PdfConverter()) + } + register(PlainTextConverter(), priority = 10.0) } - register(PlainTextConverter(), priority = 10.0) -} fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt index b93bda6..9f11ac2 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt @@ -5,9 +5,9 @@ import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.Document import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell +import java.io.InputStreamReader import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser -import java.io.InputStreamReader class CsvConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -22,10 +22,11 @@ class CsvConverter : DocumentConverter { val header = records[0].toList() if (header.isEmpty()) return Document() - val rows = records.drop(1).map { record -> - // Ragged rows are padded by the renderer; only extra columns need trimming here. - List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } - } + val rows = + records.drop(1).map { record -> + // Ragged rows are padded by the renderer; only extra columns need trimming here. + List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } + } return Document(blocks = listOf(Table(header = header.map { TableCell(it) }, rows = rows))) } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt index b1bc8e6..79a154a 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt @@ -16,15 +16,15 @@ import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.model.plainText import io.github.lemcoder.mikromarkdown.model.styled +import java.util.Base64 import org.apache.poi.xwpf.usermodel.XWPFDocument import org.apache.poi.xwpf.usermodel.XWPFParagraph import org.apache.poi.xwpf.usermodel.XWPFTable -import java.util.Base64 class DocxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "docx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -94,11 +94,12 @@ class DocxConverter : DocumentConverter { val mime = data.pictureTypeEnum.contentType val id = data.fileName ?: "image-${assets.size + 1}" assets += Asset(id = id, mediaType = mime, bytes = data.data, name = data.fileName) - out += Image( - alt = alt, - url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", - assetId = id, - ) + out += + Image( + alt = alt, + url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", + assetId = id, + ) } continue } @@ -126,11 +127,12 @@ class DocxConverter : DocumentConverter { itemLevel < level -> break itemLevel == level -> { index++ - val children = if (index < items.size && items[index].first > level) { - listOf(ListBlock(ordered = false, items = build(items[index].first))) - } else { - emptyList() - } + val children = + if (index < items.size && items[index].first > level) { + listOf(ListBlock(ordered = false, items = build(items[index].first))) + } else { + emptyList() + } result += ListItem(listOf(Paragraph(content)) + children) } // A deeper first item without a parent: promote it to this level. diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt index 9c80056..2542a64 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt @@ -8,20 +8,21 @@ import io.github.lemcoder.mikromarkdown.model.Paragraph import io.github.lemcoder.mikromarkdown.model.Strong import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument -import org.w3c.dom.Element -import org.xml.sax.InputSource import java.io.StringReader import java.util.zip.ZipInputStream import javax.xml.parsers.DocumentBuilderFactory +import org.w3c.dom.Element +import org.xml.sax.InputSource class EpubConverter : DocumentConverter { - private val metaFields = listOf( - "title" to "Title", - "creator" to "Authors", - "language" to "Language", - "description" to "Description", - "identifier" to "Identifier", - ) + private val metaFields = + listOf( + "title" to "Title", + "creator" to "Authors", + "language" to "Language", + "description" to "Description", + "identifier" to "Identifier", + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "epub" || info.mimetype == "application/epub+zip" @@ -100,8 +101,7 @@ class EpubConverter : DocumentConverter { val id = item.getAttribute("id") val href = item.getAttribute("href") val mediaType = item.getAttribute("media-type") - if (id.isNotEmpty() && href.isNotEmpty() && - (mediaType.contains("html") || mediaType.contains("xhtml"))) { + if (id.isNotEmpty() && href.isNotEmpty() && isReadableChapter(mediaType)) { manifest[id] = href } } @@ -117,12 +117,16 @@ class EpubConverter : DocumentConverter { return Triple(manifest, spine, metadata) } - private fun parseXml(bytes: ByteArray) = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = false - factory.isExpandEntityReferences = false - factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) - } catch (_: Exception) { - null - } + /** Only the spine's (X)HTML documents carry text; images and styles are skipped. */ + private fun isReadableChapter(mediaType: String): Boolean = mediaType.contains("html") + + private fun parseXml(bytes: ByteArray) = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = false + factory.isExpandEntityReferences = false + factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) + } catch (_: Exception) { + null + } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt index fb5dffd..b716b5d 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt @@ -7,8 +7,7 @@ import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument class HtmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { - return info.extension in setOf("html", "htm") || - info.mimetype in setOf("text/html", "application/xhtml+xml") + return info.extension in setOf("html", "htm") || info.mimetype in setOf("text/html", "application/xhtml+xml") } override fun parse(bytes: ByteArray, info: StreamInfo): Document = diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt index 026c9d1..9b4dbf4 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt @@ -11,16 +11,21 @@ import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document class JsonConverter : DocumentConverter { - private val writer = ObjectMapper().apply { - registerKotlinModule() - }.writer(object : DefaultPrettyPrinter() { - init { - indentArraysWith(DefaultIndenter(" ", "\n")) - indentObjectsWith(DefaultIndenter(" ", "\n")) - } - override fun createInstance() = this - override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") - }) + private val writer = + ObjectMapper() + .apply { registerKotlinModule() } + .writer( + object : DefaultPrettyPrinter() { + init { + indentArraysWith(DefaultIndenter(" ", "\n")) + indentObjectsWith(DefaultIndenter(" ", "\n")) + } + + override fun createInstance() = this + + override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") + } + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "json" || info.mimetype in setOf("application/json", "text/json") @@ -28,11 +33,12 @@ class JsonConverter : DocumentConverter { override fun parse(bytes: ByteArray, info: StreamInfo): Document { val json = bytes.toString(Charsets.UTF_8) - val pretty = try { - writer.writeValueAsString(ObjectMapper().readTree(json)) - } catch (_: Exception) { - json - } + val pretty = + try { + writer.writeValueAsString(ObjectMapper().readTree(json)) + } catch (_: Exception) { + json + } return Document(blocks = listOf(CodeBlock(pretty, "json"))) } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt index 262e38b..9eb0647 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt @@ -18,11 +18,12 @@ class PdfConverter : DocumentConverter { val title = doc.documentInformation?.title?.trim()?.ifBlank { null } // Paragraph markers let the text blocks split on real paragraph breaks // instead of collapsing a page into one block. - val stripper = PDFTextStripper().apply { - sortByPosition = true - setAddMoreFormatting(true) - paragraphStart = "\n" - } + val stripper = + PDFTextStripper().apply { + sortByPosition = true + setAddMoreFormatting(true) + paragraphStart = "\n" + } return Document(blocks = plainTextBlocks(stripper.getText(doc)), title = title) } finally { doc.close() diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt index 4c363e5..78578a7 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt @@ -23,15 +23,20 @@ import org.apache.poi.xslf.usermodel.XSLFPictureShape import org.apache.poi.xslf.usermodel.XSLFSimpleShape import org.apache.poi.xslf.usermodel.XSLFTable import org.apache.poi.xslf.usermodel.XSLFTextShape +import org.openxmlformats.schemas.drawingml.x2006.chart.CTAreaSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTAxDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTBarSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTLineSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTNumDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTPieSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTScatterSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTSerTx import org.openxmlformats.schemas.presentationml.x2006.main.CTPicture class PptxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "pptx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" + info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -44,12 +49,14 @@ class PptxConverter : DocumentConverter { blocks += HtmlComment("Slide number: ${index + 1}") blocks += shapeBlocks(slide.shapes) { if (index == 0 && title == null) title = it } - val notes = slide.notes?.shapes - ?.filterIsInstance() - ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } - ?.joinToString("\n") { it.text } - ?.trim() - .orEmpty() + val notes = + slide.notes + ?.shapes + ?.filterIsInstance() + ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } + ?.joinToString("\n") { it.text } + ?.trim() + .orEmpty() if (notes.isNotBlank()) { blocks += Heading(3, listOf(Text("Notes:"))) blocks += notes.lines().filter { it.isNotBlank() }.map { Paragraph(listOf(Text(it.trim()))) } @@ -118,49 +125,83 @@ class PptxConverter : DocumentConverter { val blocks = mutableListOf() blocks += Heading(3, listOf(Text(listOfNotNull("Chart", chartTitle(chart)).joinToString(": ")))) - val series = try { - seriesOf(chart) - } catch (_: Exception) { - blocks += Paragraph(listOf(Text("[unsupported chart]"))) - return blocks - } + val series = + try { + seriesOf(chart) + } catch (_: Exception) { + blocks += Paragraph(listOf(Text("[unsupported chart]"))) + return blocks + } if (series.isEmpty()) return blocks val rowCount = series.maxOf { it.categories.size } - blocks += Table( - header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, - rows = (0 until rowCount).map { row -> - val category = series.first().categories.getOrElse(row) { "" } - (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } - }, - ) + blocks += + Table( + header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, + rows = + (0 until rowCount).map { row -> + val category = series.first().categories.getOrElse(row) { "" } + (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } + }, + ) return blocks } - private fun chartTitle(chart: XSLFChart): String? = try { - val ctChart = chart.ctChart - if (!ctChart.isSetTitle) { + private fun chartTitle(chart: XSLFChart): String? = + try { + val ctChart = chart.ctChart + if (!ctChart.isSetTitle) { + null + } else { + val tx = ctChart.title?.tx + when { + tx?.isSetRich == true -> + tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") + tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v + else -> null + }?.ifBlank { null } + } + } catch (_: Exception) { null - } else { - val tx = ctChart.title?.tx - when { - tx?.isSetRich == true -> - tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") - tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v - else -> null - }?.ifBlank { null } } - } catch (_: Exception) { - null - } private data class Series(val name: String, val categories: List, val values: List) private fun seriesOf(chart: XSLFChart): List { - val plotArea = chart.ctChart.plotArea - val out = mutableListOf() + val plot = chart.ctChart.plotArea + // bar/bar3D, line/line3D and area/area3D each share one generated series type. + val all = + plot.barChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.bar3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.lineChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.line3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.areaChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.area3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.scatterChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.pieChartList.flatMap { it.serList.map { s -> s.toSeries() } } + return all.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } + } + + private fun CTBarSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTLineSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTAreaSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTPieSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) - fun categoryValues(cat: CTAxDataSource?): List = when { + private fun CTScatterSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetXVal) xVal else null, if (isSetYVal) yVal else null) + + private fun series(tx: CTSerTx?, categories: CTAxDataSource?, values: CTNumDataSource?) = + Series(seriesName(tx), categoryValues(categories), numericValues(values)) + + private fun categoryValues(cat: CTAxDataSource?): List = + when { cat == null -> emptyList() cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() @@ -169,64 +210,22 @@ class PptxConverter : DocumentConverter { else -> emptyList() } - fun numericValues(v: CTNumDataSource?): List = when { + private fun numericValues(v: CTNumDataSource?): List = + when { v == null -> emptyList() v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() else -> emptyList() } - fun seriesName(tx: CTSerTx?): String = when { + private fun seriesName(tx: CTSerTx?): String = + when { tx == null -> "" tx.isSetV -> tx.v tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v.orEmpty() else -> "" } - for (c in plotArea.barChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.bar3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.lineChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.line3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.areaChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.area3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.scatterChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetXVal) s.xVal else null), - numericValues(if (s.isSetYVal) s.yVal else null), - ) - for (c in plotArea.pieChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - - return out.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } - } - private fun table(table: XSLFTable): Table? { val rows = table.rows if (rows.isEmpty()) return null diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt index 984dec0..e601f5c 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt @@ -8,18 +8,18 @@ import io.github.lemcoder.mikromarkdown.model.Heading import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text +import kotlin.math.floor import org.apache.poi.ss.usermodel.Cell import org.apache.poi.ss.usermodel.CellType import org.apache.poi.ss.usermodel.DataFormatter import org.apache.poi.xssf.usermodel.XSSFWorkbook -import kotlin.math.floor class XlsxConverter : DocumentConverter { private val formatter = DataFormatter() override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "xlsx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -35,12 +35,12 @@ class XlsxConverter : DocumentConverter { if (columns == 0) continue blocks += Heading(2, listOf(Text(sheet.sheetName))) - blocks += Table( - header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, - rows = rows.drop(1).map { row -> - (0 until columns).map { TableCell(cellValue(row.getCell(it))) } - }, - ) + blocks += + Table( + header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, + rows = + rows.drop(1).map { row -> (0 until columns).map { TableCell(cellValue(row.getCell(it))) } }, + ) } return Document(blocks = blocks) @@ -54,8 +54,7 @@ class XlsxConverter : DocumentConverter { return when (cell.cellType) { CellType.NUMERIC -> { val v = cell.numericCellValue - if (v == floor(v) && !v.isInfinite()) v.toLong().toString() - else formatter.formatCellValue(cell) + if (v == floor(v) && !v.isInfinite()) v.toLong().toString() else formatter.formatCellValue(cell) } CellType.BLANK -> "" else -> formatter.formatCellValue(cell).trim() diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt index 8979d57..7d3e6ac 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt @@ -4,7 +4,6 @@ import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document -import org.xml.sax.InputSource import java.io.StringReader import java.io.StringWriter import javax.xml.parsers.DocumentBuilderFactory @@ -12,6 +11,7 @@ import javax.xml.transform.OutputKeys import javax.xml.transform.TransformerFactory import javax.xml.transform.dom.DOMSource import javax.xml.transform.stream.StreamResult +import org.xml.sax.InputSource class XmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -23,23 +23,24 @@ class XmlConverter : DocumentConverter { return Document(blocks = listOf(CodeBlock(pretty, "xml"))) } - private fun prettyPrint(xml: String): String = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = true - factory.isIgnoringElementContentWhitespace = true - val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) + private fun prettyPrint(xml: String): String = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = true + factory.isIgnoringElementContentWhitespace = true + val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) - val tf = TransformerFactory.newInstance() - tf.setAttribute("indent-number", 2) - val transformer = tf.newTransformer() - transformer.setOutputProperty(OutputKeys.INDENT, "yes") - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") + val tf = TransformerFactory.newInstance() + tf.setAttribute("indent-number", 2) + val transformer = tf.newTransformer() + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") - val writer = StringWriter() - transformer.transform(DOMSource(doc), StreamResult(writer)) - writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") - } catch (_: Exception) { - xml - } + val writer = StringWriter() + transformer.transform(DOMSource(doc), StreamResult(writer)) + writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") + } catch (_: Exception) { + xml + } } diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt index d7c9296..510e036 100644 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt +++ b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt @@ -29,22 +29,49 @@ import org.jsoup.nodes.TextNode /** * Walks an HTML DOM into the shared document model. * - * Replaces the previous HTML → Markdown string conversion: tables, lists and inline - * emphasis become model nodes, so the Markdown renderer owns all syntax decisions. + * Replaces the previous HTML → Markdown string conversion: tables, lists and inline emphasis become model nodes, so the + * Markdown renderer owns all syntax decisions. */ object HtmlToDocument { - private val DROPPED_TAGS = setOf( - "script", "style", "noscript", "template", "button", "svg", "iframe", "form", "input", "select", - ) + private val DROPPED_TAGS = + setOf( + "script", + "style", + "noscript", + "template", + "button", + "svg", + "iframe", + "form", + "input", + "select", + ) private val HEADINGS = mapOf("h1" to 1, "h2" to 2, "h3" to 3, "h4" to 4, "h5" to 5, "h6" to 6) /** Tags that only group other content; their children are lifted into the parent block flow. */ - private val CONTAINERS = setOf( - "div", "section", "article", "main", "header", "footer", "aside", "nav", "body", "html", - "figure", "details", "summary", "fieldset", "center", "hgroup", "picture", "colgroup", - ) + private val CONTAINERS = + setOf( + "div", + "section", + "article", + "main", + "header", + "footer", + "aside", + "nav", + "body", + "html", + "figure", + "details", + "summary", + "fieldset", + "center", + "hgroup", + "picture", + "colgroup", + ) fun parse(html: String, baseUri: String = ""): Document { val doc = Jsoup.parse(html, baseUri) @@ -91,49 +118,66 @@ object HtmlToDocument { return out } - private fun Element.isBlockLevel(): Boolean = when (tagName()) { - in HEADINGS, "p", "ul", "ol", "table", "pre", "blockquote", "hr", "dl", "li", "figcaption" -> true - in CONTAINERS -> true - else -> false - } - - private fun blockFor(element: Element): List = when (val tag = element.tagName()) { - in HEADINGS -> listOfNotNull( - inlines(element).trimEdges() - .takeIf { it.isNotEmpty() } - ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) }, - ) + private fun Element.isBlockLevel(): Boolean = + when (tagName()) { + in HEADINGS, + "p", + "ul", + "ol", + "table", + "pre", + "blockquote", + "hr", + "dl", + "li", + "figcaption" -> true + in CONTAINERS -> true + else -> false + } - "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) - "ul", "ol" -> listBlock(element) - "table" -> table(element) - "pre" -> listOf(codeBlock(element)) - "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() - "hr" -> listOf(ThematicBreak) - "dl" -> definitionList(element) - "figcaption" -> listOfNotNull( - inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) }, - ) - // Containers and stray
  • outside a list contribute their children directly. - else -> blocks(element) - } + private fun blockFor(element: Element): List = + when (val tag = element.tagName()) { + in HEADINGS -> + listOfNotNull( + inlines(element) + .trimEdges() + .takeIf { it.isNotEmpty() } + ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) } + ) + + "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) + "ul", + "ol" -> listBlock(element) + "table" -> table(element) + "pre" -> listOf(codeBlock(element)) + "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() + "hr" -> listOf(ThematicBreak) + "dl" -> definitionList(element) + "figcaption" -> + listOfNotNull( + inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) } + ) + // Containers and stray
  • outside a list contribute their children directly. + else -> blocks(element) + } private fun listBlock(element: Element): List { val ordered = element.tagName() == "ol" val start = element.attr("start").toIntOrNull() ?: 1 - val items = element.children() - .filter { it.tagName() == "li" } - .map { li -> ListItem(blocks = blocks(li)) } - .filter { it.blocks.isNotEmpty() } + val items = + element + .children() + .filter { it.tagName() == "li" } + .map { li -> ListItem(blocks = blocks(li)) } + .filter { it.blocks.isNotEmpty() } if (items.isEmpty()) return emptyList() return listOf(ListBlock(ordered = ordered, items = items, start = start)) } private fun codeBlock(element: Element): Block { val code = element.selectFirst("code") ?: element - val language = code.classNames() - .firstOrNull { it.startsWith("language-") || it.startsWith("lang-") } - ?.substringAfter('-') + val language = + code.classNames().firstOrNull { it.startsWith("language-") || it.startsWith("lang-") }?.substringAfter('-') return CodeBlock(code.wholeText().trimEnd(), language) } @@ -158,20 +202,22 @@ object HtmlToDocument { val body = mutableListOf>() for ((index, tr) in rows.withIndex()) { - val cells = tr.children() - .filter { it.tagName() == "th" || it.tagName() == "td" } - .map { cell -> - TableCell( - content = inlines(cell).trimEdges(), - colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - ) - } + val cells = + tr.children() + .filter { it.tagName() == "th" || it.tagName() == "td" } + .map { cell -> + TableCell( + content = inlines(cell).trimEdges(), + colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + ) + } if (cells.isEmpty()) continue - val isHeaderRow = index == 0 && - tr.children().all { it.tagName() == "th" } && - (tr.parent()?.tagName() == "thead" || header.isEmpty()) + val isHeaderRow = + index == 0 && + tr.children().all { it.tagName() == "th" } && + (tr.parent()?.tagName() == "thead" || header.isEmpty()) if (isHeaderRow && header.isEmpty()) header = cells else body += cells } @@ -196,35 +242,45 @@ object HtmlToDocument { if (text.isNotEmpty()) add(Text(text)) } - is Element -> when (node.tagName()) { - "br" -> add(LineBreak) - "img" -> { - val src = node.attr("abs:src").ifBlank { node.attr("src") } - val alt = node.attr("alt") - if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) - } + is Element -> + when (node.tagName()) { + "br" -> add(LineBreak) + "img" -> { + val src = node.attr("abs:src").ifBlank { node.attr("src") } + val alt = node.attr("alt") + if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) + } - "a" -> { - val href = node.attr("abs:href").ifBlank { node.attr("href") } - val content = inlines(node).trimEdges() - when { - content.isEmpty() -> Unit - href.isBlank() || href.startsWith("javascript:") -> addAll(content) - else -> add(Link(content, href, node.attr("title").ifBlank { null })) + "a" -> { + val href = node.attr("abs:href").ifBlank { node.attr("href") } + val content = inlines(node).trimEdges() + when { + content.isEmpty() -> Unit + href.isBlank() || href.startsWith("javascript:") -> addAll(content) + else -> add(Link(content, href, node.attr("title").ifBlank { null })) + } } - } - "strong", "b" -> wrapped(node) { Strong(it) } - "em", "i", "cite", "var" -> wrapped(node) { Emphasis(it) } - "del", "s", "strike" -> wrapped(node) { Strikethrough(it) } - "code", "kbd", "samp", "tt" -> { - val code = node.wholeText().trim() - if (code.isNotEmpty()) add(CodeSpan(code)) - } + "strong", + "b" -> wrapped(node) { Strong(it) } + "em", + "i", + "cite", + "var" -> wrapped(node) { Emphasis(it) } + "del", + "s", + "strike" -> wrapped(node) { Strikethrough(it) } + "code", + "kbd", + "samp", + "tt" -> { + val code = node.wholeText().trim() + if (code.isNotEmpty()) add(CodeSpan(code)) + } - // Block-level content encountered inline (e.g. a
    inside a ): keep its text. - else -> addAll(inlines(node)) - } + // Block-level content encountered inline (e.g. a
    inside a ): keep its text. + else -> addAll(inlines(node)) + } } } @@ -239,14 +295,11 @@ object HtmlToDocument { /** HTML collapses runs of whitespace; do the same before the text reaches the model. */ // Non-breaking spaces are not collapsible whitespace in HTML, so they survive verbatim. - private fun TextNode.normalizedText(): String = - wholeText.replace(Regex("\\s+"), " ") + private fun TextNode.normalizedText(): String = wholeText.replace(Regex("\\s+"), " ") - private fun List.startsWithSpace(): Boolean = - (firstOrNull() as? Text)?.value?.startsWith(" ") == true + private fun List.startsWithSpace(): Boolean = (firstOrNull() as? Text)?.value?.startsWith(" ") == true - private fun List.endsWithSpace(): Boolean = - (lastOrNull() as? Text)?.value?.endsWith(" ") == true + private fun List.endsWithSpace(): Boolean = (lastOrNull() as? Text)?.value?.endsWith(" ") == true /** Drops leading/trailing whitespace-only text so emphasis markers hug their content. */ private fun List.trimEdges(): List { diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt index 07310a2..ba8cc28 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/DocumentConverter.kt @@ -9,5 +9,6 @@ import io.github.lemcoder.mikromarkdown.model.Document */ interface DocumentConverter { fun accepts(bytes: ByteArray, info: StreamInfo): Boolean + fun parse(bytes: ByteArray, info: StreamInfo): Document } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt index 80f3b6a..9d25f11 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/Exceptions.kt @@ -1,7 +1,7 @@ package io.github.lemcoder.mikromarkdown -sealed class MarkItDownException(message: String, cause: Throwable? = null) : Exception(message, cause) +sealed class MikroMarkdownException(message: String, cause: Throwable? = null) : Exception(message, cause) -class UnsupportedFormatException(message: String) : MarkItDownException(message) +class UnsupportedFormatException(message: String) : MikroMarkdownException(message) -class FileConversionException(message: String, cause: Throwable? = null) : MarkItDownException(message, cause) +class FileConversionException(message: String, cause: Throwable? = null) : MikroMarkdownException(message, cause) diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt index 0514b9f..d6d2c40 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdown.kt @@ -34,25 +34,25 @@ class MikroMarkdown( } fun parse(bytes: ByteArray, info: StreamInfo): Document { - val sorted = converters.sortedBy { it.second } - for ((converter, _) in sorted) { - if (!converter.accepts(bytes, info)) continue - return try { - converter.parse(bytes, info) - } catch (e: MarkItDownException) { - throw e - } catch (e: Exception) { - throw FileConversionException( - "Conversion failed with ${converter::class.simpleName}: ${e.message}", - e, + val converter = + converters.sortedBy { it.second }.firstOrNull { (candidate, _) -> candidate.accepts(bytes, info) }?.first + ?: throw UnsupportedFormatException( + "No converter found for: ${info.extension ?: info.mimetype ?: "unknown"}" ) - } - } - throw UnsupportedFormatException( - "No converter found for: ${info.extension ?: info.mimetype ?: "unknown"}", - ) + + return converter.parseOrFail(bytes, info) } + /** Converter failures surface as [FileConversionException]; our own exceptions pass through. */ + private fun DocumentConverter.parseOrFail(bytes: ByteArray, info: StreamInfo): Document = + try { + parse(bytes, info) + } catch (e: MikroMarkdownException) { + throw e + } catch (e: Exception) { + throw FileConversionException("Conversion failed with ${this::class.simpleName}: ${e.message}", e) + } + fun render(document: Document): ConversionResult = ConversionResult( markdown = renderer.render(document), diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt index cc095d5..9649640 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PlainTextConverter.kt @@ -7,8 +7,7 @@ import io.github.lemcoder.mikromarkdown.model.RawBlock class PlainTextConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { - return info.extension in setOf("txt", "log", "text") || - info.mimetype == "text/plain" + return info.extension in setOf("txt", "log", "text") || info.mimetype == "text/plain" } override fun parse(bytes: ByteArray, info: StreamInfo): Document = diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt index 6b6c205..2f33e52 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt @@ -3,9 +3,8 @@ package io.github.lemcoder.mikromarkdown.model /** * Format-independent document model. * - * Every converter parses its input into a [Document]; a single renderer turns - * documents into Markdown. Output quirks are therefore fixed once, in the - * renderer, rather than per format. + * Every converter parses its input into a [Document]; a single renderer turns documents into Markdown. Output quirks + * are therefore fixed once, in the renderer, rather than per format. */ data class Document( val blocks: List = emptyList(), @@ -70,7 +69,12 @@ data class TableCell( constructor(text: String) : this(if (text.isEmpty()) emptyList() else listOf(Text(text))) } -enum class Alignment { NONE, LEFT, CENTER, RIGHT } +enum class Alignment { + NONE, + LEFT, + CENTER, + RIGHT, +} data object ThematicBreak : Block diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt index a5fd5fa..4a57e40 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt @@ -84,16 +84,17 @@ class DocumentBuilder { header = header.map { TableCell(it) }, rows = rows.map { row -> row.map { TableCell(it) } }, alignments = alignments, - ), + ) ) } - fun build(): Document = Document( - blocks = blocks.toList(), - title = title, - metadata = metadata.toMap(), - assets = assets.toList(), - ) + fun build(): Document = + Document( + blocks = blocks.toList(), + title = title, + metadata = metadata.toMap(), + assets = assets.toList(), + ) } /** Builds a list of inlines without repeating `listOf(...)` wrappers in parsers. */ diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt index d457376..2285c9c 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt @@ -54,9 +54,7 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De fun render(document: Document): String { val body = renderBlocks(document.blocks) if (!options.frontMatter || document.metadata.isEmpty()) return body - val front = document.metadata.entries.joinToString("\n") { (k, v) -> - "$k: ${v.replace("\n", " ")}" - } + val front = document.metadata.entries.joinToString("\n") { (k, v) -> "$k: ${v.replace("\n", " ")}" } return "---\n$front\n---\n\n$body".trimEnd() } @@ -73,49 +71,51 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return chunks.joinToString("\n\n").trim() } - private fun renderBlock(block: Block): String = when (block) { - is Heading -> { - val text = inlines(block.content, TextContext.HEADING).collapseLines() - if (text.isBlank()) "" else "${"#".repeat(block.level.coerceIn(1, options.maxHeadingLevel))} $text" - } + private fun renderBlock(block: Block): String = + when (block) { + is Heading -> { + val text = inlines(block.content, TextContext.HEADING).collapseLines() + if (text.isBlank()) "" else "${"#".repeat(block.level.coerceIn(1, options.maxHeadingLevel))} $text" + } - is Paragraph -> inlines(block.content, TextContext.BLOCK).trimEnd() + is Paragraph -> inlines(block.content, TextContext.BLOCK).trimEnd() - is CodeBlock -> { - val fence = "`".repeat(maxOf(3, longestBacktickRun(block.code) + 1)) - "$fence${block.language.orEmpty()}\n${block.code.trimEnd('\n')}\n$fence" - } + is CodeBlock -> { + val fence = "`".repeat(maxOf(3, longestBacktickRun(block.code) + 1)) + "$fence${block.language.orEmpty()}\n${block.code.trimEnd('\n')}\n$fence" + } - is BlockQuote -> renderBlocks(block.blocks) - .lines() - .joinToString("\n") { if (it.isEmpty()) ">" else "> $it" } + is BlockQuote -> + renderBlocks(block.blocks).lines().joinToString("\n") { if (it.isEmpty()) ">" else "> $it" } - is ListBlock -> renderList(block, indent = "") + is ListBlock -> renderList(block, indent = "") - is Table -> renderTable(block) + is Table -> renderTable(block) - ThematicBreak -> "---" + ThematicBreak -> "---" - is HtmlComment -> "" + is HtmlComment -> "" - // Already-Markdown content: only whitespace is normalized, never syntax. - is RawBlock -> block.text - .replace("\r\n", "\n") - .lines() - .joinToString("\n") { it.trimEnd() } - .replace(Regex("\n{3,}"), "\n\n") - .trim() - } + // Already-Markdown content: only whitespace is normalized, never syntax. + is RawBlock -> + block.text + .replace("\r\n", "\n") + .lines() + .joinToString("\n") { it.trimEnd() } + .replace(Regex("\n{3,}"), "\n\n") + .trim() + } private fun renderList(list: ListBlock, indent: String): String { val lines = mutableListOf() list.items.forEachIndexed { index, item -> val marker = if (list.ordered) "${list.start + index}. " else "${options.bullet} " - val checkbox = when (item.checked) { - true -> "[x] " - false -> "[ ] " - null -> "" - } + val checkbox = + when (item.checked) { + true -> "[x] " + false -> "[ ] " + null -> "" + } val childIndent = indent + " ".repeat(marker.length) val body = renderItemBlocks(item, childIndent) val firstLine = body.firstOrNull().orEmpty() @@ -131,10 +131,11 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De val lines = mutableListOf() item.blocks.forEachIndexed { index, block -> if (index > 0) lines += "" - val rendered = when (block) { - is ListBlock -> renderList(block, childIndent) - else -> renderBlock(block).lines().joinToString("\n") { if (it.isEmpty()) it else childIndent + it } - } + val rendered = + when (block) { + is ListBlock -> renderList(block, childIndent) + else -> renderBlock(block).lines().joinToString("\n") { if (it.isEmpty()) it else childIndent + it } + } if (rendered.isEmpty()) return@forEachIndexed lines += rendered.lines() } @@ -153,17 +154,18 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De val rows = bodyRows.map { pad(it, columns) } val alignments = List(columns) { table.alignments.getOrElse(it) { Alignment.NONE } } - val widths = if (options.padTableColumns) { - List(columns) { col -> - maxOf( - 3, - header[col].length, - rows.maxOfOrNull { it[col].length } ?: 0, - ) + val widths = + if (options.padTableColumns) { + List(columns) { col -> + maxOf( + 3, + header[col].length, + rows.maxOfOrNull { it[col].length } ?: 0, + ) + } + } else { + null } - } else { - null - } val out = StringBuilder() out.append(row(header, widths)).append('\n') @@ -173,7 +175,8 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De if (index != rows.lastIndex) out.append('\n') } if (table.caption.isNotEmpty()) { - out.append("\n\n").append(options.emphasisMarker) + out.append("\n\n") + .append(options.emphasisMarker) .append(inlines(table.caption, TextContext.INLINE).collapseLines()) .append(options.emphasisMarker) } @@ -201,20 +204,22 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De if (cells.size >= columns) cells.take(columns) else cells + List(columns - cells.size) { "" } private fun row(cells: List, widths: List?): String = - cells.mapIndexed { index, cell -> - if (widths == null) cell else cell.padEnd(widths[index]) - }.joinToString(" | ", "| ", " |") + cells + .mapIndexed { index, cell -> if (widths == null) cell else cell.padEnd(widths[index]) } + .joinToString(" | ", "| ", " |") private fun delimiterRow(alignments: List, widths: List?): String = - alignments.mapIndexed { index, alignment -> - val width = widths?.get(index) ?: 3 - when (alignment) { - Alignment.NONE -> "-".repeat(width) - Alignment.LEFT -> ":" + "-".repeat(width - 1) - Alignment.RIGHT -> "-".repeat(width - 1) + ":" - Alignment.CENTER -> ":" + "-".repeat(width - 2) + ":" + alignments + .mapIndexed { index, alignment -> + val width = widths?.get(index) ?: 3 + when (alignment) { + Alignment.NONE -> "-".repeat(width) + Alignment.LEFT -> ":" + "-".repeat(width - 1) + Alignment.RIGHT -> "-".repeat(width - 1) + ":" + Alignment.CENTER -> ":" + "-".repeat(width - 2) + ":" + } } - }.joinToString(" | ", "| ", " |") + .joinToString(" | ", "| ", " |") private fun inlines(inlines: List, context: TextContext): String { val sb = StringBuilder() @@ -283,46 +288,54 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return if (context == TextContext.TABLE) text.replace("|", "\\|") else text } return buildString(text.length) { - var lineStart = atLineStart - for ((index, ch) in text.withIndex()) { - val prev = text.getOrNull(index - 1) - val next = text.getOrNull(index + 1) - when { - ch == '\n' -> { - append(ch) - lineStart = true - continue - } - - ch == '\\' -> append("\\\\") - ch == '|' && context == TextContext.TABLE -> append("\\|") - ch == '*' -> append("\\*") - ch == '`' -> append("\\`") - ch == '[' || ch == ']' -> append('\\').append(ch) - // Intraword underscores (snake_case) are not emphasis in CommonMark. - ch == '_' && !(prev?.isLetterOrDigit() == true && next?.isLetterOrDigit() == true) -> - append("\\_") - - ch == '<' && next?.let { it.isLetter() || it == '/' || it == '!' } == true -> append("\\<") - ch == '&' && looksLikeEntity(text, index) -> append("\\&") - - lineStart && (ch == '#' || ch == '>' || ch == '=') -> append('\\').append(ch) - lineStart && (ch == '-' || ch == '+') && next?.isWhitespace() != false -> append('\\').append(ch) - lineStart && ch.isDigit() && startsOrderedList(text, index) -> { - append(ch) - // Escape the delimiter instead of the digits: "1\. item". - } - - lineStart && (ch == '.' || ch == ')') && prev?.isDigit() == true && - startsOrderedList(text, index - 1) -> append('\\').append(ch) - - else -> append(ch) - } - if (!ch.isWhitespace()) lineStart = false + for (index in text.indices) { + if (needsEscape(text, index, context, atLineStart)) append('\\') + append(text[index]) } } } + private fun needsEscape(text: String, index: Int, context: TextContext, atLineStart: Boolean): Boolean = + when (val ch = text[index]) { + '\\', + '*', + '`', + '[', + ']' -> true + '|' -> context == TextContext.TABLE + // Intraword underscores (snake_case) are not emphasis in CommonMark. + '_' -> !isIntraword(text, index) + '<' -> text.getOrNull(index + 1)?.let { it.isLetter() || it == '/' || it == '!' } == true + '&' -> looksLikeEntity(text, index) + // The rest only introduce block syntax at the start of a line. + '#', + '>', + '=' -> startsLine(text, index, atLineStart) + '-', + '+' -> startsLine(text, index, atLineStart) && text.getOrNull(index + 1)?.isWhitespace() != false + // "1. item" in running text would become a list; escape the delimiter, not the digits. + '.', + ')' -> followsOrderedListMarker(text, index, atLineStart) + else -> false + } + + private fun isIntraword(text: String, index: Int): Boolean = + text.getOrNull(index - 1)?.isLetterOrDigit() == true && text.getOrNull(index + 1)?.isLetterOrDigit() == true + + /** True when only indentation separates [index] from the start of its line. */ + private fun startsLine(text: String, index: Int, atLineStart: Boolean): Boolean { + var i = index - 1 + while (i >= 0 && (text[i] == ' ' || text[i] == '\t')) i-- + return if (i < 0) atLineStart else text[i] == '\n' + } + + private fun followsOrderedListMarker(text: String, index: Int, atLineStart: Boolean): Boolean { + var digits = index - 1 + while (digits >= 0 && text[digits].isDigit()) digits-- + if (digits == index - 1) return false + return startsLine(text, digits + 1, atLineStart) && startsOrderedList(text, index - 1) + } + private fun startsOrderedList(text: String, digitIndex: Int): Boolean { var start = digitIndex while (start > 0 && text[start - 1].isDigit()) start-- @@ -341,8 +354,7 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return text.substring(index + 1, semicolon).all { it.isLetterOrDigit() || it == '#' } } - private fun encodeUrl(url: String): String = - url.replace(" ", "%20").replace("(", "%28").replace(")", "%29") + private fun encodeUrl(url: String): String = url.replace(" ", "%20").replace("(", "%28").replace(")", "%29") private fun longestBacktickRun(text: String): Int { var longest = 0 @@ -358,10 +370,14 @@ class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOptions.De return longest } - private fun String.collapseLines(): String = - replace("\r\n", "\n").lines().joinToString(" ") { it.trim() }.trim() + private fun String.collapseLines(): String = replace("\r\n", "\n").lines().joinToString(" ") { it.trim() }.trim() - private enum class TextContext { BLOCK, INLINE, HEADING, TABLE } + private enum class TextContext { + BLOCK, + INLINE, + HEADING, + TABLE, + } companion object { val Default = MarkdownRenderer() diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt index 87810ed..bdbc49d 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TextBlocks.kt @@ -7,8 +7,8 @@ import io.github.lemcoder.mikromarkdown.model.Text /** * Turns extracted plain text (PDF pages, speaker notes, …) into paragraph blocks. * - * Blank lines separate paragraphs; soft-wrapped lines inside a paragraph are rejoined, - * so the Markdown does not inherit the source layout's line breaks. + * Blank lines separate paragraphs; soft-wrapped lines inside a paragraph are rejoined, so the Markdown does not inherit + * the source layout's line breaks. */ fun plainTextBlocks(text: String, reflow: Boolean = true): List { // Form feeds mark PDF page breaks; treat them as paragraph boundaries. @@ -32,21 +32,18 @@ fun plainTextBlocks(text: String, reflow: Boolean = true): List { return paragraphs.map { Paragraph(listOf(Text(it))) } } -private fun String.endsWithWordBreak(): Boolean = - endsWith("-") && length > 1 && this[length - 2].isLetter() +private fun String.endsWithWordBreak(): Boolean = endsWith("-") && length > 1 && this[length - 2].isLetter() private val WORD = Regex("[\\p{L}]{2,}") /** Words the document uses on their own; the de-hyphenation heuristic consults this. */ -private fun wordsIn(text: String): Set = - WORD.findAll(text).map { it.value.lowercase() }.toSet() +private fun wordsIn(text: String): Set = WORD.findAll(text).map { it.value.lowercase() }.toSet() /** * Rejoins soft-wrapped lines. * - * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are - * words the document uses elsewhere on their own (`conversation-` + `centric`), the hyphen is a - * real compound and stays. + * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are words the document uses + * elsewhere on their own (`conversation-` + `centric`), the hyphen is a real compound and stays. */ private fun joinWrappedLines(lines: List, vocabulary: Set): String { val sb = StringBuilder() diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt index bb6065e..076e5aa 100644 --- a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt @@ -5,10 +5,10 @@ import kotlin.test.Test import kotlin.test.assertFalse import kotlin.test.assertTrue -expect fun testMarkItDown(): MikroMarkdown +expect fun testMikroMarkdown(): MikroMarkdown class FileIntegrationTest { - private val mid = testMarkItDown() + private val mid = testMikroMarkdown() private fun assertConversion( filename: String, @@ -28,75 +28,84 @@ class FileIntegrationTest { } @Test - fun testDocx() = assertConversion( - filename = "test.docx", - mustInclude = listOf( - "314b0a30-5b04-470b-b9f7-eed2c2bec74a", - "49e168b7-d2ae-407f-a055-2167576f39a1", - "# Abstract", - "# Introduction", - "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", - ), - ) + fun testDocx() = + assertConversion( + filename = "test.docx", + mustInclude = + listOf( + "314b0a30-5b04-470b-b9f7-eed2c2bec74a", + "49e168b7-d2ae-407f-a055-2167576f39a1", + "# Abstract", + "# Introduction", + "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", + ), + ) @Test - fun testXlsx() = assertConversion( - filename = "test.xlsx", - mustInclude = listOf( - "09060124-b5e7-4717-9d07-3c046eb", - "6ff4173b-42a5-4784-9b19-f49caff4d93d", - "affc7dad-52dc-4b98-9b5d-51e65d8a8ad0", - ), - ) + fun testXlsx() = + assertConversion( + filename = "test.xlsx", + mustInclude = + listOf( + "09060124-b5e7-4717-9d07-3c046eb", + "6ff4173b-42a5-4784-9b19-f49caff4d93d", + "affc7dad-52dc-4b98-9b5d-51e65d8a8ad0", + ), + ) @Test - fun testPptx() = assertConversion( - filename = "test.pptx", - mustInclude = listOf( - "2cdda5c8-e50e-4db4-b5f0-9722a649f455", - "04191ea8-5c73-4215-a1d3-1cfb43aaaf12", - "44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a", - "1b92870d-e3b5-4e65-8153-919f4ff45592", - "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", - ), - ) + fun testPptx() = + assertConversion( + filename = "test.pptx", + mustInclude = + listOf( + "2cdda5c8-e50e-4db4-b5f0-9722a649f455", + "04191ea8-5c73-4215-a1d3-1cfb43aaaf12", + "44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a", + "1b92870d-e3b5-4e65-8153-919f4ff45592", + "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation", + ), + ) @Test - fun testBlogHtml() = assertConversion( - filename = "test_blog.html", - mustInclude = listOf( - "Large language models (LLMs) are powerful tools that can generate natural language texts for various applications", - "an example where high cost can easily prevent a generic complex", - ), - ) + fun testBlogHtml() = + assertConversion( + filename = "test_blog.html", + mustInclude = + listOf( + "Large language models (LLMs) are powerful tools that can generate natural language texts", + "an example where high cost can easily prevent a generic complex", + ), + ) @Test - fun testWikipediaHtml() = assertConversion( - filename = "test_wikipedia.html", - mustInclude = listOf( - "Microsoft entered the operating system (OS) business in 1980", - ), - mustNotInclude = listOf( - "move to sidebar", - ), - ) + fun testWikipediaHtml() = + assertConversion( + filename = "test_wikipedia.html", + mustInclude = listOf("Microsoft entered the operating system (OS) business in 1980"), + mustNotInclude = listOf("move to sidebar"), + ) @Test - fun testJson() = assertConversion( - filename = "test.json", - mustInclude = listOf( - "5b64c88c-b3c3-4510-bcb8-da0b200602d8", - "9700dc99-6685-40b4-9a3a-5e406dcb37f3", - ), - ) + fun testJson() = + assertConversion( + filename = "test.json", + mustInclude = + listOf( + "5b64c88c-b3c3-4510-bcb8-da0b200602d8", + "9700dc99-6685-40b4-9a3a-5e406dcb37f3", + ), + ) @Test - fun testEpub() = assertConversion( - filename = "test.epub", - mustInclude = listOf( - "A test EPUB document for MarkItDown testing", - "Chapter 1", - "Chapter 2", - ), - ) + fun testEpub() = + assertConversion( + filename = "test.epub", + mustInclude = + listOf( + "A test EPUB document for MarkItDown testing", + "Chapter 1", + "Chapter 2", + ), + ) } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt similarity index 61% rename from library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt rename to library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 6b72477..d08cb64 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MarkItDown.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -1,6 +1,5 @@ package io.github.lemcoder.mikromarkdown -import io.github.lemcoder.mikromarkdown.utils.TikaMimeDetector import io.github.lemcoder.mikromarkdown.converters.CsvConverter import io.github.lemcoder.mikromarkdown.converters.DocxConverter import io.github.lemcoder.mikromarkdown.converters.EpubConverter @@ -12,20 +11,23 @@ import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter import io.github.lemcoder.mikromarkdown.converters.PptxConverter import io.github.lemcoder.mikromarkdown.converters.XlsxConverter import io.github.lemcoder.mikromarkdown.converters.XmlConverter +import io.github.lemcoder.mikromarkdown.utils.TikaMimeDetector import java.io.File -fun MarkItDown(): MikroMarkdown = MikroMarkdown(TikaMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(DocxConverter()) - register(XlsxConverter()) - register(PptxConverter()) - register(EpubConverter()) - register(PdfConverter()) - register(PlainTextConverter(), priority = 10.0) -} +/** A [MikroMarkdown] with every JVM converter registered and Tika-based format detection. */ +fun MikroMarkdown(): MikroMarkdown = + MikroMarkdown(TikaMimeDetector).apply { + register(MarkdownPassthroughConverter()) + register(HtmlConverter()) + register(CsvConverter()) + register(JsonConverter()) + register(XmlConverter()) + register(DocxConverter()) + register(XlsxConverter()) + register(PptxConverter()) + register(EpubConverter()) + register(PdfConverter()) + register(PlainTextConverter(), priority = 10.0) + } fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt index b93bda6..9f11ac2 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/CsvConverter.kt @@ -5,9 +5,9 @@ import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.Document import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell +import java.io.InputStreamReader import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser -import java.io.InputStreamReader class CsvConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -22,10 +22,11 @@ class CsvConverter : DocumentConverter { val header = records[0].toList() if (header.isEmpty()) return Document() - val rows = records.drop(1).map { record -> - // Ragged rows are padded by the renderer; only extra columns need trimming here. - List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } - } + val rows = + records.drop(1).map { record -> + // Ragged rows are padded by the renderer; only extra columns need trimming here. + List(header.size) { col -> TableCell(record.takeIf { col < it.size() }?.get(col) ?: "") } + } return Document(blocks = listOf(Table(header = header.map { TableCell(it) }, rows = rows))) } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt index b1bc8e6..79a154a 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/DocxConverter.kt @@ -16,15 +16,15 @@ import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.model.plainText import io.github.lemcoder.mikromarkdown.model.styled +import java.util.Base64 import org.apache.poi.xwpf.usermodel.XWPFDocument import org.apache.poi.xwpf.usermodel.XWPFParagraph import org.apache.poi.xwpf.usermodel.XWPFTable -import java.util.Base64 class DocxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "docx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + info.mimetype == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -94,11 +94,12 @@ class DocxConverter : DocumentConverter { val mime = data.pictureTypeEnum.contentType val id = data.fileName ?: "image-${assets.size + 1}" assets += Asset(id = id, mediaType = mime, bytes = data.data, name = data.fileName) - out += Image( - alt = alt, - url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", - assetId = id, - ) + out += + Image( + alt = alt, + url = "data:$mime;base64,${Base64.getEncoder().encodeToString(data.data)}", + assetId = id, + ) } continue } @@ -126,11 +127,12 @@ class DocxConverter : DocumentConverter { itemLevel < level -> break itemLevel == level -> { index++ - val children = if (index < items.size && items[index].first > level) { - listOf(ListBlock(ordered = false, items = build(items[index].first))) - } else { - emptyList() - } + val children = + if (index < items.size && items[index].first > level) { + listOf(ListBlock(ordered = false, items = build(items[index].first))) + } else { + emptyList() + } result += ListItem(listOf(Paragraph(content)) + children) } // A deeper first item without a parent: promote it to this level. diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt index 9c80056..2542a64 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/EpubConverter.kt @@ -8,20 +8,21 @@ import io.github.lemcoder.mikromarkdown.model.Paragraph import io.github.lemcoder.mikromarkdown.model.Strong import io.github.lemcoder.mikromarkdown.model.Text import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument -import org.w3c.dom.Element -import org.xml.sax.InputSource import java.io.StringReader import java.util.zip.ZipInputStream import javax.xml.parsers.DocumentBuilderFactory +import org.w3c.dom.Element +import org.xml.sax.InputSource class EpubConverter : DocumentConverter { - private val metaFields = listOf( - "title" to "Title", - "creator" to "Authors", - "language" to "Language", - "description" to "Description", - "identifier" to "Identifier", - ) + private val metaFields = + listOf( + "title" to "Title", + "creator" to "Authors", + "language" to "Language", + "description" to "Description", + "identifier" to "Identifier", + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "epub" || info.mimetype == "application/epub+zip" @@ -100,8 +101,7 @@ class EpubConverter : DocumentConverter { val id = item.getAttribute("id") val href = item.getAttribute("href") val mediaType = item.getAttribute("media-type") - if (id.isNotEmpty() && href.isNotEmpty() && - (mediaType.contains("html") || mediaType.contains("xhtml"))) { + if (id.isNotEmpty() && href.isNotEmpty() && isReadableChapter(mediaType)) { manifest[id] = href } } @@ -117,12 +117,16 @@ class EpubConverter : DocumentConverter { return Triple(manifest, spine, metadata) } - private fun parseXml(bytes: ByteArray) = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = false - factory.isExpandEntityReferences = false - factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) - } catch (_: Exception) { - null - } + /** Only the spine's (X)HTML documents carry text; images and styles are skipped. */ + private fun isReadableChapter(mediaType: String): Boolean = mediaType.contains("html") + + private fun parseXml(bytes: ByteArray) = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = false + factory.isExpandEntityReferences = false + factory.newDocumentBuilder().parse(InputSource(StringReader(bytes.toString(Charsets.UTF_8)))) + } catch (_: Exception) { + null + } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt index fb5dffd..b716b5d 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/HtmlConverter.kt @@ -7,8 +7,7 @@ import io.github.lemcoder.mikromarkdown.utils.HtmlToDocument class HtmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { - return info.extension in setOf("html", "htm") || - info.mimetype in setOf("text/html", "application/xhtml+xml") + return info.extension in setOf("html", "htm") || info.mimetype in setOf("text/html", "application/xhtml+xml") } override fun parse(bytes: ByteArray, info: StreamInfo): Document = diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt index 026c9d1..9b4dbf4 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/JsonConverter.kt @@ -11,16 +11,21 @@ import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document class JsonConverter : DocumentConverter { - private val writer = ObjectMapper().apply { - registerKotlinModule() - }.writer(object : DefaultPrettyPrinter() { - init { - indentArraysWith(DefaultIndenter(" ", "\n")) - indentObjectsWith(DefaultIndenter(" ", "\n")) - } - override fun createInstance() = this - override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") - }) + private val writer = + ObjectMapper() + .apply { registerKotlinModule() } + .writer( + object : DefaultPrettyPrinter() { + init { + indentArraysWith(DefaultIndenter(" ", "\n")) + indentObjectsWith(DefaultIndenter(" ", "\n")) + } + + override fun createInstance() = this + + override fun writeObjectFieldValueSeparator(g: JsonGenerator) = g.writeRaw(": ") + } + ) override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "json" || info.mimetype in setOf("application/json", "text/json") @@ -28,11 +33,12 @@ class JsonConverter : DocumentConverter { override fun parse(bytes: ByteArray, info: StreamInfo): Document { val json = bytes.toString(Charsets.UTF_8) - val pretty = try { - writer.writeValueAsString(ObjectMapper().readTree(json)) - } catch (_: Exception) { - json - } + val pretty = + try { + writer.writeValueAsString(ObjectMapper().readTree(json)) + } catch (_: Exception) { + json + } return Document(blocks = listOf(CodeBlock(pretty, "json"))) } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt index 816bcf4..94543a8 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PdfConverter.kt @@ -18,11 +18,12 @@ class PdfConverter : DocumentConverter { val title = doc.documentInformation?.title?.trim()?.ifBlank { null } // Paragraph markers let the text blocks split on real paragraph breaks // instead of collapsing a page into one block. - val stripper = PDFTextStripper().apply { - sortByPosition = true - setAddMoreFormatting(true) - paragraphStart = "\n" - } + val stripper = + PDFTextStripper().apply { + sortByPosition = true + setAddMoreFormatting(true) + paragraphStart = "\n" + } return Document(blocks = plainTextBlocks(stripper.getText(doc)), title = title) } finally { doc.close() diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt index 4c363e5..78578a7 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/PptxConverter.kt @@ -23,15 +23,20 @@ import org.apache.poi.xslf.usermodel.XSLFPictureShape import org.apache.poi.xslf.usermodel.XSLFSimpleShape import org.apache.poi.xslf.usermodel.XSLFTable import org.apache.poi.xslf.usermodel.XSLFTextShape +import org.openxmlformats.schemas.drawingml.x2006.chart.CTAreaSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTAxDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTBarSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTLineSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTNumDataSource +import org.openxmlformats.schemas.drawingml.x2006.chart.CTPieSer +import org.openxmlformats.schemas.drawingml.x2006.chart.CTScatterSer import org.openxmlformats.schemas.drawingml.x2006.chart.CTSerTx import org.openxmlformats.schemas.presentationml.x2006.main.CTPicture class PptxConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "pptx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" + info.mimetype == "application/vnd.openxmlformats-officedocument.presentationml.presentation" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -44,12 +49,14 @@ class PptxConverter : DocumentConverter { blocks += HtmlComment("Slide number: ${index + 1}") blocks += shapeBlocks(slide.shapes) { if (index == 0 && title == null) title = it } - val notes = slide.notes?.shapes - ?.filterIsInstance() - ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } - ?.joinToString("\n") { it.text } - ?.trim() - .orEmpty() + val notes = + slide.notes + ?.shapes + ?.filterIsInstance() + ?.filter { (it as? XSLFSimpleShape)?.placeholder != Placeholder.SLIDE_IMAGE } + ?.joinToString("\n") { it.text } + ?.trim() + .orEmpty() if (notes.isNotBlank()) { blocks += Heading(3, listOf(Text("Notes:"))) blocks += notes.lines().filter { it.isNotBlank() }.map { Paragraph(listOf(Text(it.trim()))) } @@ -118,49 +125,83 @@ class PptxConverter : DocumentConverter { val blocks = mutableListOf() blocks += Heading(3, listOf(Text(listOfNotNull("Chart", chartTitle(chart)).joinToString(": ")))) - val series = try { - seriesOf(chart) - } catch (_: Exception) { - blocks += Paragraph(listOf(Text("[unsupported chart]"))) - return blocks - } + val series = + try { + seriesOf(chart) + } catch (_: Exception) { + blocks += Paragraph(listOf(Text("[unsupported chart]"))) + return blocks + } if (series.isEmpty()) return blocks val rowCount = series.maxOf { it.categories.size } - blocks += Table( - header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, - rows = (0 until rowCount).map { row -> - val category = series.first().categories.getOrElse(row) { "" } - (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } - }, - ) + blocks += + Table( + header = (listOf("Category") + series.map { it.name }).map { TableCell(it) }, + rows = + (0 until rowCount).map { row -> + val category = series.first().categories.getOrElse(row) { "" } + (listOf(category) + series.map { it.values.getOrElse(row) { "" } }).map { TableCell(it) } + }, + ) return blocks } - private fun chartTitle(chart: XSLFChart): String? = try { - val ctChart = chart.ctChart - if (!ctChart.isSetTitle) { + private fun chartTitle(chart: XSLFChart): String? = + try { + val ctChart = chart.ctChart + if (!ctChart.isSetTitle) { + null + } else { + val tx = ctChart.title?.tx + when { + tx?.isSetRich == true -> + tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") + tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v + else -> null + }?.ifBlank { null } + } + } catch (_: Exception) { null - } else { - val tx = ctChart.title?.tx - when { - tx?.isSetRich == true -> - tx.rich.pList.flatMap { p -> p.rList.map { r -> r.t.orEmpty() } }.joinToString("") - tx?.isSetStrRef == true -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v - else -> null - }?.ifBlank { null } } - } catch (_: Exception) { - null - } private data class Series(val name: String, val categories: List, val values: List) private fun seriesOf(chart: XSLFChart): List { - val plotArea = chart.ctChart.plotArea - val out = mutableListOf() + val plot = chart.ctChart.plotArea + // bar/bar3D, line/line3D and area/area3D each share one generated series type. + val all = + plot.barChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.bar3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.lineChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.line3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.areaChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.area3DChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.scatterChartList.flatMap { it.serList.map { s -> s.toSeries() } } + + plot.pieChartList.flatMap { it.serList.map { s -> s.toSeries() } } + return all.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } + } + + private fun CTBarSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTLineSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTAreaSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) + + private fun CTPieSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetCat) cat else null, if (isSetVal) `val` else null) - fun categoryValues(cat: CTAxDataSource?): List = when { + private fun CTScatterSer.toSeries() = + series(if (isSetTx) tx else null, if (isSetXVal) xVal else null, if (isSetYVal) yVal else null) + + private fun series(tx: CTSerTx?, categories: CTAxDataSource?, values: CTNumDataSource?) = + Series(seriesName(tx), categoryValues(categories), numericValues(values)) + + private fun categoryValues(cat: CTAxDataSource?): List = + when { cat == null -> emptyList() cat.isSetStrRef -> cat.strRef?.strCache?.ptList?.sortedBy { it.idx }?.map { it.v } ?: emptyList() cat.isSetNumRef -> cat.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() @@ -169,64 +210,22 @@ class PptxConverter : DocumentConverter { else -> emptyList() } - fun numericValues(v: CTNumDataSource?): List = when { + private fun numericValues(v: CTNumDataSource?): List = + when { v == null -> emptyList() v.isSetNumRef -> v.numRef?.numCache?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() v.isSetNumLit -> v.numLit?.ptList?.sortedBy { it.idx }?.map { it.v.orEmpty() } ?: emptyList() else -> emptyList() } - fun seriesName(tx: CTSerTx?): String = when { + private fun seriesName(tx: CTSerTx?): String = + when { tx == null -> "" tx.isSetV -> tx.v tx.isSetStrRef -> tx.strRef?.strCache?.ptList?.firstOrNull()?.v.orEmpty() else -> "" } - for (c in plotArea.barChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.bar3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.lineChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.line3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.areaChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.area3DChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - for (c in plotArea.scatterChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetXVal) s.xVal else null), - numericValues(if (s.isSetYVal) s.yVal else null), - ) - for (c in plotArea.pieChartList) for (s in c.serList) out += Series( - seriesName(if (s.isSetTx) s.tx else null), - categoryValues(if (s.isSetCat) s.cat else null), - numericValues(if (s.isSetVal) s.`val` else null), - ) - - return out.filter { it.categories.isNotEmpty() || it.values.isNotEmpty() } - } - private fun table(table: XSLFTable): Table? { val rows = table.rows if (rows.isEmpty()) return null diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt index 984dec0..e601f5c 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XlsxConverter.kt @@ -8,18 +8,18 @@ import io.github.lemcoder.mikromarkdown.model.Heading import io.github.lemcoder.mikromarkdown.model.Table import io.github.lemcoder.mikromarkdown.model.TableCell import io.github.lemcoder.mikromarkdown.model.Text +import kotlin.math.floor import org.apache.poi.ss.usermodel.Cell import org.apache.poi.ss.usermodel.CellType import org.apache.poi.ss.usermodel.DataFormatter import org.apache.poi.xssf.usermodel.XSSFWorkbook -import kotlin.math.floor class XlsxConverter : DocumentConverter { private val formatter = DataFormatter() override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { return info.extension == "xlsx" || - info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + info.mimetype == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" } override fun parse(bytes: ByteArray, info: StreamInfo): Document { @@ -35,12 +35,12 @@ class XlsxConverter : DocumentConverter { if (columns == 0) continue blocks += Heading(2, listOf(Text(sheet.sheetName))) - blocks += Table( - header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, - rows = rows.drop(1).map { row -> - (0 until columns).map { TableCell(cellValue(row.getCell(it))) } - }, - ) + blocks += + Table( + header = (0 until columns).map { TableCell(cellValue(rows[0].getCell(it))) }, + rows = + rows.drop(1).map { row -> (0 until columns).map { TableCell(cellValue(row.getCell(it))) } }, + ) } return Document(blocks = blocks) @@ -54,8 +54,7 @@ class XlsxConverter : DocumentConverter { return when (cell.cellType) { CellType.NUMERIC -> { val v = cell.numericCellValue - if (v == floor(v) && !v.isInfinite()) v.toLong().toString() - else formatter.formatCellValue(cell) + if (v == floor(v) && !v.isInfinite()) v.toLong().toString() else formatter.formatCellValue(cell) } CellType.BLANK -> "" else -> formatter.formatCellValue(cell).trim() diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt index 8979d57..7d3e6ac 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/converters/XmlConverter.kt @@ -4,7 +4,6 @@ import io.github.lemcoder.mikromarkdown.DocumentConverter import io.github.lemcoder.mikromarkdown.StreamInfo import io.github.lemcoder.mikromarkdown.model.CodeBlock import io.github.lemcoder.mikromarkdown.model.Document -import org.xml.sax.InputSource import java.io.StringReader import java.io.StringWriter import javax.xml.parsers.DocumentBuilderFactory @@ -12,6 +11,7 @@ import javax.xml.transform.OutputKeys import javax.xml.transform.TransformerFactory import javax.xml.transform.dom.DOMSource import javax.xml.transform.stream.StreamResult +import org.xml.sax.InputSource class XmlConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean { @@ -23,23 +23,24 @@ class XmlConverter : DocumentConverter { return Document(blocks = listOf(CodeBlock(pretty, "xml"))) } - private fun prettyPrint(xml: String): String = try { - val factory = DocumentBuilderFactory.newInstance() - factory.isNamespaceAware = true - factory.isIgnoringElementContentWhitespace = true - val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) + private fun prettyPrint(xml: String): String = + try { + val factory = DocumentBuilderFactory.newInstance() + factory.isNamespaceAware = true + factory.isIgnoringElementContentWhitespace = true + val doc = factory.newDocumentBuilder().parse(InputSource(StringReader(xml))) - val tf = TransformerFactory.newInstance() - tf.setAttribute("indent-number", 2) - val transformer = tf.newTransformer() - transformer.setOutputProperty(OutputKeys.INDENT, "yes") - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") + val tf = TransformerFactory.newInstance() + tf.setAttribute("indent-number", 2) + val transformer = tf.newTransformer() + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes") + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") - val writer = StringWriter() - transformer.transform(DOMSource(doc), StreamResult(writer)) - writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") - } catch (_: Exception) { - xml - } + val writer = StringWriter() + transformer.transform(DOMSource(doc), StreamResult(writer)) + writer.toString().trim().lines().filter { it.isNotBlank() }.joinToString("\n") + } catch (_: Exception) { + xml + } } diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt index d7c9296..510e036 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/HtmlToDocument.kt @@ -29,22 +29,49 @@ import org.jsoup.nodes.TextNode /** * Walks an HTML DOM into the shared document model. * - * Replaces the previous HTML → Markdown string conversion: tables, lists and inline - * emphasis become model nodes, so the Markdown renderer owns all syntax decisions. + * Replaces the previous HTML → Markdown string conversion: tables, lists and inline emphasis become model nodes, so the + * Markdown renderer owns all syntax decisions. */ object HtmlToDocument { - private val DROPPED_TAGS = setOf( - "script", "style", "noscript", "template", "button", "svg", "iframe", "form", "input", "select", - ) + private val DROPPED_TAGS = + setOf( + "script", + "style", + "noscript", + "template", + "button", + "svg", + "iframe", + "form", + "input", + "select", + ) private val HEADINGS = mapOf("h1" to 1, "h2" to 2, "h3" to 3, "h4" to 4, "h5" to 5, "h6" to 6) /** Tags that only group other content; their children are lifted into the parent block flow. */ - private val CONTAINERS = setOf( - "div", "section", "article", "main", "header", "footer", "aside", "nav", "body", "html", - "figure", "details", "summary", "fieldset", "center", "hgroup", "picture", "colgroup", - ) + private val CONTAINERS = + setOf( + "div", + "section", + "article", + "main", + "header", + "footer", + "aside", + "nav", + "body", + "html", + "figure", + "details", + "summary", + "fieldset", + "center", + "hgroup", + "picture", + "colgroup", + ) fun parse(html: String, baseUri: String = ""): Document { val doc = Jsoup.parse(html, baseUri) @@ -91,49 +118,66 @@ object HtmlToDocument { return out } - private fun Element.isBlockLevel(): Boolean = when (tagName()) { - in HEADINGS, "p", "ul", "ol", "table", "pre", "blockquote", "hr", "dl", "li", "figcaption" -> true - in CONTAINERS -> true - else -> false - } - - private fun blockFor(element: Element): List = when (val tag = element.tagName()) { - in HEADINGS -> listOfNotNull( - inlines(element).trimEdges() - .takeIf { it.isNotEmpty() } - ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) }, - ) + private fun Element.isBlockLevel(): Boolean = + when (tagName()) { + in HEADINGS, + "p", + "ul", + "ol", + "table", + "pre", + "blockquote", + "hr", + "dl", + "li", + "figcaption" -> true + in CONTAINERS -> true + else -> false + } - "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) - "ul", "ol" -> listBlock(element) - "table" -> table(element) - "pre" -> listOf(codeBlock(element)) - "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() - "hr" -> listOf(ThematicBreak) - "dl" -> definitionList(element) - "figcaption" -> listOfNotNull( - inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) }, - ) - // Containers and stray
  • outside a list contribute their children directly. - else -> blocks(element) - } + private fun blockFor(element: Element): List = + when (val tag = element.tagName()) { + in HEADINGS -> + listOfNotNull( + inlines(element) + .trimEdges() + .takeIf { it.isNotEmpty() } + ?.let { Heading(HEADINGS.getValue(tag), it, element.id().ifBlank { null }) } + ) + + "p" -> listOfNotNull(inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(it) }) + "ul", + "ol" -> listBlock(element) + "table" -> table(element) + "pre" -> listOf(codeBlock(element)) + "blockquote" -> blocks(element).takeIf { it.isNotEmpty() }?.let { listOf(BlockQuote(it)) } ?: emptyList() + "hr" -> listOf(ThematicBreak) + "dl" -> definitionList(element) + "figcaption" -> + listOfNotNull( + inlines(element).trimEdges().takeIf { it.isNotEmpty() }?.let { Paragraph(listOf(Emphasis(it))) } + ) + // Containers and stray
  • outside a list contribute their children directly. + else -> blocks(element) + } private fun listBlock(element: Element): List { val ordered = element.tagName() == "ol" val start = element.attr("start").toIntOrNull() ?: 1 - val items = element.children() - .filter { it.tagName() == "li" } - .map { li -> ListItem(blocks = blocks(li)) } - .filter { it.blocks.isNotEmpty() } + val items = + element + .children() + .filter { it.tagName() == "li" } + .map { li -> ListItem(blocks = blocks(li)) } + .filter { it.blocks.isNotEmpty() } if (items.isEmpty()) return emptyList() return listOf(ListBlock(ordered = ordered, items = items, start = start)) } private fun codeBlock(element: Element): Block { val code = element.selectFirst("code") ?: element - val language = code.classNames() - .firstOrNull { it.startsWith("language-") || it.startsWith("lang-") } - ?.substringAfter('-') + val language = + code.classNames().firstOrNull { it.startsWith("language-") || it.startsWith("lang-") }?.substringAfter('-') return CodeBlock(code.wholeText().trimEnd(), language) } @@ -158,20 +202,22 @@ object HtmlToDocument { val body = mutableListOf>() for ((index, tr) in rows.withIndex()) { - val cells = tr.children() - .filter { it.tagName() == "th" || it.tagName() == "td" } - .map { cell -> - TableCell( - content = inlines(cell).trimEdges(), - colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, - ) - } + val cells = + tr.children() + .filter { it.tagName() == "th" || it.tagName() == "td" } + .map { cell -> + TableCell( + content = inlines(cell).trimEdges(), + colSpan = cell.attr("colspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + rowSpan = cell.attr("rowspan").toIntOrNull()?.coerceIn(1, 100) ?: 1, + ) + } if (cells.isEmpty()) continue - val isHeaderRow = index == 0 && - tr.children().all { it.tagName() == "th" } && - (tr.parent()?.tagName() == "thead" || header.isEmpty()) + val isHeaderRow = + index == 0 && + tr.children().all { it.tagName() == "th" } && + (tr.parent()?.tagName() == "thead" || header.isEmpty()) if (isHeaderRow && header.isEmpty()) header = cells else body += cells } @@ -196,35 +242,45 @@ object HtmlToDocument { if (text.isNotEmpty()) add(Text(text)) } - is Element -> when (node.tagName()) { - "br" -> add(LineBreak) - "img" -> { - val src = node.attr("abs:src").ifBlank { node.attr("src") } - val alt = node.attr("alt") - if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) - } + is Element -> + when (node.tagName()) { + "br" -> add(LineBreak) + "img" -> { + val src = node.attr("abs:src").ifBlank { node.attr("src") } + val alt = node.attr("alt") + if (src.isNotBlank()) add(Image(alt, src, node.attr("title").ifBlank { null })) + } - "a" -> { - val href = node.attr("abs:href").ifBlank { node.attr("href") } - val content = inlines(node).trimEdges() - when { - content.isEmpty() -> Unit - href.isBlank() || href.startsWith("javascript:") -> addAll(content) - else -> add(Link(content, href, node.attr("title").ifBlank { null })) + "a" -> { + val href = node.attr("abs:href").ifBlank { node.attr("href") } + val content = inlines(node).trimEdges() + when { + content.isEmpty() -> Unit + href.isBlank() || href.startsWith("javascript:") -> addAll(content) + else -> add(Link(content, href, node.attr("title").ifBlank { null })) + } } - } - "strong", "b" -> wrapped(node) { Strong(it) } - "em", "i", "cite", "var" -> wrapped(node) { Emphasis(it) } - "del", "s", "strike" -> wrapped(node) { Strikethrough(it) } - "code", "kbd", "samp", "tt" -> { - val code = node.wholeText().trim() - if (code.isNotEmpty()) add(CodeSpan(code)) - } + "strong", + "b" -> wrapped(node) { Strong(it) } + "em", + "i", + "cite", + "var" -> wrapped(node) { Emphasis(it) } + "del", + "s", + "strike" -> wrapped(node) { Strikethrough(it) } + "code", + "kbd", + "samp", + "tt" -> { + val code = node.wholeText().trim() + if (code.isNotEmpty()) add(CodeSpan(code)) + } - // Block-level content encountered inline (e.g. a
    inside a ): keep its text. - else -> addAll(inlines(node)) - } + // Block-level content encountered inline (e.g. a
    inside a ): keep its text. + else -> addAll(inlines(node)) + } } } @@ -239,14 +295,11 @@ object HtmlToDocument { /** HTML collapses runs of whitespace; do the same before the text reaches the model. */ // Non-breaking spaces are not collapsible whitespace in HTML, so they survive verbatim. - private fun TextNode.normalizedText(): String = - wholeText.replace(Regex("\\s+"), " ") + private fun TextNode.normalizedText(): String = wholeText.replace(Regex("\\s+"), " ") - private fun List.startsWithSpace(): Boolean = - (firstOrNull() as? Text)?.value?.startsWith(" ") == true + private fun List.startsWithSpace(): Boolean = (firstOrNull() as? Text)?.value?.startsWith(" ") == true - private fun List.endsWithSpace(): Boolean = - (lastOrNull() as? Text)?.value?.endsWith(" ") == true + private fun List.endsWithSpace(): Boolean = (lastOrNull() as? Text)?.value?.endsWith(" ") == true /** Drops leading/trailing whitespace-only text so emphasis markers hug their content. */ private fun List.trimEdges(): List { diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt index 40cdc01..f819614 100644 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt +++ b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/utils/TikaMimeDetector.kt @@ -2,15 +2,20 @@ package io.github.lemcoder.mikromarkdown.utils import io.github.lemcoder.mikromarkdown.MimeDetector import io.github.lemcoder.mikromarkdown.StreamInfo -import org.apache.tika.Tika import java.io.File +import org.apache.tika.Tika object TikaMimeDetector : MimeDetector { private val tika = Tika() override fun detect(path: String): StreamInfo { val file = File(path) - val mimetype = try { tika.detect(file) } catch (_: Exception) { null } + val mimetype = + try { + tika.detect(file) + } catch (_: Exception) { + null + } val extension = file.extension.lowercase().ifEmpty { null } return StreamInfo( mimetype = mimetype, diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt new file mode 100644 index 0000000..a604680 --- /dev/null +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt @@ -0,0 +1,63 @@ +package io.github.lemcoder.mikromarkdown + +import com.lemonappdev.konsist.api.Konsist +import com.lemonappdev.konsist.api.architecture.KoArchitectureCreator.assertArchitecture +import com.lemonappdev.konsist.api.architecture.Layer +import com.lemonappdev.konsist.api.verify.assertFalse +import com.lemonappdev.konsist.api.verify.assertTrue +import kotlin.test.Test + +/** + * Guards the pipeline's boundaries: converters parse into the model, the renderer serializes it, and neither reaches + * across. Without these, Markdown syntax leaks back into converters — the exact drift the document model was introduced + * to end. + */ +class ArchitectureTest { + + private val production = Konsist.scopeFromProject(sourceSetName = null).files.filterNot { it.path.contains("Test") } + + @Test + fun `layers depend in one direction only`() { + Konsist.scopeFromProject().assertArchitecture { + val model = Layer("Model", "io.github.lemcoder.mikromarkdown.model..") + val render = Layer("Render", "io.github.lemcoder.mikromarkdown.render..") + val converters = Layer("Converters", "io.github.lemcoder.mikromarkdown.converters..") + + // The model is the shared vocabulary: it may not know about its producers or consumers. + model.dependsOnNothing() + render.dependsOn(model) + converters.dependsOn(model) + } + } + + @Test + fun `converters do not reach into the renderer`() { + production + .filter { it.packagee?.name?.contains(".converters") == true } + .assertFalse { file -> file.hasImport { it.name.contains(".render.") } } + } + + @Test + fun `markdown syntax only lives in the renderer`() { + val markdownLiterals = Regex("""\"(\|[^"]*\||#{1,6} |\*\*|!\[|```)""") + + production + .filterNot { it.packagee?.name?.contains(".render") == true } + .assertFalse { file -> markdownLiterals.containsMatchIn(file.text) } + } + + @Test + fun `converters implement DocumentConverter and are named accordingly`() { + Konsist.scopeFromProject() + .classes() + .filter { it.resideInPackage("..converters..") && it.name.endsWith("Converter") } + .assertTrue { it.hasParentInterface { parent -> parent.name == "DocumentConverter" } } + } + + @Test + fun `the model stays free of platform dependencies`() { + production + .filter { it.packagee?.name?.contains(".model") == true } + .assertFalse { file -> file.hasImport { it.name.startsWith("java.") || it.name.startsWith("android.") } } + } +} diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt index 951583d..84c9627 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/DumpOutputTest.kt @@ -1,14 +1,23 @@ package io.github.lemcoder.mikromarkdown -import org.junit.jupiter.api.Test import java.io.File +import org.junit.jupiter.api.Test class DumpOutputTest { - private val mid = MarkItDown() + private val mid = MikroMarkdown() @Test fun dumpAll() { - for (name in listOf("test.docx","test.xlsx","test.pptx","test.epub","test.json","test_blog.html","test_wikipedia.html")) { + for (name in + listOf( + "test.docx", + "test.xlsx", + "test.pptx", + "test.epub", + "test.json", + "test_blog.html", + "test_wikipedia.html", + )) { val url = javaClass.classLoader.getResource("test_files/$name") ?: continue val output = mid.convert(File(url.toURI()).absolutePath).markdown File("/tmp/kt_$name.md").writeText(output) diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt index 4befd5c..7c1314e 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt @@ -1,3 +1,3 @@ package io.github.lemcoder.mikromarkdown -actual fun testMarkItDown(): MikroMarkdown = MarkItDown() \ No newline at end of file +actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt index ce1faa1..b744e55 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/PythonComparisonTest.kt @@ -1,38 +1,32 @@ package io.github.lemcoder.mikromarkdown -import org.junit.jupiter.api.Assumptions.assumeTrue -import org.junit.jupiter.api.BeforeAll -import org.junit.jupiter.api.Test import java.io.File import java.util.concurrent.TimeUnit import kotlin.test.assertTrue +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test class PythonComparisonTest { - @Test - fun testDocx() = compare("test.docx") + @Test fun testDocx() = compare("test.docx") - @Test - fun testXlsx() = compare("test.xlsx") + @Test fun testXlsx() = compare("test.xlsx") - @Test - fun testPptx() = compare("test.pptx") + @Test fun testPptx() = compare("test.pptx") - @Test - fun testEpub() = compare("test.epub") + @Test fun testEpub() = compare("test.epub") - @Test - fun testJson() = compare("test.json") + @Test fun testJson() = compare("test.json") - @Test - fun testBlogHtml() = compare("test_blog.html") + @Test fun testBlogHtml() = compare("test_blog.html") - @Test - fun testWikipediaHtml() = compare("test_wikipedia.html") + @Test fun testWikipediaHtml() = compare("test_wikipedia.html") private fun compare(filename: String) { - val url = javaClass.classLoader.getResource("test_files/$filename") - ?: error("Resource not found: test_files/$filename") + val url = + javaClass.classLoader.getResource("test_files/$filename") + ?: error("Resource not found: test_files/$filename") val file = File(url.toURI()) val pythonOutput = runMarkitdown(file.absolutePath) @@ -49,9 +43,9 @@ class PythonComparisonTest { // Normalize unicode characters to ASCII equivalents so both converters compare fairly private fun normalize(text: String): String = text - .replace("–", "-") // en dash - .replace("—", "-") // em dash - .replace(" ", " ") // non-breaking space + .replace("–", "-") // en dash + .replace("—", "-") // em dash + .replace(" ", " ") // non-breaking space .replace(Regex("%([0-9A-Fa-f]{2})")) { mr -> val cp = mr.groupValues[1].toInt(16) if (cp in 0x20..0x7E) cp.toChar().toString() else mr.value @@ -67,10 +61,7 @@ class PythonComparisonTest { // Tokens longer than 3 chars, lowercased, deduped private fun tokenize(text: String): Set = - normalize(text).lowercase() - .split(Regex("[\\s\\p{Punct}]+")) - .filter { it.length > 3 } - .toSet() + normalize(text).lowercase().split(Regex("[\\s\\p{Punct}]+")).filter { it.length > 3 }.toSet() private fun tokenSimilarity(reference: String, actual: String): Double { val refTokens = tokenize(reference) @@ -95,24 +86,26 @@ class PythonComparisonTest { markitdownCmd.isNotEmpty(), "markitdown CLI not available — install via `pip install markitdown` or `uv tool install markitdown`", ) - mid = MarkItDown() + mid = MikroMarkdown() } private fun resolveMarkitdownCmd(): List { - val candidates = listOf( - listOf("markitdown"), - listOf("python", "-m", "markitdown"), - listOf("python3", "-m", "markitdown"), - listOf("uvx", "markitdown[all]"), - listOf("uvx", "markitdown"), - ) + val candidates = + listOf( + listOf("markitdown"), + listOf("python", "-m", "markitdown"), + listOf("python3", "-m", "markitdown"), + listOf("uvx", "markitdown[all]"), + listOf("uvx", "markitdown"), + ) return candidates.firstOrNull { cmd -> try { - val process = ProcessBuilder(cmd + "--help") - .redirectErrorStream(true) - .start() + val process = ProcessBuilder(cmd + "--help").redirectErrorStream(true).start() val finished = process.waitFor(10, TimeUnit.SECONDS) - if (!finished) { process.destroyForcibly(); return@firstOrNull false } + if (!finished) { + process.destroyForcibly() + return@firstOrNull false + } process.exitValue() == 0 } catch (_: Exception) { false @@ -121,12 +114,14 @@ class PythonComparisonTest { } fun runMarkitdown(path: String): String { - val process = ProcessBuilder(markitdownCmd + path) - .start() + val process = ProcessBuilder(markitdownCmd + path).start() val stdout = process.inputStream.bufferedReader().readText() val finished = process.waitFor(30, TimeUnit.SECONDS) assumeTrue(finished, "markitdown timed out on $path") - assumeTrue(process.exitValue() == 0, "markitdown failed on $path: ${process.errorStream.bufferedReader().readText()}") + assumeTrue( + process.exitValue() == 0, + "markitdown failed on $path: ${process.errorStream.bufferedReader().readText()}", + ) return stdout } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 5fc871d..c3b33ad 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,5 +14,7 @@ dependencyResolutionManagement { } rootProject.name = "mikromarkdown" + include(":library") + include(":cli")