From d0843c72723e1ede6e035b0636fff59a0ee638b7 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Wed, 19 Aug 2026 13:56:48 +0200 Subject: [PATCH 1/9] Delete what the earlier passes made redundant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing here changes output. Every fixture, PDF included, renders byte for byte what it rendered before. The builder DSL had no users. It was written for the office converters, which built documents statement by statement; the converters that remain construct Document directly or come out of HtmlToDocument, and the only mention left was the README. A Heading(level, text) and Paragraph(text) constructor cover what the DSL was pleasant for, in four lines rather than a hundred and sixty of public API. There were three copies of the factory — JVM, Android, macOS — registering the same seven converters, which is what a converter list looks like once every converter is common. One copy in commonMain. The File overloads they carried went with them: nothing called either, not even the JVM tests, which had always used convert(file.absolutePath). TikaMimeDetector was unreferenced and untested, and it put tika-core on the classpath of every JVM consumer to provide an implementation of a fun interface whose whole purpose is that a caller can supply their own. With it gone the library has no third-party JVM dependency at all, and no jvmMain source set — production code is entirely common. The de-hyphenation heuristic existed twice. PdfiumConverter decided whether U+FFFE was the author's hyphen or the typesetter's, and plainTextBlocks decided the same thing for a hyphen at a line break, each with its own vocabulary and its own copy of the word regex. Since the unmapped glyph is a hyphen at a line break, writing it as one leaves a single rule. That also fixed the copy that was wrong: plainTextBlocks built its vocabulary from the whole text, so a fragment appearing only at the break vouched for itself and kept a hyphen it should have dropped. Only pdfium calls it, and pdfium's own copy did exclude the halves, so no output moves. Also gone: HtmlComment, which nothing produced once the builder went and which RawBlock already covers; docx/xlsx/pptx and OLE in the signature detector, formats we deliberately dropped; four library entries and two plugins in the version catalog left behind by POI, Jackson, commons-csv and kotlinx-serialization; and .codegraph, which is no longer used. Two things the cleanup turned up rather than removed. Konsist's rule against a production file existing in two source sets had an allowlist naming MikroMarkdownFactory and PdfConverter — the second has not existed since the office removal. The allowlist is now empty and the rule absolute. And macosArm64's test compilation had been switched off because the shared integration tests expected JVM-only formats; they have not since the converters moved, so the native target runs the same five tests as the JVM. Co-Authored-By: Claude Opus 5 --- .codegraph/.gitignore | 16 -- .codegraph/config.json | 143 ---------------- README.md | 36 ++-- .../lemcoder/mikromarkdown/benchmark/Main.kt | 2 +- .../github/lemcoder/mikromarkdown/cli/Main.kt | 3 +- gradle/libs.versions.toml | 15 -- library/build.gradle.kts | 13 +- .../mikromarkdown/MikroMarkdownFactory.kt | 29 ---- .../mikromarkdown/MikroMarkdownFactory.kt | 7 +- .../mikromarkdown/SignatureMimeDetector.kt | 23 +-- .../lemcoder/mikromarkdown/model/Document.kt | 13 +- .../mikromarkdown/model/DocumentBuilder.kt | 160 ------------------ .../mikromarkdown/model/TextBlocks.kt | 31 +++- .../mikromarkdown/render/MarkdownRenderer.kt | 3 - .../mikromarkdown/MikroMarkdownFactory.kt | 29 ---- .../mikromarkdown/TikaMimeDetector.kt | 32 ---- .../mikromarkdown/ArchitectureTest.kt | 12 +- .../FileIntegrationTest.macos.kt | 3 + .../mikromarkdown/pdf/PdfiumConverter.kt | 47 +---- 19 files changed, 85 insertions(+), 532 deletions(-) delete mode 100644 .codegraph/.gitignore delete mode 100644 .codegraph/config.json delete mode 100644 library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt rename library/src/{macosMain => commonMain}/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt (75%) delete mode 100644 library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt delete mode 100644 library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt delete mode 100644 library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/TikaMimeDetector.kt create mode 100644 library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt diff --git a/.codegraph/.gitignore b/.codegraph/.gitignore deleted file mode 100644 index 9de0f16..0000000 --- a/.codegraph/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -# CodeGraph data files -# These are local to each machine and should not be committed - -# Database -*.db -*.db-wal -*.db-shm - -# Cache -cache/ - -# Logs -*.log - -# Hook markers -.dirty diff --git a/.codegraph/config.json b/.codegraph/config.json deleted file mode 100644 index 7af60ad..0000000 --- a/.codegraph/config.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "version": 1, - "include": [ - "**/*.ts", - "**/*.tsx", - "**/*.js", - "**/*.jsx", - "**/*.py", - "**/*.go", - "**/*.rs", - "**/*.java", - "**/*.c", - "**/*.h", - "**/*.cpp", - "**/*.hpp", - "**/*.cc", - "**/*.cxx", - "**/*.cs", - "**/*.php", - "**/*.rb", - "**/*.swift", - "**/*.kt", - "**/*.kts", - "**/*.dart", - "**/*.svelte", - "**/*.vue", - "**/*.liquid", - "**/*.pas", - "**/*.dpr", - "**/*.dpk", - "**/*.lpr", - "**/*.dfm", - "**/*.fmx", - "**/*.scala", - "**/*.sc" - ], - "exclude": [ - "**/.git/**", - "**/node_modules/**", - "**/vendor/**", - "**/Pods/**", - "**/dist/**", - "**/build/**", - "**/out/**", - "**/bin/**", - "**/obj/**", - "**/target/**", - "**/*.min.js", - "**/*.bundle.js", - "**/.next/**", - "**/.nuxt/**", - "**/.svelte-kit/**", - "**/.output/**", - "**/.turbo/**", - "**/.cache/**", - "**/.parcel-cache/**", - "**/.vite/**", - "**/.astro/**", - "**/.docusaurus/**", - "**/.gatsby/**", - "**/.webpack/**", - "**/.nx/**", - "**/.yarn/cache/**", - "**/.pnpm-store/**", - "**/storybook-static/**", - "**/.expo/**", - "**/web-build/**", - "**/ios/Pods/**", - "**/ios/build/**", - "**/android/build/**", - "**/android/.gradle/**", - "**/__pycache__/**", - "**/.venv/**", - "**/venv/**", - "**/site-packages/**", - "**/dist-packages/**", - "**/.pytest_cache/**", - "**/.mypy_cache/**", - "**/.ruff_cache/**", - "**/.tox/**", - "**/.nox/**", - "**/*.egg-info/**", - "**/.eggs/**", - "**/go/pkg/mod/**", - "**/target/debug/**", - "**/target/release/**", - "**/.gradle/**", - "**/.m2/**", - "**/generated-sources/**", - "**/.kotlin/**", - "**/.dart_tool/**", - "**/.vs/**", - "**/.nuget/**", - "**/artifacts/**", - "**/publish/**", - "**/cmake-build-*/**", - "**/CMakeFiles/**", - "**/bazel-*/**", - "**/vcpkg_installed/**", - "**/.conan/**", - "**/Debug/**", - "**/Release/**", - "**/x64/**", - "**/.pio/**", - "**/release/**", - "**/*.app/**", - "**/*.asar", - "**/DerivedData/**", - "**/.build/**", - "**/.swiftpm/**", - "**/xcuserdata/**", - "**/Carthage/Build/**", - "**/SourcePackages/**", - "**/__history/**", - "**/__recovery/**", - "**/*.dcu", - "**/.composer/**", - "**/storage/framework/**", - "**/bootstrap/cache/**", - "**/.bundle/**", - "**/tmp/cache/**", - "**/public/assets/**", - "**/public/packs/**", - "**/.yardoc/**", - "**/coverage/**", - "**/htmlcov/**", - "**/.nyc_output/**", - "**/test-results/**", - "**/.coverage/**", - "**/.idea/**", - "**/logs/**", - "**/tmp/**", - "**/temp/**", - "**/_build/**", - "**/docs/_build/**", - "**/site/**" - ], - "languages": [], - "frameworks": [], - "maxFileSize": 1048576, - "extractDocstrings": true, - "trackCallSites": true -} \ No newline at end of file diff --git a/README.md b/README.md index 9c7061a..56f0ab7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![Test](https://github.com/lemcoder/MikroMarkdown/actions/workflows/gradle.yml/badge.svg)](https://github.com/lemcoder/MikroMarkdown/actions/workflows/gradle.yml) -Kotlin Multiplatform (JVM + Android) library that converts documents to Markdown. Port of Microsoft's [MarkItDown](https://github.com/microsoft/markitdown). +Kotlin Multiplatform library that converts documents to Markdown, on the JVM, Android and Kotlin/Native. Port of +Microsoft's [MarkItDown](https://github.com/microsoft/markitdown). ## Supported formats @@ -37,8 +38,10 @@ bytes ──► MimeDetector ──► DocumentConverter.parse ──► Documen ``` Converters contain no Markdown syntax, so escaping, table shaping, list indentation and spacing are -fixed once for all formats. Every converter lives in `commonMain` and runs on every target; only PDF -is platform-specific, and it lives in its own module because it needs a native library. The model is +fixed once for all formats. Every converter lives in `commonMain` and runs on every target, so the +library has one registration list rather than one per platform and no third-party JVM dependency at +all; only PDF is platform-specific, and it lives in its own module because it needs a native +library. The model is public: `mid.parse(path)` returns the `Document`, and `ConversionResult.document` exposes it alongside the rendered Markdown. @@ -82,15 +85,25 @@ println(result.title) // nullable, extracted from document metadata ### Android +Identical — `MikroMarkdown()` is one common function, and every converter it registers is common +code. + ```kotlin import io.github.lemcoder.mikromarkdown.MikroMarkdown -// pass Context to enable PDF support -val mid = MikroMarkdown(context) +val mid = MikroMarkdown() val result = mid.convert(file.absolutePath) ``` +### PDF + +PDF needs pdfium, so it ships as `:pdfium` and the caller opts in: + +```kotlin +val mid = MikroMarkdown().apply { register(PdfiumConverter()) } +``` + ## Custom converters Implement `DocumentConverter` and register it: @@ -100,10 +113,8 @@ class MyConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean = info.extension == "xyz" - override fun parse(bytes: ByteArray, info: StreamInfo): Document = document { - heading(1, "Custom") - paragraph(bytes.decodeToString()) - } + override fun parse(bytes: ByteArray, info: StreamInfo): Document = + Document(blocks = listOf(Heading(1, "Custom"), Paragraph(bytes.decodeToString()))) } val mid = MikroMarkdown() @@ -115,7 +126,10 @@ Lower priority runs first. `PlainTextConverter` uses `10.0` so it acts as a fall ## Custom MIME detection -`MimeDetector` is a `fun interface` — pass a lambda or implement it: +`MimeDetector` is a `fun interface` — pass a lambda or implement it. The default, +`SignatureMimeDetector`, reads the leading bytes and falls back to the extension; content sniffing +for extension-less text formats is where a full MIME registry such as Apache Tika goes, as your +dependency rather than the library's: ```kotlin val mid = MikroMarkdown(MimeDetector { path -> @@ -155,7 +169,7 @@ import the renderer or each other; Markdown syntax appears only under `render/`. every `DocumentConverter` is named `*Converter` and lives in `converters`. *Hygiene* — no wildcard imports, no printing from library code, and no source file duplicated -between source sets. +between source sets — a rule that now has no exceptions, since every production file is common. ktfmt-gradle only derives tasks for the common and JVM source sets, so `library/build.gradle.kts` registers matching tasks for the Android ones. diff --git a/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt b/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt index a68c38c..7f0a265 100644 --- a/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt +++ b/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt @@ -10,7 +10,7 @@ import kotlin.system.measureNanoTime * * The CLI's wall clock is dominated by JVM startup and class loading, which says nothing about the pipeline itself. * This measures the stages separately on a warmed-up JVM, and separately reports the first conversion in a fresh JVM — - * the one that pays for loading POI, PDFBox and Tika. + * the one that pays for class loading. * * Usage: ./gradlew :benchmark:run --args="[fixtureDir] [warmup] [iterations]" */ diff --git a/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt b/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt index 665be14..377c70e 100644 --- a/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt +++ b/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt @@ -6,8 +6,7 @@ import io.github.lemcoder.mikromarkdown.pdf.PdfiumConverter import kotlin.system.exitProcess /** - * Minimal native entry point, kept deliberately bare so its timings measure conversion rather than an argument parser. - * The JVM CLI remains the full one. + * The command line tool, kept deliberately bare so its timings measure conversion rather than an argument parser. * * Several files may be given: a document boundary is the one point where everything the previous conversion allocated * is dead, which is what makes a manual collection policy possible at all. The default collector wins on measurement, diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3f394ec..6e90a5f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,19 +3,12 @@ agp = "9.2.1" kotlin = "2.3.21" android-minSdk = "26" android-compileSdk = "37" -vanniktechMavenPublish = "0.36.0" kotlinx-io = "0.9.0" -kotlinx-serialization = "1.9.0" ksoup = "0.2.6" korlibs = "6.0.0" konanplugin = "1.2.0-alpha06" kotlinx-resources = "0.15.0" - -commons-csv = "1.14.1" -jackson = "2.21.3" junit = "6.1.0" -tika = "3.3.0" -coreKtx = "1.7.0" detekt = "1.23.8" ktfmt = "0.27.0" konsist = "0.17.3" @@ -25,22 +18,14 @@ kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotl kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" } korlibs-compression = { module = "com.soywiz:korlibs-compression", version.ref = "korlibs" } -kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" } kotlinx-resources = { module = "com.goncalossilva:resources", version.ref = "kotlinx-resources" } - -commons-csv = { module = "org.apache.commons:commons-csv", version.ref = "commons-csv" } -jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" } junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" } -tika-core = { module = "org.apache.tika:tika-core", version.ref = "tika" } -core-ktx = { group = "androidx.test", name = "core-ktx", version.ref = "coreKtx" } konsist = { module = "com.lemonappdev:konsist", version.ref = "konsist" } [plugins] android-kotlin-multiplatform-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } kotlinMultiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } -kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } -vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version.ref = "vanniktechMavenPublish" } kotlinx-resources = { id = "com.goncalossilva.resources", version.ref = "kotlinx-resources" } detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } konanplugin = { id = "io.github.lemcoder.konanplugin", version.ref = "konanplugin" } diff --git a/library/build.gradle.kts b/library/build.gradle.kts index f750228..9cd46cb 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -21,10 +21,7 @@ kotlin { testRuns["test"].executionTask.configure { useJUnitPlatform() } } - // Spike: a native target to see how close a real binary gets to the Rust implementation. - // The shared integration tests expect JVM-only formats, so native test compilation stays off - // until the native target carries real converters. - macosArm64 { compilations.getByName("test") { compileTaskProvider.configure { enabled = false } } } + macosArm64() androidLibrary { namespace = "io.github.lemcoder.mikromarkdown" @@ -40,13 +37,11 @@ kotlin { sourceSets { commonMain.dependencies { implementation(libs.kotlinx.io.core) - // Phase 0 of the commonMain migration: HTML parsing and the inflate that ZIP needs. + // HTML parsing, and the inflate that EPUB's ZIP container needs. implementation(libs.ksoup) implementation(libs.korlibs.compression) } - jvmMain { dependencies { implementation(libs.tika.core) } } - commonTest.dependencies { implementation(libs.kotlin.test) implementation(libs.kotlinx.resources) @@ -60,8 +55,8 @@ kotlin { } } -// ktfmt-gradle only derives tasks for the common and JVM source sets, so the Android -// ones — where half the converters live — would go unformatted and unchecked. +// ktfmt-gradle only derives tasks for the common and JVM source sets, so the Android test +// source sets would go unformatted and unchecked. run { val androidSources = fileTree("src") { include("android*/**/*.kt") } val template = tasks.named("ktfmtFormatKmpCommonMain") diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt deleted file mode 100644 index 8463eae..0000000 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ /dev/null @@ -1,29 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -import io.github.lemcoder.mikromarkdown.converters.CsvConverter -import io.github.lemcoder.mikromarkdown.converters.EpubConverter -import io.github.lemcoder.mikromarkdown.converters.HtmlConverter -import io.github.lemcoder.mikromarkdown.converters.JsonConverter -import io.github.lemcoder.mikromarkdown.converters.MarkdownPassthroughConverter -import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter -import io.github.lemcoder.mikromarkdown.converters.XmlConverter -import java.io.File - -/** - * A [MikroMarkdown] with every Android converter registered. - * - * PDF is not among them: it needs a native library, so the `:pdfium` module provides it and the caller opts in with - * `register(PdfiumConverter())`. - */ -public fun MikroMarkdown(): MikroMarkdown = - MikroMarkdown(SignatureMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(EpubConverter()) - register(PlainTextConverter(), priority = 10.0) - } - -public fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/macosMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt similarity index 75% rename from library/src/macosMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt rename to library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 0d9432b..52f4aad 100644 --- a/library/src/macosMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -9,17 +9,18 @@ import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter import io.github.lemcoder.mikromarkdown.converters.XmlConverter /** - * A [MikroMarkdown] with the converters that need no platform library. + * A [MikroMarkdown] with every converter the library ships. * - * PDF is the only format still missing here; everything else the library converts is shared. + * One list for every target, because every converter is now common. PDF is the exception and is not here: it costs a + * native library, so the `:pdfium` module provides it and the caller opts in with `register(PdfiumConverter())`. */ public fun MikroMarkdown(): MikroMarkdown = MikroMarkdown(SignatureMimeDetector).apply { register(MarkdownPassthroughConverter()) register(HtmlConverter()) - register(EpubConverter()) register(CsvConverter()) register(JsonConverter()) register(XmlConverter()) + register(EpubConverter()) register(PlainTextConverter(), priority = 10.0) } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt index cf8b04d..19873bd 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt @@ -10,11 +10,11 @@ import kotlinx.io.readByteArray * * This is the default because it costs microseconds: it reads a handful of bytes and consults a fixed table, where a * full MIME registry (Tika) spends ~90 ms building itself on first use — more than the entire conversion for most - * documents. Content still wins over the extension, so a mislabelled `.txt` that is really a PDF or an OOXML package is + * documents. Content still wins over the extension, so a mislabelled `.txt` that is really a PDF or a ZIP package is * identified correctly. * - * Formats that are plain text with no signature (CSV, JSON, XML, HTML, Markdown) are recognised by extension. Pass - * [TikaMimeDetector] to `MikroMarkdown` if you need content sniffing for those too. + * Formats that are plain text with no signature (CSV, JSON, XML, HTML, Markdown) are recognised by extension. Content + * sniffing for those is a [MimeDetector] away — it is a `fun interface`, and the pipeline takes any implementation. */ public object SignatureMimeDetector : MimeDetector { @@ -23,7 +23,6 @@ public object SignatureMimeDetector : MimeDetector { private val byExtension = mapOf( "csv" to "text/csv", - "docx" to "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "epub" to "application/epub+zip", "htm" to "text/html", "html" to "text/html", @@ -31,15 +30,13 @@ public object SignatureMimeDetector : MimeDetector { "md" to "text/markdown", "markdown" to "text/markdown", "pdf" to "application/pdf", - "pptx" to "application/vnd.openxmlformats-officedocument.presentationml.presentation", "txt" to "text/plain", "log" to "text/plain", - "xlsx" to "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xml" to "application/xml", ) /** ZIP-based formats are told apart by extension; the signature only proves it is a package. */ - private val zipExtensions = setOf("docx", "xlsx", "pptx", "epub", "zip") + private val zipExtensions = setOf("epub", "zip") override fun detect(path: String): StreamInfo = describe(path, readSignature(path)) @@ -61,11 +58,8 @@ public object SignatureMimeDetector : MimeDetector { private fun mimetypeOf(signature: ByteArray, extension: String?): String? = when { signature.startsWith("%PDF") -> "application/pdf" - // Every OOXML container and EPUB is a ZIP; the extension says which one. - signature.startsWith("PK") -> - if (extension in zipExtensions) byExtension[extension] ?: "application/zip" else "application/zip" - // Legacy OLE compound files: .doc/.xls/.ppt, which no converter handles yet. - signature.startsWithBytes(0xD0, 0xCF, 0x11, 0xE0) -> "application/x-ole-storage" + // EPUB and every OOXML container is a ZIP; the extension says which one, and only EPUB is ours. + signature.startsWith("PK") -> byExtension[extension.takeIf { it in zipExtensions }] ?: "application/zip" else -> byExtension[extension] } @@ -84,9 +78,4 @@ public object SignatureMimeDetector : MimeDetector { if (size < prefix.length) return false return prefix.indices.all { this[it].toInt().toChar() == prefix[it] } } - - private fun ByteArray.startsWithBytes(vararg prefix: Int): Boolean { - if (size < prefix.size) return false - return prefix.indices.all { this[it].toInt() and 0xFF == prefix[it] } - } } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt index be2d2e5..0643384 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt @@ -31,10 +31,16 @@ public sealed interface Block public data class Heading( val level: Int, val content: List, + /** The source's own id for this heading, where it had one; HTML fills it from `id`. */ val anchor: String? = null, -) : Block +) : Block { + /** The common case: a heading whose content is one run of text. */ + public constructor(level: Int, text: String) : this(level, listOf(Text(text))) +} -public data class Paragraph(val content: List) : Block +public data class Paragraph(val content: List) : Block { + public constructor(text: String) : this(listOf(Text(text))) +} public data class CodeBlock(val code: String, val language: String? = null) : Block @@ -107,9 +113,6 @@ public enum class Alignment { public data object ThematicBreak : Block -/** Rendered verbatim as an HTML comment. Used for structural markers such as slide numbers. */ -public data class HtmlComment(val text: String) : Block - /** Escape hatch for content that is already Markdown (or must not be touched). */ public data class RawBlock(val text: String) : Block diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt deleted file mode 100644 index af3684c..0000000 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt +++ /dev/null @@ -1,160 +0,0 @@ -package io.github.lemcoder.mikromarkdown.model - -/** - * Small builder used by converters so parsing code stays free of Markdown syntax. - * - * ``` - * document { - * heading(1, "Report") - * paragraph("Body text") - * table(header = listOf("A", "B"), rows = listOf(listOf("1", "2"))) - * } - * ``` - */ -/** Restricts builder receivers so an inline block cannot silently call document-level methods. */ -@DslMarker public annotation class DocumentDsl - -@DocumentDsl public fun document(block: DocumentBuilder.() -> Unit): Document = DocumentBuilder().apply(block).build() - -@DocumentDsl -public class DocumentBuilder { - private val blocks = mutableListOf() - private val assets = mutableListOf() - private val metadata = mutableMapOf() - private var title: String? = null - - public fun add(block: Block) { - blocks += block - } - - public fun addAll(blocks: Iterable) { - this.blocks += blocks - } - - public fun asset(asset: Asset) { - assets += asset - } - - public fun meta(key: String, value: String?) { - if (!value.isNullOrBlank()) metadata[key] = value - } - - /** Records [text] as the document title unless one was already found. */ - public fun titleIfAbsent(text: String?) { - if (title == null && !text.isNullOrBlank()) title = text - } - - /** The title recorded so far, if any. */ - public fun title(): String? = title - - public fun heading(level: Int, text: String) { - if (text.isNotBlank()) add(Heading(level, listOf(Text(text)))) - } - - public fun heading(level: Int, content: List) { - if (content.isNotEmpty()) add(Heading(level, content)) - } - - public fun paragraph(text: String) { - if (text.isNotBlank()) add(Paragraph(listOf(Text(text)))) - } - - public fun paragraph(content: List) { - if (content.isNotEmpty()) add(Paragraph(content)) - } - - public fun code(code: String, language: String? = null) { - add(CodeBlock(code, language)) - } - - public fun comment(text: String) { - add(HtmlComment(text)) - } - - public fun raw(text: String) { - if (text.isNotBlank()) add(RawBlock(text)) - } - - public fun bulletList(items: List) { - if (items.isEmpty()) return - add(ListBlock(ordered = false, items = items.map { ListItem(listOf(Paragraph(listOf(Text(it))))) })) - } - - public fun table( - header: List, - rows: List>, - alignments: List = emptyList(), - ) { - if (header.isEmpty() && rows.isEmpty()) return - add( - Table( - header = header.map { TableCell(it) }, - rows = rows.map { row -> row.map { TableCell(it) } }, - alignments = alignments, - ) - ) - } - - public fun build(): Document = - Document( - blocks = blocks.toList(), - title = title, - metadata = metadata.toMap(), - assets = assets.toList(), - ) -} - -/** Builds a list of inlines without repeating `listOf(...)` wrappers in parsers. */ -@DocumentDsl public fun inlines(block: InlineBuilder.() -> Unit): List = InlineBuilder().apply(block).build() - -@DocumentDsl -public class InlineBuilder { - private val items = mutableListOf() - - public fun text(value: String) { - if (value.isNotEmpty()) items += Text(value) - } - - public fun strong(value: String) { - if (value.isNotEmpty()) items += Strong(listOf(Text(value))) - } - - public fun emphasis(value: String) { - if (value.isNotEmpty()) items += Emphasis(listOf(Text(value))) - } - - public fun code(value: String) { - if (value.isNotEmpty()) items += CodeSpan(value) - } - - public fun link(text: String, url: String) { - items += Link(listOf(Text(text)), url) - } - - public fun image(alt: String, url: String, assetId: String? = null) { - items += Image(alt, url, assetId = assetId) - } - - public fun lineBreak() { - items += LineBreak - } - - public operator fun plusAssign(inline: Inline) { - items += inline - } - - public operator fun plusAssign(inlines: List) { - items += inlines - } - - public fun build(): List = items.toList() -} - -/** Wraps [content] in the emphasis combination described by the flags. */ -public fun styled(content: List, bold: Boolean, italic: Boolean, strike: Boolean = false): List { - var result = content - if (strike) result = listOf(Strikethrough(result)) - if (italic) result = listOf(Emphasis(result)) - if (bold) result = listOf(Strong(result)) - return result -} diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt index e80aa3f..4cb2dae 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt @@ -10,8 +10,15 @@ package io.github.lemcoder.mikromarkdown.model * the source layout's line breaks. */ public fun plainTextBlocks(text: String, reflow: Boolean = true): List { - // Form feeds mark PDF page breaks; treat them as paragraph boundaries. - val normalized = text.replace("\r\n", "\n").replace('\r', '\n').replace('\u000C', '\n') + val normalized = + text + // Form feeds mark PDF page breaks; treat them as paragraph boundaries. + .replace("\r\n", "\n") + .replace('\r', '\n') + .replace('\u000C', '\n') + // pdfium emits U+FFFE where a glyph has no Unicode mapping, which in typeset text is all but always the + // hyphen at a line break. Spelling it as one leaves a single rule to decide what becomes of it, below. + .replace(UNMAPPED_GLYPH, "-\n") val vocabulary = if (reflow) wordsIn(normalized) else emptySet() val paragraphs = mutableListOf() @@ -35,16 +42,30 @@ private fun String.endsWithWordBreak(): Boolean = endsWith("-") && length > 1 && private val PARAGRAPH_BREAK = Regex("\n[ \t]*\n") +private const val UNMAPPED_GLYPH = "\uFFFE" + private val WORD = Regex("[\\p{L}]{2,}") -/** Words the document uses on their own; the de-hyphenation heuristic consults this. */ -private fun wordsIn(text: String): Set = WORD.findAll(text).map { it.value.lowercase() }.toSet() +private val HYPHEN_BREAK = Regex("\\p{L}+-\n\\p{L}+") + +/** + * Words the document uses on their own; the de-hyphenation heuristic consults this. + * + * The two halves of a broken word are cut out first: they are only in the text because the break put them there, and + * counting them would let every fragment vouch for itself. + */ +private fun wordsIn(text: String): Set = + WORD.findAll(text.replace(HYPHEN_BREAK, " ")).map { it.value.lowercase() }.toSet() /** * Rejoins soft-wrapped lines. * * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are words the document uses - * elsewhere on their own (`conversation-` + `centric`), the hyphen is a real compound and stays. + * elsewhere on their own (`conversation-` + `centric`), the hyphen is the author's and stays. Dropping it always fused + * real compounds — "chat-optimized" became "chatoptimized". + * + * This is a heuristic standing in for geometry. A hyphenation hyphen ends a line and a compound hyphen does not, which + * `FPDFText_GetCharBox` would answer outright. */ private fun joinWrappedLines(lines: List, vocabulary: Set): String { val sb = StringBuilder() diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt index 2aae945..eff63ce 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt @@ -8,7 +8,6 @@ import io.github.lemcoder.mikromarkdown.model.CodeSpan import io.github.lemcoder.mikromarkdown.model.Document import io.github.lemcoder.mikromarkdown.model.Emphasis import io.github.lemcoder.mikromarkdown.model.Heading -import io.github.lemcoder.mikromarkdown.model.HtmlComment import io.github.lemcoder.mikromarkdown.model.Image import io.github.lemcoder.mikromarkdown.model.Inline import io.github.lemcoder.mikromarkdown.model.LineBreak @@ -121,8 +120,6 @@ public class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOpt ThematicBreak -> out.append("---") - is HtmlComment -> out.append("") - // Already-Markdown content: only whitespace is normalized, never syntax. is RawBlock -> appendLines( diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt deleted file mode 100644 index 9695eb7..0000000 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ /dev/null @@ -1,29 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -import io.github.lemcoder.mikromarkdown.converters.CsvConverter -import io.github.lemcoder.mikromarkdown.converters.EpubConverter -import io.github.lemcoder.mikromarkdown.converters.HtmlConverter -import io.github.lemcoder.mikromarkdown.converters.JsonConverter -import io.github.lemcoder.mikromarkdown.converters.MarkdownPassthroughConverter -import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter -import io.github.lemcoder.mikromarkdown.converters.XmlConverter -import java.io.File - -/** - * A [MikroMarkdown] with every JVM converter registered. - * - * PDF is not among them: it needs a native library, so the `:pdfium` module provides it and the caller opts in with - * `register(PdfiumConverter())`. - */ -public fun MikroMarkdown(): MikroMarkdown = - MikroMarkdown(SignatureMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(EpubConverter()) - register(PlainTextConverter(), priority = 10.0) - } - -public fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/TikaMimeDetector.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/TikaMimeDetector.kt deleted file mode 100644 index 95823d4..0000000 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/TikaMimeDetector.kt +++ /dev/null @@ -1,32 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -import java.io.File -import org.apache.tika.Tika - -/** - * Full content sniffing through Apache Tika's MIME registry. - * - * Slower to start than [SignatureMimeDetector] — building the registry costs about 90 ms, more than converting most - * documents — but it recognises text formats by content rather than by extension. - */ -public object TikaMimeDetector : MimeDetector { - // Building Tika's MIME registry is expensive; convert(bytes, info) never needs it. - private val tika by lazy { Tika() } - - override fun detect(path: String): StreamInfo { - val file = File(path) - val mimetype = - try { - tika.detect(file) - } catch (_: Exception) { - null - } - val extension = file.extension.lowercase().ifEmpty { null } - return StreamInfo( - mimetype = mimetype, - extension = extension, - filename = file.name, - localPath = path, - ) - } -} diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt index 2f4093a..75b657d 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt @@ -119,18 +119,14 @@ class ArchitectureTest { } /** - * JVM and Android share one source set; only the factory and the PDF converter differ, because pdfbox-android is a - * separate library. Any other same-named file in two source sets means a copy that will drift. + * Every converter is common, so no production file has a per-target copy — the rule has no exceptions left. A + * same-named file in two source sets means a copy that will drift. */ @Test fun `production files are not copied between source sets`() { - val expectedPerTarget = setOf("MikroMarkdownFactory", "PdfConverter") + val copied = production.filter { it.path.contains("/src/") }.groupBy { it.name }.filterValues { it.size > 1 } - val copied = - production.filter { it.path.contains("/src/") }.groupBy { it.name }.filterValues { it.size > 1 }.keys - - expectedPerTarget - - assertEquals(emptySet(), copied, "these files exist in more than one source set") + assertEquals(emptySet(), copied.keys, "these files exist in more than one source set") } private fun productionIn(packageFragment: String) = production.filter { diff --git a/library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt b/library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt new file mode 100644 index 0000000..7c1314e --- /dev/null +++ b/library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt @@ -0,0 +1,3 @@ +package io.github.lemcoder.mikromarkdown + +actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() diff --git a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt index 273d4e5..6e98fdb 100644 --- a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt +++ b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt @@ -15,7 +15,8 @@ import io.github.lemcoder.mikromarkdown.model.plainTextBlocks * ``` * * The extraction itself is per-platform — cinterop on native, generated JNI bridges on the JVM — but both reach the - * same pdfium, and everything above [extractText] is shared. + * same pdfium, and everything above [extractText] is shared. Turning that text into paragraphs, including the + * de-hyphenation that pdfium's unmapped glyphs call for, is `plainTextBlocks`' job rather than a second copy here. */ public class PdfiumConverter : DocumentConverter { @@ -24,49 +25,7 @@ public class PdfiumConverter : DocumentConverter { } override fun parse(bytes: ByteArray, info: StreamInfo): Document = - Document(blocks = plainTextBlocks(extractText(bytes).joinHyphenatedWords())) - - /** - * pdfium emits U+FFFE where a glyph has no Unicode mapping, which in a typeset document is nearly always the hyphen - * at a line break. - * - * Dropping it always would fuse real compounds — "chat-optimized" became "chatoptimized" — so the document decides: - * if both halves appear elsewhere as words in their own right the hyphen was the author's and is restored, - * otherwise the halves are one broken word and are joined. The halves themselves are cut from the vocabulary first, - * since they are only in the text because the break put them there. - * - * This is a heuristic standing in for geometry. A hyphenation hyphen ends a line and a compound hyphen does not, - * which `FPDFText_GetCharBox` would answer outright. - */ - private fun String.joinHyphenatedWords(): String { - if (indexOf(UNMAPPED_GLYPH) < 0) return this - - val vocabulary = WORD.findAll(replace(HYPHEN_BREAK, " ")).map { it.value.lowercase() }.toSet() - val out = StringBuilder(length) - for (index in indices) { - val char = this[index] - if (char != UNMAPPED_GLYPH) { - out.append(char) - continue - } - var wordStart = out.length - while (wordStart > 0 && out[wordStart - 1].isLetter()) wordStart-- - val left = out.subSequence(wordStart, out.length).toString().lowercase() - val right = substring(index + 1).takeWhile { it.isLetter() }.lowercase() - if (isRealCompound(left, right, vocabulary)) out.append('-') - } - return out.toString() - } - - /** A hyphen the author wrote, rather than one the typesetter added at a line break. */ - private fun isRealCompound(left: String, right: String, vocabulary: Set): Boolean = - left.isNotEmpty() && right.isNotEmpty() && left in vocabulary && right in vocabulary - - private companion object { - const val UNMAPPED_GLYPH = '\uFFFE' - val WORD = Regex("[\\p{L}]{2,}") - val HYPHEN_BREAK = Regex("\\p{L}+\uFFFE\\p{L}+") - } + Document(blocks = plainTextBlocks(extractText(bytes))) } /** Every page's text, concatenated, one page per line. */ From 21c353f2481ef89b3b6aff4300dc78fc13648efd Mon Sep 17 00:00:00 2001 From: mikolaj Date: Thu, 20 Aug 2026 00:55:09 +0200 Subject: [PATCH 2/9] Fix what the review of this branch turned up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detector's ZIP branch had collapsed to a constant: "zip" was in the extension set but absent from the table, so only "epub" ever changed the answer and the takeIf/elvis around it decided nothing. The OLE signature that names a legacy .doc/.xls/.ppt went with it, leaving those files as "No converter found for: unknown". Neither was caught, because nothing exercised SignatureMimeDetector at all — the integration tests hand converters a StreamInfo they build themselves and never reach detection. The expect/actual testMikroMarkdown() indirection outlived its reason when the factory moved to commonMain: four identical actuals returning MikroMarkdown(), which commonTest can call directly. CI compiled neither the native target nor the pdfium bindings, so the macosArm64 source set and every pdfium test could break unnoticed. ktfmt named the Android source sets explicitly and so would miss any target added later; it now covers everything under src/ that ktfmt does not derive for itself. The benchmark converted every fixture in the directory with no try/catch while the directory still holds the office formats, so the documented `./gradlew :benchmark:run` died on the second file. README claimed no third-party JVM dependency at all, which three implementation dependencies contradict, and pointed at a Maven Central coordinate nothing publishes: the plugin has not been applied since 5113f53, so publish.yml could only ever fail. Co-Authored-By: Claude Opus 5 --- .github/workflows/gradle.yml | 15 ++++- .github/workflows/publish.yml | 25 -------- README.md | 16 +++-- .../lemcoder/mikromarkdown/benchmark/Main.kt | 22 ++++++- library/build.gradle.kts | 26 ++++---- .../FileIntegrationTest.android.kt | 5 -- .../mikromarkdown/TestMikroMarkdown.kt | 3 - .../mikromarkdown/SignatureMimeDetector.kt | 18 ++++-- .../mikromarkdown/FileIntegrationTest.kt | 4 +- .../SignatureMimeDetectorTest.kt | 63 +++++++++++++++++++ .../mikromarkdown/FileIntegrationTest.jvm.kt | 3 - .../FileIntegrationTest.macos.kt | 3 - 12 files changed, 129 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/publish.yml delete mode 100644 library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt delete mode 100644 library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt create mode 100644 library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetectorTest.kt delete mode 100644 library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt delete mode 100644 library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 2c10c1d..7181fe1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -12,13 +12,22 @@ permissions: jobs: test: + name: ${{ matrix.name }} strategy: + fail-fast: false matrix: include: - - target: jvmTest + - name: jvm + tasks: ":library:jvmTest" os: ubuntu-latest - - target: testAndroidHostTest + - name: android + tasks: ":library:testAndroidHostTest" os: ubuntu-latest + # The native target and the pdfium bindings are built against the mac-arm64 pdfium, so both + # legs need a macOS runner. Without this job the native source sets are never even compiled. + - name: native and pdfium + tasks: ":library:macosArm64Test :pdfium:jvmTest" + os: macos-latest runs-on: ${{ matrix.os }} steps: @@ -32,7 +41,7 @@ jobs: distribution: 'temurin' - name: Run tests uses: gradle/actions/setup-gradle@v4 - - run: ./gradlew :library:${{ matrix.target }} + - run: ./gradlew ${{ matrix.tasks }} lint: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 515213f..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish -on: - release: - types: [released, prereleased] -jobs: - publish: - name: Release build and publish - runs-on: macOS-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Set up JDK 21 - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: 21 - - name: Publish to MavenCentral - run: ./gradlew publishToMavenCentral --no-configuration-cache - env: - ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.MAVEN_CENTRAL_USERNAME }} - ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} - ORG_GRADLE_PROJECT_signingInMemoryKeyId: ${{ secrets.SIGNING_KEY_ID }} - ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.SIGNING_PASSWORD }} - ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.GPG_KEY_CONTENTS }} - diff --git a/README.md b/README.md index 56f0ab7..62889ed 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,10 @@ bytes ──► MimeDetector ──► DocumentConverter.parse ──► Documen Converters contain no Markdown syntax, so escaping, table shaping, list indentation and spacing are fixed once for all formats. Every converter lives in `commonMain` and runs on every target, so the -library has one registration list rather than one per platform and no third-party JVM dependency at -all; only PDF is platform-specific, and it lives in its own module because it needs a native -library. The model is +library has one registration list rather than one per platform and one dependency set rather than a +JVM-only one on top: Tika, PDFBox and POI are gone, and what is left — kotlinx-io, Ksoup and +korlibs-compression — is multiplatform and shared by every target. Only PDF is platform-specific, +and it lives in its own module because it needs a native library. The model is public: `mid.parse(path)` returns the `Document`, and `ConversionResult.document` exposes it alongside the rendered Markdown. @@ -56,12 +57,9 @@ println(compact.render(document)) ## Setup -```kotlin -// build.gradle.kts -dependencies { - implementation("io.github.lemcoder:mikromarkdown:0.1.0") -} -``` +Nothing is published yet: the project builds and tests from source, and no module is configured to +publish. Build it with `./gradlew build`, and depend on `:library` from a composite build (PDF adds +`:pdfium`) until a release is wired up. ## Usage diff --git a/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt b/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt index 7f0a265..d5887df 100644 --- a/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt +++ b/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt @@ -1,6 +1,7 @@ package io.github.lemcoder.mikromarkdown.benchmark import io.github.lemcoder.mikromarkdown.MikroMarkdown +import io.github.lemcoder.mikromarkdown.MikroMarkdownException import io.github.lemcoder.mikromarkdown.StreamInfo import java.io.File import kotlin.system.measureNanoTime @@ -30,11 +31,14 @@ fun main(args: Array) { require(files.isNotEmpty()) { "no fixtures in ${fixtures.absolutePath}" } val first = files.first() - val coldStart = measureNanoTime { MikroMarkdown().convert(first.absolutePath) } - report("first conversion in a fresh JVM, class loading included (${first.name})", coldStart) + // The cold number is the first conversion in the process or it is nothing: a second attempt would run warm. + runCatching { measureNanoTime { MikroMarkdown().convert(first.absolutePath) } } + .onSuccess { report("first conversion in a fresh JVM, class loading included (${first.name})", it) } + .onFailure { println("no cold-start timing: ${first.name} has no registered converter") } println() val mikroMarkdown = MikroMarkdown() + val skipped = mutableListOf() println("Best of $iterations runs after $warmup warmup runs, milliseconds.") println() @@ -45,7 +49,14 @@ fun main(args: Array) { val bytes = file.readBytes() val info = StreamInfo(extension = file.extension, filename = file.name, localPath = file.absolutePath) - repeat(warmup) { mikroMarkdown.convert(bytes, info) } + // The directory keeps fixtures no converter accepts: the office formats, and PDF until `:pdfium` is + // registered. They are named below rather than dropped, so a missing row never reads as a fast one. + try { + repeat(warmup) { mikroMarkdown.convert(bytes, info) } + } catch (e: MikroMarkdownException) { + skipped += "${file.name} (${e.message})" + continue + } val parse = best(iterations) { mikroMarkdown.parse(bytes, info) } val convertBytes = best(iterations) { mikroMarkdown.convert(bytes, info) } @@ -58,6 +69,11 @@ fun main(args: Array) { "${convertBytes.ms()} | ${convertPath.ms()} |" ) } + + if (skipped.isNotEmpty()) { + println() + println("Skipped: ${skipped.joinToString()}") + } } private fun coldBytes(file: File) { diff --git a/library/build.gradle.kts b/library/build.gradle.kts index 9cd46cb..ee9e54d 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -55,26 +55,28 @@ kotlin { } } -// ktfmt-gradle only derives tasks for the common and JVM source sets, so the Android test -// source sets would go unformatted and unchecked. +// ktfmt-gradle only derives tasks for the common and JVM source sets, so every other one — Android's +// today, a native one tomorrow — would go unformatted and unchecked. Everything under src/ that ktfmt +// does not already cover is named here by exclusion, so a new target needs no edit to this block. run { - val androidSources = fileTree("src") { include("android*/**/*.kt") } + val derived = listOf("commonMain", "commonTest", "jvmMain", "jvmTest") + val remainingSources = fileTree("src") { include("**/*.kt").exclude(derived.map { "$it/**" }) } val template = tasks.named("ktfmtFormatKmpCommonMain") - val formatAndroid = - tasks.register("ktfmtFormatAndroidSourceSets") { + val formatRemaining = + tasks.register("ktfmtFormatRemainingSourceSets") { ktfmtClasspath.from(template.map { it.ktfmtClasspath }) formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) - setSource(androidSources) + setSource(remainingSources) } - val checkAndroid = - tasks.register("ktfmtCheckAndroidSourceSets") { + val checkRemaining = + tasks.register("ktfmtCheckRemainingSourceSets") { ktfmtClasspath.from(template.map { it.ktfmtClasspath }) formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) - setSource(androidSources) + setSource(remainingSources) } - tasks.named("ktfmtFormat") { dependsOn(formatAndroid) } - tasks.named("ktfmtCheck") { dependsOn(checkAndroid) } - tasks.named("check") { dependsOn(checkAndroid) } + tasks.named("ktfmtFormat") { dependsOn(formatRemaining) } + tasks.named("ktfmtCheck") { dependsOn(checkRemaining) } + tasks.named("check") { dependsOn(checkRemaining) } } diff --git a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt deleted file mode 100644 index 15c3cc4..0000000 --- a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt +++ /dev/null @@ -1,5 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMikroMarkdown(): MikroMarkdown { - return MikroMarkdown() -} diff --git a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt deleted file mode 100644 index 7c1314e..0000000 --- a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt +++ /dev/null @@ -1,3 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt index 19873bd..79138be 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt @@ -35,9 +35,6 @@ public object SignatureMimeDetector : MimeDetector { "xml" to "application/xml", ) - /** ZIP-based formats are told apart by extension; the signature only proves it is a package. */ - private val zipExtensions = setOf("epub", "zip") - override fun detect(path: String): StreamInfo = describe(path, readSignature(path)) override fun detect(path: String, bytes: ByteArray): StreamInfo = @@ -58,8 +55,13 @@ public object SignatureMimeDetector : MimeDetector { private fun mimetypeOf(signature: ByteArray, extension: String?): String? = when { signature.startsWith("%PDF") -> "application/pdf" - // EPUB and every OOXML container is a ZIP; the extension says which one, and only EPUB is ours. - signature.startsWith("PK") -> byExtension[extension.takeIf { it in zipExtensions }] ?: "application/zip" + // EPUB and every OOXML container is a ZIP; the extension says which one, and only EPUB is ours. The rest + // are reported as the package they are, which is what leaves the caller a message naming a real format. + signature.startsWith("PK") -> if (extension == "epub") "application/epub+zip" else "application/zip" + // Legacy OLE compound files: .doc/.xls/.ppt, which no converter handles. Naming the container is what + // leaves the caller a message about a format rather than "No converter found for: unknown"; the OOXML + // mimetypes are not restored with it, because their extension already reaches the caller in [StreamInfo]. + signature.startsWithBytes(0xD0, 0xCF, 0x11, 0xE0) -> "application/x-ole-storage" else -> byExtension[extension] } @@ -78,4 +80,10 @@ public object SignatureMimeDetector : MimeDetector { if (size < prefix.length) return false return prefix.indices.all { this[it].toInt().toChar() == prefix[it] } } + + /** Signatures outside ASCII are spelled as the bytes they are. */ + private fun ByteArray.startsWithBytes(vararg prefix: Int): Boolean { + if (size < prefix.size) return false + return prefix.indices.all { this[it] == prefix[it].toByte() } + } } diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt index 5473077..6a2e058 100644 --- a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt @@ -6,10 +6,8 @@ import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertTrue -expect fun testMikroMarkdown(): MikroMarkdown - class FileIntegrationTest { - private val mid = testMikroMarkdown() + private val mid = MikroMarkdown() private fun assertConversion( filename: String, diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetectorTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetectorTest.kt new file mode 100644 index 0000000..3adb6b1 --- /dev/null +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetectorTest.kt @@ -0,0 +1,63 @@ +package io.github.lemcoder.mikromarkdown + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * Pins what the leading bytes decide and what the extension decides. + * + * The integration tests hand converters a [StreamInfo] they build themselves, so detection is only covered here. + */ +class SignatureMimeDetectorTest { + + private fun detect(filename: String, vararg signature: Int): String? = + SignatureMimeDetector.detect(filename, ByteArray(signature.size) { signature[it].toByte() }).mimetype + + private fun detect(filename: String, signature: String): String? = + SignatureMimeDetector.detect(filename, signature.encodeToByteArray()).mimetype + + @Test + fun `content wins over a mislabelled extension`() { + assertEquals("application/pdf", detect("report.txt", "%PDF-1.7")) + } + + @Test + fun `a ZIP package is told apart by its extension`() { + assertEquals("application/epub+zip", detect("book.epub", "PK\u0003\u0004")) + // Every other OOXML container is a package this library does not read; it is reported as one. + assertEquals("application/zip", detect("sheet.xlsx", "PK\u0003\u0004")) + assertEquals("application/zip", detect("archive.zip", "PK\u0003\u0004")) + } + + @Test + fun `a legacy OLE compound file is named rather than left unknown`() { + assertEquals("application/x-ole-storage", detect("memo.doc", 0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1)) + } + + @Test + fun `formats with no signature are recognised by extension`() { + assertEquals("text/csv", detect("data.csv", "a,b,c\n1,2,3")) + assertEquals("text/markdown", detect("notes.md", "# Title")) + } + + @Test + fun `an unknown extension with no signature is left undecided`() { + assertNull(detect("mystery.bin", "not a signature")) + assertNull(detect("noextension", "")) + } + + @Test + fun `a file shorter than the signature is read whole rather than failing`() { + assertEquals("text/plain", detect("tiny.txt", "hi")) + } + + @Test + fun `the path is split into filename and extension`() { + val info = SignatureMimeDetector.detect("/tmp/some dir/Book.EPUB", "PK\u0003\u0004".encodeToByteArray()) + + assertEquals("Book.EPUB", info.filename) + assertEquals("epub", info.extension) + assertEquals("application/epub+zip", info.mimetype) + } +} diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt deleted file mode 100644 index 7c1314e..0000000 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt +++ /dev/null @@ -1,3 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() diff --git a/library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt b/library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt deleted file mode 100644 index 7c1314e..0000000 --- a/library/src/macosArm64Test/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.macos.kt +++ /dev/null @@ -1,3 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() From 7fb82600a578ef13e347b0eaa06a4562794632d6 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Thu, 20 Aug 2026 00:55:45 +0200 Subject: [PATCH 3/9] Read PDF wraps from pdfium's geometry, and keep the hyphen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit U+FFFE is not what the comments said. FPDFText_HasUnicodeMapError is 0 at every one of the eleven occurrences in the AutoGen fixture and FPDFText_IsHyphen is 1: the marker stands for a hyphen pdfium removed while joining a wrapped line, not for a glyph it could not decode. So the two halves arrive adjacent with nothing to say a line ever ended between them, which is why the old rule had to guess. FPDFText_GetCharBox answers the one question that is left. Every marker sits at the right margin with the following character on a lower line, so the break pdfium discarded can be put back — and a marker the boxes do not confirm was some other undecodable character, which now leaves nothing behind instead of a stray hyphen. That is the whole of what the geometry settles: it is equally true of con-|firming and chat-|optimized, so it cannot say which hyphen was the author's. Nothing tries to. Weighing the halves against the rest of the document was tried and cost more than it returned — a compound whose halves appear nowhere else came out as "chatoptimized", a word in no document anywhere. Keeping the hyphen writes "con-firming" where the page meant "confirming", which reads a little worse and destroys nothing. That deletes the vocabulary, the two regexes and the compound test that served it, and takes TextBlocks from 113 lines to 71. The pdfium sentinel leaves the model with it. model/ is pinned by ArchitectureTest as depending on nothing, and it now knows only that a line can end mid-word — in any of the three hyphens a document may write, since :pdfium restores the one its own text uses rather than substituting U+002D. Co-Authored-By: Claude Opus 5 --- .../mikromarkdown/model/TextBlocks.kt | 76 ++++--------- .../mikromarkdown/model/TextBlocksTest.kt | 73 +++++++++++++ .../mikromarkdown/pdf/Pdfium.android.kt | 99 ++++++++++++----- .../lemcoder/mikromarkdown/pdf/PageText.kt | 66 ++++++++++++ .../mikromarkdown/pdf/PdfiumConverter.kt | 11 +- .../lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt | 97 ++++++++++++----- .../mikromarkdown/pdf/PageTextTest.kt | 61 +++++++++++ .../mikromarkdown/pdf/PdfiumConverterTest.kt | 26 ++++- .../mikromarkdown/pdf/Pdfium.macos.kt | 102 +++++++++++++----- 9 files changed, 471 insertions(+), 140 deletions(-) create mode 100644 library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocksTest.kt create mode 100644 pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageText.kt create mode 100644 pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageTextTest.kt diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt index 4cb2dae..bffddaf 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt @@ -12,90 +12,60 @@ package io.github.lemcoder.mikromarkdown.model public fun plainTextBlocks(text: String, reflow: Boolean = true): List { val normalized = text - // Form feeds mark PDF page breaks; treat them as paragraph boundaries. .replace("\r\n", "\n") .replace('\r', '\n') - .replace('\u000C', '\n') - // pdfium emits U+FFFE where a glyph has no Unicode mapping, which in typeset text is all but always the - // hyphen at a line break. Spelling it as one leaves a single rule to decide what becomes of it, below. - .replace(UNMAPPED_GLYPH, "-\n") - val vocabulary = if (reflow) wordsIn(normalized) else emptySet() + // Form feeds mark PDF page breaks; a paragraph boundary is a blank line, which takes two newlines. + .replace(FORM_FEED, "\n\n") val paragraphs = mutableListOf() for (chunk in normalized.split(PARAGRAPH_BREAK)) { val lines = chunk.lines().map { it.trim() }.filter { it.isNotEmpty() } if (lines.isEmpty()) continue - val joined = if (reflow) joinWrappedLines(lines, vocabulary) else lines.joinToString("\n") + val joined = if (reflow) joinWrappedLines(lines) else lines.joinToString("\n") // A paragraph ending mid-word means the break was a layout artifact, not a real one. val previous = paragraphs.lastOrNull() if (reflow && previous != null && previous.endsWithWordBreak()) { - paragraphs[paragraphs.lastIndex] = joinWrappedLines(listOf(previous, joined), vocabulary) + paragraphs[paragraphs.lastIndex] = joinWrappedLines(listOf(previous, joined)) } else { paragraphs += joined } } - return paragraphs.map { Paragraph(listOf(Text(it))) } + return paragraphs.map { Paragraph(it) } } -private fun String.endsWithWordBreak(): Boolean = endsWith("-") && length > 1 && this[length - 2].isLetter() - private val PARAGRAPH_BREAK = Regex("\n[ \t]*\n") -private const val UNMAPPED_GLYPH = "\uFFFE" - -private val WORD = Regex("[\\p{L}]{2,}") - -private val HYPHEN_BREAK = Regex("\\p{L}+-\n\\p{L}+") - -/** - * Words the document uses on their own; the de-hyphenation heuristic consults this. - * - * The two halves of a broken word are cut out first: they are only in the text because the break put them there, and - * counting them would let every fragment vouch for itself. - */ -private fun wordsIn(text: String): Set = - WORD.findAll(text.replace(HYPHEN_BREAK, " ")).map { it.value.lowercase() }.toSet() +private const val FORM_FEED = "\u000C" /** * Rejoins soft-wrapped lines. * - * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are words the document uses - * elsewhere on their own (`conversation-` + `centric`), the hyphen is the author's and stays. Dropping it always fused - * real compounds — "chat-optimized" became "chatoptimized". + * A line ending mid-word continues straight into the next; any other line break was the space the wrap replaced. * - * This is a heuristic standing in for geometry. A hyphenation hyphen ends a line and a compound hyphen does not, which - * `FPDFText_GetCharBox` would answer outright. + * The hyphen stays either way, and nothing here tries to work out which kind it was. Geometry cannot say — + * `FPDFText_GetCharBox` settles only that the line ended at the hyphen, which is as true of `chat-` `optimized` as of + * `con-` `firming`. Weighing it against the rest of the document was tried and cost more than it returned: a compound + * whose halves appear nowhere else, `chat-optimized` among them, came out fused as `chatoptimized`, a word that is in + * no document anywhere. Keeping the hyphen writes `con-firming` where the page meant `confirming`, which reads a little + * worse and destroys nothing — the join is still there for a reader, and for anything downstream that knows more than + * we do. */ -private fun joinWrappedLines(lines: List, vocabulary: Set): String { +private fun joinWrappedLines(lines: List): String { val sb = StringBuilder() for ((index, line) in lines.withIndex()) { - if (index == 0) { - sb.append(line) - continue - } - val head = sb.lastFragment() - val tail = line.takeWhile { it.isLetter() }.lowercase() - val hyphenated = sb.isNotEmpty() && sb.last() == '-' && head.isNotEmpty() && tail.isNotEmpty() - val realCompound = hyphenated && head in vocabulary && tail in vocabulary - when { - hyphenated && !realCompound -> { - sb.setLength(sb.length - 1) - sb.append(line) - } - - hyphenated -> sb.append(line) + index == 0 -> sb.append(line) + sb.endsWithWordBreak() && line.first().isLetter() -> sb.append(line) else -> sb.append(' ').append(line) } } return sb.toString() } -/** The word immediately before a trailing hyphen, lowercased. */ -private fun StringBuilder.lastFragment(): String { - if (isEmpty() || last() != '-') return "" - var start = length - 1 - while (start > 0 && this[start - 1].isLetter()) start-- - return substring(start, length - 1).lowercase() -} +/** A trailing hyphen with a letter before it: the mark of a word the layout cut in half. */ +private fun CharSequence.endsWithWordBreak(): Boolean = + length > 1 && this[length - 1].isHyphen() && this[length - 2].isLetter() + +/** The hyphens a line break can end on; a document may write any of them. */ +private fun Char.isHyphen(): Boolean = this == '-' || this == '\u2010' || this == '\u2011' diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocksTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocksTest.kt new file mode 100644 index 0000000..82a93a4 --- /dev/null +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocksTest.kt @@ -0,0 +1,73 @@ +package io.github.lemcoder.mikromarkdown.model + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** Pins how a trailing hyphen at a line break is read, and the paragraph structure it must not disturb. */ +class TextBlocksTest { + + /** The page break a PDF text extractor leaves behind. */ + private val formFeed = "\u000C" + + private fun paragraphs(text: String, reflow: Boolean = true): List = + plainTextBlocks(text, reflow).map { (it as Paragraph).content.plainText() } + + private fun onlyParagraph(text: String, reflow: Boolean = true): String = paragraphs(text, reflow).single() + + @Test + fun `a hyphen at a line break keeps its hyphen and loses the break`() { + val text = "an inter-\nnational body\nand nothing else" + + // The page meant "international". Which kind of hyphen this was cannot be known, so it is left where it is. + assertEquals("an inter-national body and nothing else", onlyParagraph(text)) + } + + @Test + fun `a compound broken across lines survives intact`() { + val text = "a well-\nknown case.\nIt reads the same as any other." + + assertEquals("a well-known case. It reads the same as any other.", onlyParagraph(text)) + } + + @Test + fun `a line break with no hyphen is the space the wrap replaced`() { + val text = "an ordinary\nwrapped line" + + assertEquals("an ordinary wrapped line", onlyParagraph(text)) + } + + @Test + fun `a trailing hyphen with no letter before it is not a word break`() { + val text = "a dash -\nthen more" + + assertEquals("a dash - then more", onlyParagraph(text)) + } + + @Test + fun `a form feed separates paragraphs rather than merging the pages around it`() { + val text = "End of page one.${formFeed}Start of page two." + + assertEquals(listOf("End of page one.", "Start of page two."), paragraphs(text)) + } + + @Test + fun `a U+2010 hyphen is a word break too and keeps the character the document wrote`() { + val text = "an inter\u2010\nnational body\nand nothing else" + + assertEquals("an inter\u2010national body and nothing else", onlyParagraph(text)) + } + + @Test + fun `without reflow the source layout is kept`() { + val text = "an inter-\nnational body" + + assertEquals(listOf("an inter-\nnational body"), paragraphs(text, reflow = false)) + } + + @Test + fun `a paragraph ending mid-word continues into the next`() { + val text = "a word broken by a page-\n\nbreak and nothing else" + + assertEquals(listOf("a word broken by a page-break and nothing else"), paragraphs(text)) + } +} diff --git a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt index 243f89a..b662583 100644 --- a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt +++ b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt @@ -3,62 +3,96 @@ package io.github.lemcoder.mikromarkdown.pdf import pdfium.kniBridge1 import pdfium.kniBridge15 import pdfium.kniBridge16 -import pdfium.kniBridge2 import pdfium.kniBridge26 import pdfium.kniBridge27 import pdfium.kniBridge5 import pdfium.kniBridge52 import pdfium.kniBridge53 import pdfium.kniBridge54 +import pdfium.kniBridge58 +import pdfium.kniBridge66 import pdfium.kniBridge71 /** - * The JVM half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. + * pdfium is initialised once per process and never destroyed. + * + * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like: pairing them per call makes + * extraction differ between one process and the next, where initialising once gives every process the same answer. + */ +private val pdfiumLibrary: Lazy = lazy { initLibrary() } + +/** + * The Android half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. * * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here * with the name from its doc comment, and nothing else in the module sees them. */ -internal actual fun extractText(bytes: ByteArray): String { - val text = StringBuilder() +internal actual fun extractPages(bytes: ByteArray): List { + val pages = mutableListOf() - initLibrary() + pdfiumLibrary.value + val document = loadDocument(bytes, null) + if (document == 0L) return emptyList() try { - val document = loadDocument(bytes, null) - if (document == 0L) return "" - try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) - if (page == 0L) continue - val textPage = loadTextPage(page) - if (textPage != 0L) { - text.append(pageText(textPage)) - text.append('\n') - closeTextPage(textPage) - } - closePage(page) + for (index in 0 until pageCount(document)) { + val page = loadPage(document, index) + if (page == 0L) continue + val textPage = loadTextPage(page) + if (textPage != 0L) { + pages += pageText(textPage) + closeTextPage(textPage) } - } finally { - closeDocument(document) + closePage(page) } } finally { - destroyLibrary() + closeDocument(document) } - return text.toString() + return pages } /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ -private fun pageText(textPage: Long): String { +private fun pageText(textPage: Long): PageText { val count = charCount(textPage) - if (count <= 0) return "" + if (count <= 0) return PageText("", IntArray(0)) val buffer = ShortArray(count + 1) val written = readText(textPage, 0, count, buffer) - return if (written <= 1) "" else CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + if (written <= 1) return PageText("", IntArray(0)) + // One character in, one character out, so a text index is a pdfium character index. + val text = CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + return PageText(text, hyphenWraps(textPage, text)) } -private fun initLibrary() = kniBridge1() +/** + * The collapsed wraps, as [PageText.hyphenWraps] describes them. + * + * `FPDFText_IsHyphen` marks every hyphen pdfium removed, and the character box is what proves the line ended there + * rather than the marker standing for something else on the same line. + */ +private fun hyphenWraps(textPage: Long, text: String): IntArray { + val wraps = mutableListOf() + for (index in text.indices) { + if (text[index] == HYPHEN_MARKER && isHyphen(textPage, index) != 0 && startsLowerLine(textPage, index)) { + wraps += index + } + } + return wraps.toIntArray() +} + +/** Whether the character after [index] sits below it — which is what makes [index] the end of a line. */ +private fun startsLowerLine(textPage: Long, index: Int): Boolean { + val left = DoubleArray(1) + val right = DoubleArray(1) + val bottom = DoubleArray(1) + val top = DoubleArray(1) + + if (charBox(textPage, index, left, right, bottom, top) == 0) return false + val hyphenBottom = bottom[0] + if (charBox(textPage, index + 1, left, right, bottom, top) == 0) return false + return top[0] < hyphenBottom +} -private fun destroyLibrary() = kniBridge2() +private fun initLibrary() = kniBridge1() private fun loadDocument(bytes: ByteArray, password: String?): Long = kniBridge5(bytes, bytes.size, password) @@ -76,5 +110,16 @@ private fun closeTextPage(textPage: Long) = kniBridge53(textPage) private fun charCount(textPage: Long): Int = kniBridge54(textPage) +private fun isHyphen(textPage: Long, index: Int): Int = kniBridge58(textPage, index) + +private fun charBox( + textPage: Long, + index: Int, + left: DoubleArray, + right: DoubleArray, + bottom: DoubleArray, + top: DoubleArray, +): Int = kniBridge66(textPage, index, left, right, bottom, top) + private fun readText(textPage: Long, start: Int, count: Int, buffer: ShortArray): Int = kniBridge71(textPage, start, count, buffer) diff --git a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageText.kt b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageText.kt new file mode 100644 index 0000000..2b414b2 --- /dev/null +++ b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageText.kt @@ -0,0 +1,66 @@ +package io.github.lemcoder.mikromarkdown.pdf + +/** + * One page's text, and the line breaks pdfium collapsed while producing it. + * + * A hyphenated wrap does not reach [text] as a hyphen and a line break. pdfium recognises the wrap, joins the two + * lines, and leaves [HYPHEN_MARKER] standing where the hyphen was — so `con-` at the right margin and `firming` at the + * start of the next line arrive adjacent, with the marker between them and nothing to say a line ever ended there. + * + * [hyphenWraps] are the marker positions that `FPDFText_GetCharBox` confirms: the character after the marker really + * does begin a lower line. That is the one thing the geometry answers outright, and it is what [restored] needs. + */ +internal class PageText(val text: String, val hyphenWraps: IntArray) + +/** pdfium's stand-in for a hyphen it removed at a line break. */ +internal const val HYPHEN_MARKER: Char = '\uFFFE' + +/** + * Puts the wrap back the way the page shows it. + * + * Whether the hyphen was the author's (`chat-optimized`, broken across lines by chance) or the typesetter's + * (`con-firming`) is not something the geometry can answer: both sit at the right margin with the next line below. So + * the hyphen is written back as the page had it and the question is left open — [plainTextBlocks] + * [io.github.lemcoder.mikromarkdown.model.plainTextBlocks] closes the break without touching it. + * + * A marker the geometry did not confirm was not a wrap; there is no character it stands for, so nothing is written. + */ +internal fun List.restored(): String { + val hyphen = hyphenWritten() + return joinToString("\n") { it.restored(hyphen) } +} + +private fun PageText.restored(hyphen: Char): String { + if (HYPHEN_MARKER !in text) return text + + val wraps = hyphenWraps.toHashSet() + return buildString(text.length + hyphenWraps.size) { + for ((index, char) in text.withIndex()) { + when { + char != HYPHEN_MARKER -> append(char) + index in wraps -> append(hyphen).append('\n') + } + } + } +} + +/** + * The hyphen this document writes between words. + * + * The marker does not say which character it replaced — pdfium keeps no record of it, and the one call that could + * answer, `FPDFTextObj_GetText`, needs a header we do not bind. So the document is asked instead: whichever hyphen it + * uses in the compounds it did not break is the one to put back, rather than U+002D on a page set in U+2010. + */ +private fun List.hyphenWritten(): Char { + val counts = mutableMapOf() + for (page in this) { + for (match in WRITTEN_HYPHEN.findAll(page.text)) { + val hyphen = match.value[1] + counts[hyphen] = (counts[hyphen] ?: 0) + 1 + } + } + return counts.maxByOrNull { it.value }?.key ?: '-' +} + +/** A hyphen between two letters, which is one the document meant rather than one a line break produced. */ +private val WRITTEN_HYPHEN = Regex("\\p{L}[-\u2010\u2011]\\p{L}") diff --git a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt index 6e98fdb..891ccc2 100644 --- a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt +++ b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt @@ -15,8 +15,9 @@ import io.github.lemcoder.mikromarkdown.model.plainTextBlocks * ``` * * The extraction itself is per-platform — cinterop on native, generated JNI bridges on the JVM — but both reach the - * same pdfium, and everything above [extractText] is shared. Turning that text into paragraphs, including the - * de-hyphenation that pdfium's unmapped glyphs call for, is `plainTextBlocks`' job rather than a second copy here. + * same pdfium, and everything above [extractPages] is shared: each leg reports what pdfium saw, and one copy of the + * rules here turns it into a document. Deciding what a wrapped hyphen meant belongs to `plainTextBlocks`, which reads + * the whole document rather than one page. */ public class PdfiumConverter : DocumentConverter { @@ -25,8 +26,8 @@ public class PdfiumConverter : DocumentConverter { } override fun parse(bytes: ByteArray, info: StreamInfo): Document = - Document(blocks = plainTextBlocks(extractText(bytes))) + Document(blocks = plainTextBlocks(extractPages(bytes).restored())) } -/** Every page's text, concatenated, one page per line. */ -internal expect fun extractText(bytes: ByteArray): String +/** Every page, in order; [PageText] carries the text and the wraps pdfium collapsed. */ +internal expect fun extractPages(bytes: ByteArray): List diff --git a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt index 243f89a..020a32e 100644 --- a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt +++ b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt @@ -3,62 +3,96 @@ package io.github.lemcoder.mikromarkdown.pdf import pdfium.kniBridge1 import pdfium.kniBridge15 import pdfium.kniBridge16 -import pdfium.kniBridge2 import pdfium.kniBridge26 import pdfium.kniBridge27 import pdfium.kniBridge5 import pdfium.kniBridge52 import pdfium.kniBridge53 import pdfium.kniBridge54 +import pdfium.kniBridge58 +import pdfium.kniBridge66 import pdfium.kniBridge71 +/** + * pdfium is initialised once per process and never destroyed. + * + * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like: pairing them per call makes + * extraction differ between one process and the next, where initialising once gives every process the same answer. + */ +private val pdfiumLibrary: Lazy = lazy { initLibrary() } + /** * The JVM half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. * * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here * with the name from its doc comment, and nothing else in the module sees them. */ -internal actual fun extractText(bytes: ByteArray): String { - val text = StringBuilder() +internal actual fun extractPages(bytes: ByteArray): List { + val pages = mutableListOf() - initLibrary() + pdfiumLibrary.value + val document = loadDocument(bytes, null) + if (document == 0L) return emptyList() try { - val document = loadDocument(bytes, null) - if (document == 0L) return "" - try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) - if (page == 0L) continue - val textPage = loadTextPage(page) - if (textPage != 0L) { - text.append(pageText(textPage)) - text.append('\n') - closeTextPage(textPage) - } - closePage(page) + for (index in 0 until pageCount(document)) { + val page = loadPage(document, index) + if (page == 0L) continue + val textPage = loadTextPage(page) + if (textPage != 0L) { + pages += pageText(textPage) + closeTextPage(textPage) } - } finally { - closeDocument(document) + closePage(page) } } finally { - destroyLibrary() + closeDocument(document) } - return text.toString() + return pages } /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ -private fun pageText(textPage: Long): String { +private fun pageText(textPage: Long): PageText { val count = charCount(textPage) - if (count <= 0) return "" + if (count <= 0) return PageText("", IntArray(0)) val buffer = ShortArray(count + 1) val written = readText(textPage, 0, count, buffer) - return if (written <= 1) "" else CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + if (written <= 1) return PageText("", IntArray(0)) + // One character in, one character out, so a text index is a pdfium character index. + val text = CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + return PageText(text, hyphenWraps(textPage, text)) } -private fun initLibrary() = kniBridge1() +/** + * The collapsed wraps, as [PageText.hyphenWraps] describes them. + * + * `FPDFText_IsHyphen` marks every hyphen pdfium removed, and the character box is what proves the line ended there + * rather than the marker standing for something else on the same line. + */ +private fun hyphenWraps(textPage: Long, text: String): IntArray { + val wraps = mutableListOf() + for (index in text.indices) { + if (text[index] == HYPHEN_MARKER && isHyphen(textPage, index) != 0 && startsLowerLine(textPage, index)) { + wraps += index + } + } + return wraps.toIntArray() +} + +/** Whether the character after [index] sits below it — which is what makes [index] the end of a line. */ +private fun startsLowerLine(textPage: Long, index: Int): Boolean { + val left = DoubleArray(1) + val right = DoubleArray(1) + val bottom = DoubleArray(1) + val top = DoubleArray(1) + + if (charBox(textPage, index, left, right, bottom, top) == 0) return false + val hyphenBottom = bottom[0] + if (charBox(textPage, index + 1, left, right, bottom, top) == 0) return false + return top[0] < hyphenBottom +} -private fun destroyLibrary() = kniBridge2() +private fun initLibrary() = kniBridge1() private fun loadDocument(bytes: ByteArray, password: String?): Long = kniBridge5(bytes, bytes.size, password) @@ -76,5 +110,16 @@ private fun closeTextPage(textPage: Long) = kniBridge53(textPage) private fun charCount(textPage: Long): Int = kniBridge54(textPage) +private fun isHyphen(textPage: Long, index: Int): Int = kniBridge58(textPage, index) + +private fun charBox( + textPage: Long, + index: Int, + left: DoubleArray, + right: DoubleArray, + bottom: DoubleArray, + top: DoubleArray, +): Int = kniBridge66(textPage, index, left, right, bottom, top) + private fun readText(textPage: Long, start: Int, count: Int, buffer: ShortArray): Int = kniBridge71(textPage, start, count, buffer) diff --git a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageTextTest.kt b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageTextTest.kt new file mode 100644 index 0000000..adfdb12 --- /dev/null +++ b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageTextTest.kt @@ -0,0 +1,61 @@ +package io.github.lemcoder.mikromarkdown.pdf + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** Pins what [restored] does with pdfium's hyphen marker, without needing a PDF that happens to contain one. */ +class PageTextTest { + + private val marker = HYPHEN_MARKER + + @Test + fun `a confirmed wrap gets its hyphen and its line break back`() { + val page = PageText("con${marker}firming it", intArrayOf(3)) + + assertEquals("con-\nfirming it", listOf(page).restored()) + } + + @Test + fun `the hyphen put back is the one the document writes`() { + // U+2010 in the compound the document did not break, so U+2010 is what the wrap gets. + val page = PageText("con${marker}firming a well\u2010known case", intArrayOf(3)) + + assertEquals("con\u2010\nfirming a well\u2010known case", listOf(page).restored()) + } + + @Test + fun `a document with no written hyphen falls back to U+002D`() { + val page = PageText("con${marker}firming it", intArrayOf(3)) + + assertEquals('-', listOf(page).restored()[3]) + } + + @Test + fun `the hyphen is chosen across every page, not one at a time`() { + val first = PageText("a well\u2010known case", IntArray(0)) + val second = PageText("con${marker}firming it", intArrayOf(3)) + + assertEquals("a well\u2010known case\ncon\u2010\nfirming it", listOf(first, second).restored()) + } + + @Test + fun `a marker the geometry did not confirm stands for nothing`() { + val page = PageText("Section $marker 3", IntArray(0)) + + assertEquals("Section 3", listOf(page).restored()) + } + + @Test + fun `each marker is judged on its own`() { + val page = PageText("con${marker}firming and Section $marker 3", intArrayOf(3)) + + assertEquals("con-\nfirming and Section 3", listOf(page).restored()) + } + + @Test + fun `text without markers is returned as it is`() { + val page = PageText("nothing to restore", IntArray(0)) + + assertEquals("nothing to restore", listOf(page).restored()) + } +} diff --git a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt index fa85c96..e3351f9 100644 --- a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt +++ b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt @@ -4,6 +4,7 @@ import io.github.lemcoder.mikromarkdown.StreamInfo import java.io.File import kotlin.test.Test import kotlin.test.assertContains +import kotlin.test.assertEquals import kotlin.test.assertTrue class PdfiumConverterTest { @@ -17,8 +18,27 @@ class PdfiumConverterTest { assertTrue(text.length > 1000, "expected a page of text, got ${text.length} characters") assertContains(text, "Introduction") - // The de-hyphenation ran: pdfium reports a broken word with U+FFFE between the halves. - assertContains(text, "confirming") - assertTrue('￾' !in text, "unmapped glyphs should not reach the model") + // A wrap the character boxes confirmed: the break is gone and the hyphen the page shows is still there. + assertContains(text, "con-firming") + // The other kind, the author's own, comes through the same way rather than fused into "chatoptimized". + assertContains(text, "chat-optimized") + assertTrue(HYPHEN_MARKER !in text, "pdfium's hyphen marker should not reach the model") + } + + /** + * Every marker in this paper is a wrap, and the character boxes say so. + * + * The assertion is the geometry's, not the text's: it is what separates "pdfium removed a hyphen here" from "the + * line ended here", and only the second one earns a break back. + */ + @Test + fun `every hyphen marker in the fixture is confirmed by its character box`() { + val pages = extractPages(fixture.readBytes()) + + val markers = pages.sumOf { page -> page.text.count { it == HYPHEN_MARKER } } + val confirmed = pages.sumOf { it.hyphenWraps.size } + + assertTrue(markers > 0, "the fixture is supposed to contain hyphenated wraps") + assertEquals(markers, confirmed, "a marker the geometry did not confirm would be dropped silently") } } diff --git a/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt b/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt index 655e0f1..3b0ed36 100644 --- a/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt +++ b/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt @@ -1,63 +1,113 @@ package io.github.lemcoder.mikromarkdown.pdf +import kotlinx.cinterop.DoubleVar import kotlinx.cinterop.ExperimentalForeignApi import kotlinx.cinterop.UShortVar import kotlinx.cinterop.addressOf +import kotlinx.cinterop.alloc import kotlinx.cinterop.allocArray import kotlinx.cinterop.get import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr import kotlinx.cinterop.usePinned +import kotlinx.cinterop.value import pdfium.FPDFText_ClosePage import pdfium.FPDFText_CountChars +import pdfium.FPDFText_GetCharBox import pdfium.FPDFText_GetText +import pdfium.FPDFText_IsHyphen import pdfium.FPDFText_LoadPage import pdfium.FPDF_CloseDocument import pdfium.FPDF_ClosePage -import pdfium.FPDF_DestroyLibrary import pdfium.FPDF_GetPageCount import pdfium.FPDF_InitLibrary import pdfium.FPDF_LoadMemDocument import pdfium.FPDF_LoadPage +/** + * pdfium is initialised once per process and never destroyed. + * + * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like. Pairing them per call makes + * extraction differ between one process and the next; initialising once gives every process the same sequence, which is + * the only thing this buys. + * + * It does not make extraction repeatable. pdfium carries state across document loads that the first conversion in a + * process does not see, and from the second conversion on the AutoGen fixture comes back four generated spaces short — + * `AutoGen uses` reads as `AutoGenuses`. Destroying the library between conversions does not reset it. Unfixed, and + * below this line rather than in it. + */ +@OptIn(ExperimentalForeignApi::class) private val pdfiumLibrary: Lazy = lazy { FPDF_InitLibrary() } + /** The native half, over the cinterop bindings. */ @OptIn(ExperimentalForeignApi::class) -internal actual fun extractText(bytes: ByteArray): String { - val text = StringBuilder() - - FPDF_InitLibrary() - try { - bytes.usePinned { pinned -> - val document = FPDF_LoadMemDocument(pinned.addressOf(0), bytes.size, null) ?: return@usePinned - try { - for (index in 0 until FPDF_GetPageCount(document)) { - val page = FPDF_LoadPage(document, index) ?: continue - val textPage = FPDFText_LoadPage(page) - if (textPage != null) { - text.append(pageText(textPage)) - text.append('\n') - FPDFText_ClosePage(textPage) - } - FPDF_ClosePage(page) +internal actual fun extractPages(bytes: ByteArray): List { + val pages = mutableListOf() + + pdfiumLibrary.value + bytes.usePinned { pinned -> + val document = FPDF_LoadMemDocument(pinned.addressOf(0), bytes.size, null) ?: return@usePinned + try { + for (index in 0 until FPDF_GetPageCount(document)) { + val page = FPDF_LoadPage(document, index) ?: continue + val textPage = FPDFText_LoadPage(page) + if (textPage != null) { + pages += pageText(textPage) + FPDFText_ClosePage(textPage) } - } finally { - FPDF_CloseDocument(document) + FPDF_ClosePage(page) } + } finally { + FPDF_CloseDocument(document) } - } finally { - FPDF_DestroyLibrary() } - return text.toString() + return pages } /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ @OptIn(ExperimentalForeignApi::class) -private fun pageText(textPage: pdfium.FPDF_TEXTPAGE): String { +private fun pageText(textPage: pdfium.FPDF_TEXTPAGE): PageText { val count = FPDFText_CountChars(textPage) - if (count <= 0) return "" - return memScoped { + if (count <= 0) return PageText("", IntArray(0)) + val text = memScoped { val buffer = allocArray(count + 1) val written = FPDFText_GetText(textPage, 0, count, buffer) + // One character in, one character out, so a text index is a pdfium character index. if (written <= 1) "" else CharArray(written - 1) { Char(buffer[it].toInt()) }.concatToString() } + if (text.isEmpty()) return PageText("", IntArray(0)) + return PageText(text, hyphenWraps(textPage, text)) +} + +/** + * The collapsed wraps, as [PageText.hyphenWraps] describes them. + * + * `FPDFText_IsHyphen` marks every hyphen pdfium removed, and the character box is what proves the line ended there + * rather than the marker standing for something else on the same line. + */ +@OptIn(ExperimentalForeignApi::class) +private fun hyphenWraps(textPage: pdfium.FPDF_TEXTPAGE, text: String): IntArray { + val wraps = mutableListOf() + for (index in text.indices) { + if ( + text[index] == HYPHEN_MARKER && FPDFText_IsHyphen(textPage, index) != 0 && startsLowerLine(textPage, index) + ) { + wraps += index + } + } + return wraps.toIntArray() +} + +/** Whether the character after [index] sits below it — which is what makes [index] the end of a line. */ +@OptIn(ExperimentalForeignApi::class) +private fun startsLowerLine(textPage: pdfium.FPDF_TEXTPAGE, index: Int): Boolean = memScoped { + val left = alloc() + val right = alloc() + val bottom = alloc() + val top = alloc() + + if (FPDFText_GetCharBox(textPage, index, left.ptr, right.ptr, bottom.ptr, top.ptr) == 0) return false + val hyphenBottom = bottom.value + if (FPDFText_GetCharBox(textPage, index + 1, left.ptr, right.ptr, bottom.ptr, top.ptr) == 0) return false + top.value < hyphenBottom } From a3b45e7c11b493d8ddccf09eeef7d450cfa45e93 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Thu, 20 Aug 2026 00:56:09 +0200 Subject: [PATCH 4/9] Load PDFs through a file pdfium owns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FPDF_LoadMemDocument keeps the caller's pointer and reads through it for as long as the document is open. The generated bridge does not: jbyte* p0 = (*jniEnv)->GetByteArrayElements(jniEnv, j0, 0); jlong kniResult = (jlong)FPDF_LoadMemDocument((void*)p0, p1, ...); (*jniEnv)->ReleaseByteArrayElements(jniEnv, j0, p0, 0); return kniResult; It releases before returning, so every page load and text extraction afterwards reads memory the JVM has taken back. Filling the array with zeros under an open document drops extraction to nothing, and forcing malloc to hand that block to something else does the same — pdfium was reading our array directly, not a copy of it. It mostly worked, which is the worst way to fail. The first conversion in a process reads a region nothing has touched yet and is correct; later ones read reused memory and come back four generated spaces short, so "AutoGen uses" reads as "AutoGenuses". Repeated extraction of one fixture gave 5220 / 5216 / 5220 / 5216 characters within a single process and a different sequence in the next. Through FPDF_LoadDocument, which owns everything it reads, it is 5220 every time in every process. The temporary file costs 0.12 ms against 2.66 ms of parse for a 90 KB PDF. It is created before the load and deleted only after FPDF_CloseDocument, because pdfium reads a path as lazily as it reads a buffer. The native leg needs none of this and is untouched: usePinned there holds the array for the document's whole lifetime, which is why only the JVM legs were wrong. Pairing FPDF_InitLibrary with FPDF_DestroyLibrary per call turned out to make extraction differ between one process and the next, so the library is initialised once and left for the process to reclaim. Co-Authored-By: Claude Opus 5 --- .../mikromarkdown/pdf/Pdfium.android.kt | 42 +++++++++++++------ .../lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt | 42 +++++++++++++------ .../mikromarkdown/pdf/PdfiumConverterTest.kt | 17 ++++++++ 3 files changed, 75 insertions(+), 26 deletions(-) diff --git a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt index b662583..87e2723 100644 --- a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt +++ b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt @@ -1,11 +1,12 @@ package io.github.lemcoder.mikromarkdown.pdf +import java.io.File import pdfium.kniBridge1 import pdfium.kniBridge15 import pdfium.kniBridge16 import pdfium.kniBridge26 import pdfium.kniBridge27 -import pdfium.kniBridge5 +import pdfium.kniBridge4 import pdfium.kniBridge52 import pdfium.kniBridge53 import pdfium.kniBridge54 @@ -26,26 +27,41 @@ private val pdfiumLibrary: Lazy = lazy { initLibrary() } * * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here * with the name from its doc comment, and nothing else in the module sees them. + * + * The bytes reach pdfium as a file rather than as a buffer. `FPDF_LoadMemDocument` keeps the caller's pointer and + * reads through it for as long as the document is open, but the generated bridge pairs `GetByteArrayElements` with + * `ReleaseByteArrayElements` and lets go before it returns, so every page load after that reads memory the JVM has + * taken back. It mostly works, which is the worst way to fail: the first conversion in a process reads an untouched + * region and is correct, later ones come back four generated spaces short and `AutoGen uses` reads as `AutoGenuses`. + * Zeroing the array under an open document drops extraction to nothing, which is what proved it. `FPDF_LoadDocument` + * owns everything it reads. The native leg needs none of this: `usePinned` holds the array for the document's life. */ internal actual fun extractPages(bytes: ByteArray): List { val pages = mutableListOf() pdfiumLibrary.value - val document = loadDocument(bytes, null) - if (document == 0L) return emptyList() + // The file has to outlive the document: pdfium reads it lazily, the same way it would read a buffer. + val file = File.createTempFile("mikromarkdown", ".pdf") try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) - if (page == 0L) continue - val textPage = loadTextPage(page) - if (textPage != 0L) { - pages += pageText(textPage) - closeTextPage(textPage) + file.writeBytes(bytes) + val document = loadDocument(file.absolutePath, null) + if (document == 0L) return emptyList() + try { + for (index in 0 until pageCount(document)) { + val page = loadPage(document, index) + if (page == 0L) continue + val textPage = loadTextPage(page) + if (textPage != 0L) { + pages += pageText(textPage) + closeTextPage(textPage) + } + closePage(page) } - closePage(page) + } finally { + closeDocument(document) } } finally { - closeDocument(document) + file.delete() } return pages @@ -94,7 +110,7 @@ private fun startsLowerLine(textPage: Long, index: Int): Boolean { private fun initLibrary() = kniBridge1() -private fun loadDocument(bytes: ByteArray, password: String?): Long = kniBridge5(bytes, bytes.size, password) +private fun loadDocument(path: String, password: String?): Long = kniBridge4(path, password) private fun pageCount(document: Long): Int = kniBridge15(document) diff --git a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt index 020a32e..4c1c878 100644 --- a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt +++ b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt @@ -1,11 +1,12 @@ package io.github.lemcoder.mikromarkdown.pdf +import java.io.File import pdfium.kniBridge1 import pdfium.kniBridge15 import pdfium.kniBridge16 import pdfium.kniBridge26 import pdfium.kniBridge27 -import pdfium.kniBridge5 +import pdfium.kniBridge4 import pdfium.kniBridge52 import pdfium.kniBridge53 import pdfium.kniBridge54 @@ -26,26 +27,41 @@ private val pdfiumLibrary: Lazy = lazy { initLibrary() } * * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here * with the name from its doc comment, and nothing else in the module sees them. + * + * The bytes reach pdfium as a file rather than as a buffer. `FPDF_LoadMemDocument` keeps the caller's pointer and reads + * through it for as long as the document is open, but the generated bridge pairs `GetByteArrayElements` with + * `ReleaseByteArrayElements` and lets go before it returns, so every page load after that reads memory the JVM has + * taken back. It mostly works, which is the worst way to fail: the first conversion in a process reads an untouched + * region and is correct, later ones come back four generated spaces short and `AutoGen uses` reads as `AutoGenuses`. + * Zeroing the array under an open document drops extraction to nothing, which is what proved it. `FPDF_LoadDocument` + * owns everything it reads. The native leg needs none of this: `usePinned` holds the array for the document's life. */ internal actual fun extractPages(bytes: ByteArray): List { val pages = mutableListOf() pdfiumLibrary.value - val document = loadDocument(bytes, null) - if (document == 0L) return emptyList() + // The file has to outlive the document: pdfium reads it lazily, the same way it would read a buffer. + val file = File.createTempFile("mikromarkdown", ".pdf") try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) - if (page == 0L) continue - val textPage = loadTextPage(page) - if (textPage != 0L) { - pages += pageText(textPage) - closeTextPage(textPage) + file.writeBytes(bytes) + val document = loadDocument(file.absolutePath, null) + if (document == 0L) return emptyList() + try { + for (index in 0 until pageCount(document)) { + val page = loadPage(document, index) + if (page == 0L) continue + val textPage = loadTextPage(page) + if (textPage != 0L) { + pages += pageText(textPage) + closeTextPage(textPage) + } + closePage(page) } - closePage(page) + } finally { + closeDocument(document) } } finally { - closeDocument(document) + file.delete() } return pages @@ -94,7 +110,7 @@ private fun startsLowerLine(textPage: Long, index: Int): Boolean { private fun initLibrary() = kniBridge1() -private fun loadDocument(bytes: ByteArray, password: String?): Long = kniBridge5(bytes, bytes.size, password) +private fun loadDocument(path: String, password: String?): Long = kniBridge4(path, password) private fun pageCount(document: Long): Int = kniBridge15(document) diff --git a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt index e3351f9..4043762 100644 --- a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt +++ b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt @@ -25,6 +25,23 @@ class PdfiumConverterTest { assertTrue(HYPHEN_MARKER !in text, "pdfium's hyphen marker should not reach the model") } + /** + * Converting twice in one process must read the same both times. + * + * It did not. `FPDF_LoadMemDocument` reads the caller's buffer for as long as the document is open, and the + * generated bridge released it before returning, so pdfium spent every page load reading memory the JVM had taken + * back. The first conversion in a process was right and the rest lost four generated spaces. + */ + @Test + fun `converting the same bytes twice gives the same document`() { + val bytes = fixture.readBytes() + val first = PdfiumConverter().parse(bytes, StreamInfo(extension = "pdf")) + val second = PdfiumConverter().parse(bytes, StreamInfo(extension = "pdf")) + + assertEquals(first.blocks, second.blocks) + assertContains(first.blocks.joinToString { it.toString() }, "AutoGen uses") + } + /** * Every marker in this paper is a wrap, and the character boxes say so. * From b3dc095890f0a2d6ffb685147ecdbdf6dec86901 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Thu, 20 Aug 2026 00:56:24 +0200 Subject: [PATCH 5/9] Pin anydoc to v0.1.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark and comparison figures are measured against whatever the submodule happens to be checked out at, and nothing recorded which version that was. v0.1.8 changes strikethrough and footnote markers in its PDF snapshot; it does not change the hyphen handling the comparison turns on — wrapped lines are still joined with an unconditional space, so eleven of the twelve "word- word" sites in its output of our fixture are broken words. That is pdf-inspector's doing rather than anydoc's, and it is unreported upstream. README now names both pinned versions and the fetch a fresh clone needs. Co-Authored-By: Claude Opus 5 --- README.md | 8 ++++++++ third-party/anydoc | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 62889ed..fbd4a8e 100644 --- a/README.md +++ b/README.md @@ -218,3 +218,11 @@ python3 scripts/benchmark.py Engines whose CLI is missing are skipped. anydoc only handles binary formats, so it sits out the HTML/JSON/XML fixtures. + +Both comparison engines are submodules pinned to the versions the figures were measured against — +anydoc v0.1.8, markitdown v0.1.6 — so a clone needs them fetched before either script has anything +to compare with: + +```bash +git submodule update --init +``` diff --git a/third-party/anydoc b/third-party/anydoc index 4a45add..4e3089b 160000 --- a/third-party/anydoc +++ b/third-party/anydoc @@ -1 +1 @@ -Subproject commit 4a45addbd607e8b59f0c263bca26aab228e10370 +Subproject commit 4e3089b1ed43404241a303109f81e2c7933040b2 From 8ef6d16a65edb4c6fb2ecd64b2cac8cd2a48fd18 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Thu, 20 Aug 2026 01:02:51 +0200 Subject: [PATCH 6/9] Order the Kotlin/Native download before the JNI generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The macOS CI leg added a commit ago failed on a clean runner: Execution failed for task ':pdfium:generateJvmInteropPdfium' > No Kotlin/Native distribution found. The Kotlin plugin downloads one when it first compiles a native target, so build one of those first. The generator reads that distribution, and the Kotlin plugin fetches it only as a side effect of compiling a native target. Every machine that had built one already — mine, and any working tree that had run the native target once — hid it. It is not a CI problem: `./gradlew :pdfium:jvmTest` in a fresh clone fails the same way, which is why the ordering goes in the build rather than in the workflow. Co-Authored-By: Claude Opus 5 --- pdfium/build.gradle.kts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pdfium/build.gradle.kts b/pdfium/build.gradle.kts index 25ad398..eebdd3e 100644 --- a/pdfium/build.gradle.kts +++ b/pdfium/build.gradle.kts @@ -102,3 +102,11 @@ tasks.matching { it.name.startsWith("cinteropPdfium") }.configureEach { dependsO tasks .matching { it.name.startsWith("generateJvmInterop") || it.name.startsWith("cmakeConfigure") } .configureEach { dependsOn("downloadPdfium") } + +// The JNI generator reads the Kotlin/Native distribution, and the Kotlin plugin fetches that only when +// something compiles a native target. On a machine that has never built one — a CI runner, a fresh clone +// running `:pdfium:jvmTest` on its own — generating the bindings fails with "No Kotlin/Native distribution +// found" instead, so the download is ordered before it rather than left to whatever else ran first. +tasks + .matching { it.name.startsWith("generateJvmInterop") } + .configureEach { dependsOn("downloadKotlinNativeDistribution") } From 615b6ffb3fb713c27259463377c86c571a5d23e5 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Thu, 20 Aug 2026 01:38:08 +0200 Subject: [PATCH 7/9] Drop the JVM-CLI baseline from the comparison benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `python3 scripts/benchmark.py`, as the README documents it, died on mikro = next(e for e in engines if e.name == "mikromarkdown") StopIteration The engine was renamed to "native" when the JVM CLI went and the Kotlin/ Native binary replaced it. Two references were left behind, both there to measure JVM startup and subtract it from the CLI's timing — a correction with nothing left to correct, since the native binary starts no runtime. The header line that reported the baseline goes with them. Co-Authored-By: Claude Opus 5 --- scripts/benchmark.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 35b6406..80a3e48 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -18,7 +18,7 @@ structure headings / tables / list items / links / images / code fences table health share of table rows whose column count matches the header hygiene 3+ blank-line runs, trailing whitespace, unescaped pipes - speed best of N CLI runs, and the same minus measured JVM startup + speed best of N CLI runs """ from __future__ import annotations @@ -196,15 +196,6 @@ def main() -> int: state = "ready" if engine.command else f"MISSING ({engine.note})" print(f"{engine.name:14} {state}", file=sys.stderr) - # JVM startup cost, so the Kotlin CLI's timing can be reported with and without it. - jvm_baseline = 0.0 - mikro = next(e for e in engines if e.name == "mikromarkdown") - if mikro.command: - probe = out_dir / "_probe.txt" - probe.write_text("x\n") - _, jvm_baseline = mikro.run(probe) - probe.unlink() - results: dict[str, dict[str, object]] = {} for fixture in fixtures: print(f"converting {fixture.name}", file=sys.stderr) @@ -218,8 +209,7 @@ def main() -> int: results[fixture.name] = per_engine lines: list[str] = ["# Conversion benchmark", ""] - lines.append(f"Fixtures: {len(fixtures)} · runs per file: {RUNS} (best kept) · " - f"measured JVM startup: {jvm_baseline * 1000:.0f} ms") + lines.append(f"Fixtures: {len(fixtures)} · runs per file: {RUNS} (best kept)") lines.append("") lines.append("Recall is against the *consensus* vocabulary: tokens (>3 chars) that at least two " "engines emit for the same file. Missing tokens mean dropped content.") @@ -248,11 +238,7 @@ def main() -> int: recall = len(own & consensus) / len(consensus) if consensus else 1.0 unique = len(own - set().union(*[v for k, v in token_sets.items() if k != engine_name]) ) \ if len(token_sets) > 1 else len(own) - ms = data["seconds"] * 1000 - if engine_name == "mikromarkdown": - ms_display = f"{ms:.0f} ({max(0.0, ms - jvm_baseline * 1000):.0f} warm)" - else: - ms_display = f"{ms:.0f}" + ms_display = f"{data['seconds'] * 1000:.0f}" lines.append( f"| {engine_name} | {recall * 100:.1f}% | {unique} | {len(text)} | {s['headings']} | " f"{s['list_items']} | {s['links']} | {s['images']} | {s['tables']} | {s['table_rows']} | " From 23eadd9f4a64001ee73d537f6a172f1b5cd460f9 Mon Sep 17 00:00:00 2001 From: mikolaj Date: Thu, 20 Aug 2026 02:03:51 +0200 Subject: [PATCH 8/9] Let the pdfium binding carry its own linker options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `:cli-native` named pdfium a second time — a `-L`, a `-lpdfium` and an rpath repeated in the consumer, with a comment saying Kotlin/Native does not carry a klib's linker options to the binary that uses it. It does, through one channel only: `linkerOpts` in the cinterop `.def`, which cinterop writes into the klib manifest and passes to whatever links the binding. Two things it will not do, both tried and both refused. `extraOpts ("-linker-option", ...)` earns "-linker-option(s)/-linkerOpts option is not supported by cinterop. Please add linker options to .def file or binary compilation instead." And `libraryPaths`, which the klib does record, is cinterop's own search path: the consumer's linker never sees it, and the link fails with "library 'pdfium' not found". Nor can the paths be relative. kotlin-native#2314 reported that relative paths in a .def resolve against the working directory the compiler runs in rather than the file's own location, and that still holds on 2.3.21 — the same .def indexes cleanly from its own directory and fails with "'dummy.h' file not found" from the repository root. An rpath has to hold at run time regardless of where the binary is started from, so it could not be relative even if the def resolved sensibly. That leaves absolute paths, and pdfium is unpacked under build/, so the .def is written at build time from the checked-in one. The checked-in one is unchanged: the JVM and Android legs bind it, and a macOS library path has no business in it. Static linking would be the better answer, and the mechanism is there — `staticLibraries` packs the archive into the klib, verified against a dummy library that lands at targets//included/libdummy.a, and a consumer then needs no options at all. pdfium-binaries publishes no static build, all 46 assets of chromium/8009 being shared libraries, so the 0.8 ms of dyld work the 7.2 MB dylib costs stays where it is. Co-Authored-By: Claude Opus 5 --- cli-native/build.gradle.kts | 12 +++--------- pdfium/build.gradle.kts | 38 ++++++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/cli-native/build.gradle.kts b/cli-native/build.gradle.kts index f554494..50711d0 100644 --- a/cli-native/build.gradle.kts +++ b/cli-native/build.gradle.kts @@ -2,15 +2,9 @@ plugins { alias(libs.plugins.kotlinMultiplatform) } kotlin { macosArm64 { - // Kotlin/Native does not carry a klib's linker options to the binary that uses it, so the - // consumer names pdfium itself. Worth turning into a shared convention if a second - // consumer appears. - val pdfiumLib = rootProject.layout.projectDirectory.dir("pdfium/build/pdfium/mac-arm64/lib").asFile - - binaries.executable { - entryPoint = "io.github.lemcoder.mikromarkdown.cli.main" - linkerOpts("-L${pdfiumLib.absolutePath}", "-lpdfium", "-rpath", pdfiumLib.absolutePath) - } + // Nothing about pdfium here: its cinterop .def records where the library is, and cinterop + // hands those options to whatever links the binding. + binaries.executable { entryPoint = "io.github.lemcoder.mikromarkdown.cli.main" } compilerOptions { // Worth about 8% on large inputs and nothing on small ones. Measured, not assumed: diff --git a/pdfium/build.gradle.kts b/pdfium/build.gradle.kts index eebdd3e..68353c3 100644 --- a/pdfium/build.gradle.kts +++ b/pdfium/build.gradle.kts @@ -18,13 +18,45 @@ apply(from = "pdfium-binaries.gradle.kts") @Suppress("UNCHECKED_CAST") val pdfiumAbis = extra["pdfiumAbis"] as Map +val pdfiumLibDir = pdfiumRoot.dir("mac-arm64/lib").asFile.absolutePath + +val sourceDef = layout.projectDirectory.file("src/nativeInterop/cinterop/pdfium.def") + +val linkedDef = layout.buildDirectory.file("cinterop/pdfium-linked.def") + +/** + * The .def the native target binds, with pdfium's location written into it. + * + * cinterop records a .def's `linkerOpts` in the klib and hands them to whatever links against it, which is what lets a + * consumer use the binding without naming pdfium a second time. The paths have to be absolute, for two reasons: a + * relative one in a .def resolves against the working directory the compiler happens to run in rather than the file's + * own location (kotlin-native#2314, still true on 2.3.21), and an rpath has to hold at run time wherever the binary is + * started from. `libraryPaths` is no way round it either — the klib records it, but it is cinterop's own search path + * and never reaches the consumer's linker, which then fails with "library 'pdfium' not found". + * + * So the file is generated. The JVM and Android legs bind the checked-in .def, which stays free of a macOS path. + */ +val writeLinkedDef by tasks.registering { + val from = sourceDef + val into = linkedDef + val libDir = pdfiumLibDir + inputs.file(from) + inputs.property("libDir", libDir) + outputs.file(into) + doLast { + val target = into.get().asFile + target.parentFile.mkdirs() + // The archive ships a dylib, so the binary carries an rpath to find it at run time. + target.writeText(from.asFile.readText().trimEnd() + "\nlinkerOpts = -L$libDir -lpdfium -rpath $libDir\n") + } +} + kotlin { macosArm64 { val platform = pdfiumRoot.dir("mac-arm64") compilations.getByName("main").cinterops.create("pdfium") { - defFile("src/nativeInterop/cinterop/pdfium.def") + defFile(linkedDef.get().asFile) includeDirs(platform.dir("include")) - // The archive ships a dylib, so the binary carries an rpath to find it at run time. extraOpts("-libraryPath", platform.dir("lib").asFile.absolutePath) } } @@ -97,7 +129,7 @@ kotlin.targets.getByName("android").compilations.getByName("main").jvmInterops { } // Every binding path needs the headers and the library unpacked first. -tasks.matching { it.name.startsWith("cinteropPdfium") }.configureEach { dependsOn("downloadPdfium") } +tasks.matching { it.name.startsWith("cinteropPdfium") }.configureEach { dependsOn("downloadPdfium", writeLinkedDef) } tasks .matching { it.name.startsWith("generateJvmInterop") || it.name.startsWith("cmakeConfigure") } From b1731e3732752c839559eb8ac4410f2b2bab8ece Mon Sep 17 00:00:00 2001 From: mikolaj Date: Fri, 28 Aug 2026 03:08:38 +0200 Subject: [PATCH 9/9] Call pdfium's functions by their names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Konan plugin numbered the bridges, so this module imported `kniBridge54` and wrapped it in `charCount(textPage)` to give it a name — eleven such wrappers, existing only to undo the numbering. konanplugin 1.2.0-alpha07 names each binding after the C function it calls, so the wrappers have nothing left to do and the code reads as pdfium's own API: val document = FPDF_LoadDocument(file.absolutePath, null) for (index in 0 until FPDF_GetPageCount(document)) { That is 62 lines out of the two platform legs, and a stack frame or linker error now names something that can be looked up in pdfium's headers. Co-Authored-By: Claude Opus 5 --- gradle/libs.versions.toml | 2 +- .../mikromarkdown/pdf/Pdfium.android.kt | 88 ++++++------------- .../lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt | 88 ++++++------------- 3 files changed, 58 insertions(+), 120 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 6e90a5f..835b0ed 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,7 +6,7 @@ android-compileSdk = "37" kotlinx-io = "0.9.0" ksoup = "0.2.6" korlibs = "6.0.0" -konanplugin = "1.2.0-alpha06" +konanplugin = "1.2.0-alpha07" kotlinx-resources = "0.15.0" junit = "6.1.0" detekt = "1.23.8" diff --git a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt index 87e2723..9e26afd 100644 --- a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt +++ b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt @@ -1,18 +1,18 @@ package io.github.lemcoder.mikromarkdown.pdf import java.io.File -import pdfium.kniBridge1 -import pdfium.kniBridge15 -import pdfium.kniBridge16 -import pdfium.kniBridge26 -import pdfium.kniBridge27 -import pdfium.kniBridge4 -import pdfium.kniBridge52 -import pdfium.kniBridge53 -import pdfium.kniBridge54 -import pdfium.kniBridge58 -import pdfium.kniBridge66 -import pdfium.kniBridge71 +import pdfium.FPDF_InitLibrary +import pdfium.FPDF_GetPageCount +import pdfium.FPDF_LoadPage +import pdfium.FPDF_ClosePage +import pdfium.FPDF_CloseDocument +import pdfium.FPDF_LoadDocument +import pdfium.FPDFText_LoadPage +import pdfium.FPDFText_ClosePage +import pdfium.FPDFText_CountChars +import pdfium.FPDFText_IsHyphen +import pdfium.FPDFText_GetCharBox +import pdfium.FPDFText_GetText /** * pdfium is initialised once per process and never destroyed. @@ -20,13 +20,13 @@ import pdfium.kniBridge71 * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like: pairing them per call makes * extraction differ between one process and the next, where initialising once gives every process the same answer. */ -private val pdfiumLibrary: Lazy = lazy { initLibrary() } +private val pdfiumLibrary: Lazy = lazy { FPDF_InitLibrary() } /** * The Android half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. * - * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here - * with the name from its doc comment, and nothing else in the module sees them. + * The bindings carry the names of the C functions they call, so this reads as pdfium's own API and needs no wrapper + * layer in between. * * The bytes reach pdfium as a file rather than as a buffer. `FPDF_LoadMemDocument` keeps the caller's pointer and * reads through it for as long as the document is open, but the generated bridge pairs `GetByteArrayElements` with @@ -44,21 +44,21 @@ internal actual fun extractPages(bytes: ByteArray): List { val file = File.createTempFile("mikromarkdown", ".pdf") try { file.writeBytes(bytes) - val document = loadDocument(file.absolutePath, null) + val document = FPDF_LoadDocument(file.absolutePath, null) if (document == 0L) return emptyList() try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) + for (index in 0 until FPDF_GetPageCount(document)) { + val page = FPDF_LoadPage(document, index) if (page == 0L) continue - val textPage = loadTextPage(page) + val textPage = FPDFText_LoadPage(page) if (textPage != 0L) { pages += pageText(textPage) - closeTextPage(textPage) + FPDFText_ClosePage(textPage) } - closePage(page) + FPDF_ClosePage(page) } } finally { - closeDocument(document) + FPDF_CloseDocument(document) } } finally { file.delete() @@ -69,10 +69,10 @@ internal actual fun extractPages(bytes: ByteArray): List { /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ private fun pageText(textPage: Long): PageText { - val count = charCount(textPage) + val count = FPDFText_CountChars(textPage) if (count <= 0) return PageText("", IntArray(0)) val buffer = ShortArray(count + 1) - val written = readText(textPage, 0, count, buffer) + val written = FPDFText_GetText(textPage, 0, count, buffer) if (written <= 1) return PageText("", IntArray(0)) // One character in, one character out, so a text index is a pdfium character index. val text = CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() @@ -88,9 +88,8 @@ private fun pageText(textPage: Long): PageText { private fun hyphenWraps(textPage: Long, text: String): IntArray { val wraps = mutableListOf() for (index in text.indices) { - if (text[index] == HYPHEN_MARKER && isHyphen(textPage, index) != 0 && startsLowerLine(textPage, index)) { - wraps += index - } + if (text[index] != HYPHEN_MARKER) continue + if (FPDFText_IsHyphen(textPage, index) != 0 && startsLowerLine(textPage, index)) wraps += index } return wraps.toIntArray() } @@ -102,40 +101,9 @@ private fun startsLowerLine(textPage: Long, index: Int): Boolean { val bottom = DoubleArray(1) val top = DoubleArray(1) - if (charBox(textPage, index, left, right, bottom, top) == 0) return false + if (FPDFText_GetCharBox(textPage, index, left, right, bottom, top) == 0) return false val hyphenBottom = bottom[0] - if (charBox(textPage, index + 1, left, right, bottom, top) == 0) return false + if (FPDFText_GetCharBox(textPage, index + 1, left, right, bottom, top) == 0) return false return top[0] < hyphenBottom } -private fun initLibrary() = kniBridge1() - -private fun loadDocument(path: String, password: String?): Long = kniBridge4(path, password) - -private fun pageCount(document: Long): Int = kniBridge15(document) - -private fun loadPage(document: Long, index: Int): Long = kniBridge16(document, index) - -private fun closePage(page: Long) = kniBridge26(page) - -private fun closeDocument(document: Long) = kniBridge27(document) - -private fun loadTextPage(page: Long): Long = kniBridge52(page) - -private fun closeTextPage(textPage: Long) = kniBridge53(textPage) - -private fun charCount(textPage: Long): Int = kniBridge54(textPage) - -private fun isHyphen(textPage: Long, index: Int): Int = kniBridge58(textPage, index) - -private fun charBox( - textPage: Long, - index: Int, - left: DoubleArray, - right: DoubleArray, - bottom: DoubleArray, - top: DoubleArray, -): Int = kniBridge66(textPage, index, left, right, bottom, top) - -private fun readText(textPage: Long, start: Int, count: Int, buffer: ShortArray): Int = - kniBridge71(textPage, start, count, buffer) diff --git a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt index 4c1c878..2686cb0 100644 --- a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt +++ b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt @@ -1,18 +1,18 @@ package io.github.lemcoder.mikromarkdown.pdf import java.io.File -import pdfium.kniBridge1 -import pdfium.kniBridge15 -import pdfium.kniBridge16 -import pdfium.kniBridge26 -import pdfium.kniBridge27 -import pdfium.kniBridge4 -import pdfium.kniBridge52 -import pdfium.kniBridge53 -import pdfium.kniBridge54 -import pdfium.kniBridge58 -import pdfium.kniBridge66 -import pdfium.kniBridge71 +import pdfium.FPDFText_ClosePage +import pdfium.FPDFText_CountChars +import pdfium.FPDFText_GetCharBox +import pdfium.FPDFText_GetText +import pdfium.FPDFText_IsHyphen +import pdfium.FPDFText_LoadPage +import pdfium.FPDF_CloseDocument +import pdfium.FPDF_ClosePage +import pdfium.FPDF_GetPageCount +import pdfium.FPDF_InitLibrary +import pdfium.FPDF_LoadDocument +import pdfium.FPDF_LoadPage /** * pdfium is initialised once per process and never destroyed. @@ -20,13 +20,13 @@ import pdfium.kniBridge71 * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like: pairing them per call makes * extraction differ between one process and the next, where initialising once gives every process the same answer. */ -private val pdfiumLibrary: Lazy = lazy { initLibrary() } +private val pdfiumLibrary: Lazy = lazy { FPDF_InitLibrary() } /** * The JVM half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. * - * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here - * with the name from its doc comment, and nothing else in the module sees them. + * The bindings carry the names of the C functions they call, so this reads as pdfium's own API and needs no wrapper + * layer in between. * * The bytes reach pdfium as a file rather than as a buffer. `FPDF_LoadMemDocument` keeps the caller's pointer and reads * through it for as long as the document is open, but the generated bridge pairs `GetByteArrayElements` with @@ -44,21 +44,21 @@ internal actual fun extractPages(bytes: ByteArray): List { val file = File.createTempFile("mikromarkdown", ".pdf") try { file.writeBytes(bytes) - val document = loadDocument(file.absolutePath, null) + val document = FPDF_LoadDocument(file.absolutePath, null) if (document == 0L) return emptyList() try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) + for (index in 0 until FPDF_GetPageCount(document)) { + val page = FPDF_LoadPage(document, index) if (page == 0L) continue - val textPage = loadTextPage(page) + val textPage = FPDFText_LoadPage(page) if (textPage != 0L) { pages += pageText(textPage) - closeTextPage(textPage) + FPDFText_ClosePage(textPage) } - closePage(page) + FPDF_ClosePage(page) } } finally { - closeDocument(document) + FPDF_CloseDocument(document) } } finally { file.delete() @@ -69,10 +69,10 @@ internal actual fun extractPages(bytes: ByteArray): List { /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ private fun pageText(textPage: Long): PageText { - val count = charCount(textPage) + val count = FPDFText_CountChars(textPage) if (count <= 0) return PageText("", IntArray(0)) val buffer = ShortArray(count + 1) - val written = readText(textPage, 0, count, buffer) + val written = FPDFText_GetText(textPage, 0, count, buffer) if (written <= 1) return PageText("", IntArray(0)) // One character in, one character out, so a text index is a pdfium character index. val text = CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() @@ -88,7 +88,9 @@ private fun pageText(textPage: Long): PageText { private fun hyphenWraps(textPage: Long, text: String): IntArray { val wraps = mutableListOf() for (index in text.indices) { - if (text[index] == HYPHEN_MARKER && isHyphen(textPage, index) != 0 && startsLowerLine(textPage, index)) { + if ( + text[index] == HYPHEN_MARKER && FPDFText_IsHyphen(textPage, index) != 0 && startsLowerLine(textPage, index) + ) { wraps += index } } @@ -102,40 +104,8 @@ private fun startsLowerLine(textPage: Long, index: Int): Boolean { val bottom = DoubleArray(1) val top = DoubleArray(1) - if (charBox(textPage, index, left, right, bottom, top) == 0) return false + if (FPDFText_GetCharBox(textPage, index, left, right, bottom, top) == 0) return false val hyphenBottom = bottom[0] - if (charBox(textPage, index + 1, left, right, bottom, top) == 0) return false + if (FPDFText_GetCharBox(textPage, index + 1, left, right, bottom, top) == 0) return false return top[0] < hyphenBottom } - -private fun initLibrary() = kniBridge1() - -private fun loadDocument(path: String, password: String?): Long = kniBridge4(path, password) - -private fun pageCount(document: Long): Int = kniBridge15(document) - -private fun loadPage(document: Long, index: Int): Long = kniBridge16(document, index) - -private fun closePage(page: Long) = kniBridge26(page) - -private fun closeDocument(document: Long) = kniBridge27(document) - -private fun loadTextPage(page: Long): Long = kniBridge52(page) - -private fun closeTextPage(textPage: Long) = kniBridge53(textPage) - -private fun charCount(textPage: Long): Int = kniBridge54(textPage) - -private fun isHyphen(textPage: Long, index: Int): Int = kniBridge58(textPage, index) - -private fun charBox( - textPage: Long, - index: Int, - left: DoubleArray, - right: DoubleArray, - bottom: DoubleArray, - top: DoubleArray, -): Int = kniBridge66(textPage, index, left, right, bottom, top) - -private fun readText(textPage: Long, start: Int, count: Int, buffer: ShortArray): Int = - kniBridge71(textPage, start, count, buffer)