diff --git a/.codegraph/.gitignore b/.codegraph/.gitignore deleted file mode 100644 index 9de0f16..0000000 --- a/.codegraph/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -# CodeGraph data files -# These are local to each machine and should not be committed - -# Database -*.db -*.db-wal -*.db-shm - -# Cache -cache/ - -# Logs -*.log - -# Hook markers -.dirty diff --git a/.codegraph/config.json b/.codegraph/config.json deleted file mode 100644 index 7af60ad..0000000 --- a/.codegraph/config.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "version": 1, - "include": [ - "**/*.ts", - "**/*.tsx", - "**/*.js", - "**/*.jsx", - "**/*.py", - "**/*.go", - "**/*.rs", - "**/*.java", - "**/*.c", - "**/*.h", - "**/*.cpp", - "**/*.hpp", - "**/*.cc", - "**/*.cxx", - "**/*.cs", - "**/*.php", - "**/*.rb", - "**/*.swift", - "**/*.kt", - "**/*.kts", - "**/*.dart", - "**/*.svelte", - "**/*.vue", - "**/*.liquid", - "**/*.pas", - "**/*.dpr", - "**/*.dpk", - "**/*.lpr", - "**/*.dfm", - "**/*.fmx", - "**/*.scala", - "**/*.sc" - ], - "exclude": [ - "**/.git/**", - "**/node_modules/**", - "**/vendor/**", - "**/Pods/**", - "**/dist/**", - "**/build/**", - "**/out/**", - "**/bin/**", - "**/obj/**", - "**/target/**", - "**/*.min.js", - "**/*.bundle.js", - "**/.next/**", - "**/.nuxt/**", - "**/.svelte-kit/**", - "**/.output/**", - "**/.turbo/**", - "**/.cache/**", - "**/.parcel-cache/**", - "**/.vite/**", - "**/.astro/**", - "**/.docusaurus/**", - "**/.gatsby/**", - "**/.webpack/**", - "**/.nx/**", - "**/.yarn/cache/**", - "**/.pnpm-store/**", - "**/storybook-static/**", - "**/.expo/**", - "**/web-build/**", - "**/ios/Pods/**", - "**/ios/build/**", - "**/android/build/**", - "**/android/.gradle/**", - "**/__pycache__/**", - "**/.venv/**", - "**/venv/**", - "**/site-packages/**", - "**/dist-packages/**", - "**/.pytest_cache/**", - "**/.mypy_cache/**", - "**/.ruff_cache/**", - "**/.tox/**", - "**/.nox/**", - "**/*.egg-info/**", - "**/.eggs/**", - "**/go/pkg/mod/**", - "**/target/debug/**", - "**/target/release/**", - "**/.gradle/**", - "**/.m2/**", - "**/generated-sources/**", - "**/.kotlin/**", - "**/.dart_tool/**", - "**/.vs/**", - "**/.nuget/**", - "**/artifacts/**", - "**/publish/**", - "**/cmake-build-*/**", - "**/CMakeFiles/**", - "**/bazel-*/**", - "**/vcpkg_installed/**", - "**/.conan/**", - "**/Debug/**", - "**/Release/**", - "**/x64/**", - "**/.pio/**", - "**/release/**", - "**/*.app/**", - "**/*.asar", - "**/DerivedData/**", - "**/.build/**", - "**/.swiftpm/**", - "**/xcuserdata/**", - "**/Carthage/Build/**", - "**/SourcePackages/**", - "**/__history/**", - "**/__recovery/**", - "**/*.dcu", - "**/.composer/**", - "**/storage/framework/**", - "**/bootstrap/cache/**", - "**/.bundle/**", - "**/tmp/cache/**", - "**/public/assets/**", - "**/public/packs/**", - "**/.yardoc/**", - "**/coverage/**", - "**/htmlcov/**", - "**/.nyc_output/**", - "**/test-results/**", - "**/.coverage/**", - "**/.idea/**", - "**/logs/**", - "**/tmp/**", - "**/temp/**", - "**/_build/**", - "**/docs/_build/**", - "**/site/**" - ], - "languages": [], - "frameworks": [], - "maxFileSize": 1048576, - "extractDocstrings": true, - "trackCallSites": true -} \ No newline at end of file diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 2c10c1d..7181fe1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -12,13 +12,22 @@ permissions: jobs: test: + name: ${{ matrix.name }} strategy: + fail-fast: false matrix: include: - - target: jvmTest + - name: jvm + tasks: ":library:jvmTest" os: ubuntu-latest - - target: testAndroidHostTest + - name: android + tasks: ":library:testAndroidHostTest" os: ubuntu-latest + # The native target and the pdfium bindings are built against the mac-arm64 pdfium, so both + # legs need a macOS runner. Without this job the native source sets are never even compiled. + - name: native and pdfium + tasks: ":library:macosArm64Test :pdfium:jvmTest" + os: macos-latest runs-on: ${{ matrix.os }} steps: @@ -32,7 +41,7 @@ jobs: distribution: 'temurin' - name: Run tests uses: gradle/actions/setup-gradle@v4 - - run: ./gradlew :library:${{ matrix.target }} + - run: ./gradlew ${{ matrix.tasks }} lint: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 515213f..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish -on: - release: - types: [released, prereleased] -jobs: - publish: - name: Release build and publish - runs-on: macOS-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Set up JDK 21 - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: 21 - - name: Publish to MavenCentral - run: ./gradlew publishToMavenCentral --no-configuration-cache - env: - ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.MAVEN_CENTRAL_USERNAME }} - ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} - ORG_GRADLE_PROJECT_signingInMemoryKeyId: ${{ secrets.SIGNING_KEY_ID }} - ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.SIGNING_PASSWORD }} - ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.GPG_KEY_CONTENTS }} - diff --git a/README.md b/README.md index 9c7061a..fbd4a8e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![Test](https://github.com/lemcoder/MikroMarkdown/actions/workflows/gradle.yml/badge.svg)](https://github.com/lemcoder/MikroMarkdown/actions/workflows/gradle.yml) -Kotlin Multiplatform (JVM + Android) library that converts documents to Markdown. Port of Microsoft's [MarkItDown](https://github.com/microsoft/markitdown). +Kotlin Multiplatform library that converts documents to Markdown, on the JVM, Android and Kotlin/Native. Port of +Microsoft's [MarkItDown](https://github.com/microsoft/markitdown). ## Supported formats @@ -37,8 +38,11 @@ bytes ──► MimeDetector ──► DocumentConverter.parse ──► Documen ``` Converters contain no Markdown syntax, so escaping, table shaping, list indentation and spacing are -fixed once for all formats. Every converter lives in `commonMain` and runs on every target; only PDF -is platform-specific, and it lives in its own module because it needs a native library. The model is +fixed once for all formats. Every converter lives in `commonMain` and runs on every target, so the +library has one registration list rather than one per platform and one dependency set rather than a +JVM-only one on top: Tika, PDFBox and POI are gone, and what is left — kotlinx-io, Ksoup and +korlibs-compression — is multiplatform and shared by every target. Only PDF is platform-specific, +and it lives in its own module because it needs a native library. The model is public: `mid.parse(path)` returns the `Document`, and `ConversionResult.document` exposes it alongside the rendered Markdown. @@ -53,12 +57,9 @@ println(compact.render(document)) ## Setup -```kotlin -// build.gradle.kts -dependencies { - implementation("io.github.lemcoder:mikromarkdown:0.1.0") -} -``` +Nothing is published yet: the project builds and tests from source, and no module is configured to +publish. Build it with `./gradlew build`, and depend on `:library` from a composite build (PDF adds +`:pdfium`) until a release is wired up. ## Usage @@ -82,15 +83,25 @@ println(result.title) // nullable, extracted from document metadata ### Android +Identical — `MikroMarkdown()` is one common function, and every converter it registers is common +code. + ```kotlin import io.github.lemcoder.mikromarkdown.MikroMarkdown -// pass Context to enable PDF support -val mid = MikroMarkdown(context) +val mid = MikroMarkdown() val result = mid.convert(file.absolutePath) ``` +### PDF + +PDF needs pdfium, so it ships as `:pdfium` and the caller opts in: + +```kotlin +val mid = MikroMarkdown().apply { register(PdfiumConverter()) } +``` + ## Custom converters Implement `DocumentConverter` and register it: @@ -100,10 +111,8 @@ class MyConverter : DocumentConverter { override fun accepts(bytes: ByteArray, info: StreamInfo): Boolean = info.extension == "xyz" - override fun parse(bytes: ByteArray, info: StreamInfo): Document = document { - heading(1, "Custom") - paragraph(bytes.decodeToString()) - } + override fun parse(bytes: ByteArray, info: StreamInfo): Document = + Document(blocks = listOf(Heading(1, "Custom"), Paragraph(bytes.decodeToString()))) } val mid = MikroMarkdown() @@ -115,7 +124,10 @@ Lower priority runs first. `PlainTextConverter` uses `10.0` so it acts as a fall ## Custom MIME detection -`MimeDetector` is a `fun interface` — pass a lambda or implement it: +`MimeDetector` is a `fun interface` — pass a lambda or implement it. The default, +`SignatureMimeDetector`, reads the leading bytes and falls back to the extension; content sniffing +for extension-less text formats is where a full MIME registry such as Apache Tika goes, as your +dependency rather than the library's: ```kotlin val mid = MikroMarkdown(MimeDetector { path -> @@ -155,7 +167,7 @@ import the renderer or each other; Markdown syntax appears only under `render/`. every `DocumentConverter` is named `*Converter` and lives in `converters`. *Hygiene* — no wildcard imports, no printing from library code, and no source file duplicated -between source sets. +between source sets — a rule that now has no exceptions, since every production file is common. ktfmt-gradle only derives tasks for the common and JVM source sets, so `library/build.gradle.kts` registers matching tasks for the Android ones. @@ -206,3 +218,11 @@ python3 scripts/benchmark.py Engines whose CLI is missing are skipped. anydoc only handles binary formats, so it sits out the HTML/JSON/XML fixtures. + +Both comparison engines are submodules pinned to the versions the figures were measured against — +anydoc v0.1.8, markitdown v0.1.6 — so a clone needs them fetched before either script has anything +to compare with: + +```bash +git submodule update --init +``` diff --git a/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt b/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt index a68c38c..d5887df 100644 --- a/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt +++ b/benchmark/src/main/kotlin/io/github/lemcoder/mikromarkdown/benchmark/Main.kt @@ -1,6 +1,7 @@ package io.github.lemcoder.mikromarkdown.benchmark import io.github.lemcoder.mikromarkdown.MikroMarkdown +import io.github.lemcoder.mikromarkdown.MikroMarkdownException import io.github.lemcoder.mikromarkdown.StreamInfo import java.io.File import kotlin.system.measureNanoTime @@ -10,7 +11,7 @@ import kotlin.system.measureNanoTime * * The CLI's wall clock is dominated by JVM startup and class loading, which says nothing about the pipeline itself. * This measures the stages separately on a warmed-up JVM, and separately reports the first conversion in a fresh JVM — - * the one that pays for loading POI, PDFBox and Tika. + * the one that pays for class loading. * * Usage: ./gradlew :benchmark:run --args="[fixtureDir] [warmup] [iterations]" */ @@ -30,11 +31,14 @@ fun main(args: Array) { require(files.isNotEmpty()) { "no fixtures in ${fixtures.absolutePath}" } val first = files.first() - val coldStart = measureNanoTime { MikroMarkdown().convert(first.absolutePath) } - report("first conversion in a fresh JVM, class loading included (${first.name})", coldStart) + // The cold number is the first conversion in the process or it is nothing: a second attempt would run warm. + runCatching { measureNanoTime { MikroMarkdown().convert(first.absolutePath) } } + .onSuccess { report("first conversion in a fresh JVM, class loading included (${first.name})", it) } + .onFailure { println("no cold-start timing: ${first.name} has no registered converter") } println() val mikroMarkdown = MikroMarkdown() + val skipped = mutableListOf() println("Best of $iterations runs after $warmup warmup runs, milliseconds.") println() @@ -45,7 +49,14 @@ fun main(args: Array) { val bytes = file.readBytes() val info = StreamInfo(extension = file.extension, filename = file.name, localPath = file.absolutePath) - repeat(warmup) { mikroMarkdown.convert(bytes, info) } + // The directory keeps fixtures no converter accepts: the office formats, and PDF until `:pdfium` is + // registered. They are named below rather than dropped, so a missing row never reads as a fast one. + try { + repeat(warmup) { mikroMarkdown.convert(bytes, info) } + } catch (e: MikroMarkdownException) { + skipped += "${file.name} (${e.message})" + continue + } val parse = best(iterations) { mikroMarkdown.parse(bytes, info) } val convertBytes = best(iterations) { mikroMarkdown.convert(bytes, info) } @@ -58,6 +69,11 @@ fun main(args: Array) { "${convertBytes.ms()} | ${convertPath.ms()} |" ) } + + if (skipped.isNotEmpty()) { + println() + println("Skipped: ${skipped.joinToString()}") + } } private fun coldBytes(file: File) { diff --git a/cli-native/build.gradle.kts b/cli-native/build.gradle.kts index f554494..50711d0 100644 --- a/cli-native/build.gradle.kts +++ b/cli-native/build.gradle.kts @@ -2,15 +2,9 @@ plugins { alias(libs.plugins.kotlinMultiplatform) } kotlin { macosArm64 { - // Kotlin/Native does not carry a klib's linker options to the binary that uses it, so the - // consumer names pdfium itself. Worth turning into a shared convention if a second - // consumer appears. - val pdfiumLib = rootProject.layout.projectDirectory.dir("pdfium/build/pdfium/mac-arm64/lib").asFile - - binaries.executable { - entryPoint = "io.github.lemcoder.mikromarkdown.cli.main" - linkerOpts("-L${pdfiumLib.absolutePath}", "-lpdfium", "-rpath", pdfiumLib.absolutePath) - } + // Nothing about pdfium here: its cinterop .def records where the library is, and cinterop + // hands those options to whatever links the binding. + binaries.executable { entryPoint = "io.github.lemcoder.mikromarkdown.cli.main" } compilerOptions { // Worth about 8% on large inputs and nothing on small ones. Measured, not assumed: diff --git a/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt b/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt index 665be14..377c70e 100644 --- a/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt +++ b/cli-native/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/cli/Main.kt @@ -6,8 +6,7 @@ import io.github.lemcoder.mikromarkdown.pdf.PdfiumConverter import kotlin.system.exitProcess /** - * Minimal native entry point, kept deliberately bare so its timings measure conversion rather than an argument parser. - * The JVM CLI remains the full one. + * The command line tool, kept deliberately bare so its timings measure conversion rather than an argument parser. * * Several files may be given: a document boundary is the one point where everything the previous conversion allocated * is dead, which is what makes a manual collection policy possible at all. The default collector wins on measurement, diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3f394ec..835b0ed 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,19 +3,12 @@ agp = "9.2.1" kotlin = "2.3.21" android-minSdk = "26" android-compileSdk = "37" -vanniktechMavenPublish = "0.36.0" kotlinx-io = "0.9.0" -kotlinx-serialization = "1.9.0" ksoup = "0.2.6" korlibs = "6.0.0" -konanplugin = "1.2.0-alpha06" +konanplugin = "1.2.0-alpha07" kotlinx-resources = "0.15.0" - -commons-csv = "1.14.1" -jackson = "2.21.3" junit = "6.1.0" -tika = "3.3.0" -coreKtx = "1.7.0" detekt = "1.23.8" ktfmt = "0.27.0" konsist = "0.17.3" @@ -25,22 +18,14 @@ kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotl kotlinx-io-core = { module = "org.jetbrains.kotlinx:kotlinx-io-core", version.ref = "kotlinx-io" } ksoup = { module = "com.fleeksoft.ksoup:ksoup", version.ref = "ksoup" } korlibs-compression = { module = "com.soywiz:korlibs-compression", version.ref = "korlibs" } -kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-serialization" } kotlinx-resources = { module = "com.goncalossilva:resources", version.ref = "kotlinx-resources" } - -commons-csv = { module = "org.apache.commons:commons-csv", version.ref = "commons-csv" } -jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "jackson" } junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" } -tika-core = { module = "org.apache.tika:tika-core", version.ref = "tika" } -core-ktx = { group = "androidx.test", name = "core-ktx", version.ref = "coreKtx" } konsist = { module = "com.lemonappdev:konsist", version.ref = "konsist" } [plugins] android-kotlin-multiplatform-library = { id = "com.android.kotlin.multiplatform.library", version.ref = "agp" } kotlinMultiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } -kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } -vanniktech-mavenPublish = { id = "com.vanniktech.maven.publish", version.ref = "vanniktechMavenPublish" } kotlinx-resources = { id = "com.goncalossilva.resources", version.ref = "kotlinx-resources" } detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } konanplugin = { id = "io.github.lemcoder.konanplugin", version.ref = "konanplugin" } diff --git a/library/build.gradle.kts b/library/build.gradle.kts index f750228..ee9e54d 100644 --- a/library/build.gradle.kts +++ b/library/build.gradle.kts @@ -21,10 +21,7 @@ kotlin { testRuns["test"].executionTask.configure { useJUnitPlatform() } } - // Spike: a native target to see how close a real binary gets to the Rust implementation. - // The shared integration tests expect JVM-only formats, so native test compilation stays off - // until the native target carries real converters. - macosArm64 { compilations.getByName("test") { compileTaskProvider.configure { enabled = false } } } + macosArm64() androidLibrary { namespace = "io.github.lemcoder.mikromarkdown" @@ -40,13 +37,11 @@ kotlin { sourceSets { commonMain.dependencies { implementation(libs.kotlinx.io.core) - // Phase 0 of the commonMain migration: HTML parsing and the inflate that ZIP needs. + // HTML parsing, and the inflate that EPUB's ZIP container needs. implementation(libs.ksoup) implementation(libs.korlibs.compression) } - jvmMain { dependencies { implementation(libs.tika.core) } } - commonTest.dependencies { implementation(libs.kotlin.test) implementation(libs.kotlinx.resources) @@ -60,26 +55,28 @@ kotlin { } } -// ktfmt-gradle only derives tasks for the common and JVM source sets, so the Android -// ones — where half the converters live — would go unformatted and unchecked. +// ktfmt-gradle only derives tasks for the common and JVM source sets, so every other one — Android's +// today, a native one tomorrow — would go unformatted and unchecked. Everything under src/ that ktfmt +// does not already cover is named here by exclusion, so a new target needs no edit to this block. run { - val androidSources = fileTree("src") { include("android*/**/*.kt") } + val derived = listOf("commonMain", "commonTest", "jvmMain", "jvmTest") + val remainingSources = fileTree("src") { include("**/*.kt").exclude(derived.map { "$it/**" }) } val template = tasks.named("ktfmtFormatKmpCommonMain") - val formatAndroid = - tasks.register("ktfmtFormatAndroidSourceSets") { + val formatRemaining = + tasks.register("ktfmtFormatRemainingSourceSets") { ktfmtClasspath.from(template.map { it.ktfmtClasspath }) formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) - setSource(androidSources) + setSource(remainingSources) } - val checkAndroid = - tasks.register("ktfmtCheckAndroidSourceSets") { + val checkRemaining = + tasks.register("ktfmtCheckRemainingSourceSets") { ktfmtClasspath.from(template.map { it.ktfmtClasspath }) formattingOptionsBean.set(template.flatMap { it.formattingOptionsBean }) - setSource(androidSources) + setSource(remainingSources) } - tasks.named("ktfmtFormat") { dependsOn(formatAndroid) } - tasks.named("ktfmtCheck") { dependsOn(checkAndroid) } - tasks.named("check") { dependsOn(checkAndroid) } + tasks.named("ktfmtFormat") { dependsOn(formatRemaining) } + tasks.named("ktfmtCheck") { dependsOn(checkRemaining) } + tasks.named("check") { dependsOn(checkRemaining) } } diff --git a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt b/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt deleted file mode 100644 index 15c3cc4..0000000 --- a/library/src/androidDeviceTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.android.kt +++ /dev/null @@ -1,5 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMikroMarkdown(): MikroMarkdown { - return MikroMarkdown() -} diff --git a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt b/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt deleted file mode 100644 index 7c1314e..0000000 --- a/library/src/androidHostTest/kotlin/io/github/lemcoder/mikromarkdown/TestMikroMarkdown.kt +++ /dev/null @@ -1,3 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() diff --git a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt deleted file mode 100644 index 8463eae..0000000 --- a/library/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ /dev/null @@ -1,29 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -import io.github.lemcoder.mikromarkdown.converters.CsvConverter -import io.github.lemcoder.mikromarkdown.converters.EpubConverter -import io.github.lemcoder.mikromarkdown.converters.HtmlConverter -import io.github.lemcoder.mikromarkdown.converters.JsonConverter -import io.github.lemcoder.mikromarkdown.converters.MarkdownPassthroughConverter -import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter -import io.github.lemcoder.mikromarkdown.converters.XmlConverter -import java.io.File - -/** - * A [MikroMarkdown] with every Android converter registered. - * - * PDF is not among them: it needs a native library, so the `:pdfium` module provides it and the caller opts in with - * `register(PdfiumConverter())`. - */ -public fun MikroMarkdown(): MikroMarkdown = - MikroMarkdown(SignatureMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(EpubConverter()) - register(PlainTextConverter(), priority = 10.0) - } - -public fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/macosMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt similarity index 75% rename from library/src/macosMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt rename to library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt index 0d9432b..52f4aad 100644 --- a/library/src/macosMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt @@ -9,17 +9,18 @@ import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter import io.github.lemcoder.mikromarkdown.converters.XmlConverter /** - * A [MikroMarkdown] with the converters that need no platform library. + * A [MikroMarkdown] with every converter the library ships. * - * PDF is the only format still missing here; everything else the library converts is shared. + * One list for every target, because every converter is now common. PDF is the exception and is not here: it costs a + * native library, so the `:pdfium` module provides it and the caller opts in with `register(PdfiumConverter())`. */ public fun MikroMarkdown(): MikroMarkdown = MikroMarkdown(SignatureMimeDetector).apply { register(MarkdownPassthroughConverter()) register(HtmlConverter()) - register(EpubConverter()) register(CsvConverter()) register(JsonConverter()) register(XmlConverter()) + register(EpubConverter()) register(PlainTextConverter(), priority = 10.0) } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt index cf8b04d..79138be 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetector.kt @@ -10,11 +10,11 @@ import kotlinx.io.readByteArray * * This is the default because it costs microseconds: it reads a handful of bytes and consults a fixed table, where a * full MIME registry (Tika) spends ~90 ms building itself on first use — more than the entire conversion for most - * documents. Content still wins over the extension, so a mislabelled `.txt` that is really a PDF or an OOXML package is + * documents. Content still wins over the extension, so a mislabelled `.txt` that is really a PDF or a ZIP package is * identified correctly. * - * Formats that are plain text with no signature (CSV, JSON, XML, HTML, Markdown) are recognised by extension. Pass - * [TikaMimeDetector] to `MikroMarkdown` if you need content sniffing for those too. + * Formats that are plain text with no signature (CSV, JSON, XML, HTML, Markdown) are recognised by extension. Content + * sniffing for those is a [MimeDetector] away — it is a `fun interface`, and the pipeline takes any implementation. */ public object SignatureMimeDetector : MimeDetector { @@ -23,7 +23,6 @@ public object SignatureMimeDetector : MimeDetector { private val byExtension = mapOf( "csv" to "text/csv", - "docx" to "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "epub" to "application/epub+zip", "htm" to "text/html", "html" to "text/html", @@ -31,16 +30,11 @@ public object SignatureMimeDetector : MimeDetector { "md" to "text/markdown", "markdown" to "text/markdown", "pdf" to "application/pdf", - "pptx" to "application/vnd.openxmlformats-officedocument.presentationml.presentation", "txt" to "text/plain", "log" to "text/plain", - "xlsx" to "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xml" to "application/xml", ) - /** ZIP-based formats are told apart by extension; the signature only proves it is a package. */ - private val zipExtensions = setOf("docx", "xlsx", "pptx", "epub", "zip") - override fun detect(path: String): StreamInfo = describe(path, readSignature(path)) override fun detect(path: String, bytes: ByteArray): StreamInfo = @@ -61,10 +55,12 @@ public object SignatureMimeDetector : MimeDetector { private fun mimetypeOf(signature: ByteArray, extension: String?): String? = when { signature.startsWith("%PDF") -> "application/pdf" - // Every OOXML container and EPUB is a ZIP; the extension says which one. - signature.startsWith("PK") -> - if (extension in zipExtensions) byExtension[extension] ?: "application/zip" else "application/zip" - // Legacy OLE compound files: .doc/.xls/.ppt, which no converter handles yet. + // EPUB and every OOXML container is a ZIP; the extension says which one, and only EPUB is ours. The rest + // are reported as the package they are, which is what leaves the caller a message naming a real format. + signature.startsWith("PK") -> if (extension == "epub") "application/epub+zip" else "application/zip" + // Legacy OLE compound files: .doc/.xls/.ppt, which no converter handles. Naming the container is what + // leaves the caller a message about a format rather than "No converter found for: unknown"; the OOXML + // mimetypes are not restored with it, because their extension already reaches the caller in [StreamInfo]. signature.startsWithBytes(0xD0, 0xCF, 0x11, 0xE0) -> "application/x-ole-storage" else -> byExtension[extension] } @@ -85,8 +81,9 @@ public object SignatureMimeDetector : MimeDetector { return prefix.indices.all { this[it].toInt().toChar() == prefix[it] } } + /** Signatures outside ASCII are spelled as the bytes they are. */ private fun ByteArray.startsWithBytes(vararg prefix: Int): Boolean { if (size < prefix.size) return false - return prefix.indices.all { this[it].toInt() and 0xFF == prefix[it] } + return prefix.indices.all { this[it] == prefix[it].toByte() } } } diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt index be2d2e5..0643384 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/Document.kt @@ -31,10 +31,16 @@ public sealed interface Block public data class Heading( val level: Int, val content: List, + /** The source's own id for this heading, where it had one; HTML fills it from `id`. */ val anchor: String? = null, -) : Block +) : Block { + /** The common case: a heading whose content is one run of text. */ + public constructor(level: Int, text: String) : this(level, listOf(Text(text))) +} -public data class Paragraph(val content: List) : Block +public data class Paragraph(val content: List) : Block { + public constructor(text: String) : this(listOf(Text(text))) +} public data class CodeBlock(val code: String, val language: String? = null) : Block @@ -107,9 +113,6 @@ public enum class Alignment { public data object ThematicBreak : Block -/** Rendered verbatim as an HTML comment. Used for structural markers such as slide numbers. */ -public data class HtmlComment(val text: String) : Block - /** Escape hatch for content that is already Markdown (or must not be touched). */ public data class RawBlock(val text: String) : Block diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt deleted file mode 100644 index af3684c..0000000 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/DocumentBuilder.kt +++ /dev/null @@ -1,160 +0,0 @@ -package io.github.lemcoder.mikromarkdown.model - -/** - * Small builder used by converters so parsing code stays free of Markdown syntax. - * - * ``` - * document { - * heading(1, "Report") - * paragraph("Body text") - * table(header = listOf("A", "B"), rows = listOf(listOf("1", "2"))) - * } - * ``` - */ -/** Restricts builder receivers so an inline block cannot silently call document-level methods. */ -@DslMarker public annotation class DocumentDsl - -@DocumentDsl public fun document(block: DocumentBuilder.() -> Unit): Document = DocumentBuilder().apply(block).build() - -@DocumentDsl -public class DocumentBuilder { - private val blocks = mutableListOf() - private val assets = mutableListOf() - private val metadata = mutableMapOf() - private var title: String? = null - - public fun add(block: Block) { - blocks += block - } - - public fun addAll(blocks: Iterable) { - this.blocks += blocks - } - - public fun asset(asset: Asset) { - assets += asset - } - - public fun meta(key: String, value: String?) { - if (!value.isNullOrBlank()) metadata[key] = value - } - - /** Records [text] as the document title unless one was already found. */ - public fun titleIfAbsent(text: String?) { - if (title == null && !text.isNullOrBlank()) title = text - } - - /** The title recorded so far, if any. */ - public fun title(): String? = title - - public fun heading(level: Int, text: String) { - if (text.isNotBlank()) add(Heading(level, listOf(Text(text)))) - } - - public fun heading(level: Int, content: List) { - if (content.isNotEmpty()) add(Heading(level, content)) - } - - public fun paragraph(text: String) { - if (text.isNotBlank()) add(Paragraph(listOf(Text(text)))) - } - - public fun paragraph(content: List) { - if (content.isNotEmpty()) add(Paragraph(content)) - } - - public fun code(code: String, language: String? = null) { - add(CodeBlock(code, language)) - } - - public fun comment(text: String) { - add(HtmlComment(text)) - } - - public fun raw(text: String) { - if (text.isNotBlank()) add(RawBlock(text)) - } - - public fun bulletList(items: List) { - if (items.isEmpty()) return - add(ListBlock(ordered = false, items = items.map { ListItem(listOf(Paragraph(listOf(Text(it))))) })) - } - - public fun table( - header: List, - rows: List>, - alignments: List = emptyList(), - ) { - if (header.isEmpty() && rows.isEmpty()) return - add( - Table( - header = header.map { TableCell(it) }, - rows = rows.map { row -> row.map { TableCell(it) } }, - alignments = alignments, - ) - ) - } - - public fun build(): Document = - Document( - blocks = blocks.toList(), - title = title, - metadata = metadata.toMap(), - assets = assets.toList(), - ) -} - -/** Builds a list of inlines without repeating `listOf(...)` wrappers in parsers. */ -@DocumentDsl public fun inlines(block: InlineBuilder.() -> Unit): List = InlineBuilder().apply(block).build() - -@DocumentDsl -public class InlineBuilder { - private val items = mutableListOf() - - public fun text(value: String) { - if (value.isNotEmpty()) items += Text(value) - } - - public fun strong(value: String) { - if (value.isNotEmpty()) items += Strong(listOf(Text(value))) - } - - public fun emphasis(value: String) { - if (value.isNotEmpty()) items += Emphasis(listOf(Text(value))) - } - - public fun code(value: String) { - if (value.isNotEmpty()) items += CodeSpan(value) - } - - public fun link(text: String, url: String) { - items += Link(listOf(Text(text)), url) - } - - public fun image(alt: String, url: String, assetId: String? = null) { - items += Image(alt, url, assetId = assetId) - } - - public fun lineBreak() { - items += LineBreak - } - - public operator fun plusAssign(inline: Inline) { - items += inline - } - - public operator fun plusAssign(inlines: List) { - items += inlines - } - - public fun build(): List = items.toList() -} - -/** Wraps [content] in the emphasis combination described by the flags. */ -public fun styled(content: List, bold: Boolean, italic: Boolean, strike: Boolean = false): List { - var result = content - if (strike) result = listOf(Strikethrough(result)) - if (italic) result = listOf(Emphasis(result)) - if (bold) result = listOf(Strong(result)) - return result -} diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt index e80aa3f..bffddaf 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocks.kt @@ -10,71 +10,62 @@ package io.github.lemcoder.mikromarkdown.model * the source layout's line breaks. */ public fun plainTextBlocks(text: String, reflow: Boolean = true): List { - // Form feeds mark PDF page breaks; treat them as paragraph boundaries. - val normalized = text.replace("\r\n", "\n").replace('\r', '\n').replace('\u000C', '\n') - val vocabulary = if (reflow) wordsIn(normalized) else emptySet() + val normalized = + text + .replace("\r\n", "\n") + .replace('\r', '\n') + // Form feeds mark PDF page breaks; a paragraph boundary is a blank line, which takes two newlines. + .replace(FORM_FEED, "\n\n") val paragraphs = mutableListOf() for (chunk in normalized.split(PARAGRAPH_BREAK)) { val lines = chunk.lines().map { it.trim() }.filter { it.isNotEmpty() } if (lines.isEmpty()) continue - val joined = if (reflow) joinWrappedLines(lines, vocabulary) else lines.joinToString("\n") + val joined = if (reflow) joinWrappedLines(lines) else lines.joinToString("\n") // A paragraph ending mid-word means the break was a layout artifact, not a real one. val previous = paragraphs.lastOrNull() if (reflow && previous != null && previous.endsWithWordBreak()) { - paragraphs[paragraphs.lastIndex] = joinWrappedLines(listOf(previous, joined), vocabulary) + paragraphs[paragraphs.lastIndex] = joinWrappedLines(listOf(previous, joined)) } else { paragraphs += joined } } - return paragraphs.map { Paragraph(listOf(Text(it))) } + return paragraphs.map { Paragraph(it) } } -private fun String.endsWithWordBreak(): Boolean = endsWith("-") && length > 1 && this[length - 2].isLetter() - private val PARAGRAPH_BREAK = Regex("\n[ \t]*\n") -private val WORD = Regex("[\\p{L}]{2,}") - -/** Words the document uses on their own; the de-hyphenation heuristic consults this. */ -private fun wordsIn(text: String): Set = WORD.findAll(text).map { it.value.lowercase() }.toSet() +private const val FORM_FEED = "\u000C" /** * Rejoins soft-wrapped lines. * - * A trailing hyphen is dropped only when it looks like a wrap artifact: if both fragments are words the document uses - * elsewhere on their own (`conversation-` + `centric`), the hyphen is a real compound and stays. + * A line ending mid-word continues straight into the next; any other line break was the space the wrap replaced. + * + * The hyphen stays either way, and nothing here tries to work out which kind it was. Geometry cannot say — + * `FPDFText_GetCharBox` settles only that the line ended at the hyphen, which is as true of `chat-` `optimized` as of + * `con-` `firming`. Weighing it against the rest of the document was tried and cost more than it returned: a compound + * whose halves appear nowhere else, `chat-optimized` among them, came out fused as `chatoptimized`, a word that is in + * no document anywhere. Keeping the hyphen writes `con-firming` where the page meant `confirming`, which reads a little + * worse and destroys nothing — the join is still there for a reader, and for anything downstream that knows more than + * we do. */ -private fun joinWrappedLines(lines: List, vocabulary: Set): String { +private fun joinWrappedLines(lines: List): String { val sb = StringBuilder() for ((index, line) in lines.withIndex()) { - if (index == 0) { - sb.append(line) - continue - } - val head = sb.lastFragment() - val tail = line.takeWhile { it.isLetter() }.lowercase() - val hyphenated = sb.isNotEmpty() && sb.last() == '-' && head.isNotEmpty() && tail.isNotEmpty() - val realCompound = hyphenated && head in vocabulary && tail in vocabulary - when { - hyphenated && !realCompound -> { - sb.setLength(sb.length - 1) - sb.append(line) - } - - hyphenated -> sb.append(line) + index == 0 -> sb.append(line) + sb.endsWithWordBreak() && line.first().isLetter() -> sb.append(line) else -> sb.append(' ').append(line) } } return sb.toString() } -/** The word immediately before a trailing hyphen, lowercased. */ -private fun StringBuilder.lastFragment(): String { - if (isEmpty() || last() != '-') return "" - var start = length - 1 - while (start > 0 && this[start - 1].isLetter()) start-- - return substring(start, length - 1).lowercase() -} +/** A trailing hyphen with a letter before it: the mark of a word the layout cut in half. */ +private fun CharSequence.endsWithWordBreak(): Boolean = + length > 1 && this[length - 1].isHyphen() && this[length - 2].isLetter() + +/** The hyphens a line break can end on; a document may write any of them. */ +private fun Char.isHyphen(): Boolean = this == '-' || this == '\u2010' || this == '\u2011' diff --git a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt index 2aae945..eff63ce 100644 --- a/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt +++ b/library/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/render/MarkdownRenderer.kt @@ -8,7 +8,6 @@ import io.github.lemcoder.mikromarkdown.model.CodeSpan import io.github.lemcoder.mikromarkdown.model.Document import io.github.lemcoder.mikromarkdown.model.Emphasis import io.github.lemcoder.mikromarkdown.model.Heading -import io.github.lemcoder.mikromarkdown.model.HtmlComment import io.github.lemcoder.mikromarkdown.model.Image import io.github.lemcoder.mikromarkdown.model.Inline import io.github.lemcoder.mikromarkdown.model.LineBreak @@ -121,8 +120,6 @@ public class MarkdownRenderer(private val options: MarkdownOptions = MarkdownOpt ThematicBreak -> out.append("---") - is HtmlComment -> out.append("") - // Already-Markdown content: only whitespace is normalized, never syntax. is RawBlock -> appendLines( diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt index 5473077..6a2e058 100644 --- a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.kt @@ -6,10 +6,8 @@ import kotlin.test.assertFailsWith import kotlin.test.assertFalse import kotlin.test.assertTrue -expect fun testMikroMarkdown(): MikroMarkdown - class FileIntegrationTest { - private val mid = testMikroMarkdown() + private val mid = MikroMarkdown() private fun assertConversion( filename: String, diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetectorTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetectorTest.kt new file mode 100644 index 0000000..3adb6b1 --- /dev/null +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/SignatureMimeDetectorTest.kt @@ -0,0 +1,63 @@ +package io.github.lemcoder.mikromarkdown + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * Pins what the leading bytes decide and what the extension decides. + * + * The integration tests hand converters a [StreamInfo] they build themselves, so detection is only covered here. + */ +class SignatureMimeDetectorTest { + + private fun detect(filename: String, vararg signature: Int): String? = + SignatureMimeDetector.detect(filename, ByteArray(signature.size) { signature[it].toByte() }).mimetype + + private fun detect(filename: String, signature: String): String? = + SignatureMimeDetector.detect(filename, signature.encodeToByteArray()).mimetype + + @Test + fun `content wins over a mislabelled extension`() { + assertEquals("application/pdf", detect("report.txt", "%PDF-1.7")) + } + + @Test + fun `a ZIP package is told apart by its extension`() { + assertEquals("application/epub+zip", detect("book.epub", "PK\u0003\u0004")) + // Every other OOXML container is a package this library does not read; it is reported as one. + assertEquals("application/zip", detect("sheet.xlsx", "PK\u0003\u0004")) + assertEquals("application/zip", detect("archive.zip", "PK\u0003\u0004")) + } + + @Test + fun `a legacy OLE compound file is named rather than left unknown`() { + assertEquals("application/x-ole-storage", detect("memo.doc", 0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1)) + } + + @Test + fun `formats with no signature are recognised by extension`() { + assertEquals("text/csv", detect("data.csv", "a,b,c\n1,2,3")) + assertEquals("text/markdown", detect("notes.md", "# Title")) + } + + @Test + fun `an unknown extension with no signature is left undecided`() { + assertNull(detect("mystery.bin", "not a signature")) + assertNull(detect("noextension", "")) + } + + @Test + fun `a file shorter than the signature is read whole rather than failing`() { + assertEquals("text/plain", detect("tiny.txt", "hi")) + } + + @Test + fun `the path is split into filename and extension`() { + val info = SignatureMimeDetector.detect("/tmp/some dir/Book.EPUB", "PK\u0003\u0004".encodeToByteArray()) + + assertEquals("Book.EPUB", info.filename) + assertEquals("epub", info.extension) + assertEquals("application/epub+zip", info.mimetype) + } +} diff --git a/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocksTest.kt b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocksTest.kt new file mode 100644 index 0000000..82a93a4 --- /dev/null +++ b/library/src/commonTest/kotlin/io/github/lemcoder/mikromarkdown/model/TextBlocksTest.kt @@ -0,0 +1,73 @@ +package io.github.lemcoder.mikromarkdown.model + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** Pins how a trailing hyphen at a line break is read, and the paragraph structure it must not disturb. */ +class TextBlocksTest { + + /** The page break a PDF text extractor leaves behind. */ + private val formFeed = "\u000C" + + private fun paragraphs(text: String, reflow: Boolean = true): List = + plainTextBlocks(text, reflow).map { (it as Paragraph).content.plainText() } + + private fun onlyParagraph(text: String, reflow: Boolean = true): String = paragraphs(text, reflow).single() + + @Test + fun `a hyphen at a line break keeps its hyphen and loses the break`() { + val text = "an inter-\nnational body\nand nothing else" + + // The page meant "international". Which kind of hyphen this was cannot be known, so it is left where it is. + assertEquals("an inter-national body and nothing else", onlyParagraph(text)) + } + + @Test + fun `a compound broken across lines survives intact`() { + val text = "a well-\nknown case.\nIt reads the same as any other." + + assertEquals("a well-known case. It reads the same as any other.", onlyParagraph(text)) + } + + @Test + fun `a line break with no hyphen is the space the wrap replaced`() { + val text = "an ordinary\nwrapped line" + + assertEquals("an ordinary wrapped line", onlyParagraph(text)) + } + + @Test + fun `a trailing hyphen with no letter before it is not a word break`() { + val text = "a dash -\nthen more" + + assertEquals("a dash - then more", onlyParagraph(text)) + } + + @Test + fun `a form feed separates paragraphs rather than merging the pages around it`() { + val text = "End of page one.${formFeed}Start of page two." + + assertEquals(listOf("End of page one.", "Start of page two."), paragraphs(text)) + } + + @Test + fun `a U+2010 hyphen is a word break too and keeps the character the document wrote`() { + val text = "an inter\u2010\nnational body\nand nothing else" + + assertEquals("an inter\u2010national body and nothing else", onlyParagraph(text)) + } + + @Test + fun `without reflow the source layout is kept`() { + val text = "an inter-\nnational body" + + assertEquals(listOf("an inter-\nnational body"), paragraphs(text, reflow = false)) + } + + @Test + fun `a paragraph ending mid-word continues into the next`() { + val text = "a word broken by a page-\n\nbreak and nothing else" + + assertEquals(listOf("a word broken by a page-break and nothing else"), paragraphs(text)) + } +} diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt deleted file mode 100644 index 9695eb7..0000000 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/MikroMarkdownFactory.kt +++ /dev/null @@ -1,29 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -import io.github.lemcoder.mikromarkdown.converters.CsvConverter -import io.github.lemcoder.mikromarkdown.converters.EpubConverter -import io.github.lemcoder.mikromarkdown.converters.HtmlConverter -import io.github.lemcoder.mikromarkdown.converters.JsonConverter -import io.github.lemcoder.mikromarkdown.converters.MarkdownPassthroughConverter -import io.github.lemcoder.mikromarkdown.converters.PlainTextConverter -import io.github.lemcoder.mikromarkdown.converters.XmlConverter -import java.io.File - -/** - * A [MikroMarkdown] with every JVM converter registered. - * - * PDF is not among them: it needs a native library, so the `:pdfium` module provides it and the caller opts in with - * `register(PdfiumConverter())`. - */ -public fun MikroMarkdown(): MikroMarkdown = - MikroMarkdown(SignatureMimeDetector).apply { - register(MarkdownPassthroughConverter()) - register(HtmlConverter()) - register(CsvConverter()) - register(JsonConverter()) - register(XmlConverter()) - register(EpubConverter()) - register(PlainTextConverter(), priority = 10.0) - } - -public fun MikroMarkdown.convert(file: File): ConversionResult = convert(file.absolutePath) diff --git a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/TikaMimeDetector.kt b/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/TikaMimeDetector.kt deleted file mode 100644 index 95823d4..0000000 --- a/library/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/TikaMimeDetector.kt +++ /dev/null @@ -1,32 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -import java.io.File -import org.apache.tika.Tika - -/** - * Full content sniffing through Apache Tika's MIME registry. - * - * Slower to start than [SignatureMimeDetector] — building the registry costs about 90 ms, more than converting most - * documents — but it recognises text formats by content rather than by extension. - */ -public object TikaMimeDetector : MimeDetector { - // Building Tika's MIME registry is expensive; convert(bytes, info) never needs it. - private val tika by lazy { Tika() } - - override fun detect(path: String): StreamInfo { - val file = File(path) - val mimetype = - try { - tika.detect(file) - } catch (_: Exception) { - null - } - val extension = file.extension.lowercase().ifEmpty { null } - return StreamInfo( - mimetype = mimetype, - extension = extension, - filename = file.name, - localPath = path, - ) - } -} diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt index 2f4093a..75b657d 100644 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt +++ b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/ArchitectureTest.kt @@ -119,18 +119,14 @@ class ArchitectureTest { } /** - * JVM and Android share one source set; only the factory and the PDF converter differ, because pdfbox-android is a - * separate library. Any other same-named file in two source sets means a copy that will drift. + * Every converter is common, so no production file has a per-target copy — the rule has no exceptions left. A + * same-named file in two source sets means a copy that will drift. */ @Test fun `production files are not copied between source sets`() { - val expectedPerTarget = setOf("MikroMarkdownFactory", "PdfConverter") + val copied = production.filter { it.path.contains("/src/") }.groupBy { it.name }.filterValues { it.size > 1 } - val copied = - production.filter { it.path.contains("/src/") }.groupBy { it.name }.filterValues { it.size > 1 }.keys - - expectedPerTarget - - assertEquals(emptySet(), copied, "these files exist in more than one source set") + assertEquals(emptySet(), copied.keys, "these files exist in more than one source set") } private fun productionIn(packageFragment: String) = production.filter { diff --git a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt b/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt deleted file mode 100644 index 7c1314e..0000000 --- a/library/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/FileIntegrationTest.jvm.kt +++ /dev/null @@ -1,3 +0,0 @@ -package io.github.lemcoder.mikromarkdown - -actual fun testMikroMarkdown(): MikroMarkdown = MikroMarkdown() diff --git a/pdfium/build.gradle.kts b/pdfium/build.gradle.kts index 25ad398..68353c3 100644 --- a/pdfium/build.gradle.kts +++ b/pdfium/build.gradle.kts @@ -18,13 +18,45 @@ apply(from = "pdfium-binaries.gradle.kts") @Suppress("UNCHECKED_CAST") val pdfiumAbis = extra["pdfiumAbis"] as Map +val pdfiumLibDir = pdfiumRoot.dir("mac-arm64/lib").asFile.absolutePath + +val sourceDef = layout.projectDirectory.file("src/nativeInterop/cinterop/pdfium.def") + +val linkedDef = layout.buildDirectory.file("cinterop/pdfium-linked.def") + +/** + * The .def the native target binds, with pdfium's location written into it. + * + * cinterop records a .def's `linkerOpts` in the klib and hands them to whatever links against it, which is what lets a + * consumer use the binding without naming pdfium a second time. The paths have to be absolute, for two reasons: a + * relative one in a .def resolves against the working directory the compiler happens to run in rather than the file's + * own location (kotlin-native#2314, still true on 2.3.21), and an rpath has to hold at run time wherever the binary is + * started from. `libraryPaths` is no way round it either — the klib records it, but it is cinterop's own search path + * and never reaches the consumer's linker, which then fails with "library 'pdfium' not found". + * + * So the file is generated. The JVM and Android legs bind the checked-in .def, which stays free of a macOS path. + */ +val writeLinkedDef by tasks.registering { + val from = sourceDef + val into = linkedDef + val libDir = pdfiumLibDir + inputs.file(from) + inputs.property("libDir", libDir) + outputs.file(into) + doLast { + val target = into.get().asFile + target.parentFile.mkdirs() + // The archive ships a dylib, so the binary carries an rpath to find it at run time. + target.writeText(from.asFile.readText().trimEnd() + "\nlinkerOpts = -L$libDir -lpdfium -rpath $libDir\n") + } +} + kotlin { macosArm64 { val platform = pdfiumRoot.dir("mac-arm64") compilations.getByName("main").cinterops.create("pdfium") { - defFile("src/nativeInterop/cinterop/pdfium.def") + defFile(linkedDef.get().asFile) includeDirs(platform.dir("include")) - // The archive ships a dylib, so the binary carries an rpath to find it at run time. extraOpts("-libraryPath", platform.dir("lib").asFile.absolutePath) } } @@ -97,8 +129,16 @@ kotlin.targets.getByName("android").compilations.getByName("main").jvmInterops { } // Every binding path needs the headers and the library unpacked first. -tasks.matching { it.name.startsWith("cinteropPdfium") }.configureEach { dependsOn("downloadPdfium") } +tasks.matching { it.name.startsWith("cinteropPdfium") }.configureEach { dependsOn("downloadPdfium", writeLinkedDef) } tasks .matching { it.name.startsWith("generateJvmInterop") || it.name.startsWith("cmakeConfigure") } .configureEach { dependsOn("downloadPdfium") } + +// The JNI generator reads the Kotlin/Native distribution, and the Kotlin plugin fetches that only when +// something compiles a native target. On a machine that has never built one — a CI runner, a fresh clone +// running `:pdfium:jvmTest` on its own — generating the bindings fails with "No Kotlin/Native distribution +// found" instead, so the download is ordered before it rather than left to whatever else ran first. +tasks + .matching { it.name.startsWith("generateJvmInterop") } + .configureEach { dependsOn("downloadKotlinNativeDistribution") } diff --git a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt index 243f89a..9e26afd 100644 --- a/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt +++ b/pdfium/src/androidMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.android.kt @@ -1,80 +1,109 @@ package io.github.lemcoder.mikromarkdown.pdf -import pdfium.kniBridge1 -import pdfium.kniBridge15 -import pdfium.kniBridge16 -import pdfium.kniBridge2 -import pdfium.kniBridge26 -import pdfium.kniBridge27 -import pdfium.kniBridge5 -import pdfium.kniBridge52 -import pdfium.kniBridge53 -import pdfium.kniBridge54 -import pdfium.kniBridge71 +import java.io.File +import pdfium.FPDF_InitLibrary +import pdfium.FPDF_GetPageCount +import pdfium.FPDF_LoadPage +import pdfium.FPDF_ClosePage +import pdfium.FPDF_CloseDocument +import pdfium.FPDF_LoadDocument +import pdfium.FPDFText_LoadPage +import pdfium.FPDFText_ClosePage +import pdfium.FPDFText_CountChars +import pdfium.FPDFText_IsHyphen +import pdfium.FPDFText_GetCharBox +import pdfium.FPDFText_GetText /** - * The JVM half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. + * pdfium is initialised once per process and never destroyed. * - * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here - * with the name from its doc comment, and nothing else in the module sees them. + * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like: pairing them per call makes + * extraction differ between one process and the next, where initialising once gives every process the same answer. */ -internal actual fun extractText(bytes: ByteArray): String { - val text = StringBuilder() +private val pdfiumLibrary: Lazy = lazy { FPDF_InitLibrary() } - initLibrary() +/** + * The Android half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. + * + * The bindings carry the names of the C functions they call, so this reads as pdfium's own API and needs no wrapper + * layer in between. + * + * The bytes reach pdfium as a file rather than as a buffer. `FPDF_LoadMemDocument` keeps the caller's pointer and + * reads through it for as long as the document is open, but the generated bridge pairs `GetByteArrayElements` with + * `ReleaseByteArrayElements` and lets go before it returns, so every page load after that reads memory the JVM has + * taken back. It mostly works, which is the worst way to fail: the first conversion in a process reads an untouched + * region and is correct, later ones come back four generated spaces short and `AutoGen uses` reads as `AutoGenuses`. + * Zeroing the array under an open document drops extraction to nothing, which is what proved it. `FPDF_LoadDocument` + * owns everything it reads. The native leg needs none of this: `usePinned` holds the array for the document's life. + */ +internal actual fun extractPages(bytes: ByteArray): List { + val pages = mutableListOf() + + pdfiumLibrary.value + // The file has to outlive the document: pdfium reads it lazily, the same way it would read a buffer. + val file = File.createTempFile("mikromarkdown", ".pdf") try { - val document = loadDocument(bytes, null) - if (document == 0L) return "" + file.writeBytes(bytes) + val document = FPDF_LoadDocument(file.absolutePath, null) + if (document == 0L) return emptyList() try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) + for (index in 0 until FPDF_GetPageCount(document)) { + val page = FPDF_LoadPage(document, index) if (page == 0L) continue - val textPage = loadTextPage(page) + val textPage = FPDFText_LoadPage(page) if (textPage != 0L) { - text.append(pageText(textPage)) - text.append('\n') - closeTextPage(textPage) + pages += pageText(textPage) + FPDFText_ClosePage(textPage) } - closePage(page) + FPDF_ClosePage(page) } } finally { - closeDocument(document) + FPDF_CloseDocument(document) } } finally { - destroyLibrary() + file.delete() } - return text.toString() + return pages } /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ -private fun pageText(textPage: Long): String { - val count = charCount(textPage) - if (count <= 0) return "" +private fun pageText(textPage: Long): PageText { + val count = FPDFText_CountChars(textPage) + if (count <= 0) return PageText("", IntArray(0)) val buffer = ShortArray(count + 1) - val written = readText(textPage, 0, count, buffer) - return if (written <= 1) "" else CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + val written = FPDFText_GetText(textPage, 0, count, buffer) + if (written <= 1) return PageText("", IntArray(0)) + // One character in, one character out, so a text index is a pdfium character index. + val text = CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + return PageText(text, hyphenWraps(textPage, text)) } -private fun initLibrary() = kniBridge1() - -private fun destroyLibrary() = kniBridge2() - -private fun loadDocument(bytes: ByteArray, password: String?): Long = kniBridge5(bytes, bytes.size, password) - -private fun pageCount(document: Long): Int = kniBridge15(document) - -private fun loadPage(document: Long, index: Int): Long = kniBridge16(document, index) - -private fun closePage(page: Long) = kniBridge26(page) - -private fun closeDocument(document: Long) = kniBridge27(document) - -private fun loadTextPage(page: Long): Long = kniBridge52(page) - -private fun closeTextPage(textPage: Long) = kniBridge53(textPage) +/** + * The collapsed wraps, as [PageText.hyphenWraps] describes them. + * + * `FPDFText_IsHyphen` marks every hyphen pdfium removed, and the character box is what proves the line ended there + * rather than the marker standing for something else on the same line. + */ +private fun hyphenWraps(textPage: Long, text: String): IntArray { + val wraps = mutableListOf() + for (index in text.indices) { + if (text[index] != HYPHEN_MARKER) continue + if (FPDFText_IsHyphen(textPage, index) != 0 && startsLowerLine(textPage, index)) wraps += index + } + return wraps.toIntArray() +} -private fun charCount(textPage: Long): Int = kniBridge54(textPage) +/** Whether the character after [index] sits below it — which is what makes [index] the end of a line. */ +private fun startsLowerLine(textPage: Long, index: Int): Boolean { + val left = DoubleArray(1) + val right = DoubleArray(1) + val bottom = DoubleArray(1) + val top = DoubleArray(1) + + if (FPDFText_GetCharBox(textPage, index, left, right, bottom, top) == 0) return false + val hyphenBottom = bottom[0] + if (FPDFText_GetCharBox(textPage, index + 1, left, right, bottom, top) == 0) return false + return top[0] < hyphenBottom +} -private fun readText(textPage: Long, start: Int, count: Int, buffer: ShortArray): Int = - kniBridge71(textPage, start, count, buffer) diff --git a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageText.kt b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageText.kt new file mode 100644 index 0000000..2b414b2 --- /dev/null +++ b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageText.kt @@ -0,0 +1,66 @@ +package io.github.lemcoder.mikromarkdown.pdf + +/** + * One page's text, and the line breaks pdfium collapsed while producing it. + * + * A hyphenated wrap does not reach [text] as a hyphen and a line break. pdfium recognises the wrap, joins the two + * lines, and leaves [HYPHEN_MARKER] standing where the hyphen was — so `con-` at the right margin and `firming` at the + * start of the next line arrive adjacent, with the marker between them and nothing to say a line ever ended there. + * + * [hyphenWraps] are the marker positions that `FPDFText_GetCharBox` confirms: the character after the marker really + * does begin a lower line. That is the one thing the geometry answers outright, and it is what [restored] needs. + */ +internal class PageText(val text: String, val hyphenWraps: IntArray) + +/** pdfium's stand-in for a hyphen it removed at a line break. */ +internal const val HYPHEN_MARKER: Char = '\uFFFE' + +/** + * Puts the wrap back the way the page shows it. + * + * Whether the hyphen was the author's (`chat-optimized`, broken across lines by chance) or the typesetter's + * (`con-firming`) is not something the geometry can answer: both sit at the right margin with the next line below. So + * the hyphen is written back as the page had it and the question is left open — [plainTextBlocks] + * [io.github.lemcoder.mikromarkdown.model.plainTextBlocks] closes the break without touching it. + * + * A marker the geometry did not confirm was not a wrap; there is no character it stands for, so nothing is written. + */ +internal fun List.restored(): String { + val hyphen = hyphenWritten() + return joinToString("\n") { it.restored(hyphen) } +} + +private fun PageText.restored(hyphen: Char): String { + if (HYPHEN_MARKER !in text) return text + + val wraps = hyphenWraps.toHashSet() + return buildString(text.length + hyphenWraps.size) { + for ((index, char) in text.withIndex()) { + when { + char != HYPHEN_MARKER -> append(char) + index in wraps -> append(hyphen).append('\n') + } + } + } +} + +/** + * The hyphen this document writes between words. + * + * The marker does not say which character it replaced — pdfium keeps no record of it, and the one call that could + * answer, `FPDFTextObj_GetText`, needs a header we do not bind. So the document is asked instead: whichever hyphen it + * uses in the compounds it did not break is the one to put back, rather than U+002D on a page set in U+2010. + */ +private fun List.hyphenWritten(): Char { + val counts = mutableMapOf() + for (page in this) { + for (match in WRITTEN_HYPHEN.findAll(page.text)) { + val hyphen = match.value[1] + counts[hyphen] = (counts[hyphen] ?: 0) + 1 + } + } + return counts.maxByOrNull { it.value }?.key ?: '-' +} + +/** A hyphen between two letters, which is one the document meant rather than one a line break produced. */ +private val WRITTEN_HYPHEN = Regex("\\p{L}[-\u2010\u2011]\\p{L}") diff --git a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt index 273d4e5..891ccc2 100644 --- a/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt +++ b/pdfium/src/commonMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverter.kt @@ -15,7 +15,9 @@ import io.github.lemcoder.mikromarkdown.model.plainTextBlocks * ``` * * The extraction itself is per-platform — cinterop on native, generated JNI bridges on the JVM — but both reach the - * same pdfium, and everything above [extractText] is shared. + * same pdfium, and everything above [extractPages] is shared: each leg reports what pdfium saw, and one copy of the + * rules here turns it into a document. Deciding what a wrapped hyphen meant belongs to `plainTextBlocks`, which reads + * the whole document rather than one page. */ public class PdfiumConverter : DocumentConverter { @@ -24,50 +26,8 @@ public class PdfiumConverter : DocumentConverter { } override fun parse(bytes: ByteArray, info: StreamInfo): Document = - Document(blocks = plainTextBlocks(extractText(bytes).joinHyphenatedWords())) - - /** - * pdfium emits U+FFFE where a glyph has no Unicode mapping, which in a typeset document is nearly always the hyphen - * at a line break. - * - * Dropping it always would fuse real compounds — "chat-optimized" became "chatoptimized" — so the document decides: - * if both halves appear elsewhere as words in their own right the hyphen was the author's and is restored, - * otherwise the halves are one broken word and are joined. The halves themselves are cut from the vocabulary first, - * since they are only in the text because the break put them there. - * - * This is a heuristic standing in for geometry. A hyphenation hyphen ends a line and a compound hyphen does not, - * which `FPDFText_GetCharBox` would answer outright. - */ - private fun String.joinHyphenatedWords(): String { - if (indexOf(UNMAPPED_GLYPH) < 0) return this - - val vocabulary = WORD.findAll(replace(HYPHEN_BREAK, " ")).map { it.value.lowercase() }.toSet() - val out = StringBuilder(length) - for (index in indices) { - val char = this[index] - if (char != UNMAPPED_GLYPH) { - out.append(char) - continue - } - var wordStart = out.length - while (wordStart > 0 && out[wordStart - 1].isLetter()) wordStart-- - val left = out.subSequence(wordStart, out.length).toString().lowercase() - val right = substring(index + 1).takeWhile { it.isLetter() }.lowercase() - if (isRealCompound(left, right, vocabulary)) out.append('-') - } - return out.toString() - } - - /** A hyphen the author wrote, rather than one the typesetter added at a line break. */ - private fun isRealCompound(left: String, right: String, vocabulary: Set): Boolean = - left.isNotEmpty() && right.isNotEmpty() && left in vocabulary && right in vocabulary - - private companion object { - const val UNMAPPED_GLYPH = '\uFFFE' - val WORD = Regex("[\\p{L}]{2,}") - val HYPHEN_BREAK = Regex("\\p{L}+\uFFFE\\p{L}+") - } + Document(blocks = plainTextBlocks(extractPages(bytes).restored())) } -/** Every page's text, concatenated, one page per line. */ -internal expect fun extractText(bytes: ByteArray): String +/** Every page, in order; [PageText] carries the text and the wraps pdfium collapsed. */ +internal expect fun extractPages(bytes: ByteArray): List diff --git a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt index 243f89a..2686cb0 100644 --- a/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt +++ b/pdfium/src/jvmMain/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.jvm.kt @@ -1,80 +1,111 @@ package io.github.lemcoder.mikromarkdown.pdf -import pdfium.kniBridge1 -import pdfium.kniBridge15 -import pdfium.kniBridge16 -import pdfium.kniBridge2 -import pdfium.kniBridge26 -import pdfium.kniBridge27 -import pdfium.kniBridge5 -import pdfium.kniBridge52 -import pdfium.kniBridge53 -import pdfium.kniBridge54 -import pdfium.kniBridge71 +import java.io.File +import pdfium.FPDFText_ClosePage +import pdfium.FPDFText_CountChars +import pdfium.FPDFText_GetCharBox +import pdfium.FPDFText_GetText +import pdfium.FPDFText_IsHyphen +import pdfium.FPDFText_LoadPage +import pdfium.FPDF_CloseDocument +import pdfium.FPDF_ClosePage +import pdfium.FPDF_GetPageCount +import pdfium.FPDF_InitLibrary +import pdfium.FPDF_LoadDocument +import pdfium.FPDF_LoadPage + +/** + * pdfium is initialised once per process and never destroyed. + * + * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like: pairing them per call makes + * extraction differ between one process and the next, where initialising once gives every process the same answer. + */ +private val pdfiumLibrary: Lazy = lazy { FPDF_InitLibrary() } /** * The JVM half, over the JNI bridges the Konan plugin generates from the same `.def` cinterop binds. * - * The bridges are numbered rather than named — that is what a runtime-free binding looks like — so each is wrapped here - * with the name from its doc comment, and nothing else in the module sees them. + * The bindings carry the names of the C functions they call, so this reads as pdfium's own API and needs no wrapper + * layer in between. + * + * The bytes reach pdfium as a file rather than as a buffer. `FPDF_LoadMemDocument` keeps the caller's pointer and reads + * through it for as long as the document is open, but the generated bridge pairs `GetByteArrayElements` with + * `ReleaseByteArrayElements` and lets go before it returns, so every page load after that reads memory the JVM has + * taken back. It mostly works, which is the worst way to fail: the first conversion in a process reads an untouched + * region and is correct, later ones come back four generated spaces short and `AutoGen uses` reads as `AutoGenuses`. + * Zeroing the array under an open document drops extraction to nothing, which is what proved it. `FPDF_LoadDocument` + * owns everything it reads. The native leg needs none of this: `usePinned` holds the array for the document's life. */ -internal actual fun extractText(bytes: ByteArray): String { - val text = StringBuilder() +internal actual fun extractPages(bytes: ByteArray): List { + val pages = mutableListOf() - initLibrary() + pdfiumLibrary.value + // The file has to outlive the document: pdfium reads it lazily, the same way it would read a buffer. + val file = File.createTempFile("mikromarkdown", ".pdf") try { - val document = loadDocument(bytes, null) - if (document == 0L) return "" + file.writeBytes(bytes) + val document = FPDF_LoadDocument(file.absolutePath, null) + if (document == 0L) return emptyList() try { - for (index in 0 until pageCount(document)) { - val page = loadPage(document, index) + for (index in 0 until FPDF_GetPageCount(document)) { + val page = FPDF_LoadPage(document, index) if (page == 0L) continue - val textPage = loadTextPage(page) + val textPage = FPDFText_LoadPage(page) if (textPage != 0L) { - text.append(pageText(textPage)) - text.append('\n') - closeTextPage(textPage) + pages += pageText(textPage) + FPDFText_ClosePage(textPage) } - closePage(page) + FPDF_ClosePage(page) } } finally { - closeDocument(document) + FPDF_CloseDocument(document) } } finally { - destroyLibrary() + file.delete() } - return text.toString() + return pages } /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ -private fun pageText(textPage: Long): String { - val count = charCount(textPage) - if (count <= 0) return "" +private fun pageText(textPage: Long): PageText { + val count = FPDFText_CountChars(textPage) + if (count <= 0) return PageText("", IntArray(0)) val buffer = ShortArray(count + 1) - val written = readText(textPage, 0, count, buffer) - return if (written <= 1) "" else CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + val written = FPDFText_GetText(textPage, 0, count, buffer) + if (written <= 1) return PageText("", IntArray(0)) + // One character in, one character out, so a text index is a pdfium character index. + val text = CharArray(written - 1) { Char(buffer[it].toInt() and 0xFFFF) }.concatToString() + return PageText(text, hyphenWraps(textPage, text)) } -private fun initLibrary() = kniBridge1() - -private fun destroyLibrary() = kniBridge2() - -private fun loadDocument(bytes: ByteArray, password: String?): Long = kniBridge5(bytes, bytes.size, password) - -private fun pageCount(document: Long): Int = kniBridge15(document) - -private fun loadPage(document: Long, index: Int): Long = kniBridge16(document, index) - -private fun closePage(page: Long) = kniBridge26(page) - -private fun closeDocument(document: Long) = kniBridge27(document) - -private fun loadTextPage(page: Long): Long = kniBridge52(page) - -private fun closeTextPage(textPage: Long) = kniBridge53(textPage) - -private fun charCount(textPage: Long): Int = kniBridge54(textPage) +/** + * The collapsed wraps, as [PageText.hyphenWraps] describes them. + * + * `FPDFText_IsHyphen` marks every hyphen pdfium removed, and the character box is what proves the line ended there + * rather than the marker standing for something else on the same line. + */ +private fun hyphenWraps(textPage: Long, text: String): IntArray { + val wraps = mutableListOf() + for (index in text.indices) { + if ( + text[index] == HYPHEN_MARKER && FPDFText_IsHyphen(textPage, index) != 0 && startsLowerLine(textPage, index) + ) { + wraps += index + } + } + return wraps.toIntArray() +} -private fun readText(textPage: Long, start: Int, count: Int, buffer: ShortArray): Int = - kniBridge71(textPage, start, count, buffer) +/** Whether the character after [index] sits below it — which is what makes [index] the end of a line. */ +private fun startsLowerLine(textPage: Long, index: Int): Boolean { + val left = DoubleArray(1) + val right = DoubleArray(1) + val bottom = DoubleArray(1) + val top = DoubleArray(1) + + if (FPDFText_GetCharBox(textPage, index, left, right, bottom, top) == 0) return false + val hyphenBottom = bottom[0] + if (FPDFText_GetCharBox(textPage, index + 1, left, right, bottom, top) == 0) return false + return top[0] < hyphenBottom +} diff --git a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageTextTest.kt b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageTextTest.kt new file mode 100644 index 0000000..adfdb12 --- /dev/null +++ b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PageTextTest.kt @@ -0,0 +1,61 @@ +package io.github.lemcoder.mikromarkdown.pdf + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** Pins what [restored] does with pdfium's hyphen marker, without needing a PDF that happens to contain one. */ +class PageTextTest { + + private val marker = HYPHEN_MARKER + + @Test + fun `a confirmed wrap gets its hyphen and its line break back`() { + val page = PageText("con${marker}firming it", intArrayOf(3)) + + assertEquals("con-\nfirming it", listOf(page).restored()) + } + + @Test + fun `the hyphen put back is the one the document writes`() { + // U+2010 in the compound the document did not break, so U+2010 is what the wrap gets. + val page = PageText("con${marker}firming a well\u2010known case", intArrayOf(3)) + + assertEquals("con\u2010\nfirming a well\u2010known case", listOf(page).restored()) + } + + @Test + fun `a document with no written hyphen falls back to U+002D`() { + val page = PageText("con${marker}firming it", intArrayOf(3)) + + assertEquals('-', listOf(page).restored()[3]) + } + + @Test + fun `the hyphen is chosen across every page, not one at a time`() { + val first = PageText("a well\u2010known case", IntArray(0)) + val second = PageText("con${marker}firming it", intArrayOf(3)) + + assertEquals("a well\u2010known case\ncon\u2010\nfirming it", listOf(first, second).restored()) + } + + @Test + fun `a marker the geometry did not confirm stands for nothing`() { + val page = PageText("Section $marker 3", IntArray(0)) + + assertEquals("Section 3", listOf(page).restored()) + } + + @Test + fun `each marker is judged on its own`() { + val page = PageText("con${marker}firming and Section $marker 3", intArrayOf(3)) + + assertEquals("con-\nfirming and Section 3", listOf(page).restored()) + } + + @Test + fun `text without markers is returned as it is`() { + val page = PageText("nothing to restore", IntArray(0)) + + assertEquals("nothing to restore", listOf(page).restored()) + } +} diff --git a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt index fa85c96..4043762 100644 --- a/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt +++ b/pdfium/src/jvmTest/kotlin/io/github/lemcoder/mikromarkdown/pdf/PdfiumConverterTest.kt @@ -4,6 +4,7 @@ import io.github.lemcoder.mikromarkdown.StreamInfo import java.io.File import kotlin.test.Test import kotlin.test.assertContains +import kotlin.test.assertEquals import kotlin.test.assertTrue class PdfiumConverterTest { @@ -17,8 +18,44 @@ class PdfiumConverterTest { assertTrue(text.length > 1000, "expected a page of text, got ${text.length} characters") assertContains(text, "Introduction") - // The de-hyphenation ran: pdfium reports a broken word with U+FFFE between the halves. - assertContains(text, "confirming") - assertTrue('￾' !in text, "unmapped glyphs should not reach the model") + // A wrap the character boxes confirmed: the break is gone and the hyphen the page shows is still there. + assertContains(text, "con-firming") + // The other kind, the author's own, comes through the same way rather than fused into "chatoptimized". + assertContains(text, "chat-optimized") + assertTrue(HYPHEN_MARKER !in text, "pdfium's hyphen marker should not reach the model") + } + + /** + * Converting twice in one process must read the same both times. + * + * It did not. `FPDF_LoadMemDocument` reads the caller's buffer for as long as the document is open, and the + * generated bridge released it before returning, so pdfium spent every page load reading memory the JVM had taken + * back. The first conversion in a process was right and the rest lost four generated spaces. + */ + @Test + fun `converting the same bytes twice gives the same document`() { + val bytes = fixture.readBytes() + val first = PdfiumConverter().parse(bytes, StreamInfo(extension = "pdf")) + val second = PdfiumConverter().parse(bytes, StreamInfo(extension = "pdf")) + + assertEquals(first.blocks, second.blocks) + assertContains(first.blocks.joinToString { it.toString() }, "AutoGen uses") + } + + /** + * Every marker in this paper is a wrap, and the character boxes say so. + * + * The assertion is the geometry's, not the text's: it is what separates "pdfium removed a hyphen here" from "the + * line ended here", and only the second one earns a break back. + */ + @Test + fun `every hyphen marker in the fixture is confirmed by its character box`() { + val pages = extractPages(fixture.readBytes()) + + val markers = pages.sumOf { page -> page.text.count { it == HYPHEN_MARKER } } + val confirmed = pages.sumOf { it.hyphenWraps.size } + + assertTrue(markers > 0, "the fixture is supposed to contain hyphenated wraps") + assertEquals(markers, confirmed, "a marker the geometry did not confirm would be dropped silently") } } diff --git a/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt b/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt index 655e0f1..3b0ed36 100644 --- a/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt +++ b/pdfium/src/macosArm64Main/kotlin/io/github/lemcoder/mikromarkdown/pdf/Pdfium.macos.kt @@ -1,63 +1,113 @@ package io.github.lemcoder.mikromarkdown.pdf +import kotlinx.cinterop.DoubleVar import kotlinx.cinterop.ExperimentalForeignApi import kotlinx.cinterop.UShortVar import kotlinx.cinterop.addressOf +import kotlinx.cinterop.alloc import kotlinx.cinterop.allocArray import kotlinx.cinterop.get import kotlinx.cinterop.memScoped +import kotlinx.cinterop.ptr import kotlinx.cinterop.usePinned +import kotlinx.cinterop.value import pdfium.FPDFText_ClosePage import pdfium.FPDFText_CountChars +import pdfium.FPDFText_GetCharBox import pdfium.FPDFText_GetText +import pdfium.FPDFText_IsHyphen import pdfium.FPDFText_LoadPage import pdfium.FPDF_CloseDocument import pdfium.FPDF_ClosePage -import pdfium.FPDF_DestroyLibrary import pdfium.FPDF_GetPageCount import pdfium.FPDF_InitLibrary import pdfium.FPDF_LoadMemDocument import pdfium.FPDF_LoadPage +/** + * pdfium is initialised once per process and never destroyed. + * + * `FPDF_InitLibrary` and `FPDF_DestroyLibrary` are not the matching pair they look like. Pairing them per call makes + * extraction differ between one process and the next; initialising once gives every process the same sequence, which is + * the only thing this buys. + * + * It does not make extraction repeatable. pdfium carries state across document loads that the first conversion in a + * process does not see, and from the second conversion on the AutoGen fixture comes back four generated spaces short — + * `AutoGen uses` reads as `AutoGenuses`. Destroying the library between conversions does not reset it. Unfixed, and + * below this line rather than in it. + */ +@OptIn(ExperimentalForeignApi::class) private val pdfiumLibrary: Lazy = lazy { FPDF_InitLibrary() } + /** The native half, over the cinterop bindings. */ @OptIn(ExperimentalForeignApi::class) -internal actual fun extractText(bytes: ByteArray): String { - val text = StringBuilder() - - FPDF_InitLibrary() - try { - bytes.usePinned { pinned -> - val document = FPDF_LoadMemDocument(pinned.addressOf(0), bytes.size, null) ?: return@usePinned - try { - for (index in 0 until FPDF_GetPageCount(document)) { - val page = FPDF_LoadPage(document, index) ?: continue - val textPage = FPDFText_LoadPage(page) - if (textPage != null) { - text.append(pageText(textPage)) - text.append('\n') - FPDFText_ClosePage(textPage) - } - FPDF_ClosePage(page) +internal actual fun extractPages(bytes: ByteArray): List { + val pages = mutableListOf() + + pdfiumLibrary.value + bytes.usePinned { pinned -> + val document = FPDF_LoadMemDocument(pinned.addressOf(0), bytes.size, null) ?: return@usePinned + try { + for (index in 0 until FPDF_GetPageCount(document)) { + val page = FPDF_LoadPage(document, index) ?: continue + val textPage = FPDFText_LoadPage(page) + if (textPage != null) { + pages += pageText(textPage) + FPDFText_ClosePage(textPage) } - } finally { - FPDF_CloseDocument(document) + FPDF_ClosePage(page) } + } finally { + FPDF_CloseDocument(document) } - } finally { - FPDF_DestroyLibrary() } - return text.toString() + return pages } /** pdfium writes UTF-16 into a caller-supplied buffer and counts the terminating NUL. */ @OptIn(ExperimentalForeignApi::class) -private fun pageText(textPage: pdfium.FPDF_TEXTPAGE): String { +private fun pageText(textPage: pdfium.FPDF_TEXTPAGE): PageText { val count = FPDFText_CountChars(textPage) - if (count <= 0) return "" - return memScoped { + if (count <= 0) return PageText("", IntArray(0)) + val text = memScoped { val buffer = allocArray(count + 1) val written = FPDFText_GetText(textPage, 0, count, buffer) + // One character in, one character out, so a text index is a pdfium character index. if (written <= 1) "" else CharArray(written - 1) { Char(buffer[it].toInt()) }.concatToString() } + if (text.isEmpty()) return PageText("", IntArray(0)) + return PageText(text, hyphenWraps(textPage, text)) +} + +/** + * The collapsed wraps, as [PageText.hyphenWraps] describes them. + * + * `FPDFText_IsHyphen` marks every hyphen pdfium removed, and the character box is what proves the line ended there + * rather than the marker standing for something else on the same line. + */ +@OptIn(ExperimentalForeignApi::class) +private fun hyphenWraps(textPage: pdfium.FPDF_TEXTPAGE, text: String): IntArray { + val wraps = mutableListOf() + for (index in text.indices) { + if ( + text[index] == HYPHEN_MARKER && FPDFText_IsHyphen(textPage, index) != 0 && startsLowerLine(textPage, index) + ) { + wraps += index + } + } + return wraps.toIntArray() +} + +/** Whether the character after [index] sits below it — which is what makes [index] the end of a line. */ +@OptIn(ExperimentalForeignApi::class) +private fun startsLowerLine(textPage: pdfium.FPDF_TEXTPAGE, index: Int): Boolean = memScoped { + val left = alloc() + val right = alloc() + val bottom = alloc() + val top = alloc() + + if (FPDFText_GetCharBox(textPage, index, left.ptr, right.ptr, bottom.ptr, top.ptr) == 0) return false + val hyphenBottom = bottom.value + if (FPDFText_GetCharBox(textPage, index + 1, left.ptr, right.ptr, bottom.ptr, top.ptr) == 0) return false + top.value < hyphenBottom } diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 35b6406..80a3e48 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -18,7 +18,7 @@ structure headings / tables / list items / links / images / code fences table health share of table rows whose column count matches the header hygiene 3+ blank-line runs, trailing whitespace, unescaped pipes - speed best of N CLI runs, and the same minus measured JVM startup + speed best of N CLI runs """ from __future__ import annotations @@ -196,15 +196,6 @@ def main() -> int: state = "ready" if engine.command else f"MISSING ({engine.note})" print(f"{engine.name:14} {state}", file=sys.stderr) - # JVM startup cost, so the Kotlin CLI's timing can be reported with and without it. - jvm_baseline = 0.0 - mikro = next(e for e in engines if e.name == "mikromarkdown") - if mikro.command: - probe = out_dir / "_probe.txt" - probe.write_text("x\n") - _, jvm_baseline = mikro.run(probe) - probe.unlink() - results: dict[str, dict[str, object]] = {} for fixture in fixtures: print(f"converting {fixture.name}", file=sys.stderr) @@ -218,8 +209,7 @@ def main() -> int: results[fixture.name] = per_engine lines: list[str] = ["# Conversion benchmark", ""] - lines.append(f"Fixtures: {len(fixtures)} · runs per file: {RUNS} (best kept) · " - f"measured JVM startup: {jvm_baseline * 1000:.0f} ms") + lines.append(f"Fixtures: {len(fixtures)} · runs per file: {RUNS} (best kept)") lines.append("") lines.append("Recall is against the *consensus* vocabulary: tokens (>3 chars) that at least two " "engines emit for the same file. Missing tokens mean dropped content.") @@ -248,11 +238,7 @@ def main() -> int: recall = len(own & consensus) / len(consensus) if consensus else 1.0 unique = len(own - set().union(*[v for k, v in token_sets.items() if k != engine_name]) ) \ if len(token_sets) > 1 else len(own) - ms = data["seconds"] * 1000 - if engine_name == "mikromarkdown": - ms_display = f"{ms:.0f} ({max(0.0, ms - jvm_baseline * 1000):.0f} warm)" - else: - ms_display = f"{ms:.0f}" + ms_display = f"{data['seconds'] * 1000:.0f}" lines.append( f"| {engine_name} | {recall * 100:.1f}% | {unique} | {len(text)} | {s['headings']} | " f"{s['list_items']} | {s['links']} | {s['images']} | {s['tables']} | {s['table_rows']} | " diff --git a/third-party/anydoc b/third-party/anydoc index 4a45add..4e3089b 160000 --- a/third-party/anydoc +++ b/third-party/anydoc @@ -1 +1 @@ -Subproject commit 4a45addbd607e8b59f0c263bca26aab228e10370 +Subproject commit 4e3089b1ed43404241a303109f81e2c7933040b2