diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f029b2437..812dec3a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,6 +306,8 @@ jobs: npm-alias-matrix: ${{ steps.released.outputs.npm-alias-matrix }} attest-matrix: ${{ steps.released.outputs.attest-matrix }} sea-matrix: ${{ steps.released.outputs.sea-matrix }} + container-matrix: ${{ steps.released.outputs.container-matrix }} + container-image-matrix: ${{ steps.released.outputs.container-image-matrix }} steps: # main requires status checks to merge, and the default GITHUB_TOKEN has no bypass for that -- the orchestrator's release commits, dependency-bump commits, and tags are direct pushes to main, so they need a token from an actor the branch ruleset explicitly allows to bypass (see ExaDev/.github and this repo's own ruleset). Everything else in this job (release notes, GitHub Release creation, issue/PR comments, OIDC npm publish) keeps using secrets.GITHUB_TOKEN below, unaffected -- only checkout's own git credentials need the elevated token, since that's what the later pushes inherit. - name: Generate a token for the release pushes @@ -343,7 +345,7 @@ jobs: NODE_AUTH_TOKEN: "" - name: Collect the packages this run released id: released - # Builds the four matrices the post-release jobs fan out over: one attestation leg per released published package, one GitHub Packages leg per released package the map mirrors there, one npm-alias leg per (released package, alias) pair, and one SEA-binary leg per (SEA-eligible released package, platform) pair. The version comes from each package's own package.json in this working tree -- the orchestrator has already committed the released version by the time this step runs, so it is the same version the tag names. The web UI is private and publishes nothing, so it is filtered out. + # Builds the matrices the post-release jobs fan out over: one attestation leg per released published package, one GitHub Packages leg per released package the map mirrors there, one npm-alias leg per (released package, alias) pair, one SEA-binary leg per (SEA-eligible released package, platform) pair, one container-image leg per (container-eligible released package, Linux architecture) pair, and one container-manifest leg per container-eligible released package. The version comes from each package's own package.json in this working tree -- the orchestrator has already committed the released version by the time this step runs, so it is the same version the tag names. The web UI is private and publishes nothing, so it is filtered out. run: | git ls-remote --tags origin | sed 's|.*refs/tags/||' | grep -v '\^{}' | sort > "$RUNNER_TEMP/release-tags-after.txt" comm -13 "$RUNNER_TEMP/release-tags-before.txt" "$RUNNER_TEMP/release-tags-after.txt" > "$RUNNER_TEMP/release-tags-new.txt" @@ -351,6 +353,8 @@ jobs: NPM_ALIASES='[]' ATTEST='[]' SEA='[]' + CONTAINER='[]' + CONTAINER_IMAGES='[]' while read -r TAG; do [ -n "$TAG" ] || continue NAME=${TAG%@*} @@ -383,12 +387,32 @@ jobs: done ;; esac + # The two packages this workspace runs as long-lived servers rather than a one-shot CLI/TUI -- document-cli is deliberately excluded, per the root README's Releases section. This needs two shapes: one leg per (package, architecture) for build-container-images (its own per-arch build+push), and one leg per package for publish-container-manifests (which merges both architectures' images into one manifest) -- there's no way to dedupe the former into the latter inside a job's own `fromJSON(...)` matrix expression, so both are built here directly. isLatest is "true" here (a genuine release); collect-backfill-matrix's own copy of this block sets it "false", since a historical catch-up run must never regress the floating `latest` tag to an older version. + case "$NAME" in + document-mcp | document-rest) + IMAGE="" + [ "$NAME" = "document-rest" ] && IMAGE="documents.js" + [ "$NAME" = "document-mcp" ] && IMAGE="document-mcp" + CONTAINER_IMAGES=$(echo "$CONTAINER_IMAGES" | jq -c --arg name "$NAME" --arg version "$VERSION" --arg tag "$TAG" --arg image "$IMAGE" \ + '. + [{name: $name, version: $version, tag: $tag, image: $image, isLatest: "true"}]') + for OS in ubuntu-latest ubuntu-24.04-arm; do + case "$OS" in + ubuntu-latest) DOCKER_ARCH=amd64; NODE_ARCH=x64 ;; + ubuntu-24.04-arm) DOCKER_ARCH=arm64; NODE_ARCH=arm64 ;; + esac + CONTAINER=$(echo "$CONTAINER" | jq -c --arg name "$NAME" --arg version "$VERSION" --arg tag "$TAG" --arg image "$IMAGE" --arg os "$OS" --arg dockerArch "$DOCKER_ARCH" --arg nodeArch "$NODE_ARCH" \ + '. + [{name: $name, version: $version, tag: $tag, image: $image, os: $os, dockerArch: $dockerArch, nodeArch: $nodeArch}]') + done + ;; + esac done < "$RUNNER_TEMP/release-tags-new.txt" { echo "github-packages-matrix=$GITHUB_PACKAGES" echo "npm-alias-matrix=$NPM_ALIASES" echo "attest-matrix=$ATTEST" echo "sea-matrix=$SEA" + echo "container-matrix=$CONTAINER" + echo "container-image-matrix=$CONTAINER_IMAGES" } >> "$GITHUB_OUTPUT" # Everything the release job gates on (commitlint/lint/typecheck/test/test-workers/test-smoke/knip) validates the commit BEFORE release bumps any versions -- never the workspace state AFTER. The orchestrator's own release/dependency-bump commits are pushed with `[skip ci]` (deliberately: a push authenticated this way would otherwise re-trigger this same workflow and risk a concurrent release racing itself, per the release job's own comment above), so nothing else ever runs CI against post-release main either. A package released with a breaking (major) change is therefore never checked against its own workspace-local dependents until whichever of THEIR test suites next happens to run for an unrelated reason -- exactly what caught document-schema.js's 7.0.0 major landing with documents.js's own committed example dumps still pinned to 6.0.0 (ExaDev/documents.js, 2026-09-07): main was red from the moment that release commit landed, silently, until a later PR's own turbo --affected run (which doesn't even touch documents.js on an unrelated branch) happened not to catch it either. This job closes that gap: re-run every check that already exists above, against the real post-release commit on main, so a release that breaks a sibling package inside this same workspace fails LOUDLY on this same workflow run's own summary rather than staying invisible until someone else's unrelated work trips over it. It cannot un-publish a release that already went out -- nothing running after the fact can -- but a red run here is exactly the loud, immediate signal that should prompt a same-day follow-up fix, the way this file's own commit history already shows one landing for the 7.0.0 case. @@ -436,6 +460,8 @@ jobs: npm-alias-matrix: ${{ steps.collect.outputs.npm-alias-matrix }} attest-matrix: ${{ steps.collect.outputs.attest-matrix }} sea-matrix: ${{ steps.collect.outputs.sea-matrix }} + container-matrix: ${{ steps.collect.outputs.container-matrix }} + container-image-matrix: ${{ steps.collect.outputs.container-image-matrix }} steps: - uses: actions/checkout@v7 with: @@ -451,7 +477,9 @@ jobs: NPM_ALIASES='[]' ATTEST='[]' SEA='[]' - # Process substitution, not a pipe: a pipe would run the loop in a subshell, where GITHUB_PACKAGES/NPM_ALIASES/ATTEST/SEA's accumulated values never escape back to the `>> "$GITHUB_OUTPUT"` below. + CONTAINER='[]' + CONTAINER_IMAGES='[]' + # Process substitution, not a pipe: a pipe would run the loop in a subshell, where GITHUB_PACKAGES/NPM_ALIASES/ATTEST/SEA/CONTAINER/CONTAINER_IMAGES's accumulated values never escape back to the `>> "$GITHUB_OUTPUT"` below. while IFS= read -r RAW; do TAG=$(echo "$RAW" | xargs) [ -n "$TAG" ] || continue @@ -489,6 +517,24 @@ jobs: done ;; esac + # Mirrors the release job's own identical container-matrix case statement, with one deliberate difference: isLatest is "false" here, never "true" -- a backfill exists specifically to catch up a possibly-old, already-superseded tag after the fact, so floating the registry's `latest` tag onto it would regress `latest` to an older version if newer releases have shipped since. + case "$NAME" in + document-mcp | document-rest) + IMAGE="" + [ "$NAME" = "document-rest" ] && IMAGE="documents.js" + [ "$NAME" = "document-mcp" ] && IMAGE="document-mcp" + CONTAINER_IMAGES=$(echo "$CONTAINER_IMAGES" | jq -c --arg name "$NAME" --arg version "$VERSION" --arg tag "$TAG" --arg image "$IMAGE" \ + '. + [{name: $name, version: $version, tag: $tag, image: $image, isLatest: "false"}]') + for OS in ubuntu-latest ubuntu-24.04-arm; do + case "$OS" in + ubuntu-latest) DOCKER_ARCH=amd64; NODE_ARCH=x64 ;; + ubuntu-24.04-arm) DOCKER_ARCH=arm64; NODE_ARCH=arm64 ;; + esac + CONTAINER=$(echo "$CONTAINER" | jq -c --arg name "$NAME" --arg version "$VERSION" --arg tag "$TAG" --arg image "$IMAGE" --arg os "$OS" --arg dockerArch "$DOCKER_ARCH" --arg nodeArch "$NODE_ARCH" \ + '. + [{name: $name, version: $version, tag: $tag, image: $image, os: $os, dockerArch: $dockerArch, nodeArch: $nodeArch}]') + done + ;; + esac # printf '%s\n', not '%s': `tr` never adds a trailing newline of its own, and `read` silently drops the last line of a while-read loop when it hits EOF without one -- confirmed the hard way, the final tag in a comma-separated backfill_tags list was dropped from every matrix with no warning or error. The trailing \n here survives the comma-to-newline translation as the terminator the last tag needs. done < <(printf '%s\n' "$BACKFILL_TAGS" | tr ',' '\n') { @@ -496,6 +542,8 @@ jobs: echo "npm-alias-matrix=$NPM_ALIASES" echo "sea-matrix=$SEA" echo "attest-matrix=$ATTEST" + echo "container-matrix=$CONTAINER" + echo "container-image-matrix=$CONTAINER_IMAGES" } >> "$GITHUB_OUTPUT" # The three jobs below restore what the monorepo migration lost (ExaDev/documents.js#732): each standalone repository's own pipeline republished its package under alternate npm names and a @exadev/ scope on GitHub Packages, and signed an SPDX SBOM plus build-provenance attestation against the packed release tarball. They run after the release job (or, for a backfill, the collect-backfill-matrix job above) keyed off whichever of the two produced the matrices, and nothing depends on them -- a failure here can never block or fail the release itself or the Pages deploy. @@ -787,6 +835,88 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh release upload "${{ matrix.tag }}" --clobber "${{ steps.build.outputs.binary-path }}" + build-container-images: + name: Build container image (${{ matrix.image }}, ${{ matrix.dockerArch }}) + needs: [release, collect-backfill-matrix, build-sea-binaries] + # build-sea-binaries' own result is checked for "attempted", not "success": its own matrix spans every SEA-eligible package and OS at once (including document-cli and every non-Linux leg), so an unrelated leg failing there must never block this job -- the gh release download step below fails loudly and specifically instead if the one Linux asset this leg actually needs wasn't uploaded. + if: always() && (needs.release.result == 'success' || needs.collect-backfill-matrix.result == 'success') && (needs.release.outputs.container-matrix || needs.collect-backfill-matrix.outputs.container-matrix) != '[]' && (needs.build-sea-binaries.result == 'success' || needs.build-sea-binaries.result == 'failure') + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.release.outputs.container-matrix || needs.collect-backfill-matrix.outputs.container-matrix) }} + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ matrix.tag }} # the release commit the orchestrator tagged, not whatever main has moved on to by now + fetch-depth: 0 + - name: Download this leg's Linux SEA binary from the release + # Reuses the binary build-sea-binaries already built and uploaded for this exact (package, arch) pair rather than rebuilding it a second time here. + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p "packages/${{ matrix.name }}/dist-sea" + gh release download "${{ matrix.tag }}" \ + --pattern "${{ matrix.name }}-linux-${{ matrix.nodeArch }}" \ + --dir "packages/${{ matrix.name }}/dist-sea" \ + --clobber + mv "packages/${{ matrix.name }}/dist-sea/${{ matrix.name }}-linux-${{ matrix.nodeArch }}" \ + "packages/${{ matrix.name }}/dist-sea/${{ matrix.name }}" + # The distroless base image this package's own Dockerfile builds FROM has no shell to chmod this with, so the execute bit has to already be set on disk before `docker build` runs -- COPY preserves it. + chmod +x "packages/${{ matrix.name }}/dist-sea/${{ matrix.name }}" + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push this architecture's image + uses: docker/build-push-action@v6 + with: + context: packages/${{ matrix.name }} + platforms: linux/${{ matrix.dockerArch }} + push: true + tags: ghcr.io/exadev/${{ matrix.image }}:${{ matrix.version }}-${{ matrix.dockerArch }} + + publish-container-manifests: + name: Publish container manifest (${{ matrix.image }} ${{ matrix.version }}) + needs: [release, collect-backfill-matrix, build-container-images] + # build-container-images' own result is likewise checked for "attempted": its own matrix can span both document-rest and document-mcp at once, so one package's arch leg failing must never block merging the other package's manifest -- imagetools create below fails specifically if either of *this* image's two per-arch tags doesn't actually exist. + if: always() && (needs.release.result == 'success' || needs.collect-backfill-matrix.result == 'success') && (needs.release.outputs.container-image-matrix || needs.collect-backfill-matrix.outputs.container-image-matrix) != '[]' && (needs.build-container-images.result == 'success' || needs.build-container-images.result == 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.release.outputs.container-image-matrix || needs.collect-backfill-matrix.outputs.container-image-matrix) }} + steps: + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Merge the per-architecture images into one multi-arch manifest + run: | + docker buildx imagetools create \ + -t "ghcr.io/exadev/${{ matrix.image }}:${{ matrix.version }}" \ + "ghcr.io/exadev/${{ matrix.image }}:${{ matrix.version }}-amd64" \ + "ghcr.io/exadev/${{ matrix.image }}:${{ matrix.version }}-arm64" + - name: Float latest onto this manifest (never for a backfill run) + # A backfill run sets isLatest "false" specifically because it exists to catch up an old, already-superseded tag -- floating `latest` onto it would regress the tag if newer releases have shipped since. + if: matrix.isLatest == 'true' + run: | + docker buildx imagetools create \ + -t "ghcr.io/exadev/${{ matrix.image }}:latest" \ + "ghcr.io/exadev/${{ matrix.image }}:${{ matrix.version }}" + deploy-site: name: Build and deploy the web UI to Pages # After release, so the deploy is built from the release commit the orchestrator just pushed (it bumps packages/web/package.json and tags it). For a commit that releases nothing, no new tag is created and this builds the current tip of main. test-e2e is in the gate so a main-branch e2e failure blocks deploying the site the suite drives a real browser through, matching the required role e2e already plays on pull requests via the ruleset. diff --git a/README.md b/README.md index ecafb87fb..9b8bdf569 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,9 @@ Release configuration is **only** at the root. The orchestrator sets `tagFormat` **SEA binaries:** a fourth post-release matrix builds a Node [single-executable application](https://nodejs.org/api/single-executable-applications.html) binary for `document-cli`, `document-mcp`, and `document-rest` — the three packages whose own `tsdown.config.ts` produces a fully-bundled, dependency-free entry point for it (see each package's own README for what the resulting binary covers) — for Linux, Windows, and macOS, each on both its standard architecture and its less common one (x64 and arm64 for Linux and Windows; Apple Silicon and Intel for macOS), and attaches each one to that release's GitHub Release assets under a name qualified by both OS and architecture (`document-cli-macos-arm64`, `document-cli-linux-x64`, and so on), since three of the six build legs share an OS with another leg and would otherwise collide on upload. Every non-standard-architecture leg (`macos-26-intel`, `ubuntu-24.04-arm`, `windows-11-arm`) is still a standard GitHub-hosted runner, not a paid "larger runner". Node's own `--build-sea` (Node 25.5.0+) copies whichever `node` binary builds it, so the job installs a second, newer Node (beyond the workspace's own pinned version) immediately before that step, specifically to get the native ESM `mainFormat` support `document-cli`'s Ink-driven bundle needs; `.github/scripts/build-sea-binary.ts` holds the platform-specific ad-hoc `codesign` signing step macOS needs (Linux and Windows ship unsigned). Like the three jobs above, nothing depends on it and a failure there can never block the release. -If a release job is itself cancelled by its own timeout after the per-package publish work has already completed but before it collects the released tags into those four matrices, a plain re-run cannot recover it: the collection step diffs tags against a before/after snapshot taken within that same run, so a later run sees every already-existing tag as pre-existing, not new. `ci.yml`'s `workflow_dispatch` trigger takes an optional `backfill_tags` input (comma- or newline-separated `name@version` tags) for exactly this case: a `collect-backfill-matrix` job builds the identical four matrices directly from the given tags, without re-running the release itself, and the four post-release jobs consume whichever of it or the normal release job produced output. Trigger it with `gh workflow run ci.yml --field backfill_tags="pkg-a@1.2.3,pkg-b@4.5.6"`. +**Container images:** a fifth and sixth post-release job pair build a multi-arch (`linux/amd64` + `linux/arm64`) container image for `document-mcp` and `document-rest` only — the two packages that run as long-lived servers rather than a one-shot CLI/TUI (`document-cli` is deliberately excluded) — and publish it to GitHub Container Registry as `ghcr.io/exadev/document-mcp` and `ghcr.io/exadev/documents.js` respectively (the latter matching the package family's own project name rather than its npm package name `document-rest`). Each image wraps the identical Linux SEA binary the fourth matrix already built and uploaded as a release asset — downloaded via `gh release download`, never rebuilt a second time — on a [distroless](https://github.com/GoogleContainerTools/distroless) `cc-debian12` base, since the SEA binaries are glibc-linked, not musl, and cannot run on an Alpine or `FROM scratch` base; a small multi-stage build step extracts the one library (`libatomic.so.1`) the SEA binary needs that `cc-debian12` doesn't ship on its own. Each architecture builds natively on its own runner (`ubuntu-latest` for amd64, `ubuntu-24.04-arm` for arm64) rather than cross-compiling under QEMU: `build-container-images` builds and pushes each architecture under an arch-suffixed tag (`:1.2.0-amd64`, `:1.2.0-arm64`, left in the registry rather than deleted), and `publish-container-manifests` merges the two into one multi-arch manifest under the real version tag via `docker buildx imagetools create`, plus a floating `latest` tag for a genuine new release (never for a backfill run, so a historical catch-up can never regress `latest` to an older version). A brand-new GHCR package defaults to private visibility on creation regardless of this repository's own visibility — a user with org package admin needs to flip it to public once per image after its first publish (github.com → Organization ExaDev → Packages → `documents.js`/`document-mcp` → Package settings → Change visibility). Like the fourth matrix, nothing depends on this pair and a failure here can never block the release. + +If a release job is itself cancelled by its own timeout after the per-package publish work has already completed but before it collects the released tags into those matrices, a plain re-run cannot recover it: the collection step diffs tags against a before/after snapshot taken within that same run, so a later run sees every already-existing tag as pre-existing, not new. `ci.yml`'s `workflow_dispatch` trigger takes an optional `backfill_tags` input (comma- or newline-separated `name@version` tags) for exactly this case: a `collect-backfill-matrix` job builds the identical matrices directly from the given tags, without re-running the release itself, and the post-release jobs consume whichever of it or the normal release job produced output. Trigger it with `gh workflow run ci.yml --field backfill_tags="pkg-a@1.2.3,pkg-b@4.5.6"`. Every alias name's trusted publisher is registered against this repository and workflow (trusted publishing is registered per package name, so each alias needed its own one-time registration; all eleven landed 2026-09-10 and the backfill run [ExaDev/documents.js#34449796133](https://github.com/ExaDev/documents.js/actions/runs/34449796133) republished every alias at its package's then-current version — registry-confirmed). Until an alias is registered, its publish leg fails with the exact one-time registration instructions in its annotation — a deliberate signal rather than a graceful skip, because a silent skip let every alias stay stranded from the monorepo migration until a manual audit found them; the failure still cannot block the release or the Pages deploy (nothing depends on the post-release jobs), it only makes the outstanding registration visible the moment a release runs. The legacy GitHub Packages mirror keeps its graceful skip (a legacy mirror package's Actions-access list still names only its archived standalone repository — grant this repository Write on the package's settings page once, and the next release mirrors automatically); the registration is: sign in to [npmjs.com](https://www.npmjs.com) as a user with access to the package's settings → **Packages** → the alias (e.g. `document-bytes`) → **Settings** → **Trusted publisher** → select **GitHub Actions**, then enter Organization `ExaDev`, Repository `documents.js`, Workflow filename `ci.yml` (filename only — it must exist in `.github/workflows/`), leave the environment empty, and allow the `npm publish` action ([npm's trusted-publishers documentation](https://docs.npmjs.com/trusted-publishers) has the canonical form). The first release after that publishes under the alias automatically. The formerly stranded aliases' individual threads are [ExaDev/documents.js#727](https://github.com/ExaDev/documents.js/issues/727), [#728](https://github.com/ExaDev/documents.js/issues/728), [#729](https://github.com/ExaDev/documents.js/issues/729), [#730](https://github.com/ExaDev/documents.js/issues/730), [#731](https://github.com/ExaDev/documents.js/issues/731), and [#770](https://github.com/ExaDev/documents.js/issues/770), each carrying its registration evidence. @@ -181,7 +183,7 @@ Every alias name's trusted publisher is registered against this repository and w ## CI -`.github/workflows/ci.yml` holds one job per task — Commitlint, Lint, Typecheck, Test, Test (workerd), Smoke test, E2E test — each running that task once across the workspace through turbo, alongside Knip, Dependency versions, Audit, and `npm registration` (see Releases above for what that last one guards against), followed by Release, its four post-release republish/attestation/SEA-binary matrices (see Releases above), and the web UI's Pages deploy on `main`. Mutation testing runs in its own `.github/workflows/mutation.yml` (sharded, incrementally cached, serialised through one concurrency queue so it can never starve these jobs of runners), reporting one aggregate "Mutation testing result" check the repository ruleset can require once every package's run is green behind its threshold. On a pull request every turbo task runs with `--affected`, restricting work to the packages the branch changed and their dependents; on `main` the full workspace runs, so the caches later runs restore from are complete and the release gate covers everything. Each job restores turbo's cache keyed by task, so an unchanged package costs a cache replay rather than a rebuild. The Typecheck job additionally runs `attw --pack` across every published package after building it, checking that each package's declared types resolve under every module resolution mode — the web UI is excluded, since it publishes nothing and exposes no types. +`.github/workflows/ci.yml` holds one job per task — Commitlint, Lint, Typecheck, Test, Test (workerd), Smoke test, E2E test — each running that task once across the workspace through turbo, alongside Knip, Dependency versions, Audit, and `npm registration` (see Releases above for what that last one guards against), followed by Release, its post-release republish/attestation/SEA-binary/container-image matrices (see Releases above), and the web UI's Pages deploy on `main`. Mutation testing runs in its own `.github/workflows/mutation.yml` (sharded, incrementally cached, serialised through one concurrency queue so it can never starve these jobs of runners), reporting one aggregate "Mutation testing result" check the repository ruleset can require once every package's run is green behind its threshold. On a pull request every turbo task runs with `--affected`, restricting work to the packages the branch changed and their dependents; on `main` the full workspace runs, so the caches later runs restore from are complete and the release gate covers everything. Each job restores turbo's cache keyed by task, so an unchanged package costs a cache replay rather than a rebuild. The Typecheck job additionally runs `attw --pack` across every published package after building it, checking that each package's declared types resolve under every module resolution mode — the web UI is excluded, since it publishes nothing and exposes no types. Dependabot covers the root manifest and every package's, batching minor and patch updates into one pull request and leaving majors individual; `.github/workflows/dependabot-auto-merge.yml` auto-merges the former once CI is green. The cross-repository `sibling-released` dispatch the separate repositories used to propagate version bumps between themselves is gone: the orchestrator does that inside a single run now, in dependency order, without a pull request per bump. diff --git a/packages/document-mcp/Dockerfile b/packages/document-mcp/Dockerfile new file mode 100644 index 000000000..a3e01dc87 --- /dev/null +++ b/packages/document-mcp/Dockerfile @@ -0,0 +1,25 @@ +# Only used to obtain libatomic.so.1: cc-debian12 below doesn't ship it, but the SEA binary's own Node runtime dynamically links against it (confirmed by direct verification against document-rest's identical Dockerfile -- the binary otherwise fails at startup with "libatomic.so.1: cannot open shared object file"). Nothing else from this stage is ever copied forward -- no pnpm, no Node, no apt cache. +FROM debian:bookworm-slim AS libatomic +RUN apt-get update && apt-get install --no-install-recommends -y libatomic1 \ + && rm -rf /var/lib/apt/lists/* \ + && cp -P /usr/lib/*/libatomic.so.1* /usr/lib/ + +# Distroless, not a general-purpose base like debian:bookworm-slim: this image never needs a shell, package manager, or any other OS tooling at runtime -- only the dynamic linker and libraries the SEA binary actually links against. cc-debian12 already carries glibc, libgcc, and libstdc++ (Node's own official distribution ships a separate -musl build alongside the default linux-x64/linux-arm64 tarball, confirming the default -- and therefore this binary -- is glibc-linked, and V8 is C++, requiring libstdc++); libatomic above is the one runtime dependency it's missing. The :nonroot tag runs as a built-in non-root user (UID 65532) with no groupadd/useradd step needed. +FROM gcr.io/distroless/cc-debian12:nonroot + +COPY --from=libatomic /usr/lib/libatomic.so.1* /usr/lib/ +# cc-debian12's own glibc searches its own arch-specific multiarch directory by default, not bare /usr/lib -- rather than hardcode that directory's name (aarch64-linux-gnu vs x86_64-linux-gnu, yet another arch-naming split like Docker's amd64/arm64 vs Node's x64/arm64 elsewhere in this project's own CI), LD_LIBRARY_PATH makes the linker check /usr/lib too, which every architecture's build of this image copies libatomic.so.1 into identically. +ENV LD_LIBRARY_PATH=/usr/lib + +# The Linux SEA binary this exact CI leg already downloaded from the release (see build-container-images in .github/workflows/ci.yml), not built here -- this image runs no pnpm install/build of its own. Already executable: distroless has no shell to chmod it with, so the CI job sets the execute bit on the runner's own filesystem before `docker build` runs, and COPY preserves that permission bit. +COPY dist-sea/document-mcp /usr/local/bin/document-mcp + +EXPOSE 3000 + +LABEL org.opencontainers.image.source="https://github.com/ExaDev/documents.js" +LABEL org.opencontainers.image.description="document-mcp: an MCP server over documents.js's document-conversion, .odb, metadata, and font tooling (--transport http only in this image)" +LABEL org.opencontainers.image.licenses="MIT" + +# --transport http is fixed: stdio mode is exec'd directly by an MCP client as a subprocess, which a container has no role in. --host 0.0.0.0 for the identical reason as document-rest's own Dockerfile -- the binary defaults to loopback-only, unreachable from outside a container regardless of `-p`. --port stays a CMD default so `docker run -p 3000:3000 ... --port 3000` still works, and any other --port a caller passes as trailing docker run args overrides it. +ENTRYPOINT ["/usr/local/bin/document-mcp", "--transport", "http", "--host", "0.0.0.0"] +CMD ["--port", "3000"] diff --git a/packages/document-mcp/README.md b/packages/document-mcp/README.md index f51783c67..48f16b9a2 100644 --- a/packages/document-mcp/README.md +++ b/packages/document-mcp/README.md @@ -191,6 +191,16 @@ Then add the server URL (e.g., `https://your-host:3000/mcp`) as a connector in C Every release also attaches a Node [single-executable application](https://nodejs.org/api/single-executable-applications.html) build for Linux (x64 and arm64), Windows (x64 and arm64), and macOS (Apple Silicon and Intel) to that release's own GitHub Release assets — the entire server and its dependencies embedded in one file, needing no Node.js install or `npx` at all. It supports both `stdio` and `--transport http` exactly as above; point an MCP client's `command` at the downloaded binary directly instead of `npx`/`node`. Download the asset matching your platform from the package's tag on the [Releases page](https://github.com/ExaDev/documents.js/releases) and run it directly (`chmod +x` on Linux/macOS first). +### Container image + +Every release also publishes a multi-arch (`linux/amd64` + `linux/arm64`) container image to GitHub Container Registry, wrapping the identical standalone `--transport http` binary above on a minimal [distroless](https://github.com/GoogleContainerTools/distroless) base rather than a Node install: + +```sh +docker run -p 3000:3000 ghcr.io/exadev/document-mcp:VERSION --port 3000 +``` + +The image always runs in `--transport http` mode — `stdio` mode is exec'd directly by an MCP client as a subprocess, which a container has no role in — and binds to `0.0.0.0` inside the container regardless of `--port`, so `-p :` reaches `/mcp` directly. Replace `VERSION` with the package's own exact release version; `latest` also tracks the newest release, matching `document-rest`'s identical image. See [Remote transport](#remote-transport-http) above for this listener's own lack of authentication. + ### Development Requires Node.js `>=20` and pnpm `11.6.0` (pinned via `packageManager` in `package.json`). diff --git a/packages/document-mcp/src/cli.test.ts b/packages/document-mcp/src/cli.test.ts index 441105f2d..9efe723dd 100644 --- a/packages/document-mcp/src/cli.test.ts +++ b/packages/document-mcp/src/cli.test.ts @@ -81,4 +81,30 @@ describe("main", () => { process.argv = ["node", "bin.js", "--transport"]; await expect(main()).rejects.toThrow(/--transport requires a value/); }); + + it("binds the given --host instead of the loopback default", async () => { + process.argv = [ + "node", + "bin.js", + "--transport", + "http", + "--host", + "0.0.0.0", + "--port", + "0", + ]; + server = await main(); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("document-mcp listening on http://0.0.0.0:"), + ); + const address = server?.address(); + if ( + address === null || + address === undefined || + typeof address === "string" + ) { + throw new Error("expected a TCP address"); + } + expect(address.address).toBe("0.0.0.0"); + }); }); diff --git a/packages/document-mcp/src/cli.ts b/packages/document-mcp/src/cli.ts index 7e2dd7032..842fb2c11 100644 --- a/packages/document-mcp/src/cli.ts +++ b/packages/document-mcp/src/cli.ts @@ -56,7 +56,8 @@ export async function main(): Promise { if (transport === "http") { const portArg = readFlag(args, "port"); const port = portArg === undefined ? DEFAULT_HTTP_PORT : parsePort(portArg); - const httpServer = await serveHttp(port); + const host = readFlag(args, "host") ?? "127.0.0.1"; + const httpServer = await serveHttp(port, host); const address = httpServer.address(); if (address === null || typeof address === "string") { throw new Error( @@ -64,7 +65,7 @@ export async function main(): Promise { ); } console.error( - `document-mcp listening on http://127.0.0.1:${String(address.port)}${MCP_HTTP_PATH}`, + `document-mcp listening on http://${host}:${String(address.port)}${MCP_HTTP_PATH}`, ); return httpServer; } diff --git a/packages/document-mcp/src/serve-http.ts b/packages/document-mcp/src/serve-http.ts index 29dae6f94..9a8f40066 100644 --- a/packages/document-mcp/src/serve-http.ts +++ b/packages/document-mcp/src/serve-http.ts @@ -9,8 +9,8 @@ import { createServer } from "./server"; // The path an HTTP/SSE-only client (Claude Web, Claude Mobile, ChatGPT) is told to add as a connector -- see the README's remote transport section. A GET/DELETE (or POST) against this exact path falls through to createMcpHandler's own routing (legacy session operations, 405s, and so on); every other path on this listener 404s before nodeHandler ever sees it. export const MCP_HTTP_PATH = "/mcp"; -// Binds a plain node:http listener over the same server factory src/bin.ts's stdio path uses (see src/server.ts), so both transports register the identical tool set from one place. Built on the SDK's own createMcpHandler + toNodeHandler composition rather than a hand-rolled `NodeStreamableHTTPServerTransport` per request: createMcpHandler already serves both the current protocol era and the older HTTP+SSE era's stateless fallback from one factory, which a hand-wired transport would otherwise have to reimplement to stay spec-compliant. createMcpHandler performs no Host/Origin validation of its own by design (see its own doc comment) -- appropriate here because this listener's whole purpose, per the README, is remote access through an operator-supplied tunnel or reverse proxy presenting its own public hostname, which a localhost-only allowlist would reject outright. Binding to the loopback interface is the actual network boundary: only a same-machine tunnel process (or a reverse proxy explicitly configured to forward here) can ever reach the socket. -export function serveHttp(port: number): Promise { +// Binds a plain node:http listener over the same server factory src/bin.ts's stdio path uses (see src/server.ts), so both transports register the identical tool set from one place. Built on the SDK's own createMcpHandler + toNodeHandler composition rather than a hand-rolled `NodeStreamableHTTPServerTransport` per request: createMcpHandler already serves both the current protocol era and the older HTTP+SSE era's stateless fallback from one factory, which a hand-wired transport would otherwise have to reimplement to stay spec-compliant. createMcpHandler performs no Host/Origin validation of its own by design (see its own doc comment) -- appropriate here because this listener's whole purpose, per the README, is remote access through an operator-supplied tunnel or reverse proxy presenting its own public hostname, which a localhost-only allowlist would reject outright. `host` defaults to the loopback interface as the actual network boundary in every ordinary case (only a same-machine tunnel process, or a reverse proxy explicitly configured to forward here, can ever reach the socket), but a container's own ENTRYPOINT passes 0.0.0.0 instead: a loopback bind is unreachable from outside a container's network namespace no matter what port a `docker run -p` maps. +export function serveHttp(port: number, host: string): Promise { const handler = createMcpHandler(createServer); const nodeHandler = toNodeHandler(handler); @@ -28,7 +28,7 @@ export function serveHttp(port: number): Promise { }); return new Promise((resolve) => { - httpServer.listen(port, "127.0.0.1", () => { + httpServer.listen(port, host, () => { resolve(httpServer); }); }); diff --git a/packages/document-rest/Dockerfile b/packages/document-rest/Dockerfile new file mode 100644 index 000000000..48aa53ee5 --- /dev/null +++ b/packages/document-rest/Dockerfile @@ -0,0 +1,25 @@ +# Only used to obtain libatomic.so.1: cc-debian12 below doesn't ship it, but the SEA binary's own Node runtime dynamically links against it (confirmed by direct verification -- the binary otherwise fails at startup with "libatomic.so.1: cannot open shared object file"). Nothing else from this stage is ever copied forward -- no pnpm, no Node, no apt cache. +FROM debian:bookworm-slim AS libatomic +RUN apt-get update && apt-get install --no-install-recommends -y libatomic1 \ + && rm -rf /var/lib/apt/lists/* \ + && cp -P /usr/lib/*/libatomic.so.1* /usr/lib/ + +# Distroless, not a general-purpose base like debian:bookworm-slim: this image never needs a shell, package manager, or any other OS tooling at runtime -- only the dynamic linker and libraries the SEA binary actually links against. cc-debian12 already carries glibc, libgcc, and libstdc++ (Node's own official distribution ships a separate -musl build alongside the default linux-x64/linux-arm64 tarball, confirming the default -- and therefore this binary -- is glibc-linked, and V8 is C++, requiring libstdc++); libatomic above is the one runtime dependency it's missing. The :nonroot tag runs as a built-in non-root user (UID 65532) with no groupadd/useradd step needed. +FROM gcr.io/distroless/cc-debian12:nonroot + +COPY --from=libatomic /usr/lib/libatomic.so.1* /usr/lib/ +# cc-debian12's own glibc searches its own arch-specific multiarch directory by default, not bare /usr/lib -- rather than hardcode that directory's name (aarch64-linux-gnu vs x86_64-linux-gnu, yet another arch-naming split like Docker's amd64/arm64 vs Node's x64/arm64 elsewhere in this project's own CI), LD_LIBRARY_PATH makes the linker check /usr/lib too, which every architecture's build of this image copies libatomic.so.1 into identically. +ENV LD_LIBRARY_PATH=/usr/lib + +# The Linux SEA binary this exact CI leg already downloaded from the release (see build-container-images in .github/workflows/ci.yml), not built here -- this image runs no pnpm install/build of its own. Already executable: distroless has no shell to chmod it with, so the CI job sets the execute bit on the runner's own filesystem before `docker build` runs, and COPY preserves that permission bit. +COPY dist-sea/document-rest /usr/local/bin/document-rest + +EXPOSE 3100 + +LABEL org.opencontainers.image.source="https://github.com/ExaDev/documents.js" +LABEL org.opencontainers.image.description="document-rest: a plain REST API server over documents.js's document-conversion, editing, inspection, and .odb operations" +LABEL org.opencontainers.image.licenses="MIT" + +# --host 0.0.0.0 is fixed, not overridable via `docker run` args: the binary defaults to loopback-only for the ordinary npx/standalone-binary case, which is unreachable from outside a container regardless of `-p`. --port stays a CMD default so `docker run -p 3100:3100 ... --port 3100` still works, and any other --port a caller passes as trailing docker run args overrides it (Docker concatenates ENTRYPOINT with CMD-or-the-caller's-override, never replacing ENTRYPOINT itself). +ENTRYPOINT ["/usr/local/bin/document-rest", "--host", "0.0.0.0"] +CMD ["--port", "3100"] diff --git a/packages/document-rest/README.md b/packages/document-rest/README.md index 3989c8b98..ad7a54121 100644 --- a/packages/document-rest/README.md +++ b/packages/document-rest/README.md @@ -22,6 +22,16 @@ This binds a plain `node:http` listener to `127.0.0.1` (loopback only) on the gi Every release also attaches a Node [single-executable application](https://nodejs.org/api/single-executable-applications.html) build for Linux (x64 and arm64), Windows (x64 and arm64), and macOS (Apple Silicon and Intel) to that release's own GitHub Release assets — the entire server and its dependencies embedded in one file, needing no Node.js install or `npx` at all. For the "a caller with no Node runtime of its own" case this package exists for in the first place, this removes the last Node dependency too: download the asset matching your platform from the package's tag on the [Releases page](https://github.com/ExaDev/documents.js/releases), run it directly (`chmod +x` on Linux/macOS first), and it takes the identical `--port` flag. +### Container image + +Every release also publishes a multi-arch (`linux/amd64` + `linux/arm64`) container image to GitHub Container Registry, wrapping the identical standalone binary above on a minimal [distroless](https://github.com/GoogleContainerTools/distroless) base rather than a Node install: + +```sh +docker run -p 3100:3100 ghcr.io/exadev/documents.js:VERSION --port 3100 +``` + +Replace `VERSION` with the package's own exact release version (e.g. `1.2.0`) — matching the version-pinned convention every other artifact in this repository uses, this image is never published under a loose major/minor tag such as `1` or `1.2`, though `latest` does track the newest release. The image binds to `0.0.0.0` inside the container regardless of `--port`, so `-p :` is all that's needed to reach it; see the security note above about this listener having no authentication of its own — publishing the container's port makes it reachable by anything that can reach the host, so put a reverse proxy or firewall in front of it before exposing it beyond your own machine. + ## API **`GET /`** lists every available operation: diff --git a/packages/document-rest/src/cli.test.ts b/packages/document-rest/src/cli.test.ts index fd2e5cdde..e476fc01a 100644 --- a/packages/document-rest/src/cli.test.ts +++ b/packages/document-rest/src/cli.test.ts @@ -59,4 +59,17 @@ describe("main", () => { process.argv = ["node", "bin.js", "--port"]; await expect(main()).rejects.toThrow(/--port requires a value/); }); + + it("binds the given --host instead of the loopback default", async () => { + process.argv = ["node", "bin.js", "--host", "0.0.0.0", "--port", "0"]; + server = await main(); + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("document-rest listening on http://0.0.0.0:"), + ); + const address = server.address(); + if (address === null || typeof address === "string") { + throw new Error("expected a TCP address"); + } + expect(address.address).toBe("0.0.0.0"); + }); }); diff --git a/packages/document-rest/src/cli.ts b/packages/document-rest/src/cli.ts index e4a8225df..b8f631470 100644 --- a/packages/document-rest/src/cli.ts +++ b/packages/document-rest/src/cli.ts @@ -45,11 +45,12 @@ export async function main(): Promise { const args = process.argv.slice(2); const portArg = readFlag(args, "port"); const port = portArg === undefined ? DEFAULT_PORT : parsePort(portArg); + // Defaults to loopback-only, matching document-mcp's own --transport http listener: this process has no authentication or Host/Origin allowlisting of its own, so binding to 127.0.0.1 is the actual network boundary -- whatever fronts it for remote access (a tunnel, a reverse proxy) is responsible for authenticating callers before traffic ever reaches this process. --host exists so a container's own ENTRYPOINT can bind 0.0.0.0 instead: a loopback bind is unreachable from outside a container's network namespace no matter what port a `docker run -p` maps, since Docker's port mapping reaches the container's external interface, not its loopback. + const host = readFlag(args, "host") ?? "127.0.0.1"; const server = createRestServer(); await new Promise((resolve) => { - // Loopback-only, matching document-mcp's own --transport http listener: this process has no authentication or Host/Origin allowlisting of its own, so binding to 127.0.0.1 is the actual network boundary -- whatever fronts it for remote access (a tunnel, a reverse proxy) is responsible for authenticating callers before traffic ever reaches this process. - server.listen(port, "127.0.0.1", resolve); + server.listen(port, host, resolve); }); const address = server.address(); if (address === null || typeof address === "string") { @@ -58,7 +59,7 @@ export async function main(): Promise { ); } console.error( - `document-rest listening on http://127.0.0.1:${String(address.port)}`, + `document-rest listening on http://${host}:${String(address.port)}`, ); return server; }