diff --git a/.github/workflows/preview-release.yml b/.github/workflows/preview-release.yml new file mode 100644 index 0000000..1313adf --- /dev/null +++ b/.github/workflows/preview-release.yml @@ -0,0 +1,276 @@ +name: preview release + +# Publishes a preview as GitHub Release assets, one zip per platform, because `org.chdb` is +# not on Maven Central yet and a native package is too large to commit. +# +# Same four runners, same build-native.sh and same integration tests as release.yml; only the +# last step differs. The version lives in git here too: no `versions:set` in CI, so the POMs +# at the tagged commit already carry the tag's version. The first attempt at v1.0.0-preview.1 +# broke that rule and shipped a side branch 67 commits behind main, which is what every check +# in `preflight` is for. It was withdrawn, so the tag name is in use again here. +on: + push: + tags: ["v*-preview.*"] + workflow_dispatch: + inputs: + tag: + description: An existing preview tag to build and publish + required: true + type: string + +concurrency: + group: preview-release-${{ inputs.tag || github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + actions: read + +env: + MAVEN_ARGS: "--batch-mode --no-transfer-progress" + +jobs: + preflight: + name: preflight + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.resolve.outputs.tag }} + version: ${{ steps.resolve.outputs.version }} + sha: ${{ steps.resolve.outputs.sha }} + steps: + - uses: actions/checkout@v4 + with: + # On a dispatch `github.sha` is the branch, not what gets built. + ref: ${{ inputs.tag || github.ref }} + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + cache: maven + + - name: Resolve the tag and check it against the POMs + id: resolve + env: + TAG: ${{ inputs.tag || github.ref_name }} + run: | + set -euo pipefail + + case "$TAG" in + v*-preview.*) ;; + *) + echo "::error::expected a tag like v1.0.0-preview.1, got $TAG" + exit 1 + ;; + esac + TAG_VERSION="${TAG#v}" + + # From Maven, so an inherited or property-substituted version reads correctly. + VERSION=$(mvn $MAVEN_ARGS -q -DforceStdout help:evaluate -Dexpression=project.version) + if [ "$TAG_VERSION" != "$VERSION" ]; then + echo "::error::tag $TAG implies version $TAG_VERSION but the POMs say $VERSION. Commit the preview version, then tag that commit." + exit 1 + fi + case "$VERSION" in + *-SNAPSHOT) + echo "::error::the POMs are at $VERSION. A published preview needs a non-SNAPSHOT version committed and tagged." + exit 1 + ;; + esac + + SHA=$(git rev-parse HEAD) + echo "tag $TAG, version $VERSION, commit $SHA" + { + echo "tag=$TAG" + echo "version=$VERSION" + echo "sha=$SHA" + } >> "$GITHUB_OUTPUT" + + - name: The tag must point at the commit being built + # release.yml's script, for the same guarantee: what is published is what a commit says. + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + scripts/check-release-tag.sh \ + "${{ github.repository }}" \ + "${{ steps.resolve.outputs.version }}" \ + "${{ steps.resolve.outputs.sha }}" \ + | tee -a "$GITHUB_STEP_SUMMARY" + + - name: The tagged commit must be on main + # Green on a branch says the tree works, not that it is the tree main has. + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + if ! gh api "repos/${{ github.repository }}/compare/main...${{ steps.resolve.outputs.sha }}" \ + --jq '.status' | grep -qx 'identical\|behind'; then + echo "::error::${{ steps.resolve.outputs.sha }} is not an ancestor of main. Merge the release commit to main, then tag it there." + exit 1 + fi + + - name: The `build` workflow must have passed for this commit + # `build` filters on branches, so a tag push does not run it. + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + CONCLUSION=$(gh api \ + "repos/${{ github.repository }}/actions/runs?head_sha=${{ steps.resolve.outputs.sha }}&per_page=100" \ + --jq '[.workflow_runs[] | select(.name == "build")] | first | .conclusion // "none"') + echo "build workflow for ${{ steps.resolve.outputs.sha }}: $CONCLUSION" + if [ "$CONCLUSION" != "success" ]; then + echo "::error::the build workflow for this commit concluded '$CONCLUSION'. Let the matrix go green on main, then tag that commit." + exit 1 + fi + + - name: The engine version this preview carries + run: | + set -euo pipefail + ENGINE=$(sed -n 's/^engine.version=//p' scripts/engine.properties | head -1) + echo "engine $ENGINE, pinned by SHA-256 in scripts/engine.properties" >> "$GITHUB_STEP_SUMMARY" + + # One job per platform: build-native.sh refuses to cross-build. + bundle: + name: bundle ${{ matrix.platform }} + needs: preflight + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x86_64-gnu + runner: ubuntu-22.04 + - platform: linux-aarch64-gnu + runner: ubuntu-22.04-arm + - platform: macos-aarch64 + runner: macos-15 + # macos-15-intel, not macos-15: the plain label is Apple Silicon. + - platform: macos-x86_64 + runner: macos-15-intel + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.preflight.outputs.sha }} + + - uses: actions/setup-java@v4 + with: + # The floor, deliberately: it is what makes maven.compiler.release=11 a fact. + distribution: temurin + java-version: "11" + cache: maven + + - name: Cache the pinned engine download + uses: actions/cache@v4 + with: + path: target/engine/download + key: chdb-engine-${{ matrix.platform }}-${{ hashFiles('scripts/engine.properties') }} + + - name: Compile the Java side and run its unit tests + run: mvn $MAVEN_ARGS -pl chdb-jdbc -am test + + - name: Fetch the engine, build the shim, stage the platform package + run: | + case "${{ matrix.platform }}" in + linux-*) scripts/build-native-in-container.sh ${{ matrix.platform }} ;; + *) scripts/build-native.sh ${{ matrix.platform }} ;; + esac + + - name: Integration tests against the staged package + # The bytes about to be zipped are new bytes, whatever `build` concluded for the commit. + run: | + set -eu + # stdin closed: chDB reads a non-TTY stdin with bytes on it as external data for an + # INSERT. See the same step in build.yml. + exec > "$GITHUB_STEP_SUMMARY" + + - name: Publish the release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.preflight.outputs.tag }} + EXPECTED_SHA: ${{ needs.preflight.outputs.sha }} + run: | + set -euo pipefail + + # Re-resolved here, not just in preflight: staging takes tens of minutes and a tag + # can be moved or deleted inside that window, which would upload these bundles under + # a tag naming a different commit. --verify-tag only checks that the tag exists. + REMOTE_SHA=$(gh api "repos/${{ github.repository }}/commits/$TAG" --jq '.sha') + if [ "$REMOTE_SHA" != "$EXPECTED_SHA" ]; then + echo "::error::$TAG now points at $REMOTE_SHA, not the $EXPECTED_SHA these bundles were built from" + exit 1 + fi + + # --prerelease so a preview never becomes "Latest release"; --draft=false because an + # existing draft would otherwise take the uploads and stay invisible. + if gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1; then + gh release edit "$TAG" --repo "${{ github.repository }}" --prerelease --draft=false + else + gh release create "$TAG" --repo "${{ github.repository }}" \ + --title "chdb-java $TAG" --prerelease --verify-tag --generate-notes + fi + gh release upload "$TAG" --repo "${{ github.repository }}" \ + release-assets/*.zip release-assets/SHA256SUMS --clobber diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 96cc6b6..19eaddb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,12 @@ on: # A release is a tag on a commit whose POMs already carry the release version. `build` does # not run on tags (it filters on branches), which is why preflight checks that `build` # succeeded for this exact commit rather than assuming a tag implies a tested tree. - tags: ["v*"] + tags: + - "v*" + # Never a preview: `v1.0.0-preview.1` matches `v*` and preflight would accept it, so + # the Central path would stage, sign and offer a preview in the portal, where a release + # cannot be unpublished. preview-release.yml publishes those. + - "!v*-preview.*" workflow_dispatch: inputs: channel: diff --git a/CHDB_JAVA_V1_WORK_PLAN.md b/CHDB_JAVA_V1_WORK_PLAN.md index 0fed8dc..d0aaede 100644 --- a/CHDB_JAVA_V1_WORK_PLAN.md +++ b/CHDB_JAVA_V1_WORK_PLAN.md @@ -201,36 +201,34 @@ META-INF/sbom/ ### 4.3 Versioning rules -The scheme is "full engine version plus binding revision": - -```text -. -``` - -- `engine-version` keeps the chDB Core release version verbatim, `rc` qualifier included. -- `binding-revision` is the trailing positive integer covering Java, JNI, loader and platform packaging revisions. It restarts at `1` for every new engine version. -- This is an engine-aligned scheme. Do not read it as Java SemVer. - -Examples: - -| Case | Maven version | Meaning | -|---|---|---| -| First binding against stable engine 26.7.0 | `26.7.0.1` | engine=`26.7.0`, binding revision=`1` | -| Java/JNI/loader fix only, same stable engine | `26.7.0.2` | engine unchanged, binding revision incremented | -| First binding against engine 26.7.2-rc.2 | `26.7.2-rc.2.1` | engine=`26.7.2-rc.2`, binding revision=`1` | -| Re-release on the same rc.2 after an addon change | `26.7.2-rc.2.2` | engine unchanged, binding revision incremented | -| Engine moves to rc.3 | `26.7.2-rc.3.1` | new engine, binding revision back to `1` | -| Engine reaches stable 26.7.2 | `26.7.2.1` | first binding release on that stable engine | +The binding is versioned on its own, in SemVer — `MAJOR.MINOR.PATCH`, first release `1.0.0`. +The number describes the Java API, which is the question a consumer asks it. + +The engine version is not in it. It is recorded in each native package's +`manifest.properties`, pinned with its SHA-256 in `scripts/engine.properties`, and named in +the release notes; the ABI check refuses any other engine build, so the pairing is enforced +rather than spelled. + +This replaces `.`, which read the wrong way round in both +directions: `26.7.2-rc.2.1` → `26.7.3.1` looked major and was not the binding's doing, while +a break in the Java API could ship as a trailing `.2`. + +| Case | Version | +|---|---| +| First release | `1.0.0` | +| Fix in the driver, loader or JNI shim | `1.0.1` | +| New engine baseline, no Java API change | `1.1.0` | +| Breaking change to the Java API | `2.0.0` | +| Preview of `1.0.0` | `1.0.0-preview.1` | Release rules: -- A release artifact in a Maven repository is immutable. Never overwrite `26.7.2-rc.2.1`. Any addon, JNI, Java, POM, loader, checksum or single-platform fix ships as `.2`. -- Within one binding release, `chdb-jdbc`, the four platform packages and `chdb-bom` carry exactly the same version. They ship together even when some platform content did not change, so the BOM and the platform packages never end up on mixed versions. -- Moving the engine from one RC to another, or from an RC to a stable release, counts as a new engine version, and the binding revision restarts at `1`. -- A Java artifact built on an engine RC is itself a preview and cannot be the engine dependency of V1 GA. V1 GA has to bind a stable chDB Core release. -- Development builds may use `26.7.2-rc.2.2-SNAPSHOT`, but `SNAPSHOT` never enters a Maven Central release. -- If the Java binding on a stable engine needs its own release candidates, use `26.7.0.1-rc.1`, `26.7.0.1-rc.2`, with `26.7.0.1` as the final GA. A candidate and the GA never reuse the same immutable artifact. -- Any engine change produces a new Maven version and a full platform test run. +- A published artifact is immutable. Never overwrite `1.0.0`; any fix ships as `1.0.1`. +- `chdb-jdbc`, the four platform packages and `chdb-bom` always carry the same version. +- An engine change is a `MINOR` bump when it changes what the driver can do and a `PATCH` when it does not. It is never invisible: the manifest and the release notes name the engine. +- A binding built on an engine RC is a preview and cannot be V1 GA. +- `-SNAPSHOT` is for development and never reaches Maven Central. +- **`-preview.` sorts *above* the release it previews.** Maven's `ComparableVersion` orders unknown qualifiers after the final release, so `1.0.0-preview.1` compares newer than `1.0.0` — measured, not assumed. It costs nothing here because previews are installed by hand into a local repository and never published beside a GA, and consumers name exact versions. If a preview ever has to live in a shared repository, use `-rc.`, which Maven does order below the release. To remove string-parsing ambiguity, every artifact manifest records these separately: @@ -524,7 +522,7 @@ Exit condition: a Java user who knows nothing about the implementation can insta - [ ] Freeze the public Java API and the JNI ABI. - [ ] Write the release notes and the known limitations. - [ ] Publish an RC and hold a soak and external validation window of at least one week. -- [ ] Once every V1 release gate passes, publish the first release aligned to a stable engine, for example `26.7.0.1`. +- [ ] Once every V1 release gate passes, publish the first release, `1.0.0`, built on a stable engine. ## 6. Milestones diff --git a/README.md b/README.md index ee093ac..6903a21 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,10 @@ ClickHouse. It runs the engine in your JVM's process — no server, no network streaming, forward-only result sets over ClickHouse SQL. > **Status: pre-release.** V1 is under construction against the work plan in -> [`CHDB_JAVA_V1_WORK_PLAN.md`](CHDB_JAVA_V1_WORK_PLAN.md). Nothing is published to Maven -> Central yet, and the public API is not frozen. See [What works today](#what-works-today). +> [`CHDB_JAVA_V1_WORK_PLAN.md`](CHDB_JAVA_V1_WORK_PLAN.md). Nothing is on Maven Central yet +> and the public API is not frozen; releases ship as +> [preview bundles](#a-preview-from-a-github-release). See +> [What works today](#what-works-today). ```java try (Connection connection = DriverManager.getConnection("jdbc:chdb::memory:"); @@ -58,10 +60,34 @@ on. The native package pulls in the driver, so declaring it alone is enough. org.chdb chdb-native-linux-x86_64-gnu - 26.7.3.1 + 1.0.0 ``` +**These coordinates do not resolve yet, and `org.chdb` is provisional** — nothing is on Maven +Central, and the published group id may end up being `com.clickhouse`. See +[docs/release-readiness.md](docs/release-readiness.md). Until then, install a preview. + +### A preview from a GitHub Release + +Previews ship as one zip per platform, holding the same artifacts a Central release would: + +```bash +curl -fsSL -o /tmp/install-chdb-java-preview.sh \ + https://raw.githubusercontent.com/chdb-io/chdb-java/main/scripts/install-preview.sh +chmod +x /tmp/install-chdb-java-preview.sh +/tmp/install-chdb-java-preview.sh v1.0.0-preview.1 +``` + +It downloads the bundle for this platform, verifies it against the release's `SHA256SUMS`, +installs it with `mvn install-file`, and prints the coordinates. Use `--maven-repo PATH` for a +repository other than `~/.m2/repository`. Then declare the dependency as above with +`1.0.0-preview.1`; because the bundle carries its own POMs, a later change of group id does +not strand an installed preview. + +Each preview tag is a commit on `main`, built and tested on all four platforms by +[`preview-release.yml`](.github/workflows/preview-release.yml). + Building for several platforms — a CI matrix, or a distribution your users install on either architecture — declare the driver plus each native package you need: @@ -71,7 +97,7 @@ architecture — declare the driver plus each native package you need: org.chdb chdb-bom - 26.7.3.1 + 1.0.0 pom import @@ -102,12 +128,15 @@ unpacked. There is no all-platforms package, on purpose: it would be the sum of ### Versioning -`.`, so `26.7.3.1` is the first Java release built -against engine 26.7.3 — the engine version is carried through verbatim, `rc` qualifier -included when there is one. A new engine always means a new version, and moving from one RC to -another, or from an RC to a stable release, counts as a new engine and resets the binding -revision to `1`: that is why the move off `26.7.2-rc.2.1` is `26.7.3.1` and not `26.7.2-rc.2.2`. -This is not SemVer; see [work plan §4.3](CHDB_JAVA_V1_WORK_PLAN.md). +SemVer, on the binding alone: `1.0.0` is the first release and a major bump means a breaking +change to the Java API. The engine version is not part of it — it is in each package's +`manifest.properties`, pinned in [`scripts/engine.properties`](scripts/engine.properties) and +named in the release notes (`26.7.3` today), and the driver refuses to load any other build. +See [work plan §4.3](CHDB_JAVA_V1_WORK_PLAN.md). + +A preview is that version with a `-preview.` qualifier. Maven orders an unknown qualifier +*after* the release, so `1.0.0-preview.1` compares newer than `1.0.0`: name the version you +want rather than a range, and note that previews are never published beside a GA. ## Connecting diff --git a/chdb-bom/pom.xml b/chdb-bom/pom.xml index 033a9b4..d27ae3c 100644 --- a/chdb-bom/pom.xml +++ b/chdb-bom/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-bom diff --git a/chdb-examples/pom.xml b/chdb-examples/pom.xml index 284c0a2..9135439 100644 --- a/chdb-examples/pom.xml +++ b/chdb-examples/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-examples diff --git a/chdb-integration-tests/pom.xml b/chdb-integration-tests/pom.xml index 4a774bd..b5cdb5a 100644 --- a/chdb-integration-tests/pom.xml +++ b/chdb-integration-tests/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-integration-tests diff --git a/chdb-jdbc/pom.xml b/chdb-jdbc/pom.xml index 36acf8c..02a86c5 100644 --- a/chdb-jdbc/pom.xml +++ b/chdb-jdbc/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-jdbc diff --git a/chdb-native-linux-aarch64-gnu/pom.xml b/chdb-native-linux-aarch64-gnu/pom.xml index f2904da..989551c 100644 --- a/chdb-native-linux-aarch64-gnu/pom.xml +++ b/chdb-native-linux-aarch64-gnu/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-native-linux-aarch64-gnu diff --git a/chdb-native-linux-x86_64-gnu/pom.xml b/chdb-native-linux-x86_64-gnu/pom.xml index 135afaa..408fe78 100644 --- a/chdb-native-linux-x86_64-gnu/pom.xml +++ b/chdb-native-linux-x86_64-gnu/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-native-linux-x86_64-gnu diff --git a/chdb-native-macos-aarch64/pom.xml b/chdb-native-macos-aarch64/pom.xml index 20e3568..6dfd182 100644 --- a/chdb-native-macos-aarch64/pom.xml +++ b/chdb-native-macos-aarch64/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-native-macos-aarch64 diff --git a/chdb-native-macos-x86_64/pom.xml b/chdb-native-macos-x86_64/pom.xml index 71be5b5..4d600dc 100644 --- a/chdb-native-macos-x86_64/pom.xml +++ b/chdb-native-macos-x86_64/pom.xml @@ -7,7 +7,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 chdb-native-macos-x86_64 diff --git a/docs/publishing.md b/docs/publishing.md index a5fe316..03cb52e 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -4,6 +4,9 @@ Everything needed to put `org.chdb:chdb-jdbc` on Maven Central, in the order it marked by who can do it. The short version: the mechanics are done and tested, the namespace needs one DNS record, and one licence question needs somebody with authority to answer it. +Until those land there is nothing for `mvn` to resolve, so releases go out as preview bundles +on GitHub — section 8. + --- ## Status @@ -17,6 +20,7 @@ needs one DNS record, and one licence question needs somebody with authority to | Third-party licence inventory | **done** — generated from the engine, shipped in the package, drift-tested | | `org.chdb` namespace | **not started** — needs a DNS TXT record, and there is no fallback | | A way to stage all four platforms for one release | **done** — `.github/workflows/release.yml`, issue #15 | +| A way to ship before the namespace exists | **done** — preview bundles, section 8 | | Proof that the artifacts work when consumed from a repository | **done, locally** — `scripts/verify-consumer.sh`, issue #10 | | GPG key for the project | **not started** — needs a decision about whose key | | Position on the LGPL components | **not started** — needs chdb-io | @@ -39,7 +43,7 @@ scripts/build-native.sh macos-x86_64 # on an Intel Mac JAVA_HOME=/path/to/linux-jdk scripts/build-native-in-container.sh linux-x86_64-gnu JAVA_HOME=/path/to/linux-jdk scripts/build-native-in-container.sh linux-aarch64-gnu -mvn versions:set -DnewVersion=26.7.0.1 # a release, not the -SNAPSHOT in the POM +mvn versions:set -DnewVersion=1.0.0 # a release, not the -SNAPSHOT in the POM mvn -Prelease deploy ``` @@ -103,7 +107,7 @@ workflow's own header and summarised here: What the workflow does instead is record what it shipped: every staged library's SHA-256 goes into the job summary and into `manifest.properties` inside the JAR. -The version matters too: `deploy` on the `26.7.0.1-SNAPSHOT` currently in the POM publishes a +The version matters too: `deploy` on the `-SNAPSHOT` currently in the POM publishes a snapshot, which goes to a different place and is never validated or promoted. A Central release bundle needs a non-`SNAPSHOT` version. @@ -384,3 +388,35 @@ Each step can invalidate the next, so: | A publishing-limit exception | Sonatype support | yes, if the numbers need it | | **Position on the twelve copyleft components** | **chdb-io / ClickHouse legal** | **yes — the only real gate** | | Whether chDB is "commercial" for Central | whoever owns the Sonatype relationship | yes | + +--- + +## 8. Previews, until Central exists + +A preview is this release published somewhere reachable today. Same runners, same +`build-native.sh`, same integration tests; the difference is one workflow: +`.github/workflows/preview-release.yml` uploads one zip per platform to a GitHub Release, and +`scripts/install-preview.sh` installs a zip into a local Maven repository. The POMs in the +bundle are the ones the build produced. A GitHub Release rather than files in the repository, +because a native package is 112–167 MB. + +Cutting one is cutting a release, because it is one: + +```bash +mvn versions:set -DnewVersion=1.0.0-preview.1 -DgenerateBackupPoms=false +git commit -am "Set the version for the 1.0.0-preview.1 release" +# merge to main, let build.yml go green on the merge commit +git tag -a v1.0.0-preview.1 -m "chdb-java 1.0.0-preview.1" +git push origin v1.0.0-preview.1 +``` + +The workflow refuses the tag unless the POMs at that commit carry that version, the tag points +at that commit, the commit is an ancestor of `main`, and `build` was green for it. The first +attempt at `v1.0.0-preview.1` failed all four: tagged on a side branch 67 commits behind +`main`, it published binaries missing thirteen merged fixes. Nothing had downloaded it, so it +was deleted and the number reused. Afterwards the usual back-to-development commit returns the +POMs to a `-SNAPSHOT`. + +A preview tag cannot start the Central path — `release.yml` excludes it — and the release is +always marked pre-release. + diff --git a/docs/release-readiness.md b/docs/release-readiness.md index 53f3bab..ece860a 100644 --- a/docs/release-readiness.md +++ b/docs/release-readiness.md @@ -136,6 +136,11 @@ in CI. Afterwards, the ordinary "back to development" commit returns the POMs to a `-SNAPSHOT`. +**Before any of this, a preview can go out.** Step 1 gates Central, not shipping: a preview is +the same commit and the same four-runner build, published as GitHub Release assets and +installed with `scripts/install-preview.sh`. No namespace, no GPG key, no portal account. The +runbook is [docs/publishing.md](publishing.md) section 8. + --- ## Track 1 — Signing and publishing mechanics diff --git a/pom.xml b/pom.xml index 60ed114..4e0c499 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ org.chdb chdb-java-parent - 26.7.3.1-SNAPSHOT + 1.0.0-preview.1 pom chDB Java Binding (parent) diff --git a/scripts/install-preview.sh b/scripts/install-preview.sh new file mode 100755 index 0000000..9305de7 --- /dev/null +++ b/scripts/install-preview.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# +# Installs a published preview into a local Maven repository: downloads the asset for this +# platform, verifies it against the release's SHA256SUMS, and installs the jars with the POMs +# the build produced. Needs curl, unzip and mvn. +# +# install-preview.sh v1.0.0-preview.1 [--repo OWNER/REPO] [--maven-repo PATH] +# +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: install-preview.sh [--repo OWNER/REPO] [--maven-repo PATH] + +Downloads the matching platform bundle from a GitHub Release and installs its +POMs and JARs into the local Maven repository. +EOF +} + +die() { + printf 'error: %s\n' "$1" >&2 + exit 1 +} + +TAG='' +REPO='chdb-io/chdb-java' +MAVEN_REPO=${MAVEN_REPO:-"${HOME:-$PWD}/.m2/repository"} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --repo) + [[ $# -ge 2 ]] || die "--repo requires OWNER/REPO" + REPO=$2 + shift 2 + ;; + --repo=*) + REPO=${1#*=} + shift + ;; + --maven-repo) + [[ $# -ge 2 ]] || die "--maven-repo requires a path" + MAVEN_REPO=$2 + shift 2 + ;; + --maven-repo=*) + MAVEN_REPO=${1#*=} + shift + ;; + --*) + die "unknown option: $1" + ;; + '') + die "release tag cannot be empty" + ;; + *) + [[ -z "$TAG" ]] || die "only one release tag may be supplied" + TAG=$1 + shift + ;; + esac +done + +[[ -n "$TAG" ]] || { usage; exit 2; } +# Before any cd: install_file runs Maven from $WORK, which the EXIT trap deletes. +[[ "$MAVEN_REPO" = /* ]] || MAVEN_REPO="$PWD/$MAVEN_REPO" +[[ "$REPO" =~ ^[^/]+/[^/]+$ ]] || die "repository must look like OWNER/REPO: $REPO" +case "$TAG" in + v*-preview.*) ;; + *) die "release tag must look like v1.0.0-preview.1: $TAG" ;; +esac +VERSION=${TAG#v} + +case "$(uname -s):$(uname -m)" in + Darwin:arm64|Darwin:aarch64) + PLATFORM='macos-aarch64' + ;; + Darwin:x86_64|Darwin:amd64) + PLATFORM='macos-x86_64' + ;; + Linux:aarch64|Linux:arm64) + PLATFORM='linux-aarch64-gnu' + ;; + Linux:x86_64|Linux:amd64) + PLATFORM='linux-x86_64-gnu' + ;; + *) + die "unsupported host $(uname -s)/$(uname -m); chdb-java preview supports Linux and macOS on x86_64/aarch64" + ;; +esac + +for command_name in curl unzip mvn; do + command -v "$command_name" >/dev/null 2>&1 || die "required command is not installed: $command_name" +done + +ASSET="chdb-java-${VERSION}-${PLATFORM}.zip" +BASE_URL="https://github.com/${REPO}/releases/download/${TAG}" +WORK=$(mktemp -d "${TMPDIR:-/tmp}/chdb-java-preview.XXXXXX") +trap 'rm -rf "$WORK"' EXIT + +# curl's "error: 22" does not say whether the tag or the platform asset is the missing one. +download() { + local name=$1 + curl --fail --location --retry 3 --silent --show-error \ + --output "$WORK/$name" "$BASE_URL/$name" && return 0 + die "cannot download $name from $BASE_URL. +Check that the release exists and publishes this asset: https://github.com/${REPO}/releases/tag/${TAG}" +} + +download "$ASSET" +download SHA256SUMS + +EXPECTED=$(awk -v asset="$ASSET" '$2 == asset || $3 == asset { print $1; exit }' "$WORK/SHA256SUMS") +[[ "$EXPECTED" =~ ^[[:xdigit:]]{64}$ ]] || die "no valid checksum found for $ASSET" + +if command -v sha256sum >/dev/null 2>&1; then + ACTUAL=$(sha256sum "$WORK/$ASSET" | awk '{print $1}') +else + command -v shasum >/dev/null 2>&1 || die "sha256sum or shasum is required to verify the bundle" + ACTUAL=$(shasum -a 256 "$WORK/$ASSET" | awk '{print $1}') +fi +[[ "$EXPECTED" == "$ACTUAL" ]] || die "checksum mismatch for $ASSET" + +unzip -q "$WORK/$ASSET" -d "$WORK/extracted" +BUNDLE_ROOT="$WORK/extracted/chdb-java-${VERSION}-${PLATFORM}" +[[ -d "$BUNDLE_ROOT" ]] || die "unexpected bundle layout in $ASSET" + +PARENT_POM="$BUNDLE_ROOT/maven-poms/chdb-java-parent.pom" +DRIVER_POM="$BUNDLE_ROOT/maven-poms/chdb-jdbc.pom" +NATIVE_POM="$BUNDLE_ROOT/maven-poms/chdb-native-${PLATFORM}.pom" +BOM_POM="$BUNDLE_ROOT/maven-poms/chdb-bom.pom" +PREVIEW_PROPERTIES="$BUNDLE_ROOT/preview.properties" +DRIVER_JAR=$(find "$BUNDLE_ROOT/lib" -maxdepth 1 -type f -name 'chdb-jdbc-*.jar' -print -quit) +NATIVE_JAR=$(find "$BUNDLE_ROOT/lib" -maxdepth 1 -type f -name "chdb-native-${PLATFORM}-*.jar" -print -quit) + +for required_file in "$PARENT_POM" "$DRIVER_POM" "$NATIVE_POM" "$BOM_POM" "$PREVIEW_PROPERTIES" "$DRIVER_JAR" "$NATIVE_JAR"; do + [[ -n "$required_file" && -f "$required_file" ]] || die "bundle is missing $required_file" +done + +read_property() { + awk -F= -v key="$1" '$1 == key { print $2; exit }' "$PREVIEW_PROPERTIES" +} + +GROUP_ID=$(read_property chdb.java.preview.groupId) +[[ -n "$GROUP_ID" ]] || die "bundle does not declare a Maven groupId" + +# The asset name comes from the tag, so a mismatched bundle would install under the wrong version. +BUNDLE_VERSION=$(read_property chdb.java.preview.version) +[[ "$BUNDLE_VERSION" == "$VERSION" ]] || \ + die "bundle declares version $BUNDLE_VERSION but $TAG names $VERSION; the release asset does not match its tag" + +install_file() { + local file=$1 + local pom=$2 + local packaging=${3:-jar} + + ( + # From $WORK so Maven does not read the caller's own project POM. + cd "$WORK" + mvn -q -B -Dmaven.repo.local="$MAVEN_REPO" \ + org.apache.maven.plugins:maven-install-plugin:3.1.2:install-file \ + -Dfile="$file" \ + -DpomFile="$pom" \ + -Dpackaging="$packaging" \ + -DgeneratePom=false + ) +} + +# Parent first: the module POMs keep their parent relationship. +install_file "$PARENT_POM" "$PARENT_POM" pom +install_file "$DRIVER_JAR" "$DRIVER_POM" +install_file "$NATIVE_JAR" "$NATIVE_POM" +install_file "$BOM_POM" "$BOM_POM" pom + +printf 'Installed chdb-java preview %s (%s) into %s\n' "$TAG" "$PLATFORM" "$MAVEN_REPO" +printf 'Driver: %s:chdb-jdbc:%s\n' "$GROUP_ID" "$VERSION" +printf 'Native: %s:chdb-native-%s:%s\n' "$GROUP_ID" "$PLATFORM" "$VERSION" diff --git a/scripts/package-preview.sh b/scripts/package-preview.sh new file mode 100755 index 0000000..91152d9 --- /dev/null +++ b/scripts/package-preview.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# +# Packages one platform's preview bundle: the jars Maven just built plus the POMs that +# describe them, zipped for a GitHub Release. The POMs are copied, not generated, so +# install-file keeps the parent relationship and the driver dependency. +# +# The layout is scripts/install-preview.sh's contract: +# +# chdb-java--/ +# lib/ chdb-jdbc and chdb-native- jars +# maven-poms/ parent, driver, native, BOM +# preview.properties groupId, version, platform +# LICENSE, README.md +# +# scripts/package-preview.sh +# +set -euo pipefail + +usage() { + printf 'Usage: %s \n' "$(basename "$0")" >&2 + printf 'Platforms: macos-aarch64, macos-x86_64, linux-aarch64-gnu, linux-x86_64-gnu\n' >&2 +} + +die() { + printf 'error: %s\n' "$1" >&2 + exit 1 +} + +if [[ $# -ne 3 ]]; then + usage + exit 2 +fi + +VERSION=$1 +PLATFORM=$2 +OUTPUT_DIR=$3 +ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) + +case "$VERSION" in + ''|*[!A-Za-z0-9._-]*) die "version contains unsupported characters: $VERSION" ;; +esac + +case "$PLATFORM" in + macos-aarch64|macos-x86_64|linux-aarch64-gnu|linux-x86_64-gnu) ;; + *) die "unsupported platform: $PLATFORM" ;; +esac + +case "$PLATFORM" in + macos-aarch64) NATIVE_OS=macos; NATIVE_ARCH=aarch64; JNI_SUFFIX=dylib ;; + macos-x86_64) NATIVE_OS=macos; NATIVE_ARCH=x86_64; JNI_SUFFIX=dylib ;; + linux-aarch64-gnu) NATIVE_OS=linux; NATIVE_ARCH=aarch64; JNI_SUFFIX=so ;; + linux-x86_64-gnu) NATIVE_OS=linux; NATIVE_ARCH=x86_64; JNI_SUFFIX=so ;; +esac + +DRIVER_JAR="$ROOT/chdb-jdbc/target/chdb-jdbc-${VERSION}.jar" +NATIVE_JAR="$ROOT/chdb-native-${PLATFORM}/target/chdb-native-${PLATFORM}-${VERSION}.jar" +PARENT_POM="$ROOT/pom.xml" +DRIVER_POM="$ROOT/chdb-jdbc/pom.xml" +NATIVE_POM="$ROOT/chdb-native-${PLATFORM}/pom.xml" +BOM_POM="$ROOT/chdb-bom/pom.xml" + +for required_file in "$DRIVER_JAR" "$NATIVE_JAR" "$PARENT_POM" "$DRIVER_POM" "$NATIVE_POM" "$BOM_POM" "$ROOT/LICENSE" "$ROOT/README.md"; do + [[ -f "$required_file" ]] || die "required file does not exist: $required_file" +done + +GROUP_ID=$(awk ' + // { + sub(/^.*/, "") + sub(/<\/groupId>.*$/, "") + print + exit + } +' "$PARENT_POM") +[[ -n "$GROUP_ID" ]] || die "could not determine the Maven groupId from $PARENT_POM" + +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR=$(cd -- "$OUTPUT_DIR" && pwd) +ASSET="$OUTPUT_DIR/chdb-java-${VERSION}-${PLATFORM}.zip" +WORK=$(mktemp -d "${TMPDIR:-/tmp}/chdb-java-preview.XXXXXX") +trap 'rm -rf "$WORK"' EXIT + +BUNDLE_ROOT="$WORK/chdb-java-${VERSION}-${PLATFORM}" +mkdir -p "$BUNDLE_ROOT/lib" "$BUNDLE_ROOT/maven-poms" + +cp "$DRIVER_JAR" "$BUNDLE_ROOT/lib/" +cp "$NATIVE_JAR" "$BUNDLE_ROOT/lib/" +cp "$PARENT_POM" "$BUNDLE_ROOT/maven-poms/chdb-java-parent.pom" +cp "$DRIVER_POM" "$BUNDLE_ROOT/maven-poms/chdb-jdbc.pom" +cp "$NATIVE_POM" "$BUNDLE_ROOT/maven-poms/chdb-native-${PLATFORM}.pom" +cp "$BOM_POM" "$BUNDLE_ROOT/maven-poms/chdb-bom.pom" +cp "$ROOT/LICENSE" "$BUNDLE_ROOT/LICENSE" +cp "$ROOT/README.md" "$BUNDLE_ROOT/README.md" + +printf 'chdb.java.preview.groupId=%s\nchdb.java.preview.version=%s\nchdb.java.preview.platform=%s\n' \ + "$GROUP_ID" "$VERSION" "$PLATFORM" > "$BUNDLE_ROOT/preview.properties" + +if ! command -v jar >/dev/null 2>&1; then + die "the Java jar tool is required to create the preview bundle" +fi +if ! command -v unzip >/dev/null 2>&1; then + die "unzip is required to validate the native JAR" +fi + +NATIVE_ROOT="META-INF/chdb/native/${NATIVE_OS}/${NATIVE_ARCH}" +NATIVE_CONTENTS="$WORK/native-jar-contents.txt" +unzip -l "$NATIVE_JAR" > "$NATIVE_CONTENTS" +for entry in \ + "$NATIVE_ROOT/libchdb.so" \ + "$NATIVE_ROOT/libchdb_java_jni.${JNI_SUFFIX}" \ + "$NATIVE_ROOT/manifest.properties" \ + "$NATIVE_ROOT/sha256sums.txt" \ + "META-INF/sbom/bom.json"; do + grep -Fq "$entry" "$NATIVE_CONTENTS" || die "native JAR is missing $entry: $NATIVE_JAR" +done +grep -Fq 'META-INF/licenses/' "$NATIVE_CONTENTS" || die "native JAR is missing its license inventory: $NATIVE_JAR" + +rm -f "$ASSET" +jar --create --file="$ASSET" -C "$WORK" "$(basename "$BUNDLE_ROOT")" + +printf 'Created %s (%s bytes)\n' "$ASSET" "$(wc -c < "$ASSET" | tr -d ' ')" diff --git a/scripts/verify-preview-bundle.sh b/scripts/verify-preview-bundle.sh new file mode 100755 index 0000000..b555b89 --- /dev/null +++ b/scripts/verify-preview-bundle.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# +# Consumes a preview bundle the way a user will: installs it into an empty local Maven +# repository, then builds a project outside this checkout against it. +# +# A classpath smoke test would not do. What install-file can break lives in the POMs -- the +# parent relationship, the BOM, the native package's dependency on the driver -- so the +# consumer here declares the native package with no version and lets resolution supply the +# rest, then runs a query in memory and one that has to survive a reopen. +# +# scripts/verify-preview-bundle.sh +# +set -euo pipefail + +die() { printf 'verify-preview-bundle: %s\n' "$*" >&2; exit 1; } +step() { printf '\n== %s\n' "$*"; } +ok() { printf ' ok: %s\n' "$*"; } + +BUNDLE_ZIP=${1:-} +[[ -n "$BUNDLE_ZIP" ]] || { printf 'usage: %s \n' "$0" >&2; exit 2; } +[[ -f "$BUNDLE_ZIP" ]] || die "no such bundle: $BUNDLE_ZIP" +BUNDLE_ZIP=$(cd "$(dirname "$BUNDLE_ZIP")" && printf '%s/%s' "$(pwd)" "$(basename "$BUNDLE_ZIP")") + +for command_name in unzip mvn java; do + command -v "$command_name" >/dev/null 2>&1 || die "required command is not installed: $command_name" +done + +MVN_FLAGS=(--batch-mode --no-transfer-progress) +WORK="$(cd "$(mktemp -d "${TMPDIR:-/tmp}/chdb-verify-preview.XXXXXX")" && pwd)" +cleanup() { + status=$? + if [ "$status" -eq 0 ]; then + rm -rf "$WORK" + else + printf '\nverify-preview-bundle: left %s in place for diagnosis\n' "$WORK" >&2 + fi +} +trap cleanup EXIT + +M2="$WORK/m2" +CONSUMER="$WORK/consumer" +mkdir -p "$M2" "$CONSUMER/src/main/java" + +step "Unpacking the bundle" +unzip -q "$BUNDLE_ZIP" -d "$WORK/extracted" +# By preview.properties, not the first directory: `jar --create` writes its own META-INF. +PROPERTIES=$(find "$WORK/extracted" -mindepth 2 -maxdepth 2 -name preview.properties -print -quit) +[[ -n "$PROPERTIES" ]] || die "the bundle has no preview.properties" +BUNDLE_ROOT=$(dirname "$PROPERTIES") + +read_property() { awk -F= -v key="$1" '$1 == key { print $2; exit }' "$PROPERTIES"; } +GROUP_ID=$(read_property chdb.java.preview.groupId) +VERSION=$(read_property chdb.java.preview.version) +PLATFORM=$(read_property chdb.java.preview.platform) +[[ -n "$GROUP_ID" && -n "$VERSION" && -n "$PLATFORM" ]] || die "preview.properties is incomplete" +GROUP_PATH=${GROUP_ID//./\/} +ok "$GROUP_ID:$VERSION for $PLATFORM" + +# The bundle is per-platform, and the whole point of the run is to execute a query. +case "$(uname -s):$(uname -m)" in + Darwin:arm64|Darwin:aarch64) HOST=macos-aarch64 ;; + Darwin:x86_64|Darwin:amd64) HOST=macos-x86_64 ;; + Linux:aarch64|Linux:arm64) HOST=linux-aarch64-gnu ;; + Linux:x86_64|Linux:amd64) HOST=linux-x86_64-gnu ;; + *) die "unsupported host $(uname -s)/$(uname -m)" ;; +esac +[[ "$PLATFORM" == "$HOST" ]] || die "this is a $PLATFORM bundle and the host is $HOST" + +step "Installing it into an empty local repository" +install_file() { + ( cd "$WORK" && mvn "${MVN_FLAGS[@]}" -q -Dmaven.repo.local="$M2" \ + org.apache.maven.plugins:maven-install-plugin:3.1.2:install-file \ + -Dfile="$1" -DpomFile="$2" -Dpackaging="${3:-jar}" -DgeneratePom=false ) +} +POMS="$BUNDLE_ROOT/maven-poms" +install_file "$POMS/chdb-java-parent.pom" "$POMS/chdb-java-parent.pom" pom +install_file "$BUNDLE_ROOT/lib/chdb-jdbc-${VERSION}.jar" "$POMS/chdb-jdbc.pom" +install_file "$BUNDLE_ROOT/lib/chdb-native-${PLATFORM}-${VERSION}.jar" "$POMS/chdb-native-${PLATFORM}.pom" +install_file "$POMS/chdb-bom.pom" "$POMS/chdb-bom.pom" pom +ok "parent, driver, native package and BOM installed" + +step "Building a project outside this checkout against it" +# One dependency, no version: the BOM supplies it and the native POM brings in the driver. +cat > "$CONSUMER/pom.xml" < + 4.0.0 + org.chdb.verify + preview-consumer + 1 + + 11 + UTF-8 + + + + + ${GROUP_ID} + chdb-bom + ${VERSION} + pom + import + + + + + + ${GROUP_ID} + chdb-native-${PLATFORM} + + + +POM + +cat > "$CONSUMER/src/main/java/PreviewConsumer.java" <<'JAVA' +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; + +/** No Class.forName, one in-memory query, one that persists. */ +public final class PreviewConsumer { + public static void main(String[] args) throws Exception { + try (Connection connection = DriverManager.getConnection("jdbc:chdb::memory:"); + Statement statement = connection.createStatement(); + ResultSet results = statement.executeQuery("SELECT 42, chdb()")) { + results.next(); + if (results.getInt(1) != 42) { + throw new IllegalStateException("SELECT 42 returned " + results.getInt(1)); + } + System.out.println("engine " + results.getString(2)); + } + + Path storage = Path.of(args[0]); + try (Connection connection = DriverManager.getConnection("jdbc:chdb:" + storage); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS verify"); + statement.execute("CREATE TABLE IF NOT EXISTS verify.t (id UInt32) ENGINE = MergeTree ORDER BY id"); + statement.execute("INSERT INTO verify.t SELECT number FROM numbers(1000)"); + } + try (Connection connection = DriverManager.getConnection("jdbc:chdb:" + storage); + Statement statement = connection.createStatement(); + ResultSet results = statement.executeQuery("SELECT count() FROM verify.t")) { + results.next(); + if (results.getLong(1) != 1000) { + throw new IllegalStateException("reopened storage holds " + results.getLong(1) + " rows"); + } + } + System.out.println("PREVIEW BUNDLE OK"); + } +} +JAVA + +( cd "$CONSUMER" && mvn "${MVN_FLAGS[@]}" -q -Dmaven.repo.local="$M2" package "$WORK/build.log" 2>&1 || { tail -40 "$WORK/build.log"; die "the consumer project did not build; see $WORK/build.log"; } +ok "resolved and compiled with only the BOM and the native package declared" + +( cd "$CONSUMER" && mvn "${MVN_FLAGS[@]}" -q -Dmaven.repo.local="$M2" \ + dependency:build-classpath -Dmdep.outputFile="$WORK/cp.txt" ) \ + > "$WORK/resolve.log" 2>&1 || { tail -40 "$WORK/resolve.log"; die "could not resolve the consumer's classpath"; } + +# Transitive, not asked for. +grep -q "chdb-jdbc-${VERSION}.jar" "$WORK/cp.txt" \ + || die "chdb-jdbc is not on the classpath: the native package's POM did not bring it in" +ok "chdb-jdbc arrived transitively" + +# Nothing may have come from anywhere but the bundle's own repository. +while IFS= read -r entry; do + case "$entry" in + *"/${GROUP_PATH}/"*) + [[ "$entry" == "$M2/"* ]] || die "$entry did not come from $M2" + ;; + esac +done < <(tr ':' '\n' < "$WORK/cp.txt") +ok "every $GROUP_ID artifact resolved from the bundle's repository" + +step "Running a query through it" +# stdin closed: chDB reads a non-TTY stdin with bytes on it as external data for an INSERT, +# and this consumer inserts. Same reason as the integration-test step in build.yml. +OUTPUT=$( cd "$CONSUMER" && java -cp "target/classes:$(cat "$WORK/cp.txt")" \ + -Dchdb.cache.dir="$WORK/native-cache" PreviewConsumer "$WORK/storage"