diff --git a/.github/workflows/rc-release.yml b/.github/workflows/rc-release.yml index b991e3a..47312b2 100644 --- a/.github/workflows/rc-release.yml +++ b/.github/workflows/rc-release.yml @@ -1,8 +1,7 @@ name: Maven RC release -# RCs are published to GitHub Packages while Maven Central Publisher Pro is being arranged. -# The permanent Maven coordinates are used from the first RC, so the later Central migration -# changes only the repository URL, not users' dependencies. +# RCs are staged as a standard Maven repository for upload to the public Vercel Blob store +# behind maven.chdb.io. Stable releases will move to Maven Central without changing coordinates. on: push: tags: @@ -88,12 +87,18 @@ jobs: # shellcheck disable=SC2086 mvn $MAVEN_ARGS -pl chdb-jdbc,chdb-native-${{ matrix.platform }} package -DskipTests mkdir -p "maven/${{ matrix.platform }}" - cp "chdb-jdbc/target/chdb-jdbc-${RC_VERSION}.jar" "maven/${{ matrix.platform }}/" cp "chdb-native-${{ matrix.platform }}/target/chdb-native-${{ matrix.platform }}-${RC_VERSION}.jar" "maven/${{ matrix.platform }}/" - cp pom.xml "maven/${{ matrix.platform }}/chdb-java-parent.pom" - cp chdb-jdbc/pom.xml "maven/${{ matrix.platform }}/chdb-jdbc.pom" cp "chdb-native-${{ matrix.platform }}/pom.xml" "maven/${{ matrix.platform }}/chdb-native-${{ matrix.platform }}.pom" - cp chdb-bom/pom.xml "maven/${{ matrix.platform }}/chdb-bom.pom" + + # Build the pure-Java artifacts once. JAR ZIP timestamps differ across matrix jobs, + # so accepting four interchangeable copies would make the published bytes depend on + # artifact download order. + if [ '${{ matrix.platform }}' = linux-x86_64-gnu ]; then + cp "chdb-jdbc/target/chdb-jdbc-${RC_VERSION}.jar" "maven/${{ matrix.platform }}/" + cp pom.xml "maven/${{ matrix.platform }}/chdb-java-parent.pom" + cp chdb-jdbc/pom.xml "maven/${{ matrix.platform }}/chdb-jdbc.pom" + cp chdb-bom/pom.xml "maven/${{ matrix.platform }}/chdb-bom.pom" + fi - uses: actions/upload-artifact@v4 with: @@ -102,13 +107,10 @@ jobs: if-no-files-found: error retention-days: 14 - publish: - name: publish RC to GitHub Packages + stage: + name: stage public Maven repository needs: build runs-on: ubuntu-latest - permissions: - contents: read - packages: write steps: - uses: actions/checkout@v4 with: @@ -118,9 +120,6 @@ jobs: with: distribution: temurin java-version: '11' - server-id: github - server-username: GITHUB_ACTOR - server-password: GITHUB_TOKEN - name: Resolve RC version run: | @@ -136,37 +135,70 @@ jobs: pattern: maven-* path: maven - - name: Publish Maven artifacts - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_ACTOR: ${{ github.actor }} + - name: Build the Maven repository directory run: | set -eu - REPOSITORY_URL="https://maven.pkg.github.com/${GITHUB_REPOSITORY_OWNER,,}/chdb-java" + REPOSITORY_URL="file://${GITHUB_WORKSPACE}/repository" PARENT_POM=$(find maven -name chdb-java-parent.pom -print -quit) JDBC_POM=$(find maven -name chdb-jdbc.pom -print -quit) BOM_POM=$(find maven -name chdb-bom.pom -print -quit) + # shellcheck disable=SC2086 mvn $MAVEN_ARGS org.apache.maven.plugins:maven-deploy-plugin:3.1.3:deploy-file \ -Dfile="$PARENT_POM" -DgroupId=com.clickhouse.chdb -DartifactId=chdb-java-parent \ -Dversion="$RC_VERSION" -Dpackaging=pom -DgeneratePom=false \ - -DrepositoryId=github -Durl="$REPOSITORY_URL" + -DrepositoryId=chdb-rc -Durl="$REPOSITORY_URL" + # shellcheck disable=SC2086 mvn $MAVEN_ARGS org.apache.maven.plugins:maven-deploy-plugin:3.1.3:deploy-file \ -Dfile="$(find maven -name "chdb-jdbc-${RC_VERSION}.jar" -print -quit)" \ -DpomFile="$JDBC_POM" -DgeneratePom=false \ - -DrepositoryId=github -Durl="$REPOSITORY_URL" + -DrepositoryId=chdb-rc -Durl="$REPOSITORY_URL" for platform in macos-aarch64 macos-x86_64 linux-x86_64-gnu linux-aarch64-gnu; do NATIVE_POM=$(find maven -name "chdb-native-${platform}.pom" -print -quit) NATIVE_JAR=$(find maven -name "chdb-native-${platform}-${RC_VERSION}.jar" -print -quit) + # shellcheck disable=SC2086 mvn $MAVEN_ARGS org.apache.maven.plugins:maven-deploy-plugin:3.1.3:deploy-file \ -Dfile="$NATIVE_JAR" -DpomFile="$NATIVE_POM" -DgeneratePom=false \ - -DrepositoryId=github -Durl="$REPOSITORY_URL" + -DrepositoryId=chdb-rc -Durl="$REPOSITORY_URL" done + # shellcheck disable=SC2086 mvn $MAVEN_ARGS org.apache.maven.plugins:maven-deploy-plugin:3.1.3:deploy-file \ -Dfile="$BOM_POM" -DpomFile="$BOM_POM" -Dpackaging=pom \ -DgroupId=com.clickhouse.chdb -DartifactId=chdb-bom -Dversion="$RC_VERSION" \ -DgeneratePom=false \ - -DrepositoryId=github -Durl="$REPOSITORY_URL" + -DrepositoryId=chdb-rc -Durl="$REPOSITORY_URL" + + # Version discovery is intentionally not supported by this temporary RC repository. + # Mutable artifact-level metadata would collide with the next immutable RC upload. + find repository -type f -name 'maven-metadata*' -delete + + find repository -type f \( -name '*.jar' -o -name '*.pom' \) -print0 | + while IFS= read -r -d '' file; do + md5sum "$file" | awk '{print $1}' > "$file.md5" + sha1sum "$file" | awk '{print $1}' > "$file.sha1" + sha256sum "$file" | awk '{print $1}' > "$file.sha256" + sha512sum "$file" | awk '{print $1}' > "$file.sha512" + done + + - uses: actions/upload-artifact@v4 + with: + name: maven-repository + path: repository + if-no-files-found: error + retention-days: 14 + + - name: Record the publication handoff + run: | + { + echo '### RC repository staged' + echo + # shellcheck disable=SC2016 + echo 'Download the `maven-repository` artifact and publish it with:' + echo + echo '```console' + echo 'scripts/publish-rc-repository.sh /path/to/maven-repository' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 9f30bc3..000f786 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ ClickHouse. It runs the engine in your JVM's process — no server, no network streaming, forward-only result sets over ClickHouse SQL. > **Status: release candidate.** The first Maven release candidate is `v1.0.0-rc.1`, built with -> chDB Core `26.7.3`. RC artifacts are published to GitHub Packages until Maven Central is ready; -> the public API is not frozen. See +> chDB Core `26.7.3`. RC artifacts are available from the public chDB Maven repository; the +> public API is not frozen. See > [What works today](#what-works-today). ```java @@ -42,13 +42,7 @@ and is loaded from a real packaged JAR, in CI: floor rather than infer it from symbol versions. - **Engine:** chDB Core **26.7.3**, pinned. The C ABI is version-locked, so the driver refuses to run against a different engine build rather than risking a struct-layout mismatch. - That release is a stable chdb-core release, which is what [work plan - §4.3](CHDB_JAVA_V1_WORK_PLAN.md) requires of a V1 GA engine: the previous baseline, - `26.7.2-rc.2`, was a pre-release and made every binding built on it a preview. -- **Not supported:** Windows, musl (Alpine), 32-bit, GraalVM Native Image, Android. See - [work plan §2.3](CHDB_JAVA_V1_WORK_PLAN.md). - -[docs/v1-progress.md](docs/v1-progress.md) has the phase-by-phase status and what is left. +- **Not supported:** Windows, musl (Alpine), 32-bit, GraalVM Native Image, Android. ## Installing @@ -63,26 +57,12 @@ on. The native package pulls in the driver, so declaring it alone is enough. ``` -GitHub Packages requires a GitHub classic PAT with `read:packages` in `~/.m2/settings.xml`: - -```xml - - - - github - YOUR_GITHUB_USERNAME - YOUR_GITHUB_PAT - - - -``` - -Add the repository to the consuming project: +Add the public RC repository to the consuming project. It does not require a username or token: ```xml - github - https://maven.pkg.github.com/chdb-io/chdb-java + chdb-rc + https://maven.chdb.io ``` @@ -95,7 +75,7 @@ architecture — declare the driver plus each native package you need: com.clickhouse.chdb chdb-bom - 1.0.0 + 1.0.0-rc.1 pom import @@ -130,11 +110,10 @@ SemVer, on the binding alone: `1.0.0` is the first release and a major bump mean change to the Java API. The engine version is not part of it — it is in each package's `manifest.properties`, pinned in [`scripts/engine.properties`](scripts/engine.properties) and named in the release notes (`26.7.3` today), and the driver refuses to load any other build. -See [work plan §4.3](CHDB_JAVA_V1_WORK_PLAN.md). The first test version is the RC `1.0.0-rc.1`; later candidates increment the final number, and the first stable Maven release is `1.0.0`. RCs use the permanent `com.clickhouse.chdb` groupId, -so moving stable releases to Maven Central will not change dependency coordinates. +so publishing stable releases to Maven Central will not change dependency coordinates. ## Connecting @@ -192,7 +171,7 @@ Use a second `Connection` — they can share the storage path — or close the f The engine runs in your process. There is no crash isolation: if it segfaults, your JVM dies with it, and no Java `catch` can intervene. That is the price of an in-process binding, and it is the reason not to embed chDB in a service where that is unacceptable. Subprocess isolation -is a post-V1 idea, not something V1 offers. +is not supported. The related trade is that the driver switches off chDB's own crash handlers, because they overwrite the ones HotSpot needs to function. You keep a working JVM and lose ClickHouse-format @@ -227,9 +206,8 @@ engine. In Tomcat, Spark or Flink this decides where the driver goes. See | ✅ | Bounded memory on results far larger than the heap | | ✅ | Host JVM signal handlers preserved — see [signal handlers](docs/signal-handlers.md) | | ✅ | Native loading from the platform JAR, or a directory you point at | -| 🚧 | Framework smoke tests — HikariCP, MyBatis and jOOQ pass, and nothing on the full `DatabaseMetaData` surface throws; Spring `JdbcTemplate` is next, and ShardingSphere cannot parse a `jdbc:chdb:` URL at all — see [under a framework](docs/unsupported.md#under-a-framework) | +| 🚧 | Framework smoke tests — HikariCP, MyBatis and jOOQ pass, and nothing on the full `DatabaseMetaData` surface throws; Spring `JdbcTemplate` is not yet verified, and ShardingSphere cannot parse a `jdbc:chdb:` URL at all — see [under a framework](docs/unsupported.md#under-a-framework) | | 🚧 | Soak tests; full-process ASan, which needs an upstream sanitizer build of chdb-core | -| 🚧 | Maven Central publishing — nothing is released yet | | ❌ | Transactions, batch updates, scrollable/updatable result sets, `CallableStatement` | | ❌ | Stored procedures, generated keys, `Blob`/`Clob`/`Array`/`SQLXML` | | ❌ | `Array`, `Map`, `Tuple`, `Nested`, `Variant`, `JSON`, `Dynamic` columns | @@ -265,7 +243,7 @@ detection. | Storage path | many connections on one path; a second path refused with a usable diagnosis; rebinding after the last close; a failed connect leaving nothing pinned | | Loader | five failure paths: no platform package, a bad override, a missing shim, a corrupted cache and a tampered checksum | | Packaging | each platform JAR is built, then the engine is loaded back out of it and a query run, on every platform | -| Version floors | the full suite again on AlmaLinux 8 — glibc 2.28, RHEL 8's base — against the artefacts the release job would publish; and the build fails if a platform's measured floor rises above its ceiling | +| Version floors | the full suite again on AlmaLinux 8 — glibc 2.28, RHEL 8's base — against the packaged artifacts; and the build fails if a platform's measured floor rises above its ceiling | **Sanitizers**, on both a Linux and a macOS toolchain: UBSan over the whole integration suite in a real JVM against the real engine, and ASan plus UBSan over a 107-check harness for the shim's @@ -287,7 +265,6 @@ but not exercised on an old macOS, because no such runner exists. | [Memory](docs/memory.md) | Why `-Xmx` does not bound chDB, and what does | | [ClassLoaders](docs/classloaders.md) | Tomcat, Spark, Flink: where to put the driver | | [Upstream findings](docs/upstream-findings.md) | Engine behaviours this binding works around, with reproductions | -| [V1 progress](docs/v1-progress.md) | Phase-by-phase status against the work plan, and what to do next | ## Building from source @@ -339,7 +316,6 @@ scripts/engine.properties the pinned engine version and its checksums scripts/fetch-libchdb.sh downloads and verifies the pinned engine scripts/build-native.sh builds the shim and stages a platform package scripts/verify-consumer.sh resolves the driver from a repository, outside this checkout -scripts/check-release-tag.sh refuses a release whose tag does not name the commit ``` ## Reporting a problem diff --git a/deploy/vercel-maven/.gitignore b/deploy/vercel-maven/.gitignore new file mode 100644 index 0000000..245259b --- /dev/null +++ b/deploy/vercel-maven/.gitignore @@ -0,0 +1,2 @@ +.vercel +.env* diff --git a/deploy/vercel-maven/README.md b/deploy/vercel-maven/README.md new file mode 100644 index 0000000..1e6efda --- /dev/null +++ b/deploy/vercel-maven/README.md @@ -0,0 +1,40 @@ +# chDB public Maven repository gateway + +This Vercel project gives RC consumers the stable, anonymous repository URL +`https://maven.chdb.io`. Maven files live in the public `chdb-maven-blob` store; the gateway +rewrites `/com/...` to that store. + +The gateway deliberately disables Vercel's external-rewrite cache. Vercel Blob already caches +the immutable artifacts, while a second cache at the rewrite layer can incorrectly reuse one +HTTP Range response for a different range of the same large JAR. + +## Deploy the gateway + +The local directory is linked to the ClickHouse team project `chdb-maven`: + +```console +cd deploy/vercel-maven +npx --yes vercel@latest build --prod --scope clickhouse +npx --yes vercel@latest deploy --prebuilt --prod --scope clickhouse +``` + +## Publish an RC + +The `Maven RC release` workflow builds a `maven-repository` artifact. Download and publish it +from an account with Developer access to the ClickHouse Vercel team: + +```console +gh run download RUN_ID --name maven-repository --dir /tmp/chdb-maven-repository +cd deploy/vercel-maven +npx --yes vercel@latest link --project chdb-maven --scope clickhouse +npx --yes vercel@latest env pull .env.local --environment=development +cd ../.. +scripts/publish-rc-repository.sh /tmp/chdb-maven-repository +``` + +The environment file is ignored by Git. It supplies a short-lived Vercel OIDC credential; no +personal or long-lived Blob token is stored in GitHub. RC paths are immutable, and the upload +script refuses to overwrite an existing file. Before writing anything, it checks every destination +path. If an upload fails or is interrupted, it removes the blobs written by that run so the complete +RC can be retried safely. If rollback itself fails, remove the paths reported by the script before +retrying; the next preflight will refuse to overwrite them. diff --git a/deploy/vercel-maven/index.html b/deploy/vercel-maven/index.html new file mode 100644 index 0000000..5ad212a --- /dev/null +++ b/deploy/vercel-maven/index.html @@ -0,0 +1,16 @@ + + + + + + chDB Maven repository + + +
+

chDB Maven repository

+

Public release-candidate artifacts for chDB Java.

+

Use https://maven.chdb.io as the repository URL. Authentication is not required.

+

Documentation and source

+
+ + diff --git a/deploy/vercel-maven/vercel.json b/deploy/vercel-maven/vercel.json new file mode 100644 index 0000000..3b2688e --- /dev/null +++ b/deploy/vercel-maven/vercel.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "rewrites": [ + { + "source": "/com/:path*", + "destination": "https://ygrvrz9pslxmp8da.public.blob.vercel-storage.com/com/:path*" + } + ], + "headers": [ + { + "source": "/com/:path*", + "headers": [ + { + "key": "x-vercel-enable-rewrite-caching", + "value": "0" + }, + { + "key": "Vercel-CDN-Cache-Control", + "value": "no-store" + }, + { + "key": "X-Content-Type-Options", + "value": "nosniff" + } + ] + } + ] +} diff --git a/scripts/publish-rc-repository.sh b/scripts/publish-rc-repository.sh new file mode 100755 index 0000000..c6319bf --- /dev/null +++ b/scripts/publish-rc-repository.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash + +# Upload an already-staged Maven repository to the public chDB Vercel Blob store. +# Run `vercel env pull .env.local --environment=development` from +# deploy/vercel-maven immediately before this command so the short-lived OIDC +# credential is fresh. Published RC paths are immutable: this script never +# overwrites an existing blob. + +set -euo pipefail + +usage() { + printf 'Usage: %s [--dry-run] \n' "$(basename "$0")" >&2 + exit 2 +} + +DRY_RUN=false +if [[ ${1:-} == --dry-run ]]; then + DRY_RUN=true + shift +fi +[[ $# -eq 1 ]] || usage + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROJECT_DIR=$(cd "$SCRIPT_DIR/.." && pwd) +GATEWAY_DIR="$PROJECT_DIR/deploy/vercel-maven" +REPOSITORY_DIR=$(cd "$1" && pwd) +ENV_FILE=${CHDB_VERCEL_ENV_FILE:-"$GATEWAY_DIR/.env.local"} + +[[ -d "$REPOSITORY_DIR/com/clickhouse/chdb" ]] || { + printf 'Expected com/clickhouse/chdb below %s\n' "$REPOSITORY_DIR" >&2 + exit 1 +} + +if find "$REPOSITORY_DIR" -type l -print -quit | grep -q .; then + printf 'Refusing to publish a repository containing symbolic links\n' >&2 + exit 1 +fi + +FILES=() +RELATIVE_PATHS=() +while IFS= read -r -d '' file; do + relative=${file#"$REPOSITORY_DIR/"} + if [[ ! "$relative" =~ ^com/clickhouse/chdb/[^/]+/[^/]+/[^/]+\.(jar|pom)(\.(md5|sha1|sha256|sha512))?$ ]]; then + printf 'Refusing to publish a non-versioned Maven artifact path: %s\n' "$relative" >&2 + exit 1 + fi + FILES+=("$file") + RELATIVE_PATHS+=("$relative") +done < <(find "$REPOSITORY_DIR" -type f -print0) + +file_count=${#FILES[@]} +(( file_count > 0 )) || { + printf 'No files found below %s\n' "$REPOSITORY_DIR" >&2 + exit 1 +} + +if [[ "$DRY_RUN" == true ]]; then + printf 'Validated %d immutable Maven files below %s\n' "$file_count" "$REPOSITORY_DIR" + exit 0 +fi + +[[ -f "$ENV_FILE" ]] || { + printf 'Missing %s. Run these commands first:\n' "$ENV_FILE" >&2 + printf ' cd %q\n' "$GATEWAY_DIR" >&2 + printf ' npx --yes vercel@latest link --project chdb-maven --scope clickhouse\n' >&2 + printf ' npx --yes vercel@latest env pull .env.local --environment=development\n' >&2 + exit 1 +} + +set -a +# shellcheck disable=SC1090 +. "$ENV_FILE" +set +a + +: "${BLOB_STORE_ID:?BLOB_STORE_ID is missing from $ENV_FILE}" +: "${VERCEL_OIDC_TOKEN:?VERCEL_OIDC_TOKEN is missing from $ENV_FILE}" + +if command -v vercel >/dev/null 2>&1; then + VERCEL=(vercel) +else + VERCEL=(npx --yes vercel@latest) +fi + +BLOB_AUTH_ARGS=( + --store-id "$BLOB_STORE_ID" + --oidc-token "$VERCEL_OIDC_TOKEN" +) + +blob_exists() { + local target=$1 + local output + + if ! output=$("${VERCEL[@]}" blob list \ + --prefix "$target" \ + --limit 1000 \ + --mode expanded \ + --no-color \ + "${BLOB_AUTH_ARGS[@]}"); then + printf 'Failed to check whether %s already exists\n' "$target" >&2 + return 2 + fi + + awk -v target="$target" ' + { + for (field = 1; field <= NF; field++) { + if ($field == target) { + found = 1 + } + } + } + END { exit found ? 0 : 1 } + ' <<<"$output" +} + +printf 'Checking %d destination paths before upload\n' "$file_count" +conflict_count=0 +for relative in "${RELATIVE_PATHS[@]}"; do + if blob_exists "$relative"; then + printf 'Refusing to overwrite existing blob: %s\n' "$relative" >&2 + conflict_count=$((conflict_count + 1)) + else + result=$? + (( result == 1 )) || exit "$result" + fi +done + +(( conflict_count == 0 )) || { + printf 'Preflight found %d existing destination path(s); nothing was uploaded\n' \ + "$conflict_count" >&2 + exit 1 +} + +UPLOADED_PATHS=() +uploaded_count=0 +rollback() { + local status=$1 + local rollback_failed=false + + trap - ERR HUP INT TERM + set +e + + if (( uploaded_count > 0 )); then + printf 'Upload did not complete; removing %d blob(s) written by this run\n' \ + "$uploaded_count" >&2 + + for relative in "${UPLOADED_PATHS[@]}"; do + if ! "${VERCEL[@]}" blob del "$relative" "${BLOB_AUTH_ARGS[@]}"; then + printf 'Rollback failed for %s; remove it before retrying\n' "$relative" >&2 + rollback_failed=true + fi + done + fi + + if [[ "$rollback_failed" == true ]]; then + printf 'Rollback was incomplete; the next preflight will refuse to publish over remaining blobs\n' >&2 + fi + + exit "$status" +} + +trap 'rollback $?' ERR +trap 'rollback 129' HUP +trap 'rollback 130' INT +trap 'rollback 143' TERM + +printf 'Uploading %d immutable files to store %s\n' "$file_count" "$BLOB_STORE_ID" + +for index in "${!FILES[@]}"; do + file=${FILES[$index]} + relative=${RELATIVE_PATHS[$index]} + "${VERCEL[@]}" blob put "$file" \ + --pathname "$relative" \ + --access public \ + --add-random-suffix false \ + --allow-overwrite false \ + --cache-control-max-age 31536000 \ + --multipart true \ + "${BLOB_AUTH_ARGS[@]}" + UPLOADED_PATHS+=("$relative") + uploaded_count=$((uploaded_count + 1)) +done + +trap - ERR HUP INT TERM + +printf 'Published %d files. Verify the RC from https://maven.chdb.io with a fresh Maven cache.\n' "$file_count"