Merge remote-tracking branch 'root/main' into feat_refactor_tag

Author: zhyass

.github/actions/test_sqllogic_stage/action.yml

@@ -17,6 +17,10 @@ inputs:
     description: "path type for dedup when copy, choices: full_path,sub_path"
     required: true
     default: ""
+  size:
+    description: "stage parquet read size mode, choices: small,large"
+    required: true
+    default: ""
 runs:
   using: "composite"
   steps:
@@ -34,4 +38,5 @@ runs:
         TEST_HANDLERS: ${{ inputs.handlers }}
         TEST_STAGE_STORAGE: ${{ inputs.storage }}
         TEST_STAGE_DEDUP: ${{ inputs.dedup}}
+        TEST_STAGE_SIZE: ${{ inputs.size }}
       run: bash ./scripts/ci/ci-run-sqllogic-tests-without-sandbox.sh ${{ inputs.dirs }}

.github/actions/test_stateful_iceberg_catalogs_standalone/action.yml

@@ -10,10 +10,10 @@ runs:
       run: |
         DOCKER_DEFAULT_PLATFORM=`docker info --format {{.OSType}}/{{.Architecture}}`
         docker-compose -f "./docker/it-iceberg-catalogs/docker-compose.yaml" up --quiet-pull -d
-        hms_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq --filter "name=hive-metastore"))
+        # hms_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq --filter "name=hive-metastore"))
 
         docker ps -a | grep iceberg
-        echo "HMS in $hms_ip"
+        # echo "HMS in $hms_ip"
 
     - name: Run Stateful Tests with Standalone mode
       shell: bash

.github/workflows/release.yml

@@ -65,6 +65,29 @@ jobs:
           script: |
             const script = require('./.github/scripts/bump_version.js')
             await script({ github, context, core })
+      - name: Generate release notes
+        id: notes
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = '${{ steps.bump.outputs.tag }}';
+            const previous = '${{ steps.bump.outputs.previous }}';
+            const sha = '${{ steps.bump.outputs.sha }}';
+            const { data } = await github.rest.repos.generateReleaseNotes({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              tag_name: tag,
+              target_commitish: sha,
+              previous_tag_name: previous,
+            });
+            const MAX_LEN = 124000;
+            let body = data.body;
+            if (body.length > MAX_LEN) {
+              core.warning(`Release notes too long (${body.length} chars), truncating to ${MAX_LEN}`);
+              body = body.substring(0, MAX_LEN) + '\n\n> **Note**: changelog truncated due to GitHub size limits. See commit history for full details.';
+            }
+            const fs = require('fs');
+            fs.writeFileSync('/tmp/release_notes.md', body);
       - name: Create release
         env:
           # we need workflow:write permission to create release if there were any workflow changes
@@ -73,13 +96,13 @@ jobs:
         run: |
           case "${{ steps.bump.outputs.type }}" in
             nightly)
-              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag ${{ steps.bump.outputs.previous }} --prerelease --draft
+              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --prerelease --draft
               ;;
             stable)
-              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag ${{ steps.bump.outputs.previous }} --latest --draft
+              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --latest --draft
               ;;
             patch)
-              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag ${{ steps.bump.outputs.previous }} --prerelease --draft
+              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --prerelease --draft
               ;;
             *)
               echo "Invalid release type: ${{ steps.bump.outputs.type }}"
@@ -96,6 +119,11 @@ jobs:
         with:
           repository: databendlabs/databend-docs
           ref: main
+      - name: Checkout Scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+          path: databend
       - name: Get date
         id: date
         shell: bash
@@ -107,8 +135,10 @@ jobs:
           DATE: ${{ steps.date.outputs.DATE }}
         with:
           script: |
-            const script = require('./.github/scripts/generate_release_note.js')
+            const script = require('./databend/.github/scripts/generate_release_note.js')
             await script({ github, context, core })
+      - name: Clean up temporary checkout
+        run: rm -rf databend
       - name: Add Release Note
         run: |
           git add docs/release-stable

.github/workflows/reuse.sqllogic.yml

@@ -323,6 +323,7 @@ jobs:
           dirs: stage
           handlers: ${{ matrix.handler }}
           dedup: ${{ matrix.dedup }}
+          size: ${{ matrix.size }}
       - name: Upload failure
         if: failure()
         uses: ./.github/actions/artifact_failure

Cargo.lock

@@ -64,6 +64,7 @@ default-members = [
 # Workspace dependencies
 [workspace.dependencies]
 databend-common-ast = { path = "src/query/ast" }
+databend-common-ast-visit-derive = { path = "src/query/ast/visit_derive" }
 databend-common-base = { path = "src/common/base" }
 databend-common-binaries = { path = "src/binaries" }
 databend-common-building = { path = "src/common/building" }
@@ -293,6 +294,8 @@ hyper = "1"
 hyper-util = { version = "0.1.9", features = ["client", "client-legacy", "tokio", "service"] }
 lru = "0.12"
 perf-event2 = "0.7.4"
+proc-macro2 = "1.0"
+quote = "1.0"
 
 ## in branch dev
 iceberg = { version = "0.8.0", git = "https://github.com/databendlabs/iceberg-rust", rev = "6ccaa60e", features = [
@@ -464,6 +467,7 @@ strength_reduce = "0.2.4"
 stringslice = "0.2.0"
 strum = "0.24.1"
 sub-cache = "0.2.1"
+syn = { version = "2.0", features = ["full"] }
 sys-info = "0.9"
 sysinfo = "0.34.2"
 tantivy = "0.26.0"

Cargo.toml

@@ -60,17 +60,17 @@ services:
     entrypoint: >
       /bin/sh -c " until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; /usr/bin/mc rm -r --force minio/warehouse; /usr/bin/mc mb minio/warehouse; /usr/bin/mc policy set public minio/warehouse; tail -f /dev/null "
 
-  hive-metastore:
-    image: iceberg-hive-metastore
-    build: ./
-    platform: ${DOCKER_DEFAULT_PLATFORM}
-    ports:
-      - "9083:9083"
-    expose:
-      - 9083
-    environment:
-      SERVICE_NAME: "metastore"
-      SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/"
+  # hive-metastore:
+  #   image: iceberg-hive-metastore
+  #   build: ./
+  #   platform: ${DOCKER_DEFAULT_PLATFORM}
+  #   ports:
+  #     - "9083:9083"
+  #   expose:
+  #     - 9083
+  #   environment:
+  #     SERVICE_NAME: "metastore"
+  #     SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/"
 
   moto:
     image: motoserver/moto:5.0.3

docker/it-iceberg-catalogs/docker-compose.yaml

@@ -452,3 +452,9 @@ impl From<tokio::task::JoinError> for ErrorCode {
         }
     }
 }
+
+impl From<!> for ErrorCode {
+    fn from(_: !) -> Self {
+        unreachable!()
+    }
+}

src/common/exception/src/exception_into.rs

@@ -14,6 +14,7 @@
 
 #![allow(clippy::uninlined_format_args)]
 #![allow(clippy::collapsible_if)]
+#![feature(never_type)]
 
 extern crate core;
 

src/common/exception/src/lib.rs

@@ -146,6 +146,9 @@ pub struct OutputFormatSettings {
     pub binary_format: BinaryDisplayFormat,
     pub http_json_result_mode: HttpHandlerDataFormat,
     pub headers: u8,
+    // used only for compat with old bendSQL driver.
+    pub http_arrow_use_jsonb: bool,
+
     pub json_compact: bool,
     pub json_strings: bool,
 
@@ -161,6 +164,7 @@ impl Default for OutputFormatSettings {
             binary_format: BinaryDisplayFormat::Hex,
             http_json_result_mode: HttpHandlerDataFormat::Display,
             headers: 0,
+            http_arrow_use_jsonb: false,
             json_compact: false,
             json_strings: false,
             format_null_as_str: false,