Merge remote-tracking branch 'root/main' into feat_refactor_tag

Author: zhyass

.github/actions/artifact_meta_container_failure/action.yml

@@ -11,11 +11,25 @@ runs:
       shell: bash
       run: |
         mkdir -p logs
+
+        echo "=== Collecting k8s diagnostics ==="
+
+        # Pod status and events
+        kubectl get pods -n databend -o wide > logs/pods-status.txt 2>&1 || true
+        kubectl describe pods -n databend > logs/pods-describe.txt 2>&1 || true
+        kubectl get events -n databend --sort-by='.lastTimestamp' > logs/events.txt 2>&1 || true
+
+        # Chaos mesh resources
+        kubectl get iochaos -n databend -o yaml > logs/iochaos.txt 2>&1 || true
+
+        # Container logs (kubectl logs) and file logs
         for name in "test-databend-meta-0" "test-databend-meta-1" "test-databend-meta-2" "databend-metaverifier"
         do
-          echo "cat logs of $name"
-          kubectl exec -i $name -n databend -- /cat-logs.sh > logs/$name.log
+          echo "collecting logs of $name"
+          kubectl logs "$name" -n databend --tail=2000 > "logs/${name}-kubectl.log" 2>&1 || true
+          kubectl exec -i "$name" -n databend -- /cat-logs.sh > "logs/${name}-file.log" 2>&1 || true
         done
+
         tar -zcf logs/failure-${{ inputs.name }}.tar.gz logs/*
     - uses: actions/upload-artifact@v4
       with:

.github/actions/io_delay_chaos/action.yml

@@ -7,9 +7,3 @@ runs:
       shell: bash
       run: |
         ./scripts/ci/meta-chaos/ci-io-delay-meta-test.sh
-
-    - name: Upload failure
-      if: failure()
-      uses: ./.github/actions/artifact_meta_container_failure
-      with:
-        name: meta-io-delay-chaos

.github/workflows/meta.yml

@@ -79,3 +79,8 @@ jobs:
           artifacts: meta,metactl,metaverifier
       - uses: ./.github/actions/io_delay_chaos
         timeout-minutes: 20
+      - name: Upload failure artifacts
+        if: failure() || cancelled()
+        uses: ./.github/actions/artifact_meta_container_failure
+        with:
+          name: meta-io-delay-chaos

.github/workflows/release.yml

@@ -96,13 +96,13 @@ jobs:
         run: |
           case "${{ steps.bump.outputs.type }}" in
             nightly)
-              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --prerelease --draft
+              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --title ${{ steps.bump.outputs.tag }} --prerelease --draft
               ;;
             stable)
-              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --latest --draft
+              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --title ${{ steps.bump.outputs.tag }} --latest --draft
               ;;
             patch)
-              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --prerelease --draft
+              gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --notes-file /tmp/release_notes.md --title ${{ steps.bump.outputs.tag }} --prerelease --draft
               ;;
             *)
               echo "Invalid release type: ${{ steps.bump.outputs.type }}"
@@ -667,6 +667,11 @@ jobs:
           artifacts: meta,metactl,metaverifier
       - uses: ./.github/actions/io_delay_chaos
         timeout-minutes: 20
+      - name: Upload failure artifacts
+        if: failure() || cancelled()
+        uses: ./.github/actions/artifact_meta_container_failure
+        with:
+          name: meta-io-delay-chaos
       - name: Notify failure
         if: failure()
         uses: actions/github-script@v7

.gitignore

@@ -26,6 +26,7 @@ target/
 # profile
 flamegraph.*
 perf.*
+!perf.rs
 
 # tests
 *.stderr

Cargo.lock

@@ -73,6 +73,7 @@ databend-common-catalog = { path = "src/query/catalog" }
 databend-common-cloud-control = { path = "src/common/cloud_control" }
 databend-common-codegen = { path = "src/query/codegen" }
 databend-common-column = { path = "src/common/column" }
+databend-common-component = { path = "src/query/common/component" }
 databend-common-compress = { path = "src/common/compress" }
 databend-common-config = { path = "src/query/config" }
 databend-common-datavalues = { path = "src/query/datavalues" }

Cargo.toml

@@ -135,6 +135,7 @@ impl CaptureLogSettings {
 
 pub struct TrackingPayload {
     pub query_id: Option<String>,
+    pub warehouse_id: Option<String>,
     pub profile: Option<Arc<Profile>>,
     pub mem_stat: Option<Arc<MemStat>>,
     pub metrics: Option<Arc<ScopedRegistry>>,
@@ -204,6 +205,7 @@ impl Clone for TrackingPayload {
     fn clone(&self) -> Self {
         TrackingPayload {
             query_id: self.query_id.clone(),
+            warehouse_id: self.warehouse_id.clone(),
             profile: self.profile.clone(),
             mem_stat: self.mem_stat.clone(),
             metrics: self.metrics.clone(),
@@ -291,6 +293,7 @@ impl ThreadTracker {
                 metrics: None,
                 mem_stat: None,
                 query_id: None,
+                warehouse_id: None,
                 capture_log_settings: None,
                 time_series_profile: None,
                 local_time_series_profile: None,
@@ -410,6 +413,19 @@ impl ThreadTracker {
             .unwrap_or(None)
     }
 
+    pub fn warehouse_id() -> Option<&'static String> {
+        TRACKER
+            .try_with(|tracker| {
+                tracker
+                    .borrow()
+                    .payload
+                    .warehouse_id
+                    .as_ref()
+                    .map(|warehouse_id| unsafe { &*(warehouse_id as *const String) })
+            })
+            .unwrap_or(None)
+    }
+
     pub fn capture_log_settings() -> Option<&'static Arc<CaptureLogSettings>> {
         TRACKER
             .try_with(|tracker| {

src/common/base/src/runtime/runtime_tracker.rs

@@ -38,11 +38,21 @@ pub const DEFAULT_HISTOGRAM_BUCKETS: usize = 100;
 pub struct Histogram {
     pub accuracy: bool,
     pub buckets: Vec<HistogramBucket>,
+    /// Bucket width: (max - min) / num_buckets.
+    /// Only set when accuracy == false (generated from NDV + min/max).
+    /// Used to detect range distortion caused by outlier values inflating min/max.
+    /// A very large bucket_width means linear interpolation within buckets is unreliable.
+    #[serde(default)]
+    pub avg_spacing: Option<f64>,
 }
 
 impl Histogram {
     pub fn new(buckets: Vec<HistogramBucket>, accuracy: bool) -> Self {
-        Self { accuracy, buckets }
+        Self {
+            accuracy,
+            buckets,
+            avg_spacing: None,
+        }
     }
 
     /// Get number of buckets

src/common/statistics/src/histogram.rs

@@ -58,7 +58,6 @@ use crate::loggers::collect_kvs;
 pub struct RemoteLog {
     cluster_id: String,
     node_id: String,
-    warehouse_id: Option<String>,
     buffer: Arc<LogBuffer>,
 }
 
@@ -113,13 +112,9 @@ impl RemoteLog {
         let node_id = labels.get("node_id").cloned().unwrap_or_default();
         let rt = Runtime::with_worker_threads(2, Some("remote-log-writer".to_string()))?;
         let (tx, rx) = bounded(1);
-        // warehouse_id need to be specified after `create warehouse`
-        // TODO: inject warehouse_id like query_id
-        let warehouse_id = None;
         let remote_log = RemoteLog {
             cluster_id: labels.get("cluster_id").cloned().unwrap_or_default(),
             node_id: node_id.clone(),
-            warehouse_id,
             buffer: Arc::new(LogBuffer::new(tx.clone(), interval as u64)),
         };
         rt.spawn(async move { RemoteLog::work(rx, &stage_name).await });
@@ -209,6 +204,7 @@ impl RemoteLog {
     pub fn prepare_log_element(&self, record: &Record) -> RemoteLogElement {
         let timestamp = Timestamp::now().as_microsecond();
         let query_id = ThreadTracker::query_id().cloned();
+        let warehouse_id = ThreadTracker::warehouse_id().cloned();
         let target = record.target().to_string();
         let message = record.args().to_string();
         let fields = Map::from_iter(collect_kvs(record.key_values()));
@@ -232,7 +228,7 @@ impl RemoteLog {
             target,
             cluster_id: self.cluster_id.clone(),
             node_id: self.node_id.clone(),
-            warehouse_id: self.warehouse_id.clone(),
+            warehouse_id,
             query_id,
             log_level,
             message,