Skip to content

[fix](statistics) Quote identifiers in statistics queries #67703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 2 commits into from
Sep 11, 2026
Merged

[fix](statistics) Quote identifiers in statistics queries #67703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a72d605
[fix](fe) Quote identifiers in statistics queries
morrySnow Sep 9, 2026
1491ba2
[fix](statistics) Update tests for quoted identifiers
morrySnow Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 37 additions & 36 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.qe.AuditLogHelper;
Expand Down Expand Up @@ -91,8 +92,8 @@ public enum AnalyzeSampleAlgorithm {
public static final String ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER = "ANALYZE_SKIP_LONG_STRING_COLUMN";

protected static final String FULL_ANALYZE_TEMPLATE = "WITH cte1 AS ("
+ "SELECT `${colName}`${lengthAssert} "
+ "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}), "
+ "SELECT ${colName}${lengthAssert} "
+ "FROM ${catalogName}.${dbName}.${tblName} ${index}), "
+ "cte2 AS ("
+ "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
+ "${catalogId} AS `catalog_id`, "
Expand All @@ -102,10 +103,10 @@ public enum AnalyzeSampleAlgorithm {
+ "'${colId}' AS `col_id`, "
+ "NULL AS `part_id`, "
+ "COUNT(1) AS `row_count`, "
+ "NDV(`${colName}`) AS `ndv`, "
+ "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
+ "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`, "
+ "NDV(${colName}) AS `ndv`, "
+ "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() "
+ "FROM cte1), "
Expand All @@ -116,9 +117,9 @@ public enum AnalyzeSampleAlgorithm {
+ "as `hot_value` "
+ "FROM ("
+ "SELECT ${subStringColName} as `hash_value`, "
+ "MAX(`${colName}`) as `column_key`, "
+ "MAX(${colName}) as `column_key`, "
+ "COUNT(1) AS `count` "
+ "FROM cte1 WHERE `${colName}` IS NOT NULL "
+ "FROM cte1 WHERE ${colName} IS NOT NULL "
+ "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT ${hotValueCollectCount}) t) "
+ "SELECT * FROM cte2 CROSS JOIN cte3";

Expand All @@ -131,19 +132,19 @@ public enum AnalyzeSampleAlgorithm {
+ "'${colId}' AS `col_id`, "
+ "NULL AS `part_id`, "
+ "COUNT(1) AS `row_count`, "
+ "NDV(`${colName}`) AS `ndv`, "
+ "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
+ "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`, "
+ "NDV(${colName}) AS `ndv`, "
+ "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() AS `update_time`, "
+ "null as `hot_value` "
+ "FROM (SELECT `${colName}`${lengthAssert} "
+ "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}) __lc_t";
+ "FROM (SELECT ${colName}${lengthAssert} "
+ "FROM ${catalogName}.${dbName}.${tblName} ${index}) __lc_t";

protected static final String LINEAR_ANALYZE_TEMPLATE = "WITH cte1 AS ("
+ "SELECT `${colName}`${lengthAssert} "
+ "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index} ${sampleHints} ${limit} ${preAggHint}), "
+ "SELECT ${colName}${lengthAssert} "
+ "FROM ${catalogName}.${dbName}.${tblName} ${index} ${sampleHints} ${limit} ${preAggHint}), "
+ "cte2 AS ("
+ "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
+ "${catalogId} AS `catalog_id`, "
Expand All @@ -154,7 +155,7 @@ public enum AnalyzeSampleAlgorithm {
+ "NULL AS `part_id`, "
+ "${rowCount} AS `row_count`, "
+ "${ndvFunction} as `ndv`, "
+ "ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) * ${scaleFactor}) AS `null_count`, "
+ "ROUND(SUM(CASE WHEN ${colName} IS NULL THEN 1 ELSE 0 END) * ${scaleFactor}) AS `null_count`, "
+ "SUBSTRING(CAST(${min} AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(${max} AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} * ${scaleFactor} AS `data_size`, "
Expand All @@ -166,9 +167,9 @@ public enum AnalyzeSampleAlgorithm {
+ "as `hot_value` "
+ "FROM ("
+ "SELECT ${subStringColName} as `hash_value`, "
+ "MAX(`${colName}`) as `column_key`, "
+ "MAX(${colName}) as `column_key`, "
+ "COUNT(1) AS `count` "
+ "FROM cte1 WHERE `${colName}` IS NOT NULL "
+ "FROM cte1 WHERE ${colName} IS NOT NULL "
+ "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT ${hotValueCollectCount}) t) "
+ "SELECT * FROM cte2 CROSS JOIN cte3";

Expand All @@ -177,9 +178,9 @@ public enum AnalyzeSampleAlgorithm {
+ "FROM "
+ "(SELECT "
+ "${subStringColName} AS `hash_value`, "
+ "`${colName}` AS `col_value`, "
+ "LENGTH(`${colName}`) as `len`${lengthAssert} "
+ "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index} ${sampleHints} ${limit}) as `t0` "
+ "${colName} AS `col_value`, "
+ "LENGTH(${colName}) as `len`${lengthAssert} "
+ "FROM ${catalogName}.${dbName}.${tblName} ${index} ${sampleHints} ${limit}) as `t0` "
+ "${preAggHint} GROUP BY `t0`.`hash_value`), "
+ "cte2 AS ( "
+ "SELECT CONCAT('${tblId}', '-', '${idxId}', '-', '${colId}') AS `id`, "
Expand Down Expand Up @@ -241,13 +242,13 @@ public enum AnalyzeSampleAlgorithm {
+ "${partId} AS `part_id`, "
+ "'${colId}' AS `col_id`, "
+ "COUNT(1) AS `row_count`, "
+ "HLL_UNION(HLL_HASH(`${colName}`)) as ndv, "
+ "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
+ "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`, "
+ "HLL_UNION(HLL_HASH(${colName})) as ndv, "
+ "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() AS `update_time` "
+ "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index} ${partitionInfo}";
+ "FROM ${catalogName}.${dbName}.${tblName} ${index} ${partitionInfo}";

protected static final String MERGE_PARTITION_TEMPLATE =
"SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
Expand Down Expand Up @@ -418,7 +419,7 @@ protected String getDataSizeFunction(Column column, boolean useDuj1) {
}
} else {
if (column.getType().isStringType()) {
return "SUM(LENGTH(`${colName}`))";
return "SUM(LENGTH(${colName}))";
} else {
return "COUNT(1) * " + column.getType().getSlotSize();
}
Expand All @@ -427,15 +428,15 @@ protected String getDataSizeFunction(Column column, boolean useDuj1) {

protected String getStringTypeColName(Column column) {
if (column.getType().isStringType()) {
return "xxhash_64(SUBSTRING(CAST(`${colName}` AS STRING), 1, 1024))";
return "xxhash_64(SUBSTRING(CAST(${colName} AS STRING), 1, 1024))";
} else {
return "`${colName}`";
return "${colName}";
}
}

protected String getMinFunction() {
if (tableSample == null) {
return "CAST(MIN(`${colName}`) as ${type}) ";
return "CAST(MIN(${colName}) as ${type}) ";
} else {
// Min value is not accurate while sample, so set it to NULL to avoid optimizer generate bad plan.
return "NULL";
Expand All @@ -456,7 +457,7 @@ protected String getNdvFunction(String totalRows) {
// Max value is not accurate while sample, so set it to NULL to avoid optimizer generate bad plan.
protected String getMaxFunction() {
if (tableSample == null) {
return "CAST(MAX(`${colName}`) as ${type}) ";
return "CAST(MAX(${colName}) as ${type}) ";
} else {
return "NULL";
}
Expand Down Expand Up @@ -562,7 +563,7 @@ protected void doPartitionTable() throws Exception {
}
}
}
params.put("partName", "'" + StatisticsUtil.escapeColumnName(part) + "'");
params.put("partName", StatisticsUtil.quote(StatisticsUtil.escapeSQL(part)));
params.put("partitionInfo", getPartitionInfo(part));
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
sqls.add(stringSubstitutor.replace(PARTITION_ANALYZE_TEMPLATE));
Expand Down Expand Up @@ -646,11 +647,11 @@ protected Map<String, String> buildSqlParams() {
protected void addLengthAssertParam(Map<String, String> params) {
long maxLen = org.apache.doris.common.Config.statistics_max_string_column_length;
if (col != null && col.getType().isStringType() && maxLen > 0) {
String escapedColName = StatisticsUtil.escapeColumnName(String.valueOf(info.colName));
String quotedColName = SqlUtils.getIdentSql(String.valueOf(info.colName));
// The StringSubstitutor used by callers already has ${colName} populated,
// so we inline the escaped column name directly here.
// so we inline the quoted column name directly here.
params.put("lengthAssert",
", assert_true(`" + escapedColName + "` IS NULL OR LENGTH(`" + escapedColName + "`) <= "
", assert_true(" + quotedColName + " IS NULL OR LENGTH(" + quotedColName + ") <= "
+ maxLen + ", '" + ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER + "') AS `__lc`");
} else {
params.put("lengthAssert", "");
Expand Down
14 changes: 8 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.statistics.util.StatisticsUtil;

import org.apache.commons.text.StringSubstitutor;

Expand Down Expand Up @@ -68,7 +70,7 @@ protected void doFull() throws Exception {
if (shouldCollectHotValue()) {
params.put("hotValueCollectCount", String.valueOf(SessionVariable.getHotValueCollectCount()));
params.put("subStringColName", getStringTypeColName(col));
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)");
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)");
template = FULL_ANALYZE_TEMPLATE;
} else {
template = FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE;
Expand All @@ -87,11 +89,11 @@ protected Map<String, String> buildSqlParams() {
params.put("dbId", String.valueOf(db.getId()));
params.put("tblId", String.valueOf(tbl.getId()));
params.put("idxId", "-1");
params.put("colName", info.colName);
params.put("colId", info.colName);
params.put("catalogName", catalog.getName());
params.put("dbName", db.getFullName());
params.put("tblName", tbl.getName());
params.put("colName", SqlUtils.getIdentSql(info.colName));
params.put("colId", StatisticsUtil.escapeSQL(info.colName));
params.put("catalogName", SqlUtils.getIdentSql(catalog.getName()));
params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
params.put("tblName", SqlUtils.getIdentSql(tbl.getName()));
params.put("sampleHints", getSampleHint());
params.put("limit", "");
params.put("scaleFactor", "1");
Expand Down
17 changes: 9 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.catalog.Env;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.statistics.analysis.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.util.StatisticsUtil;
Expand All @@ -44,10 +45,10 @@ public class HistogramTask extends BaseAnalysisTask {
+ " ${idxId} AS idx_id, "
+ " '${colId}' AS col_id, "
+ " ${sampleRate} AS sample_rate, "
+ " HISTOGRAM(`${colName}`, ${maxBucketNum}) AS buckets, "
+ " HISTOGRAM(${colName}, ${maxBucketNum}) AS buckets, "
+ " NOW() AS create_time "
+ "FROM "
+ " `${dbName}`.`${tblName}`";
+ " ${dbName}.${tblName}";

public HistogramTask(AnalysisInfo info) {
super(info);
Expand All @@ -56,16 +57,16 @@ public HistogramTask(AnalysisInfo info) {
@Override
public void doExecute() throws Exception {
Map<String, String> params = new HashMap<>();
params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
params.put("histogramStatTbl", StatisticConstants.HISTOGRAM_TBL_NAME);
params.put("internalDB", SqlUtils.getIdentSql(FeConstants.INTERNAL_DB_NAME));
params.put("histogramStatTbl", SqlUtils.getIdentSql(StatisticConstants.HISTOGRAM_TBL_NAME));
params.put("catalogId", String.valueOf(catalog.getId()));
params.put("dbId", String.valueOf(db.getId()));
params.put("tblId", String.valueOf(tbl.getId()));
params.put("idxId", String.valueOf(info.indexId));
params.put("colId", String.valueOf(info.colName));
params.put("dbName", db.getFullName());
params.put("tblName", tbl.getName());
params.put("colName", String.valueOf(info.colName));
params.put("colId", StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
params.put("tblName", SqlUtils.getIdentSql(tbl.getName()));
params.put("colName", SqlUtils.getIdentSql(String.valueOf(info.colName)));
params.put("sampleRate", getSampleRateFunction());
params.put("maxBucketNum", String.valueOf(info.maxBucketNum));

Expand Down
25 changes: 13 additions & 12 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
Expand Down Expand Up @@ -67,9 +68,9 @@
public class OlapAnalysisTask extends BaseAnalysisTask {

private static final String BASIC_STATS_TEMPLATE = "SELECT "
+ "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) as min, "
+ "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) as max "
+ "FROM `${dbName}`.`${tblName}` ${index}";
+ "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) as min, "
+ "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) as max "
+ "FROM ${dbName}.${tblName} ${index}";

private boolean keyColumnSampleTooManyRows = false;
private boolean partitionColumnSampleTooManyRows = false;
Expand Down Expand Up @@ -365,12 +366,12 @@ protected void setSampleParamsByAlgorithm(Map<String, String> params, long table
params.put("dataSizeFunction", getDataSizeFunction(col, true));
params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE `col_value` IS NOT NULL)");
} else {
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)");
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)");
// For single unique key, use count as ndv.
if (isSingleUniqueKey()) {
params.put("ndvFunction", String.valueOf(tableRowCount));
} else {
params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
params.put("ndvFunction", "ROUND(NDV(${colName}) * ${scaleFactor})");
}
}
}
Expand All @@ -387,7 +388,7 @@ protected void doFull() throws Exception {
if (shouldCollectHotValue()) {
params.put("hotValueCollectCount", String.valueOf(SessionVariable.getHotValueCollectCount()));
params.put("subStringColName", getStringTypeColName(col));
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)");
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)");
runQuery(stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE));
} else {
runQuery(stringSubstitutor.replace(FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE));
Expand Down Expand Up @@ -446,7 +447,7 @@ protected void deleteNotExistPartitionStats(AnalysisInfo jobInfo) throws DdlExce

@Override
protected String getPartitionInfo(String partitionName) {
return "partition " + partitionName;
return "partition " + SqlUtils.getIdentSql(partitionName);
}

@Override
Expand All @@ -460,10 +461,10 @@ protected Map<String, String> buildSqlParams() {
params.put("idxId", String.valueOf(info.indexId));
params.put("colId", StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
params.put("dataSizeFunction", getDataSizeFunction(col, false));
params.put("catalogName", catalog.getName());
params.put("dbName", db.getFullName());
params.put("colName", StatisticsUtil.escapeColumnName(String.valueOf(info.colName)));
params.put("tblName", String.valueOf(tbl.getName()));
params.put("catalogName", SqlUtils.getIdentSql(catalog.getName()));
params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
params.put("colName", SqlUtils.getIdentSql(String.valueOf(info.colName)));
params.put("tblName", SqlUtils.getIdentSql(String.valueOf(tbl.getName())));
params.put("index", getIndex());
params.put("preAggHint", "");
addLengthAssertParam(params);
Expand All @@ -475,7 +476,7 @@ protected String getIndex() {
return "";
} else {
OlapTable olapTable = (OlapTable) this.tbl;
return "index `" + olapTable.getIndexNameById(info.indexId) + "`";
return "index " + SqlUtils.getIdentSql(olapTable.getIndexNameById(info.indexId));
}
}

Expand Down
4 changes: 2 additions & 2 deletions ...re/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ protected void doSample() {
if (distributionColumns.size() == 1 && distributionColumns.contains(col.getName().toLowerCase())) {
bucketFlag = true;
sb.append(LINEAR_ANALYZE_TEMPLATE);
params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})");
params.put("ndvFunction", "ROUND(NDV(${colName}) * ${scaleFactor})");
params.put("rowCount", "ROUND(COUNT(1) * ${scaleFactor})");
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)");
params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)");
} else {
sb.append(DUJ1_ANALYZE_TEMPLATE);
params.put("subStringColName", getStringTypeColName(col));
Expand Down
Loading
Loading