From a72d6054561eb6f162138e32912e646ecf8aea28 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Wed, 9 Sep 2026 12:31:53 +0800 Subject: [PATCH 1/2] [fix](fe) Quote identifiers in statistics queries Problem Summary: Statistics collection SQL placed object names inside template-owned backticks without escaping embedded backticks. Because these statements run with an elevated internal identity, a crafted object name could alter the generated query and expose data outside the caller's privileges. Solution: Generate complete quoted identifier fragments with SqlUtils.getIdentSql for catalogs, databases, tables, columns, indexes, and partitions across OLAP, external, plugin-sample, and histogram analysis paths. Escape identifier-derived values separately when stored as string literals, and add coverage for embedded backticks and apostrophes. Issue Number: None --- .../statistics/analysis/BaseAnalysisTask.java | 73 ++++++++--------- .../analysis/ExternalAnalysisTask.java | 14 ++-- .../statistics/analysis/HistogramTask.java | 17 ++-- .../statistics/analysis/OlapAnalysisTask.java | 25 +++--- .../PluginDrivenSampleAnalysisTask.java | 4 +- .../analysis/OlapAnalysisTaskTest.java | 79 ++++++++++++++----- 6 files changed, 128 insertions(+), 84 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java index 52a1077aa8eaca..5bbb96e1134c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/BaseAnalysisTask.java @@ -28,6 +28,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.Status; import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.metric.MetricRepo; import org.apache.doris.qe.AuditLogHelper; @@ -91,8 +92,8 @@ public enum AnalyzeSampleAlgorithm { public static final String ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER = "ANALYZE_SKIP_LONG_STRING_COLUMN"; protected static final String FULL_ANALYZE_TEMPLATE = "WITH cte1 AS (" - + "SELECT `${colName}`${lengthAssert} " - + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}), " + + "SELECT ${colName}${lengthAssert} " + + "FROM ${catalogName}.${dbName}.${tblName} ${index}), " + "cte2 AS (" + "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, " + "${catalogId} AS `catalog_id`, " @@ -102,10 +103,10 @@ public enum AnalyzeSampleAlgorithm { + "'${colId}' AS `col_id`, " + "NULL AS `part_id`, " + "COUNT(1) AS `row_count`, " - + "NDV(`${colName}`) AS `ndv`, " - + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, " - + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`, " - + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`, " + + "NDV(${colName}) AS `ndv`, " + + "COUNT(1) - COUNT(${colName}) AS `null_count`, " + + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, " + + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, " + "${dataSizeFunction} AS `data_size`, " + "NOW() " + "FROM cte1), " @@ -116,9 +117,9 @@ public enum AnalyzeSampleAlgorithm { + "as `hot_value` " + "FROM (" + "SELECT ${subStringColName} as `hash_value`, " - + "MAX(`${colName}`) as `column_key`, " + + "MAX(${colName}) as `column_key`, " + "COUNT(1) AS `count` " - + "FROM cte1 WHERE `${colName}` IS NOT NULL " + + "FROM cte1 WHERE ${colName} IS NOT NULL " + "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT ${hotValueCollectCount}) t) " + "SELECT * FROM cte2 CROSS JOIN cte3"; @@ -131,19 +132,19 @@ public enum AnalyzeSampleAlgorithm { + "'${colId}' AS `col_id`, " + "NULL AS `part_id`, " + "COUNT(1) AS `row_count`, " - + "NDV(`${colName}`) AS `ndv`, " - + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, " - + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`, " - + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`, " + + "NDV(${colName}) AS `ndv`, " + + "COUNT(1) - COUNT(${colName}) AS `null_count`, " + + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, " + + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, " + "${dataSizeFunction} AS `data_size`, " + "NOW() AS `update_time`, " + "null as `hot_value` " - + "FROM (SELECT `${colName}`${lengthAssert} " - + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}) __lc_t"; + + "FROM (SELECT ${colName}${lengthAssert} " + + "FROM ${catalogName}.${dbName}.${tblName} ${index}) __lc_t"; protected static final String LINEAR_ANALYZE_TEMPLATE = "WITH cte1 AS (" - + "SELECT `${colName}`${lengthAssert} " - + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index} ${sampleHints} ${limit} ${preAggHint}), " + + "SELECT ${colName}${lengthAssert} " + + "FROM ${catalogName}.${dbName}.${tblName} ${index} ${sampleHints} ${limit} ${preAggHint}), " + "cte2 AS (" + "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, " + "${catalogId} AS `catalog_id`, " @@ -154,7 +155,7 @@ public enum AnalyzeSampleAlgorithm { + "NULL AS `part_id`, " + "${rowCount} AS `row_count`, " + "${ndvFunction} as `ndv`, " - + "ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END) * ${scaleFactor}) AS `null_count`, " + + "ROUND(SUM(CASE WHEN ${colName} IS NULL THEN 1 ELSE 0 END) * ${scaleFactor}) AS `null_count`, " + "SUBSTRING(CAST(${min} AS STRING), 1, 1024) AS `min`, " + "SUBSTRING(CAST(${max} AS STRING), 1, 1024) AS `max`, " + "${dataSizeFunction} * ${scaleFactor} AS `data_size`, " @@ -166,9 +167,9 @@ public enum AnalyzeSampleAlgorithm { + "as `hot_value` " + "FROM (" + "SELECT ${subStringColName} as `hash_value`, " - + "MAX(`${colName}`) as `column_key`, " + + "MAX(${colName}) as `column_key`, " + "COUNT(1) AS `count` " - + "FROM cte1 WHERE `${colName}` IS NOT NULL " + + "FROM cte1 WHERE ${colName} IS NOT NULL " + "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT ${hotValueCollectCount}) t) " + "SELECT * FROM cte2 CROSS JOIN cte3"; @@ -177,9 +178,9 @@ public enum AnalyzeSampleAlgorithm { + "FROM " + "(SELECT " + "${subStringColName} AS `hash_value`, " - + "`${colName}` AS `col_value`, " - + "LENGTH(`${colName}`) as `len`${lengthAssert} " - + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index} ${sampleHints} ${limit}) as `t0` " + + "${colName} AS `col_value`, " + + "LENGTH(${colName}) as `len`${lengthAssert} " + + "FROM ${catalogName}.${dbName}.${tblName} ${index} ${sampleHints} ${limit}) as `t0` " + "${preAggHint} GROUP BY `t0`.`hash_value`), " + "cte2 AS ( " + "SELECT CONCAT('${tblId}', '-', '${idxId}', '-', '${colId}') AS `id`, " @@ -241,13 +242,13 @@ public enum AnalyzeSampleAlgorithm { + "${partId} AS `part_id`, " + "'${colId}' AS `col_id`, " + "COUNT(1) AS `row_count`, " - + "HLL_UNION(HLL_HASH(`${colName}`)) as ndv, " - + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, " - + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`, " - + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`, " + + "HLL_UNION(HLL_HASH(${colName})) as ndv, " + + "COUNT(1) - COUNT(${colName}) AS `null_count`, " + + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, " + + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, " + "${dataSizeFunction} AS `data_size`, " + "NOW() AS `update_time` " - + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index} ${partitionInfo}"; + + "FROM ${catalogName}.${dbName}.${tblName} ${index} ${partitionInfo}"; protected static final String MERGE_PARTITION_TEMPLATE = "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, " @@ -418,7 +419,7 @@ protected String getDataSizeFunction(Column column, boolean useDuj1) { } } else { if (column.getType().isStringType()) { - return "SUM(LENGTH(`${colName}`))"; + return "SUM(LENGTH(${colName}))"; } else { return "COUNT(1) * " + column.getType().getSlotSize(); } @@ -427,15 +428,15 @@ protected String getDataSizeFunction(Column column, boolean useDuj1) { protected String getStringTypeColName(Column column) { if (column.getType().isStringType()) { - return "xxhash_64(SUBSTRING(CAST(`${colName}` AS STRING), 1, 1024))"; + return "xxhash_64(SUBSTRING(CAST(${colName} AS STRING), 1, 1024))"; } else { - return "`${colName}`"; + return "${colName}"; } } protected String getMinFunction() { if (tableSample == null) { - return "CAST(MIN(`${colName}`) as ${type}) "; + return "CAST(MIN(${colName}) as ${type}) "; } else { // Min value is not accurate while sample, so set it to NULL to avoid optimizer generate bad plan. return "NULL"; @@ -456,7 +457,7 @@ protected String getNdvFunction(String totalRows) { // Max value is not accurate while sample, so set it to NULL to avoid optimizer generate bad plan. protected String getMaxFunction() { if (tableSample == null) { - return "CAST(MAX(`${colName}`) as ${type}) "; + return "CAST(MAX(${colName}) as ${type}) "; } else { return "NULL"; } @@ -562,7 +563,7 @@ protected void doPartitionTable() throws Exception { } } } - params.put("partName", "'" + StatisticsUtil.escapeColumnName(part) + "'"); + params.put("partName", StatisticsUtil.quote(StatisticsUtil.escapeSQL(part))); params.put("partitionInfo", getPartitionInfo(part)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); sqls.add(stringSubstitutor.replace(PARTITION_ANALYZE_TEMPLATE)); @@ -646,11 +647,11 @@ protected Map buildSqlParams() { protected void addLengthAssertParam(Map params) { long maxLen = org.apache.doris.common.Config.statistics_max_string_column_length; if (col != null && col.getType().isStringType() && maxLen > 0) { - String escapedColName = StatisticsUtil.escapeColumnName(String.valueOf(info.colName)); + String quotedColName = SqlUtils.getIdentSql(String.valueOf(info.colName)); // The StringSubstitutor used by callers already has ${colName} populated, - // so we inline the escaped column name directly here. + // so we inline the quoted column name directly here. params.put("lengthAssert", - ", assert_true(`" + escapedColName + "` IS NULL OR LENGTH(`" + escapedColName + "`) <= " + ", assert_true(" + quotedColName + " IS NULL OR LENGTH(" + quotedColName + ") <= " + maxLen + ", '" + ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER + "') AS `__lc`"); } else { params.put("lengthAssert", ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java index d9e4ce9237fce8..d21deaff687e98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/ExternalAnalysisTask.java @@ -20,9 +20,11 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.NotImplementedException; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.StatisticConstants; +import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.commons.text.StringSubstitutor; @@ -68,7 +70,7 @@ protected void doFull() throws Exception { if (shouldCollectHotValue()) { params.put("hotValueCollectCount", String.valueOf(SessionVariable.getHotValueCollectCount())); params.put("subStringColName", getStringTypeColName(col)); - params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)"); + params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)"); template = FULL_ANALYZE_TEMPLATE; } else { template = FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE; @@ -87,11 +89,11 @@ protected Map buildSqlParams() { params.put("dbId", String.valueOf(db.getId())); params.put("tblId", String.valueOf(tbl.getId())); params.put("idxId", "-1"); - params.put("colName", info.colName); - params.put("colId", info.colName); - params.put("catalogName", catalog.getName()); - params.put("dbName", db.getFullName()); - params.put("tblName", tbl.getName()); + params.put("colName", SqlUtils.getIdentSql(info.colName)); + params.put("colId", StatisticsUtil.escapeSQL(info.colName)); + params.put("catalogName", SqlUtils.getIdentSql(catalog.getName())); + params.put("dbName", SqlUtils.getIdentSql(db.getFullName())); + params.put("tblName", SqlUtils.getIdentSql(tbl.getName())); params.put("sampleHints", getSampleHint()); params.put("limit", ""); params.put("scaleFactor", "1"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java index ed843c05cc97ab..46cdc0d3225815 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/HistogramTask.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.statistics.StatisticConstants; import org.apache.doris.statistics.analysis.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.util.StatisticsUtil; @@ -44,10 +45,10 @@ public class HistogramTask extends BaseAnalysisTask { + " ${idxId} AS idx_id, " + " '${colId}' AS col_id, " + " ${sampleRate} AS sample_rate, " - + " HISTOGRAM(`${colName}`, ${maxBucketNum}) AS buckets, " + + " HISTOGRAM(${colName}, ${maxBucketNum}) AS buckets, " + " NOW() AS create_time " + "FROM " - + " `${dbName}`.`${tblName}`"; + + " ${dbName}.${tblName}"; public HistogramTask(AnalysisInfo info) { super(info); @@ -56,16 +57,16 @@ public HistogramTask(AnalysisInfo info) { @Override public void doExecute() throws Exception { Map params = new HashMap<>(); - params.put("internalDB", FeConstants.INTERNAL_DB_NAME); - params.put("histogramStatTbl", StatisticConstants.HISTOGRAM_TBL_NAME); + params.put("internalDB", SqlUtils.getIdentSql(FeConstants.INTERNAL_DB_NAME)); + params.put("histogramStatTbl", SqlUtils.getIdentSql(StatisticConstants.HISTOGRAM_TBL_NAME)); params.put("catalogId", String.valueOf(catalog.getId())); params.put("dbId", String.valueOf(db.getId())); params.put("tblId", String.valueOf(tbl.getId())); params.put("idxId", String.valueOf(info.indexId)); - params.put("colId", String.valueOf(info.colName)); - params.put("dbName", db.getFullName()); - params.put("tblName", tbl.getName()); - params.put("colName", String.valueOf(info.colName)); + params.put("colId", StatisticsUtil.escapeSQL(String.valueOf(info.colName))); + params.put("dbName", SqlUtils.getIdentSql(db.getFullName())); + params.put("tblName", SqlUtils.getIdentSql(tbl.getName())); + params.put("colName", SqlUtils.getIdentSql(String.valueOf(info.colName))); params.put("sampleRate", getSampleRateFunction()); params.put("maxBucketNum", String.valueOf(info.maxBucketNum)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java index 9644aadfb009bc..606a2424d1035c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/OlapAnalysisTask.java @@ -34,6 +34,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.qe.AutoCloseConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.StmtExecutor; @@ -67,9 +68,9 @@ public class OlapAnalysisTask extends BaseAnalysisTask { private static final String BASIC_STATS_TEMPLATE = "SELECT " - + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) as min, " - + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) as max " - + "FROM `${dbName}`.`${tblName}` ${index}"; + + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) as min, " + + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) as max " + + "FROM ${dbName}.${tblName} ${index}"; private boolean keyColumnSampleTooManyRows = false; private boolean partitionColumnSampleTooManyRows = false; @@ -365,12 +366,12 @@ protected void setSampleParamsByAlgorithm(Map params, long table params.put("dataSizeFunction", getDataSizeFunction(col, true)); params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE `col_value` IS NOT NULL)"); } else { - params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)"); + params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)"); // For single unique key, use count as ndv. if (isSingleUniqueKey()) { params.put("ndvFunction", String.valueOf(tableRowCount)); } else { - params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})"); + params.put("ndvFunction", "ROUND(NDV(${colName}) * ${scaleFactor})"); } } } @@ -387,7 +388,7 @@ protected void doFull() throws Exception { if (shouldCollectHotValue()) { params.put("hotValueCollectCount", String.valueOf(SessionVariable.getHotValueCollectCount())); params.put("subStringColName", getStringTypeColName(col)); - params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)"); + params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)"); runQuery(stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE)); } else { runQuery(stringSubstitutor.replace(FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE)); @@ -446,7 +447,7 @@ protected void deleteNotExistPartitionStats(AnalysisInfo jobInfo) throws DdlExce @Override protected String getPartitionInfo(String partitionName) { - return "partition " + partitionName; + return "partition " + SqlUtils.getIdentSql(partitionName); } @Override @@ -460,10 +461,10 @@ protected Map buildSqlParams() { params.put("idxId", String.valueOf(info.indexId)); params.put("colId", StatisticsUtil.escapeSQL(String.valueOf(info.colName))); params.put("dataSizeFunction", getDataSizeFunction(col, false)); - params.put("catalogName", catalog.getName()); - params.put("dbName", db.getFullName()); - params.put("colName", StatisticsUtil.escapeColumnName(String.valueOf(info.colName))); - params.put("tblName", String.valueOf(tbl.getName())); + params.put("catalogName", SqlUtils.getIdentSql(catalog.getName())); + params.put("dbName", SqlUtils.getIdentSql(db.getFullName())); + params.put("colName", SqlUtils.getIdentSql(String.valueOf(info.colName))); + params.put("tblName", SqlUtils.getIdentSql(String.valueOf(tbl.getName()))); params.put("index", getIndex()); params.put("preAggHint", ""); addLengthAssertParam(params); @@ -475,7 +476,7 @@ protected String getIndex() { return ""; } else { OlapTable olapTable = (OlapTable) this.tbl; - return "index `" + olapTable.getIndexNameById(info.indexId) + "`"; + return "index " + SqlUtils.getIdentSql(olapTable.getIndexNameById(info.indexId)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java index 6f55158189fc25..91690df1eb029a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/analysis/PluginDrivenSampleAnalysisTask.java @@ -89,9 +89,9 @@ protected void doSample() { if (distributionColumns.size() == 1 && distributionColumns.contains(col.getName().toLowerCase())) { bucketFlag = true; sb.append(LINEAR_ANALYZE_TEMPLATE); - params.put("ndvFunction", "ROUND(NDV(`${colName}`) * ${scaleFactor})"); + params.put("ndvFunction", "ROUND(NDV(${colName}) * ${scaleFactor})"); params.put("rowCount", "ROUND(COUNT(1) * ${scaleFactor})"); - params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)"); + params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)"); } else { sb.append(DUJ1_ANALYZE_TEMPLATE); params.put("subStringColName", getStringTypeColName(col)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java index c4c387e033e979..b6976a5d41fb6c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/OlapAnalysisTaskTest.java @@ -39,6 +39,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DebugPointUtil; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.SessionVariable; @@ -142,6 +143,7 @@ public void testKeyColumnUseLimitAndNot() { Mockito.when(catalogIf.getId()).thenReturn(10001L); Mockito.when(catalogIf.getName()).thenReturn("catalogName"); Mockito.when(databaseIf.getId()).thenReturn(20001L); + Mockito.when(databaseIf.getFullName()).thenReturn("dbName"); OlapAnalysisTask olapAnalysisTask = Mockito.spy(new OlapAnalysisTask()); Mockito.doReturn(new ResultRow(Lists.newArrayList("1", "2"))).when(olapAnalysisTask).collectMinMax(); @@ -153,7 +155,7 @@ public void testKeyColumnUseLimitAndNot() { Mockito.doReturn(true).when(olapAnalysisTask).useLinearAnalyzeTemplate(); Mockito.doAnswer(inv -> { String sql = inv.getArgument(0); - Assertions.assertEquals("WITH cte1 AS (SELECT `null` FROM `catalogName`.`${dbName}`.`null` " + Assertions.assertEquals("WITH cte1 AS (SELECT `null` FROM `catalogName`.`dbName`.`null` " + "${sampleHints} ${limit} ), cte2 AS (SELECT CONCAT(30001, '-', -1, '-', 'null') AS `id`, " + "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`, " + "'null' AS `col_id`, NULL AS `part_id`, ${rowCount} AS `row_count`, ${ndvFunction} as `ndv`, " @@ -185,7 +187,7 @@ public void testKeyColumnUseLimitAndNot() { String sql = inv.getArgument(0); Assertions.assertEquals("WITH cte1 AS (SELECT MAX(t0.`col_value`) as `col_value`, COUNT(1) as `count`," + " SUM(`len`) as `column_length` FROM (SELECT ${subStringColName} AS `hash_value`, " - + "`null` AS `col_value`, LENGTH(`null`) as `len` FROM `catalogName`.`${dbName}`.`null` " + + "`null` AS `col_value`, LENGTH(`null`) as `len` FROM `catalogName`.`dbName`.`null` " + "${sampleHints} ${limit}) as `t0` GROUP BY `t0`.`hash_value`), " + "cte2 AS ( SELECT CONCAT('30001', '-', '-1', '-', 'null') AS `id`, 10001 AS `catalog_id`, " + "20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`, 'null' AS `col_id`, NULL AS `part_id`, " @@ -320,9 +322,9 @@ public void testGetSampleParams() { new OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null)); Assertions.assertEquals("1", params.get("scaleFactor")); Assertions.assertEquals("", params.get("sampleHints")); - Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)", + Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)", params.get("rowCount2")); - Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})", params.get("ndvFunction")); + Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})", params.get("ndvFunction")); Assertions.assertNull(params.get("preAggHint")); Assertions.assertEquals("COUNT(1)", params.get("rowCount")); params.clear(); @@ -332,9 +334,9 @@ public void testGetSampleParams() { new OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR, Pair.of(Lists.newArrayList(1L, 2L), 100L))); Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints")); - Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}` IS NOT NULL)", + Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE ${colName} IS NOT NULL)", params.get("rowCount2")); - Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})", params.get("ndvFunction")); + Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})", params.get("ndvFunction")); params.clear(); // DUJ1 algorithm with sample tablets: rowCount2 and ndvFunction must reference the cte1 @@ -366,7 +368,7 @@ public void testGetSampleParams() { Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints")); Assertions.assertEquals("SUM(`t1`.`count`) * COUNT(`t1`.`col_value`) / (SUM(`t1`.`count`) - SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null, 1, 0)) + SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null, 1, 0)) * SUM(`t1`.`count`) / 1000)", params.get("ndvFunction")); Assertions.assertEquals("SUM(t1.count) * 4", params.get("dataSizeFunction")); - Assertions.assertEquals("`${colName}`", params.get("subStringColName")); + Assertions.assertEquals("${colName}", params.get("subStringColName")); Assertions.assertEquals("/*+PREAGGOPEN*/", params.get("preAggHint")); params.clear(); @@ -417,7 +419,7 @@ public void testGetSampleParams() { Pair.of(Lists.newArrayList(1L, 2L), 100L))); Assertions.assertEquals("10.0", params.get("scaleFactor")); Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints")); - Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})", params.get("ndvFunction")); + Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})", params.get("ndvFunction")); params.clear(); task = Mockito.spy(new OlapAnalysisTask()); @@ -774,6 +776,43 @@ public void testAddLengthAssertParamConfigDisabled() { } } + @Test + public void testBuildSqlParamsQuotesIdentifiers() { + CatalogIf catalog = Mockito.mock(CatalogIf.class); + DatabaseIf database = Mockito.mock(DatabaseIf.class); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(catalog.getId()).thenReturn(1L); + Mockito.when(catalog.getName()).thenReturn("cat`alog"); + Mockito.when(database.getId()).thenReturn(2L); + Mockito.when(database.getFullName()).thenReturn("db`name"); + Mockito.when(table.getId()).thenReturn(3L); + Mockito.when(table.getName()).thenReturn("table`name"); + Mockito.when(table.getIndexNameById(4L)).thenReturn("index`name"); + + OlapAnalysisTask task = new OlapAnalysisTask(); + task.catalog = catalog; + task.db = database; + task.tbl = table; + task.col = new Column("col`'name", PrimitiveType.INT); + task.info = new AnalysisInfoBuilder() + .setIndexId(4L) + .setColName("col`'name") + .build(); + + Map params = task.buildSqlParams(); + Assertions.assertEquals("`cat``alog`", params.get("catalogName")); + Assertions.assertEquals("`db``name`", params.get("dbName")); + Assertions.assertEquals("`table``name`", params.get("tblName")); + Assertions.assertEquals("`col``'name`", params.get("colName")); + Assertions.assertEquals("col`''name", params.get("colId")); + Assertions.assertEquals("index `index``name`", params.get("index")); + Assertions.assertEquals("partition `part``name`", task.getPartitionInfo("part`name")); + + String sql = new StringSubstitutor(params).replace(BaseAnalysisTask.FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE); + Assertions.assertTrue(sql.contains("FROM (SELECT `col``'name` FROM " + + "`cat``alog`.`db``name`.`table``name` index `index``name`) __lc_t"), sql); + } + @Test public void testFullAnalyzeTemplateRendersLengthAssert() { // Confirm the rendered FULL_ANALYZE_TEMPLATE wraps the base table in a subquery @@ -787,10 +826,10 @@ public void testFullAnalyzeTemplateRendersLengthAssert() { params.put("idxId", "3"); params.put("colId", "s"); params.put("dataSizeFunction", "100"); - params.put("catalogName", "internal"); - params.put("dbName", "db1"); - params.put("colName", "s"); - params.put("tblName", "tbl1"); + params.put("catalogName", SqlUtils.getIdentSql("internal")); + params.put("dbName", SqlUtils.getIdentSql("db1")); + params.put("colName", SqlUtils.getIdentSql("s")); + params.put("tblName", SqlUtils.getIdentSql("tbl1")); params.put("index", ""); params.put("lengthAssert", ", assert_true(`s` IS NULL OR LENGTH(`s`) <= 1024, '" @@ -814,10 +853,10 @@ public void testFullAnalyzeTemplateRendersWithoutLengthAssert() { params.put("idxId", "3"); params.put("colId", "id"); params.put("dataSizeFunction", "100"); - params.put("catalogName", "internal"); - params.put("dbName", "db1"); - params.put("colName", "id"); - params.put("tblName", "tbl1"); + params.put("catalogName", SqlUtils.getIdentSql("internal")); + params.put("dbName", SqlUtils.getIdentSql("db1")); + params.put("colName", SqlUtils.getIdentSql("id")); + params.put("tblName", SqlUtils.getIdentSql("tbl1")); params.put("index", ""); params.put("lengthAssert", ""); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); @@ -836,11 +875,11 @@ public void testFullAnalyzeTemplateSql() { params.put("tblId", "2"); params.put("idxId", "3"); params.put("colId", "col1"); - params.put("colName", "col1"); + params.put("colName", SqlUtils.getIdentSql("col1")); params.put("dataSizeFunction", "SUM(LENGTH(`col1`))"); - params.put("catalogName", "internal"); - params.put("dbName", "db1"); - params.put("tblName", "tbl1"); + params.put("catalogName", SqlUtils.getIdentSql("internal")); + params.put("dbName", SqlUtils.getIdentSql("db1")); + params.put("tblName", SqlUtils.getIdentSql("tbl1")); params.put("index", ""); params.put("hotValueCollectCount", "10"); params.put("subStringColName", "`col1`"); From 1491ba281ec4a55350f5c58f2770bb5817ed164e Mon Sep 17 00:00:00 2001 From: morrySnow Date: Thu, 10 Sep 2026 12:31:39 +0800 Subject: [PATCH 2/2] [fix](statistics) Update tests for quoted identifiers Issue Number: None Keep the statistics task test fixtures and template expectations consistent with pre-quoted identifier substitutions. --- .../org/apache/doris/statistics/analysis/AnalyzeTest.java | 3 +++ .../doris/statistics/analysis/BaseAnalysisTaskTest.java | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java index 4e9658d4f53a21..0b82a21bd0f973 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/AnalyzeTest.java @@ -91,6 +91,9 @@ public void testJobExecution() throws Exception { Database database = Mockito.mock(Database.class); OlapTable olapTable = Mockito.mock(OlapTable.class); + Mockito.when(catalog.getName()).thenReturn("internal"); + Mockito.when(database.getFullName()).thenReturn("analysis_job_test"); + Mockito.when(olapTable.getName()).thenReturn("t1"); Mockito.when(olapTable.getColumn(Mockito.anyString())) .thenReturn(new Column("col1", PrimitiveType.INT)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java index ad23fb6155b9e9..597c23da2d52b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/analysis/BaseAnalysisTaskTest.java @@ -39,7 +39,7 @@ public void testGetFunctions() { String dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column, true); Assertions.assertEquals("SUM(`column_length`)", dataSizeFunction); dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column, false); - Assertions.assertEquals("SUM(LENGTH(`${colName}`))", dataSizeFunction); + Assertions.assertEquals("SUM(LENGTH(${colName}))", dataSizeFunction); column = new Column("int_column", PrimitiveType.INT); dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column, false); @@ -48,14 +48,14 @@ public void testGetFunctions() { Assertions.assertEquals("SUM(t1.count) * 4", dataSizeFunction); String minFunction = olapAnalysisTask.getMinFunction(); - Assertions.assertEquals("CAST(MIN(`${colName}`) as ${type}) ", minFunction); + Assertions.assertEquals("CAST(MIN(${colName}) as ${type}) ", minFunction); olapAnalysisTask.tableSample = new TableSample(true, 20L); minFunction = olapAnalysisTask.getMinFunction(); Assertions.assertEquals("NULL", minFunction); olapAnalysisTask.tableSample = null; String maxFunction = olapAnalysisTask.getMaxFunction(); - Assertions.assertEquals("CAST(MAX(`${colName}`) as ${type}) ", maxFunction); + Assertions.assertEquals("CAST(MAX(${colName}) as ${type}) ", maxFunction); olapAnalysisTask.tableSample = new TableSample(true, 20L); maxFunction = olapAnalysisTask.getMaxFunction(); Assertions.assertEquals("NULL", maxFunction);