Skip to content

fix: MySQL comprehensions generated SQL that silently matched nothing #44

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
richardwooding merged 1 commit into from
Aug 22, 2026
Merged

fix: MySQL comprehensions generated SQL that silently matched nothing #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions ...egrationTest/java/com/spandigital/cel2sql/integration/AbstractDialectIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ protected List<IntegrationTestCase> testCatalog() {
// --- Arrays: native (skip MySQL/SQLite) ---
cases.add(IntegrationTestCase.expr("array_index_literal", "[1, 2, 3][0] == 1", "array_native"));

// --- Comprehensions (skip MySQL, skip SQLite) ---
// --- Comprehensions ---
cases.add(IntegrationTestCase.where("comp_all", "string_list.all(x, x != \"bad\")", "comprehension", 2, 4, 6));
cases.add(IntegrationTestCase.where("comp_exists", "string_list.exists(x, x == \"good\")", "comprehension", 1, 2, 4, 5));
cases.add(IntegrationTestCase.where("comp_exists_one", "string_list.exists_one(x, x == \"unique\")", "comprehension", 4, 5));
Expand Down Expand Up @@ -114,9 +114,6 @@ private void applyAssumptions(IntegrationTestCase tc) {
if ("array_native".equals(tc.category())) {
Assumptions.assumeTrue(dialect.supportsNativeArrays(), getDialectName() + ": no native array support");
}
if ("comprehension".equals(tc.category())) {
Assumptions.assumeTrue(dialect.name() != DialectName.MYSQL, "MySQL: comprehensions not supported");
}
}

// ===== Test factories =====
Expand Down
21 changes: 11 additions & 10 deletions src/main/java/com/spandigital/cel2sql/Converter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2208,11 +2208,12 @@ private void visitComprehensionAll(CelComprehension comp) throws ConversionExcep
String iterVar = comp.iterVar();
CelExpr predicate = extractComprehensionPredicate(comp.loopCondition(), comp.loopStep());

str.append("NOT EXISTS (SELECT 1 FROM ");
dialect.writeComprehensionSource(str, () -> visit(iterRange), iterVar);
str.append(" WHERE NOT (");
visit(predicate);
str.append("))");
dialect.writeComprehensionNotExists(str, () -> {
dialect.writeComprehensionSource(str, () -> visit(iterRange), iterVar);
str.append(" WHERE NOT (");
visit(predicate);
str.append(')');
});
}

/**
Expand All @@ -2224,11 +2225,11 @@ private void visitComprehensionExists(CelComprehension comp) throws ConversionEx
String iterVar = comp.iterVar();
CelExpr predicate = extractComprehensionPredicate(comp.loopCondition(), comp.loopStep());

str.append("EXISTS (SELECT 1 FROM ");
dialect.writeComprehensionSource(str, () -> visit(iterRange), iterVar);
str.append(" WHERE ");
visit(predicate);
str.append(')');
dialect.writeComprehensionExists(str, () -> {
dialect.writeComprehensionSource(str, () -> visit(iterRange), iterVar);
str.append(" WHERE ");
visit(predicate);
});
}

/**
Expand Down
19 changes: 19 additions & 0 deletions src/main/java/com/spandigital/cel2sql/dialect/Dialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,25 @@ default void writeComprehensionSource(StringBuilder w, SqlWriter writeSource, St
w.append(" AS ").append(iterVar);
}

/**
* Wraps a comprehension's existential subquery: {@code EXISTS (SELECT 1 FROM <body>)}.
* MySQL overrides this with a COUNT comparison because its 8.x optimizer turns a
* correlated EXISTS into a semijoin and loses the correlation to a JSON_TABLE source,
* silently matching nothing.
*/
default void writeComprehensionExists(StringBuilder w, SqlWriter writeBody) throws ConversionException {
w.append("EXISTS (SELECT 1 FROM ");
writeBody.write();
w.append(')');
}

/** Negation of {@link #writeComprehensionExists}: {@code NOT EXISTS (SELECT 1 FROM <body>)}. */
default void writeComprehensionNotExists(StringBuilder w, SqlWriter writeBody) throws ConversionException {
w.append("NOT EXISTS (SELECT 1 FROM ");
writeBody.write();
w.append(')');
}

/** Writes the prefix before the transform expression in an array-building subquery. */
void writeArraySubqueryOpen(StringBuilder w);

Expand Down
26 changes: 26 additions & 0 deletions src/main/java/com/spandigital/cel2sql/dialect/mysql/MySqlDialect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,32 @@ public void writeUnnest(StringBuilder w, SqlWriter writeSource) throws Conversio
w.append(", '$[*]' COLUMNS(value TEXT PATH '$'))");
}

@Override
public void writeComprehensionSource(StringBuilder w, SqlWriter writeSource, String iterVar) throws ConversionException {
// JSON_TABLE is table-valued, so the value column is renamed to iterVar
// through a derived table (JSON_TABLE itself requires an alias).
w.append("(SELECT value AS ").append(iterVar).append(" FROM ");
writeUnnest(w, writeSource);
w.append(" AS jt) AS _t");
}

@Override
public void writeComprehensionExists(StringBuilder w, SqlWriter writeBody) throws ConversionException {
// Not EXISTS: the MySQL 8.x optimizer turns a correlated EXISTS into a
// semijoin and loses the correlation to JSON_TABLE, silently matching
// nothing (works from 9.x). COUNT comparisons are never transformed.
w.append("(SELECT COUNT(*) FROM ");
writeBody.write();
w.append(") > 0");
}

@Override
public void writeComprehensionNotExists(StringBuilder w, SqlWriter writeBody) throws ConversionException {
w.append("(SELECT COUNT(*) FROM ");
writeBody.write();
w.append(") = 0");
}

@Override
public void writeArraySubqueryOpen(StringBuilder w) {
w.append("(SELECT JSON_ARRAYAGG(");
Expand Down
15 changes: 13 additions & 2 deletions src/test/java/com/spandigital/cel2sql/Cel2SqlComprehensionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.spandigital.cel2sql.dialect.Dialect;
import com.spandigital.cel2sql.dialect.bigquery.BigQueryDialect;
import com.spandigital.cel2sql.dialect.duckdb.DuckDbDialect;
import com.spandigital.cel2sql.dialect.mysql.MySqlDialect;
import com.spandigital.cel2sql.dialect.postgres.PostgresDialect;
import com.spandigital.cel2sql.dialect.sqlite.SqliteDialect;
import com.spandigital.cel2sql.testutil.CelHelper;
Expand All @@ -16,13 +17,13 @@

/**
* Comprehension tests covering all(), exists(), exists_one(), filter(), and map()
* macros on lists across PostgreSQL, SQLite, DuckDB, and BigQuery.
* MySQL is skipped (no comprehension support in the Go reference).
* macros on lists across PostgreSQL, MySQL, SQLite, DuckDB, and BigQuery.
* Mirrors the test cases from Go's testcases/comprehension_tests.go.
*/
class Cel2SqlComprehensionTest {

private static final Dialect PG = new PostgresDialect();
private static final Dialect MYSQL = new MySqlDialect();
private static final Dialect SQLITE = new SqliteDialect();
private static final Dialect DUCKDB = new DuckDbDialect();
private static final Dialect BQ = new BigQueryDialect();
Expand All @@ -32,6 +33,8 @@ static Stream<Arguments> comprehensionTests() {
// all: NOT EXISTS with UNNEST/json_each
Arguments.of("all", "string_list.all(x, x != \"bad\")", "PostgreSQL", PG,
"NOT EXISTS (SELECT 1 FROM UNNEST(string_list) AS x WHERE NOT (x != 'bad'))"),
Arguments.of("all", "string_list.all(x, x != \"bad\")", "MySQL", MYSQL,
"(SELECT COUNT(*) FROM (SELECT value AS x FROM JSON_TABLE(string_list, '$[*]' COLUMNS(value TEXT PATH '$')) AS jt) AS _t WHERE NOT (x != 'bad')) = 0"),
Arguments.of("all", "string_list.all(x, x != \"bad\")", "SQLite", SQLITE,
"NOT EXISTS (SELECT 1 FROM (SELECT value AS x FROM json_each(string_list)) AS _t WHERE NOT (x != 'bad'))"),
Arguments.of("all", "string_list.all(x, x != \"bad\")", "DuckDB", DUCKDB,
Expand All @@ -42,6 +45,8 @@ static Stream<Arguments> comprehensionTests() {
// exists: EXISTS with UNNEST/json_each
Arguments.of("exists", "string_list.exists(x, x == \"good\")", "PostgreSQL", PG,
"EXISTS (SELECT 1 FROM UNNEST(string_list) AS x WHERE x = 'good')"),
Arguments.of("exists", "string_list.exists(x, x == \"good\")", "MySQL", MYSQL,
"(SELECT COUNT(*) FROM (SELECT value AS x FROM JSON_TABLE(string_list, '$[*]' COLUMNS(value TEXT PATH '$')) AS jt) AS _t WHERE x = 'good') > 0"),
Arguments.of("exists", "string_list.exists(x, x == \"good\")", "SQLite", SQLITE,
"EXISTS (SELECT 1 FROM (SELECT value AS x FROM json_each(string_list)) AS _t WHERE x = 'good')"),
Arguments.of("exists", "string_list.exists(x, x == \"good\")", "DuckDB", DUCKDB,
Expand All @@ -52,6 +57,8 @@ static Stream<Arguments> comprehensionTests() {
// exists_one: COUNT subquery
Arguments.of("exists_one", "string_list.exists_one(x, x == \"unique\")", "PostgreSQL", PG,
"(SELECT COUNT(*) FROM UNNEST(string_list) AS x WHERE x = 'unique') = 1"),
Arguments.of("exists_one", "string_list.exists_one(x, x == \"unique\")", "MySQL", MYSQL,
"(SELECT COUNT(*) FROM (SELECT value AS x FROM JSON_TABLE(string_list, '$[*]' COLUMNS(value TEXT PATH '$')) AS jt) AS _t WHERE x = 'unique') = 1"),
Arguments.of("exists_one", "string_list.exists_one(x, x == \"unique\")", "SQLite", SQLITE,
"(SELECT COUNT(*) FROM (SELECT value AS x FROM json_each(string_list)) AS _t WHERE x = 'unique') = 1"),
Arguments.of("exists_one", "string_list.exists_one(x, x == \"unique\")", "DuckDB", DUCKDB,
Expand All @@ -62,6 +69,8 @@ static Stream<Arguments> comprehensionTests() {
// filter: ARRAY subquery / json_group_array
Arguments.of("filter", "string_list.filter(x, x != \"bad\")", "PostgreSQL", PG,
"ARRAY(SELECT x FROM UNNEST(string_list) AS x WHERE x != 'bad')"),
Arguments.of("filter", "string_list.filter(x, x != \"bad\")", "MySQL", MYSQL,
"(SELECT JSON_ARRAYAGG(x) FROM (SELECT value AS x FROM JSON_TABLE(string_list, '$[*]' COLUMNS(value TEXT PATH '$')) AS jt) AS _t WHERE x != 'bad')"),
Arguments.of("filter", "string_list.filter(x, x != \"bad\")", "SQLite", SQLITE,
"(SELECT json_group_array(x) FROM (SELECT value AS x FROM json_each(string_list)) AS _t WHERE x != 'bad')"),
Arguments.of("filter", "string_list.filter(x, x != \"bad\")", "DuckDB", DUCKDB,
Expand All @@ -72,6 +81,8 @@ static Stream<Arguments> comprehensionTests() {
// map_transform: ARRAY subquery with transform / json_group_array
Arguments.of("map_transform", "string_list.map(x, x + \"_suffix\")", "PostgreSQL", PG,
"ARRAY(SELECT x || '_suffix' FROM UNNEST(string_list) AS x)"),
Arguments.of("map_transform", "string_list.map(x, x + \"_suffix\")", "MySQL", MYSQL,
"(SELECT JSON_ARRAYAGG(CONCAT(x, '_suffix')) FROM (SELECT value AS x FROM JSON_TABLE(string_list, '$[*]' COLUMNS(value TEXT PATH '$')) AS jt) AS _t)"),
Arguments.of("map_transform", "string_list.map(x, x + \"_suffix\")", "SQLite", SQLITE,
"(SELECT json_group_array(x || '_suffix') FROM (SELECT value AS x FROM json_each(string_list)) AS _t)"),
Arguments.of("map_transform", "string_list.map(x, x + \"_suffix\")", "DuckDB", DUCKDB,
Expand Down
Loading