From 7defdb4c856a77fb09371084d747739bd0674c90 Mon Sep 17 00:00:00 2001
From: PranavKTiwari <pranav.tiwari@mariadb.com>
Date: Fri, 19 Jun 2026 15:32:17 +0530
Subject: [PATCH] MDEV-36990: SIGFPE in
 get_max_range_rowid_filter_elems_for_table
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Problem:
ALTER TABLE allowing a column defined as BINARY(0) to be used as part of a PRIMARY KEY or UNIQUE KEY can lead to a zero-length index column. This results in tab->file->ref_length becoming 0 inside the storage engine, which later causes a division-by-zero crash in get_max_range_rowid_filter_elems_for_table() during optimizer cost estimation for range queries.

Cause:
During key initialization in init_key_part_spec(), validation relied on column->flags & NOT_NULL_FLAG to detect non-nullable key columns. However, in ALTER TABLE ... CHANGE COLUMN, the Create_field is only partially initialized and does not yet propagate implicit constraints such as PRIMARY KEY ⇒ NOT NULL. As a result, NOT_NULL_FLAG may be 0 even for primary key columns, allowing key_part_length == 0 cases to bypass validation.

This leads to invalid index definitions where a primary key column contributes zero bytes to the index.

Fix:
Strengthen key validation in init_key_part_spec() by rejecting zero-length
key parts for columns that are effectively non-nullable in key context,
including PRIMARY KEY columns whose implicit NOT NULL property may not yet
be reflected in Create_field::flags during ALTER TABLE processing.

This prevents creation of invalid zero-length index definitions and avoids
propagation of ref_length=0 metadata that can later trigger optimizer
crashes.
---
 mysql-test/main/create_select.result | 13 +++++++++++++
 mysql-test/main/create_select.test   | 13 +++++++++++++
 sql/sql_table.cc                     |  4 ++--
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/mysql-test/main/create_select.result b/mysql-test/main/create_select.result
index cbb276b61d31b..b8c1f7c9096eb 100644
--- a/mysql-test/main/create_select.result
+++ b/mysql-test/main/create_select.result
@@ -75,3 +75,16 @@ t2	CREATE TABLE `t2` (
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1, t2;
 # End of 12.1 tests
+#
+# MDEV-36990: SIGFPE in get_max_range_rowid_filter_elems_for_table
+#
+DROP TABLE IF EXISTS t;
+Warnings:
+Note	1051	Unknown table 'test.t'
+CREATE TABLE t (a INT,b INT,PRIMARY KEY(a),INDEX (b))Engine=InnoDB;
+ALTER TABLE t CHANGE COLUMN a a BINARY (0);
+ERROR 42000: The storage engine InnoDB can't index column `a`
+SELECT * FROM t WHERE b>-1;
+a	b
+DROP TABLE IF EXISTS t;
+# End of 10.11 tests
diff --git a/mysql-test/main/create_select.test b/mysql-test/main/create_select.test
index c464744390bd4..4d9e99983bf4c 100644
--- a/mysql-test/main/create_select.test
+++ b/mysql-test/main/create_select.test
@@ -79,3 +79,16 @@ show create table t2;
 drop table t1, t2;
 
 --echo # End of 12.1 tests
+
+--echo #
+--echo # MDEV-36990: SIGFPE in get_max_range_rowid_filter_elems_for_table
+--echo #
+
+DROP TABLE IF EXISTS t;
+CREATE TABLE t (a INT,b INT,PRIMARY KEY(a),INDEX (b))Engine=InnoDB;
+--error ER_WRONG_KEY_COLUMN
+ALTER TABLE t CHANGE COLUMN a a BINARY (0);
+SELECT * FROM t WHERE b>-1;
+
+DROP TABLE IF EXISTS t;
+--echo # End of 10.11 tests
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index e86ebba561288..05e3fd626bc77 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -2963,8 +2963,8 @@ my_bool init_key_part_spec(THD *thd, Alter_info *alter_info,
     else if (!(file->ha_table_flags() & HA_NO_PREFIX_CHAR_KEYS))
       key_part_length= kp.length;
   }
-  else if (key_part_length == 0 && (column->flags & NOT_NULL_FLAG) &&
-           !*is_hash_field_needed)
+  else if (key_part_length == 0 && ((column->flags & NOT_NULL_FLAG) ||
+    key.type == Key::PRIMARY) && !*is_hash_field_needed)
   {
     my_error(ER_WRONG_KEY_COLUMN, MYF(0), file->table_type(), field_name.str);
     DBUG_RETURN(TRUE);