Skip to content

Commit 66f2cb4

Browse files
TeddyCrTeddyCr
authored
fix: add lowcardinality support (#24921)
1 parent 8c855c6 commit 66f2cb4

10 files changed

Lines changed: 137 additions & 8 deletions

‎bootstrap/sql/migrations/native/1.11.4/mysql/postDataMigrationSQLScript.sql‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
-- Add LOWCARDINALITY to supportedDataTypes for test definitions that already support STRING
2+
UPDATE test_definition
3+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
4+
WHERE name = 'columnValueLengthsToBeBetween'
5+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
6+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
7+
8+
UPDATE test_definition
9+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
10+
WHERE name = 'columnValuesMissingCount'
11+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
12+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
13+
14+
UPDATE test_definition
15+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
16+
WHERE name = 'columnValuesToBeInSet'
17+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
18+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
19+
20+
UPDATE test_definition
21+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
22+
WHERE name = 'columnValuesToBeNotInSet'
23+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
24+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
25+
26+
UPDATE test_definition
27+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
28+
WHERE name = 'columnValuesToBeNotNull'
29+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
30+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
31+
32+
UPDATE test_definition
33+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
34+
WHERE name = 'columnValuesToBeUnique'
35+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
36+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
37+
38+
UPDATE test_definition
39+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
40+
WHERE name = 'columnValuesToMatchRegex'
41+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
42+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');
43+
44+
UPDATE test_definition
45+
SET json = JSON_ARRAY_APPEND(json, '$.supportedDataTypes', 'LOWCARDINALITY')
46+
WHERE name = 'columnValuesToNotMatchRegex'
47+
AND JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"STRING"')
48+
AND NOT JSON_CONTAINS(JSON_EXTRACT(json, '$.supportedDataTypes'), '"LOWCARDINALITY"');

‎bootstrap/sql/migrations/native/1.11.4/postgres/postDataMigrationSQLScript.sql‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
-- Add LOWCARDINALITY to supportedDataTypes for test definitions that already support STRING
2+
UPDATE test_definition
3+
SET json = jsonb_set(
4+
json::jsonb,
5+
'{supportedDataTypes}',
6+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
7+
)
8+
WHERE name = 'columnValueLengthsToBeBetween'
9+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
10+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
11+
12+
UPDATE test_definition
13+
SET json = jsonb_set(
14+
json::jsonb,
15+
'{supportedDataTypes}',
16+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
17+
)
18+
WHERE name = 'columnValuesMissingCount'
19+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
20+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
21+
22+
UPDATE test_definition
23+
SET json = jsonb_set(
24+
json::jsonb,
25+
'{supportedDataTypes}',
26+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
27+
)
28+
WHERE name = 'columnValuesToBeInSet'
29+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
30+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
31+
32+
UPDATE test_definition
33+
SET json = jsonb_set(
34+
json::jsonb,
35+
'{supportedDataTypes}',
36+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
37+
)
38+
WHERE name = 'columnValuesToBeNotInSet'
39+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
40+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
41+
42+
UPDATE test_definition
43+
SET json = jsonb_set(
44+
json::jsonb,
45+
'{supportedDataTypes}',
46+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
47+
)
48+
WHERE name = 'columnValuesToBeNotNull'
49+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
50+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
51+
52+
UPDATE test_definition
53+
SET json = jsonb_set(
54+
json::jsonb,
55+
'{supportedDataTypes}',
56+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
57+
)
58+
WHERE name = 'columnValuesToBeUnique'
59+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
60+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
61+
62+
UPDATE test_definition
63+
SET json = jsonb_set(
64+
json::jsonb,
65+
'{supportedDataTypes}',
66+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
67+
)
68+
WHERE name = 'columnValuesToMatchRegex'
69+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
70+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;
71+
72+
UPDATE test_definition
73+
SET json = jsonb_set(
74+
json::jsonb,
75+
'{supportedDataTypes}',
76+
(json->'supportedDataTypes')::jsonb || '["LOWCARDINALITY"]'::jsonb
77+
)
78+
WHERE name = 'columnValuesToNotMatchRegex'
79+
AND json->'supportedDataTypes' @> '"STRING"'::jsonb
80+
AND NOT json->'supportedDataTypes' @> '"LOWCARDINALITY"'::jsonb;

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"description": "This schema defines the test ColumnValueLengthsToBeBetween. Test the value lengths in a column to be between minimum and maximum value. ",
66
"entityType": "COLUMN",
77
"testPlatforms": ["OpenMetadata"],
8-
"supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "ARRAY"],
8+
"supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "ARRAY", "LOWCARDINALITY"],
99
"parameterDefinition": [
1010
{
1111
"name": "minLength",

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"description": "This schema defines the test ColumnValuesMissingCount. Test the column values missing count to be equal to given number. ",
66
"entityType": "COLUMN",
77
"testPlatforms": ["OpenMetadata"],
8-
"supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
8+
"supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON","LOWCARDINALITY"],
99
"parameterDefinition": [
1010
{
1111
"name": "missingCountValue",

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"description": "This schema defines the test ColumnValuesToBeInSet. Test the column values are in the set.",
66
"entityType": "COLUMN",
77
"testPlatforms": ["OpenMetadata"],
8-
"supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN"],
8+
"supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN", "LOWCARDINALITY"],
99
"parameterDefinition": [
1010
{
1111
"name": "allowedValues",

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"description": "This schema defines the test ColumnValuesToBeNotInSet. Test the column values to not be in the set. ",
66
"entityType": "COLUMN",
77
"testPlatforms": ["OpenMetadata"],
8-
"supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN"],
8+
"supportedDataTypes": ["NUMBER", "INT", "FLOAT", "DOUBLE", "DECIMAL", "TINYINT", "SMALLINT", "BIGINT", "BYTEINT", "BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "BOOLEAN", "LOWCARDINALITY"],
99
"parameterDefinition": [
1010
{
1111
"name": "forbiddenValues",

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"description": "This schema defines the test ColumnValuesToBeNotNull. Test the number of values in a column are null. Values must be explicitly null. Empty strings don't count as null. ",
66
"entityType": "COLUMN",
77
"testPlatforms": ["OpenMetadata"],
8-
"supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
8+
"supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON","LOWCARDINALITY"],
99
"supportsRowLevelPassedFailed": true,
1010
"provider": "system",
1111
"dataQualityDimension": "Completeness"

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@
4646
"VARIANT",
4747
"GEOMETRY",
4848
"POINT",
49-
"POLYGON"
49+
"POLYGON",
50+
"LOWCARDINALITY"
5051
],
5152
"supportsRowLevelPassedFailed": true,
5253
"provider": "system",

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"description": "This schema defines the test ColumnValuesToMatchRegex. Test the values in a column to match a given regular expression. ",
66
"entityType": "COLUMN",
77
"testPlatforms": ["OpenMetadata"],
8-
"supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"],
8+
"supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "LOWCARDINALITY"],
99
"parameterDefinition": [
1010
{
1111
"name": "regex",

‎openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"description": "This schema defines the test ColumnValuesToNotMatchRegex. Test the values in a column to not match a given regular expression. ",
66
"entityType": "COLUMN",
77
"testPlatforms": ["OpenMetadata"],
8-
"supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR"],
8+
"supportedDataTypes": ["BYTES", "STRING", "MEDIUMTEXT", "TEXT", "CHAR", "VARCHAR", "LOWCARDINALITY"],
99
"parameterDefinition": [
1010
{
1111
"name": "forbiddenRegex",

0 commit comments

Comments
 (0)