From 031a31012478152481451ec3d1377280bed4caad Mon Sep 17 00:00:00 2001 From: Evgen Bielozorov Date: Tue, 15 Sep 2026 07:52:00 +0200 Subject: [PATCH] fix(q): convert minute/second/month units on decode; null out-of-range datetime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decoder copied every 4-byte q temporal raw and re-tagged it as the rayforce type of the same width, which is only correct for date (KD) and time (KT). minute (KU) counts minutes and second (KV) counts seconds since midnight, but both landed in TIME (i32 milliseconds) unscaled — 09:30 came back as 00:00:00.570 and 09:30:15 as 00:00:34.215. month (KM) counts months since 2000.01 but landed in DATE (i32 days) — 2024.02m came back as 2000.10.16. Every result carried the correct type code, so nothing downstream could tell the values were wrong. Decode KU and KV by scaling to ms, and KM to the DATE of the month's first day (what q's `date$ does with a month). Values the target type cannot hold — the q infinities 0W/-0W, a minute/second count past the i32 ms range — decode to the typed null, and the null sentinels map to the null as before. The datetime (KZ) conversion had the mirror problem at the edges: it fed any double through llround and an i64 multiply, so a datetime past the TIMESTAMP nanosecond range (or 0Wz) overflowed into a plausible-looking timestamp. Range-check the day count and decode those to the null timestamp. timespan (KN) is unchanged and still decodes to a TIMESTAMP anchored at 2000.01.01: rayforce has no duration type, and that mapping is what kdb-tick style `time` columns rely on today. Tests: 05_temporal.rfl covers atoms, vectors, nulls, infinities and the null gate on aggregates against a real q; the codec selftest checks the raw wire bytes so the conversion is also verified where no q is available. --- q.c | 112 ++++++++++++++++++++++++++++++-- test/driver.c | 44 +++++++++++++ test/rfl/client/05_temporal.rfl | 39 +++++++++++ 3 files changed, 190 insertions(+), 5 deletions(-) diff --git a/q.c b/q.c index 0669269..5ea3a48 100644 --- a/q.c +++ b/q.c @@ -821,10 +821,99 @@ static ray_t *q_des_vec_i(uint8_t **buf, int64_t *len, int8_t ray_type, return vec; } +/* q temporal payloads that are NOT byte-compatible with a rayforce type and + * need a unit conversion instead of a raw copy + re-tag: + * + * KZ datetime double days since 2000.01.01 -> TIMESTAMP (i64 ns) + * KU minute i32 minutes since midnight -> TIME (i32 ms) + * KV second i32 seconds since midnight -> TIME (i32 ms) + * KM month i32 months since 2000.01 -> DATE (i32 days, 1st of month) + * + * A value the target cannot represent (q infinities 0W/-0W, a datetime past + * the i64 nanosecond range, a minute/second count that overflows i32 ms) + * decodes to the typed null rather than wrapping into a plausible-looking + * wrong value; the null sentinel itself (0Nz/0Nu/0Nv/0Nm) maps to the null. */ + +/* |ms| must stay below INT64_MAX / 1e6 for the ns product to fit. */ +#define Q_KZ_MAX_MS 9223372036854.0 + static inline int64_t q_kz_days_to_nanos(double days) { if (isnan(days)) return NULL_I64; - return (int64_t)llround(days * 86400000.0) * 1000000LL; + double ms = days * 86400000.0; + if (!(ms > -Q_KZ_MAX_MS && ms < Q_KZ_MAX_MS)) + return NULL_I64; /* +-0Wz, or out of TIMESTAMP range */ + return (int64_t)llround(ms) * 1000000LL; +} + +/* minute/second -> ms. `scale` is 60000 (KU) or 1000 (KV). */ +static int32_t q_i32_scaled_to_ms(int32_t v, int32_t scale) { + if (v == NULL_I32) + return NULL_I32; + if (v > INT32_MAX / scale || v < -(INT32_MAX / scale)) + return NULL_I32; /* +-0Wu/0Wv, or past the i32 ms range */ + return v * scale; +} + +/* Days since 1970-01-01 for the first day of (y, m), m in 1..12. + * Howard Hinnant's days_from_civil, integral in the proleptic Gregorian + * calendar for any year. */ +static inline int64_t q_days_from_civil(int64_t y, int m) { + y -= m <= 2; + int64_t era = (y >= 0 ? y : y - 399) / 400; + int64_t yoe = y - era * 400; /* [0, 399] */ + int64_t doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5; /* day 1 of month */ + int64_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; /* [0, 146096] */ + return era * 146097 + doe - 719468; +} + +/* months since 2000.01 -> days since 2000.01.01 of that month's first day, + * i.e. what q's `date$ does with a month (2024.02m -> 2024.02.01). The + * second argument only exists to share q_des_vec_i32_conv's callback shape. */ +static int32_t q_km_months_to_days(int32_t months, int32_t unused) { + (void)unused; + if (months == NULL_I32) + return NULL_I32; + if (months == INT32_MAX || months == -INT32_MAX) + return NULL_I32; /* +-0Wm */ + int64_t m0 = months; + int64_t y = 2000 + (m0 >= 0 ? m0 / 12 : -((-m0 + 11) / 12)); + int mon = (int)(m0 - (y - 2000) * 12) + 1; /* 1..12 */ + int64_t days = q_days_from_civil(y, mon) - q_days_from_civil(2000, 1); + if (days < INT32_MIN + 1 || days > INT32_MAX) + return NULL_I32; + return (int32_t)days; +} + +/* Decode an i32 vector through a per-element conversion (KU/KV/KM). */ +static ray_t *q_des_vec_i32_conv(uint8_t **buf, int64_t *len, int8_t ray_type, + int32_t (*conv)(int32_t, int32_t), + int32_t arg) { + int32_t n; + if (q_read_vec_header(buf, len, &n) < 0) + return ray_error("q: buffer underflow", NULL); + if (n < 0) + return ray_error("q: negative vector length", NULL); + int64_t bytes = (int64_t)n * 4; + if (*len < bytes) + return ray_error("q: buffer underflow (vec body)", NULL); + ray_t *vec = ray_vec_new(ray_type, n); + if (vec == NULL || RAY_IS_ERR(vec)) { + if (vec) + ray_release(vec); + return ray_error("q: vector alloc failed", NULL); + } + int32_t *out = (int32_t *)ray_data(vec); + for (int32_t i = 0; i < n; i++) { + int32_t v; + memcpy(&v, *buf + (int64_t)i * 4, 4); + out[i] = conv(v, arg); + } + vec->len = n; + *buf += bytes; + *len -= bytes; + q_flag_nulls(vec); + return vec; } static ray_t *q_des_obj(uint8_t **buf, int64_t *len) { @@ -851,12 +940,22 @@ static ray_t *q_des_obj(uint8_t **buf, int64_t *len) { case -Q_KN: return q_des_atom_i(buf, len, RAY_TIMESTAMP, 8); case -Q_KD: - case -Q_KM: return q_des_atom_i(buf, len, RAY_DATE, 4); case -Q_KT: + return q_des_atom_i(buf, len, RAY_TIME, 4); case -Q_KU: case -Q_KV: - return q_des_atom_i(buf, len, RAY_TIME, 4); + case -Q_KM: { + /* Same width as the target, different unit — convert, don't re-tag. */ + Q_NEED(4); + int32_t v; + memcpy(&v, *buf, 4); + *buf += 4; + *len -= 4; + if (type == -Q_KM) + return ray_date(q_km_months_to_days(v, 0)); + return ray_time(q_i32_scaled_to_ms(v, type == -Q_KU ? 60000 : 1000)); + } case -Q_KZ: { Q_NEED(8); double d; @@ -922,12 +1021,15 @@ static ray_t *q_des_obj(uint8_t **buf, int64_t *len) { case Q_KN: return q_des_vec_i(buf, len, RAY_TIMESTAMP, 8); case Q_KD: - case Q_KM: return q_des_vec_i(buf, len, RAY_DATE, 4); case Q_KT: + return q_des_vec_i(buf, len, RAY_TIME, 4); case Q_KU: + return q_des_vec_i32_conv(buf, len, RAY_TIME, q_i32_scaled_to_ms, 60000); case Q_KV: - return q_des_vec_i(buf, len, RAY_TIME, 4); + return q_des_vec_i32_conv(buf, len, RAY_TIME, q_i32_scaled_to_ms, 1000); + case Q_KM: + return q_des_vec_i32_conv(buf, len, RAY_DATE, q_km_months_to_days, 0); case Q_KZ: { int32_t n; if (q_read_vec_header(buf, len, &n) < 0) diff --git a/test/driver.c b/test/driver.c index 8bdf204..7f02c70 100644 --- a/test/driver.c +++ b/test/driver.c @@ -338,6 +338,50 @@ static int run_codec_selftest(void) { failures++; } + /* Unit-converted temporals, raw wire bytes so no q is needed: the q + * type byte, then the little-endian payload. */ + struct { + const char *name; + uint8_t body[16]; + int64_t body_len; + int8_t want_type; + int64_t want_val; /* i32 sentinel-compared through the atom union */ + } temporal[] = { + /* -17 minute 570 -> 09:30 = 34200000 ms */ + {"minute atom", {0xEF, 0x3A, 0x02, 0x00, 0x00}, 5, -RAY_TIME, 34200000}, + /* -18 second 34215 -> 09:30:15 = 34215000 ms */ + {"second atom", {0xEE, 0xA7, 0x85, 0x00, 0x00}, 5, -RAY_TIME, 34215000}, + /* -17 minute 0Wu (INT32_MAX) has no i32-ms form -> null time */ + {"minute inf", {0xEF, 0xFF, 0xFF, 0xFF, 0x7F}, 5, -RAY_TIME, NULL_I32}, + /* -13 month 289 (2024.02m) -> 2024.02.01 = 8797 days since 2000 */ + {"month atom", {0xF3, 0x21, 0x01, 0x00, 0x00}, 5, -RAY_DATE, 8797}, + /* -13 month -1 (1999.12m) -> 1999.12.01 = -31 days */ + {"month neg", {0xF3, 0xFF, 0xFF, 0xFF, 0xFF}, 5, -RAY_DATE, -31}, + /* -15 datetime 1e6 days (~2737 years): past the ns range -> null */ + {"datetime range", + {0xF1, 0x00, 0x00, 0x00, 0x00, 0x80, 0x84, 0x2E, 0x41}, + 9, + -RAY_TIMESTAMP, + NULL_I64}, + }; + for (size_t i = 0; i < sizeof temporal / sizeof temporal[0]; i++) { + err[0] = '\0'; + r = q_decode(temporal[i].body, temporal[i].body_len, 0, err, sizeof err); + int64_t got = 0; + if (r != NULL && !RAY_IS_ERR(r)) + got = (r->type == -RAY_TIMESTAMP) ? r->i64 : (int64_t)r->i32; + if (r == NULL || RAY_IS_ERR(r) || r->type != temporal[i].want_type || + got != temporal[i].want_val) { + fprintf(stderr, + "codec selftest: %s decoded wrong (type %d val %lld, want " + "type %d val %lld) %s\n", + temporal[i].name, r ? (int)r->type : 0, (long long)got, + (int)temporal[i].want_type, (long long)temporal[i].want_val, err); + failures++; + } + release_any(r); + } + ray_poll_t *poll = ray_poll_create(); if (poll == NULL) { fprintf(stderr, "codec selftest: failed to create poll\n"); diff --git a/test/rfl/client/05_temporal.rfl b/test/rfl/client/05_temporal.rfl index 4721aaf..13dc5c2 100644 --- a/test/rfl/client/05_temporal.rfl +++ b/test/rfl/client/05_temporal.rfl @@ -18,4 +18,43 @@ (.q.send h "2025.01.01T12:00:00.000") -- 2025.01.01D12:00:00.000000000 (.q.send h "2026.07.10T09:30:00.000 2025.01.01T00:00:00.000") -- [2026.07.10D09:30:00.000000000 2025.01.01D00:00:00.000000000] +;; A datetime the i64-nanosecond TIMESTAMP cannot hold (here ~2737 years +;; out) and the q infinities decode to the null timestamp, not to an +;; overflowed but plausible-looking value. +(nil? (.q.send h "\"z\"$1e6")) -- true +(nil? (.q.send h "0Wz")) -- true +(nil? (.q.send h "-0Wz")) -- true +(.q.send h "(2025.01.01T12:00:00.000;0Wz)") -- [2025.01.01D12:00:00.000000000 0Np] + +;; minute (KU) and second (KV) share TIME's i32 width but count minutes / +;; seconds since midnight, not milliseconds — they must be scaled, not +;; re-tagged (09:30 used to come back as 00:00:00.570). +(.q.send h "09:30") -- 09:30:00.000 +(.q.send h "09:30 10:45") -- [09:30:00.000 10:45:00.000] +(.q.send h "09:30:15") -- 09:30:15.000 +(.q.send h "09:30:15 23:59:59") -- [09:30:15.000 23:59:59.000] +(.q.send h "-09:30") -- -09:30:00.000 +(nil? (.q.send h "0Nu")) -- true +(nil? (.q.send h "0Nv")) -- true +(nil? (at (.q.send h "`minute$570 0N 645") 1)) -- true +;; the infinities have no i32-ms representation -> null +(nil? (.q.send h "0Wu")) -- true +(nil? (.q.send h "-0Wv")) -- true +(nil? (at (.q.send h "09:30:15 0Wv") 1)) -- true +;; decoded vectors carry the null gate, so aggregates skip the nulls +(min (.q.send h "`minute$645 0N 570")) -- 09:30:00.000 +(max (.q.send h "`second$34215 0W 34216")) -- 09:30:16.000 + +;; month (KM) counts months since 2000.01; it decodes to the DATE of the +;; month's first day, as q's `date$ does (2024.02m used to be 2000.10.16). +(.q.send h "2024.02m") -- 2024.02.01 +(.q.send h "2024.02 2024.03m") -- [2024.02.01 2024.03.01] +(.q.send h "2000.01m") -- 2000.01.01 +(.q.send h "1999.12m") -- 1999.12.01 +(.q.send h "1999.01 1998.12 2100.02m") -- [1999.01.01 1998.12.01 2100.02.01] +(.q.send h "`date$2024.02m") -- 2024.02.01 +(nil? (.q.send h "0Nm")) -- true +(nil? (.q.send h "0Wm")) -- true +(nil? (at (.q.send h "`month$289 0N 290") 1)) -- true + (.q.close h)