From 058d6079ec598f0861c599b11be39f73f3f47ae8 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Thu, 1 Oct 2026 12:48:56 +0200 Subject: [PATCH] State the codec each class reads and writes its values with typeEncodings states, for each class, the public functions a binding reads and writes a value with and the trailing inputs each takes, read once the SQL signatures and the bound literals are in the catalog (parser/codecs.py, after the bound literals resolve). A class several SQL types share reads and writes each type through that type's own public function: readers and writers map the SQL type a function's signature returns or takes to the function, Set {text: {intset: intset_in, floatset: floatset_in, ...}}, and that encoding has no single decoder or encoder. Within one SQL type the function the encoding table names first wins (cbuffer_out before cbuffer_as_ewkt), then the one serving fewer SQL types (cbufferset_out before spatialset_out); two that tie stop the catalog. An encoding whose functions carry no SQL signature keeps the function the C shapes give it, as GSERIALIZED does, its geometry and geography being PostGIS's types. The hex-WKB writer is a class's wkb encoder, its size an out-parameter of the function's shape.outParams. Its variant is the value the type's own send binds, WKB_EXTENDED or 0, else the value its SQL hex writer passes when the byte order is left out: asHexWKB(raster, endian DEFAULT '') passes 0, so Raster writes through raster_as_hexwkb. Every decoder, encoder, reader and writer states its trailing inputs by name with the value a binding passes, in decoderAux, encoderAux, readerAux and writerAux, and the byte codec its variant in bytes.encoderAux: a binding fills each parameter by name and refuses one the catalog does not fill. A slot whose header names a type of its own over a width, a typedef neither PostgreSQL nor the C standard defines whose chain reaches a C standard integer, keeps that name as typedef beside canonical, which states the width: {canonical: uint64_t, typedef: H3Index}. H3Index, Quadbin and S2CellId are classes of their own, read and written by their own functions. Why. A binding read every set from text through bigintset_in and wrote every temporal value through the internal temporal_out: the per-class pick was right for one SQL type and wrong for the others, and the generic functions that would serve all of them are internal. A cell read as a bare uint64_t accepts values no cell is. Measured. Over MobilityDB 72abdf86c2: Set reads and writes 18 SQL types, Span and SpanSet 5 each, Temporal writes 20 and reads 17 from text and 5 from MF-JSON; the classes stating a hex-WKB writer go from 2 to 19; H3Index, Quadbin and S2CellId join the 27 classes; 229 slots carry a typedef, every one a cell id. GSERIALIZED, TimeTzADT, NumericData, TInstant, TSequence and TSequenceSet keep the codec they state, and beside typeEncodings, struct serialization and typedef the catalog is byte for byte the one derived without this change. tpose_in, tposechain_in, trgeometry_in and 15 of the 20 *_from_mfjson readers carry no SQL signature, so no SQL type keys them. Witness. tests/test_codecs.py reads and writes each set type through its own function, a type through its narrower writer, the hex writer with the variant its send binds and, without a send, the SQL writer's default, a cell as a class of its own, and stops on two readers alike for one type; over the catalog, every stated trailing input names a parameter of its function, carries a value and is no out-parameter. tests/test_typerecover.py finds typedef on the three cell ids alone. tests/test_engine_integration.py reads and writes tbool, tint, tfloat and ttext through the reader and writer the catalog states for each. The suite floor goes from 368 to 377. --- .github/workflows/pytest.yml | 2 +- docs/enrichment.md | 78 ++++++---- parser/codecs.py | 258 +++++++++++++++++++++++++++++++ parser/typerecover.py | 21 +++ run.py | 10 ++ tests/test_codecs.py | 182 ++++++++++++++++++++++ tests/test_engine_integration.py | 90 +++++------ tests/test_typerecover.py | 12 ++ 8 files changed, 571 insertions(+), 82 deletions(-) create mode 100644 parser/codecs.py create mode 100644 tests/test_codecs.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 6951d1c..e40d4cc 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -96,7 +96,7 @@ jobs: # carries, or a change to them is not exercised until after it merges. # Consumers use the action; this repository owns the rules. - name: Refuse a skip, and a suite that shrank - run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 368 + run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 377 # The rules earn their place by refusing a log that carries what they # name. Both fixtures are written here rather than tracked, and the diff --git a/docs/enrichment.md b/docs/enrichment.md index 388cf09..6f5ec6f 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -33,44 +33,60 @@ Each function gets one `category` (first matching rule wins): ## 2. `typeEncodings` -A top-level map: opaque C type → how it round-trips to the wire. Built by -scanning the catalog for the type's own in/out functions. +A top-level map: class → the functions a binding reads and writes its values with, and +the trailing inputs each takes. `parser/enrich.py` builds the map from the shape of the +C functions; `parser/codecs.py` completes it once the SQL signatures and the bound +literals are in the catalog. ```json "typeEncodings": { - "Temporal": { - "encodings": ["mfjson", "text", "wkb"], - "decoders": { "text": "temporal_in", "mfjson": "temporal_from_mfjson" }, - "encoders": { "text": "temporal_out", "wkb": "temporal_as_hexwkb" }, - "in": "temporal_in", - "out": "temporal_out" + "Set": { + "encodings": ["text", "wkb"], + "decoders": { "wkb": "set_from_hexwkb" }, + "decoderAux": { "wkb": [] }, + "encoders": { "wkb": "set_as_hexwkb" }, + "encoderAux": { "wkb": [{"name": "variant", "kind": "integer", "default": 4}] }, + "readers": { "text": { "intset": "intset_in", "floatset": "floatset_in", "...": "..." } }, + "readerAux": { "text": { "intset": [], "floatset": [], "...": "..." } }, + "writers": { "text": { "intset": "intset_out", "floatset": "floatset_out", "...": "..." } }, + "writerAux": { "text": { "floatset": [{"name": "maxdd", "kind": "integer", "default": 15}] } }, + "in": "set_from_hexwkb", "in_aux": [], + "out": "set_as_hexwkb", "out_aux": [{"name": "variant", "kind": "integer", "default": 4}], + "bytes": { "decoder": "set_from_wkb", "encoder": "set_as_wkb", + "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] } } } ``` -- **decoder** — `const char * (+ aux) → T *` (`*_in`, `*_from_mfjson`, …) -- **encoder** — `const T * (+ aux) → char *` (`*_out`, `*_as_mfjson`, …) -- `in`/`out` — the preferred decoder/encoder, `text` > `mfjson` > `wkb`; - among candidates the **generic root** (`_in`/`_out`) is preferred - (so `temporal_out` serialises *every* subtype), else a deterministic - alphabetical pick. `in_aux`/`out_aux` carry the trailing args. - -> **Auxiliary arguments.** Real MEOS in/out wrappers (the public functions -> in the `*_meos.c` files) are not pure `(str)->T` / `(T)->str`: they take -> trailing *formatting* scalars — `temporal_out(temp, int maxdd)`, -> `*_as_mfjson(temp, with_bbox, flags, precision, srs)`. Those are safe to -> default (`maxdd`/`precision` → 15, flags/bbox → 0, `srs` → NULL), so the -> wrapper still satisfies the stateless contract; the defaults are recorded -> in `in_aux`/`out_aux` for the runtime to pass. A trailing arg that is -> *not* a defaultable formatting scalar disqualifies the wrapper: a -> semantic `*type` tag (`temporal_in`'s `temptype` — tagged -> `@ingroup meos_internal` in MEOS) or a pointer/array (`*_as_wkb`'s -> `size_out`). So polymorphic `Temporal` *decoding* resolves to a typed -> wrapper (`tbool_in`, …) — subtype-narrow on input; carrying the subtype -> on the wire for a universal decode is future work. *Encoding* is already -> universal via the generic `temporal_out`. - -The same data is folded onto each `structs[*]` entry as `serialization`. +- **decoder** — a public `const char * (+ aux) → T` (`*_in`, `*_from_mfjson`, + `*_from_hexwkb`, …); **encoder** — a public `T (+ aux) → char *` (`*_out`, + `*_as_mfjson`, `*_as_hexwkb`, …). A size the function writes back is an + out-parameter, stated in its `shape.outParams` as for any other function. +- **`readers` / `writers`** — a class several SQL types share (`Set`, `Span`, + `SpanSet`, `Temporal`) reads and writes each type through that type's own public + function, keyed by the SQL type its signature returns (a reader) or takes (a writer). + An encoding with a `readers` or `writers` entry has no single `decoders` or + `encoders` entry. Within one SQL type, the function the encoding table names first + wins (`cbuffer_out` before `cbuffer_as_ewkt`), then the narrower one + (`cbufferset_out` before `spatialset_out`); two that tie stop the catalog. An + encoding whose functions carry no SQL signature (`GSERIALIZED`, whose geometry and + geography are PostGIS's types) keeps the function the C shapes give it. +- **`in` / `out`** — the class's single decoder and encoder in the order `text` > + `mfjson` > `wkb`, absent when no single function serves the whole class. +- **Trailing inputs** — every decoder, encoder, reader and writer states the inputs + after its value by name with the value a binding passes, in `decoderAux`, + `encoderAux`, `readerAux` and `writerAux` (and `in_aux` / `out_aux`, + `bytes.encoderAux`). A binding builds each call from the function's own parameters, + filling each trailing input by name, and refuses one the catalog does not fill. +- **`variant`** of a WKB writer — the value the type's own `send` binds + (`WKB_EXTENDED`, 4, keeping the SRID), else the value its SQL hex writer passes when + the byte order is left out (`asHexWKB(raster, endian DEFAULT '')` passes 0). +- **A type of its own** — a value whose slot carries a `typedef` (`H3Index`, + `Quadbin`, `S2CellId`, each a `uint64_t`) is a class keyed by that name, read and + written by its own functions. + +The class's `encodings`, `in` and `out` are folded onto each `structs[*]` entry as +`serialization`. ## 3. `network` and `wire` diff --git a/parser/codecs.py b/parser/codecs.py new file mode 100644 index 0000000..ab8eb7a --- /dev/null +++ b/parser/codecs.py @@ -0,0 +1,258 @@ +"""State the codec of every class a binding carries, from the functions MEOS declares and +the SQL surface that registers them. + +#build_type_encodings of parser/enrich.py reads a class's readers and writers from the +shape of the C functions alone, before any SQL fact is in the catalog, and keeps one of +each per encoding. This pass completes each class once the SQL signatures and the bound +literals are attached: + +- A class that several SQL types share reads each type's text or MF-JSON form through that + type's own public reader, so it states them in ``readers`` keyed by the SQL type each + reader returns (``Set: {text: {intset: intset_in, bigintset: bigintset_in, ...}}``) and + states no single decoder, nor ``in``, for that encoding; it writes each type through + that type's own public writer, stated in ``writers`` keyed by the SQL type each writer + takes (``Set: {text: {intset: intset_out, ...}}``), with no single encoder, nor + ``out``, for that encoding. +- The hex-WKB writer is the class's ``wkb`` encoder. Its size is an out-parameter, stated + in the function's ``shape.outParams`` as for any other function, and its ``variant`` is + the value the type's own ``send`` binds (``WKB_EXTENDED``), else the value its SQL hex + writer passes when the call leaves the byte order out (``asHexWKB(raster, endian + DEFAULT '')`` passes 0, as ``wkb_variant_from_endian`` reads an empty order). +- Every decoder and encoder states its trailing inputs by name with the value a binding + passes, in ``decoderAux`` and ``encoderAux`` keyed by encoding, every reader and writer + of a SQL type in ``readerAux`` and ``writerAux`` keyed as ``readers`` and ``writers`` + are, and the byte codec its own in ``bytes.encoderAux``: a binding fills each parameter + by name and refuses one the catalog does not fill. +- A value whose slot carries a ``typedef`` (``H3Index``, ``Quadbin``, ``S2CellId``) is a + class of its own, keyed by that name. +""" +import re + +from parser.enrich import _DECODERS, _ENCODERS, _STRING_PTR_BASES, _aux_specs, _base + +_ORDER = ("text", "mfjson", "wkb") + + +def _cls(slot): + """The class of a slot: the type of its own it names, else its base type.""" + return slot.get("typedef") or _base(slot.get("canonical") or slot.get("c") + or slot.get("cType") or "") + + +def _encoding(name, table): + for rx, encoding in table: + if rx.search(name): + return encoding + return None + + +def _literal(value, macros): + """The number a bound literal names: a macro's value or an integer literal.""" + if value in macros: + return macros[value] + return int(value) if re.fullmatch(r"-?\d+", str(value)) else None + + +def _variant(cls, fns, macros, errors): + """The WKB variant a binding passes for ``cls``: the one the type's own ``send`` + binds, else the one its SQL hex writer passes when the byte order is left out.""" + bound = set() + for f in fns: + ps = f.get("params") or [] + if (f.get("api") == "public" and f["name"].endswith("_as_wkb") and ps + and _cls(ps[0]) == cls): + for s in f.get("sqlSignatures") or (): + v = (s.get("boundArgs") or {}).get("variant") + if v is not None and (s.get("sqlName") or "").endswith("_send"): + bound.add(_literal(v, macros)) + if len(bound) > 1: + errors.append(f"{cls}: its send functions bind variants {sorted(bound)}") + if len(bound) == 1: + return bound.pop() + for f in fns: + ps = f.get("params") or [] + if (f.get("api") == "public" and f["name"].endswith("_as_hexwkb") and ps + and _cls(ps[0]) == cls): + for s in f.get("sqlSignatures") or (): + dflts = s.get("argDefaults") or [] + if len(dflts) > 1 and dflts[1] == "''": + return 0 + return None + + +def _trailing(fn, skip_first=True): + """The aux of ``fn``'s trailing inputs, its out-parameters set aside, or None when + one is not defaultable (#_aux_specs of parser/enrich.py).""" + out = set(((fn.get("shape") or {}).get("outParams")) or ()) + params = [p for p in (fn.get("params") or [])[1 if skip_first else 0:] + if p["name"] not in out] + return _aux_specs(params) + + +def state_type_encodings(idl): + """(idl, errors): ``idl["typeEncodings"]`` and each struct's ``serialization`` stated + as this module's docstring describes, and the classes whose codec contradicts + itself.""" + fns = [f for f in idl.get("functions", []) if f.get("api") == "public"] + macros = {m["name"]: m.get("value") for m in idl.get("macros", [])} + encs = idl.setdefault("typeEncodings", {}) + errors = [] + + decoders, encoders = {}, {} + for f in fns: + ps = f.get("params") or [] + if not ps: + continue + ret, p0 = f["returnType"], ps[0] + # a reader: one string in, a value of the class out + enc = _encoding(f["name"], _DECODERS) + if (enc and "*" in (p0.get("cType") or "") and _base(p0["cType"]) in _STRING_PTR_BASES + and (("*" in (ret.get("c") or "")) or ret.get("typedef"))): + aux = _trailing(f) + if aux is not None: + decoders.setdefault(_cls(ret), {}).setdefault(enc, []).append((f, aux)) + # a writer: a value of the class in, a string out + enc = _encoding(f["name"], _ENCODERS) + if (enc and _base(ret.get("c") or "") in _STRING_PTR_BASES and "*" in (ret.get("c") or "") + and ("*" in (p0.get("cType") or "") or p0.get("typedef"))): + aux = _trailing(f) + if aux is not None: + encoders.setdefault(_cls(p0), {}).setdefault(enc, []).append((f, aux)) + + def keyed(cls, cands, sql_type, side, table): + """(one function and its aux, or None; {SQL type: function name}) for the readers + or writers ``cands`` of ``cls`` in one encoding, replacing the one pick + #build_type_encodings of parser/enrich.py makes per encoding. Each is keyed by + the SQL type its signatures read or write (``sql_type``); several functions keyed + so make the map, one makes the class's codec. A function no signature keys stands + for the class only when it is the encoding's one candidate (``interval_in``). Two + functions of one encoding for one SQL type rank by ``table``, whose first match + names the encoding (``cbuffer_out`` before ``cbuffer_as_text`` and + ``cbuffer_as_ewkt``), then by how many SQL types each serves, the narrower first, + as PostgreSQL resolves an overload to its most specific candidate + (``cbufferset_out`` before ``spatialset_out`` for a ``cbufferset``). Two that tie + on both contradict each other.""" + def types_of(f): + """The SQL types ``f``'s signatures serve; #build_type_encodings of + parser/enrich.py reads none, having no SQL signature yet.""" + return {sql_type(s) for s in f.get("sqlSignatures") or ()} - {None} + + def rank(f): + """(the index of ``f``'s pattern in ``table``, the encoding table + #build_type_encodings of parser/enrich.py classifies with; how many SQL types + ``f`` serves).""" + return (next(i for i, (rx, _) in enumerate(table) if rx.search(f["name"])), + len(types_of(f))) + by_type, unkeyed = {}, [] + for f, aux in cands: + types = types_of(f) + for t in types: + held = by_type.get(t) + if held and held[0] is not f: + if rank(held[0]) == rank(f): + errors.append(f"{cls}: {t} {side} by {held[0]['name']} " + f"and {f['name']}") + if rank(held[0]) <= rank(f): + continue + by_type[t] = (f, aux) + if not types: + unkeyed.append((f, aux)) + named = {f["name"]: (f, aux) for f, aux in by_type.values()} + if len(named) > 1: + return None, {t: (f["name"], aux) for t, (f, aux) in sorted(by_type.items())} + if named: + return next(iter(named.values())), {} + return None, {} + + def resolve(cls, enc, cands, prior, sql_type, side, table): + """(function and aux, or None; {SQL type: (name, aux)}) of one encoding: what the + SQL signatures key (#keyed), else the function #build_type_encodings of + parser/enrich.py states for the encoding, else the encoding's one public + candidate. A class whose readers and writers carry no SQL signature + (``GSERIALIZED``, whose geometry and geography are PostGIS's types) keeps what + enrich states.""" + pick, by_type = keyed(cls, cands, sql_type, side, table) if cands else (None, {}) + if pick or by_type: + return pick, by_type + if prior in every: + aux = _trailing(every[prior]) + return ((every[prior], aux) if aux is not None else None), {} + return (cands[0] if len(cands) == 1 else None), {} + + every = {f["name"]: f for f in idl.get("functions", [])} + for cls in sorted(set(encs) | {c for c in decoders if c in encs or _is_identity(c, fns)}): + e = encs.setdefault(cls, {"encodings": [], "decoders": {}, "encoders": {}}) + prior_dec, prior_enc = dict(e.get("decoders") or {}), dict(e.get("encoders") or {}) + dec, decaux, readers, readaux = {}, {}, {}, {} + for enc in sorted(set(decoders.get(cls) or {}) | set(prior_dec)): + pick, by_type = resolve(cls, enc, (decoders.get(cls) or {}).get(enc, []), + prior_dec.get(enc), lambda s: s.get("ret"), "read", + _DECODERS) + if by_type: + readers[enc] = {t: name for t, (name, _) in by_type.items()} + readaux[enc] = {t: aux for t, (_, aux) in by_type.items()} + elif pick: + dec[enc], decaux[enc] = pick[0]["name"], pick[1] + encd, encaux, writers, writeaux = {}, {}, {}, {} + for enc in sorted(set(encoders.get(cls) or {}) | set(prior_enc)): + pick, by_type = resolve(cls, enc, (encoders.get(cls) or {}).get(enc, []), + prior_enc.get(enc), + lambda s: (s.get("args") or [None])[0], "written", + _ENCODERS) + if by_type: + writers[enc] = {t: name for t, (name, _) in by_type.items()} + writeaux[enc] = {t: aux for t, (_, aux) in by_type.items()} + continue + if not pick: + continue + f, aux = pick + if enc == "wkb": + v = _variant(cls, fns, macros, errors) + aux = [dict(a, default=v) if a["name"] == "variant" else a for a in aux] + encd[enc], encaux[enc] = f["name"], aux + e["decoders"], e["decoderAux"], e["encoders"], e["encoderAux"] = dec, decaux, encd, encaux + for key, val in (("readers", readers), ("readerAux", readaux), + ("writers", writers), ("writerAux", writeaux)): + if val: + e[key] = val + else: + e.pop(key, None) + if "bytes" not in e and _is_identity(cls, fns): + bytes_ = _byte_codec(cls, fns) + if bytes_: + e["bytes"] = bytes_ + e["encodings"] = sorted(set(dec) | set(encd) | set(readers) | set(writers)) + in_e = next((x for x in _ORDER if x in dec), None) + out_e = next((x for x in _ORDER if x in encd), None) + e["in"], e["in_aux"] = (dec[in_e], decaux[in_e]) if in_e else (None, []) + e["out"], e["out_aux"] = (encd[out_e], encaux[out_e]) if out_e else (None, []) + if "bytes" in e: + v = _variant(cls, fns, macros, errors) + e["bytes"]["encoderAux"] = [{"name": "variant", "kind": "integer", "default": v}] + + for struct in idl.get("structs", []): + te = encs.get(struct["name"]) + if te: + struct["serialization"] = {"encodings": te["encodings"], "in": te["in"], + "out": te["out"]} + return idl, errors + + +def _byte_codec(cls, fns): + """{decoder, encoder} of a type of its own (``H3Index``): the reader ``T f(const + uint8_t *, size_t)`` and the writer ``uint8_t *f(T, uint8_t, size_t *)``, the shapes + #build_type_encodings of parser/enrich.py reads for a structure.""" + dec = [f["name"] for f in fns if f["returnType"].get("typedef") == cls + and [_base(p["cType"]) for p in f.get("params") or []] == ["uint8_t", "size_t"]] + enc = [f["name"] for f in fns if (f.get("params") or [{}])[0].get("typedef") == cls + and _base(f["returnType"].get("c")) == "uint8_t" + and len(f.get("params") or []) == 3] + if len(dec) == 1 and len(enc) == 1: + return {"decoder": dec[0], "encoder": enc[0]} + return None + + +def _is_identity(cls, fns): + """Whether ``cls`` is a type of its own some slot names (``H3Index``).""" + return any(p.get("typedef") == cls for f in fns + for p in [f["returnType"]] + (f.get("params") or [])) diff --git a/parser/typerecover.py b/parser/typerecover.py index 4a9c05c..1921d88 100644 --- a/parser/typerecover.py +++ b/parser/typerecover.py @@ -161,6 +161,9 @@ def _apply(slot, recovery): # resolved platform spelling ("unsigned long") -> fall through to normalize canonical. if cur not in recoverable and cur != _nospace(recovered): return 0 + # the name the header declares, which #normalize_canonical states as the slot's + # `typedef` when it names a type of its own + slot["_declared"] = _base_name(original) rewrote = slot.get(key) != recovered slot[key] = recovered canon = _nospace(slot.get("canonical")) @@ -287,10 +290,25 @@ def normalize_canonical(idl, pg_names=frozenset()): ``int *``). Complements ``recover_collapsed_types``: that recovers a ``cType`` the preprocessor erased to ``int``; this trusts a faithful ``cType`` and only re-spells ``canonical``. + + A slot whose header names a type of its own over a width, a name neither PostgreSQL + nor the C standard defines whose chain (#scalar_spelling) reaches a C standard + integer (``H3Index``, ``Quadbin``, ``S2CellId`` over ``uint64_t``), keeps that name + as ``typedef``: ``canonical`` states the width, ``typedef`` what the value is, which + only its own reader produces. """ fixed = 0 typedefs = idl.pop("_typedefs", None) or {} + def identity(name): + """``name`` when it is a type of its own over a C standard integer, read through + #scalar_spelling.""" + if (not name or name not in typedefs or name in pg_names + or _C_STANDARD.match(name)): + return None + reached = scalar_spelling(name, typedefs, pg_names) + return name if reached and _C_STANDARD.match(reached) else None + def want(ctype): base = _base_name(ctype) mapped = (scalar_spelling(base, typedefs, pg_names) if base in typedefs @@ -306,6 +324,9 @@ def fix(slot): if not (isinstance(slot, dict) and "canonical" in slot): return ctype = slot.get("cType") or slot.get("c") + own = identity(slot.pop("_declared", None) or _base_name(ctype)) + if own: + slot["typedef"] = own w = want(ctype) if ctype else None if w and _nospace(slot["canonical"]) != _nospace(w): slot["canonical"] = w diff --git a/run.py b/run.py index d35a4ac..afb3dd6 100644 --- a/run.py +++ b/run.py @@ -18,6 +18,7 @@ from parser.boundargs import merge_boundargs, resolve_bound_names from parser.compositions import attach_compositions from parser.enrich import enrich_idl +from parser.codecs import state_type_encodings from parser.sqlfn import (attach_sqlfn_map, attach_aggfn_map, attach_row_sources, attach_sqlaggfn_map, lint_container_family_csqlfn, lint_ea_sqlfn, lint_positional_sqlfn, @@ -314,6 +315,15 @@ def main(): f"{', '.join(unlisted)} — classify them explicitly so bindings " f"handle them uniformly", file=sys.stderr) + # State the codec of every class once the SQL signatures and the bound literals are + # in: the reader of each SQL type a class serves, the hex-WKB writer with the + # variant its type's send binds, and every trailing input by name with its value. + # A class whose codec contradicts itself stops the catalog. + idl, codec_errors = state_type_encodings(idl) + if codec_errors: + raise ValueError("type encodings that contradict themselves:\n " + + "\n ".join(codec_errors)) + # 6. Attach the temporal-covering descriptor (Parquet/Iceberg projection) print(f" Attaching temporal covering from {COVERING_PATH}...", file=sys.stderr) diff --git a/tests/test_codecs.py b/tests/test_codecs.py new file mode 100644 index 0000000..46e28cb --- /dev/null +++ b/tests/test_codecs.py @@ -0,0 +1,182 @@ +"""The codec each class states: its readers and writers per SQL type, its hex-WKB writer +with the variant its type binds, and every trailing input by name with its value. + +The synthetic catalog follows the shape #ExposabilityTests of tests/test_enrich.py builds +(``fn`` with ``c`` / ``cType`` / ``canonical``), carrying the SQL signatures and bound +literals the pass reads; the contract tests read the generated catalog, as +#TypeRecoverTests of tests/test_typerecover.py does. Plain unittest, no pytest dependency. +""" +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from parser.codecs import state_type_encodings + +IDL = Path(__file__).resolve().parents[1] / "output" / "meos-idl.json" +SET, RAST = "const Set *", "const Raster *" + + +def fn(name, ret, params, sigs=(), api="public", shape=None, typedef=None): + f = {"name": name, "api": api, + "returnType": {"c": ret, "canonical": ret}, + "params": [{"name": n, "cType": t, "canonical": t, + **({"typedef": typedef} if typedef and t == "uint64_t" else {})} + for t, n in params], + "sqlSignatures": list(sigs)} + if typedef and ret == "uint64_t": + f["returnType"]["typedef"] = typedef + if shape: + f["shape"] = shape + return f + + +def sig(name, args, ret, **kw): + return {"sqlName": name, "args": args, "ret": ret, **kw} + + +FUNCTIONS = [ + # one reader and one writer per set type, keyed by the SQL type each serves + fn("intset_in", "Set *", [("const char *", "str")], + [sig("intset_in", ["cstring"], "intset")]), + fn("floatset_in", "Set *", [("const char *", "str")], + [sig("floatset_in", ["cstring"], "floatset")]), + fn("intset_out", "char *", [(SET, "s")], + [sig("intset_out", ["intset"], "cstring"), sig("asText", ["intset"], "text")]), + fn("floatset_out", "char *", [(SET, "s"), ("int", "maxdd")], + [sig("floatset_out", ["floatset"], "cstring")]), + # a generic writer serving several types loses to each type's own + fn("spatialset_out", "char *", [(SET, "s"), ("int", "maxdd")], + [sig("geomset_out", ["geomset"], "cstring"), sig("intset_out", ["intset"], "cstring")]), + # the generic reader of one internal programmer surface is no codec + fn("set_in", "Set *", [("const char *", "str")], + [sig("intset_in", ["cstring"], "intset")], api="internal"), + # the hex-WKB pair, its size an out-parameter, its variant what send binds + fn("set_from_hexwkb", "Set *", [("const char *", "hexwkb")], + [sig("intsetFromHexWKB", ["text"], "intset"), + sig("floatsetFromHexWKB", ["text"], "floatset")]), + fn("set_as_hexwkb", "char *", + [(SET, "s"), ("uint8_t", "variant"), ("size_t *", "size_out")], + [sig("asHexWKB", ["intset", "text"], "text")], shape={"outParams": ["size_out"]}), + fn("set_as_wkb", "uint8_t *", + [(SET, "s"), ("uint8_t", "variant"), ("size_t *", "size_out")], + [sig("intset_send", ["intset"], "bytea", boundArgs={"variant": "WKB_EXTENDED"})], + shape={"outParams": ["size_out"]}), + # a type with no send of its own: its SQL hex writer leaves the byte order empty + fn("raster_from_hexwkb", "Raster *", [("const char *", "hexwkb")], + [sig("rasterFromHexWKB", ["text"], "raster")]), + fn("raster_as_hexwkb", "char *", + [(RAST, "rast"), ("uint8_t", "variant"), ("size_t *", "size_out")], + [sig("asHexWKB", ["raster", "text"], "text", argDefaults=[None, "''"])], + shape={"outParams": ["size_out"]}), + # a cell: a value of its own over uint64_t, read and written by its own functions + fn("h3index_in", "uint64_t", [("const char *", "str")], typedef="H3Index"), + fn("h3index_out", "char *", [("uint64_t", "cell")], typedef="H3Index"), +] + + +def _idl(functions=FUNCTIONS): + return {"functions": json.loads(json.dumps(functions)), + "macros": [{"name": "WKB_EXTENDED", "value": 4}], + "structs": [{"name": "Set", "fields": []}, {"name": "Raster", "fields": []}], + "typeEncodings": {"Set": {}, "Raster": {}}} + + +class CodecTests(unittest.TestCase): + """#state_type_encodings of parser/codecs.py over a synthetic catalog.""" + + def setUp(self): + idl, self.errors = state_type_encodings(_idl()) + self.te = idl["typeEncodings"] + self.structs = {s["name"]: s for s in idl["structs"]} + + def test_each_type_is_read_and_written_by_its_own_function(self): + s = self.te["Set"] + self.assertEqual(s["readers"]["text"], {"intset": "intset_in", + "floatset": "floatset_in"}) + self.assertEqual(s["writers"]["text"], {"intset": "intset_out", + "floatset": "floatset_out", + "geomset": "spatialset_out"}) + self.assertNotIn("text", s["decoders"]) + self.assertNotIn("text", s["encoders"]) + self.assertEqual(s["writerAux"]["text"]["floatset"], + [{"name": "maxdd", "kind": "integer", "default": 15}]) + self.assertEqual(self.errors, []) + + def test_one_function_for_every_type_is_the_class_codec(self): + s = self.te["Set"] + self.assertEqual(s["decoders"]["wkb"], "set_from_hexwkb") + self.assertEqual(s["in"], "set_from_hexwkb") + self.assertEqual(self.structs["Set"]["serialization"]["in"], "set_from_hexwkb") + + def test_the_hex_writer_takes_the_variant_its_send_binds(self): + s = self.te["Set"] + self.assertEqual(s["encoders"]["wkb"], "set_as_hexwkb") + self.assertEqual(s["encoderAux"]["wkb"], + [{"name": "variant", "kind": "integer", "default": 4}]) + + def test_without_a_send_the_variant_is_the_sql_writers_default(self): + r = self.te["Raster"] + self.assertEqual(r["encoders"], {"wkb": "raster_as_hexwkb"}) + self.assertEqual(r["out"], "raster_as_hexwkb") + self.assertEqual(r["encoderAux"]["wkb"][0]["default"], 0) + + def test_a_cell_is_a_class_of_its_own(self): + h = self.te["H3Index"] + self.assertEqual((h["in"], h["out"]), ("h3index_in", "h3index_out")) + + def test_two_functions_alike_for_one_type_stop_the_catalog(self): + twin = fn("intset_in2", "Set *", [("const char *", "str")], + [sig("intset_in", ["cstring"], "intset")]) + twin["name"] = "intsetx_in" + _, errors = state_type_encodings(_idl(FUNCTIONS + [twin])) + self.assertEqual(errors, ["Set: intset read by intset_in and intsetx_in"]) + + +class CodecContractTests(unittest.TestCase): + """Over the generated catalog: every trailing input a codec states names a parameter + of its function, carries the value a binding passes, and is never an out-parameter.""" + + def setUp(self): + if not IDL.exists(): + self.skipTest(f"{IDL} not generated; run `python run.py` first") + self.idl = json.loads(IDL.read_text()) + self.fns = {f["name"]: f for f in self.idl["functions"]} + + def _stated(self): + for cls, e in self.idl["typeEncodings"].items(): + for side, names in (("decoderAux", e.get("decoders") or {}), + ("encoderAux", e.get("encoders") or {})): + for enc, name in names.items(): + yield cls, name, (e.get(side) or {}).get(enc, []) + for side, aux in (("readers", "readerAux"), ("writers", "writerAux")): + for enc, by_type in (e.get(side) or {}).items(): + for t, name in by_type.items(): + yield cls, name, e[aux][enc][t] + if e.get("bytes", {}).get("encoderAux") is not None: + yield cls, e["bytes"]["encoder"], e["bytes"]["encoderAux"] + + def test_every_trailing_input_is_named_valued_and_no_out_parameter(self): + bad = [] + for cls, name, aux in self._stated(): + f = self.fns[name] + params = {p["name"] for p in f["params"][1:]} + out = set((f.get("shape") or {}).get("outParams") or ()) + for a in aux: + if a["name"] not in params or "default" not in a or a["name"] in out: + bad.append((cls, name, a)) + self.assertEqual(bad, []) + + def test_the_classes_a_binding_reads(self): + te = self.idl["typeEncodings"] + self.assertEqual(te["Set"]["readers"]["text"]["intset"], "intset_in") + self.assertEqual(te["Temporal"]["writers"]["text"]["tfloat"], "tfloat_out") + self.assertEqual(te["Raster"]["out"], "raster_as_hexwkb") + for cell in ("H3Index", "Quadbin", "S2CellId"): + self.assertIn(cell, te) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_engine_integration.py b/tests/test_engine_integration.py index b3dfdd9..7a8f2dd 100644 --- a/tests/test_engine_integration.py +++ b/tests/test_engine_integration.py @@ -5,11 +5,11 @@ MEOS_LIBRARY_PATH=/usr/local/lib/libmeos.so python3 tests/test_engine_integration.py -It drives the exact path the server uses, including the catalog's -``in_aux``/``out_aux`` defaults (so the *generic* ``temporal_out(temp, -maxdd=15)`` is called correctly — proving it serialises any subtype), and -asserts that bad input raises ``MeosError`` instead of terminating the -process (MEOS's default handler calls ``exit()``). +It drives the exact path the server uses: a temporal value is read and +written through the reader and writer the catalog states for its SQL type, +each called with the trailing inputs the catalog states by name +(``tfloat_out(temp, maxdd=15)``), and bad input raises ``MeosError`` instead of +terminating the process (MEOS's default handler calls ``exit()``). """ import json @@ -28,18 +28,13 @@ _TBOOL = "{t@2000-01-01, f@2000-01-03, t@2000-01-05}" _TFLOAT = "{1.5@2000-01-01, 3.5@2000-01-03}" -# A three-instant literal per temporal subtype, keyed by the decoder that reads -# it. Which decoder the catalog selects for the opaque `Temporal` is settled by -# an alphabetical tiebreak that build_type_encodings() documents as arbitrary: -# there is no generic `temporal_in`, so the pick is whichever subtype sorts -# first, and it MOVES when MobilityDB gains a type (tbigint displaced tbool). -# The fixture therefore follows the pick instead of naming it. -_LITERAL_BY_DECODER = { - "tbool_in": _TBOOL, - "tint_in": "{1@2000-01-01, 2@2000-01-03, 1@2000-01-05}", - "tbigint_in": "{1@2000-01-01, 2@2000-01-03, 1@2000-01-05}", - "tfloat_in": "{1.5@2000-01-01, 3.5@2000-01-03, 1.5@2000-01-05}", - "ttext_in": "{AA@2000-01-01, BB@2000-01-03, AA@2000-01-05}", +# A temporal value of each SQL type, read through the reader the catalog states +# for that type: `Temporal` serves twenty types and has no generic public reader. +_LITERAL_BY_TYPE = { + "tbool": _TBOOL, + "tint": "{1@2000-01-01, 2@2000-01-03, 1@2000-01-05}", + "tfloat": "{1.5@2000-01-01, 3.5@2000-01-03, 1.5@2000-01-05}", + "ttext": "{AA@2000-01-01, BB@2000-01-03, AA@2000-01-05}", } _KIND_TAG = {"integer": "int", "number": "double", @@ -58,41 +53,36 @@ def setUpClass(cls): te = (json.loads(_CATALOG.read_text()).get("typeEncodings", {}) if _CATALOG.exists() else {}) t = te.get("Temporal", {}) - cls.tin = t.get("in", "tbool_in") - cls.tout = t.get("out", "tbool_out") - cls.in_aux = _aux(t.get("in_aux", [])) - cls.out_aux = _aux(t.get("out_aux", [])) - cls.tin_literal = _LITERAL_BY_DECODER.get(cls.tin) - - def test_catalog_selected_in_out(self): - # Decoding stays a typed wrapper (subtype-narrow); encoding is the - # generic temporal_out with a defaulted maxdd. - # - # WHICH subtype decodes is not asserted: no `temporal_in` exists, so - # build_type_encodings() falls back to an alphabetical pick it calls - # arbitrary, and that pick moves when MobilityDB gains a type. What the - # design does guarantee is asserted instead — the decoder is one of the - # subtype-narrow readers, and the encoder IS the generic root. - self.assertIn(self.tin, _LITERAL_BY_DECODER, - f"catalog selected {self.tin!r} as the Temporal decoder; " - f"add its literal to _LITERAL_BY_DECODER") - self.assertEqual(self.tout, "temporal_out") - self.assertEqual(self.out_aux, [("int", 15)]) + cls.readers = (t.get("readers") or {}).get("text", {}) + cls.reader_aux = (t.get("readerAux") or {}).get("text", {}) + cls.writers = (t.get("writers") or {}).get("text", {}) + cls.writer_aux = (t.get("writerAux") or {}).get("text", {}) + + def read(self, sqltype, literal): + return self.eng.decode(self.readers[sqltype], literal, + _aux(self.reader_aux[sqltype])) + + def write(self, sqltype, handle): + return self.eng.encode(self.writers[sqltype], handle, + _aux(self.writer_aux[sqltype])) + + def test_each_type_states_its_own_reader_and_writer(self): + for sqltype in _LITERAL_BY_TYPE: + self.assertEqual(self.readers[sqltype], sqltype + "_in") + self.assertEqual(self.writers[sqltype], sqltype + "_out") + self.assertEqual(_aux(self.writer_aux["tfloat"]), [("int", 15)]) def test_decode_invoke_scalar(self): - h = self.eng.decode(self.tin, self.tin_literal, self.in_aux) - self.assertTrue(h) - n = self.eng.invoke("temporal_num_instants", [("ptr", h)], "int") - self.assertEqual(n, 3) - - def test_generic_encoder_round_trips_any_subtype(self): - # The whole point of the gap fix: temporal_out(+maxdd) serialises - # a tbool AND a tfloat — a subtype-narrow tbool_out could not. - hb = self.eng.decode("tbool_in", _TBOOL) - ob = self.eng.encode(self.tout, hb, self.out_aux) + for sqltype, literal in _LITERAL_BY_TYPE.items(): + h = self.read(sqltype, literal) + self.assertTrue(h, sqltype) + n = self.eng.invoke("temporal_num_instants", [("ptr", h)], "int") + self.assertEqual(n, 3, sqltype) + + def test_each_type_round_trips_through_its_own_writer(self): + ob = self.write("tbool", self.read("tbool", _TBOOL)) self.assertIn("@", ob) - hf = self.eng.decode("tfloat_in", _TFLOAT) - of = self.eng.encode(self.tout, hf, self.out_aux) + of = self.write("tfloat", self.read("tfloat", _TFLOAT)) self.assertIn("@", of) self.assertIn("1.5", of) @@ -137,7 +127,7 @@ def test_input_array_builder_round_trip(self): "temporal_merge_array", [("ptrarray", [h1, h2]), ("int", 2)], "ptr") self.assertTrue(merged) - out = self.eng.encode(self.tout, merged, self.out_aux) + out = self.write("tbool", merged) self.assertIn("@", out) self.assertIn("2000-01-03", out) # both instants merged in diff --git a/tests/test_typerecover.py b/tests/test_typerecover.py index cbce880..2446f98 100644 --- a/tests/test_typerecover.py +++ b/tests/test_typerecover.py @@ -243,6 +243,18 @@ def canon(name, pname): self.assertEqual(canon("pg_time_in", "typmod"), "int32_t") self.assertEqual(canon("set_as_wkb", "variant"), "uint8_t") + def test_a_type_of_its_own_keeps_its_name_beside_its_width(self): + # #test_cell_id_canonical_normalized_uniform holds the cells' width; typedef + # states what the value is, on the three cell ids and on no PostgreSQL type or + # width name + idl = json.loads(IDL.read_text()) + slots = [s for f in idl["functions"] + for s in [f["returnType"]] + f.get("params", [])] + named = {(s["typedef"], _base(s["canonical"])) for s in slots if "typedef" in s} + self.assertEqual(named, {("H3Index", "uint64_t"), ("Quadbin", "uint64_t"), + ("S2CellId", "uint64_t")}) + self.assertEqual(self.by_name["h3index_in"]["returnType"]["typedef"], "H3Index") + # C's own integer names: what a typedef must never be stated as. _C_INTEGERS = {