diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 6951d1c..e40d4cc 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -96,7 +96,7 @@ jobs: # carries, or a change to them is not exercised until after it merges. # Consumers use the action; this repository owns the rules. - name: Refuse a skip, and a suite that shrank - run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 368 + run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 377 # The rules earn their place by refusing a log that carries what they # name. Both fixtures are written here rather than tracked, and the diff --git a/docs/enrichment.md b/docs/enrichment.md index 388cf09..6f5ec6f 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -33,44 +33,60 @@ Each function gets one `category` (first matching rule wins): ## 2. `typeEncodings` -A top-level map: opaque C type → how it round-trips to the wire. Built by -scanning the catalog for the type's own in/out functions. +A top-level map: class → the functions a binding reads and writes its values with, and +the trailing inputs each takes. `parser/enrich.py` builds the map from the shape of the +C functions; `parser/codecs.py` completes it once the SQL signatures and the bound +literals are in the catalog. ```json "typeEncodings": { - "Temporal": { - "encodings": ["mfjson", "text", "wkb"], - "decoders": { "text": "temporal_in", "mfjson": "temporal_from_mfjson" }, - "encoders": { "text": "temporal_out", "wkb": "temporal_as_hexwkb" }, - "in": "temporal_in", - "out": "temporal_out" + "Set": { + "encodings": ["text", "wkb"], + "decoders": { "wkb": "set_from_hexwkb" }, + "decoderAux": { "wkb": [] }, + "encoders": { "wkb": "set_as_hexwkb" }, + "encoderAux": { "wkb": [{"name": "variant", "kind": "integer", "default": 4}] }, + "readers": { "text": { "intset": "intset_in", "floatset": "floatset_in", "...": "..." } }, + "readerAux": { "text": { "intset": [], "floatset": [], "...": "..." } }, + "writers": { "text": { "intset": "intset_out", "floatset": "floatset_out", "...": "..." } }, + "writerAux": { "text": { "floatset": [{"name": "maxdd", "kind": "integer", "default": 15}] } }, + "in": "set_from_hexwkb", "in_aux": [], + "out": "set_as_hexwkb", "out_aux": [{"name": "variant", "kind": "integer", "default": 4}], + "bytes": { "decoder": "set_from_wkb", "encoder": "set_as_wkb", + "encoderAux": [{"name": "variant", "kind": "integer", "default": 4}] } } } ``` -- **decoder** — `const char * (+ aux) → T *` (`*_in`, `*_from_mfjson`, …) -- **encoder** — `const T * (+ aux) → char *` (`*_out`, `*_as_mfjson`, …) -- `in`/`out` — the preferred decoder/encoder, `text` > `mfjson` > `wkb`; - among candidates the **generic root** (`_in`/`_out`) is preferred - (so `temporal_out` serialises *every* subtype), else a deterministic - alphabetical pick. `in_aux`/`out_aux` carry the trailing args. - -> **Auxiliary arguments.** Real MEOS in/out wrappers (the public functions -> in the `*_meos.c` files) are not pure `(str)->T` / `(T)->str`: they take -> trailing *formatting* scalars — `temporal_out(temp, int maxdd)`, -> `*_as_mfjson(temp, with_bbox, flags, precision, srs)`. Those are safe to -> default (`maxdd`/`precision` → 15, flags/bbox → 0, `srs` → NULL), so the -> wrapper still satisfies the stateless contract; the defaults are recorded -> in `in_aux`/`out_aux` for the runtime to pass. A trailing arg that is -> *not* a defaultable formatting scalar disqualifies the wrapper: a -> semantic `*type` tag (`temporal_in`'s `temptype` — tagged -> `@ingroup meos_internal` in MEOS) or a pointer/array (`*_as_wkb`'s -> `size_out`). So polymorphic `Temporal` *decoding* resolves to a typed -> wrapper (`tbool_in`, …) — subtype-narrow on input; carrying the subtype -> on the wire for a universal decode is future work. *Encoding* is already -> universal via the generic `temporal_out`. - -The same data is folded onto each `structs[*]` entry as `serialization`. +- **decoder** — a public `const char * (+ aux) → T` (`*_in`, `*_from_mfjson`, + `*_from_hexwkb`, …); **encoder** — a public `T (+ aux) → char *` (`*_out`, + `*_as_mfjson`, `*_as_hexwkb`, …). A size the function writes back is an + out-parameter, stated in its `shape.outParams` as for any other function. +- **`readers` / `writers`** — a class several SQL types share (`Set`, `Span`, + `SpanSet`, `Temporal`) reads and writes each type through that type's own public + function, keyed by the SQL type its signature returns (a reader) or takes (a writer). + An encoding with a `readers` or `writers` entry has no single `decoders` or + `encoders` entry. Within one SQL type, the function the encoding table names first + wins (`cbuffer_out` before `cbuffer_as_ewkt`), then the narrower one + (`cbufferset_out` before `spatialset_out`); two that tie stop the catalog. An + encoding whose functions carry no SQL signature (`GSERIALIZED`, whose geometry and + geography are PostGIS's types) keeps the function the C shapes give it. +- **`in` / `out`** — the class's single decoder and encoder in the order `text` > + `mfjson` > `wkb`, absent when no single function serves the whole class. +- **Trailing inputs** — every decoder, encoder, reader and writer states the inputs + after its value by name with the value a binding passes, in `decoderAux`, + `encoderAux`, `readerAux` and `writerAux` (and `in_aux` / `out_aux`, + `bytes.encoderAux`). A binding builds each call from the function's own parameters, + filling each trailing input by name, and refuses one the catalog does not fill. +- **`variant`** of a WKB writer — the value the type's own `send` binds + (`WKB_EXTENDED`, 4, keeping the SRID), else the value its SQL hex writer passes when + the byte order is left out (`asHexWKB(raster, endian DEFAULT '')` passes 0). +- **A type of its own** — a value whose slot carries a `typedef` (`H3Index`, + `Quadbin`, `S2CellId`, each a `uint64_t`) is a class keyed by that name, read and + written by its own functions. + +The class's `encodings`, `in` and `out` are folded onto each `structs[*]` entry as +`serialization`. ## 3. `network` and `wire` diff --git a/parser/codecs.py b/parser/codecs.py new file mode 100644 index 0000000..ab8eb7a --- /dev/null +++ b/parser/codecs.py @@ -0,0 +1,258 @@ +"""State the codec of every class a binding carries, from the functions MEOS declares and +the SQL surface that registers them. + +#build_type_encodings of parser/enrich.py reads a class's readers and writers from the +shape of the C functions alone, before any SQL fact is in the catalog, and keeps one of +each per encoding. This pass completes each class once the SQL signatures and the bound +literals are attached: + +- A class that several SQL types share reads each type's text or MF-JSON form through that + type's own public reader, so it states them in ``readers`` keyed by the SQL type each + reader returns (``Set: {text: {intset: intset_in, bigintset: bigintset_in, ...}}``) and + states no single decoder, nor ``in``, for that encoding; it writes each type through + that type's own public writer, stated in ``writers`` keyed by the SQL type each writer + takes (``Set: {text: {intset: intset_out, ...}}``), with no single encoder, nor + ``out``, for that encoding. +- The hex-WKB writer is the class's ``wkb`` encoder. Its size is an out-parameter, stated + in the function's ``shape.outParams`` as for any other function, and its ``variant`` is + the value the type's own ``send`` binds (``WKB_EXTENDED``), else the value its SQL hex + writer passes when the call leaves the byte order out (``asHexWKB(raster, endian + DEFAULT '')`` passes 0, as ``wkb_variant_from_endian`` reads an empty order). +- Every decoder and encoder states its trailing inputs by name with the value a binding + passes, in ``decoderAux`` and ``encoderAux`` keyed by encoding, every reader and writer + of a SQL type in ``readerAux`` and ``writerAux`` keyed as ``readers`` and ``writers`` + are, and the byte codec its own in ``bytes.encoderAux``: a binding fills each parameter + by name and refuses one the catalog does not fill. +- A value whose slot carries a ``typedef`` (``H3Index``, ``Quadbin``, ``S2CellId``) is a + class of its own, keyed by that name. +""" +import re + +from parser.enrich import _DECODERS, _ENCODERS, _STRING_PTR_BASES, _aux_specs, _base + +_ORDER = ("text", "mfjson", "wkb") + + +def _cls(slot): + """The class of a slot: the type of its own it names, else its base type.""" + return slot.get("typedef") or _base(slot.get("canonical") or slot.get("c") + or slot.get("cType") or "") + + +def _encoding(name, table): + for rx, encoding in table: + if rx.search(name): + return encoding + return None + + +def _literal(value, macros): + """The number a bound literal names: a macro's value or an integer literal.""" + if value in macros: + return macros[value] + return int(value) if re.fullmatch(r"-?\d+", str(value)) else None + + +def _variant(cls, fns, macros, errors): + """The WKB variant a binding passes for ``cls``: the one the type's own ``send`` + binds, else the one its SQL hex writer passes when the byte order is left out.""" + bound = set() + for f in fns: + ps = f.get("params") or [] + if (f.get("api") == "public" and f["name"].endswith("_as_wkb") and ps + and _cls(ps[0]) == cls): + for s in f.get("sqlSignatures") or (): + v = (s.get("boundArgs") or {}).get("variant") + if v is not None and (s.get("sqlName") or "").endswith("_send"): + bound.add(_literal(v, macros)) + if len(bound) > 1: + errors.append(f"{cls}: its send functions bind variants {sorted(bound)}") + if len(bound) == 1: + return bound.pop() + for f in fns: + ps = f.get("params") or [] + if (f.get("api") == "public" and f["name"].endswith("_as_hexwkb") and ps + and _cls(ps[0]) == cls): + for s in f.get("sqlSignatures") or (): + dflts = s.get("argDefaults") or [] + if len(dflts) > 1 and dflts[1] == "''": + return 0 + return None + + +def _trailing(fn, skip_first=True): + """The aux of ``fn``'s trailing inputs, its out-parameters set aside, or None when + one is not defaultable (#_aux_specs of parser/enrich.py).""" + out = set(((fn.get("shape") or {}).get("outParams")) or ()) + params = [p for p in (fn.get("params") or [])[1 if skip_first else 0:] + if p["name"] not in out] + return _aux_specs(params) + + +def state_type_encodings(idl): + """(idl, errors): ``idl["typeEncodings"]`` and each struct's ``serialization`` stated + as this module's docstring describes, and the classes whose codec contradicts + itself.""" + fns = [f for f in idl.get("functions", []) if f.get("api") == "public"] + macros = {m["name"]: m.get("value") for m in idl.get("macros", [])} + encs = idl.setdefault("typeEncodings", {}) + errors = [] + + decoders, encoders = {}, {} + for f in fns: + ps = f.get("params") or [] + if not ps: + continue + ret, p0 = f["returnType"], ps[0] + # a reader: one string in, a value of the class out + enc = _encoding(f["name"], _DECODERS) + if (enc and "*" in (p0.get("cType") or "") and _base(p0["cType"]) in _STRING_PTR_BASES + and (("*" in (ret.get("c") or "")) or ret.get("typedef"))): + aux = _trailing(f) + if aux is not None: + decoders.setdefault(_cls(ret), {}).setdefault(enc, []).append((f, aux)) + # a writer: a value of the class in, a string out + enc = _encoding(f["name"], _ENCODERS) + if (enc and _base(ret.get("c") or "") in _STRING_PTR_BASES and "*" in (ret.get("c") or "") + and ("*" in (p0.get("cType") or "") or p0.get("typedef"))): + aux = _trailing(f) + if aux is not None: + encoders.setdefault(_cls(p0), {}).setdefault(enc, []).append((f, aux)) + + def keyed(cls, cands, sql_type, side, table): + """(one function and its aux, or None; {SQL type: function name}) for the readers + or writers ``cands`` of ``cls`` in one encoding, replacing the one pick + #build_type_encodings of parser/enrich.py makes per encoding. Each is keyed by + the SQL type its signatures read or write (``sql_type``); several functions keyed + so make the map, one makes the class's codec. A function no signature keys stands + for the class only when it is the encoding's one candidate (``interval_in``). Two + functions of one encoding for one SQL type rank by ``table``, whose first match + names the encoding (``cbuffer_out`` before ``cbuffer_as_text`` and + ``cbuffer_as_ewkt``), then by how many SQL types each serves, the narrower first, + as PostgreSQL resolves an overload to its most specific candidate + (``cbufferset_out`` before ``spatialset_out`` for a ``cbufferset``). Two that tie + on both contradict each other.""" + def types_of(f): + """The SQL types ``f``'s signatures serve; #build_type_encodings of + parser/enrich.py reads none, having no SQL signature yet.""" + return {sql_type(s) for s in f.get("sqlSignatures") or ()} - {None} + + def rank(f): + """(the index of ``f``'s pattern in ``table``, the encoding table + #build_type_encodings of parser/enrich.py classifies with; how many SQL types + ``f`` serves).""" + return (next(i for i, (rx, _) in enumerate(table) if rx.search(f["name"])), + len(types_of(f))) + by_type, unkeyed = {}, [] + for f, aux in cands: + types = types_of(f) + for t in types: + held = by_type.get(t) + if held and held[0] is not f: + if rank(held[0]) == rank(f): + errors.append(f"{cls}: {t} {side} by {held[0]['name']} " + f"and {f['name']}") + if rank(held[0]) <= rank(f): + continue + by_type[t] = (f, aux) + if not types: + unkeyed.append((f, aux)) + named = {f["name"]: (f, aux) for f, aux in by_type.values()} + if len(named) > 1: + return None, {t: (f["name"], aux) for t, (f, aux) in sorted(by_type.items())} + if named: + return next(iter(named.values())), {} + return None, {} + + def resolve(cls, enc, cands, prior, sql_type, side, table): + """(function and aux, or None; {SQL type: (name, aux)}) of one encoding: what the + SQL signatures key (#keyed), else the function #build_type_encodings of + parser/enrich.py states for the encoding, else the encoding's one public + candidate. A class whose readers and writers carry no SQL signature + (``GSERIALIZED``, whose geometry and geography are PostGIS's types) keeps what + enrich states.""" + pick, by_type = keyed(cls, cands, sql_type, side, table) if cands else (None, {}) + if pick or by_type: + return pick, by_type + if prior in every: + aux = _trailing(every[prior]) + return ((every[prior], aux) if aux is not None else None), {} + return (cands[0] if len(cands) == 1 else None), {} + + every = {f["name"]: f for f in idl.get("functions", [])} + for cls in sorted(set(encs) | {c for c in decoders if c in encs or _is_identity(c, fns)}): + e = encs.setdefault(cls, {"encodings": [], "decoders": {}, "encoders": {}}) + prior_dec, prior_enc = dict(e.get("decoders") or {}), dict(e.get("encoders") or {}) + dec, decaux, readers, readaux = {}, {}, {}, {} + for enc in sorted(set(decoders.get(cls) or {}) | set(prior_dec)): + pick, by_type = resolve(cls, enc, (decoders.get(cls) or {}).get(enc, []), + prior_dec.get(enc), lambda s: s.get("ret"), "read", + _DECODERS) + if by_type: + readers[enc] = {t: name for t, (name, _) in by_type.items()} + readaux[enc] = {t: aux for t, (_, aux) in by_type.items()} + elif pick: + dec[enc], decaux[enc] = pick[0]["name"], pick[1] + encd, encaux, writers, writeaux = {}, {}, {}, {} + for enc in sorted(set(encoders.get(cls) or {}) | set(prior_enc)): + pick, by_type = resolve(cls, enc, (encoders.get(cls) or {}).get(enc, []), + prior_enc.get(enc), + lambda s: (s.get("args") or [None])[0], "written", + _ENCODERS) + if by_type: + writers[enc] = {t: name for t, (name, _) in by_type.items()} + writeaux[enc] = {t: aux for t, (_, aux) in by_type.items()} + continue + if not pick: + continue + f, aux = pick + if enc == "wkb": + v = _variant(cls, fns, macros, errors) + aux = [dict(a, default=v) if a["name"] == "variant" else a for a in aux] + encd[enc], encaux[enc] = f["name"], aux + e["decoders"], e["decoderAux"], e["encoders"], e["encoderAux"] = dec, decaux, encd, encaux + for key, val in (("readers", readers), ("readerAux", readaux), + ("writers", writers), ("writerAux", writeaux)): + if val: + e[key] = val + else: + e.pop(key, None) + if "bytes" not in e and _is_identity(cls, fns): + bytes_ = _byte_codec(cls, fns) + if bytes_: + e["bytes"] = bytes_ + e["encodings"] = sorted(set(dec) | set(encd) | set(readers) | set(writers)) + in_e = next((x for x in _ORDER if x in dec), None) + out_e = next((x for x in _ORDER if x in encd), None) + e["in"], e["in_aux"] = (dec[in_e], decaux[in_e]) if in_e else (None, []) + e["out"], e["out_aux"] = (encd[out_e], encaux[out_e]) if out_e else (None, []) + if "bytes" in e: + v = _variant(cls, fns, macros, errors) + e["bytes"]["encoderAux"] = [{"name": "variant", "kind": "integer", "default": v}] + + for struct in idl.get("structs", []): + te = encs.get(struct["name"]) + if te: + struct["serialization"] = {"encodings": te["encodings"], "in": te["in"], + "out": te["out"]} + return idl, errors + + +def _byte_codec(cls, fns): + """{decoder, encoder} of a type of its own (``H3Index``): the reader ``T f(const + uint8_t *, size_t)`` and the writer ``uint8_t *f(T, uint8_t, size_t *)``, the shapes + #build_type_encodings of parser/enrich.py reads for a structure.""" + dec = [f["name"] for f in fns if f["returnType"].get("typedef") == cls + and [_base(p["cType"]) for p in f.get("params") or []] == ["uint8_t", "size_t"]] + enc = [f["name"] for f in fns if (f.get("params") or [{}])[0].get("typedef") == cls + and _base(f["returnType"].get("c")) == "uint8_t" + and len(f.get("params") or []) == 3] + if len(dec) == 1 and len(enc) == 1: + return {"decoder": dec[0], "encoder": enc[0]} + return None + + +def _is_identity(cls, fns): + """Whether ``cls`` is a type of its own some slot names (``H3Index``).""" + return any(p.get("typedef") == cls for f in fns + for p in [f["returnType"]] + (f.get("params") or [])) diff --git a/parser/typerecover.py b/parser/typerecover.py index 4a9c05c..1921d88 100644 --- a/parser/typerecover.py +++ b/parser/typerecover.py @@ -161,6 +161,9 @@ def _apply(slot, recovery): # resolved platform spelling ("unsigned long") -> fall through to normalize canonical. if cur not in recoverable and cur != _nospace(recovered): return 0 + # the name the header declares, which #normalize_canonical states as the slot's + # `typedef` when it names a type of its own + slot["_declared"] = _base_name(original) rewrote = slot.get(key) != recovered slot[key] = recovered canon = _nospace(slot.get("canonical")) @@ -287,10 +290,25 @@ def normalize_canonical(idl, pg_names=frozenset()): ``int *``). Complements ``recover_collapsed_types``: that recovers a ``cType`` the preprocessor erased to ``int``; this trusts a faithful ``cType`` and only re-spells ``canonical``. + + A slot whose header names a type of its own over a width, a name neither PostgreSQL + nor the C standard defines whose chain (#scalar_spelling) reaches a C standard + integer (``H3Index``, ``Quadbin``, ``S2CellId`` over ``uint64_t``), keeps that name + as ``typedef``: ``canonical`` states the width, ``typedef`` what the value is, which + only its own reader produces. """ fixed = 0 typedefs = idl.pop("_typedefs", None) or {} + def identity(name): + """``name`` when it is a type of its own over a C standard integer, read through + #scalar_spelling.""" + if (not name or name not in typedefs or name in pg_names + or _C_STANDARD.match(name)): + return None + reached = scalar_spelling(name, typedefs, pg_names) + return name if reached and _C_STANDARD.match(reached) else None + def want(ctype): base = _base_name(ctype) mapped = (scalar_spelling(base, typedefs, pg_names) if base in typedefs @@ -306,6 +324,9 @@ def fix(slot): if not (isinstance(slot, dict) and "canonical" in slot): return ctype = slot.get("cType") or slot.get("c") + own = identity(slot.pop("_declared", None) or _base_name(ctype)) + if own: + slot["typedef"] = own w = want(ctype) if ctype else None if w and _nospace(slot["canonical"]) != _nospace(w): slot["canonical"] = w diff --git a/run.py b/run.py index d35a4ac..afb3dd6 100644 --- a/run.py +++ b/run.py @@ -18,6 +18,7 @@ from parser.boundargs import merge_boundargs, resolve_bound_names from parser.compositions import attach_compositions from parser.enrich import enrich_idl +from parser.codecs import state_type_encodings from parser.sqlfn import (attach_sqlfn_map, attach_aggfn_map, attach_row_sources, attach_sqlaggfn_map, lint_container_family_csqlfn, lint_ea_sqlfn, lint_positional_sqlfn, @@ -314,6 +315,15 @@ def main(): f"{', '.join(unlisted)} — classify them explicitly so bindings " f"handle them uniformly", file=sys.stderr) + # State the codec of every class once the SQL signatures and the bound literals are + # in: the reader of each SQL type a class serves, the hex-WKB writer with the + # variant its type's send binds, and every trailing input by name with its value. + # A class whose codec contradicts itself stops the catalog. + idl, codec_errors = state_type_encodings(idl) + if codec_errors: + raise ValueError("type encodings that contradict themselves:\n " + + "\n ".join(codec_errors)) + # 6. Attach the temporal-covering descriptor (Parquet/Iceberg projection) print(f" Attaching temporal covering from {COVERING_PATH}...", file=sys.stderr) diff --git a/tests/test_codecs.py b/tests/test_codecs.py new file mode 100644 index 0000000..46e28cb --- /dev/null +++ b/tests/test_codecs.py @@ -0,0 +1,182 @@ +"""The codec each class states: its readers and writers per SQL type, its hex-WKB writer +with the variant its type binds, and every trailing input by name with its value. + +The synthetic catalog follows the shape #ExposabilityTests of tests/test_enrich.py builds +(``fn`` with ``c`` / ``cType`` / ``canonical``), carrying the SQL signatures and bound +literals the pass reads; the contract tests read the generated catalog, as +#TypeRecoverTests of tests/test_typerecover.py does. Plain unittest, no pytest dependency. +""" +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from parser.codecs import state_type_encodings + +IDL = Path(__file__).resolve().parents[1] / "output" / "meos-idl.json" +SET, RAST = "const Set *", "const Raster *" + + +def fn(name, ret, params, sigs=(), api="public", shape=None, typedef=None): + f = {"name": name, "api": api, + "returnType": {"c": ret, "canonical": ret}, + "params": [{"name": n, "cType": t, "canonical": t, + **({"typedef": typedef} if typedef and t == "uint64_t" else {})} + for t, n in params], + "sqlSignatures": list(sigs)} + if typedef and ret == "uint64_t": + f["returnType"]["typedef"] = typedef + if shape: + f["shape"] = shape + return f + + +def sig(name, args, ret, **kw): + return {"sqlName": name, "args": args, "ret": ret, **kw} + + +FUNCTIONS = [ + # one reader and one writer per set type, keyed by the SQL type each serves + fn("intset_in", "Set *", [("const char *", "str")], + [sig("intset_in", ["cstring"], "intset")]), + fn("floatset_in", "Set *", [("const char *", "str")], + [sig("floatset_in", ["cstring"], "floatset")]), + fn("intset_out", "char *", [(SET, "s")], + [sig("intset_out", ["intset"], "cstring"), sig("asText", ["intset"], "text")]), + fn("floatset_out", "char *", [(SET, "s"), ("int", "maxdd")], + [sig("floatset_out", ["floatset"], "cstring")]), + # a generic writer serving several types loses to each type's own + fn("spatialset_out", "char *", [(SET, "s"), ("int", "maxdd")], + [sig("geomset_out", ["geomset"], "cstring"), sig("intset_out", ["intset"], "cstring")]), + # the generic reader of one internal programmer surface is no codec + fn("set_in", "Set *", [("const char *", "str")], + [sig("intset_in", ["cstring"], "intset")], api="internal"), + # the hex-WKB pair, its size an out-parameter, its variant what send binds + fn("set_from_hexwkb", "Set *", [("const char *", "hexwkb")], + [sig("intsetFromHexWKB", ["text"], "intset"), + sig("floatsetFromHexWKB", ["text"], "floatset")]), + fn("set_as_hexwkb", "char *", + [(SET, "s"), ("uint8_t", "variant"), ("size_t *", "size_out")], + [sig("asHexWKB", ["intset", "text"], "text")], shape={"outParams": ["size_out"]}), + fn("set_as_wkb", "uint8_t *", + [(SET, "s"), ("uint8_t", "variant"), ("size_t *", "size_out")], + [sig("intset_send", ["intset"], "bytea", boundArgs={"variant": "WKB_EXTENDED"})], + shape={"outParams": ["size_out"]}), + # a type with no send of its own: its SQL hex writer leaves the byte order empty + fn("raster_from_hexwkb", "Raster *", [("const char *", "hexwkb")], + [sig("rasterFromHexWKB", ["text"], "raster")]), + fn("raster_as_hexwkb", "char *", + [(RAST, "rast"), ("uint8_t", "variant"), ("size_t *", "size_out")], + [sig("asHexWKB", ["raster", "text"], "text", argDefaults=[None, "''"])], + shape={"outParams": ["size_out"]}), + # a cell: a value of its own over uint64_t, read and written by its own functions + fn("h3index_in", "uint64_t", [("const char *", "str")], typedef="H3Index"), + fn("h3index_out", "char *", [("uint64_t", "cell")], typedef="H3Index"), +] + + +def _idl(functions=FUNCTIONS): + return {"functions": json.loads(json.dumps(functions)), + "macros": [{"name": "WKB_EXTENDED", "value": 4}], + "structs": [{"name": "Set", "fields": []}, {"name": "Raster", "fields": []}], + "typeEncodings": {"Set": {}, "Raster": {}}} + + +class CodecTests(unittest.TestCase): + """#state_type_encodings of parser/codecs.py over a synthetic catalog.""" + + def setUp(self): + idl, self.errors = state_type_encodings(_idl()) + self.te = idl["typeEncodings"] + self.structs = {s["name"]: s for s in idl["structs"]} + + def test_each_type_is_read_and_written_by_its_own_function(self): + s = self.te["Set"] + self.assertEqual(s["readers"]["text"], {"intset": "intset_in", + "floatset": "floatset_in"}) + self.assertEqual(s["writers"]["text"], {"intset": "intset_out", + "floatset": "floatset_out", + "geomset": "spatialset_out"}) + self.assertNotIn("text", s["decoders"]) + self.assertNotIn("text", s["encoders"]) + self.assertEqual(s["writerAux"]["text"]["floatset"], + [{"name": "maxdd", "kind": "integer", "default": 15}]) + self.assertEqual(self.errors, []) + + def test_one_function_for_every_type_is_the_class_codec(self): + s = self.te["Set"] + self.assertEqual(s["decoders"]["wkb"], "set_from_hexwkb") + self.assertEqual(s["in"], "set_from_hexwkb") + self.assertEqual(self.structs["Set"]["serialization"]["in"], "set_from_hexwkb") + + def test_the_hex_writer_takes_the_variant_its_send_binds(self): + s = self.te["Set"] + self.assertEqual(s["encoders"]["wkb"], "set_as_hexwkb") + self.assertEqual(s["encoderAux"]["wkb"], + [{"name": "variant", "kind": "integer", "default": 4}]) + + def test_without_a_send_the_variant_is_the_sql_writers_default(self): + r = self.te["Raster"] + self.assertEqual(r["encoders"], {"wkb": "raster_as_hexwkb"}) + self.assertEqual(r["out"], "raster_as_hexwkb") + self.assertEqual(r["encoderAux"]["wkb"][0]["default"], 0) + + def test_a_cell_is_a_class_of_its_own(self): + h = self.te["H3Index"] + self.assertEqual((h["in"], h["out"]), ("h3index_in", "h3index_out")) + + def test_two_functions_alike_for_one_type_stop_the_catalog(self): + twin = fn("intset_in2", "Set *", [("const char *", "str")], + [sig("intset_in", ["cstring"], "intset")]) + twin["name"] = "intsetx_in" + _, errors = state_type_encodings(_idl(FUNCTIONS + [twin])) + self.assertEqual(errors, ["Set: intset read by intset_in and intsetx_in"]) + + +class CodecContractTests(unittest.TestCase): + """Over the generated catalog: every trailing input a codec states names a parameter + of its function, carries the value a binding passes, and is never an out-parameter.""" + + def setUp(self): + if not IDL.exists(): + self.skipTest(f"{IDL} not generated; run `python run.py` first") + self.idl = json.loads(IDL.read_text()) + self.fns = {f["name"]: f for f in self.idl["functions"]} + + def _stated(self): + for cls, e in self.idl["typeEncodings"].items(): + for side, names in (("decoderAux", e.get("decoders") or {}), + ("encoderAux", e.get("encoders") or {})): + for enc, name in names.items(): + yield cls, name, (e.get(side) or {}).get(enc, []) + for side, aux in (("readers", "readerAux"), ("writers", "writerAux")): + for enc, by_type in (e.get(side) or {}).items(): + for t, name in by_type.items(): + yield cls, name, e[aux][enc][t] + if e.get("bytes", {}).get("encoderAux") is not None: + yield cls, e["bytes"]["encoder"], e["bytes"]["encoderAux"] + + def test_every_trailing_input_is_named_valued_and_no_out_parameter(self): + bad = [] + for cls, name, aux in self._stated(): + f = self.fns[name] + params = {p["name"] for p in f["params"][1:]} + out = set((f.get("shape") or {}).get("outParams") or ()) + for a in aux: + if a["name"] not in params or "default" not in a or a["name"] in out: + bad.append((cls, name, a)) + self.assertEqual(bad, []) + + def test_the_classes_a_binding_reads(self): + te = self.idl["typeEncodings"] + self.assertEqual(te["Set"]["readers"]["text"]["intset"], "intset_in") + self.assertEqual(te["Temporal"]["writers"]["text"]["tfloat"], "tfloat_out") + self.assertEqual(te["Raster"]["out"], "raster_as_hexwkb") + for cell in ("H3Index", "Quadbin", "S2CellId"): + self.assertIn(cell, te) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_engine_integration.py b/tests/test_engine_integration.py index b3dfdd9..7a8f2dd 100644 --- a/tests/test_engine_integration.py +++ b/tests/test_engine_integration.py @@ -5,11 +5,11 @@ MEOS_LIBRARY_PATH=/usr/local/lib/libmeos.so python3 tests/test_engine_integration.py -It drives the exact path the server uses, including the catalog's -``in_aux``/``out_aux`` defaults (so the *generic* ``temporal_out(temp, -maxdd=15)`` is called correctly — proving it serialises any subtype), and -asserts that bad input raises ``MeosError`` instead of terminating the -process (MEOS's default handler calls ``exit()``). +It drives the exact path the server uses: a temporal value is read and +written through the reader and writer the catalog states for its SQL type, +each called with the trailing inputs the catalog states by name +(``tfloat_out(temp, maxdd=15)``), and bad input raises ``MeosError`` instead of +terminating the process (MEOS's default handler calls ``exit()``). """ import json @@ -28,18 +28,13 @@ _TBOOL = "{t@2000-01-01, f@2000-01-03, t@2000-01-05}" _TFLOAT = "{1.5@2000-01-01, 3.5@2000-01-03}" -# A three-instant literal per temporal subtype, keyed by the decoder that reads -# it. Which decoder the catalog selects for the opaque `Temporal` is settled by -# an alphabetical tiebreak that build_type_encodings() documents as arbitrary: -# there is no generic `temporal_in`, so the pick is whichever subtype sorts -# first, and it MOVES when MobilityDB gains a type (tbigint displaced tbool). -# The fixture therefore follows the pick instead of naming it. -_LITERAL_BY_DECODER = { - "tbool_in": _TBOOL, - "tint_in": "{1@2000-01-01, 2@2000-01-03, 1@2000-01-05}", - "tbigint_in": "{1@2000-01-01, 2@2000-01-03, 1@2000-01-05}", - "tfloat_in": "{1.5@2000-01-01, 3.5@2000-01-03, 1.5@2000-01-05}", - "ttext_in": "{AA@2000-01-01, BB@2000-01-03, AA@2000-01-05}", +# A temporal value of each SQL type, read through the reader the catalog states +# for that type: `Temporal` serves twenty types and has no generic public reader. +_LITERAL_BY_TYPE = { + "tbool": _TBOOL, + "tint": "{1@2000-01-01, 2@2000-01-03, 1@2000-01-05}", + "tfloat": "{1.5@2000-01-01, 3.5@2000-01-03, 1.5@2000-01-05}", + "ttext": "{AA@2000-01-01, BB@2000-01-03, AA@2000-01-05}", } _KIND_TAG = {"integer": "int", "number": "double", @@ -58,41 +53,36 @@ def setUpClass(cls): te = (json.loads(_CATALOG.read_text()).get("typeEncodings", {}) if _CATALOG.exists() else {}) t = te.get("Temporal", {}) - cls.tin = t.get("in", "tbool_in") - cls.tout = t.get("out", "tbool_out") - cls.in_aux = _aux(t.get("in_aux", [])) - cls.out_aux = _aux(t.get("out_aux", [])) - cls.tin_literal = _LITERAL_BY_DECODER.get(cls.tin) - - def test_catalog_selected_in_out(self): - # Decoding stays a typed wrapper (subtype-narrow); encoding is the - # generic temporal_out with a defaulted maxdd. - # - # WHICH subtype decodes is not asserted: no `temporal_in` exists, so - # build_type_encodings() falls back to an alphabetical pick it calls - # arbitrary, and that pick moves when MobilityDB gains a type. What the - # design does guarantee is asserted instead — the decoder is one of the - # subtype-narrow readers, and the encoder IS the generic root. - self.assertIn(self.tin, _LITERAL_BY_DECODER, - f"catalog selected {self.tin!r} as the Temporal decoder; " - f"add its literal to _LITERAL_BY_DECODER") - self.assertEqual(self.tout, "temporal_out") - self.assertEqual(self.out_aux, [("int", 15)]) + cls.readers = (t.get("readers") or {}).get("text", {}) + cls.reader_aux = (t.get("readerAux") or {}).get("text", {}) + cls.writers = (t.get("writers") or {}).get("text", {}) + cls.writer_aux = (t.get("writerAux") or {}).get("text", {}) + + def read(self, sqltype, literal): + return self.eng.decode(self.readers[sqltype], literal, + _aux(self.reader_aux[sqltype])) + + def write(self, sqltype, handle): + return self.eng.encode(self.writers[sqltype], handle, + _aux(self.writer_aux[sqltype])) + + def test_each_type_states_its_own_reader_and_writer(self): + for sqltype in _LITERAL_BY_TYPE: + self.assertEqual(self.readers[sqltype], sqltype + "_in") + self.assertEqual(self.writers[sqltype], sqltype + "_out") + self.assertEqual(_aux(self.writer_aux["tfloat"]), [("int", 15)]) def test_decode_invoke_scalar(self): - h = self.eng.decode(self.tin, self.tin_literal, self.in_aux) - self.assertTrue(h) - n = self.eng.invoke("temporal_num_instants", [("ptr", h)], "int") - self.assertEqual(n, 3) - - def test_generic_encoder_round_trips_any_subtype(self): - # The whole point of the gap fix: temporal_out(+maxdd) serialises - # a tbool AND a tfloat — a subtype-narrow tbool_out could not. - hb = self.eng.decode("tbool_in", _TBOOL) - ob = self.eng.encode(self.tout, hb, self.out_aux) + for sqltype, literal in _LITERAL_BY_TYPE.items(): + h = self.read(sqltype, literal) + self.assertTrue(h, sqltype) + n = self.eng.invoke("temporal_num_instants", [("ptr", h)], "int") + self.assertEqual(n, 3, sqltype) + + def test_each_type_round_trips_through_its_own_writer(self): + ob = self.write("tbool", self.read("tbool", _TBOOL)) self.assertIn("@", ob) - hf = self.eng.decode("tfloat_in", _TFLOAT) - of = self.eng.encode(self.tout, hf, self.out_aux) + of = self.write("tfloat", self.read("tfloat", _TFLOAT)) self.assertIn("@", of) self.assertIn("1.5", of) @@ -137,7 +127,7 @@ def test_input_array_builder_round_trip(self): "temporal_merge_array", [("ptrarray", [h1, h2]), ("int", 2)], "ptr") self.assertTrue(merged) - out = self.eng.encode(self.tout, merged, self.out_aux) + out = self.write("tbool", merged) self.assertIn("@", out) self.assertIn("2000-01-03", out) # both instants merged in diff --git a/tests/test_typerecover.py b/tests/test_typerecover.py index cbce880..2446f98 100644 --- a/tests/test_typerecover.py +++ b/tests/test_typerecover.py @@ -243,6 +243,18 @@ def canon(name, pname): self.assertEqual(canon("pg_time_in", "typmod"), "int32_t") self.assertEqual(canon("set_as_wkb", "variant"), "uint8_t") + def test_a_type_of_its_own_keeps_its_name_beside_its_width(self): + # #test_cell_id_canonical_normalized_uniform holds the cells' width; typedef + # states what the value is, on the three cell ids and on no PostgreSQL type or + # width name + idl = json.loads(IDL.read_text()) + slots = [s for f in idl["functions"] + for s in [f["returnType"]] + f.get("params", [])] + named = {(s["typedef"], _base(s["canonical"])) for s in slots if "typedef" in s} + self.assertEqual(named, {("H3Index", "uint64_t"), ("Quadbin", "uint64_t"), + ("S2CellId", "uint64_t")}) + self.assertEqual(self.by_name["h3index_in"]["returnType"]["typedef"], "H3Index") + # C's own integer names: what a typedef must never be stated as. _C_INTEGERS = {