From e59a53caa7f059419e011cbaa06ec380d30339d4 Mon Sep 17 00:00:00 2001 From: Matthew Date: Tue, 9 Jun 2026 09:58:08 +0100 Subject: [PATCH 1/4] Independent module using log dependency. With passing unit tests --- di/html/html.md | 165 ++++++++++++++++++++++++++++++++++++++ di/html/html.q | 201 +++++++++++++++++++++++++++++++++++++++++++++++ di/html/init.q | 4 + di/html/test.csv | 40 ++++++++++ 4 files changed, 410 insertions(+) create mode 100644 di/html/html.md create mode 100644 di/html/html.q create mode 100644 di/html/init.q create mode 100644 di/html/test.csv diff --git a/di/html/html.md b/di/html/html.md new file mode 100644 index 00000000..c1d98d59 --- /dev/null +++ b/di/html/html.md @@ -0,0 +1,165 @@ +# di.html + +WebSocket pub/sub and HTML page serving module, extracted from TorQ. + +## Overview + +This module does two things: + +1. **Pub/sub over WebSockets** — allows browser clients to subscribe to kdb+ tables and receive live updates as data is published. +2. **HTML page serving** — reads HTML files from a configured directory and serves them over HTTP, optionally replacing server/port placeholders so the page can connect back to itself. + +## Usage + +```q +html:use`di.html + +/ minimal setup - log dep is required, handlers dep is optional +logdep:`info`warn`error!( + {[c;m] -1 "INFO ",(string c)," ",m;}; + {[c;m] -1 "WARN ",(string c)," ",m;}; + {[c;m] -2 "ERROR ",(string c)," ",m;}) +html.init[(enlist `homedir)!enlist "/opt/app/html";enlist[`log]!enlist logdep] + +/ register tables for pub/sub +html.addtables[`trades`quotes] + +/ publish data to subscribers +html.pub[`trades;newdata] +``` + +## init + +```q +html.init[config;deps] +``` + +### config keys + +| Key | Type | Description | Required | +|---|---|---|---| +| `homedir` | string | Path to the directory containing HTML files | Yes | + +### deps keys + +| Key | Description | If absent | +|---|---|---| +| `` `log `` | Logging function dict with keys `` `info`warn`error `` | **Required** — `init` signals an error | +| `` `handlers `` | Handler registration dict with key `` `register `` | Assigns `.z.wc` and `.z.ws` directly | + +## Exported functions + +### addtables + +```q +html.addtables[tablelist] +``` + +Registers a list of table names for pub/sub. Can be called multiple times to add new tables. Sets a default modifier that JSON-encodes updates before sending to subscribers. + +### pub + +```q +html.pub[tbl;data] +``` + +Publishes `data` for `tbl` to all currently subscribed handles. + +### sub + +```q +html.sub[tbl;syms] +``` + +Subscribes the current handle (`.z.w`) to `tbl`. Pass `` ` `` as `syms` to receive all data. Pass `` ` `` as `tbl` to subscribe to all registered tables. Returns `(tablename; current data)` so the subscriber can initialise their local copy of the table. + +### wssub + +```q +html.wssub[tbl] +``` + +Calls `sub[tbl;`` ` ``]`. Pass `` ` `` as `tbl` to subscribe to all registered tables. Returns nothing. + +### end + +```q +html.end[eodval] +``` + +Broadcasts an end-of-day message to all subscriber handles. + +### readpage + +```q +html.readpage[filename] +``` + +Reads the file at `homedir/filename` and returns its contents as a string. Returns an error message string if the file is not found. + +### readpagereplaceHP + +```q +html.readpagereplaceHP[filename] +``` + +Reads the file at `homedir/filename` and replaces `MYKDBSERVER` and `MYKDBPORT` tokens with the process's current IP address and port. Used to serve self-referencing HTML pages over HTTP. + +### evaluate + +```q +html.evaluate[inputdict] +``` + +Takes a q dictionary (already deserialised from JSON), extracts the `func` key, calls the named function with any additional keys as arguments, and returns the result. Used internally by the `.z.ws` handler — the handler does the JSON deserialisation before calling this function. + +## WebSocket handler + +The module registers a `.z.ws` handler that receives bytes from the browser, deserialises them to a q dict, calls `evaluate`, JSON-encodes the result, and sends it back. A `.z.wc` handler cleans up subscriptions when a connection closes. + +## Logging + +The module logs at three points: + +- On `init`: confirms the module started and which `homedir` was set +- On `readpage`: warns if a requested file is not found +- On `evaluate`: logs at error level when a WebSocket-invoked function fails (the error is also re-thrown to the caller) + +The `log` dependency is required — `init` signals an error if it is absent. Internally the loggers are stored on `.z.m` as `lginfo`/`lgwarn`/`lgerr` (not `log`, which is a q built-in) and every call site invokes them through `.z.m`. + +## Example with injected dependencies + +The `log` dep is required and the `handlers` dep is optional; both are supplied by +the host application. The `log` functions are called as `(ctx;msg)` and the +`handlers` registry's `register` is called as `(.z event name; label; handler)`. + +```q +/ wire up logging on top of the kx.log module +/ kx.log loggers take a single message, so wrap them to the (ctx;msg) shape +logger:use`kx.log +kxlog:logger.createLog[] +logdep:`info`warn`error!( + {[c;m] kxlog.info[(string c),": ",m]}; + {[c;m] kxlog.warn[(string c),": ",m]}; + {[c;m] kxlog.error[(string c),": ",m]}) + +/ wire up handlers via a host-provided registry +/ a real registry composes handlers so several modules can share .z.ws / .z.wc +/ omit this dep entirely to have the module assign .z.ws / .z.wc directly +hnddep:enlist[`register]!enlist {[zname;label;fn] zname set fn} + +/ initialise html module +html:use`di.html +html.init[(enlist `homedir)!enlist "/opt/app/html";`log`handlers!(logdep;hnddep)] +``` + +## Testing + +```q +k4unit:use`di.k4unit +k4unit.moduletest`di.html +``` + +All exported functions are covered. The one path not unit-tested is `pub` physically +delivering a message over a live WebSocket connection, which requires a real client +handle. diff --git a/di/html/html.q b/di/html/html.q new file mode 100644 index 00000000..57c0e384 --- /dev/null +++ b/di/html/html.q @@ -0,0 +1,201 @@ +/ di.html - websocket pub/sub and html page serving + +/ list of table names registered for pub/sub +subtables:`symbol$(); + +/ subscriber list per table: each entry is (handle; sym-filter) +subs:()!(); + +/ modifier function per table: transforms data before sending to subscriber +modifier:()!(); + +/ cache of resolved ip addresses: int -> symbol +ipacache:(`int$())!`symbol$(); + +/ home directory for html files - set by init +homedir:""; + +/ logging functions - required injected deps, wired by init via .z.m + +/ converts a list of timestamps or dates to iso 8601 strings e.g. "2024-01-02T12:00:00Z" +jstsiso8601:{[x] {("-" sv "." vs string `date$x),"T",string[`second$x],"Z"} each x}; + +/ converts a list of dates to javascript epoch milliseconds +/ kdb dates are days since 2000-01-01; 10957 is the day offset from 1970-01-01 to 2000-01-01 +jstsfromd:{[x] "j"$86400000 * 10957 + `long$x}; + +/ converts a list of time, second, or minute values to milliseconds since midnight +jstsfromt:{[x] "j"$"t"$x}; + +/ converts a list of months to javascript epoch milliseconds via the first day of each month +jstsfromm:{[x] jstsfromd `date$x}; + +/ maps kdb type shorts to their javascript converter function +/ types not listed here are left unchanged by jsformat +typemap:12 13 14 16 17 18 19h!(jstsiso8601;jstsfromm;jstsfromd;jstsfromt;jstsfromt;jstsfromt;jstsfromt); + +jsformat:{[tbl] + / applies the correct javascript converter to each column of a table that needs it + / columns whose type is not in the typemap are passed through unchanged (via ::) + coldict:flip 0!tbl; + k:key coldict; + colvals:value coldict; + converters:typemap type each colvals; + :flip k!converters@'colvals; + }; + +updformat:{[msgtype;msgdata] + / wraps an upd message into a name/data dictionary with javascript-formatted table data + formatteddata:(key msgdata)!(msgdata`tablename;jsformat msgdata`tabledata); + :(`name`data)!(msgtype;formatteddata); + }; + +/ filter applied before sending data to a subscriber - returns full table (no filtering) +sel:{[tbl;syms] tbl}; + +del:{[tbl;handle] + / removes a handle from the subscriber list for a table + / if handle is not found, ? returns count of list and drop has no effect + idx:subs[tbl;;0]?handle; + .z.m.subs:@[subs;tbl;_;idx]; + }; + +add:{[tbl;syms] + / adds the current handle to the subscriber list for a table + / if already subscribed, updates the sym filter by taking union with new syms + / if not subscribed, appends a new (handle; syms) pair to the list + / returns (tablename; current data) so the subscriber can initialise their local copy + i:subs[tbl;;0]?.z.w; + .z.m.subs:$[(count subs tbl)>i; + .[subs;(tbl;i;1);union;syms]; + @[subs;tbl;,;enlist(.z.w;syms)] + ]; + :(tbl;$[99=type v:value tbl;sel[v;syms];@[0#v;`sym;`g#]]); + }; + +closehandle:{[handle] + / removes the given handle from all subscriber lists when a connection closes + del[;handle] each subtables; + }; + +replace:{[str;findreplace] + / applies each find->replace pair to the string in sequence using ssr + :(ssr/)[str;string key findreplace;value findreplace]; + }; + +ipa:{[ipint] + / resolves an ip address integer to a hostname symbol, caching results + / tries .Q.host first; falls back to converting the ip bytes manually if that fails + if[not `~r:ipacache ipint;:r]; + hostname:.Q.host ipint; + r:$[`~hostname;`$"."sv string "i"$0x0 vs ipint;hostname]; + .z.m.ipacache:@[ipacache;ipint;:;r]; + :r; + }; + +/ returns the current listen port as a string +getport:{[] string system "p"}; + +execdict:{[inputdict] + / extracts the func key and any additional args from a dictionary and calls the function + / args are passed to the function in the order the keys appear after func + if[not `func in key inputdict;'"no func in dictionary"]; + f:value inputdict`func; + args:value inputdict _ `func; + :$[1=count key inputdict;f @ 1;f . args]; + }; + +/ websocket message handler - module-level so it carries the module context for evaluate +wshandler:{neg[.z.w] -8!.j.j[evaluate[.j.k -9!x]];}; + +addtables:{[tablelist] + / registers a list of tables for pub/sub and sets their default modifier + / can be called multiple times; already-registered tables are ignored + tablelist,:(); + new:tablelist except subtables; + .z.m.subtables:subtables,new; + .z.m.subs:subs,new!(count new)#(); + .z.m.modifier:modifier,new!(count new)#{-8!.j.j updformat["upd";`tablename`tabledata!(x 1;x 2)]}; + if[count new;.z.m.lginfo[`html;"registered tables: ",", " sv string new]]; + }; + +pub:{[tbl;data] + / publishes data to all current subscribers of a table + / applies the table modifier before sending (default modifier json-encodes the data) + {[tbl;data;s] + if[count data:sel[data;s 1]; + (neg first s) modifier[tbl]@(`upd;tbl;data)]; + }[tbl;data] each subs tbl; + }; + +sub:{[tbl;syms] + / subscribes the current handle to a table with an optional sym filter + / pass backtick as tbl to subscribe to all registered tables + / removes any existing subscription for this handle before re-adding + if[tbl~`;:sub[;syms] each subtables]; + if[not tbl in subtables;'tbl]; + del[tbl;.z.w]; + :add[tbl;syms]; + }; + +wssub:{[tbl] + / subscribes via websocket, no return value + sub[tbl;`]; + }; + +end:{[eodval] + / broadcasts end-of-day message to all subscriber handles across all tables + (neg union/[subs[;;0]])@\:(`.u.end;eodval); + }; + +readpage:{[filename] + / reads an html file from the configured home directory and returns it as a string + / returns a "not found" message string if the file does not exist + p:homedir,"/",filename; + r:@[read1;`$":",p;""]; + if[not count r;.z.m.lgwarn[`html;p,": not found"]]; + :$[count r;"c"$r;p,": not found"]; + }; + +readpagereplaceHP:{[filename] + / reads a page and replaces MYKDBSERVER and MYKDBPORT tokens with live server values + :replace[readpage filename;`MYKDBSERVER`MYKDBPORT!("\"",(string ipa .z.a),"\"";getport[])]; + }; + +evaluate:{[inputdict] + / safely calls execdict on the input, logging then re-throwing any errors with context + :@[execdict;inputdict;{[d;e] m:"failed to execute ",(-3!d)," : ",e;.z.m.lgerr[`html;m];'m}[inputdict]]; + }; + +init:{[config;deps] + / sets up module state from config and registers websocket handlers + / log dep is required; handlers dep is optional + + / log dependency is required - guard that deps is a dict, log key is present and not (::) + logdict:$[99h=type deps;$[(`log in key deps)and not(::)~deps`log;deps`log;()!()];()!()]; + if[not count logdict; + '"di.html: log dependency is required; pass `info`warn`error functions - see di.log for a default implementation"; + ]; + .z.m.lginfo:logdict`info; + .z.m.lgwarn:logdict`warn; + .z.m.lgerr:logdict`error; + + / store config + .z.m.homedir:config`homedir; + + / register .h content type handlers - protected in case not available in kdb-x + @[{.h.tx[`non]:{enlist x};.h.ty[`non]:"text/html"};`;{[e]}]; + + / register handlers via dep if provided + if[`handlers in key deps; + deps[`handlers][`register][`.z.ws;`html.ws;wshandler]; + deps[`handlers][`register][`.z.wc;`html.close;closehandle]; + :()]; + + / no handlers dep: assign directly, wrapping any existing .z.wc to preserve it + .z.ws:wshandler; + .z.wc:@[value;`.z.wc;{{}}]; + .z.wc:{[existing;h] closehandle h; existing h}[.z.wc;]; + + .z.m.lginfo[`html;"initialised with homedir: ",homedir]; + }; diff --git a/di/html/init.q b/di/html/init.q new file mode 100644 index 00000000..c86244a9 --- /dev/null +++ b/di/html/init.q @@ -0,0 +1,4 @@ +/ entry point for di.html module +\l ::html.q + +export:([init;addtables;pub;sub;wssub;end;readpage;readpagereplaceHP;evaluate]) diff --git a/di/html/test.csv b/di/html/test.csv new file mode 100644 index 00000000..583aa277 --- /dev/null +++ b/di/html/test.csv @@ -0,0 +1,40 @@ +action,ms,bytes,lang,code,repeat,minver,comment +before,0,0,q,html:use`di.html,1,,load di.html module +before,0,0,q,"mocklog:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,,mock log dep that discards all messages +before,0,0,q,"html.init[(enlist `homedir)!enlist ""/tmp"";enlist[`log]!enlist[mocklog]]",1,,initialise module with /tmp as html home directory +before,0,0,q,"(`:/tmp/di_html_test.html) 1: ""test""",1,,write test html file using q file io +before,0,0,q,"(`:/tmp/di_html_hp_test.html) 1: ""MYKDBSERVER:MYKDBPORT""",1,,write test file containing server and port tokens +before,0,0,q,htmltestfn:{x-y},1,,helper function for multi-arg evaluate test +before,0,0,q,"trades:([]sym:`g#`symbol$();px:`float$())",1,,real global table so sub success path can read its contents +before,0,0,q,"quotes:([]sym:`g#`symbol$();bid:`float$())",1,,real global table so subscribing to all tables can read its contents +before,0,0,q,"hns:` sv `.m.di,first (key `.m.di) where (key `.m.di) like ""*html""",1,,resolve the html module's internal namespace (avoids hardcoding the load-order prefix) +before,0,0,q,"jsf:get ` sv hns,`jsformat",1,,grab the internal jsformat converter so its date logic can be tested directly +before,0,0,q,"convtest:([]ts:enlist 2024.01.02D12:00:00.000000000;d:enlist 2024.01.02;tm:enlist 12:00:00.000;sym:enlist `AAPL)",1,,sample table with one column of each time type plus an untyped sym column + +run,0,0,q,"html.addtables[`trades`quotes]",1,,register two tables for pub/sub +run,0,0,q,"html.addtables[`trades]",1,,re-registering an existing table does not error +run,0,0,q,"html.pub[`trades;([]a:enlist 1i)]",1,,pub with no subscribers completes without error + +fail,0,0,q,"html.sub[`notregistered;`]",1,,subscribing to an unregistered table throws an error +run,0,0,q,"html.end[`eod]",1,,end is a clean no-op when there are no subscribers (tested before any console sub) +true,0,0,q,"`trades~first html.sub[`trades;`]",1,,sub success path returns the table name as first element +true,0,0,q,"0=count last html.sub[`trades;`]",1,,sub success path returns the current (empty) table contents +run,0,0,q,"html.wssub[`trades]",1,,wssub delegates to sub without error +run,0,0,q,"html.sub[`;`]",1,,subscribing with backtick subscribes to all registered tables + +true,0,0,q,"-42~html.evaluate[`func`arg1!(""neg"";42)]",1,,evaluate calls a function with one arg correctly +true,0,0,q,"-1~html.evaluate[`func`arg1`arg2!(""htmltestfn"";1;2)]",1,,evaluate calls a function with multiple args correctly +true,0,0,q,"-1~html.evaluate[(enlist `func)!enlist ""neg""]",1,,evaluate with only func key calls f@1 - preserving TorQ behaviour +fail,0,0,q,"html.evaluate[(enlist `arg1)!enlist 1]",1,,evaluate throws when the func key is missing + +true,0,0,q,"""test""~html.readpage[""di_html_test.html""]",1,,readpage returns file contents as a string +true,0,0,q,"""/tmp/di_html_notfound.html: not found""~html.readpage[""di_html_notfound.html""]",1,,readpage returns a not-found message string for a missing file +true,0,0,q,"0=count html.readpagereplaceHP[""di_html_hp_test.html""] ss ""MYKDBSERVER""",1,,readpagereplaceHP replaces the MYKDBSERVER token +true,0,0,q,"0=count html.readpagereplaceHP[""di_html_hp_test.html""] ss ""MYKDBPORT""",1,,readpagereplaceHP replaces the MYKDBPORT token + +true,0,0,q,"""2024-01-02T12:00:00Z""~first (jsf convtest)`ts",1,,jsformat converts a timestamp column to an iso 8601 string +true,0,0,q,"1704153600000~first (jsf convtest)`d",1,,jsformat converts a date column to javascript epoch milliseconds +true,0,0,q,"43200000~first (jsf convtest)`tm",1,,jsformat converts a time column to milliseconds since midnight +true,0,0,q,"`AAPL~first (jsf convtest)`sym",1,,jsformat leaves columns whose type is not in the typemap unchanged + +after,0,0,q,"system ""rm /tmp/di_html_test.html /tmp/di_html_hp_test.html""",1,,remove test html files created during setup From 259738e0670bf085f06d1954b5a8a2ff422d9b40 Mon Sep 17 00:00:00 2001 From: Matthew Date: Thu, 11 Jun 2026 15:02:39 +0100 Subject: [PATCH 2/4] Changed html to be in line with other modules --- di/html/html.md | 59 ++++++++++++++++++++++----------------- di/html/html.q | 72 ++++++++++++++++++++++++++++++++---------------- di/html/init.q | 2 +- di/html/test.csv | 14 ++++++++-- 4 files changed, 96 insertions(+), 51 deletions(-) diff --git a/di/html/html.md b/di/html/html.md index c1d98d59..d538af4d 100644 --- a/di/html/html.md +++ b/di/html/html.md @@ -14,12 +14,9 @@ This module does two things: ```q html:use`di.html -/ minimal setup - log dep is required, handlers dep is optional -logdep:`info`warn`error!( - {[c;m] -1 "INFO ",(string c)," ",m;}; - {[c;m] -1 "WARN ",(string c)," ",m;}; - {[c;m] -2 "ERROR ",(string c)," ",m;}) -html.init[(enlist `homedir)!enlist "/opt/app/html";enlist[`log]!enlist logdep] +/ minimal setup - all config is optional +/ homedir defaults to the KDBHTML env var (else "html"), logging defaults to the console +html.init[] / register tables for pub/sub html.addtables[`trades`quotes] @@ -28,24 +25,29 @@ html.addtables[`trades`quotes] html.pub[`trades;newdata] ``` +This mirrors the original TorQ deployment: set `KDBHTML`, initialise, register tables. + ## init ```q -html.init[config;deps] +html.init[] +html.init[configs] ``` -### config keys +`configs` is an optional dictionary — call `init[]` to use the defaults. Only recognised keys are picked up: -| Key | Type | Description | Required | +| Key | Type | Description | Default | |---|---|---|---| -| `homedir` | string | Path to the directory containing HTML files | Yes | +| `homedir` | string | Path to the directory containing HTML files | `KDBHTML` env var, else `"html"` (TorQ behaviour) | +| `` `log `` | dict | Logging functions with keys `` `info`warn`error ``, each called as `(ctx;msg)` | Console loggers — info/warn to stdout, error to stderr | +| `` `handlers `` | dict | Handler registry with key `` `register `` | Assigns `.z.ws`, `.z.wc` and `.z.pc` directly | -### deps keys +```q +/ override config explicitly +html.init[`homedir`log!("/opt/app/html";logdict)] +``` -| Key | Description | If absent | -|---|---|---| -| `` `log `` | Logging function dict with keys `` `info`warn`error `` | **Required** — `init` signals an error | -| `` `handlers `` | Handler registration dict with key `` `register `` | Assigns `.z.wc` and `.z.ws` directly | +`init` also sets `.h.HOME` to `homedir` (protected, skipped if `.h` is unavailable) so the default HTTP handler serves static assets (css/js/img) from the same directory — the equivalent of TorQ's `KDBHTML` behaviour. ## Exported functions @@ -89,6 +91,14 @@ html.end[eodval] Broadcasts an end-of-day message to all subscriber handles. +### dataformat + +```q +html.dataformat[msgtype;msgdata] +``` + +Wraps a message into a `` `name`data `` dictionary, javascript-formatting each table in `msgdata` (a list or dictionary of tables). Used by host data functions that the front end requests over the websocket, e.g. TorQ's monitor `start` call returning several tables at once. + ### readpage ```q @@ -115,7 +125,7 @@ Takes a q dictionary (already deserialised from JSON), extracts the `func` key, ## WebSocket handler -The module registers a `.z.ws` handler that receives bytes from the browser, deserialises them to a q dict, calls `evaluate`, JSON-encodes the result, and sends it back. A `.z.wc` handler cleans up subscriptions when a connection closes. +The module registers a `.z.ws` handler that receives bytes from the browser, deserialises them to a q dict, calls `evaluate`, JSON-encodes the result, and sends it back. Subscriptions are cleaned up when a connection closes via both `.z.wc` (websocket) and `.z.pc` (IPC), as in TorQ — `sub` can also be called over a plain IPC handle. In the direct-assignment path (no `handlers` config) any existing `.z.wc`/`.z.pc` handlers are preserved by wrapping, and the wiring happens only once across repeated `init` calls. ## Logging @@ -125,12 +135,11 @@ The module logs at three points: - On `readpage`: warns if a requested file is not found - On `evaluate`: logs at error level when a WebSocket-invoked function fails (the error is also re-thrown to the caller) -The `log` dependency is required — `init` signals an error if it is absent. Internally the loggers are stored on `.z.m` as `lginfo`/`lgwarn`/`lgerr` (not `log`, which is a q built-in) and every call site invokes them through `.z.m`. +Logging defaults to the console (info/warn to stdout, error to stderr); pass a `log` dict to `init` to integrate with a real logging module. Internally the loggers are stored on `.z.m` as `lginfo`/`lgwarn`/`lgerr` (not `log`, which is a q built-in) and every call site invokes them through `.z.m`. -## Example with injected dependencies +## Example with custom log and handlers config -The `log` dep is required and the `handlers` dep is optional; both are supplied by -the host application. The `log` functions are called as `(ctx;msg)` and the +Both keys are optional overrides. The `log` functions are called as `(ctx;msg)` and the `handlers` registry's `register` is called as `(.z event name; label; handler)`. ```q @@ -138,19 +147,19 @@ the host application. The `log` functions are called as `(ctx;msg)` and the / kx.log loggers take a single message, so wrap them to the (ctx;msg) shape logger:use`kx.log kxlog:logger.createLog[] -logdep:`info`warn`error!( +logdict:`info`warn`error!( {[c;m] kxlog.info[(string c),": ",m]}; {[c;m] kxlog.warn[(string c),": ",m]}; {[c;m] kxlog.error[(string c),": ",m]}) / wire up handlers via a host-provided registry -/ a real registry composes handlers so several modules can share .z.ws / .z.wc -/ omit this dep entirely to have the module assign .z.ws / .z.wc directly -hnddep:enlist[`register]!enlist {[zname;label;fn] zname set fn} +/ a real registry composes handlers so several modules can share .z.ws / .z.wc / .z.pc +/ omit this key entirely to have the module assign .z.ws / .z.wc / .z.pc directly +hnddict:enlist[`register]!enlist {[zname;label;fn] zname set fn} / initialise html module html:use`di.html -html.init[(enlist `homedir)!enlist "/opt/app/html";`log`handlers!(logdep;hnddep)] +html.init[`homedir`log`handlers!("/opt/app/html";logdict;hnddict)] ``` ## Testing diff --git a/di/html/html.q b/di/html/html.q index 57c0e384..c981d048 100644 --- a/di/html/html.q +++ b/di/html/html.q @@ -15,7 +15,16 @@ ipacache:(`int$())!`symbol$(); / home directory for html files - set by init homedir:""; -/ logging functions - required injected deps, wired by init via .z.m +/ flag so direct .z handler wiring happens only once across repeated init calls +zwired:0b; + +/ logging functions - default to the console, overridable via the log config key on init + +/ default console loggers, called as (ctx;msg) +deflog:`info`warn`error!( + {[c;m] -1 "INFO ",(string c)," ",m;}; + {[c;m] -1 "WARN ",(string c)," ",m;}; + {[c;m] -2 "ERROR ",(string c)," ",m;}); / converts a list of timestamps or dates to iso 8601 strings e.g. "2024-01-02T12:00:00Z" jstsiso8601:{[x] {("-" sv "." vs string `date$x),"T",string[`second$x],"Z"} each x}; @@ -32,7 +41,7 @@ jstsfromm:{[x] jstsfromd `date$x}; / maps kdb type shorts to their javascript converter function / types not listed here are left unchanged by jsformat -typemap:12 13 14 16 17 18 19h!(jstsiso8601;jstsfromm;jstsfromd;jstsfromt;jstsfromt;jstsfromt;jstsfromt); +typemap:12 13 14 15 16 17 18 19h!(jstsiso8601;jstsfromm;jstsfromd;jstsiso8601;jstsfromt;jstsfromt;jstsfromt;jstsfromt); jsformat:{[tbl] / applies the correct javascript converter to each column of a table that needs it @@ -50,6 +59,12 @@ updformat:{[msgtype;msgdata] :(`name`data)!(msgtype;formatteddata); }; +dataformat:{[msgtype;msgdata] + / wraps a message into a name/data dictionary, javascript-formatting each table in msgdata + / msgdata is a list or dictionary of tables - used by host data functions requested from the front end + :(`name`data)!(msgtype;jsformat each msgdata); + }; + / filter applied before sending data to a subscriber - returns full table (no filtering) sel:{[tbl;syms] tbl}; @@ -167,35 +182,46 @@ evaluate:{[inputdict] :@[execdict;inputdict;{[d;e] m:"failed to execute ",(-3!d)," : ",e;.z.m.lgerr[`html;m];'m}[inputdict]]; }; -init:{[config;deps] - / sets up module state from config and registers websocket handlers - / log dep is required; handlers dep is optional +init:{[configs] + / sets up module state and registers websocket handlers + / configs is an optional dict - recognised keys are homedir, log and handlers + / defaults: homedir from the KDBHTML env var (else "html"), console logging, direct .z handler assignment + + / default configuration values + hd:$[count e:getenv`KDBHTML;e;"html"]; + logdict:deflog; + hnd:(::); + + / set custom config values - only recognised keys are picked up + if[not configs~(::); + if[`homedir in key configs;hd:configs`homedir]; + if[`log in key configs;logdict:configs`log]; + if[`handlers in key configs;hnd:configs`handlers]]; - / log dependency is required - guard that deps is a dict, log key is present and not (::) - logdict:$[99h=type deps;$[(`log in key deps)and not(::)~deps`log;deps`log;()!()];()!()]; - if[not count logdict; - '"di.html: log dependency is required; pass `info`warn`error functions - see di.log for a default implementation"; - ]; .z.m.lginfo:logdict`info; .z.m.lgwarn:logdict`warn; .z.m.lgerr:logdict`error; + .z.m.homedir:hd; - / store config - .z.m.homedir:config`homedir; + / register .h content type handlers and static file root - protected in case not available in kdb-x + / .h.HOME lets the default http handler serve static assets (css/js/img) from homedir, as TorQ does via KDBHTML + @[{.h.HOME:x;.h.tx[`non]:{enlist x};.h.ty[`non]:"text/html"};homedir;{[e]}]; - / register .h content type handlers - protected in case not available in kdb-x - @[{.h.tx[`non]:{enlist x};.h.ty[`non]:"text/html"};`;{[e]}]; + .z.m.lginfo[`html;"initialised with homedir: ",homedir]; - / register handlers via dep if provided - if[`handlers in key deps; - deps[`handlers][`register][`.z.ws;`html.ws;wshandler]; - deps[`handlers][`register][`.z.wc;`html.close;closehandle]; + / register handlers via the handlers config if provided + / closehandle is registered for both websocket (.z.wc) and ipc (.z.pc) closes, as in TorQ + if[not hnd~(::); + hnd[`register][`.z.ws;`html.ws;wshandler]; + hnd[`register][`.z.wc;`html.close;closehandle]; + hnd[`register][`.z.pc;`html.close;closehandle]; :()]; - / no handlers dep: assign directly, wrapping any existing .z.wc to preserve it + / no handlers config: assign directly, wrapping any existing handlers to preserve them + / wire only once so repeated init calls do not stack the wrappers + if[zwired;:()]; .z.ws:wshandler; - .z.wc:@[value;`.z.wc;{{}}]; - .z.wc:{[existing;h] closehandle h; existing h}[.z.wc;]; - - .z.m.lginfo[`html;"initialised with homedir: ",homedir]; + .z.wc:{[existing;h] closehandle h; existing h}[@[value;`.z.wc;{{[x]}}];]; + .z.pc:{[existing;h] closehandle h; existing h}[@[value;`.z.pc;{{[x]}}];]; + .z.m.zwired:1b; }; diff --git a/di/html/init.q b/di/html/init.q index c86244a9..b13d5faf 100644 --- a/di/html/init.q +++ b/di/html/init.q @@ -1,4 +1,4 @@ / entry point for di.html module \l ::html.q -export:([init;addtables;pub;sub;wssub;end;readpage;readpagereplaceHP;evaluate]) +export:([init;addtables;pub;sub;wssub;end;dataformat;readpage;readpagereplaceHP;evaluate]) diff --git a/di/html/test.csv b/di/html/test.csv index 583aa277..6974123c 100644 --- a/di/html/test.csv +++ b/di/html/test.csv @@ -1,7 +1,7 @@ action,ms,bytes,lang,code,repeat,minver,comment before,0,0,q,html:use`di.html,1,,load di.html module -before,0,0,q,"mocklog:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,,mock log dep that discards all messages -before,0,0,q,"html.init[(enlist `homedir)!enlist ""/tmp"";enlist[`log]!enlist[mocklog]]",1,,initialise module with /tmp as html home directory +before,0,0,q,"mocklog:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,,mock log config that discards all messages +before,0,0,q,"html.init[`homedir`log!(""/tmp"";mocklog)]",1,,initialise module with /tmp as html home directory before,0,0,q,"(`:/tmp/di_html_test.html) 1: ""test""",1,,write test html file using q file io before,0,0,q,"(`:/tmp/di_html_hp_test.html) 1: ""MYKDBSERVER:MYKDBPORT""",1,,write test file containing server and port tokens before,0,0,q,htmltestfn:{x-y},1,,helper function for multi-arg evaluate test @@ -32,9 +32,19 @@ true,0,0,q,"""/tmp/di_html_notfound.html: not found""~html.readpage[""di_html_no true,0,0,q,"0=count html.readpagereplaceHP[""di_html_hp_test.html""] ss ""MYKDBSERVER""",1,,readpagereplaceHP replaces the MYKDBSERVER token true,0,0,q,"0=count html.readpagereplaceHP[""di_html_hp_test.html""] ss ""MYKDBPORT""",1,,readpagereplaceHP replaces the MYKDBPORT token +true,0,0,q,"""/tmp""~.h.HOME",1,,init sets .h.HOME to homedir so the default http handler serves static assets +true,0,0,q,"""start""~html.dataformat[""start"";enlist convtest][`name]",1,,dataformat returns the message type as name +true,0,0,q,"1704153600000~first (first html.dataformat[""start"";enlist convtest][`data])[`d]",1,,dataformat javascript-formats each table in a list +true,0,0,q,"`t1`t2~key html.dataformat[""start"";`t1`t2!(convtest;convtest)][`data]",1,,dataformat preserves table names when given a dictionary of tables +true,0,0,q,"""2024-01-02T12:00:00Z""~first html.dataformat[""start"";`t1`t2!(convtest;convtest)][`data][`t1][`ts]",1,,dataformat javascript-formats each table in a dictionary true,0,0,q,"""2024-01-02T12:00:00Z""~first (jsf convtest)`ts",1,,jsformat converts a timestamp column to an iso 8601 string true,0,0,q,"1704153600000~first (jsf convtest)`d",1,,jsformat converts a date column to javascript epoch milliseconds true,0,0,q,"43200000~first (jsf convtest)`tm",1,,jsformat converts a time column to milliseconds since midnight true,0,0,q,"`AAPL~first (jsf convtest)`sym",1,,jsformat leaves columns whose type is not in the typemap unchanged +run,0,0,q,"setenv[`KDBHTML;""/tmp""]",1,,set KDBHTML so default init picks up the TorQ env var +run,0,0,q,"html.init[enlist[`log]!enlist mocklog]",1,,init without homedir config applies defaults without error +true,0,0,q,"""/tmp""~.h.HOME",1,,default init takes homedir from the KDBHTML env var like TorQ +true,0,0,q,"""test""~html.readpage[""di_html_test.html""]",1,,readpage works against the env var derived homedir + after,0,0,q,"system ""rm /tmp/di_html_test.html /tmp/di_html_hp_test.html""",1,,remove test html files created during setup From 5f9a3fe79e904f57de0754d902ae5ad1e73a6cd4 Mon Sep 17 00:00:00 2001 From: Matthew Date: Fri, 12 Jun 2026 13:58:04 +0100 Subject: [PATCH 3/4] Changed in line with dexters comment about required logging dependency --- di/html/html.md | 19 ++++++++++--------- di/html/html.q | 41 ++++++++++++++++++++++++----------------- di/html/test.csv | 8 ++++++++ 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/di/html/html.md b/di/html/html.md index d538af4d..a5064892 100644 --- a/di/html/html.md +++ b/di/html/html.md @@ -13,10 +13,12 @@ This module does two things: ```q html:use`di.html +log:use`di.log -/ minimal setup - all config is optional -/ homedir defaults to the KDBHTML env var (else "html"), logging defaults to the console -html.init[] +/ minimal setup - the log dependency is required +/ homedir defaults to the KDBHTML env var (else "html") +logdep:`info`warn`error!(log.info;log.warn;log.error) +html.init[enlist[`log]!enlist logdep] / register tables for pub/sub html.addtables[`trades`quotes] @@ -30,16 +32,15 @@ This mirrors the original TorQ deployment: set `KDBHTML`, initialise, register t ## init ```q -html.init[] html.init[configs] ``` -`configs` is an optional dictionary — call `init[]` to use the defaults. Only recognised keys are picked up: +`configs` is a dictionary. The `` `log `` key is required — `init` signals an error if it is missing or does not contain `` `info`warn`error `` functions. Only recognised keys are picked up: | Key | Type | Description | Default | |---|---|---|---| +| `` `log `` | dict | **Required.** Logging functions with keys `` `info`warn`error ``, each called as `(ctx;msg)` — e.g. from `di.log` | — | | `homedir` | string | Path to the directory containing HTML files | `KDBHTML` env var, else `"html"` (TorQ behaviour) | -| `` `log `` | dict | Logging functions with keys `` `info`warn`error ``, each called as `(ctx;msg)` | Console loggers — info/warn to stdout, error to stderr | | `` `handlers `` | dict | Handler registry with key `` `register `` | Assigns `.z.ws`, `.z.wc` and `.z.pc` directly | ```q @@ -135,12 +136,12 @@ The module logs at three points: - On `readpage`: warns if a requested file is not found - On `evaluate`: logs at error level when a WebSocket-invoked function fails (the error is also re-thrown to the caller) -Logging defaults to the console (info/warn to stdout, error to stderr); pass a `log` dict to `init` to integrate with a real logging module. Internally the loggers are stored on `.z.m` as `lginfo`/`lgwarn`/`lgerr` (not `log`, which is a q built-in) and every call site invokes them through `.z.m`. +There is no built-in default logger — the `log` dict must be injected via `init`, typically from `di.log`. Internally the loggers are stored on `.z.m` as `lginfo`/`lgwarn`/`lgerr` (not `log`, which is a q built-in) and every call site invokes them through `.z.m`. ## Example with custom log and handlers config -Both keys are optional overrides. The `log` functions are called as `(ctx;msg)` and the -`handlers` registry's `register` is called as `(.z event name; label; handler)`. +The `log` functions are called as `(ctx;msg)` and the `handlers` registry's `register` +is called as `(.z event name; label; handler)`. ```q / wire up logging on top of the kx.log module diff --git a/di/html/html.q b/di/html/html.q index c981d048..e1c3cbd7 100644 --- a/di/html/html.q +++ b/di/html/html.q @@ -18,16 +18,17 @@ homedir:""; / flag so direct .z handler wiring happens only once across repeated init calls zwired:0b; -/ logging functions - default to the console, overridable via the log config key on init - -/ default console loggers, called as (ctx;msg) -deflog:`info`warn`error!( - {[c;m] -1 "INFO ",(string c)," ",m;}; - {[c;m] -1 "WARN ",(string c)," ",m;}; - {[c;m] -2 "ERROR ",(string c)," ",m;}); - -/ converts a list of timestamps or dates to iso 8601 strings e.g. "2024-01-02T12:00:00Z" -jstsiso8601:{[x] {("-" sv "." vs string `date$x),"T",string[`second$x],"Z"} each x}; +jstsiso8601:{[x] + / converts a list of timestamps or datetimes to iso 8601 strings e.g. "2024-01-02T12:00:00Z" + / vectorised: stringifies the date and time parts in bulk rather than per element; nulls return "" + i:where 10=count each d:string `date$x; + r:(count d)#enlist ""; + if[not count i;:r]; + dd:d i; + dd[;4 7]:"-"; + r[i]:dd,'("T",/:string `second$x i),\:"Z"; + :r; + }; / converts a list of dates to javascript epoch milliseconds / kdb dates are days since 2000-01-01; 10957 is the day offset from 1970-01-01 to 2000-01-01 @@ -184,19 +185,25 @@ evaluate:{[inputdict] init:{[configs] / sets up module state and registers websocket handlers - / configs is an optional dict - recognised keys are homedir, log and handlers - / defaults: homedir from the KDBHTML env var (else "html"), console logging, direct .z handler assignment + / configs is a dict - recognised keys are homedir, log and handlers + / log is required: `info`warn`error!(...) where each is a {[ctx;msg]} function, e.g. from di.log + / defaults: homedir from the KDBHTML env var (else "html"), direct .z handler assignment + + / log dependency is required - fail loudly rather than falling back to a default logger + logdict:$[99h=type configs;$[(`log in key configs) and not (::)~configs`log;configs`log;()!()];()!()]; + if[not count logdict; + '"di.html: log dependency is required; pass `info`warn`error functions - see di.log or refer to confluence documentation"]; + if[not 99h=type logdict;'"di.html: log must be a dict of `info`warn`error!(logging functions)"]; + if[count missing:`info`warn`error except key logdict; + '"di.html: log dict missing key(s): ",", " sv string missing]; / default configuration values hd:$[count e:getenv`KDBHTML;e;"html"]; - logdict:deflog; hnd:(::); / set custom config values - only recognised keys are picked up - if[not configs~(::); - if[`homedir in key configs;hd:configs`homedir]; - if[`log in key configs;logdict:configs`log]; - if[`handlers in key configs;hnd:configs`handlers]]; + if[`homedir in key configs;hd:configs`homedir]; + if[`handlers in key configs;hnd:configs`handlers]; .z.m.lginfo:logdict`info; .z.m.lgwarn:logdict`warn; diff --git a/di/html/test.csv b/di/html/test.csv index 6974123c..519e8490 100644 --- a/di/html/test.csv +++ b/di/html/test.csv @@ -41,10 +41,18 @@ true,0,0,q,"""2024-01-02T12:00:00Z""~first (jsf convtest)`ts",1,,jsformat conver true,0,0,q,"1704153600000~first (jsf convtest)`d",1,,jsformat converts a date column to javascript epoch milliseconds true,0,0,q,"43200000~first (jsf convtest)`tm",1,,jsformat converts a time column to milliseconds since midnight true,0,0,q,"`AAPL~first (jsf convtest)`sym",1,,jsformat leaves columns whose type is not in the typemap unchanged +true,0,0,q,"""""~first (jsf ([]ts:enlist 0Np))`ts",1,,jsformat converts null timestamps to empty strings +true,0,0,q,"0=count (jsf 0#convtest)`ts",1,,jsformat handles empty tables run,0,0,q,"setenv[`KDBHTML;""/tmp""]",1,,set KDBHTML so default init picks up the TorQ env var run,0,0,q,"html.init[enlist[`log]!enlist mocklog]",1,,init without homedir config applies defaults without error true,0,0,q,"""/tmp""~.h.HOME",1,,default init takes homedir from the KDBHTML env var like TorQ true,0,0,q,"""test""~html.readpage[""di_html_test.html""]",1,,readpage works against the env var derived homedir +fail,0,0,q,"html.init[(::)]",1,,init without a configs dict throws - the log dependency is required +fail,0,0,q,"html.init[enlist[`homedir]!enlist ""/tmp""]",1,,init with a configs dict missing the log key throws +fail,0,0,q,"html.init[enlist[`log]!enlist 42]",1,,init throws when the log value is not a dict +fail,0,0,q,"html.init[enlist[`log]!enlist enlist[`info]!enlist {[c;m]}]",1,,init throws when the log dict is missing the warn and error keys +true,0,0,q,"(@[html.init;(::);{x}]) like ""*log*""",1,,the missing log dependency error message names the log key + after,0,0,q,"system ""rm /tmp/di_html_test.html /tmp/di_html_hp_test.html""",1,,remove test html files created during setup From 4556fb19398012ae54441348ee61abc9c714573f Mon Sep 17 00:00:00 2001 From: Matthew Date: Mon, 15 Jun 2026 10:58:56 +0100 Subject: [PATCH 4/4] Removed camel case and fixed remaining unit tests --- di/html/html.md | 4 ++-- di/html/html.q | 2 +- di/html/init.q | 2 +- di/html/test.csv | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/di/html/html.md b/di/html/html.md index a5064892..571039b1 100644 --- a/di/html/html.md +++ b/di/html/html.md @@ -108,10 +108,10 @@ html.readpage[filename] Reads the file at `homedir/filename` and returns its contents as a string. Returns an error message string if the file is not found. -### readpagereplaceHP +### readpagereplacehostport ```q -html.readpagereplaceHP[filename] +html.readpagereplacehostport[filename] ``` Reads the file at `homedir/filename` and replaces `MYKDBSERVER` and `MYKDBPORT` tokens with the process's current IP address and port. Used to serve self-referencing HTML pages over HTTP. diff --git a/di/html/html.q b/di/html/html.q index e1c3cbd7..6584a82b 100644 --- a/di/html/html.q +++ b/di/html/html.q @@ -173,7 +173,7 @@ readpage:{[filename] :$[count r;"c"$r;p,": not found"]; }; -readpagereplaceHP:{[filename] +readpagereplacehostport:{[filename] / reads a page and replaces MYKDBSERVER and MYKDBPORT tokens with live server values :replace[readpage filename;`MYKDBSERVER`MYKDBPORT!("\"",(string ipa .z.a),"\"";getport[])]; }; diff --git a/di/html/init.q b/di/html/init.q index b13d5faf..a31e306e 100644 --- a/di/html/init.q +++ b/di/html/init.q @@ -1,4 +1,4 @@ / entry point for di.html module \l ::html.q -export:([init;addtables;pub;sub;wssub;end;dataformat;readpage;readpagereplaceHP;evaluate]) +export:([init;addtables;pub;sub;wssub;end;dataformat;readpage;readpagereplacehostport;evaluate]) diff --git a/di/html/test.csv b/di/html/test.csv index 519e8490..a5a78946 100644 --- a/di/html/test.csv +++ b/di/html/test.csv @@ -29,8 +29,8 @@ fail,0,0,q,"html.evaluate[(enlist `arg1)!enlist 1]",1,,evaluate throws when the true,0,0,q,"""test""~html.readpage[""di_html_test.html""]",1,,readpage returns file contents as a string true,0,0,q,"""/tmp/di_html_notfound.html: not found""~html.readpage[""di_html_notfound.html""]",1,,readpage returns a not-found message string for a missing file -true,0,0,q,"0=count html.readpagereplaceHP[""di_html_hp_test.html""] ss ""MYKDBSERVER""",1,,readpagereplaceHP replaces the MYKDBSERVER token -true,0,0,q,"0=count html.readpagereplaceHP[""di_html_hp_test.html""] ss ""MYKDBPORT""",1,,readpagereplaceHP replaces the MYKDBPORT token +true,0,0,q,"0=count html.readpagereplacehostport[""di_html_hp_test.html""] ss ""MYKDBSERVER""",1,,readpagereplacehostport replaces the MYKDBSERVER token +true,0,0,q,"0=count html.readpagereplacehostport[""di_html_hp_test.html""] ss ""MYKDBPORT""",1,,readpagereplacehostport replaces the MYKDBPORT token true,0,0,q,"""/tmp""~.h.HOME",1,,init sets .h.HOME to homedir so the default http handler serves static assets true,0,0,q,"""start""~html.dataformat[""start"";enlist convtest][`name]",1,,dataformat returns the message type as name