diff --git a/lib/mq.ex b/lib/mq.ex index 722c925..40e58d9 100644 --- a/lib/mq.ex +++ b/lib/mq.ex @@ -227,6 +227,7 @@ defmodule Mq do defdelegate to_array(query), to: Query defdelegate to_bytes(query), to: Query defdelegate to_markdown_string(query), to: Query + defdelegate to_boolean(query), to: Query # Collection operations defdelegate length(query), to: Query @@ -257,20 +258,38 @@ defmodule Mq do defdelegate insert(query, idx, val), to: Query defdelegate repeat(query, n), to: Query + # Type-check filters + defdelegate strings(query), to: Query + defdelegate dicts(query), to: Query + defdelegate nones(query), to: Query + defdelegate bytes(query), to: Query + defdelegate iterables(query), to: Query + defdelegate scalars(query), to: Query + + # Dict entry helpers + defdelegate has(query, key), to: Query + defdelegate from_entries(query), to: Query + defdelegate with_entries(query, filter), to: Query + defdelegate walk(query, filter), to: Query + # String operations defdelegate trim(query), to: Query defdelegate ltrim(query), to: Query defdelegate rtrim(query), to: Query defdelegate downcase(query), to: Query + defdelegate ascii_downcase(query), to: Query defdelegate upcase(query), to: Query + defdelegate ascii_upcase(query), to: Query defdelegate explode(query), to: Query defdelegate implode(query), to: Query defdelegate url_encode(query), to: Query + defdelegate url_decode(query), to: Query defdelegate intern(query), to: Query defdelegate gsub(query, pattern, replacement), to: Query defdelegate replace(query, from, to), to: Query defdelegate test(query, pattern), to: Query defdelegate capture(query, pattern), to: Query + defdelegate scan(query, pattern), to: Query # Math operations defdelegate abs(query), to: Query @@ -304,6 +323,20 @@ defmodule Mq do defdelegate from_hex(query), to: Query defdelegate to_hex(query), to: Query + # Random / UUID generation + defdelegate uuid(), to: Query + defdelegate uuid(query), to: Query + defdelegate uuid_v4(), to: Query + defdelegate uuid_v4(query), to: Query + defdelegate uuid_v7(), to: Query + defdelegate uuid_v7(query), to: Query + defdelegate rand(), to: Query + defdelegate rand(query), to: Query + defdelegate rand_int(min, max), to: Query + defdelegate rand_int(query, min, max), to: Query + defdelegate shuffle(query), to: Query + defdelegate sample(query, n), to: Query + # Path operations defdelegate basename(query), to: Query defdelegate dirname(query), to: Query diff --git a/lib/mq/query.ex b/lib/mq/query.ex index 6616137..e54e6fc 100644 --- a/lib/mq/query.ex +++ b/lib/mq/query.ex @@ -278,6 +278,9 @@ defmodule Mq.Query do @doc "Convert to a Markdown string (serialized)." def to_markdown_string(%__MODULE__{} = q), do: pipe_expr(q, "to_markdown_string()") + @doc "Convert to a boolean value." + def to_boolean(%__MODULE__{} = q), do: pipe_expr(q, "to_boolean()") + # Collection operations @doc "Return the length of the current value." def length(%__MODULE__{} = q), do: pipe_expr(q, "len()") @@ -360,6 +363,53 @@ defmodule Mq.Query do @doc "Repeat the current value `n` times." def repeat(%__MODULE__{} = q, n), do: pipe_expr(q, "repeat(#{n})") + @doc "Filter to string values (like `select(is_string(v))`)." + def strings(%__MODULE__{} = q), do: pipe_expr(q, "strings()") + + @doc "Filter to dict values (like `select(is_dict(v))`)." + def dicts(%__MODULE__{} = q), do: pipe_expr(q, "dicts()") + + @doc "Filter to none/null values." + def nones(%__MODULE__{} = q), do: pipe_expr(q, "nones()") + + @doc "Filter to byte values." + def bytes(%__MODULE__{} = q), do: pipe_expr(q, "bytes()") + + @doc "Filter to iterable values (arrays and dicts)." + def iterables(%__MODULE__{} = q), do: pipe_expr(q, "iterables()") + + @doc "Filter to scalar (non-array, non-dict) values." + def scalars(%__MODULE__{} = q), do: pipe_expr(q, "scalars()") + + @doc "Check whether a dict has `key`, or an array has an element at index `key`." + def has(%__MODULE__{} = q, key), do: pipe_expr(q, "has(#{inspect(key)})") + + @doc "Build a dict from an array of `[key, value]` pairs, as produced by `entries/1`." + def from_entries(%__MODULE__{} = q), do: pipe_expr(q, "from_entries()") + + @doc """ + Transform each `[key, value]` pair of a dict with `filter`, then rebuild a dict + from the resulting pairs. + + Accepts an `Mq.Filter` or a raw mq expression string, e.g. a `fn(entry): ...;` + lambda. + """ + def with_entries(%__MODULE__{} = q, %Mq.Filter{expr: expr}), + do: pipe_expr(q, "with_entries(#{expr})") + + def with_entries(%__MODULE__{} = q, filter) when is_binary(filter), + do: pipe_expr(q, "with_entries(#{filter})") + + @doc """ + Recursively walk a value (Markdown node, array, or dict), applying `filter` to + every leaf value and rebuilding the structure with the results. + + Accepts an `Mq.Filter` or a raw mq expression string, e.g. a `fn(x): ...;` + lambda. + """ + def walk(%__MODULE__{} = q, %Mq.Filter{expr: expr}), do: pipe_expr(q, "walk(#{expr})") + def walk(%__MODULE__{} = q, filter) when is_binary(filter), do: pipe_expr(q, "walk(#{filter})") + # String operations @doc "Trim leading and trailing whitespace." def trim(%__MODULE__{} = q), do: pipe_expr(q, "trim()") @@ -373,9 +423,15 @@ defmodule Mq.Query do @doc "Convert to lowercase (Unicode-aware)." def downcase(%__MODULE__{} = q), do: pipe_expr(q, "downcase()") + @doc "Convert ASCII uppercase letters (A-Z) to lowercase, leaving other characters unchanged." + def ascii_downcase(%__MODULE__{} = q), do: pipe_expr(q, "ascii_downcase()") + @doc "Convert to uppercase (Unicode-aware)." def upcase(%__MODULE__{} = q), do: pipe_expr(q, "upcase()") + @doc "Convert ASCII lowercase letters (a-z) to uppercase, leaving other characters unchanged." + def ascii_upcase(%__MODULE__{} = q), do: pipe_expr(q, "ascii_upcase()") + @doc "Explode a string into codepoints." def explode(%__MODULE__{} = q), do: pipe_expr(q, "explode()") @@ -385,6 +441,9 @@ defmodule Mq.Query do @doc "URL-encode the current value." def url_encode(%__MODULE__{} = q), do: pipe_expr(q, "url_encode()") + @doc "URL-decode the current value." + def url_decode(%__MODULE__{} = q), do: pipe_expr(q, "url_decode()") + @doc "Intern the current string." def intern(%__MODULE__{} = q), do: pipe_expr(q, "intern()") @@ -402,6 +461,9 @@ defmodule Mq.Query do @doc "Capture groups from `pattern` (regex)." def capture(%__MODULE__{} = q, pattern), do: pipe_expr(q, "capture(#{inspect(pattern)})") + @doc "Find all matches of `pattern` (regex) in the current value." + def scan(%__MODULE__{} = q, pattern), do: pipe_expr(q, "scan(#{inspect(pattern)})") + # Math operations @doc "Absolute value." def abs(%__MODULE__{} = q), do: pipe_expr(q, "abs()") @@ -483,6 +545,43 @@ defmodule Mq.Query do @doc "Encode to hex." def to_hex(%__MODULE__{} = q), do: pipe_expr(q, "to_hex()") + # Random / UUID generation + @doc "Standalone: generate a random (version 4) UUID string." + def uuid, do: new("uuid()") + + @doc "Chained: replace the current value with a random (version 4) UUID string." + def uuid(%__MODULE__{} = q), do: pipe_expr(q, "uuid()") + + @doc "Standalone: generate a random (version 4) UUID string. Alias of `uuid/0`." + def uuid_v4, do: new("uuid_v4()") + + @doc "Chained: replace the current value with a random (version 4) UUID string. Alias of `uuid/1`." + def uuid_v4(%__MODULE__{} = q), do: pipe_expr(q, "uuid_v4()") + + @doc "Standalone: generate a time-ordered (version 7) UUID string." + def uuid_v7, do: new("uuid_v7()") + + @doc "Chained: replace the current value with a time-ordered (version 7) UUID string." + def uuid_v7(%__MODULE__{} = q), do: pipe_expr(q, "uuid_v7()") + + @doc "Standalone: generate a pseudo-random number in `[0, 1)`. Not cryptographically secure." + def rand, do: new("rand()") + + @doc "Chained: replace the current value with a pseudo-random number in `[0, 1)`." + def rand(%__MODULE__{} = q), do: pipe_expr(q, "rand()") + + @doc "Standalone: generate a pseudo-random integer in `[min, max]` (inclusive)." + def rand_int(min, max), do: new("rand_int(#{min}, #{max})") + + @doc "Chained: replace the current value with a pseudo-random integer in `[min, max]` (inclusive)." + def rand_int(%__MODULE__{} = q, min, max), do: pipe_expr(q, "rand_int(#{min}, #{max})") + + @doc "Shuffle the current array's elements into a uniformly random order." + def shuffle(%__MODULE__{} = q), do: pipe_expr(q, "shuffle()") + + @doc "Sample `n` elements from the current array without replacement, in random order." + def sample(%__MODULE__{} = q, n), do: pipe_expr(q, "sample(#{n})") + # Path operations @doc "Return the basename of a path." def basename(%__MODULE__{} = q), do: pipe_expr(q, "basename()") diff --git a/native/mq_nif/Cargo.lock b/native/mq_nif/Cargo.lock index afb24f0..7c4cbf4 100644 --- a/native/mq_nif/Cargo.lock +++ b/native/mq_nif/Cargo.lock @@ -376,6 +376,31 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + [[package]] name = "half" version = "2.7.1" @@ -755,15 +780,16 @@ dependencies = [ [[package]] name = "mq-lang" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1212fc83c06eef62bf76ccea314ae51cfa72304bd73445bae64960281835c229" +checksum = "1e3607ac3570b7ac0f12ac385f43689689d7af123c47ef822d9a6812a5cac112" dependencies = [ "base64", "chrono", "ciborium", "csv", "dirs", + "getrandom 0.3.4", "hcl-rs", "itertools", "md5", @@ -774,6 +800,7 @@ dependencies = [ "nom_locate", "percent-encoding", "quick-xml", + "rand", "regex-lite", "rustc-hash", "serde", @@ -787,15 +814,16 @@ dependencies = [ "toml", "toon-format", "url", + "uuid", "web-sys", "yaml-rust2", ] [[package]] name = "mq-macros" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e8b069c44f6802457aac42a52b2a44953b55b128ad1cfd7960d263bb3a318d" +checksum = "9efab4192caf4e9bf91f840db36c8a3b420a844e857122dd07a05593ad59e08e" dependencies = [ "proc-macro2", "quote", @@ -804,9 +832,9 @@ dependencies = [ [[package]] name = "mq-markdown" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97cb203323d6225628fc2dfbd1c208e914b85aabf96250e217fc4e883cef16e9" +checksum = "6d2cc321c5808b511b9b06ffb67b504ee6128a064cabfef16a2a0994ea8dce25" dependencies = [ "ego-tree", "itertools", @@ -965,6 +993,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "pratt" version = "0.4.0" @@ -1004,6 +1041,47 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1019,7 +1097,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ - "getrandom", + "getrandom 0.2.16", "libredox", "thiserror", ] @@ -1470,6 +1548,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "vecmap-rs" version = "0.2.4" @@ -1485,6 +1574,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.106" @@ -1629,6 +1727,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.2" diff --git a/native/mq_nif/Cargo.toml b/native/mq_nif/Cargo.toml index bfd5c8a..91de899 100644 --- a/native/mq_nif/Cargo.toml +++ b/native/mq_nif/Cargo.toml @@ -11,8 +11,8 @@ crate-type = ["cdylib"] name = "mq_nif" [dependencies] -mq-lang = {version = "0.6.3"} -mq-markdown = {version = "0.6.3"} +mq-lang = {version = "0.6.4"} +mq-markdown = {version = "0.6.4"} rustler = "0.37.0" [profile.release] diff --git a/test/mq_test.exs b/test/mq_test.exs index 4f86167..b6b5291 100644 --- a/test/mq_test.exs +++ b/test/mq_test.exs @@ -263,6 +263,8 @@ defmodule MqTest do assert to_string(Query.text() |> Query.to_markdown_string()) == ".text | to_markdown_string()" + + assert to_string(Query.text() |> Query.to_boolean()) == ".text | to_boolean()" end test "chained attribute selectors" do @@ -296,7 +298,9 @@ defmodule MqTest do assert to_string(Query.text() |> Query.ltrim()) == ".text | ltrim()" assert to_string(Query.text() |> Query.rtrim()) == ".text | rtrim()" assert to_string(Query.text() |> Query.downcase()) == ".text | downcase()" + assert to_string(Query.text() |> Query.ascii_downcase()) == ".text | ascii_downcase()" assert to_string(Query.text() |> Query.upcase()) == ".text | upcase()" + assert to_string(Query.text() |> Query.ascii_upcase()) == ".text | ascii_upcase()" assert to_string(Query.text() |> Query.len()) == ".text | len()" assert to_string(Query.text() |> Query.utf8bytelen()) == ".text | utf8bytelen()" @@ -311,6 +315,12 @@ defmodule MqTest do assert to_string(Query.text() |> Query.slice(0, 5)) == ".text | slice(0, 5)" assert to_string(Query.text() |> Query.index("foo")) == ".text | index(\"foo\")" assert to_string(Query.text() |> Query.rindex("foo")) == ".text | rindex(\"foo\")" + assert to_string(Query.text() |> Query.url_decode()) == ".text | url_decode()" + + assert to_string(Query.text() |> Query.capture("(?P\\w+)")) == + ".text | capture(\"(?P\\\\w+)\")" + + assert to_string(Query.text() |> Query.scan("\\w+")) == ".text | scan(\"\\\\w+\")" end test "collection operations" do @@ -336,6 +346,44 @@ defmodule MqTest do assert to_string(Query.list() |> Query.insert(0, "new")) == ".[] | insert(0, \"new\")" end + test "type-check filters (mq 0.6.4)" do + assert to_string(Query.list() |> Query.strings()) == ".[] | strings()" + assert to_string(Query.list() |> Query.dicts()) == ".[] | dicts()" + assert to_string(Query.list() |> Query.nones()) == ".[] | nones()" + assert to_string(Query.list() |> Query.bytes()) == ".[] | bytes()" + assert to_string(Query.list() |> Query.iterables()) == ".[] | iterables()" + assert to_string(Query.list() |> Query.scalars()) == ".[] | scalars()" + end + + test "dict entry helpers (mq 0.6.4)" do + assert to_string(Query.text() |> Query.has("key")) == ".text | has(\"key\")" + assert to_string(Query.text() |> Query.from_entries()) == ".text | from_entries()" + + assert to_string(Query.text() |> Query.with_entries("fn(e): e;")) == + ".text | with_entries(fn(e): e;)" + + assert to_string(Query.text() |> Query.with_entries(Filter.eq(1))) == + ".text | with_entries(eq(1))" + + assert to_string(Query.text() |> Query.walk("fn(x): upcase(x);")) == + ".text | walk(fn(x): upcase(x);)" + end + + test "random / uuid generation (mq 0.6.4)" do + assert to_string(Query.uuid()) == "uuid()" + assert to_string(Query.text() |> Query.uuid()) == ".text | uuid()" + assert to_string(Query.uuid_v4()) == "uuid_v4()" + assert to_string(Query.text() |> Query.uuid_v4()) == ".text | uuid_v4()" + assert to_string(Query.uuid_v7()) == "uuid_v7()" + assert to_string(Query.text() |> Query.uuid_v7()) == ".text | uuid_v7()" + assert to_string(Query.rand()) == "rand()" + assert to_string(Query.text() |> Query.rand()) == ".text | rand()" + assert to_string(Query.rand_int(1, 10)) == "rand_int(1, 10)" + assert to_string(Query.text() |> Query.rand_int(1, 10)) == ".text | rand_int(1, 10)" + assert to_string(Query.list() |> Query.shuffle()) == ".[] | shuffle()" + assert to_string(Query.list() |> Query.sample(2)) == ".[] | sample(2)" + end + test "math operations" do assert to_string(Query.text() |> Query.abs()) == ".text | abs()" assert to_string(Query.text() |> Query.ceil()) == ".text | ceil()" @@ -549,6 +597,30 @@ defmodule MqTest do assert {:ok, result} = Mq.run(".h2", content) assert result.values == ["## Features", "## Installation"] end + + test "ascii_upcase transformation (mq 0.6.4)" do + md = "Hello world" + assert {:ok, result} = Mq.run(Query.text() |> Query.ascii_upcase(), md) + assert result.values == ["HELLO WORLD"] + end + + test "url_decode transformation (mq 0.6.4)" do + md = "hello%20world" + assert {:ok, result} = Mq.run(Query.text() |> Query.url_decode(), md) + assert result.values == ["hello world"] + end + + test "scan finds all regex matches (mq 0.6.4)" do + md = "Hello world" + assert {:ok, result} = Mq.run(Query.text() |> Query.scan("\\w+"), md) + assert result.values == ["Hello\nworld"] + end + + test "standalone uuid generates a UUID string (mq 0.6.4)" do + assert {:ok, result} = Mq.run(Query.uuid(), "# x") + assert [uuid] = result.values + assert String.match?(uuid, ~r/^[0-9a-f-]{36}$/) + end end describe "Mq.Filter" do