diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..d40bef65 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,73 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +MistQL is a miniature query language for computations on JSON-like structures, with implementations in JavaScript/TypeScript, Python, Rust, and Kotlin. All implementations must produce identical behavior — cross-platform correctness is critical. + +## Build & Test Commands + +The project uses `talc` (a task CLI defined in `talc.yaml`) for orchestration: + +```bash +talc install # Install all dependencies +talc test all # Test all implementations +talc test js # JS tests (cd js && npm test) +talc test py # Python tests (cd py && uv run pytest && uv run mypy mistql) +talc build all # Build all +talc build js # JS build via rollup (cd js && npm run build) +talc build py # Python build (cd py && uv build) +``` + +### Running tests directly + +```bash +# JavaScript - ts-mocha, tests are colocated as *.spec.ts +cd js && npm test +cd js && npx ts-mocha -p tsconfig.tests.json 'src/builtins/count.spec.ts' # single test file + +# Python - pytest +cd py && uv run pytest +cd py && uv run pytest tests/test_shared.py -k "test_name" # single test +cd py && uv run mypy mistql # type checking + +# Rust +cd rust && cargo test + +# Kotlin - Gradle + JUnit 5 +cd kt && ./gradlew test +``` + +## Architecture + +### Monorepo layout +- `js/` — TypeScript implementation (lexer → parser → executor pipeline) +- `py/` — Python implementation (Lark grammar-based parser) +- `rust/` — Rust implementation (nom parser combinators) +- `kt/` — Kotlin/JVM implementation (lexer → parser → executor pipeline) +- `shared/` — Language-independent test suite (`testdata.json`) and sample data +- `docs/` — Docusaurus 3 site (mistql.com) + +### Shared test suite +`shared/testdata.json` defines the canonical test cases that all implementations must pass. When adding or modifying language features, update this file and ensure all implementations pass. + +### JS implementation (`js/src/`) +Pipeline: `lexer.ts` → `parser.ts` → `executor.ts`. Builtin functions are individual files in `builtins/`, each exporting a function registered in `builtins/index.ts`. Tests live alongside source as `*.spec.ts`. + +### Python implementation (`py/mistql/`) +Uses a Lark grammar (`grammar.lark`) for parsing. All builtins are in a single `builtins.py`. The runtime type system is in `runtime_value.py`. Tests are in `py/tests/`. + +### Rust implementation (`rust/src/`) +Uses nom parser combinators in `parser.rs`. Builtins in `builtins.rs`, type system in `types.rs`. + +### Kotlin implementation (`kt/src/main/kotlin/com/mistql/`) +Follows the JS architecture: `parser/Parser.kt` (lexer + token-based parser with precedence resolution) → `runtime/Executor.kt`. Builtins are split across files in `builtins/` (ArrayFunctions, StringFunctions, ObjectFunctions, MathFunctions, UtilityFunctions, Operators). Type system in `runtime/RuntimeValue.kt`. Uses Gradle with kotlinx-serialization-json. Tests in `kt/src/test/kotlin/com/mistql/`. + +## Key Conventions + +- The `@` symbol refers to the current context value; `$` accesses builtins/root +- Functions use contextualized expressions (e.g., `filter`, `map`, `sortby` change what `@` refers to) +- JS bundle target is ~5.3kB gzipped — monitor with `cd js && npm run bundlesize` +- Python uses `uv` as the package manager +- Version metadata is in `meta.json` at the repo root diff --git a/kt/.gitignore b/kt/.gitignore new file mode 100644 index 00000000..67bcc2f7 --- /dev/null +++ b/kt/.gitignore @@ -0,0 +1,2 @@ +.gradle/ +build/ diff --git a/kt/build.gradle.kts b/kt/build.gradle.kts new file mode 100644 index 00000000..fa419b41 --- /dev/null +++ b/kt/build.gradle.kts @@ -0,0 +1,34 @@ +plugins { + kotlin("jvm") version "1.9.22" + kotlin("plugin.serialization") version "1.9.22" + application +} + +group = "com.mistql" +version = "0.5.0-beta.1" + +repositories { + mavenCentral() +} + +dependencies { + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.2") + + testImplementation(kotlin("test")) + testImplementation("org.junit.jupiter:junit-jupiter:5.10.1") + testImplementation("org.junit.jupiter:junit-jupiter-params:5.10.1") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +tasks.test { + useJUnitPlatform() + workingDir = rootProject.projectDir.parentFile +} + +application { + mainClass.set("com.mistql.cli.MainKt") +} + +kotlin { + jvmToolchain(21) +} diff --git a/kt/gradle/wrapper/gradle-wrapper.jar b/kt/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..8bdaf60c Binary files /dev/null and b/kt/gradle/wrapper/gradle-wrapper.jar differ diff --git a/kt/gradle/wrapper/gradle-wrapper.properties b/kt/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..2e111328 --- /dev/null +++ b/kt/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/kt/gradlew b/kt/gradlew new file mode 100755 index 00000000..adff685a --- /dev/null +++ b/kt/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/kt/gradlew.bat b/kt/gradlew.bat new file mode 100644 index 00000000..e509b2dd --- /dev/null +++ b/kt/gradlew.bat @@ -0,0 +1,93 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/kt/settings.gradle.kts b/kt/settings.gradle.kts new file mode 100644 index 00000000..5694e910 --- /dev/null +++ b/kt/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "mistql-kt" diff --git a/kt/src/main/kotlin/com/mistql/MistQL.kt b/kt/src/main/kotlin/com/mistql/MistQL.kt new file mode 100644 index 00000000..d51f785b --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/MistQL.kt @@ -0,0 +1,37 @@ +package com.mistql + +import com.mistql.parser.Parser +import com.mistql.runtime.ExecutionContext +import com.mistql.runtime.Executor +import com.mistql.runtime.RuntimeValue +import kotlinx.serialization.json.JsonElement + +sealed class MistQLError(message: String) : Exception(message) { + class Parse(message: String) : MistQLError("Parse error: $message") + class Runtime(message: String) : MistQLError("Runtime error: $message") + class Type(message: String) : MistQLError("Type error: $message") +} + +object MistQL { + fun query(queryStr: String, data: Any?): Any? { + val runtimeData = RuntimeValue.fromAny(data) + val expr = Parser.parse(queryStr) + val ctx = ExecutionContext(runtimeData) + val result = Executor.execute(expr, ctx) + return result.toAny() + } + + fun queryRuntime(queryStr: String, data: RuntimeValue): RuntimeValue { + val expr = Parser.parse(queryStr) + val ctx = ExecutionContext(data) + return Executor.execute(expr, ctx) + } + + fun queryJson(queryStr: String, data: JsonElement): JsonElement { + val runtimeData = RuntimeValue.fromJson(data) + val expr = Parser.parse(queryStr) + val ctx = ExecutionContext(runtimeData) + val result = Executor.execute(expr, ctx) + return result.toJson() + } +} diff --git a/kt/src/main/kotlin/com/mistql/ast/Expression.kt b/kt/src/main/kotlin/com/mistql/ast/Expression.kt new file mode 100644 index 00000000..85e6e7d1 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/ast/Expression.kt @@ -0,0 +1,13 @@ +package com.mistql.ast + +import com.mistql.runtime.RuntimeValue + +sealed class Expression { + data class Value(val value: RuntimeValue) : Expression() + data class Ref(val name: String, val internal: Boolean = false) : Expression() + data class FnCall(val function: Expression, val arguments: List, val isExplicitCall: Boolean = false) : Expression() + data class Pipe(val stages: List) : Expression() + data class ArrayLiteral(val items: List) : Expression() + data class ObjectLiteral(val entries: List>) : Expression() + data class Paren(val expression: Expression) : Expression() +} diff --git a/kt/src/main/kotlin/com/mistql/builtins/ArrayFunctions.kt b/kt/src/main/kotlin/com/mistql/builtins/ArrayFunctions.kt new file mode 100644 index 00000000..2d36803d --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/builtins/ArrayFunctions.kt @@ -0,0 +1,239 @@ +package com.mistql.builtins + +import com.mistql.MistQLError +import com.mistql.ast.Expression +import com.mistql.runtime.ExecutionContext +import com.mistql.runtime.RuntimeValue + +object ArrayFunctions { + val all: Map = mapOf( + "count" to ::count, + "filter" to ::filter, + "map" to ::map, + "find" to ::find, + "reverse" to ::reverse, + "flatten" to ::flatten, + "sum" to ::sum, + "sort" to ::sort, + "sortby" to ::sortby, + "reduce" to ::reduce, + "groupby" to ::groupby, + "withindices" to ::withindices, + "sequence" to ::sequence, + ) + + private fun count(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("count", args, 1, 1) + val arr = assertArr(eval(args[0], ctx)) + return RuntimeValue.Num(arr.size.toDouble()) + } + + private fun filter(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("filter", args, 2, 2) + val condition = args[0] + val arr = assertArr(eval(args[1], ctx)) + + val result = mutableListOf() + for (item in arr) { + ctx.pushContext(item) + val condResult = evalContextualized(condition, ctx) + ctx.popContext() + if (condResult.truthy()) { + result.add(item) + } + } + return RuntimeValue.Arr(result) + } + + private fun map(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("map", args, 2, 2) + val transform = args[0] + val arr = assertArr(eval(args[1], ctx)) + + val result = mutableListOf() + for (item in arr) { + ctx.pushContext(item) + val transformed = evalContextualized(transform, ctx) + ctx.popContext() + result.add(transformed) + } + return RuntimeValue.Arr(result) + } + + private fun find(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("find", args, 2, 2) + val condition = args[0] + val arr = assertArr(eval(args[1], ctx)) + + for (item in arr) { + ctx.pushContext(item) + val condResult = evalContextualized(condition, ctx) + ctx.popContext() + if (condResult.truthy()) { + return item + } + } + return RuntimeValue.Null + } + + private fun reverse(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("reverse", args, 1, 1) + return RuntimeValue.Arr(assertArr(eval(args[0], ctx)).reversed()) + } + + private fun flatten(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("flatten", args, 1, 1) + val arr = assertArr(eval(args[0], ctx)) + val result = mutableListOf() + for (item in arr) { + when (item) { + is RuntimeValue.Arr -> result.addAll(item.items) + else -> throw MistQLError.Type("flatten: all elements must be arrays, got ${item.typeName()}") + } + } + return RuntimeValue.Arr(result) + } + + private fun sum(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("sum", args, 1, 1) + val arr = assertArr(eval(args[0], ctx)) + var total = 0.0 + for (item in arr) { + total += assertNum(item) + } + return RuntimeValue.Num(total) + } + + private fun sort(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("sort", args, 1, 1) + val arr = assertArr(eval(args[0], ctx)) + if (arr.isEmpty()) return RuntimeValue.Arr(arr) + + val firstType = arr[0].typeName() + for (item in arr) { + if (item.typeName() != firstType) { + throw MistQLError.Type("sort: cannot sort non-homogeneous arrays (found $firstType and ${item.typeName()})") + } + if (!item.comparable()) { + throw MistQLError.Runtime("sort: Cannot sort non-comparable values") + } + } + + return RuntimeValue.Arr(arr.sortedWith { a, b -> a.compare(b) }) + } + + private fun sortby(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("sortby", args, 2, 2) + val keyExpr = args[0] + val arr = assertArr(eval(args[1], ctx)) + + val withKeys = arr.map { item -> + ctx.pushContext(item) + val key = evalContextualized(keyExpr, ctx) + ctx.popContext() + Pair(key, item) + } + + // Validate all keys are comparable and same type + if (withKeys.isNotEmpty()) { + val firstType = withKeys[0].first.typeName() + for ((key, _) in withKeys) { + if (!key.comparable()) { + throw MistQLError.Runtime("Cannot compare MistQL values of type ${key.typeName()}") + } + if (key.typeName() != firstType) { + throw MistQLError.Runtime("Cannot compare MistQL values of different types") + } + } + } + + return RuntimeValue.Arr( + withKeys.sortedWith { a, b -> a.first.compare(b.first) }.map { it.second } + ) + } + + private fun reduce(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("reduce", args, 3, 3) + val reducerExpr = args[0] + var accumulator = eval(args[1], ctx) + val arr = assertArr(eval(args[2], ctx)) + + for (item in arr) { + val accCur = RuntimeValue.Arr(listOf(accumulator, item)) + ctx.pushContext(accCur) + accumulator = eval(reducerExpr, ctx) + ctx.popContext() + } + return accumulator + } + + private fun groupby(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("groupby", args, 2, 2) + val keyExpr = args[0] + val arr = assertArr(eval(args[1], ctx)) + + val groups = LinkedHashMap>() + for (item in arr) { + ctx.pushContext(item) + val key = evalContextualized(keyExpr, ctx) + ctx.popContext() + val keyStr = key.toStringDisplay() + groups.getOrPut(keyStr) { mutableListOf() }.add(item) + } + + val result = LinkedHashMap() + for ((key, values) in groups) { + result[key] = RuntimeValue.Arr(values) + } + return RuntimeValue.Obj(result) + } + + private fun withindices(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("withindices", args, 1, 1) + val arr = assertArr(eval(args[0], ctx)) + return RuntimeValue.Arr(arr.mapIndexed { index, item -> + RuntimeValue.Arr(listOf(RuntimeValue.Num(index.toDouble()), item)) + }) + } + + private fun sequence(args: List, ctx: ExecutionContext): RuntimeValue { + if (args.size < 2) throw MistQLError.Runtime("sequence takes at least 2 arguments") + val predicates = args.subList(0, args.size - 1) + val arr = assertArr(eval(args.last(), ctx)) + + // Create bitmasks for each predicate + val bitmasks = predicates.map { predicate -> + arr.map { item -> + ctx.pushContext(item) + val result = eval(predicate, ctx) + ctx.popContext() + result.truthy() + } + } + + val indices = sequenceHelper(bitmasks, 0) + return RuntimeValue.Arr(indices.map { idxList -> + RuntimeValue.Arr(idxList.map { idx -> arr[idx] }) + }) + } + + private fun sequenceHelper(bitmasks: List>, start: Int): List> { + if (bitmasks.isEmpty()) return emptyList() + val first = bitmasks[0] + val result = mutableListOf>() + + for (idx in start until first.size) { + if (first[idx]) { + if (bitmasks.size == 1) { + result.add(listOf(idx)) + } else { + val subResults = sequenceHelper(bitmasks.subList(1, bitmasks.size), idx + 1) + for (sub in subResults) { + result.add(listOf(idx) + sub) + } + } + } + } + return result + } +} diff --git a/kt/src/main/kotlin/com/mistql/builtins/Builtins.kt b/kt/src/main/kotlin/com/mistql/builtins/Builtins.kt new file mode 100644 index 00000000..df1db2d5 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/builtins/Builtins.kt @@ -0,0 +1,57 @@ +package com.mistql.builtins + +import com.mistql.MistQLError +import com.mistql.ast.Expression +import com.mistql.runtime.ExecutionContext +import com.mistql.runtime.Executor +import com.mistql.runtime.RuntimeValue + +typealias BuiltinFn = (args: List, ctx: ExecutionContext) -> RuntimeValue + +object Builtins { + private val registry: Map = buildMap { + putAll(Operators.all) + putAll(ArrayFunctions.all) + putAll(ObjectFunctions.all) + putAll(StringFunctions.all) + putAll(MathFunctions.all) + putAll(UtilityFunctions.all) + } + + fun get(name: String): BuiltinFn? = registry[name] + fun allNames(): Set = registry.keys + fun isBuiltin(name: String): Boolean = name in registry +} + +// Helpers +fun validateArgs(name: String, args: List, min: Int, max: Int? = null) { + if (args.size < min) { + throw MistQLError.Runtime("$name takes at least $min arguments, got ${args.size}") + } + if (max != null && args.size > max) { + throw MistQLError.Runtime("$name takes at most $max arguments, got ${args.size}") + } +} + +fun eval(expr: Expression, ctx: ExecutionContext): RuntimeValue = Executor.execute(expr, ctx) +fun evalContextualized(expr: Expression, ctx: ExecutionContext): RuntimeValue = Executor.executeContextualized(expr, ctx) + +fun assertNum(v: RuntimeValue): Double = when (v) { + is RuntimeValue.Num -> v.value + else -> throw MistQLError.Type("Expected number, got ${v.typeName()}") +} + +fun assertArr(v: RuntimeValue): List = when (v) { + is RuntimeValue.Arr -> v.items + else -> throw MistQLError.Type("Expected array, got ${v.typeName()}") +} + +fun assertStr(v: RuntimeValue): String = when (v) { + is RuntimeValue.Str -> v.value + else -> throw MistQLError.Type("Expected string, got ${v.typeName()}") +} + +fun assertObj(v: RuntimeValue): LinkedHashMap = when (v) { + is RuntimeValue.Obj -> v.entries + else -> throw MistQLError.Type("Expected object, got ${v.typeName()}") +} diff --git a/kt/src/main/kotlin/com/mistql/builtins/MathFunctions.kt b/kt/src/main/kotlin/com/mistql/builtins/MathFunctions.kt new file mode 100644 index 00000000..9a8b705e --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/builtins/MathFunctions.kt @@ -0,0 +1,96 @@ +package com.mistql.builtins + +import com.mistql.MistQLError +import com.mistql.ast.Expression +import com.mistql.runtime.ExecutionContext +import com.mistql.runtime.RuntimeValue +import kotlin.math.sqrt + +object MathFunctions { + val all: Map = mapOf( + "range" to ::range, + "summarize" to ::summarize, + ) + + private fun range(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("range", args, 1, 3) + + val start: Long + val stop: Long + val step: Long + + when (args.size) { + 1 -> { + start = 0 + stop = assertInteger(eval(args[0], ctx), "stop") + step = 1 + } + 2 -> { + start = assertInteger(eval(args[0], ctx), "start") + stop = assertInteger(eval(args[1], ctx), "stop") + step = 1 + } + else -> { + start = assertInteger(eval(args[0], ctx), "start") + stop = assertInteger(eval(args[1], ctx), "stop") + step = assertInteger(eval(args[2], ctx), "step") + } + } + + if (step == 0L) throw MistQLError.Runtime("range: step must not be 0") + + val result = mutableListOf() + if (step > 0) { + var current = start + while (current < stop) { + result.add(RuntimeValue.Num(current.toDouble())) + current += step + } + } else { + var current = start + while (current > stop) { + result.add(RuntimeValue.Num(current.toDouble())) + current += step + } + } + return RuntimeValue.Arr(result) + } + + private fun summarize(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("summarize", args, 1, 1) + val arr = assertArr(eval(args[0], ctx)) + if (arr.isEmpty()) throw MistQLError.Runtime("summarize: cannot summarize empty array") + + val numbers = arr.map { assertNum(it) } + if (numbers.size < 2) throw MistQLError.Runtime("summarize: requires at least 2 numbers") + + val max = numbers.max() + val min = numbers.min() + val mean = numbers.sum() / numbers.size + val sorted = numbers.sorted() + val median = if (sorted.size % 2 == 0) { + (sorted[sorted.size / 2 - 1] + sorted[sorted.size / 2]) / 2.0 + } else { + sorted[sorted.size / 2] + } + val variance = numbers.sumOf { (it - mean) * (it - mean) } / (numbers.size - 1) + val stddev = sqrt(variance) + + val result = LinkedHashMap() + result["max"] = RuntimeValue.Num(max) + result["min"] = RuntimeValue.Num(min) + result["mean"] = RuntimeValue.Num(mean) + result["median"] = RuntimeValue.Num(median) + result["variance"] = RuntimeValue.Num(variance) + result["stddev"] = RuntimeValue.Num(stddev) + return RuntimeValue.Obj(result) + } + + private fun assertInteger(v: RuntimeValue, name: String): Long { + val n = assertNum(v) + if (n != kotlin.math.floor(n)) { + throw MistQLError.Type("range: $name must be an integer, got $n") + } + return n.toLong() + } +} diff --git a/kt/src/main/kotlin/com/mistql/builtins/ObjectFunctions.kt b/kt/src/main/kotlin/com/mistql/builtins/ObjectFunctions.kt new file mode 100644 index 00000000..39d452d5 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/builtins/ObjectFunctions.kt @@ -0,0 +1,119 @@ +package com.mistql.builtins + +import com.mistql.ast.Expression +import com.mistql.runtime.ExecutionContext +import com.mistql.runtime.RuntimeValue + +object ObjectFunctions { + val all: Map = mapOf( + "keys" to ::keys, + "values" to ::values, + "entries" to ::entries, + "fromentries" to ::fromentries, + "mapkeys" to ::mapkeys, + "mapvalues" to ::mapvalues, + "filterkeys" to ::filterkeys, + "filtervalues" to ::filtervalues, + ) + + private fun keys(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("keys", args, 1, 1) + val obj = assertObj(eval(args[0], ctx)) + val sortedKeys = obj.keys.sorted() + return RuntimeValue.Arr(sortedKeys.map { RuntimeValue.Str(it) }) + } + + private fun values(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("values", args, 1, 1) + val v = eval(args[0], ctx) + if (v !is RuntimeValue.Obj) return RuntimeValue.Arr(emptyList()) + val sortedKeys = v.entries.keys.sorted() + return RuntimeValue.Arr(sortedKeys.map { v.entries[it]!! }) + } + + private fun entries(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("entries", args, 1, 1) + val obj = assertObj(eval(args[0], ctx)) + val sortedKeys = obj.keys.sorted() + return RuntimeValue.Arr(sortedKeys.map { key -> + RuntimeValue.Arr(listOf(RuntimeValue.Str(key), obj[key]!!)) + }) + } + + private fun fromentries(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("fromentries", args, 1, 1) + val arr = assertArr(eval(args[0], ctx)) + val result = LinkedHashMap() + for (entry in arr) { + val entryArr = assertArr(entry) + val key = if (entryArr.isNotEmpty()) entryArr[0].toStringDisplay() else "null" + val value = if (entryArr.size > 1) entryArr[1] else RuntimeValue.Null + result[key] = value + } + return RuntimeValue.Obj(result) + } + + private fun mapkeys(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("mapkeys", args, 2, 2) + val transform = args[0] + val obj = assertObj(eval(args[1], ctx)) + val result = LinkedHashMap() + for ((key, value) in obj) { + ctx.pushContext(RuntimeValue.Str(key)) + val newKey = evalContextualized(transform, ctx) + ctx.popContext() + val keyStr = when (newKey) { + is RuntimeValue.Obj, is RuntimeValue.Arr -> newKey.toStringSerialize() + else -> newKey.toStringDisplay() + } + result[keyStr] = value + } + return RuntimeValue.Obj(result) + } + + private fun mapvalues(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("mapvalues", args, 2, 2) + val transform = args[0] + val obj = assertObj(eval(args[1], ctx)) + val result = LinkedHashMap() + for ((key, value) in obj) { + ctx.pushContext(value) + val newValue = evalContextualized(transform, ctx) + ctx.popContext() + result[key] = newValue + } + return RuntimeValue.Obj(result) + } + + private fun filterkeys(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("filterkeys", args, 2, 2) + val condition = args[0] + val obj = assertObj(eval(args[1], ctx)) + val result = LinkedHashMap() + for ((key, value) in obj) { + ctx.pushContext(RuntimeValue.Str(key)) + val condResult = evalContextualized(condition, ctx) + ctx.popContext() + if (condResult.truthy()) { + result[key] = value + } + } + return RuntimeValue.Obj(result) + } + + private fun filtervalues(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("filtervalues", args, 2, 2) + val condition = args[0] + val obj = assertObj(eval(args[1], ctx)) + val result = LinkedHashMap() + for ((key, value) in obj) { + ctx.pushContext(value) + val condResult = evalContextualized(condition, ctx) + ctx.popContext() + if (condResult.truthy()) { + result[key] = value + } + } + return RuntimeValue.Obj(result) + } +} diff --git a/kt/src/main/kotlin/com/mistql/builtins/Operators.kt b/kt/src/main/kotlin/com/mistql/builtins/Operators.kt new file mode 100644 index 00000000..7e23bd75 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/builtins/Operators.kt @@ -0,0 +1,147 @@ +package com.mistql.builtins + +import com.mistql.MistQLError +import com.mistql.runtime.RuntimeValue + +object Operators { + val all: Map = mapOf( + "!/unary" to ::notUnary, + "-/unary" to ::negUnary, + "||" to ::or, + "&&" to ::and, + "==" to ::eq, + "!=" to ::neq, + "=~" to ::regexMatch, + ">" to ::gt, + "<" to ::lt, + ">=" to ::gte, + "<=" to ::lte, + "+" to ::add, + "-" to ::sub, + "*" to ::mul, + "/" to ::div, + "%" to ::mod, + "." to ::dot, + ) + + private fun notUnary(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("!/unary", args, 1, 1) + return RuntimeValue.Bool(!eval(args[0], ctx).truthy()) + } + + private fun negUnary(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("-/unary", args, 1, 1) + val v = eval(args[0], ctx) + return RuntimeValue.Num(-assertNum(v)) + } + + private fun or(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("||", args, 2, 2) + val left = eval(args[0], ctx) + return if (left.truthy()) left else eval(args[1], ctx) + } + + private fun and(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("&&", args, 2, 2) + val left = eval(args[0], ctx) + return if (left.truthy()) eval(args[1], ctx) else left + } + + private fun eq(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("==", args, 2, 2) + return RuntimeValue.Bool(eval(args[0], ctx).equal(eval(args[1], ctx))) + } + + private fun neq(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("!=", args, 2, 2) + return RuntimeValue.Bool(!eval(args[0], ctx).equal(eval(args[1], ctx))) + } + + private fun regexMatch(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("=~", args, 2, 2) + val target = eval(args[0], ctx) + val pattern = eval(args[1], ctx) + val targetStr = assertStr(target) + return when (pattern) { + is RuntimeValue.Regex -> RuntimeValue.Bool(pattern.compiled.containsMatchIn(targetStr)) + is RuntimeValue.Str -> { + val regex = Regex(pattern.value) + RuntimeValue.Bool(regex.containsMatchIn(targetStr)) + } + else -> throw MistQLError.Type("=~ requires regex or string pattern, got ${pattern.typeName()}") + } + } + + private fun gt(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs(">", args, 2, 2) + return RuntimeValue.Bool(eval(args[0], ctx).compare(eval(args[1], ctx)) > 0) + } + + private fun lt(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("<", args, 2, 2) + return RuntimeValue.Bool(eval(args[0], ctx).compare(eval(args[1], ctx)) < 0) + } + + private fun gte(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs(">=", args, 2, 2) + return RuntimeValue.Bool(eval(args[0], ctx).compare(eval(args[1], ctx)) >= 0) + } + + private fun lte(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("<=", args, 2, 2) + return RuntimeValue.Bool(eval(args[0], ctx).compare(eval(args[1], ctx)) <= 0) + } + + private fun add(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("+", args, 2, 2) + val left = eval(args[0], ctx) + val right = eval(args[1], ctx) + return when { + left is RuntimeValue.Num && right is RuntimeValue.Num -> RuntimeValue.Num(left.value + right.value) + left is RuntimeValue.Str && right is RuntimeValue.Str -> RuntimeValue.Str(left.value + right.value) + left is RuntimeValue.Arr && right is RuntimeValue.Arr -> RuntimeValue.Arr(left.items + right.items) + else -> throw MistQLError.Type("Cannot add ${left.typeName()} and ${right.typeName()}") + } + } + + private fun sub(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("-", args, 2, 2) + return RuntimeValue.Num(assertNum(eval(args[0], ctx)) - assertNum(eval(args[1], ctx))) + } + + private fun mul(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("*", args, 2, 2) + return RuntimeValue.Num(assertNum(eval(args[0], ctx)) * assertNum(eval(args[1], ctx))) + } + + private fun div(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("/", args, 2, 2) + val left = assertNum(eval(args[0], ctx)) + val right = assertNum(eval(args[1], ctx)) + if (right == 0.0) throw MistQLError.Runtime("Division by zero") + return RuntimeValue.Num(left / right) + } + + private fun mod(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs("%", args, 2, 2) + val left = assertNum(eval(args[0], ctx)) + val right = assertNum(eval(args[1], ctx)) + if (right == 0.0) throw MistQLError.Runtime("Modulo by zero") + return RuntimeValue.Num(left % right) + } + + private fun dot(args: List, ctx: com.mistql.runtime.ExecutionContext): RuntimeValue { + validateArgs(".", args, 2, 2) + val obj = eval(args[0], ctx) + val fieldExpr = args[1] + val fieldName = when (fieldExpr) { + is com.mistql.ast.Expression.Ref -> fieldExpr.name + else -> assertStr(eval(fieldExpr, ctx)) + } + return when (obj) { + is RuntimeValue.Obj -> obj.entries[fieldName] ?: RuntimeValue.Null + is RuntimeValue.Null -> RuntimeValue.Null + else -> throw MistQLError.Type("Cannot access property '$fieldName' on type ${obj.typeName()}") + } + } +} diff --git a/kt/src/main/kotlin/com/mistql/builtins/StringFunctions.kt b/kt/src/main/kotlin/com/mistql/builtins/StringFunctions.kt new file mode 100644 index 00000000..79eb4f02 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/builtins/StringFunctions.kt @@ -0,0 +1,195 @@ +package com.mistql.builtins + +import com.mistql.MistQLError +import com.mistql.ast.Expression +import com.mistql.runtime.ExecutionContext +import com.mistql.runtime.RuntimeValue + +object StringFunctions { + val all: Map = mapOf( + "split" to ::split, + "stringjoin" to ::stringjoin, + "replace" to ::replace, + "match" to ::matchFn, + "regex" to ::regex, + "index" to ::index, + ) + + private fun split(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("split", args, 2, 2) + val delimiter = eval(args[0], ctx) + val str = assertStr(eval(args[1], ctx)) + + val parts: List = when (delimiter) { + is RuntimeValue.Str -> { + if (delimiter.value.isEmpty()) { + // Split by code points, not chars + val codePoints = str.codePoints().toArray() + codePoints.map { String(intArrayOf(it), 0, 1) } + } else { + str.split(delimiter.value) + } + } + is RuntimeValue.Regex -> { + delimiter.compiled.split(str) + } + else -> throw MistQLError.Type("split: delimiter must be string or regex, got ${delimiter.typeName()}") + } + return RuntimeValue.Arr(parts.map { RuntimeValue.Str(it) }) + } + + private fun stringjoin(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("stringjoin", args, 2, 2) + val delimiter = assertStr(eval(args[0], ctx)) + val arr = assertArr(eval(args[1], ctx)) + val strings = arr.map { item -> + when (item) { + is RuntimeValue.Str -> item.value + is RuntimeValue.Func -> throw MistQLError.Type("Cannot cast function to string") + is RuntimeValue.Regex -> throw MistQLError.Type("Cannot cast regex to string") + else -> item.toStringSerialize() + } + } + return RuntimeValue.Str(strings.joinToString(delimiter)) + } + + private fun replace(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("replace", args, 3, 3) + val pattern = eval(args[0], ctx) + val replacement = assertStr(eval(args[1], ctx)) + val target = assertStr(eval(args[2], ctx)) + + return when (pattern) { + is RuntimeValue.Str -> { + // Replace first occurrence only + val idx = target.indexOf(pattern.value) + val result = if (idx >= 0) { + target.substring(0, idx) + replacement + target.substring(idx + pattern.value.length) + } else { + target + } + RuntimeValue.Str(result) + } + is RuntimeValue.Regex -> { + val result = if (pattern.flags.contains('g')) { + pattern.compiled.replace(target, Regex.escapeReplacement(replacement)) + } else { + pattern.compiled.replaceFirst(target, Regex.escapeReplacement(replacement)) + } + RuntimeValue.Str(result) + } + else -> throw MistQLError.Type("replace: pattern must be string or regex, got ${pattern.typeName()}") + } + } + + private fun matchFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("match", args, 2, 2) + val pattern = eval(args[0], ctx) + val target = assertStr(eval(args[1], ctx)) + + val matches = when (pattern) { + is RuntimeValue.Str -> { + val regex = Regex(pattern.value) + regex.containsMatchIn(target) + } + is RuntimeValue.Regex -> { + pattern.compiled.containsMatchIn(target) + } + else -> throw MistQLError.Type("match: pattern must be string or regex, got ${pattern.typeName()}") + } + return RuntimeValue.Bool(matches) + } + + private fun regex(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("regex", args, 1, 2) + val pattern = assertStr(eval(args[0], ctx)) + val flags = if (args.size == 2) assertStr(eval(args[1], ctx)) else "" + return RuntimeValue.createRegex(pattern, flags) + } + + private fun index(args: List, ctx: ExecutionContext): RuntimeValue { + return when (args.size) { + 2 -> indexSingle(args, ctx) + 3 -> indexSlice(args, ctx) + else -> throw MistQLError.Runtime("index takes 2 or 3 arguments, got ${args.size}") + } + } + + private fun indexSingle(args: List, ctx: ExecutionContext): RuntimeValue { + val indexVal = eval(args[0], ctx) + val operand = eval(args[1], ctx) + + return when (operand) { + is RuntimeValue.Arr -> { + val num = assertNum(indexVal) + if (num != kotlin.math.floor(num)) throw MistQLError.Runtime("index: Non-integers cannot be used on arrays") + val idx = num.toInt() + val len = operand.items.size + val actual = if (idx < 0) len + idx else idx + if (actual in 0 until len) operand.items[actual] else RuntimeValue.Null + } + is RuntimeValue.Str -> { + val num = assertNum(indexVal) + if (num != kotlin.math.floor(num)) throw MistQLError.Runtime("index: Non-integers cannot be used on strings") + val idx = num.toInt() + val codePoints = operand.value.codePoints().toArray() + val len = codePoints.size + val actual = if (idx < 0) len + idx else idx + if (actual in 0 until len) RuntimeValue.Str(String(intArrayOf(codePoints[actual]), 0, 1)) else RuntimeValue.Null + } + is RuntimeValue.Obj -> { + val key = assertStr(indexVal) + operand.entries[key] ?: RuntimeValue.Null + } + is RuntimeValue.Null -> { + // Indexing null returns null, but only with number or string indices + when (indexVal) { + is RuntimeValue.Num, is RuntimeValue.Str -> RuntimeValue.Null + else -> throw MistQLError.Type("Cannot index null with ${indexVal.typeName()}") + } + } + else -> throw MistQLError.Type("Cannot index ${operand.typeName()}") + } + } + + private fun indexSlice(args: List, ctx: ExecutionContext): RuntimeValue { + val startVal = eval(args[0], ctx) + val endVal = eval(args[1], ctx) + val operand = eval(args[2], ctx) + + return when (operand) { + is RuntimeValue.Arr -> { + val len = operand.items.size + val start = resolveSliceIndex(startVal, 0, len) + val end = resolveSliceIndex(endVal, len, len) + val actualStart = start.coerceIn(0, len) + val actualEnd = end.coerceIn(0, len) + if (actualStart >= actualEnd) RuntimeValue.Arr(emptyList()) + else RuntimeValue.Arr(operand.items.subList(actualStart, actualEnd)) + } + is RuntimeValue.Str -> { + val codePoints = operand.value.codePoints().toArray() + val len = codePoints.size + val start = resolveSliceIndex(startVal, 0, len) + val end = resolveSliceIndex(endVal, len, len) + val actualStart = start.coerceIn(0, len) + val actualEnd = end.coerceIn(0, len) + if (actualStart >= actualEnd) RuntimeValue.Str("") + else RuntimeValue.Str(String(codePoints, actualStart, actualEnd - actualStart)) + } + is RuntimeValue.Null -> RuntimeValue.Null + else -> throw MistQLError.Type("Cannot slice ${operand.typeName()}") + } + } + + private fun resolveSliceIndex(value: RuntimeValue, default: Int, len: Int): Int { + return when (value) { + is RuntimeValue.Null -> default + is RuntimeValue.Num -> { + val idx = value.value.toInt() + if (idx < 0) len + idx else idx + } + else -> throw MistQLError.Type("Slice index must be number or null, got ${value.typeName()}") + } + } +} diff --git a/kt/src/main/kotlin/com/mistql/builtins/UtilityFunctions.kt b/kt/src/main/kotlin/com/mistql/builtins/UtilityFunctions.kt new file mode 100644 index 00000000..b02785dd --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/builtins/UtilityFunctions.kt @@ -0,0 +1,106 @@ +package com.mistql.builtins + +import com.mistql.MistQLError +import com.mistql.ast.Expression +import com.mistql.runtime.ExecutionContext +import com.mistql.runtime.RuntimeValue + +object UtilityFunctions { + val all: Map = mapOf( + "log" to ::log, + "if" to ::ifFn, + "apply" to ::apply, + "string" to ::stringFn, + "float" to ::floatFn, + "type" to ::typeFn, + "not" to ::notFn, + "empty" to ::emptyFn, + "error" to ::errorFn, + ) + + private fun log(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("log", args, 1, 1) + val value = eval(args[0], ctx) + println("MistQL: ${value.toStringSerialize()}") + return value + } + + private fun ifFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("if", args, 3, 3) + val condition = eval(args[0], ctx) + return if (condition.truthy()) eval(args[1], ctx) else eval(args[2], ctx) + } + + private fun apply(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("apply", args, 2, 2) + val expression = args[0] + val value = eval(args[1], ctx) + ctx.pushContext(value) + val result = evalContextualized(expression, ctx) + ctx.popContext() + return result + } + + private fun stringFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("string", args, 1, 1) + val value = eval(args[0], ctx) + return when (value) { + is RuntimeValue.Str -> value + is RuntimeValue.Func -> throw MistQLError.Type("Cannot cast type function to string") + is RuntimeValue.Regex -> throw MistQLError.Type("Cannot cast type regex to string") + else -> { + assertNoFunctionsOrRegexes(value) + RuntimeValue.Str(value.toStringSerialize()) + } + } + } + + private fun assertNoFunctionsOrRegexes(value: RuntimeValue) { + when (value) { + is RuntimeValue.Func -> throw MistQLError.Type("Cannot cast type function to string") + is RuntimeValue.Regex -> throw MistQLError.Type("Cannot cast type regex to string") + is RuntimeValue.Arr -> value.items.forEach { assertNoFunctionsOrRegexes(it) } + is RuntimeValue.Obj -> value.entries.values.forEach { assertNoFunctionsOrRegexes(it) } + else -> {} + } + } + + private fun floatFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("float", args, 1, 1) + val value = eval(args[0], ctx) + val num = when (value) { + is RuntimeValue.Num -> value.value + is RuntimeValue.Str -> { + val trimmed = value.value.trim() + trimmed.toDoubleOrNull() + ?: throw MistQLError.Type("Cannot convert string to float: ${value.value}") + } + is RuntimeValue.Bool -> if (value.value) 1.0 else 0.0 + is RuntimeValue.Null -> 0.0 + else -> throw MistQLError.Type("Cannot convert ${value.typeName()} to float") + } + return RuntimeValue.Num(num) + } + + private fun typeFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("type", args, 1, 1) + return RuntimeValue.Str(eval(args[0], ctx).typeName()) + } + + private fun notFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("not", args, 1, 1) + return RuntimeValue.Bool(!eval(args[0], ctx).truthy()) + } + + private fun emptyFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("empty", args, 1, 1) + val value = eval(args[0], ctx) + return RuntimeValue.Bool(!value.truthy()) + } + + private fun errorFn(args: List, ctx: ExecutionContext): RuntimeValue { + validateArgs("error", args, 1, 1) + val msg = assertStr(eval(args[0], ctx)) + throw MistQLError.Runtime(msg) + } +} diff --git a/kt/src/main/kotlin/com/mistql/cli/Main.kt b/kt/src/main/kotlin/com/mistql/cli/Main.kt new file mode 100644 index 00000000..6403eb1e --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/cli/Main.kt @@ -0,0 +1,27 @@ +package com.mistql.cli + +import com.mistql.MistQL +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement + +fun main(args: Array) { + if (args.isEmpty()) { + System.err.println("Usage: mq [file]") + System.exit(1) + } + + val queryStr = args[0] + val json = Json { ignoreUnknownKeys = true } + + val input = if (args.size > 1) { + java.io.File(args[1]).readText() + } else { + System.`in`.bufferedReader().readText() + } + + val data: JsonElement = json.parseToJsonElement(input) + val result = MistQL.queryJson(queryStr, data) + + val output = json.encodeToString(JsonElement.serializer(), result) + println(output) +} diff --git a/kt/src/main/kotlin/com/mistql/parser/Parser.kt b/kt/src/main/kotlin/com/mistql/parser/Parser.kt new file mode 100644 index 00000000..b5164f53 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/parser/Parser.kt @@ -0,0 +1,587 @@ +package com.mistql.parser + +import com.mistql.MistQLError +import com.mistql.ast.Expression +import com.mistql.runtime.RuntimeValue + +/** + * Parser for MistQL that follows the JS implementation's approach: + * 1. Lex into tokens (value, ref, special) + * 2. Collect items and joiners (operators) in a flat pass + * 3. Resolve precedence by collapsing operators from highest to lowest + * 4. Handle "amalgamating" operators (space for function application, | for pipeline) + */ +class Parser(private val input: String) { + private var pos: Int = 0 + + companion object { + fun parse(input: String): Expression { + val parser = Parser(input) + val tokens = parser.lex() + return parseTokens(tokens, input) + } + + // Operator precedence: highest first + private val SIMPLE_BINARY_OPERATORS = listOf( + listOf("."), + listOf("*", "/", "%"), + listOf("+", "-"), + listOf("<", ">", "<=", ">="), + listOf("==", "!=", "=~"), + listOf("&&"), + listOf("||"), + ) + + private val AMALGAMATING_OPERATORS = listOf(" ", "|") + + private val ALL_BINARY_OPS: Set = buildSet { + for (level in SIMPLE_BINARY_OPERATORS) addAll(level) + addAll(AMALGAMATING_OPERATORS) + } + + private val UNARY_OPS = setOf("-", "!") + + private val SPECIALS: Set = buildSet { + addAll(ALL_BINARY_OPS) + addAll(UNARY_OPS) + addAll(listOf("(", ")", "[", "]", "{", "}", ":", ",")) + } + + private val BUILTIN_VALUES = mapOf("true" to true, "false" to false, "null" to null) + private val NUM_REGEX = Regex("^(0|([1-9][0-9]*))(\\.[0-9]*)?([eE][+-]?[0-9]+)?") + + // Operators that vacuum whitespace on right side + private val VACUUM_RIGHT = setOf("(", "{", "[") + // Operators that vacuum whitespace on left side + private val VACUUM_LEFT = setOf(")", "}", "]") + // Operators that vacuum whitespace on both sides + private val VACUUM_BOTH: Set = buildSet { + add(".") + add(":") + add("|") + add(",") + for (level in SIMPLE_BINARY_OPERATORS) addAll(level) + addAll(AMALGAMATING_OPERATORS.filter { it != " " }) + } + + private fun vacuumsWhitespace(token: String, direction: String): Boolean { + if (token in VACUUM_BOTH) return true + return when (direction) { + "l" -> token in VACUUM_LEFT + "r" -> token in VACUUM_RIGHT + else -> false + } + } + + // ────────── Token-based parsing ────────── + + private fun parseTokens(tokens: List, rawQuery: String): Expression { + val (result, offset) = consumeExpression(tokens, 0, rawQuery) + if (offset != tokens.size) { + throw MistQLError.Parse("Unexpected token ${tokens[offset].value}, expected EOF") + } + return result + } + + private fun consumeExpression(tokens: List, startOffset: Int, rawQuery: String): Pair { + val items = mutableListOf() + val joiners = mutableListOf() + var offset = startOffset + + while (offset < tokens.size) { + val next = tokens[offset] + + // Determine if we should parse a unary operator + val unaryProcessors = mutableListOf<(Expression) -> Expression>() + if (next.type == TokenType.SPECIAL && next.value.toString() in UNARY_OPS && joiners.size + 1 != items.size) { + // Collect consecutive unary operators + var i = offset + while (i < tokens.size && tokens[i].type == TokenType.SPECIAL && tokens[i].value.toString() in UNARY_OPS) { + val op = tokens[i].value.toString() + unaryProcessors.add { operand -> + Expression.FnCall(Expression.Ref("${op}/unary", internal = true), listOf(operand)) + } + i++ + } + offset = i + if (offset >= tokens.size) { + throw MistQLError.Parse("Expected expression after unary operator") + } + } + + val token = if (offset < tokens.size) tokens[offset] else break + + when { + // Parenthetical + token.type == TokenType.SPECIAL && token.value == "(" -> { + guardItem(items, joiners, token, rawQuery) + val (result, newOffset) = consumeParenthetical(tokens, offset, rawQuery) + items.add(applyUnaries(result, unaryProcessors)) + offset = newOffset + } + + // Dot access - postfix on previous item + token.type == TokenType.SPECIAL && token.value == "." -> { + if (items.isEmpty()) throw MistQLError.Parse("Unexpected '.'") + val (result, newOffset) = consumeDotAccess(items.removeLast(), tokens, offset, rawQuery) + items.add(result) + offset = newOffset + } + + // [ - either array literal or indexer + token.type == TokenType.SPECIAL && token.value == "[" -> { + if (joiners.size == items.size) { + // Item position -> array literal + val (result, newOffset) = consumeArray(tokens, offset, rawQuery) + items.add(applyUnaries(result, unaryProcessors)) + offset = newOffset + } else { + // Postfix position -> indexer on previous item + val (indexApp, newOffset) = consumeIndexer(tokens, offset, rawQuery) + val prev = items.removeLast() + // indexApp is FnCall(Ref("index"), indexArgs) - append prev as last arg + val indexCall = indexApp as Expression.FnCall + items.add(Expression.FnCall(indexCall.function, indexCall.arguments + prev)) + offset = newOffset + } + } + + // Object literal + token.type == TokenType.SPECIAL && token.value == "{" -> { + guardItem(items, joiners, token, rawQuery) + val (result, newOffset) = consumeStruct(tokens, offset, rawQuery) + items.add(applyUnaries(result, unaryProcessors)) + offset = newOffset + } + + // Value literal + token.type == TokenType.VALUE -> { + guardItem(items, joiners, token, rawQuery) + val expr = valueLiteral(token.value) + items.add(applyUnaries(expr, unaryProcessors)) + offset++ + } + + // Reference + token.type == TokenType.REF -> { + guardItem(items, joiners, token, rawQuery) + val expr = Expression.Ref(token.value.toString()) + items.add(applyUnaries(expr, unaryProcessors)) + offset++ + } + + // Binary operator + token.type == TokenType.SPECIAL && token.value.toString() in ALL_BINARY_OPS -> { + if (unaryProcessors.isNotEmpty()) { + throw MistQLError.Parse("Unexpected token ${token.value}") + } + guardJoiner(items, joiners, token, rawQuery) + joiners.add(token.value.toString()) + offset++ + } + + else -> break + } + } + + // Also check for postfix indexer after resolving + val result = resolvePrecedence(items, joiners) + + // Check for trailing indexer + if (offset < tokens.size && tokens[offset].type == TokenType.SPECIAL && tokens[offset].value == "[") { + val (indexApp, newOffset) = consumeIndexer(tokens, offset, rawQuery) + val indexCall = indexApp as Expression.FnCall + return Pair(Expression.FnCall(indexCall.function, indexCall.arguments + result), newOffset) + } + + return Pair(result, offset) + } + + private fun applyUnaries(expr: Expression, processors: List<(Expression) -> Expression>): Expression { + var result = expr + for (proc in processors.reversed()) { + result = proc(result) + } + return result + } + + private fun guardItem(items: List, joiners: List, token: Token, rawQuery: String) { + if (joiners.size != items.size) { + throw MistQLError.Parse("Unexpected token ${token.value}") + } + } + + private fun guardJoiner(items: List, joiners: List, token: Token, rawQuery: String) { + if (joiners.size + 1 != items.size) { + throw MistQLError.Parse("Unexpected token ${token.value}") + } + } + + private fun valueLiteral(value: Any?): Expression { + return when (value) { + null -> Expression.Value(RuntimeValue.Null) + is Boolean -> Expression.Value(RuntimeValue.Bool(value)) + is Double -> Expression.Value(RuntimeValue.Num(value)) + is String -> Expression.Value(RuntimeValue.Str(value)) + else -> throw MistQLError.Parse("Unknown value type: $value") + } + } + + // ────── Precedence resolution ────── + + private fun resolvePrecedence(items: List, joiners: List): Expression { + if (items.isEmpty()) throw MistQLError.Parse("Tried to parse empty expression!") + if (items.size == 1 && joiners.isEmpty()) return items[0] + if (items.size - 1 != joiners.size) throw MistQLError.Parse("Invalid binary expression sequence!") + + var currentItems = items.toMutableList() + var currentJoiners = joiners.toMutableList() + + // Stage 1: Simple binary operators (highest precedence first) + for (precedenceLevel in SIMPLE_BINARY_OPERATORS) { + val newItems = mutableListOf(currentItems[0]) + val newJoiners = mutableListOf() + + for (j in currentJoiners.indices) { + newItems.add(currentItems[j + 1]) + if (currentJoiners[j] in precedenceLevel) { + val right = newItems.removeLast() + val left = newItems.removeLast() + val op = currentJoiners[j] + newItems.add(Expression.FnCall( + Expression.Ref(op, internal = true), + listOf(left, right) + )) + } else { + newJoiners.add(currentJoiners[j]) + } + } + currentItems = newItems + currentJoiners = newJoiners + } + + // Stage 2: Amalgamating operators (space, then pipe) + for (amalgOp in AMALGAMATING_OPERATORS) { + val newItems = mutableListOf(currentItems[0]) + val newJoiners = mutableListOf() + var streak = mutableListOf() + + val flushStreak = { + if (streak.isNotEmpty()) { + newItems.add(amalgamate(amalgOp, streak)) + streak = mutableListOf() + } + } + + for (j in currentJoiners.indices) { + if (currentJoiners[j] == amalgOp) { + if (streak.isEmpty()) { + streak.add(currentItems[j]) + newItems.removeLast() + } + streak.add(currentItems[j + 1]) + } else { + flushStreak() + newItems.add(currentItems[j + 1]) + newJoiners.add(currentJoiners[j]) + } + } + flushStreak() + + currentItems = newItems + currentJoiners = newJoiners + } + + if (currentJoiners.isNotEmpty()) { + throw MistQLError.Parse("Expected expression following binary expression ${currentJoiners[0]}") + } + return currentItems[0] + } + + private fun amalgamate(op: String, items: List): Expression { + return when (op) { + " " -> { + // Function application: first item is function, rest are arguments + Expression.FnCall(items[0], items.subList(1, items.size), isExplicitCall = true) + } + "|" -> { + // Pipeline: wrap non-explicit-call stages + val stages = mutableListOf(items[0]) + for (i in 1 until items.size) { + val expr = items[i] + if (expr is Expression.FnCall && expr.isExplicitCall) { + // Explicit function call (from space) — keep as-is + stages.add(expr) + } else { + // Non-explicit (operator result, bare ref, etc.) — wrap in zero-arg application + stages.add(Expression.FnCall(expr, emptyList(), isExplicitCall = true)) + } + } + Expression.Pipe(stages) + } + else -> throw MistQLError.Parse("Unknown amalgamating operator: $op") + } + } + + // ────── Composite parsers ────── + + private fun consumeParenthetical(tokens: List, startOffset: Int, rawQuery: String): Pair { + var offset = startOffset + 1 // skip ( + val (result, newOffset) = consumeExpression(tokens, offset, rawQuery) + offset = newOffset + if (offset >= tokens.size || tokens[offset].value != ")") { + throw MistQLError.Parse("Expected ')'") + } + offset++ + return Pair(Expression.Paren(result), offset) + } + + private fun consumeArray(tokens: List, startOffset: Int, rawQuery: String): Pair { + var offset = startOffset + 1 // skip [ + val entries = mutableListOf() + while (true) { + if (offset < tokens.size && tokens[offset].value == "]") { + offset++ + break + } + val (result, newOffset) = consumeExpression(tokens, offset, rawQuery) + entries.add(result) + offset = newOffset + if (offset < tokens.size && tokens[offset].value == ",") { + offset++ + continue + } else if (offset < tokens.size && tokens[offset].value == "]") { + offset++ + break + } else { + throw MistQLError.Parse("Expected ',' or ']'") + } + } + return Pair(Expression.ArrayLiteral(entries), offset) + } + + private fun consumeStruct(tokens: List, startOffset: Int, rawQuery: String): Pair { + var offset = startOffset + 1 // skip { + val entries = mutableListOf>() + while (true) { + if (offset < tokens.size && tokens[offset].value == "}") { + offset++ + break + } + if (offset >= tokens.size) throw MistQLError.Parse("Unexpected EOF") + + // Key: ref or string value + val key: String = when { + tokens[offset].type == TokenType.REF -> tokens[offset].value.toString() + tokens[offset].type == TokenType.VALUE && tokens[offset].value is String -> tokens[offset].value.toString() + else -> throw MistQLError.Parse("Expected object key, got ${tokens[offset].value}") + } + offset++ + + // Colon + if (offset >= tokens.size || tokens[offset].value != ":") { + throw MistQLError.Parse("Expected ':'") + } + offset++ + + // Value + val (value, newOffset) = consumeExpression(tokens, offset, rawQuery) + entries.add(Pair(key, value)) + offset = newOffset + + if (offset < tokens.size && tokens[offset].value == ",") { + offset++ + continue + } else if (offset < tokens.size && tokens[offset].value == "}") { + offset++ + break + } else { + throw MistQLError.Parse("Expected ',' or '}'") + } + } + return Pair(Expression.ObjectLiteral(entries), offset) + } + + private fun consumeDotAccess(left: Expression, tokens: List, startOffset: Int, rawQuery: String): Pair { + var offset = startOffset + 1 // skip . + if (offset >= tokens.size || tokens[offset].type != TokenType.REF) { + throw MistQLError.Parse("Expected identifier after '.'") + } + val name = tokens[offset].value.toString() + offset++ + return Pair( + Expression.FnCall( + Expression.Ref(".", internal = true), + listOf(left, Expression.Ref(name)) + ), + offset + ) + } + + private fun consumeIndexer(tokens: List, startOffset: Int, rawQuery: String): Pair { + var offset = startOffset + 1 // skip [ + val entries = mutableListOf() + while (true) { + if (offset < tokens.size && tokens[offset].value == ":") { + offset++ + entries.add(Expression.Value(RuntimeValue.Null)) + continue + } else if (offset < tokens.size && tokens[offset].value == "]") { + offset++ + entries.add(Expression.Value(RuntimeValue.Null)) + break + } + val (result, newOffset) = consumeExpression(tokens, offset, rawQuery) + entries.add(result) + offset = newOffset + if (offset < tokens.size && tokens[offset].value == ":") { + offset++ + continue + } else if (offset < tokens.size && tokens[offset].value == "]") { + offset++ + break + } else { + throw MistQLError.Parse("Expected ':', ']'") + } + } + return Pair( + Expression.FnCall(Expression.Ref("index", internal = true), entries), + offset + ) + } + } + + // ────────── Lexer ────────── + + enum class TokenType { VALUE, REF, SPECIAL } + data class Token(val type: TokenType, val value: Any?, val position: Int) + + private fun lex(): List { + val tokens = mutableListOf() + while (pos < input.length) { + val position = pos + val c = input[pos] + + when { + // Number + c.isDigit() -> { + val numStr = input.substring(pos) + val matchResult = NUM_REGEX.find(numStr) + ?: throw MistQLError.Parse("Invalid number at position $pos") + val matchStr = matchResult.value + tokens.add(Token(TokenType.VALUE, matchStr.toDouble(), position)) + pos += matchStr.length + } + + // Whitespace + c.isWhitespace() -> { + while (pos < input.length && input[pos].isWhitespace()) pos++ + tokens.add(Token(TokenType.SPECIAL, " ", position)) + } + + // String + c == '"' -> { + tokens.add(Token(TokenType.VALUE, parseString(), position)) + } + + // @ or $ + c == '@' || c == '$' -> { + tokens.add(Token(TokenType.REF, c.toString(), position)) + pos++ + } + + // Identifier + c.isLetter() || c == '_' -> { + val start = pos + pos++ + while (pos < input.length && (input[pos].isLetterOrDigit() || input[pos] == '_')) pos++ + val name = input.substring(start, pos) + if (name in BUILTIN_VALUES) { + tokens.add(Token(TokenType.VALUE, BUILTIN_VALUES[name], position)) + } else { + tokens.add(Token(TokenType.REF, name, position)) + } + } + + // Special characters (operators, punctuation) + else -> { + // Try matching longest special first + var matched: String? = null + for (special in SPECIALS.sortedByDescending { it.length }) { + if (input.startsWith(special, pos)) { + matched = special + break + } + } + if (matched != null) { + // Handle whitespace vacuuming + if (vacuumsWhitespace(matched, "r")) { + pos += matched.length + while (pos < input.length && input[pos].isWhitespace()) pos++ + } else { + pos += matched.length + } + if (vacuumsWhitespace(matched, "l") && tokens.isNotEmpty() + && tokens.last().type == TokenType.SPECIAL && tokens.last().value == " ") { + tokens.removeLast() + } + tokens.add(Token(TokenType.SPECIAL, matched, position)) + } else { + throw MistQLError.Parse("Unexpected character '$c' at position $pos") + } + } + } + } + + // Trim leading/trailing whitespace tokens + if (tokens.isNotEmpty() && tokens.first().type == TokenType.SPECIAL && tokens.first().value == " ") { + tokens.removeFirst() + } + if (tokens.isNotEmpty() && tokens.last().type == TokenType.SPECIAL && tokens.last().value == " ") { + tokens.removeLast() + } + + return tokens + } + + private fun parseString(): String { + pos++ // consume opening '"' + val sb = StringBuilder() + while (pos < input.length) { + val c = input[pos] + if (c == '"') { + pos++ + return sb.toString() + } + if (c == '\\') { + pos++ + if (pos >= input.length) throw MistQLError.Parse("Unterminated string escape") + when (val escaped = input[pos]) { + '"' -> sb.append('"') + '\\' -> sb.append('\\') + '/' -> sb.append('/') + 'b' -> sb.append('\b') + 'f' -> sb.append('\u000C') + 'n' -> sb.append('\n') + 'r' -> sb.append('\r') + 't' -> sb.append('\t') + 'u' -> { + if (pos + 4 >= input.length) throw MistQLError.Parse("Invalid unicode escape") + val hex = input.substring(pos + 1, pos + 5) + sb.append(hex.toInt(16).toChar()) + pos += 4 + } + else -> { + sb.append('\\') + sb.append(escaped) + } + } + } else { + sb.append(c) + } + pos++ + } + throw MistQLError.Parse("Unterminated string literal") + } + +} diff --git a/kt/src/main/kotlin/com/mistql/runtime/ExecutionContext.kt b/kt/src/main/kotlin/com/mistql/runtime/ExecutionContext.kt new file mode 100644 index 00000000..50113790 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/runtime/ExecutionContext.kt @@ -0,0 +1,80 @@ +package com.mistql.runtime + +import com.mistql.MistQLError +import com.mistql.builtins.Builtins + +class ExecutionContext(val rootData: RuntimeValue) { + private val stack: MutableList> = mutableListOf() + + init { + // Frame 0: $ frame containing builtins + root data + val dollarFrame = mutableMapOf() + dollarFrame["@"] = rootData + for (name in Builtins.allNames()) { + dollarFrame[name] = RuntimeValue.Func(name) + } + + val rootFrame = mutableMapOf() + rootFrame["$"] = RuntimeValue.Obj(LinkedHashMap(dollarFrame)) + stack.add(rootFrame) + + // Frame 1: Data context + pushContext(rootData) + } + + fun pushContext(value: RuntimeValue) { + val newFrame = mutableMapOf() + newFrame["@"] = value + + // If value is an object, populate keys as variables + if (value is RuntimeValue.Obj) { + for ((key, v) in value.entries) { + if (isValidIdentifier(key)) { + newFrame[key] = v + } + } + } + stack.add(newFrame) + } + + fun popContext() { + if (stack.size <= 2) { + throw MistQLError.Runtime("Cannot pop initial stack frames") + } + stack.removeAt(stack.lastIndex) + } + + fun findVariable(name: String, internal: Boolean): RuntimeValue { + if (internal) { + // For internal references, check builtins directly + if (Builtins.isBuiltin(name)) { + return RuntimeValue.Func(name) + } + throw MistQLError.Runtime("Could not find referenced variable $name") + } + + // Search from top to bottom + for (i in stack.indices.reversed()) { + val frame = stack[i] + val value = frame[name] + if (value != null) { + return value + } + } + + // Check builtins as fallback + if (Builtins.isBuiltin(name)) { + return RuntimeValue.Func(name) + } + + throw MistQLError.Runtime("Could not find referenced variable $name") + } + + companion object { + private val identifierRegex = Regex("^[a-zA-Z_][a-zA-Z_0-9]*$") + + fun isValidIdentifier(s: String): Boolean { + return identifierRegex.matches(s) + } + } +} diff --git a/kt/src/main/kotlin/com/mistql/runtime/Executor.kt b/kt/src/main/kotlin/com/mistql/runtime/Executor.kt new file mode 100644 index 00000000..0d15d8b8 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/runtime/Executor.kt @@ -0,0 +1,74 @@ +package com.mistql.runtime + +import com.mistql.MistQLError +import com.mistql.ast.Expression +import com.mistql.builtins.Builtins + +object Executor { + fun execute(expr: Expression, ctx: ExecutionContext): RuntimeValue { + return when (expr) { + is Expression.Value -> expr.value + is Expression.Ref -> ctx.findVariable(expr.name, expr.internal) + is Expression.FnCall -> executeFnCall(expr, ctx) + is Expression.Pipe -> executePipeline(expr.stages, ctx) + is Expression.ArrayLiteral -> RuntimeValue.Arr(expr.items.map { execute(it, ctx) }) + is Expression.ObjectLiteral -> { + val map = LinkedHashMap() + for ((k, v) in expr.entries) { + map[k] = execute(v, ctx) + } + RuntimeValue.Obj(map) + } + is Expression.Paren -> execute(expr.expression, ctx) + } + } + + private fun executeFnCall(expr: Expression.FnCall, ctx: ExecutionContext): RuntimeValue { + val funcValue = execute(expr.function, ctx) + if (funcValue !is RuntimeValue.Func) { + throw MistQLError.Runtime("Attempted to call a variable of type \"${funcValue.typeName()}\". Only functions are callable") + } + val builtin = Builtins.get(funcValue.name) + ?: throw MistQLError.Runtime("Unknown function: ${funcValue.name}") + return builtin(expr.arguments, ctx) + } + + private fun executePipeline(stages: List, ctx: ExecutionContext): RuntimeValue { + if (stages.isEmpty()) { + throw MistQLError.Runtime("Empty pipeline") + } + + var data = execute(stages[0], ctx) + + for (i in 1 until stages.size) { + val stage = stages[i] as Expression.FnCall + ctx.pushContext(data) + + // Append @ reference as the last argument (like JS does) + val atRef = Expression.Ref("@") + val newCall = Expression.FnCall(stage.function, stage.arguments + atRef) + val result = execute(newCall, ctx) + + ctx.popContext() + data = result + } + + return data + } + + /** + * Execute an expression in a contextualized way. + * If the expression is a bare function reference, call it with @ as argument. + * Otherwise, evaluate the expression directly. + */ + fun executeContextualized(expr: Expression, ctx: ExecutionContext): RuntimeValue { + if (expr is Expression.Ref && !expr.internal) { + val value = try { ctx.findVariable(expr.name, false) } catch (_: Exception) { null } + if (value is RuntimeValue.Func) { + val call = Expression.FnCall(expr, listOf(Expression.Ref("@"))) + return execute(call, ctx) + } + } + return execute(expr, ctx) + } +} diff --git a/kt/src/main/kotlin/com/mistql/runtime/RuntimeValue.kt b/kt/src/main/kotlin/com/mistql/runtime/RuntimeValue.kt new file mode 100644 index 00000000..0a085652 --- /dev/null +++ b/kt/src/main/kotlin/com/mistql/runtime/RuntimeValue.kt @@ -0,0 +1,273 @@ +package com.mistql.runtime + +import com.mistql.MistQLError +import kotlinx.serialization.json.* + +sealed class RuntimeValue { + object Null : RuntimeValue() { + override fun toString(): String = "null" + } + data class Bool(val value: Boolean) : RuntimeValue() + data class Num(val value: Double) : RuntimeValue() + data class Str(val value: String) : RuntimeValue() + data class Arr(val items: List) : RuntimeValue() + data class Obj(val entries: LinkedHashMap) : RuntimeValue() + data class Func(val name: String) : RuntimeValue() + data class Regex(val pattern: String, val flags: String, val compiled: kotlin.text.Regex) : RuntimeValue() { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Regex) return false + return pattern == other.pattern && flags == other.flags + } + override fun hashCode(): Int = pattern.hashCode() * 31 + flags.hashCode() + } + + fun truthy(): Boolean = when (this) { + is Null -> false + is Bool -> value + is Num -> value != 0.0 + is Str -> value.isNotEmpty() + is Arr -> items.isNotEmpty() + is Obj -> entries.isNotEmpty() + is Func -> true + is Regex -> true + } + + fun typeName(): String = when (this) { + is Null -> "null" + is Bool -> "boolean" + is Num -> "number" + is Str -> "string" + is Arr -> "array" + is Obj -> "object" + is Func -> "function" + is Regex -> "regex" + } + + fun comparable(): Boolean = this is Bool || this is Num || this is Str + + fun compare(other: RuntimeValue): Int { + if (this.typeName() != other.typeName()) { + throw MistQLError.Runtime("Cannot compare MistQL values of different types") + } + if (!this.comparable()) { + throw MistQLError.Runtime("Cannot compare MistQL values of type ${this.typeName()}") + } + return when { + this is Num && other is Num -> this.value.compareTo(other.value) + this is Str && other is Str -> this.value.compareTo(other.value) + this is Bool && other is Bool -> this.value.compareTo(other.value) + else -> throw MistQLError.Runtime("Cannot compare MistQL values of type ${this.typeName()}") + } + } + + fun equal(other: RuntimeValue): Boolean { + if (this.typeName() != other.typeName()) return false + return when { + this is Null && other is Null -> true + this is Bool && other is Bool -> this.value == other.value + this is Num && other is Num -> this.value == other.value + this is Str && other is Str -> this.value == other.value + this is Arr && other is Arr -> { + if (this.items.size != other.items.size) false + else this.items.zip(other.items).all { (a, b) -> a.equal(b) } + } + this is Obj && other is Obj -> { + if (this.entries.size != other.entries.size) false + else { + val aKeys = this.entries.keys.sorted() + val bKeys = other.entries.keys.sorted() + if (aKeys != bKeys) false + else aKeys.all { key -> this.entries[key]!!.equal(other.entries[key]!!) } + } + } + this is Func && other is Func -> this.name == other.name + this is Regex && other is Regex -> this.pattern == other.pattern && this.flags == other.flags + else -> false + } + } + + fun toStringDisplay(): String = when (this) { + is Null -> "null" + is Bool -> value.toString() + is Num -> formatNumber(value) + is Str -> value + is Arr -> "[${items.joinToString(",") { it.toStringSerialize() }}]" + is Obj -> "{${entries.entries.joinToString(",") { "\"${it.key}\":${it.value.toStringSerialize()}" }}}" + is Func -> "[function]" + is Regex -> "[regex]" + } + + fun toStringSerialize(): String = when (this) { + is Null -> "null" + is Bool -> value.toString() + is Num -> formatNumber(value) + is Str -> "\"${escapeJsonString(value)}\"" + is Arr -> "[${items.joinToString(",") { it.toStringSerialize() }}]" + is Obj -> "{${entries.entries.joinToString(",") { "\"${escapeJsonString(it.key)}\":${it.value.toStringSerialize()}" }}}" + is Func -> "[function]" + is Regex -> "[regex]" + } + + fun toJson(): JsonElement = when (this) { + is Null -> JsonNull + is Bool -> JsonPrimitive(value) + is Num -> JsonPrimitive(value) + is Str -> JsonPrimitive(value) + is Arr -> JsonArray(items.map { it.toJson() }) + is Obj -> JsonObject(entries.mapValues { it.value.toJson() }) + is Func -> throw MistQLError.Runtime("Cannot convert function to JSON") + is Regex -> throw MistQLError.Runtime("Cannot convert regex to JSON") + } + + fun toAny(): Any? = when (this) { + is Null -> null + is Bool -> value + is Num -> if (value == kotlin.math.floor(value) && !value.isInfinite() && kotlin.math.abs(value) < Long.MAX_VALUE.toDouble()) value.toLong() else value + is Str -> value + is Arr -> items.map { it.toAny() } + is Obj -> entries.mapValues { it.value.toAny() } + is Func -> throw MistQLError.Runtime("Cannot return function as output") + is Regex -> throw MistQLError.Runtime("Cannot return regex as output") + } + + companion object { + fun fromJson(element: JsonElement): RuntimeValue = when (element) { + is JsonNull -> Null + is JsonPrimitive -> when { + element.isString -> Str(element.content) + element.content == "true" -> Bool(true) + element.content == "false" -> Bool(false) + else -> Num(element.double) + } + is JsonArray -> Arr(element.map { fromJson(it) }) + is JsonObject -> Obj(LinkedHashMap().also { map -> + element.forEach { (k, v) -> map[k] = fromJson(v) } + }) + } + + fun fromAny(value: Any?): RuntimeValue = when (value) { + null -> Null + is Boolean -> Bool(value) + is Number -> { + val d = value.toDouble() + if (d.isNaN() || d.isInfinite()) Null + else Num(d) + } + is String -> Str(value) + is List<*> -> Arr(value.map { fromAny(it) }) + is Map<*, *> -> Obj(LinkedHashMap().also { map -> + value.forEach { (k, v) -> map[k.toString()] = fromAny(v) } + }) + is JsonElement -> fromJson(value) + else -> throw MistQLError.Type("Cannot convert ${value::class.simpleName} to RuntimeValue") + } + + fun formatNumber(n: Double): String { + if (n.isNaN()) return "NaN" + if (n.isInfinite()) return if (n > 0) "Infinity" else "-Infinity" + + // Use Double.toString() for the canonical representation, then reformat + // to match JavaScript's Number.toString() output + val javaStr = n.toString() + + // Handle scientific notation from Java (e.g., "3.141592653589793E50") + val eIdx = javaStr.indexOf('E') + if (eIdx >= 0) { + val mantissa = javaStr.substring(0, eIdx) + val exp = javaStr.substring(eIdx + 1).toInt() + + val abs = kotlin.math.abs(n) + if (abs >= 1e21 || (abs != 0.0 && abs < 1e-6)) { + // JS uses scientific notation + val cleanMantissa = mantissa.trimEnd('0').trimEnd('.') + val expStr = if (exp >= 0) "e+$exp" else "e$exp" + return "$cleanMantissa$expStr" + } + + // Otherwise expand to fixed notation + return formatFixed(n, abs) + } + + // No scientific notation from Java + val abs = kotlin.math.abs(n) + if (abs != 0.0 && abs < 1e-6) { + // JS would use scientific notation + return formatToScientific(n) + } + if (abs >= 1e21) { + return formatToScientific(n) + } + + // Regular number - Java's toString is usually fine + // But remove trailing ".0" for integers + if (n == kotlin.math.floor(n) && abs < 1e15 && javaStr.endsWith(".0")) { + return javaStr.dropLast(2) + } + return javaStr + } + + private fun formatFixed(n: Double, abs: Double): String { + // For numbers in the range [1e-6, 1e21), format as fixed-point + return if (n == kotlin.math.floor(n) && abs < 1e15) { + "%.0f".format(n) + } else { + val formatted = "%.20f".format(n) + formatted.trimEnd('0').trimEnd('.') + } + } + + private fun formatToScientific(n: Double): String { + // Convert a double to JS-style scientific notation + val javaStr = n.toString() + val eIdx = javaStr.indexOf('E') + if (eIdx >= 0) { + val mantissa = javaStr.substring(0, eIdx).trimEnd('0').trimEnd('.') + val exp = javaStr.substring(eIdx + 1).toInt() + val expStr = if (exp >= 0) "e+$exp" else "e$exp" + return "$mantissa$expStr" + } + // Shouldn't happen, but fallback + return javaStr + } + + private fun escapeJsonString(s: String): String { + val sb = StringBuilder() + for (c in s) { + when (c) { + '"' -> sb.append("\\\"") + '\\' -> sb.append("\\\\") + '\b' -> sb.append("\\b") + '\u000C' -> sb.append("\\f") + '\n' -> sb.append("\\n") + '\r' -> sb.append("\\r") + '\t' -> sb.append("\\t") + else -> { + if (c.code < 0x20) { + sb.append("\\u%04x".format(c.code)) + } else { + sb.append(c) + } + } + } + } + return sb.toString() + } + + fun createRegex(pattern: String, flags: String): Regex { + val options = mutableSetOf() + for (flag in flags) { + when (flag) { + 'i' -> options.add(RegexOption.IGNORE_CASE) + 'm' -> options.add(RegexOption.MULTILINE) + 's' -> options.add(RegexOption.DOT_MATCHES_ALL) + 'g' -> {} // handled at call site + 'x' -> options.add(RegexOption.COMMENTS) + else -> throw MistQLError.Runtime("Invalid regex flag: $flag") + } + } + val compiled = kotlin.text.Regex(pattern, options) + return Regex(pattern, flags, compiled) + } + } +} diff --git a/kt/src/test/kotlin/com/mistql/DiagnosticTest.kt b/kt/src/test/kotlin/com/mistql/DiagnosticTest.kt new file mode 100644 index 00000000..6c6d5da1 --- /dev/null +++ b/kt/src/test/kotlin/com/mistql/DiagnosticTest.kt @@ -0,0 +1,50 @@ +package com.mistql + +import org.junit.jupiter.api.Test +import kotlin.test.assertEquals + +class DiagnosticTest { + @Test + fun testBasicFunctionCalls() { + // string 1 should call string(1) and return "1" + assertEquals("1", MistQL.query("string 1", null)) + } + + @Test + fun testCount() { + assertEquals(3L, MistQL.query("count @", listOf(1, 2, 3))) + } + + @Test + fun testArithmetic() { + assertEquals(3L, MistQL.query("1 + 2", null)) + } + + @Test + fun testPipe() { + assertEquals(3L, MistQL.query("[1, 2, 3] | count", null)) + } + + @Test + fun testFilter() { + assertEquals(listOf(2L, 3L), MistQL.query("[1, 2, 3] | filter @ > 1", null)) + } + + @Test + fun testDotAccess() { + val data = mapOf("name" to "John") + assertEquals("John", MistQL.query("name", data)) + } + + @Test + fun testDollarAccess() { + val data = mapOf("filter" to "hp") + // $.filter should access the builtin filter function, not the data "filter" key + // This requires $ handling + } + + @Test + fun testRegexLiteral() { + assertEquals(true, MistQL.query("!!(regex \"hi\")", null)) + } +} diff --git a/kt/src/test/kotlin/com/mistql/SharedTestRunner.kt b/kt/src/test/kotlin/com/mistql/SharedTestRunner.kt new file mode 100644 index 00000000..b56cc14c --- /dev/null +++ b/kt/src/test/kotlin/com/mistql/SharedTestRunner.kt @@ -0,0 +1,120 @@ +package com.mistql + +import com.mistql.runtime.RuntimeValue +import kotlinx.serialization.json.* +import org.junit.jupiter.api.Assumptions +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.api.TestFactory +import java.io.File +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.test.assertFailsWith + +class SharedTestRunner { + companion object { + private const val SELF_LANG_ID = "kt" + } + + @TestFactory + fun sharedTests(): List { + val testdataFile = File("shared/testdata.json") + if (!testdataFile.exists()) { + return listOf(DynamicTest.dynamicTest("shared/testdata.json not found") { + Assumptions.assumeTrue(false, "shared/testdata.json not found") + }) + } + + val json = Json { ignoreUnknownKeys = true } + val testdata = json.parseToJsonElement(testdataFile.readText()).jsonObject + val dataBlocks = testdata["data"]!!.jsonArray + + val tests = mutableListOf() + + for (block in dataBlocks) { + val blockDescribe = block.jsonObject["describe"]!!.jsonPrimitive.content + val blockCases = block.jsonObject["cases"]!!.jsonArray + + for (innerBlock in blockCases) { + val innerDescribe = innerBlock.jsonObject["describe"]!!.jsonPrimitive.content + val innerCases = innerBlock.jsonObject["cases"]!!.jsonArray + + for (testCase in innerCases) { + val itDescription = testCase.jsonObject["it"]!!.jsonPrimitive.content + val skip = testCase.jsonObject["skip"]?.jsonArray?.map { it.jsonPrimitive.content } + val assertions = testCase.jsonObject["assertions"]!!.jsonArray + + val testName = "$blockDescribe > $innerDescribe > $itDescription" + + if (skip?.contains(SELF_LANG_ID) == true) { + tests.add(DynamicTest.dynamicTest("[SKIP] $testName") { + Assumptions.assumeTrue(false, "Skipped for $SELF_LANG_ID") + }) + continue + } + + for ((i, assertion) in assertions.withIndex()) { + val query = assertion.jsonObject["query"]!!.jsonPrimitive.content + val data = assertion.jsonObject["data"]!! + val expected = assertion.jsonObject["expected"] + val expectedSet = assertion.jsonObject["expectedSet"]?.jsonArray + val throws = assertion.jsonObject["throws"]?.jsonPrimitive?.booleanOrNull + + tests.add(DynamicTest.dynamicTest("$testName #${i + 1}: $query") { + if (throws == true) { + assertFailsWith { + MistQL.queryJson(query, data) + } + } else if (expectedSet != null) { + val result = MistQL.queryJson(query, data) + val matches = expectedSet.any { jsonElementsEqual(result, it) } + assertTrue(matches, "Result $result did not match any of $expectedSet") + } else if (expected != null) { + val result = MistQL.queryJson(query, data) + assertTrue( + jsonElementsEqual(result, expected), + "Expected $expected but got $result for query: $query" + ) + } + }) + } + } + } + } + + return tests + } + + /** + * Deep equality for JsonElement that handles number comparison correctly. + * JSON numbers like 6 and 6.0 should be considered equal. + */ + private fun jsonElementsEqual(a: JsonElement, b: JsonElement): Boolean { + return when { + a is JsonNull && b is JsonNull -> true + a is JsonPrimitive && b is JsonPrimitive -> { + if (a.isString && b.isString) { + a.content == b.content + } else if (!a.isString && !b.isString) { + // Both are non-string primitives (numbers or booleans) + // Try numeric comparison first + val aDouble = a.doubleOrNull + val bDouble = b.doubleOrNull + if (aDouble != null && bDouble != null) { + aDouble == bDouble + } else { + a.content == b.content + } + } else { + false + } + } + a is JsonArray && b is JsonArray -> { + a.size == b.size && a.zip(b).all { (x, y) -> jsonElementsEqual(x, y) } + } + a is JsonObject && b is JsonObject -> { + a.keys == b.keys && a.keys.all { key -> jsonElementsEqual(a[key]!!, b[key]!!) } + } + else -> false + } + } +}