diff --git a/.github/workflows/glm53-tool-parser-validation.yml b/.github/workflows/glm53-tool-parser-validation.yml new file mode 100644 index 000000000000..c5ec96757cf5 --- /dev/null +++ b/.github/workflows/glm53-tool-parser-validation.yml @@ -0,0 +1,25 @@ +name: GLM 5.3 parser validation + +on: + pull_request: + paths: + - 'common/chat-auto-parser-generator.cpp' + - 'models/templates/GLM-5.3-Flash.jinja' + - 'tests/test-log.cpp' + - '.github/workflows/glm53-tool-parser-validation.yml' + workflow_dispatch: + +jobs: + windows-x64: + runs-on: windows-2025 + steps: + - uses: actions/checkout@v6 + - name: Configure + shell: cmd + run: cmake -S . -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_TESTS=ON -DBUILD_SHARED_LIBS=OFF + - name: Build focused test + shell: cmd + run: cmake --build build --target test-log -j %NUMBER_OF_PROCESSORS% + - name: Run focused test + shell: cmd + run: build\bin\test-log.exe diff --git a/tests/test-glm53-tool-parser.cpp b/tests/test-glm53-tool-parser.cpp new file mode 100644 index 000000000000..81d338e9f352 --- /dev/null +++ b/tests/test-glm53-tool-parser.cpp @@ -0,0 +1,226 @@ +#include "chat-auto-parser.h" +#include "chat-peg-parser.h" +#include "chat.h" +#include "testing.h" + +#include +#include +#include +#include +#include + +using namespace autoparser; + +static std::string read_text_file(const std::string & path) { + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) { + throw std::runtime_error("Could not open file: " + path); + } + std::ostringstream out; + out << in.rdbuf(); + return out.str(); +} + +static common_chat_template load_glm53_template() { + return common_chat_template( + read_text_file("models/templates/GLM-5.3-Flash.jinja"), + "", + ""); +} + +static json build_exec_shell_tools() { + json properties = json::object(); + properties["command"] = json::object({ { "type", "string" } }); + properties["description"] = json::object({ { "type", "string" } }); + properties["workdir"] = json::object({ { "type", "string" } }); + + json parameters = json::object(); + parameters["type"] = "object"; + parameters["properties"] = properties; + parameters["required"] = json::array({ "command" }); + + json function = json::object(); + function["name"] = "exec_shell_command"; + function["description"] = "Execute a shell command"; + function["parameters"] = parameters; + + return json::array({ + json::object({ + { "type", "function" }, + { "function", function }, + }), + }); +} + +static common_peg_arena build_glm53_tool_parser(const common_chat_template & tmpl, const generation_params & inputs, + autoparser::autoparser & analysis) { + analysis.analyze_template(tmpl); + return build_chat_peg_parser([&](common_chat_peg_builder & p) { + parser_build_context ctx(p, inputs); + ctx.reasoning_parser = p.eps(); + ctx.extracting_reasoning = false; + ctx.reasoning = &analysis.reasoning; + ctx.content = &analysis.content; + return analysis.tools.build_parser(ctx); + }); +} + +static common_chat_msg parse_complete(testing & t, const common_peg_arena & parser, const std::string & label, + const std::string & input) { + common_peg_parse_context ctx(input); + auto result = parser.parse(ctx); + if (!t.assert_true(label + ": parse success", result.success())) { + return {}; + } + + common_chat_msg msg; + common_chat_peg_mapper mapper(msg); + mapper.from_ast(ctx.ast, result); + return msg; +} + +static void assert_exec_call(testing & t, const std::string & label, const common_chat_msg & msg, + const std::string & expected_command) { + if (!t.assert_equal(label + ": one tool call", size_t(1), msg.tool_calls.size())) { + return; + } + + t.assert_equal(label + ": function name", std::string("exec_shell_command"), msg.tool_calls[0].name); + + try { + auto args = json::parse(msg.tool_calls[0].arguments); + t.assert_equal(label + ": command", expected_command, args.at("command").get()); + } catch (const std::exception & e) { + t.assert_true(label + ": arguments must be valid JSON: " + std::string(e.what()), false); + } +} + +static void test_tool_dialects(testing & t) { + auto tmpl = load_glm53_template(); + + generation_params inputs; + inputs.tools = build_exec_shell_tools(); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.parallel_tool_calls = true; + inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE; + inputs.enable_thinking = true; + + autoparser::autoparser analysis; + auto parser = build_glm53_tool_parser(tmpl, inputs, analysis); + + t.assert_equal("GLM 5.3 template is analyzed as TAG_WITH_TAGGED", + tool_format::TAG_WITH_TAGGED, analysis.tools.format.mode); + + const std::string command = "cd \"L:/AI_pictures_generate/Manual LLAMA_CPP\" && ls -la | head -40"; + + const std::string canonical = + "exec_shell_command" + "command" + "cd \"L:/AI_pictures_generate/Manual LLAMA_CPP\" && ls -la | head -40" + ""; + + const std::string tag_with_json = + "exec_shell_command{\"command\":\"cd \\\"L:/AI_pictures_generate/Manual LLAMA_CPP\\\" && ls -la | head -40\"}"; + + const std::string flat_json = + "{\"function-name\":\"exec_shell_command\",\"command\":\"cd \\\"L:/AI_pictures_generate/Manual LLAMA_CPP\\\" && ls -la | head -40\"}"; + + assert_exec_call(t, "canonical tagged", parse_complete(t, parser, "canonical tagged", canonical), command); + assert_exec_call(t, "name plus JSON", parse_complete(t, parser, "name plus JSON", tag_with_json), command); + assert_exec_call(t, "flat JSON", parse_complete(t, parser, "flat JSON", flat_json), command); +} + +static void test_streaming_prefixes_do_not_become_content(testing & t) { + auto tmpl = load_glm53_template(); + + generation_params inputs; + inputs.tools = build_exec_shell_tools(); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.parallel_tool_calls = true; + inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE; + inputs.enable_thinking = true; + + autoparser::autoparser analysis; + auto parser = build_glm53_tool_parser(tmpl, inputs, analysis); + + const std::string full = "exec_shell_command{\"command\":\"pwd\"}"; + const std::string flat = "{\"function-name\":\"exec_shell_command\",\"command\":\"pwd\"}"; + + for (const auto & sample : { full, flat }) { + for (size_t i = 1; i <= sample.size(); ++i) { + common_peg_parse_context ctx(sample.substr(0, i), COMMON_PEG_PARSE_FLAG_LENIENT); + auto result = parser.parse(ctx); + if (!result.success()) { + continue; + } + + common_chat_msg msg; + common_chat_peg_mapper mapper(msg); + mapper.from_ast(ctx.ast, result); + + if (sample.substr(0, i).find("") != std::string::npos) { + t.assert_true("recognized tool prefix must not leak into content", + msg.content.find("") == std::string::npos); + } + } + } +} + +static void test_reasoning_can_end_at_tool_call(testing & t) { + auto tmpl = load_glm53_template(); + + generation_params inputs; + inputs.tools = build_exec_shell_tools(); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.parallel_tool_calls = true; + inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + inputs.enable_thinking = true; + + autoparser::autoparser analysis; + analysis.analyze_template(tmpl); + auto parser = analysis.build_parser(inputs, ""); + + const std::string input = + "Need to inspect the repository." + "exec_shell_command{\"command\":\"pwd\"}"; + + auto msg = parse_complete(t, parser, "implicit reasoning boundary", input); + t.assert_equal("reasoning before tool call", std::string("Need to inspect the repository."), msg.reasoning_content); + assert_exec_call(t, "implicit reasoning boundary", msg, "pwd"); +} + +static void test_reasoning_effort_none(testing & t) { + auto tmpl = load_glm53_template(); + + generation_params inputs; + inputs.messages = json::array({ + json::object({ { "role", "user" }, { "content", "Hello" } }), + }); + inputs.add_generation_prompt = true; + inputs.extra_context["reasoning_effort"] = "none"; + + const std::string rendered = common_chat_template_direct_apply(tmpl, inputs); + + t.assert_true("none does not emit Reasoning Effort system line", + rendered.find("Reasoning Effort: None") == std::string::npos); + t.assert_true("none closes thinking in generation prompt", + rendered.size() >= std::string("<|assistant|>").size() && + rendered.rfind("<|assistant|>") == + rendered.size() - std::string("<|assistant|>").size()); +} + +int main(int argc, char ** argv) { + testing t(std::cout); + t.verbose = true; + + if (argc > 1) { + t.set_filter(argv[1]); + } + + t.test("glm53.tool_dialects", test_tool_dialects); + t.test("glm53.streaming_prefixes", test_streaming_prefixes_do_not_become_content); + t.test("glm53.reasoning_tool_boundary", test_reasoning_can_end_at_tool_call); + t.test("glm53.reasoning_effort_none", test_reasoning_effort_none); + + return t.summary(); +} diff --git a/tests/test-log.cpp b/tests/test-log.cpp index ae4a6606bd3e..10baf4c4fa16 100644 --- a/tests/test-log.cpp +++ b/tests/test-log.cpp @@ -1,43 +1,197 @@ -#include "log.h" +#include "chat-auto-parser.h" +#include "chat-peg-parser.h" +#include "chat.h" +#include "testing.h" #include -#include +#include +#include +#include +#include +#include +#include -int main() { - const int n_thread = 8; - - std::thread threads[n_thread]; - for (int i = 0; i < n_thread; i++) { - threads[i] = std::thread([i]() { - const int n_msg = 1000; - - for (int j = 0; j < n_msg; j++) { - const int log_type = std::rand() % 4; - - switch (log_type) { - case 0: LOG_INF("Thread %d: %d\n", i, j); break; - case 1: LOG_WRN("Thread %d: %d\n", i, j); break; - case 2: LOG_ERR("Thread %d: %d\n", i, j); break; - case 3: LOG_DBG("Thread %d: %d\n", i, j); break; - default: - break; - } - - if (rand () % 10 < 5) { - common_log_set_timestamps(common_log_main(), rand() % 2); - common_log_set_prefix (common_log_main(), rand() % 2); - } - } - }); +using namespace autoparser; + +static std::string read_text_file(const std::filesystem::path & path) { + std::ifstream in(path, std::ios::binary); + if (!in.is_open()) { + throw std::runtime_error("Could not open file: " + path.string()); + } + std::ostringstream out; + out << in.rdbuf(); + return out.str(); +} + +static common_chat_template load_glm53_template() { + std::filesystem::path root; + if (const char * workspace = std::getenv("GITHUB_WORKSPACE")) { + root = workspace; + } else { + root = std::filesystem::path(__FILE__).parent_path().parent_path(); + } + return common_chat_template(read_text_file(root / "models/templates/GLM-5.3-Flash.jinja"), "", ""); +} + +static json build_tools() { + json properties = json::object(); + properties["command"] = json::object({ { "type", "string" } }); + properties["description"] = json::object({ { "type", "string" } }); + properties["workdir"] = json::object({ { "type", "string" } }); + + json parameters = json::object(); + parameters["type"] = "object"; + parameters["properties"] = properties; + parameters["required"] = json::array({ "command" }); + + return json::array({ + json::object({ + { "type", "function" }, + { "function", json::object({ + { "name", "exec_shell_command" }, + { "description", "Execute a shell command" }, + { "parameters", parameters }, + }) }, + }), + }); +} + +static common_chat_msg parse(testing & t, const common_peg_arena & parser, const std::string & label, + const std::string & input) { + common_peg_parse_context ctx(input); + auto result = parser.parse(ctx); + if (!t.assert_true(label + ": parse success", result.success())) { + return {}; } + common_chat_msg msg; + common_chat_peg_mapper mapper(msg); + mapper.from_ast(ctx.ast, result); + return msg; +} + +static void assert_call(testing & t, const std::string & label, const common_chat_msg & msg, + const std::string & command) { + if (!t.assert_equal(label + ": one call", size_t(1), msg.tool_calls.size())) { + return; + } + t.assert_equal(label + ": name", std::string("exec_shell_command"), msg.tool_calls[0].name); + try { + auto args = json::parse(msg.tool_calls[0].arguments); + t.assert_equal(label + ": command", command, args.at("command").get()); + } catch (const std::exception & e) { + t.assert_true(label + ": valid args JSON: " + std::string(e.what()), false); + } +} - for (int i = 0; i < n_thread; i++) { - threads[i].join(); +static common_peg_arena build_tool_only_parser(const common_chat_template & tmpl, const generation_params & inputs, + autoparser::autoparser & analysis) { + analysis.analyze_template(tmpl); + return build_chat_peg_parser([&](common_chat_peg_builder & p) { + parser_build_context ctx(p, inputs); + ctx.reasoning_parser = p.eps(); + ctx.extracting_reasoning = false; + ctx.reasoning = &analysis.reasoning; + ctx.content = &analysis.content; + return analysis.tools.build_parser(ctx); + }); +} + +static void test_dialects(testing & t) { + auto tmpl = load_glm53_template(); + generation_params inputs; + inputs.tools = build_tools(); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.parallel_tool_calls = true; + inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE; + inputs.enable_thinking = true; + + autoparser::autoparser analysis; + auto parser = build_tool_only_parser(tmpl, inputs, analysis); + t.assert_equal("format", tool_format::TAG_WITH_TAGGED, analysis.tools.format.mode); + + const std::string command = "cd \"L:/AI_pictures_generate/Manual LLAMA_CPP\" && ls -la | head -40"; + const std::string canonical = + "exec_shell_command" + "command" + "cd \"L:/AI_pictures_generate/Manual LLAMA_CPP\" && ls -la | head -40" + ""; + const std::string name_json = + "exec_shell_command{\"command\":\"cd \\\"L:/AI_pictures_generate/Manual LLAMA_CPP\\\" && ls -la | head -40\"}"; + const std::string flat_json = + "{\"function-name\":\"exec_shell_command\",\"command\":\"cd \\\"L:/AI_pictures_generate/Manual LLAMA_CPP\\\" && ls -la | head -40\"}"; + + assert_call(t, "canonical", parse(t, parser, "canonical", canonical), command); + assert_call(t, "name+json", parse(t, parser, "name+json", name_json), command); + assert_call(t, "flat-json", parse(t, parser, "flat-json", flat_json), command); +} + +static void test_streaming(testing & t) { + auto tmpl = load_glm53_template(); + generation_params inputs; + inputs.tools = build_tools(); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.parallel_tool_calls = true; + inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE; + inputs.enable_thinking = true; + + autoparser::autoparser analysis; + auto parser = build_tool_only_parser(tmpl, inputs, analysis); + + const std::string a = "exec_shell_command{\"command\":\"pwd\"}"; + const std::string b = "{\"function-name\":\"exec_shell_command\",\"command\":\"pwd\"}"; + for (const auto & sample : { a, b }) { + for (size_t i = 1; i <= sample.size(); ++i) { + common_peg_parse_context ctx(sample.substr(0, i), COMMON_PEG_PARSE_FLAG_LENIENT); + auto result = parser.parse(ctx); + if (!result.success()) { + continue; + } + common_chat_msg msg; + common_chat_peg_mapper mapper(msg); + mapper.from_ast(ctx.ast, result); + if (sample.substr(0, i).find("") != std::string::npos) { + t.assert_true("tool marker never leaks into content", msg.content.find("") == std::string::npos); + } + } } +} - common_log_flush(common_log_main()); - // We explicitly free the logger singleton to avoid hanging on Windows - // related to timing issues of thread startup and DLL teardown - common_log_free(common_log_main()); - return 0; +static void test_reasoning_boundary(testing & t) { + auto tmpl = load_glm53_template(); + generation_params inputs; + inputs.tools = build_tools(); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.parallel_tool_calls = true; + inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + inputs.enable_thinking = true; + + autoparser::autoparser analysis; + analysis.analyze_template(tmpl); + auto parser = analysis.build_parser(inputs, ""); + auto msg = parse(t, parser, "reasoning boundary", + "Need to inspect the repository.exec_shell_command{\"command\":\"pwd\"}"); + t.assert_equal("reasoning", std::string("Need to inspect the repository."), msg.reasoning_content); + assert_call(t, "reasoning boundary", msg, "pwd"); +} + +static void test_none(testing & t) { + auto tmpl = load_glm53_template(); + generation_params inputs; + inputs.messages = json::array({ json::object({ { "role", "user" }, { "content", "Hello" } }) }); + inputs.add_generation_prompt = true; + inputs.extra_context["reasoning_effort"] = "none"; + const std::string rendered = common_chat_template_direct_apply(tmpl, inputs); + t.assert_true("no Reasoning Effort: None", rendered.find("Reasoning Effort: None") == std::string::npos); + const std::string suffix = "<|assistant|>"; + t.assert_true("none closes think", rendered.size() >= suffix.size() && rendered.rfind(suffix) == rendered.size() - suffix.size()); +} + +int main() { + testing t(std::cout); + t.verbose = true; + t.test("glm53 dialects", test_dialects); + t.test("glm53 streaming", test_streaming); + t.test("glm53 reasoning boundary", test_reasoning_boundary); + t.test("glm53 reasoning none", test_none); + return t.summary(); }