From cb146b635c6fdf7867670f3ec3944119b0f4af26 Mon Sep 17 00:00:00 2001 From: ShiroKSH Date: Wed, 8 Jul 2026 19:58:12 +0300 Subject: [PATCH 1/2] fix: harden runner edge cases --- LLM_GUIDE.md | 4 +- QUICKSTART.md | 2 +- README.md | 24 +++-- docker/base_server.cfg | 2 +- docker/entrypoint.sh | 4 +- lua/gluatest/expectations/negative.lua | 12 ++- lua/gluatest/expectations/positive.lua | 13 +-- .../expectations/utils/table_diff.lua | 2 +- lua/gluatest/loader.lua | 13 ++- lua/gluatest/runner/helpers.lua | 83 +++++++++------- lua/gluatest/runner/runner.lua | 10 +- lua/gluatest/runner/test_case_runner.lua | 30 ++++-- lua/gluatest/stubs/stub_maker.lua | 6 +- lua/gluatest/types.lua | 4 +- lua/tests/gluatest/expectations/negative.lua | 48 +++++++++ lua/tests/gluatest/expectations/positive.lua | 45 +++++++++ .../expectations/utils/table_diff.lua | 17 ++++ .../gluatest/loader/checkSendToClients.lua | 19 ++++ lua/tests/gluatest/loader/simpleError.lua | 28 ++++++ .../gluatest/runner/test_case_runner.lua | 97 +++++++++++++++++++ lua/tests/gluatest/stubs/stub_maker.lua | 17 ++++ 21 files changed, 403 insertions(+), 77 deletions(-) create mode 100644 lua/tests/gluatest/expectations/negative.lua create mode 100644 lua/tests/gluatest/expectations/positive.lua create mode 100644 lua/tests/gluatest/expectations/utils/table_diff.lua create mode 100644 lua/tests/gluatest/loader/simpleError.lua create mode 100644 lua/tests/gluatest/runner/test_case_runner.lua create mode 100644 lua/tests/gluatest/stubs/stub_maker.lua diff --git a/LLM_GUIDE.md b/LLM_GUIDE.md index 078e44f..e3acb1f 100644 --- a/LLM_GUIDE.md +++ b/LLM_GUIDE.md @@ -9,7 +9,7 @@ Your job when writing tests: read the code under test first, then produce test f - Test files live at `lua/tests//` (any name for ``, e.g. your addon's name). Subdirectories are scanned recursively; every `.lua` file found is loaded as one test group. - Files placed directly in `lua/tests/` (not inside a project folder) are **not** loaded. - Each test file must `return` a single test group table. A file whose returned table has no `cases` field is ignored with a warning. File-local helpers and constants above the `return` are fine. -- Tests run **serverside**. +- Tests run serverside by default. Cases marked `clientside = true` or `shared = true` can run on clients when `gluatest_client_enable 1` is set. Minimal runnable skeleton: @@ -56,6 +56,8 @@ This is the complete list of case fields: | `cleanup` | `function( state )` | — | Runs after the case, **even if it failed or errored** | | `when` | boolean, function, or list of either | — | Case runs only if every condition is/returns `true` | | `skip` | boolean or function | — | Case is skipped if `true`/returns `true`; takes precedence over `when` | +| `clientside` | boolean | `false` | Run on clients when clientside tests are enabled | +| `shared` | boolean | `false` | Run serverside and on clients when clientside tests are enabled | Notes: - `when = false` (or a function returning anything but `true`) skips the case. A list form is supported: `when = { system.IsLinux(), function() return game.IsDedicated() end }`. diff --git a/QUICKSTART.md b/QUICKSTART.md index a320b45..06ee096 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -80,4 +80,4 @@ jobs: That's it! You have a working automated test runner for your project. -Check out the README for more details about [before/after functions](https://github.com/CFC-Servers/GLuaTest/blob/main/README.md#before--after-functions) and [how to make async tests](https://github.com/CFC-Servers/GLuaTest/blob/main/README.md#async-tests-and-the-done-function). +Check out the README for more details about [before/after functions](https://github.com/CFC-Servers/GLuaTest/blob/main/README.md#before--after-functions) and [how to make async tests](https://github.com/CFC-Servers/GLuaTest/blob/main/README.md#async-tests-and-the-donefail-functions). diff --git a/README.md b/README.md index bede35e..4be292d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ GLuaTest takes a lot of inspiration from both Ruby's [RSpec](https://rspec.info/ - [Troubleshooting](#troubleshooting-) - [Developers](#developers-) -![GLuaLint](https://github.com/CFC-Servers/GLuaTest/actions/workflows/glualint.yml/badge.svg) +![GLuaLint](https://github.com/CFC-Servers/GLuaTest/actions/workflows/lint.yml/badge.svg) --- _(Are you an impatient software developer? Check out the [quickstart](https://github.com/CFC-Servers/GLuaTest/blob/main/QUICKSTART.md) guide to go fast)_ @@ -340,8 +340,8 @@ jobs: | `branch` | Which GMod branch to run your tests on | `public`|`prerelease`|`dev`|`x86-64` | | `gluatest-ref` | Which tag/branch of GLuaTest to run | `main`|`feature/new-feature-branch` | | `custom-overrides` | An absolute path with custom files to copy to the server directly. Structure should match the contents of `garrysmod/` | `$GITHUB_WORKSPACE/my_overrides` | -| `download-artifact` | A URL path to a .tar.gz file that will be unpacked in the root directory | `https://github.com/RaphaelIT7/gmod-holylib/releases/download/Release0.7/gmsv_holylib_linux_packed.zip` | -| `additional-setup` | If specificed, executes the given string as a script after all setup is complete, allowing additional setup | `echo "Hello, this is a test!"` | +| `download-artifact` | A URL path to a .tar.gz file that will be unpacked in the root directory | `https://example.com/my-artifact.tar.gz` | +| `additional-setup` | If specified, executes the given string as a script after all setup is complete, allowing additional setup | `echo "Hello, this is a test!"` | @@ -405,13 +405,21 @@ You can do that with simple environment variables, i.e.: export REQUIREMENTS=/absolute/path/to/requirements.txt export CUSTOM_SERVER_CONFIG=/absolute/path/to/server.cfg export PROJECT_DIR=/home/me/Code/my_project +export GMOD_BRANCH="public" +export GMOD_ARTIFACT_DIR=/tmp/gluatest-artifacts export GAMEMODE="sandbox" export COLLECTION_ID="12345" export SSH_PRIVATE_KEY="the-entire-private-key" export GITHUB_TOKEN="a-personal-access-token" ``` - - You can skip the `REQUIREMENTS` and `CUSTOM_SERVER_CONFIG` if you don't need them, but you must set the `PROJECT_DIR` variable. + - The `REQUIREMENTS` and `CUSTOM_SERVER_CONFIG` files can be empty, but Docker Compose needs them to exist. + + - The `PROJECT_DIR` variable must point to the project or override directory you want copied into the test server. + + - The `GMOD_BRANCH` variable selects the Docker image tag. Use `public` unless you need `x86-64`, `prerelease`, or `dev`. + + - The `GMOD_ARTIFACT_DIR` variable must point to an existing directory. It can be empty. - The `GAMEMODE` variable defaults to `"sandbox"`, so you can omit it if that's appropriate for your tests. @@ -514,6 +522,8 @@ Each Test Case is a table with the following keys:
+`clientside` and `shared` are also supported for test cases that should run on clients when `gluatest_client_enable 1` is set. + ### The `expect` function The heart of a test is the _expectation_. You did a thing, and now you expect a result. @@ -541,7 +551,7 @@ There are a number of different expectations you can use. | **`deepEqual`** | Expects that two tables are deeply equal | `expect( {{ Entity(1) }} ).to.deepEqual( {{ Entity(1) }} )` | | **`beLessThan`** | Basic `<` comparison | `expect( 5 ).to.beLessThan( 6 )` | | **`beGreaterThan`** | Basic `>` comparison | `expect( 10 ).to.beGreaterThan( 1 )` | -| **`beBetween`** | Expects the subject to be less than min, and greater than max | `expect( 5 ).to.beBetween( 3, 7 )` | +| **`beBetween`** | Expects the subject to be between the lower and upper bounds | `expect( 5 ).to.beBetween( 3, 7 )` | | **`beTrue`** | Expects the subject to literally be `true` | `expect( Entity( 1 ):IsPlayer() ).to.beTrue()` | | **`beFalse`** | Expects the subject to literally be `false` | `expect( istable( "test" ) ).to.beFalse()` | | **`beValid`** | Expects `IsValid( value )` to return `true` | `expect( ply ).to.beValid()` | @@ -593,7 +603,7 @@ For example, to run your test case only on the `x86-64` branch: Skipping is also handy if you want to disable a test but keep the code: ```lua { - name = "Broken test (but I'll definitely fix it some day 100%), + name = "Broken test (but I'll definitely fix it some day 100%)", skip = true, func = function() error() end } @@ -722,7 +732,7 @@ return { }, { name = "Should check if user exists in database", - func = function() + func = function( state ) local dbCheck = stub( MyProject, "UserExistsInDatabase" ).returns( true ) MyProject.CheckUser( state.validUser ) diff --git a/docker/base_server.cfg b/docker/base_server.cfg index fc7899b..f468995 100644 --- a/docker/base_server.cfg +++ b/docker/base_server.cfg @@ -31,4 +31,4 @@ sv_cacheencodedents 0 net_compresspackets 0 sv_stats 0 -gluatest_enable 1 +gluatest_server_enable 1 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a5d02bf..4aee72e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -19,8 +19,8 @@ echo "Copying serverfiles overrides..." rsync --verbose --archive $home/garrysmod_override/ $server/ # Any additional files -if [ -n "$(ls -A $home/_gluatest_artifacts/_gluatest_artifacts/)" ]; then # Only execute if there are any artifacts or else tar will complain - cp $home/_gluatest_artifacts/_gluatest_artifacts/* $gmodroot/ +if [ -n "$(ls -A $home/_gluatest_artifacts/)" ]; then # Only execute if there are any artifacts or else tar will complain + cp $home/_gluatest_artifacts/* $gmodroot/ for file in $gmodroot/*.tar.gz; do \ tar --extract --verbose --ungzip --file="$file" --directory="$gmodroot" && \ diff --git a/lua/gluatest/expectations/negative.lua b/lua/gluatest/expectations/negative.lua index b5a938f..1fd69f4 100644 --- a/lua/gluatest/expectations/negative.lua +++ b/lua/gluatest/expectations/negative.lua @@ -8,7 +8,7 @@ local GetDiff = include( "utils/table_diff.lua" ) -- Inverse checks return function( subject, ... ) -- Args that are passed after the subject, i.e. expect( subject, arg1, arg2 ) - local args = { ... } + local args = { n = select( "#", ... ), ... } -- Wrap the subject in quotes if if's a string local fmtPrefix = "Expectation Failed: Expected %s " @@ -178,7 +178,7 @@ return function( subject, ... ) function expectations.succeed() assert( TypeID( subject ) == TYPE_FUNCTION, ".succeed expects a function" ) - local success = pcall( subject, unpack( args ) ) + local success = pcall( subject, unpack( args, 1, args.n ) ) if success ~= false then i.expected( "to not succeed" ) @@ -189,7 +189,7 @@ return function( subject, ... ) function expectations.err() assert( TypeID( subject ) == TYPE_FUNCTION, ".err expects a function" ) - local success = pcall( subject, unpack( args ) ) + local success = pcall( subject, unpack( args, 1, args.n ) ) if success ~= true then i.expected( "to not error" ) @@ -202,10 +202,10 @@ return function( subject, ... ) assert( TypeID( subject ) == TYPE_FUNCTION, ".errWith expects a function" ) assert( isstring( comparison ), "errWith expects a string" ) - local success, err = pcall( subject, unpack( args ) ) + local success, err = pcall( subject, unpack( args, 1, args.n ) ) if success == true then - i.expected( "to error" ) + return else if string.StartsWith( err, "lua/" ) or string.StartsWith( err, "addons/" ) then local _, endOfPath = string.find( err, ":%d+: ", 1 ) @@ -225,6 +225,8 @@ return function( subject, ... ) --- the Positive expectation lets you specify how many times it --- should have been called, but this one does not function expectations.called() + assert( subject.IsStub, ".called expects a stub" ) + local callCount = subject.callCount if callCount > 0 then i.expected( "to not have been called, got: %d", callCount ) diff --git a/lua/gluatest/expectations/positive.lua b/lua/gluatest/expectations/positive.lua index 92464d9..b7ae64b 100644 --- a/lua/gluatest/expectations/positive.lua +++ b/lua/gluatest/expectations/positive.lua @@ -8,7 +8,7 @@ local GetDiff = include( "utils/table_diff.lua" ) -- Positive checks return function( subject, ... ) -- Args that are passed after the subject, i.e. expect( subject, arg1, arg2 ) - local args = { ... } + local args = { n = select( "#", ... ), ... } -- Wrap the subject in quotes if if's a string local fmtPrefix = "Expectation Failed: Expected %s " @@ -69,6 +69,7 @@ return function( subject, ... ) --- @param tolerance? number Tolerance for the comparison function expectations.aboutEqual( comparison, tolerance ) assert( TypeID( subject ) == TYPE_NUMBER, ".aboutEqual expects a number" ) + assert( TypeID( comparison ) == TYPE_NUMBER, ".aboutEqual expects a number" ) tolerance = tolerance or 0.00001 local difference = math.abs( subject - comparison ) @@ -169,7 +170,7 @@ return function( subject, ... ) local class = type( subject ) if class ~= comparison then - i.expected( "to not be an '%s'", comparison ) + i.expected( "to be an '%s'", comparison ) end end @@ -177,7 +178,7 @@ return function( subject, ... ) function expectations.succeed() assert( TypeID( subject ) == TYPE_FUNCTION, ".succeed expects a function" ) - local success, err = pcall( subject, unpack( args ) ) + local success, err = pcall( subject, unpack( args, 1, args.n ) ) if success == false then i.expected( "to succeed, got: %s", err ) @@ -188,7 +189,7 @@ return function( subject, ... ) function expectations.err() assert( TypeID( subject ) == TYPE_FUNCTION, ".err expects a function" ) - local success = pcall( subject, unpack( args ) ) + local success = pcall( subject, unpack( args, 1, args.n ) ) if success == true then i.expected( "to error" ) @@ -201,7 +202,7 @@ return function( subject, ... ) assert( TypeID( subject ) == TYPE_FUNCTION, ".errWith expects a function" ) assert( TypeID( comparison ) == TYPE_STRING, ".errWith expects a string" ) - local success, err = pcall( subject, unpack( args ) ) + local success, err = pcall( subject, unpack( args, 1, args.n ) ) if success == true then i.expected( "to error with '%s'", comparison ) @@ -231,7 +232,7 @@ return function( subject, ... ) i.expected( "to have been called at least once " ) end else - if callCount < n then + if callCount ~= n then i.expected( "to have been called exactly %d times, got: %d", n, callCount ) end end diff --git a/lua/gluatest/expectations/utils/table_diff.lua b/lua/gluatest/expectations/utils/table_diff.lua index f821b50..1e6b0bb 100644 --- a/lua/gluatest/expectations/utils/table_diff.lua +++ b/lua/gluatest/expectations/utils/table_diff.lua @@ -12,7 +12,7 @@ local function stringifyKey( key ) key = "\"" .. key:gsub( "\"", "\\\"" ) .. "\"" end - return "[" .. key .. "]" + return "[" .. tostring( key ) .. "]" end local function GetDiff( t1, t2, path ) diff --git a/lua/gluatest/loader.lua b/lua/gluatest/loader.lua index 9fd1181..c35839c 100644 --- a/lua/gluatest/loader.lua +++ b/lua/gluatest/loader.lua @@ -5,10 +5,11 @@ local Loader = {} --- @param filePath string --- @param cases GLuaTest_TestCase[] function Loader.checkSendToClients( filePath, cases ) + if not SERVER then return end if not GLuaTest.RunClientsideConVar:GetBool() then return end for _, case in ipairs( cases ) do - if case.clientside then + if case.clientside or case.shared then return AddCSLuaFile( filePath ) end end @@ -26,8 +27,12 @@ end --- @param filePath string --- @return table function Loader.simpleError( reason, filePath ) + reason = tostring( reason or "file did not return a test group" ) + local linePrefixStart = string.find( reason, ":%d+:" ) + if linePrefixStart then reason = string.sub( reason, linePrefixStart + 1 ) end + return { - reason = string.sub( reason, string.find( reason, ":" ) + 1 ), + reason = reason, sourceFile = filePath, lineNumber = -1, locals = {} @@ -44,6 +49,10 @@ function Loader.processFile( dir, fileName, groups ) local filePath = dir .. "/" .. fileName local success, result = pcall( function( givenFilePath ) local fileContent = file.Read( givenFilePath, "LUA" ) + if not fileContent then + return Loader.simpleError( "Unable to read file: " .. givenFilePath, givenFilePath ) + end + local compiled = CompileString( fileContent, "lua/" .. givenFilePath, false ) if not isfunction( compiled ) then diff --git a/lua/gluatest/runner/helpers.lua b/lua/gluatest/runner/helpers.lua index d94b6a4..6b1935b 100644 --- a/lua/gluatest/runner/helpers.lua +++ b/lua/gluatest/runner/helpers.lua @@ -30,7 +30,7 @@ function Helpers.makeHookTable() if not trackedHooks[event] then trackedHooks[event] = {} end table.insert( trackedHooks[event], name ) - if not isfunction( func ) and func.IsStub then + if istable( func ) and func.IsStub then local givenStub = func func = function( ... ) givenStub( ... ) @@ -183,10 +183,10 @@ function Helpers.findStackInfo() -- Step up through the stacks to find the error we care about for stack = 1, 12 do - local info = debug.getinfo( stack, "lnS" ) + local info = debug.getinfo( stack, "lnSn" ) if not info then break end - local emptyName = #info.namewhat == 0 + local emptyName = #( info.namewhat or "" ) == 0 local notGluatest = not string.match( info.short_src, "/lua/gluatest/" ) if emptyName and notGluatest then @@ -259,33 +259,36 @@ function Helpers.MakeAsyncEnv( done, fail, onFailedExpectation ) -- they're called in an async function expect = function( ... ) local built = expect( ... ) - local expected = built.to.expected - local recordedFailure = false - -- Wrap the error-throwing function - -- and handle the error with the correct context - -- (and to only record the first failure) - built.to.expected = function( ... ) - if recordedFailure then return end + local function wrapExpected( expectations ) + local expected = expectations.expected + local recordedFailure = false - local stack = debug.getinfo( 3, "lnS" ) - local locals = Helpers.getLocals( 4 ) + expectations.expected = function( ... ) + if recordedFailure then return end - local _, errInfo = xpcall( expected, Helpers.FailCallback, ... ) - assert( errInfo ) + local stack = debug.getinfo( 3, "lnS" ) + local locals = Helpers.getLocals( 4 ) - local final = { - reason = errInfo.reason, - sourceFile = stack.short_src, - lineNumber = stack.currentline, - locals = locals - } + local _, errInfo = xpcall( expected, Helpers.FailCallback, ... ) + assert( errInfo ) - onFailedExpectation( final --[[@as GLuaTest_FailCallbackInfo]] ) + local final = { + reason = errInfo.reason, + sourceFile = stack.short_src, + lineNumber = stack.currentline, + locals = locals + } - recordedFailure = true + onFailedExpectation( final --[[@as GLuaTest_FailCallbackInfo]] ) + + recordedFailure = true + end end + wrapExpected( built.to ) + wrapExpected( built.notTo ) + return built end, @@ -323,23 +326,35 @@ function Helpers.SafeRunWithEnv( defaultEnv, before, func, state ) return ogExpect( ... ) end - -- Before - if before then - setfenv( before, testEnv ) - before( state ) - setfenv( before, defaultEnv ) + local function cleanup() + for _, cleanupFunc in ipairs( cleanupFuncs ) do + cleanupFunc() + end end - -- Run - setfenv( func, testEnv ) - local success, output = xpcall( func, Helpers.FailCallback, state ) - setfenv( func, defaultEnv ) + local function runWithEnv( funcToRun ) + setfenv( funcToRun, testEnv ) + local success, output = xpcall( funcToRun, Helpers.FailCallback, state ) + setfenv( funcToRun, defaultEnv ) - -- Cleanup - for _, cleanup in ipairs( cleanupFuncs ) do - cleanup() + return success, output end + if before then + local beforeSuccess, beforeOutput = runWithEnv( before ) + if not beforeSuccess then + cleanup() + + return { + result = "failure", + errInfo = beforeOutput + } + end + end + + local success, output = runWithEnv( func ) + cleanup() + if success then -- If it succeeded but never ran `expect`, it's an empty test if not ranExpect then diff --git a/lua/gluatest/runner/runner.lua b/lua/gluatest/runner/runner.lua index 2ce04b0..a8fdf54 100644 --- a/lua/gluatest/runner/runner.lua +++ b/lua/gluatest/runner/runner.lua @@ -28,12 +28,13 @@ end --- @param testGroups GLuaTest_TestGroup[] function TestRunner:Complete( testGroups ) local duration = SysTime() - self.startTime + local results = self.results - hook.Run( "GLuaTest_Finished", testGroups, self.results, duration ) + hook.Run( "GLuaTest_Finished", testGroups, results, duration ) -- Some logs may be printed after our completion message, we need to wait a bit timer.Simple( 0.1, function() - LogTestsComplete( testGroups, self.results, duration ) + LogTestsComplete( testGroups, results, duration ) end ) end @@ -46,6 +47,7 @@ function TestRunner:Run( testGroups ) hook.Run( "GLuaTest_StartedTestRun", testGroups ) self.startTime = SysTime() + self.results = {} --- @type GLuaTest_TestGroupRunner[] local runners = {} @@ -55,9 +57,11 @@ function TestRunner:Run( testGroups ) table.insert( runners, runner ) end + local runnerIndex = 1 local function runNext() --- @type GLuaTest_TestGroupRunner - local nextRunner = table.remove( runners ) + local nextRunner = runners[runnerIndex] + runnerIndex = runnerIndex + 1 if not nextRunner then return self:Complete( testGroups ) diff --git a/lua/gluatest/runner/test_case_runner.lua b/lua/gluatest/runner/test_case_runner.lua index b49d567..20a148f 100644 --- a/lua/gluatest/runner/test_case_runner.lua +++ b/lua/gluatest/runner/test_case_runner.lua @@ -96,12 +96,19 @@ function GLuaTest.TestCaseRunner( TestGroupRunner, case ) --- @param cb fun(): nil The function to run once the test is complete function TCR:RunAsync( cb ) local isDone = false - local expectationFailure = false + local resultRecorded = false local asyncCleanup = function() ErrorNoHaltWithStack( "Running an empty Async Cleanup func" ) end + local function recordFailure( errInfo ) + if resultRecorded then return end + + TestGroupRunner:SetFailed( case, errInfo ) + resultRecorded = true + end + local function testComplete() isDone = true @@ -123,8 +130,9 @@ function GLuaTest.TestCaseRunner( TestGroupRunner, case ) local function done() if isDone then return end - if not expectationFailure then + if not resultRecorded then TestGroupRunner:SetSucceeded( case ) + resultRecorded = true end testComplete() @@ -136,7 +144,7 @@ function GLuaTest.TestCaseRunner( TestGroupRunner, case ) local function fail( reason ) if isDone then return end - TestGroupRunner:SetFailed( case, { reason = reason or "fail() called" } ) + recordFailure( { reason = reason or "fail() called" } ) testComplete() end @@ -149,10 +157,8 @@ function GLuaTest.TestCaseRunner( TestGroupRunner, case ) --- @param errInfo GLuaTest_FailCallbackInfo local function onFailedExpectation( errInfo ) if isDone then return end - if expectationFailure then return end - TestGroupRunner:SetFailed( case, errInfo ) - expectationFailure = true + recordFailure( errInfo ) end local asyncEnv, asyncCleanupFuncs = Helpers.MakeAsyncEnv( done, fail, onFailedExpectation ) @@ -165,12 +171,18 @@ function GLuaTest.TestCaseRunner( TestGroupRunner, case ) local beforeEach = group.beforeEach if beforeEach then setfenv( beforeEach, asyncEnv ) - beforeEach( case.state ) + local beforeSuccess, beforeErrInfo = xpcall( beforeEach, Helpers.FailCallback, case.state ) setfenv( beforeEach, defaultEnv ) + + if not beforeSuccess then + recordFailure( beforeErrInfo ) + testComplete() + return + end end local function setTimedOut() - TestGroupRunner:SetTimedOut( case ) + recordFailure( { reason = "Timeout" } ) testComplete() end @@ -185,7 +197,7 @@ function GLuaTest.TestCaseRunner( TestGroupRunner, case ) -- (Async expectation failures handled in asyncEnv.expect) -- (Async unhandled failures handled with timeouts) if not success then - TestGroupRunner:SetFailed( case, errInfo ) + recordFailure( errInfo ) testComplete() return diff --git a/lua/gluatest/stubs/stub_maker.lua b/lua/gluatest/stubs/stub_maker.lua index feafdcc..aa61a81 100644 --- a/lua/gluatest/stubs/stub_maker.lua +++ b/lua/gluatest/stubs/stub_maker.lua @@ -1,4 +1,4 @@ ---- @alias StubFunction fun(tbl: table, key: any): GLuaTest_Stub +--- @alias StubFunction fun(tbl?: table, key?: any): GLuaTest_Stub --- @alias GLuaTest_StubMaker fun(): StubFunction, fun(): nil return function() @@ -60,9 +60,9 @@ return function() function stubTbl.returns( ... ) assert( stubTbl.stubbedFunc == nil, "Stub already set" ) - local args = { ... } + local args = { n = select( "#", ... ), ... } stubTbl.stubbedFunc = function() - return unpack( args ) + return unpack( args, 1, args.n ) end return stubTbl diff --git a/lua/gluatest/types.lua b/lua/gluatest/types.lua index f11561f..67f3fb0 100644 --- a/lua/gluatest/types.lua +++ b/lua/gluatest/types.lua @@ -71,8 +71,8 @@ function expect( subject, ... ) end --- Create a stub function ---- @param tbl table The table to stub ---- @param key any The key to stub +--- @param tbl? table The table to stub +--- @param key? any The key to stub --- @return GLuaTest_Stub function stub( tbl, key ) -- _ is used to make the GLuaLinter happy, as else it'll complain that the variables are unused. diff --git a/lua/tests/gluatest/expectations/negative.lua b/lua/tests/gluatest/expectations/negative.lua new file mode 100644 index 0000000..b71b555 --- /dev/null +++ b/lua/tests/gluatest/expectations/negative.lua @@ -0,0 +1,48 @@ +local Helpers = include( "gluatest/runner/helpers.lua" ) + +return { + groupName = "Negative Expectations", + + cases = { + { + name = "called requires a stub", + func = function() + expect( function() + expect( {} ).wasNot.called() + end ).to.errWith( ".called expects a stub" ) + end + }, + { + name = "errWith passes when the function succeeds", + func = function() + expect( function() + expect( function() end ).notTo.errWith( "boom" ) + end ).to.succeed() + end + }, + { + name = "async negated expectation failures are recorded", + func = function() + local recordedFailure + local env, cleanupFuncs = Helpers.MakeAsyncEnv( function() end, function() end, function( errInfo ) + recordedFailure = errInfo + end ) + local defaultEnv = getfenv( 1 ) + local testFunc = function() + expect( true ).notTo.beTrue() + end + + setfenv( testFunc, env ) + testFunc() + setfenv( testFunc, defaultEnv ) + + for _, cleanup in ipairs( cleanupFuncs ) do + cleanup() + end + + expect( recordedFailure ).to.exist() + expect( recordedFailure.reason ).to.equal( "Expected true to not be true" ) + end + } + } +} diff --git a/lua/tests/gluatest/expectations/positive.lua b/lua/tests/gluatest/expectations/positive.lua new file mode 100644 index 0000000..8cd4d6d --- /dev/null +++ b/lua/tests/gluatest/expectations/positive.lua @@ -0,0 +1,45 @@ +return { + groupName = "Positive Expectations", + + cases = { + { + name = "called with a count fails when the stub was called too many times", + func = function() + local testStub = stub() + + testStub() + testStub() + + expect( function() + expect( testStub ).was.called( 1 ) + end ).to.errWith( "Expectation Failed: Expected GLuaTest::Stub to have been called exactly 1 times, got: 2" ) + end + }, + { + name = "aboutEqual requires a numeric comparison", + func = function() + expect( function() + expect( 1 ).to.aboutEqual( "1" ) + end ).to.errWith( ".aboutEqual expects a number" ) + end + }, + { + name = "succeed forwards trailing nil arguments", + func = function() + local function receivesTrailingNil( ... ) + expect( select( "#", ... ) ).to.equal( 2 ) + end + + expect( receivesTrailingNil, "value", nil ).to.succeed() + end + }, + { + name = "beAn failure message uses positive wording", + func = function() + expect( function() + expect( "value" ).to.beAn( "number" ) + end ).to.errWith( "Expectation Failed: Expected 'value' to be an 'number'" ) + end + } + } +} diff --git a/lua/tests/gluatest/expectations/utils/table_diff.lua b/lua/tests/gluatest/expectations/utils/table_diff.lua new file mode 100644 index 0000000..0ef7cc1 --- /dev/null +++ b/lua/tests/gluatest/expectations/utils/table_diff.lua @@ -0,0 +1,17 @@ +local GetDiff = include( "gluatest/expectations/utils/table_diff.lua" ) + +return { + groupName = "Table Diff", + + cases = { + { + name = "Formats boolean keys in diff paths", + func = function() + local isDifferent, path = GetDiff( { [true] = 1 }, { [true] = 2 } ) + + expect( isDifferent ).to.beTrue() + expect( path ).to.equal( "tableA[true]" ) + end + } + } +} diff --git a/lua/tests/gluatest/loader/checkSendToClients.lua b/lua/tests/gluatest/loader/checkSendToClients.lua index b151d24..355735c 100644 --- a/lua/tests/gluatest/loader/checkSendToClients.lua +++ b/lua/tests/gluatest/loader/checkSendToClients.lua @@ -65,6 +65,25 @@ return { expect( AddCSLuaFileStub ).wasNot.called() end, + cleanup = function( state ) + GLuaTest.RunClientsideConVar:SetBool( state.currentRunClientside ) + end + }, + + { + name = "Sends shared cases when the gluatest_client_enable ConVar is enabled", + func = function( state ) + local Loader = state.Loader + state.currentRunClientside = GLuaTest.RunClientsideConVar:GetBool() + + GLuaTest.RunClientsideConVar:SetBool( true ) + local AddCSLuaFileStub = stub( _G, "AddCSLuaFile" ) + + local cases = { { shared = true } } + Loader.checkSendToClients( "test.lua", cases ) + expect( AddCSLuaFileStub ).was.called() + end, + cleanup = function( state ) GLuaTest.RunClientsideConVar:SetBool( state.currentRunClientside ) end diff --git a/lua/tests/gluatest/loader/simpleError.lua b/lua/tests/gluatest/loader/simpleError.lua new file mode 100644 index 0000000..512cbfd --- /dev/null +++ b/lua/tests/gluatest/loader/simpleError.lua @@ -0,0 +1,28 @@ +return { + groupName = "simpleError", + + beforeEach = function( state ) + state.Loader = include( "gluatest/loader.lua" ) + end, + + cases = { + { + name = "Handles errors without a file prefix", + func = function( state ) + local errInfo = state.Loader.simpleError( "plain failure", "tests/project/file.lua" ) + + expect( errInfo.reason ).to.equal( "plain failure" ) + expect( errInfo.sourceFile ).to.equal( "tests/project/file.lua" ) + expect( errInfo.lineNumber ).to.equal( -1 ) + end + }, + { + name = "Trims prefixed file errors", + func = function( state ) + local errInfo = state.Loader.simpleError( "tests/project/file.lua:12: prefixed failure", "tests/project/file.lua" ) + + expect( errInfo.reason ).to.equal( "12: prefixed failure" ) + end + } + } +} diff --git a/lua/tests/gluatest/runner/test_case_runner.lua b/lua/tests/gluatest/runner/test_case_runner.lua new file mode 100644 index 0000000..a6f578c --- /dev/null +++ b/lua/tests/gluatest/runner/test_case_runner.lua @@ -0,0 +1,97 @@ +include( "gluatest/runner/test_case_runner.lua" ) + +local noop = function() end + +local function makeTestGroupRunner( group ) + local results = {} + local runner = { + group = group, + results = results + } + + function runner:SetSucceeded( case ) + table.insert( results, { case = case, success = true } ) + end + + function runner:SetFailed( case, errInfo ) + table.insert( results, { case = case, success = false, errInfo = errInfo } ) + end + + function runner:SetEmpty( case ) + table.insert( results, { case = case, empty = true } ) + end + + return runner +end + +local function makeCase( name, func ) + return { + name = name, + func = func, + cleanup = noop, + id = name, + state = {} + } +end + +return { + groupName = "TestCaseRunner", + + cases = { + { + name = "Records sync beforeEach errors as failures", + func = function() + local caseRan = false + local group = { + beforeEach = function() + error( "sync beforeEach failed" ) + end + } + local testGroupRunner = makeTestGroupRunner( group ) + local case = makeCase( "sync_before_each_failure", function() + caseRan = true + end ) + local caseRunner = GLuaTest.TestCaseRunner( testGroupRunner, case ) + local completed = false + + caseRunner:RunSync( function() + completed = true + end ) + + expect( completed ).to.beTrue() + expect( caseRan ).to.beFalse() + expect( #testGroupRunner.results ).to.equal( 1 ) + expect( testGroupRunner.results[1].success ).to.beFalse() + expect( testGroupRunner.results[1].errInfo.reason ).to.equal( "Unhandled: sync beforeEach failed" ) + end + }, + { + name = "Records async beforeEach errors as failures", + func = function() + local caseRan = false + local group = { + beforeEach = function() + error( "async beforeEach failed" ) + end + } + local testGroupRunner = makeTestGroupRunner( group ) + local case = makeCase( "async_before_each_failure", function() + caseRan = true + done() + end ) + local caseRunner = GLuaTest.TestCaseRunner( testGroupRunner, case ) + local completed = false + + caseRunner:RunAsync( function() + completed = true + end ) + + expect( completed ).to.beTrue() + expect( caseRan ).to.beFalse() + expect( #testGroupRunner.results ).to.equal( 1 ) + expect( testGroupRunner.results[1].success ).to.beFalse() + expect( testGroupRunner.results[1].errInfo.reason ).to.equal( "Unhandled: async beforeEach failed" ) + end + } + } +} diff --git a/lua/tests/gluatest/stubs/stub_maker.lua b/lua/tests/gluatest/stubs/stub_maker.lua new file mode 100644 index 0000000..a538fbb --- /dev/null +++ b/lua/tests/gluatest/stubs/stub_maker.lua @@ -0,0 +1,17 @@ +return { + groupName = "Stub Maker", + + cases = { + { + name = "returns preserves trailing nil values", + func = function() + local testStub = stub().returns( "value", nil ) + local value, trailingNil = testStub() + + expect( value ).to.equal( "value" ) + expect( trailingNil ).to.beNil() + expect( select( "#", testStub() ) ).to.equal( 2 ) + end + } + } +} From efd921b5e71a75086af7a75e4b5972b9f5a52ea6 Mon Sep 17 00:00:00 2001 From: ShiroKSH Date: Wed, 15 Jul 2026 17:09:01 +0300 Subject: [PATCH 2/2] fix: address runner review feedback --- lua/gluatest/expectations/negative.lua | 12 +++++------- lua/gluatest/expectations/positive.lua | 8 ++++---- lua/gluatest/stubs/stub_maker.lua | 4 ++-- lua/tests/gluatest/expectations/positive.lua | 10 ---------- .../gluatest/loader/checkSendToClients.lua | 16 ++++++++-------- lua/tests/gluatest/runner/test_case_runner.lua | 4 ++-- lua/tests/gluatest/stubs/stub_maker.lua | 17 ----------------- 7 files changed, 21 insertions(+), 50 deletions(-) delete mode 100644 lua/tests/gluatest/stubs/stub_maker.lua diff --git a/lua/gluatest/expectations/negative.lua b/lua/gluatest/expectations/negative.lua index 1fd69f4..db97f8e 100644 --- a/lua/gluatest/expectations/negative.lua +++ b/lua/gluatest/expectations/negative.lua @@ -8,7 +8,7 @@ local GetDiff = include( "utils/table_diff.lua" ) -- Inverse checks return function( subject, ... ) -- Args that are passed after the subject, i.e. expect( subject, arg1, arg2 ) - local args = { n = select( "#", ... ), ... } + local args = { ... } -- Wrap the subject in quotes if if's a string local fmtPrefix = "Expectation Failed: Expected %s " @@ -178,7 +178,7 @@ return function( subject, ... ) function expectations.succeed() assert( TypeID( subject ) == TYPE_FUNCTION, ".succeed expects a function" ) - local success = pcall( subject, unpack( args, 1, args.n ) ) + local success = pcall( subject, unpack( args ) ) if success ~= false then i.expected( "to not succeed" ) @@ -189,7 +189,7 @@ return function( subject, ... ) function expectations.err() assert( TypeID( subject ) == TYPE_FUNCTION, ".err expects a function" ) - local success = pcall( subject, unpack( args, 1, args.n ) ) + local success = pcall( subject, unpack( args ) ) if success ~= true then i.expected( "to not error" ) @@ -202,11 +202,9 @@ return function( subject, ... ) assert( TypeID( subject ) == TYPE_FUNCTION, ".errWith expects a function" ) assert( isstring( comparison ), "errWith expects a string" ) - local success, err = pcall( subject, unpack( args, 1, args.n ) ) + local success, err = pcall( subject, unpack( args ) ) - if success == true then - return - else + if success == false then if string.StartsWith( err, "lua/" ) or string.StartsWith( err, "addons/" ) then local _, endOfPath = string.find( err, ":%d+: ", 1 ) assert( endOfPath, "Could not find end of path in error message: " .. err ) diff --git a/lua/gluatest/expectations/positive.lua b/lua/gluatest/expectations/positive.lua index b7ae64b..c644251 100644 --- a/lua/gluatest/expectations/positive.lua +++ b/lua/gluatest/expectations/positive.lua @@ -8,7 +8,7 @@ local GetDiff = include( "utils/table_diff.lua" ) -- Positive checks return function( subject, ... ) -- Args that are passed after the subject, i.e. expect( subject, arg1, arg2 ) - local args = { n = select( "#", ... ), ... } + local args = { ... } -- Wrap the subject in quotes if if's a string local fmtPrefix = "Expectation Failed: Expected %s " @@ -178,7 +178,7 @@ return function( subject, ... ) function expectations.succeed() assert( TypeID( subject ) == TYPE_FUNCTION, ".succeed expects a function" ) - local success, err = pcall( subject, unpack( args, 1, args.n ) ) + local success, err = pcall( subject, unpack( args ) ) if success == false then i.expected( "to succeed, got: %s", err ) @@ -189,7 +189,7 @@ return function( subject, ... ) function expectations.err() assert( TypeID( subject ) == TYPE_FUNCTION, ".err expects a function" ) - local success = pcall( subject, unpack( args, 1, args.n ) ) + local success = pcall( subject, unpack( args ) ) if success == true then i.expected( "to error" ) @@ -202,7 +202,7 @@ return function( subject, ... ) assert( TypeID( subject ) == TYPE_FUNCTION, ".errWith expects a function" ) assert( TypeID( comparison ) == TYPE_STRING, ".errWith expects a string" ) - local success, err = pcall( subject, unpack( args, 1, args.n ) ) + local success, err = pcall( subject, unpack( args ) ) if success == true then i.expected( "to error with '%s'", comparison ) diff --git a/lua/gluatest/stubs/stub_maker.lua b/lua/gluatest/stubs/stub_maker.lua index aa61a81..e94000a 100644 --- a/lua/gluatest/stubs/stub_maker.lua +++ b/lua/gluatest/stubs/stub_maker.lua @@ -60,9 +60,9 @@ return function() function stubTbl.returns( ... ) assert( stubTbl.stubbedFunc == nil, "Stub already set" ) - local args = { n = select( "#", ... ), ... } + local args = { ... } stubTbl.stubbedFunc = function() - return unpack( args, 1, args.n ) + return unpack( args ) end return stubTbl diff --git a/lua/tests/gluatest/expectations/positive.lua b/lua/tests/gluatest/expectations/positive.lua index 8cd4d6d..3815385 100644 --- a/lua/tests/gluatest/expectations/positive.lua +++ b/lua/tests/gluatest/expectations/positive.lua @@ -23,16 +23,6 @@ return { end ).to.errWith( ".aboutEqual expects a number" ) end }, - { - name = "succeed forwards trailing nil arguments", - func = function() - local function receivesTrailingNil( ... ) - expect( select( "#", ... ) ).to.equal( 2 ) - end - - expect( receivesTrailingNil, "value", nil ).to.succeed() - end - }, { name = "beAn failure message uses positive wording", func = function() diff --git a/lua/tests/gluatest/loader/checkSendToClients.lua b/lua/tests/gluatest/loader/checkSendToClients.lua index 355735c..3c59c5b 100644 --- a/lua/tests/gluatest/loader/checkSendToClients.lua +++ b/lua/tests/gluatest/loader/checkSendToClients.lua @@ -18,7 +18,7 @@ return { name = "Sends clientside cases when the gluatest_client_enable ConVar is enabled", func = function( state ) local Loader = state.Loader - state.currentRunClientside = GLuaTest.RunClientsideConVar:GetBool() + state.previousRunClientside = GLuaTest.RunClientsideConVar:GetBool() GLuaTest.RunClientsideConVar:SetBool( true ) local AddCSLuaFileStub = stub( _G, "AddCSLuaFile" ) @@ -28,7 +28,7 @@ return { expect( AddCSLuaFileStub ).was.called() end, cleanup = function( state ) - GLuaTest.RunClientsideConVar:SetBool( state.currentRunClientside ) + GLuaTest.RunClientsideConVar:SetBool( state.previousRunClientside ) end }, @@ -36,7 +36,7 @@ return { name = "Does not send clientside cases if the gluatest_client_enable ConVar is disabled", func = function( state ) local Loader = state.Loader - state.currentRunClientside = GLuaTest.RunClientsideConVar:GetBool() + state.previousRunClientside = GLuaTest.RunClientsideConVar:GetBool() GLuaTest.RunClientsideConVar:SetBool( false ) local AddCSLuaFileStub = stub( _G, "AddCSLuaFile" ) @@ -47,7 +47,7 @@ return { end, cleanup = function( state ) - GLuaTest.RunClientsideConVar:SetBool( state.currentRunClientside ) + GLuaTest.RunClientsideConVar:SetBool( state.previousRunClientside ) end }, @@ -55,7 +55,7 @@ return { name = "Does not send non-clientside cases when the gluatest_client_enable ConVar is enabled", func = function( state ) local Loader = state.Loader - state.currentRunClientside = GLuaTest.RunClientsideConVar:GetBool() + state.previousRunClientside = GLuaTest.RunClientsideConVar:GetBool() GLuaTest.RunClientsideConVar:SetBool( true ) local AddCSLuaFileStub = stub( _G, "AddCSLuaFile" ) @@ -66,7 +66,7 @@ return { end, cleanup = function( state ) - GLuaTest.RunClientsideConVar:SetBool( state.currentRunClientside ) + GLuaTest.RunClientsideConVar:SetBool( state.previousRunClientside ) end }, @@ -74,7 +74,7 @@ return { name = "Sends shared cases when the gluatest_client_enable ConVar is enabled", func = function( state ) local Loader = state.Loader - state.currentRunClientside = GLuaTest.RunClientsideConVar:GetBool() + state.previousRunClientside = GLuaTest.RunClientsideConVar:GetBool() GLuaTest.RunClientsideConVar:SetBool( true ) local AddCSLuaFileStub = stub( _G, "AddCSLuaFile" ) @@ -85,7 +85,7 @@ return { end, cleanup = function( state ) - GLuaTest.RunClientsideConVar:SetBool( state.currentRunClientside ) + GLuaTest.RunClientsideConVar:SetBool( state.previousRunClientside ) end } } diff --git a/lua/tests/gluatest/runner/test_case_runner.lua b/lua/tests/gluatest/runner/test_case_runner.lua index a6f578c..3c929ba 100644 --- a/lua/tests/gluatest/runner/test_case_runner.lua +++ b/lua/tests/gluatest/runner/test_case_runner.lua @@ -39,7 +39,7 @@ return { cases = { { - name = "Records sync beforeEach errors as failures", + name = "Fails the case when beforeEach errors", func = function() local caseRan = false local group = { @@ -66,7 +66,7 @@ return { end }, { - name = "Records async beforeEach errors as failures", + name = "Fails the async case when beforeEach errors", func = function() local caseRan = false local group = { diff --git a/lua/tests/gluatest/stubs/stub_maker.lua b/lua/tests/gluatest/stubs/stub_maker.lua deleted file mode 100644 index a538fbb..0000000 --- a/lua/tests/gluatest/stubs/stub_maker.lua +++ /dev/null @@ -1,17 +0,0 @@ -return { - groupName = "Stub Maker", - - cases = { - { - name = "returns preserves trailing nil values", - func = function() - local testStub = stub().returns( "value", nil ) - local value, trailingNil = testStub() - - expect( value ).to.equal( "value" ) - expect( trailingNil ).to.beNil() - expect( select( "#", testStub() ) ).to.equal( 2 ) - end - } - } -}