From 72d6d106b9e25691bb0d82d0cb01ff9688ee38d0 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Thu, 13 Aug 2026 19:16:32 +0200 Subject: [PATCH 01/12] Skip ignored files in Docker sync Build an exclude manifest from Git's standard ignore rules and pass it to rsync when copying a package into the Docker test environment. Retain the existing sync behavior outside Git repositories and for callers that do not provide a manifest. --- docker/README.md | 1 + docker/entrypoint.sh | 6 ++++- docker/run_tests.py | 59 ++++++++++++++++++++++++++++++++++++++------ sbin/ci.sh | 7 +++++- 4 files changed, 63 insertions(+), 10 deletions(-) diff --git a/docker/README.md b/docker/README.md index c55e549b..c900814a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -29,6 +29,7 @@ By default it: - runs UnitTesting through the same CI shell entrypoints - stores Sublime install/cache in docker volume `unittesting-home` - synchronizes only changed files into `Packages/` using `rsync` +- excludes files ignored by Git, including repository-local and global rules ## Manual docker usage diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 05dfa2d6..27f260b3 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -99,7 +99,11 @@ fi if [ -d "$UNITTESTING_SOURCE/sbin" ]; then # Ensure UnitTesting comes from the local checkout running this script, # so first runs do not depend on tagged upstream releases. - (cd "$UNITTESTING_SOURCE" && PACKAGE=UnitTesting /docker.sh copy_tested_package overwrite) + ( + cd "$UNITTESTING_SOURCE" + PACKAGE=UnitTesting UNITTESTING_IGNORE_MANIFEST= \ + /docker.sh copy_tested_package overwrite + ) # Normalize CRLF in shell scripts copied from Windows workspaces. if [ -d "$ST_PACKAGES_DIR/UnitTesting/sbin" ]; then diff --git a/docker/run_tests.py b/docker/run_tests.py index 23ad2418..c3e08f39 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -85,6 +85,7 @@ def main(argv: list[str] | None = None) -> int: lock_enabled = should_lock_cache(args) runner_name = docker_cache_runner_name(args.cache_volume) if lock_enabled else None + ignore_manifest = make_git_ignore_manifest(package_root) command = build_docker_run_command( package_root=package_root, unit_testing_root=unit_testing_root, @@ -92,6 +93,7 @@ def main(argv: list[str] | None = None) -> int: image=image, cache_volume=args.cache_volume, container_name=runner_name, + ignore_manifest=ignore_manifest, scheduler_delay_ms=args.scheduler_delay_ms, coverage=args.coverage, failfast=args.failfast, @@ -116,16 +118,22 @@ def main(argv: list[str] | None = None) -> int: print("Cache refresh: enabled") if tests_dir and pattern: print(f"Test target: {tests_dir}/{pattern}") + if ignore_manifest: + print("Package sync: Git-ignored files excluded") - if lock_enabled: - with CacheVolumeLock(args.cache_volume, args.lock_timeout): - wait_for_cache_volume_idle(args.cache_volume, args.lock_timeout) - ensure_runner_container_name_available(args.cache_volume, args.lock_timeout) - return call_docker_run_with_name_retry( - command, args.cache_volume, args.lock_timeout - ) + try: + if lock_enabled: + with CacheVolumeLock(args.cache_volume, args.lock_timeout): + wait_for_cache_volume_idle(args.cache_volume, args.lock_timeout) + ensure_runner_container_name_available(args.cache_volume, args.lock_timeout) + return call_docker_run_with_name_retry( + command, args.cache_volume, args.lock_timeout + ) - return subprocess.call(command) + return subprocess.call(command) + finally: + if ignore_manifest: + ignore_manifest.unlink(missing_ok=True) def parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -377,6 +385,35 @@ def resolve_test_target( return resolved_tests_dir, resolved_pattern +def make_git_ignore_manifest(package_root: Path) -> Path | None: + if not shutil.which("git"): + return None + + result = subprocess.run( + [ + "git", + "-C", + str(package_root), + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + "-z", + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + return None + + with tempfile.NamedTemporaryFile( + prefix="unittesting-ignore-", suffix=".files", delete=False + ) as manifest: + manifest.write(result.stdout) + return Path(manifest.name) + + def build_docker_run_command( package_root: Path, unit_testing_root: Path, @@ -384,6 +421,7 @@ def build_docker_run_command( image: str, cache_volume: str | None, container_name: str | None, + ignore_manifest: Path | None, scheduler_delay_ms: int, coverage: bool, failfast: bool, @@ -409,6 +447,11 @@ def build_docker_run_command( command.extend(["-v", f"{package_root}:/project"]) command.extend(["-v", f"{unit_testing_root}:/unittesting"]) + if ignore_manifest: + manifest_target = "/tmp/unittesting-ignore.files" + command.extend(["-e", f"UNITTESTING_IGNORE_MANIFEST={manifest_target}"]) + command.extend(["-v", f"{ignore_manifest}:{manifest_target}:ro"]) + if cache_volume: command.extend(["-v", f"{cache_volume}:/root"]) diff --git a/sbin/ci.sh b/sbin/ci.sh index b0ae90a8..573e19dc 100644 --- a/sbin/ci.sh +++ b/sbin/ci.sh @@ -123,7 +123,12 @@ CopyTestedPackage() { if [ -n "$OverwriteExisting" ] && command -v rsync >/dev/null 2>&1; then echo "sync package into sublime package directory" - rsync -a --delete --exclude .git ./ "$STP/$PACKAGE/" + if [ -n "${UNITTESTING_IGNORE_MANIFEST:-}" ] && [ -f "$UNITTESTING_IGNORE_MANIFEST" ]; then + rsync -a --delete --delete-excluded --from0 --exclude .git \ + --exclude-from="$UNITTESTING_IGNORE_MANIFEST" ./ "$STP/$PACKAGE/" + else + rsync -a --delete --exclude .git ./ "$STP/$PACKAGE/" + fi return fi From 2139cfa55da3c28eb46a258b3fbddc283d141c01 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 09:54:35 +0200 Subject: [PATCH 02/12] Encapsulate temporary ignore manifest Move temporary-file cleanup into a context-managed manifest object and use its optional path when constructing the Docker command. --- docker/run_tests.py | 111 +++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 48 deletions(-) diff --git a/docker/run_tests.py b/docker/run_tests.py index c3e08f39..e80fdc12 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -86,42 +86,42 @@ def main(argv: list[str] | None = None) -> int: lock_enabled = should_lock_cache(args) runner_name = docker_cache_runner_name(args.cache_volume) if lock_enabled else None ignore_manifest = make_git_ignore_manifest(package_root) - command = build_docker_run_command( - package_root=package_root, - unit_testing_root=unit_testing_root, - package_name=package_name, - image=image, - cache_volume=args.cache_volume, - container_name=runner_name, - ignore_manifest=ignore_manifest, - scheduler_delay_ms=args.scheduler_delay_ms, - coverage=args.coverage, - failfast=args.failfast, - reload_package_on_testing=args.reload_package_on_testing, - dry_run=args.dry_run, - color=args.color, - tests_dir=tests_dir, - pattern=pattern, - ) + with ignore_manifest: + command = build_docker_run_command( + package_root=package_root, + unit_testing_root=unit_testing_root, + package_name=package_name, + image=image, + cache_volume=args.cache_volume, + container_name=runner_name, + ignore_manifest=ignore_manifest, + scheduler_delay_ms=args.scheduler_delay_ms, + coverage=args.coverage, + failfast=args.failfast, + reload_package_on_testing=args.reload_package_on_testing, + dry_run=args.dry_run, + color=args.color, + tests_dir=tests_dir, + pattern=pattern, + ) - print(f"Package root: {package_root}") - print(f"Package name: {package_name}") - print(f"Docker image: {image}") - print(f"Scheduler delay: {args.scheduler_delay_ms}ms") - if args.refresh_image: - print("Image refresh: enabled") - if args.cache_volume: - print(f"Cache volume: {args.cache_volume}") - if lock_enabled: - print("Cache lock: enabled") - if args.refresh_cache: - print("Cache refresh: enabled") - if tests_dir and pattern: - print(f"Test target: {tests_dir}/{pattern}") - if ignore_manifest: - print("Package sync: Git-ignored files excluded") + print(f"Package root: {package_root}") + print(f"Package name: {package_name}") + print(f"Docker image: {image}") + print(f"Scheduler delay: {args.scheduler_delay_ms}ms") + if args.refresh_image: + print("Image refresh: enabled") + if args.cache_volume: + print(f"Cache volume: {args.cache_volume}") + if lock_enabled: + print("Cache lock: enabled") + if args.refresh_cache: + print("Cache refresh: enabled") + if tests_dir and pattern: + print(f"Test target: {tests_dir}/{pattern}") + if ignore_manifest: + print("Package sync: Git-ignored files excluded") - try: if lock_enabled: with CacheVolumeLock(args.cache_volume, args.lock_timeout): wait_for_cache_volume_idle(args.cache_volume, args.lock_timeout) @@ -131,9 +131,6 @@ def main(argv: list[str] | None = None) -> int: ) return subprocess.call(command) - finally: - if ignore_manifest: - ignore_manifest.unlink(missing_ok=True) def parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -385,9 +382,31 @@ def resolve_test_target( return resolved_tests_dir, resolved_pattern -def make_git_ignore_manifest(package_root: Path) -> Path | None: +class GitIgnoreManifest: + def __init__(self, contents: bytes | None = None) -> None: + self.path: Path | None = None + if contents is not None: + with tempfile.NamedTemporaryFile( + prefix="unittesting-ignore-", suffix=".files", delete=False + ) as manifest: + manifest.write(contents) + self.path = Path(manifest.name) + + def __bool__(self) -> bool: + return self.path is not None + + def __enter__(self) -> "GitIgnoreManifest": + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + if self.path: + self.path.unlink(missing_ok=True) + self.path = None + + +def make_git_ignore_manifest(package_root: Path) -> GitIgnoreManifest: if not shutil.which("git"): - return None + return GitIgnoreManifest() result = subprocess.run( [ @@ -405,13 +424,9 @@ def make_git_ignore_manifest(package_root: Path) -> Path | None: stderr=subprocess.DEVNULL, ) if result.returncode != 0: - return None + return GitIgnoreManifest() - with tempfile.NamedTemporaryFile( - prefix="unittesting-ignore-", suffix=".files", delete=False - ) as manifest: - manifest.write(result.stdout) - return Path(manifest.name) + return GitIgnoreManifest(result.stdout) def build_docker_run_command( @@ -421,7 +436,7 @@ def build_docker_run_command( image: str, cache_volume: str | None, container_name: str | None, - ignore_manifest: Path | None, + ignore_manifest: GitIgnoreManifest, scheduler_delay_ms: int, coverage: bool, failfast: bool, @@ -447,10 +462,10 @@ def build_docker_run_command( command.extend(["-v", f"{package_root}:/project"]) command.extend(["-v", f"{unit_testing_root}:/unittesting"]) - if ignore_manifest: + if ignore_manifest.path: manifest_target = "/tmp/unittesting-ignore.files" command.extend(["-e", f"UNITTESTING_IGNORE_MANIFEST={manifest_target}"]) - command.extend(["-v", f"{ignore_manifest}:{manifest_target}:ro"]) + command.extend(["-v", f"{ignore_manifest.path}:{manifest_target}:ro"]) if cache_volume: command.extend(["-v", f"{cache_volume}:/root"]) From 488f9996bac50c774117a00f998bc3f228c73b5a Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 11:06:34 +0200 Subject: [PATCH 03/12] Preserve selected ignored test files Translate Git's ignored paths into rsync filter rules and include an explicitly selected test file without copying its ignored siblings. --- docker/run_tests.py | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/docker/run_tests.py b/docker/run_tests.py index e80fdc12..17671441 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -74,7 +74,7 @@ def main(argv: list[str] | None = None) -> int: return 2 package_name = args.package_name or package_root.name - tests_dir, pattern = resolve_test_target( + tests_dir, pattern, selected_file = resolve_test_target( package_root, args.file, args.tests_dir, args.pattern ) @@ -85,7 +85,7 @@ def main(argv: list[str] | None = None) -> int: lock_enabled = should_lock_cache(args) runner_name = docker_cache_runner_name(args.cache_volume) if lock_enabled else None - ignore_manifest = make_git_ignore_manifest(package_root) + ignore_manifest = make_git_ignore_manifest(package_root, selected_file) with ignore_manifest: command = build_docker_run_command( package_root=package_root, @@ -359,9 +359,9 @@ def resolve_test_target( test_file: str | None, tests_dir: str | None, pattern: str | None, -) -> tuple[str | None, str | None]: +) -> tuple[str | None, str | None, str | None]: if not test_file: - return tests_dir, pattern + return tests_dir, pattern, None file_path = Path(test_file) if not file_path.is_absolute(): @@ -379,7 +379,7 @@ def resolve_test_target( rel_parent = rel_file_path.parent.as_posix() resolved_tests_dir = rel_parent if rel_parent else "." resolved_pattern = rel_file_path.name - return resolved_tests_dir, resolved_pattern + return resolved_tests_dir, resolved_pattern, rel_file_path.as_posix() class GitIgnoreManifest: @@ -404,7 +404,9 @@ def __exit__(self, exc_type, exc_value, traceback) -> None: self.path = None -def make_git_ignore_manifest(package_root: Path) -> GitIgnoreManifest: +def make_git_ignore_manifest( + package_root: Path, selected_file: str | None = None +) -> GitIgnoreManifest: if not shutil.which("git"): return GitIgnoreManifest() @@ -426,7 +428,36 @@ def make_git_ignore_manifest(package_root: Path) -> GitIgnoreManifest: if result.returncode != 0: return GitIgnoreManifest() - return GitIgnoreManifest(result.stdout) + return GitIgnoreManifest(make_rsync_ignore_filter(result.stdout, selected_file)) + + +def make_rsync_ignore_filter(paths: bytes, selected_file: str | None) -> bytes: + entries = [path for path in paths.split(b"\0") if path] + rules: list[bytes] = [] + + if selected_file: + selected_path = os.fsencode(selected_file) + ignored_directories = [ + path + for path in entries + if path.endswith(b"/") and selected_path.startswith(path) + ] + if selected_path in entries or ignored_directories: + parts = selected_path.split(b"/") + # rsync uses the first matching rule and does not descend into an + # excluded directory. Include the selected file and its ancestors + # first... + rules.extend( + b"+ /" + b"/".join(parts[:i]) + b"/" + for i in range(1, len(parts)) + ) + rules.append(b"+ /" + selected_path) + # ... then exclude everything else under the ignored ancestor + # directories (`/***`). + rules.extend(b"- /" + path + b"***" for path in ignored_directories) + + rules.extend(b"- /" + path for path in entries) + return b"\0".join(rules) + (b"\0" if rules else b"") def build_docker_run_command( From 558526ce5372f59f6933ccf94faf569ff57dd346 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 09:25:45 +0200 Subject: [PATCH 04/12] Reject conflicting Docker test targets --- docker/run_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/run_tests.py b/docker/run_tests.py index 17671441..0695b950 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -235,6 +235,9 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: if args.file and args.pattern: parser.error("--file and --pattern are mutually exclusive") + if args.file and args.tests_dir: + parser.error("--file and --tests-dir are mutually exclusive") + if args.refresh_cache and not args.cache_volume: parser.error("--refresh-cache requires a cache volume (omit --no-cache-volume)") From 2cd697e9be50e926c3691f05e8365a5c32b29db6 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 09:26:51 +0200 Subject: [PATCH 05/12] Reorder manual Docker usage docs --- docker/README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/README.md b/docker/README.md index c900814a..f82a952a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -31,20 +31,6 @@ By default it: - synchronizes only changed files into `Packages/` using `rsync` - excludes files ignored by Git, including repository-local and global rules -## Manual docker usage - -```sh -# build from UnitTesting/docker -docker build -t unittesting-local . - -# run from package root -docker run --rm -it \ - -e PACKAGE=$PACKAGE \ - -v $PWD:/project \ - -v unittesting-home:/root \ - unittesting-local run_tests -``` - ## Fast reruns The container entrypoint writes a marker in `/root/.cache/unittesting`. @@ -121,7 +107,21 @@ Use `--color` to control ANSI colors in test output: ut-run-tests . --color always ``` -## Run a single test file +## Manual docker usage + +```sh +# build from UnitTesting/docker +docker build -t unittesting-local . + +# run from package root +docker run --rm -it \ + -e PACKAGE=$PACKAGE \ + -v $PWD:/project \ + -v unittesting-home:/root \ + unittesting-local run_tests +``` + +Run a single test file ```sh docker run --rm -it \ From e6250f1bb55e6909575e209b691cbdc2790ac4ff Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 09:34:33 +0200 Subject: [PATCH 06/12] Clarify Docker runner concurrency --- docker/README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docker/README.md b/docker/README.md index f82a952a..61e41b7b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -37,28 +37,29 @@ The container entrypoint writes a marker in `/root/.cache/unittesting`. With `-v unittesting-home:/root`, bootstrap/install runs once and later runs only refresh your package files and execute tests. -## Serialized runs +## Concurrent runs and races The shared cache volume contains the Sublime data directory, including -`Packages`, `Lib`, UnitTesting schedules and test output files. Concurrent -runs against the same volume are serialized by default to avoid races while +`Packages`, `Lib`, UnitTesting schedules and test output files. *Concurrent +runs against the same volume are serialized by default* to avoid races while copying packages, writing schedules and syncing Package Control libraries. +This is a speed-versus-space trade-off: tests are likely to run quickly, +keeping wait times low, and a shared volume uses less disk space than +multiple volumes. Use `--lock-timeout SECONDS` to control how long a runner waits for the cache volume lock. Use `--no-lock` only if you know the selected cache volume is not shared by another runner. -## Concurrent runs - -You can control concurrency by choosing how many cache volumes you use. The -default single volume serializes all runs. A stable volume per package allows -different packages to run concurrently while still keeping warm caches: +You can increase concurrency by choosing how many cache volumes you use. For +example, a stable volume per package allows different packages to run +concurrently while still keeping warm caches: ```sh ut-run-tests . --cache-volume unittesting-home-gitsavvy ``` -To maximize concurrency, use a stable volume per checkout directory. For +To *maximize* concurrency, use a stable volume per checkout directory. For example, in a POSIX shell: ```sh From f31798d5b24b3a3de4c9243eb2b018eba113e1bd Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 11:03:15 +0200 Subject: [PATCH 07/12] Reject repeated Docker file options Report repeated --file options instead of silently using only the final value. --- docker/run_tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker/run_tests.py b/docker/run_tests.py index 0695b950..03c72862 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -147,7 +147,11 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: ) test_group = parser.add_argument_group("test options") - test_group.add_argument("--file", help="Run only tests from this file.") + test_group.add_argument( + "--file", + action="append", + help="Run only tests from this file (may be specified once).", + ) test_group.add_argument("--pattern", help="Custom unittest discovery pattern.") test_group.add_argument("--tests-dir", help="Custom tests directory.") test_group.add_argument("--package-name", help="Override package name.") @@ -232,6 +236,10 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: args = parser.parse_args(argv) + if args.file and len(args.file) > 1: + parser.error("--file may only be specified once") + args.file = args.file[0] if args.file else None + if args.file and args.pattern: parser.error("--file and --pattern are mutually exclusive") From ea5fd7564e86295c8d5ab6bf732525b1c99ae56a Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 11:35:15 +0200 Subject: [PATCH 08/12] Filter syntax resources by test options Forward tests_dir and pattern through scheduled syntax runs and use them to narrow syntax test and compatibility resources within the selected package. --- tests/test_3141596.py | 27 ++++++++++++++++++++++++++- unittesting/scheduler.py | 16 +++++++++++++--- unittesting/syntax.py | 28 ++++++++++++++++++++++++---- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/tests/test_3141596.py b/tests/test_3141596.py index d7b253c9..9cf3c421 100644 --- a/tests/test_3141596.py +++ b/tests/test_3141596.py @@ -39,7 +39,8 @@ def cleanup_package(package): def with_package(package, output=None, syntax_test=False, syntax_compatibility=False, - color_scheme_test=False, wait_timeout=5000): + color_scheme_test=False, wait_timeout=5000, pattern=None, + tests_dir=None): def wrapper(func): @wraps(func) def real_wrapper(self): @@ -56,6 +57,10 @@ def real_wrapper(self): yield AWAIT_WORKER kwargs = {"package": package} + if pattern is not None: + kwargs["pattern"] = pattern + if tests_dir is not None: + kwargs["tests_dir"] = tests_dir if outfile: # Command kwargs have the highest precedence. Passing down # 'None' is not what we want, the intention is to omit it @@ -181,6 +186,14 @@ def test_fail_syntax(self, txt): def test_success_syntax(self, txt): self.assertOk(txt) + @with_package("_Syntax_Success", syntax_test=True, pattern="missing*") + def test_syntax_pattern(self, txt): + self.assertRegexContains(txt, r'^ERROR: No syntax_test') + + @with_package("_Syntax_Success", syntax_test=True, tests_dir="missing") + def test_syntax_tests_dir(self, txt): + self.assertRegexContains(txt, r'^ERROR: No syntax_test') + @with_package("_Syntax_Error", syntax_test=True) def test_error_syntax(self, txt): self.assertRegexContains(txt, r'^ERROR: No syntax_test') @@ -193,6 +206,18 @@ def test_fail_syntax_compatibility(self, txt): def test_success_syntax_compatibility(self, txt): self.assertOk(txt) + @with_package( + "_Syntax_Compat_Success", syntax_compatibility=True, pattern="missing*" + ) + def test_syntax_compatibility_pattern(self, txt): + self.assertRegexContains(txt, r'^ERROR: No sublime-syntax') + + @with_package( + "_Syntax_Compat_Success", syntax_compatibility=True, tests_dir="missing" + ) + def test_syntax_compatibility_tests_dir(self, txt): + self.assertRegexContains(txt, r'^ERROR: No sublime-syntax') + def has_colorschemeunit(): return ( diff --git a/unittesting/scheduler.py b/unittesting/scheduler.py index 871c2171..32054a97 100644 --- a/unittesting/scheduler.py +++ b/unittesting/scheduler.py @@ -76,16 +76,26 @@ def run(self): ) elif self.syntax_test: sublime.active_window().run_command( - "unit_testing_syntax", {"package": self.package, "output": self.output} + "unit_testing_syntax", self.syntax_testing_args() ) elif self.syntax_compatibility: sublime.active_window().run_command( - "unit_testing_syntax_compatibility", - {"package": self.package, "output": self.output}, + "unit_testing_syntax_compatibility", self.syntax_testing_args() ) else: sublime.active_window().run_command("unit_testing", self.unit_testing_args()) + def syntax_testing_args(self): + args = {"package": self.package, "output": self.output} + args.update( + { + key: self.unit_testing_options[key] + for key in ("pattern", "tests_dir") + if key in self.unit_testing_options + } + ) + return args + def unit_testing_args(self): args = { "package": self.package, diff --git a/unittesting/syntax.py b/unittesting/syntax.py index 20f60bce..79fdc87a 100644 --- a/unittesting/syntax.py +++ b/unittesting/syntax.py @@ -20,8 +20,11 @@ def run(self, package=None, **kwargs): failed_assertions = 0 try: - tests = sublime.find_resources("syntax_test*") - tests = [t for t in tests if t.startswith("Packages/%s/" % package)] + tests = find_package_resources( + package, + kwargs.get("pattern", "syntax_test*"), + kwargs.get("tests_dir"), + ) if not tests: raise RuntimeError("No syntax_test files are found in %s!" % package) @@ -66,8 +69,11 @@ def run(self, package=None, **kwargs): stream = self.load_stream(package, settings) try: - syntaxes = sublime.find_resources("*.sublime-syntax") - syntaxes = [s for s in syntaxes if s.startswith("Packages/%s/" % package)] + syntaxes = find_package_resources( + package, + kwargs.get("pattern", "*.sublime-syntax"), + kwargs.get("tests_dir"), + ) if not syntaxes: raise RuntimeError("No sublime-syntax files found in %s!" % package) @@ -109,3 +115,17 @@ def run(self, package=None, **kwargs): stream.write("\n") stream.write(DONE_MESSAGE) stream.close() + + +def find_package_resources(package, pattern, tests_dir=None): + resource_prefix = "Packages/%s/" % package + if tests_dir: + tests_dir = tests_dir.replace("\\", "/").strip("/") + if tests_dir != ".": + resource_prefix += tests_dir + "/" + + return [ + resource + for resource in sublime.find_resources(pattern) + if resource.startswith(resource_prefix) + ] From 53df6318a83c3ad4a15bc3f469d14f798ae028d1 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 11:37:09 +0200 Subject: [PATCH 09/12] Allow empty syntax resource sets Add an opt-in fail_if_no_resources setting while preserving the existing failure behavior by default. Forward it through scheduled syntax runs. --- tests/test_3141596.py | 18 +++++++++++++++++- unittesting/scheduler.py | 16 +++++++++------- unittesting/syntax.py | 8 ++++---- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/test_3141596.py b/tests/test_3141596.py index 9cf3c421..6954f68f 100644 --- a/tests/test_3141596.py +++ b/tests/test_3141596.py @@ -40,7 +40,7 @@ def cleanup_package(package): def with_package(package, output=None, syntax_test=False, syntax_compatibility=False, color_scheme_test=False, wait_timeout=5000, pattern=None, - tests_dir=None): + tests_dir=None, fail_if_no_resources=None): def wrapper(func): @wraps(func) def real_wrapper(self): @@ -61,6 +61,8 @@ def real_wrapper(self): kwargs["pattern"] = pattern if tests_dir is not None: kwargs["tests_dir"] = tests_dir + if fail_if_no_resources is not None: + kwargs["fail_if_no_resources"] = fail_if_no_resources if outfile: # Command kwargs have the highest precedence. Passing down # 'None' is not what we want, the intention is to omit it @@ -198,6 +200,12 @@ def test_syntax_tests_dir(self, txt): def test_error_syntax(self, txt): self.assertRegexContains(txt, r'^ERROR: No syntax_test') + @with_package( + "_Syntax_Error", syntax_test=True, fail_if_no_resources=False + ) + def test_empty_syntax_allowed(self, txt): + self.assertOk(txt) + @with_package("_Syntax_Compat_Failure", syntax_compatibility=True) def test_fail_syntax_compatibility(self, txt): self.assertRegexContains(txt, r'^FAILED: 3 errors in 1 of 1 syntax$') @@ -218,6 +226,14 @@ def test_syntax_compatibility_pattern(self, txt): def test_syntax_compatibility_tests_dir(self, txt): self.assertRegexContains(txt, r'^ERROR: No sublime-syntax') + @with_package( + "_Syntax_Error", + syntax_compatibility=True, + fail_if_no_resources=False, + ) + def test_empty_syntax_compatibility_allowed(self, txt): + self.assertOk(txt) + def has_colorschemeunit(): return ( diff --git a/unittesting/scheduler.py b/unittesting/scheduler.py index 32054a97..0dc03efc 100644 --- a/unittesting/scheduler.py +++ b/unittesting/scheduler.py @@ -42,6 +42,11 @@ def save(self, data, indent=4): class Unit: + SYNTAX_TESTING_OPTION_KEYS = ( + "fail_if_no_resources", + "pattern", + "tests_dir", + ) UNIT_TESTING_OPTION_KEYS = ( "capture_console", "condition_timeout", @@ -64,6 +69,9 @@ def __init__(self, s): self.syntax_compatibility = s.get("syntax_compatibility", False) self.color_scheme_test = s.get("color_scheme_test", False) self.coverage = s.get("coverage", False) + self.syntax_testing_options = { + key: s[key] for key in self.SYNTAX_TESTING_OPTION_KEYS if key in s + } self.unit_testing_options = { key: s[key] for key in self.UNIT_TESTING_OPTION_KEYS if key in s } @@ -87,13 +95,7 @@ def run(self): def syntax_testing_args(self): args = {"package": self.package, "output": self.output} - args.update( - { - key: self.unit_testing_options[key] - for key in ("pattern", "tests_dir") - if key in self.unit_testing_options - } - ) + args.update(self.syntax_testing_options) return args def unit_testing_args(self): diff --git a/unittesting/syntax.py b/unittesting/syntax.py index 79fdc87a..d15a0f9d 100644 --- a/unittesting/syntax.py +++ b/unittesting/syntax.py @@ -26,7 +26,7 @@ def run(self, package=None, **kwargs): kwargs.get("tests_dir"), ) - if not tests: + if not tests and kwargs.get("fail_if_no_resources", True): raise RuntimeError("No syntax_test files are found in %s!" % package) for t in tests: assertions, test_output_lines = sublime_api.run_syntax_test(t) @@ -36,7 +36,7 @@ def run(self, package=None, **kwargs): for line in test_output_lines: stream.write(line + "\n") - file_noun = "files" if len(tests) > 1 else "file" + file_noun = "files" if len(tests) != 1 else "file" if failed_assertions > 0: stream.write( "FAILED: %d of %d assertions in %d %s failed\n" @@ -75,7 +75,7 @@ def run(self, package=None, **kwargs): kwargs.get("tests_dir"), ) - if not syntaxes: + if not syntaxes and kwargs.get("fail_if_no_resources", True): raise RuntimeError("No sublime-syntax files found in %s!" % package) total_errors = 0 @@ -93,7 +93,7 @@ def run(self, package=None, **kwargs): total_failed_syntaxes += 1 error_noun = "errors" if total_errors > 1 else "error" - syntax_noun = "syntaxes" if len(syntaxes) > 1 else "syntax" + syntax_noun = "syntaxes" if len(syntaxes) != 1 else "syntax" if total_errors: stream.write( "FAILED: %d %s in %d of %d %s\n" From 8a7bc657ac4c99741b11667c03adf805f89466f9 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 11:42:30 +0200 Subject: [PATCH 10/12] Include syntax tests execution in Docker Run unit tests, syntax tests and syntax compatibility checks by default. For selected files (`--file`), naivly infer the category. Allow categories to be disabled (the `--no-*` args) and coordinate category runs through the shared CI scripts. --- README.md | 12 +++- docker/README.md | 22 +++++++ docker/run_tests.py | 77 ++++++++++++++++++++++- docker/tests/test_run_tests.py | 110 +++++++++++++++++++++++++++++++++ sbin/ci.sh | 59 +++++++++++++++++- sbin/run_tests.py | 9 +++ 6 files changed, 284 insertions(+), 5 deletions(-) create mode 100644 docker/tests/test_run_tests.py diff --git a/README.md b/README.md index f2f0fbfb..dfa7a8d2 100644 --- a/README.md +++ b/README.md @@ -132,12 +132,20 @@ ut-run-tests . This launcher calls `docker/run_tests.py`, which runs tests in a Docker container (headless), streams output to stdout/stderr and keeps a cache -volume so repeated runs are fast. +volume so repeated runs are fast. By default it runs Python unit tests, +syntax tests and syntax compatibility checks. + +`--file` chooses the runner from the selected file: Python files run as unit +tests, `syntax_test*` files run as syntax tests, and `.sublime-syntax` files +run compatibility checks. Useful options: - `--file tests/test_foo.py` -- `--pattern test_foo.py --tests-dir tests/subdir` +- `--pattern test_foo.py --tests-dir tests/subdir` (unit tests only) +- `--no-unit-tests` +- `--no-syntax-tests` +- `--no-syntax-compatibility-checks` - `--coverage` - `--failfast` - `--reload-package-on-testing` (default: off) diff --git a/docker/README.md b/docker/README.md index 61e41b7b..ec17a4ae 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,11 +26,33 @@ By default it: - builds `unittesting-local` image from `./docker` if missing - mounts your repo as `/project` +- runs Python unit tests, syntax tests and syntax compatibility checks - runs UnitTesting through the same CI shell entrypoints - stores Sublime install/cache in docker volume `unittesting-home` - synchronizes only changed files into `Packages/` using `rsync` - excludes files ignored by Git, including repository-local and global rules +A category with no matching resources is reported and succeeds. Disable +categories that are not needed with: + +```sh +ut-run-tests . --no-unit-tests +ut-run-tests . --no-syntax-tests +ut-run-tests . --no-syntax-compatibility-checks +``` + +`--file` selects its category automatically. Python files run as unit tests, +files whose names start with `syntax_test` run as syntax tests, and +`.sublime-syntax` files run compatibility checks: + +```sh +ut-run-tests . --file tests/test_example.py +ut-run-tests . --file syntax_test_example +ut-run-tests . --file Example.sublime-syntax +``` + +`--pattern` and `--tests-dir` select unit tests only. + ## Fast reruns The container entrypoint writes a marker in `/root/.cache/unittesting`. diff --git a/docker/run_tests.py b/docker/run_tests.py index 03c72862..c61af050 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -23,6 +23,14 @@ DEFAULT_IMAGE = "unittesting-local" DEFAULT_CACHE_VOLUME = "unittesting-home" DEFAULT_LOCK_TIMEOUT = 3600 +UNIT_TESTS = "unit-tests" +SYNTAX_TESTS = "syntax-tests" +SYNTAX_COMPATIBILITY_CHECKS = "syntax-compatibility-checks" +ALL_TEST_CATEGORIES = ( + UNIT_TESTS, + SYNTAX_TESTS, + SYNTAX_COMPATIBILITY_CHECKS, +) DOCKER_CONTEXT_HASH_LABEL = "org.sublimetext.unittesting.context-hash" DOCKER_CONTEXT_INPUTS = ( "Dockerfile", @@ -77,6 +85,7 @@ def main(argv: list[str] | None = None) -> int: tests_dir, pattern, selected_file = resolve_test_target( package_root, args.file, args.tests_dir, args.pattern ) + test_categories = resolve_test_categories(args, selected_file) maybe_build_image(image, refresh=False) @@ -95,6 +104,7 @@ def main(argv: list[str] | None = None) -> int: cache_volume=args.cache_volume, container_name=runner_name, ignore_manifest=ignore_manifest, + test_categories=test_categories, scheduler_delay_ms=args.scheduler_delay_ms, coverage=args.coverage, failfast=args.failfast, @@ -117,6 +127,7 @@ def main(argv: list[str] | None = None) -> int: print("Cache lock: enabled") if args.refresh_cache: print("Cache refresh: enabled") + print(f"Test categories: {', '.join(test_categories)}") if tests_dir and pattern: print(f"Test target: {tests_dir}/{pattern}") if ignore_manifest: @@ -155,6 +166,21 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: test_group.add_argument("--pattern", help="Custom unittest discovery pattern.") test_group.add_argument("--tests-dir", help="Custom tests directory.") test_group.add_argument("--package-name", help="Override package name.") + test_group.add_argument( + "--no-unit-tests", + action="store_true", + help="Do not run Python unit tests.", + ) + test_group.add_argument( + "--no-syntax-tests", + action="store_true", + help="Do not run syntax tests.", + ) + test_group.add_argument( + "--no-syntax-compatibility-checks", + action="store_true", + help="Do not run syntax compatibility checks.", + ) test_group.add_argument("--coverage", action="store_true", help="Enable coverage.") test_group.add_argument("--failfast", action="store_true", help="Stop on first failure.") test_group.add_argument( @@ -246,6 +272,20 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: if args.file and args.tests_dir: parser.error("--file and --tests-dir are mutually exclusive") + category_options = ( + args.no_unit_tests, + args.no_syntax_tests, + args.no_syntax_compatibility_checks, + ) + if args.file and any(category_options): + parser.error("--file cannot be combined with --no-* test category options") + + if args.no_unit_tests and (args.pattern or args.tests_dir): + parser.error("--pattern and --tests-dir require unit tests") + + if all(category_options): + parser.error("all test categories are disabled") + if args.refresh_cache and not args.cache_volume: parser.error("--refresh-cache requires a cache volume (omit --no-cache-volume)") @@ -393,6 +433,38 @@ def resolve_test_target( return resolved_tests_dir, resolved_pattern, rel_file_path.as_posix() +def resolve_test_categories( + args: argparse.Namespace, selected_file: str | None +) -> tuple[str, ...]: + if selected_file: + return (test_category_for_file(selected_file),) + + if args.pattern or args.tests_dir: + return (UNIT_TESTS,) + + disabled_categories = { + UNIT_TESTS: args.no_unit_tests, + SYNTAX_TESTS: args.no_syntax_tests, + SYNTAX_COMPATIBILITY_CHECKS: args.no_syntax_compatibility_checks, + } + return tuple( + category + for category in ALL_TEST_CATEGORIES + if not disabled_categories[category] + ) + + +def test_category_for_file(test_file: str) -> str: + file_name = Path(test_file).name + if file_name.startswith("syntax_test"): + return SYNTAX_TESTS + if file_name.endswith(".sublime-syntax"): + return SYNTAX_COMPATIBILITY_CHECKS + if file_name.endswith(".py"): + return UNIT_TESTS + raise SystemExit(f"Error: unsupported test file type: {test_file}") + + class GitIgnoreManifest: def __init__(self, contents: bytes | None = None) -> None: self.path: Path | None = None @@ -479,6 +551,7 @@ def build_docker_run_command( cache_volume: str | None, container_name: str | None, ignore_manifest: GitIgnoreManifest, + test_categories: tuple[str, ...], scheduler_delay_ms: int, coverage: bool, failfast: bool, @@ -513,7 +586,9 @@ def build_docker_run_command( command.extend(["-v", f"{cache_volume}:/root"]) command.append(image) - command.append("run_tests") + command.append("run_test_categories") + command.extend(f"--{category}" for category in test_categories) + command.append("--") if coverage: command.append("--coverage") diff --git a/docker/tests/test_run_tests.py b/docker/tests/test_run_tests.py new file mode 100644 index 00000000..8c377ca9 --- /dev/null +++ b/docker/tests/test_run_tests.py @@ -0,0 +1,110 @@ +import importlib.util +import io +import unittest +from contextlib import redirect_stderr +from pathlib import Path + + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "run_tests.py" +SPEC = importlib.util.spec_from_file_location("docker_run_tests", RUNNER_PATH) +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +class ParseArgsTests(unittest.TestCase): + def test_rejects_repeated_file(self): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + runner.parse_args(["--file", "a.py", "--file", "b.py"]) + + self.assertEqual(error.exception.code, 2) + + def test_rejects_disabling_every_category(self): + with redirect_stderr(io.StringIO()), self.assertRaises(SystemExit) as error: + runner.parse_args( + [ + "--no-unit-tests", + "--no-syntax-tests", + "--no-syntax-compatibility-checks", + ] + ) + + self.assertEqual(error.exception.code, 2) + + +class TestCategoryTests(unittest.TestCase): + def test_runs_every_category_by_default(self): + args = runner.parse_args([]) + + self.assertEqual( + runner.resolve_test_categories(args, None), + runner.ALL_TEST_CATEGORIES, + ) + + def test_skips_disabled_categories(self): + args = runner.parse_args(["--no-syntax-tests"]) + + self.assertEqual( + runner.resolve_test_categories(args, None), + (runner.UNIT_TESTS, runner.SYNTAX_COMPATIBILITY_CHECKS), + ) + + def test_unit_discovery_options_select_unit_tests(self): + args = runner.parse_args(["--tests-dir", "specs", "--pattern", "spec*.py"]) + + self.assertEqual( + runner.resolve_test_categories(args, None), + (runner.UNIT_TESTS,), + ) + + def test_infers_category_from_file(self): + args = runner.parse_args([]) + cases = { + "tests/test_example.py": runner.UNIT_TESTS, + "syntax/syntax_test_example": runner.SYNTAX_TESTS, + "syntaxes/Example.sublime-syntax": runner.SYNTAX_COMPATIBILITY_CHECKS, + } + + for test_file, category in cases.items(): + with self.subTest(test_file=test_file): + self.assertEqual( + runner.resolve_test_categories(args, test_file), + (category,), + ) + + def test_rejects_unsupported_file_type(self): + args = runner.parse_args([]) + + with self.assertRaisesRegex(SystemExit, "unsupported test file type"): + runner.resolve_test_categories(args, "tests/example.txt") + + +class DockerCommandTests(unittest.TestCase): + def test_passes_selected_categories_to_container_runner(self): + command = runner.build_docker_run_command( + package_root=Path("/package"), + unit_testing_root=Path("/unittesting"), + package_name="Example", + image="image", + cache_volume=None, + container_name=None, + ignore_manifest=runner.GitIgnoreManifest(), + test_categories=(runner.UNIT_TESTS, runner.SYNTAX_TESTS), + scheduler_delay_ms=0, + coverage=False, + failfast=False, + reload_package_on_testing=False, + dry_run=False, + color="never", + tests_dir=None, + pattern=None, + ) + + image_index = command.index("image") + self.assertEqual( + command[image_index + 1 : image_index + 5], + ["run_test_categories", "--unit-tests", "--syntax-tests", "--"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/sbin/ci.sh b/sbin/ci.sh index 573e19dc..937006ae 100644 --- a/sbin/ci.sh +++ b/sbin/ci.sh @@ -180,20 +180,72 @@ InstallPackageControl() { sh "$STP/UnitTesting/sbin/install_package_control.sh" "--st" "$SUBLIME_TEXT_VERSION" } +RunTestCategories() { + local RunUnitTests=false + local RunSyntaxTests=false + local RunSyntaxCompatibilityChecks=false + + while [ "$#" -gt 0 ]; do + case "$1" in + "--unit-tests") + RunUnitTests=true + ;; + "--syntax-tests") + RunSyntaxTests=true + ;; + "--syntax-compatibility-checks") + RunSyntaxCompatibilityChecks=true + ;; + "--") + shift + break + ;; + *) + echo "Unknown test category: $1" >&2 + return 2 + ;; + esac + shift + done + + if [ "$RunUnitTests" = false ] && [ "$RunSyntaxTests" = false ] && \ + [ "$RunSyntaxCompatibilityChecks" = false ]; then + echo "No test categories selected" >&2 + return 2 + fi + + local Status=0 + if [ "$RunUnitTests" = true ]; then + echo "Run unit tests" + RunTests "$@" || Status=$? + fi + if [ "$RunSyntaxTests" = true ]; then + echo "Run syntax tests" + RunTests --syntax-test --no-fail-if-no-resources "$@" || Status=$? + fi + if [ "$RunSyntaxCompatibilityChecks" = true ]; then + echo "Run syntax compatibility checks" + RunTests --syntax-compatibility --no-fail-if-no-resources "$@" || Status=$? + fi + return "$Status" +} + RunTests() { # if [ -n "$(echo "$@" | grep -e '--coverage\b')" ] && [ "$SUBLIME_TEXT_VERSION" -eq 4 ]; then # echo "Coverage is not yet supported in Sublime Text 4" # exit 1 # fi + local Status=0 if [ -z "$1" ]; then - python "$STP/UnitTesting/sbin/run_tests.py" "$PACKAGE" + python "$STP/UnitTesting/sbin/run_tests.py" "$PACKAGE" || Status=$? else - python "$STP/UnitTesting/sbin/run_tests.py" "$@" "$PACKAGE" + python "$STP/UnitTesting/sbin/run_tests.py" "$@" "$PACKAGE" || Status=$? fi pkill "[Ss]ubl" || true pkill 'plugin_host' || true sleep 1 + return "$Status" } @@ -222,6 +274,9 @@ case $COMMAND in "run_tests") RunTests "$@" ;; + "run_test_categories") + RunTestCategories "$@" + ;; "run_syntax_tests") RunTests "--syntax-test" "$@" ;; diff --git a/sbin/run_tests.py b/sbin/run_tests.py index 742151e9..f0e1c580 100644 --- a/sbin/run_tests.py +++ b/sbin/run_tests.py @@ -361,6 +361,12 @@ def main(default_schedule_info, dry_run=False, color='auto'): parser.add_option('--coverage', action='store_true') parser.add_option('--pattern') parser.add_option('--tests-dir') + parser.add_option( + '--no-fail-if-no-resources', + action='store_false', + dest='fail_if_no_resources', + default=True, + ) parser.add_option('--failfast', action='store_true') parser.add_option('--reload-package-on-testing', action='store_true') parser.add_option('--dry-run', action='store_true') @@ -395,6 +401,9 @@ def main(default_schedule_info, dry_run=False, color='auto'): if options.tests_dir: default_schedule_info['tests_dir'] = options.tests_dir + if not options.fail_if_no_resources: + default_schedule_info['fail_if_no_resources'] = False + if options.failfast: default_schedule_info['failfast'] = True From 10df26d0ea0dd7f97feba511a99b5cd088608aa8 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 12:45:24 +0200 Subject: [PATCH 11/12] Apply selectors to enabled categories Let tests_dir and pattern constrain every enabled test category instead of implicitly selecting unit tests. --- README.md | 2 +- docker/README.md | 3 ++- docker/run_tests.py | 6 ------ docker/tests/test_run_tests.py | 21 ++++++++++++++++++--- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index dfa7a8d2..b11109cc 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ run compatibility checks. Useful options: - `--file tests/test_foo.py` -- `--pattern test_foo.py --tests-dir tests/subdir` (unit tests only) +- `--pattern test_foo.py --tests-dir tests/subdir` - `--no-unit-tests` - `--no-syntax-tests` - `--no-syntax-compatibility-checks` diff --git a/docker/README.md b/docker/README.md index ec17a4ae..150489f5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -51,7 +51,8 @@ ut-run-tests . --file syntax_test_example ut-run-tests . --file Example.sublime-syntax ``` -`--pattern` and `--tests-dir` select unit tests only. +`--pattern` and `--tests-dir` try every enabled category. Use the +`--no-*` options to avoid running unrelated categories when desired. ## Fast reruns diff --git a/docker/run_tests.py b/docker/run_tests.py index c61af050..fa09e953 100644 --- a/docker/run_tests.py +++ b/docker/run_tests.py @@ -280,9 +280,6 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: if args.file and any(category_options): parser.error("--file cannot be combined with --no-* test category options") - if args.no_unit_tests and (args.pattern or args.tests_dir): - parser.error("--pattern and --tests-dir require unit tests") - if all(category_options): parser.error("all test categories are disabled") @@ -439,9 +436,6 @@ def resolve_test_categories( if selected_file: return (test_category_for_file(selected_file),) - if args.pattern or args.tests_dir: - return (UNIT_TESTS,) - disabled_categories = { UNIT_TESTS: args.no_unit_tests, SYNTAX_TESTS: args.no_syntax_tests, diff --git a/docker/tests/test_run_tests.py b/docker/tests/test_run_tests.py index 8c377ca9..74663315 100644 --- a/docker/tests/test_run_tests.py +++ b/docker/tests/test_run_tests.py @@ -48,12 +48,27 @@ def test_skips_disabled_categories(self): (runner.UNIT_TESTS, runner.SYNTAX_COMPATIBILITY_CHECKS), ) - def test_unit_discovery_options_select_unit_tests(self): - args = runner.parse_args(["--tests-dir", "specs", "--pattern", "spec*.py"]) + def test_discovery_options_apply_to_every_category(self): + args = runner.parse_args(["--tests-dir", "specs", "--pattern", "spec*"]) self.assertEqual( runner.resolve_test_categories(args, None), - (runner.UNIT_TESTS,), + runner.ALL_TEST_CATEGORIES, + ) + + def test_discovery_options_apply_to_enabled_categories(self): + args = runner.parse_args( + [ + "--tests-dir", + "syntax/test", + "--no-unit-tests", + "--no-syntax-compatibility-checks", + ] + ) + + self.assertEqual( + runner.resolve_test_categories(args, None), + (runner.SYNTAX_TESTS,), ) def test_infers_category_from_file(self): From d76043c2654270bd85739351fc30d3f431693961 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 14 Aug 2026 13:50:46 +0200 Subject: [PATCH 12/12] Run categories in one Sublime instance Write selected categories to one schedule with distinct result files. Run synchronous checks before potentially deferred unit tests, aggregate their output and stop Sublime only after every category completes. This removes repeated startup and shutdown costs from the default Docker run while preserving standalone category behavior. --- sbin/ci.sh | 18 ++-- sbin/run_tests.py | 172 +++++++++++++++++++++------------ sbin/tests/test_sbin_runner.py | 146 ++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 69 deletions(-) create mode 100644 sbin/tests/test_sbin_runner.py diff --git a/sbin/ci.sh b/sbin/ci.sh index 937006ae..aadfefa7 100644 --- a/sbin/ci.sh +++ b/sbin/ci.sh @@ -214,20 +214,22 @@ RunTestCategories() { return 2 fi - local Status=0 + local CategoryOptions=() if [ "$RunUnitTests" = true ]; then - echo "Run unit tests" - RunTests "$@" || Status=$? + CategoryOptions+=("--unit-test") fi if [ "$RunSyntaxTests" = true ]; then - echo "Run syntax tests" - RunTests --syntax-test --no-fail-if-no-resources "$@" || Status=$? + CategoryOptions+=("--syntax-test") fi if [ "$RunSyntaxCompatibilityChecks" = true ]; then - echo "Run syntax compatibility checks" - RunTests --syntax-compatibility --no-fail-if-no-resources "$@" || Status=$? + CategoryOptions+=("--syntax-compatibility") fi - return "$Status" + if [ "$RunSyntaxTests" = true ] || \ + [ "$RunSyntaxCompatibilityChecks" = true ]; then + CategoryOptions+=("--no-fail-if-no-resources") + fi + + RunTests "${CategoryOptions[@]}" "$@" } RunTests() { diff --git a/sbin/run_tests.py b/sbin/run_tests.py index f0e1c580..d2584074 100644 --- a/sbin/run_tests.py +++ b/sbin/run_tests.py @@ -26,6 +26,7 @@ UT_SBIN_PATH = os.path.realpath(os.path.join(PACKAGES_DIR_PATH, 'UnitTesting', 'sbin')) SCHEDULE_RUNNER_SOURCE = os.path.join(UT_SBIN_PATH, "run_scheduler.py") SCHEDULE_RUNNER_TARGET = os.path.join(UT_DIR_PATH, "zzz_run_scheduler.py") +DONE_MESSAGE = "UnitTesting: Done.\n" RX_RESULT = re.compile(r'^(?POK|FAILED|ERROR)', re.MULTILINE) RX_DONE = re.compile(r'^UnitTesting: Done\.$', re.MULTILINE) RX_TEST_STATUS = re.compile(r'\.\.\. (ok|FAIL|ERROR|skipped)(\b.*)$') @@ -56,7 +57,7 @@ def copy_file_if_not_exists(source, target): shutil.copyfile(source, target) -def create_schedule(package, output_file, default_schedule): +def create_schedules(package, named_schedules): schedule = [] try: @@ -65,16 +66,8 @@ def create_schedule(package, output_file, default_schedule): except Exception: pass - print('Schedule:') - for k, v in default_schedule.items(): - print(' %s: %s' % (k, v)) - - for idx, item in enumerate(schedule): - if item.get('package') == package: - schedule[idx] = default_schedule - break - else: - schedule.append(default_schedule) + schedule = [item for item in schedule if item.get('package') != package] + schedule.extend(default_schedule for _, default_schedule in named_schedules) with open(SCHEDULE_FILE_PATH, 'w') as f: f.write(json.dumps(schedule, ensure_ascii=False, indent=True)) @@ -83,7 +76,6 @@ def create_schedule(package, output_file, default_schedule): def wait_for_output(path, schedule, timeout=10, poll_interval=0.2): start_time = time.time() last_dot = 0 - needs_newline = False def check_has_timed_out(): return time.time() - start_time > timeout @@ -99,7 +91,6 @@ def check_is_output_available(): if now - last_dot >= 1: print(".", end="") sys.stdout.flush() - needs_newline = True last_dot = now if check_has_timed_out(): @@ -109,8 +100,7 @@ def check_is_output_available(): time.sleep(poll_interval) else: - if needs_newline: - print() + print() def start_sublime_text(): @@ -122,7 +112,7 @@ def kill_sublime_text(): subprocess.Popen("pkill plugin_host || true", shell=True) -def read_output(path, color='auto'): +def read_output(path, color='auto', show_done=True): # todo: use notification instead of polling success = None use_color = should_use_color(color) @@ -143,11 +133,12 @@ def check_is_done(result): result = f.read() if result: + display_result = result if show_done else result.replace(DONE_MESSAGE, "") if use_color: - rendered, pending = colorize_output_chunk(result, pending) + rendered, pending = colorize_output_chunk(display_result, pending) print(rendered, end="") else: - print(result, end="") + print(display_result, end="") sys.stdout.flush() # Keep checking while we don't have a definite result. @@ -307,34 +298,34 @@ def detect_package_control_version(): return str(version) if version else None -def main(default_schedule_info, dry_run=False, color='auto'): - package_under_test = default_schedule_info['package'] +def main(named_schedules, dry_run=False, color='auto'): + package_under_test = named_schedules[0][1]['package'] output_dir = os.path.join(UT_OUTPUT_DIR_PATH, package_under_test) - output_file = os.path.join(output_dir, "result") coverage_file = os.path.join(output_dir, "coverage") - - default_schedule_info['output'] = output_file + output_files = configure_schedule_outputs(named_schedules, output_dir) print_runtime_metadata() + print_schedules(named_schedules) if dry_run: create_dir_if_not_exists(output_dir) - delete_file_if_exists(output_file) + delete_files(output_files) delete_file_if_exists(coverage_file) - create_schedule(package_under_test, output_file, default_schedule_info) + create_schedules(package_under_test, named_schedules) return for i in range(3): create_dir_if_not_exists(output_dir) - delete_file_if_exists(output_file) + delete_files(output_files) delete_file_if_exists(coverage_file) - create_schedule(package_under_test, output_file, default_schedule_info) + create_schedules(package_under_test, named_schedules) delete_file_if_exists(SCHEDULE_RUNNER_TARGET) copy_file_if_not_exists(SCHEDULE_RUNNER_SOURCE, SCHEDULE_RUNNER_TARGET) start_sublime_text() try: - print("Wait for tests output...", end="") - wait_for_output(output_file, SCHEDULE_RUNNER_TARGET) + for name, output_file in output_files: + print("Wait for %s output..." % name, end="") + wait_for_output(output_file, SCHEDULE_RUNNER_TARGET) break except ValueError: if i == 2: @@ -343,18 +334,105 @@ def main(default_schedule_info, dry_run=False, color='auto'): "is being written to the wrong file.") delete_file_if_exists(SCHEDULE_RUNNER_TARGET) sys.exit(1) + print("Retrying after Sublime Text did not produce test output.") kill_sublime_text() time.sleep(2) - print("Start to read output...") - if not read_output(output_file, color=color): + success = True + show_category_done = len(output_files) == 1 + for name, output_file in output_files: + print("=== %s OUTPUT ===" % name.upper()) + if not read_output(output_file, color=color, show_done=show_category_done): + success = False + + if not show_category_done: + print(DONE_MESSAGE, end="") + + if not success: sys.exit(1) restore_coverage_file(coverage_file, package_under_test) delete_file_if_exists(SCHEDULE_RUNNER_TARGET) +def print_schedules(named_schedules): + for name, schedule in named_schedules: + heading = 'Schedule:' if len(named_schedules) == 1 else 'Schedule (%s):' % name + print(heading) + for key, value in schedule.items(): + print(' %s: %s' % (key, value)) + + +def configure_schedule_outputs(named_schedules, output_dir): + output_files = [] + for name, schedule in named_schedules: + output_name = ( + "result" + if len(named_schedules) == 1 + else "result-" + name.replace(" ", "-") + ) + output_file = os.path.join(output_dir, output_name) + schedule['output'] = output_file + output_files.append((name, output_file)) + return output_files + + +def delete_files(named_files): + for _, path in named_files: + delete_file_if_exists(path) + + +def build_named_schedules(options, package): + schedule_options = { + 'package': package, + 'coverage': options.coverage, + 'reload_package_on_testing': bool(options.reload_package_on_testing), + } + + if options.pattern: + schedule_options['pattern'] = options.pattern + if options.tests_dir: + schedule_options['tests_dir'] = options.tests_dir + if not options.fail_if_no_resources: + schedule_options['fail_if_no_resources'] = False + if options.failfast: + schedule_options['failfast'] = True + + named_schedules = [] + explicit_category = any( + ( + options.unit_test, + options.syntax_test, + options.syntax_compatibility, + options.color_scheme_test, + ) + ) + if options.syntax_test: + named_schedules.append( + ('syntax tests', dict(schedule_options, syntax_test=True)) + ) + if options.syntax_compatibility: + named_schedules.append( + ( + 'syntax compatibility checks', + dict(schedule_options, syntax_compatibility=True), + ) + ) + if options.color_scheme_test: + named_schedules.append( + ('color scheme tests', dict(schedule_options, color_scheme_test=True)) + ) + + # Unit tests may continue through deferred callbacks after their command + # returns, so keep them last to avoid overlapping another category. + if options.unit_test or not explicit_category: + named_schedules.append(('unit tests', schedule_options)) + + return named_schedules + + if __name__ == '__main__': parser = optparse.OptionParser() + parser.add_option('--unit-test', action='store_true') parser.add_option('--syntax-test', action='store_true') parser.add_option('--syntax-compatibility', action='store_true') parser.add_option('--color-scheme-test', action='store_true') @@ -380,34 +458,6 @@ def main(default_schedule_info, dry_run=False, color='auto'): options, remainder = parser.parse_args() - syntax_test = options.syntax_test - syntax_compatibility = options.syntax_compatibility - color_scheme_test = options.color_scheme_test - coverage = options.coverage package_under_test = remainder[0] if len(remainder) > 0 else "UnitTesting" - - default_schedule_info = { - 'package': package_under_test, - 'syntax_test': syntax_test, - 'syntax_compatibility': syntax_compatibility, - 'color_scheme_test': color_scheme_test, - 'coverage': coverage, - 'reload_package_on_testing': False, - } - - if options.pattern: - default_schedule_info['pattern'] = options.pattern - - if options.tests_dir: - default_schedule_info['tests_dir'] = options.tests_dir - - if not options.fail_if_no_resources: - default_schedule_info['fail_if_no_resources'] = False - - if options.failfast: - default_schedule_info['failfast'] = True - - if options.reload_package_on_testing: - default_schedule_info['reload_package_on_testing'] = True - - main(default_schedule_info, dry_run=options.dry_run, color=options.color) + named_schedules = build_named_schedules(options, package_under_test) + main(named_schedules, dry_run=options.dry_run, color=options.color) diff --git a/sbin/tests/test_sbin_runner.py b/sbin/tests/test_sbin_runner.py new file mode 100644 index 00000000..226547bd --- /dev/null +++ b/sbin/tests/test_sbin_runner.py @@ -0,0 +1,146 @@ +import importlib.util +import io +import json +import os +import tempfile +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "run_tests.py" +SPEC = importlib.util.spec_from_file_location("sbin_run_tests", RUNNER_PATH) +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +def options(**overrides): + values = { + "color_scheme_test": False, + "coverage": False, + "fail_if_no_resources": True, + "failfast": False, + "pattern": None, + "reload_package_on_testing": False, + "syntax_compatibility": False, + "syntax_test": False, + "tests_dir": None, + "unit_test": False, + } + values.update(overrides) + return SimpleNamespace(**values) + + +class BuildSchedulesTests(unittest.TestCase): + def test_defaults_to_unit_tests(self): + self.assertEqual( + runner.build_named_schedules(options(), "Example"), + [ + ( + "unit tests", + { + "package": "Example", + "coverage": False, + "reload_package_on_testing": False, + }, + ) + ], + ) + + def test_builds_synchronous_categories_before_unit_tests(self): + schedules = runner.build_named_schedules( + options( + unit_test=True, + syntax_test=True, + syntax_compatibility=True, + fail_if_no_resources=False, + pattern="selected*", + tests_dir="syntax/test", + ), + "Example", + ) + + self.assertEqual( + [name for name, _ in schedules], + ["syntax tests", "syntax compatibility checks", "unit tests"], + ) + for _, schedule in schedules: + self.assertEqual(schedule["pattern"], "selected*") + self.assertEqual(schedule["tests_dir"], "syntax/test") + self.assertFalse(schedule["fail_if_no_resources"]) + + def test_assigns_distinct_outputs_to_multiple_schedules(self): + schedules = runner.build_named_schedules( + options(unit_test=True, syntax_test=True), "Example" + ) + + output_files = runner.configure_schedule_outputs(schedules, "/output") + + self.assertEqual( + output_files, + [ + ("syntax tests", os.path.join("/output", "result-syntax-tests")), + ("unit tests", os.path.join("/output", "result-unit-tests")), + ], + ) + + +class OutputTests(unittest.TestCase): + def test_wait_heading_ends_with_newline_when_output_already_exists(self): + with tempfile.NamedTemporaryFile() as output: + output.write(b"ready") + output.flush() + rendered = io.StringIO() + with redirect_stdout(rendered): + print("Wait for output...", end="") + runner.wait_for_output(output.name, "unused") + + self.assertEqual(rendered.getvalue(), "Wait for output...\n") + + def test_can_hide_category_done_message(self): + with tempfile.NamedTemporaryFile(mode="w", delete=False) as output: + output.write("OK\n\n" + runner.DONE_MESSAGE) + output_path = output.name + + try: + rendered = io.StringIO() + with redirect_stdout(rendered): + success = runner.read_output( + output_path, color="never", show_done=False + ) + finally: + Path(output_path).unlink() + + self.assertTrue(success) + self.assertEqual(rendered.getvalue(), "OK\n\n") + + +class CreateSchedulesTests(unittest.TestCase): + def test_replaces_package_with_every_selected_schedule(self): + schedules = runner.build_named_schedules( + options(unit_test=True, syntax_test=True), "Example" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + schedule_file = Path(temp_dir) / "schedule.json" + schedule_file.write_text( + json.dumps( + [ + {"package": "Other"}, + {"package": "Example", "stale": True}, + ] + ) + ) + with mock.patch.object(runner, "SCHEDULE_FILE_PATH", str(schedule_file)): + with redirect_stdout(io.StringIO()): + runner.create_schedules("Example", schedules) + saved = json.loads(schedule_file.read_text()) + + self.assertEqual(saved[0], {"package": "Other"}) + self.assertEqual(saved[1:], [schedule for _, schedule in schedules]) + + +if __name__ == "__main__": + unittest.main()