From eee9886935855bd4c290954676fead3122c16358 Mon Sep 17 00:00:00 2001 From: Seth Bromberger Date: Fri, 19 Jun 2026 16:49:41 +0000 Subject: [PATCH 1/6] tries to reduce stderr by deduplicating lines --- py/src/clippy/backends/fs/execution.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/py/src/clippy/backends/fs/execution.py b/py/src/clippy/backends/fs/execution.py index 023935f..9e0ff5d 100644 --- a/py/src/clippy/backends/fs/execution.py +++ b/py/src/clippy/backends/fs/execution.py @@ -11,6 +11,7 @@ import select import subprocess import sys +import collections from ... import cfg from ...clippy_types import AnyDict @@ -130,7 +131,16 @@ def _stream_exec( stderr_lines.append(stderr_buffer) print(stderr_buffer.rstrip(), flush=True) - stderr = "".join(stderr_lines) if stderr_lines else None + # this relies on defaultdict preserving insertion order. + stderr_map: dict[str, int] = collections.defaultdict(int) + for line in stderr_lines: + stderr_map[line] += 1 + + if not stderr_lines: + stderr = None # type: ignore + else: + stderr: str = "".join(stderr_map.keys()) + if progress is not None: progress.close() # if proc.returncode: From d129d28ce310935d2d429b144e9e6c9fcd6b8205 Mon Sep 17 00:00:00 2001 From: Seth Bromberger Date: Fri, 19 Jun 2026 19:33:52 +0000 Subject: [PATCH 2/6] CLIPPY_FULL_STDERR env variable dictates whether stderr is deduplicated (default yes) --- py/src/clippy/backends/fs/execution.py | 27 ++++++++++++++++---------- py/src/clippy/config.py | 1 + test/run_ipython.sh | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/py/src/clippy/backends/fs/execution.py b/py/src/clippy/backends/fs/execution.py index 9e0ff5d..a71f291 100644 --- a/py/src/clippy/backends/fs/execution.py +++ b/py/src/clippy/backends/fs/execution.py @@ -45,6 +45,8 @@ def _stream_exec( already be set. """ + full_stderr: bool = cfg.get("full_stderr") == "YES" + logger.debug(f"Submission = {submission_dict}") # PP support passing objects # ~ cmd_stdin = json.dumps(submission_dict) @@ -112,7 +114,7 @@ def _stream_exec( while "\n" in stderr_buffer: line, stderr_buffer = stderr_buffer.split("\n", 1) stderr_lines.append(line + "\n") - print(line, flush=True) + # print(line, flush=True) except BlockingIOError: # No data available right now continue @@ -129,18 +131,23 @@ def _stream_exec( if stderr_buffer.strip(): stderr_lines.append(stderr_buffer) - print(stderr_buffer.rstrip(), flush=True) - - # this relies on defaultdict preserving insertion order. - stderr_map: dict[str, int] = collections.defaultdict(int) - for line in stderr_lines: - stderr_map[line] += 1 + # print(stderr_buffer.rstrip(), flush=True) if not stderr_lines: stderr = None # type: ignore - else: - stderr: str = "".join(stderr_map.keys()) - + else: # we have stderr - process it + if not full_stderr: # deduplicate + logger.debug("stderr deduplication enabled") + # this relies on defaultdict preserving insertion order. + stderr_map: dict[str, int] = collections.defaultdict(int) + for line in stderr_lines: + stderr_map[line] += 1 + stderr: str = "".join(stderr_map.keys()) + else: # use all the lines + stderr: str = "".join(stderr_lines) + + if stderr is not None: + print(stderr) if progress is not None: progress.close() # if proc.returncode: diff --git a/py/src/clippy/config.py b/py/src/clippy/config.py index 85d5834..24c092f 100644 --- a/py/src/clippy/config.py +++ b/py/src/clippy/config.py @@ -26,4 +26,5 @@ "%(asctime)s [%(filename)s:%(lineno)d (%(funcName)s) %(levelname)s: %(message)s", ), "logname": ("CLIPPY_LOGNAME", __name__), + "full_stderr": ("CLIPPY_FULL_STDERR", "YES"), } diff --git a/test/run_ipython.sh b/test/run_ipython.sh index 849661c..ee46a2c 100755 --- a/test/run_ipython.sh +++ b/test/run_ipython.sh @@ -3,4 +3,4 @@ PROJ_ROOT_DIR=$(pwd)/.. CPP_BUILD_DIR=$PROJ_ROOT_DIR/cpp/build export PYTHONPATH=$PROJ_ROOT_DIR/py/src:$PYTHONPATH export CLIPPY_BACKEND_PATH=$CPP_BUILD_DIR/examples -CLIPPY_LOGLEVEL=DEBUG ipython +CLIPPY_LOGLEVEL=CRITICAL ipython From 96eb6175e20fffc7bb403e19d251a4736ed075a4 Mon Sep 17 00:00:00 2001 From: Seth Bromberger Date: Fri, 19 Jun 2026 19:36:37 +0000 Subject: [PATCH 3/6] CLIPPY_FULL_STDERR env variable dictates whether stderr is deduplicated (default yes) --- py/src/clippy/backends/fs/execution.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/src/clippy/backends/fs/execution.py b/py/src/clippy/backends/fs/execution.py index a71f291..394b189 100644 --- a/py/src/clippy/backends/fs/execution.py +++ b/py/src/clippy/backends/fs/execution.py @@ -106,9 +106,10 @@ def _stream_exec( try: d = json.loads(line, object_hook=decode_clippy_json) except json.JSONDecodeError: - warning = f"Warning: invalid JSON on stdout: {line!r}" - stderr_lines.append(warning + "\n") - print(warning, file=sys.stderr, flush=True) + # warning = f"Warning: invalid JSON on stdout: {line!r}" + # logger.debug(warning + "\n") + # if it's not valid JSON, let's print it. + print(line, flush=True) elif fd == stderr_fd: stderr_buffer += text while "\n" in stderr_buffer: From 6044eb209dbdf77da772fc42c3e1230955383eec Mon Sep 17 00:00:00 2001 From: Seth Bromberger Date: Fri, 19 Jun 2026 19:43:37 +0000 Subject: [PATCH 4/6] ruff --- py/src/clippy/backends/fs/execution.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/py/src/clippy/backends/fs/execution.py b/py/src/clippy/backends/fs/execution.py index 394b189..4daf46c 100644 --- a/py/src/clippy/backends/fs/execution.py +++ b/py/src/clippy/backends/fs/execution.py @@ -4,14 +4,13 @@ from __future__ import annotations +import collections import contextlib import json import logging import os import select import subprocess -import sys -import collections from ... import cfg from ...clippy_types import AnyDict From 66c51e33edeacf487dde9359e08a527ba243e563 Mon Sep 17 00:00:00 2001 From: Seth Bromberger Date: Fri, 19 Jun 2026 19:45:16 +0000 Subject: [PATCH 5/6] mypy --- py/src/clippy/backends/fs/execution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/src/clippy/backends/fs/execution.py b/py/src/clippy/backends/fs/execution.py index 4daf46c..b5f0dac 100644 --- a/py/src/clippy/backends/fs/execution.py +++ b/py/src/clippy/backends/fs/execution.py @@ -142,9 +142,9 @@ def _stream_exec( stderr_map: dict[str, int] = collections.defaultdict(int) for line in stderr_lines: stderr_map[line] += 1 - stderr: str = "".join(stderr_map.keys()) + stderr = "".join(stderr_map.keys()) else: # use all the lines - stderr: str = "".join(stderr_lines) + stderr = "".join(stderr_lines) if stderr is not None: print(stderr) From 66cc7aeb9bd3e4f60e8fb1148ca7ffa453afb7be Mon Sep 17 00:00:00 2001 From: Seth Bromberger Date: Fri, 19 Jun 2026 19:56:17 +0000 Subject: [PATCH 6/6] removed test_noise since we're not sending invalid json warnings to stderr --- test/test_clippy.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_clippy.py b/test/test_clippy.py index a274032..27d8592 100644 --- a/test/test_clippy.py +++ b/test/test_clippy.py @@ -211,7 +211,3 @@ def test_graph(examplegraph): assert "c" in c_e_only and "e" in c_e_only and len(c_e_only) == 2 -def test_noise(examplefn, capsys): - assert examplefn.returns_noisy_int() == 42 - captured = capsys.readouterr() - assert "Warning: invalid JSON on stdout" in captured.err