From c38ee20f8c20c5bab8eafd6ee6cd1a86cf4266ac Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 18 Sep 2026 22:41:01 +0300 Subject: [PATCH] Revision&Refactoring --- src/exceptions.py | 2 +- src/local_ops.py | 47 ++++++++++----------- src/os_ops.py | 8 ++-- src/raise_error.py | 26 +++++++++--- src/remote_ops.py | 58 +++++++++++++------------- tests/test_os_ops_common.py | 81 ++++++++++++++++++++++++++----------- tests/test_os_ops_local.py | 15 +++++-- 7 files changed, 150 insertions(+), 87 deletions(-) diff --git a/src/exceptions.py b/src/exceptions.py index 1f305b5..d6743d2 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -147,7 +147,7 @@ def __init__( timeout: T_OS_TIMEOUT, output: typing.Optional[T_OUT_DATA] = None, error: typing.Optional[T_ERR_DATA] = None, - source: typing.Optional[str] = None + source: typing.Optional[str] = None, ): assert type(cmd) in [str, list] assert type(timeout) in [int, float] diff --git a/src/local_ops.py b/src/local_ops.py index 66e4b1c..aaa5ac8 100644 --- a/src/local_ops.py +++ b/src/local_ops.py @@ -1,5 +1,24 @@ from __future__ import annotations +from .exceptions import ExecUtilException +from .exceptions import ExecTimeoutException +from .exceptions import InvalidOperationException +from .os_ops import ConnectionParams, OsOperations, get_default_encoding +from .os_ops import OsProcessController +from .os_ops import OsCommandResult +from .os_ops import T_OS_CMD +from .os_ops import T_OS_SIGNAL +from .os_ops import T_OS_TIMEOUT +from .os_ops import T_OS_IO +from .os_ops import T_OS_IO_ID +from .os_ops import T_OS_EXEC_INPUT +from .os_ops import T_OS_EXEC_ENV +from .raise_error import RaiseError +from .helpers import Helpers + +from shutil import which as find_executable +from shutil import rmtree + import getpass import logging import os @@ -21,25 +40,6 @@ import pathlib import io -from .exceptions import ExecUtilException -from .exceptions import ExecTimeoutException -from .exceptions import InvalidOperationException -from .os_ops import ConnectionParams, OsOperations, get_default_encoding -from .os_ops import OsProcessController -from .os_ops import OsCommandResult -from .os_ops import T_OS_CMD -from .os_ops import T_OS_SIGNAL -from .os_ops import T_OS_TIMEOUT -from .os_ops import T_OS_IO -from .os_ops import T_OS_IO_ID -from .os_ops import T_OS_EXEC_INPUT -from .os_ops import T_OS_EXEC_ENV -from .raise_error import RaiseError -from .helpers import Helpers - -from shutil import which as find_executable -from shutil import rmtree - CMD_TIMEOUT_SEC = 60 @@ -385,7 +385,7 @@ def _run_command__generic( cwd=cwd, text=get_process and (encoding is not None), encoding=encoding if get_process else None, - **extParams + **extParams, ) assert process is not None assert isinstance(process, subprocess.Popen) @@ -498,7 +498,8 @@ def exec_command( exit_code=run_r[0], msg_arg=run_r[2] or run_r[1], error=run_r[2], - out=run_r[1]) + out=run_r[1], + ) if verbose: return run_r @@ -825,8 +826,8 @@ def write( truncate: bool = False, binary: bool = False, read_and_write: bool = False, - encoding: typing.Optional[str] = None - ): + encoding: typing.Optional[str] = None, + ) -> None: """ Write data to a file locally Args: diff --git a/src/os_ops.py b/src/os_ops.py index d5b4baf..ee99764 100644 --- a/src/os_ops.py +++ b/src/os_ops.py @@ -195,7 +195,7 @@ def exec_command( timeout=None, ignore_errors=False, exec_env: typing.Optional[dict] = None, - cwd: typing.Optional[str] = None + cwd: typing.Optional[str] = None, ) -> T_EXEC_COMMAND_RESULT: assert type(cmd) is str or type(cmd) is list assert type(verbose) is bool @@ -218,7 +218,7 @@ def popen( stdout: typing.Optional[T_OS_IO_ID] = None, stderr: typing.Optional[T_OS_IO_ID] = None, exec_env: typing.Optional[T_OS_EXEC_ENV] = None, - cwd: typing.Optional[str] = None + cwd: typing.Optional[str] = None, ) -> OsProcessController: assert type(cmd) is str or type(cmd) is list assert text is None or type(text) is bool @@ -394,7 +394,7 @@ def write( binary: bool = False, read_and_write: bool = False, encoding: typing.Optional[str] = None - ): + ) -> None: assert type(filename) is str assert encoding is None or type(encoding) is str assert data is not None @@ -491,7 +491,7 @@ def get_process_children(self, pid: int) -> typing.List: assert type(pid) is int RaiseError.MethodIsNotImplemented(__class__, "get_process_children") - def is_port_free(self, number: int): + def is_port_free(self, number: int) -> bool: assert type(number) is int RaiseError.MethodIsNotImplemented(__class__, "is_port_free") diff --git a/src/raise_error.py b/src/raise_error.py index 337662f..17a4dc0 100644 --- a/src/raise_error.py +++ b/src/raise_error.py @@ -34,7 +34,13 @@ def PropertyIsNotImplemented( raise NotImplementedError(err_msg) @staticmethod - def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out) -> typing.NoReturn: + def UtilityExitedWithNonZeroCode( + cmd, + exit_code, + msg_arg, + error, + out, + ) -> typing.NoReturn: assert type(exit_code) is int msg_arg_s = __class__._TranslateDataIntoString(msg_arg) @@ -50,10 +56,17 @@ def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out) -> typing. command=cmd, exit_code=exit_code, out=out, - error=error) + error=error, + ) @staticmethod - def CommandExecutionError(cmd, exit_code, message, error, out) -> typing.NoReturn: + def CommandExecutionError( + cmd, + exit_code, + message, + error, + out, + ) -> typing.NoReturn: assert type(exit_code) is int assert type(message) is str assert message != "" @@ -63,7 +76,8 @@ def CommandExecutionError(cmd, exit_code, message, error, out) -> typing.NoRetur command=cmd, exit_code=exit_code, out=out, - error=error) + error=error, + ) @staticmethod def _TranslateDataIntoString(data): @@ -71,7 +85,9 @@ def _TranslateDataIntoString(data): return "" if type(data) is bytes: - return __class__._TranslateDataIntoString__FromBinary(data) + return __class__._TranslateDataIntoString__FromBinary( + data, + ) return str(data) diff --git a/src/remote_ops.py b/src/remote_ops.py index 5c83bb9..84f2d0d 100644 --- a/src/remote_ops.py +++ b/src/remote_ops.py @@ -1,21 +1,5 @@ from __future__ import annotations -import getpass -import os -import posixpath -import subprocess -import io -import logging -import typing -import copy -import re -import signal as os_signal -import time -import datetime -import shlex -import threading -import warnings - from .exceptions import ExecUtilException from .exceptions import ExecTimeoutException from .exceptions import InvalidOperationException @@ -33,6 +17,22 @@ from .helpers import Helpers from .static_config import OsOperationStaticConfig +import getpass +import os +import posixpath +import subprocess +import io +import logging +import typing +import copy +import re +import signal as os_signal +import time +import datetime +import shlex +import threading +import warnings + class PsUtilProcessProxy: def __init__(self, ssh, pid): @@ -40,12 +40,14 @@ def __init__(self, ssh, pid): assert type(pid) is int self.ssh = ssh self.pid = pid + return def kill(self): assert isinstance(self.ssh, RemoteOperations) assert type(self.pid) is int command = ["kill", str(self.pid)] self.ssh._transport_run(command, encoding=get_default_encoding()) + return def cmdline(self): assert isinstance(self.ssh, RemoteOperations) @@ -425,7 +427,7 @@ def exec_command( timeout=None, ignore_errors=False, exec_env: typing.Optional[T_ENVS] = None, - cwd: typing.Optional[str] = None + cwd: typing.Optional[str] = None, ) -> OsOperations.T_EXEC_COMMAND_RESULT: """ Execute a command in the SSH session. @@ -533,7 +535,8 @@ def exec_command( exit_code=run_r[0], msg_arg=error, error=run_r[2], - out=run_r[1]) + out=run_r[1], + ) if verbose: return run_r @@ -550,7 +553,7 @@ def popen( stdout: typing.Optional[T_OS_IO_ID] = None, stderr: typing.Optional[T_OS_IO_ID] = None, exec_env: typing.Optional[T_OS_EXEC_ENV] = None, - cwd: typing.Optional[str] = None + cwd: typing.Optional[str] = None, ) -> OsProcessController: assert type(cmd) is str or type(cmd) is list assert text is None or type(text) is bool @@ -639,7 +642,7 @@ def popen( ] ping_pong_script2_s = self._join_command_arguments( - ping_pong_script2 + ping_pong_script2, ) # Run script within isolated env to get a true return codes of kill/terminate @@ -1021,7 +1024,7 @@ def makedirs( "&&", "mkdir", "-p", - path_q + path_q, ] else: cmd_p = [ @@ -1077,7 +1080,7 @@ def rmdirs( "elif", "[", "-e", path_q, "]", ";", "then", "{", "echo", "cannot remove " + path_q + ": it is not a directory", ">&2", ";", "exit", "20", ";", "}", ";", "else", "{", "echo", "directory " + path_q + " does not exist", ">&2", ";", "exit", "2", ";", "}", ";", - "fi" + "fi", ] cmd1 = " ".join(cmd1_p) @@ -1160,7 +1163,8 @@ def path_exists(self, path: str) -> bool: errMsg = "Test operation returns an unknown result code: {0}. Path is [{1}].".format( exec_r.returncode, - path) + path, + ) RaiseError.CommandExecutionError( cmd=command, @@ -1290,8 +1294,8 @@ def write( truncate: bool = False, binary: bool = False, read_and_write: bool = False, - encoding: typing.Optional[str] = None - ): + encoding: typing.Optional[str] = None, + ) -> None: assert type(filename) is str assert encoding is None or type(encoding) is str assert data is not None @@ -1580,7 +1584,7 @@ def get_process_children(self, pid: int) -> typing.List: if exec_r.returncode == 100: err_msg = "Failed to get process children. Reason: No such process with PID {}.".format( - pid + pid, ) raise ExecUtilException( @@ -1787,7 +1791,7 @@ def _transport_popen( stdout: typing.Optional[T_OS_IO_ID] = None, stderr: typing.Optional[T_OS_IO_ID] = None, exec_env: typing.Optional[T_OS_EXEC_ENV] = None, - cwd: typing.Optional[str] = None + cwd: typing.Optional[str] = None, ) -> subprocess.Popen: assert type(cmd) in [str, list] assert text is None or type(text) is bool diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 1522ed5..abe0819 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -1351,8 +1351,9 @@ def test_read_binary__spec__negative_size( filename = os_ops.mkstemp(name_with_surprize.value) with pytest.raises( - ValueError, - match=re.escape("Negative 'size' is not supported.")): + ValueError, + match=re.escape("Negative 'size' is not supported."), + ): os_ops.read_binary(filename, 0, size=-1) os_ops.remove_file(filename) @@ -1998,17 +1999,23 @@ def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): LocalCheck.check_path_exists(os_ops, lock_dir) assert os_ops.path_exists(lock_dir) is True - def MAKE_PATH(os_ops: OsOperations, lock_dir: str, num: int) -> str: + def MAKE_PATH( + os_ops: OsOperations, + lock_dir: str, + num: int, + ) -> str: assert isinstance(os_ops, OsOperations) assert type(lock_dir) is str assert type(num) is int return os_ops.build_path(lock_dir, str(num) + ".lock") - def LOCAL_WORKER(os_ops: OsOperations, - workerID: int, - lock_dir: str, - cNumbers: int, - reservedNumbers: typing.Set[int]) -> None: + def LOCAL_WORKER( + os_ops: OsOperations, + workerID: int, + lock_dir: str, + cNumbers: int, + reservedNumbers: typing.Set[int], + ) -> None: assert isinstance(os_ops, OsOperations) assert type(workerID) is int assert type(lock_dir) is str @@ -2064,7 +2071,7 @@ def LOG_INFO(template: str, *args) -> None: threadPool = ThreadPoolExecutor( max_workers=N_WORKERS, - thread_name_prefix="ex_creator" + thread_name_prefix="ex_creator", ) class tadWorkerData: @@ -2089,7 +2096,7 @@ class tadWorkerData: n, lock_dir, N_NUMBERS, - workerDatas[n].reservedNumbers + workerDatas[n].reservedNumbers, ) assert workerDatas[n].future is not None @@ -2152,7 +2159,7 @@ class tadWorkerData: nErrors += 1 logging.error("Number {} was already reserved by worker #{}".format( n, - reservedNumbers[n] + reservedNumbers[n], )) else: reservedNumbers[n] = i @@ -2198,7 +2205,7 @@ class tadWorkerData: logging.error("Cannot delete directory [{}]. Error ({}): {}".format( file_path, type(e).__name__, - str(e) + str(e), )) continue @@ -2215,7 +2222,7 @@ class tadWorkerData: logging.error("Cannot delete directory [{}]. Error ({}): {}".format( lock_dir, type(e).__name__, - str(e) + str(e), )) logging.info("Test is finished! Total error count is {}.".format(nErrors)) @@ -2278,7 +2285,7 @@ def test_kill( "python3", "-u", "-c", - "import os, time; print(os.getpid());time.sleep(300);print('EXIT')" + "import os, time; print(os.getpid());time.sleep(300);print('EXIT')", ] logging.info("Local test process is creating ...") @@ -2532,7 +2539,10 @@ def test_is_abs_path__yes( assert actual_value is True return - def test_is_abs_path__no(self, os_ops_descr: OsOpsDescr): + def test_is_abs_path__no( + self, + os_ops_descr: OsOpsDescr, + ): assert type(os_ops_descr) is OsOpsDescr assert isinstance(os_ops_descr.os_ops, OsOperations) @@ -3085,7 +3095,10 @@ class tagReadLinesData_TXT: for x in sm_ReadLinesData_TXT ] ) - def readlines_data_txt(self, request: pytest.FixtureRequest) -> tagReadLinesData_TXT: + def readlines_data_txt( + self, + request: pytest.FixtureRequest, + ) -> tagReadLinesData_TXT: assert isinstance(request, pytest.FixtureRequest) return request.param @@ -3143,7 +3156,7 @@ def test_readlines__BIN( def test_prove_environment_isolation( self, - os_ops_descr: OsOpsDescr + os_ops_descr: OsOpsDescr, ): # # Author: Marg G. (mark@google.com) @@ -4540,7 +4553,7 @@ def test_popen_returncode_active( controller = os_ops.popen( fx_data_wait_timeout.cmd, - shell=type(fx_data_wait_timeout.cmd) is str + shell=type(fx_data_wait_timeout.cmd) is str, ) assert isinstance(controller, OsProcessController) @@ -4855,7 +4868,7 @@ def test_popen_garbage_collection( local_p = controller._local_process else: raise RuntimeError("Unknown controller type: {}.".format( - type(controller).__name__ + type(controller).__name__, )) assert local_p is not None @@ -5150,6 +5163,8 @@ def test_popen_cwd( s = controller.stderr.read() assert s == "" + return + def test_popen_communicate_timeout( self, os_ops_descr: OsOpsDescr, @@ -6313,11 +6328,20 @@ def test_run_check_exception2__list(self, os_ops_descr: OsOpsDescr): RunConditions.skip_if_windows() os_ops = os_ops_descr.os_ops - cmd = ["sh", "-c", "echo normal_out && echo error_err >&2 && exit 1"] + cmd = [ + "sh", + "-c", + "echo normal_out && echo error_err >&2 && exit 1", + ] # 1. Check default behavior (check=True) with pytest.raises(expected_exception=ExecUtilException) as x: - os_ops.run(cmd, text=True, encoding="utf-8", check=True) + os_ops.run( + cmd, + text=True, + encoding="utf-8", + check=True, + ) assert x.type is ExecUtilException assert type(x.value.out) is str @@ -6343,7 +6367,12 @@ def test_run_check_exception2__list(self, os_ops_descr: OsOpsDescr): ) # 2. Test the negative scenario with validation disabled (check=False) - result = os_ops.run(cmd, text=True, encoding="utf-8", check=False) + result = os_ops.run( + cmd, + text=True, + encoding="utf-8", + check=False, + ) assert isinstance(result, OsCommandResult) assert result.returncode == 1 @@ -6364,7 +6393,13 @@ def test_run_check_exception3__str(self, os_ops_descr: OsOpsDescr): # 1. Check default behavior (check=True) with pytest.raises(expected_exception=ExecUtilException) as x: - os_ops.run(cmd, text=True, encoding="utf-8", shell=True, check=True) + os_ops.run( + cmd, + text=True, + encoding="utf-8", + shell=True, + check=True, + ) assert x.type is ExecUtilException assert type(x.value.out) is str diff --git a/tests/test_os_ops_local.py b/tests/test_os_ops_local.py index 1909824..4fafe56 100644 --- a/tests/test_os_ops_local.py +++ b/tests/test_os_ops_local.py @@ -28,7 +28,10 @@ def test_read__unknown_file( os_ops = os_ops_descr.os_ops assert isinstance(os_ops, OsOperations) - with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + with pytest.raises( + FileNotFoundError, + match=re.escape("[Errno 2] No such file or directory: '/dummy'"), + ): os_ops.read("/dummy") return @@ -46,8 +49,9 @@ def test_read_binary__spec__unk_file( assert isinstance(os_ops, OsOperations) with pytest.raises( - FileNotFoundError, - match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + FileNotFoundError, + match=re.escape("[Errno 2] No such file or directory: '/dummy'"), + ): os_ops.read_binary("/dummy", 0) return @@ -64,7 +68,10 @@ def test_get_file_size__unk_file( os_ops = os_ops_descr.os_ops assert isinstance(os_ops, OsOperations) - with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + with pytest.raises( + FileNotFoundError, + match=re.escape("[Errno 2] No such file or directory: '/dummy'"), + ): os_ops.get_file_size("/dummy") return