Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions bin/start.spec
Original file line number Diff line number Diff line change
@@ -1,19 +1,26 @@
# -*- mode: python -*-

from PyInstaller.utils.hooks import collect_data_files

block_cipher = None

# eth_account.hdaccount reads BIP39 wordlists from disk (hdaccount/wordlist/*.txt).
# hiddenimports only bundles Python modules; collect_data_files includes those data files.
eth_account_datas = collect_data_files("eth_account")

OCTOBOT_PACKAGES_FILES = REQUIRED = [s.strip() for s in open('bin/octobot_packages_files.txt').readlines()]
# hiddenimports=['numpy.core._dtype_ctypes'] from https://github.com/pyinstaller/pyinstaller/issues/3982
a = Analysis(
['../start.py'],
pathex=['../'],
datas=[
('../octobot/config', 'octobot/config'),
('../octobot/strategy_optimizer/optimizer_data_files', 'octobot/strategy_optimizer/optimizer_data_files')
],
('../octobot/strategy_optimizer/optimizer_data_files', 'octobot/strategy_optimizer/optimizer_data_files'),
] + eth_account_datas, # required for node wallet mnemonic generation (web3.Account.create_with_mnemonic)
hiddenimports=[
"colorlog", "numpy.core._dtype_ctypes", "dotenv",
"pgpy", "imghdr",
"web3", "eth_account",
"aiosqlite", "aiohttp",
"pyarrow", "pyiceberg",
"psutil",
Expand All @@ -22,6 +29,7 @@ a = Analysis(
"asyncpraw", "simplifiedpytrends", "simplifiedpytrends.exceptions", "simplifiedpytrends.request",
"pyngrok", "pyngrok.ngrok", "openai",
"flask", "flask_login", "flask_wtf", "flask_caching", "flask_compress", "flask_socketio", "flask_cors",
"werkzeug.middleware", "werkzeug.middleware.proxy_fix",
"wtforms", "wtforms.fields", "gevent", "geventwebsocket",
"vaderSentiment", "vaderSentiment.vaderSentiment",
"coingecko_openapi_client",
Expand Down
5 changes: 3 additions & 2 deletions octobot/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import octobot_commons.logging as logging
import octobot_commons.constants as commons_constants
import octobot_commons.errors as commons_errors
import octobot_commons.os_util as os_util
import octobot_commons.aiohttp_util as aiohttp_util

import octobot_tentacles_manager.api as tentacles_manager_api
Expand Down Expand Up @@ -343,7 +344,7 @@ def get_bot_file():


def restart_bot():
argv = (f'{a}' for a in sys.argv if a not in IGNORED_COMMAND_WHEN_RESTART)
argv = [f'{argument}' for argument in sys.argv if argument not in IGNORED_COMMAND_WHEN_RESTART]
if get_bot_file().endswith(".py"):
os.execl(sys.executable, f'{sys.executable}', *argv)
elif get_bot_file().endswith(constants.PROJECT_NAME):
Expand All @@ -353,7 +354,7 @@ def restart_bot():
# restart from binary
# from https://pyinstaller.org/en/stable/common-issues-and-pitfalls.html#using-sys-executable-to-spawn-subprocesses-that-outlive-the-application-process-implementing-application-restart
# Restart the application
subprocess.Popen([sys.executable], env={**os.environ, "PYINSTALLER_RESET_ENVIRONMENT": "1"})
subprocess.Popen(argv, env={**os.environ, **os_util.PYINSTALLER_RESET_ENVIRONMENT_VARS})
# force stop and restart
os._exit(0)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,19 @@ def spawn_subprocess(
self.pid = proc.pid
return proc

def bind_authoritative_child_pid(
self,
authoritative_pid: int,
*,
spawn_pid: typing.Optional[int] = None,
) -> None:
"""Point this operator and the managed-child registry at the authoritative app pid."""
previous_pid = spawn_pid if spawn_pid is not None else (self.pid or 0)
if previous_pid == authoritative_pid and self.pid == authoritative_pid:
return
process_util.rebind_managed_child_pid(previous_pid, authoritative_pid)
self.pid = authoritative_pid

@staticmethod
def reject_user_path_segment(path_value: str) -> None:
"""Reject obvious path traversal in user-supplied relative paths."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,33 @@ def unregister(self, pid: int) -> None:
self._pids.discard(pid)
self._logger.debug("Unregistered managed child pid=%s", pid)

def rebind_managed_child_pid(self, spawn_pid: int, authoritative_pid: int) -> None:
"""
Replace a bootstrap/spawn pid with the authoritative app pid (e.g. PyInstaller child).

Unregisters ``spawn_pid`` when it differs from ``authoritative_pid``, then registers
the authoritative pid. No-op when ``authoritative_pid`` is not positive or already bound.
"""
if authoritative_pid <= 0:
return
with self._lock:
authoritative_already_registered = authoritative_pid in self._pids
spawn_in_registry = spawn_pid > 0 and spawn_pid in self._pids
if authoritative_already_registered and (
spawn_pid <= 0 or spawn_pid == authoritative_pid
):
return
if spawn_pid > 0 and spawn_pid != authoritative_pid and spawn_in_registry:
self.unregister(spawn_pid)
if not authoritative_already_registered:
self.register(authoritative_pid)
if spawn_pid > 0 and spawn_pid != authoritative_pid:
self._logger.debug(
"Rebound managed child pid from %s to %s",
spawn_pid,
authoritative_pid,
)

def snapshot_running_pids(self) -> frozenset[int]:
"""Prune dead pids, return a point-in-time copy of still-running entries."""
with self._lock:
Expand Down
18 changes: 17 additions & 1 deletion packages/commons/octobot_commons/os_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,22 @@ def get_octobot_type():
return enums.OctoBotTypes.BINARY.value


# PyInstaller 6.9+: environment entries for spawning an independent frozen child process.
# Required when re-execing or subprocess-spawning the same frozen executable so each child
# unpacks and runs as its own instance (see PyInstaller common issues and pitfalls).
PYINSTALLER_RESET_ENVIRONMENT_VARS = {"PYINSTALLER_RESET_ENVIRONMENT": "1"}


def is_frozen_binary_octobot() -> bool:
"""
Return whether OctoBot is running as a PyInstaller frozen binary.

Uses PyInstaller's ``sys.frozen`` flag (not ``get_octobot_type()``), so ``python -m pytest``
and other interpreter invocations are not misclassified as frozen binaries.
"""
return bool(getattr(sys, "frozen", False))


def get_os():
"""
Return the OS name
Expand Down Expand Up @@ -194,7 +210,7 @@ def tcp_port_is_free(bind_host: str, port: int) -> bool:
return True


_HOST_WIDE_LISTENER_PROBE_HOSTS = ("0.0.0.0", "127.0.0.1")
_HOST_WIDE_LISTENER_PROBE_HOSTS = ("0.0.0.0", "127.0.0.1") # bind probes when psutil is unavailable


def tcp_port_has_listener_on_host(port: int) -> bool:
Expand Down
9 changes: 9 additions & 0 deletions packages/commons/octobot_commons/process_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,15 @@ async def wait_until_pid_stopped_async(
)


def rebind_managed_child_pid(spawn_pid: int, authoritative_pid: int) -> None:
"""Replace spawn pid with authoritative app pid in the managed-child registry."""
import octobot_commons.managed_child_process_registry as managed_child_process_registry
managed_child_process_registry.ManagedChildProcessRegistry.instance().rebind_managed_child_pid(
spawn_pid,
authoritative_pid,
)


async def graceful_stop_managed_children(
*,
timeout_seconds: float,
Expand Down
69 changes: 69 additions & 0 deletions packages/commons/tests/test_managed_child_process_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,75 @@ def test_unregister_unknown_pid_is_no_op(self):
assert registry.snapshot_running_pids() == frozenset()


class TestRebindManagedChildPid:
def test_rebind_replaces_spawn_pid_with_authoritative_pid(self):
registry = managed_child_process_registry.ManagedChildProcessRegistry.instance()
with mock.patch.object(process_util, "pid_is_running", return_value=True):
registry.register(20520)
registry.rebind_managed_child_pid(20520, 25044)
assert registry.snapshot_running_pids() == frozenset({25044})

def test_rebind_no_op_for_non_positive_authoritative_pid(self):
registry = managed_child_process_registry.ManagedChildProcessRegistry.instance()
with mock.patch.object(process_util, "pid_is_running", return_value=True):
registry.register(101)
registry.rebind_managed_child_pid(101, 0)
assert registry.snapshot_running_pids() == frozenset({101})

def test_rebind_same_pid_is_idempotent(self):
registry = managed_child_process_registry.ManagedChildProcessRegistry.instance()
with mock.patch.object(process_util, "pid_is_running", return_value=True):
registry.register(505)
with mock.patch.object(registry, "register") as register_mock:
registry.rebind_managed_child_pid(505, 505)
register_mock.assert_not_called()
assert registry.snapshot_running_pids() == frozenset({505})

def test_second_rebind_to_same_authoritative_pid_is_no_op(self):
registry = managed_child_process_registry.ManagedChildProcessRegistry.instance()
with mock.patch.object(process_util, "pid_is_running", return_value=True):
registry.register(20520)
registry.rebind_managed_child_pid(20520, 25044)
with mock.patch.object(registry, "register") as register_mock:
registry.rebind_managed_child_pid(20520, 25044)
register_mock.assert_not_called()
assert registry.snapshot_running_pids() == frozenset({25044})

@pytest.mark.asyncio
async def test_graceful_stop_all_signals_authoritative_pid_after_rebind(self):
registry = managed_child_process_registry.ManagedChildProcessRegistry.instance()
stop_requested = False

def pid_is_running_side_effect(pid):
if pid == 25044 and stop_requested:
return False
return True

def sigterm_side_effect(pid, *, logger=None):
nonlocal stop_requested
stop_requested = True
return {"status": "stopped", "signal": "sigterm"}

with (
mock.patch.object(
managed_child_process_registry.process_util,
"pid_is_running",
side_effect=pid_is_running_side_effect,
),
mock.patch.object(
managed_child_process_registry.process_util,
"request_graceful_stop_via_sigterm",
side_effect=sigterm_side_effect,
) as sigterm_mock,
):
registry.register(20520)
registry.rebind_managed_child_pid(20520, 25044)
outcomes = await registry.graceful_stop_all(timeout_seconds=0.1, poll_interval=0.01)
sigterm_mock.assert_called_once_with(25044, logger=registry._logger)
assert outcomes[25044] == "stopped"
assert 20520 not in outcomes


class TestManagedChildProcessRegistrySnapshotRunningPids:
def test_lazy_prunes_dead_pids(self):
registry = managed_child_process_registry.ManagedChildProcessRegistry.instance()
Expand Down
23 changes: 23 additions & 0 deletions packages/commons/tests/test_os_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# License along with this library.
import mock
import socket
import sys
import pytest

import octobot_commons.os_util as os_util
Expand All @@ -30,6 +31,28 @@ def test_get_cpu_and_ram_usage():
assert unique_ram > 0


class TestIsFrozenBinaryOctobot:
def test_returns_true_when_sys_frozen_is_set(self):
with mock.patch.object(sys, "frozen", True, create=True):
assert os_util.is_frozen_binary_octobot() is True

def test_returns_false_for_interpreter_argv_without_sys_frozen(self):
with mock.patch.object(sys, "argv", [sys.executable]), mock.patch.object(
sys, "frozen", False, create=True
):
assert os_util.is_frozen_binary_octobot() is False

def test_returns_false_when_sys_frozen_absent(self):
original_frozen = getattr(sys, "frozen", None)
if hasattr(sys, "frozen"):
delattr(sys, "frozen")
try:
assert os_util.is_frozen_binary_octobot() is False
finally:
if original_frozen is not None:
sys.frozen = original_frozen


class TestTcpPortIsFree:
def test_returns_false_when_tcp_listener_holds_port(self):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
Expand Down
Loading
Loading