diff --git a/BUILD b/BUILD index f6816fb2a3..dc913a0ee2 100644 --- a/BUILD +++ b/BUILD @@ -161,7 +161,7 @@ python_distribution( ] + PACKAGE_SOURCES + PACKAGE_REQS + PACKAGE_FULL_REQS, provides=python_artifact( name="octobot", - version="2.1.1", + version="3.0.0-beta1", url='https://github.com/Drakkar-Software/OctoBot', license='GPL-3.0', author="Drakkar-Software", diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ac8394ee7..334b50b091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 *It is strongly advised to perform an update of your tentacles after updating OctoBot. (start.py tentacles --install --all)* +## [3.0.0-beta1] - 2026-07-26 +### Start of the OctoBot 3.0.0 beta +OctoBot 3.0.0 a new system for portfolio automation from desktop and mobile. +You run it on your computer or server in node mode (the new default OctoBot mode), which acts as the backend for the new OctoBot interface. + +OctoBot now stats as node by default and lets users start as many OctoBot as they want. Those bots can be started either: +1. From the new OctoBot UI, available from the web and as a mobile app +2. From the node directly, for expert OctoBots enabling backtesting and advanced analysis (using the classic OctoBot UI) + +### Added +- [Node mode] The new OctoBot default: use the node as the orchestrator of your OctoBot, use it from the new OctoBot UI or app +- [UI] Brand new Octobot UI. More private, flexible, easy to use and secure than ever. Available at https://new.mobile.octobot.cloud/auth/unlock +- [Mobile apps] Start of the beta of the Android and iOS new apps with all the features of the Octobot UI, built into your phone +- [Multi OctoBot support] Starting as many bots as you need, directly from the Octobot interface +- [Kraken] Fully support Kraken exchange spot trading + + ## [2.1.1] - 2026-03-28 ### Added - [GridTrading] Add reinvest_profits config option diff --git a/README.md b/README.md index c1279a9295..1b700abc26 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,31 @@ # OctoBot - Free Open Source Crypto Trading Bot -[![PyPI](https://img.shields.io/pypi/v/OctoBot.svg?logo=pypi)](https://pypi.org/project/OctoBot) -[![Downloads](https://pepy.tech/badge/octobot/month)](https://pepy.tech/project/octobot) -[![Dockerhub](https://img.shields.io/docker/pulls/drakkarsoftware/octobot.svg?logo=docker)](https://hub.docker.com/r/drakkarsoftware/octobot) -[![OctoBot-CI](https://github.com/Drakkar-Software/OctoBot/workflows/OctoBot-CI/badge.svg)](https://github.com/Drakkar-Software/OctoBot/actions) -[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/Drakkar-Software/OctoBot) - -

- OctoBot free open source trading bot web user interface showcase -

- -[![OctoBot](https://img.shields.io/badge/dynamic/json.svg?&url=https://octobot.cloud/api/community/stats&query=$.total_bots&color=blue&label=Installed%20OctoBots)]() -[![Telegram Chat](https://img.shields.io/badge/telegram-chat-green.svg?logo=telegram&label=Telegram)](https://t.me/octobot_trading) -[![Discord](https://img.shields.io/discord/530629985661222912.svg?logo=discord&label=Discord)](https://discord.com/invite/vHkcb8W) -[![Telegram News](https://img.shields.io/badge/telegram-news-blue.svg?logo=telegram&label=Telegram)](https://t.me/OctoBot_Project) -[![Twitter](https://img.shields.io/twitter/follow/DrakkarsOctobot.svg?label=twitter&style=social)](https://x.com/DrakkarsOctoBot) -[![YouTube](https://img.shields.io/youtube/channel/views/UC2YAaBeWY8y_Olqs79b_X8A?label=youtube&style=social)](https://www.youtube.com/@octobot1134) + +[PyPI](https://pypi.org/project/OctoBot) +[Downloads](https://pepy.tech/project/octobot) +[Dockerhub](https://hub.docker.com/r/drakkarsoftware/octobot) +[OctoBot-CI](https://github.com/Drakkar-Software/OctoBot/actions) +[Ask DeepWiki](https://deepwiki.com/Drakkar-Software/OctoBot) + +[Telegram Chat](https://t.me/octobot_trading) +[Discord](https://discord.com/invite/vHkcb8W) +[Telegram News](https://t.me/OctoBot_Project) +[Twitter](https://x.com/DrakkarsOctoBot) +[YouTube](https://www.youtube.com/@octobot1134) + +## The new OctoBot beta is live + +The latest release runs in **node mode** by default. Your node acts as the backend for the [new OctoBot interface](https://new.mobile.octobot.cloud/home?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_new_interface) and mobile app, where you manage wallets, automations, and multiple strategies from a unified and secure dashboard. + +From the node interface, you can also start a **manual OctoBot** instance for backtesting, Telegram, TradingView, and other classic OctoBot workflows. + +Read the [full beta announcement](https://www.octobot.cloud/en/blog/next-octobot-beta-testing-starts?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=full_beta_announcement) to learn more about the largest OctoBot update ever. + +Please note that the new OctoBot beta is a work in progress. You may hit bugs or incomplete features. ## Open source crypto trading bot with a visual user interface [OctoBot](https://www.octobot.cloud/trading-bot?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_what_is_octobot) is an open source cryptocurrency trading robot designed for crypto investors who want to automate their investment strategies. The bot can automate strategies using built-in: + - Strategies such as [grids](#grid-trading-bot), [DCA strategies](#dca-trading-bot), [crypto baskets](#crypto-basket-trading-bot) and much more, which can all be configured - [AI connectors](#ai-trading-bot) to trade using any [OpenAI](https://openai.com/) or [Ollama](https://ollama.com/) model such as ChatGPT, llama or any [custom model](https://ollama.com/search) running on an Ollama server - [TradingView connectors](#tradingview-trading-bot) to automate trades from [TradingView](https://www.tradingview.com/?aff_id=27595) indicators or strategies @@ -26,65 +33,59 @@ - Technical analysis indicators such as RSI, Moving Averages or MACD - [15+ exchange integrations](#your-trading-bot-for-binance-coinbase-hyperliquid-and-15-other-exchanges) including Binance, Coinbase, MEXC and Hyperliquid +The trading bot is written in Python and built as a free open source software since 2018. It can be [installed on your system or executed on a cloud provider](#installing-octobot-open-source-crypto-trading-bot). + +### Control OctoBot from desktop, mobile, web, or Telegram + +Are you looking for a bot you can set up from the peaceful environment of your home computer and that you can follow from anywhere using your phone? + +#### OctoBot beta: node mode and the new interface + +Install the latest release and run OctoBot on your computer or server. Connect to it from the [new OctoBot interface](https://new.mobile.octobot.cloud/home?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_new_interface) in your browser, or from the mobile app. -

- OctoBot - Open Source Crypto Trading Bot Introduction Video from the official OctoBot YouTube Channel -

+From one node, run multiple **automations** (DCA, grids, crypto baskets, TradingView, and more) and monitor your full portfolio in one dashboard. -The trading bot is written in Python being built and improved as a free open source software since 2018. It can be [installed on your system or executed on a cloud provider](#installing-octobot-open-source-crypto-trading-bot). +The [new OctoBot's philosophy](https://www.octobot.cloud/en/blog/next-octobot-one-platform-for-all?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=new_octobots_philosophy) revolves around 3 pillars: +1. Multi-portfolio analysis and automation +2. Self custody: your keys never leave your devices +3. Zero knowledge privacy: all your data is end-to-end encrypted, no one can read them except you -### An easy to use trading bot with a Mobile App, Web and Telegram user interfaces -Are you looking for a bot you can setup from the peaceful environment of your home computer and that you can follow from anywhere using your phone? +#### Classic OctoBot: web interface and Telegram OctoBot is designed for crypto investors who want to automate their trading strategies in a simple way, using a graphic interface to: + - Configure the details of their strategy and its traded markets and exchange(s) - Test and optimize the strategy using [backtesting](https://www.octobot.cloud/en/guides/octobot-usage/backtesting?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=backtesting) - Live test it with paper money - Execute it on a real exchange account, by automatically sending orders to the exchange -Once started, an OctoBot can be followed using its web interface, making it reachable when running on a cloud server. -OctoBot can also be connected to a Telegram bot, therefore turning OctoBot into a [Telegram trading bot](https://www.octobot.cloud/en/guides/octobot-interfaces/telegram?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=telegram). +A **manual OctoBot** instance can still be configured and followed using its built-in web interface, making it reachable when running on a cloud server. +OctoBot can also be connected to a Telegram bot, turning OctoBot into a [Telegram trading bot](https://www.octobot.cloud/en/guides/octobot-interfaces/telegram?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=telegram). -You can also follow your trading bot from the OctoBot mobile app, which is designed to automate [octobot.cloud](https://www.octobot.cloud/trading-bot?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=octobot_app_section) strategies, the app can also be used to follow your open source OctoBot's profits, portfolio, open orders and trades. -

- octobot mobile app dashboard with portfolio value and two live bots with news -      - octobot mobile app bot view with portfolio content recent activities and historical profits -

+You can also follow your trading bot from the OctoBot mobile app, which is designed to automate [octobot.cloud](https://www.octobot.cloud/trading-bot?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=octobot_app_section) strategies. The app can also be used to follow your open source OctoBot's profits, portfolio, open orders and trades. -

- Get it on the Apple Play Store - Get it on Google Play -

+The public (non-beta) OctoBot mobile app is designed for [octobot.cloud](https://www.octobot.cloud/trading-bot?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=octobot_app_section) strategies. To manage your open source OctoBot in beta, use the new OctoBot interface or mobile beta app instead. ### Live and backtesting trading strategies automation -OctoBot is more than just a strategy execution engine, it can also simulate investments using [risk-free paper trading](https://www.octobot.cloud/en/guides/octobot-usage/simulator?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=risk-free_paper_trading). +OctoBot is more than just a strategy execution engine, it can also simulate investments using [risk-free paper trading](https://www.octobot.cloud/en/guides/octobot-usage/simulator?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=risk-free_paper_trading). Even better, the trading robot comes with its [built-in backtesting engine](https://www.octobot.cloud/en/guides/octobot-usage/backtesting?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=backtesting_engine) to help you test and optimize your strategy over a long period of time with a simulated portfolio and historical exchange data. -

- Backtesting report using grid trading on eth btc with 8 percent profit -

- Backtesting your trading strategy and portfolio will give you accurate insights on the past performance and behavior of your trading strategy starting from its initial portfolio. This analysis tool gives you all the metrics to create the best version of your strategy before automating it with your real funds, on your exchange account. -### Your trading bot for Binance, Coinbase, Hyperliquid and 15+ other exchanges -OctoBot supports the vast majority of crypto exchanges thanks to the great [CCXT library](https://github.com/ccxt/ccxt). - -

- list of octobot supported exchanges including binance coinbase hyperliquid mexc and more -

+### Your trading bot for Binance, Coinbase, Hyperliquid and 15+ other exchanges +OctoBot supports the vast majority of crypto exchanges thanks to the great [CCXT library](https://github.com/ccxt/ccxt). This wide range of supported exchanges makes it easy to create investment strategies on any crypto, from Bitcoin, Ethereum or Solana to altcoins from the darkest depths of the altcoin forest. - Supported exchanges notably include: + - [Binance](https://accounts.binance.com/en/register?ref=528112221) spot and futures trading using the REST and websocket APIs - [Coinbase](https://www.coinbase.com/) spot trading using the REST and websocket APIs -- [Bybit](https://www.bybit.com/en-US/invite?ref=QW6O5) spot and futures trading using the REST and websocket APIs Note: due to a recent update, the Bybit API will soon be available again on OctoBot +- [Bybit](https://www.bybit.com/en-US/invite?ref=QW6O5) spot and futures trading using the REST and websocket APIs. Note: due to a recent update, the Bybit API will soon be available again on OctoBot - [Hyperliquid](https://app.hyperliquid.xyz/) spot trading (with API Keys) using the REST and websocket APIs - [MEXC](https://www.mexc.com/register?inviteCode=1fqGu) spot trading using the REST and websocket APIs - [Kucoin](https://www.kucoin.com/ucenter/signup?rcode=rJ2Q2T3) spot and futures trading using the REST and websocket APIs @@ -92,70 +93,78 @@ Supported exchanges notably include: - Many other such as OKX, Binance US, Crypto.com, HTX, Bitget, BingX, CoinEx, BitMart, Phemex, Gate.io, Ascendex and more on the [full list of supported exchanges](https://www.octobot.cloud/en/guides/exchanges?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=exchanges_full_list). ## Automate your trading strategies + OctoBot is designed as a one-stop-shop for crypto trading strategies. If you think of a crypto trading strategy, it can most likely be automated by OctoBot, unless it requires very complex custom mechanisms. ### AI trading bot -OctoBot can be an AI trading bot using [OpenAI](https://openai.com/) model such as ChatGPT. The [ChatGPT trading mode](https://www.octobot.cloud/en/guides/octobot-trading-modes/chatgpt-trading?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=ai-trading-bot) is the dedicated OctoBot configuration to give market context to a LLM model, ask for its opinion and trade accordingly. + +OctoBot can be an AI trading bot using [OpenAI](https://openai.com/) models such as ChatGPT. The [ChatGPT trading mode](https://www.octobot.cloud/en/guides/octobot-trading-modes/chatgpt-trading?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=ai-trading-bot) is the dedicated OctoBot configuration to give market context to a LLM model, ask for its opinion and trade accordingly. Local LLM models, such as [Ollama](https://ollama.com/) llama or any [custom model](https://ollama.com/search) running on your Ollama server can also be used by the bot for deeper customization and cost management. ### Grid trading bot + Grid trading is a strategy that extracts value from volatility. Unlike most strategies, it relies on pure math and no statistics. It will "simply" create and maintain many buy and sell orders at regular intervals and generate profits every time both buy and sell orders are executed. [The OctoBot grid trading bot](https://www.octobot.cloud/en/guides/octobot-trading-modes/grid-trading-mode?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=grid-trading-bot) can be heavily customized and optimized to be perfectly adapted to your market and exchange. ### DCA trading bot + Dollar Cost Averaging (DCA) is a well known investment strategy where you buy on a regular basis in order to profit from local price drops. It allows investors to reduce their overall buying costs. As a [DCA trading bot](https://www.octobot.cloud/en/guides/octobot-trading-modes/dca-trading-mode?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=dca-trading-bot), OctoBot can optimize your investment strategies for short or long term gains with heavy customization and backtesting capabilities. ### TradingView trading bot + Use OctoBot as your [TradingView trading bot](https://www.octobot.cloud/en/guides/octobot-trading-modes/tradingview-trading-mode?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=tradingview-trading-bot) and simply emit alerts from your TradingView indicators or strategies. Use these alerts to trade any crypto market, on any exchange from your TradingView native strategy. Whether it's from a visual TradingView indicator or a heavily optimized Pine Script strategy, your trades can be automated. ### Crypto basket trading bot + Crypto baskets are similar to stock indexes or ETFs. They enable you to invest into many cryptocurrencies, all at once, in a simple way. A crypto basket is a simple way to invest in the whole crypto market at once, or follow coin categories, such as AI or RWA coins. OctoBot can be used as a [crypto basket trading bot](https://www.octobot.cloud/en/guides/octobot-trading-modes/index-trading-mode?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=crypto-basket-trading-bot) and make it simple to invest in customized crypto indexes or follow baskets from the wide range of [OctoBot cloud's crypto baskets](https://www.octobot.cloud/features/crypto-basket?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=crypto-basket-trading-bot). ### Market Making trading bot -OctoBot can also [automate market making strategies](https://github.com/Drakkar-Software/OctoBot-market-making) to help token creators provide liquidity to their markets. -

- octobot market making dashboard with buy and sell orders -

+OctoBot can also [automate market making strategies](https://github.com/Drakkar-Software/OctoBot-market-making) to help token creators provide liquidity to their markets. Advanced market making strategies can be automated on [market-making.octobot.cloud](https://market-making.octobot.cloud?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=market-making), the self-service market making platform based on OctoBot. Feel free to contact us if you have any questions about it. ### Institutional custom strategies -If you represent an institution that is interested in a commercial license or custom development to suit your specific needs or strategy please contact us at contact@drakkar.software. + +If you represent an institution that is interested in a commercial license or custom development to suit your specific needs or strategy, please contact us at [contact@drakkar.software](mailto:contact@drakkar.software). ## Installing OctoBot, open source crypto trading bot -

- A man relaxing in his couch while octobot the free open source crypto trading bot is making money by automating cryptocurrency strategies -

+Starting the [latest release](https://github.com/Drakkar-Software/OctoBot/releases/latest) runs OctoBot in **node mode**. See the [getting-started guide](https://www.octobot.cloud/en/guides/octobot-beta-getting-started?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_install_beta) to connect the new OctoBot interface to your node. OctoBot can be deployed on the cloud or for free on your computer, server or [Raspberry Pi](https://www.raspberrypi.com/). ### Deploying OctoBot with one click on DigitalOcean + OctoBot can be easily launched in the cloud from the [DigitalOcean Marketplace](https://digitalocean.pxf.io/octobot-app). -[![Deploy on DigitalOcean](https://mp-assets1.sfo2.digitaloceanspaces.com/deploy-to-do/do-btn-blue.svg)](https://digitalocean.pxf.io/start-octobot) +[Deploy on DigitalOcean](https://digitalocean.pxf.io/start-octobot) ### Using the OctoBot Executable + This is the easiest way to download and install OctoBot on your computer or server. Here is [our executable installation guide](https://www.octobot.cloud/en/guides/octobot-installation/install-octobot-on-your-computer?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_local_installation). Note: The latest release executables for Windows, MacOS, Linux and Raspberry Pi are automatically built and pushed to the [releases](https://github.com/Drakkar-Software/OctoBot/releases) page. ### Using the OctoBot Docker image + You can also install OctoBot using the [OctoBot Docker image](https://hub.docker.com/r/drakkarsoftware/octobot). Here is our [using Docker installation guide](https://www.octobot.cloud/en/guides/octobot-installation/install-octobot-with-docker-video?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_docker_installation). -Docker install in one liner: +Docker install in one-liner: + ```sh docker run -itd --name OctoBot -p 80:5001 -v $(pwd)/user:/octobot/user -v $(pwd)/tentacles:/octobot/tentacles -v $(pwd)/logs:/octobot/logs drakkarsoftware/octobot:stable ``` ### Installing OctoBot using Python -If you want to install OctoBot from Python, for example in order to edit the code or contribute, [here is our python installation guide](https://www.octobot.cloud/en/guides/octobot-installation/install-octobot-with-python-and-git?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_python_installation). + +If you want to install OctoBot from Python, for example in order to edit the code or contribute, [here is our Python installation guide](https://www.octobot.cloud/en/guides/octobot-installation/install-octobot-with-python-and-git?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=readme_python_installation). You might also want to look at our [contributing guide](CONTRIBUTING.md) to quickly understand how OctoBot is architected. -### Minimum hardware requirements +### Minimum hardware requirements + - CPU : 1 Core / 1GHz - RAM : 250 MB - Disk : 1 GB @@ -166,15 +175,18 @@ Would you like to add or improve something in OctoBot? We welcome your pull requ Please have a look at our [contributing guide](CONTRIBUTING.md) to read our guidelines. ### Contribute to OctoBot from a local IDE -We recommend using a [VSCode](https://code.visualstudio.com/)-based IDE to contribute to OctoBot however [PyCharm](https://www.jetbrains.com/pycharm/) can also be used. -As the OctoBot code is split into different repositories, we created a [developer installation guide](https://www.octobot.cloud/en/guides/octobot-developers-environment/setup-your-environment?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=setup_your_environment) to help setting up a VSCode or PyCharm environment. + +We recommend using a [VSCode](https://code.visualstudio.com/)-based IDE to contribute to OctoBot, however [PyCharm](https://www.jetbrains.com/pycharm/) can also be used. +As the OctoBot code is split into different repositories, we created a [developer installation guide](https://www.octobot.cloud/en/guides/octobot-developers-environment/setup-your-environment?utm_source=github&utm_medium=dk&utm_campaign=regular_open_source_content&utm_content=setup_your_environment) to help you set up a VSCode or PyCharm environment. ### Contribute to OctoBot from Ona (formerly Gitpod) -Make changes and contribute to OctoBot in a single click with an **already setup and ready to code developer environment** using Ona. -[![Contribute from Ona](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/Drakkar-Software/OctoBot) +Make changes and contribute to OctoBot in a single click with an **already set up and ready to code developer environment** using Ona. + +[Contribute from Ona](https://gitpod.io/#https://github.com/Drakkar-Software/OctoBot) ## Disclaimer + Do not risk money which you are afraid to lose. USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. Always start by running a trading bot in simulation mode and do not engage money @@ -183,34 +195,29 @@ before you understand how it works and what profit/loss you should expect. Please feel free to read the source code and understand the mechanism of this bot. ## Sponsors - - - - - - - - - - - - - -
Special thanks to Chatwoot for helping us assist the users of OctoBot.

Huge thank you to Scaleway for hosting OctoBot's cloud services.

A big thank you to Sentry for helping us identify and understand errors in OctoBot to make it better.

+ + +| | | +| ----------------------------------------------------------------------------------------------------------------------------------- | --- | +| Special thanks to [Chatwoot](https://www.chatwoot.com/) for helping us assist the users of OctoBot. | | +| Huge thank you to [Scaleway](https://www.scaleway.com) for hosting OctoBot's cloud services. | | +| A big thank you to [Sentry](https://sentry.io/welcome/) for helping us identify and understand errors in OctoBot to make it better. | | + ## License + GNU General Public License v3.0 or later. See [GPL-3.0 LICENSE](https://github.com/Drakkar-Software/OctoBot/blob/master/LICENSE) to see the full text. - ## Give a boost to OctoBot + Do you like what we are building with OctoBot? Consider giving us a star ⭐ to boost the project's visibility! -And join us on the OctoBot channels +And join us on the OctoBot channels. -[![Telegram Chat](https://img.shields.io/badge/telegram-chat-green.svg?logo=telegram&label=Telegram)](https://t.me/octobot_trading) -[![Discord](https://img.shields.io/discord/530629985661222912.svg?logo=discord&label=Discord)](https://discord.com/invite/vHkcb8W) -[![Telegram News](https://img.shields.io/badge/telegram-news-blue.svg?logo=telegram&label=Telegram)](https://t.me/OctoBot_Project) -[![Twitter](https://img.shields.io/twitter/follow/DrakkarsOctobot.svg?label=twitter&style=social)](https://x.com/DrakkarsOctoBot) -[![YouTube](https://img.shields.io/youtube/channel/views/UC2YAaBeWY8y_Olqs79b_X8A?label=youtube&style=social)](https://www.youtube.com/@octobot1134) +[Telegram Chat](https://t.me/octobot_trading) +[Discord](https://discord.com/invite/vHkcb8W) +[Telegram News](https://t.me/OctoBot_Project) +[Twitter](https://x.com/DrakkarsOctoBot) +[YouTube](https://www.youtube.com/@octobot1134) \ No newline at end of file diff --git a/bin/start.spec b/bin/start.spec index a4dfa8c62c..18c0e03875 100644 --- a/bin/start.spec +++ b/bin/start.spec @@ -1,7 +1,13 @@ # -*- mode: python -*- +from PyInstaller.utils.hooks import collect_data_files + block_cipher = None +# eth_account.hdaccount reads BIP39 wordlists from disk (hdaccount/wordlist/*.txt). +# hiddenimports only bundles Python modules; collect_data_files includes those data files. +eth_account_datas = collect_data_files("eth_account") + OCTOBOT_PACKAGES_FILES = REQUIRED = [s.strip() for s in open('bin/octobot_packages_files.txt').readlines()] # hiddenimports=['numpy.core._dtype_ctypes'] from https://github.com/pyinstaller/pyinstaller/issues/3982 a = Analysis( @@ -9,11 +15,12 @@ a = Analysis( pathex=['../'], datas=[ ('../octobot/config', 'octobot/config'), - ('../octobot/strategy_optimizer/optimizer_data_files', 'octobot/strategy_optimizer/optimizer_data_files') - ], + ('../octobot/strategy_optimizer/optimizer_data_files', 'octobot/strategy_optimizer/optimizer_data_files'), + ] + eth_account_datas, # required for node wallet mnemonic generation (web3.Account.create_with_mnemonic) hiddenimports=[ "colorlog", "numpy.core._dtype_ctypes", "dotenv", "pgpy", "imghdr", + "web3", "eth_account", "aiosqlite", "aiohttp", "pyarrow", "pyiceberg", "psutil", @@ -22,6 +29,7 @@ a = Analysis( "asyncpraw", "simplifiedpytrends", "simplifiedpytrends.exceptions", "simplifiedpytrends.request", "pyngrok", "pyngrok.ngrok", "openai", "flask", "flask_login", "flask_wtf", "flask_caching", "flask_compress", "flask_socketio", "flask_cors", + "werkzeug.middleware", "werkzeug.middleware.proxy_fix", "wtforms", "wtforms.fields", "gevent", "geventwebsocket", "vaderSentiment", "vaderSentiment.vaderSentiment", "coingecko_openapi_client", diff --git a/docs/content/developers/packages/node.md b/docs/content/developers/packages/node.md index 11b42c8087..c64eff559a 100644 --- a/docs/content/developers/packages/node.md +++ b/docs/content/developers/packages/node.md @@ -24,6 +24,18 @@ User-triggered actions — things like a manual override sent through the API Log messages emitted inside any workflow or step are routed to a per-workflow file under `logs/automations/`. Child workflows share their parent's log file, keyed by the first 36 characters of the workflow ID. +### DBOS application version and recovery + +DBOS only dequeues and recovers workflows whose `application_version` matches the runtime. OctoBot Node uses a stable code constant — `SCHEDULER_APPLICATION_VERSION` in `octobot_node.constants` (currently `octobot_node_v1`) — instead of the OctoBot release version, so automations survive OctoBot upgrades. + +When `ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION` is enabled, scheduler startup runs `migrate_stranded_workflow_versions()` before `launch()`. That re-tags any `ENQUEUED` or `PENDING` workflows in the system database to `SCHEDULER_APPLICATION_VERSION`, unblocking workflows stranded by older releases that tagged rows with the OctoBot version string. The flag is read from the environment variable of the same name (`true` / `false`); it defaults to `false`. + +| Setting | Default | Purpose | +|---|---|---| +| `ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION` | `false` | Run startup migration of stranded workflow `application_version` values | + +Bump `SCHEDULER_APPLICATION_VERSION` only when workflow step order or semantics change in a breaking way. For branching old vs new code paths within the same version, use [DBOS patching](https://docs.dbos.dev/python/tutorials/upgrading-workflows). + ## Encryption Task payloads are optionally encrypted using a hybrid RSA/AES-GCM/ECDSA scheme. Each encryption call generates a fresh AES-256-GCM key and IV; the AES key is wrapped with RSA-OAEP so the bulk payload never travels under the asymmetric key directly. An ECDSA signature over the ciphertext — computed as `ciphertext + encrypted_aes_key + iv` concatenated — is verified before any decryption attempt, preventing chosen-ciphertext attacks. diff --git a/docs/content/guides/octobot-configuration/profiles.md b/docs/content/guides/octobot-configuration/profiles.md index 475f2dbfe8..d3f1b29ea2 100644 --- a/docs/content/guides/octobot-configuration/profiles.md +++ b/docs/content/guides/octobot-configuration/profiles.md @@ -6,10 +6,15 @@ sidebar_position: 1 # Profiles -OctoBot's trading configuration is using profiles (located into -user/profiles). This allows for quick switches between previously set +OctoBot's trading configuration is using profiles. This allows for quick switches between previously set configurations. Each profile defines a [Trading Mode](/guides/octobot-trading-modes/trading-modes) configuration as well as other settings. +Bundled default profiles (for example `default` and `non-trading`) remain on the filesystem under +`user/profiles/`. When a wallet is configured, user-created profiles are stored in the sync +`StrategyProvider` collection as `GenericProcessConfiguration.profile_data` (same profile id as the +strategy). OctoBot still exposes the same profile API to the web UI and configuration layer; only the +profile module selects the storage backend. + ![octobot trading mode details from profiles](/images/guides/configuration/octobot-trading-mode-details-from-profiles.png) Profiles include: diff --git a/octobot/__init__.py b/octobot/__init__.py index 54d9202257..0ddcf06eac 100644 --- a/octobot/__init__.py +++ b/octobot/__init__.py @@ -16,5 +16,5 @@ PROJECT_NAME = "OctoBot" AUTHOR = "Drakkar-Software" -VERSION = "2.1.1" # major.minor.revision +VERSION = "3.0.0-beta1" # major.minor.revision LONG_VERSION = f"{VERSION}" diff --git a/octobot/cli.py b/octobot/cli.py index f246102d5b..158f901442 100644 --- a/octobot/cli.py +++ b/octobot/cli.py @@ -38,12 +38,16 @@ import octobot_tentacles_manager.api as tentacles_manager_api import octobot_tentacles_manager.cli as tentacles_manager_cli import octobot_tentacles_manager.constants as tentacles_manager_constants + import octobot_tentacles_manager + + import octobot_backtesting.constants as backtesting_constants # make tentacles importable sys.path.append(os.path.dirname(sys.executable)) import octobot.octobot as octobot_class import octobot.octobot_node as octobot_node_class + import octobot_node.config import octobot.commands as commands import octobot.configuration_manager as configuration_manager import octobot.octobot_backtesting_factory as octobot_backtesting @@ -64,13 +68,6 @@ sys.exit(-1) def update_config_with_args(starting_args, config: configuration.Configuration, logger): - try: - import octobot_backtesting.constants as backtesting_constants - except ImportError as e: - octobot_commons.logging.get_logger().error( - "Can't start backtesting without the octobot_backtesting package properly installed.") - raise e - if starting_args.backtesting: if starting_args.backtesting_files: config.config[backtesting_constants.CONFIG_BACKTESTING][ @@ -134,23 +131,30 @@ def _create_configuration(): return config -def _create_startup_config(logger, default_config_file): +def _create_startup_config(logger, default_config_file, *, is_process_child: bool = False): logger.info("Loading config files...") config = _create_configuration() is_first_startup = config.is_config_file_empty_or_missing() if is_first_startup: - logger.info("No configuration found creating default configuration...") - configuration_manager.init_config(from_config_file=default_config_file) - config.read(should_raise=False) + user_config_path = configuration.get_user_config() + if is_process_child: + raise errors.ConfigError( + f"Process OctoBot child expected prepared {common_constants.CONFIG_FILE} at " + f"{user_config_path!r} (under {common_constants.USER_AUTOMATIONS_FOLDER}//). " + f"The executor must materialize the automation layout before spawn; if this file is missing, " + f"check ensure_user_profile_and_layout timing or parallel functional test races." + ) + logger.info( + f"No configuration found in {user_config_path}. " + f"Creating default configuration..." + ) + configuration_manager.init_config( + config_file=user_config_path, + from_config_file=default_config_file, + ) + config.read(should_raise=False, activate_profile=False) else: - _read_config(config, logger) - try: - commands.ensure_profile(config) - _validate_config(config, logger) - except (errors.NoProfileError, errors.ConfigError): - # real issue if tentacles exist otherwise continue - if os.path.isdir(tentacles_manager_constants.TENTACLES_PATH): - raise + _read_config(config, logger, activate_profile=False) distribution = configuration_manager.get_distribution(config.config) if distribution is not enums.OctoBotDistribution.DEFAULT: logger.info(f"Using {distribution.value} OctoBot distribution.") @@ -183,10 +187,10 @@ async def _apply_db_bot_config(logger, config, community_auth) -> bool: constants.COMMUNITY_BOT_ID, constants.USER_AUTH_KEY, ) - profile = await profiles.import_profile_data_as_profile( + profile = await config.profile_storage.import_profile_data( profile_data, constants.PROFILE_FILE_SCHEMA, - None, + os.getcwd(), name=profile_data.profile_details.name, auto_update=False, force_simulator=False, @@ -277,17 +281,31 @@ def _apply_forced_configs(community_auth, logger, config, is_first_startup): _apply_env_variables_to_config(logger, config) -def _read_config(config, logger): +def _read_config(config, logger, *, activate_profile=True): try: - config.read(should_raise=True, fill_missing_fields=True) + config.read( + should_raise=True, + fill_missing_fields=True, + activate_profile=activate_profile, + ) except errors.NoProfileError: _repair_with_default_profile(config, logger) config = _create_configuration() - config.read(should_raise=False, fill_missing_fields=True) + config.read( + should_raise=False, + fill_missing_fields=True, + activate_profile=activate_profile, + ) except Exception as e: raise errors.ConfigError(e) +def _activate_saved_profile_after_sync(config, logger): + config.activate_saved_profile() + commands.ensure_profile(config) + _validate_config(config, logger) + + def _validate_config(config, logger): try: config.validate() @@ -305,19 +323,33 @@ def _repair_with_default_profile(config, logger): config.load_profiles_if_possible_and_necessary() -def _load_or_create_tentacles(community_auth, config, logger): +def _load_or_create_tentacles(community_auth, config, logger, *, is_process_child: bool = False): # add tentacles folder to Python path sys.path.append(os.path.realpath(os.getcwd())) + # Process children store readonly_reference_tentacles_path in config.json and have no + # local reference_tentacles_config/. Apply the override before probing whether the + # reference config file exists, so the path resolves to the master tree. + config.apply_readonly_reference_tentacles_override() + if os.path.isfile( user_root_folder_provider.get_user_reference_tentacle_config_file_path() ): # when tentacles folder already exists config.load_profiles_if_possible_and_necessary() - tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config( - config.get_tentacles_config_path() - ) - commands.run_update_or_repair_tentacles_if_necessary(community_auth, config, tentacles_setup_config) + tentacles_setup_config = config.get_active_tentacles_setup_config() + if ( + is_process_child + and config.config.get(common_constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH) + ): + # Process children share the master reference tree; skip repair that would write to it. + if not tentacles_manager_api.load_tentacles(verbose=True): + logger.error("OctoBot tentacles failed to load for process child.") + config.save(schema_file=config.config_schema_path) + else: + commands.run_update_or_repair_tentacles_if_necessary( + community_auth, config, tentacles_setup_config + ) else: # when no tentacles folder has been found logger.info("OctoBot tentacles can't be found. Installing default tentacles ...") @@ -338,6 +370,112 @@ def _init_cli_overriden_folders(args): return overrides, logs_folder +def _assert_process_child_folder_overrides(args) -> None: + """When ``--dump-state`` is set (DSL process children), require an automation-scoped ``--user-folder``.""" + if not args.dump_state or not str(args.dump_state).strip(): + return + user_folder = args.user_folder + if not user_folder or not str(user_folder).strip(): + raise errors.ConfigError( + "Process OctoBot children require --user-folder under " + f"{common_constants.USER_AUTOMATIONS_FOLDER}// when --dump-state is set." + ) + path_segments = tuple( + segment + for segment in str(user_folder).replace("\\", "/").split("/") + if segment + ) + expected_prefix = ( + common_constants.USER_FOLDER, + common_constants.AUTOMATIONS_FOLDER, + ) + if len(path_segments) < len(expected_prefix) + 1: + raise errors.ConfigError( + "Process OctoBot children require --user-folder under " + f"{common_constants.USER_AUTOMATIONS_FOLDER}//, got " + f"{user_folder!r}." + ) + if path_segments[: len(expected_prefix)] != expected_prefix: + raise errors.ConfigError( + "Process OctoBot children require --user-folder to start with " + f"{common_constants.USER_AUTOMATIONS_FOLDER}/, got {user_folder!r}." + ) + if ".." in path_segments: + raise errors.ConfigError( + f"Invalid --user-folder for process child: parent segments are not allowed ({user_folder!r})." + ) + +def _configure_profile_sync_user(config, community_auth, *, is_process_child: bool = False): + if is_process_child: + sync_user_id = constants.PROCESS_BOT_SYNC_USER_ID + if not sync_user_id: + raise errors.ConfigError( + "Process OctoBot children require " + f"{constants.ENV_PROCESS_BOT_SYNC_USER_ID} to be set by the executor." + ) + config.profile_storage.bind_process_child_sync_user_id(sync_user_id) + return + if community_auth is not None and community_auth.auto_init_sync_client(): + config.profile_storage.configure_sync_user(community_auth.sync_user_id) + + +def _apply_env_vars_to_startup_args(args) -> None: + if constants.FORCE_OCTOBOT_STANDALONE: + args.standalone = True + + +def _validate_startup_mode_args(args) -> None: + if args.standalone and (args.master or args.consumer_only): + raise errors.ConfigError( + "--standalone cannot be used with --master or --consumer_only." + ) + + +def _apply_node_cli_settings(args) -> None: + if args.master: + octobot_node.config.settings.IS_MASTER_MODE = True + octobot_node.config.settings.CONSUMER_ONLY = args.consumer_only + + +def _apply_node_startup_settings(args) -> None: + constants.FORCED_DISTRIBUTION = enums.OctoBotDistribution.NODE.value + args.no_web = True + + +def _apply_standalone_startup_settings() -> None: + constants.FORCED_DISTRIBUTION = enums.OctoBotDistribution.DEFAULT.value + octobot_node.config.settings.IS_MASTER_MODE = False + octobot_node.config.settings.CONSUMER_ONLY = False + + +def _apply_startup_distribution_mode(args) -> None: + if args.standalone or args.backtesting: + _apply_standalone_startup_settings() + else: + _apply_node_cli_settings(args) + _apply_node_startup_settings(args) + + +def _log_startup_distribution_mode(logger, args) -> None: + if args.backtesting: + startup_message = "Starting OctoBot in backtesting mode." + elif args.standalone: + startup_message = ( + "Starting OctoBot in standalone mode: will trade using the selected profile." + ) + else: + startup_message = ( + "Starting OctoBot in node mode: use the interface to start OctoBots that trade " + "from your selected strategy on the exchanges of your choice." + ) + if not args.backtesting: + if args.master: + startup_message += " Master scheduler enabled." + if args.consumer_only: + startup_message += " Consumer-only worker mode enabled." + logger.info(startup_message) + + def start_octobot(args, default_config_file=None): logger = None try: @@ -346,7 +484,12 @@ def start_octobot(args, default_config_file=None): return overrides, logs_folder = _init_cli_overriden_folders(args) + _assert_process_child_folder_overrides(args) logger = octobot_logger.init_logger(logs_folder=logs_folder) + _apply_env_vars_to_startup_args(args) + _validate_startup_mode_args(args) + _apply_startup_distribution_mode(args) + _log_startup_distribution_mode(logger, args) startup_messages = [] # Version @@ -361,25 +504,25 @@ def start_octobot(args, default_config_file=None): # Current running environment _log_environment(logger) - octobot_community.init_sentry_tracker() - # load configuration + is_process_child = bool(args.dump_state and str(args.dump_state).strip()) config, is_first_startup = _create_startup_config( - logger, default_config_file or constants.DEFAULT_CONFIG_FILE + logger, + default_config_file or constants.DEFAULT_CONFIG_FILE, + is_process_child=is_process_child, ) # check config loading if not config.is_loaded(): raise errors.ConfigError + octobot_community.ActivityMetrics.initialize_tracker(config) + # Handle utility methods before bot initializing if possible if args.encrypter: commands.exchange_keys_encrypter() return - # add args to config - update_config_with_args(args, config, logger) - # show terms _log_terms_if_unaccepted(config, logger) @@ -387,13 +530,29 @@ def start_octobot(args, default_config_file=None): _get_authenticated_community_if_possible(config, logger) ) - # tries to load, install or repair tentacles - _load_or_create_tentacles(community_auth, config, logger) + # Startup order matters: sync user and tentacles/community config must run before + # profile activation. First boot (empty user/) has no profiles until tentacles + # install and _apply_forced_configs import them; ensure_profile() here would fail too early. + _configure_profile_sync_user( + config, + community_auth, + is_process_child=is_process_child, + ) + + # Install or repair tentacles before selecting a profile (may create default profiles). + # Process children: readonly_reference_tentacles_path from config.json is applied here. + _load_or_create_tentacles(community_auth, config, logger, is_process_child=is_process_child) - # patch setup with forced values + # Cloud/community forced profiles and env overrides (first startup imports remote profile). if not args.backtesting: _apply_forced_configs(community_auth, logger, config, is_first_startup) + # Select saved profile and validate config only after profiles are available. + _activate_saved_profile_after_sync(config, logger) + + # apply CLI overrides after profile activation (trader keys come from profile) + update_config_with_args(args, config, logger) + # Can now perform config health check (some checks require a loaded profile) configuration_manager.config_health_check(config, args.backtesting) @@ -417,7 +576,7 @@ def start_octobot(args, default_config_file=None): startup_messages=startup_messages) if not args.backtesting: - path = getattr(args, "dump_state", None) + path = args.dump_state if path: bot.dump_state_path = os.path.normpath(path) @@ -553,6 +712,13 @@ def octobot_parser(parser, default_config_file=None): 'test. Example: -o TechnicalAnalysisStrategyEvaluator' ' Warning: this process may take a long time.', nargs='+') + parser.add_argument( + '--standalone', + help='Start standalone OctoBot with the trading web interface (default startup is node mode). ' + 'Can also be enabled with OCTOBOT_STANDALONE=true.', + action='store_true', + ) + _register_node_arguments(parser) parser.set_defaults(func= lambda args: start_octobot(args, default_config_file)) # add sub commands @@ -565,13 +731,6 @@ def octobot_parser(parser, default_config_file=None): tentacles_manager_cli.register_tentacles_manager_arguments(tentacles_parser) tentacles_parser.set_defaults(func=commands.call_tentacles_manager) - # node manager - node_parser = subparsers.add_parser("node", help='Start OctoBot in node mode.\n' - 'Use "node --help" to get the ' - 'node manager help.') - _register_node_arguments(node_parser) - node_parser.set_defaults(func=lambda args: start_node(args, default_config_file)) - def _register_node_arguments(parser): parser.add_argument( '--host', @@ -598,19 +757,6 @@ def _register_node_arguments(parser): ) -def start_node(args, default_config_file=None): - import octobot_node.config - - constants.FORCED_DISTRIBUTION = enums.OctoBotDistribution.NODE.value - if args.master: - octobot_node.config.settings.IS_MASTER_MODE = True - octobot_node.config.settings.CONSUMER_ONLY = args.consumer_only - args.no_web = True - start_octobot(args, default_config_file) - - - - def start_background_octobot_with_args( version=False, update=False, @@ -632,6 +778,11 @@ def start_background_octobot_with_args( default_config_file=None, user_folder=None, log_folder=None, + standalone=True, + master=False, + consumer_only=False, + host=None, + port=None, ): if backtesting_files is None: backtesting_files = [] @@ -652,7 +803,13 @@ def start_background_octobot_with_args( risk=risk, reset_trading_history=reset_trading_history, user_folder=user_folder, - log_folder=log_folder) + log_folder=log_folder, + standalone=standalone, + master=master, + consumer_only=consumer_only, + host=host, + port=port, + dump_state=None) if in_subprocess: bot_process = multiprocessing.Process(target=start_octobot, args=(args, default_config_file)) bot_process.start() @@ -671,9 +828,7 @@ def main(args=None, default_config_file=None): # check compatible tentacle manager try: - from octobot_tentacles_manager import VERSION - - if packaging_version.Version(VERSION) < packaging_version.Version(MIN_TENTACLE_MANAGER_VERSION): + if packaging_version.Version(octobot_tentacles_manager.VERSION) < packaging_version.Version(MIN_TENTACLE_MANAGER_VERSION): print("OctoBot requires OctoBot-Tentacles-Manager in a minimum version of " + MIN_TENTACLE_MANAGER_VERSION + " you can install and update OctoBot-Tentacles-Manager using the following command: " "python3 -m pip install -U OctoBot-Tentacles-Manager", file=sys.stderr) diff --git a/octobot/commands.py b/octobot/commands.py index 25f4261582..64897320f5 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -28,6 +28,7 @@ import octobot_commons.logging as logging import octobot_commons.constants as commons_constants import octobot_commons.errors as commons_errors +import octobot_commons.os_util as os_util import octobot_commons.aiohttp_util as aiohttp_util import octobot_tentacles_manager.api as tentacles_manager_api @@ -333,7 +334,7 @@ async def start_bot(bot, logger, catch=False): def stop_bot(bot, force=False): - bot.task_manager.stop_tasks() + bot.task_manager.stop_tasks(stop_managed_child_processes=True, force=force) if force: os._exit(0) @@ -343,7 +344,7 @@ def get_bot_file(): def restart_bot(): - argv = (f'{a}' for a in sys.argv if a not in IGNORED_COMMAND_WHEN_RESTART) + argv = [f'{argument}' for argument in sys.argv if argument not in IGNORED_COMMAND_WHEN_RESTART] if get_bot_file().endswith(".py"): os.execl(sys.executable, f'{sys.executable}', *argv) elif get_bot_file().endswith(constants.PROJECT_NAME): @@ -353,7 +354,7 @@ def restart_bot(): # restart from binary # from https://pyinstaller.org/en/stable/common-issues-and-pitfalls.html#using-sys-executable-to-spawn-subprocesses-that-outlive-the-application-process-implementing-application-restart # Restart the application - subprocess.Popen([sys.executable], env={**os.environ, "PYINSTALLER_RESET_ENVIRONMENT": "1"}) + subprocess.Popen(argv, env={**os.environ, **os_util.PYINSTALLER_RESET_ENVIRONMENT_VARS}) # force stop and restart os._exit(0) diff --git a/octobot/community/__init__.py b/octobot/community/__init__.py index 28b78b76c2..63be4d8210 100644 --- a/octobot/community/__init__.py +++ b/octobot/community/__init__.py @@ -54,7 +54,6 @@ ) from octobot.community import local_authenticator from octobot.community import community_analysis -from octobot.community import community_manager from octobot.community import authentication from octobot.community import graphql_requests from octobot.community import feeds @@ -65,8 +64,8 @@ get_current_octobots_stats, can_read_metrics, ) -from octobot.community.community_manager import ( - CommunityManager, +from octobot.community.activity_analysis.activity_metrics import ( + ActivityMetrics, ) from octobot.community.authentication import ( CommunityAuthentication, @@ -124,7 +123,7 @@ "get_community_metrics", "get_current_octobots_stats", "can_read_metrics", - "CommunityManager", + "ActivityMetrics", "CommunityAuthentication", "CommunityTentaclesPackage", "CommunitySupports", diff --git a/octobot/community/activity_analysis/__init__.py b/octobot/community/activity_analysis/__init__.py new file mode 100644 index 0000000000..0b90ee9a4f --- /dev/null +++ b/octobot/community/activity_analysis/__init__.py @@ -0,0 +1,39 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +from octobot.community.activity_analysis.activity_metrics import ActivityMetrics +from octobot.community.activity_analysis.bot_id_resolver import ( + BotIdResolution, + ensure_activity_bot_id, +) +from octobot.community.activity_analysis.config_path_binding import ( + PathBoundValueResolution, + ensure_config_path_fingerprint, + fingerprint_config_path, + get_bound_config_path, + path_binding_is_stale, +) + +__all__ = [ + "ActivityMetrics", + "BotIdResolution", + "ensure_activity_bot_id", + "PathBoundValueResolution", + "ensure_config_path_fingerprint", + "fingerprint_config_path", + "get_bound_config_path", + "path_binding_is_stale", +] diff --git a/octobot/community/activity_analysis/activity_metrics.py b/octobot/community/activity_analysis/activity_metrics.py new file mode 100644 index 0000000000..901bb93cfb --- /dev/null +++ b/octobot/community/activity_analysis/activity_metrics.py @@ -0,0 +1,113 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import asyncio + +import octobot_commons.logging as logging +import octobot_commons.configuration as configuration +import octobot_commons.authentication as authentication +import octobot_commons.constants as common_constants + +import octobot_trading.api as trading_api + +import octobot.community.activity_analysis.bot_id_resolver as bot_id_resolver +import octobot.community.errors_upload.sentry_tracker as tracker +import octobot.constants as constants +import octobot.enums as enums + + +class ActivityMetrics: + + def __init__(self, octobot_api): + self.octobot_api = octobot_api + self.edited_config: configuration.Configuration = octobot_api.get_edited_config(dict_only=False) + self.enabled = constants.IS_CLOUD_ENV or self.edited_config.get_metrics_enabled() + self.logger = logging.get_logger(self.__class__.__name__) + self.keep_running = True + + @staticmethod + def initialize_tracker(config: configuration.Configuration) -> None: + tracker.init_sentry_tracker(metrics_enabled=config.get_metrics_enabled()) + + @staticmethod + def clear_activity_bot_id(config: configuration.Configuration) -> None: + metrics_section = config.config.setdefault(common_constants.CONFIG_METRICS, {}) + if isinstance(metrics_section, dict): + metrics_section[common_constants.CONFIG_METRICS_ACTIVITY_BOT_ID] = "" + + def setup_activity_tracking(self, distribution: enums.OctoBotDistribution) -> None: + if not self.enabled: + return + resolution = bot_id_resolver.ensure_activity_bot_id(self.edited_config) + if tracker.activity_tracking_is_active(): + tracker.update_tracker_bot_id(resolution.bot_id) + if distribution is enums.OctoBotDistribution.NODE and resolution.was_created: + tracker.track_usage_event( + "node_first_start", + distribution="node", + version=constants.LONG_VERSION, + ) + + @staticmethod + def report_child_octobot_first_start() -> None: + if not tracker.has_tracker_bot_id(): + return + tracker.track_usage_event("child_octobot_first_start") + + async def start_community_task(self): + if not self.enabled: + return + try: + while self.keep_running: + await asyncio.sleep(common_constants.TIMER_BETWEEN_METRICS_UPTIME_UPDATE) + try: + await self._update_authenticated_bot() + except Exception as err: + self.logger.debug(f"Exception when handling community data : {err}") + except asyncio.CancelledError: + pass + except Exception as err: + self.logger.debug(f"Exception when handling community registration: {err}") + + async def stop_task(self): + self.logger.debug("Stopping ...") + self.keep_running = False + self.logger.debug("Stopped ...") + + async def _update_authenticated_bot(self): + try: + if authentication.Authenticator.instance().is_logged_in(): + await authentication.Authenticator.instance().update_bot_config_and_stats( + self._get_profitability() + ) + except Exception as err: + self.logger.debug(f"Exception when pushing config and stats : {err}") + + def _get_profitability(self): + total_origin_values = 0 + total_profitability = 0 + + for exchange_manager in self._get_exchange_managers(): + if trading_api.is_exchange_trading(exchange_manager): + profitability, _, _, _, _ = trading_api.get_profitability_stats(exchange_manager) + total_profitability += float(profitability) + total_origin_values += float(trading_api.get_origin_portfolio_value(exchange_manager)) + + return (total_profitability * 100 / total_origin_values) if total_origin_values > 0 else 0 + + def _get_exchange_managers(self): + return trading_api.get_exchange_managers_from_exchange_ids( + self.octobot_api.get_exchange_manager_ids() + ) diff --git a/octobot/community/activity_analysis/bot_id_resolver.py b/octobot/community/activity_analysis/bot_id_resolver.py new file mode 100644 index 0000000000..ec3387e489 --- /dev/null +++ b/octobot/community/activity_analysis/bot_id_resolver.py @@ -0,0 +1,50 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import dataclasses +import uuid + +import octobot_commons.configuration as configuration +import octobot_commons.constants as commons_constants + + +@dataclasses.dataclass(frozen=True) +class BotIdResolution: + bot_id: str + was_created: bool + + +def _get_metrics_section(config: configuration.Configuration) -> dict: + metrics_section = config.config.setdefault(commons_constants.CONFIG_METRICS, {}) + if not isinstance(metrics_section, dict): + raise ValueError(f"{commons_constants.CONFIG_METRICS} must be a mapping in config") + return metrics_section + + +def ensure_activity_bot_id(config: configuration.Configuration) -> BotIdResolution: + metrics_section = _get_metrics_section(config) + stored_bot_id = metrics_section.get(commons_constants.CONFIG_METRICS_ACTIVITY_BOT_ID) + if stored_bot_id: + return BotIdResolution( + bot_id=str(stored_bot_id), + was_created=False, + ) + new_bot_id = str(uuid.uuid4()) + metrics_section[commons_constants.CONFIG_METRICS_ACTIVITY_BOT_ID] = new_bot_id + config.save() + return BotIdResolution( + bot_id=new_bot_id, + was_created=True, + ) diff --git a/octobot/community/activity_analysis/config_path_binding.py b/octobot/community/activity_analysis/config_path_binding.py new file mode 100644 index 0000000000..e97ebf3198 --- /dev/null +++ b/octobot/community/activity_analysis/config_path_binding.py @@ -0,0 +1,76 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import dataclasses +import hashlib +import os + +import octobot_commons.configuration as configuration + +import octobot.constants as constants + + +@dataclasses.dataclass(frozen=True) +class PathBoundValueResolution: + value: str + bound_path: str + was_regenerated: bool + + +def get_bound_config_path(config_path: str) -> str: + return os.path.normpath(os.path.abspath(config_path)) + + +def path_binding_is_stale(stored_bound_path: str | None, config_path: str) -> bool: + if not stored_bound_path: + return True + return stored_bound_path != get_bound_config_path(config_path) + + +def fingerprint_config_path(config_path: str) -> str: + bound_path = get_bound_config_path(config_path) + return hashlib.sha256(bound_path.encode()).hexdigest() + + +def _get_community_section(config: configuration.Configuration) -> dict: + community_section = config.config.setdefault(constants.CONFIG_COMMUNITY, {}) + if not isinstance(community_section, dict): + raise ValueError(f"{constants.CONFIG_COMMUNITY} must be a mapping in config") + return community_section + + +def ensure_config_path_fingerprint( + config: configuration.Configuration, + *, + persist: bool = True, +) -> PathBoundValueResolution: + bound_path = get_bound_config_path(config.config_path) + current_fingerprint = fingerprint_config_path(config.config_path) + community_section = _get_community_section(config) + stored_fingerprint = community_section.get(constants.CONFIG_COMMUNITY_LOCAL_DATA_IDENTIFIER) + if stored_fingerprint == current_fingerprint: + return PathBoundValueResolution( + value=current_fingerprint, + bound_path=bound_path, + was_regenerated=False, + ) + community_section[constants.CONFIG_COMMUNITY_LOCAL_DATA_IDENTIFIER] = current_fingerprint + if persist: + config.save() + return PathBoundValueResolution( + value=current_fingerprint, + bound_path=bound_path, + was_regenerated=True, + ) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index db5a35049a..ac4bf8cd15 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -16,11 +16,12 @@ import asyncio import contextlib import json +import os import time import threading import typing -import hashlib -import os +import octobot.community.activity_analysis.config_path_binding as config_path_binding +import octobot.community.activity_analysis.activity_metrics as activity_metrics import decimal import octobot.constants as constants @@ -44,6 +45,7 @@ import octobot_commons.authentication as authentication import octobot_commons.configuration as commons_configuration import octobot_commons.profiles as commons_profiles +import octobot_commons.user_root_folder_provider as user_root_folder_provider import octobot_trading.enums as trading_enums import octobot_sync.client as sync_client import octobot_sync.chain as sync_chain @@ -129,10 +131,10 @@ def __init__(self, config=None, backend_url=None, backend_key=None, use_as_singl self._fetch_account_task: typing.Optional[asyncio.Task] = None self._sync_client = None - self._sync_user_id: str = "" + self.sync_user_id: str = "" self._sync_client_lock = threading.Lock() self._wallet_backend: wallet_backend.WalletBackend = wallet_backend.WalletBackend( - self.configuration_storage.sync_storage, self.logger + self._get_wallet_sync_storage(), self.logger ) @staticmethod @@ -144,6 +146,25 @@ def create(configuration: commons_configuration.Configuration, **kwargs): def update(self, configuration: commons_configuration.Configuration): self.configuration_storage.set_configuration(configuration) + self._wallet_backend = wallet_backend.WalletBackend( + self._get_wallet_sync_storage(), self.logger + ) + + def _get_wallet_sync_storage(self): + sync_data_root = os.path.normpath(user_root_folder_provider.get_sync_data_root()) + user_root = os.path.normpath(user_root_folder_provider.get_user_root_folder()) + if sync_data_root != user_root: + # This is a child octobot process using the master sync data root, + # configure storage accordingly + master_config_path = os.path.join(sync_data_root, commons_constants.CONFIG_FILE) + master_config = commons_configuration.Configuration( + master_config_path, + os.path.join(sync_data_root, commons_constants.PROFILES_FOLDER), + ) + if os.path.isfile(master_config_path): + master_config.read(should_raise=False, activate_profile=False) + return supabase_backend.SyncConfigurationStorage(master_config) + return self.configuration_storage.sync_storage def get_logged_in_email(self): if self.user_account.has_user_data(): @@ -557,6 +578,7 @@ def _clear_bot_scoped_config(self): "and webhook url will be different on this bot." ) self._save_bot_id("") + activity_metrics.ActivityMetrics.clear_activity_bot_id(self.config) self.save_tradingview_email("") # also reset mqtt id to force a new mqtt id creation self._save_mqtt_device_uuid("") @@ -567,20 +589,11 @@ def clear_local_data_if_necessary(self): if constants.IS_CLOUD_ENV: # disabled on cloud environments return - previous_local_identifier = self._get_saved_bot_scoped_data_identifier() - current_local_identifier = self._get_bot_scoped_data_identifier() - if not previous_local_identifier: - self._save_bot_scoped_data_identifier(current_local_identifier) - # nothing to clear - return - if current_local_identifier != previous_local_identifier: + community_section = self.config.config.setdefault(constants.CONFIG_COMMUNITY, {}) + previous_local_identifier = community_section.get(constants.CONFIG_COMMUNITY_LOCAL_DATA_IDENTIFIER) + resolution = config_path_binding.ensure_config_path_fingerprint(self.config) + if resolution.was_regenerated and previous_local_identifier is not None: self._clear_bot_scoped_config() - self._save_bot_scoped_data_identifier(current_local_identifier) - - def _get_bot_scoped_data_identifier(self) -> str: - # identifier is based on the path to the local bot to ensure the same data are not re-used - # when copy/pasting a bot config to another bot - return hashlib.sha256(os.getcwd().encode()).hexdigest() async def stop(self): self.logger.debug("Stopping ...") @@ -712,7 +725,7 @@ def init_sync_client_for_wallet(self, address: str) -> None: self.logger.debug("No sync server URL configured, skipping sync client init") return wallet = self.get_wallet(address) - self._sync_client, self._sync_user_id = sync_client.create_sync_client( + self._sync_client, self.sync_user_id = sync_client.create_sync_client( private_key=wallet.private_key, sync_url=sync_url, ) @@ -745,7 +758,7 @@ def auto_init_sync_client(self) -> bool: "No sync server URL configured, skipping auto sync client init" ) return False - self._sync_client, self._sync_user_id = sync_client.create_sync_client( + self._sync_client, self.sync_user_id = sync_client.create_sync_client( private_key=wallet.private_key, sync_url=sync_url, ) diff --git a/octobot/community/community_manager.py b/octobot/community/community_manager.py deleted file mode 100644 index b4eeaa2a47..0000000000 --- a/octobot/community/community_manager.py +++ /dev/null @@ -1,326 +0,0 @@ -# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) -# Copyright (c) 2025 Drakkar-Software, All rights reserved. -# -# OctoBot is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# OctoBot is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with OctoBot. If not, see . -import time -import asyncio -import json -import requests -import threading - -import octobot_commons.logging as logging -import octobot_commons.configuration as configuration -import octobot_commons.os_util as os_util -import octobot_commons.symbols.symbol_util as symbol_util -import octobot_commons.authentication as authentication - -import octobot_commons.constants as common_constants - -import octobot_evaluators.api as evaluator_api -import octobot_evaluators.enums as evaluator_enums - -import octobot_services.constants as service_constants - -import octobot_trading.api as trading_api - -import octobot.community.models.community_fields as community_fields -import octobot.constants as constants - - -class CommunityManager: - _headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} - - def __init__(self, octobot_api): - self.octobot_api = octobot_api - self.edited_config: configuration.Configuration = octobot_api.get_edited_config(dict_only=False) - self.enabled = constants.IS_CLOUD_ENV or self.edited_config.get_metrics_enabled() - self.reference_market = trading_api.get_reference_market(self.edited_config.config) - self.logger = logging.get_logger(self.__class__.__name__) - self.current_config = None - self.keep_running = True - self.session = octobot_api.get_aiohttp_session() - try: - self.bot_id = self.edited_config.get_metrics_id() - except KeyError: - self.bot_id = None - - # these attributes will be set at the last moment to ensure relevance and let time for everything to startup - self.has_real_trader = None - self.has_simulator = None - - def _init_community_config(self): - self.has_real_trader = trading_api.is_trader_enabled_in_config(self.edited_config.config) - self.has_simulator = trading_api.is_trader_simulator_enabled_in_config(self.edited_config.config) - - async def start_community_task(self): - if self.enabled: - try: - # first ensure this session is not just a configuration test: register after a timer - await asyncio.sleep( - constants.CLOUD_FIRST_METRICS_UPDATE_TIME - if constants.IS_CLOUD_ENV else common_constants.TIMER_BEFORE_METRICS_REGISTRATION_SECONDS - ) - self._init_community_config() - # await self.register_session() # waiting for metrics migration - await self._update_authenticated_bot() - while self.keep_running: - # send a keepalive at periodic intervals - await asyncio.sleep(common_constants.TIMER_BETWEEN_METRICS_UPTIME_UPDATE) - try: - # await self._update_session() # waiting for metrics migration - await self._update_authenticated_bot() - except Exception as e: - self.logger.debug(f"Exception when handling community data : {e}") - except asyncio.CancelledError: - pass - except Exception as e: - self.logger.debug(f"Exception when handling community registration: {e}") - - async def stop_task(self): - self.logger.debug("Stopping ...") - self.keep_running = False - await self.session.close() - self.logger.debug("Stopped ...") - - @staticmethod - def should_register_bot(config: configuration.Configuration): - try: - config.get_metrics_id() - return True - except KeyError: - return False - - @staticmethod - def background_get_id_and_register_bot(octobot_api): - community_manager = CommunityManager(octobot_api) - threading.Thread(target=community_manager._blocking_get_id_and_register, name="CommunityManagerGetId").start() - - def _blocking_get_id_and_register(self): - try: - resp = requests.get(f"{common_constants.METRICS_URL}{common_constants.METRICS_ROUTE_GEN_BOT_ID}", - headers=self._headers) - text = resp.text - if resp.status_code != 200: - self.logger.debug(f"Impossible to get bot id: status code: {resp.status_code}, text: {text}") - else: - self.bot_id = json.loads(text) - self._save_bot_id() - community = self._get_bot_community() - requests.post(f"{common_constants.METRICS_URL}{common_constants.METRICS_ROUTE_REGISTER}", - json=community, headers=self._headers) - except Exception as e: - self.logger.debug(f"Error when handling community: {e}") - - async def register_session(self, retry_on_error=True): - self.current_config = await self._get_current_community_config() - await self._post_community_data(common_constants.METRICS_ROUTE_REGISTER, self.current_config, retry_on_error) - - async def _update_session(self, retry_on_error=True): - self.current_config[community_fields.CommunityFields.CURRENT_SESSION.value][ - community_fields.CommunityFields.UP_TIME.value] = int(time.time() - self.octobot_api.get_start_time()) - self.current_config[community_fields.CommunityFields.CURRENT_SESSION.value][ - community_fields.CommunityFields.PROFITABILITY.value] = self._get_profitability() - self.current_config[community_fields.CommunityFields.CURRENT_SESSION.value][ - community_fields.CommunityFields.TRADED_VOLUMES.value] = self._get_traded_volumes() - await self._post_community_data(common_constants.METRICS_ROUTE_UPTIME, self.current_config, retry_on_error) - - async def _update_authenticated_bot(self): - try: - if authentication.Authenticator.instance().is_logged_in(): - await authentication.Authenticator.instance().update_bot_config_and_stats( - self._get_profitability() - ) - except Exception as err: - self.logger.debug(f"Exception when pushing config and stats : {err}") - - async def _get_current_community_config(self): - if not self.bot_id: - await self._init_bot_id() - if self.bot_id: - return self._get_bot_community() - - def _get_bot_community(self): - return { - community_fields.CommunityFields.ID.value: self.bot_id, - community_fields.CommunityFields.CURRENT_SESSION.value: { - community_fields.CommunityFields.STARTED_AT.value: int(self.octobot_api.get_start_time()), - community_fields.CommunityFields.UP_TIME.value: int(time.time() - self.octobot_api.get_start_time()), - community_fields.CommunityFields.VERSION.value: constants.LONG_VERSION, - community_fields.CommunityFields.SIMULATOR.value: self.has_simulator, - community_fields.CommunityFields.TRADER.value: self.has_real_trader, - community_fields.CommunityFields.EVAL_CONFIG.value: self._get_eval_config(), - community_fields.CommunityFields.PAIRS.value: self._get_traded_pairs(), - community_fields.CommunityFields.EXCHANGES.value: list(trading_api.get_exchange_names()), - community_fields.CommunityFields.EXCHANGE_TYPES.value: self._get_exchange_types(), - community_fields.CommunityFields.NOTIFICATIONS.value: self._get_notification_types(), - community_fields.CommunityFields.TYPE.value: os_util.get_octobot_type(), - community_fields.CommunityFields.PLATFORM.value: os_util.get_current_platform(), - community_fields.CommunityFields.REFERENCE_MARKET.value: self.reference_market, - community_fields.CommunityFields.PORTFOLIO_VALUE.value: self._get_real_portfolio_value(), - community_fields.CommunityFields.PROFITABILITY.value: self._get_profitability(), - community_fields.CommunityFields.TRADED_VOLUMES.value: self._get_traded_volumes(), - community_fields.CommunityFields.SUPPORTS.value: self._get_supports(), - community_fields.CommunityFields.SIGNAL_EMITTER.value: - authentication.Authenticator.instance().get_is_signal_emitter(), - community_fields.CommunityFields.SIGNAL_RECEIVER.value: - authentication.Authenticator.instance().get_is_signal_receiver(), - community_fields.CommunityFields.PROFILE_NAME.value: self.edited_config.profile.name, - community_fields.CommunityFields.PROFILE_ID.value: self.edited_config.profile.profile_id, - community_fields.CommunityFields.PROFILE_IMPORTED.value: self.edited_config.profile.imported, - } - } - - def _get_exchange_types(self): - return [ - trading_api.get_exchange_type(exchange_manager).value - for exchange_manager in self._get_exchange_managers() - ] - - def _get_profitability(self): - total_origin_values = 0 - total_profitability = 0 - - for exchange_manager in self._get_exchange_managers(): - if trading_api.is_exchange_trading(exchange_manager): - profitability, _, _, _, _ = trading_api.get_profitability_stats(exchange_manager) - total_profitability += float(profitability) - total_origin_values += float(trading_api.get_origin_portfolio_value(exchange_manager)) - - return (total_profitability * 100 / total_origin_values) if total_origin_values > 0 else 0 - - def _get_traded_volumes(self): - volume_by_currency = {} - if self.has_real_trader: - trades = [] - for exchange_manager in self._get_exchange_managers(): - trades += trading_api.get_trade_history(exchange_manager, since=self.octobot_api.get_start_time()) - for trade in trades: - # cost is in quote currency for a traded pair - currency = symbol_util.parse_symbol(trade.symbol).quote - if currency in volume_by_currency: - volume_by_currency[currency] += float(trade.total_cost) - else: - volume_by_currency[currency] = float(trade.total_cost) - return volume_by_currency - - def _get_supports(self): - supporting_exchanges = [] - for exchange_manager in self._get_exchange_managers(): - exchange_name = trading_api.get_exchange_name(exchange_manager) - if self.has_real_trader \ - and trading_api.is_sponsoring(exchange_name) \ - and trading_api.is_broker_enabled(exchange_manager): - supporting_exchanges.append(exchange_name) - supports = authentication.Authenticator.instance().user_account.supports - return { - community_fields.CommunityFields.EXCHANGES.value: supporting_exchanges, - community_fields.CommunityFields.ROLES.value: [supports.support_role], - community_fields.CommunityFields.DONATIONS.value: [str(donation) for donation in supports.donations] - } - - def _get_real_portfolio_value(self): - if self.has_real_trader: - total_value = 0 - for exchange_manager in self._get_exchange_managers(): - current_value = trading_api.get_current_portfolio_value(exchange_manager) - # current_value might be 0 if no trades have been made / canceled => use origin value - if current_value == 0: - current_value = trading_api.get_origin_portfolio_value(exchange_manager) - total_value += current_value - return float(total_value) - else: - return 0 - - def _get_traded_pairs(self): - pairs = set() - for exchange_manager in self._get_exchange_managers(): - pairs = pairs.union(trading_api.get_trading_pairs(exchange_manager)) - return list(pairs) - - def _get_notification_types(self): - has_notifications = service_constants.CONFIG_CATEGORY_NOTIFICATION in self.edited_config.config \ - and service_constants.CONFIG_NOTIFICATION_TYPE in self.edited_config.config[ - service_constants.CONFIG_CATEGORY_NOTIFICATION] - return self.edited_config.config[service_constants.CONFIG_CATEGORY_NOTIFICATION][ - service_constants.CONFIG_NOTIFICATION_TYPE] if has_notifications else [] - - def _get_eval_config(self): - tentacle_setup_config = self.octobot_api.get_tentacles_setup_config() - # trading mode - config_eval = [] - if (trading_mode := self.octobot_api.get_trading_mode()) is not None: - config_eval.append(trading_mode.get_name()) - - # strategies - for strategy in evaluator_api.get_evaluator_classes_from_type( - evaluator_enums.EvaluatorMatrixTypes.STRATEGIES.value, - tentacle_setup_config): - config_eval.append(strategy.get_name()) - - # evaluators - evaluators = evaluator_api.get_evaluator_classes_from_type(evaluator_enums.EvaluatorMatrixTypes.TA.value, - tentacle_setup_config) - evaluators += evaluator_api.get_evaluator_classes_from_type(evaluator_enums.EvaluatorMatrixTypes.SOCIAL.value, - tentacle_setup_config) - evaluators += evaluator_api.get_evaluator_classes_from_type( - evaluator_enums.EvaluatorMatrixTypes.REAL_TIME.value, - tentacle_setup_config) - for evaluator in evaluators: - config_eval.append(evaluator.get_name()) - return config_eval - - async def _init_bot_id(self): - try: - async with self.session.get(f"{common_constants.METRICS_URL}{common_constants.METRICS_ROUTE_GEN_BOT_ID}", - headers=self._headers) as resp: - text = await resp.text() - if resp.status != 200: - self.logger.debug(f"Impossible to get bot id: status code: {resp.status}, text: {text}") - else: - self.bot_id = json.loads(text) - self._save_bot_id() - except Exception as e: - self.logger.debug(f"Error when handling community data : {e}") - - def _save_bot_id(self): - if common_constants.CONFIG_METRICS not in self.edited_config.config \ - or not self.edited_config.config[common_constants.CONFIG_METRICS]: - self.edited_config.config[common_constants.CONFIG_METRICS] = {common_constants.CONFIG_ENABLED_OPTION: True} - self.edited_config.config[common_constants.CONFIG_METRICS][common_constants.CONFIG_METRICS_BOT_ID] = self.bot_id - self.edited_config.save() - - async def _post_community_data(self, route, bot, retry_on_error): - try: - async with self.session.post(f"{common_constants.METRICS_URL}{route}", json=bot, - headers=self._headers) as resp: - await self._handle_post_error(resp, retry_on_error) - except Exception as e: - self.logger.debug(f"Error when handling community data : {e}") - - async def _handle_post_error(self, resp, retry_on_error): - if resp.status != 200: - if resp.status == 404: - # did not found bot with id in config: generate new id and register new bot - if retry_on_error: - await self._init_bot_id() - await self.register_session(retry_on_error=False) - else: - self.logger.debug(f"Impossible to send community data : " - f"status code: {resp.status}, " - f"text: {await resp.text()}") - - def _get_exchange_managers(self): - return trading_api.get_exchange_managers_from_exchange_ids( - self.octobot_api.get_exchange_manager_ids() - ) diff --git a/octobot/community/errors_upload/__init__.py b/octobot/community/errors_upload/__init__.py index 84551bf1f3..6af2d0b4a7 100644 --- a/octobot/community/errors_upload/__init__.py +++ b/octobot/community/errors_upload/__init__.py @@ -18,6 +18,8 @@ from octobot.community.errors_upload.sentry_tracker import ( init_sentry_tracker, flush_tracker, + update_tracker_bot_id, + track_usage_event, ) __all__ = [ diff --git a/octobot/community/errors_upload/sentry_tracker.py b/octobot/community/errors_upload/sentry_tracker.py index 056d18a930..1e015ad107 100644 --- a/octobot/community/errors_upload/sentry_tracker.py +++ b/octobot/community/errors_upload/sentry_tracker.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import time +import typing + import sentry_sdk import octobot_commons.constants @@ -22,71 +24,128 @@ import octobot.constants -def init_sentry_tracker(): +_sentry_initialized = False +_activity_tracking_active = False +_tracker_bot_id_set: bool = False + + +def init_sentry_tracker(metrics_enabled: bool) -> None: """ - Will upload errors to octobot.constants.ERROR_TRACKER_DSN if its value is set + Initialize Sentry when ACTIVITY_TRACKER_DSN (metrics enabled) or ERROR_TRACKER_DSN is set. + Tracker DSN takes priority when metrics are enabled. """ + global _activity_tracking_active, _sentry_initialized logger = octobot_commons.logging.get_logger("sentry_tracker") - if not octobot.constants.ERROR_TRACKER_DSN: - logger.debug(f"Error tracker disabled") + + activity_dsn = octobot.constants.ACTIVITY_TRACKER_DSN + error_dsn = octobot.constants.ERROR_TRACKER_DSN + use_activity_dsn = bool(activity_dsn and metrics_enabled) + active_dsn = activity_dsn if use_activity_dsn else error_dsn + + if not active_dsn: + logger.debug("Sentry tracker disabled: no applicable DSN") + _activity_tracking_active = False + _sentry_initialized = False + return + + if _sentry_initialized: + _activity_tracking_active = use_activity_dsn return + environment = "cloud" if octobot.constants.IS_CLOUD_ENV else "self hosted" app_name = f"{octobot.constants.PROJECT_NAME} open source" - sentry_sdk.init( - dsn=octobot.constants.ERROR_TRACKER_DSN, - + init_kwargs: dict[str, typing.Any] = { + "dsn": active_dsn, # Percent of error events to send to the server. # 0.5 would be 50%. Defaults to 1.0. - sample_rate=1, - + "sample_rate": 1, # Set traces_sample_rate to 1.0 to capture 100% # of transactions for performance monitoring. # We recommend adjusting this value in production. - traces_sample_rate=0, - + "traces_sample_rate": 0, # By default the SDK will try to use the SENTRY_RELEASE # environment variable, or infer a git commit # SHA as release, however you may want to set # something more human-readable. - release=octobot.constants.LONG_VERSION, - - include_local_variables=False, # careful not to upload sensitive data - max_breadcrumbs=5, # breadcrumbs from multiple tasks and activities are mixed, only take the last ones - - environment=environment, - before_send=_before_send, - ) + "release": octobot.constants.LONG_VERSION, + "include_local_variables": False, # careful not to upload sensitive data + # breadcrumbs from multiple tasks and activities are mixed, only take the last ones + "max_breadcrumbs": 5, + "environment": environment, + } + if use_activity_dsn: + init_kwargs["sample_rate"] = 0 + # Activity-only: no default integrations (especially LoggingIntegration). + # Only explicit track_usage_event calls send data. + init_kwargs["default_integrations"] = False + init_kwargs["integrations"] = [] + else: + # Error DSN: auto-capture log errors; strip bot log prefix before upload. + init_kwargs["before_send"] = _before_send + + sentry_sdk.init(**init_kwargs) + _activity_tracking_active = use_activity_dsn + _sentry_initialized = True + + mode = "activity" if use_activity_dsn else "error" logger.info( - f"Initialized error tracking with environment: {environment}, " - f"release: {octobot.constants.LONG_VERSION}, dns: {octobot.constants.ERROR_TRACKER_DSN}" + f"Initialized {mode} tracking with environment: {environment}, " + f"release: {octobot.constants.LONG_VERSION}, dsn: {active_dsn}" ) sentry_sdk.set_context("app", { "app_start_time": str(round(time.time())), "app_name": app_name, }) sentry_sdk.set_tag("app", app_name) - if octobot.constants.COMMUNITY_BOT_ID: - sentry_sdk.set_tag("bot_id", octobot.constants.COMMUNITY_BOT_ID) -def flush_tracker(): - if octobot.constants.ERROR_TRACKER_DSN: - delay = 2 - octobot_commons.logging.get_logger("sentry_tracker").info(f"Flushing trackers: shutting down in {delay} seconds ...") - sentry_sdk.flush() - # let trackers upload errors - time.sleep(delay) +def flush_tracker() -> None: + if not _sentry_initialized: + return + delay = 2 + octobot_commons.logging.get_logger("sentry_tracker").info( + f"Flushing trackers: shutting down in {delay} seconds ..." + ) + sentry_sdk.flush() + time.sleep(delay) + + +def activity_tracking_is_active() -> bool: + return _activity_tracking_active + + +def has_tracker_bot_id() -> bool: + return _tracker_bot_id_set + + +def update_tracker_bot_id(bot_id: str) -> None: + global _tracker_bot_id_set + _tracker_bot_id_set = True + sentry_sdk.set_user({"id": bot_id}) + sentry_sdk.set_tag("bot_id", bot_id) + + +def track_usage_event(event_name: str, **attributes: typing.Any) -> None: + metric_attributes = {"event": event_name} + for attribute_key, attribute_value in attributes.items(): + if attribute_value is not None: + metric_attributes[attribute_key] = str(attribute_value) + sentry_sdk.metrics.count("octobot.usage", 1, attributes=metric_attributes) + octobot_commons.logging.get_logger("sentry_tracker").debug( + "Tracked usage event %s with attributes %s", + event_name, + metric_attributes, + ) def _get_log_prefix() -> str: return f"[{octobot.constants.COMMUNITY_BOT_ID}] " if octobot.constants.COMMUNITY_BOT_ID else "[self-hosted]" + def _before_send(event: dict, hint: dict): if event.get("extra", {}).get(octobot_commons.constants.IS_EXCEPTION_DESC, False): - # error already sent with exception return - # do not include log_prefix in log entry message try: message = event["logentry"]["message"] log_prefix = _get_log_prefix() diff --git a/octobot/config/config_schema.json b/octobot/config/config_schema.json index 9bc11700d8..a4aefdff21 100644 --- a/octobot/config/config_schema.json +++ b/octobot/config/config_schema.json @@ -13,6 +13,9 @@ "accepted_terms": { "type": "boolean" }, + "octobot_name": { + "type": "string" + }, "distribution": { "type": "string" }, @@ -33,6 +36,9 @@ "bot_id": { "type": "string" }, + "local_data_identifier": { + "type": "string" + }, "supabase.auth.token": { "type": "string" }, @@ -74,7 +80,7 @@ "enabled": { "type": "boolean" }, - "metrics-bot-id": { + "activity_bot_id": { "type": "string" } } diff --git a/octobot/configuration_manager.py b/octobot/configuration_manager.py index 8b2e9d9dd9..087b3878de 100644 --- a/octobot/configuration_manager.py +++ b/octobot/configuration_manager.py @@ -115,7 +115,7 @@ def config_health_check(config: configuration.Configuration, in_backtesting: boo def init_config( - config_file=configuration.get_user_config(), + config_file=None, from_config_file=constants.DEFAULT_CONFIG_FILE ): """ @@ -123,6 +123,8 @@ def init_config( :param config_file: the config file path :param from_config_file: the default config file path """ + if config_file is None: + config_file = configuration.get_user_config() try: user_root = user_root_folder_provider.get_user_root_folder() if not os.path.exists(user_root): diff --git a/octobot/constants.py b/octobot/constants.py index 5d20b2b858..916bfa2c08 100644 --- a/octobot/constants.py +++ b/octobot/constants.py @@ -115,7 +115,8 @@ SYNC_NAMESPACE=os.getenv("SYNC_NAMESPACE", "dk") SYNC_CHAIN_ID = os.getenv("SYNC_CHAIN_ID", "evm:8453") -ERROR_TRACKER_DSN = os.getenv("ERROR_TRACKER_DSN") +ERROR_TRACKER_DSN = os.getenv("ERROR_TRACKER_DSN") # has priority over ACTIVITY_TRACKER_DSN if set +ACTIVITY_TRACKER_DSN = os.getenv("ACTIVITY_TRACKER_DSN", "https://c2d3481420c3d1d5f3017c5508e7802f@o4508733888331776.ingest.de.sentry.io/4508733892132944") CONFIG_COMMUNITY = "community" CONFIG_COMMUNITY_BOT_ID = "bot_id" @@ -215,6 +216,9 @@ # Process bot state JSON next to user config (--dump-state); liveness for run_octobot_process PROCESS_BOT_STATE_FILE_NAME = "process_bot_state.json" ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = "OCTOBOT_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS" +ENV_PROCESS_BOT_SYNC_USER_ID = "OCTOBOT_PROCESS_BOT_SYNC_USER_ID" +PROCESS_BOT_SYNC_USER_ID = os.environ.get(ENV_PROCESS_BOT_SYNC_USER_ID, "").strip() + PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS = float( os.getenv(ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "30") ) @@ -223,6 +227,7 @@ # distribution FORCED_DISTRIBUTION = os.getenv("DISTRIBUTION") +FORCE_OCTOBOT_STANDALONE = os_util.parse_boolean_environment_var("FORCE_OCTOBOT_STANDALONE", "False") # system ENABLE_CLOCK_SYNCH = os_util.parse_boolean_environment_var("ENABLE_CLOCK_SYNCH", "True") @@ -292,6 +297,15 @@ ENABLE_RUN_DATABASE_LIMIT = os_util.parse_boolean_environment_var("ENABLE_RUN_DATABASE_LIMIT", "True") MAX_TOTAL_RUN_DATABASES_SIZE = int(os.getenv("MAX_TOTAL_RUN_DATABASES_SIZE", DEFAULT_MAX_TOTAL_RUN_DATABASES_SIZE)) +# Managed child OctoBot processes (run_octobot_process) +MANAGED_CHILD_GRACEFUL_STOP_TIMEOUT_SECONDS = float( + os.getenv("OCTOBOT_MANAGED_CHILD_STOP_TIMEOUT", "10.0") +) + +OCTOBOT_STOP_TIMEOUT_SECONDS = float( + os.getenv("OCTOBOT_STOP_TIMEOUT_SECONDS", "30.0") +) + # Channel OCTOBOT_CHANNEL = "OctoBot" diff --git a/octobot/initializer.py b/octobot/initializer.py index ced0aa1628..ccfcbfab11 100644 --- a/octobot/initializer.py +++ b/octobot/initializer.py @@ -13,7 +13,6 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . -import octobot_tentacles_manager.api as tentacles_manager_api import octobot.constants as constants import octobot_commons.databases as databases import octobot_commons.logging as logging @@ -31,9 +30,8 @@ def __init__(self, octobot): async def create(self, init_bot_storage): # initialize tentacle configuration - tentacles_config_path = self.octobot.get_startup_config(constants.CONFIG_KEY, dict_only=False).\ - get_tentacles_config_path() - self.octobot.tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(tentacles_config_path) + startup_config = self.octobot.get_startup_config(constants.CONFIG_KEY, dict_only=False) + self.octobot.tentacles_setup_config = startup_config.get_active_tentacles_setup_config() if init_bot_storage: try: diff --git a/octobot/limits.py b/octobot/limits.py index 71277dccc3..1c8206fcc1 100644 --- a/octobot/limits.py +++ b/octobot/limits.py @@ -21,7 +21,6 @@ import octobot_commons.enums as common_enums import octobot_commons.logging as logging import octobot_commons.time_frame_manager as time_frame_manager -import octobot_tentacles_manager.api as tentacles_manager_api import octobot_evaluators.api as evaluators_api import octobot_trading.api as trading_api @@ -79,7 +78,7 @@ def _apply_symbols_limits(dict_config, logger, limit): def _apply_time_frames_limits(full_config, logger, limit): - tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(full_config.get_tentacles_config_path()) + tentacles_setup_config = full_config.get_active_tentacles_setup_config() has_disabled_time_frames = False all_enabled_time_frames = [] # patch time frames config diff --git a/octobot/logger.py b/octobot/logger.py index 659c24dfd6..69711db41b 100644 --- a/octobot/logger.py +++ b/octobot/logger.py @@ -75,6 +75,8 @@ def init_logger(logs_folder: str = constants.DEFAULT_LOGS_FOLDER): ) os._exit(-1) + # log once rolling over is done to save in the correct file + logger.info(f"Saving logs in '{os.path.join(os.getcwd(), logs_folder)}' folder") sys.excepthook = _log_uncaught_exceptions return logger @@ -95,7 +97,6 @@ def _load_logger_config(logs_folder: str): shutil.copyfile(constants.LOGGING_CONFIG_FILE, configuration_manager.get_user_local_config_file()) logging.config.fileConfig(configuration_manager.get_user_local_config_file()) logger = logging.getLogger("Logging Configuration") - logger.info(f"Saving logs in '{os.path.join(os.getcwd(), logs_folder)}' folder") if logs_folder != constants.DEFAULT_LOGS_FOLDER: _set_log_folder(os.path.join(os.getcwd(), logs_folder)) if constants.FORCED_LOG_LEVEL: diff --git a/octobot/octobot.py b/octobot/octobot.py index 6fa46382e4..ce2e44ed67 100644 --- a/octobot/octobot.py +++ b/octobot/octobot.py @@ -79,7 +79,7 @@ def __init__(self, config: configuration.Configuration, community_authenticator= self._aiohttp_session = None # community if enabled - self.community_handler = None + self.activity_metrics = None # use edited config in community authentication community_config = self.get_edited_config(constants.CONFIG_KEY, dict_only=False) @@ -243,7 +243,9 @@ async def _start_tools_tasks(self): await self.task_manager.start_tools_tasks() def _init_community(self): - self.community_handler = community.CommunityManager(self.octobot_api) + self.activity_metrics = community.ActivityMetrics(self.octobot_api) + distribution = configuration_manager.get_distribution(self.config) + self.activity_metrics.setup_activity_tracking(distribution) async def _ensure_clock(self): if trading_api.is_trader_enabled_in_config(self.config) and constants.ENABLE_CLOCK_SYNCH: diff --git a/octobot/octobot_api.py b/octobot/octobot_api.py index 7503ba4d9a..493842b9c6 100644 --- a/octobot/octobot_api.py +++ b/octobot/octobot_api.py @@ -91,6 +91,9 @@ def get_aiohttp_session(self) -> object: def get_automation(self) -> automation.Automation: return self._octobot.automation + def get_activity_metrics(self): + return self._octobot.activity_metrics + def get_interface(self, interface_class): for interface in self._octobot.interface_producer.interfaces: if isinstance(interface, interface_class): diff --git a/octobot/storage/process_bot_state_dumper.py b/octobot/storage/process_bot_state_dumper.py index ff51f7a9f1..a2ed8ea752 100644 --- a/octobot/storage/process_bot_state_dumper.py +++ b/octobot/storage/process_bot_state_dumper.py @@ -56,12 +56,12 @@ def _synced_exchange_account_elements_for_first_trading_exchange( elements.sync_from_exchange_manager(first_exchange_manager, []) for skipped_exchange_manager in trading_managers[1:]: _get_logger().error( - "process bot state dump includes only the first trading exchange; dumping %s (%s). " - "Skipping additional trading exchange %s (%s).", - trading_api.get_exchange_name(first_exchange_manager), - trading_api.get_exchange_manager_id(first_exchange_manager), - trading_api.get_exchange_name(skipped_exchange_manager), - trading_api.get_exchange_manager_id(skipped_exchange_manager), + f"process bot state dump includes only the first trading exchange; dumping " + f"{trading_api.get_exchange_name(first_exchange_manager)} " + f"({trading_api.get_exchange_manager_id(first_exchange_manager)}). " + f"Skipping additional trading exchange " + f"{trading_api.get_exchange_name(skipped_exchange_manager)} " + f"({trading_api.get_exchange_manager_id(skipped_exchange_manager)})." ) return elements @@ -82,7 +82,7 @@ async def _write_state_file_async( bot, ), ) - content = state.to_dict(include_default_values=False) + content = json_util.sanitize(state.to_dict(include_default_values=False)) str_content = json_util.dump_formatted_json(content) full_path = os.path.abspath(state_file_path) directory = os.path.dirname(full_path) @@ -113,7 +113,7 @@ async def run_periodic_dump_loop(state_file_path: str, bot: "octobot.octobot.Oct except asyncio.CancelledError: raise except Exception as err: # pylint: disable=broad-except - _get_logger().exception(err, True, "process bot state dump failed: %s", err) + _get_logger().exception(err, True, f"process bot state dump failed: {err}") try: await asyncio.sleep(interval) except asyncio.CancelledError: diff --git a/octobot/task_manager.py b/octobot/task_manager.py index af81034491..f921fb66ea 100644 --- a/octobot/task_manager.py +++ b/octobot/task_manager.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import asyncio +import os import threading import concurrent.futures as thread import traceback @@ -22,6 +23,7 @@ import octobot_commons.asyncio_tools as asyncio_tools import octobot_commons.logging as logging import octobot_commons.constants as commons_constants +import octobot_commons.managed_child_process_registry as managed_child_process_registry import octobot.constants as constants import octobot.storage.process_bot_state_dumper as process_bot_state_dumper @@ -59,8 +61,8 @@ def init_async_loop(self): async def start_tools_tasks(self): task_list = [] - if self.octobot.community_handler: - task_list.append(self.octobot.community_handler.start_community_task()) + if self.octobot.activity_metrics: + task_list.append(self.octobot.activity_metrics.start_community_task()) self.octobot.async_loop = self.async_loop self.ready = True @@ -93,7 +95,7 @@ def run_forever(self, coroutine): while self.loop_forever_thread.is_alive(): self.loop_forever_thread.join(timeout=1) - def stop_tasks(self, stop_octobot=True): + def stop_tasks(self, stop_octobot=True, stop_managed_child_processes=False, force=False): self.logger.info("Stopping tasks...") async def stop_timeout(timeout): @@ -101,7 +103,9 @@ async def stop_timeout(timeout): stop_coroutines = [] if stop_octobot: - allowed_seconds_to_stop = 10 + allowed_seconds_to_stop = constants.OCTOBOT_STOP_TIMEOUT_SECONDS + if stop_managed_child_processes: + stop_coroutines.append(self._graceful_stop_managed_child_processes()) stop_coroutines.append(self.octobot.stop()) stop_coroutines.append(stop_timeout(allowed_seconds_to_stop)) @@ -113,23 +117,26 @@ async def stop_timeout(timeout): self._process_bot_state_dump_task = None # close community session - if self.octobot.community_handler: - stop_coroutines.append(self.octobot.community_handler.stop_task()) + if self.octobot.activity_metrics: + stop_coroutines.append(self.octobot.activity_metrics.stop_task()) async def _await_timeouted_gather(tasks): # await this gather to be sure to complete each stop call or timeout try: await asyncio.gather(*tasks) - except asyncio.exceptions.TimeoutError: + except TimeoutError: self.logger.warning(f"Timeout while stopping tasks, forcing stop.") raise if stop_coroutines: try: asyncio_tools.run_coroutine_in_asyncio_loop(_await_timeouted_gather(stop_coroutines), self.async_loop) - except asyncio.exceptions.TimeoutError: + except TimeoutError: self.logger.info(f"Remaining threads: {self._get_remaining_threads()}") - sys.exit(-1) + if force: + os._exit(1) + else: + sys.exit(-1) self.async_loop.stop() # ensure there is at least one element in the event loop tasks # not to block on base_event.py#self._selector.select(timeout) which prevents run_forever() from completing @@ -138,6 +145,11 @@ async def _await_timeouted_gather(tasks): self.logger.debug(f"Remaining threads: {self._get_remaining_threads()}") self.logger.info("Tasks stopped.") + async def _graceful_stop_managed_child_processes(self): + await managed_child_process_registry.ManagedChildProcessRegistry.instance().graceful_stop_all( + timeout_seconds=constants.MANAGED_CHILD_GRACEFUL_STOP_TIMEOUT_SECONDS, + ) + def _get_remaining_threads(self): return [ f"{alive_thread.name}{'[daemon]' if alive_thread.daemon else ''}" diff --git a/packages/commons/full_requirements.txt b/packages/commons/full_requirements.txt index 70329ea2e7..537f8ea5e3 100644 --- a/packages/commons/full_requirements.txt +++ b/packages/commons/full_requirements.txt @@ -2,7 +2,7 @@ jsonschema==4.26.0 psutil==7.2.2 -certifi==2026.1.4 +certifi==2026.6.17 tinydb==4.8.2 diff --git a/packages/commons/octobot_commons/authentication.py b/packages/commons/octobot_commons/authentication.py index 937b3f59e0..37f403c43d 100644 --- a/packages/commons/octobot_commons/authentication.py +++ b/packages/commons/octobot_commons/authentication.py @@ -20,6 +20,7 @@ import octobot_commons.logging as bot_logging import octobot_commons.singleton as singleton +import octobot_sync.chain as sync_chain class Authenticator(singleton.Singleton): @@ -178,6 +179,12 @@ def has_open_source_package(self) -> bool: """ raise NotImplementedError + def get_wallet_by_user_id(self, user_id: str) -> sync_chain.Wallet: + """ + Returns the wallet by user id + """ + raise NotImplementedError + @staticmethod async def wait_and_check_has_open_source_package(raise_on_timeout=False) -> bool: """ diff --git a/packages/commons/octobot_commons/configuration/config_file_manager.py b/packages/commons/octobot_commons/configuration/config_file_manager.py index f7348a0cb7..3fe7565464 100644 --- a/packages/commons/octobot_commons/configuration/config_file_manager.py +++ b/packages/commons/octobot_commons/configuration/config_file_manager.py @@ -88,6 +88,7 @@ def dump( ) raise global_exception + logging.get_logger(LOGGER_NAME).info(f"Saving config to {config_file}") json_util.safe_dump(config, config_file) diff --git a/packages/commons/octobot_commons/configuration/configuration.py b/packages/commons/octobot_commons/configuration/configuration.py index 5d0deb0f8a..766d86553c 100644 --- a/packages/commons/octobot_commons/configuration/configuration.py +++ b/packages/commons/octobot_commons/configuration/configuration.py @@ -1,4 +1,4 @@ -# pylint: disable=R0913, R0902, W0703 +# pylint: disable=R0913,R0902,W0703,R0904,C0415 # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -17,15 +17,17 @@ import os import functools import copy -import shutil +import typing import octobot_commons.logging as logging import octobot_commons.errors as errors import octobot_commons.constants as commons_constants import octobot_commons.profiles as profiles +import octobot_commons.profiles.profile_storage as profile_storage_module import octobot_commons.json_util as json_util import octobot_commons.configuration.config_file_manager as config_file_manager import octobot_commons.configuration.config_operations as config_operations +import octobot_commons.user_root_folder_provider as user_root_folder_provider class Configuration: @@ -54,6 +56,10 @@ def __init__( self._read_config: dict = None self.profile: profiles.Profile = None self.profile_by_id: dict = {} + self.profile_storage = profile_storage_module.ProfileStorage( + profiles_path, + profile_schema_path, + ) def validate(self) -> None: """ @@ -63,13 +69,20 @@ def validate(self) -> None: json_util.validate(self._read_config, self.config_schema_path) self.profile.validate() - def read(self, should_raise=True, fill_missing_fields=False) -> None: + def read( + self, + should_raise=True, + fill_missing_fields=False, + *, + activate_profile=True, + ) -> None: """ Reads the configuration from self.config_path and load the current profile Overall config is stored into self.config and consists of a merger from the user config and activated profile :param should_raise: will raise upon exception when True :param fill_missing_fields: will try to fill in missing fields when true + :param activate_profile: when False, only load config.json without selecting a profile :return: None """ self._read_config = config_file_manager.load( @@ -78,6 +91,13 @@ def read(self, should_raise=True, fill_missing_fields=False) -> None: fill_missing_fields=fill_missing_fields, ) self.config = copy.deepcopy(self._read_config) + if activate_profile: + self.load_profiles_if_possible_and_necessary() + + def activate_saved_profile(self) -> None: + """ + Load all profiles and select CONFIG_PROFILE from the last read config.json. + """ self.load_profiles_if_possible_and_necessary() def load_profiles_if_possible_and_necessary(self) -> None: @@ -97,6 +117,7 @@ def select_profile(self, profile_id) -> None: """ self.config[commons_constants.CONFIG_PROFILE] = profile_id self.profile = self.profile_by_id[profile_id] + self.profile_storage.activate_profile(self.profile) self.logger.info(f"Using {self.profile.name} profile.") self._generate_config_from_user_config_and_profile() @@ -110,8 +131,13 @@ def remove_profile(self, profile_id: str) -> None: if profile.read_only and not profile.imported: raise errors.ProfileRemovalError(f"{profile.name} profile can't be removed") try: - shutil.rmtree(profile.path) + self.profile_storage.delete_profile( + profile_id, + profile=profile, + ) self.profile_by_id.pop(profile_id, None) + except errors.ProfileRemovalError: + raise except Exception as err: raise errors.ProfileRemovalError() from err @@ -128,7 +154,8 @@ def _generate_config_from_user_config_and_profile(self): def save( self, schema_file=None, - sync_all_profiles=False, + sync_all_profiles: bool = False, + save_profile: typing.Optional[bool] = None, ) -> None: """ Save the current self.config and self.profile. @@ -141,11 +168,52 @@ def save( config_to_save, schema_file=schema_file, ) - if self.profile is not None: + if save_profile is None: + save_profile = self._profile_managed_elements_changed() + if ( + save_profile + and self.profile is not None + and not self.profile_storage.is_readonly_master_overlay_profile(self.profile) + ): self.profile.save_config(self.config) if sync_all_profiles: self._sync_other_profiles() + def _profile_managed_elements_changed(self) -> bool: + if self.profile is None: + return False + for element in self.profile.FULLY_MANAGED_ELEMENTS: + if element in self.config: + if self.config[element] != self.profile.config.get(element): + return True + for element in self.profile.PARTIALLY_MANAGED_ELEMENTS: + if self._partially_managed_element_would_change(element): + return True + return False + + def _partially_managed_element_would_change(self, element: str) -> bool: + if element not in self.config: + return False + allowed_keys = profiles.Profile.PARTIALLY_MANAGED_ELEMENTS_ALLOWED_KEYS.get( + element + ) + if allowed_keys is None: + return self.config[element] != self.profile.config.get(element) + global_element = self.config[element] + profile_element = self.profile.config.get(element, {}) + for exchange_name, global_exchange_config in global_element.items(): + if not isinstance(global_exchange_config, dict): + continue + profile_exchange_config = profile_element.get(exchange_name, {}) + if not isinstance(profile_exchange_config, dict): + return True + for allowed_key in allowed_keys: + if global_exchange_config.get(allowed_key) != profile_exchange_config.get( + allowed_key + ): + return True + return False + def _sync_other_profiles(self): """ Update profile partially managed elements for all profiles except self.profile @@ -193,8 +261,12 @@ def are_profiles_empty_or_missing(self) -> bool: Checks if self.profiles_path exists and contains folders :return: True if profiles folder is not empty """ + self._prepare_profile_storage() return not ( - os.path.isdir(self.profiles_path) and os.listdir(self.profiles_path) + self.profile_storage.has_any_profiles() + or ( + os.path.isdir(self.profiles_path) and os.listdir(self.profiles_path) + ) ) def get_non_imported_profiles(self) -> list: @@ -211,6 +283,24 @@ def get_tentacles_config_path(self) -> str: """ return self.profile.get_tentacles_config_path() + def get_active_tentacles_setup_config(self): + """ + :return: The tentacles setup config for the activated profile. + Profile-data-backed profiles use in-memory setup; filesystem profiles use their config path. + """ + if self.profile.is_profile_data_tentacle_backed(): + if self.profile.tentacles_setup_config is None: + self.profile.init_tentacles_setup_config() + tentacles_setup_config = self.profile.tentacles_setup_config + return self.profile.bind_tentacles_setup_config(tentacles_setup_config) + # create it from filesystem config path + import octobot_tentacles_manager.api as tentacles_manager_api + + return tentacles_manager_api.get_tentacles_setup_config( + self.get_tentacles_config_path(), + profile=self.profile, + ) + def get_metrics_enabled(self) -> bool: """ Check if metrics are enabled @@ -222,14 +312,6 @@ def get_metrics_enabled(self) -> bool: ) ) - def get_metrics_id(self) -> str: - """ - :return: The current user's metrics id - """ - return self.config[commons_constants.CONFIG_METRICS][ - commons_constants.CONFIG_METRICS_BOT_ID - ] - def accepted_terms(self) -> bool: """ Check if terms has been accepted @@ -237,6 +319,14 @@ def accepted_terms(self) -> bool: """ return self.config.get(commons_constants.CONFIG_ACCEPTED_TERMS, False) + def octobot_name(self) -> str | None: + """ + Optional display name for DSL-spawned child OctoBot instances. + :return: the configured name, or None when unset or blank + """ + value = self.config.get(commons_constants.CONFIG_OCTOBOT_NAME) + return value if value and str(value).strip() else None + def accept_terms(self, accepted) -> None: """ Perform terms acceptation @@ -302,19 +392,82 @@ def _get_selected_profile(self): selected_profile_id != commons_constants.DEFAULT_PROFILE and commons_constants.DEFAULT_PROFILE in self.profile_by_id ): + self.logger.warning( + "Profile %r from config.json is not available yet; falling back to %r. " + "This can happen when sync profiles are not loaded.", + selected_profile_id, + commons_constants.DEFAULT_PROFILE, + ) return commons_constants.DEFAULT_PROFILE raise errors.NoProfileError + def apply_readonly_reference_tentacles_override(self) -> None: + """ + Point ``UserRootFolderProvider`` at the master ``reference_tentacles_config`` tree + when ``readonly_reference_tentacles_path`` is set in user ``config.json`` (DSL process + children share the executor's reference tentacles instead of copying them locally). + """ + if not isinstance(self.config, dict): + return + readonly_reference_tentacles_path = self.config.get( + commons_constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH + ) + if readonly_reference_tentacles_path: + user_root_folder_provider.instance().configure_readonly_reference_tentacles_path( + readonly_reference_tentacles_path + ) + + def _prepare_profile_storage(self) -> None: + self.profile_storage.configure_paths( + self.profiles_path, self.profile_schema_path + ) + if self.config: + readonly_profiles_path = self.config.get( + commons_constants.CONFIG_READONLY_PROFILES_PATH + ) + if readonly_profiles_path: + self.profile_storage.configure_readonly_profiles_path( + readonly_profiles_path + ) + # Same config.json readonly overlay as profiles above: profile load and tentacles + # setup read reference paths via UserRootFolderProvider, so apply it whenever + # profile storage is prepared (load_profiles, refresh_sync_profiles, etc.). + self.apply_readonly_reference_tentacles_override() + def load_profiles(self) -> None: """ Loads the available profiles :return: None """ - for profile in profiles.Profile.get_all_profiles( - self.profiles_path, schema_path=self.profile_schema_path - ): - if profile.profile_id not in self.profile_by_id: - self.profile_by_id[profile.profile_id] = profile + self._prepare_profile_storage() + loaded_profiles = self.profile_storage.load_all_profiles() + for profile_id, profile in loaded_profiles.items(): + if profile_id not in self.profile_by_id: + self.profile_by_id[profile_id] = profile + + def refresh_sync_profiles(self) -> None: + """ + Reload sync-backed profiles from storage into profile_by_id. + Used when the sync collection may have changed externally. + """ + if not self.profile_storage.is_sync_available(): + return + self._prepare_profile_storage() + loaded_sync_profiles = self.profile_storage.list_sync_profiles() + loaded_sync_profile_ids = set(loaded_sync_profiles) + for profile_id, profile in loaded_sync_profiles.items(): + self.profile_by_id[profile_id] = profile + removed_sync_profile_ids = [ + profile_id + for profile_id, profile in list(self.profile_by_id.items()) + if profile.is_sync_backed() and profile_id not in loaded_sync_profile_ids + ] + for profile_id in removed_sync_profile_ids: + self.profile_by_id.pop(profile_id, None) + if self.profile is not None and self.profile.is_sync_backed(): + refreshed_profile = self.profile_by_id.get(self.profile.profile_id) + if refreshed_profile is not None: + self.profile = refreshed_profile def _get_config_without_profile_elements(self) -> dict: filtered_config = copy.deepcopy(self.config) diff --git a/packages/commons/octobot_commons/configuration/user_inputs.py b/packages/commons/octobot_commons/configuration/user_inputs.py index 21f87a4d8a..0f92b922fb 100644 --- a/packages/commons/octobot_commons/configuration/user_inputs.py +++ b/packages/commons/octobot_commons/configuration/user_inputs.py @@ -333,6 +333,10 @@ def _find_parent_config_node(tentacle_config, parent_input_name, array_indexes): if found and isinstance(nested_parent, dict): return nested_parent if found and isinstance(nested_parent, list) and array_indexes: + # Nested OBJECT_ARRAY item inputs may create placeholder objects for + # schema/default seeding on empty config. + while len(nested_parent) <= array_indexes[-1]: + nested_parent.append({}) return nested_parent[array_indexes[-1]] # non dict or list with array_indexes nested parents are not supported return None diff --git a/packages/commons/octobot_commons/constants.py b/packages/commons/octobot_commons/constants.py index 650686fcc8..3f77df957e 100644 --- a/packages/commons/octobot_commons/constants.py +++ b/packages/commons/octobot_commons/constants.py @@ -76,7 +76,13 @@ def parse_boolean_environment_var(env_key: str, default_value: str) -> bool: # profiles PROFILES_FOLDER = "profiles" +PROFILES_MIGRATED_FOLDER = "profiles_migrated" USER_PROFILES_FOLDER = f"{USER_FOLDER}/{PROFILES_FOLDER}" +USER_PROFILES_MIGRATED_FOLDER = f"{USER_FOLDER}/profiles_migrated" +SYNC_PROFILE_RUNTIME_FOLDER = f"{USER_FOLDER}/sync_profile_runtime" +ENV_OCTOBOT_SYNC_DATA_ROOT = "OCTOBOT_SYNC_DATA_ROOT" +CONFIG_READONLY_PROFILES_PATH = "readonly_profiles_path" +CONFIG_READONLY_REFERENCE_TENTACLES_PATH = "readonly_reference_tentacles_path" PROFILE_CONFIG_FILE = "profile.json" CONFIG_PROFILE = "profile" CONFIG_BACKTESTING_PROFILE = "backtesting_profile" @@ -205,13 +211,16 @@ def parse_boolean_environment_var(env_key: str, default_value: str) -> bool: # terms of service CONFIG_ACCEPTED_TERMS = "accepted_terms" +# child-process display name (DSL-spawned OctoBot instances) +CONFIG_OCTOBOT_NAME = "octobot_name" + # distribution DEFAULT_DISTRIBUTION = "default" CONFIG_DISTRIBUTION = "distribution" # metrics CONFIG_METRICS = "metrics" -CONFIG_METRICS_BOT_ID = "metrics-bot-id" +CONFIG_METRICS_ACTIVITY_BOT_ID = "activity_bot_id" TIMER_BEFORE_METRICS_REGISTRATION_SECONDS = 600 TIMER_BETWEEN_METRICS_UPTIME_UPDATE = float( os.getenv("TIMER_BETWEEN_METRICS_UPTIME_UPDATE", str(3600 * 4)) diff --git a/packages/commons/octobot_commons/dict_util.py b/packages/commons/octobot_commons/dict_util.py index 37d537b6bb..33f24d75db 100644 --- a/packages/commons/octobot_commons/dict_util.py +++ b/packages/commons/octobot_commons/dict_util.py @@ -44,16 +44,16 @@ def find_nested_value(dict_, field, list_indexes=None): def _find_nested_value_in_list(list_value, field, list_indexes): if list_indexes: # list_indexes is provided: only look at the given index - try: - item = list_value[list_indexes[0]] - if isinstance(item, dict): - found_value, possible_value = find_nested_value( - item, field, list_indexes=list_indexes[1:] - ) - if found_value: - return found_value, possible_value - except IndexError: - pass + target_index = list_indexes[0] + while len(list_value) <= target_index: + list_value.append({}) + item = list_value[target_index] + if isinstance(item, dict): + found_value, possible_value = find_nested_value( + item, field, list_indexes=list_indexes[1:] + ) + if found_value: + return found_value, possible_value else: for item in list_value: if isinstance(item, dict): diff --git a/packages/commons/octobot_commons/dsl_interpreter/__init__.py b/packages/commons/octobot_commons/dsl_interpreter/__init__.py index ead290ce2c..dbc00ff00c 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/__init__.py +++ b/packages/commons/octobot_commons/dsl_interpreter/__init__.py @@ -26,7 +26,10 @@ ) from octobot_commons.dsl_interpreter.operator_parameter import ( OperatorParameter, + OperatorParameterOption, + TIME_FRAME_OPERATOR_PARAMETER_OPTIONS, UNINITIALIZED_VALUE, + dsl_value_type_for_user_input, ) from octobot_commons.dsl_interpreter.operator_docs import OperatorDocs from octobot_commons.dsl_interpreter.operators import ( @@ -71,7 +74,10 @@ "Interpreter", "Operator", "OperatorParameter", + "OperatorParameterOption", + "TIME_FRAME_OPERATOR_PARAMETER_OPTIONS", "UNINITIALIZED_VALUE", + "dsl_value_type_for_user_input", "OperatorDocs", "BinaryOperator", "UnaryOperator", diff --git a/packages/commons/octobot_commons/dsl_interpreter/operator.py b/packages/commons/octobot_commons/dsl_interpreter/operator.py index c9425e8f28..5d39b915ca 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operator.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operator.py @@ -48,6 +48,8 @@ class Operator: ) DESCRIPTION: str = "" # description of the operator EXAMPLE: str = "" # example of the operator in the DSL + LABEL: str = "" # human-readable label; defaults to name when empty + CATEGORY: str = "" # composition role; must match protocol DslKeywordCategory when set def __init__(self, *parameters: OperatorParameterType, **kwargs: typing.Any): self._validate_parameters(parameters, kwargs) @@ -115,12 +117,16 @@ def get_docs(cls) -> dsl_interpreter_operator_docs.OperatorDocs: """ Get the documentation of the operator. """ + operator_name = cls.NAME or cls.get_name() return dsl_interpreter_operator_docs.OperatorDocs( - name=cls.NAME or cls.get_name(), + name=operator_name, description=cls.DESCRIPTION, type=cls.get_library(), example=cls.EXAMPLE, parameters=cls.get_parameters(), + label=cls.LABEL or operator_name, + category=cls.CATEGORY, + return_values=cls.get_return_values(), ) @classmethod @@ -130,6 +136,33 @@ def get_parameters(cls) -> list[dsl_interpreter_operator_parameter.OperatorParam """ return [] + @classmethod + def get_return_values( + cls, + ) -> list[dsl_interpreter_operator_parameter.OperatorParameter]: + """ + return: catalog metadata for the operator result value(s). + """ + return [] + + @classmethod + def result_return_value( + cls, + value_type: str, + description: str = "Operator result", + ) -> list[dsl_interpreter_operator_parameter.OperatorParameter]: + """ + Build a single-result return_values list for catalog metadata. + """ + return [ + dsl_interpreter_operator_parameter.OperatorParameter( + name="result", + description=description, + required=True, + type=value_type, + ) + ] + async def pre_compute(self) -> None: # rename pre_compute """ Refreshes the operator data, override if necessary. diff --git a/packages/commons/octobot_commons/dsl_interpreter/operator_docs.py b/packages/commons/octobot_commons/dsl_interpreter/operator_docs.py index 962dca08f8..14e347269c 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operator_docs.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operator_docs.py @@ -20,10 +20,10 @@ @dataclasses.dataclass -class OperatorDocs: +class OperatorDocs: # pylint: disable=too-many-instance-attributes """ Operator documentation class, used to store operators metadata to - generate an operator documentation. + generate an operator documentation and protocol keyword conversion. """ name: str @@ -31,6 +31,11 @@ class OperatorDocs: type: str example: str parameters: list[dsl_interpreter_operator_parameter.OperatorParameter] + label: str = "" + category: str = "" + return_values: list[dsl_interpreter_operator_parameter.OperatorParameter] = ( + dataclasses.field(default_factory=list) + ) def to_json(self) -> dict: """ @@ -42,4 +47,9 @@ def to_json(self) -> dict: "type": self.type, "example": self.example, "parameters": [parameter.to_json() for parameter in self.parameters], + "label": self.label or self.name, + "category": self.category, + "return_values": [ + return_value.to_json() for return_value in self.return_values + ], } diff --git a/packages/commons/octobot_commons/dsl_interpreter/operator_parameter.py b/packages/commons/octobot_commons/dsl_interpreter/operator_parameter.py index 7293f0f2df..2576c8241c 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operator_parameter.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operator_parameter.py @@ -1,4 +1,4 @@ -# pylint: disable=too-many-branches,too-many-return-statements +# pylint: disable=too-many-branches,too-many-return-statements,too-many-instance-attributes # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -17,34 +17,84 @@ import dataclasses import typing +import octobot_commons.enums as commons_enums + UNSET_VALUE = "UNSET_VALUE" UNINITIALIZED_VALUE = object() +@dataclasses.dataclass +class OperatorParameterOption: + """Selectable value for an operator parameter (protocol options).""" + + value: str + label: str + + def to_json(self) -> dict: + """Convert the option to a JSON serializable dict.""" + return dataclasses.asdict(self) + + +TIME_FRAME_OPERATOR_PARAMETER_OPTIONS = [ + OperatorParameterOption(value=time_frame.value, label=time_frame.value) + for time_frame in commons_enums.TimeFrames +] + + +def dsl_value_type_for_user_input(input_type: str) -> str: + """Map a user input type string to its DSL value type.""" + try: + return commons_enums.USER_INPUT_TYPE_TO_DSL_VALUE_TYPE[input_type] + except KeyError as error: + raise ValueError( + f"Unsupported user input type {input_type!r} for DSL catalog type" + ) from error + + @dataclasses.dataclass class OperatorParameter: name: str description: str required: bool - type: typing.Type[typing.Any] + type: str # DslValueType string default: typing.Any = UNSET_VALUE + options: typing.Optional[list[OperatorParameterOption]] = None + minimum: typing.Optional[float] = None + maximum: typing.Optional[float] = None + step: typing.Optional[float] = None + multiple: typing.Optional[bool] = None + primary: typing.Optional[bool] = None def __repr__(self) -> str: default_str = f' (default: {self.default})' if self.default is not UNSET_VALUE else '' return ( f"{self.name}{' (required)' if self.required else default_str}" - f"[{self.type.__name__}] - {self.description}" + f"[{self.type}] - {self.description}" ) def to_json(self) -> dict: """ Convert the operator parameter to a JSON serializable dict. """ - return { + payload = { "name": self.name, "description": self.description, "required": self.required, - "type": self.type.__name__, + "type": self.type, "default": self.default, + "label": self.name, } + if self.options is not None: + payload["options"] = [option.to_json() for option in self.options] + if self.minimum is not None: + payload["minimum"] = self.minimum + if self.maximum is not None: + payload["maximum"] = self.maximum + if self.step is not None: + payload["step"] = self.step + if self.multiple is not None: + payload["multiple"] = self.multiple + if self.primary is not None: + payload["primary"] = self.primary + return payload diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/binary_operator.py b/packages/commons/octobot_commons/dsl_interpreter/operators/binary_operator.py index 4adeee9c9c..18bff54300 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/binary_operator.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/binary_operator.py @@ -15,7 +15,9 @@ # License along with this library. import typing +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator +import octobot_commons.dsl_interpreter.operator_parameter as dsl_interpreter_operator_parameter class BinaryOperator( @@ -26,6 +28,8 @@ class BinaryOperator( Binary operators have two operands. """ + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value + def __init__( self, left: dsl_interpreter_operator.OperatorParameterType, @@ -37,6 +41,15 @@ def __init__( """ super().__init__(left, right, **kwargs) + @classmethod + def get_return_values( + cls, + ) -> list[dsl_interpreter_operator_parameter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Binary operation result", + ) + def get_computed_left_and_right_parameters( self, ) -> typing.Tuple[ diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/compare_operator.py b/packages/commons/octobot_commons/dsl_interpreter/operators/compare_operator.py index a02689a418..c80d71dd97 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/compare_operator.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/compare_operator.py @@ -15,7 +15,9 @@ # License along with this library. import typing +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator +import octobot_commons.dsl_interpreter.operator_parameter as dsl_interpreter_operator_parameter class CompareOperator( @@ -26,6 +28,8 @@ class CompareOperator( Compare operators have two operands. """ + CATEGORY = commons_enums.DslKeywordCategory.CONDITION.value + def __init__( self, left: dsl_interpreter_operator.OperatorParameterType, @@ -37,6 +41,15 @@ def __init__( """ super().__init__(left, right, **kwargs) + @classmethod + def get_return_values( + cls, + ) -> list[dsl_interpreter_operator_parameter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.BOOLEAN.value, + description="Comparison result", + ) + def get_computed_left_and_right_parameters( self, ) -> typing.Tuple[ @@ -44,7 +57,7 @@ def get_computed_left_and_right_parameters( dsl_interpreter_operator.ComputedOperatorParameterType, ]: """ - Get the computed left and right computed operands. + Get the computed left and right parameters of the compare operator. """ computed_parameters = self.get_computed_parameters() return computed_parameters[0], computed_parameters[1] diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/dynamic_dependencies_operator_mixin.py b/packages/commons/octobot_commons/dsl_interpreter/operators/dynamic_dependencies_operator_mixin.py index b43cb1956c..6bf2e7ea8a 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/dynamic_dependencies_operator_mixin.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/dynamic_dependencies_operator_mixin.py @@ -19,6 +19,7 @@ import octobot_commons.dataclasses import octobot_commons.dsl_interpreter.operator_parameter as operator_parameter +import octobot_commons.enums as commons_enums @dataclasses.dataclass @@ -61,7 +62,7 @@ def get_dynamic_dependencies_parameters(cls) -> list[operator_parameter.Operator name=cls.DYNAMIC_DEPENDENCIES_KEY, description="Injected upstream operator results (list of DynamicDependency JSON objects)", required=False, - type=list, + type=commons_enums.DslValueType.ANY.value, default=None, ), ] diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/n_ary_operator.py b/packages/commons/octobot_commons/dsl_interpreter/operators/n_ary_operator.py index bb4b8816d5..87ae874e67 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/n_ary_operator.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/n_ary_operator.py @@ -13,7 +13,9 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator +import octobot_commons.dsl_interpreter.operator_parameter as dsl_interpreter_operator_parameter class NaryOperator( @@ -23,3 +25,14 @@ class NaryOperator( Base class for n-ary operators. N-ary operators have one or more operands. """ + + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value + + @classmethod + def get_return_values( + cls, + ) -> list[dsl_interpreter_operator_parameter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.BOOLEAN.value, + description="N-ary boolean operation result", + ) diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py b/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py index ec8bb2a242..e8069a3ac6 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/process_bound_operator_mixin.py @@ -14,10 +14,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. -import asyncio import pathlib import subprocess -import time import typing import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator @@ -52,7 +50,10 @@ def request_graceful_stop( raise commons_errors.DSLInterpreterError( "No process id set; cannot request graceful stop." ) - return process_util.request_graceful_stop_via_sigterm(self.pid, logger=logger) + try: + return process_util.request_graceful_stop_via_sigterm(self.pid, logger=logger) + except commons_errors.ProcessError as err: + raise commons_errors.DSLInterpreterError(str(err)) from err async def wait_until_pid_stopped( self, @@ -63,27 +64,15 @@ async def wait_until_pid_stopped( poll_interval: float = 0.2, ) -> None: """Poll until ``pid`` is gone or ``timeout_seconds`` elapses (after e.g. SIGTERM).""" - resolved_logger = logger or commons_logging.get_logger(self.__class__.__name__) - if pid <= 0: - resolved_logger.info( - "wait_until_pid_stopped: pid=%s treated as already stopped (non-positive)", + try: + await process_util.wait_until_pid_stopped_async( pid, + logger=logger or commons_logging.get_logger(self.__class__.__name__), + timeout_seconds=timeout_seconds, + poll_interval=poll_interval, ) - return - resolved_logger.info( - "wait_until_pid_stopped: waiting for pid=%s to exit (timeout=%ss)", - pid, - timeout_seconds, - ) - deadline = time.monotonic() + timeout_seconds - while time.monotonic() < deadline: - if not process_util.pid_is_running(pid): - resolved_logger.info("wait_until_pid_stopped: pid=%s exited", pid) - return - await asyncio.sleep(poll_interval) - raise commons_errors.DSLInterpreterError( - f"Timed out after {timeout_seconds}s waiting for pid={pid} to exit." - ) + except commons_errors.ProcessError as err: + raise commons_errors.DSLInterpreterError(str(err)) from err def spawn_subprocess( self, @@ -103,6 +92,19 @@ def spawn_subprocess( self.pid = proc.pid return proc + def bind_authoritative_child_pid( + self, + authoritative_pid: int, + *, + spawn_pid: typing.Optional[int] = None, + ) -> None: + """Point this operator and the managed-child registry at the authoritative app pid.""" + previous_pid = spawn_pid if spawn_pid is not None else (self.pid or 0) + if previous_pid == authoritative_pid and self.pid == authoritative_pid: + return + process_util.rebind_managed_child_pid(previous_pid, authoritative_pid) + self.pid = authoritative_pid + @staticmethod def reject_user_path_segment(path_value: str) -> None: """Reject obvious path traversal in user-supplied relative paths.""" diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/re_callable_operator_mixin.py b/packages/commons/octobot_commons/dsl_interpreter/operators/re_callable_operator_mixin.py index c67b57a186..f67b30596a 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/re_callable_operator_mixin.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/re_callable_operator_mixin.py @@ -21,6 +21,7 @@ import octobot_commons.dataclasses import octobot_commons.dsl_interpreter.operator_parameter as operator_parameter import octobot_commons.dsl_interpreter.parameters_util as parameters_util +import octobot_commons.enums as commons_enums class ReCallingOperatorResultKeys(str, enum.Enum): @@ -94,7 +95,7 @@ def get_re_callable_parameters(cls) -> list[operator_parameter.OperatorParameter name=cls.LAST_EXECUTION_RESULT_KEY, description="the return value of the previous call", required=False, - type=dict, + type=commons_enums.DslValueType.DICT.value, default=None, ), ] diff --git a/packages/commons/octobot_commons/dsl_interpreter/operators/unary_operator.py b/packages/commons/octobot_commons/dsl_interpreter/operators/unary_operator.py index e08eca4172..af616170f7 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/operators/unary_operator.py +++ b/packages/commons/octobot_commons/dsl_interpreter/operators/unary_operator.py @@ -15,7 +15,9 @@ # License along with this library. import typing +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator +import octobot_commons.dsl_interpreter.operator_parameter as dsl_interpreter_operator_parameter class UnaryOperator( @@ -26,6 +28,8 @@ class UnaryOperator( Unary operators have one operand. """ + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value + def __init__( self, operand: dsl_interpreter_operator.OperatorParameterType, @@ -36,11 +40,19 @@ def __init__( """ super().__init__(operand, **kwargs) + @classmethod + def get_return_values( + cls, + ) -> list[dsl_interpreter_operator_parameter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Unary operation result", + ) + def get_computed_operand( self, ) -> dsl_interpreter_operator.ComputedOperatorParameterType: """ Get the computed operand of the unary operator. """ - computed_parameters = self.get_computed_parameters() - return computed_parameters[0] + return self.get_computed_parameters()[0] diff --git a/packages/commons/octobot_commons/enums.py b/packages/commons/octobot_commons/enums.py index 70fcdcfa32..7a9d599608 100644 --- a/packages/commons/octobot_commons/enums.py +++ b/packages/commons/octobot_commons/enums.py @@ -450,6 +450,44 @@ class UserInputTypes(enum.Enum): STRING_ARRAY = "string_array" +class DslKeywordCategory(str, enum.Enum): + """Composition role of a DSL keyword (must match protocol DslKeywordCategory).""" + + TRIGGER = "trigger" + SOURCE = "source" + CONDITION = "condition" + LOGIC = "logic" + ACTION = "action" + + +class DslValueType(str, enum.Enum): + """Typed value carried by a DSL keyword parameter (must match protocol DslValueType).""" + + BOOLEAN = "boolean" + NUMBER = "number" + TEXT = "text" + SERIES = "series" + SIGNAL = "signal" + DICT = "dict" + ORDER = "order" + TIME_FRAME = "time_frame" + ANY = "any" + + +# Map tentacle UserInputTypes values to catalog DslValueType values. +USER_INPUT_TYPE_TO_DSL_VALUE_TYPE = { + UserInputTypes.INT.value: DslValueType.NUMBER.value, + UserInputTypes.FLOAT.value: DslValueType.NUMBER.value, + UserInputTypes.BOOLEAN.value: DslValueType.BOOLEAN.value, + UserInputTypes.TEXT.value: DslValueType.TEXT.value, + UserInputTypes.OPTIONS.value: DslValueType.TEXT.value, + UserInputTypes.OBJECT.value: DslValueType.DICT.value, + UserInputTypes.OBJECT_ARRAY.value: DslValueType.ANY.value, + UserInputTypes.STRING_ARRAY.value: DslValueType.ANY.value, + UserInputTypes.MULTIPLE_OPTIONS.value: DslValueType.ANY.value, +} + + class UserInputEditorOptionsTypes(enum.Enum): # source for the available options: # https://github.com/json-editor/json-editor#editor-options @@ -523,6 +561,12 @@ class ProfileType(enum.Enum): BACKTESTING = "backtesting" +class ProfileSource(enum.Enum): + FILESYSTEM = "filesystem" + SYNC = "sync" + EPHEMERAL = "ephemeral" + + class SignalHistoryTypes(enum.Enum): GPT = "gpt" diff --git a/packages/commons/octobot_commons/errors.py b/packages/commons/octobot_commons/errors.py index 9b1254c17a..ff01a254c1 100644 --- a/packages/commons/octobot_commons/errors.py +++ b/packages/commons/octobot_commons/errors.py @@ -193,3 +193,9 @@ class AmbiguousTradedSymbolsTradingTypeError(ValueError): """ Raised when traded symbols map to more than one exchange trading type. """ + + +class ProcessError(Exception): + """ + Raised when a process error occurs + """ diff --git a/packages/commons/octobot_commons/managed_child_process_registry.py b/packages/commons/octobot_commons/managed_child_process_registry.py new file mode 100644 index 0000000000..8db1f3c4d0 --- /dev/null +++ b/packages/commons/octobot_commons/managed_child_process_registry.py @@ -0,0 +1,163 @@ +# pylint: disable=R0401 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import asyncio +import threading +import time +import typing + +import octobot_commons.errors as commons_errors +import octobot_commons.logging as commons_logging +import octobot_commons.process_util as process_util +import octobot_commons.singleton.singleton_class as singleton_class + + +class ManagedChildProcessRegistry(singleton_class.Singleton): + """ + Thread-safe registry of OS pids spawned by spawn_managed_subprocess. + Used at parent shutdown to SIGTERM (then force-kill) child OctoBot processes. + """ + + def __init__(self) -> None: + self._pids: set[int] = set() + self._lock = threading.Lock() + self._logger = commons_logging.get_logger(self.__class__.__name__) + + def register(self, pid: int) -> None: + """Record a managed child pid. No-op for pid <= 0.""" + if pid <= 0: + return + with self._lock: + self._pids.add(pid) + registered_count = len(self._pids) + self._logger.debug( + "Registered managed child pid=%s (count=%s)", + pid, + registered_count, + ) + + def unregister(self, pid: int) -> None: + """Remove a pid from the registry. No-op if not present.""" + with self._lock: + self._pids.discard(pid) + self._logger.debug("Unregistered managed child pid=%s", pid) + + def rebind_managed_child_pid(self, spawn_pid: int, authoritative_pid: int) -> None: + """ + Replace a bootstrap/spawn pid with the authoritative app pid (e.g. PyInstaller child). + + Unregisters ``spawn_pid`` when it differs from ``authoritative_pid``, then registers + the authoritative pid. No-op when ``authoritative_pid`` is not positive or already bound. + """ + if authoritative_pid <= 0: + return + with self._lock: + authoritative_already_registered = authoritative_pid in self._pids + spawn_in_registry = spawn_pid > 0 and spawn_pid in self._pids + if authoritative_already_registered and ( + spawn_pid <= 0 or spawn_pid == authoritative_pid + ): + return + if spawn_pid > 0 and spawn_pid != authoritative_pid and spawn_in_registry: + self.unregister(spawn_pid) + if not authoritative_already_registered: + self.register(authoritative_pid) + if spawn_pid > 0 and spawn_pid != authoritative_pid: + self._logger.debug( + "Rebound managed child pid from %s to %s", + spawn_pid, + authoritative_pid, + ) + + def snapshot_running_pids(self) -> frozenset[int]: + """Prune dead pids, return a point-in-time copy of still-running entries.""" + with self._lock: + dead_pids = { + pid for pid in self._pids if not process_util.pid_is_running(pid) + } + self._pids -= dead_pids + return frozenset(self._pids) + + async def graceful_stop_all( + self, + *, + timeout_seconds: float, + poll_interval: float = 0.2, + ) -> dict[int, str]: + """SIGTERM all snapshot pids, wait, force-kill survivors. Returns per-pid outcome.""" + pids = self.snapshot_running_pids() + if not pids: + return {} + + self._logger.info( + "Graceful stop for %s managed child process(es): %s", + len(pids), + sorted(pids), + ) + + signal_outcomes: dict[int, dict[str, typing.Any]] = {} + for pid in pids: + try: + signal_outcomes[pid] = process_util.request_graceful_stop_via_sigterm( + pid, + logger=self._logger, + ) + except commons_errors.ProcessError as err: + self._logger.warning( + "Failed to send SIGTERM to managed child pid=%s: %s", + pid, + err, + ) + signal_outcomes[pid] = {"status": "failed", "reason": str(err)} + + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if not any(process_util.pid_is_running(pid) for pid in pids): + break + await asyncio.sleep(poll_interval) + + outcomes: dict[int, str] = {} + for pid in pids: + if not process_util.pid_is_running(pid): + sig_status = signal_outcomes.get(pid, {}).get("status", "stopped") + outcomes[pid] = ( + "already_stopped" if sig_status == "already_stopped" else "stopped" + ) + self.unregister(pid) + continue + try: + process_util.request_force_kill(pid, logger=self._logger) + except commons_errors.ProcessError as err: + self._logger.error( + "Force kill failed for managed child pid=%s: %s", + pid, + err, + ) + outcomes[pid] = "failed" + continue + if not process_util.pid_is_running(pid): + outcomes[pid] = "force_killed" + self.unregister(pid) + else: + self._logger.error( + "Managed child pid=%s still running after force kill", + pid, + ) + outcomes[pid] = "failed" + + self._logger.info("Managed child graceful stop outcomes: %s", outcomes) + return outcomes diff --git a/packages/commons/octobot_commons/network.py b/packages/commons/octobot_commons/network.py new file mode 100644 index 0000000000..754cc0c670 --- /dev/null +++ b/packages/commons/octobot_commons/network.py @@ -0,0 +1,126 @@ +# Drakkar-Software OctoBot +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import ipaddress +import socket + +import psutil + +PREFERRED_LOCAL_IPV4_PREFIX = "192." +UDP_ROUTE_PROBE_HOST = "8.8.8.8" +UDP_ROUTE_PROBE_PORT = 80 +LOCAL_HOST_IP = "127.0.0.1" + + +def _is_private_ipv4(ip_address: str) -> bool: + try: + parsed = ipaddress.ip_address(ip_address) + except ValueError: + return False + return parsed.version == 4 and parsed.is_private and not parsed.is_loopback and not parsed.is_link_local + + +def _is_preferred_local_ipv4(ip_address: str) -> bool: + return _is_private_ipv4(ip_address) and ip_address.startswith(PREFERRED_LOCAL_IPV4_PREFIX) + + +def _pick_preferred_local_ipv4(candidates: list[str]) -> str | None: + private_candidates = [ + candidate for candidate in candidates if _is_private_ipv4(candidate) + ] + if not private_candidates: + return None + for candidate in private_candidates: + if _is_preferred_local_ipv4(candidate): + return candidate + return private_candidates[0] + + +def _udp_route_local_ipv4() -> str | None: + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as udp_socket: + udp_socket.connect((UDP_ROUTE_PROBE_HOST, UDP_ROUTE_PROBE_PORT)) + candidate = udp_socket.getsockname()[0] + except OSError: + return None + if _is_private_ipv4(candidate): + return candidate + return None + + +def _hostname_private_ipv4_candidates() -> list[str]: + candidates: list[str] = [] + try: + address_infos = socket.getaddrinfo(socket.gethostname(), None, family=socket.AF_INET) + except OSError: + return candidates + for address_info in address_infos: + sockaddr = address_info[4] + if not sockaddr: + continue + candidate = sockaddr[0] + if _is_private_ipv4(candidate) and candidate not in candidates: + candidates.append(candidate) + return candidates + + +def _get_local_network_ipv4() -> str | None: + candidates: list[str] = [] + udp_candidate = _udp_route_local_ipv4() + if udp_candidate is not None: + candidates.append(udp_candidate) + for hostname_candidate in _hostname_private_ipv4_candidates(): + if hostname_candidate not in candidates: + candidates.append(hostname_candidate) + return _pick_preferred_local_ipv4(candidates) + + +def get_interface_ipv4_by_name_substring(interface_name_substring: str) -> str | None: + """ + Return the first IPv4 address on a network interface whose name contains the given substring. + Interface name matching is case-insensitive. + :param interface_name_substring: substring to match against interface names + :return: the IPv4 address string, or None if no matching interface or address is found + """ + for interface_name, addresses in psutil.net_if_addrs().items(): + if interface_name_substring not in interface_name.lower(): + continue + for address in addresses: + if address.family == socket.AF_INET: + return address.address + return None + + +def get_interface_ipv4_by_prefix(ipv4_prefix: str) -> str | None: + """ + Return the first IPv4 address on any network interface that starts with the given prefix. + :param ipv4_prefix: prefix to match against IPv4 address strings (e.g. "100.") + :return: the IPv4 address string, or None if none is found + """ + for addresses in psutil.net_if_addrs().values(): + for address in addresses: + if address.family == socket.AF_INET and address.address.startswith(ipv4_prefix): + return address.address + return None + + +def get_local_network_ip() -> str | None: + """ + Return the preferred private IPv4 address for local network access. + Uses UDP route probing and hostname resolution; prefers 192.* addresses when multiple candidates exist. + :return: the local network IPv4 address string, or None if none is found + """ + return _get_local_network_ipv4() diff --git a/packages/commons/octobot_commons/os_util.py b/packages/commons/octobot_commons/os_util.py index ecb434b4c2..e82fad513b 100644 --- a/packages/commons/octobot_commons/os_util.py +++ b/packages/commons/octobot_commons/os_util.py @@ -95,6 +95,22 @@ def get_octobot_type(): return enums.OctoBotTypes.BINARY.value +# PyInstaller 6.9+: environment entries for spawning an independent frozen child process. +# Required when re-execing or subprocess-spawning the same frozen executable so each child +# unpacks and runs as its own instance (see PyInstaller common issues and pitfalls). +PYINSTALLER_RESET_ENVIRONMENT_VARS = {"PYINSTALLER_RESET_ENVIRONMENT": "1"} + + +def is_frozen_binary_octobot() -> bool: + """ + Return whether OctoBot is running as a PyInstaller frozen binary. + + Uses PyInstaller's ``sys.frozen`` flag (not ``get_octobot_type()``), so ``python -m pytest`` + and other interpreter invocations are not misclassified as frozen binaries. + """ + return bool(getattr(sys, "frozen", False)) + + def get_os(): """ Return the OS name @@ -194,7 +210,7 @@ def tcp_port_is_free(bind_host: str, port: int) -> bool: return True -_HOST_WIDE_LISTENER_PROBE_HOSTS = ("0.0.0.0", "127.0.0.1") +_HOST_WIDE_LISTENER_PROBE_HOSTS = ("0.0.0.0", "127.0.0.1") # bind probes when psutil is unavailable def tcp_port_has_listener_on_host(port: int) -> bool: diff --git a/packages/commons/octobot_commons/process_util.py b/packages/commons/octobot_commons/process_util.py index 08b7d61b97..834db51af5 100644 --- a/packages/commons/octobot_commons/process_util.py +++ b/packages/commons/octobot_commons/process_util.py @@ -1,3 +1,4 @@ +# pylint: disable=C0415,R1732 # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -14,10 +15,12 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. +import asyncio import os import signal import subprocess import sys +import time import typing import octobot_commons.errors as commons_errors @@ -60,7 +63,7 @@ def spawn_managed_subprocess( else: child_stdout = subprocess.DEVNULL child_stderr = subprocess.DEVNULL - return subprocess.Popen( + proc = subprocess.Popen( argv, cwd=working_directory, env=resolved_env, @@ -68,6 +71,9 @@ def spawn_managed_subprocess( stdout=child_stdout, stderr=child_stderr, ) + import octobot_commons.managed_child_process_registry as managed_child_process_registry + managed_child_process_registry.ManagedChildProcessRegistry.instance().register(proc.pid) + return proc def pid_is_running(pid: int) -> bool: # pylint: disable=too-many-return-statements @@ -108,12 +114,12 @@ def request_graceful_stop_via_sigterm( """ resolved_logger = logger or commons_logging.get_logger(__name__) if pid <= 0: - raise commons_errors.DSLInterpreterError( + raise commons_errors.ProcessError( "Invalid pid for graceful stop via SIGTERM." ) sigterm = getattr(signal, "SIGTERM", None) if sigterm is None: - raise commons_errors.DSLInterpreterError( + raise commons_errors.ProcessError( "SIGTERM is not available on this platform." ) if not pid_is_running(pid): @@ -135,8 +141,100 @@ def request_graceful_stop_via_sigterm( resolved_logger.warning( "Graceful stop: failed to signal pid=%s: %s", pid, err ) - raise commons_errors.DSLInterpreterError( + raise commons_errors.ProcessError( f"Failed to send stop signal to pid={pid}: {err}" ) from err resolved_logger.info("Sent graceful stop signal (sigterm) to pid=%s", pid) return {"status": "stopped", "signal": "sigterm"} + + +def request_force_kill( + pid: int, + *, + logger: typing.Optional[typing.Any] = None, +) -> dict[str, typing.Any]: + """Force-kill the process identified by ``pid`` (SIGKILL / TerminateProcess).""" + resolved_logger = logger or commons_logging.get_logger(__name__) + if pid <= 0: + raise commons_errors.ProcessError("Invalid pid for force kill.") + if not pid_is_running(pid): + resolved_logger.info( + "Force kill: pid=%s not running, treating as already stopped", + pid, + ) + return {"status": "already_stopped", "reason": "not_running"} + try: + psutil.Process(pid).kill() + except psutil.NoSuchProcess: + resolved_logger.info( + "Force kill: pid=%s gone before kill", + pid, + ) + return {"status": "already_stopped", "reason": "not_running"} + except Exception as err: + if not pid_is_running(pid): + resolved_logger.info( + "Force kill: pid=%s gone after failed kill: %s", + pid, + err, + ) + return {"status": "already_stopped", "reason": str(err)} + resolved_logger.warning("Force kill failed for pid=%s: %s", pid, err) + raise commons_errors.ProcessError( + f"Failed to force kill pid={pid}: {err}" + ) from err + resolved_logger.info("Force killed pid=%s", pid) + return {"status": "force_killed"} + + +async def wait_until_pid_stopped_async( + pid: int, + *, + logger: typing.Optional[typing.Any] = None, + timeout_seconds: float, + poll_interval: float = 0.2, +) -> None: + """Poll until ``pid`` is gone or ``timeout_seconds`` elapses.""" + resolved_logger = logger or commons_logging.get_logger(__name__) + if pid <= 0: + resolved_logger.info( + "wait_until_pid_stopped_async: pid=%s treated as already stopped (non-positive)", + pid, + ) + return + resolved_logger.info( + "wait_until_pid_stopped_async: waiting for pid=%s to exit (timeout=%ss)", + pid, + timeout_seconds, + ) + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if not pid_is_running(pid): + resolved_logger.info("wait_until_pid_stopped_async: pid=%s exited", pid) + return + await asyncio.sleep(poll_interval) + raise commons_errors.ProcessError( + f"Timed out after {timeout_seconds}s waiting for pid={pid} to exit." + ) + + +def rebind_managed_child_pid(spawn_pid: int, authoritative_pid: int) -> None: + """Replace spawn pid with authoritative app pid in the managed-child registry.""" + import octobot_commons.managed_child_process_registry as managed_child_process_registry + managed_child_process_registry.ManagedChildProcessRegistry.instance().rebind_managed_child_pid( + spawn_pid, + authoritative_pid, + ) + + +async def graceful_stop_managed_children( + *, + timeout_seconds: float, + poll_interval: float = 0.2, +) -> dict[int, str]: + """Gracefully stop all registered managed children (see ManagedChildProcessRegistry).""" + import octobot_commons.managed_child_process_registry as managed_child_process_registry + return await managed_child_process_registry.ManagedChildProcessRegistry.instance().graceful_stop_all( + timeout_seconds=timeout_seconds, + poll_interval=poll_interval, + ) diff --git a/packages/commons/octobot_commons/profiles/__init__.py b/packages/commons/octobot_commons/profiles/__init__.py index 8c5b764d06..0f3612d912 100644 --- a/packages/commons/octobot_commons/profiles/__init__.py +++ b/packages/commons/octobot_commons/profiles/__init__.py @@ -1,3 +1,4 @@ +# pylint: disable=R0401 # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -15,11 +16,8 @@ # License along with this library. -from octobot_commons.profiles import profile - -from octobot_commons.profiles.profile import ( - Profile, -) +from octobot_commons.profiles.profile_types import Profile +from octobot_commons.profiles.profile_types import EphemeralProfile from octobot_commons.profiles import profile_sharing from octobot_commons.profiles.profile_sharing import ( @@ -41,9 +39,15 @@ OptionsData, ) -from octobot_commons.profiles import profile_sync +from octobot_commons.profiles import profile_storage + +from octobot_commons.profiles.profile_storage import ( + ProfileStorage, +) + +from octobot_commons.profiles import profile_synchronizer -from octobot_commons.profiles.profile_sync import ( +from octobot_commons.profiles.profile_synchronizer import ( start_profile_synchronizer, stop_profile_synchronizer, ) @@ -69,6 +73,8 @@ __all__ = [ "Profile", + "EphemeralProfile", + "ProfileStorage", "export_profile", "install_profile", "import_profile", diff --git a/packages/commons/octobot_commons/profiles/backends/__init__.py b/packages/commons/octobot_commons/profiles/backends/__init__.py new file mode 100644 index 0000000000..7221cb5a40 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/__init__.py @@ -0,0 +1,19 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from octobot_commons.profiles.backends.abstract_profile_backend import AbstractProfileBackend +from octobot_commons.profiles.backends.filesystem_profile_backend import FilesystemProfileBackend +from octobot_commons.profiles.backends.sync_profile_backend import SyncProfileBackend diff --git a/packages/commons/octobot_commons/profiles/backends/abstract_profile_backend.py b/packages/commons/octobot_commons/profiles/backends/abstract_profile_backend.py new file mode 100644 index 0000000000..cab7883c7a --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/abstract_profile_backend.py @@ -0,0 +1,94 @@ +# pylint: disable=C0116,R0801 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import abc +import typing + +import octobot_commons.enums as enums +import octobot_commons.profiles.profile_types.profile as profile_module + + +class AbstractProfileBackend(abc.ABC): + def __init__( + self, + profiles_path: str = None, + profile_schema_path: str = None, + sync_user_id: str = None, + ) -> None: + self._profiles_path = profiles_path + self._profile_schema_path = profile_schema_path + self._sync_user_id = sync_user_id + + def _is_sync_available(self) -> bool: + return bool(self._sync_user_id and str(self._sync_user_id).strip()) + + def _resolve_schema_path(self, schema_path: str = None) -> str: + return schema_path if schema_path is not None else self._profile_schema_path + + @property + @abc.abstractmethod + def source(self) -> enums.ProfileSource: + raise NotImplementedError + + @abc.abstractmethod + def list_profiles( + self, + schema_path: str = None, + ) -> dict[str, profile_module.Profile]: + raise NotImplementedError + + @abc.abstractmethod + def get_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + raise NotImplementedError + + @abc.abstractmethod + def save_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + raise NotImplementedError + + @abc.abstractmethod + def delete_profile( + self, + profile_id: str, + profile: typing.Optional[profile_module.Profile] = None, + ) -> None: + raise NotImplementedError + + @abc.abstractmethod + def list_profile_ids( + self, + ignore: str = None, + schema_path: str = None, + ) -> list[str]: + raise NotImplementedError + + @abc.abstractmethod + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + schema_path: str = None, + ) -> profile_module.Profile: + raise NotImplementedError diff --git a/packages/commons/octobot_commons/profiles/backends/filesystem_profile_backend.py b/packages/commons/octobot_commons/profiles/backends/filesystem_profile_backend.py new file mode 100644 index 0000000000..8ef0272699 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/filesystem_profile_backend.py @@ -0,0 +1,194 @@ +# pylint: disable=C0116,W0718 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import os +import shutil +import typing + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.enums as enums +import octobot_commons.json_util as json_util +import octobot_commons.logging as commons_logging +import octobot_commons.profiles.backends.abstract_profile_backend as abstract_profile_backend_module +import octobot_commons.profiles.profile_types.profile as profile_module + + +class FilesystemProfileBackend(abstract_profile_backend_module.AbstractProfileBackend): + @property + def source(self) -> enums.ProfileSource: + return enums.ProfileSource.FILESYSTEM + + @staticmethod + def config_file_path(profile_path: str) -> str: + return os.path.join(profile_path, constants.PROFILE_CONFIG_FILE) + + @staticmethod + def tentacles_config_path(profile_path: str) -> str: + return os.path.join(profile_path, constants.CONFIG_TENTACLES_FILE) + + def list_profiles( + self, + schema_path: str = None, + ) -> dict[str, profile_module.Profile]: + resolved_schema_path = self._resolve_schema_path(schema_path) + profiles = {} + for profile in self._scan_profiles(self._profiles_path, resolved_schema_path): + profiles[profile.profile_id] = profile + return profiles + + def get_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + resolved_schema_path = self._resolve_schema_path(schema_path) + for profile in self._scan_profiles(self._profiles_path, resolved_schema_path): + if profile.profile_id == profile_id: + return profile + return None + + def load_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> profile_module.Profile: + profile = self.get_profile(profile_id, schema_path) + if profile is None: + raise errors.NoProfileError(f"No profile with id: {profile_id}") + return profile + + def list_profile_ids( + self, + ignore: str = None, + schema_path: str = None, + ) -> list[str]: + resolved_schema_path = self._resolve_schema_path(schema_path) + return [ + profile.profile_id + for profile in self._scan_profiles( + self._profiles_path, resolved_schema_path, ignore=ignore + ) + ] + + def filesystem_profile_ids(self) -> set[str]: + if not os.path.isdir(self._profiles_path): + return set() + return { + entry + for entry in os.listdir(self._profiles_path) + if os.path.isdir(os.path.join(self._profiles_path, entry)) + } + + def read_profile_from_path( + self, + profile_path: str, + schema_path: str = None, + ) -> profile_module.Profile: + profile = self._load_profile_from_folder(profile_path, schema_path) + if profile is None: + raise errors.ProfileDataError( + f"No profile configuration found at '{profile_path}'" + ) + return profile + + def write_profile_config(self, profile: profile_module.Profile) -> None: + if profile.is_sync_backed(): + raise errors.ProfileDataError( + "FilesystemProfileBackend cannot write sync-backed profiles" + ) + json_util.safe_dump(profile.as_dict(), self.config_file_path(profile.path)) + + def resolve_avatar_path(self, profile: profile_module.Profile) -> None: + if profile.avatar and profile.path: + avatar_path = os.path.join(profile.path, profile.avatar) + if os.path.isfile(avatar_path): + profile.avatar_path = avatar_path + + def save_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + self.write_profile_config(profile) + + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + schema_path: str = None, + ) -> profile_module.Profile: + raise NotImplementedError("FilesystemProfileBackend cannot duplicate profiles") + + def delete_profile( + self, + profile_id: str, + profile: typing.Optional[profile_module.Profile] = None, + ) -> None: + if profile is not None and not profile.is_sync_backed(): + shutil.rmtree(profile.path) + return + profile_path = os.path.join(self._profiles_path, profile_id) + if os.path.isdir(profile_path): + shutil.rmtree(profile_path) + + def _scan_profiles( + self, + profiles_path: str, + schema_path: str = None, + ignore: str = None, + ) -> list[profile_module.Profile]: + profiles = [] + if not os.path.isdir(profiles_path): + return profiles + ignored_path = None if ignore is None else os.path.normpath(ignore) + for profile_entry in os.scandir(profiles_path): + if ( + ignored_path is not None + and os.path.normpath(profile_entry.path) == ignored_path + ): + continue + profile = self._load_profile_from_folder(profile_entry.path, schema_path) + if profile is not None: + profiles.append(profile) + return profiles + + def _load_profile_from_folder( + self, + profile_path: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + logger = commons_logging.get_logger("ProfileExplorer") + config_path = self.config_file_path(profile_path) + if not os.path.isfile(config_path): + logger.debug( + f"Ignored {profile_path} as it does not contain a profile configuration" + ) + return None + profile = profile_module.Profile(profile_path, schema_path=schema_path) + try: + profile.from_dict(json_util.read_file(config_path)) + self.resolve_avatar_path(profile) + return profile + except Exception as err: + logger.exception( + err, + True, + f"Ignored profile due to an error upon reading '{profile_path}': {err}", + ) + return None diff --git a/packages/commons/octobot_commons/profiles/backends/sync_profile_backend.py b/packages/commons/octobot_commons/profiles/backends/sync_profile_backend.py new file mode 100644 index 0000000000..44df25a658 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/backends/sync_profile_backend.py @@ -0,0 +1,347 @@ +# pylint: disable=C0116,C0415,W0212,W0603,W0718,R0913,W0613,C0412 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import datetime +import os +import typing +import uuid + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.enums as enums +import octobot_commons.logging as logging +import octobot_commons.profiles.backends.abstract_profile_backend as abstract_profile_backend_module +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.sync_profile as sync_profile_module + + +_LOGGER = None + + +def _get_logger(): + global _LOGGER + if _LOGGER is None: + _LOGGER = logging.get_logger("SyncProfileBackend") + return _LOGGER + + +class SyncProfileBackend(abstract_profile_backend_module.AbstractProfileBackend): + @property + def source(self) -> enums.ProfileSource: + return enums.ProfileSource.SYNC + + def list_profiles( + self, + schema_path: str = None, + ) -> dict[str, profile_module.Profile]: + if not self._is_sync_available(): + return {} + resolved_schema_path = self._resolve_schema_path(schema_path) + profiles = {} + try: + for strategy in self._list_profile_strategies(): + profile = self._strategy_to_profile(strategy, resolved_schema_path) + if profile is not None: + profiles[profile.profile_id] = profile + except Exception as err: + _get_logger().exception( + "Failed to list sync profiles for user %r: %s", + self._sync_user_id, + err, + ) + return {} + return profiles + + def get_profile( + self, + profile_id: str, + schema_path: str = None, + ) -> typing.Optional[profile_module.Profile]: + if not self._is_sync_available(): + return None + strategy = self._get_strategy(profile_id) + if strategy is None: + return None + return self._strategy_to_profile(strategy, self._resolve_schema_path(schema_path)) + + def save_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile save requires a configured wallet user id" + ) + if not profile.is_sync_backed(): + raise errors.ProfileDataError( + "SyncProfileBackend cannot save filesystem profiles" + ) + profile_data = self._profile_to_profile_data(profile, global_config) + self._validate_profile_data(profile_data) + strategy = self._profile_data_to_strategy(profile_data, profile) + strategy_provider = self._get_strategy_provider() + import octobot_sync.sync.collection_backend.errors as collection_errors + + try: + strategy_provider.update_item(self._sync_user_id, strategy) + except collection_errors.ItemNotFoundError: + strategy_provider.create_item(self._sync_user_id, strategy) + sync_profile = typing.cast(sync_profile_module.SyncProfile, profile) + sync_profile.set_profile_data(profile_data) + + def delete_profile( + self, + profile_id: str, + profile: typing.Optional[profile_module.Profile] = None, + ) -> None: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile delete requires a configured wallet user id" + ) + self._get_strategy_provider().delete_item(self._sync_user_id, profile_id) + + def list_profile_ids( + self, + ignore: str = None, + schema_path: str = None, + ) -> list[str]: + if not self._is_sync_available(): + return [] + return [strategy.id for strategy in self._list_profile_strategies()] + + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + schema_path: str = None, + ) -> sync_profile_module.SyncProfile: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile duplicate requires a configured wallet user id" + ) + if profile.is_sync_backed(): + profile_data = self._profile_to_profile_data( + profile, profile._global_config_from_profile() + ) + else: + profile_data = profile_data_module.ProfileData.from_filesystem_profile(profile) + profile_data.profile_details.id = uuid.uuid4().hex + duplicated_name = name or profile.name + duplicated_description = description if description is not None else profile.description + duplicate = self.import_profile_data( + profile_data, + schema_path=schema_path, + name=duplicated_name, + description=duplicated_description, + risk=profile.risk, + auto_update=False, + slug=profile.slug, + ) + duplicate.read_only = False + duplicate.imported = False + duplicate.origin_url = None + duplicate.description = duplicated_description + return duplicate + + def import_profile_data( + self, + profile_data: profile_data_module.ProfileData, + schema_path: str = None, + name: str = None, + description: str = None, + risk=None, + auto_update: bool = False, + slug: str = None, + force_simulator: bool = False, + ) -> sync_profile_module.SyncProfile: + if not self._is_sync_available(): + raise errors.ProfileDataError( + "Sync profile import requires a configured wallet user id" + ) + resolved_schema_path = self._resolve_schema_path(schema_path) + if profile_data.profile_details.id is None: + profile_data.profile_details.id = uuid.uuid4().hex + if name: + profile_data.profile_details.name = name + profile = self._profile_data_to_runtime_profile( + profile_data, resolved_schema_path + ) + if description is not None: + profile.description = description + if risk is not None: + profile.risk = risk + if slug is not None: + profile.slug = slug + profile.auto_update = auto_update + if force_simulator: + profile.config[constants.CONFIG_TRADER][ + constants.CONFIG_ENABLED_OPTION + ] = False + profile.config[constants.CONFIG_SIMULATOR][ + constants.CONFIG_ENABLED_OPTION + ] = True + self.save_profile(profile, profile._global_config_from_profile()) + return profile + + def _get_strategy_provider(self): + import octobot_sync.sync.collection_providers as collection_providers + + return collection_providers.StrategyProvider.instance() + + def _list_profile_strategies(self): + import octobot_sync.sync.collection_backend.errors as collection_errors + + try: + strategies = self._get_strategy_provider().list_items(self._sync_user_id) + except collection_errors.CollectionNoDataError: + return [] + return [ + strategy + for strategy in strategies + if self._is_profile_strategy(strategy) + ] + + def _get_strategy(self, profile_id: str): + try: + strategy = self._get_strategy_provider().get_item( + self._sync_user_id, profile_id + ) + except Exception: + return None + if not self._is_profile_strategy(strategy): + return None + return strategy + + def _is_profile_strategy(self, strategy) -> bool: + configuration = strategy.configuration + if configuration is None or configuration.actual_instance is None: + return False + import octobot_protocol.models.generic_process_configuration as generic_process_configuration + + return isinstance( + configuration.actual_instance, + generic_process_configuration.GenericProcessConfiguration, + ) and configuration.actual_instance.profile_data is not None + + def _strategy_to_profile( + self, + strategy, + schema_path: str, + ) -> typing.Optional[sync_profile_module.SyncProfile]: + configuration = strategy.configuration.actual_instance + profile_data_dict = configuration.profile_data + if profile_data_dict is None: + return None + profile_data = profile_data_module.ProfileData.from_dict(profile_data_dict) + if profile_data.profile_details.id != strategy.id: + profile_data.profile_details.id = strategy.id + runtime_path = self._runtime_profile_path(strategy.id) + profile = sync_profile_module.SyncProfile( + profile_data, + runtime_path, + schema_path=schema_path, + strategy_version=strategy.version, + ) + if strategy.name: + profile.name = strategy.name + if strategy.description: + profile.description = strategy.description + return profile + + def _profile_data_to_runtime_profile( + self, + profile_data: profile_data_module.ProfileData, + schema_path: str, + ) -> sync_profile_module.SyncProfile: + runtime_path = self._runtime_profile_path( + profile_data.profile_details.id + ) + return sync_profile_module.SyncProfile( + profile_data, runtime_path, schema_path=schema_path + ) + + def _profile_data_to_strategy( + self, + profile_data: profile_data_module.ProfileData, + profile: profile_module.Profile, + ): + import octobot_protocol.models as protocol_models + import octobot_protocol.models.action_configuration_type as action_configuration_type + import octobot_protocol.models.generic_process_configuration as generic_process_configuration + import octobot_protocol.models.strategy_configuration as strategy_configuration + + strategy_version = "1" + if isinstance(profile, sync_profile_module.SyncProfile): + strategy_version = profile.get_strategy_version() + generic_configuration = generic_process_configuration.GenericProcessConfiguration( + configuration_type=action_configuration_type.ActionConfigurationType.GENERIC_PROCESS, + profile_data=profile_data.to_dict(), + ) + now = datetime.datetime.now(datetime.timezone.utc) + return protocol_models.Strategy( + id=profile_data.profile_details.id, + version=strategy_version, + name=profile.name or profile_data.profile_details.name, + description=profile.description, + created_at=now, + updated_at=now, + reference_market=profile_data.trading.reference_market + or constants.DEFAULT_REFERENCE_MARKET, + configuration=strategy_configuration.StrategyConfiguration( + actual_instance=generic_configuration + ), + ) + + def _profile_to_profile_data( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> profile_data_module.ProfileData: + profile_data = profile_data_module.ProfileData.from_profile(profile) + profile_data.profile_details.id = profile.profile_id + profile_data.profile_details.name = profile.name + tentacles_data = profile.get_tentacles_data() + if tentacles_data is not None: + profile_data.tentacles = tentacles_data + elif profile.is_sync_backed(): + sync_profile = typing.cast(sync_profile_module.SyncProfile, profile) + profile_data.tentacles = list(sync_profile.get_profile_data().tentacles) + return profile_data + + def _validate_profile_data( + self, profile_data: profile_data_module.ProfileData + ) -> None: + if not profile_data.profile_details.id: + raise errors.ProfileDataError("profile_data.profile_details.id is required") + if ( + profile_data.profile_details.name is None + or profile_data.profile_details.name == "" + ): + raise errors.ProfileDataError("profile_data.profile_details.name is required") + + def _runtime_profile_path(self, profile_id: str) -> str: + import octobot_commons.user_root_folder_provider as user_root_folder_provider + + user_root = user_root_folder_provider.get_sync_data_root() + return os.path.join( + user_root, + constants.SYNC_PROFILE_RUNTIME_FOLDER, + profile_id, + ) diff --git a/packages/commons/octobot_commons/profiles/profile_data.py b/packages/commons/octobot_commons/profiles/profile_data.py index 8676b557eb..cf63781796 100644 --- a/packages/commons/octobot_commons/profiles/profile_data.py +++ b/packages/commons/octobot_commons/profiles/profile_data.py @@ -1,4 +1,4 @@ -# pylint: disable=C0103,R0902,C0301 +# pylint: disable=C0103,R0902,C0301,C0415,R0401 # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -18,7 +18,7 @@ import dataclasses import typing -import octobot_commons.profiles.profile as profile_import +import octobot_commons.profiles.profile_types.profile as profile_import import octobot_commons.dataclasses import octobot_commons.constants as constants @@ -154,6 +154,7 @@ class TentaclesData( ): name: typing.Optional[str] = None config: dict = dataclasses.field(default_factory=dict) + activated: bool = True @dataclasses.dataclass @@ -221,13 +222,33 @@ def __post_init__(self): ) @classmethod - def from_profile(cls, profile: profile_import.Profile): + def exchanges_from_profile_config(cls, profile_config: dict) -> list[ExchangeData]: + """ + Build enabled exchanges from a profile config dict. + """ + exchanges_config = profile_config.get(constants.CONFIG_EXCHANGES, {}) + enabled_exchanges = [] + for exchange_name, exchange_details in exchanges_config.items(): + if not exchange_details.get(constants.CONFIG_ENABLED_OPTION, False): + continue + enabled_exchanges.append( + ExchangeData( + internal_name=exchange_name, + exchange_type=exchange_details.get( + constants.CONFIG_EXCHANGE_TYPE, constants.DEFAULT_EXCHANGE_TYPE + ), + ) + ) + return enabled_exchanges + + @classmethod + def from_profile(cls, profile: "profile_import.Profile"): """ Creates a cls instance from the given profile """ profile_dict = profile.as_dict() content = profile_dict[constants.PROFILE_CONFIG] - return cls.from_dict( + profile_data = cls.from_dict( { "profile_details": { "id": profile_dict[constants.CONFIG_PROFILE][constants.CONFIG_ID], @@ -275,14 +296,27 @@ def from_profile(cls, profile: profile_import.Profile): "tentacles": [], } ) + profile_data.exchanges = cls.exchanges_from_profile_config(content) + return profile_data - def to_profile(self, to_create_profile_path: str) -> profile_import.Profile: + @classmethod + def from_filesystem_profile(cls, profile: "profile_import.Profile") -> "ProfileData": """ - Returns a new Profile from self + Build a complete ProfileData from a filesystem profile, including tentacles. """ - profile = profile_import.Profile(to_create_profile_path) - profile.from_dict(self._to_profile_dict()) - return profile + profile_data = cls.from_profile(profile) + profile_data.profile_details.id = profile.profile_id + profile_data.profile_details.name = profile.name + try: + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + except ImportError: + return profile_data + tentacles = profile_tentacles_util.collect_tentacles_data_from_filesystem_profile( + profile + ) + if tentacles is not None: + profile_data.tentacles = tentacles + return profile_data def set_tentacles_config(self, config_by_tentacle: dict): """ diff --git a/packages/commons/octobot_commons/profiles/profile_data_import.py b/packages/commons/octobot_commons/profiles/profile_data_import.py index 24082d07ce..04ad09419c 100644 --- a/packages/commons/octobot_commons/profiles/profile_data_import.py +++ b/packages/commons/octobot_commons/profiles/profile_data_import.py @@ -1,4 +1,4 @@ -# pylint: disable=R0913,W0718,W0706,C0415 +# pylint: disable=R0913,W0718,W0706,C0415,R0914 # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -19,9 +19,9 @@ import uuid import octobot_commons.profiles.profile_data as profile_data_import -import octobot_commons.profiles.profile as profile_import +import octobot_commons.profiles.profile_types.profile as profile_import +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.logging as bot_logging -import octobot_commons.json_util as json_util import octobot_commons.constants as constants import octobot_commons.aiohttp_util as aiohttp_util import octobot_commons.enums as enums @@ -86,14 +86,21 @@ async def convert_profile_data_to_profile_directory( # when updating profile, keep existing registered tentacles import_registered_tentacles = profile_to_update is not None # tentacles_config.json - tentacles_setup_config = _get_tentacles_setup_config( - profile_data, output_path, import_registered_tentacles - ) - if tentacles_setup_config.save_config(is_config_update=True): - changed = True - # specific_config - if _save_specific_config(profile_data, output_path, bool(profile_to_update)): - changed = True + try: + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + tentacles_setup_config = profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, output_path, import_registered_tentacles + ) + if tentacles_setup_config.save_config(is_config_update=True): + changed = True + # specific_config + if profile_tentacles_util.write_specific_configs_to_profile_folder( + profile_data, output_path, bool(profile_to_update) + ): + changed = True + except ImportError: + raise # avatar file if avatar_url: try: @@ -106,7 +113,11 @@ async def convert_profile_data_to_profile_directory( ) # finish with profile.json to include edits from previous methods if changed: - profile.save() + profile_storage = profile.get_profile_storage() + if profile_storage is not None: + profile.save() + else: + profile_backends_module.FilesystemProfileBackend().write_profile_config(profile) return changed @@ -119,14 +130,14 @@ def _get_profile( slug: str, force_simulator: bool, ): - profile = profile_data.to_profile(output_path) + profile = profile_import.Profile.from_profile_data(profile_data, output_path) if force_simulator: profile.config[constants.CONFIG_TRADER][constants.CONFIG_ENABLED_OPTION] = False profile.config[constants.CONFIG_SIMULATOR][ constants.CONFIG_ENABLED_OPTION ] = True profile.description = description - profile.risk = risk + profile.risk = risk if risk is not None else enums.ProfileRisk.MODERATE profile.auto_update = auto_update profile.slug = slug profile.profile_id = str(uuid.uuid4().hex) @@ -146,7 +157,7 @@ def get_updated_profile( :param profile_data: the profile_data to get the update from :return: True if something changed in the updated profile """ - updated_profile = profile_data.to_profile("") + updated_profile = profile_import.Profile.from_profile_data(profile_data, "") changed = False # update traded currencies (add new currencies) origin_currencies = copy.deepcopy( @@ -179,78 +190,6 @@ def get_updated_profile( return changed -def _get_tentacles_setup_config( - profile_data: profile_data_import.ProfileData, - output_path: str, - import_registered_tentacles: bool, -): - try: - import octobot_tentacles_manager.api - import octobot_tentacles_manager.constants - - classes = [ - octobot_tentacles_manager.api.get_tentacle_class_from_string( - tentacle_data.name - ).__name__ - for tentacle_data in profile_data.tentacles - if tentacle_data.name not in ( - octobot_tentacles_manager.constants.IGNORED_TENTACLES_NAMES_IN_TENTACLES_SETUP_CONFIG - ) - ] - config_path = os.path.join(output_path, constants.CONFIG_TENTACLES_FILE) - tentacles_setup_config = ( - octobot_tentacles_manager.api.create_tentacles_setup_config_with_tentacles( - *classes, config_path=config_path - ) - ) - use_reference_registered_tentacles = ( - not tentacles_setup_config.registered_tentacles - ) - octobot_tentacles_manager.api.fill_with_installed_tentacles( - tentacles_setup_config, - import_registered_tentacles=import_registered_tentacles, - use_reference_registered_tentacles=use_reference_registered_tentacles, - ) - return tentacles_setup_config - except ImportError: - raise - - -def _save_specific_config( - profile_data: profile_data_import.ProfileData, - output_path: str, - is_config_update: bool, -) -> bool: - changed = False - try: - import octobot_tentacles_manager.constants - - specific_config_dir = os.path.join( - output_path, - octobot_tentacles_manager.constants.TENTACLES_SPECIFIC_CONFIG_FOLDER, - ) - if not os.path.exists(specific_config_dir): - os.mkdir(specific_config_dir) - for tentacle_config in profile_data.tentacles: - file_path = os.path.join( - specific_config_dir, - f"{tentacle_config.name}{octobot_tentacles_manager.constants.CONFIG_EXT}", - ) - if is_config_update and json_util.has_same_content( - file_path, tentacle_config.config - ): - # nothing to do - continue - changed = True - json_util.safe_dump( - tentacle_config.config, - file_path, - ) - except ImportError: - raise - return changed - - async def _download_and_set_avatar( profile, avatar_url, output_path: str, aiohttp_session ): diff --git a/packages/commons/octobot_commons/profiles/profile_migration.py b/packages/commons/octobot_commons/profiles/profile_migration.py new file mode 100644 index 0000000000..e890c2915e --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_migration.py @@ -0,0 +1,71 @@ +# pylint: disable=W0212,C0415,R0401 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from __future__ import annotations + +import os +import shutil + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_storage as profile_storage_module + + +def migrate_user_profiles_to_sync( + profile_storage: profile_storage_module.ProfileStorage, +) -> list[str]: + """Migrate local filesystem profiles into the sync backend for the configured wallet.""" + if not profile_storage.is_sync_available(): + raise errors.ProfileDataError( + "Profile migration requires a configured wallet user id" + ) + migrated_profile_ids = [] + sync_backend = profile_storage._sync_backend + filesystem_ids = profile_storage.filesystem_profile_ids() + for profile_id in list(filesystem_ids): + profile = profile_storage.find_profile(profile_id) + if profile is None or profile.is_sync_backed(): + continue + if profile.read_only and not profile.imported: + continue + profile_data = profile_data_module.ProfileData.from_filesystem_profile(profile) + profile_data.profile_details.id = profile.profile_id + sync_backend.import_profile_data( + profile_data, schema_path=profile_storage.profile_schema_path + ) + _archive_filesystem_profile(profile_storage.profiles_path, profile_id) + migrated_profile_ids.append(profile_id) + return migrated_profile_ids + + +def _archive_filesystem_profile( + profiles_path: str, + profile_id: str, +) -> None: + source_path = os.path.join(profiles_path, profile_id) + if not os.path.isdir(source_path): + return + import octobot_commons.user_root_folder_provider as user_root_folder_provider + + user_root = user_root_folder_provider.get_sync_data_root() + migrated_root = os.path.join(user_root, constants.PROFILES_MIGRATED_FOLDER) + os.makedirs(migrated_root, exist_ok=True) + destination_path = os.path.join(migrated_root, profile_id) + if os.path.exists(destination_path): + shutil.rmtree(destination_path) + shutil.move(source_path, destination_path) diff --git a/packages/commons/octobot_commons/profiles/profile_sharing.py b/packages/commons/octobot_commons/profiles/profile_sharing.py index b978485e3f..7891b32049 100644 --- a/packages/commons/octobot_commons/profiles/profile_sharing.py +++ b/packages/commons/octobot_commons/profiles/profile_sharing.py @@ -1,4 +1,4 @@ -# pylint: disable=R0913,W0703 +# pylint: disable=R0913,W0703,R0914,R0401 # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -44,9 +44,11 @@ def __init__(self, *args): else: raise # avoid cyclic import -from octobot_commons.profiles.profile import Profile +from octobot_commons.profiles.profile_types.profile import Profile +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_data as profile_data_import import octobot_commons.profiles.profile_data_import as profile_data_importer +import octobot_commons.profiles.profile_storage as profile_storage_module import octobot_commons.user_root_folder_provider as user_root_folder_provider @@ -121,7 +123,10 @@ def install_profile( if not quite: logger.info(f"{action}ing {profile_name} profile.") _import_profile_files(import_path, target_import_path) - profile = Profile(target_import_path, schema_path=profile_schema).read_config() + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + profile = filesystem_backend.read_profile_from_path( + target_import_path, schema_path=profile_schema + ) profile.imported = is_imported profile.origin_url = origin_url _ensure_unique_profile_id(profile) @@ -133,7 +138,7 @@ def install_profile( raise errors.ProfileImportError( f"Invalid imported profile: {err.message} in '{'/'.join(err.absolute_path)}'" ) from err - profile.save() + filesystem_backend.write_profile_config(profile) if not quite: logger.info(f"{action}ed {profile.name} ({profile_name}) profile.") return profile @@ -165,8 +170,6 @@ def import_profile( origin_url=origin_url, profile_schema=profile_schema, ) - if profile.name != temp_profile_name: - profile.rename_folder(_get_unique_profile_folder_from_name(profile), False) return profile @@ -182,6 +185,7 @@ async def import_profile_data_as_profile( logo_url: str = None, auto_update: bool = False, force_simulator: bool = False, + profile_storage=None, ) -> Profile: """ Imports the given ProfileData into the user's profile directory with the "imported_" prefix @@ -198,6 +202,22 @@ async def import_profile_data_as_profile( :param force_simulator: True if trader simulator should be forced in config :return: The created profile """ + + if profile_storage is not None and profile_storage.is_sync_available(): + return await profile_storage.import_profile_data( + profile_data, + profile_schema, + bot_install_path, + name=name, + description=description, + risk=risk, + auto_update=auto_update, + slug=profile_data.profile_details.name, + logo_url=logo_url, + force_simulator=force_simulator, + aiohttp_session=aiohttp_session, + origin_url=origin_url, + ) logger = bot_logging.get_logger("ProfileSharing") import_path = f"{name}-{uuid.uuid4().hex}" try: @@ -423,10 +443,10 @@ def _ensure_unique_profile_id(profile) -> None: :param profile: the installed profile :return: None """ - ids = Profile.get_all_profiles_ids( - pathlib.Path(profile.path).parent, ignore=profile.path - ) + profiles_parent = str(pathlib.Path(profile.path).parent) + profile_storage = profile_storage_module.ProfileStorage(profiles_parent) + existing_ids = set(profile_storage.list_profile_ids(ignore=profile.path)) iteration = 1 - while profile.profile_id in ids and iteration < 100: + while profile.profile_id in existing_ids and iteration < 100: profile.profile_id = str(uuid.uuid4()) iteration += 1 diff --git a/packages/commons/octobot_commons/profiles/profile_storage.py b/packages/commons/octobot_commons/profiles/profile_storage.py new file mode 100644 index 0000000000..4021f036ae --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_storage.py @@ -0,0 +1,326 @@ +# pylint: disable=C0116,W0212,R0904,R0913,R0401 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import os +import typing + +import octobot_commons.authentication as authentication_module +import octobot_commons.enums as enums +import octobot_commons.errors as errors +import octobot_commons.logging as logging +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_migration as profile_migration +import octobot_commons.profiles.profile_sharing as profile_sharing + + +class ProfileStorage: + def __init__( + self, + profiles_path: str, + profile_schema_path: str = None, + filesystem_backend: profile_backends_module.FilesystemProfileBackend = None, + sync_backend: profile_backends_module.SyncProfileBackend = None, + ) -> None: + self._profiles_path = profiles_path + self._profile_schema_path = profile_schema_path + self._sync_user_id: typing.Optional[str] = None + self._readonly_profiles_path: typing.Optional[str] = None + self._readonly_filesystem_backend: typing.Optional[ + profile_backends_module.FilesystemProfileBackend + ] = None + self._filesystem_backend = ( + filesystem_backend + or profile_backends_module.FilesystemProfileBackend( + profiles_path, profile_schema_path + ) + ) + self._sync_backend = sync_backend or profile_backends_module.SyncProfileBackend( + profiles_path, profile_schema_path, sync_user_id=None + ) + + @property + def profiles_path(self) -> str: + return self._profiles_path + + @property + def profile_schema_path(self) -> str: + return self._profile_schema_path + + def configure_sync_user(self, user_id: str) -> None: + try: + authentication_module.Authenticator.instance().get_wallet_by_user_id(user_id) + except Exception as error: + raise errors.ProfileDataError( + f"Unknown sync user id: {user_id}" + ) from error + self._sync_user_id = user_id + self._sync_backend._sync_user_id = user_id + + def bind_process_child_sync_user_id(self, user_id: str) -> None: + if not user_id or not str(user_id).strip(): + raise errors.ProfileDataError("Process child sync user id must be non-empty") + self._sync_user_id = str(user_id) + self._sync_backend._sync_user_id = self._sync_user_id + + def is_master_overlay_profile(self, profile: profile_module.Profile) -> bool: + if not self._readonly_profiles_path or profile.path is None: + return False + normalized_profile_path = os.path.normpath(profile.path) + readonly_prefix = self._readonly_profiles_path + if not readonly_prefix.endswith(os.sep): + readonly_prefix = f"{readonly_prefix}{os.sep}" + return normalized_profile_path == self._readonly_profiles_path or normalized_profile_path.startswith( + readonly_prefix + ) + + def is_readonly_master_overlay_profile(self, profile: profile_module.Profile) -> bool: + return self.is_master_overlay_profile(profile) and profile.read_only + + def configure_readonly_profiles_path(self, path: str) -> None: + normalized_path = os.path.normpath(path) + self._readonly_profiles_path = normalized_path + self._readonly_filesystem_backend = profile_backends_module.FilesystemProfileBackend( + normalized_path, self._profile_schema_path + ) + + def configure_paths( + self, + profiles_path: str, + profile_schema_path: str = None, + ) -> None: + self._profiles_path = profiles_path + if profile_schema_path is not None: + self._profile_schema_path = profile_schema_path + self._filesystem_backend._profiles_path = profiles_path + self._sync_backend._profiles_path = profiles_path + if profile_schema_path is not None: + self._filesystem_backend._profile_schema_path = profile_schema_path + self._sync_backend._profile_schema_path = profile_schema_path + if self._readonly_filesystem_backend is not None: + self._readonly_filesystem_backend._profile_schema_path = profile_schema_path + + def is_sync_available(self) -> bool: + return bool(self._sync_user_id and str(self._sync_user_id).strip()) + + def load_all_profiles(self) -> dict[str, profile_module.Profile]: + if self._profiles_path is None: + raise errors.ProfileDataError("profiles_path is required to load profiles") + loaded_profiles = self._list_profiles() + for profile in loaded_profiles.values(): + profile.bind_profile_storage(self) + return loaded_profiles + + def list_sync_profiles(self) -> dict[str, profile_module.Profile]: + profiles = self._sync_backend.list_profiles(self._profile_schema_path) + for profile in profiles.values(): + profile.bind_profile_storage(self) + return profiles + + def find_profile( + self, + profile_id: str, + ) -> typing.Optional[profile_module.Profile]: + return self._resolve_profile(profile_id) + + def get_profile( + self, + profile_id: str, + ) -> typing.Optional[profile_module.Profile]: + profile = self.find_profile(profile_id) + if profile is not None: + profile.bind_profile_storage(self) + return profile + + def load_profile_by_id(self, profile_id: str) -> profile_module.Profile: + profile = self.get_profile(profile_id) + if profile is None: + raise errors.NoProfileError(f"No profile with id: {profile_id}") + return profile + + def filesystem_profile_ids(self) -> set[str]: + return self._filesystem_backend.filesystem_profile_ids() + + def list_profile_ids(self, ignore: str = None) -> list[str]: + filesystem_ids = self._filesystem_backend.list_profile_ids(ignore=ignore) + overlay_ids = self._master_overlay_profile_ids(ignore=ignore) + sync_ids = self._sync_backend.list_profile_ids(ignore=ignore) + return list(dict.fromkeys(filesystem_ids + overlay_ids + sync_ids)) + + def duplicate_profile( + self, + profile: profile_module.Profile, + name: str = None, + description: str = None, + ) -> profile_module.Profile: + if not self.is_sync_available(): + raise errors.ProfileDataError( + "Profile duplicate requires a configured wallet user id" + ) + clone = self._sync_backend.duplicate_profile( + profile, + name=name, + description=description, + ) + clone.bind_profile_storage(self) + return clone + + def activate_profile(self, profile: profile_module.Profile) -> None: + profile.bind_profile_storage(self) + profile.init_tentacles_setup_config() + + def save_active_profile( + self, + profile: profile_module.Profile, + global_config: dict, + ) -> None: + if self.is_readonly_master_overlay_profile(profile): + raise errors.ProfileDataError( + f"{profile.name} profile is shared from the master and can't be saved" + ) + backend = self._get_backend_for_profile(profile) + logging.get_logger(self.__class__.__name__).info( + f"Saving {profile.name} {profile.__class__.__name__} with " + f"{backend.__class__.__name__}" + ) + backend.save_profile(profile, global_config) + if profile.get_storage_source() == enums.ProfileSource.FILESYSTEM: + tentacles_setup_config = profile.tentacles_setup_config + if tentacles_setup_config is not None: + tentacles_setup_config.save_config(is_config_update=True) + + def delete_profile( + self, + profile_id: str, + profile: profile_module.Profile = None, + ) -> None: + if profile is None: + profile = self.find_profile(profile_id) + if profile is None: + raise errors.ProfileRemovalError(f"Profile {profile_id} not found") + if self.is_readonly_master_overlay_profile(profile): + raise errors.ProfileRemovalError( + f"{profile.name} profile is shared from the master and can't be removed" + ) + backend = self._get_backend_for_profile(profile) + if profile.read_only and not profile.imported: + raise errors.ProfileRemovalError(f"{profile.name} profile can't be removed") + backend.delete_profile(profile_id, profile=profile) + + def has_any_profiles(self) -> bool: + if self._profiles_path is None: + return False + return self._has_any_profiles() + + async def import_profile_data( + self, + profile_data: profile_data_module.ProfileData, + profile_schema: str, + bot_install_path: str, + name: str = None, + description: str = None, + risk=enums.ProfileRisk.MODERATE, + auto_update: bool = False, + slug: str = None, + logo_url: str = None, + force_simulator: bool = False, + aiohttp_session=None, + origin_url: str = None, + ) -> profile_module.Profile: + if self.is_sync_available(): + profile = self._sync_backend.import_profile_data( + profile_data, + schema_path=profile_schema, + name=name, + description=description, + risk=risk, + auto_update=auto_update, + slug=slug, + force_simulator=force_simulator, + ) + profile.bind_profile_storage(self) + return profile + return await profile_sharing.import_profile_data_as_profile( + profile_data, + profile_schema, + aiohttp_session, + name=name, + description=description, + risk=risk, + bot_install_path=bot_install_path, + origin_url=origin_url, + logo_url=logo_url, + auto_update=auto_update, + force_simulator=force_simulator, + profile_storage=self, + ) + + def migrate_filesystem_profiles_to_sync(self) -> list[str]: + return profile_migration.migrate_user_profiles_to_sync(self) + + def _master_overlay_profiles(self) -> dict[str, profile_module.Profile]: + if self._readonly_filesystem_backend is None: + return {} + return self._readonly_filesystem_backend.list_profiles() + + def _master_overlay_profile_ids(self, ignore: str = None) -> list[str]: + overlay_ids = [] + for profile_id in self._master_overlay_profiles(): + if ignore is not None and profile_id == ignore: + continue + overlay_ids.append(profile_id) + return overlay_ids + + def _list_profiles(self) -> dict[str, profile_module.Profile]: + profiles = self._sync_backend.list_profiles() + for profile_id, profile in self._master_overlay_profiles().items(): + if profile_id not in profiles: + profiles[profile_id] = profile + profiles.update(self._filesystem_backend.list_profiles()) + return profiles + + def _resolve_profile( + self, + profile_id: str, + ) -> typing.Optional[profile_module.Profile]: + filesystem_profile = self._filesystem_backend.get_profile(profile_id) + if filesystem_profile is not None: + return filesystem_profile + sync_profile = self._sync_backend.get_profile(profile_id) + if sync_profile is not None: + return sync_profile + if self._readonly_filesystem_backend is None: + return None + return self._readonly_filesystem_backend.get_profile(profile_id) + + def _ensure_profile_persistable(self, profile: profile_module.Profile) -> None: + if profile.get_storage_source() == enums.ProfileSource.EPHEMERAL: + raise errors.ProfileDataError("Ephemeral profiles cannot be persisted") + + def _get_backend_for_profile( + self, profile: profile_module.Profile + ) -> profile_backends_module.AbstractProfileBackend: + self._ensure_profile_persistable(profile) + if profile.is_sync_backed(): + return self._sync_backend + if self.is_master_overlay_profile(profile) and self._readonly_filesystem_backend is not None: + return self._readonly_filesystem_backend + return self._filesystem_backend + + def _has_any_profiles(self) -> bool: + return bool(self._list_profiles()) diff --git a/packages/commons/octobot_commons/profiles/profile_sync.py b/packages/commons/octobot_commons/profiles/profile_synchronizer.py similarity index 99% rename from packages/commons/octobot_commons/profiles/profile_sync.py rename to packages/commons/octobot_commons/profiles/profile_synchronizer.py index a6d5b73029..e36b737ea8 100644 --- a/packages/commons/octobot_commons/profiles/profile_sync.py +++ b/packages/commons/octobot_commons/profiles/profile_synchronizer.py @@ -88,6 +88,7 @@ def stop(self): self.sync_job.stop() + async def start_profile_synchronizer(current_config, on_profile_change): """ Start the clock synchronization loop if possible on this system diff --git a/packages/commons/octobot_commons/profiles/profile_types/__init__.py b/packages/commons/octobot_commons/profiles/profile_types/__init__.py new file mode 100644 index 0000000000..3c1d933f88 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/__init__.py @@ -0,0 +1,29 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from octobot_commons.profiles.profile_types.profile import Profile +from octobot_commons.profiles.profile_types.profile_data_backed_profile import ( + ProfileDataBackedProfile, +) +from octobot_commons.profiles.profile_types.sync_profile import SyncProfile +from octobot_commons.profiles.profile_types.ephemeral_profile import EphemeralProfile + +__all__ = [ + "Profile", + "ProfileDataBackedProfile", + "SyncProfile", + "EphemeralProfile", +] diff --git a/packages/commons/octobot_commons/profiles/profile_types/ephemeral_profile.py b/packages/commons/octobot_commons/profiles/profile_types/ephemeral_profile.py new file mode 100644 index 0000000000..3f8e7432cc --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/ephemeral_profile.py @@ -0,0 +1,62 @@ +# pylint: disable=W0237 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from __future__ import annotations + +import octobot_commons.enums as enums +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.profile_data_backed_profile as profile_data_backed_profile_module + + +class EphemeralProfile(profile_data_backed_profile_module.ProfileDataBackedProfile): + """ + Short-lived RAM-only profile backed by ProfileData. + """ + + @classmethod + def from_profile_data( + cls, + profile_data: profile_data_module.ProfileData, + schema_path: str = None, + ) -> EphemeralProfile: + return cls(profile_data, schema_path=schema_path) + + def __init__( + self, + profile_data: profile_data_module.ProfileData, + schema_path: str = None, + ): + super().__init__(profile_data, profile_path=None, schema_path=schema_path) + + def is_sync_backed(self) -> bool: + return False + + def get_storage_source(self) -> enums.ProfileSource: + return enums.ProfileSource.EPHEMERAL + + def validate_and_save_config(self) -> None: + raise errors.ProfileDataError("Ephemeral profiles cannot be saved") + + def save(self) -> None: + raise errors.ProfileDataError("Ephemeral profiles cannot be saved") + + def delete(self) -> None: + raise errors.ProfileDataError("Ephemeral profiles cannot be deleted") + + def duplicate(self, name: str = None, description: str = None): + raise errors.ProfileDataError("Ephemeral profiles cannot be duplicated") diff --git a/packages/commons/octobot_commons/profiles/profile.py b/packages/commons/octobot_commons/profiles/profile_types/profile.py similarity index 71% rename from packages/commons/octobot_commons/profiles/profile.py rename to packages/commons/octobot_commons/profiles/profile_types/profile.py index 98a0b693c1..c2173f05b2 100644 --- a/packages/commons/octobot_commons/profiles/profile.py +++ b/packages/commons/octobot_commons/profiles/profile_types/profile.py @@ -1,4 +1,4 @@ -# pylint: disable=R0902, W0703 +# pylint: disable=R0902,W0703,C0116,W0212,C0415,R0904,R0401 # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # @@ -14,9 +14,11 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +from __future__ import annotations + import copy import os -import shutil +import typing import uuid import octobot_commons.constants as constants import octobot_commons.enums as enums @@ -79,13 +81,39 @@ def __init__(self, profile_path: str, schema_path: str = None): self.extra_backtesting_time_frames = [] self.config: dict = {} + self.tentacles_setup_config = None + self._profile_storage = None - def read_config(self): - """ - Reads a profile from self.path - :return: self - """ - return self.from_dict(json_util.read_file(self.config_file())) + def bind_profile_storage(self, profile_storage) -> None: + self._profile_storage = profile_storage + + def get_profile_storage(self): + return self._profile_storage + + def _require_profile_storage(self): + profile_storage = self._profile_storage + if profile_storage is None: + raise errors.ProfileDataError("ProfileStorage is not bound to this profile") + return profile_storage + + def is_sync_backed(self) -> bool: + return False + + def is_profile_data_tentacle_backed(self) -> bool: + return False + + def get_storage_source(self): + return enums.ProfileSource.FILESYSTEM + + @classmethod + def from_profile_data( + cls, + profile_data, + to_create_profile_path: str, + ) -> Profile: + profile = cls(to_create_profile_path) + profile.from_dict(profile_data._to_profile_dict()) + return profile def from_dict(self, profile_dict: dict): """ @@ -104,24 +132,21 @@ def from_dict(self, profile_dict: dict): self.hidden = profile_config.get(constants.CONFIG_HIDDEN, False) self.imported = profile_config.get(constants.CONFIG_IMPORTED, False) self.complexity = enums.ProfileComplexity( - profile_config.get( - constants.CONFIG_COMPLEXITY, enums.ProfileComplexity.MEDIUM.value - ) + profile_config.get(constants.CONFIG_COMPLEXITY) + or enums.ProfileComplexity.MEDIUM.value ) self.risk = enums.ProfileRisk( - profile_config.get(constants.CONFIG_RISK, enums.ProfileRisk.MODERATE.value) + profile_config.get(constants.CONFIG_RISK) + or enums.ProfileRisk.MODERATE.value ) self.profile_type = enums.ProfileType( - profile_config.get(constants.CONFIG_TYPE, enums.ProfileType.LIVE.value) + profile_config.get(constants.CONFIG_TYPE) + or enums.ProfileType.LIVE.value ) self.extra_backtesting_time_frames = profile_config.get( constants.CONFIG_EXTRA_BACKTESTING_TIME_FRAMES, [] ) self.config = self.apply_default_values(profile_dict[constants.PROFILE_CONFIG]) - if self.avatar and self.path: - avatar_path = os.path.join(self.path, self.avatar) - if os.path.isfile(avatar_path): - self.avatar_path = avatar_path return self def save_config(self, global_config: dict): @@ -134,7 +159,7 @@ def save_config(self, global_config: dict): if element in global_config: self.config[element] = global_config[element] self.sync_partially_managed_elements(global_config) - self.validate_and_save_config() + self._save_through_profile_storage(global_config) def remove_deleted_elements(self, global_config): """ @@ -177,42 +202,18 @@ def validate(self): def validate_and_save_config(self) -> None: """ - JSON validates this profile and then saves its configuration file + JSON validates this profile and then saves its configuration :return: None """ self.validate() - self.save() + self._save_through_profile_storage(self._global_config_from_profile()) def save(self) -> None: """ - Saves the current profile configuration file + Saves the current profile configuration :return: None """ - json_util.safe_dump(self.as_dict(), self.config_file()) - - def rename_folder(self, new_name, should_raise) -> str: - """ - rename the profile folder - :param new_name: name of the new folder - :param should_raise: raises ProfileConflictError if the profile can't be renamed - :return: the new profile path - """ - new_path = os.path.join(os.path.split(self.path)[0], new_name) - if os.path.exists(new_path): - if should_raise: - raise errors.ProfileConflictError( - "Skipping folder renaming: a profile already exists at this path" - ) - return self.path - try: - os.rename(self.path, new_path) - self.path = new_path - except Exception as err: - commons_logging.get_logger("ProfileRenamer").error( - f"Error when renaming profile: {err}" - ) - raise errors.ProfileConflictError from err - return self.path + self.validate_and_save_config() def duplicate(self, name: str = None, description: str = None): """ @@ -221,25 +222,22 @@ def duplicate(self, name: str = None, description: str = None): :param description: description of the profile to create, uses the original's one by default :return: the created profile """ - clone = copy.deepcopy(self) - clone.name = name or clone.name - clone.description = description or clone.description - clone.profile_id = str(uuid.uuid4()) - clone.read_only = False - clone.imported = False - clone.origin_url = None - clone.auto_update = False - try: - clone.path = os.path.join( - os.path.split(self.path)[0], f"{clone.name}_{clone.profile_id}" - ) - shutil.copytree(self.path, clone.path) - except OSError: - # invalid profile name for a filename - clone.path = os.path.join(os.path.split(self.path)[0], clone.profile_id) - shutil.copytree(self.path, clone.path) - clone.save() - return clone + return self._require_profile_storage().duplicate_profile(self, name=name, description=description) + + def delete(self) -> None: + self._require_profile_storage().delete_profile(self.profile_id, profile=self) + + def _save_through_profile_storage(self, global_config: dict) -> None: + self._require_profile_storage().save_active_profile(self, global_config) + + def _global_config_from_profile(self) -> dict: + global_config = {} + for element in self.FULLY_MANAGED_ELEMENTS: + global_config[element] = self.config.get(element, {}) + for element in self.PARTIALLY_MANAGED_ELEMENTS: + if element in self.config: + global_config[element] = self.config[element] + return global_config def get_tentacles_config_path(self) -> str: """ @@ -247,6 +245,51 @@ def get_tentacles_config_path(self) -> str: """ return os.path.join(self.path, constants.CONFIG_TENTACLES_FILE) + def init_tentacles_setup_config(self) -> None: + setup = self._build_tentacles_setup_config() + self.bind_tentacles_setup_config(setup) + + def bind_tentacles_setup_config(self, tentacles_setup_config): + """ + Link profile and setup config both ways (setup.profile + profile.tentacles_setup_config). + """ + tentacles_setup_config.profile = self + self.tentacles_setup_config = tentacles_setup_config + return tentacles_setup_config + + def get_tentacles_data(self) -> typing.Optional[list]: + tentacles_setup_config = self.tentacles_setup_config + if tentacles_setup_config is None: + return None + try: + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + except ImportError: + return self._get_tentacles_data_without_tentacles_manager() + return profile_tentacles_util.collect_tentacles_data_from_setup( + tentacles_setup_config + ) + + def _get_tentacles_data_without_tentacles_manager(self) -> typing.Optional[list]: + return None + + def _build_tentacles_setup_config(self): + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + os.makedirs(self.path, exist_ok=True) + tentacles_config_path = self.get_tentacles_config_path() + if os.path.isfile(tentacles_config_path): + return profile_tentacles_util.load_setup_config_from_profile_path( + tentacles_config_path + ) + try: + profile_data = profile_data_module.ProfileData.from_filesystem_profile(self) + except (KeyError, OSError, TypeError): + profile_data = profile_data_module.ProfileData() + return profile_tentacles_util.build_setup_config_from_profile_data( + profile_data, self.path, import_registered_tentacles=True + ) + def as_dict(self) -> dict: """ :return: A dict representation of this profile configuration @@ -275,12 +318,6 @@ def as_dict(self) -> dict: constants.PROFILE_CONFIG: self.config, } - def config_file(self): - """ - :return: the path to this profile config file - """ - return os.path.join(self.path, constants.PROFILE_CONFIG_FILE) - def merge_partially_managed_element_into_config(self, config: dict, element: str): """ Merge this profile configuration's partially managed element into the given config @@ -373,71 +410,6 @@ def _filter_fill_elements( if key in allowed_keys: profile_config[element][key] = value - @staticmethod - def load_profile(profiles_path, profile_id, schema_path: str = None): - """ - :param profiles_path: the path to look for the profile - :param profile_id: the required profile id - :return: the loaded profile - """ - for profile in Profile.get_all_profiles(profiles_path, schema_path=schema_path): - if profile.profile_id == profile_id: - return profile - raise errors.NoProfileError(f"No profile with id: {profile_id}") - - @staticmethod - def get_all_profiles(profiles_path, ignore: str = None, schema_path: str = None): - """ - Loads profiles found in the given directory - :param profiles_path: Path to a directory containing profiles - :param ignore: A profile path to ignore - :param schema_path: Path to the json schema to pass to the created profile instances - :return: the profile instances list - """ - profiles = [] - ignored_path = None if ignore is None else os.path.normpath(ignore) - for profile_entry in os.scandir(profiles_path): - if ( - ignored_path is None - or os.path.normpath(profile_entry.path) != ignored_path - ): - profile = Profile._load_profile(profile_entry.path, schema_path) - if profile is not None: - profiles.append(profile) - return profiles - - @staticmethod - def _load_profile(profile_path: str, schema_path: str): - logger = commons_logging.get_logger("ProfileExplorer") - profile = Profile(profile_path, schema_path) - try: - if os.path.isfile(profile.config_file()): - profile.read_config() - return profile - logger.debug( - f"Ignored {profile_path} as it does not contain a profile configuration" - ) - except Exception as err: - logger.exception( - err, - True, - f"Ignored profile due to an error upon reading '{profile_path}': {err}", - ) - return None - - @staticmethod - def get_all_profiles_ids(profiles_path, ignore: str = None): - """ - Get ids of profiles found in the given directory - :param profiles_path: Path to a directory containing profiles - :param ignore: A profile path to ignore in ids listing - :return: the profile ids list - """ - return [ - profile.profile_id - for profile in Profile.get_all_profiles(profiles_path, ignore) - ] - @staticmethod def apply_default_values(config: dict) -> dict: """ diff --git a/packages/commons/octobot_commons/profiles/profile_types/profile_data_backed_profile.py b/packages/commons/octobot_commons/profiles/profile_types/profile_data_backed_profile.py new file mode 100644 index 0000000000..859160ce93 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/profile_data_backed_profile.py @@ -0,0 +1,84 @@ +# pylint: disable=C0116,C0415,C0412 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from __future__ import annotations + +import typing + +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.profile_data as profile_data_module + + +class ProfileDataBackedProfile(profile_module.Profile): + """ + Profile facade with tentacle config stored in profile_data (RAM only). + """ + + def __init__( + self, + profile_data: profile_data_module.ProfileData, + profile_path: str = None, + schema_path: str = None, + ): + super().__init__(profile_path, schema_path=schema_path) + self._profile_data = profile_data + self.from_dict(profile_data._to_profile_dict()) + if profile_data.profile_details.id: + self.profile_id = profile_data.profile_details.id + if profile_data.profile_details.name: + self.name = profile_data.profile_details.name + + def is_profile_data_tentacle_backed(self) -> bool: + return True + + def get_profile_data(self) -> profile_data_module.ProfileData: + return self._profile_data + + def get_tentacles_config_path(self) -> str: + raise errors.ProfileDataError( + "Profile data backed profiles have no filesystem tentacles config path" + ) + + def _build_tentacles_setup_config(self): + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + return profile_tentacles_util.build_setup_config_from_profile_data( + self.get_profile_data(), output_path=None, import_registered_tentacles=False + ) + + def _get_tentacles_data_without_tentacles_manager(self) -> list: + return list(self.get_profile_data().tentacles) + + def get_tentacles_data(self) -> typing.Optional[list]: + tentacles_setup_config = self.tentacles_setup_config + if tentacles_setup_config is None: + return None + import octobot_tentacles_manager.configuration.profile_tentacles_util as profile_tentacles_util + + specific_configs_by_tentacle_name = { + tentacle_data.name: tentacle_data.config or {} + for tentacle_data in self.get_profile_data().tentacles + } + collected_tentacles_data = profile_tentacles_util.collect_tentacles_data_from_setup( + tentacles_setup_config, + specific_configs_by_tentacle_name=specific_configs_by_tentacle_name, + ) + return profile_tentacles_util.merge_inactive_tentacles_data_from_profile( + collected_tentacles_data, + self.get_profile_data(), + ) diff --git a/packages/commons/octobot_commons/profiles/profile_types/sync_profile.py b/packages/commons/octobot_commons/profiles/profile_types/sync_profile.py new file mode 100644 index 0000000000..ec5c706627 --- /dev/null +++ b/packages/commons/octobot_commons/profiles/profile_types/sync_profile.py @@ -0,0 +1,51 @@ +# pylint: disable=C0116,W0212,R0902 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import octobot_commons.enums as enums +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.profile_data_backed_profile as profile_data_backed_profile_module + + +class SyncProfile(profile_data_backed_profile_module.ProfileDataBackedProfile): + """ + Profile facade backed by StrategyProvider profile_data. + """ + + def __init__( + self, + profile_data: profile_data_module.ProfileData, + runtime_path: str, + schema_path: str = None, + strategy_version: str = "1", + ): + super().__init__(profile_data, profile_path=runtime_path, schema_path=schema_path) + self._strategy_version = strategy_version + + def is_sync_backed(self) -> bool: + return True + + def get_storage_source(self) -> enums.ProfileSource: + return enums.ProfileSource.SYNC + + def get_strategy_version(self) -> str: + return self._strategy_version + + def set_profile_data(self, profile_data: profile_data_module.ProfileData) -> None: + self._profile_data = profile_data + self.from_dict(profile_data._to_profile_dict()) + if profile_data.profile_details.id: + self.profile_id = profile_data.profile_details.id diff --git a/packages/commons/octobot_commons/tentacles_management/class_inspector.py b/packages/commons/octobot_commons/tentacles_management/class_inspector.py index 1108225dff..f19bb6606d 100644 --- a/packages/commons/octobot_commons/tentacles_management/class_inspector.py +++ b/packages/commons/octobot_commons/tentacles_management/class_inspector.py @@ -93,12 +93,13 @@ def get_deep_class_from_parent_subclasses(class_string, parent): return None -def get_class_from_string( +def get_class_from_string( # pylint: disable=too-many-arguments class_string: str, parent, module, parent_inspection=default_parent_inspection, error_when_not_found: bool = False, + case_insensitive: bool = False, ): """ Search a class from a class string in a specified module for a specified parent @@ -107,16 +108,22 @@ def get_class_from_string( :param module: the class expected module :param parent_inspection: the parent inspection :param error_when_not_found: if errors should be raised + :param case_insensitive: if True, match class names without case sensitivity :return: the class if found else None """ + def _name_matches(member_name: str) -> bool: + if case_insensitive: + return member_name.lower() == class_string.lower() + return member_name == class_string + if tentacle_class_by_name := { - m[0]: m[1] - for m in inspect.getmembers(module) - if (m[0] == class_string) - and hasattr(m[1], "__bases__") - and parent_inspection(m[1], parent) + member_name: member_class + for member_name, member_class in inspect.getmembers(module) + if _name_matches(member_name) + and hasattr(member_class, "__bases__") + and parent_inspection(member_class, parent) }: - return tentacle_class_by_name[class_string] + return next(iter(tentacle_class_by_name.values())) if error_when_not_found: raise ModuleNotFoundError(f"Cant find {class_string} module") return None # no class found diff --git a/packages/commons/octobot_commons/user_root_folder_provider.py b/packages/commons/octobot_commons/user_root_folder_provider.py index fd659a1d1a..09d0585360 100644 --- a/packages/commons/octobot_commons/user_root_folder_provider.py +++ b/packages/commons/octobot_commons/user_root_folder_provider.py @@ -34,6 +34,7 @@ class UserRootFolderProvider(singleton_class.Singleton): def __init__(self) -> None: """Initialize with no explicit root (``get_root`` falls back to ``USER_FOLDER``).""" self._root: typing.Optional[str] = None + self._readonly_reference_tentacles_path: typing.Optional[str] = None def get_root(self) -> str: """Return the configured user data root, or ``commons_constants.USER_FOLDER`` if unset.""" @@ -49,8 +50,17 @@ def get_user_profiles_folder(self) -> str: """Return the profiles folder path under the user root.""" return os.path.join(self.get_root(), commons_constants.PROFILES_FOLDER) + def configure_readonly_reference_tentacles_path(self, path: str) -> None: + """Use a shared reference tentacles config directory (e.g. master user root).""" + if not path or not str(path).strip(): + self._readonly_reference_tentacles_path = None + return + self._readonly_reference_tentacles_path = os.path.normpath(path) + def get_user_reference_tentacle_config_path(self) -> str: """Return the reference tentacles config directory under the user root.""" + if self._readonly_reference_tentacles_path: + return self._readonly_reference_tentacles_path return os.path.join(self.get_root(), _REFERENCE_TENTACLES_CONFIG_DIR) def get_user_reference_tentacle_config_file_path(self) -> str: @@ -78,6 +88,17 @@ def get_user_root_folder() -> str: return UserRootFolderProvider.instance().get_root() +def get_sync_data_root() -> str: + """ + Master sync data root (StrategyProvider storage, sync profile runtime, master wallets). + Child automation processes set ENV_OCTOBOT_SYNC_DATA_ROOT to the master user/ path. + """ + sync_data_root = os.getenv(commons_constants.ENV_OCTOBOT_SYNC_DATA_ROOT) + if sync_data_root: + return sync_data_root + return get_user_root_folder() + + def get_user_profiles_folder() -> str: """Module-level helper: profiles folder under the user root.""" return UserRootFolderProvider.instance().get_user_profiles_folder() diff --git a/packages/commons/tests/configuration/conftest.py b/packages/commons/tests/configuration/conftest.py new file mode 100644 index 0000000000..ca70ddde5f --- /dev/null +++ b/packages/commons/tests/configuration/conftest.py @@ -0,0 +1,11 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import pytest + +import octobot_commons.constants as constants + + +@pytest.fixture(autouse=True) +def reset_sync_data_root_env(monkeypatch): + monkeypatch.delenv(constants.ENV_OCTOBOT_SYNC_DATA_ROOT, raising=False) diff --git a/packages/commons/tests/configuration/test_configuration.py b/packages/commons/tests/configuration/test_configuration.py index f18ff01395..771350c93a 100644 --- a/packages/commons/tests/configuration/test_configuration.py +++ b/packages/commons/tests/configuration/test_configuration.py @@ -23,6 +23,7 @@ import octobot_commons.json_util import octobot_commons.configuration as configuration import octobot_commons.profiles as profiles +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.constants as constants import octobot_commons.tests.test_config as test_config from ..profiles import get_profiles_path @@ -38,6 +39,15 @@ def get_profile_path(): return test_config.TEST_CONFIG_FOLDER +def _load_test_profile(config, profile_path=None): + resolved_profile_path = profile_path or get_profile_path() + loaded_profile = profile_backends_module.FilesystemProfileBackend().read_profile_from_path( + resolved_profile_path + ) + loaded_profile.bind_profile_storage(config.profile_storage) + return loaded_profile + + @pytest.fixture def config(): return configuration.Configuration(get_fake_config_path(), get_profile_path()) @@ -62,24 +72,31 @@ def test_validate(config): def test_read(default_config): - with mock.patch.object(default_config, "load_profiles", mock.Mock()) as load_profiles_mock, \ - mock.patch.object(default_config, "_get_selected_profile", mock.Mock()) as _select_mock, \ - mock.patch.object(default_config, "select_profile", - mock.Mock()) as select_profile_mock: + with mock.patch.object( + default_config, + "load_profiles_if_possible_and_necessary", + mock.Mock(), + ) as load_profiles_mock: default_config.read() assert isinstance(default_config._read_config, dict) assert isinstance(default_config.config, dict) load_profiles_mock.assert_called_once() - _select_mock.assert_called_once() - select_profile_mock.assert_called_once() + with mock.patch.object( + default_config, + "load_profiles_if_possible_and_necessary", + mock.Mock(), + ) as load_profiles_mock: + default_config.read(activate_profile=False) + load_profiles_mock.assert_not_called() + assert default_config.profile is None -def test_select_profile(config): +def test_select_profile(config, tmp_path): with mock.patch.object(config, "_generate_config_from_user_config_and_profile", mock.Mock()) as _generate_config_from_user_config_and_profile_mock: config.profile_by_id = { - "1": profiles.Profile("plop"), - "hoo": profiles.Profile("ah") + "1": profiles.Profile(str(tmp_path / "plop")), + "hoo": profiles.Profile(str(tmp_path / "ah")), } config.profile_by_id["1"].name = "ploup" config.config = {} @@ -89,8 +106,7 @@ def test_select_profile(config): def test_remove_profile(config): - config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) - config.profile.read_config() + config.profile = _load_test_profile(config) config.profile.read_only = True config.profile_by_id[config.profile.profile_id] = config.profile # id not in loaded profiles @@ -113,8 +129,7 @@ def test_remove_profile(config): def test_generate_config_from_user_config_and_profile(config): with open(DEFAULT_CONFIG) as config_file: config._read_config = json.load(config_file) - config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) - config.profile.read_config() + config.profile = _load_test_profile(config) for key in config.profile.FULLY_MANAGED_ELEMENTS: assert key not in config._read_config for key in config.profile.PARTIALLY_MANAGED_ELEMENTS: @@ -128,6 +143,145 @@ def test_generate_config_from_user_config_and_profile(config): assert config.config is not config._read_config +def _align_config_with_profile(config): + config.config = copy.deepcopy(config._read_config) if config._read_config else {} + config._generate_config_from_user_config_and_profile() + + +class TestConfigurationProfileManagedElementsChanged: + def test_returns_false_when_profile_unset(self, config): + config.profile = None + config.config = {"community": {"token": "value"}} + assert config._profile_managed_elements_changed() is False + + def test_returns_false_when_only_community_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config["community"] = {"token": "updated"} + assert config._profile_managed_elements_changed() is False + + def test_returns_true_when_fully_managed_element_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config[constants.CONFIG_CRYPTO_CURRENCIES] = {"Updated": {"pairs": ["ETH/USDT"]}} + assert config._profile_managed_elements_changed() is True + + def test_returns_true_when_partial_exchange_allowed_key_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + exchange_name = next(iter(config.config[constants.CONFIG_EXCHANGES])) + config.config[constants.CONFIG_EXCHANGES][exchange_name][ + constants.CONFIG_ENABLED_OPTION + ] = not config.profile.config[constants.CONFIG_EXCHANGES][exchange_name][ + constants.CONFIG_ENABLED_OPTION + ] + assert config._profile_managed_elements_changed() is True + + def test_returns_false_when_partial_exchange_non_allowed_key_differs(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + exchange_name = next(iter(config.config[constants.CONFIG_EXCHANGES])) + config.config[constants.CONFIG_EXCHANGES][exchange_name][ + constants.CONFIG_EXCHANGE_KEY + ] = "updated-api-key" + assert config._profile_managed_elements_changed() is False + + +class TestConfigurationSaveSkipUnchangedProfile: + def test_skips_profile_save_when_only_non_profile_config_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config["community"] = {"token": "updated"} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save() + save_profile_config_mock.assert_not_called() + + def test_saves_profile_when_managed_elements_changed(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config[constants.CONFIG_CRYPTO_CURRENCIES] = {"Updated": {"pairs": ["ETH/USDT"]}} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save() + save_profile_config_mock.assert_called_once_with(config.config) + + def test_save_profile_true_forces_persist(self, config): + config.profile = _load_test_profile(config) + config.config = {"community": {"token": "updated"}} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save(save_profile=True) + save_profile_config_mock.assert_called_once_with(config.config) + + def test_sync_all_profiles_still_runs_when_active_profile_unchanged(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config["community"] = {"token": "updated"} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ), mock.patch.object( + config, + "_sync_other_profiles", + mock.Mock(), + ) as sync_other_profiles_mock: + config.save(sync_all_profiles=True) + sync_other_profiles_mock.assert_called_once_with() + + def test_save(config): save_file = "saved_config.json" config.config_path = save_file @@ -139,13 +293,13 @@ def test_save(config): with open(DEFAULT_CONFIG) as config_file: config._read_config = json.load(config_file) # add profile data - config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) - config.profile.read_config() + config.profile = _load_test_profile(config) with mock.patch.object(config, "_get_config_without_profile_elements", mock.Mock(return_value=config._read_config)) as _filter_mock, \ - mock.patch.object(config.profile, "save_config", mock.Mock()) as _save_profile_mock: - config.save() + mock.patch.object(config.profile, "save_config", mock.Mock()) as save_profile_config_mock: + config.save(save_profile=True) assert os.path.isfile(save_file) + save_profile_config_mock.assert_called_once_with(config.config) with open(save_file) as config_file: saved_config = json.load(config_file) assert saved_config == config._read_config @@ -178,6 +332,48 @@ def test_get_tentacles_config_path(config): constants.CONFIG_TENTACLES_FILE) +class TestGetActiveTentaclesSetupConfigProfileDataBacked: + def test_returns_in_memory_setup_without_filesystem_path(self, config): + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module + + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + profile.init_tentacles_setup_config() + config.profile = profile + setup_config = config.get_active_tentacles_setup_config() + assert setup_config is profile.tentacles_setup_config + assert setup_config.profile is profile + + def test_init_setup_when_missing(self, config): + import octobot_commons.profiles.profile_data as profile_data_module + import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module + + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + config.profile = profile + assert profile.tentacles_setup_config is None + setup_config = config.get_active_tentacles_setup_config() + assert setup_config is not None + assert setup_config.profile is profile + + +class TestGetActiveTentaclesSetupConfigFilesystemBacked: + def test_delegates_to_tentacles_manager_api(self, config): + config.profile = profiles.Profile(get_profile_path(), config.profile_schema_path) + expected_setup = mock.Mock() + with mock.patch( + "octobot_tentacles_manager.api.get_tentacles_setup_config", + mock.Mock(return_value=expected_setup), + ) as get_setup_mock: + setup_config = config.get_active_tentacles_setup_config() + get_setup_mock.assert_called_once_with( + config.get_tentacles_config_path(), + profile=config.profile, + ) + assert setup_config is expected_setup + + def test_get_metrics_enabled(config): config.config = {} assert config.get_metrics_enabled() is True @@ -300,7 +496,9 @@ def test_get_selected_profile(config): assert config._get_selected_profile() == "55" # missing profile config._read_config[constants.CONFIG_PROFILE] = "66" - assert config._get_selected_profile() == "default" + with mock.patch.object(config.logger, "warning", mock.Mock()) as warning_mock: + assert config._get_selected_profile() == "default" + warning_mock.assert_called_once() # no default config.profile_by_id.pop("default") config._read_config[constants.CONFIG_PROFILE] = "66" @@ -311,6 +509,57 @@ def test_get_selected_profile(config): assert config._get_selected_profile() == "default" +class TestConfigurationSave: + def test_calls_profile_save_config_with_live_config(self, config): + config.profile = _load_test_profile(config) + with open(DEFAULT_CONFIG) as config_file: + config._read_config = json.load(config_file) + _align_config_with_profile(config) + config.config[constants.CONFIG_CRYPTO_CURRENCIES] = {"Updated": {"pairs": ["ETH/USDT"]}} + with mock.patch( + "octobot_commons.configuration.configuration.config_file_manager.dump", + mock.Mock(), + ), mock.patch.object( + config, + "_get_config_without_profile_elements", + mock.Mock(return_value={}), + ), mock.patch.object( + config.profile, + "save_config", + mock.Mock(), + ) as save_profile_config_mock: + config.save() + save_profile_config_mock.assert_called_once_with(config.config) + + +class TestConfigurationDeferredProfileActivation: + def test_activate_saved_profile_loads_and_selects(self, config): + sync_profile_id = "sync-only-profile-id" + config._read_config = {constants.CONFIG_PROFILE: sync_profile_id} + config.config = copy.deepcopy(config._read_config) + sync_profile = profiles.Profile("sync-path") + sync_profile.profile_id = sync_profile_id + sync_profile.name = "AAAA" + with mock.patch.object(config, "load_profiles", mock.Mock()) as load_profiles_mock, \ + mock.patch.object(config, "_get_selected_profile", mock.Mock(return_value=sync_profile_id)) as get_selected_mock, \ + mock.patch.object(config, "select_profile", mock.Mock()) as select_profile_mock: + config.activate_saved_profile() + load_profiles_mock.assert_called_once() + get_selected_mock.assert_called_once() + select_profile_mock.assert_called_once_with(sync_profile_id) + + def test_read_without_activate_profile_leaves_profile_unset(self, default_config): + with mock.patch.object( + default_config, + "load_profiles_if_possible_and_necessary", + mock.Mock(), + ) as load_mock: + default_config.read(activate_profile=False) + load_mock.assert_not_called() + assert default_config.profile is None + assert default_config.config is not None + + def test_load_profiles(config): config.profiles_path = get_profiles_path() nb_profiles = 1 @@ -322,6 +571,115 @@ def test_load_profiles(config): assert config.profile_by_id["default"] is loaded_profile +class TestConfigurationRefreshSyncProfiles: + def test_no_op_when_sync_unavailable(self, config): + config.profile_by_id = {"sync-profile-id": mock.Mock()} + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=False), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(), + ) as list_sync_profiles_mock: + config.refresh_sync_profiles() + list_sync_profiles_mock.assert_not_called() + assert "sync-profile-id" in config.profile_by_id + + def test_adds_new_sync_profile(self, config): + new_sync_profile = mock.Mock() + new_sync_profile.is_sync_backed.return_value = True + config.profile_by_id = {} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={"new-sync-profile-id": new_sync_profile}), + ): + config.refresh_sync_profiles() + assert config.profile_by_id["new-sync-profile-id"] is new_sync_profile + + def test_replaces_updated_sync_profile(self, config): + stale_sync_profile = mock.Mock() + stale_sync_profile.is_sync_backed.return_value = True + refreshed_sync_profile = mock.Mock() + refreshed_sync_profile.is_sync_backed.return_value = True + config.profile_by_id = {"sync-profile-id": stale_sync_profile} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={"sync-profile-id": refreshed_sync_profile}), + ): + config.refresh_sync_profiles() + assert config.profile_by_id["sync-profile-id"] is refreshed_sync_profile + + def test_removes_deleted_sync_profile(self, config): + removed_sync_profile = mock.Mock() + removed_sync_profile.is_sync_backed.return_value = True + config.profile_by_id = {"removed-sync-profile-id": removed_sync_profile} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={}), + ): + config.refresh_sync_profiles() + assert "removed-sync-profile-id" not in config.profile_by_id + + def test_leaves_filesystem_profiles_untouched(self, config): + filesystem_profile = mock.Mock() + filesystem_profile.is_sync_backed.return_value = False + config.profile_by_id = {"filesystem-profile-id": filesystem_profile} + config.profile = None + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock(return_value={}), + ): + config.refresh_sync_profiles() + assert config.profile_by_id["filesystem-profile-id"] is filesystem_profile + + def test_updates_active_profile_when_sync_backed(self, config): + stale_active_profile = mock.Mock() + stale_active_profile.is_sync_backed.return_value = True + stale_active_profile.profile_id = "active-sync-profile-id" + refreshed_active_profile = mock.Mock() + refreshed_active_profile.is_sync_backed.return_value = True + config.profile_by_id = {"active-sync-profile-id": stale_active_profile} + config.profile = stale_active_profile + with mock.patch.object( + config.profile_storage, + "is_sync_available", + mock.Mock(return_value=True), + ), mock.patch.object( + config.profile_storage, + "list_sync_profiles", + mock.Mock( + return_value={"active-sync-profile-id": refreshed_active_profile} + ), + ): + config.refresh_sync_profiles() + assert config.profile is refreshed_active_profile + + def test_get_config_without_profile_elements(config): config.profile = profiles.Profile(config.profiles_path) config.config = { @@ -335,3 +693,162 @@ def test_get_config_without_profile_elements(config): "plip": True, next(iter(profiles.Profile.PARTIALLY_MANAGED_ELEMENTS)): "tt" } + + +class TestConfigurationReadonlyProfileOverlay: + def _write_readonly_master_profile( + self, + profile_folder_path: str, + profile_id: str, + ) -> None: + self._write_master_profile(profile_folder_path, profile_id, read_only=True) + + def _write_master_profile( + self, + profile_folder_path: str, + profile_id: str, + *, + read_only: bool, + ) -> None: + import octobot_commons.json_util as json_util_module + + os.makedirs(profile_folder_path, exist_ok=True) + profile_file = { + constants.CONFIG_PROFILE: { + constants.CONFIG_ID: profile_id, + constants.CONFIG_NAME: "Non-Trading", + constants.CONFIG_READ_ONLY: read_only, + }, + constants.PROFILE_CONFIG: { + constants.CONFIG_CRYPTO_CURRENCIES: {}, + constants.CONFIG_EXCHANGES: {}, + constants.CONFIG_TRADER: {constants.CONFIG_ENABLED_OPTION: False}, + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {}, + constants.CONFIG_SIMULATOR_FEES: {}, + }, + constants.CONFIG_TRADING: { + constants.CONFIG_TRADER_REFERENCE_MARKET: constants.DEFAULT_REFERENCE_MARKET, + constants.CONFIG_TRADER_RISK: 1, + }, + constants.CONFIG_DISTRIBUTION: constants.DEFAULT_DISTRIBUTION, + }, + } + json_util_module.safe_dump( + profile_file, + os.path.join(profile_folder_path, constants.PROFILE_CONFIG_FILE), + ) + + def _child_config_with_readonly_overlay( + self, + tmp_path, + ) -> tuple[configuration.Configuration, str]: + child_user_root = tmp_path / "child" / "user" + child_profiles_path = child_user_root / constants.PROFILES_FOLDER + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + self._write_readonly_master_profile( + str(master_profiles_path / "non-trading"), + constants.DEFAULT_PROFILE, + ) + child_user_root.mkdir(parents=True) + config_path = child_user_root / constants.CONFIG_FILE + config_data = { + constants.CONFIG_PROFILE: "non-trading", + constants.CONFIG_READONLY_PROFILES_PATH: str(master_profiles_path), + constants.CONFIG_ACCEPTED_TERMS: True, + } + with open(config_path, "w", encoding="utf-8") as config_file: + json.dump(config_data, config_file) + bot_config = configuration.Configuration( + str(config_path), + str(child_profiles_path), + ) + return bot_config, str(master_profiles_path) + + def _child_config_with_editable_overlay( + self, + tmp_path, + ) -> tuple[configuration.Configuration, str]: + child_user_root = tmp_path / "child" / "user" + child_profiles_path = child_user_root / constants.PROFILES_FOLDER + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + self._write_master_profile( + str(master_profiles_path / "editable-strategy"), + "editable-strategy", + read_only=False, + ) + child_user_root.mkdir(parents=True) + config_path = child_user_root / constants.CONFIG_FILE + config_data = { + constants.CONFIG_PROFILE: "editable-strategy", + constants.CONFIG_READONLY_PROFILES_PATH: str(master_profiles_path), + constants.CONFIG_ACCEPTED_TERMS: True, + } + with open(config_path, "w", encoding="utf-8") as config_file: + json.dump(config_data, config_file) + bot_config = configuration.Configuration( + str(config_path), + str(child_profiles_path), + ) + return bot_config, str(master_profiles_path) + + def test_are_profiles_empty_or_missing_false_with_readonly_overlay(self, tmp_path): + bot_config, _master_profiles_path = self._child_config_with_readonly_overlay(tmp_path) + bot_config.read(should_raise=False, fill_missing_fields=True) + assert bot_config.are_profiles_empty_or_missing() is False + + def test_read_loads_profile_from_readonly_overlay(self, tmp_path): + bot_config, _master_profiles_path = self._child_config_with_readonly_overlay(tmp_path) + bot_config.read(should_raise=False, fill_missing_fields=True) + assert bot_config.profile is not None + assert bot_config.profile.profile_id == constants.DEFAULT_PROFILE + assert bot_config.config[constants.CONFIG_PROFILE] == constants.DEFAULT_PROFILE + + def test_save_skips_master_overlay_profile_persist(self, tmp_path): + bot_config, _master_profiles_path = self._child_config_with_readonly_overlay(tmp_path) + bot_config.read(should_raise=False, fill_missing_fields=True) + with mock.patch.object( + bot_config.profile_storage, + "save_active_profile", + mock.Mock(), + ) as save_active_profile_mock: + bot_config.save() + save_active_profile_mock.assert_not_called() + + def test_save_persists_editable_master_overlay_profile(self, tmp_path): + bot_config, _master_profiles_path = self._child_config_with_editable_overlay(tmp_path) + bot_config.read(should_raise=False, fill_missing_fields=True) + with mock.patch.object( + bot_config.profile_storage, + "save_active_profile", + mock.Mock(), + ) as save_active_profile_mock: + bot_config.save(save_profile=True) + save_active_profile_mock.assert_called_once() + + def test_read_configures_readonly_reference_tentacles_path(self, tmp_path): + import octobot_commons.user_root_folder_provider as user_root_folder_provider + + child_user_root = tmp_path / "child" / "user" + child_profiles_path = child_user_root / constants.PROFILES_FOLDER + master_reference_path = tmp_path / "master" / "reference_tentacles_config" + master_reference_path.mkdir(parents=True) + (master_reference_path / constants.CONFIG_TENTACLES_FILE).write_text("{}", encoding="utf-8") + child_user_root.mkdir(parents=True) + config_path = child_user_root / constants.CONFIG_FILE + config_data = { + constants.CONFIG_PROFILE: constants.DEFAULT_PROFILE, + constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH: str(master_reference_path), + constants.CONFIG_ACCEPTED_TERMS: True, + } + with open(config_path, "w", encoding="utf-8") as config_file: + json.dump(config_data, config_file) + provider = user_root_folder_provider.UserRootFolderProvider.instance() + provider.configure_readonly_reference_tentacles_path("") + bot_config = configuration.Configuration( + str(config_path), + str(child_profiles_path), + ) + bot_config.read(should_raise=False, fill_missing_fields=True) + assert provider.get_user_reference_tentacle_config_path() == str(master_reference_path) diff --git a/packages/commons/tests/configuration/test_user_inputs.py b/packages/commons/tests/configuration/test_user_inputs.py new file mode 100644 index 0000000000..5af39ffca4 --- /dev/null +++ b/packages/commons/tests/configuration/test_user_inputs.py @@ -0,0 +1,34 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import octobot_commons.configuration.user_inputs as user_inputs + + +class TestFindParentConfigNode: + def test_extends_empty_object_array_with_placeholder(self): + tentacle_config = {"items": []} + parent_node = user_inputs._find_parent_config_node( + tentacle_config, "items", [0] + ) + assert parent_node == {} + assert tentacle_config["items"] == [{}] + + def test_extends_object_array_until_requested_index(self): + tentacle_config = {"items": [{"existing": True}]} + parent_node = user_inputs._find_parent_config_node( + tentacle_config, "items", [2] + ) + assert parent_node == {} + assert tentacle_config["items"] == [{"existing": True}, {}, {}] diff --git a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py b/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py index 9eeab0bd12..394d01e904 100644 --- a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py +++ b/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py @@ -19,6 +19,7 @@ import asyncio import sqlite3 import contextlib +import tempfile import octobot_commons.asyncio_tools as asyncio_tools @@ -58,7 +59,8 @@ async def get_temp_empty_database(): async def test_invalid_file(): - file_name = "plop" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name db = databases.SQLiteDatabase(file_name) try: await db.initialize() diff --git a/packages/commons/tests/dsl_interpreter/operators/test_dynamic_dependencies_operator_mixin.py b/packages/commons/tests/dsl_interpreter/operators/test_dynamic_dependencies_operator_mixin.py index 04b64c26ad..0068d162bd 100644 --- a/packages/commons/tests/dsl_interpreter/operators/test_dynamic_dependencies_operator_mixin.py +++ b/packages/commons/tests/dsl_interpreter/operators/test_dynamic_dependencies_operator_mixin.py @@ -11,6 +11,7 @@ import pytest import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.operators.dynamic_dependencies_operator_mixin as dynamic_dependencies_operator_mixin @@ -50,7 +51,7 @@ def test_get_dynamic_dependencies_parameters(self): parameters = _DynamicDependenciesOperator.get_dynamic_dependencies_parameters() assert len(parameters) == 1 assert parameters[0].name == "_dynamic_dependencies" - assert parameters[0].type is list + assert parameters[0].type == commons_enums.DslValueType.ANY.value def test_dsl_statement_uses_dynamic_dependencies_true(self): unresolved_placeholder = commons_constants.UNRESOLVED_PARAMETER_PLACEHOLDER diff --git a/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py b/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py index 6393161fca..53d13d004e 100644 --- a/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py +++ b/packages/commons/tests/dsl_interpreter/operators/test_process_bound_operator_mixin.py @@ -76,6 +76,17 @@ def test_delegates_to_process_util(self): assert result == expected stop_mock.assert_called_once_with(99, logger=mock.sentinel.log) + def test_wraps_process_error_as_dsl_interpreter_error(self): + bound = process_bound_operator_mixin.ProcessBoundOperatorMixin() + bound.pid = 99 + with mock.patch.object( + process_util, + "request_graceful_stop_via_sigterm", + side_effect=commons_errors.ProcessError("failed to signal"), + ): + with pytest.raises(commons_errors.DSLInterpreterError, match="failed to signal"): + bound.request_graceful_stop() + @pytest.mark.asyncio class TestWaitUntilPidStopped: @@ -100,6 +111,16 @@ async def test_timeout_raises_dsl_error(self): poll_interval=0.01, ) + async def test_wraps_process_error_as_dsl_interpreter_error(self): + bound = process_bound_operator_mixin.ProcessBoundOperatorMixin() + with mock.patch.object( + process_util, + "wait_until_pid_stopped_async", + side_effect=commons_errors.ProcessError("wait failed"), + ): + with pytest.raises(commons_errors.DSLInterpreterError, match="wait failed"): + await bound.wait_until_pid_stopped(99, timeout_seconds=1.0) + class TestSpawnSubprocess: def test_sets_self_pid_from_child_and_returns_popen(self): diff --git a/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py b/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py index 8c82b74194..45d3fe31e7 100644 --- a/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py +++ b/packages/commons/tests/dsl_interpreter/operators/test_re_callable_operator_mixin.py @@ -17,6 +17,7 @@ import time import octobot_commons.dsl_interpreter as dsl_interpreter +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.operator_parameter as operator_parameter import octobot_commons.dsl_interpreter.operators.re_callable_operator_mixin as re_callable_operator_mixin @@ -113,13 +114,13 @@ def get_parameters(cls): name="seconds", description="wait duration", required=True, - type=float, + type=commons_enums.DslValueType.NUMBER.value, ), operator_parameter.OperatorParameter( name="label", description="optional label", required=False, - type=str, + type=commons_enums.DslValueType.TEXT.value, ), ] diff --git a/packages/commons/tests/dsl_interpreter/test_custom_operators.py b/packages/commons/tests/dsl_interpreter/test_custom_operators.py index 1b4728dbd8..a4f5592a98 100644 --- a/packages/commons/tests/dsl_interpreter/test_custom_operators.py +++ b/packages/commons/tests/dsl_interpreter/test_custom_operators.py @@ -54,8 +54,8 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="data", description="the data to compute the sum of", required=True, type=int), - dsl_interpreter.OperatorParameter(name="data2", description="the data to compute the sum of", required=False, type=int), + dsl_interpreter.OperatorParameter(name="data", description="the data to compute the sum of", required=True, type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter(name="data2", description="the data to compute the sum of", required=False, type=commons_enums.DslValueType.NUMBER.value), ] async def pre_compute(self) -> None: @@ -212,8 +212,8 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="left", description="the left operand", required=True, type=int), - dsl_interpreter.OperatorParameter(name="right", description="the right operand", required=True, type=int), + dsl_interpreter.OperatorParameter(name="left", description="the left operand", required=True, type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter(name="right", description="the right operand", required=True, type=commons_enums.DslValueType.NUMBER.value), ] def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: @@ -228,8 +228,8 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="a", description="first value", required=True, type=int), - dsl_interpreter.OperatorParameter(name="b", description="second value", required=True, type=int), + dsl_interpreter.OperatorParameter(name="a", description="first value", required=True, type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter(name="b", description="second value", required=True, type=commons_enums.DslValueType.NUMBER.value), ] async def pre_compute(self) -> None: @@ -246,10 +246,10 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="value1", description="the first value", required=True, type=int), - dsl_interpreter.OperatorParameter(name="value2", description="the second value", required=False, type=int, default=0), - dsl_interpreter.OperatorParameter(name="added_extra_value", description="value to add to the result", required=False, type=int, default=0), - dsl_interpreter.OperatorParameter(name="substracted_extra_value", description="value to substract from the result", required=False, type=int, default=0), + dsl_interpreter.OperatorParameter(name="value1", description="the first value", required=True, type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter(name="value2", description="the second value", required=False, type=commons_enums.DslValueType.NUMBER.value, default=0), + dsl_interpreter.OperatorParameter(name="added_extra_value", description="value to add to the result", required=False, type=commons_enums.DslValueType.NUMBER.value, default=0), + dsl_interpreter.OperatorParameter(name="substracted_extra_value", description="value to substract from the result", required=False, type=commons_enums.DslValueType.NUMBER.value, default=0), ] def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: @@ -270,8 +270,8 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="p1", description="the first value", required=True, type=int), - dsl_interpreter.OperatorParameter(name="p2", description="the second value", required=True, type=int), + dsl_interpreter.OperatorParameter(name="p1", description="the first value", required=True, type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter(name="p2", description="the second value", required=True, type=commons_enums.DslValueType.NUMBER.value), ] def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: @@ -287,7 +287,7 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="values", description="the dictionary to sum the values of", required=True, type=dict), + dsl_interpreter.OperatorParameter(name="values", description="the dictionary to sum the values of", required=True, type=commons_enums.DslValueType.ANY.value), ] def nested_sum(self, values: dict) -> float: @@ -499,8 +499,8 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="x", description="first parameter", required=True, type=int), - dsl_interpreter.OperatorParameter(name="y", description="second parameter", required=False, type=int), + dsl_interpreter.OperatorParameter(name="x", description="first parameter", required=True, type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter(name="y", description="second parameter", required=False, type=commons_enums.DslValueType.NUMBER.value), ] def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: @@ -582,12 +582,12 @@ def test_get_docs_with_parameters(): assert docs.parameters[0].name == "x" assert docs.parameters[0].description == "first parameter" assert docs.parameters[0].required - assert docs.parameters[0].type == int + assert docs.parameters[0].type == commons_enums.DslValueType.NUMBER.value assert isinstance(docs.parameters[1], dsl_interpreter.OperatorParameter) assert docs.parameters[1].name == "y" assert docs.parameters[1].description == "second parameter" assert not docs.parameters[1].required - assert docs.parameters[1].type == int + assert docs.parameters[1].type == commons_enums.DslValueType.NUMBER.value def test_get_docs_without_parameters(): @@ -639,11 +639,11 @@ def test_get_docs_to_json(): assert json_data["parameters"][0]["name"] == "x" assert json_data["parameters"][0]["description"] == "first parameter" assert json_data["parameters"][0]["required"] is True - assert json_data["parameters"][0]["type"] == "int" + assert json_data["parameters"][0]["type"] == commons_enums.DslValueType.NUMBER.value assert json_data["parameters"][1]["name"] == "y" assert json_data["parameters"][1]["description"] == "second parameter" assert json_data["parameters"][1]["required"] is False - assert json_data["parameters"][1]["type"] == "int" + assert json_data["parameters"][1]["type"] == commons_enums.DslValueType.NUMBER.value @pytest.mark.asyncio diff --git a/packages/commons/tests/dsl_interpreter/test_parameters_util.py b/packages/commons/tests/dsl_interpreter/test_parameters_util.py index f7fb6bfbc1..4e27c4ddba 100644 --- a/packages/commons/tests/dsl_interpreter/test_parameters_util.py +++ b/packages/commons/tests/dsl_interpreter/test_parameters_util.py @@ -19,6 +19,7 @@ import pytest import octobot_commons.constants as constants +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.parameters_util as parameters_util import octobot_commons.dsl_interpreter.operator_parameter as operator_parameter import octobot_commons.errors as commons_errors @@ -93,10 +94,10 @@ def test_empty_params_and_empty_values(self): def test_required_params_only(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=True, type=str + name="b", description="second", required=True, type=commons_enums.DslValueType.TEXT.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b] @@ -106,7 +107,7 @@ def test_required_params_only(self): def test_optional_params_only(self): param_x = operator_parameter.OperatorParameter( - name="x", description="optional", required=False, type=int + name="x", description="optional", required=False, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_x] @@ -116,10 +117,10 @@ def test_optional_params_only(self): def test_mixed_required_and_optional(self): param_req = operator_parameter.OperatorParameter( - name="req", description="required", required=True, type=int + name="req", description="required", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_opt = operator_parameter.OperatorParameter( - name="opt", description="optional", required=False, type=str + name="opt", description="optional", required=False, type=commons_enums.DslValueType.TEXT.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_req, param_opt] @@ -129,10 +130,10 @@ def test_mixed_required_and_optional(self): def test_skips_missing_params(self): param_req = operator_parameter.OperatorParameter( - name="req", description="required", required=True, type=int + name="req", description="required", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_opt = operator_parameter.OperatorParameter( - name="opt", description="optional", required=False, type=str + name="opt", description="optional", required=False, type=commons_enums.DslValueType.TEXT.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_req, param_opt] @@ -142,7 +143,7 @@ def test_skips_missing_params(self): def test_extra_values_ignored(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a] @@ -163,10 +164,10 @@ def test_empty_params_returns_unchanged(self): def test_positional_args_only(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=True, type=int + name="b", description="second", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b] @@ -178,10 +179,10 @@ def test_positional_args_only(self): def test_positional_arg_as_keyword_arg(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=True, type=int + name="b", description="second", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b] @@ -193,13 +194,13 @@ def test_positional_arg_as_keyword_arg(self): def test_positional_arg_as_keyword_arg_in_a_wrong_order(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=True, type=int + name="b", description="second", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_c = operator_parameter.OperatorParameter( - name="c", description="third", required=True, type=int + name="c", description="third", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b, param_c] @@ -211,13 +212,13 @@ def test_positional_arg_as_keyword_arg_in_a_wrong_order(self): def test_positional_all_args_as_keywords_in_a_wrong_order(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=True, type=int + name="b", description="second", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_c = operator_parameter.OperatorParameter( - name="c", description="third", required=True, type=int + name="c", description="third", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b, param_c] @@ -229,10 +230,10 @@ def test_positional_all_args_as_keywords_in_a_wrong_order(self): def test_kwargs_only(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=True, type=int + name="b", description="second", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b] @@ -244,13 +245,13 @@ def test_kwargs_only(self): def test_mixed_args_and_kwargs(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=True, type=int + name="b", description="second", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_c = operator_parameter.OperatorParameter( - name="c", description="optional", required=False, type=int + name="c", description="optional", required=False, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b, param_c] @@ -262,7 +263,7 @@ def test_mixed_args_and_kwargs(self): def test_extra_kwargs_preserved(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a] @@ -274,7 +275,7 @@ def test_extra_kwargs_preserved(self): def test_raises_when_too_many_positional_args(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a] @@ -287,10 +288,10 @@ def test_raises_when_too_many_positional_args(self): def test_partial_params_allowed(self): param_a = operator_parameter.OperatorParameter( - name="a", description="first", required=True, type=int + name="a", description="first", required=True, type=commons_enums.DslValueType.NUMBER.value ) param_b = operator_parameter.OperatorParameter( - name="b", description="second", required=False, type=int + name="b", description="second", required=False, type=commons_enums.DslValueType.NUMBER.value ) operator_class = mock.Mock() operator_class.get_parameters.return_value = [param_a, param_b] diff --git a/packages/commons/tests/profiles/__init__.py b/packages/commons/tests/profiles/__init__.py index 95b67fe133..05a6b50be4 100644 --- a/packages/commons/tests/profiles/__init__.py +++ b/packages/commons/tests/profiles/__init__.py @@ -1,11 +1,6 @@ # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. # -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU @@ -13,27 +8,5 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import os - -import pytest -import pathlib -import octobot_commons.profiles as profiles -import octobot_commons.tests.test_config as test_config - - -def get_profile_path(): - return test_config.TEST_CONFIG_FOLDER - - -def get_profiles_path(): - return pathlib.Path(get_profile_path()).parent - - -@pytest.fixture -def profile(): - return profiles.Profile(get_profile_path()) - -@pytest.fixture -def invalid_profile(): - return profiles.Profile(os.path.join(get_profile_path(), "invalid_profile")) +from tests.profiles.conftest import get_profile_path, get_profiles_path diff --git a/packages/commons/tests/profiles/conftest.py b/packages/commons/tests/profiles/conftest.py index 223511bb1e..ecfc40e880 100644 --- a/packages/commons/tests/profiles/conftest.py +++ b/packages/commons/tests/profiles/conftest.py @@ -1,42 +1,49 @@ # Drakkar-Software OctoBot-Commons # Copyright (c) Drakkar-Software, All rights reserved. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. -import pathlib -import shutil + +import os import pytest +import pathlib -import tests.profiles as profiles_tests +import octobot_commons.constants as constants +import octobot_commons.profiles as profiles +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.profiles.profile_storage as profile_storage_module +import octobot_commons.tests.test_config as test_config PROFILES_FS_XDIST_GROUP = "profiles_fs" -_EPHEMERAL_PROFILE_DIRECTORIES = ( - "second_profile", - "other_profile", -) - - -@pytest.fixture(autouse=True) -def _clean_ephemeral_test_profile_directories(): - profiles_path = pathlib.Path(profiles_tests.get_profiles_path()) - for directory_name in _EPHEMERAL_PROFILE_DIRECTORIES: - directory_path = profiles_path.joinpath(directory_name) - if directory_path.is_dir(): - shutil.rmtree(directory_path) - yield - for directory_name in _EPHEMERAL_PROFILE_DIRECTORIES: - directory_path = profiles_path.joinpath(directory_name) - if directory_path.is_dir(): - shutil.rmtree(directory_path) + +def get_profile_path(): + return test_config.TEST_CONFIG_FOLDER + + +def get_profiles_path(): + return pathlib.Path(get_profile_path()).parent + + +@pytest.fixture +def profile_storage(tmp_path): + profiles_path = tmp_path / constants.PROFILES_FOLDER + profiles_path.mkdir() + storage = profile_storage_module.ProfileStorage(str(profiles_path), None) + yield storage + + +@pytest.fixture +def profile_storage_for_tests(): + return profile_storage_module.ProfileStorage(str(get_profiles_path()), None) + + +@pytest.fixture +def profile(profile_storage_for_tests): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + loaded_profile = filesystem_backend.read_profile_from_path(get_profile_path()) + loaded_profile.bind_profile_storage(profile_storage_for_tests) + return loaded_profile + + +@pytest.fixture +def invalid_profile(): + return profiles.Profile(os.path.join(get_profile_path(), "invalid_profile")) diff --git a/packages/commons/tests/profiles/test_ephemeral_profile.py b/packages/commons/tests/profiles/test_ephemeral_profile.py new file mode 100644 index 0000000000..d5e986bda3 --- /dev/null +++ b/packages/commons/tests/profiles/test_ephemeral_profile.py @@ -0,0 +1,56 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock +import pytest + +import octobot_commons.enums as enums +import octobot_commons.errors as errors +import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module +import octobot_commons.profiles.profile_data as profile_data_module + + +class TestEphemeralProfileFromProfileData: + def test_from_profile_data_returns_ephemeral_profile(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + assert isinstance(profile, ephemeral_profile_module.EphemeralProfile) + assert profile.path is None + assert profile.get_storage_source() == enums.ProfileSource.EPHEMERAL + assert profile.is_profile_data_tentacle_backed() is True + assert profile.is_sync_backed() is False + + +class TestEphemeralProfileInitTentaclesSetupConfig: + def test_init_tentacles_setup_config_does_not_require_filesystem_path(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + profile.init_tentacles_setup_config() + assert profile.tentacles_setup_config is not None + + +class TestEphemeralProfileBindTentaclesSetupConfig: + def test_sets_both_profile_and_setup_references(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + setup = mock.Mock() + returned_setup = profile.bind_tentacles_setup_config(setup) + assert returned_setup is setup + assert setup.profile is profile + assert profile.tentacles_setup_config is setup + + +class TestEphemeralProfileGetTentaclesConfigPath: + def test_raises_profile_data_error(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + with pytest.raises(errors.ProfileDataError): + profile.get_tentacles_config_path() + + +class TestEphemeralProfileSave: + def test_save_raises_profile_data_error(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + with pytest.raises(errors.ProfileDataError): + profile.save() diff --git a/packages/commons/tests/profiles/test_filesystem_profile_backend.py b/packages/commons/tests/profiles/test_filesystem_profile_backend.py new file mode 100644 index 0000000000..8577263003 --- /dev/null +++ b/packages/commons/tests/profiles/test_filesystem_profile_backend.py @@ -0,0 +1,122 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import json +import os + +import mock +import pytest + +import octobot_commons.constants as constants +import octobot_commons.errors as errors +import octobot_commons.json_util as json_util +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.tests.test_config as test_config + +import tests.profiles.conftest as profiles_conftest +from tests.profiles import get_profile_path, get_profiles_path + + +class TestFilesystemProfileBackendReadWrite: + def test_read_profile_from_path(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + profile = filesystem_backend.read_profile_from_path(get_profile_path()) + assert profile.profile_id == "default" + assert profile.name == "default" + assert profile.description == "OctoBot default profile." + assert profile.avatar == "default_profile.png" + assert profile.avatar_path == os.path.join( + test_config.TEST_CONFIG_FOLDER, "default_profile.png" + ) + assert profile.origin_url == "https://default.url" + assert profile.config[constants.CONFIG_DISTRIBUTION] == constants.DEFAULT_DISTRIBUTION + assert len(profile.config) == 6 + + def test_read_profile_from_path_raises_when_missing(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + with pytest.raises(errors.ProfileDataError): + filesystem_backend.read_profile_from_path("") + + def test_write_profile_config(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + profile = filesystem_backend.read_profile_from_path(get_profile_path()) + save_file = "profile_config.json" + if os.path.isfile(save_file): + os.remove(save_file) + try: + profile.config = {"a": 1} + with mock.patch.object( + filesystem_backend, + "config_file_path", + mock.Mock(return_value=save_file), + ): + filesystem_backend.write_profile_config(profile) + with open(save_file) as config_file: + saved_profile = json.load(config_file) + assert saved_profile == profile.as_dict() + finally: + if os.path.isfile(save_file): + os.remove(save_file) + + def test_config_file_path(self): + assert profile_backends_module.FilesystemProfileBackend.config_file_path( + get_profile_path() + ) == os.path.join(get_profile_path(), constants.PROFILE_CONFIG_FILE) + + +class TestFilesystemProfileBackendDiscovery: + def test_scan_profiles(self): + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + with mock.patch.object( + filesystem_backend, + "_load_profile_from_folder", + mock.Mock(), + ) as load_profile_mock: + nb_files = len(os.listdir(get_profiles_path())) + assert nb_files > 1 + filesystem_backend._scan_profiles(str(get_profiles_path())) + assert load_profile_mock.call_count == nb_files + + def test_load_profile_from_folder(self): + schema_path = "schema_path" + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + with mock.patch.object( + json_util, + "read_file", + mock.Mock(return_value={ + constants.CONFIG_PROFILE: {}, + constants.PROFILE_CONFIG: {}, + }), + ) as read_file_mock: + profile = filesystem_backend._load_profile_from_folder( + test_config.TEST_CONFIG_FOLDER, schema_path + ) + assert profile.path == test_config.TEST_CONFIG_FOLDER + assert profile.schema_path == schema_path + read_file_mock.assert_called_once() + + def test_load_profile(self): + profiles_path = str(get_profiles_path()) + filesystem_backend = profile_backends_module.FilesystemProfileBackend( + profiles_path, None + ) + profile = filesystem_backend.load_profile("default") + assert profile.profile_id == "default" + + @pytest.mark.xdist_group(name=profiles_conftest.PROFILES_FS_XDIST_GROUP) + def test_list_profile_ids(self, profile): + profiles_path = str(get_profiles_path()) + filesystem_backend = profile_backends_module.FilesystemProfileBackend( + profiles_path, None + ) + assert filesystem_backend.list_profile_ids() == ["default"] + assert filesystem_backend.list_profile_ids(ignore=profile.path) == [] + + +class TestFilesystemProfileBackendMutations: + def test_duplicate_profile_raises_not_implemented(self, profile): + filesystem_backend = profile_backends_module.FilesystemProfileBackend( + str(get_profiles_path()), None + ) + with pytest.raises(NotImplementedError): + filesystem_backend.duplicate_profile(profile) diff --git a/packages/commons/tests/profiles/test_profile.py b/packages/commons/tests/profiles/test_profile.py index 9b88eddc05..9d755914bc 100644 --- a/packages/commons/tests/profiles/test_profile.py +++ b/packages/commons/tests/profiles/test_profile.py @@ -13,56 +13,32 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import os import copy -import json -import shutil import pytest import mock import octobot_commons.json_util import octobot_commons.profiles as profiles import octobot_commons.constants as constants import octobot_commons.enums as enums +import octobot_commons.errors as errors import octobot_commons.tests.test_config as test_config -import tests.profiles.conftest as profiles_conftest -from tests.profiles import profile, get_profile_path, get_profiles_path - - -def test_read_config(profile): - save_ref = profile - assert profile.read_config() is save_ref - assert profile.profile_id == "default" - assert profile.name == "default" - assert profile.description == "OctoBot default profile." - assert profile.avatar == "default_profile.png" - assert profile.avatar_path == os.path.join(test_config.TEST_CONFIG_FOLDER, "default_profile.png") - assert profile.origin_url == "https://default.url" - # default value: distribution is not in profile config - assert profile.config[constants.CONFIG_DISTRIBUTION] == constants.DEFAULT_DISTRIBUTION - assert len(profile.config) == 6 - assert isinstance(profile.config, dict) - - profile.path = "" - with pytest.raises(FileNotFoundError): - profile.read_config() +from tests.profiles import get_profile_path def test_save_config(profile): - with mock.patch.object(profile, "validate_and_save_config", mock.Mock()) as validate_and_save_config_mock, \ + with mock.patch.object(profile, "_save_through_profile_storage", mock.Mock()) as save_mock, \ mock.patch.object(profile, "_filter_fill_elements", mock.Mock()) as _filter_fill_elements_mock: profile.config = {} - # nothing to operate on global_config = {} profile.save_config(global_config) assert profile.config == {} - validate_and_save_config_mock.assert_called_once() + save_mock.assert_called_once_with(global_config) _filter_fill_elements_mock.assert_not_called() - validate_and_save_config_mock.reset_mock() + save_mock.reset_mock() _filter_fill_elements_mock.reset_mock() profile.config = {} - # things in config global_config = { profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip", @@ -73,13 +49,12 @@ def test_save_config(profile): profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip" } - validate_and_save_config_mock.assert_called_once() + save_mock.assert_called_once_with(global_config) _filter_fill_elements_mock.assert_not_called() - validate_and_save_config_mock.reset_mock() + save_mock.reset_mock() _filter_fill_elements_mock.reset_mock() profile.config = {} - # things in config global_config = { profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip", @@ -91,7 +66,7 @@ def test_save_config(profile): profile.FULLY_MANAGED_ELEMENTS[0]: "plop", profile.FULLY_MANAGED_ELEMENTS[1]: "plip", } - validate_and_save_config_mock.assert_called_once() + save_mock.assert_called_once_with(global_config) _filter_fill_elements_mock.assert_called_once_with(global_config, profile.config, next(iter(profile.PARTIALLY_MANAGED_ELEMENTS)), @@ -107,61 +82,45 @@ def test_validate(profile): def test_validate_and_save_config(profile): - save_file = "profile_config.json" with mock.patch.object(profile, "validate", mock.Mock()) as validate_mock, \ - mock.patch.object(profile, "config_file", mock.Mock(return_value=save_file)), \ - mock.patch.object(profile, "save", mock.Mock()) as save_mock: + mock.patch.object(profile, "_save_through_profile_storage", mock.Mock()) as save_mock: profile.validate_and_save_config() validate_mock.assert_called_once() save_mock.assert_called_once() def test_save(profile): - save_file = "profile_config.json" - if os.path.isfile(save_file): - os.remove(save_file) - try: - profile.read_config() - with mock.patch.object(profile, "config_file", mock.Mock(return_value=save_file)): - profile.save() - with open(save_file) as config_file: - saved_profile = json.load(config_file) - assert saved_profile == profile.as_dict() - finally: - if os.path.isfile(save_file): - os.remove(save_file) + with mock.patch.object(profile, "validate_and_save_config", mock.Mock()) as validate_and_save_mock: + profile.save() + validate_and_save_mock.assert_called_once() + + +def test_save_requires_profile_storage(): + unbound_profile = profiles.Profile(get_profile_path()) + with pytest.raises(errors.ProfileDataError): + unbound_profile.save() def test_duplicate(profile): - with mock.patch.object(shutil, "copytree", mock.Mock()) as copytree_mock, \ - mock.patch.object(profiles.Profile, "save", mock.Mock()) as save_mock: + clone = mock.Mock() + with mock.patch.object( + profile.get_profile_storage(), + "duplicate_profile", + mock.Mock(return_value=clone), + ) as duplicate_mock: profile.read_only = True profile.imported = True profile.origin_url = "hello" - clone = profile.duplicate() - assert clone.name == profile.name - assert clone.description == profile.description - assert clone.profile_id != profile.description - assert clone.path != profile.path - assert clone.profile_id in clone.path - assert clone.profile_id is not None - # duplicates are not read_only - assert clone.read_only is False - # duplicates are never imported nor have an origin url - assert clone.imported is False - assert clone.origin_url is None - copytree_mock.assert_called_with(profile.path, clone.path) - save_mock.assert_called_once() + assert profile.duplicate() is clone + duplicate_mock.assert_called_once_with(profile, name=None, description=None) - clone = profile.duplicate(name="123", description="456") - assert clone.name == "123" - assert clone.name != profile.name - assert clone.description == "456" - assert clone.description != profile.description + profile.duplicate(name="123", description="456") + duplicate_mock.assert_called_with(profile, name="123", description="456") def test_as_dict(profile): - assert profile.as_dict() == { + empty_profile = profiles.Profile(get_profile_path()) + assert empty_profile.as_dict() == { constants.CONFIG_PROFILE: { constants.CONFIG_ID: None, constants.CONFIG_NAME: None, @@ -180,8 +139,6 @@ def test_as_dict(profile): }, constants.PROFILE_CONFIG: {}, } - profile.read_config() - # do not test read config profile.config = {"a": 1} profile.imported = True profile.complexity = enums.ProfileComplexity.DIFFICULT @@ -213,10 +170,6 @@ def test_as_dict(profile): } -def test_config_file(profile): - assert profile.config_file() == os.path.join(get_profile_path(), constants.PROFILE_CONFIG_FILE) - - def test_merge_partially_managed_element_into_config(profile): with mock.patch.object(profiles.Profile, "_merge_partially_managed_element", mock.Mock()) as _merge_mock: config = {} @@ -228,7 +181,6 @@ def test_merge_partially_managed_element_into_config(profile): def test_merge_partially_managed_element(profile): - profile.read_config() element = next(iter(profile.PARTIALLY_MANAGED_ELEMENTS)) template = profile.PARTIALLY_MANAGED_ELEMENTS[element] config = { @@ -239,7 +191,6 @@ def test_merge_partially_managed_element(profile): } } } - # add constants.CONFIG_ENABLED_OPTION profile._merge_partially_managed_element(config, profile.config, element, template) assert config == { constants.CONFIG_EXCHANGES: { @@ -254,7 +205,6 @@ def test_merge_partially_managed_element(profile): constants.CONFIG_EXCHANGES: {} } profile.config[constants.CONFIG_EXCHANGES]["binance"][constants.CONFIG_ENABLED_OPTION] = False - # add whole exchange profile._merge_partially_managed_element(config, profile.config, element, template) assert config == { constants.CONFIG_EXCHANGES: { @@ -268,7 +218,6 @@ def test_merge_partially_managed_element(profile): } } config = {} - # add whole exchange and exchanges key with 2 exchanges in profile profile.config[constants.CONFIG_EXCHANGES]["kucoin"] = { constants.CONFIG_ENABLED_OPTION: True, constants.CONFIG_EXCHANGE_TYPE: constants.CONFIG_EXCHANGE_FUTURE @@ -301,7 +250,6 @@ def test_merge_partially_managed_element(profile): } } } - # add constants.CONFIG_ENABLED_OPTION with 2 exchanges in profile, update constants.CONFIG_ENABLED_OPTION profile._merge_partially_managed_element(config, profile.config, element, template) assert config == { constants.CONFIG_EXCHANGES: { @@ -322,7 +270,6 @@ def test_merge_partially_managed_element(profile): def test_remove_deleted_elements(profile): - profile.read_config() element = next(iter(profile.PARTIALLY_MANAGED_ELEMENTS)) config = { constants.CONFIG_EXCHANGES: { @@ -335,13 +282,11 @@ def test_remove_deleted_elements(profile): } before_sync_elements_count = len(profile.config[element]) profile.remove_deleted_elements(config) - # did not remove any element assert before_sync_elements_count == len(profile.config[element]) profile.config[element]["plop"] = config[constants.CONFIG_EXCHANGES]["binance"] assert len(profile.config[element]) == before_sync_elements_count + 1 profile.remove_deleted_elements(config) assert before_sync_elements_count == len(profile.config[element]) - # removed "plop" element assert list(profile.config[element]) == ["binance"] @@ -356,7 +301,6 @@ def test_get_element_from_template(profile): def test_filter_fill_elements(profile): - profile.read_config() config = { constants.CONFIG_EXCHANGES: { "binance": { @@ -377,24 +321,21 @@ def test_filter_fill_elements(profile): } -def test_get_all_profiles(): - with mock.patch.object(profiles.Profile, "_load_profile", mock.Mock()) as _load_profile_mock: - nb_files = len(os.listdir(get_profiles_path())) - assert nb_files > 1 - profiles.Profile.get_all_profiles(get_profiles_path()) - assert _load_profile_mock.call_count == nb_files - - -def test_load_profile(): - schema_path = "schema_path" - with mock.patch.object(profiles.Profile, "read_config", mock.Mock()) as read_config_mock: - profile = profiles.Profile._load_profile(test_config.TEST_CONFIG_FOLDER, schema_path) - assert profile.path == test_config.TEST_CONFIG_FOLDER - assert profile.schema_path == schema_path - read_config_mock.assert_called_once() - - -@pytest.mark.xdist_group(name=profiles_conftest.PROFILES_FS_XDIST_GROUP) -def test_get_existing_profiles_ids(profile): - assert profiles.Profile.get_all_profiles_ids(get_profiles_path()) == ["default"] - assert profiles.Profile.get_all_profiles_ids(get_profiles_path(), ignore=profile.path) == [] +class TestProfileFromDict: + @pytest.mark.parametrize( + "field_key,attribute_name,expected", + [ + (constants.CONFIG_RISK, "risk", enums.ProfileRisk.MODERATE), + (constants.CONFIG_COMPLEXITY, "complexity", enums.ProfileComplexity.MEDIUM), + (constants.CONFIG_TYPE, "profile_type", enums.ProfileType.LIVE), + ], + ) + def test_null_enum_field_uses_default(self, field_key, attribute_name, expected): + profile = profiles.Profile(get_profile_path()) + profile.from_dict( + { + constants.CONFIG_PROFILE: {field_key: None}, + constants.PROFILE_CONFIG: {}, + } + ) + assert getattr(profile, attribute_name) == expected diff --git a/packages/commons/tests/profiles/test_profile_data.py b/packages/commons/tests/profiles/test_profile_data.py index c80e734edf..f76b0e327d 100644 --- a/packages/commons/tests/profiles/test_profile_data.py +++ b/packages/commons/tests/profiles/test_profile_data.py @@ -22,7 +22,7 @@ import octobot_commons.constants as constants import octobot_commons.enums as enums -from tests.profiles import get_profile_path, profile +from tests.profiles import get_profile_path @pytest.fixture @@ -104,6 +104,7 @@ def profile_data_dict(): { 'name': 'plopEvaluator', 'config': {}, + 'activated': True, }, { 'name': 'plopEvaluator', @@ -114,6 +115,7 @@ def profile_data_dict(): 'n': None, } }, + 'activated': True, }, ], 'options': { 'values': { @@ -176,12 +178,14 @@ def min_profile_data_dict(): def test_from_profile(profile): - profile_data = profiles.ProfileData.from_profile(profile.read_config()) + profile_data = profiles.ProfileData.from_profile(profile) # check one element per attribute to be sure it's all parsed assert profile_data.distribution == "default" assert profile_data.profile_details.name == "default" assert profile_data.crypto_currencies[0].trading_pairs == ['BTC/USDT'] - assert profile_data.exchanges == [] + assert len(profile_data.exchanges) == 1 + assert profile_data.exchanges[0].internal_name == "binance" + assert profile_data.exchanges[0].exchange_type == constants.DEFAULT_EXCHANGE_TYPE assert profile_data.trader.enabled is False assert profile_data.trader_simulator.enabled is True assert profile_data.trader_simulator.starting_portfolio == {'BTC': 10, 'USDT': 1000} @@ -189,14 +193,48 @@ def test_from_profile(profile): assert profile_data.tentacles == [] -def test_to_profile(profile): - profile_data = profiles.ProfileData.from_profile(profile.read_config()) - created_profile = profile_data.to_profile("plop_path") +class TestProfileDataExchangesFromProfileConfig: + def test_maps_only_enabled_exchanges(self): + profile_config = { + constants.CONFIG_EXCHANGES: { + "binance": { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_EXCHANGE_TYPE: "spot", + }, + "kucoin": { + constants.CONFIG_ENABLED_OPTION: False, + constants.CONFIG_EXCHANGE_TYPE: "spot", + }, + } + } + exchanges = profiles.ProfileData.exchanges_from_profile_config(profile_config) + assert len(exchanges) == 1 + assert exchanges[0].internal_name == "binance" + assert exchanges[0].exchange_type == "spot" + + def test_uses_default_exchange_type_when_missing(self): + profile_config = { + constants.CONFIG_EXCHANGES: { + "binance": { + constants.CONFIG_ENABLED_OPTION: True, + }, + } + } + exchanges = profiles.ProfileData.exchanges_from_profile_config(profile_config) + assert exchanges[0].exchange_type == constants.DEFAULT_EXCHANGE_TYPE + + +def test_from_profile_data(profile): + profile_data = profiles.ProfileData.from_profile(profile) + created_profile = profiles.Profile.from_profile_data(profile_data, "plop_path") # force missing values for crypto_data in profile.config[constants.CONFIG_CRYPTO_CURRENCIES].values(): crypto_data[constants.CONFIG_ENABLED_OPTION] = crypto_data.get(constants.CONFIG_ENABLED_OPTION, True) + for exchange_data in profile.config[constants.CONFIG_EXCHANGES].values(): + exchange_data[constants.CONFIG_EXCHANGE_TYPE] = exchange_data.get( + constants.CONFIG_EXCHANGE_TYPE, constants.DEFAULT_EXCHANGE_TYPE + ) # remove not stored values - profile.config[constants.CONFIG_EXCHANGES] = {} profile.avatar = profile.description = "" profile.complexity = enums.ProfileComplexity.MEDIUM profile.risk = enums.ProfileRisk.MODERATE diff --git a/packages/commons/tests/profiles/test_profile_data_backed_profile.py b/packages/commons/tests/profiles/test_profile_data_backed_profile.py new file mode 100644 index 0000000000..2773ba5421 --- /dev/null +++ b/packages/commons/tests/profiles/test_profile_data_backed_profile.py @@ -0,0 +1,50 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock + +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_types.profile_data_backed_profile as profile_data_backed_profile_module + + +class TestProfileDataBackedProfileGetTentaclesData: + def test_merges_inactive_tentacle_configs_on_save(self): + profile_data = profile_data_module.ProfileData() + profile_data.tentacles = [ + profile_data_module.TentaclesData( + name="GridTradingMode", + config={"flat_spread": 2}, + activated=False, + ), + ] + profile = profile_data_backed_profile_module.ProfileDataBackedProfile( + profile_data, profile_path="/tmp/profile" + ) + tentacles_setup_config = mock.Mock() + profile.tentacles_setup_config = tentacles_setup_config + collected = [ + profile_data_module.TentaclesData( + name="IndexTradingMode", + config={"refresh_interval": 0}, + activated=True, + ), + ] + + with mock.patch( + "octobot_tentacles_manager.configuration.profile_tentacles_util.collect_tentacles_data_from_setup", + mock.Mock(return_value=collected), + ), mock.patch( + "octobot_tentacles_manager.configuration.profile_tentacles_util.merge_inactive_tentacles_data_from_profile", + mock.Mock( + return_value=[ + collected[0], + profile_data.tentacles[0], + ] + ), + ) as merge_mock: + result = profile.get_tentacles_data() + + merge_mock.assert_called_once_with(collected, profile_data) + assert len(result) == 2 + assert result[1].name == "GridTradingMode" + assert result[1].activated is False diff --git a/packages/commons/tests/profiles/test_profile_data_import.py b/packages/commons/tests/profiles/test_profile_data_import.py new file mode 100644 index 0000000000..96eda3415a --- /dev/null +++ b/packages/commons/tests/profiles/test_profile_data_import.py @@ -0,0 +1,27 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_commons.constants as constants +import octobot_commons.enums as enums +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_data_import as profile_data_import_module + + +class TestGetProfile: + def test_none_risk_defaults_to_moderate(self): + profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"name": "fetched_config", "id": "profile-id"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + } + ) + profile = profile_data_import_module._get_profile( + profile_data, + description=None, + risk=None, + output_path="/tmp/fetched_config", + auto_update=False, + slug="fetched_config", + force_simulator=False, + ) + assert profile.risk == enums.ProfileRisk.MODERATE diff --git a/packages/commons/tests/profiles/test_profile_sharing.py b/packages/commons/tests/profiles/test_profile_sharing.py index bae651c36b..828f0c497b 100644 --- a/packages/commons/tests/profiles/test_profile_sharing.py +++ b/packages/commons/tests/profiles/test_profile_sharing.py @@ -26,17 +26,23 @@ import octobot_commons.user_root_folder_provider as user_root_folder_provider import octobot_commons.errors as commons_errors import octobot_commons.profiles as profiles +import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_sharing as profile_sharing +import octobot_commons.profiles.profile_storage as profile_storage_module from octobot_commons.profiles.profile_sharing import _get_unique_profile_folder, _ensure_unique_profile_id, \ _get_profile_name import octobot_commons.tests.test_config as test_config import tests.profiles.conftest as profiles_conftest -from tests.profiles import profile, get_profile_path, invalid_profile +from tests.profiles import get_profile_path pytestmark = pytest.mark.xdist_group(name=profiles_conftest.PROFILES_FS_XDIST_GROUP) +def _profile_config_file(profile): + return profile_backends_module.FilesystemProfileBackend.config_file_path(profile.path) + + def test_export_profile(profile): export_path = "exported" exported_file = f"{export_path}.zip" @@ -49,10 +55,10 @@ def test_export_profile(profile): dir1=spec_tentacles_config, dir2=other_profile): # create fake tentacles config - shutil.copy(profile.config_file(), tentacles_config) + shutil.copy(_profile_config_file(profile), tentacles_config) os.mkdir(spec_tentacles_config) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t1.json")) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t2.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t1.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t2.json")) with mock.patch.object(os, "remove", mock.Mock()) as remove_mock: profiles.export_profile(profile, export_path) remove_mock.assert_not_called() @@ -81,11 +87,11 @@ def test_export_profile_with_existing_file(profile): dir1=spec_tentacles_config, dir2=other_profile): # create fake tentacles config - shutil.copy(profile.config_file(), tentacles_config) + shutil.copy(_profile_config_file(profile), tentacles_config) os.mkdir(spec_tentacles_config) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t1.json")) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t2.json")) - shutil.copy(profile.config_file(), f"{export_path}.{constants.PROFILE_EXPORT_FORMAT}") + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t1.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t2.json")) + shutil.copy(_profile_config_file(profile), f"{export_path}.{constants.PROFILE_EXPORT_FORMAT}") with mock.patch.object(os, "remove", mock.Mock()) as remove_mock: profiles.export_profile(profile, export_path) remove_mock.assert_called_once_with(f"{export_path}.{constants.PROFILE_EXPORT_FORMAT}") @@ -115,35 +121,35 @@ def test_import_install_profile(profile, invalid_profile): dir2=user_root_folder_provider.get_user_root_folder(), dir3=spec_tentacles_config): # create fake tentacles config - shutil.copy(profile.config_file(), tentacles_config) + shutil.copy(_profile_config_file(profile), tentacles_config) os.mkdir(spec_tentacles_config) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t1.json")) - shutil.copy(profile.config_file(), os.path.join(spec_tentacles_config, "t2.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t1.json")) + shutil.copy(_profile_config_file(profile), os.path.join(spec_tentacles_config, "t2.json")) profiles.export_profile(profile, export_path) - imported_profile_path = os.path.join(user_root_folder_provider.get_user_profiles_folder(), "default") with mock.patch.object(profile_sharing, "_ensure_unique_profile_id", mock.Mock()) \ as _ensure_unique_profile_id_mock: imported_profile = profiles.import_profile(exported_file, profile_schema, origin_url="plop.wow") assert isinstance(imported_profile, profiles.Profile) - profile.read_config() assert profile.name == imported_profile.name assert profile.path != imported_profile.path assert profile.imported is False assert imported_profile.imported is True assert imported_profile.origin_url == "plop.wow" _ensure_unique_profile_id_mock.assert_called_once() + imported_profile_path = imported_profile.path assert os.path.isdir(imported_profile_path) # ensure all files got imported for root, dirs, files in os.walk(profile.path): - dir_path = os.path.join(other_profile, "specific_config") if "specific_config" in root else other_profile + dir_path = os.path.join(imported_profile_path, "specific_config") if "specific_config" in root else imported_profile_path assert all( os.path.isfile(os.path.join(dir_path, f)) for f in files ) - assert isinstance(profiles.import_profile(exported_file, profile_schema), profiles.Profile) - assert os.path.isdir(f"{imported_profile_path}_2") + second_imported_profile = profiles.import_profile(exported_file, profile_schema) + assert isinstance(second_imported_profile, profiles.Profile) + assert os.path.isdir(second_imported_profile.path) + assert second_imported_profile.path != imported_profile_path assert os.path.isdir(imported_profile_path) - assert not os.path.isdir(f"{imported_profile_path}_3") # now with invalid profile profiles.export_profile(invalid_profile, export_path) @@ -152,17 +158,17 @@ def test_import_install_profile(profile, invalid_profile): def test_get_unique_profile_folder(profile): - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_2" - other_file = f"{profile.config_file()}_2" - other_file_2 = f"{profile.config_file()}_3" - other_file_3 = f"{profile.config_file()}_5" + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_2" + other_file = f"{_profile_config_file(profile)}_2" + other_file_2 = f"{_profile_config_file(profile)}_3" + other_file_3 = f"{_profile_config_file(profile)}_5" with _cleaned_tentacles(other_file, other_file_2, other_file_3): - shutil.copy(profile.config_file(), other_file) - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_3" - shutil.copy(profile.config_file(), other_file_2) - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_4" - shutil.copy(profile.config_file(), other_file_3) - assert _get_unique_profile_folder(profile.config_file()) == f"{profile.config_file()}_4" + shutil.copy(_profile_config_file(profile), other_file) + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_3" + shutil.copy(_profile_config_file(profile), other_file_2) + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_4" + shutil.copy(_profile_config_file(profile), other_file_3) + assert _get_unique_profile_folder(_profile_config_file(profile)) == f"{_profile_config_file(profile)}_4" def test_ensure_unique_profile_id(profile): @@ -171,10 +177,13 @@ def test_ensure_unique_profile_id(profile): other_profile_path = profiles_path.joinpath(other_profile) with _cleaned_tentacles(dir1=other_profile_path): shutil.copytree(profile.path, other_profile_path) - other_profile = profiles.Profile(other_profile_path).read_config() + filesystem_backend = profile_backends_module.FilesystemProfileBackend() + other_profile = filesystem_backend.read_profile_from_path(str(other_profile_path)) + other_profile.bind_profile_storage(profile.get_profile_storage()) _ensure_unique_profile_id(other_profile) - other_profile.save() - ids = profiles.Profile.get_all_profiles_ids(profiles_path) + filesystem_backend.write_profile_config(other_profile) + profile_storage = profile_storage_module.ProfileStorage(str(profiles_path)) + ids = profile_storage.list_profile_ids() assert len(ids) == 2 # changed new profile id assert ids[0] != ids[1] diff --git a/packages/commons/tests/profiles/test_profile_storage.py b/packages/commons/tests/profiles/test_profile_storage.py new file mode 100644 index 0000000000..d6fd8574bd --- /dev/null +++ b/packages/commons/tests/profiles/test_profile_storage.py @@ -0,0 +1,669 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import asyncio +import os +import copy + +import mock +import pytest + +import octobot_commons.constants as constants +import octobot_commons.enums as enums +import octobot_commons.errors as errors_module +import octobot_commons.json_util as json_util +import octobot_commons.profiles.profile_types.profile as profile_module +import octobot_commons.profiles.backends as profile_backends_module +import octobot_commons.profiles.profile_data as profile_data_module +import octobot_commons.profiles.profile_storage as profile_storage_module +import octobot_commons.profiles.profile_types.sync_profile as sync_profile_module +import octobot_commons.profiles.profile_data_import as profile_data_import_module + + +class TestProfileStorageListProfiles: + def test_filesystem_profile_wins_on_id_conflict(self, tmp_path): + profiles_path = os.path.join(tmp_path, constants.PROFILES_FOLDER) + os.makedirs(profiles_path, exist_ok=True) + default_profile_path = os.path.join(profiles_path, constants.DEFAULT_PROFILE) + os.makedirs(default_profile_path, exist_ok=True) + profile_file = { + constants.CONFIG_PROFILE: { + constants.CONFIG_ID: constants.DEFAULT_PROFILE, + constants.CONFIG_NAME: "filesystem-default", + }, + constants.PROFILE_CONFIG: { + constants.CONFIG_CRYPTO_CURRENCIES: {}, + constants.CONFIG_EXCHANGES: {}, + constants.CONFIG_TRADER: {constants.CONFIG_ENABLED_OPTION: False}, + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {}, + constants.CONFIG_SIMULATOR_FEES: {}, + }, + constants.CONFIG_TRADING: { + constants.CONFIG_TRADER_REFERENCE_MARKET: constants.DEFAULT_REFERENCE_MARKET, + constants.CONFIG_TRADER_RISK: 1, + }, + constants.CONFIG_DISTRIBUTION: constants.DEFAULT_DISTRIBUTION, + }, + } + import octobot_commons.json_util as json_util + + json_util.safe_dump(profile_file, os.path.join(default_profile_path, constants.PROFILE_CONFIG_FILE)) + + sync_profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"id": constants.DEFAULT_PROFILE, "name": "sync-default"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + } + ) + sync_profile = sync_profile_module.SyncProfile( + sync_profile_data, + os.path.join(tmp_path, "runtime", constants.DEFAULT_PROFILE), + ) + filesystem_backend = mock.Mock() + filesystem_backend.list_profiles.return_value = { + constants.DEFAULT_PROFILE: profile_module.Profile(default_profile_path) + } + filesystem_backend.list_profiles.return_value[ + constants.DEFAULT_PROFILE + ] = profile_backends_module.FilesystemProfileBackend().read_profile_from_path( + default_profile_path + ) + sync_backend = mock.Mock() + sync_backend.list_profiles.return_value = {constants.DEFAULT_PROFILE: sync_profile} + profile_storage = profile_storage_module.ProfileStorage( + profiles_path, + None, + filesystem_backend=filesystem_backend, + sync_backend=sync_backend, + ) + profiles = profile_storage._list_profiles() + assert profiles[constants.DEFAULT_PROFILE].name == "filesystem-default" + + +class TestProfileStorageConfigureSyncUser: + def test_configure_sync_user_validates_wallet(self, profile_storage, monkeypatch): + authenticator = mock.Mock() + authenticator.get_wallet_by_user_id.return_value = mock.Mock() + monkeypatch.setattr( + "octobot_commons.authentication.Authenticator.instance", + mock.Mock(return_value=authenticator), + ) + profile_storage.configure_sync_user("wallet-user") + assert profile_storage.is_sync_available() + authenticator.get_wallet_by_user_id.assert_called_once_with("wallet-user") + + +class TestProfileStorageBindProcessChildSyncUserId: + def test_bind_process_child_sync_user_id_sets_backend_without_wallet(self, profile_storage): + profile_storage.bind_process_child_sync_user_id("process-child-user") + assert profile_storage.is_sync_available() + assert profile_storage._sync_user_id == "process-child-user" + + def test_bind_process_child_sync_user_id_rejects_empty(self, profile_storage): + with pytest.raises(errors_module.ProfileDataError, match="non-empty"): + profile_storage.bind_process_child_sync_user_id("") + + +class TestProfileStorageListSyncProfiles: + def test_returns_empty_when_sync_unavailable(self, profile_storage): + assert profile_storage.list_sync_profiles() == {} + + def test_returns_sync_profiles_with_storage_bound(self, profile_storage): + sync_profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"id": "sync-profile-id", "name": "sync-profile"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + } + ) + sync_profile = sync_profile_module.SyncProfile( + sync_profile_data, + os.path.join(profile_storage.profiles_path, "runtime", "sync-profile-id"), + ) + sync_backend = mock.Mock() + sync_backend.list_profiles.return_value = {"sync-profile-id": sync_profile} + profile_storage._sync_backend = sync_backend + profile_storage.bind_process_child_sync_user_id("process-child-user") + + profiles_by_id = profile_storage.list_sync_profiles() + + sync_backend.list_profiles.assert_called_once_with(None) + assert profiles_by_id == {"sync-profile-id": sync_profile} + assert sync_profile.get_profile_storage() is profile_storage + + +class TestSyncProfileBackendImportProfileData: + def test_import_profile_data_assigns_strategy_id(self, tmp_path): + sync_backend = profile_backends_module.SyncProfileBackend( + sync_user_id="wallet-user" + ) + profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"name": "my-profile"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + } + ) + created_profile = None + + def _create_item(user_id, strategy): + nonlocal created_profile + assert user_id == "wallet-user" + assert strategy.id == profile_data.profile_details.id + created_profile = strategy + return strategy + + import octobot_sync.sync.collection_backend.errors as collection_errors + + strategy_provider = mock.Mock() + strategy_provider.update_item.side_effect = collection_errors.ItemNotFoundError( + "missing" + ) + strategy_provider.create_item.side_effect = _create_item + with mock.patch.object( + sync_backend, + "_get_strategy_provider", + mock.Mock(return_value=strategy_provider), + ): + profile = sync_backend.import_profile_data( + profile_data, + schema_path=None, + name="my-profile", + ) + assert profile.is_sync_backed() + assert profile.profile_id == profile_data.profile_details.id + assert created_profile is not None + + +class TestSyncProfileBackendListProfiles: + def test_list_profiles_logs_exception_on_failure(self): + sync_backend = profile_backends_module.SyncProfileBackend( + "/profiles", + sync_user_id="wallet-user", + ) + with mock.patch.object( + sync_backend, + "_list_profile_strategies", + mock.Mock(side_effect=RuntimeError("sync storage unavailable")), + ), mock.patch( + "octobot_commons.profiles.backends.sync_profile_backend._get_logger", + ) as get_logger_mock: + logger_mock = mock.Mock() + get_logger_mock.return_value = logger_mock + profiles_by_id = sync_backend.list_profiles() + assert profiles_by_id == {} + logger_mock.exception.assert_called_once() + + +class TestSyncProfileBackendDuplicateProfile: + def test_duplicate_profile_assigns_new_strategy_id(self, profile): + sync_backend = profile_backends_module.SyncProfileBackend( + sync_user_id="wallet-user" + ) + created_strategy_id = None + + def _create_item(user_id, strategy): + nonlocal created_strategy_id + assert user_id == "wallet-user" + created_strategy_id = strategy.id + return strategy + + import octobot_sync.sync.collection_backend.errors as collection_errors + + strategy_provider = mock.Mock() + strategy_provider.update_item.side_effect = collection_errors.ItemNotFoundError( + "missing" + ) + strategy_provider.create_item.side_effect = _create_item + with mock.patch.object( + sync_backend, + "_get_strategy_provider", + mock.Mock(return_value=strategy_provider), + ): + duplicate = sync_backend.duplicate_profile( + profile, + name="copy-name", + description="copy-desc", + ) + assert duplicate.is_sync_backed() + assert duplicate.profile_id == created_strategy_id + assert duplicate.profile_id != profile.profile_id + assert duplicate.name == "copy-name" + assert duplicate.description == "copy-desc" + assert duplicate.read_only is False + assert duplicate.imported is False + assert duplicate.origin_url is None + + +class TestProfileStorageListProfileIdsMerge: + def test_list_profile_ids_merges_filesystem_and_sync_ids(self): + filesystem_backend = mock.Mock() + filesystem_backend.list_profile_ids.return_value = ["fs-profile"] + sync_backend = mock.Mock() + sync_backend.list_profile_ids.return_value = ["sync-profile", "fs-profile"] + profile_storage = profile_storage_module.ProfileStorage( + "/profiles", + None, + filesystem_backend=filesystem_backend, + sync_backend=sync_backend, + ) + profile_ids = profile_storage.list_profile_ids() + assert profile_ids == ["fs-profile", "sync-profile"] + filesystem_backend.list_profile_ids.assert_called_once_with(ignore=None) + sync_backend.list_profile_ids.assert_called_once_with(ignore=None) + + +class TestProfileStorageDuplicateProfile: + def test_duplicate_profile_requires_sync(self, profile_storage, profile): + with pytest.raises( + errors_module.ProfileDataError, + match="configured wallet user id", + ): + profile_storage.duplicate_profile(profile) + + def test_duplicate_profile_delegates_to_sync_backend( + self, profile_storage, profile, monkeypatch + ): + authenticator = mock.Mock() + authenticator.get_wallet_by_user_id.return_value = mock.Mock() + monkeypatch.setattr( + "octobot_commons.authentication.Authenticator.instance", + mock.Mock(return_value=authenticator), + ) + profile_storage.configure_sync_user("wallet-user") + sync_duplicate = mock.Mock() + sync_duplicate.bind_profile_storage = mock.Mock() + sync_backend = mock.Mock() + sync_backend.duplicate_profile.return_value = sync_duplicate + profile_storage._sync_backend = sync_backend + result = profile_storage.duplicate_profile( + profile, name="copy", description="desc" + ) + sync_backend.duplicate_profile.assert_called_once_with( + profile, + name="copy", + description="desc", + ) + sync_duplicate.bind_profile_storage.assert_called_once_with(profile_storage) + assert result is sync_duplicate + + +class TestProfileStorageMasterOverlay: + def _write_profile_file(self, profile_path: str, profile_id: str, *, read_only: bool) -> None: + import octobot_commons.json_util as json_util + + os.makedirs(profile_path, exist_ok=True) + profile_file = { + constants.CONFIG_PROFILE: { + constants.CONFIG_ID: profile_id, + constants.CONFIG_NAME: profile_id, + constants.CONFIG_READ_ONLY: read_only, + }, + constants.PROFILE_CONFIG: { + constants.CONFIG_CRYPTO_CURRENCIES: {}, + constants.CONFIG_EXCHANGES: {}, + constants.CONFIG_TRADER: {constants.CONFIG_ENABLED_OPTION: False}, + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {}, + constants.CONFIG_SIMULATOR_FEES: {}, + }, + constants.CONFIG_TRADING: { + constants.CONFIG_TRADER_REFERENCE_MARKET: constants.DEFAULT_REFERENCE_MARKET, + constants.CONFIG_TRADER_RISK: 1, + }, + constants.CONFIG_DISTRIBUTION: constants.DEFAULT_DISTRIBUTION, + }, + } + json_util.safe_dump(profile_file, os.path.join(profile_path, constants.PROFILE_CONFIG_FILE)) + + def test_overlay_exposes_read_only_profiles(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + readonly_profile_id = "readonly-strategy" + self._write_profile_file( + os.path.join(master_profiles_path, readonly_profile_id), + readonly_profile_id, + read_only=True, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + profiles = profile_storage.load_all_profiles() + assert readonly_profile_id in profiles + assert profiles[readonly_profile_id].read_only is True + + def test_overlay_exposes_editable_profiles(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + editable_profile_id = "editable-strategy" + self._write_profile_file( + os.path.join(master_profiles_path, editable_profile_id), + editable_profile_id, + read_only=False, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + profiles = profile_storage.load_all_profiles() + assert editable_profile_id in profiles + assert profiles[editable_profile_id].read_only is False + + def test_local_filesystem_profile_wins_on_id_conflict(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + profile_id = "shared-id" + child_profiles_path.mkdir(parents=True, exist_ok=True) + self._write_profile_file( + os.path.join(master_profiles_path, profile_id), + profile_id, + read_only=True, + ) + self._write_profile_file( + os.path.join(child_profiles_path, profile_id), + profile_id, + read_only=False, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + profiles = profile_storage.load_all_profiles() + assert profiles[profile_id].name == profile_id + assert profiles[profile_id].path == os.path.join(child_profiles_path, profile_id) + + def test_save_active_profile_blocks_master_overlay_profile(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + readonly_profile_id = "non-trading" + self._write_profile_file( + os.path.join(master_profiles_path, readonly_profile_id), + readonly_profile_id, + read_only=True, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + overlay_profile = profile_storage.get_profile(readonly_profile_id) + with pytest.raises( + errors_module.ProfileDataError, + match="shared from the master", + ): + profile_storage.save_active_profile(overlay_profile, {}) + + def test_save_active_profile_persists_editable_master_overlay_on_master_path(self, tmp_path): + import octobot_commons.json_util as json_util + + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + editable_profile_id = "editable-strategy" + master_profile_path = os.path.join(master_profiles_path, editable_profile_id) + self._write_profile_file( + master_profile_path, + editable_profile_id, + read_only=False, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + overlay_profile = profile_storage.get_profile(editable_profile_id) + overlay_profile.config[constants.CONFIG_TRADER] = { + constants.CONFIG_ENABLED_OPTION: True, + } + profile_storage.save_active_profile(overlay_profile, {}) + master_profile_file = json_util.read_file( + os.path.join(master_profile_path, constants.PROFILE_CONFIG_FILE) + ) + assert ( + master_profile_file[constants.PROFILE_CONFIG][constants.CONFIG_TRADER][ + constants.CONFIG_ENABLED_OPTION + ] + is True + ) + assert not os.path.isdir(os.path.join(child_profiles_path, editable_profile_id)) + + def test_delete_profile_blocks_master_overlay_profile(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + readonly_profile_id = "readonly-strategy" + self._write_profile_file( + os.path.join(master_profiles_path, readonly_profile_id), + readonly_profile_id, + read_only=True, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + overlay_profile = profile_storage.get_profile(readonly_profile_id) + with pytest.raises( + errors_module.ProfileRemovalError, + match="shared from the master", + ): + profile_storage.delete_profile(readonly_profile_id, profile=overlay_profile) + + def test_delete_profile_removes_editable_master_overlay_from_master_path(self, tmp_path): + child_profiles_path = tmp_path / "child" / constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True) + master_profiles_path = tmp_path / "master" / constants.PROFILES_FOLDER + editable_profile_id = "editable-strategy" + master_profile_path = os.path.join(master_profiles_path, editable_profile_id) + self._write_profile_file( + master_profile_path, + editable_profile_id, + read_only=False, + ) + profile_storage = profile_storage_module.ProfileStorage(str(child_profiles_path), None) + profile_storage.configure_readonly_profiles_path(str(master_profiles_path)) + overlay_profile = profile_storage.get_profile(editable_profile_id) + profile_storage.delete_profile(editable_profile_id, profile=overlay_profile) + assert not os.path.isdir(master_profile_path) + assert not os.path.isdir(os.path.join(child_profiles_path, editable_profile_id)) + + +class TestSyncProfileBackendSaveProfile: + def test_save_profile_persists_exchanges_and_portfolio(self, tmp_path): + sync_backend = profile_backends_module.SyncProfileBackend( + sync_user_id="wallet-user" + ) + initial_profile_data = profile_data_module.ProfileData.from_dict( + { + "profile_details": {"id": "aaaa", "name": "AAAA"}, + "trading": {"reference_market": constants.DEFAULT_REFERENCE_MARKET}, + "trader_simulator": { + "enabled": True, + "starting_portfolio": {"USDT": 100}, + }, + } + ) + profile = sync_profile_module.SyncProfile( + initial_profile_data, + str(tmp_path / "runtime"), + ) + profile.profile_id = "aaaa" + profile.name = "AAAA" + profile_storage = profile_storage_module.ProfileStorage( + str(tmp_path / constants.PROFILES_FOLDER), + None, + sync_backend=sync_backend, + ) + profile.bind_profile_storage(profile_storage) + global_config = { + constants.CONFIG_CRYPTO_CURRENCIES: copy.deepcopy( + profile.config[constants.CONFIG_CRYPTO_CURRENCIES] + ), + constants.CONFIG_DISTRIBUTION: profile.config[constants.CONFIG_DISTRIBUTION], + constants.CONFIG_TRADING: copy.deepcopy(profile.config[constants.CONFIG_TRADING]), + constants.CONFIG_TRADER: copy.deepcopy(profile.config[constants.CONFIG_TRADER]), + constants.CONFIG_SIMULATOR: { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_STARTING_PORTFOLIO: {"BTC": 5, "USDT": 5000}, + constants.CONFIG_SIMULATOR_FEES: { + constants.CONFIG_SIMULATOR_FEES_MAKER: 0.1, + constants.CONFIG_SIMULATOR_FEES_TAKER: 0.1, + }, + }, + constants.CONFIG_EXCHANGES: { + "binance": { + constants.CONFIG_ENABLED_OPTION: True, + constants.CONFIG_EXCHANGE_TYPE: "spot", + }, + }, + } + strategy_provider = mock.Mock() + strategy_provider.update_item = mock.Mock() + with mock.patch.object( + sync_backend, + "_get_strategy_provider", + mock.Mock(return_value=strategy_provider), + ): + profile.save_config(global_config) + saved_profile_data = profile.get_profile_data() + assert saved_profile_data.trader_simulator.starting_portfolio == { + "BTC": 5, + "USDT": 5000, + } + assert len(saved_profile_data.exchanges) == 1 + assert saved_profile_data.exchanges[0].internal_name == "binance" + assert saved_profile_data.exchanges[0].exchange_type == "spot" + strategy_provider.update_item.assert_called_once() + + +def _profile_schema_path() -> str: + return os.path.join( + os.path.dirname(__file__), + "..", + "static", + "profile_schema.json", + ) + + +def _cloud_like_fetched_profile_data() -> profile_data_module.ProfileData: + return profile_data_module.ProfileData.from_dict( + { + "profile_details": { + "name": "proud-bear_fetched_config", + "id": "test-profile-id", + "bot_id": "test-bot-id", + }, + "trading": {"reference_market": "USDT"}, + "trader": {"enabled": True}, + "trader_simulator": {"enabled": False}, + "exchanges": [ + { + "internal_name": "lbank", + "exchange_type": "spot", + } + ], + } + ) + + +class TestProfileStorageImportProfileDataFunctional: + def test_import_profile_data_round_trips_full_profile_shape(self, tmp_path, monkeypatch): + profile_name = "proud-bear_fetched_config" + original_slug = profile_name + bot_install_path = str(tmp_path) + user_folder = tmp_path / constants.USER_FOLDER + profiles_path = user_folder / constants.PROFILES_FOLDER + profiles_path.mkdir(parents=True) + monkeypatch.chdir(bot_install_path) + profile_schema = os.path.abspath(_profile_schema_path()) + profile_storage = profile_storage_module.ProfileStorage(str(profiles_path), None) + assert profile_storage.is_sync_available() is False + profile_data = _cloud_like_fetched_profile_data() + tentacles_setup_config_mock = mock.Mock() + tentacles_setup_config_mock.save_config.return_value = True + + with mock.patch( + "octobot_tentacles_manager.configuration.profile_tentacles_util.build_setup_config_from_profile_data", + mock.Mock(return_value=tentacles_setup_config_mock), + ), mock.patch( + "octobot_tentacles_manager.configuration.profile_tentacles_util.write_specific_configs_to_profile_folder", + mock.Mock(return_value=False), + ): + imported_profile = asyncio.run( + profile_storage.import_profile_data( + profile_data, + profile_schema, + bot_install_path, + name=profile_name, + ) + ) + + profile_folder = profiles_path / profile_name + profile_config_path = profile_backends_module.FilesystemProfileBackend.config_file_path( + str(profile_folder) + ) + + assert profile_folder.is_dir() + assert os.path.isfile(profile_config_path) + assert not os.path.isfile( + os.path.join(profile_folder, constants.CONFIG_TENTACLES_FILE) + ) + + on_disk_profile = json_util.read_file(profile_config_path) + on_disk_metadata = on_disk_profile[constants.CONFIG_PROFILE] + assert on_disk_metadata[constants.CONFIG_RISK] is not None + assert on_disk_metadata[constants.CONFIG_COMPLEXITY] is not None + assert on_disk_metadata[constants.CONFIG_TYPE] is not None + assert on_disk_metadata[constants.CONFIG_NAME] == profile_name + assert on_disk_metadata[constants.CONFIG_SLUG] == original_slug + assert on_disk_metadata[constants.CONFIG_READ_ONLY] is True + assert on_disk_metadata[constants.CONFIG_IMPORTED] is True + assert on_disk_metadata[constants.CONFIG_AUTO_UPDATE] is False + assert on_disk_metadata[constants.CONFIG_RISK] == enums.ProfileRisk.MODERATE.value + assert on_disk_metadata[constants.CONFIG_COMPLEXITY] == enums.ProfileComplexity.MEDIUM.value + assert on_disk_metadata[constants.CONFIG_TYPE] == enums.ProfileType.LIVE.value + assert on_disk_metadata[constants.CONFIG_EXTRA_BACKTESTING_TIME_FRAMES] == [ + profile_data_import_module.IMPORTED_PROFILES_DEFAULT_EXTRA_BACKTESTING_TIMEFRAME + ] + assert on_disk_metadata[constants.CONFIG_ID] + + on_disk_config = on_disk_profile[constants.PROFILE_CONFIG] + assert on_disk_config[constants.CONFIG_EXCHANGES]["lbank"][constants.CONFIG_ENABLED_OPTION] is True + assert on_disk_config[constants.CONFIG_TRADER][constants.CONFIG_ENABLED_OPTION] is True + assert on_disk_config[constants.CONFIG_SIMULATOR][constants.CONFIG_ENABLED_OPTION] is False + assert ( + on_disk_config[constants.CONFIG_TRADING][constants.CONFIG_TRADER_REFERENCE_MARKET] + == "USDT" + ) + + self._assert_imported_profile_metadata( + imported_profile, + profile_name=profile_name, + original_slug=original_slug, + ) + + reloaded_profile = profile_backends_module.FilesystemProfileBackend().read_profile_from_path( + str(profile_folder), + schema_path=profile_schema, + ) + self._assert_imported_profile_metadata( + reloaded_profile, + profile_name=profile_name, + original_slug=original_slug, + ) + assert imported_profile.as_dict()[constants.CONFIG_PROFILE] == reloaded_profile.as_dict()[ + constants.CONFIG_PROFILE + ] + assert imported_profile.as_dict()[constants.PROFILE_CONFIG] == reloaded_profile.as_dict()[ + constants.PROFILE_CONFIG + ] + + def _assert_imported_profile_metadata( + self, + profile: profile_module.Profile, + *, + profile_name: str, + original_slug: str, + ): + assert profile.name == profile_name + assert profile.slug == original_slug + assert profile.read_only is True + assert profile.imported is True + assert profile.auto_update is False + assert profile.risk == enums.ProfileRisk.MODERATE + assert profile.complexity == enums.ProfileComplexity.MEDIUM + assert profile.profile_type == enums.ProfileType.LIVE + assert profile.extra_backtesting_time_frames == [ + profile_data_import_module.IMPORTED_PROFILES_DEFAULT_EXTRA_BACKTESTING_TIMEFRAME + ] + assert profile.profile_id diff --git a/packages/commons/tests/static/profile.json b/packages/commons/tests/static/profile.json index caa51ba337..b5992503f7 100644 --- a/packages/commons/tests/static/profile.json +++ b/packages/commons/tests/static/profile.json @@ -1,43 +1,53 @@ { - "profile": { - "avatar": "default_profile.png", - "description": "OctoBot default profile.", - "id": "default", - "name": "default", - "origin_url": "https://default.url" - }, - "config": { - "crypto-currencies": { - "Bitcoin": { - "pairs": [ - "BTC/USDT" - ] - } + "config": { + "crypto-currencies": { + "Bitcoin": { + "pairs": [ + "BTC/USDT" + ] + } + }, + "distribution": "default", + "exchanges": { + "binance": { + "enabled": true + } + }, + "trader": { + "enabled": false, + "load-trade-history": true + }, + "trader-simulator": { + "enabled": true, + "fees": { + "maker": 0.1, + "taker": 0.1 + }, + "starting-portfolio": { + "BTC": 10, + "USDT": 1000 + } + }, + "trading": { + "paused": false, + "reference-market": "BTC", + "risk": 0.5 + } }, - "exchanges": { - "binance": { - "enabled": true - } - }, - "trading": { - "paused": false, - "reference-market": "BTC", - "risk": 0.5 - }, - "trader": { - "enabled": false, - "load-trade-history": true - }, - "trader-simulator": { - "enabled": true, - "fees": { - "maker": 0.1, - "taker": 0.1 - }, - "starting-portfolio": { - "BTC": 10, - "USDT": 1000 - } + "profile": { + "auto_update": false, + "avatar": "default_profile.png", + "complexity": 2, + "description": "OctoBot default profile.", + "extra_backtesting_time_frames": [], + "hidden": false, + "id": "default", + "imported": false, + "name": "default", + "origin_url": "https://default.url", + "read_only": false, + "risk": 2, + "slug": "", + "type": "live" } - } -} +} \ No newline at end of file diff --git a/packages/commons/tests/test_managed_child_process_registry.py b/packages/commons/tests/test_managed_child_process_registry.py new file mode 100644 index 0000000000..2d5b7f2757 --- /dev/null +++ b/packages/commons/tests/test_managed_child_process_registry.py @@ -0,0 +1,235 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import sys +import time + +import mock +import pytest + +import octobot_commons.managed_child_process_registry as managed_child_process_registry +import octobot_commons.process_util as process_util +import octobot_commons.singleton.singleton_class as singleton_class + + +@pytest.fixture(autouse=True) +def reset_managed_child_process_registry(): + yield + singleton_class.Singleton._instances.pop( + managed_child_process_registry.ManagedChildProcessRegistry, + None, + ) + + +class TestManagedChildProcessRegistryInstance: + def test_instance_returns_same_object(self): + first = managed_child_process_registry.ManagedChildProcessRegistry.instance() + second = managed_child_process_registry.ManagedChildProcessRegistry.instance() + assert first is second + + +class TestManagedChildProcessRegistryRegister: + def test_register_ignores_non_positive_pid(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + registry.register(0) + registry.register(-3) + assert registry.snapshot_running_pids() == frozenset() + + def test_register_adds_pid_to_snapshot(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object( + managed_child_process_registry.process_util, + "pid_is_running", + return_value=True, + ): + registry.register(101) + assert registry.snapshot_running_pids() == frozenset({101}) + + def test_register_is_idempotent(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object( + managed_child_process_registry.process_util, + "pid_is_running", + return_value=True, + ): + registry.register(202) + registry.register(202) + assert registry.snapshot_running_pids() == frozenset({202}) + + +class TestManagedChildProcessRegistryUnregister: + def test_unregister_removes_pid(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(303) + registry.unregister(303) + assert registry.snapshot_running_pids() == frozenset() + + def test_unregister_unknown_pid_is_no_op(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + registry.unregister(99999) + assert registry.snapshot_running_pids() == frozenset() + + +class TestRebindManagedChildPid: + def test_rebind_replaces_spawn_pid_with_authoritative_pid(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(20520) + registry.rebind_managed_child_pid(20520, 25044) + assert registry.snapshot_running_pids() == frozenset({25044}) + + def test_rebind_no_op_for_non_positive_authoritative_pid(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(101) + registry.rebind_managed_child_pid(101, 0) + assert registry.snapshot_running_pids() == frozenset({101}) + + def test_rebind_same_pid_is_idempotent(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(505) + with mock.patch.object(registry, "register") as register_mock: + registry.rebind_managed_child_pid(505, 505) + register_mock.assert_not_called() + assert registry.snapshot_running_pids() == frozenset({505}) + + def test_second_rebind_to_same_authoritative_pid_is_no_op(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(20520) + registry.rebind_managed_child_pid(20520, 25044) + with mock.patch.object(registry, "register") as register_mock: + registry.rebind_managed_child_pid(20520, 25044) + register_mock.assert_not_called() + assert registry.snapshot_running_pids() == frozenset({25044}) + + @pytest.mark.asyncio + async def test_graceful_stop_all_signals_authoritative_pid_after_rebind(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + stop_requested = False + + def pid_is_running_side_effect(pid): + if pid == 25044 and stop_requested: + return False + return True + + def sigterm_side_effect(pid, *, logger=None): + nonlocal stop_requested + stop_requested = True + return {"status": "stopped", "signal": "sigterm"} + + with ( + mock.patch.object( + managed_child_process_registry.process_util, + "pid_is_running", + side_effect=pid_is_running_side_effect, + ), + mock.patch.object( + managed_child_process_registry.process_util, + "request_graceful_stop_via_sigterm", + side_effect=sigterm_side_effect, + ) as sigterm_mock, + ): + registry.register(20520) + registry.rebind_managed_child_pid(20520, 25044) + outcomes = await registry.graceful_stop_all(timeout_seconds=0.1, poll_interval=0.01) + sigterm_mock.assert_called_once_with(25044, logger=registry._logger) + assert outcomes[25044] == "stopped" + assert 20520 not in outcomes + + +class TestManagedChildProcessRegistrySnapshotRunningPids: + def test_lazy_prunes_dead_pids(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + registry.register(404) + with mock.patch.object(process_util, "pid_is_running", return_value=False): + assert registry.snapshot_running_pids() == frozenset() + assert registry.snapshot_running_pids() == frozenset() + + +class TestManagedChildProcessRegistryGracefulStopAll: + @pytest.mark.asyncio + async def test_stops_spawned_child_via_sigterm(self, tmp_path): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + child = process_util.spawn_managed_subprocess( + [ + sys.executable, + "-c", + "import time; time.sleep(30)", + ], + working_directory=str(tmp_path), + ) + try: + assert child.pid in registry.snapshot_running_pids() + outcomes = await registry.graceful_stop_all(timeout_seconds=15.0) + assert outcomes[child.pid] in {"stopped", "already_stopped"} + deadline = time.monotonic() + 15.0 + while child.poll() is None and time.monotonic() < deadline: + await asyncio.sleep(0.05) + assert child.poll() is not None + assert child.pid not in registry.snapshot_running_pids() + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=10) + + @pytest.mark.asyncio + async def test_force_kills_child_when_graceful_stop_times_out(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + force_kill_requested = False + + def pid_is_running_side_effect(_pid): + return not force_kill_requested + + def request_force_kill_side_effect(_pid, *, logger=None): + nonlocal force_kill_requested + force_kill_requested = True + return {"status": "force_killed"} + + with ( + mock.patch.object( + managed_child_process_registry.process_util, + "pid_is_running", + side_effect=pid_is_running_side_effect, + ), + mock.patch.object( + managed_child_process_registry.process_util, + "request_graceful_stop_via_sigterm", + return_value={"status": "stopped", "signal": "sigterm"}, + ), + mock.patch.object( + managed_child_process_registry.process_util, + "request_force_kill", + side_effect=request_force_kill_side_effect, + ) as force_kill_mock, + ): + registry.register(777) + outcomes = await registry.graceful_stop_all( + timeout_seconds=0.1, + poll_interval=0.01, + ) + assert outcomes[777] == "force_killed" + force_kill_mock.assert_called_once_with(777, logger=registry._logger) + assert registry.snapshot_running_pids() == frozenset() + + @pytest.mark.asyncio + async def test_returns_empty_when_no_registered_children(self): + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + outcomes = await registry.graceful_stop_all(timeout_seconds=1.0) + assert outcomes == {} diff --git a/packages/commons/tests/test_network.py b/packages/commons/tests/test_network.py new file mode 100644 index 0000000000..dda25df451 --- /dev/null +++ b/packages/commons/tests/test_network.py @@ -0,0 +1,258 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import ipaddress +import socket + +import mock +import psutil +import pytest + +import octobot_commons.network as network + + +NETWORK_MODULE = "octobot_commons.network" + + +def _first_interface_ipv4(): + for interface_name, addresses in psutil.net_if_addrs().items(): + for address in addresses: + if address.family == socket.AF_INET: + return interface_name, address.address + return None, None + + +class TestGetInterfaceIpv4ByNameSubstring: + def test_returns_ipv4_when_interface_name_contains_substring(self): + tailscale_address = mock.Mock() + tailscale_address.family = socket.AF_INET + tailscale_address.address = "100.64.0.1" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={"tailscale0": [tailscale_address]}, + ): + assert network.get_interface_ipv4_by_name_substring("tailscale") == "100.64.0.1" + + def test_returns_none_when_no_matching_interface(self): + with mock.patch(f"{NETWORK_MODULE}.psutil.net_if_addrs", return_value={}): + assert network.get_interface_ipv4_by_name_substring("tailscale") is None + + def test_matches_substring_case_insensitively(self): + tailscale_address = mock.Mock() + tailscale_address.family = socket.AF_INET + tailscale_address.address = "100.64.0.2" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={"Tailscale0": [tailscale_address]}, + ): + assert network.get_interface_ipv4_by_name_substring("tailscale") == "100.64.0.2" + + def test_skips_interfaces_without_substring(self): + ethernet_address = mock.Mock() + ethernet_address.family = socket.AF_INET + ethernet_address.address = "192.168.0.5" + tailscale_address = mock.Mock() + tailscale_address.family = socket.AF_INET + tailscale_address.address = "100.64.0.3" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={ + "eth0": [ethernet_address], + "tailscale0": [tailscale_address], + }, + ): + assert network.get_interface_ipv4_by_name_substring("tailscale") == "100.64.0.3" + + def test_skips_non_af_inet_addresses(self): + ipv6_address = mock.Mock() + ipv6_address.family = socket.AF_INET6 + ipv6_address.address = "fe80::1" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={"tailscale0": [ipv6_address]}, + ): + assert network.get_interface_ipv4_by_name_substring("tailscale") is None + + def test_returns_first_af_inet_on_matching_interface(self): + first_address = mock.Mock() + first_address.family = socket.AF_INET + first_address.address = "100.64.0.4" + second_address = mock.Mock() + second_address.family = socket.AF_INET + second_address.address = "100.64.0.5" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={"tailscale0": [first_address, second_address]}, + ): + assert network.get_interface_ipv4_by_name_substring("tailscale") == "100.64.0.4" + + +class TestGetInterfaceIpv4ByPrefix: + def test_returns_first_ipv4_matching_prefix_across_interfaces(self): + ethernet_address = mock.Mock() + ethernet_address.family = socket.AF_INET + ethernet_address.address = "192.168.0.5" + tailscale_address = mock.Mock() + tailscale_address.family = socket.AF_INET + tailscale_address.address = "100.115.92.45" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={ + "eth0": [ethernet_address], + "utun3": [tailscale_address], + }, + ): + assert network.get_interface_ipv4_by_prefix("100.") == "100.115.92.45" + + def test_returns_none_when_no_address_matches_prefix(self): + ethernet_address = mock.Mock() + ethernet_address.family = socket.AF_INET + ethernet_address.address = "192.168.0.5" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={"eth0": [ethernet_address]}, + ): + assert network.get_interface_ipv4_by_prefix("100.") is None + + def test_skips_non_af_inet_addresses(self): + ipv6_address = mock.Mock() + ipv6_address.family = socket.AF_INET6 + ipv6_address.address = "fe80::1" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={"utun3": [ipv6_address]}, + ): + assert network.get_interface_ipv4_by_prefix("100.") is None + + def test_returns_first_matching_ipv4_when_multiple_exist(self): + first_address = mock.Mock() + first_address.family = socket.AF_INET + first_address.address = "100.64.0.4" + second_address = mock.Mock() + second_address.family = socket.AF_INET + second_address.address = "100.64.0.5" + with mock.patch( + f"{NETWORK_MODULE}.psutil.net_if_addrs", + return_value={ + "utun3": [first_address, second_address], + }, + ): + assert network.get_interface_ipv4_by_prefix("100.") == "100.64.0.4" + + +class TestGetInterfaceIpv4ByNameSubstringPsutilIntegration: + def test_psutil_net_if_addrs_runs_without_error(self): + interface_addresses = psutil.net_if_addrs() + assert isinstance(interface_addresses, dict) + + def test_returns_none_for_nonexistent_substring(self): + assert network.get_interface_ipv4_by_name_substring("__octobot_no_such_iface__") is None + + def test_returns_valid_ipv4_for_real_interface(self): + interface_name, expected_ipv4 = _first_interface_ipv4() + if interface_name is None: + pytest.skip("No AF_INET network interface on this host") + result = network.get_interface_ipv4_by_name_substring(interface_name.lower()) + assert result == expected_ipv4 + assert ipaddress.ip_address(result).version == 4 + + +class TestGetLocalNetworkIp: + def test_returns_udp_route_private_ip(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("192.168.1.42", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + assert network.get_local_network_ip() == "192.168.1.42" + + def test_falls_back_to_hostname_private_ip(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("8.8.8.8", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + with mock.patch( + f"{NETWORK_MODULE}.socket.getaddrinfo", + return_value=[(None, None, None, None, ("10.0.0.5", 0))], + ): + assert network.get_local_network_ip() == "10.0.0.5" + + def test_returns_none_when_no_private_ip_found(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("127.0.0.1", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + with mock.patch( + f"{NETWORK_MODULE}.socket.getaddrinfo", + side_effect=OSError, + ): + assert network.get_local_network_ip() is None + + def test_prefers_192_address_over_udp_10_address(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("10.0.0.5", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + with mock.patch( + f"{NETWORK_MODULE}.socket.getaddrinfo", + return_value=[(None, None, None, None, ("192.168.1.42", 0))], + ): + assert network.get_local_network_ip() == "192.168.1.42" + + def test_prefers_192_address_from_hostname_candidates(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("8.8.8.8", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + with mock.patch( + f"{NETWORK_MODULE}.socket.getaddrinfo", + return_value=[ + (None, None, None, None, ("10.0.0.5", 0)), + (None, None, None, None, ("192.168.0.10", 0)), + ], + ): + assert network.get_local_network_ip() == "192.168.0.10" + + def test_uses_udp_probe_constants(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("192.168.1.42", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + network.get_local_network_ip() + mock_socket.connect.assert_called_once_with( + (network.UDP_ROUTE_PROBE_HOST, network.UDP_ROUTE_PROBE_PORT), + ) + + def test_ignores_udp_non_private_candidate(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("8.8.8.8", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + with mock.patch( + f"{NETWORK_MODULE}.socket.getaddrinfo", + return_value=[(None, None, None, None, ("10.0.0.5", 0))], + ): + assert network.get_local_network_ip() == "10.0.0.5" + + def test_deduplicates_udp_and_hostname_candidates(self): + mock_socket = mock.MagicMock() + mock_socket.getsockname.return_value = ("192.168.1.42", 54321) + mock_socket.__enter__.return_value = mock_socket + with mock.patch(f"{NETWORK_MODULE}.socket.socket", return_value=mock_socket): + with mock.patch( + f"{NETWORK_MODULE}.socket.getaddrinfo", + return_value=[(None, None, None, None, ("192.168.1.42", 0))], + ): + assert network.get_local_network_ip() == "192.168.1.42" diff --git a/packages/commons/tests/test_os_util.py b/packages/commons/tests/test_os_util.py index 011cab57ee..378d22c5fd 100644 --- a/packages/commons/tests/test_os_util.py +++ b/packages/commons/tests/test_os_util.py @@ -15,6 +15,7 @@ # License along with this library. import mock import socket +import sys import pytest import octobot_commons.os_util as os_util @@ -30,6 +31,28 @@ def test_get_cpu_and_ram_usage(): assert unique_ram > 0 +class TestIsFrozenBinaryOctobot: + def test_returns_true_when_sys_frozen_is_set(self): + with mock.patch.object(sys, "frozen", True, create=True): + assert os_util.is_frozen_binary_octobot() is True + + def test_returns_false_for_interpreter_argv_without_sys_frozen(self): + with mock.patch.object(sys, "argv", [sys.executable]), mock.patch.object( + sys, "frozen", False, create=True + ): + assert os_util.is_frozen_binary_octobot() is False + + def test_returns_false_when_sys_frozen_absent(self): + original_frozen = getattr(sys, "frozen", None) + if hasattr(sys, "frozen"): + delattr(sys, "frozen") + try: + assert os_util.is_frozen_binary_octobot() is False + finally: + if original_frozen is not None: + sys.frozen = original_frozen + + class TestTcpPortIsFree: def test_returns_false_when_tcp_listener_holds_port(self): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: diff --git a/packages/commons/tests/test_process_util.py b/packages/commons/tests/test_process_util.py index eba9fb9393..5453274eb6 100644 --- a/packages/commons/tests/test_process_util.py +++ b/packages/commons/tests/test_process_util.py @@ -23,12 +23,23 @@ import pytest import octobot_commons.errors as commons_errors +import octobot_commons.managed_child_process_registry as managed_child_process_registry import octobot_commons.process_util as process_util +import octobot_commons.singleton.singleton_class as singleton_class class TestSpawnManagedSubprocess: + @pytest.fixture(autouse=True) + def reset_managed_child_process_registry(self): + yield + singleton_class.Singleton._instances.pop( + managed_child_process_registry.ManagedChildProcessRegistry, + None, + ) + def test_popen_called_with_argv_cwd_env(self): fake_handle = mock.Mock(spec=subprocess.Popen) + fake_handle.pid = 1001 with mock.patch.object( process_util.subprocess, "Popen", @@ -54,12 +65,14 @@ def test_uses_os_environ_copy_when_environment_missing(self): {"EXISTING_ENV_KEY": "1"}, clear=False, ): + popen_mock.return_value.pid = 1002 process_util.spawn_managed_subprocess([], working_directory="/w") _args, keywords = popen_mock.call_args assert keywords["env"]["EXISTING_ENV_KEY"] == "1" def test_creationflags_hide_console_on_windows(self): fake_handle = mock.Mock(spec=subprocess.Popen) + fake_handle.pid = 1003 with mock.patch.object(process_util.sys, "platform", "win32"), mock.patch.object( process_util.subprocess, "Popen", @@ -73,6 +86,19 @@ def test_creationflags_hide_console_on_windows(self): 0, ) + def test_registers_child_pid_with_managed_child_process_registry(self): + fake_handle = mock.Mock(spec=subprocess.Popen) + fake_handle.pid = 4242 + with mock.patch.object( + process_util.subprocess, + "Popen", + return_value=fake_handle, + ): + process_util.spawn_managed_subprocess(["x"], working_directory="/work") + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + assert fake_handle.pid in registry.snapshot_running_pids() + class TestPidIsRunning: def test_non_positive_pid_is_false(self): @@ -126,14 +152,14 @@ def test_with_psutil_no_such_process_from_is_running(self): class TestRequestGracefulStopViaSigterm: def test_invalid_pid_raises(self): - with pytest.raises(commons_errors.DSLInterpreterError, match="Invalid pid"): + with pytest.raises(commons_errors.ProcessError, match="Invalid pid"): process_util.request_graceful_stop_via_sigterm(0) def test_raises_when_sigterm_unavailable(self): sentinel_signal_module = mock.Mock() sentinel_signal_module.SIGTERM = None with mock.patch.object(process_util, "signal", sentinel_signal_module): - with pytest.raises(commons_errors.DSLInterpreterError, match="SIGTERM is not available"): + with pytest.raises(commons_errors.ProcessError, match="SIGTERM is not available"): process_util.request_graceful_stop_via_sigterm(10) def test_returns_already_stopped_when_pid_not_running(self): @@ -155,12 +181,45 @@ def test_os_kill_failure_when_still_running_wraps_error(self): mock.patch.object(process_util.os, "kill", side_effect=OSError("perm denied")), ): with pytest.raises( - commons_errors.DSLInterpreterError, + commons_errors.ProcessError, match=r"Failed to send stop signal to pid=88", ): process_util.request_graceful_stop_via_sigterm(88) +class TestRequestForceKill: + def test_invalid_pid_raises(self): + with pytest.raises(commons_errors.ProcessError, match="Invalid pid"): + process_util.request_force_kill(0) + + def test_returns_already_stopped_when_pid_not_running(self): + with mock.patch.object(process_util, "pid_is_running", return_value=False): + result = process_util.request_force_kill(55) + assert result["status"] == "already_stopped" + + def test_kills_running_process(self): + fake_process = mock.Mock() + with ( + mock.patch.object(process_util, "pid_is_running", return_value=True), + mock.patch.object(process_util.psutil, "Process", return_value=fake_process), + ): + result = process_util.request_force_kill(66) + fake_process.kill.assert_called_once() + assert result["status"] == "force_killed" + + def test_no_such_process_returns_already_stopped(self): + with ( + mock.patch.object(process_util, "pid_is_running", return_value=True), + mock.patch.object( + process_util.psutil, + "Process", + side_effect=process_util.psutil.NoSuchProcess(77), + ), + ): + result = process_util.request_force_kill(77) + assert result["status"] == "already_stopped" + + class TestSpawnManagedSubprocessGracefulStopIntegration: def test_spawned_sleeping_child_can_be_stopped_by_request_graceful_stop_via_sigterm( self, diff --git a/packages/commons/tests/test_user_root_folder_provider.py b/packages/commons/tests/test_user_root_folder_provider.py new file mode 100644 index 0000000000..9f2d514fd3 --- /dev/null +++ b/packages/commons/tests/test_user_root_folder_provider.py @@ -0,0 +1,47 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. + +import os + +import octobot_commons.constants as constants +import octobot_commons.user_root_folder_provider as user_root_folder_provider + + +class TestUserRootFolderProviderReadonlyReferenceTentaclesPath: + def setup_method(self): + provider = user_root_folder_provider.UserRootFolderProvider.instance() + provider.set_root("user/automations/test-bot") + provider.configure_readonly_reference_tentacles_path("") + + def teardown_method(self): + provider = user_root_folder_provider.UserRootFolderProvider.instance() + provider.set_root(None) # type: ignore[arg-type] + provider.configure_readonly_reference_tentacles_path("") + + def test_uses_override_for_reference_directory(self, tmp_path): + master_reference_path = tmp_path / "master" / "reference_tentacles_config" + master_reference_path.mkdir(parents=True) + provider = user_root_folder_provider.UserRootFolderProvider.instance() + provider.configure_readonly_reference_tentacles_path(str(master_reference_path)) + assert provider.get_user_reference_tentacle_config_path() == str(master_reference_path) + + def test_derives_reference_file_and_specific_paths_from_override(self, tmp_path): + master_reference_path = tmp_path / "master" / "reference_tentacles_config" + master_reference_path.mkdir(parents=True) + provider = user_root_folder_provider.UserRootFolderProvider.instance() + provider.configure_readonly_reference_tentacles_path(str(master_reference_path)) + assert provider.get_user_reference_tentacle_config_file_path() == str( + master_reference_path / constants.CONFIG_TENTACLES_FILE + ) + assert provider.get_user_reference_tentacle_specific_config_path() == str( + master_reference_path / "specific_config" + ) + + def test_falls_back_to_user_root_when_override_unset(self): + provider = user_root_folder_provider.UserRootFolderProvider.instance() + provider.set_root("user/automations/child-bot") + provider.configure_readonly_reference_tentacles_path("") + assert provider.get_user_reference_tentacle_config_path() == os.path.join( + "user/automations/child-bot", + "reference_tentacles_config", + ) diff --git a/packages/copy/octobot_copy/constants.py b/packages/copy/octobot_copy/constants.py index 24d6f50493..3942c18e19 100644 --- a/packages/copy/octobot_copy/constants.py +++ b/packages/copy/octobot_copy/constants.py @@ -33,6 +33,7 @@ # Exchange / order lifecycle (seconds) FILL_ORDER_TIMEOUT = 60 +OPEN_ORDER_POLL_INTERVAL = 0.5 # Mirrored orphan grace: max |simulated_copier_pair_share − ref_pair_share| to allow deferral DEFAULT_MIRRORED_ORPHAN_GRACE_PAIR_RATIO_MAX_DELTA = decimal.Decimal("0.02") # 2% diff --git a/packages/copy/octobot_copy/copiers/account_copier.py b/packages/copy/octobot_copy/copiers/account_copier.py index 54485839c1..96bc0b069c 100644 --- a/packages/copy/octobot_copy/copiers/account_copier.py +++ b/packages/copy/octobot_copy/copiers/account_copier.py @@ -120,6 +120,12 @@ async def _resync_if_mirrored_open_order_grace_period_elapsed(self) -> None: self._orders_synchronizer.abort_mirrored_orphan_grace() await self._orders_synchronizer.cancel_orders_pending_synchronization(None) await self._copier_exchange_interface.portfolio.refresh_portfolio() + reconciled_count = await self._orders_synchronizer.reconcile_open_orders_with_reference() + if reconciled_count: + self._get_logger().info( + f"Reconciled {reconciled_count} stray open order(s) after grace-abort portfolio refresh " + f"on [{self._copier_exchange_interface.exchange_name}]" + ) async def _synchronize_reference_open_orders(self) -> list[trading_personal_data.Order]: return await self._orders_synchronizer.synchronize() diff --git a/packages/copy/octobot_copy/exchange/orders.py b/packages/copy/octobot_copy/exchange/orders.py index 1f0c87a948..e781fa1446 100644 --- a/packages/copy/octobot_copy/exchange/orders.py +++ b/packages/copy/octobot_copy/exchange/orders.py @@ -1,5 +1,6 @@ import asyncio import decimal +import time import typing import octobot_commons.constants as commons_constants @@ -186,6 +187,105 @@ async def wait_for_orders_to_fill(self, orders: list) -> None: self._exchange_manager, orders, copy_constants.FILL_ORDER_TIMEOUT, True, temp_refresh_portfolio_on_static_wait=False ) + async def wait_for_orders_to_open( + self, + orders: list, + symbol: str, + *, + timeout: float = trading_constants.INDIVIDUAL_ORDER_SYNC_TIMEOUT, + poll_interval: float = copy_constants.OPEN_ORDER_POLL_INTERVAL, + ) -> None: + if not orders: + return + exchange_id_column = trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value + start_time = time.monotonic() + logger = commons_logging.get_logger(self.__class__.__name__) + exchange = self._exchange_manager.exchange + total_order_count = len(orders) + exchange_name = self._exchange_manager.exchange_name + poll_iteration = 0 + started_wait_log = False + deadline = start_time + timeout + pending_orders = [ + order for order in orders if order.is_pending_creation() + ] + while pending_orders and time.monotonic() < deadline: + if not started_wait_log: + logger.info( + "Waiting for %s mirrored order(s) on %s to open on %s (timeout=%ss)", + len(pending_orders), + symbol, + exchange_name, + timeout, + ) + started_wait_log = True + poll_iteration += 1 + raw_open_orders = await exchange.get_open_orders(symbol=symbol) + exchange_open_order_ids = { + raw_order[exchange_id_column] + for raw_order in raw_open_orders + if raw_order.get(exchange_id_column) is not None + } + promoted_count = 0 + for order in pending_orders: + if not order.exchange_order_id: + continue + if order.exchange_order_id not in exchange_open_order_ids: + continue + await self._promote_pending_order_to_open(order) + promoted_count += 1 + pending_orders = [ + order for order in orders if order.is_pending_creation() + ] + remaining_pending_count = len(pending_orders) + open_count = total_order_count - remaining_pending_count + if promoted_count: + logger.info( + "Promoted %s mirrored order(s) to open on %s for %s (%s/%s open, %s remaining)", + promoted_count, + exchange_name, + symbol, + open_count, + total_order_count, + remaining_pending_count, + ) + else: + logger.debug( + "Still waiting for %s mirrored order(s) on %s to open on %s (poll %s)", + remaining_pending_count, + symbol, + exchange_name, + poll_iteration, + ) + await asyncio.sleep(poll_interval) + if pending_orders: + pending_exchange_ids = [ + order.exchange_order_id for order in pending_orders if order.exchange_order_id + ] + logger.warning( + "Timed out waiting for %s mirrored order(s) to open on %s: %s", + len(pending_orders), + exchange_name, + pending_exchange_ids, + ) + else: + logger.info( + "All %s mirrored order(s) open on %s for %s after %.1fs", + total_order_count, + exchange_name, + symbol, + time.monotonic() - start_time, + ) + + async def _promote_pending_order_to_open(self, order) -> None: + orders_manager = self._exchange_manager.exchange_personal_data.orders_manager + for index, pending_order in enumerate(orders_manager.pending_creation_orders): + if pending_order is order or pending_order.order_id == order.order_id: + orders_manager.pending_creation_orders.pop(index) + break + order.status = trading_enums.OrderStatus.OPEN + await order.on_open(force_open=True, is_from_exchange_data=False) + def get_open_orders(self, symbol: typing.Optional[str] = None, active: typing.Optional[bool] = None) -> list: return trading_api.get_open_orders(self._exchange_manager, symbol=symbol, active=active) @@ -269,6 +369,7 @@ async def cancel_order( ignored_order: object = None, wait_for_cancelling: bool = True, dependencies: typing.Optional[commons_signals.SignalDependencies] = None, + skip_pending_cancel_status: bool = True, ) -> tuple[bool, commons_signals.SignalDependencies]: if self._trading_mode is not None: return await self._trading_mode.cancel_order( @@ -276,6 +377,7 @@ async def cancel_order( ignored_order=ignored_order, wait_for_cancelling=wait_for_cancelling, dependencies=dependencies, + skip_pending_cancel_status=skip_pending_cancel_status, ) return await signal_creation.cancel_order( self._exchange_manager, @@ -284,6 +386,7 @@ async def cancel_order( ignored_order=ignored_order, wait_for_cancelling=wait_for_cancelling, dependencies=dependencies, + skip_pending_cancel_status=skip_pending_cancel_status, ) async def cancel_symbol_open_orders( diff --git a/packages/copy/octobot_copy/orders_mirroring/mirrored_order_replication_failure_util.py b/packages/copy/octobot_copy/orders_mirroring/mirrored_order_replication_failure_util.py index 0a1a364af9..7e59a89006 100644 --- a/packages/copy/octobot_copy/orders_mirroring/mirrored_order_replication_failure_util.py +++ b/packages/copy/octobot_copy/orders_mirroring/mirrored_order_replication_failure_util.py @@ -85,14 +85,16 @@ def format_order_mirror_completion_message( *, orphan_cancelled_count: int, replaced_cancelled_count: int, + reconciled_cancelled_count: int = 0, total_created: int, already_synchronized_count: int, replication_failures: list[mirrored_order_replication_failure.MirroredOrderReplicationFailure], ) -> str: - total_cancelled = orphan_cancelled_count + replaced_cancelled_count + total_cancelled = orphan_cancelled_count + replaced_cancelled_count + reconciled_cancelled_count completion_message = ( f"Order mirror completed: {total_cancelled} cancelled " - f"[{orphan_cancelled_count} orphan(s), {replaced_cancelled_count} replaced], " + f"[{orphan_cancelled_count} orphan(s), {replaced_cancelled_count} replaced, " + f"{reconciled_cancelled_count} reconciled], " f"{total_created} created, " f"{already_synchronized_count} already synchronized orders." ) diff --git a/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py b/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py index 7507b27e7c..0dd04d45ef 100644 --- a/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py +++ b/packages/copy/octobot_copy/orders_mirroring/orders_synchronizer.py @@ -33,6 +33,8 @@ def __init__( self._copy_settings = copy_settings self._force_immediate_orphan_cancel_next: bool = False self._mirrored_orphan_cancel_was_deferred_in_episode: bool = False + self._claimed_reference_order_ids: set[str] = set() + self._claimed_exchange_order_ids: set[str] = set() def _get_replicable_reference_orders_from( self, @@ -497,10 +499,22 @@ async def synchronize(self) -> list: async with self._exchange_interface.portfolio.mirror_sync_available_updates(): return await self._synchronize_impl() + async def reconcile_open_orders_with_reference(self) -> int: + """Cancel open limits in the order manager that do not match reference price levels.""" + replicable = self._get_replicable_reference_orders() + return await self._reconcile_open_orders_with_reference(replicable) + async def _synchronize_impl(self) -> list: """Align copier open orders with reference_account.orders (synched mirror rows).""" + self._claimed_reference_order_ids = set() + self._claimed_exchange_order_ids = set() replicable = self._get_replicable_reference_orders() + # Compute grace skip symbols before pre-sync reconcile so grace symbols use stray_only. skip_symbols_for_upsert = self._reference_symbols_skipped_while_grace_orphans_uncancelled(replicable) + reconciled_cancelled_count = await self._reconcile_open_orders_with_reference( + replicable, + stray_only_symbols=skip_symbols_for_upsert, + ) skip_symbols_for_upsert = self._maybe_bypass_grace_for_missing_mirrored_reference_orders( replicable, skip_symbols_for_upsert ) @@ -556,6 +570,11 @@ async def _synchronize_impl(self) -> list: order, "creation_error" ) ) + # Always post-sync reconcile; same grace rules as pre-sync (bypass may have cleared skip set). + reconciled_cancelled_count += await self._reconcile_open_orders_with_reference( + replicable, + stray_only_symbols=skip_symbols_for_upsert, + ) if skipped_grace_upserts: skipped_summary = ", ".join( mirrored_order_replication_failure_util.format_replication_failure_entry(failure) @@ -570,11 +589,19 @@ async def _synchronize_impl(self) -> list: completion_message = mirrored_order_replication_failure_util.format_order_mirror_completion_message( orphan_cancelled_count=orphan_cancelled_count, replaced_cancelled_count=replaced_cancelled_count, + reconciled_cancelled_count=reconciled_cancelled_count, total_created=len(created), already_synchronized_count=already_synchronized_count, replication_failures=replication_failures, ) self._get_logger().info(completion_message) + if created and not self._exchange_interface.orders.automatically_synchronize_orders(): + symbols = {order.symbol for order in created} + for symbol in symbols: + symbol_created = [order for order in created if order.symbol == symbol] + await self._exchange_interface.orders.wait_for_orders_to_open(symbol_created, symbol) + # After wait (or when wait is skipped): count is confirmed against local open orders. + self._check_open_limit_order_count_invariant(replicable) return created def _format_grace_deferral_order_details( @@ -752,6 +779,212 @@ async def _cancel_mirrored_orphan_order_list( ) return cancelled_count + def _is_open_limit_order(self, order: trading_personal_data.Order) -> bool: + return trading_personal_data.get_trade_order_type(order.order_type) is not trading_enums.TradeOrderType.MARKET + + def _reference_price_level_for_open_order( + self, + order: trading_personal_data.Order, + replicable: list[protocol_models.Order], + ) -> typing.Optional[tuple[str, trading_enums.TradeOrderSide, decimal.Decimal]]: + for reference_order in replicable: + if reference_order.symbol != order.symbol: + continue + if order.side is None: + continue + raw = trading_personal_data.exchange_columns_dict_from_protocol_order(reference_order) + reference_side, trader_order_type = trading_personal_data.parse_order_type(raw) + if reference_side is None or reference_side != order.side: + continue + if trading_personal_data.get_trade_order_type(trader_order_type) is trading_enums.TradeOrderType.MARKET: + continue + reference_price_value = reference_order.price + if reference_price_value in (None, ""): + continue + reference_price = decimal.Decimal(str(reference_price_value)) + price_threshold = self._limit_order_price_match_threshold(reference_price) + if abs(order.origin_price - reference_price) <= price_threshold: + return (order.symbol, reference_side, reference_price) + return None + + async def _cancel_reconciled_stray_order(self, order: trading_personal_data.Order) -> int: + try: + await self._exchange_interface.orders.cancel_order(order) + self._get_logger().info( + "Reconciled stray open order: symbol=%s exchange_id=%s side=%s price=%s order_id=%s", + order.symbol, + order.exchange_order_id, + order.side, + order.origin_price, + order.order_id, + ) + return 1 + except trading_errors.UnexpectedExchangeSideOrderStateError as err: + self._get_logger().exception( + err, + True, + f"Skipped reconciled stray cancel: {err}, order: {order}", + ) + return 0 + + def _count_reference_orders_at_level( + self, + replicable: list[protocol_models.Order], + level: tuple[str, trading_enums.TradeOrderSide, decimal.Decimal], + ) -> int: + symbol, side, reference_price = level + price_threshold = self._limit_order_price_match_threshold(reference_price) + matching_count = 0 + for reference_order in replicable: + if reference_order.symbol != symbol: + continue + raw = trading_personal_data.exchange_columns_dict_from_protocol_order(reference_order) + reference_side, trader_order_type = trading_personal_data.parse_order_type(raw) + if reference_side != side: + continue + if trading_personal_data.get_trade_order_type(trader_order_type) is trading_enums.TradeOrderType.MARKET: + continue + reference_price_value = reference_order.price + if reference_price_value in (None, ""): + continue + candidate_price = decimal.Decimal(str(reference_price_value)) + if abs(candidate_price - reference_price) <= price_threshold: + matching_count += 1 + return matching_count + + def _rank_orders_at_reference_level( + self, + orders_at_level: list[trading_personal_data.Order], + active_reference_ids: set[str], + ) -> list[trading_personal_data.Order]: + def sort_key(open_order: trading_personal_data.Order) -> tuple: + has_active_reference_id = str(open_order.order_id) in active_reference_ids + has_mirrored_tag = open_order.tag == copy_constants.MIRRORED_ORDER_TAG + exchange_order_id = str(open_order.exchange_order_id or "") + return ( + 0 if has_active_reference_id else 1, + 0 if has_mirrored_tag else 1, + exchange_order_id, + ) + + return sorted(orders_at_level, key=sort_key) + + async def _reconcile_open_orders_with_reference( + self, + replicable: list[protocol_models.Order], + stray_only_symbols: typing.Optional[set[str]] = None, + ) -> int: + """ + Cancel open limits that do not match reference price levels. + + Wrong-price strays are cancelled except during grace (``stray_only_symbols``): mirrored + orphans are kept so late-fill grace can resolve them. Untagged / non-orphan wrong-price + opens are still cancelled during grace. Duplicate-at-price extras are cancelled unless + ``symbol`` is in ``stray_only_symbols`` and force-abort is not set. + """ + if not replicable: + return 0 + symbols = {order.symbol for order in replicable} + active_reference_ids = self._active_reference_order_ids(replicable) + grace_stray_only_symbols = stray_only_symbols or set() + # Force-abort clears grace: full reconcile even on previously skipped symbols. + if self._force_immediate_orphan_cancel_next: + grace_stray_only_symbols = set() + cancelled_count = 0 + for symbol in symbols: + # Load open limits from the pre-loaded order manager (no exchange fetch here). + open_limit_orders = [ + order + for order in self._exchange_interface.orders.get_open_orders(symbol=symbol) + if self._is_open_limit_order(order) + ] + # Bucket each open onto a reference (symbol, side, price) level, or mark as wrong-price. + orders_by_level: dict[ + tuple[str, trading_enums.TradeOrderSide, decimal.Decimal], + list[trading_personal_data.Order], + ] = {} + stray_orders: list[trading_personal_data.Order] = [] + for open_order in open_limit_orders: + matched_level = self._reference_price_level_for_open_order(open_order, replicable) + if matched_level is None: + stray_orders.append(open_order) + continue + orders_by_level.setdefault(matched_level, []).append(open_order) + # Cancel wrong-price strays. During grace, leave mirrored orphans for grace/late-fill. + for stray_order in stray_orders: + if symbol in grace_stray_only_symbols: + if ( + stray_order.tag == copy_constants.MIRRORED_ORDER_TAG + and str(stray_order.order_id) not in active_reference_ids + ): + continue + cancelled_count += await self._cancel_reconciled_stray_order(stray_order) + # During grace, skip duplicate-at-price cleanup so copier-ahead extras are not removed. + if symbol in grace_stray_only_symbols: + continue + # Full mode: cancel opens beyond the reference count at each price level. + for level, orders_at_level in orders_by_level.items(): + expected_count = self._count_reference_orders_at_level(replicable, level) + if len(orders_at_level) <= expected_count: + continue + ranked_orders = self._rank_orders_at_reference_level( + orders_at_level, + active_reference_ids, + ) + for duplicate_order in ranked_orders[expected_count:]: + cancelled_count += await self._cancel_reconciled_stray_order(duplicate_order) + return cancelled_count + + def _check_open_limit_order_count_invariant( + self, + replicable: list[protocol_models.Order], + ) -> None: + """ + Log when open limit count diverges from reference after adjusting for late-fill candidates. + + Late fills mean the copier correctly has fewer opens than reference still lists as OPEN. + Called after wait_for_orders_to_open (when used) so pending creations are already open. + Does not call reconcile — post-sync reconcile already ran. + """ + active_reference_ids = self._active_reference_order_ids(replicable) + orphan_orders = self._mirrored_orphan_open_orders(active_reference_ids) + late_fill_orders = ( + [] if self._force_immediate_orphan_cancel_next + else self._late_reference_fill_candidate_orders(replicable, orphan_orders, None) + ) + late_fill_count_by_symbol: dict[str, int] = {} + for late_fill_order in late_fill_orders: + late_fill_count_by_symbol[late_fill_order.symbol] = ( + late_fill_count_by_symbol.get(late_fill_order.symbol, 0) + 1 + ) + symbols = {order.symbol for order in replicable} + for symbol in symbols: + reference_count = sum(1 for order in replicable if order.symbol == symbol) + expected_count = reference_count - late_fill_count_by_symbol.get(symbol, 0) + open_limit_orders = [ + order + for order in self._exchange_interface.orders.get_open_orders(symbol=symbol) + if self._is_open_limit_order(order) + ] + actual_count = len(open_limit_orders) + if actual_count == expected_count: + continue + exchange_order_ids = [ + str(order.exchange_order_id) + for order in open_limit_orders + if order.exchange_order_id is not None + ] + self._get_logger().error( + "Open limit order count mismatch on %s: expected=%s actual=%s " + "reference=%s late_fills=%s exchange_ids=%s", + symbol, + expected_count, + actual_count, + reference_count, + late_fill_count_by_symbol.get(symbol, 0), + exchange_order_ids, + ) + def _scale_mirrored_order_quantity( self, order: protocol_models.Order, @@ -779,6 +1012,155 @@ def _find_open_order_by_bot_order_id(self, order_id: str) -> typing.Optional[tra return order return None + def _claim_mirrored_open_order(self, order: trading_personal_data.Order) -> None: + self._claimed_reference_order_ids.add(str(order.order_id)) + if order.exchange_order_id: + self._claimed_exchange_order_ids.add(str(order.exchange_order_id)) + + def _limit_order_price_match_threshold(self, reference_price: decimal.Decimal) -> decimal.Decimal: + price_tolerance = reference_price * self._copy_settings.mirrored_order_price_ratio_threshold + return max(price_tolerance, decimal.Decimal("1e-12")) + + def _open_orders_matching_symbol_side_price( + self, + symbol: str, + side: trading_enums.TradeOrderSide, + reference_price: decimal.Decimal, + trader_order_type: trading_enums.TraderOrderType, + ) -> list[trading_personal_data.Order]: + if trading_personal_data.get_trade_order_type(trader_order_type) is trading_enums.TradeOrderType.MARKET: + return [] + price_threshold = self._limit_order_price_match_threshold(reference_price) + candidates: list[trading_personal_data.Order] = [] + for open_order in self._exchange_interface.orders.get_open_orders(symbol=symbol): + if open_order.symbol != symbol: + continue + if open_order.side != side: + continue + if trading_personal_data.get_trade_order_type(open_order.order_type) is trading_enums.TradeOrderType.MARKET: + continue + if abs(open_order.origin_price - reference_price) > price_threshold: + continue + candidates.append(open_order) + return candidates + + def _is_order_claimed_by_another_reference( + self, + order: trading_personal_data.Order, + reference_order_id: str, + active_reference_ids: set[str], + ) -> bool: + order_id = str(order.order_id) + if order_id in active_reference_ids and order_id != reference_order_id: + return True + exchange_order_id = str(order.exchange_order_id) if order.exchange_order_id else None + if exchange_order_id is not None and exchange_order_id in self._claimed_exchange_order_ids: + return True + return False + + def _relink_open_order_to_reference( + self, + order: trading_personal_data.Order, + reference_order_id: str, + ) -> trading_personal_data.Order: + previous_id = order.order_id + order.order_id = reference_order_id + if order.tag != copy_constants.MIRRORED_ORDER_TAG: + order.tag = copy_constants.MIRRORED_ORDER_TAG + orders_manager = self._exchange_interface.orders._exchange_manager.exchange_personal_data.orders_manager + orders_manager.replace_order(previous_id, order) + self._get_logger().info( + "Mapped unmapped open order exchange_id=%s previous_bot_id=%s reference_id=%s", + order.exchange_order_id, + previous_id, + reference_order_id, + ) + return order + + def _pick_best_unmapped_open_order_candidate( + self, + candidates: list[trading_personal_data.Order], + scaled_reference_quantity: decimal.Decimal, + ) -> trading_personal_data.Order: + def sort_key(open_order: trading_personal_data.Order) -> tuple: + has_mirrored_tag = open_order.tag == copy_constants.MIRRORED_ORDER_TAG + quantity_distance = abs(open_order.origin_quantity - scaled_reference_quantity) + exchange_order_id = str(open_order.exchange_order_id or "") + return ( + 0 if has_mirrored_tag else 1, + quantity_distance, + exchange_order_id, + ) + + return min(candidates, key=sort_key) + + async def _map_unmapped_open_order_for_reference( + self, + reference_order: protocol_models.Order, + reference_order_id: str, + side: trading_enums.TradeOrderSide, + trader_order_type: trading_enums.TraderOrderType, + order_target_price: decimal.Decimal, + active_reference_ids: set[str], + scaled_reference_quantity: decimal.Decimal, + ) -> typing.Optional[trading_personal_data.Order]: + candidates = self._open_orders_matching_symbol_side_price( + reference_order.symbol, + side, + order_target_price, + trader_order_type, + ) + unclaimed_candidates = [ + candidate + for candidate in candidates + if not self._is_order_claimed_by_another_reference( + candidate, reference_order_id, active_reference_ids + ) + ] + if not unclaimed_candidates: + return None + if len(unclaimed_candidates) == 1: + mapped_order = self._relink_open_order_to_reference(unclaimed_candidates[0], reference_order_id) + self._claim_mirrored_open_order(mapped_order) + return mapped_order + chosen_candidate = self._pick_best_unmapped_open_order_candidate( + unclaimed_candidates, + scaled_reference_quantity, + ) + self._get_logger().warning( + "Ambiguous unmapped open order match for reference_id=%s symbol=%s side=%s price=%s: " + "%s candidate(s) at same price; relinking exchange_id=%s and cancelling extras", + reference_order_id, + reference_order.symbol, + side, + order_target_price, + len(unclaimed_candidates), + chosen_candidate.exchange_order_id, + ) + for duplicate_candidate in unclaimed_candidates: + if duplicate_candidate is chosen_candidate: + continue + try: + await self._exchange_interface.orders.cancel_order(duplicate_candidate) + self._get_logger().info( + "Cancelled ambiguous unmapped open order duplicate: symbol=%s exchange_id=%s " + "side=%s price=%s order_id=%s", + duplicate_candidate.symbol, + duplicate_candidate.exchange_order_id, + duplicate_candidate.side, + duplicate_candidate.origin_price, + duplicate_candidate.order_id, + ) + except trading_errors.UnexpectedExchangeSideOrderStateError as err: + self._get_logger().exception( + err, + True, + f"Skipped ambiguous duplicate cancel: {err}, order: {duplicate_candidate}", + ) + mapped_order = self._relink_open_order_to_reference(chosen_candidate, reference_order_id) + self._claim_mirrored_open_order(mapped_order) + return mapped_order + def _count_unmirrored_reference_orders(self, replicable: list[protocol_models.Order]) -> int: missing_count = 0 for order in replicable: @@ -852,10 +1234,18 @@ async def _upsert_mirrored_reference_order( ) return [], 0, 0, None reference_order_id = str(order.id) + current_price_val = order.price + order_target_price = ( + decimal.Decimal(str(current_price_val)) + if current_price_val not in (None, "") + else trading_constants.ZERO + ) existing = self._find_open_order_by_bot_order_id(reference_order_id) replicable_orders = self._get_replicable_reference_orders() active_reference_ids = self._active_reference_order_ids(replicable_orders) orphan_orders = self._mirrored_orphan_open_orders(active_reference_ids) + if existing is not None: + self._claim_mirrored_open_order(existing) if existing is None and not self._force_immediate_orphan_cancel_next and self._is_late_reference_fill_for_order(order, orphan_orders): self._get_logger().info( f"Skipping mirrored order creation (late reference fill on copier): symbol={symbol} " @@ -878,12 +1268,16 @@ async def _upsert_mirrored_reference_order( self._exchange_interface, ), ) - current_price_val = order.price - order_target_price = ( - decimal.Decimal(str(current_price_val)) - if current_price_val not in (None, "") - else trading_constants.ZERO - ) + if existing is None: + existing = await self._map_unmapped_open_order_for_reference( + reference_order=order, + reference_order_id=reference_order_id, + side=side, + trader_order_type=trader_order_type, + order_target_price=order_target_price, + active_reference_ids=active_reference_ids, + scaled_reference_quantity=scaled_quantity, + ) compute_result = await self._compute_mirrored_quantity_type_and_price( symbol, side, @@ -992,6 +1386,29 @@ async def _upsert_mirrored_reference_order( ) return out, replaced_cancelled, 0, None + def _get_locked_base_from_open_mirrored_sells( + self, + symbol: str, + exclude_order: typing.Optional[trading_personal_data.Order] = None, + ) -> decimal.Decimal: + parsed = symbol_util.parse_symbol(symbol) + base_currency = parsed.base + exclude_order_id = str(exclude_order.order_id) if exclude_order is not None else None + locked_base = trading_constants.ZERO + for order in self._exchange_interface.orders.get_open_orders(): + if order.symbol != symbol: + continue + if order.tag != copy_constants.MIRRORED_ORDER_TAG: + continue + if order.side is not trading_enums.TradeOrderSide.SELL: + continue + if exclude_order_id is not None and str(order.order_id) == exclude_order_id: + continue + if order.currency != base_currency: + continue + locked_base += self._exchange_interface.orders.get_order_locked_amount(order) + return locked_base + async def _compute_mirrored_quantity_type_and_price( self, symbol: str, @@ -1003,8 +1420,8 @@ async def _compute_mirrored_quantity_type_and_price( ) -> mirrored_quantity_compute_result.MirroredQuantityComputeResult: # Buys cap using free quote for new orders (sibling buys reserve quote). When re-checking an open # mirrored buy, add this order's locked quote back so ideal size matches portfolio semantics. - # New sells use total base (sibling sell locks still count). Open mirrored sells use available - # base plus this order's locked base for the same reason as buys. + # Sells cap using total base minus locked base from open mirrored sells (sibling locks count). + # When re-checking an open mirrored sell, exclude this order's lock from that sum. ( total_symbol_holding, _total_market_holding, @@ -1066,16 +1483,13 @@ async def _compute_mirrored_quantity_type_and_price( if effective_target_price else scaled_quantity, ) - elif ( - open_mirrored_order is not None - and open_mirrored_order.side is trading_enums.TradeOrderSide.SELL - ): - base_budget = available_symbol_holding + self._exchange_interface.orders.get_order_locked_amount( - open_mirrored_order + elif side is trading_enums.TradeOrderSide.SELL: + locked_by_siblings = self._get_locked_base_from_open_mirrored_sells( + symbol, + exclude_order=open_mirrored_order, ) + base_budget = total_symbol_holding - locked_by_siblings target_quantity = min(scaled_quantity, base_budget) - else: - target_quantity = min(scaled_quantity, total_symbol_holding) zero_short_reason: typing.Optional[str] = None if target_quantity <= trading_constants.ZERO: zero_short_reason = ( diff --git a/packages/copy/tests/python/exchange/test_orders_interface_wait_for_orders_to_open.py b/packages/copy/tests/python/exchange/test_orders_interface_wait_for_orders_to_open.py new file mode 100644 index 0000000000..61eac400e3 --- /dev/null +++ b/packages/copy/tests/python/exchange/test_orders_interface_wait_for_orders_to_open.py @@ -0,0 +1,169 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or +# (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# OctoBot. If not, see . +import logging + +import mock +import pytest + +import octobot_trading.enums as trading_enums + +import octobot_copy.exchange.orders as orders_module + +pytestmark = pytest.mark.asyncio + +_EXCHANGE_ID_COLUMN = trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value + + +def _orders_interface_with_exchange(): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + exchange_manager.exchange_personal_data.orders_manager.pending_creation_orders = [] + exchange = mock.AsyncMock() + exchange_manager.exchange = exchange + return orders_module.OrdersInterface(exchange_manager, None), exchange_manager, exchange + + +def _pending_order_stub(*, exchange_order_id: str, symbol: str = "BTC/USDC"): + order = mock.Mock() + order.symbol = symbol + order.exchange_order_id = exchange_order_id + order.order_id = f"local-{exchange_order_id}" + pending = True + + def is_pending_creation(): + return pending + + async def on_open(**_kwargs): + nonlocal pending + pending = False + + order.is_pending_creation = is_pending_creation + order.on_open = mock.AsyncMock(side_effect=on_open) + order.update_from_raw = mock.AsyncMock() + return order + + +class TestOrdersInterfaceWaitForOrdersToOpen: + async def test_returns_immediately_when_no_orders(self): + orders_interface, _, _ = _orders_interface_with_exchange() + await orders_interface.wait_for_orders_to_open([], "BTC/USDC") + + async def test_promotes_pending_orders_found_on_exchange(self): + orders_interface, exchange_manager, exchange = _orders_interface_with_exchange() + order = _pending_order_stub(exchange_order_id="ex-open-1") + exchange_manager.exchange_personal_data.orders_manager.pending_creation_orders = [order] + + async def get_open_orders(symbol=None, **kwargs): + return [{_EXCHANGE_ID_COLUMN: "ex-open-1"}] + + exchange.get_open_orders = get_open_orders + + async def promote_side_effect(promoted_order): + promoted_order.is_pending_creation = lambda: False + + with mock.patch.object( + orders_interface, + "_promote_pending_order_to_open", + mock.AsyncMock(side_effect=promote_side_effect), + ) as promote_mock: + await orders_interface.wait_for_orders_to_open( + [order], + "BTC/USDC", + poll_interval=0.01, + timeout=1, + ) + promote_mock.assert_awaited_once_with(order) + + async def test_does_not_call_update_from_raw_when_promoting(self): + orders_interface, exchange_manager, exchange = _orders_interface_with_exchange() + order = _pending_order_stub(exchange_order_id="ex-open-2") + exchange_manager.exchange_personal_data.orders_manager.pending_creation_orders = [order] + exchange.get_open_orders = mock.AsyncMock( + return_value=[{_EXCHANGE_ID_COLUMN: "ex-open-2"}], + ) + await orders_interface.wait_for_orders_to_open( + [order], + "BTC/USDC", + poll_interval=0.01, + timeout=1, + ) + order.on_open.assert_awaited() + order.update_from_raw.assert_not_awaited() + + async def test_logs_warning_on_timeout(self, caplog): + orders_interface, _, exchange = _orders_interface_with_exchange() + order = _pending_order_stub(exchange_order_id="ex-still-pending") + exchange.get_open_orders = mock.AsyncMock(return_value=[]) + with caplog.at_level(logging.WARNING): + await orders_interface.wait_for_orders_to_open( + [order], + "BTC/USDC", + poll_interval=0.01, + timeout=0.05, + ) + assert any( + "Timed out waiting for 1 mirrored order(s) to open on kraken" in record.message + for record in caplog.records + ) + + async def test_logs_start_and_completion_when_orders_open(self, caplog): + orders_interface, exchange_manager, exchange = _orders_interface_with_exchange() + order = _pending_order_stub(exchange_order_id="ex-open-3") + exchange_manager.exchange_personal_data.orders_manager.pending_creation_orders = [order] + exchange.get_open_orders = mock.AsyncMock( + return_value=[{_EXCHANGE_ID_COLUMN: "ex-open-3"}], + ) + with caplog.at_level(logging.INFO): + await orders_interface.wait_for_orders_to_open( + [order], + "BTC/USDC", + poll_interval=0.01, + timeout=1, + ) + info_messages = [record.message for record in caplog.records if record.levelno == logging.INFO] + assert any("Waiting for 1 mirrored order(s) on BTC/USDC to open on kraken" in message for message in info_messages) + assert any("All 1 mirrored order(s) open on kraken for BTC/USDC after" in message for message in info_messages) + + async def test_logs_promotion_progress(self, caplog): + orders_interface, exchange_manager, exchange = _orders_interface_with_exchange() + first_order = _pending_order_stub(exchange_order_id="ex-open-a") + second_order = _pending_order_stub(exchange_order_id="ex-open-b") + exchange_manager.exchange_personal_data.orders_manager.pending_creation_orders = [ + first_order, + second_order, + ] + poll_count = 0 + + async def get_open_orders(symbol=None, **kwargs): + nonlocal poll_count + poll_count += 1 + if poll_count == 1: + return [{_EXCHANGE_ID_COLUMN: "ex-open-a"}] + return [{_EXCHANGE_ID_COLUMN: "ex-open-b"}] + + exchange.get_open_orders = get_open_orders + with caplog.at_level(logging.INFO): + await orders_interface.wait_for_orders_to_open( + [first_order, second_order], + "BTC/USDC", + poll_interval=0.01, + timeout=1, + ) + info_messages = [record.message for record in caplog.records if record.levelno == logging.INFO] + assert any( + "Promoted 1 mirrored order(s) to open on kraken for BTC/USDC (1/2 open, 1 remaining)" in message + for message in info_messages + ) diff --git a/packages/copy/tests/python/orders_mirroring/functional_tests/test_sec_copy_grid_twenty_grace_elapsed.py b/packages/copy/tests/python/orders_mirroring/functional_tests/test_sec_copy_grid_twenty_grace_elapsed.py index 1a17a0d808..e928030666 100644 --- a/packages/copy/tests/python/orders_mirroring/functional_tests/test_sec_copy_grid_twenty_grace_elapsed.py +++ b/packages/copy/tests/python/orders_mirroring/functional_tests/test_sec_copy_grid_twenty_grace_elapsed.py @@ -350,7 +350,9 @@ def build_exchange_interface(self) -> mock.MagicMock: ) exchange_if.market.get_market_status = mock.Mock(return_value=self.symbol_market) exchange_if.market.is_market_open_for_order_type = mock.Mock(return_value=True) - exchange_if.orders.get_open_orders = mock.Mock(side_effect=lambda: list(self.open_orders)) + exchange_if.orders.get_open_orders = mock.Mock( + side_effect=lambda symbol=None: list(self.open_orders) + ) exchange_if.orders.get_pre_order_data = mock.AsyncMock(side_effect=self._get_pre_order_data) exchange_if.orders.get_order_locked_amount = order_util.get_order_locked_amount exchange_if.orders.adapt_order_quantity_and_target_price_for_order_creation = mock.Mock( diff --git a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py index c7b521ccb6..4ff573d283 100644 --- a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py +++ b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer.py @@ -1,4 +1,4 @@ -# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) # Copyright (c) 2025 Drakkar-Software, All rights reserved. # # OctoBot is free software; you can redistribute it and/or @@ -30,6 +30,7 @@ import octobot_copy.constants as copy_constants import octobot_copy.entities as copy_entities +import octobot_copy.orders_mirroring.mirrored_quantity_compute_result as mirrored_quantity_compute_result import octobot_copy.orders_mirroring.orders_synchronizer as orders_synchronizer_module @@ -49,18 +50,6 @@ def _copied_account( ) -def _reference_account_with_allocations( - base_ratio: decimal.Decimal, - quote_ratio: decimal.Decimal, -) -> protocol_models.CopiedAccount: - return _copied_account( - copied_assets=[ - protocol_models.CopiedAsset(name="ETH", total=1.0, available=1.0, ratio=float(base_ratio)), - protocol_models.CopiedAsset(name="USDT", total=10000.0, available=10000.0, ratio=float(quote_ratio)), - ], - orders=[], - ) - def _eth_usdt_pair_assets( *, @@ -94,131 +83,6 @@ def currency_total(currency: str) -> decimal.Decimal: return exchange_interface -def _order_stub(*, symbol: str, side, quantity: decimal.Decimal, price: decimal.Decimal): - order = mock.Mock() - order.symbol = symbol - order.side = side - order.origin_quantity = quantity - order.origin_price = price - return order - - -class TestOrdersSynchronizerOrphanGraceHeuristic: - def test_reference_pair_leg_share(self): - reference = _reference_account_with_allocations( - decimal.Decimal("0.25"), - decimal.Decimal("0.5"), - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - mock.MagicMock(), - copy_entities.AccountCopySettings(), - ) - expected = decimal.Decimal("0.25") / (decimal.Decimal("0.25") + decimal.Decimal("0.5")) - assert synchronizer._reference_pair_leg_share("ETH/USDT") == expected - - def test_reference_pair_leg_share_missing_quote_returns_one(self): - reference = _copied_account( - copied_assets=[ - protocol_models.CopiedAsset(name="ETH", total=1.0, available=1.0, ratio=0.5), - ], - orders=[], - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - mock.MagicMock(), - copy_entities.AccountCopySettings(), - ) - assert synchronizer._reference_pair_leg_share("ETH/USDT") == trading_constants.ONE - - def test_simulated_pair_share_buy_matches_reference_example(self): - reference = _reference_account_with_allocations( - decimal.Decimal("0.25"), - decimal.Decimal("0.5"), - ) - currency_totals = { - "ETH": decimal.Decimal("1"), - "USDT": decimal.Decimal("10000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings(), - ) - buy_order = _order_stub( - symbol="ETH/USDT", - side=trading_enums.TradeOrderSide.BUY, - quantity=decimal.Decimal("1"), - price=decimal.Decimal("2000"), - ) - reference_share = synchronizer._reference_pair_leg_share("ETH/USDT") - simulated_share = synchronizer._simulated_copier_pair_leg_share_after_orphan_fill(buy_order) - assert reference_share is not None - assert simulated_share is not None - assert simulated_share == reference_share - - def test_batch_eligible_false_when_simulated_share_mismatch(self): - reference = _reference_account_with_allocations( - decimal.Decimal("0.5"), - decimal.Decimal("0.5"), - ) - currency_totals = { - "ETH": decimal.Decimal("1"), - "USDT": decimal.Decimal("10000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings(mirrored_orphan_grace_pair_ratio_max_delta=decimal.Decimal("0.02")), - ) - buy_order = _order_stub( - symbol="ETH/USDT", - side=trading_enums.TradeOrderSide.BUY, - quantity=decimal.Decimal("1"), - price=decimal.Decimal("2000"), - ) - assert synchronizer._mirrored_orphan_batch_eligible_for_grace([buy_order]) is False - - def test_simulated_pair_share_sell(self): - reference = _reference_account_with_allocations( - decimal.Decimal("1") / decimal.Decimal("6"), - decimal.Decimal("5") / decimal.Decimal("6"), - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings(), - ) - sell_order = _order_stub( - symbol="ETH/USDT", - side=trading_enums.TradeOrderSide.SELL, - quantity=decimal.Decimal("1"), - price=decimal.Decimal("2000"), - ) - reference_share = synchronizer._reference_pair_leg_share("ETH/USDT") - simulated_share = synchronizer._simulated_copier_pair_leg_share_after_orphan_fill(sell_order) - assert reference_share is not None - assert simulated_share is not None - # Reference leg share uses CopiedAsset.ratio (float round-trip); simulated share is exact Decimal math. - assert abs(simulated_share - reference_share) <= decimal.Decimal("1e-15") - - def _replicable_buy_limit_order( *, order_id: str = "ref-late-1", @@ -304,808 +168,28 @@ def test_mirrored_orphan_open_orders_excludes_copier_market_orders(self): assert orphans == [limit_mirror] -class TestLateReferenceFillHeuristic: - def test_late_fill_true_when_copier_matches_simulated_reference_fill(self): - reference = _copied_account( - copied_assets=_eth_usdt_pair_assets(), - orders=[], - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings(), - ) - order = _replicable_buy_limit_order() - assert synchronizer._passes_late_reference_fill_heuristic(order) is True - assert synchronizer._is_late_reference_fill_for_order(order, []) is True - - def test_late_fill_false_when_new_reference_order_copier_not_yet_filled(self): - reference = _copied_account( - copied_assets=_eth_usdt_pair_assets(), - orders=[], - ) - currency_totals = { - "ETH": decimal.Decimal("1"), - "USDT": decimal.Decimal("10000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings(), - ) - order = _replicable_buy_limit_order() - assert synchronizer._passes_late_reference_fill_heuristic(order) is False - assert synchronizer._is_late_reference_fill_for_order(order, []) is False - - def test_grace_started_when_late_fill_only_no_orphans(self): - assets = _eth_usdt_pair_assets() - compliant_snapshot = _copied_account( - updated_at=time.time() - 1.0, - copied_assets=assets, - orders=[], - ) - reference = _copied_account( - updated_at=time.time(), - copied_assets=assets, - orders=[], - historical_snapshots=[compliant_snapshot], - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - copy_settings = copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=3, - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_settings, - ) - order = _replicable_buy_limit_order() - replicable = [order] - - async def run_grace(): - return await synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], replicable) - - asyncio.run(run_grace()) - assert synchronizer.get_mirrored_orphan_grace_started_at() is not None - - -class TestApplyGraceGraceEpisodeClearedLogging: - _EPISODE_CLEARED_SNIPPET = "Mirrored open-order grace episode cleared" - _CANCEL_DEFERRED_SNIPPET = "Mirrored orphan cancel deferred" - _GRACE_ELAPSED_SNIPPET = "Mirrored orphan grace elapsed after" - - def _sync_late_fill_only_defer_setup(self, *, frozen_reference_time: float): - assets = _eth_usdt_pair_assets() - compliant_snapshot = _copied_account( - updated_at=frozen_reference_time - 1.0, - copied_assets=assets, - orders=[], - ) - reference = _copied_account( - updated_at=frozen_reference_time, - copied_assets=assets, - orders=[], - historical_snapshots=[compliant_snapshot], - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - copy_settings = copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=3, - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_settings, - ) - return synchronizer, _replicable_buy_limit_order() - - def test_idle_no_episode_cleared_log_when_never_deferred(self, caplog): - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - _copied_account(), - mock.MagicMock(), - copy_entities.AccountCopySettings(), - ) - with caplog.at_level(logging.INFO): - asyncio.run(synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [])) - assert self._EPISODE_CLEARED_SNIPPET not in caplog.text - - def test_episode_cleared_log_after_defer_then_grace_total_zero(self, caplog): - frozen_t0 = 1_700_000_000.0 - synchronizer, order = self._sync_late_fill_only_defer_setup(frozen_reference_time=frozen_t0) - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - with caplog.at_level(logging.INFO): - asyncio.run( - synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [order]) - ) - assert self._EPISODE_CLEARED_SNIPPET not in caplog.text - assert self._CANCEL_DEFERRED_SNIPPET in caplog.text - assert "ref-late-1" in caplog.text - assert "late-reference-fill candidate(s):" in caplog.text - - caplog.clear() - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - with caplog.at_level(logging.INFO): - asyncio.run(synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [])) - assert caplog.text.count(self._EPISODE_CLEARED_SNIPPET) == 1 - - def test_no_episode_cleared_after_grace_elapsed_flag_reset(self, caplog): - frozen_t0 = 1_700_000_000.0 - synchronizer, order = self._sync_late_fill_only_defer_setup(frozen_reference_time=frozen_t0) - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - with caplog.at_level(logging.INFO): - asyncio.run( - synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [order]) - ) - assert self._CANCEL_DEFERRED_SNIPPET in caplog.text - - caplog.clear() - elapsed_time = frozen_t0 + 70.0 - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=elapsed_time, - ): - with caplog.at_level(logging.INFO): - asyncio.run( - synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [order]) - ) - assert self._GRACE_ELAPSED_SNIPPET in caplog.text - - caplog.clear() - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=elapsed_time, - ): - with caplog.at_level(logging.INFO): - asyncio.run(synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [])) - assert self._EPISODE_CLEARED_SNIPPET not in caplog.text - - -class TestSynchronizeGracePeriodCompletionLogging: - def _grace_period_synchronize_setup(self, *, frozen_reference_time: float): - order = _replicable_buy_limit_order() - assets = _eth_usdt_pair_assets() - compliant_snapshot = _copied_account( - updated_at=frozen_reference_time - 1.0, - copied_assets=assets, - orders=[], - ) - reference = _copied_account( - updated_at=frozen_reference_time, - copied_assets=assets, - orders=[order], - historical_snapshots=[compliant_snapshot], - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates - copy_settings = copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=3, - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_settings, - ) - return synchronizer - - def test_completion_uses_grace_summary_not_replication_failure(self, caplog): - frozen_t0 = 1_700_000_000.0 - synchronizer = self._grace_period_synchronize_setup(frozen_reference_time=frozen_t0) - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - with caplog.at_level(logging.INFO): - asyncio.run(synchronizer.synchronize()) - - completion_logs = [ - record.message - for record in caplog.records - if record.message.startswith("Order mirror completed:") - ] - assert len(completion_logs) == 1 - completion_message = completion_logs[0] - assert "Grace period active for" in completion_message - assert "ref-late-1" in completion_message - assert "Failed to replicate" not in completion_message - - -class TestIsMirroredOrphanGraceIdentified: - def _late_fill_grace_synchronizer_setup( - self, - *, - frozen_reference_time: float, - orders: list[protocol_models.Order], - copy_settings: typing.Optional[copy_entities.AccountCopySettings] = None, - ): - assets = _eth_usdt_pair_assets() - compliant_snapshot = _copied_account( - updated_at=frozen_reference_time - 1.0, - copied_assets=assets, - orders=[], - ) - reference = _copied_account( - updated_at=frozen_reference_time, - copied_assets=assets, - orders=orders, - historical_snapshots=[compliant_snapshot], - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_settings or copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=3, - ), - ) - return synchronizer - - def test_true_when_late_fill_grace_window_active(self): - frozen_t0 = 1_700_000_000.0 - synchronizer = self._late_fill_grace_synchronizer_setup( - frozen_reference_time=frozen_t0, - orders=[_replicable_buy_limit_order()], - ) - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - assert synchronizer.is_mirrored_orphan_grace_identified() is True - - def test_false_when_no_grace_items(self): - reference = _copied_account( - copied_assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), - orders=[_replicable_buy_limit_order()], - ) - currency_totals = { - "ETH": decimal.Decimal("1"), - "USDT": decimal.Decimal("10000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings(mirrored_orphan_cancel_grace_seconds=60.0), - ) - assert synchronizer.is_mirrored_orphan_grace_identified() is False - - def test_false_when_grace_window_elapsed(self): - frozen_t0 = 1_700_000_000.0 - synchronizer = self._late_fill_grace_synchronizer_setup( - frozen_reference_time=frozen_t0, - orders=[_replicable_buy_limit_order()], - ) - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0 + 120.0, - ): - assert synchronizer.is_mirrored_orphan_grace_identified() is False - - -class TestCountUnmirroredReferenceOrders: - def test_counts_only_reference_orders_without_open_copier_mirror(self): - first_order = _replicable_buy_limit_order(order_id="mirror-1") - second_order = _replicable_buy_limit_order(order_id="missing-1") - reference = _copied_account(orders=[first_order, second_order]) - exchange_if = mock.MagicMock() - exchange_if.orders.get_open_orders = mock.Mock( - return_value=[_mirrored_eth_buy_order_stub("mirror-1")] - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings(), - ) - replicable = synchronizer._get_replicable_reference_orders() - assert synchronizer._count_unmirrored_reference_orders(replicable) == 1 - +class TestMirroredOrderSelfLockCreditCompute: + """open_mirrored_order credits this line's locked funds so repeat sync does not false quantity mismatch.""" -class TestSynchronizeBypassGraceWhenTooManyMissingMirrors: - def _grace_active_synchronizer_with_orders( - self, + @staticmethod + def _mirrored_sell_order( *, - frozen_reference_time: float, - orders: list[protocol_models.Order], - abort_threshold: int, - late_fill_order_ids: set[str], + order_id: str, + symbol: str, + locked_quantity: decimal.Decimal, + price: decimal.Decimal, ): - assets = _eth_usdt_pair_assets() - compliant_snapshot = _copied_account( - updated_at=frozen_reference_time - 1.0, - copied_assets=assets, - orders=[], - ) - reference = _copied_account( - updated_at=frozen_reference_time, - copied_assets=assets, - orders=orders, - historical_snapshots=[compliant_snapshot], - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=abort_threshold, - ), - ) - - original_is_late_reference_fill = synchronizer._is_late_reference_fill_for_order - - def late_fill_side_effect(order, orphan_orders, reference_state=None): - if str(order.id) in late_fill_order_ids: - return original_is_late_reference_fill(order, orphan_orders, reference_state) - return False - - synchronizer._is_late_reference_fill_for_order = late_fill_side_effect - return synchronizer - - def test_bypasses_grace_when_missing_exceed_threshold(self, caplog): - frozen_t0 = 1_700_000_000.0 - orders = [ - _replicable_buy_limit_order(order_id=f"ref-order-{order_index}") - for order_index in range(3) - ] - synchronizer = self._grace_active_synchronizer_with_orders( - frozen_reference_time=frozen_t0, - orders=orders, - abort_threshold=2, - late_fill_order_ids={"ref-order-0"}, - ) - created_order = mock.Mock() - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - with mock.patch.object( - synchronizer, - "_upsert_mirrored_reference_order", - mock.AsyncMock(return_value=([created_order], 0, 0, None)), - ): - with caplog.at_level(logging.INFO): - created = asyncio.run(synchronizer.synchronize()) - - assert created == [created_order] * 3 - assert any( - "Bypassing mirrored orphan grace: 3 reference order(s)" in record.message - for record in caplog.records - ) - - def test_does_not_bypass_when_missing_equals_threshold(self, caplog): - frozen_t0 = 1_700_000_000.0 - orders = [ - _replicable_buy_limit_order(order_id=f"ref-order-{order_index}") - for order_index in range(2) - ] - synchronizer = self._grace_active_synchronizer_with_orders( - frozen_reference_time=frozen_t0, - orders=orders, - abort_threshold=2, - late_fill_order_ids={"ref-order-0"}, - ) - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - with mock.patch.object( - synchronizer, - "_upsert_mirrored_reference_order", - mock.AsyncMock(return_value=([], 0, 0, None)), - ): - with caplog.at_level(logging.INFO): - created = asyncio.run(synchronizer.synchronize()) - - assert created == [] - assert not any( - "Bypassing mirrored orphan grace" in record.message - for record in caplog.records - ) - assert any( - "Skipped reference mirror upsert for 2 order(s)" in record.message - for record in caplog.records - ) - - def test_does_not_bypass_when_grace_not_identified(self, caplog): - frozen_t0 = 1_700_000_000.0 - orders = [ - _replicable_buy_limit_order(order_id=f"ref-order-{order_index}") - for order_index in range(2) - ] - reference = _copied_account( - updated_at=frozen_t0, - copied_assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), - orders=orders, - ) - currency_totals = { - "ETH": decimal.Decimal("1"), - "USDT": decimal.Decimal("10000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=2, - ), - ) - abort_spy = mock.Mock(wraps=synchronizer.abort_mirrored_orphan_grace) - synchronizer.abort_mirrored_orphan_grace = abort_spy - with mock.patch.object( - synchronizer, - "_upsert_mirrored_reference_order", - mock.AsyncMock(return_value=([], 0, 0, None)), - ): - with caplog.at_level(logging.INFO): - asyncio.run(synchronizer.synchronize()) - - abort_spy.assert_not_called() - assert not any( - "Bypassing mirrored orphan grace" in record.message - for record in caplog.records - ) - - -class TestSynchronizeGridTwentyLimitsMissingMirrorGraceBypass: - """ - Reproduces Copy grid 20 R after rebalance mass-cancel + market buy: grace defers - symbol-level upserts while one late-fill candidate is active. Missing-mirror bypass - inside synchronize() (not account_copier rebalance abort) should create all 20 limits. - """ - - def _grid_post_rebalance_grace_synchronizer(self, *, frozen_reference_time: float): - # 20 reference limits on one symbol (grid); copier has none — limits were cancelled before rebalance - grid_orders = [ - _replicable_buy_limit_order( - order_id=f"ref-order-{order_index}", - price=decimal.Decimal("2000") - decimal.Decimal(order_index), - created_ts=frozen_reference_time, - ) - for order_index in range(20) - ] - assets = _eth_usdt_pair_assets() - # Compliant historical snapshot: required for grace window / pair-ratio checks - compliant_snapshot = _copied_account( - updated_at=frozen_reference_time - 1.0, - copied_assets=assets, - orders=[], - ) - reference = _copied_account( - updated_at=frozen_reference_time, - copied_assets=assets, - orders=grid_orders, - historical_snapshots=[compliant_snapshot], - ) - # Post–market-buy copier holdings: skewed vs reference snapshot so late-fill heuristic can match one order - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates - # Default abort threshold from production settings - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=2, - ), - ) - # Only one late-fill candidate (grace_total=1); mirrors log where 20 missing ≠ 20 grace items - original_is_late_reference_fill = synchronizer._is_late_reference_fill_for_order - late_fill_order_ids = {"ref-order-0"} - - def late_fill_side_effect(order, orphan_orders, reference_state=None): - if str(order.id) in late_fill_order_ids: - return original_is_late_reference_fill(order, orphan_orders, reference_state) - return False - - synchronizer._is_late_reference_fill_for_order = late_fill_side_effect - return synchronizer - - def test_creates_twenty_limits_via_missing_mirror_bypass_without_rebalance_abort(self, caplog): - frozen_t0 = 1_700_000_000.0 - synchronizer = self._grid_post_rebalance_grace_synchronizer(frozen_reference_time=frozen_t0) - abort_spy = mock.Mock(wraps=synchronizer.abort_mirrored_orphan_grace) - synchronizer.abort_mirrored_orphan_grace = abort_spy - upsert_mock = mock.AsyncMock( - side_effect=lambda order: ([mock.Mock(name=f"created-{order.id}")], 0, 0, None) - ) - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - # Grace is active before sync; no abort_mirrored_orphan_grace() — rebalance bypass path not used - assert synchronizer.is_mirrored_orphan_grace_identified() is True - with mock.patch.object(synchronizer, "_upsert_mirrored_reference_order", upsert_mock): - with caplog.at_level(logging.INFO): - created = asyncio.run(synchronizer.synchronize()) - - # synchronize() alone must bypass grace via missing_count (20) > threshold (2) - abort_spy.assert_called_once() - assert upsert_mock.await_count == 20 - # All symbol-level skips cleared after bypass; every limit upserted - assert len(created) == 20 - assert any( - "Bypassing mirrored orphan grace: 20 reference order(s) " - "missing on copier (> abort threshold 2)" in record.message - for record in caplog.records - ) - assert not any( - "Skipped reference mirror upsert for 20 order(s)" in record.message - for record in caplog.records - ) - - -class TestSynchronizeAfterAbortMirroredOrphanGrace: - def test_manual_abort_allows_upsert_while_grace_active(self, caplog): - frozen_t0 = 1_700_000_000.0 - assets = _eth_usdt_pair_assets() - compliant_snapshot = _copied_account( - updated_at=frozen_t0 - 1.0, - copied_assets=assets, - orders=[], - ) - reference = _copied_account( - updated_at=frozen_t0, - copied_assets=assets, - orders=[_replicable_buy_limit_order()], - historical_snapshots=[compliant_snapshot], - ) - currency_totals = { - "ETH": decimal.Decimal("2"), - "USDT": decimal.Decimal("8000"), - } - exchange_if = _exchange_interface_stub( - currency_totals=currency_totals, - market_price=decimal.Decimal("2000"), - ) - exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) - exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - reference, - exchange_if, - copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=3, - ), - ) - created_order = mock.Mock() - with mock.patch( - "octobot_copy.orders_mirroring.orders_synchronizer.time.time", - return_value=frozen_t0, - ): - with mock.patch.object( - synchronizer, - "_upsert_mirrored_reference_order", - mock.AsyncMock(return_value=([created_order], 0, 0, None)), - ): - with caplog.at_level(logging.INFO): - blocked_created = asyncio.run(synchronizer.synchronize()) - synchronizer.abort_mirrored_orphan_grace() - allowed_created = asyncio.run(synchronizer.synchronize()) - - assert blocked_created == [] - assert allowed_created == [created_order] - assert any( - "Skipped reference mirror upsert for 1 order(s)" in record.message - for record in caplog.records - ) - - -def _replicable_buy_limit_order_id(order_id: str) -> protocol_models.Order: - return _replicable_buy_limit_order(order_id=order_id) - - -def _mirrored_eth_buy_order_stub(order_id: str) -> mock.Mock: - mirrored = mock.Mock() - mirrored.tag = copy_constants.MIRRORED_ORDER_TAG - mirrored.order_id = order_id - mirrored.symbol = "ETH/USDT" - mirrored.side = trading_enums.TradeOrderSide.BUY - mirrored.origin_price = decimal.Decimal("2000") - mirrored.origin_quantity = decimal.Decimal("1") - return mirrored - - -class TestMissedHistoricalSignalsGraceAbort: - def test_is_aborted_when_first_compliant_snapshot_index_at_threshold(self): - order_m1 = _replicable_buy_limit_order_id("m1") - order_m2 = _replicable_buy_limit_order_id("m2") - assets = _eth_usdt_pair_assets() - empty_snapshot = _copied_account( - updated_at=time.time(), - copied_assets=assets, - orders=[], - ) - empty_snapshot_mid = _copied_account( - updated_at=time.time() - 1.0, - copied_assets=assets, - orders=[], - ) - compliant_snapshot = _copied_account( - updated_at=time.time() - 5.0, - copied_assets=assets, - orders=[order_m1, order_m2], - ) - live_reference = _copied_account( - updated_at=time.time(), - copied_assets=assets, - orders=[order_m1], - historical_snapshots=[empty_snapshot, empty_snapshot_mid, compliant_snapshot], - ) - mirror_m1 = _mirrored_eth_buy_order_stub("m1") - mirror_m2 = _mirrored_eth_buy_order_stub("m2") - exchange_if = mock.MagicMock() - exchange_if.orders.get_open_orders = mock.Mock(return_value=[mirror_m1, mirror_m2]) - exchange_if.portfolio.reference_market = "USDT" - exchange_if.portfolio.get_currency_portfolio_total = mock.Mock( - return_value=decimal.Decimal("1") - ) - exchange_if.market.get_potentially_outdated_price = mock.Mock( - return_value=(decimal.Decimal("2000"), False) - ) - copy_settings = copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=2, - missed_signals_grace_abort_threshold=2, - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - live_reference, - exchange_if, - copy_settings, - ) - assert synchronizer.is_mirrored_orphan_grace_aborted_for_missed_historical_signals() is True - - def test_apply_grace_cancels_immediately_when_missed_signals_abort(self): - order_m1 = _replicable_buy_limit_order_id("m1") - order_m2 = _replicable_buy_limit_order_id("m2") - assets = _eth_usdt_pair_assets() - empty_snapshot = _copied_account( - updated_at=time.time(), - copied_assets=assets, - orders=[], - ) - empty_snapshot_mid = _copied_account( - updated_at=time.time() - 1.0, - copied_assets=assets, - orders=[], - ) - compliant_snapshot = _copied_account( - updated_at=time.time() - 5.0, - copied_assets=assets, - orders=[order_m1, order_m2], - ) - live_reference = _copied_account( - updated_at=time.time(), - copied_assets=assets, - orders=[order_m1], - historical_snapshots=[empty_snapshot, empty_snapshot_mid, compliant_snapshot], - ) - mirror_m1 = _mirrored_eth_buy_order_stub("m1") - mirror_m2 = _mirrored_eth_buy_order_stub("m2") - exchange_if = mock.MagicMock() - # Two open mirrors so empty-order snapshots see grace_total>=threshold and stay non-compliant; - # otherwise a single orphan snapshot "complies" and missed-signals abort never triggers. - exchange_if.orders.get_open_orders = mock.Mock(return_value=[mirror_m1, mirror_m2]) - exchange_if.orders.cancel_order = mock.AsyncMock() - exchange_if.portfolio.reference_market = "USDT" - currency_totals = { - "ETH": decimal.Decimal("1"), - "USDT": decimal.Decimal("10000"), - } - exchange_if.portfolio.get_currency_portfolio_total = mock.Mock( - side_effect=lambda currency: currency_totals[currency] - ) - exchange_if.market.get_potentially_outdated_price = mock.Mock( - return_value=(decimal.Decimal("2000"), False) - ) - copy_settings = copy_entities.AccountCopySettings( - mirrored_orphan_cancel_grace_seconds=60.0, - mirrored_orphan_grace_abort_threshold=2, - missed_signals_grace_abort_threshold=2, - ) - synchronizer = orders_synchronizer_module.OrdersSynchronizer( - live_reference, - exchange_if, - copy_settings, - ) - replicable = synchronizer._get_replicable_reference_orders() - - async def run_grace(): - return await synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( - [mirror_m2], - replicable, - ) - - asyncio.run(run_grace()) - exchange_if.orders.cancel_order.assert_called_once_with(mirror_m2) - - -class TestMirroredOrderSelfLockCreditCompute: - """open_mirrored_order credits this line's locked funds so repeat sync does not false quantity mismatch.""" + order = mock.Mock() + order.order_id = order_id + order.tag = copy_constants.MIRRORED_ORDER_TAG + order.side = trading_enums.TradeOrderSide.SELL + order.symbol = symbol + order.currency = symbol.split("/")[0] + order.origin_price = price + order.is_filled = mock.Mock(return_value=False) + order.get_locked_quantity = mock.Mock(return_value=locked_quantity) + order.get_computed_fee = mock.Mock(return_value=None) + return order @staticmethod def _exchange_interface_for_compute( @@ -1115,6 +199,7 @@ def _exchange_interface_for_compute( available_symbol: decimal.Decimal, available_market: decimal.Decimal, mark_price: decimal.Decimal, + open_mirrored_sell_orders: typing.Optional[list] = None, ): symbol_market = mock.Mock() market_quantity_total = total_market / mark_price if mark_price else trading_constants.ZERO @@ -1144,6 +229,9 @@ def _exchange_interface_for_compute( side_effect=lambda symbol, quantity, limit_price: ([(quantity, limit_price)], symbol_market) ) exchange_if.orders.get_order_locked_amount = order_util.get_order_locked_amount + exchange_if.orders.get_open_orders = mock.Mock( + return_value=open_mirrored_sell_orders or [] + ) exchange_if.market.is_market_open_for_order_type = mock.Mock(return_value=True) return exchange_if @@ -1185,24 +273,31 @@ async def run_compute(open_order): def test_sell_open_mirrored_order_adds_locked_base_to_cap(self): mark_price = decimal.Decimal("2000") + open_sell = self._mirrored_sell_order( + order_id="open-sell", + symbol="ETH/USDT", + locked_quantity=decimal.Decimal("1"), + price=mark_price, + ) + sibling_sell = self._mirrored_sell_order( + order_id="sibling-sell", + symbol="ETH/USDT", + locked_quantity=decimal.Decimal("8.95"), + price=mark_price, + ) exchange_if = self._exchange_interface_for_compute( total_symbol=decimal.Decimal("10"), total_market=decimal.Decimal("10000"), available_symbol=decimal.Decimal("0.05"), available_market=decimal.Decimal("500"), mark_price=mark_price, + open_mirrored_sell_orders=[open_sell, sibling_sell], ) synchronizer = orders_synchronizer_module.OrdersSynchronizer( _copied_account(), exchange_if, copy_entities.AccountCopySettings(), ) - open_sell = mock.Mock() - open_sell.side = trading_enums.TradeOrderSide.SELL - open_sell.symbol = "ETH/USDT" - open_sell.origin_price = mark_price - open_sell.get_locked_quantity = mock.Mock(return_value=decimal.Decimal("1")) - open_sell.get_computed_fee = mock.Mock(return_value=None) async def run_compute(open_order, scaled): return await synchronizer._compute_mirrored_quantity_type_and_price( @@ -1217,9 +312,46 @@ async def run_compute(open_order, scaled): scaled = decimal.Decimal("2") ideal_without = asyncio.run(run_compute(None, scaled)).ideal_quantity ideal_with = asyncio.run(run_compute(open_sell, scaled)).ideal_quantity - assert ideal_without == decimal.Decimal("2") + assert ideal_without == decimal.Decimal("0.05") assert ideal_with == decimal.Decimal("1.05") + def test_new_sell_caps_to_total_minus_sibling_locked_base(self): + mark_price = decimal.Decimal("60300") + total_btc = decimal.Decimal("0.00753") + available_btc = decimal.Decimal("0.00068") + sibling_locked_btc = total_btc - available_btc + sibling_sell = self._mirrored_sell_order( + order_id="sibling-sell", + symbol="BTC/USDT", + locked_quantity=sibling_locked_btc, + price=decimal.Decimal("62188"), + ) + exchange_if = self._exchange_interface_for_compute( + total_symbol=total_btc, + total_market=decimal.Decimal("500"), + available_symbol=available_btc, + available_market=decimal.Decimal("15"), + mark_price=mark_price, + open_mirrored_sell_orders=[sibling_sell], + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + _copied_account(), + exchange_if, + copy_entities.AccountCopySettings(), + ) + + async def run_compute(): + return await synchronizer._compute_mirrored_quantity_type_and_price( + "BTC/USDT", + trading_enums.TradeOrderSide.SELL, + decimal.Decimal("0.00074"), + decimal.Decimal("61188"), + trading_enums.TraderOrderType.SELL_LIMIT, + open_mirrored_order=None, + ) + + assert asyncio.run(run_compute()).ideal_quantity == available_btc + class TestMirroredOrderSkipLogging: @staticmethod @@ -1393,3 +525,469 @@ def test_synchronize_summary_lists_failed_replications_with_reason(self, caplog) assert "Failed to replicate 2 order(s):" in completion_message assert "buy ETH/USDT @ 50745.57 [11111111-1111-1111-1111-111111111111] (insufficient_quote)" in completion_message assert "buy ETH/USDT @ 49245.57 [22222222-2222-2222-2222-222222222222] (insufficient_quote)" in completion_message + + +class TestOrdersSynchronizerWaitForMirroredOrdersOpen: + def _synchronizer_with_auto_sync(self, auto_sync_enabled: bool): + reference = _copied_account( + copied_assets=_eth_usdt_pair_assets(), + orders=[_replicable_buy_limit_order()], + ) + exchange_if = _exchange_interface_stub( + currency_totals={ + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + }, + market_price=decimal.Decimal("2000"), + ) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + exchange_if.orders.automatically_synchronize_orders = mock.Mock(return_value=auto_sync_enabled) + exchange_if.orders.wait_for_orders_to_open = mock.AsyncMock() + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + return synchronizer, exchange_if + + def test_waits_for_created_orders_when_auto_sync_disabled(self): + synchronizer, exchange_if = self._synchronizer_with_auto_sync(False) + created_order = mock.Mock() + created_order.symbol = "ETH/USDT" + with mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([created_order], 0, 0, None)), + ): + created = asyncio.run(synchronizer.synchronize()) + + assert created == [created_order] + exchange_if.orders.wait_for_orders_to_open.assert_awaited_once_with( + [created_order], + "ETH/USDT", + ) + + def test_skips_wait_when_auto_sync_enabled(self): + synchronizer, exchange_if = self._synchronizer_with_auto_sync(True) + created_order = mock.Mock() + created_order.symbol = "ETH/USDT" + with mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([created_order], 0, 0, None)), + ): + created = asyncio.run(synchronizer.synchronize()) + + assert created == [created_order] + exchange_if.orders.wait_for_orders_to_open.assert_not_awaited() + + def test_count_invariant_runs_after_wait_when_auto_sync_disabled(self): + synchronizer, exchange_if = self._synchronizer_with_auto_sync(False) + created_order = mock.Mock() + created_order.symbol = "ETH/USDT" + call_order: list[str] = [] + + async def wait_side_effect(orders, symbol): + call_order.append("wait") + + def invariant_side_effect(replicable): + call_order.append("invariant") + + exchange_if.orders.wait_for_orders_to_open = mock.AsyncMock(side_effect=wait_side_effect) + with mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([created_order], 0, 0, None)), + ), mock.patch.object( + synchronizer, + "_check_open_limit_order_count_invariant", + side_effect=invariant_side_effect, + ): + asyncio.run(synchronizer.synchronize()) + assert call_order == ["wait", "invariant"] + + def test_count_invariant_runs_when_wait_skipped(self): + synchronizer, exchange_if = self._synchronizer_with_auto_sync(True) + created_order = mock.Mock() + created_order.symbol = "ETH/USDT" + call_order: list[str] = [] + + def invariant_side_effect(replicable): + call_order.append("invariant") + + with mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([created_order], 0, 0, None)), + ), mock.patch.object( + synchronizer, + "_check_open_limit_order_count_invariant", + side_effect=invariant_side_effect, + ): + asyncio.run(synchronizer.synchronize()) + exchange_if.orders.wait_for_orders_to_open.assert_not_awaited() + assert call_order == ["invariant"] + + +def _btc_usdc_buy_limit_reference_order( + *, + order_id: str, + amount: decimal.Decimal = decimal.Decimal("0.0001"), + price: decimal.Decimal = decimal.Decimal("59326.7"), +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol="BTC/USDC", + price=float(price), + quantity=float(amount), + filled=0.0, + exchange_id="ref-ex", + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + trigger_above=False, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _open_limit_order_stub( + *, + order_id: str, + exchange_order_id: str, + symbol: str = "BTC/USDC", + side=trading_enums.TradeOrderSide.BUY, + quantity: decimal.Decimal, + price: decimal.Decimal, + tag: str | None = None, + order_type=trading_enums.TraderOrderType.BUY_LIMIT, +): + order = mock.Mock() + order.order_id = order_id + order.exchange_order_id = exchange_order_id + order.symbol = symbol + order.side = side + order.origin_quantity = quantity + order.origin_price = price + order.order_type = order_type + order.tag = tag + return order + + +def _synchronizer_with_open_orders( + *, + reference_orders: list[protocol_models.Order], + open_orders: list, + currency_totals: dict[str, decimal.Decimal] | None = None, +) -> tuple[orders_synchronizer_module.OrdersSynchronizer, mock.MagicMock]: + currency_totals = currency_totals or { + "BTC": decimal.Decimal("0.01"), + "USDC": decimal.Decimal("10000"), + } + reference = _copied_account( + copied_assets=[ + protocol_models.CopiedAsset(name="BTC", total=1.0, available=1.0, ratio=0.5), + protocol_models.CopiedAsset(name="USDC", total=10000.0, available=10000.0, ratio=0.5), + ], + orders=reference_orders, + ) + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("59326.7"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=open_orders) + orders_manager = mock.Mock() + exchange_if.orders._exchange_manager = mock.Mock() + exchange_if.orders._exchange_manager.exchange_personal_data.orders_manager = orders_manager + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + return synchronizer, exchange_if + + +class TestOpenOrdersMatchingSymbolSidePrice: + def test_matches_limit_order_within_price_tolerance(self): + reference_price = decimal.Decimal("59326.7") + matching_order = _open_limit_order_stub( + order_id="wrong-bot-id", + exchange_order_id="OGE3T6-NDOIV-LR6MZI", + quantity=decimal.Decimal("0.0001"), + price=reference_price, + ) + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[], + open_orders=[matching_order], + ) + candidates = synchronizer._open_orders_matching_symbol_side_price( + "BTC/USDC", + trading_enums.TradeOrderSide.BUY, + reference_price, + trading_enums.TraderOrderType.BUY_LIMIT, + ) + assert candidates == [matching_order] + + def test_excludes_wrong_price(self): + reference_price = decimal.Decimal("59326.7") + wrong_price_order = _open_limit_order_stub( + order_id="wrong-bot-id", + exchange_order_id="OTHER-TXID", + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("58326.7"), + ) + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[], + open_orders=[wrong_price_order], + ) + candidates = synchronizer._open_orders_matching_symbol_side_price( + "BTC/USDC", + trading_enums.TradeOrderSide.BUY, + reference_price, + trading_enums.TraderOrderType.BUY_LIMIT, + ) + assert candidates == [] + + def test_excludes_opposite_side(self): + reference_price = decimal.Decimal("59326.7") + sell_order = _open_limit_order_stub( + order_id="sell-id", + exchange_order_id="SELL-TXID", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.0001"), + price=reference_price, + order_type=trading_enums.TraderOrderType.SELL_LIMIT, + ) + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[], + open_orders=[sell_order], + ) + candidates = synchronizer._open_orders_matching_symbol_side_price( + "BTC/USDC", + trading_enums.TradeOrderSide.BUY, + reference_price, + trading_enums.TraderOrderType.BUY_LIMIT, + ) + assert candidates == [] + + def test_excludes_market_orders(self): + reference_price = decimal.Decimal("59326.7") + market_order = _open_limit_order_stub( + order_id="market-id", + exchange_order_id="MARKET-TXID", + quantity=decimal.Decimal("0.0001"), + price=reference_price, + order_type=trading_enums.TraderOrderType.BUY_MARKET, + ) + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[], + open_orders=[market_order], + ) + candidates = synchronizer._open_orders_matching_symbol_side_price( + "BTC/USDC", + trading_enums.TradeOrderSide.BUY, + reference_price, + trading_enums.TraderOrderType.BUY_LIMIT, + ) + assert candidates == [] + + +class TestMapUnmappedOpenOrderForReference: + def test_returns_none_when_no_candidates(self): + reference_order = _btc_usdc_buy_limit_reference_order(order_id="28c1394b-dcb7-4f90-8878-4a61827471ca") + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[], + ) + mapped = asyncio.run( + synchronizer._map_unmapped_open_order_for_reference( + reference_order=reference_order, + reference_order_id=str(reference_order.id), + side=trading_enums.TradeOrderSide.BUY, + trader_order_type=trading_enums.TraderOrderType.BUY_LIMIT, + order_target_price=decimal.Decimal("59326.7"), + active_reference_ids={str(reference_order.id)}, + scaled_reference_quantity=decimal.Decimal("0.0001"), + ) + ) + assert mapped is None + + def test_relinks_single_unmapped_candidate(self): + reference_order = _btc_usdc_buy_limit_reference_order(order_id="28c1394b-dcb7-4f90-8878-4a61827471ca") + open_order = _open_limit_order_stub( + order_id="stale-bot-id", + exchange_order_id="OGE3T6-NDOIV-LR6MZI", + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[open_order], + ) + mapped = asyncio.run( + synchronizer._map_unmapped_open_order_for_reference( + reference_order=reference_order, + reference_order_id=str(reference_order.id), + side=trading_enums.TradeOrderSide.BUY, + trader_order_type=trading_enums.TraderOrderType.BUY_LIMIT, + order_target_price=decimal.Decimal("59326.7"), + active_reference_ids={str(reference_order.id)}, + scaled_reference_quantity=decimal.Decimal("0.0001"), + ) + ) + assert mapped is open_order + assert open_order.order_id == str(reference_order.id) + exchange_if.orders._exchange_manager.exchange_personal_data.orders_manager.replace_order.assert_called_once_with( + "stale-bot-id", + open_order, + ) + + def test_ambiguous_candidates_relinks_one_and_cancels_extras(self, caplog): + caplog.set_level(logging.WARNING) + reference_order = _btc_usdc_buy_limit_reference_order(order_id="28c1394b-dcb7-4f90-8878-4a61827471ca") + first_candidate = _open_limit_order_stub( + order_id="first-bot-id", + exchange_order_id="O7GDOQ-5ALJT-5QPQE4", + quantity=decimal.Decimal("0.00009"), + price=decimal.Decimal("59326.7"), + ) + second_candidate = _open_limit_order_stub( + order_id="second-bot-id", + exchange_order_id="OGE3T6-NDOIV-LR6MZI", + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[first_candidate, second_candidate], + ) + exchange_if.orders.cancel_order = mock.AsyncMock() + mapped = asyncio.run( + synchronizer._map_unmapped_open_order_for_reference( + reference_order=reference_order, + reference_order_id=str(reference_order.id), + side=trading_enums.TradeOrderSide.BUY, + trader_order_type=trading_enums.TraderOrderType.BUY_LIMIT, + order_target_price=decimal.Decimal("59326.7"), + active_reference_ids={str(reference_order.id)}, + scaled_reference_quantity=decimal.Decimal("0.0001"), + ) + ) + assert mapped is second_candidate + assert second_candidate.order_id == str(reference_order.id) + exchange_if.orders.cancel_order.assert_awaited_once_with(first_candidate) + assert any("Ambiguous unmapped open order match" in record.message for record in caplog.records) + + def test_skips_order_claimed_by_another_reference(self): + reference_order = _btc_usdc_buy_limit_reference_order(order_id="28c1394b-dcb7-4f90-8878-4a61827471ca") + other_reference_id = "other-reference-id" + claimed_order = _open_limit_order_stub( + order_id=other_reference_id, + exchange_order_id="CLAIMED-TXID", + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[claimed_order], + ) + mapped = asyncio.run( + synchronizer._map_unmapped_open_order_for_reference( + reference_order=reference_order, + reference_order_id=str(reference_order.id), + side=trading_enums.TradeOrderSide.BUY, + trader_order_type=trading_enums.TraderOrderType.BUY_LIMIT, + order_target_price=decimal.Decimal("59326.7"), + active_reference_ids={str(reference_order.id), other_reference_id}, + scaled_reference_quantity=decimal.Decimal("0.0001"), + ) + ) + assert mapped is None + + +class TestRelinkOpenOrderToReference: + def test_replace_order_tag_and_id_updated(self): + reference_order_id = "28c1394b-dcb7-4f90-8878-4a61827471ca" + open_order = _open_limit_order_stub( + order_id="previous-bot-id", + exchange_order_id="OGE3T6-NDOIV-LR6MZI", + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=None, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[], + open_orders=[open_order], + ) + relinked = synchronizer._relink_open_order_to_reference(open_order, reference_order_id) + assert relinked.order_id == reference_order_id + assert relinked.tag == copy_constants.MIRRORED_ORDER_TAG + exchange_if.orders._exchange_manager.exchange_personal_data.orders_manager.replace_order.assert_called_once_with( + "previous-bot-id", + open_order, + ) + + +class TestUpsertMirroredReferenceOrderMapsBeforeCreate: + def test_unmapped_same_price_order_is_already_synchronized_without_create(self): + reference_order_id = "28c1394b-dcb7-4f90-8878-4a61827471ca" + reference_order = _btc_usdc_buy_limit_reference_order(order_id=reference_order_id) + open_order = _open_limit_order_stub( + order_id="stale-bot-id", + exchange_order_id="OGE3T6-NDOIV-LR6MZI", + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=None, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[open_order], + ) + exchange_if.orders.create_orders = mock.AsyncMock() + exchange_if.orders.cancel_order = mock.AsyncMock() + compute_result = mirrored_quantity_compute_result.MirroredQuantityComputeResult( + ideal_quantity=decimal.Decimal("0.0001"), + resolved_trader_order_type=trading_enums.TraderOrderType.BUY_LIMIT, + limit_price=decimal.Decimal("59326.7"), + current_price=decimal.Decimal("59326.7"), + ) + with mock.patch.object( + synchronizer, + "_is_late_reference_fill_for_order", + return_value=False, + ), mock.patch.object( + synchronizer, + "_compute_mirrored_quantity_type_and_price", + mock.AsyncMock(return_value=compute_result), + ): + created, replaced_cancelled, already_synchronized, replication_failure = asyncio.run( + synchronizer._upsert_mirrored_reference_order(reference_order) + ) + assert created == [] + assert replaced_cancelled == 0 + assert already_synchronized == 1 + assert replication_failure is None + exchange_if.orders.create_orders.assert_not_called() + exchange_if.orders.cancel_order.assert_not_called() + assert open_order.order_id == reference_order_id diff --git a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_grace_period.py b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_grace_period.py new file mode 100644 index 0000000000..d715204b21 --- /dev/null +++ b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_grace_period.py @@ -0,0 +1,1793 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or +# (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# OctoBot. If not, see . +""" +OrdersSynchronizer mirrored-orphan / late-fill grace period scenarios. + +Catalog (extend when a new grace bug appears — add a letter + class): + A Late fill only — copier filled first; skip upserts; grace window active + B Deferred tagged orphan — valid price; cancel deferred + C Orphan + late fill together — both deferred / skip upserts + D Wrong-price stray during grace — cancel stray; keep valid-price orphan; skip upserts + E Grace elapsed — cancel tagged orphans; identified False + F Explicit abort — abort_mirrored_orphan_grace then upserts proceed + G Grace disabled (grace_seconds <= 0) — immediate orphan cancel; no skip set + H grace_total >= threshold — immediate cancel + abort log + I No compliant historical snapshot — immediate cancel + J Missed historical signals abort — immediate cancel + K Pair-ratio heuristic fails — orphans cancelled; late-fill continuation if present + L Missing-mirror bypass — missing count > threshold clears skip / aborts grace + M Idle / aligned — no skip; episode cleared only after prior defer + N Duplicate at valid price with grace idle — no grace interference (see reconciliation suite) + O Duplicate at valid price during grace — stray_only keeps both; upserts skipped + P Grace keeps mirrored orphan off-grid; cancels untagged wrong-price via synchronize + +Functional / grid fixtures: see functional_tests/test_sec_copy_grid_twenty_grace_elapsed.py +""" +import asyncio +import contextlib +import decimal +import logging +import time +import typing + +import mock + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_copy.constants as copy_constants +import octobot_copy.entities as copy_entities +import octobot_copy.orders_mirroring.orders_synchronizer as orders_synchronizer_module +def _copied_account( + *, + updated_at: typing.Optional[float] = None, + copied_assets: typing.Optional[list[protocol_models.CopiedAsset]] = None, + orders: typing.Optional[list[protocol_models.Order]] = None, + historical_snapshots: typing.Optional[list[protocol_models.CopiedAccount]] = None, +) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=updated_at if updated_at is not None else time.time(), + copied_assets=copied_assets or [], + orders=orders, + historical_snapshots=historical_snapshots, + ) + + +def _reference_account_with_allocations( + base_ratio: decimal.Decimal, + quote_ratio: decimal.Decimal, +) -> protocol_models.CopiedAccount: + return _copied_account( + copied_assets=[ + protocol_models.CopiedAsset(name="ETH", total=1.0, available=1.0, ratio=float(base_ratio)), + protocol_models.CopiedAsset(name="USDT", total=10000.0, available=10000.0, ratio=float(quote_ratio)), + ], + orders=[], + ) + + +def _eth_usdt_pair_assets( + *, + eth_ratio: float = 0.25, + usdt_ratio: float = 0.5, + eth_value: float = 1.0, + usdt_value: float = 10000.0, +) -> list[protocol_models.CopiedAsset]: + return [ + protocol_models.CopiedAsset(name="ETH", total=eth_value, available=eth_value, ratio=eth_ratio), + protocol_models.CopiedAsset(name="USDT", total=usdt_value, available=usdt_value, ratio=usdt_ratio), + ] + + +@contextlib.asynccontextmanager +async def _passthrough_mirror_sync_available_updates(): + yield + + +def _exchange_interface_stub(*, currency_totals: dict[str, decimal.Decimal], market_price: decimal.Decimal): + exchange_interface = mock.MagicMock() + exchange_interface.portfolio.reference_market = "USDT" + + def currency_total(currency: str) -> decimal.Decimal: + return currency_totals[currency] + + exchange_interface.portfolio.get_currency_portfolio_total = currency_total + exchange_interface.market.get_potentially_outdated_price = mock.Mock( + return_value=(market_price, False) + ) + return exchange_interface + + +def _order_stub(*, symbol: str, side, quantity: decimal.Decimal, price: decimal.Decimal): + order = mock.Mock() + order.symbol = symbol + order.side = side + order.origin_quantity = quantity + order.origin_price = price + return order + + +class TestScenarioKPairRatioHeuristic: + """ + Scenario K — Pair-ratio heuristic helpers and batch eligibility. + + Trigger: orphan or reference pair-leg share setup for grace eligibility checks. + Expected: share helpers match reference math; batch eligibility False when delta exceeds max. + """ + def test_reference_pair_leg_share(self): + reference = _reference_account_with_allocations( + decimal.Decimal("0.25"), + decimal.Decimal("0.5"), + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + mock.MagicMock(), + copy_entities.AccountCopySettings(), + ) + expected = decimal.Decimal("0.25") / (decimal.Decimal("0.25") + decimal.Decimal("0.5")) + assert synchronizer._reference_pair_leg_share("ETH/USDT") == expected + + def test_reference_pair_leg_share_missing_quote_returns_one(self): + reference = _copied_account( + copied_assets=[ + protocol_models.CopiedAsset(name="ETH", total=1.0, available=1.0, ratio=0.5), + ], + orders=[], + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + mock.MagicMock(), + copy_entities.AccountCopySettings(), + ) + assert synchronizer._reference_pair_leg_share("ETH/USDT") == trading_constants.ONE + + def test_simulated_pair_share_buy_matches_reference_example(self): + reference = _reference_account_with_allocations( + decimal.Decimal("0.25"), + decimal.Decimal("0.5"), + ) + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + buy_order = _order_stub( + symbol="ETH/USDT", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("1"), + price=decimal.Decimal("2000"), + ) + reference_share = synchronizer._reference_pair_leg_share("ETH/USDT") + simulated_share = synchronizer._simulated_copier_pair_leg_share_after_orphan_fill(buy_order) + assert reference_share is not None + assert simulated_share is not None + assert simulated_share == reference_share + + def test_batch_eligible_false_when_simulated_share_mismatch(self): + reference = _reference_account_with_allocations( + decimal.Decimal("0.5"), + decimal.Decimal("0.5"), + ) + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(mirrored_orphan_grace_pair_ratio_max_delta=decimal.Decimal("0.02")), + ) + buy_order = _order_stub( + symbol="ETH/USDT", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("1"), + price=decimal.Decimal("2000"), + ) + assert synchronizer._mirrored_orphan_batch_eligible_for_grace([buy_order]) is False + + def test_simulated_pair_share_sell(self): + reference = _reference_account_with_allocations( + decimal.Decimal("1") / decimal.Decimal("6"), + decimal.Decimal("5") / decimal.Decimal("6"), + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + sell_order = _order_stub( + symbol="ETH/USDT", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("1"), + price=decimal.Decimal("2000"), + ) + reference_share = synchronizer._reference_pair_leg_share("ETH/USDT") + simulated_share = synchronizer._simulated_copier_pair_leg_share_after_orphan_fill(sell_order) + assert reference_share is not None + assert simulated_share is not None + # Reference leg share uses CopiedAsset.ratio (float round-trip); simulated share is exact Decimal math. + assert abs(simulated_share - reference_share) <= decimal.Decimal("1e-15") +def _replicable_buy_limit_order( + *, + order_id: str = "ref-late-1", + amount: decimal.Decimal = decimal.Decimal("1"), + price: decimal.Decimal = decimal.Decimal("2000"), + created_ts: float | None = None, +) -> protocol_models.Order: + created_ts = created_ts if created_ts is not None else time.time() + return protocol_models.Order( + id=order_id, + symbol="ETH/USDT", + price=float(price), + quantity=float(amount), + filled=0.0, + exchange_id="ex", + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + trigger_above=False, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(created_ts), + ) + + +class TestScenarioALateFillOnly: + """ + Scenario A — Late fill only (heuristic and grace start). + + Trigger: copier holdings match simulated reference fill; no tagged orphans. + Expected: late-fill heuristic True; grace start resolves when applying late-fill-only grace. + """ + def test_late_fill_true_when_copier_matches_simulated_reference_fill(self): + reference = _copied_account( + copied_assets=_eth_usdt_pair_assets(), + orders=[], + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + order = _replicable_buy_limit_order() + assert synchronizer._passes_late_reference_fill_heuristic(order) is True + assert synchronizer._is_late_reference_fill_for_order(order, []) is True + + def test_late_fill_false_when_new_reference_order_copier_not_yet_filled(self): + reference = _copied_account( + copied_assets=_eth_usdt_pair_assets(), + orders=[], + ) + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + order = _replicable_buy_limit_order() + assert synchronizer._passes_late_reference_fill_heuristic(order) is False + assert synchronizer._is_late_reference_fill_for_order(order, []) is False + + def test_grace_started_when_late_fill_only_no_orphans(self): + assets = _eth_usdt_pair_assets() + compliant_snapshot = _copied_account( + updated_at=time.time() - 1.0, + copied_assets=assets, + orders=[], + ) + reference = _copied_account( + updated_at=time.time(), + copied_assets=assets, + orders=[], + historical_snapshots=[compliant_snapshot], + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + copy_settings = copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_settings, + ) + order = _replicable_buy_limit_order() + replicable = [order] + + async def run_grace(): + return await synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], replicable) + + asyncio.run(run_grace()) + assert synchronizer.get_mirrored_orphan_grace_started_at() is not None + + +class TestScenarioMIdleAndEpisodeCleared: + """ + Scenario M — Idle / aligned episode cleared logging. + + Trigger: grace_total becomes zero after a prior deferral (or never deferred). + Expected: episode-cleared log only after prior defer; elapsed path does not clear that way. + """ + _EPISODE_CLEARED_SNIPPET = "Mirrored open-order grace episode cleared" + _CANCEL_DEFERRED_SNIPPET = "Mirrored orphan cancel deferred" + _GRACE_ELAPSED_SNIPPET = "Mirrored orphan grace elapsed after" + + def _sync_late_fill_only_defer_setup(self, *, frozen_reference_time: float): + assets = _eth_usdt_pair_assets() + compliant_snapshot = _copied_account( + updated_at=frozen_reference_time - 1.0, + copied_assets=assets, + orders=[], + ) + reference = _copied_account( + updated_at=frozen_reference_time, + copied_assets=assets, + orders=[], + historical_snapshots=[compliant_snapshot], + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + copy_settings = copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_settings, + ) + return synchronizer, _replicable_buy_limit_order() + + def test_idle_no_episode_cleared_log_when_never_deferred(self, caplog): + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + _copied_account(), + mock.MagicMock(), + copy_entities.AccountCopySettings(), + ) + with caplog.at_level(logging.INFO): + asyncio.run(synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [])) + assert self._EPISODE_CLEARED_SNIPPET not in caplog.text + + def test_episode_cleared_log_after_defer_then_grace_total_zero(self, caplog): + frozen_t0 = 1_700_000_000.0 + synchronizer, order = self._sync_late_fill_only_defer_setup(frozen_reference_time=frozen_t0) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + with caplog.at_level(logging.INFO): + asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [order]) + ) + assert self._EPISODE_CLEARED_SNIPPET not in caplog.text + assert self._CANCEL_DEFERRED_SNIPPET in caplog.text + assert "ref-late-1" in caplog.text + assert "late-reference-fill candidate(s):" in caplog.text + + caplog.clear() + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + with caplog.at_level(logging.INFO): + asyncio.run(synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [])) + assert caplog.text.count(self._EPISODE_CLEARED_SNIPPET) == 1 + + def test_no_episode_cleared_after_grace_elapsed_flag_reset(self, caplog): + frozen_t0 = 1_700_000_000.0 + synchronizer, order = self._sync_late_fill_only_defer_setup(frozen_reference_time=frozen_t0) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + with caplog.at_level(logging.INFO): + asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [order]) + ) + assert self._CANCEL_DEFERRED_SNIPPET in caplog.text + + caplog.clear() + elapsed_time = frozen_t0 + 70.0 + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=elapsed_time, + ): + with caplog.at_level(logging.INFO): + asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [order]) + ) + assert self._GRACE_ELAPSED_SNIPPET in caplog.text + + caplog.clear() + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=elapsed_time, + ): + with caplog.at_level(logging.INFO): + asyncio.run(synchronizer._apply_grace_policy_and_cancel_mirrored_orphans([], [])) + assert self._EPISODE_CLEARED_SNIPPET not in caplog.text + + +class TestScenarioAGracePeriodCompletionLogging: + """ + Scenario A — Late fill sync completion uses grace summary. + + Trigger: synchronize while late-fill grace skips upserts. + Expected: completion log says Grace period active; not Failed to replicate. + """ + def _grace_period_synchronize_setup(self, *, frozen_reference_time: float): + order = _replicable_buy_limit_order() + assets = _eth_usdt_pair_assets() + compliant_snapshot = _copied_account( + updated_at=frozen_reference_time - 1.0, + copied_assets=assets, + orders=[], + ) + reference = _copied_account( + updated_at=frozen_reference_time, + copied_assets=assets, + orders=[order], + historical_snapshots=[compliant_snapshot], + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + copy_settings = copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_settings, + ) + return synchronizer + + def test_completion_uses_grace_summary_not_replication_failure(self, caplog): + frozen_t0 = 1_700_000_000.0 + synchronizer = self._grace_period_synchronize_setup(frozen_reference_time=frozen_t0) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + with caplog.at_level(logging.INFO): + asyncio.run(synchronizer.synchronize()) + + completion_logs = [ + record.message + for record in caplog.records + if record.message.startswith("Order mirror completed:") + ] + assert len(completion_logs) == 1 + completion_message = completion_logs[0] + assert "Grace period active for" in completion_message + assert "ref-late-1" in completion_message + assert "Failed to replicate" not in completion_message + + +class TestScenarioAGraceIdentified: + """ + Scenario A — Grace identified for late-fill window. + + Trigger: late-fill grace active, idle, or wall-clock elapsed. + Expected: is_mirrored_orphan_grace_identified True only while window active with grace items. + """ + def _late_fill_grace_synchronizer_setup( + self, + *, + frozen_reference_time: float, + orders: list[protocol_models.Order], + copy_settings: typing.Optional[copy_entities.AccountCopySettings] = None, + ): + assets = _eth_usdt_pair_assets() + compliant_snapshot = _copied_account( + updated_at=frozen_reference_time - 1.0, + copied_assets=assets, + orders=[], + ) + reference = _copied_account( + updated_at=frozen_reference_time, + copied_assets=assets, + orders=orders, + historical_snapshots=[compliant_snapshot], + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_settings or copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + return synchronizer + + def test_true_when_late_fill_grace_window_active(self): + frozen_t0 = 1_700_000_000.0 + synchronizer = self._late_fill_grace_synchronizer_setup( + frozen_reference_time=frozen_t0, + orders=[_replicable_buy_limit_order()], + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + assert synchronizer.is_mirrored_orphan_grace_identified() is True + + def test_false_when_no_grace_items(self): + reference = _copied_account( + copied_assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), + orders=[_replicable_buy_limit_order()], + ) + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(mirrored_orphan_cancel_grace_seconds=60.0), + ) + assert synchronizer.is_mirrored_orphan_grace_identified() is False + + def test_false_when_grace_window_elapsed(self): + frozen_t0 = 1_700_000_000.0 + synchronizer = self._late_fill_grace_synchronizer_setup( + frozen_reference_time=frozen_t0, + orders=[_replicable_buy_limit_order()], + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0 + 120.0, + ): + assert synchronizer.is_mirrored_orphan_grace_identified() is False + + +class TestScenarioLCountUnmirroredReferenceOrders: + """ + Scenario L — Count unmirrored reference orders (bypass counter). + + Trigger: some reference ids have open mirrors, others do not. + Expected: count equals reference orders without a matching open copier mirror. + """ + def test_counts_only_reference_orders_without_open_copier_mirror(self): + first_order = _replicable_buy_limit_order(order_id="mirror-1") + second_order = _replicable_buy_limit_order(order_id="missing-1") + reference = _copied_account(orders=[first_order, second_order]) + exchange_if = mock.MagicMock() + exchange_if.orders.get_open_orders = mock.Mock( + return_value=[_mirrored_eth_buy_order_stub("mirror-1")] + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + replicable = synchronizer._get_replicable_reference_orders() + assert synchronizer._count_unmirrored_reference_orders(replicable) == 1 + + +class TestScenarioLMissingMirrorBypass: + """ + Scenario L — Missing-mirror bypass when count exceeds threshold. + + Trigger: grace active with late fill; many reference orders missing on copier. + Expected: bypass when missing > threshold; skip upserts when missing equals threshold; no bypass if grace not identified. + """ + def _grace_active_synchronizer_with_orders( + self, + *, + frozen_reference_time: float, + orders: list[protocol_models.Order], + abort_threshold: int, + late_fill_order_ids: set[str], + ): + assets = _eth_usdt_pair_assets() + compliant_snapshot = _copied_account( + updated_at=frozen_reference_time - 1.0, + copied_assets=assets, + orders=[], + ) + reference = _copied_account( + updated_at=frozen_reference_time, + copied_assets=assets, + orders=orders, + historical_snapshots=[compliant_snapshot], + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=abort_threshold, + ), + ) + + original_is_late_reference_fill = synchronizer._is_late_reference_fill_for_order + + def late_fill_side_effect(order, orphan_orders, reference_state=None): + if str(order.id) in late_fill_order_ids: + return original_is_late_reference_fill(order, orphan_orders, reference_state) + return False + + synchronizer._is_late_reference_fill_for_order = late_fill_side_effect + return synchronizer + + def test_bypasses_grace_when_missing_exceed_threshold(self, caplog): + frozen_t0 = 1_700_000_000.0 + orders = [ + _replicable_buy_limit_order(order_id=f"ref-order-{order_index}") + for order_index in range(3) + ] + synchronizer = self._grace_active_synchronizer_with_orders( + frozen_reference_time=frozen_t0, + orders=orders, + abort_threshold=2, + late_fill_order_ids={"ref-order-0"}, + ) + created_order = mock.Mock() + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + with mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([created_order], 0, 0, None)), + ): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + + assert created == [created_order] * 3 + assert any( + "Bypassing mirrored orphan grace: 3 reference order(s)" in record.message + for record in caplog.records + ) + + def test_does_not_bypass_when_missing_equals_threshold(self, caplog): + frozen_t0 = 1_700_000_000.0 + orders = [ + _replicable_buy_limit_order(order_id=f"ref-order-{order_index}") + for order_index in range(2) + ] + synchronizer = self._grace_active_synchronizer_with_orders( + frozen_reference_time=frozen_t0, + orders=orders, + abort_threshold=2, + late_fill_order_ids={"ref-order-0"}, + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + with mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([], 0, 0, None)), + ): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + + assert created == [] + assert not any( + "Bypassing mirrored orphan grace" in record.message + for record in caplog.records + ) + assert any( + "Skipped reference mirror upsert for 2 order(s)" in record.message + for record in caplog.records + ) + + def test_does_not_bypass_when_grace_not_identified(self, caplog): + frozen_t0 = 1_700_000_000.0 + orders = [ + _replicable_buy_limit_order(order_id=f"ref-order-{order_index}") + for order_index in range(2) + ] + reference = _copied_account( + updated_at=frozen_t0, + copied_assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), + orders=orders, + ) + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=2, + ), + ) + abort_spy = mock.Mock(wraps=synchronizer.abort_mirrored_orphan_grace) + synchronizer.abort_mirrored_orphan_grace = abort_spy + with mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([], 0, 0, None)), + ): + with caplog.at_level(logging.INFO): + asyncio.run(synchronizer.synchronize()) + + abort_spy.assert_not_called() + assert not any( + "Bypassing mirrored orphan grace" in record.message + for record in caplog.records + ) + + +class TestScenarioLGridTwentyMissingMirrorBypass: + """ + Scenario L — Grid twenty missing-mirror bypass without rebalance abort. + + Trigger: 20 reference limits, one late-fill candidate, zero open mirrors. + Expected: synchronize bypasses grace via missing count and upserts all 20. + """ + + def _grid_post_rebalance_grace_synchronizer(self, *, frozen_reference_time: float): + # 20 reference limits on one symbol (grid); copier has none — limits were cancelled before rebalance + grid_orders = [ + _replicable_buy_limit_order( + order_id=f"ref-order-{order_index}", + price=decimal.Decimal("2000") - decimal.Decimal(order_index), + created_ts=frozen_reference_time, + ) + for order_index in range(20) + ] + assets = _eth_usdt_pair_assets() + # Compliant historical snapshot: required for grace window / pair-ratio checks + compliant_snapshot = _copied_account( + updated_at=frozen_reference_time - 1.0, + copied_assets=assets, + orders=[], + ) + reference = _copied_account( + updated_at=frozen_reference_time, + copied_assets=assets, + orders=grid_orders, + historical_snapshots=[compliant_snapshot], + ) + # Post–market-buy copier holdings: skewed vs reference snapshot so late-fill heuristic can match one order + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + # Default abort threshold from production settings + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=2, + ), + ) + # Only one late-fill candidate (grace_total=1); mirrors log where 20 missing ≠ 20 grace items + original_is_late_reference_fill = synchronizer._is_late_reference_fill_for_order + late_fill_order_ids = {"ref-order-0"} + + def late_fill_side_effect(order, orphan_orders, reference_state=None): + if str(order.id) in late_fill_order_ids: + return original_is_late_reference_fill(order, orphan_orders, reference_state) + return False + + synchronizer._is_late_reference_fill_for_order = late_fill_side_effect + return synchronizer + + def test_creates_twenty_limits_via_missing_mirror_bypass_without_rebalance_abort(self, caplog): + frozen_t0 = 1_700_000_000.0 + synchronizer = self._grid_post_rebalance_grace_synchronizer(frozen_reference_time=frozen_t0) + abort_spy = mock.Mock(wraps=synchronizer.abort_mirrored_orphan_grace) + synchronizer.abort_mirrored_orphan_grace = abort_spy + upsert_mock = mock.AsyncMock( + side_effect=lambda order: ([mock.Mock(name=f"created-{order.id}")], 0, 0, None) + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + # Grace is active before sync; no abort_mirrored_orphan_grace() — rebalance bypass path not used + assert synchronizer.is_mirrored_orphan_grace_identified() is True + with mock.patch.object(synchronizer, "_upsert_mirrored_reference_order", upsert_mock): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + + # synchronize() alone must bypass grace via missing_count (20) > threshold (2) + abort_spy.assert_called_once() + assert upsert_mock.await_count == 20 + # All symbol-level skips cleared after bypass; every limit upserted + assert len(created) == 20 + assert any( + "Bypassing mirrored orphan grace: 20 reference order(s) " + "missing on copier (> abort threshold 2)" in record.message + for record in caplog.records + ) + assert not any( + "Skipped reference mirror upsert for 20 order(s)" in record.message + for record in caplog.records + ) + + +class TestScenarioFExplicitAbort: + """ + Scenario F — Explicit abort_mirrored_orphan_grace. + + Trigger: late-fill grace active; then abort_mirrored_orphan_grace before second sync. + Expected: first sync skips upserts; second sync after abort creates mirrors. + """ + def test_manual_abort_allows_upsert_while_grace_active(self, caplog): + frozen_t0 = 1_700_000_000.0 + assets = _eth_usdt_pair_assets() + compliant_snapshot = _copied_account( + updated_at=frozen_t0 - 1.0, + copied_assets=assets, + orders=[], + ) + reference = _copied_account( + updated_at=frozen_t0, + copied_assets=assets, + orders=[_replicable_buy_limit_order()], + historical_snapshots=[compliant_snapshot], + ) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[]) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + created_order = mock.Mock() + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + with mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([created_order], 0, 0, None)), + ): + with caplog.at_level(logging.INFO): + blocked_created = asyncio.run(synchronizer.synchronize()) + synchronizer.abort_mirrored_orphan_grace() + allowed_created = asyncio.run(synchronizer.synchronize()) + + assert blocked_created == [] + assert allowed_created == [created_order] + assert any( + "Skipped reference mirror upsert for 1 order(s)" in record.message + for record in caplog.records + ) + + +def _replicable_buy_limit_order_id(order_id: str) -> protocol_models.Order: + return _replicable_buy_limit_order(order_id=order_id) + + +def _mirrored_eth_buy_order_stub(order_id: str) -> mock.Mock: + mirrored = mock.Mock() + mirrored.tag = copy_constants.MIRRORED_ORDER_TAG + mirrored.order_id = order_id + mirrored.symbol = "ETH/USDT" + mirrored.side = trading_enums.TradeOrderSide.BUY + mirrored.origin_price = decimal.Decimal("2000") + mirrored.origin_quantity = decimal.Decimal("1") + return mirrored + + +class TestScenarioJMissedHistoricalSignals: + """ + Scenario J — Missed historical signals grace abort. + + Trigger: first compliant snapshot index at missed_signals threshold; orphan still open. + Expected: abort flag True; apply_grace cancels orphan immediately. + """ + def test_is_aborted_when_first_compliant_snapshot_index_at_threshold(self): + order_m1 = _replicable_buy_limit_order_id("m1") + order_m2 = _replicable_buy_limit_order_id("m2") + assets = _eth_usdt_pair_assets() + empty_snapshot = _copied_account( + updated_at=time.time(), + copied_assets=assets, + orders=[], + ) + empty_snapshot_mid = _copied_account( + updated_at=time.time() - 1.0, + copied_assets=assets, + orders=[], + ) + compliant_snapshot = _copied_account( + updated_at=time.time() - 5.0, + copied_assets=assets, + orders=[order_m1, order_m2], + ) + live_reference = _copied_account( + updated_at=time.time(), + copied_assets=assets, + orders=[order_m1], + historical_snapshots=[empty_snapshot, empty_snapshot_mid, compliant_snapshot], + ) + mirror_m1 = _mirrored_eth_buy_order_stub("m1") + mirror_m2 = _mirrored_eth_buy_order_stub("m2") + exchange_if = mock.MagicMock() + exchange_if.orders.get_open_orders = mock.Mock(return_value=[mirror_m1, mirror_m2]) + exchange_if.portfolio.reference_market = "USDT" + exchange_if.portfolio.get_currency_portfolio_total = mock.Mock( + return_value=decimal.Decimal("1") + ) + exchange_if.market.get_potentially_outdated_price = mock.Mock( + return_value=(decimal.Decimal("2000"), False) + ) + copy_settings = copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=2, + missed_signals_grace_abort_threshold=2, + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + live_reference, + exchange_if, + copy_settings, + ) + assert synchronizer.is_mirrored_orphan_grace_aborted_for_missed_historical_signals() is True + + def test_apply_grace_cancels_immediately_when_missed_signals_abort(self): + order_m1 = _replicable_buy_limit_order_id("m1") + order_m2 = _replicable_buy_limit_order_id("m2") + assets = _eth_usdt_pair_assets() + empty_snapshot = _copied_account( + updated_at=time.time(), + copied_assets=assets, + orders=[], + ) + empty_snapshot_mid = _copied_account( + updated_at=time.time() - 1.0, + copied_assets=assets, + orders=[], + ) + compliant_snapshot = _copied_account( + updated_at=time.time() - 5.0, + copied_assets=assets, + orders=[order_m1, order_m2], + ) + live_reference = _copied_account( + updated_at=time.time(), + copied_assets=assets, + orders=[order_m1], + historical_snapshots=[empty_snapshot, empty_snapshot_mid, compliant_snapshot], + ) + mirror_m1 = _mirrored_eth_buy_order_stub("m1") + mirror_m2 = _mirrored_eth_buy_order_stub("m2") + exchange_if = mock.MagicMock() + # Two open mirrors so empty-order snapshots see grace_total>=threshold and stay non-compliant; + # otherwise a single orphan snapshot "complies" and missed-signals abort never triggers. + exchange_if.orders.get_open_orders = mock.Mock(return_value=[mirror_m1, mirror_m2]) + exchange_if.orders.cancel_order = mock.AsyncMock() + exchange_if.portfolio.reference_market = "USDT" + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + exchange_if.portfolio.get_currency_portfolio_total = mock.Mock( + side_effect=lambda currency: currency_totals[currency] + ) + exchange_if.market.get_potentially_outdated_price = mock.Mock( + return_value=(decimal.Decimal("2000"), False) + ) + copy_settings = copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=2, + missed_signals_grace_abort_threshold=2, + ) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + live_reference, + exchange_if, + copy_settings, + ) + replicable = synchronizer._get_replicable_reference_orders() + + async def run_grace(): + return await synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( + [mirror_m2], + replicable, + ) + + asyncio.run(run_grace()) + exchange_if.orders.cancel_order.assert_called_once_with(mirror_m2) + + +def _mirrored_limit_orphan_stub( + *, + order_id: str, + symbol: str = "ETH/USDT", + side=trading_enums.TradeOrderSide.BUY, + price: decimal.Decimal = decimal.Decimal("2000"), + quantity: decimal.Decimal = decimal.Decimal("1"), +): + orphan = mock.Mock() + orphan.tag = copy_constants.MIRRORED_ORDER_TAG + orphan.order_id = order_id + orphan.exchange_order_id = f"ex-{order_id}" + orphan.symbol = symbol + orphan.side = side + orphan.origin_price = price + orphan.origin_quantity = quantity + orphan.order_type = ( + trading_enums.TraderOrderType.BUY_LIMIT + if side is trading_enums.TradeOrderSide.BUY + else trading_enums.TraderOrderType.SELL_LIMIT + ) + orphan.creation_time = time.time() - 10.0 + orphan.timestamp = orphan.creation_time + orphan.status = trading_enums.OrderStatus.OPEN + return orphan + + +def _grace_window_reference( + *, + frozen_reference_time: float, + orders: list[protocol_models.Order], + assets: typing.Optional[list] = None, +): + pair_assets = assets or _eth_usdt_pair_assets() + compliant_snapshot = _copied_account( + updated_at=frozen_reference_time - 1.0, + copied_assets=pair_assets, + orders=[], + ) + return _copied_account( + updated_at=frozen_reference_time, + copied_assets=pair_assets, + orders=orders, + historical_snapshots=[compliant_snapshot], + ) + + +class TestScenarioBDeferredTaggedOrphan: + """ + Scenario B — Deferred tagged orphan at valid reference price. + + Trigger: tagged mirrored_order with bot id not in active reference; price still on grid; + inside grace window; pair-ratio OK. + Expected: cancel_order not called for that orphan; symbol upserts skipped. + """ + + def test_orphan_cancel_deferred_while_grace_active(self): + frozen_t0 = 1_700_000_000.0 + reference_order = _replicable_buy_limit_order(order_id="ref-active-1") + orphan = _mirrored_limit_orphan_stub(order_id="stale-orphan-1", price=decimal.Decimal("2000")) + # Pair-ratio eligible: copier holdings match simulated post-orphan-fill vs reference share. + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + reference = _grace_window_reference( + frozen_reference_time=frozen_t0, + orders=[reference_order], + assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), + ) + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[orphan]) + exchange_if.orders.cancel_order = mock.AsyncMock() + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ): + cancelled = asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( + [orphan], + synchronizer._get_replicable_reference_orders(), + ) + ) + assert cancelled == 0 + exchange_if.orders.cancel_order.assert_not_called() + + +class TestScenarioCOrphanAndLateFillTogether: + """ + Scenario C — Orphan + late fill together during grace. + + Trigger: tagged orphan and late-fill candidate both under abort threshold; inside grace. + Expected: orphan cancel deferred; upserts skipped on symbol. + """ + + def test_defer_orphan_and_skip_upserts(self, caplog): + frozen_t0 = 1_700_000_000.0 + late_fill_order = _replicable_buy_limit_order(order_id="ref-late-1") + orphan = _mirrored_limit_orphan_stub(order_id="stale-orphan-1", price=decimal.Decimal("2000")) + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + reference = _grace_window_reference( + frozen_reference_time=frozen_t0, + orders=[late_fill_order], + ) + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[orphan]) + exchange_if.orders.cancel_order = mock.AsyncMock() + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ), mock.patch.object( + synchronizer, + "_mirrored_orphan_batch_eligible_for_grace", + return_value=True, + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([], 0, 0, None)), + ): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + assert created == [] + exchange_if.orders.cancel_order.assert_not_called() + assert any( + "Skipped reference mirror upsert" in record.message + for record in caplog.records + ) + + +class TestScenarioDWrongPriceStrayDuringGrace: + """ + Scenario D — Wrong-price stray during active grace. + + Trigger: late-fill grace active and an open limit at a price not in the reference grid. + Expected: stray cancelled; upserts still skipped; valid-price opens kept. + """ + + def test_synchronize_cancels_wrong_price_stray_keeps_grace_skip(self, caplog): + frozen_t0 = 1_700_000_000.0 + reference_order = _replicable_buy_limit_order(order_id="ref-late-1") + matched = _mirrored_limit_orphan_stub( + order_id=str(reference_order.id), + price=decimal.Decimal("2000"), + ) + stray = _mirrored_limit_orphan_stub( + order_id="wrong-price-1", + price=decimal.Decimal("1500"), + ) + stray.tag = None + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + reference = _grace_window_reference( + frozen_reference_time=frozen_t0, + orders=[reference_order], + ) + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + open_orders = [stray, matched] + + def get_open_orders(symbol=None): + if symbol is None or symbol == "ETH/USDT": + return list(open_orders) + return [] + + exchange_if.orders.get_open_orders = mock.Mock(side_effect=get_open_orders) + + async def cancel_side_effect(order): + if order in open_orders: + open_orders.remove(order) + + exchange_if.orders.cancel_order = mock.AsyncMock(side_effect=cancel_side_effect) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ), mock.patch.object( + synchronizer, + "_reference_symbols_skipped_while_grace_orphans_uncancelled", + return_value={"ETH/USDT"}, + ), mock.patch.object( + synchronizer, + "_maybe_bypass_grace_for_missing_mirrored_reference_orders", + side_effect=lambda replicable, skip: skip, + ), mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + assert created == [] + exchange_if.orders.cancel_order.assert_awaited_once_with(stray) + assert matched in open_orders + assert any( + "Skipped reference mirror upsert" in record.message + for record in caplog.records + ) + + +class TestScenarioEGraceElapsedCancelOrphans: + """ + Scenario E — Grace elapsed cancels tagged orphans. + + Trigger: tagged orphan still open after grace_seconds from started_at. + Expected: cancel_order called; grace elapsed log. + """ + + def test_cancels_tagged_orphan_when_grace_elapsed(self, caplog): + frozen_t0 = 1_700_000_000.0 + reference_order = _replicable_buy_limit_order(order_id="ref-active-1") + orphan = _mirrored_limit_orphan_stub(order_id="stale-orphan-1") + currency_totals = { + "ETH": decimal.Decimal("1"), + "USDT": decimal.Decimal("10000"), + } + reference = _grace_window_reference( + frozen_reference_time=frozen_t0, + orders=[reference_order], + assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), + ) + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[orphan]) + exchange_if.orders.cancel_order = mock.AsyncMock() + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0 + 70.0, + ): + with caplog.at_level(logging.INFO): + cancelled = asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( + [orphan], + synchronizer._get_replicable_reference_orders(), + ) + ) + assert cancelled == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(orphan) + assert "Mirrored orphan grace elapsed after" in caplog.text + + +class TestScenarioGGraceDisabled: + """ + Scenario G — Grace disabled (grace_seconds <= 0). + + Trigger: mirrored_orphan_cancel_grace_seconds is 0 with a tagged orphan. + Expected: immediate orphan cancel; skip symbol set empty. + """ + + def test_immediate_orphan_cancel_when_grace_seconds_zero(self): + orphan = _mirrored_limit_orphan_stub(order_id="stale-orphan-1") + reference = _copied_account( + copied_assets=_eth_usdt_pair_assets(), + orders=[_replicable_buy_limit_order()], + ) + exchange_if = _exchange_interface_stub( + currency_totals={"ETH": decimal.Decimal("1"), "USDT": decimal.Decimal("10000")}, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[orphan]) + exchange_if.orders.cancel_order = mock.AsyncMock() + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(mirrored_orphan_cancel_grace_seconds=0.0), + ) + cancelled = asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( + [orphan], + synchronizer._get_replicable_reference_orders(), + ) + ) + assert cancelled == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(orphan) + assert synchronizer._reference_symbols_skipped_while_grace_orphans_uncancelled( + synchronizer._get_replicable_reference_orders() + ) == set() + + +class TestScenarioHGraceTotalAtOrAboveThreshold: + """ + Scenario H — grace_total >= abort threshold. + + Trigger: orphan count reaches mirrored_orphan_grace_abort_threshold (empty history). + Expected: immediate cancel of orphans; threshold abort log. + """ + + def test_cancels_immediately_when_grace_total_reaches_threshold(self, caplog): + frozen_t0 = 1_700_000_000.0 + orphan_a = _mirrored_limit_orphan_stub(order_id="orphan-a") + orphan_b = _mirrored_limit_orphan_stub(order_id="orphan-b", price=decimal.Decimal("1999")) + reference_order = _replicable_buy_limit_order(order_id="ref-1") + # Empty history: not "invalid"; threshold abort is the path under test. + reference = _copied_account( + updated_at=frozen_t0, + copied_assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), + orders=[reference_order], + historical_snapshots=[], + ) + exchange_if = _exchange_interface_stub( + currency_totals={"ETH": decimal.Decimal("1"), "USDT": decimal.Decimal("10000")}, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[orphan_a, orphan_b]) + exchange_if.orders.cancel_order = mock.AsyncMock() + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=2, + ), + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ), mock.patch.object( + synchronizer, + "_mirrored_orphan_batch_eligible_for_grace", + return_value=True, + ): + with caplog.at_level(logging.INFO): + cancelled = asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( + [orphan_a, orphan_b], + synchronizer._get_replicable_reference_orders(), + ) + ) + assert cancelled == 2 + assert any( + "grace item(s) >= threshold" in record.message + for record in caplog.records + ) + + +class TestScenarioINoCompliantHistoricalSnapshot: + """ + Scenario I — No compliant historical snapshot. + + Trigger: non-empty history where no snapshot complies under grace checks. + Expected: invalid grace; immediate orphan cancel; no-compliant-snapshot log. + """ + + def test_cancels_immediately_when_no_compliant_snapshot(self, caplog): + orphan = _mirrored_limit_orphan_stub(order_id="orphan-1") + # History with orders that keep grace_total high vs threshold so nothing complies. + non_compliant = _copied_account( + updated_at=time.time() - 1.0, + copied_assets=_eth_usdt_pair_assets(), + orders=[], + ) + live = _copied_account( + updated_at=time.time(), + copied_assets=_eth_usdt_pair_assets(), + orders=[_replicable_buy_limit_order()], + historical_snapshots=[non_compliant], + ) + exchange_if = _exchange_interface_stub( + currency_totals={"ETH": decimal.Decimal("1"), "USDT": decimal.Decimal("10000")}, + market_price=decimal.Decimal("2000"), + ) + # Two orphans so empty snapshot has grace_total >= threshold and is non-compliant. + orphan_b = _mirrored_limit_orphan_stub(order_id="orphan-2", price=decimal.Decimal("1999")) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[orphan, orphan_b]) + exchange_if.orders.cancel_order = mock.AsyncMock() + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + live, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=2, + ), + ) + assert synchronizer.is_mirrored_orphan_grace_invalid_no_compliant_snapshot() is True + with caplog.at_level(logging.INFO): + cancelled = asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( + [orphan, orphan_b], + synchronizer._get_replicable_reference_orders(), + ) + ) + assert cancelled == 2 + assert any( + "no compliant reference snapshot" in record.message + for record in caplog.records + ) + + +class TestScenarioKApplyGraceWhenPairRatioFails: + """ + Scenario K — Pair-ratio heuristic fails on apply-grace path. + + Trigger: tagged orphan present; batch pair-ratio eligibility False. + Expected: orphans cancelled immediately via apply_grace. + """ + + def test_cancels_orphans_when_batch_ineligible(self, caplog): + orphan = _mirrored_limit_orphan_stub( + order_id="orphan-1", + quantity=decimal.Decimal("1"), + price=decimal.Decimal("2000"), + ) + reference = _copied_account( + copied_assets=_eth_usdt_pair_assets(eth_ratio=0.5, usdt_ratio=0.5), + orders=[_replicable_buy_limit_order()], + historical_snapshots=[ + _copied_account( + updated_at=time.time() - 1.0, + copied_assets=_eth_usdt_pair_assets(eth_ratio=0.5, usdt_ratio=0.5), + orders=[], + ) + ], + ) + # Copier holdings make simulated post-fill share mismatch reference (same as heuristic unit test). + exchange_if = _exchange_interface_stub( + currency_totals={"ETH": decimal.Decimal("1"), "USDT": decimal.Decimal("10000")}, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[orphan]) + exchange_if.orders.cancel_order = mock.AsyncMock() + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + mirrored_orphan_grace_pair_ratio_max_delta=decimal.Decimal("0.02"), + ), + ) + assert synchronizer._mirrored_orphan_batch_eligible_for_grace([orphan]) is False + with caplog.at_level(logging.INFO): + cancelled = asyncio.run( + synchronizer._apply_grace_policy_and_cancel_mirrored_orphans( + [orphan], + synchronizer._get_replicable_reference_orders(), + ) + ) + assert cancelled == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(orphan) + + +class TestScenarioNDuplicateNoGraceCrossRef: + """ + Scenario N — Duplicate at valid price with grace idle. + + Trigger: open mirrors match active reference ids (no orphan / late-fill grace items). + Expected: grace not identified; authoritative duplicate cancel coverage is reconciliation R2. + """ + + def test_grace_not_identified_when_mirrors_match_reference(self): + reference_order = _replicable_buy_limit_order(order_id="ref-1") + mirror = _mirrored_limit_orphan_stub(order_id="ref-1") + reference = _copied_account( + copied_assets=_eth_usdt_pair_assets(eth_ratio=0.25, usdt_ratio=0.5), + orders=[reference_order], + ) + exchange_if = _exchange_interface_stub( + currency_totals={"ETH": decimal.Decimal("1"), "USDT": decimal.Decimal("10000")}, + market_price=decimal.Decimal("2000"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=[mirror]) + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(mirrored_orphan_cancel_grace_seconds=60.0), + ) + assert synchronizer.is_mirrored_orphan_grace_identified() is False + + +class TestScenarioODuplicateDuringActiveGrace: + """ + Scenario O — Duplicate at valid price during grace. + + Trigger: two opens at the same reference price; symbol in grace skip set. + Expected: reconcile stray_only keeps both; upserts skipped; cancel_order not called. + """ + + def test_synchronize_keeps_same_price_duplicate_during_grace(self, caplog): + frozen_t0 = 1_700_000_000.0 + reference_order = _replicable_buy_limit_order(order_id="ref-late-1") + preferred = _mirrored_limit_orphan_stub(order_id="ref-late-1", price=decimal.Decimal("2000")) + duplicate = _mirrored_limit_orphan_stub(order_id="extra-dup", price=decimal.Decimal("2000")) + duplicate.tag = None + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + reference = _grace_window_reference( + frozen_reference_time=frozen_t0, + orders=[reference_order], + ) + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + open_orders = [preferred, duplicate] + + def get_open_orders(symbol=None): + if symbol is None or symbol == "ETH/USDT": + return list(open_orders) + return [] + + exchange_if.orders.get_open_orders = mock.Mock(side_effect=get_open_orders) + exchange_if.orders.cancel_order = mock.AsyncMock() + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ), mock.patch.object( + synchronizer, + "_reference_symbols_skipped_while_grace_orphans_uncancelled", + return_value={"ETH/USDT"}, + ), mock.patch.object( + synchronizer, + "_maybe_bypass_grace_for_missing_mirrored_reference_orders", + side_effect=lambda replicable, skip: skip, + ), mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + assert created == [] + exchange_if.orders.cancel_order.assert_not_called() + assert preferred in open_orders and duplicate in open_orders + assert any( + "Skipped reference mirror upsert" in record.message + for record in caplog.records + ) + + +class TestScenarioPGraceKeepsMirroredOrphanOffGrid: + """ + Scenario P — Grace keeps mirrored orphan off-grid; cancels untagged. + + Trigger: grace skip symbols active; matched open + mirrored orphan at off-grid price + untagged stray. + Expected: synchronize cancels only the untagged stray; mirrored orphan kept; upserts skipped. + """ + + def test_synchronize_keeps_mirrored_orphan_cancels_untagged(self, caplog): + frozen_t0 = 1_700_000_000.0 + reference_order = _replicable_buy_limit_order(order_id="ref-late-1") + matched = _mirrored_limit_orphan_stub( + order_id=str(reference_order.id), + price=decimal.Decimal("2000"), + ) + mirrored_orphan = _mirrored_limit_orphan_stub( + order_id="grid_ref_b1", + price=decimal.Decimal("1500"), + ) + untagged_stray = _mirrored_limit_orphan_stub( + order_id="wrong-price-1", + price=decimal.Decimal("1400"), + ) + untagged_stray.tag = None + currency_totals = { + "ETH": decimal.Decimal("2"), + "USDT": decimal.Decimal("8000"), + } + reference = _grace_window_reference( + frozen_reference_time=frozen_t0, + orders=[reference_order], + ) + exchange_if = _exchange_interface_stub( + currency_totals=currency_totals, + market_price=decimal.Decimal("2000"), + ) + open_orders = [matched, mirrored_orphan, untagged_stray] + + def get_open_orders(symbol=None): + if symbol is None or symbol == "ETH/USDT": + return list(open_orders) + return [] + + exchange_if.orders.get_open_orders = mock.Mock(side_effect=get_open_orders) + + async def cancel_side_effect(order): + if order in open_orders: + open_orders.remove(order) + + exchange_if.orders.cancel_order = mock.AsyncMock(side_effect=cancel_side_effect) + exchange_if.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings( + mirrored_orphan_cancel_grace_seconds=60.0, + mirrored_orphan_grace_abort_threshold=3, + ), + ) + with mock.patch( + "octobot_copy.orders_mirroring.orders_synchronizer.time.time", + return_value=frozen_t0, + ), mock.patch.object( + synchronizer, + "_reference_symbols_skipped_while_grace_orphans_uncancelled", + return_value={"ETH/USDT"}, + ), mock.patch.object( + synchronizer, + "_maybe_bypass_grace_for_missing_mirrored_reference_orders", + side_effect=lambda replicable, skip: skip, + ), mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + assert created == [] + exchange_if.orders.cancel_order.assert_awaited_once_with(untagged_stray) + assert matched in open_orders + assert mirrored_orphan in open_orders + assert untagged_stray not in open_orders + assert any( + "Skipped reference mirror upsert" in record.message + for record in caplog.records + ) diff --git a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py index c3536de837..7a1c66fe83 100644 --- a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py +++ b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_live_portfolio_sync.py @@ -45,7 +45,16 @@ def _load_copy_tests_python_helpers(): pytestmark = pytest.mark.asyncio _BTC_USDT = "BTC/USDT" -_BTC_PRICE = decimal.Decimal("60000") +_BTC_PRICE = decimal.Decimal("60300") +_RECREATE_SELL_ORDER_ID = "4bd49d83-20d5-4258-8298-55da4bac60e7" +_SIBLING_SELL_ORDER_ID = "978f4fc0-d60a-42e5-8657-8973e7a53999" +_COUPLER_BTC_TOTAL = decimal.Decimal("0.00753") +_COUPLER_BTC_AVAILABLE = decimal.Decimal("0.00068") +_SIBLING_SELL_QUANTITY = _COUPLER_BTC_TOTAL - _COUPLER_BTC_AVAILABLE +_REFERENCE_BTC_TOTAL = decimal.Decimal("0.00753") +_REFERENCE_SELL_QUANTITY = decimal.Decimal("0.00074") +_REFERENCE_SELL_PRICE = decimal.Decimal("61188") +_SELL_SCENARIO_USDT_TOTAL = decimal.Decimal("500") _REPLACE_ORDER_ID = "dabfe054-a650-4a09-a296-8d22ceb6f664" _CREATE_ORDER_ID = "7cf7e7ad-b3e4-4f6a-a2f5-1ecff2cc176f" # Copier open buy before sync (oversized vs reference target); locks ~19.8 USDT at _BTC_PRICE. @@ -183,3 +192,121 @@ async def test_creates_buy_after_downsize_replace_frees_stale_quote(self, live_t assert replace_order.origin_quantity == decimal.Decimal("0.0002") assert create_order is not None assert create_order.origin_quantity == decimal.Decimal("0.00015") + + +def _replicable_btc_sell_limit_order( + *, + order_id: str, + quantity: decimal.Decimal, + price: decimal.Decimal, +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol=_BTC_USDT, + price=float(price), + quantity=float(quantity), + filled=0.0, + exchange_id="reference-exchange-id", + side=protocol_models.Side.SELL, + type=protocol_models.OrderType.LIMIT, + trigger_above=True, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _reference_account_with_one_sell() -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[ + protocol_models.CopiedAsset( + name="BTC", + total=float(_REFERENCE_BTC_TOTAL), + available=float(_COUPLER_BTC_AVAILABLE), + ratio=0.5, + ), + protocol_models.CopiedAsset( + name="USDT", + total=float(_SELL_SCENARIO_USDT_TOTAL), + available=float(_SELL_SCENARIO_USDT_TOTAL), + ratio=0.5, + ), + ], + orders=[ + _replicable_btc_sell_limit_order( + order_id=_RECREATE_SELL_ORDER_ID, + quantity=_REFERENCE_SELL_QUANTITY, + price=_REFERENCE_SELL_PRICE, + ), + _replicable_btc_sell_limit_order( + order_id=_SIBLING_SELL_ORDER_ID, + quantity=_SIBLING_SELL_QUANTITY, + price=decimal.Decimal("62188"), + ), + ], + ) + + +class TestSynchronizeGraceElapsedSellRecreate: + @pytest.mark.parametrize("backtesting_config", ["USDT"], indirect=True) + async def test_creates_sell_after_grace_elapsed_respects_available_base(self, live_trading_trader): + # Jul 2 log: sibling sells lock most BTC; grace-elapsed resync recreates a missing mirrored + # sell. Sizing must cap to available base (total minus sibling locks), not raw total. + _config, exchange_manager, _trader = live_trading_trader + copy_tests_python_helpers.ensure_traded_symbol_pairs(exchange_manager, (_BTC_USDT,)) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + + trading_api.force_set_mark_price(exchange_manager, _BTC_USDT, _BTC_PRICE) + portfolio_manager.portfolio.update_portfolio_from_balance( + { + "BTC": { + "available": _COUPLER_BTC_TOTAL, + "total": _COUPLER_BTC_TOTAL, + }, + "USDT": { + "available": _SELL_SCENARIO_USDT_TOTAL, + "total": _SELL_SCENARIO_USDT_TOTAL, + }, + }, + True, + ) + portfolio_manager.handle_balance_updated() + portfolio_manager.portfolio_value_holder.value_converter.missing_currency_data_in_exchange.discard("USDT") + portfolio_manager.handle_mark_price_update(_BTC_USDT, _BTC_PRICE) + + exchange_interface = copy_exchange.ExchangeInterface(exchange_manager) + await exchange_interface.orders.create_order( + trading_enums.TraderOrderType.SELL_LIMIT, + _BTC_USDT, + decimal.Decimal("62188"), + _SIBLING_SELL_QUANTITY, + decimal.Decimal("62188"), + tag=copy_constants.MIRRORED_ORDER_TAG, + order_id=_SIBLING_SELL_ORDER_ID, + wait_for_creation=True, + ) + + portfolio_manager.portfolio.get_currency_portfolio("BTC").available = _COUPLER_BTC_AVAILABLE + + reference_account = _reference_account_with_one_sell() + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference_account, + exchange_interface, + copy_entities.AccountCopySettings(), + ) + synchronizer.abort_mirrored_orphan_grace() + + refresh_portfolio_mock = mock.AsyncMock(return_value=True) + with mock.patch.object( + exchange_interface.portfolio, + "refresh_portfolio", + refresh_portfolio_mock, + ): + await synchronizer.synchronize() + + created_order = _open_order_by_id(exchange_manager, _RECREATE_SELL_ORDER_ID) + assert created_order is not None + assert created_order.origin_quantity == _COUPLER_BTC_AVAILABLE diff --git a/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_reconciliation.py b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_reconciliation.py new file mode 100644 index 0000000000..17aa417541 --- /dev/null +++ b/packages/copy/tests/python/orders_mirroring/test_orders_synchronizer_reconciliation.py @@ -0,0 +1,1000 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or +# (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# OctoBot. If not, see . +""" +OrdersSynchronizer price-level open-order reconciliation cases. + +Catalog (extend when a new reconcile bug appears — add Rn + class): + R1 Wrong-price stray (OAEOCK) — cancel off-grid; keep matched + R2 Duplicate at same valid price (full) — cancel extras beyond reference count + R3 Multiple reference orders at same price — do not cancel down to 1 when ref count is 2+ + R4 stray_only mode — cancel wrong-price only; keep same-price duplicates + R5 Pre-sync + post-sync always — both run; post runs even with no creates/replaces + R6 Sync cancels untagged stray before upsert + R7 Replace waits for cancel before create + R8 Ambiguous map cancels extras + R9 Late-fill-adjusted count invariant — no error when actual == expected - late_fills + R10 Invariant does not call reconcile — log only + R11 Force-abort / public reconcile — full mode + R12 Grace symbol + wrong-price (D) — stray cancelled under stray_only + R13 Grace symbol + duplicate (O) — stray_only keeps; full after abort + R14 Count invariant after wait — no premature ERROR while pending creation + R15 Grace keeps mirrored orphan off-grid; cancels untagged wrong-price +""" +import asyncio +import contextlib +import decimal +import logging +import time +import typing + +import mock + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums + +import octobot_copy.constants as copy_constants +import octobot_copy.entities as copy_entities +import octobot_copy.orders_mirroring.mirrored_quantity_compute_result as mirrored_quantity_compute_result +import octobot_copy.orders_mirroring.orders_synchronizer as orders_synchronizer_module + + +def _copied_account( + *, + orders: typing.Optional[list[protocol_models.Order]] = None, +) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[ + protocol_models.CopiedAsset(name="BTC", total=1.0, available=1.0, ratio=0.5), + protocol_models.CopiedAsset(name="USDC", total=10000.0, available=10000.0, ratio=0.5), + ], + orders=orders, + ) + + +@contextlib.asynccontextmanager +async def _passthrough_mirror_sync_available_updates(): + yield + + +def _exchange_interface_stub(*, currency_totals: dict[str, decimal.Decimal], market_price: decimal.Decimal): + exchange_interface = mock.MagicMock() + exchange_interface.portfolio.reference_market = "USDC" + exchange_interface.portfolio.get_currency_portfolio_total = ( + lambda currency: currency_totals[currency] + ) + exchange_interface.market.get_potentially_outdated_price = mock.Mock( + return_value=(market_price, False) + ) + exchange_interface.portfolio.mirror_sync_available_updates = _passthrough_mirror_sync_available_updates + return exchange_interface + + +def _btc_usdc_limit_reference_order( + *, + order_id: str, + side: protocol_models.Side, + price: decimal.Decimal, + quantity: decimal.Decimal = decimal.Decimal("0.0001"), +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol="BTC/USDC", + price=float(price), + quantity=float(quantity), + filled=0.0, + exchange_id=f"ref-{order_id[:8]}", + side=side, + type=protocol_models.OrderType.LIMIT, + trigger_above=side is protocol_models.Side.SELL, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _open_limit_order_stub( + *, + order_id: str, + exchange_order_id: str, + side: trading_enums.TradeOrderSide, + quantity: decimal.Decimal, + price: decimal.Decimal, + tag: str | None = None, +): + order_type = ( + trading_enums.TraderOrderType.SELL_LIMIT + if side is trading_enums.TradeOrderSide.SELL + else trading_enums.TraderOrderType.BUY_LIMIT + ) + order = mock.Mock() + order.order_id = order_id + order.exchange_order_id = exchange_order_id + order.symbol = "BTC/USDC" + order.side = side + order.origin_quantity = quantity + order.origin_price = price + order.order_type = order_type + order.tag = tag + order.status = trading_enums.OrderStatus.OPEN + return order + + +def _synchronizer_with_open_orders( + *, + reference_orders: list[protocol_models.Order], + open_orders: list, +) -> tuple[orders_synchronizer_module.OrdersSynchronizer, mock.MagicMock]: + reference = _copied_account(orders=reference_orders) + exchange_if = _exchange_interface_stub( + currency_totals={ + "BTC": decimal.Decimal("0.01"), + "USDC": decimal.Decimal("10000"), + }, + market_price=decimal.Decimal("66326.7"), + ) + exchange_if.orders.get_open_orders = mock.Mock(return_value=open_orders) + exchange_if.orders.cancel_order = mock.AsyncMock() + exchange_if.orders.automatically_synchronize_orders = mock.Mock(return_value=True) + orders_manager = mock.Mock() + exchange_if.orders._exchange_manager = mock.Mock() + exchange_if.orders._exchange_manager.exchange_personal_data.orders_manager = orders_manager + synchronizer = orders_synchronizer_module.OrdersSynchronizer( + reference, + exchange_if, + copy_entities.AccountCopySettings(), + ) + return synchronizer, exchange_if + + +class TestCaseR1WrongPriceStray: + """ + R1 — Wrong-price stray (OAEOCK-class). + + Trigger: open limit at a price not in reference grid; another open matches a reference level. + Expected: cancel only the stray; keep the matched order. + """ + + def test_reconcile_cancels_stray_keeps_matched(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="2f7c0eac-cd66-42ff-96f6-80b2e6039658", + side=protocol_models.Side.SELL, + price=decimal.Decimal("66326.7"), + ) + stray_order = _open_limit_order_stub( + order_id="86fc50ab-7bc4-4632-a2e0-2c9128d0ba9e", + exchange_order_id="OAEOCK-6OQ3C-VDJ6QN", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.0002"), + price=decimal.Decimal("65326.7"), + tag=None, + ) + matched_order = _open_limit_order_stub( + order_id=str(reference_order.id), + exchange_order_id="OPXF5Q-6AKBQ-PAJXD2", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.00027"), + price=decimal.Decimal("66326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[stray_order, matched_order], + ) + cancelled_count = asyncio.run( + synchronizer._reconcile_open_orders_with_reference([reference_order]) + ) + assert cancelled_count == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(stray_order) + + +class TestCaseR2DuplicateAtSamePriceFull: + """ + R2 — Duplicate at same valid price (full reconcile). + + Trigger: two opens at one reference price level; reference count is one. + Expected: cancel the non-preferred duplicate; keep the tagged match. + """ + + def test_reconcile_cancels_duplicate_at_same_price(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="28c1394b-dcb7-4f90-8878-4a61827471ca", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + first_duplicate = _open_limit_order_stub( + order_id="first-bot-id", + exchange_order_id="O7GDOQ-5ALJT-5QPQE4", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.00009"), + price=decimal.Decimal("59326.7"), + ) + second_duplicate = _open_limit_order_stub( + order_id=str(reference_order.id), + exchange_order_id="OGE3T6-NDOIV-LR6MZI", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[first_duplicate, second_duplicate], + ) + cancelled_count = asyncio.run( + synchronizer._reconcile_open_orders_with_reference([reference_order]) + ) + assert cancelled_count == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(first_duplicate) + + +class TestCaseR3MultipleReferenceAtSamePrice: + """ + R3 — Multiple reference orders at the same price. + + Trigger: two reference orders and two opens at the same price level. + Expected: cancel count 0; do not cancel down to a single open. + """ + + def test_keeps_two_opens_when_reference_has_two_at_level(self): + first_ref = _btc_usdc_limit_reference_order( + order_id="ref-a", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + second_ref = _btc_usdc_limit_reference_order( + order_id="ref-b", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + first_open = _open_limit_order_stub( + order_id="ref-a", + exchange_order_id="EX-A", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + second_open = _open_limit_order_stub( + order_id="ref-b", + exchange_order_id="EX-B", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[first_ref, second_ref], + open_orders=[first_open, second_open], + ) + cancelled_count = asyncio.run( + synchronizer._reconcile_open_orders_with_reference([first_ref, second_ref]) + ) + assert cancelled_count == 0 + exchange_if.orders.cancel_order.assert_not_called() + + +class TestCaseR4StrayOnlyMode: + """ + R4 — stray_only mode. + + Trigger: wrong-price stray plus same-price duplicate; stray_only_symbols includes the symbol. + Expected: cancel wrong-price only; keep same-price duplicate. + """ + + def test_stray_only_cancels_wrong_price_keeps_duplicate(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.SELL, + price=decimal.Decimal("66326.7"), + ) + stray = _open_limit_order_stub( + order_id="wrong", + exchange_order_id="OAEOCK-STRAY", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.0002"), + price=decimal.Decimal("65326.7"), + ) + preferred = _open_limit_order_stub( + order_id="ref-1", + exchange_order_id="MATCHED", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.00027"), + price=decimal.Decimal("66326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + duplicate = _open_limit_order_stub( + order_id="dup", + exchange_order_id="DUP", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.0002"), + price=decimal.Decimal("66326.7"), + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[stray, preferred, duplicate], + ) + cancelled_count = asyncio.run( + synchronizer._reconcile_open_orders_with_reference( + [reference_order], + stray_only_symbols={"BTC/USDC"}, + ) + ) + assert cancelled_count == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(stray) + + +class TestCaseR5PreAndPostSyncAlways: + """ + R5 — Pre-sync and post-sync reconcile always. + + Trigger: synchronize with already-synced opens and no creates/replaces. + Expected: _reconcile_open_orders_with_reference awaited twice. + """ + + def test_post_reconcile_runs_when_no_creates_or_replaces(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.SELL, + price=decimal.Decimal("66326.7"), + ) + matched = _open_limit_order_stub( + order_id="ref-1", + exchange_order_id="MATCHED", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.00027"), + price=decimal.Decimal("66326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[matched], + ) + reconcile_mock = mock.AsyncMock(return_value=0) + with mock.patch.object( + synchronizer, + "_reconcile_open_orders_with_reference", + reconcile_mock, + ), mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([], 0, 1, None)), + ): + asyncio.run(synchronizer.synchronize()) + assert reconcile_mock.await_count == 2 + + +class TestCaseR6SyncCancelsUntaggedStray: + """ + R6 — Sync cancels untagged stray before upsert. + + Trigger: synchronize with an untagged wrong-price open and a matched mirrored open. + Expected: cancel the stray once before upserts proceed. + """ + + def test_synchronize_cancels_untagged_stray_before_upsert(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="2f7c0eac-cd66-42ff-96f6-80b2e6039658", + side=protocol_models.Side.SELL, + price=decimal.Decimal("66326.7"), + ) + stray_order = _open_limit_order_stub( + order_id="86fc50ab-7bc4-4632-a2e0-2c9128d0ba9e", + exchange_order_id="OAEOCK-6OQ3C-VDJ6QN", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.0002"), + price=decimal.Decimal("65326.7"), + tag=None, + ) + matched_order = _open_limit_order_stub( + order_id=str(reference_order.id), + exchange_order_id="OPXF5Q-6AKBQ-PAJXD2", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.00027"), + price=decimal.Decimal("66326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[stray_order, matched_order], + ) + remaining_open_orders = [stray_order, matched_order] + + def get_open_orders(symbol=None): + if symbol is None or symbol == "BTC/USDC": + return list(remaining_open_orders) + return [] + + exchange_if.orders.get_open_orders = mock.Mock(side_effect=get_open_orders) + + async def cancel_side_effect(order): + if order in remaining_open_orders: + remaining_open_orders.remove(order) + + exchange_if.orders.cancel_order = mock.AsyncMock(side_effect=cancel_side_effect) + with mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([], 0, 1, None)), + ): + asyncio.run(synchronizer.synchronize()) + exchange_if.orders.cancel_order.assert_awaited_once_with(stray_order) + + +class TestCaseR7ReplaceCancelsThenCreates: + """ + R7 — Replace cancels then creates. + + Trigger: upsert replace of an existing mirrored limit with quantity mismatch. + Expected: cancel_order then create_orders (no wait_for_order_absent). + """ + + def test_replace_cancels_then_creates(self): + reference_order_id = "2f7c0eac-cd66-42ff-96f6-80b2e6039658" + reference_order = _btc_usdc_limit_reference_order( + order_id=reference_order_id, + side=protocol_models.Side.SELL, + price=decimal.Decimal("66326.7"), + quantity=decimal.Decimal("0.00027"), + ) + existing_order = _open_limit_order_stub( + order_id=reference_order_id, + exchange_order_id="OAINUR-XFJRB-SKVMUR", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.0003071"), + price=decimal.Decimal("66326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[existing_order], + ) + exchange_if.market.get_market_status = mock.Mock(return_value=mock.Mock()) + exchange_if.orders.adapt_order_quantity_and_target_price_for_order_creation = mock.Mock( + return_value=(decimal.Decimal("66326.7"), decimal.Decimal("0.00027639")), + ) + exchange_if.orders.create_orders = mock.AsyncMock(return_value=([], False)) + compute_result = mirrored_quantity_compute_result.MirroredQuantityComputeResult( + ideal_quantity=decimal.Decimal("0.00027639"), + resolved_trader_order_type=trading_enums.TraderOrderType.SELL_LIMIT, + limit_price=decimal.Decimal("66326.7"), + current_price=decimal.Decimal("66326.7"), + ) + with mock.patch.object( + synchronizer, + "_is_late_reference_fill_for_order", + return_value=False, + ), mock.patch.object( + synchronizer, + "_compute_mirrored_quantity_type_and_price", + mock.AsyncMock(return_value=compute_result), + ): + asyncio.run(synchronizer._upsert_mirrored_reference_order(reference_order)) + exchange_if.orders.cancel_order.assert_awaited_once_with(existing_order) + exchange_if.orders.create_orders.assert_awaited_once() + + +class TestCaseR8AmbiguousMapCancelsExtras: + """ + R8 — Ambiguous map cancels extras. + + Trigger: two unmapped opens at the reference price for one reference order. + Expected: relink preferred candidate; cancel the other; warn about ambiguous match. + """ + + def test_ambiguous_candidates_relinks_one_and_cancels_extras(self, caplog): + caplog.set_level(logging.WARNING) + reference_order = _btc_usdc_limit_reference_order( + order_id="28c1394b-dcb7-4f90-8878-4a61827471ca", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + first_candidate = _open_limit_order_stub( + order_id="first-bot-id", + exchange_order_id="O7GDOQ-5ALJT-5QPQE4", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.00009"), + price=decimal.Decimal("59326.7"), + ) + second_candidate = _open_limit_order_stub( + order_id="second-bot-id", + exchange_order_id="OGE3T6-NDOIV-LR6MZI", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[first_candidate, second_candidate], + ) + mapped = asyncio.run( + synchronizer._map_unmapped_open_order_for_reference( + reference_order=reference_order, + reference_order_id=str(reference_order.id), + side=trading_enums.TradeOrderSide.BUY, + trader_order_type=trading_enums.TraderOrderType.BUY_LIMIT, + order_target_price=decimal.Decimal("59326.7"), + active_reference_ids={str(reference_order.id)}, + scaled_reference_quantity=decimal.Decimal("0.0001"), + ) + ) + assert mapped is second_candidate + assert second_candidate.order_id == str(reference_order.id) + exchange_if.orders.cancel_order.assert_awaited_once_with(first_candidate) + assert any("Ambiguous unmapped open order match" in record.message for record in caplog.records) + + +class TestCaseR9LateFillAdjustedInvariant: + """ + R9 — Late-fill-adjusted count invariant. + + Trigger: reference open count exceeds copier opens by late-fill candidate count. + Expected: no open-limit count mismatch error log. + """ + + def test_no_error_when_count_matches_after_late_fill_adjustment(self, caplog): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-late", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + # Copier has zero opens; one late-fill candidate → expected 0. + synchronizer, _exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[], + ) + with mock.patch.object( + synchronizer, + "_late_reference_fill_candidate_orders", + return_value=[reference_order], + ): + with caplog.at_level(logging.ERROR): + synchronizer._check_open_limit_order_count_invariant([reference_order]) + assert not any( + "Open limit order count mismatch" in record.message + for record in caplog.records + ) + + +class TestCaseR10InvariantDoesNotReconcile: + """ + R10 — Invariant logs only; does not reconcile. + + Trigger: actual open count differs from adjusted expected with no late fills. + Expected: error log; _reconcile_open_orders_with_reference not called. + """ + + def test_mismatch_logs_without_reconcile(self, caplog): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.SELL, + price=decimal.Decimal("66326.7"), + ) + stray = _open_limit_order_stub( + order_id="extra", + exchange_order_id="EXTRA", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.0002"), + price=decimal.Decimal("66326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + matched = _open_limit_order_stub( + order_id="ref-1", + exchange_order_id="MATCHED", + side=trading_enums.TradeOrderSide.SELL, + quantity=decimal.Decimal("0.00027"), + price=decimal.Decimal("66326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[matched, stray], + ) + reconcile_spy = mock.AsyncMock(return_value=0) + with mock.patch.object( + synchronizer, + "_reconcile_open_orders_with_reference", + reconcile_spy, + ), mock.patch.object( + synchronizer, + "_late_reference_fill_candidate_orders", + return_value=[], + ): + with caplog.at_level(logging.ERROR): + synchronizer._check_open_limit_order_count_invariant([reference_order]) + reconcile_spy.assert_not_called() + assert any( + "Open limit order count mismatch" in record.message + for record in caplog.records + ) + + +class TestCaseR11ForceAbortFullReconcile: + """ + R11 — Force-abort / public reconcile uses full mode. + + Trigger: same-price duplicate with stray_only_symbols set, or public reconcile entry. + Expected: force-abort ignores stray_only and cancels duplicate; public reconcile cancels duplicate. + """ + + def test_force_abort_ignores_stray_only_symbols(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + preferred = _open_limit_order_stub( + order_id="ref-1", + exchange_order_id="PREF", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + duplicate = _open_limit_order_stub( + order_id="dup", + exchange_order_id="DUP", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.00009"), + price=decimal.Decimal("59326.7"), + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[preferred, duplicate], + ) + synchronizer.abort_mirrored_orphan_grace() + cancelled_count = asyncio.run( + synchronizer._reconcile_open_orders_with_reference( + [reference_order], + stray_only_symbols={"BTC/USDC"}, + ) + ) + assert cancelled_count == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(duplicate) + + def test_public_reconcile_cancels_duplicate(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + preferred = _open_limit_order_stub( + order_id="ref-1", + exchange_order_id="PREF", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + duplicate = _open_limit_order_stub( + order_id="dup", + exchange_order_id="DUP", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.00009"), + price=decimal.Decimal("59326.7"), + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[preferred, duplicate], + ) + cancelled_count = asyncio.run(synchronizer.reconcile_open_orders_with_reference()) + assert cancelled_count == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(duplicate) + + +class TestCaseR12GraceSymbolWrongPrice: + """ + R12 — Grace symbol + wrong-price stray. + + Trigger: synchronize with grace skip symbols set and an off-grid open. + Expected: cancel the stray; skip upserts on the grace symbol. + """ + + def test_synchronize_cancels_stray_while_grace_skips_upserts(self, caplog): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-late", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + stray = _open_limit_order_stub( + order_id="wrong", + exchange_order_id="OAEOCK", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("50000"), + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[stray], + ) + open_orders = [stray] + + def get_open_orders(symbol=None): + if symbol is None or symbol == "BTC/USDC": + return list(open_orders) + return [] + + exchange_if.orders.get_open_orders = mock.Mock(side_effect=get_open_orders) + + async def cancel_side_effect(order): + if order in open_orders: + open_orders.remove(order) + + exchange_if.orders.cancel_order = mock.AsyncMock(side_effect=cancel_side_effect) + with mock.patch.object( + synchronizer, + "_reference_symbols_skipped_while_grace_orphans_uncancelled", + return_value={"BTC/USDC"}, + ), mock.patch.object( + synchronizer, + "_maybe_bypass_grace_for_missing_mirrored_reference_orders", + side_effect=lambda replicable, skip: skip, + ), mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ): + with caplog.at_level(logging.INFO): + created = asyncio.run(synchronizer.synchronize()) + assert created == [] + exchange_if.orders.cancel_order.assert_awaited_once_with(stray) + assert any( + "Skipped reference mirror upsert" in record.message + for record in caplog.records + ) + + +class TestCaseR13GraceSymbolDuplicate: + """ + R13 — Grace symbol + duplicate at valid price. + + Trigger: two opens at the same reference price; stray_only then force-abort. + Expected: keep both under stray_only; cancel duplicate after abort (full mode). + """ + + def test_keeps_duplicate_during_grace_cancels_after_abort(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + preferred = _open_limit_order_stub( + order_id="ref-1", + exchange_order_id="PREF", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + duplicate = _open_limit_order_stub( + order_id="dup", + exchange_order_id="DUP", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.00009"), + price=decimal.Decimal("59326.7"), + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[preferred, duplicate], + ) + cancelled_during_grace = asyncio.run( + synchronizer._reconcile_open_orders_with_reference( + [reference_order], + stray_only_symbols={"BTC/USDC"}, + ) + ) + assert cancelled_during_grace == 0 + exchange_if.orders.cancel_order.assert_not_called() + synchronizer.abort_mirrored_orphan_grace() + cancelled_after_abort = asyncio.run( + synchronizer._reconcile_open_orders_with_reference( + [reference_order], + stray_only_symbols={"BTC/USDC"}, + ) + ) + assert cancelled_after_abort == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(duplicate) + + +class TestCaseR14InvariantAfterWait: + """ + R14 — Count invariant after wait. + + Trigger: synchronize creates an order while auto-sync is off (wait path used). + Expected: wait_for_orders_to_open runs before _check_open_limit_order_count_invariant; + when wait leaves open count matching reference, no mismatch ERROR. + """ + + def test_invariant_after_wait_with_matching_count_logs_no_error(self, caplog): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + created_open = _open_limit_order_stub( + order_id="ref-1", + exchange_order_id="CREATED", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("59326.7"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=[], + ) + exchange_if.orders.automatically_synchronize_orders = mock.Mock(return_value=False) + call_order: list[str] = [] + + async def wait_side_effect(orders, symbol): + call_order.append("wait") + # Promote: after wait, local open book matches reference count. + exchange_if.orders.get_open_orders = mock.Mock(return_value=[created_open]) + + def invariant_side_effect(replicable): + call_order.append("invariant") + orders_synchronizer_module.OrdersSynchronizer._check_open_limit_order_count_invariant( + synchronizer, + replicable, + ) + + exchange_if.orders.wait_for_orders_to_open = mock.AsyncMock(side_effect=wait_side_effect) + with mock.patch.object( + synchronizer, + "cancel_orders_pending_synchronization", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_upsert_mirrored_reference_order", + mock.AsyncMock(return_value=([created_open], 0, 0, None)), + ), mock.patch.object( + synchronizer, + "_reconcile_open_orders_with_reference", + mock.AsyncMock(return_value=0), + ), mock.patch.object( + synchronizer, + "_check_open_limit_order_count_invariant", + side_effect=invariant_side_effect, + ): + with caplog.at_level(logging.ERROR): + asyncio.run(synchronizer.synchronize()) + assert call_order == ["wait", "invariant"] + exchange_if.orders.wait_for_orders_to_open.assert_awaited_once() + assert not any( + "Open limit order count mismatch" in record.message + for record in caplog.records + ) + + +class TestCaseR15GraceKeepsMirroredOrphanStray: + """ + R15 — Grace keeps mirrored orphan stray; cancels untagged. + + Trigger: stray_only_symbols set; mirrored orphan at off-grid price + untagged wrong-price open. + Expected: cancel only the untagged open; keep the mirrored orphan. Force-abort cancels both. + """ + + def test_stray_only_keeps_mirrored_orphan_cancels_untagged(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + mirrored_orphan = _open_limit_order_stub( + order_id="grid_ref_b1", + exchange_order_id="ORPHAN", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("50000"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + untagged_stray = _open_limit_order_stub( + order_id="wrong", + exchange_order_id="OAEOCK", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("45000"), + ) + open_orders = [mirrored_orphan, untagged_stray] + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=open_orders, + ) + + def get_open_orders(symbol=None): + if symbol is None or symbol == "BTC/USDC": + return list(open_orders) + return [] + + exchange_if.orders.get_open_orders = mock.Mock(side_effect=get_open_orders) + + async def cancel_side_effect(order): + if order in open_orders: + open_orders.remove(order) + + exchange_if.orders.cancel_order = mock.AsyncMock(side_effect=cancel_side_effect) + cancelled_count = asyncio.run( + synchronizer._reconcile_open_orders_with_reference( + [reference_order], + stray_only_symbols={"BTC/USDC"}, + ) + ) + assert cancelled_count == 1 + exchange_if.orders.cancel_order.assert_awaited_once_with(untagged_stray) + assert mirrored_orphan in open_orders + assert untagged_stray not in open_orders + + def test_force_abort_cancels_mirrored_orphan_and_untagged(self): + reference_order = _btc_usdc_limit_reference_order( + order_id="ref-1", + side=protocol_models.Side.BUY, + price=decimal.Decimal("59326.7"), + ) + mirrored_orphan = _open_limit_order_stub( + order_id="grid_ref_b1", + exchange_order_id="ORPHAN", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("50000"), + tag=copy_constants.MIRRORED_ORDER_TAG, + ) + untagged_stray = _open_limit_order_stub( + order_id="wrong", + exchange_order_id="OAEOCK", + side=trading_enums.TradeOrderSide.BUY, + quantity=decimal.Decimal("0.0001"), + price=decimal.Decimal("45000"), + ) + open_orders = [mirrored_orphan, untagged_stray] + synchronizer, exchange_if = _synchronizer_with_open_orders( + reference_orders=[reference_order], + open_orders=open_orders, + ) + + def get_open_orders(symbol=None): + if symbol is None or symbol == "BTC/USDC": + return list(open_orders) + return [] + + exchange_if.orders.get_open_orders = mock.Mock(side_effect=get_open_orders) + + async def cancel_side_effect(order): + if order in open_orders: + open_orders.remove(order) + + exchange_if.orders.cancel_order = mock.AsyncMock(side_effect=cancel_side_effect) + synchronizer.abort_mirrored_orphan_grace() + cancelled_count = asyncio.run( + synchronizer._reconcile_open_orders_with_reference( + [reference_order], + stray_only_symbols={"BTC/USDC"}, + ) + ) + assert cancelled_count == 2 + assert exchange_if.orders.cancel_order.await_count == 2 + assert open_orders == [] diff --git a/packages/evaluators/octobot_evaluators/evaluators/abstract_evaluator.py b/packages/evaluators/octobot_evaluators/evaluators/abstract_evaluator.py index b7c8587d0f..a028bacc3e 100644 --- a/packages/evaluators/octobot_evaluators/evaluators/abstract_evaluator.py +++ b/packages/evaluators/octobot_evaluators/evaluators/abstract_evaluator.py @@ -528,6 +528,8 @@ def is_enabled(cls, tentacles_setup_config, default) -> bool: :param default: default value if evaluator config is not found :return: evaluator config """ + if tentacles_setup_config is None: + return default try: return api.is_tentacle_activated_in_tentacles_setup_config(tentacles_setup_config, cls.get_name(), diff --git a/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py b/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py index 32ebb10679..e69b924ada 100644 --- a/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py +++ b/packages/evaluators/octobot_evaluators/evaluators/evaluator_dsl_factory.py @@ -62,7 +62,6 @@ def _create_operator_parameters_from_user_inputs( if isinstance(user_input.input_type, str) else user_input.input_type.value ) - param_type = user_inputs.USER_INPUT_TYPE_TO_PYTHON_TYPE[input_type] param_name = user_inputs.sanitize_user_input_name(user_input.name) description = user_input.title or user_input.name params.append( @@ -70,22 +69,12 @@ def _create_operator_parameters_from_user_inputs( name=param_name, description=str(description), required=False, - type=param_type, - default=user_input.def_val, - ) + type=dsl_interpreter.dsl_value_type_for_user_input(input_type), + default=user_input.def_val) ) return params -class _LocalTentaclesSetupConfig: - def is_tentacle_activated(self, _klass_name) -> bool: - return True - - -def _get_local_tentacles_setup_config(): - return _LocalTentaclesSetupConfig() - - async def _ensure_dsl_bot_storage_registered() -> None: # disable evaluator storage operations for DSL calls await commons_databases.init_bot_storage( @@ -102,10 +91,8 @@ def _create_evaluator_operator_parameters( evaluator_class: type, config: dict, ) -> list[dsl_interpreter.OperatorParameter]: - tentacles_setup_config = _get_local_tentacles_setup_config() - loaded_config = {} tentacle_instance = evaluator_class.create_local_instance( - config, tentacles_setup_config, loaded_config + config, None, {} ) created_user_inputs = {} tentacle_instance.init_user_inputs(created_user_inputs) @@ -134,6 +121,8 @@ class EvaluatorOperator( dsl_interpreter.ReCallableOperatorMixin, dsl_interpreter.DynamicDependenciesOperatorMixin, ): + CATEGORY = common_enums.DslKeywordCategory.SOURCE.value + def __init__( self, *parameters: dsl_interpreter.OperatorParameterType, @@ -145,6 +134,13 @@ def __init__( def get_library() -> str: return common_constants.CONTEXTUAL_OPERATORS_LIBRARY + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + common_enums.DslValueType.SIGNAL.value, + description="Evaluator result", + ) + def get_exchange_manager( self, ) -> typing.Optional["octobot_trading.exchanges.ExchangeManager"]: @@ -214,7 +210,6 @@ async def _create_evaluator_instance( await _ensure_dsl_bot_storage_registered() evaluator_instance = await evaluator_factory.create_dsl_evaluator( evaluator_class, - _get_local_tentacles_setup_config(), matrix_id, exchange_manager.exchange_name, tentacle_config, @@ -236,7 +231,7 @@ def _init_strategy_time_frames( ) -> None: evaluator_instance.strategy_time_frames = evaluators.StrategyEvaluator.get_required_time_frames( self.get_config(), - _get_local_tentacles_setup_config(), + None, strategy_config=tentacle_config, ) @@ -668,9 +663,8 @@ def get_evaluator_meta_parameters(cls) -> list[dsl_interpreter.OperatorParameter name=TIME_FRAMES_PARAM, description="Evaluated time frames", required=False, - type=list, - default=None, - ), + type=common_enums.DslValueType.ANY.value, + default=None), ] if not issubclass(evaluator_class, evaluators.StrategyEvaluator): meta_parameters.extend([ @@ -678,16 +672,14 @@ def get_evaluator_meta_parameters(cls) -> list[dsl_interpreter.OperatorParameter name=SYMBOLS_PARAM, description="Evaluated trading symbols", required=False, - type=list, - default=None, - ), + type=common_enums.DslValueType.ANY.value, + default=None), dsl_interpreter.OperatorParameter( name=CRYPTOCURRENCY_PARAM, description="Evaluated cryptocurrency", required=False, - type=str, - default=None, - ), + type=common_enums.DslValueType.TEXT.value, + default=None), ]) if issubclass(evaluator_class, evaluators.TAEvaluator): meta_parameters.append( @@ -695,9 +687,8 @@ def get_evaluator_meta_parameters(cls) -> list[dsl_interpreter.OperatorParameter name=INCLUDE_IN_CONSTRUCTION_CANDLE_PARAM, description="Include the in-construction candle when executing the TA evaluator", required=False, - type=bool, - default=False, - ) + type=common_enums.DslValueType.BOOLEAN.value, + default=False) ) return meta_parameters diff --git a/packages/evaluators/octobot_evaluators/evaluators/evaluator_factory.py b/packages/evaluators/octobot_evaluators/evaluators/evaluator_factory.py index 6efa886382..91ce2ee31c 100644 --- a/packages/evaluators/octobot_evaluators/evaluators/evaluator_factory.py +++ b/packages/evaluators/octobot_evaluators/evaluators/evaluator_factory.py @@ -129,7 +129,6 @@ def _parse_time_frame(time_frame): async def create_dsl_evaluator( evaluator_class, - tentacles_setup_config: object, matrix_id: str, exchange_name: str, evaluator_configuration: dict, @@ -138,11 +137,12 @@ async def create_dsl_evaluator( time_frame: typing.Optional[str] = None, cryptocurrency: typing.Optional[str] = None, ): + # DSL-referenced evaluators are always enabled; tentacles_setup_config activation does not apply. parsed_time_frame = _parse_time_frame(time_frame) time_frames = [parsed_time_frame] if parsed_time_frame is not None else None return await create_evaluator( evaluator_class, - tentacles_setup_config, + None, bot_id=bot_id, matrix_id=matrix_id, exchange_name=exchange_name, @@ -155,6 +155,7 @@ async def create_dsl_evaluator( ), time_frames=time_frames, evaluator_configuration=evaluator_configuration, + force_enabled=True, ) @@ -172,10 +173,13 @@ async def create_evaluator( all_symbols_by_crypto_currencies=None, time_frames=None, real_time_time_frames=None, - evaluator_configuration=None + evaluator_configuration=None, + force_enabled: bool = False, ): try: eval_class_instance = _instantiate_evaluator(evaluator_class, tentacles_setup_config, True) + if force_enabled: + eval_class_instance.enabled = True if api.is_relevant_evaluator(eval_class_instance, relevant_evaluators): eval_class_instance.matrix_id = matrix_id eval_class_instance.exchange_name = exchange_name if exchange_name else None diff --git a/packages/evaluators/tests/evaluators/test_evaluator_dsl_factory.py b/packages/evaluators/tests/evaluators/test_evaluator_dsl_factory.py index 3f22c8260f..66af41d434 100644 --- a/packages/evaluators/tests/evaluators/test_evaluator_dsl_factory.py +++ b/packages/evaluators/tests/evaluators/test_evaluator_dsl_factory.py @@ -186,8 +186,12 @@ def test_parameters_include_user_inputs_and_meta(self): } assert evaluator_dsl_factory.SYMBOLS_PARAM in parameters_by_name assert evaluator_dsl_factory.TIME_FRAMES_PARAM in parameters_by_name - assert parameters_by_name[evaluator_dsl_factory.SYMBOLS_PARAM].type is list - assert parameters_by_name[evaluator_dsl_factory.TIME_FRAMES_PARAM].type is list + assert parameters_by_name[evaluator_dsl_factory.SYMBOLS_PARAM].type == ( + common_enums.DslValueType.ANY.value + ) + assert parameters_by_name[evaluator_dsl_factory.TIME_FRAMES_PARAM].type == ( + common_enums.DslValueType.ANY.value + ) assert evaluator_dsl_factory.INCLUDE_IN_CONSTRUCTION_CANDLE_PARAM in parameter_names def test_strategy_operator_meta_parameters_only_time_frames(self): @@ -202,7 +206,9 @@ def test_strategy_operator_meta_parameters_only_time_frames(self): for parameter in OpCls.get_evaluator_meta_parameters() } assert set(meta_parameters_by_name) == {evaluator_dsl_factory.TIME_FRAMES_PARAM} - assert meta_parameters_by_name[evaluator_dsl_factory.TIME_FRAMES_PARAM].type is list + assert meta_parameters_by_name[evaluator_dsl_factory.TIME_FRAMES_PARAM].type == ( + common_enums.DslValueType.ANY.value + ) class TestGetDependencies: @@ -571,7 +577,7 @@ async def test_calls_matrix_callback_once_with_seeded_matrix(self, matrix_id): ) operator = OpCls() strategy_instance = fake_evaluators.FakeMatrixReadingStrategyEvaluator( - evaluator_dsl_factory._get_local_tentacles_setup_config() + None ) strategy_instance.matrix_id = matrix_id strategy_instance.matrix_callback = mock.AsyncMock( @@ -626,7 +632,7 @@ async def test_executes_one_strategy_per_dynamic_dependency_symbol(self, matrix_ async def _create_strategy_instance(*_args, **_kwargs): strategy_instance = fake_evaluators.FakeMatrixReadingStrategyEvaluator( - evaluator_dsl_factory._get_local_tentacles_setup_config() + None ) strategy_instance.matrix_id = matrix_id strategy_instance.evaluator_type = evaluators_enums.EvaluatorMatrixTypes.STRATEGIES diff --git a/packages/evaluators/tests/evaluators/test_evaluator_factory.py b/packages/evaluators/tests/evaluators/test_evaluator_factory.py index 9fcbd724e6..e213399ad3 100644 --- a/packages/evaluators/tests/evaluators/test_evaluator_factory.py +++ b/packages/evaluators/tests/evaluators/test_evaluator_factory.py @@ -271,3 +271,35 @@ def get_exchange_manager_from_exchange_name_and_id(self, *args): def get_base_currency(self, exchange_manager, symbol): return symbol_util.parse_symbol(symbol).base + + +@pytest.mark.usefixtures("event_loop", "install_tentacles") +class TestCreateDslEvaluator: + async def test_returns_enabled_instance_without_tentacles_setup_config( + self, evaluators_and_matrix_channels + ): + import tentacles + with ( + mock.patch.object( + tentacles.RSIMomentumEvaluator, + "initialize", + mock.AsyncMock(), + ), + mock.patch.object( + tentacles.RSIMomentumEvaluator, + "prepare", + mock.AsyncMock(), + ), + ): + evaluator_instance = await evaluator_factory.create_dsl_evaluator( + tentacles.RSIMomentumEvaluator, + matrix_id=evaluators_and_matrix_channels, + exchange_name=exchange_name, + evaluator_configuration={}, + bot_id=bot_id, + symbol="BTC/USDT", + time_frame=enums.TimeFrames.ONE_HOUR.value, + cryptocurrency="BTC", + ) + assert evaluator_instance is not None + assert evaluator_instance.enabled is True diff --git a/packages/evaluators/tests/evaluators/test_evaluator_factory_create_evaluators.py b/packages/evaluators/tests/evaluators/test_evaluator_factory_create_evaluators.py index 32c0d8c96e..fef91da740 100644 --- a/packages/evaluators/tests/evaluators/test_evaluator_factory_create_evaluators.py +++ b/packages/evaluators/tests/evaluators/test_evaluator_factory_create_evaluators.py @@ -206,7 +206,8 @@ async def _mocked_create_evaluator(evaluator_class, all_symbols_by_crypto_currencies=None, time_frames=None, real_time_time_frames=None, - evaluator_configuration=None): + evaluator_configuration=None, + force_enabled: bool = False): return evaluator_class(cryptocurrency, cryptocurrency_name, symbol, time_frame, all_symbols_by_crypto_currencies) diff --git a/packages/evaluators/tests/functional/test_dsl_evaluators_execution.py b/packages/evaluators/tests/functional/test_dsl_evaluators_execution.py index 95fd5df8f9..482b1abadd 100644 --- a/packages/evaluators/tests/functional/test_dsl_evaluators_execution.py +++ b/packages/evaluators/tests/functional/test_dsl_evaluators_execution.py @@ -187,7 +187,7 @@ async def test_strategy_reads_dummy_evaluator_values_from_matrix(self, matrix_id async def _create_strategy_instance(*_args, **_kwargs): strategy_instance = fake_evaluators.FakeMatrixReadingStrategyEvaluator( - evaluator_dsl_factory._get_local_tentacles_setup_config() + None ) strategy_instance.matrix_id = matrix_id strategy_instance.evaluator_type = evaluators_enums.EvaluatorMatrixTypes.STRATEGIES diff --git a/packages/evaluators/tests/static/fake_evaluators.py b/packages/evaluators/tests/static/fake_evaluators.py index 1591fac775..d318f4f82b 100644 --- a/packages/evaluators/tests/static/fake_evaluators.py +++ b/packages/evaluators/tests/static/fake_evaluators.py @@ -58,9 +58,6 @@ def get_name(cls) -> str: @classmethod def create_local_instance(cls, config, tentacles_setup_config, loaded_config): - if tentacles_setup_config is None: - import octobot_evaluators.evaluators.evaluator_dsl_factory as evaluator_dsl_factory - tentacles_setup_config = evaluator_dsl_factory._get_local_tentacles_setup_config() return evaluator_factory.create_temporary_evaluator_with_local_config( cls, tentacles_setup_config, loaded_config, False ) @@ -105,9 +102,6 @@ def get_name(cls) -> str: @classmethod def create_local_instance(cls, config, tentacles_setup_config, loaded_config): - if tentacles_setup_config is None: - import octobot_evaluators.evaluators.evaluator_dsl_factory as evaluator_dsl_factory - tentacles_setup_config = evaluator_dsl_factory._get_local_tentacles_setup_config() return evaluator_factory.create_temporary_evaluator_with_local_config( cls, tentacles_setup_config, loaded_config, False ) @@ -130,9 +124,6 @@ def get_name(cls) -> str: @classmethod def create_local_instance(cls, config, tentacles_setup_config, loaded_config): - if tentacles_setup_config is None: - import octobot_evaluators.evaluators.evaluator_dsl_factory as evaluator_dsl_factory - tentacles_setup_config = evaluator_dsl_factory._get_local_tentacles_setup_config() return evaluator_factory.create_temporary_evaluator_with_local_config( cls, tentacles_setup_config, loaded_config, False ) diff --git a/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py b/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py index fb2afecd41..5a966dfa78 100644 --- a/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py +++ b/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py @@ -138,6 +138,21 @@ def sync_orders_from_exchange_manager(self, exchange_manager: octobot_trading.ex updated_open_orders.append( octobot_trading.storage.orders_storage._format_order(order, exchange_manager) ) + for order in octobot_trading.api.get_pending_creation_orders(exchange_manager): + if not order.exchange_order_id: + continue + if order.is_self_managed(): + octobot_commons.logging.get_logger(self.__class__.__name__).error( + f"Self managed order created. This type of [{exchange_manager.exchange_name}] " + f"order is not supported, order is ignored. Order: {order}" + ) + continue + if order.exchange_order_id in updated_open_orders_exchange_ids: + continue + updated_open_orders_exchange_ids.add(order.exchange_order_id) + updated_open_orders.append( + octobot_trading.storage.orders_storage._format_order(order, exchange_manager) + ) updated_missing_orders = [ order for exchange_id, order in octobot_trading.personal_data.get_enriched_orders_by_exchange_id(previous_orders.open_orders).items() diff --git a/packages/flow/octobot_flow/entities/automations/__init__.py b/packages/flow/octobot_flow/entities/automations/__init__.py index 8d1b60bf57..aeb5c25802 100644 --- a/packages/flow/octobot_flow/entities/automations/__init__.py +++ b/packages/flow/octobot_flow/entities/automations/__init__.py @@ -23,6 +23,13 @@ NextIterationDetails, PostIterationActionsDetails, ) +from octobot_flow.entities.automations.octobot_process_state import ( + OctobotProcessState, + is_run_octobot_process_dsl_action, + parse_octobot_process_state, + recall_inner_from_action_result, +) + __all__ = [ "FetchedExchangeAccountElements", "FetchedExchangePublicData", @@ -39,4 +46,8 @@ "RefreshExchangeBotsAuthenticatedDataDetails", "NextIterationDetails", "PostIterationActionsDetails", + "OctobotProcessState", + "is_run_octobot_process_dsl_action", + "parse_octobot_process_state", + "recall_inner_from_action_result", ] diff --git a/packages/flow/octobot_flow/entities/automations/execution_details.py b/packages/flow/octobot_flow/entities/automations/execution_details.py index 06f5ded478..4c1cdc3f29 100644 --- a/packages/flow/octobot_flow/entities/automations/execution_details.py +++ b/packages/flow/octobot_flow/entities/automations/execution_details.py @@ -37,6 +37,7 @@ def was_interrupted(self) -> bool: @dataclasses.dataclass class DegradedStateDetails(octobot_commons.dataclasses.MinimizableDataclass): since: float = 0 + error: typing.Optional[str] = None reason: typing.Optional[str] = None @@ -72,3 +73,4 @@ def complete_execution(self, next_execution_scheduled_to: float): trigger_reason=octobot_flow.enums.LastTriggerReason.SCHEDULED.value, additional_actions=additional_actions_import.AdditionalActions.default_iteration(), ) + self.degraded_state = DegradedStateDetails() diff --git a/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py b/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py index 3559ee477d..6b96267559 100644 --- a/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py +++ b/packages/flow/octobot_flow/entities/automations/fetched_dependencies.py @@ -10,3 +10,4 @@ class FetchedDependencies(octobot_commons.dataclasses.MinimizableDataclass): fetched_exchange_data: typing.Optional[fetched_exchange_data_import.FetchedExchangeData] = None fetched_copy_trading_data: typing.Optional[fetched_copy_trading_data_import.FetchedCopyTradingData] = None + skip_exchange: bool = False diff --git a/packages/flow/octobot_flow/entities/automations/octobot_process_state.py b/packages/flow/octobot_flow/entities/automations/octobot_process_state.py new file mode 100644 index 0000000000..6bdce57c27 --- /dev/null +++ b/packages/flow/octobot_flow/entities/automations/octobot_process_state.py @@ -0,0 +1,63 @@ +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +import typing + +import pydantic + +import octobot_commons.dsl_interpreter as dsl_interpreter +import octobot_flow.entities.actions.action_details as action_details_module + +class OctobotProcessState(pydantic.BaseModel): + """Master-side recall payload for run_octobot_process (former EnsureOctobotProcessState).""" + + model_config = pydantic.ConfigDict(validate_assignment=True, extra="ignore") + + http_base_url: str + web_port: int + node_port: int + user_root: str + user_folder: str + log_folder: str + profile_id: str | None + pid: int # Last known child PID on the master; may lag after a child self-restart until adoption. + state_file_path: str = "" + started_waiting_at: float = 0.0 # Wall-clock when the first spawn began; used only while `init_state_ok` is False (`ping_timeout`). + init_state_ok: bool = False # True once the child reached confirmed-alive; switches from init `ping_timeout` to recall/grace rules. + executor_id: str # Required scheduler executor id at emit time; compared on recall to detect worker restart. + +def parse_octobot_process_state(raw: dict) -> OctobotProcessState | None: + """Parse recall inner dict; empty or invalid dict → None.""" + if not raw: + return None + try: + return OctobotProcessState.model_validate(raw) + except pydantic.ValidationError: + return None + +def _run_octobot_process_operator_name() -> str: + import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops + + return octobot_process_ops.RUN_OCTOBOT_PROCESS_OPERATOR_NAME + +def is_run_octobot_process_dsl_action( + flow_action: action_details_module.DSLScriptActionDetails, +) -> bool: + """True when the action DSL script invokes the run_octobot_process operator.""" + dsl_value = flow_action.resolved_dsl_script or flow_action.dsl_script + if not dsl_value: + return False + return dsl_value.strip().startswith(f"{_run_octobot_process_operator_name()}(") + +def recall_inner_from_action_result( + action_result: typing.Any, +) -> dict | None: + """Unwrap ReCallingOperatorResult.last_execution_result from a DAG action result.""" + if not isinstance(action_result, dict): + return None + if dsl_interpreter.ReCallingOperatorResult.is_re_calling_operator_result(action_result): + recall_wrapper = dsl_interpreter.ReCallingOperatorResult.from_dict( + action_result[dsl_interpreter.ReCallingOperatorResult.__name__] + ) + inner_last = recall_wrapper.last_execution_result + return inner_last if isinstance(inner_last, dict) else None + return action_result if action_result else None diff --git a/packages/flow/octobot_flow/errors.py b/packages/flow/octobot_flow/errors.py index 74efbf348f..708d93f470 100644 --- a/packages/flow/octobot_flow/errors.py +++ b/packages/flow/octobot_flow/errors.py @@ -90,3 +90,11 @@ class ActionDependencyNotFoundError(ActionDependencyError): class MissingDSLExecutorDependencyError(DSLExecutorError): """raise when a DSL executor dependency is missing""" + + +class PriorityActionError(AutomationActionError): + """raise when a priority action error occurs""" + + +class PendingPriorityActionsSkippedError(PriorityActionError): + """raise when supplied priority actions were not executed and DAG actions would run instead""" diff --git a/packages/flow/octobot_flow/jobs/automation_job.py b/packages/flow/octobot_flow/jobs/automation_job.py index 1fa3f381df..93c7d3e2ad 100644 --- a/packages/flow/octobot_flow/jobs/automation_job.py +++ b/packages/flow/octobot_flow/jobs/automation_job.py @@ -40,6 +40,9 @@ def __init__( self.automation_state: octobot_flow.entities.AutomationState = ( octobot_flow.entities.AutomationState.from_dict(automation_state) ) + self._added_priority_actions: list[octobot_flow.entities.AbstractActionDetails] = ( + list(added_priority_actions) if added_priority_actions else [] + ) if added_priority_actions: # Include added priority actions in the automation state. # All pending priority actions will be executed before any other actions. @@ -75,6 +78,15 @@ async def run(self) -> list[octobot_flow.entities.AbstractActionDetails]: ) with octobot_flow.encryption.decrypted_bots_configurations(self.automation_state): to_execute_actions, are_priority_actions = self._get_actions_to_execute() + if self._added_priority_actions and not are_priority_actions: + added_action_ids = [action.id for action in self._added_priority_actions] + self._logger.error( + f"Skipped {len(self._added_priority_actions)} supplied priority action(s) " + f"because they are already completed or duplicate in state: {added_action_ids}" + ) + raise octobot_flow.errors.PendingPriorityActionsSkippedError( + f"Supplied priority actions were not pending and were skipped: {added_action_ids}" + ) if are_priority_actions: self._logger.info(f"Running {len(to_execute_actions)} priority actions: {to_execute_actions}") self._resolve_dsl_scripts(to_execute_actions, True) @@ -202,13 +214,29 @@ async def _fetch_actions( self.fetched_actions.extend(all_actions) def _requires_initialization_run(self) -> bool: - return ( - self.automation_state.automation.execution.previous_execution.triggered_at == 0 - and ( - not self.automation_state.exchange_account_details - or not self.automation_state.exchange_account_details.exchange_details.internal_name - ) - ) + if self.automation_state.automation.execution.previous_execution.triggered_at != 0: + return False + if self.automation_state.has_exchange(): + return False + if self._has_pending_apply_configuration_actions(): + return True + if octobot_flow.logic.dsl.dag_has_only_process_bound_dsl_actions( + self.automation_state.automation.actions_dag.actions, + exchange_account_details=self.automation_state.exchange_account_details, + automation_id=self.automation_state.automation.metadata.automation_id, + ): + return False + return True + + def _has_pending_apply_configuration_actions(self) -> bool: + for action in self.automation_state.automation.actions_dag.actions: + if ( + isinstance(action, octobot_flow.entities.ConfiguredActionDetails) + and action.action == octobot_flow.enums.ActionType.APPLY_CONFIGURATION.value + and action.executed_at is None + ): + return True + return False async def _fetch_dependencies( self, @@ -232,6 +260,7 @@ async def _fetch_dependencies( return octobot_flow.entities.FetchedDependencies( fetched_exchange_data=None, fetched_copy_trading_data=None, + skip_exchange=True, ) if fetched_copy_trading_data := await self._init_all_required_copy_trading_data( maybe_community_repository, to_execute_actions, minimal_profile_data, @@ -360,10 +389,14 @@ async def _execute_automation_actions( default_next_execution_scheduled_to ) automation = self.automation_state.automation - exchange_account_desc = ( - 'simulated exchange account' if self.automation_state.exchange_account_details.is_simulated() - else 'real exchange account' - ) + if self.automation_state.has_exchange(): + exchange_account_desc = ( + 'simulated exchange account' + if self.automation_state.exchange_account_details.is_simulated() + else 'real exchange account' + ) + else: + exchange_account_desc = 'process-bound automation' automation_signature = f"{exchange_account_desc} automation {automation.metadata.automation_id}" try: self._logger.info(f"Updating {automation_signature}") diff --git a/packages/flow/octobot_flow/jobs/automation_runner_job.py b/packages/flow/octobot_flow/jobs/automation_runner_job.py index 8c7f48ddae..4eb378875b 100644 --- a/packages/flow/octobot_flow/jobs/automation_runner_job.py +++ b/packages/flow/octobot_flow/jobs/automation_runner_job.py @@ -180,7 +180,10 @@ async def actions_context( raise octobot_flow.errors.AutomationValidationError( f"A bot_id is required to run a bot. Found: {self.profile_data_provider.get_profile_data().profile_details.bot_id}" ) - async with self.exchange_manager_context(): + if self.fetched_dependencies.skip_exchange: yield self + else: + async with self.exchange_manager_context(): + yield self finally: self._to_execute_actions = None # type: ignore diff --git a/packages/flow/octobot_flow/logic/actions/actions_executor.py b/packages/flow/octobot_flow/logic/actions/actions_executor.py index fa4c15a056..f70e1a0dd9 100644 --- a/packages/flow/octobot_flow/logic/actions/actions_executor.py +++ b/packages/flow/octobot_flow/logic/actions/actions_executor.py @@ -349,7 +349,7 @@ def _sync_after_execution( ): if synchronized_exchange_account_elements: self._get_logger().info( - f"Exchange account elements are being updated from {len(synchronized_exchange_account_elements)}" + f"Exchange account elements are being updated from {len(synchronized_exchange_account_elements)} " f"synchronized exchange account elements on {[s.name for s in synchronized_exchange_account_elements]}" f"returned by actions; this iteration does not apply sync_from_exchange_manager from the " f"local exchange_manager.", diff --git a/packages/flow/octobot_flow/logic/dsl/__init__.py b/packages/flow/octobot_flow/logic/dsl/__init__.py index 6323e931a0..369602978f 100644 --- a/packages/flow/octobot_flow/logic/dsl/__init__.py +++ b/packages/flow/octobot_flow/logic/dsl/__init__.py @@ -5,12 +5,16 @@ ) from octobot_flow.logic.dsl.dsl_actions_util import ( are_all_actions_process_bound_only, + dag_has_only_process_bound_dsl_actions, + is_recallable_dsl_action, ) from octobot_flow.logic.dsl.dsl_executor import DSLExecutor from octobot_flow.logic.dsl.dsl_action_execution_context import dsl_action_execution __all__ = [ "are_all_actions_process_bound_only", + "dag_has_only_process_bound_dsl_actions", + "is_recallable_dsl_action", "get_actions_symbol_dependencies", "get_actions_time_frames_dependencies", "get_copy_trading_dependencies", diff --git a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py index 28e3bff197..2a95cf6115 100644 --- a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py +++ b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py @@ -8,6 +8,17 @@ import octobot_flow.entities import octobot_flow.enums import octobot_flow.logic.dsl.action_error_util +import octobot_flow.logic.dsl.dsl_actions_util as dsl_actions_util_module +import octobot_flow.logic.dsl.dsl_executor as dsl_executor_module + + +POSTPONE_ON_RECALLABLE_TRADING_ERRORS: tuple[type[Exception], ...] = ( + octobot_trading.errors.MissingFunds, + octobot_trading.errors.FailedRequest, + octobot_trading.errors.PortfolioNegativeValueError, + octobot_trading.errors.AuthenticationError, + octobot_trading.errors.MissingMinimalExchangeTradeVolume, +) def _dsl_action_error_call_result( @@ -23,6 +34,48 @@ def _dsl_action_error_call_result( ) +def _should_postpone_recallable_trading_error( + executor: object, + action: octobot_flow.entities.DSLScriptActionDetails, +) -> bool: + + if not isinstance(executor, dsl_executor_module.DSLExecutor): + return False + return dsl_actions_util_module.is_recallable_dsl_action(executor, action) + + +def _map_non_recallable_postpone_trading_error( + action: octobot_flow.entities.DSLScriptActionDetails, + err: Exception, +) -> octobot_commons.dsl_interpreter.DSLCallResult: + if isinstance(err, octobot_trading.errors.MissingMinimalExchangeTradeVolume): + octobot_commons.logging.get_logger("action_execution").exception( + err, True, f"Missing minimal exchange trade volume error: {err}" + ) + return _dsl_action_error_call_result( + action, + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER.value, + str(err), + ) + if isinstance(err, octobot_trading.errors.AuthenticationError): + return _dsl_action_error_call_result( + action, + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value, + str(err), + ) + if isinstance(err, octobot_trading.errors.MissingFunds): + return _dsl_action_error_call_result( + action, + octobot_flow.enums.ActionErrorStatus.NOT_ENOUGH_FUNDS.value, + str(err), + ) + return _dsl_action_error_call_result( + action, + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + str(err), + ) + + def dsl_action_execution(func): async def _action_execution_error_handler_wrapper( self, action: octobot_flow.entities.DSLScriptActionDetails, **kwargs @@ -49,13 +102,6 @@ async def _action_execution_error_handler_wrapper( octobot_flow.enums.ActionErrorStatus.DISABLED_FUNDS_TRANSFER_ERROR.value, str(err), ) - except octobot_trading.errors.MissingMinimalExchangeTradeVolume as err: - octobot_commons.logging.get_logger("action_execution").exception(err, True, f"Missing minimal exchange trade volume error: {err}") - return _dsl_action_error_call_result( - action, - octobot_flow.enums.ActionErrorStatus.INVALID_ORDER.value, - str(err), - ) except (octobot_trading.errors.UnsupportedHedgeContractError, octobot_trading.errors.InvalidPositionSide) as err: return _dsl_action_error_call_result( action, @@ -88,7 +134,12 @@ async def _action_execution_error_handler_wrapper( octobot_flow.enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR.value, str(err), ) + except POSTPONE_ON_RECALLABLE_TRADING_ERRORS as err: + if _should_postpone_recallable_trading_error(self, action): + raise + return _map_non_recallable_postpone_trading_error(action, err) except Exception as err: + # swallowed errors: warning: will stop the workflow octobot_commons.logging.get_logger("action_execution").exception( err, True, diff --git a/packages/flow/octobot_flow/logic/dsl/dsl_actions_util.py b/packages/flow/octobot_flow/logic/dsl/dsl_actions_util.py index 371270a9ba..889b3be093 100644 --- a/packages/flow/octobot_flow/logic/dsl/dsl_actions_util.py +++ b/packages/flow/octobot_flow/logic/dsl/dsl_actions_util.py @@ -8,10 +8,65 @@ import octobot_commons.profiles.profile_data as profile_data_import import octobot_flow.entities -import octobot_flow.errors +import octobot_flow.enums +import octobot_flow.logic.configuration as configuration_module import octobot_flow.logic.dsl.dsl_executor as dsl_executor_module +def dag_has_only_process_bound_dsl_actions( + dag_actions: list[octobot_flow.entities.AbstractActionDetails], + *, + exchange_account_details: octobot_flow.entities.ExchangeAccountDetails | None, + automation_id: str, +) -> bool: + non_init_actions = [ + action + for action in dag_actions + if not ( + isinstance(action, octobot_flow.entities.ConfiguredActionDetails) + and action.action == octobot_flow.enums.ActionType.APPLY_CONFIGURATION.value + ) + ] + if not non_init_actions: + return False + minimal_profile_data = configuration_module.create_profile_data( + exchange_account_details, + automation_id, + set(), + ) + return are_all_actions_process_bound_only( + minimal_profile_data, + non_init_actions, + ) + + +def is_recallable_dsl_action( + dsl_executor: "dsl_executor_module.DSLExecutor", + action: octobot_flow.entities.DSLScriptActionDetails, +) -> bool: + """ + True when the action's top-level DSL operator is a ReCallableOperatorMixin + (e.g. copy_exchange_account, wait, run_octobot_process). + """ + dsl_script = action.resolved_dsl_script or action.dsl_script + if not dsl_script: + return False + try: + dsl_executor._interpreter.prepare(dsl_script) + except commons_errors.DSLInterpreterError as err: + common_logging.get_logger(__name__).info( + "Recallable check: DSL script skipped for action %s (%s): %s", + action.id, + action.dsl_script, + err, + ) + return False + top_operator = dsl_executor.get_top_operator() + if not isinstance(top_operator, dsl_interpreter_operator.Operator): + return False + return isinstance(top_operator, dsl_interpreter_import.ReCallableOperatorMixin) + + def are_all_actions_process_bound_only( profile_data: profile_data_import.ProfileData, actions: list[octobot_flow.entities.AbstractActionDetails], diff --git a/packages/flow/octobot_flow/logic/dsl/dsl_executor.py b/packages/flow/octobot_flow/logic/dsl/dsl_executor.py index f874f0bfb8..7e6d2ce6aa 100644 --- a/packages/flow/octobot_flow/logic/dsl/dsl_executor.py +++ b/packages/flow/octobot_flow/logic/dsl/dsl_executor.py @@ -5,6 +5,7 @@ import octobot_commons.signals import octobot_commons.errors import octobot_commons.profiles +import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module import octobot_commons.logging import octobot_trading.exchanges import octobot_trading.dsl @@ -20,6 +21,7 @@ # avoid circular import from octobot_flow.logic.dsl.dsl_action_execution_context import dsl_action_execution from octobot_flow.logic.actions.abstract_action_executor import AbstractActionExecutor +from octobot_flow.logic.dsl.dsl_actions_util import is_recallable_dsl_action import tentacles.Meta.DSL_operators as dsl_operators import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops @@ -37,7 +39,9 @@ def __init__( super().__init__() self._exchange_manager = exchange_manager self._dependencies = dependencies - self._dependencies_config: dict = profile_data.to_profile("").config + self._dependencies_config: dict = ephemeral_profile_module.EphemeralProfile.from_profile_data( + profile_data + ).config self._interpreter_signals: octobot_commons.dsl_interpreter.OperatorSignals = None # type: ignore (reset when interpreter is created) self._interpreter: octobot_commons.dsl_interpreter.Interpreter = self._create_interpreter( None, executor_id @@ -111,6 +115,11 @@ def get_top_operator(self) -> typing.Union[ ]: return self._interpreter.get_top_operator() + def is_action_recallable( + self, action: octobot_flow.entities.DSLScriptActionDetails + ) -> bool: + return is_recallable_dsl_action(self, action) + @dsl_action_execution async def execute_action( self, diff --git a/packages/flow/octobot_flow/repositories/community/community_repository.py b/packages/flow/octobot_flow/repositories/community/community_repository.py index 8e751759f1..52f764ee41 100644 --- a/packages/flow/octobot_flow/repositories/community/community_repository.py +++ b/packages/flow/octobot_flow/repositories/community/community_repository.py @@ -3,6 +3,7 @@ import typing import octobot.community +import octobot_commons.logging import octobot_sync.client import octobot_flow.entities @@ -18,6 +19,27 @@ def __init__(self, authenticator: octobot.community.CommunityAuthentication, wal def from_community_repository(cls, other_repository: "CommunityRepository") -> typing.Self: return cls(other_repository.authenticator, other_repository.wallet_address) + @staticmethod + def user_id_to_evm(user_id: typing.Optional[str]) -> typing.Optional[str]: + """Return the EVM wallet address for a Starfish *user_id*, or None if unresolvable. + + The community repository (used inside OctoBotActionsJob) requires the EVM wallet + address for the community sync client. We store Starfish user_id in Task.user_id + (the sync-core identity), so we must translate back to the EVM address at the + automation-job boundary. + """ + if user_id is None: + return None + try: + return octobot.community.CommunityAuthentication.instance().get_wallet_by_user_id( + user_id + ).address + except Exception as err: + octobot_commons.logging.get_logger("CommunityRepository").warning( + f"Could not resolve EVM address for user_id={user_id!r}: {err}" + ) + return None + async def insert_bot_logs(self, log_data: list[octobot.community.BotLogData]): await asyncio.gather( *[ diff --git a/packages/flow/tests/conftest.py b/packages/flow/tests/conftest.py index 59b5518d31..eb3d4dd107 100644 --- a/packages/flow/tests/conftest.py +++ b/packages/flow/tests/conftest.py @@ -3,3 +3,13 @@ import dotenv import os dotenv.load_dotenv(dotenv_path=os.path.join(os.path.dirname(__file__), ".env")) + +import mock +import pytest + + +@pytest.fixture(autouse=True) +def _disable_auto_open_in_web_browser(): + import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops + with mock.patch.object(octobot_process_ops, "AUTO_OPEN_IN_WEB_BROWSER", False): + yield diff --git a/packages/flow/tests/entities/automations/test_octobot_process_state.py b/packages/flow/tests/entities/automations/test_octobot_process_state.py new file mode 100644 index 0000000000..65d9ad4df1 --- /dev/null +++ b/packages/flow/tests/entities/automations/test_octobot_process_state.py @@ -0,0 +1,78 @@ +import unittest + +import octobot_commons.dsl_interpreter as dsl_interpreter +import octobot_flow.entities.actions.action_details as action_details +import octobot_flow.entities.automations.octobot_process_state as octobot_process_state_module + + +class TestParseOctobotProcessState(unittest.TestCase): + def test_parse_octobot_process_state_returns_none_for_empty_dict(self): + self.assertIsNone(octobot_process_state_module.parse_octobot_process_state({})) + + def test_parse_octobot_process_state_returns_state_for_valid_recall_dict(self): + recall_dict = { + "http_base_url": "http://127.0.0.1:5002", + "web_port": 5002, + "node_port": 6002, + "user_root": "/tmp/user", + "user_folder": "automation-a", + "log_folder": "/tmp/log", + "profile_id": None, + "pid": 12345, + "init_state_ok": True, + "executor_id": "exec-1", + } + parsed_state = octobot_process_state_module.parse_octobot_process_state(recall_dict) + self.assertIsInstance(parsed_state, octobot_process_state_module.OctobotProcessState) + self.assertEqual(parsed_state.http_base_url, "http://127.0.0.1:5002") + self.assertEqual(parsed_state.web_port, 5002) + self.assertTrue(parsed_state.init_state_ok) + + +class TestIsRunOctobotProcessDslAction(unittest.TestCase): + def test_is_run_octobot_process_dsl_action_returns_true_for_run_octobot_process_script(self): + dsl_action = action_details.DSLScriptActionDetails( + id="action-1", + dsl_script='run_octobot_process("folder", profile_data={})', + ) + self.assertTrue( + octobot_process_state_module.is_run_octobot_process_dsl_action(dsl_action) + ) + + def test_is_run_octobot_process_dsl_action_returns_false_for_other_dsl_script(self): + dsl_action = action_details.DSLScriptActionDetails( + id="action-1", + dsl_script='other_operator("folder")', + ) + self.assertFalse( + octobot_process_state_module.is_run_octobot_process_dsl_action(dsl_action) + ) + + def test_is_run_octobot_process_dsl_action_uses_resolved_dsl_script_when_set(self): + dsl_action = action_details.DSLScriptActionDetails( + id="action-1", + dsl_script='placeholder', + resolved_dsl_script='run_octobot_process("folder", profile_data={})', + ) + self.assertTrue( + octobot_process_state_module.is_run_octobot_process_dsl_action(dsl_action) + ) + + +class TestRecallInnerFromActionResult(unittest.TestCase): + def test_recall_inner_from_action_result_unwraps_recalling_operator_result(self): + inner_recall = { + "http_base_url": "http://127.0.0.1:5002", + "web_port": 5002, + "pid": 12345, + } + recall_wrapper = dsl_interpreter.ReCallingOperatorResult( + keyword="run_octobot_process", + reset_to_id="action-1", + last_execution_result=inner_recall, + ) + action_result = { + dsl_interpreter.ReCallingOperatorResult.__name__: recall_wrapper.to_dict(), + } + inner = octobot_process_state_module.recall_inner_from_action_result(action_result) + self.assertEqual(inner, inner_recall) diff --git a/packages/flow/tests/entities/test_exchange_account_elements_sync_orders.py b/packages/flow/tests/entities/test_exchange_account_elements_sync_orders.py new file mode 100644 index 0000000000..e253bdb9db --- /dev/null +++ b/packages/flow/tests/entities/test_exchange_account_elements_sync_orders.py @@ -0,0 +1,92 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or +# (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# OctoBot. If not, see . +import mock + +import octobot_flow.entities.accounts.exchange_account_elements as exchange_account_elements_module + + +def _order_stub(exchange_order_id: str): + order = mock.Mock() + order.exchange_order_id = exchange_order_id + order.is_self_managed = mock.Mock(return_value=False) + return order + + +class TestExchangeAccountElementsSyncOrdersFromExchangeManager: + def test_includes_pending_creation_orders_with_exchange_id(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + open_order = _order_stub("open-order-id") + pending_order = _order_stub("pending-order-id") + with mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.api.get_open_orders", + return_value=[open_order], + ), mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.api.get_pending_creation_orders", + return_value=[pending_order], + ), mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.storage.orders_storage._format_order", + side_effect=lambda order, _exchange_manager: {"exchange_order_id": order.exchange_order_id}, + ): + elements = exchange_account_elements_module.ExchangeAccountElements() + elements.sync_orders_from_exchange_manager(exchange_manager) + + assert len(elements.orders.open_orders) == 2 + stored_exchange_ids = { + stored_order["exchange_order_id"] for stored_order in elements.orders.open_orders + } + assert stored_exchange_ids == {"open-order-id", "pending-order-id"} + + def test_deduplicates_pending_creation_when_already_open(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + shared_exchange_id = "shared-order-id" + open_order = _order_stub(shared_exchange_id) + pending_order = _order_stub(shared_exchange_id) + with mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.api.get_open_orders", + return_value=[open_order], + ), mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.api.get_pending_creation_orders", + return_value=[pending_order], + ), mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.storage.orders_storage._format_order", + side_effect=lambda order, _exchange_manager: {"exchange_order_id": order.exchange_order_id}, + ): + elements = exchange_account_elements_module.ExchangeAccountElements() + elements.sync_orders_from_exchange_manager(exchange_manager) + + assert len(elements.orders.open_orders) == 1 + + def test_skips_pending_creation_without_exchange_id(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + open_order = _order_stub("open-only") + pending_without_id = _order_stub(None) + with mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.api.get_open_orders", + return_value=[open_order], + ), mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.api.get_pending_creation_orders", + return_value=[pending_without_id], + ), mock.patch( + "octobot_flow.entities.accounts.exchange_account_elements.octobot_trading.storage.orders_storage._format_order", + side_effect=lambda order, _exchange_manager: {"exchange_order_id": order.exchange_order_id}, + ): + elements = exchange_account_elements_module.ExchangeAccountElements() + elements.sync_orders_from_exchange_manager(exchange_manager) + + assert len(elements.orders.open_orders) == 1 diff --git a/packages/flow/tests/entities/test_execution_details.py b/packages/flow/tests/entities/test_execution_details.py new file mode 100644 index 0000000000..ca645c1b8e --- /dev/null +++ b/packages/flow/tests/entities/test_execution_details.py @@ -0,0 +1,22 @@ +import pytest + +import octobot_flow.entities +import octobot_flow.enums + + +class TestExecutionDetailsCompleteExecution: + def test_clears_degraded_state_on_successful_completion(self): + execution = octobot_flow.entities.ExecutionDetails( + degraded_state=octobot_flow.entities.DegradedStateDetails( + since=100.0, + error=octobot_flow.enums.ActionErrorStatus.NOT_ENOUGH_FUNDS.value, + reason="Insufficient funds", + ), + current_execution=octobot_flow.entities.TriggerDetails(triggered_at=1.0), + ) + + execution.complete_execution(next_execution_scheduled_to=200.0) + + assert execution.degraded_state.since == 0 + assert execution.degraded_state.error is None + assert execution.degraded_state.reason is None diff --git a/packages/flow/tests/functionnal_tests/conftest.py b/packages/flow/tests/functionnal_tests/conftest.py index 1264acfeb6..a1e29fc03f 100644 --- a/packages/flow/tests/functionnal_tests/conftest.py +++ b/packages/flow/tests/functionnal_tests/conftest.py @@ -1,9 +1,71 @@ +import os +import pathlib + import pytest +import octobot_commons.constants as commons_constants +import octobot_trading.errors as trading_errors + +import octobot_flow.environment import tests.functionnal_tests as functionnal_tests +@pytest.fixture +def skip_on_exchange_proxy_error(request): + """Opt-in: convert ExchangeProxyError during the test call into a skip.""" + + class _SkipOnExchangeProxyErrorPlugin: + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + outcome = yield + report = outcome.get_result() + if item is not request.node: + return + if call.when != "call" or call.excinfo is None: + return + if not isinstance(call.excinfo.value, trading_errors.ExchangeProxyError): + return + proxy_url = os.environ.get("EXCHANGE_HTTP_PROXY_AUTHENTICATED_URL") + proxy_url_part = ( + f" (EXCHANGE_HTTP_PROXY_AUTHENTICATED_URL={proxy_url})" if proxy_url else "" + ) + skip_reason = f"Exchange proxy unavailable{proxy_url_part}: {call.excinfo.value}" + report.outcome = "skipped" + # pytest expects skipped longrepr as (path, lineno, "Skipped: reason") + line_number = item.location[1] if item.location else 0 + report.longrepr = (str(item.path), line_number, f"Skipped: {skip_reason}") + + plugin = _SkipOnExchangeProxyErrorPlugin() + request.config.pluginmanager.register(plugin) + yield + request.config.pluginmanager.unregister(plugin) + + +@pytest.fixture(autouse=True) +def _register_functional_executor_id(): + octobot_flow.environment.register_executor_id("func-test-executor") + yield + + @pytest.fixture(autouse=True) def _mock_local_user_configuration(): with functionnal_tests.mocked_local_user_configuration(): yield + + +@pytest.fixture(autouse=True) +def _assert_master_user_config_unchanged(request): + if not os.path.isfile(os.path.join(os.getcwd(), "start.py")): + yield + return + master_config_path = pathlib.Path(commons_constants.USER_FOLDER) / commons_constants.CONFIG_FILE + if not master_config_path.is_file(): + yield + return + config_bytes_before = master_config_path.read_bytes() + yield + config_bytes_after = master_config_path.read_bytes() + assert config_bytes_before == config_bytes_after, ( + f"master user config must not be modified during functional test " + f"{request.node.nodeid!r}: {master_config_path}" + ) diff --git a/packages/flow/tests/functionnal_tests/exchanges_actions/test_authenticated_exchange_actions.py b/packages/flow/tests/functionnal_tests/exchanges_actions/test_authenticated_exchange_actions.py index a3fb2357da..29fe4069f7 100644 --- a/packages/flow/tests/functionnal_tests/exchanges_actions/test_authenticated_exchange_actions.py +++ b/packages/flow/tests/functionnal_tests/exchanges_actions/test_authenticated_exchange_actions.py @@ -54,7 +54,10 @@ def init_action(): @pytest.mark.asyncio @pytest.mark.xdist_group(name=AUTHENTICATED_TEST_GROUP) async def test_execute_actions_with_limit_orders_and_empty_state( - init_action: dict, actions_with_create_limit_orders: list[dict], actions_with_cancel_limit_orders: list[dict] + init_action: dict, + actions_with_create_limit_orders: list[dict], + actions_with_cancel_limit_orders: list[dict], + skip_on_exchange_proxy_error, ): all_actions = [init_action] with ( diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py index 40e1d6085d..fdc6eeab27 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py @@ -2,9 +2,11 @@ # Shared helpers/constants for octobot process functional tests (run_octobot_process, GridTradingMode). import asyncio +import contextlib import copy import decimal import json +import mock import os import pathlib import time @@ -19,14 +21,16 @@ import pytest import octobot_flow.jobs +import octobot_flow.jobs.automation_runner_job as automation_runner_job_module import octobot_flow.entities -import octobot_flow.environment import octobot_flow.enums import tests.functionnal_tests as functionnal_tests import tests.functionnal_tests.tentacle_test_configs as tentacle_test_configs pytestmark = pytest.mark.asyncio +OCTOBOT_PROCESS_TEST_GROUP = "octobot_process_xdist_group" + # --- Timeouts and grid geometry (must match pair_settings spread / increment below) --- GLOBAL_START_TIMEOUT_SEC = 30.0 SLEEP_BETWEEN_JOB_POLLS_SEC = 2.0 @@ -50,14 +54,61 @@ WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC = 2 RECALL_SCHEDULE_TOLERANCE_SEC = 1.5 + +class ExchangeManagerContextTracker: + def __init__(self) -> None: + self.entered_count = 0 + + +@contextlib.contextmanager +def track_exchange_manager_context() -> typing.Iterator[ExchangeManagerContextTracker]: + tracker = ExchangeManagerContextTracker() + + @contextlib.asynccontextmanager + async def counting_exchange_manager_context(self): + tracker.entered_count += 1 + self._exchange_manager = mock.Mock() + yield self._exchange_manager + + with mock.patch.object( + automation_runner_job_module.AutomationRunnerJob, + "exchange_manager_context", + counting_exchange_manager_context, + ): + yield tracker + + +def assert_exchange_manager_not_initialized(tracker: ExchangeManagerContextTracker) -> None: + assert tracker.entered_count == 0, ( + "expected exchange_manager_context to be skipped for process-bound actions, " + f"but it was entered {tracker.entered_count} time(s)" + ) + + +async def run_automation_job_without_exchange_manager( + automation_state: dict, + priority_actions: list, + updated_trading_signals: list, + auth_details: dict, +) -> octobot_flow.jobs.AutomationJob: + with track_exchange_manager_context() as tracker: + async with octobot_flow.jobs.AutomationJob( + automation_state, priority_actions, updated_trading_signals, auth_details + ) as automation_job: + await automation_job.run() + assert_exchange_manager_not_initialized(tracker) + return automation_job + + EXCHANGE_BINANCEUS = "binanceus" +FUNCTIONAL_TEST_USER_ID = "wallet-user" # --- DSL / DAG action ids (fixtures, dependencies, _get_action_by_id) --- ACTION_ID_INIT = "action_init" ACTION_ID_RUN_OCTOBOT = "action_run_octobot" ACTION_ID_STOP_AUTOMATION = "action_stop_automation" -# --- Child profile for run_octobot_process: simulator (trader.enabled False) + GridTradingMode BTC/USDT 2×2 --- +# --- Child profile for run_octobot_process: simulator (trader.enabled False) + GridTradingMode BTC/USDT 2×2 --- Use profile_data= keyword (2nd positional is octobot_name). GRID_BINANCEUS_PROFILE_DATA = { "profile_details": {"name": "func_test_grid_octoprocess", "id": "func_test_grid_octoprocess"}, "crypto_currencies": [ @@ -316,12 +367,6 @@ def _tracked(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: return _tracked -@pytest.fixture(autouse=True) -def register_functional_executor_id(): - octobot_flow.environment.register_executor_id("func-test-executor") - yield - - @pytest.fixture def init_action(): # Automation apply_configuration: seed automation state to match expected exchange + portfolio. diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py index 22b3147366..3533af4a8c 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py @@ -66,6 +66,7 @@ def _assert_three_by_three_grid_ladder_orders(orders_wrapped: list[dict]) -> Non ) +@pytest.mark.xdist_group(name=octobot_process_functional_shared.OCTOBOT_PROCESS_TEST_GROUP) async def test_run_octobot_process_grid_refresh_four_to_six_orders( init_action: dict, monkeypatch: pytest.MonkeyPatch, @@ -81,7 +82,8 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( profile_2x2 = octobot_process_functional_shared._grid_binanceus_profile_data(2, 2) run_dsl = ( "run_octobot_process(" - f"{user_folder!r}, {repr(profile_2x2)}, " + f"{user_folder!r}, profile_data={repr(profile_2x2)}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) run_action = { @@ -146,8 +148,9 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None # 2) First automation pass, then poll until the child reports init_state_ok (ready to query). - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as first_poll: - await first_poll.run() + first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( first_poll.dump() ) @@ -160,8 +163,9 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( if not (inner and inner.get("init_state_ok") is True): while time.monotonic() < deadline: await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as poll_job: - await poll_job.run() + poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( poll_job.dump() ) @@ -196,9 +200,10 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( ] = None last_open_order_count = 0 while time.monotonic() < orders_deadline: - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as grid_poll_job: - await grid_poll_job.run() - job_dump_payload = grid_poll_job.dump() + grid_poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) + job_dump_payload = grid_poll_job.dump() octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( job_dump_payload ) @@ -234,7 +239,8 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( profile_3x3 = octobot_process_functional_shared._grid_binanceus_profile_data(3, 3) new_run_dsl = ( "run_octobot_process(" - f"{user_folder!r}, {repr(profile_3x3)}, " + f"{user_folder!r}, profile_data={repr(profile_3x3)}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) update_config_priority_action = { @@ -263,9 +269,10 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( last_six_count = 0 inner_after: typing.Optional[dict] = None while time.monotonic() < six_orders_deadline: - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as six_poll: - await six_poll.run() - dump_payload = six_poll.dump() + six_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) + dump_payload = six_poll.dump() octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( dump_payload ) diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py index 87db26013c..50c1a2203b 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py @@ -31,6 +31,7 @@ # --- Main lifecycle: spawn child OctoBot; EAE from job.dump() after merge; metadata from file; recall, stop --- +@pytest.mark.xdist_group(name=octobot_process_functional_shared.OCTOBOT_PROCESS_TEST_GROUP) async def test_run_octobot_process_lifecycle_grid_trading( init_action: dict, monkeypatch: pytest.MonkeyPatch, @@ -44,7 +45,8 @@ async def test_run_octobot_process_lifecycle_grid_trading( user_folder = f"functionnal_tests/octlife_{uuid.uuid4().hex[:12]}" run_dsl = ( "run_octobot_process(" - f"{user_folder!r}, {repr(octobot_process_functional_shared.GRID_BINANCEUS_PROFILE_DATA)}, " + f"{user_folder!r}, profile_data={repr(octobot_process_functional_shared.GRID_BINANCEUS_PROFILE_DATA)}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) run_action = { @@ -114,8 +116,9 @@ async def test_run_octobot_process_lifecycle_grid_trading( deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None # Run DSL job once, then optionally poll until recall payload shows init_state_ok. - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as first_poll: - await first_poll.run() + first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( first_poll.dump() ) @@ -128,8 +131,9 @@ async def test_run_octobot_process_lifecycle_grid_trading( if not (inner and inner.get("init_state_ok") is True): while time.monotonic() < deadline: await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as poll_job: - await poll_job.run() + poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( poll_job.dump() ) @@ -165,9 +169,10 @@ async def test_run_octobot_process_lifecycle_grid_trading( ] = None last_open_order_count = 0 while time.monotonic() < orders_deadline: - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as grid_poll_job: - await grid_poll_job.run() - job_dump_payload = grid_poll_job.dump() + grid_poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) + job_dump_payload = grid_poll_job.dump() octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( job_dump_payload ) @@ -240,13 +245,22 @@ async def test_run_octobot_process_lifecycle_grid_trading( exchange_account_snapshot.orders.open_orders, ) + # Grid polls update recall state (e.g. adopted pid from process_bot_state); refresh inner. + run_after_grid = octobot_process_functional_shared._get_action_by_id( + grid_poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + ) + assert run_after_grid is not None + inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_after_grid) + assert inner is not None child_pid = int(inner["pid"]) + assert child_pid == process_metadata.pid assert process_util.pid_is_running(child_pid) # 3) Second automation run: re-call path only (no second Popen; same child pid). before = popen_calls["count"] - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as idem_job: - await idem_job.run() + idem_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( idem_job.dump() ) @@ -298,6 +312,7 @@ async def test_run_octobot_process_lifecycle_grid_trading( shutil.rmtree(log_folder_guess, ignore_errors=True) +@pytest.mark.xdist_group(name=octobot_process_functional_shared.OCTOBOT_PROCESS_TEST_GROUP) async def test_run_octobot_process_lifecycle_default_config_no_profile_data( init_action: dict, monkeypatch: pytest.MonkeyPatch, @@ -329,6 +344,7 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( ] run_dsl = ( f"run_octobot_process({user_folder!r}, " + f"user_id={octobot_process_functional_shared.FUNCTIONAL_TEST_USER_ID!r}, " f"exchange_auth_data={dsl_interpreter.format_parameter_value(exchange_auth)}, " f"waiting_time={octobot_process_functional_shared.WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC}, ping_timeout=30.0)" ) @@ -391,8 +407,9 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as first_poll: - await first_poll.run() + first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( first_poll.dump() ) @@ -405,8 +422,9 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( if not (inner and inner.get("init_state_ok") is True): while time.monotonic() < deadline: await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as poll_job: - await poll_job.run() + poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( + state, [], [], {} + ) octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( poll_job.dump() ) @@ -441,13 +459,28 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( exchange_auth_entry["api_key"], exchange_auth_entry["api_secret"], ) - profile_json_path = ( + root_cfg = json.loads( + (user_root / common_constants.CONFIG_FILE).read_text(encoding="utf-8") + ) + expected_readonly_profiles_path = os.path.normpath( + os.path.join( + os.getcwd(), + common_constants.USER_FOLDER, + common_constants.PROFILES_FOLDER, + ) + ) + assert ( + root_cfg[common_constants.CONFIG_READONLY_PROFILES_PATH] + == expected_readonly_profiles_path + ) + # make sure profiles are not stored in filesystem + local_non_trading_profile_json = ( user_root / common_constants.PROFILES_FOLDER / "non-trading" / common_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not local_non_trading_profile_json.exists() # First process_bot_state dump can lag init_state_ok (see shared wait helper). state_path = octobot_process_functional_shared._process_bot_state_path(inner) diff --git a/packages/flow/tests/functionnal_tests/tentacle_test_configs.py b/packages/flow/tests/functionnal_tests/tentacle_test_configs.py index 83bb65309b..4b1885f0dd 100644 --- a/packages/flow/tests/functionnal_tests/tentacle_test_configs.py +++ b/packages/flow/tests/functionnal_tests/tentacle_test_configs.py @@ -63,6 +63,19 @@ def binanceus_dca_maximum_evaluators_config(**overrides: typing.Any) -> dict[str return _dsl_compatible_tentacle_config(config) +def binanceus_dca_time_based_config(**overrides: typing.Any) -> dict[str, typing.Any]: + config = binanceus_dca_tentacle_config( + **{ + dca_trading.DCATradingModeProducer.TRIGGER_MODE: ( + dca_trading.TriggerMode.TIME_BASED.value + ), + dca_trading.DCATradingModeProducer.MINUTES_BEFORE_NEXT_BUY: 1, + } + ) + config.update(overrides) + return _dsl_compatible_tentacle_config(config) + + def dma_evaluator_config( *, long_period_length: int = 10, diff --git a/packages/flow/tests/functionnal_tests/trading_modes_actions/authenticated/test_authenticated_grid_trading_mode_action.py b/packages/flow/tests/functionnal_tests/trading_modes_actions/authenticated/test_authenticated_grid_trading_mode_action.py index 5ce4c8525c..a4a26752ac 100644 --- a/packages/flow/tests/functionnal_tests/trading_modes_actions/authenticated/test_authenticated_grid_trading_mode_action.py +++ b/packages/flow/tests/functionnal_tests/trading_modes_actions/authenticated/test_authenticated_grid_trading_mode_action.py @@ -319,7 +319,7 @@ def init_action(): @pytest.mark.asyncio @pytest.mark.xdist_group(name=AUTHENTICATED_TEST_GROUP) -async def test_authenticated_grid_init_from_empty_state(init_action: dict): +async def test_authenticated_grid_init_from_empty_state(init_action: dict, skip_on_exchange_proxy_error): """ Same flow as the simulator grid test, but against a real authenticated account: current market price anchors the ladder (no ticker/ohlcv mocks). @@ -491,7 +491,7 @@ async def test_authenticated_grid_init_from_empty_state(init_action: dict): @pytest.mark.asyncio @pytest.mark.xdist_group(name=AUTHENTICATED_TEST_GROUP) -async def test_authenticated_copy_grid(init_action: dict): +async def test_authenticated_copy_grid(init_action: dict, skip_on_exchange_proxy_error): """ Same flow as test_simulator_copy_grid: init, then copy a synthetic reference BTC/USDC grid onto the account, then a no-op second copy run. Uses live BTC/USDC price to build valid limit prices (no mocks). diff --git a/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/test_dca_trading_mode_action.py b/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/test_dca_trading_mode_action.py index a526679762..128b5d8eb2 100644 --- a/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/test_dca_trading_mode_action.py +++ b/packages/flow/tests/functionnal_tests/trading_modes_actions/simulator/test_dca_trading_mode_action.py @@ -73,6 +73,12 @@ BINANCEUS_DCA_MAXIMUM_EVALUATORS_CONFIG = tentacle_test_configs.binanceus_dca_maximum_evaluators_config() +BINANCEUS_DCA_TIME_BASED_CONFIG = tentacle_test_configs.binanceus_dca_time_based_config( + **{ + dca_trading.DCATradingMode.TRADING_PAIRS: [BTC_USDC, ETH_USDC], + } +) + BINANCEUS_DCA_MAXIMUM_EVALUATORS_FROM_STRATEGY_SYMBOLS_CONFIG = ( tentacle_test_configs.binanceus_dca_maximum_evaluators_config( **{ @@ -256,6 +262,23 @@ def dca_trading_mode_action(dependency_action: dict) -> dict: } +def time_based_dca_trading_mode_action( + dependency_action: dict, + *, + dca_config: dict | None = None, +) -> dict: + resolved_dca_config = dca_config or BINANCEUS_DCA_TIME_BASED_CONFIG + config_parts = ", ".join( + f"{key}={dsl_interpreter.format_parameter_value(value)}" + for key, value in resolved_dca_config.items() + ) + return { + "id": "action_1", + "dsl_script": f"{DCA_TRADING_MODE_DSL_OPERATOR}({config_parts})", + "dependencies": [{"action_id": dependency_action["id"]}], + } + + def _dca_reference_storage_order( order_id: str, side: str, @@ -947,6 +970,158 @@ async def test_simulator_dca_init_from_empty_state_always_long_and_fill_buy_orde _assert_exchange_price_mocks_called(exchange_price_mock_calls) +@pytest.mark.asyncio +async def test_simulator_dca_time_based_entry_recall(init_action: dict): + """ + DCA in TIME_BASED mode on BTC/USDC + ETH/USDC: init then DCA places buy ladders, + schedules recall from minutes_before_next_buy, resets only the DCA action, and + re-applies the entry sequence on the next automation run. + """ + simulated_close_by_symbol = _default_close_prices_by_symbol() + one_minute_seconds = common_constants.MINUTE_TO_SECONDS + allowed_execution_time = 20 + + with patch_dca_simulator_exchange_prices( + lambda symbol: simulated_close_by_symbol[symbol] + ) as exchange_price_mock_calls: + all_actions = [init_action, time_based_dca_trading_mode_action(init_action)] + automation_state = automation_state_dict(resolved_actions(all_actions)) + + # Step 1 — init: only action_init completed + async with octobot_flow.jobs.AutomationJob(automation_state, [], [], {}) as automation_job: + await automation_job.run() + after_init_dump = automation_job.dump() + actions_dag = automation_job.automation_state.automation.actions_dag + # init applied configuration; DCA action is pending and depends on init + _assert_dag_snapshot(actions_dag, { + "action_init": {"completed": True, "result_is_none": True, "previous_result_is_none": True}, + "action_1": {"completed": False, "result_is_none": True, "previous_result_is_none": True}, + }) + assert {action.id for action in actions_dag.get_executable_actions()} == {"action_1"} + + # Step 2 — first DCA run: orders placed, recall scheduled, DCA reset + async with octobot_flow.jobs.AutomationJob(after_init_dump, [], [], {}) as automation_job: + await automation_job.run() + after_first_dca_dump = automation_job.dump() + actions_dag = automation_job.automation_state.automation.actions_dag + # DCA ran and was reset for recall; init stays completed (TIME_BASED does not reset the DAG to init) + _assert_dag_snapshot(actions_dag, { + "action_init": {"completed": True, "result_is_none": True, "previous_result_is_none": True}, + "action_1": {"completed": False, "result_is_none": True, "previous_result_is_none": False}, + }) + dca_action = actions_dag.get_actions_by_id()["action_1"] + # recall payload: ReCallingOperatorResult with waiting_time from minutes_before_next_buy=1 + assert isinstance(dca_action.previous_execution_result, dict) + assert dsl_interpreter.ReCallingOperatorResult.is_re_calling_operator_result( + dca_action.previous_execution_result + ) + recall_wrapper = dsl_interpreter.ReCallingOperatorResult.from_dict( + dca_action.previous_execution_result[dsl_interpreter.ReCallingOperatorResult.__name__] + ) + # reset_to_id defaults to the DCA action itself (not action_init) + assert recall_wrapper.reset_to_id is None + assert abs(recall_wrapper.last_execution_result["waiting_time"] - one_minute_seconds) < 20 + assert {action.id for action in actions_dag.get_executable_actions()} == {"action_1"} + assert not actions_dag.completed_all_actions() + + # automation is scheduled for the next recall at triggered_at + 1 minute + schedule_delay = ( + after_first_dca_dump["automation"]["execution"]["current_execution"]["scheduled_to"] + - after_first_dca_dump["automation"]["execution"]["previous_execution"]["triggered_at"] + ) + assert one_minute_seconds - allowed_execution_time < schedule_delay < one_minute_seconds + allowed_execution_time + + # TIME_BASED manual trigger placed entry ladders on both symbols (2 buys each) + open_after_first_dca = _open_orders_from_dump(after_first_dca_dump) + buy_after_first_dca = _sorted_orders_by_side( + open_after_first_dca, trading_enums.TradeOrderSide.BUY.value + ) + buy_after_first_dca_btc = _sorted_orders_by_side_and_symbol( + open_after_first_dca, trading_enums.TradeOrderSide.BUY.value, BTC_USDC + ) + buy_after_first_dca_eth = _sorted_orders_by_side_and_symbol( + open_after_first_dca, trading_enums.TradeOrderSide.BUY.value, ETH_USDC + ) + assert len(buy_after_first_dca) == 4 + assert len(buy_after_first_dca_btc) == 2 + assert len(buy_after_first_dca_eth) == 2 + _assert_dca_buy_ladder_prices(buy_after_first_dca_btc, close=_FIXED_BTC_USDC_CLOSE) + _assert_dca_buy_ladder_prices(buy_after_first_dca_eth, close=_FIXED_ETH_USDC_CLOSE) + + first_dca_buy_order_ids = { + order[trading_enums.ExchangeConstantsOrderColumns.ID.value] + for order in buy_after_first_dca + } + + # Step 3 — second DCA run after recall interval: entry sequence re-applied + scheduled_to = after_first_dca_dump["automation"]["execution"]["current_execution"]["scheduled_to"] + with mock.patch.object(time, "time", mock.Mock(return_value=scheduled_to + 1)): + async with octobot_flow.jobs.AutomationJob(after_first_dca_dump, [], [], {}) as automation_job: + await automation_job.run() + after_second_dca_dump = automation_job.dump() + actions_dag = automation_job.automation_state.automation.actions_dag + dca_action = actions_dag.get_actions_by_id()["action_1"] + # recallable actions are reset again after each run (executed_at cleared) + assert dca_action.executed_at is None + assert isinstance(dca_action.previous_execution_result, dict) + recall_wrapper = dsl_interpreter.ReCallingOperatorResult.from_dict( + dca_action.previous_execution_result[dsl_interpreter.ReCallingOperatorResult.__name__] + ) + assert abs(recall_wrapper.last_execution_result["waiting_time"] - one_minute_seconds) < 20 + + open_after_second_dca = _open_orders_from_dump(after_second_dca_dump) + buy_after_second_dca = _sorted_orders_by_side( + open_after_second_dca, trading_enums.TradeOrderSide.BUY.value + ) + assert len(buy_after_second_dca) == 4 + second_dca_buy_order_ids = { + order[trading_enums.ExchangeConstantsOrderColumns.ID.value] + for order in buy_after_second_dca + } + # cancel_open_orders_at_each_entry=True: recall re-triggered entry and replaced the ladders + assert second_dca_buy_order_ids != first_dca_buy_order_ids + + second_schedule_delay = ( + after_second_dca_dump["automation"]["execution"]["current_execution"]["scheduled_to"] + - after_second_dca_dump["automation"]["execution"]["previous_execution"]["triggered_at"] + ) + assert one_minute_seconds - allowed_execution_time < second_schedule_delay < one_minute_seconds + allowed_execution_time + + # Step 4 — fill lowest buy after recall to verify orders remain functional + lowest_buy_order = min( + buy_after_second_dca, + key=lambda order: d_order_price( + order[trading_enums.ExchangeConstantsOrderColumns.PRICE.value] + ), + ) + fill_symbol = lowest_buy_order[trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value] + lowest_buy = d_order_price( + lowest_buy_order[trading_enums.ExchangeConstantsOrderColumns.PRICE.value] + ) + simulated_close_by_symbol[fill_symbol] = float(lowest_buy - decimal.Decimal("10")) + + async with octobot_flow.jobs.AutomationJob(after_second_dca_dump, [], [], {}) as automation_job: + await automation_job.run() + after_fill_dump = automation_job.dump() + + after_fill_portfolio = _portfolio_content_from_dump(after_fill_dump) + after_second_dca_portfolio = _portfolio_content_from_dump(after_second_dca_dump) + filled_base_asset = _base_asset_from_symbol(fill_symbol) + # price drop filled the lowest buy limit on the recalled ladder + assert _portfolio_asset_total(after_fill_portfolio, filled_base_asset) > _portfolio_asset_total( + after_second_dca_portfolio, filled_base_asset + ) + + open_after_fill = _open_orders_from_dump(after_fill_dump) + sell_after_fill = _sorted_orders_by_side( + open_after_fill, trading_enums.TradeOrderSide.SELL.value + ) + # chained take-profit sell was created from the filled entry + assert len(sell_after_fill) >= 1 + + _assert_exchange_price_mocks_called(exchange_price_mock_calls) + + @pytest.mark.asyncio async def test_simulator_dca_fill_buy_then_sell_orders(init_action: dict): """ diff --git a/packages/flow/tests/jobs/test_automation_job.py b/packages/flow/tests/jobs/test_automation_job.py index 4e50da9cc6..e8b4d7cd2d 100644 --- a/packages/flow/tests/jobs/test_automation_job.py +++ b/packages/flow/tests/jobs/test_automation_job.py @@ -7,6 +7,7 @@ import octobot_protocol.models as protocol_models import octobot_flow.entities +import octobot_flow.entities.actions.action_details as action_details import octobot_flow.errors import octobot_flow.jobs.automation_job as automation_job_module import octobot_flow.logic.actions @@ -17,6 +18,9 @@ STRATEGY_ID = "test-strategy-id" +_PROCESS_BOUND_DSL_SCRIPT = ( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" +) def _minimal_automation_job() -> automation_job_module.AutomationJob: @@ -30,6 +34,41 @@ def _minimal_automation_job() -> automation_job_module.AutomationJob: return automation_job_module.AutomationJob(automation_state, [], [], auth_details) +def _dsl_action(dsl_script: str, *, action_id: str = "action_dsl") -> action_details.DSLScriptActionDetails: + return action_details.DSLScriptActionDetails( + id=action_id, + dsl_script=dsl_script, + dependencies=[], + resolved_dsl_script=dsl_script, + ) + + +def _automation_job_with_exchange_dag( + *dag_actions: action_details.AbstractActionDetails, +) -> automation_job_module.AutomationJob: + automation_state = octobot_flow.entities.AutomationState.from_dict( + { + "exchange_account_details": { + "exchange_details": {"internal_name": "binanceus"}, + "auth_details": {}, + "portfolio": {}, + }, + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": {"actions": []}, + }, + } + ) + automation_state.automation.actions_dag.actions = list(dag_actions) + user_auth_details = octobot_flow.entities.UserAuthentication(wallet_address="0xtest") + return automation_job_module.AutomationJob( + automation_state.to_dict(include_default_values=False), + [], + [], + user_auth_details, + ) + + def _minimal_copied_account() -> protocol_models.CopiedAccount: return protocol_models.CopiedAccount( version=copy_constants.COPIED_ACCOUNT_VERSION, @@ -91,3 +130,169 @@ async def test_skips_emission_and_logs_when_wallet_not_found(self): assert emitted_signal.strategy_id == STRATEGY_ID assert emitted_signal.account is copied_account error_log_mock.assert_called_once_with(f"Skipping trading signal emission: {wallet_error}") + + +class TestFetchDependencies: + @pytest.mark.asyncio + async def test_sets_skip_exchange_when_executable_dag_is_process_bound_only(self): + process_bound_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + automation_job = _automation_job_with_exchange_dag(process_bound_action) + + fetched_dependencies = await automation_job._fetch_dependencies( + None, + [process_bound_action], + ) + + assert fetched_dependencies.skip_exchange is True + assert fetched_dependencies.fetched_exchange_data is None + assert fetched_dependencies.fetched_copy_trading_data is None + + @pytest.mark.asyncio + async def test_does_not_set_skip_exchange_when_no_executable_dag_actions(self): + completed_process_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + completed_process_action.executed_at = time.time() + stop_automation_action = _dsl_action("stop_automation()", action_id="action_stop") + automation_job = _automation_job_with_exchange_dag(completed_process_action) + + fetched_dependencies = await automation_job._fetch_dependencies( + None, + [stop_automation_action], + ) + + assert fetched_dependencies.skip_exchange is False + + +class TestRequiresInitializationRun: + def _automation_job_without_exchange( + self, + *dag_actions: action_details.AbstractActionDetails, + ) -> automation_job_module.AutomationJob: + automation_state = octobot_flow.entities.AutomationState.from_dict( + { + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": {"actions": []}, + }, + } + ) + automation_state.automation.actions_dag.actions = list(dag_actions) + user_auth_details = octobot_flow.entities.UserAuthentication(wallet_address="0xtest") + return automation_job_module.AutomationJob( + automation_state.to_dict(include_default_values=False), + [], + [], + user_auth_details, + ) + + def test_skips_initialization_for_process_bound_dag_without_exchange(self): + process_bound_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + automation_job = self._automation_job_without_exchange(process_bound_action) + assert automation_job.is_initialization_run is False + + def test_requires_initialization_for_non_process_bound_dag_without_exchange(self): + stop_automation_action = _dsl_action("stop_automation()", action_id="action_stop") + automation_job = self._automation_job_without_exchange(stop_automation_action) + assert automation_job.is_initialization_run is True + + def test_requires_initialization_for_pending_apply_configuration_without_exchange(self): + init_action = action_details.ConfiguredActionDetails( + id="action_init", + action=octobot_flow.enums.ActionType.APPLY_CONFIGURATION.value, + config={ + "automation": { + "metadata": {"automation_id": "automation_1"}, + }, + }, + ) + process_bound_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + process_bound_action.dependencies = [ + { + octobot_flow.enums.ActionDependencyParameter.ACTION_ID.value: init_action.id, + } + ] + automation_job = self._automation_job_without_exchange(init_action, process_bound_action) + assert automation_job.is_initialization_run is True + + +class TestGetActionsToExecuteWithStaleCompletedPriority: + def _automation_job_with_priority_actions( + self, + priority_actions: list[action_details.AbstractActionDetails], + *, + persisted_priority_actions: list[action_details.AbstractActionDetails] | None = None, + dag_actions: list[action_details.AbstractActionDetails] | None = None, + ) -> automation_job_module.AutomationJob: + automation_state = octobot_flow.entities.AutomationState.from_dict( + { + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": {"actions": []}, + "execution": {"previous_execution": {"triggered_at": 1.0}}, + }, + "priority_actions": persisted_priority_actions or [], + } + ) + if dag_actions is not None: + automation_state.automation.actions_dag.actions = list(dag_actions) + user_auth_details = octobot_flow.entities.UserAuthentication(wallet_address="0xtest") + return automation_job_module.AutomationJob( + automation_state.to_dict(include_default_values=False), + priority_actions, + [], + user_auth_details, + ) + + def test_returns_fresh_priority_when_stale_completed_exists(self): + completed_stop_action = _dsl_action( + "stop_automation()", + action_id="action_stop_priority_ua-stop-auto-1-stale", + ) + completed_stop_action.executed_at = time.time() + fresh_stop_action = _dsl_action( + "stop_automation()", + action_id="action_stop_priority_ua-stop-auto-1-fresh", + ) + process_bound_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + automation_job = self._automation_job_with_priority_actions( + [fresh_stop_action], + persisted_priority_actions=[completed_stop_action], + dag_actions=[process_bound_action], + ) + + selected_actions, are_priority_actions = automation_job._get_actions_to_execute() + + assert are_priority_actions is True + assert len(selected_actions) == 1 + assert selected_actions[0].id == fresh_stop_action.id + + +class TestRunRaisesWhenSuppliedPriorityAlreadyCompleted: + @pytest.mark.asyncio + async def test_raises_pending_priority_actions_skipped_error(self): + completed_stop_action = _dsl_action( + "stop_automation()", + action_id="action_stop_priority_ua-stop-1", + ) + completed_stop_action.executed_at = time.time() + process_bound_action = _dsl_action(_PROCESS_BOUND_DSL_SCRIPT, action_id="action_run") + automation_state = octobot_flow.entities.AutomationState.from_dict( + { + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": {"actions": []}, + "execution": {"previous_execution": {"triggered_at": 1.0}}, + }, + "priority_actions": [completed_stop_action], + } + ) + automation_state.automation.actions_dag.actions = [process_bound_action] + user_auth_details = octobot_flow.entities.UserAuthentication() + automation_job = automation_job_module.AutomationJob( + automation_state.to_dict(include_default_values=False), + [completed_stop_action], + [], + user_auth_details, + ) + + with pytest.raises(octobot_flow.errors.PendingPriorityActionsSkippedError): + await automation_job.run() diff --git a/packages/flow/tests/jobs/test_automation_runner_job_actions_context.py b/packages/flow/tests/jobs/test_automation_runner_job_actions_context.py new file mode 100644 index 0000000000..3420f496db --- /dev/null +++ b/packages/flow/tests/jobs/test_automation_runner_job_actions_context.py @@ -0,0 +1,171 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import time +import typing + +import mock +import pytest + +import octobot_flow.entities +import octobot_flow.entities.actions.action_details as action_details +import octobot_flow.jobs.automation_runner_job as automation_runner_job_module + + +def _automation_state() -> octobot_flow.entities.AutomationState: + return octobot_flow.entities.AutomationState.from_dict( + { + "exchange_account_details": { + "exchange_details": {"internal_name": "binanceus"}, + "auth_details": {}, + "portfolio": {}, + }, + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": {"actions": []}, + }, + } + ) + + +def _fetched_dependencies(*, skip_exchange: bool = False) -> octobot_flow.entities.FetchedDependencies: + return octobot_flow.entities.FetchedDependencies(skip_exchange=skip_exchange) + + +def _runner_job( + *, + skip_exchange: bool = False, + automation_state: octobot_flow.entities.AutomationState | None = None, +) -> automation_runner_job_module.AutomationRunnerJob: + return automation_runner_job_module.AutomationRunnerJob( + automation_state or _automation_state(), + _fetched_dependencies(skip_exchange=skip_exchange), + None, + 0.0, + ) + + +def _dsl_action(dsl_script: str) -> action_details.DSLScriptActionDetails: + return action_details.DSLScriptActionDetails( + id="action_dsl", + dsl_script=dsl_script, + dependencies=[], + resolved_dsl_script=dsl_script, + ) + + +@contextlib.asynccontextmanager +async def _track_exchange_manager_context( + entered_calls: list[bool], +) -> typing.AsyncGenerator[None, None]: + @contextlib.asynccontextmanager + async def counting_exchange_manager_context(self): + entered_calls.append(True) + self._exchange_manager = mock.Mock() + yield self._exchange_manager + + with mock.patch.object( + automation_runner_job_module.AutomationRunnerJob, + "exchange_manager_context", + counting_exchange_manager_context, + ): + yield + + +def _automation_state_with_dag( + *dag_actions: action_details.AbstractActionDetails, +) -> octobot_flow.entities.AutomationState: + automation_state = _automation_state() + automation_state.automation.actions_dag.actions = list(dag_actions) + return automation_state + + +class TestAutomationRunnerJobActionsContext: + @pytest.mark.asyncio + async def test_skips_exchange_manager_for_process_bound_actions(self): + runner_job = _runner_job(skip_exchange=True) + entered_calls: list[bool] = [] + process_bound_action = _dsl_action( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" + ) + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([process_bound_action], True): + pass + + assert entered_calls == [] + assert runner_job._exchange_manager is None + + @pytest.mark.asyncio + async def test_enters_exchange_manager_for_non_process_bound_dsl_action(self): + runner_job = _runner_job() + entered_calls: list[bool] = [] + wait_action = _dsl_action("wait(1.0, 1.0)") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([wait_action], True): + pass + + assert len(entered_calls) == 1 + + @pytest.mark.asyncio + async def test_enters_exchange_manager_for_configured_action(self): + runner_job = _runner_job() + entered_calls: list[bool] = [] + configured_action = action_details.ConfiguredActionDetails(id="action_init") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([configured_action], True): + pass + + assert len(entered_calls) == 1 + + @pytest.mark.asyncio + async def test_enters_exchange_manager_for_empty_actions(self): + runner_job = _runner_job() + entered_calls: list[bool] = [] + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([], True): + pass + + assert len(entered_calls) == 1 + + @pytest.mark.asyncio + async def test_skips_exchange_manager_for_stop_automation_on_process_bound_dag(self): + process_bound_dag_action = _dsl_action( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" + ) + runner_job = _runner_job( + skip_exchange=True, + automation_state=_automation_state_with_dag(process_bound_dag_action), + ) + entered_calls: list[bool] = [] + stop_automation_action = _dsl_action("stop_automation()") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([stop_automation_action], True): + pass + + assert entered_calls == [] + assert runner_job._exchange_manager is None + + @pytest.mark.asyncio + async def test_skips_exchange_manager_for_stop_automation_when_dag_action_between_recalls(self): + process_bound_dag_action = _dsl_action( + "run_octobot_process('bots/b1', user_id='user_1', waiting_time=1.0, ping_timeout=30.0)" + ) + process_bound_dag_action.executed_at = time.time() + runner_job = _runner_job( + skip_exchange=True, + automation_state=_automation_state_with_dag(process_bound_dag_action), + ) + entered_calls: list[bool] = [] + stop_automation_action = _dsl_action("stop_automation()") + + async with _track_exchange_manager_context(entered_calls): + async with runner_job.actions_context([stop_automation_action], True): + pass + + assert entered_calls == [] + assert runner_job._exchange_manager is None diff --git a/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py new file mode 100644 index 0000000000..0b32f94f2d --- /dev/null +++ b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py @@ -0,0 +1,245 @@ +import mock +import pytest + +import octobot_commons.errors +import octobot_trading.enums +import octobot_trading.errors + +import octobot_flow.entities +import octobot_flow.enums +import octobot_flow.logic.dsl.dsl_action_execution_context + + +_PORTFOLIO_NEGATIVE_VALUE_ERROR_MESSAGE = ( + "Trying to update BTC with -0.00074 but quantity was 0.00068" +) +_DISABLED_FUNDS_TRANSFER_ERROR_MESSAGE = "Funds transfer is disabled" +_MISSING_MINIMAL_EXCHANGE_TRADE_VOLUME_MESSAGE = "Order volume below exchange minimum" +_MISSING_FUNDS_MESSAGE = "Insufficient funds for order" +_AUTHENTICATION_ERROR_MESSAGE = "Invalid API credentials" +_UNSUPPORTED_HEDGE_CONTRACT_MESSAGE = "Hedge mode is not supported for this contract" +_INVALID_POSITION_SIDE_MESSAGE = "Invalid position side for this order" +_EXCHANGE_ACCOUNT_SYMBOL_PERMISSION_MESSAGE = "Symbol is not allowed on this account" +_INVALID_PARAMETER_FORMAT_MESSAGE = "Invalid signal parameter format" +_NOT_SUPPORTED_STOP_LOSS_ORDER_MESSAGE = "STOP_LOSS orders are not supported on binance" +_NOT_SUPPORTED_BUY_MARKET_ORDER_MESSAGE = "BUY_MARKET orders are not supported on binance" +_BLOCKCHAIN_WALLET_ERROR_MESSAGE = "Blockchain wallet connection failed" +_GENERIC_EXCEPTION_MESSAGE = "Unexpected DSL execution failure" +_FAILED_REQUEST_ERROR_MESSAGE = "Exchange API request failed" + + +class TestDslActionExecutionReraisesRecallablePostponeErrors: + @pytest.mark.asyncio + @pytest.mark.parametrize( + "raised_exception,expected_exception_type", + [ + pytest.param( + octobot_trading.errors.MissingFunds(_MISSING_FUNDS_MESSAGE), + octobot_trading.errors.MissingFunds, + id="missing_funds", + ), + pytest.param( + octobot_trading.errors.FailedRequest(_FAILED_REQUEST_ERROR_MESSAGE), + octobot_trading.errors.FailedRequest, + id="failed_request", + ), + pytest.param( + octobot_trading.errors.PortfolioNegativeValueError( + _PORTFOLIO_NEGATIVE_VALUE_ERROR_MESSAGE + ), + octobot_trading.errors.PortfolioNegativeValueError, + id="portfolio_negative_value_error", + ), + pytest.param( + octobot_trading.errors.AuthenticationError(_AUTHENTICATION_ERROR_MESSAGE), + octobot_trading.errors.AuthenticationError, + id="authentication_error", + ), + pytest.param( + octobot_trading.errors.MissingMinimalExchangeTradeVolume( + _MISSING_MINIMAL_EXCHANGE_TRADE_VOLUME_MESSAGE + ), + octobot_trading.errors.MissingMinimalExchangeTradeVolume, + id="missing_minimal_exchange_trade_volume", + ), + ], + ) + async def test_reraises_when_action_is_recallable( + self, + raised_exception, + expected_exception_type, + ): + class StubExecutor: + @octobot_flow.logic.dsl.dsl_action_execution_context.dsl_action_execution + async def execute_action(self, action, **_kwargs): + raise raised_exception + + action = octobot_flow.entities.DSLScriptActionDetails( + id="copy_1", + dsl_script="copy_exchange_account()", + ) + stub_executor = StubExecutor() + + with mock.patch( + "octobot_flow.logic.dsl.dsl_action_execution_context._should_postpone_recallable_trading_error", + return_value=True, + ): + with pytest.raises(expected_exception_type) as raised_error: + await stub_executor.execute_action(action) + + assert str(raised_error.value) == str(raised_exception) + assert action.error_status is None + assert action.error_message is None + + +class TestDslActionExecutionMapsNonRecallablePostponeErrors: + @pytest.mark.asyncio + @pytest.mark.parametrize( + "raised_exception,expected_error_status", + [ + pytest.param( + octobot_trading.errors.MissingFunds(_MISSING_FUNDS_MESSAGE), + octobot_flow.enums.ActionErrorStatus.NOT_ENOUGH_FUNDS, + id="missing_funds", + ), + pytest.param( + octobot_trading.errors.FailedRequest(_FAILED_REQUEST_ERROR_MESSAGE), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, + id="failed_request", + ), + pytest.param( + octobot_trading.errors.PortfolioNegativeValueError( + _PORTFOLIO_NEGATIVE_VALUE_ERROR_MESSAGE + ), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, + id="portfolio_negative_value_error", + ), + pytest.param( + octobot_trading.errors.AuthenticationError(_AUTHENTICATION_ERROR_MESSAGE), + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR, + id="authentication_error", + ), + pytest.param( + octobot_trading.errors.MissingMinimalExchangeTradeVolume( + _MISSING_MINIMAL_EXCHANGE_TRADE_VOLUME_MESSAGE + ), + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER, + id="missing_minimal_exchange_trade_volume", + ), + ], + ) + async def test_maps_to_action_error_when_not_recallable( + self, + raised_exception, + expected_error_status, + ): + class StubExecutor: + @octobot_flow.logic.dsl.dsl_action_execution_context.dsl_action_execution + async def execute_action(self, action, **_kwargs): + raise raised_exception + + action = octobot_flow.entities.DSLScriptActionDetails( + id="action_1", + dsl_script="True", + resolved_dsl_script="True", + ) + stub_executor = StubExecutor() + + with mock.patch( + "octobot_flow.logic.dsl.dsl_action_execution_context._should_postpone_recallable_trading_error", + return_value=False, + ): + await stub_executor.execute_action(action) + + assert action.error_status == expected_error_status.value + assert action.error_message == str(raised_exception) + + +class TestDslActionExecutionMapsCaughtException: + @pytest.mark.asyncio + @pytest.mark.parametrize( + "raised_exception,expected_error_status", + [ + pytest.param( + octobot_trading.errors.DisabledFundsTransferError(_DISABLED_FUNDS_TRANSFER_ERROR_MESSAGE), + octobot_flow.enums.ActionErrorStatus.DISABLED_FUNDS_TRANSFER_ERROR, + id="disabled_funds_transfer", + ), + pytest.param( + octobot_trading.errors.MissingMinimalExchangeTradeVolume( + _MISSING_MINIMAL_EXCHANGE_TRADE_VOLUME_MESSAGE + ), + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER, + id="missing_minimal_exchange_trade_volume", + ), + pytest.param( + octobot_trading.errors.UnsupportedHedgeContractError(_UNSUPPORTED_HEDGE_CONTRACT_MESSAGE), + octobot_flow.enums.ActionErrorStatus.UNSUPPORTED_HEDGE_POSITION, + id="unsupported_hedge_contract", + ), + pytest.param( + octobot_trading.errors.InvalidPositionSide(_INVALID_POSITION_SIDE_MESSAGE), + octobot_flow.enums.ActionErrorStatus.UNSUPPORTED_HEDGE_POSITION, + id="invalid_position_side", + ), + pytest.param( + octobot_trading.errors.ExchangeAccountSymbolPermissionError( + _EXCHANGE_ACCOUNT_SYMBOL_PERMISSION_MESSAGE + ), + octobot_flow.enums.ActionErrorStatus.SYMBOL_INCOMPATIBLE_WITH_ACCOUNT, + id="exchange_account_symbol_permission", + ), + pytest.param( + octobot_commons.errors.InvalidParameterFormatError(_INVALID_PARAMETER_FORMAT_MESSAGE), + octobot_flow.enums.ActionErrorStatus.INVALID_SIGNAL_FORMAT, + id="invalid_parameter_format", + ), + pytest.param( + octobot_trading.errors.NotSupportedOrderTypeError( + _NOT_SUPPORTED_STOP_LOSS_ORDER_MESSAGE, + octobot_trading.enums.TraderOrderType.STOP_LOSS, + ), + octobot_flow.enums.ActionErrorStatus.UNSUPPORTED_STOP_ORDER, + id="not_supported_order_type_stop_loss", + ), + pytest.param( + octobot_trading.errors.NotSupportedOrderTypeError( + _NOT_SUPPORTED_BUY_MARKET_ORDER_MESSAGE, + octobot_trading.enums.TraderOrderType.BUY_MARKET, + ), + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER, + id="not_supported_order_type_buy_market", + ), + pytest.param( + octobot_trading.errors.BlockchainWalletError(_BLOCKCHAIN_WALLET_ERROR_MESSAGE), + octobot_flow.enums.ActionErrorStatus.BLOCKCHAIN_WALLET_ERROR, + id="blockchain_wallet", + ), + pytest.param( + RuntimeError(_GENERIC_EXCEPTION_MESSAGE), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, + id="generic_exception", + ), + ], + ) + async def test_maps_caught_exception_to_action_error_status( + self, + raised_exception, + expected_error_status, + ): + class StubExecutor: + @octobot_flow.logic.dsl.dsl_action_execution_context.dsl_action_execution + async def execute_action(self, action, **_kwargs): + raise raised_exception + + action = octobot_flow.entities.DSLScriptActionDetails( + id="action_1", + dsl_script="True", + resolved_dsl_script="True", + ) + stub_executor = StubExecutor() + + await stub_executor.execute_action(action) + + assert action.error_status == expected_error_status.value + assert action.error_message == str(raised_exception) diff --git a/packages/flow/tests/logic/dsl/test_dsl_actions_util.py b/packages/flow/tests/logic/dsl/test_dsl_actions_util.py new file mode 100644 index 0000000000..900e264081 --- /dev/null +++ b/packages/flow/tests/logic/dsl/test_dsl_actions_util.py @@ -0,0 +1,61 @@ +import mock +import pytest + +import octobot_commons.dsl_interpreter +import octobot_commons.profiles.profile_data as profile_data_import + +import octobot_flow.entities +import octobot_flow.logic.dsl.dsl_actions_util + + +class _RecallableTestOperator( + octobot_commons.dsl_interpreter.Operator, + octobot_commons.dsl_interpreter.ReCallableOperatorMixin, +): + @classmethod + def get_name(cls) -> str: + return "test_recallable_operator" + + +class _NonRecallableTestOperator(octobot_commons.dsl_interpreter.Operator): + @classmethod + def get_name(cls) -> str: + return "test_non_recallable_operator" + + +class TestIsRecallableDslAction: + def test_returns_true_for_recallable_top_operator(self): + dsl_executor = mock.Mock() + dsl_executor.get_top_operator.return_value = _RecallableTestOperator() + action = octobot_flow.entities.DSLScriptActionDetails( + id="action_1", + dsl_script="test_recallable_operator()", + resolved_dsl_script="test_recallable_operator()", + ) + + assert octobot_flow.logic.dsl.dsl_actions_util.is_recallable_dsl_action( + dsl_executor, action + ) is True + dsl_executor._interpreter.prepare.assert_called_once_with("test_recallable_operator()") + + def test_returns_false_for_non_recallable_top_operator(self): + dsl_executor = mock.Mock() + dsl_executor.get_top_operator.return_value = _NonRecallableTestOperator() + action = octobot_flow.entities.DSLScriptActionDetails( + id="action_1", + dsl_script="test_non_recallable_operator()", + resolved_dsl_script="test_non_recallable_operator()", + ) + + assert octobot_flow.logic.dsl.dsl_actions_util.is_recallable_dsl_action( + dsl_executor, action + ) is False + + def test_returns_false_when_dsl_script_is_empty(self): + dsl_executor = mock.Mock() + action = octobot_flow.entities.DSLScriptActionDetails(id="action_1", dsl_script="") + + assert octobot_flow.logic.dsl.dsl_actions_util.is_recallable_dsl_action( + dsl_executor, action + ) is False + dsl_executor._interpreter.prepare.assert_not_called() diff --git a/packages/flow/tests/repositories/community/test_community_repository.py b/packages/flow/tests/repositories/community/test_community_repository.py new file mode 100644 index 0000000000..36ccee7914 --- /dev/null +++ b/packages/flow/tests/repositories/community/test_community_repository.py @@ -0,0 +1,37 @@ +import mock + +import octobot_flow.repositories.community.community_repository as community_repository_module + + +class TestUserIdToEvm: + def test_user_id_to_evm_returns_none_when_user_id_is_none(self): + assert community_repository_module.CommunityRepository.user_id_to_evm(None) is None + + def test_user_id_to_evm_returns_wallet_address(self): + mock_wallet = mock.Mock(address="0xabc") + mock_auth = mock.Mock(get_wallet_by_user_id=mock.Mock(return_value=mock_wallet)) + with mock.patch.object( + community_repository_module.octobot.community.CommunityAuthentication, + "instance", + return_value=mock_auth, + ): + result = community_repository_module.CommunityRepository.user_id_to_evm("user_1") + assert result == "0xabc" + mock_auth.get_wallet_by_user_id.assert_called_once_with("user_1") + + def test_user_id_to_evm_returns_none_when_resolution_fails(self): + mock_auth = mock.Mock(get_wallet_by_user_id=mock.Mock(side_effect=ValueError("not found"))) + with ( + mock.patch.object( + community_repository_module.octobot.community.CommunityAuthentication, + "instance", + return_value=mock_auth, + ), + mock.patch.object( + community_repository_module.octobot_commons.logging, + "get_logger", + ) as mock_get_logger, + ): + result = community_repository_module.CommunityRepository.user_id_to_evm("user_1") + assert result is None + mock_get_logger.return_value.warning.assert_called_once() diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index 793cfaa23c..081b3a972b 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -30,6 +30,11 @@ AUTOMATION_WORKFLOW_MAX_ITERATION_RETRIES = int(os.getenv("AUTOMATION_WORKFLOW_MAX_ITERATION_RETRIES", 19)) AUTOMATION_WORKFLOW_BACKOFF_RATE = float(os.getenv("AUTOMATION_WORKFLOW_BACKOFF_RATE", 1.5)) +DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS = 60.0 * 60.0 # 1 hour +SCHEDULES_DEFAULT_BACKFILL_DAYS = 1 + +NODE_API_STOP_TIMEOUT_SECONDS = float(os.getenv("NODE_API_STOP_TIMEOUT_SECONDS", "5.0")) + # delay between authentication errors retry attempts of the current iteration (30 minutes) INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS = float(os.getenv("INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS", 1800)) @@ -69,4 +74,19 @@ DEFAULT_PORTFOLIO_VALUATION_UNIT = "USDT" +NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID = "non-trading-generic-process-octobot-strategy" +NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_VERSION = "1.0.0" + +USER_ACTION_WORKFLOW_RESULT_TIMEOUT_SECONDS = float( + os.getenv("USER_ACTION_WORKFLOW_RESULT_TIMEOUT_SECONDS", 120.0) +) + SCHEDULER_EXECUTOR_ID = str(uuid.uuid4()) # unique for each worker + +# Stable DBOS application_version for workflow recovery across OctoBot releases. +# Bump only when workflow step order/semantics change in a breaking way. +SCHEDULER_APPLICATION_VERSION = "octobot_node_v1" +ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION = bool(( + os.getenv("ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION") or "false" +).lower() == "true") + diff --git a/packages/node/octobot_node/enums.py b/packages/node/octobot_node/enums.py index 4f70522394..f9cc05cf4b 100644 --- a/packages/node/octobot_node/enums.py +++ b/packages/node/octobot_node/enums.py @@ -37,3 +37,4 @@ class AutomationWorkflowMessageTopics(enum.Enum): class SchedulerQueues(enum.Enum): AUTOMATION_WORKFLOW_QUEUE = "automation_workflow_queue" USER_ACTION_QUEUE = "user_action_queue" + DBOS_CLEANUP_QUEUE = "dbos_cleanup_queue" diff --git a/packages/node/octobot_node/errors.py b/packages/node/octobot_node/errors.py index 72ab30bad2..9ad9be06f9 100644 --- a/packages/node/octobot_node/errors.py +++ b/packages/node/octobot_node/errors.py @@ -104,3 +104,7 @@ class UnknownTradingTypeError(UserActionError): class AmbiguousTradingTypeError(UserActionError): """Raised when multiple trading types are found for an account.""" + + +class UnrestartableAutomationError(UserActionError): + """Raised when an automation cannot be restarted.""" diff --git a/packages/node/octobot_node/models.py b/packages/node/octobot_node/models.py index ef284cf57c..dd637788d1 100644 --- a/packages/node/octobot_node/models.py +++ b/packages/node/octobot_node/models.py @@ -21,6 +21,7 @@ from pydantic import BaseModel, Field +import octobot_protocol.models as protocol_models class UserBase(BaseModel): email: str = Field(max_length=255) @@ -62,6 +63,10 @@ class Execution(BaseModel): user_id: typing.Optional[str] = None +class TaskMetadata(BaseModel): + child_octobot_process: typing.Optional[protocol_models.ChildOctoBotProcessState] = None + + class Task(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4())) name: typing.Optional[str] = None @@ -74,6 +79,7 @@ class Task(BaseModel): error_message: typing.Optional[str] = None user_ecdsa_public_key: typing.Optional[str] = None user_id: typing.Optional[str] = None + metadata: typing.Optional[TaskMetadata] = None class Node(BaseModel): node_type: str diff --git a/packages/node/octobot_node/protocol/automations.py b/packages/node/octobot_node/protocol/automations.py index 90a21f5389..4c5f8f0e76 100644 --- a/packages/node/octobot_node/protocol/automations.py +++ b/packages/node/octobot_node/protocol/automations.py @@ -24,6 +24,7 @@ import octobot_commons.logging as octobot_commons_logging import octobot_commons.timestamp_util as octobot_commons_timestamp_util import octobot_flow.entities as flow_entities +import octobot_flow.entities.automations.octobot_process_state as octobot_process_state_module import octobot_node.models as node_models import octobot_node.scheduler.octobot_flow_client as octobot_flow_client import octobot_node.scheduler.workflows.params as workflow_params @@ -146,6 +147,15 @@ def _base_protocol_automation_state(task: node_models.Task) -> protocol_models.A ) +def _degraded_state_protocol_errors( + execution: flow_entities.ExecutionDetails, +) -> tuple[typing.Optional[str], typing.Optional[str]]: + degraded_state = execution.degraded_state + if degraded_state.since <= 0 or not degraded_state.error: + return None, None + return degraded_state.error, degraded_state.reason + + def _apply_workflow_resolution_to_automation_state( filled: protocol_models.AutomationState, task: node_models.Task, @@ -167,6 +177,8 @@ def _apply_workflow_resolution_to_automation_state( ) else: error, error_message = resolved_error, resolved_error_message + if error is None and error_message is None: + error, error_message = filled.error, filled.error_message return filled.model_copy(update={ "status": resolved_status, "error": error, @@ -326,6 +338,35 @@ def _protocol_action_from_flow( ) +def _octobot_process_state_to_child_protocol( + state: octobot_process_state_module.OctobotProcessState, +) -> protocol_models.ChildOctoBotProcessState: + """Project full recall state → slim API shape for task cards.""" + return protocol_models.ChildOctoBotProcessState( + http_base_url=state.http_base_url, + web_port=state.web_port, + init_state_ok=state.init_state_ok, + ) + + +def _child_octobot_process_from_flow_actions( + actions: typing.Iterable[flow_entities.AbstractActionDetails], +) -> protocol_models.ChildOctoBotProcessState | None: + """Find run_octobot_process recall on DAG actions and map to protocol child state.""" + for flow_action in actions: + if not isinstance(flow_action, flow_entities.DSLScriptActionDetails): + continue + if not octobot_process_state_module.is_run_octobot_process_dsl_action(flow_action): + continue + action_result = flow_action.result + if action_result is None: + action_result = flow_action.previous_execution_result + inner = octobot_process_state_module.recall_inner_from_action_result(action_result) + if parsed_state := octobot_process_state_module.parse_octobot_process_state(inner or {}): + return _octobot_process_state_to_child_protocol(parsed_state) + return None + + def _fill_protocol_automation_state( protocol_automation_state: protocol_models.AutomationState, flow_automation_state: flow_entities.AutomationState, @@ -375,9 +416,17 @@ def _fill_protocol_automation_state( ), } ) + degraded_error, degraded_error_message = _degraded_state_protocol_errors( + flow_automation_state.automation.execution + ) + child_octobot_process = _child_octobot_process_from_flow_actions( + actions_dag.actions + ) return protocol_automation_state.model_copy( update={ "status": status, + "error": degraded_error, + "error_message": degraded_error_message, "metadata": metadata, "actions": dag_actions or None, "priority_actions": priority_actions or None, @@ -387,6 +436,7 @@ def _fill_protocol_automation_state( "orders": orders, "trades": trades, "positions": positions, + "child_octobot_process": child_octobot_process, } ) diff --git a/packages/node/octobot_node/protocol/dsl.py b/packages/node/octobot_node/protocol/dsl.py new file mode 100644 index 0000000000..8007ba186d --- /dev/null +++ b/packages/node/octobot_node/protocol/dsl.py @@ -0,0 +1,172 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . +""" +Convert DSL operator documentation to protocol DslKeywordsState. + +inputs are derived from get_parameters() / OperatorDocs.parameters. +outputs are derived from get_return_values() / OperatorDocs.return_values. +""" +import typing + +import octobot_commons.dsl_interpreter.operator_docs as dsl_interpreter_operator_docs +import octobot_commons.dsl_interpreter.operator_parameter as dsl_interpreter_operator_parameter +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + + +_CATALOG_EXECUTOR_ID = "dsl-keywords-catalog" + + +def list_dsl_operator_docs() -> list[dsl_interpreter_operator_docs.OperatorDocs]: + """ + Collect operator documentation for keywords exposed by the node. + + Uses the same operator set as flow DSLExecutor.get_flow_operator_classes. + """ + # Lazy: flow → tentacles; keep module import safe for build_openapi packaging. + import octobot_flow.logic.configuration.profile_data_factory as profile_data_factory + import octobot_flow.logic.dsl.dsl_executor as dsl_executor_module + + catalog_profile_data = profile_data_factory.create_profile_data( + None, + _CATALOG_EXECUTOR_ID, + set(), + ) + catalog_executor = dsl_executor_module.DSLExecutor( + catalog_profile_data, + None, + None, + executor_id=_CATALOG_EXECUTOR_ID, + ) + operator_classes = catalog_executor.get_flow_operator_classes( + _CATALOG_EXECUTOR_ID, + ) + return [operator_class.get_docs() for operator_class in operator_classes] + + +def operator_parameter_to_dsl_parameter( + parameter: dsl_interpreter_operator_parameter.OperatorParameter, +) -> protocol_models.DslParameter: + """ + Map an OperatorParameter to a protocol DslParameter. + + Requires an authored DslValueType string on parameter.type. + """ + if not parameter.type: + raise ValueError( + f"OperatorParameter {parameter.name!r} is missing type; " + "author a DslValueType-compatible type on the operator parameter" + ) + try: + value_type = protocol_models.DslValueType(parameter.type) + except ValueError as error: + raise ValueError( + f"OperatorParameter {parameter.name!r} has unknown type " + f"{parameter.type!r}" + ) from error + + dsl_parameter_kwargs: dict[str, typing.Any] = { + "name": parameter.name, + "label": parameter.name, + "value_type": value_type, + "description": parameter.description, + "required": parameter.required, + } + if ( + parameter.default is not dsl_interpreter_operator_parameter.UNSET_VALUE + and parameter.default is not None + and isinstance(parameter.default, (bool, int, float, str)) + ): + # Protocol DslParameterDefaultValue only accepts bool | float | str (int via float). + # Skip None and structured defaults (list/dict) that the schema cannot represent. + dsl_parameter_kwargs["default_value"] = ( + protocol_models.DslParameterDefaultValue(parameter.default) + ) + if parameter.options is not None: + dsl_parameter_kwargs["options"] = [ + protocol_models.DslParameterOption( + value=option.value, + label=option.label, + ) + for option in parameter.options + ] + if parameter.minimum is not None: + dsl_parameter_kwargs["minimum"] = parameter.minimum + if parameter.maximum is not None: + dsl_parameter_kwargs["maximum"] = parameter.maximum + if parameter.step is not None: + dsl_parameter_kwargs["step"] = parameter.step + if parameter.multiple is not None: + dsl_parameter_kwargs["multiple"] = parameter.multiple + if parameter.primary is not None: + dsl_parameter_kwargs["primary"] = parameter.primary + return protocol_models.DslParameter(**dsl_parameter_kwargs) + + +def operator_docs_to_dsl_keyword( + operator_docs: dsl_interpreter_operator_docs.OperatorDocs, +) -> protocol_models.DslKeyword: + """ + Convert operator documentation to a protocol DslKeyword. + + Raises ValueError when required catalog metadata is missing. + """ + if not operator_docs.category: + raise ValueError( + f"Operator {operator_docs.name!r} is missing CATEGORY; " + "author a DslKeywordCategory-compatible CATEGORY on the operator" + ) + try: + category = protocol_models.DslKeywordCategory(operator_docs.category) + except ValueError as error: + raise ValueError( + f"Operator {operator_docs.name!r} has unknown category " + f"{operator_docs.category!r}" + ) from error + if not operator_docs.return_values: + raise ValueError( + f"Operator {operator_docs.name!r} is missing return_values; " + "author get_return_values() on the operator" + ) + + return protocol_models.DslKeyword( + name=operator_docs.name, + category=category, + label=operator_docs.label or operator_docs.name, + description=operator_docs.description, + inputs=[ + operator_parameter_to_dsl_parameter(parameter) + for parameter in operator_docs.parameters + ], + outputs=[ + operator_parameter_to_dsl_parameter(return_value) + for return_value in operator_docs.return_values + ], + ) + + +def get_dsl_keywords_state() -> protocol_models.DslKeywordsState: + """ + Return the DSL keywords state for this node. + """ + keywords = [ + operator_docs_to_dsl_keyword(operator_docs) + for operator_docs in list_dsl_operator_docs() + ] + return protocol_models.DslKeywordsState( + version=sync_constants.DSL_KEYWORDS_STATE_VERSION, + keywords=keywords, + ) diff --git a/packages/node/octobot_node/scheduler/__init__.py b/packages/node/octobot_node/scheduler/__init__.py index fa819b5d7a..58d2eb595a 100644 --- a/packages/node/octobot_node/scheduler/__init__.py +++ b/packages/node/octobot_node/scheduler/__init__.py @@ -16,13 +16,17 @@ import logging +import octobot_node.constants import octobot_node.scheduler.scheduler as scheduler_lib import octobot_node.scheduler.workflows +import octobot_node.scheduler.workflows_version_migration as workflows_version_migration scheduler_logger = logging.getLogger(__name__) SCHEDULER: scheduler_lib.Scheduler = scheduler_lib.Scheduler() +_shutdown_done = False + def is_enabled() -> bool: return SCHEDULER.is_enabled() @@ -32,17 +36,30 @@ def is_initialized() -> bool: return SCHEDULER.is_initialized() -def initialize_scheduler(): +async def initialize_scheduler(): + global _shutdown_done + _shutdown_done = False scheduler_logger.info("Initializing scheduler") SCHEDULER.create() octobot_node.scheduler.workflows.register_workflows() + if octobot_node.constants.ALWAYS_ENSURE_SCHEDULER_APPLICATION_VERSION: + workflows_version_migration.migrate_stranded_workflow_versions( + target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + import octobot_node.scheduler.schedules as schedules SCHEDULER.start() + # apply_schedules requires DBOS launch (sys_db); must run after start(). + await schedules.register_schedules(SCHEDULER) async def shutdown_scheduler_and_trading_signal_channel() -> None: + global _shutdown_done + if _shutdown_done or not is_initialized(): + return try: import octobot_flow.repositories.community.trading_signals_channel as trading_signals_channel await trading_signals_channel.shutdown_internal_trading_signal_channel() except ImportError: pass SCHEDULER.stop() + _shutdown_done = True diff --git a/packages/node/octobot_node/scheduler/api.py b/packages/node/octobot_node/scheduler/api.py index 265c48014f..d2caf6211f 100644 --- a/packages/node/octobot_node/scheduler/api.py +++ b/packages/node/octobot_node/scheduler/api.py @@ -15,12 +15,11 @@ # License along with OctoBot. If not, see . import asyncio -import logging import typing import uuid import dbos - +import octobot_commons.logging as octobot_commons_logging import octobot_protocol.models as protocol_models import octobot_node.config @@ -29,7 +28,7 @@ import octobot_node.scheduler import octobot_node.scheduler.workflows_util as workflows_util -logger = logging.getLogger(__name__) +logger = octobot_commons_logging.get_logger("octobot_node.scheduler.api") def get_node_status() -> dict[str, str | int | None | uuid.UUID]: @@ -62,8 +61,11 @@ def get_node_status() -> dict[str, str | int | None | uuid.UUID]: } -async def get_automation_states(user_id: typing.Optional[str]) -> list[protocol_models.AutomationState]: - return await octobot_node.scheduler.SCHEDULER.get_automation_states(user_id) +async def get_automation_states( + user_id: typing.Optional[str], + statuses: typing.Optional[list[dbos.WorkflowStatusString]] = None, +) -> list[protocol_models.AutomationState]: + return await octobot_node.scheduler.SCHEDULER.get_automation_states(user_id, statuses) async def list_user_actions(user_id: typing.Optional[str], active_only: bool) -> list[protocol_models.UserAction]: @@ -160,6 +162,50 @@ def _build_tasks_from_executions( return tasks +async def _enrich_tasks_with_child_octobot_process( + tasks: list[octobot_node.models.Task], + user_id: typing.Optional[str], +) -> None: + active_task_statuses = { + octobot_node.models.TaskStatus.PENDING, + octobot_node.models.TaskStatus.RUNNING, + octobot_node.models.TaskStatus.SCHEDULED, + octobot_node.models.TaskStatus.PERIODIC, + } + active_task_ids = { + task.id + for task in tasks + if task.id is not None + and (active_execution := _get_active_execution(task.executions)) is not None + and active_execution.status in active_task_statuses + } + if not active_task_ids: + return + try: + automation_states = await octobot_node.scheduler.SCHEDULER.get_automation_states( + user_id, + statuses=[ + dbos.WorkflowStatusString.ENQUEUED, + dbos.WorkflowStatusString.PENDING, + ], + ) + child_process_by_automation_id = { + automation_state.id: automation_state.child_octobot_process + for automation_state in automation_states + if automation_state.child_octobot_process is not None + and automation_state.id in active_task_ids + } + for task in tasks: + if task.id in child_process_by_automation_id: + if task.metadata is None: + task.metadata = octobot_node.models.TaskMetadata() + task.metadata.child_octobot_process = child_process_by_automation_id[task.id] + except Exception as enrich_error: + logger.exception( + enrich_error, True, "Failed to enrich tasks with child_octobot_process: %s", enrich_error + ) + + async def get_all_tasks( user_id: typing.Optional[str] = None, ) -> list[octobot_node.models.Task]: @@ -180,6 +226,7 @@ async def get_all_tasks( return [] tasks = _build_tasks_from_executions(executions) + await _enrich_tasks_with_child_octobot_process(tasks, user_id) logger.debug("Returning %d total tasks from %d executions", len(tasks), len(executions)) return tasks @@ -205,9 +252,21 @@ async def cancel_tasks(task_ids: list[str]) -> list[str]: return await octobot_node.scheduler.SCHEDULER.cancel_workflows(task_ids) +async def retrieve_workflow_handle(workflow_id: str): + return await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async(workflow_id) + + +async def await_workflow_result_from_id(workflow_id: str) -> typing.Any: + workflow_handle = await retrieve_workflow_handle(workflow_id) + return await asyncio.wait_for( + workflow_handle.get_result(), + timeout=octobot_node.constants.USER_ACTION_WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + + async def get_task_result(task_id: str): try: - handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async(task_id) + handle = await retrieve_workflow_handle(task_id) except Exception: return {"error": "task not found"} @@ -222,9 +281,9 @@ async def get_task_result(task_id: str): if wf_status == "ERROR": try: result_data = await handle.get_result() - except Exception as e: - result_data = {"error": str(e)} + except Exception as error: + result_data = {"error": str(error)} return {"status": "completed", "data": result_data} - except Exception as e: - logger.debug(f"Workflow {task_id} not yet complete: {e}") + except Exception as error: + logger.debug(f"Workflow {task_id} not yet complete: {error}") return {"status": "pending or running"} diff --git a/packages/node/octobot_node/scheduler/generic_process_octobot.py b/packages/node/octobot_node/scheduler/generic_process_octobot.py new file mode 100644 index 0000000000..ab72ba3d3a --- /dev/null +++ b/packages/node/octobot_node/scheduler/generic_process_octobot.py @@ -0,0 +1,189 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import datetime +import json +import typing +import uuid + +import octobot_protocol.models as protocol_models +import octobot_commons.constants as commons_constants + +import octobot_node.constants +import octobot_node.errors as node_errors +import octobot_node.scheduler.api as scheduler_api +import octobot_node.scheduler.tasks as scheduler_tasks +import octobot_node.scheduler.workflows.params as workflow_params +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers + + +def _wrap_user_action_configuration( + payload: typing.Any, +) -> protocol_models.UserActionConfiguration: + return protocol_models.UserActionConfiguration.from_json(payload.to_json()) + + +def _build_non_trading_generic_process_octobot_strategy() -> protocol_models.Strategy: + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + ) + return protocol_models.Strategy( + id=octobot_node.constants.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID, + version=octobot_node.constants.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_VERSION, + name="Generic process OctoBot strategy", + reference_market=commons_constants.DEFAULT_REFERENCE_MARKET, + configuration=protocol_models.StrategyConfiguration(generic_process_configuration), + ) + + +def _build_create_strategy_user_action( + strategy: protocol_models.Strategy, +) -> protocol_models.UserAction: + strategy_payload = protocol_models.CreateStrategyConfiguration( + action_type=protocol_models.UserActionType.STRATEGY_CREATE, + configuration=strategy, + ) + return protocol_models.UserAction( + id=f"ua-strategy-create-{uuid.uuid4()}", + configuration=_wrap_user_action_configuration(strategy_payload), + ) + + +def _build_create_automation_user_action( + *, + automation_user_action_id: str, + name: str, + strategy: protocol_models.Strategy, +) -> protocol_models.UserAction: + strategy_reference = protocol_models.StrategyReference( + id=strategy.id, + version=strategy.version, + emit_signals=False, + ) + automation_configuration = protocol_models.AutomationConfiguration( + id=automation_user_action_id, + name=name, + created_at=datetime.datetime.now(datetime.UTC), + strategy=strategy_reference, + accounts=[], + ) + automation_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=automation_configuration, + ) + return protocol_models.UserAction( + id=automation_user_action_id, + configuration=_wrap_user_action_configuration(automation_payload), + ) + + +def _updated_user_action_from_workflow_result( + workflow_result: typing.Any, +) -> protocol_models.UserAction: + if isinstance(workflow_result, str): + workflow_result = workflow_params.UserActionWorkflowOutput.from_dict(json.loads(workflow_result)) + elif isinstance(workflow_result, dict): + workflow_result = workflow_params.UserActionWorkflowOutput.from_dict(workflow_result) + elif not isinstance(workflow_result, workflow_params.UserActionWorkflowOutput): + raise node_errors.UserActionError( + f"Unexpected user action workflow result type: {type(workflow_result).__name__}" + ) + updated_user_action = workflow_result.updated_user_action + if updated_user_action.status == protocol_models.UserActionStatus.FAILED: + error_details = None + if updated_user_action.result and updated_user_action.result.actual_instance: + error_details = getattr( + updated_user_action.result.actual_instance, + "error_details", + None, + ) + raise node_errors.UserActionError( + error_details or f"User action {updated_user_action.id!r} failed" + ) + if updated_user_action.status != protocol_models.UserActionStatus.COMPLETED: + raise node_errors.UserActionError( + f"User action {updated_user_action.id!r} finished with status " + f"{updated_user_action.status!r}, expected completed" + ) + return updated_user_action + + +def _created_automation_id_from_user_action( + updated_user_action: protocol_models.UserAction, +) -> str: + if updated_user_action.result is None or updated_user_action.result.actual_instance is None: + raise node_errors.UserActionError( + f"Automation create user action {updated_user_action.id!r} has no result" + ) + automation_result = updated_user_action.result.actual_instance + if not isinstance(automation_result, protocol_models.AutomationActionResult): + raise node_errors.UserActionError( + f"Automation create user action {updated_user_action.id!r} returned " + f"{type(automation_result).__name__}, expected AutomationActionResult" + ) + if not automation_result.created_automation_id: + raise node_errors.UserActionError( + f"Automation create user action {updated_user_action.id!r} has no created_automation_id" + ) + return automation_result.created_automation_id + + +async def _execute_user_action_and_await( + user_id: str, + user_action: protocol_models.UserAction, +) -> protocol_models.UserAction: + workflow_id = await scheduler_tasks.trigger_user_action_workflow(user_action, user_id) + workflow_result = await scheduler_api.await_workflow_result_from_id(workflow_id) + return _updated_user_action_from_workflow_result(workflow_result) + + +async def _ensure_non_trading_generic_process_octobot_strategy( + user_id: str, +) -> protocol_models.Strategy: + strategy_provider = collection_providers.StrategyProvider.instance() + try: + return strategy_provider.get_item( + user_id, + octobot_node.constants.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID, + ) + except collection_errors.ItemNotFoundError: + strategy = _build_non_trading_generic_process_octobot_strategy() + create_strategy_user_action = _build_create_strategy_user_action(strategy) + await _execute_user_action_and_await(user_id, create_strategy_user_action) + return strategy_provider.get_item( + user_id, + octobot_node.constants.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID, + ) + + +async def create_generic_process_bot( + user_id: str, + name: str, + automation_id: str | None = None, +) -> str: + stored_strategy = await _ensure_non_trading_generic_process_octobot_strategy(user_id) + resolved_automation_id = automation_id or str(uuid.uuid4()) + create_automation_user_action = _build_create_automation_user_action( + automation_user_action_id=resolved_automation_id, + name=name, + strategy=stored_strategy, + ) + updated_user_action = await _execute_user_action_and_await( + user_id, + create_automation_user_action, + ) + return _created_automation_id_from_user_action(updated_user_action) diff --git a/packages/node/octobot_node/scheduler/octobot_flow_client.py b/packages/node/octobot_node/scheduler/octobot_flow_client.py index bf9209de24..c6566c6820 100644 --- a/packages/node/octobot_node/scheduler/octobot_flow_client.py +++ b/packages/node/octobot_node/scheduler/octobot_flow_client.py @@ -94,8 +94,9 @@ def _include_actions_in_automation_state(self, automation_id: str, actions: "oct automation_state.upsert_automation_actions(actions.actions) self.state = automation_state.to_dict(include_default_values=False) - def get_next_execution_time(self) -> float: - return self.state["automation"]["execution"]["current_execution"]["scheduled_to"] + @staticmethod + def get_next_execution_time(state: dict) -> float: + return state["automation"]["execution"]["current_execution"]["scheduled_to"] @dataclasses.dataclass @@ -149,12 +150,10 @@ async def run(self) -> None: self.updated_trading_signals, self.description.auth_details, ) as automation_job: - selected_actions = ( - self.priority_user_actions - or automation_job.automation_state.automation.actions_dag.get_executable_actions() - ) - octobot_commons.logging.get_logger(self.__class__.__name__).info(f"Running automation actions: {selected_actions}") executed_actions = await automation_job.run() + octobot_commons.logging.get_logger(self.__class__.__name__).info( + f"Running automation actions: {executed_actions}" + ) self.after_execution_state = automation_job.automation_state post_execution_state_dump = automation_job.dump() next_actions_description, has_next_actions = self.get_next_actions_description(post_execution_state_dump) diff --git a/packages/node/octobot_node/scheduler/scheduler.py b/packages/node/octobot_node/scheduler/scheduler.py index e5359c3291..eb52515d47 100644 --- a/packages/node/octobot_node/scheduler/scheduler.py +++ b/packages/node/octobot_node/scheduler/scheduler.py @@ -16,13 +16,13 @@ import contextlib import datetime +import asyncio import dbos import json import logging import typing import decimal import enum -import sqlalchemy import octobot_commons.logging import octobot_commons.timestamp_util as timestamp_util @@ -32,23 +32,19 @@ import octobot_node.models import octobot_node.constants import octobot_node.scheduler.workflows_util as workflows_util +import octobot_node.scheduler.workflows_retention as workflows_retention import octobot_node.scheduler.workflows.params as workflow_params import octobot_node.scheduler.user_actions.user_action_util as user_action_util import octobot_node.scheduler.encryption as encryption import octobot_node.scheduler.task_context as task_context import octobot_node.protocol.automations as automations_protocol -try: - from octobot import VERSION -except ImportError: - VERSION = "unknown" - DEFAULT_NAME = "octobot_node" _BASE_CONFIG = dbos.DBOSConfig( name=DEFAULT_NAME, max_executor_threads=octobot_node.config.settings.SCHEDULER_MAX_EXECUTOR_THREADS, - application_version=VERSION, # octobot version + application_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, # executor_id=..., # a constant executor_id is required for DBOS workflow recovery: leave its init to DBOS ) @@ -69,6 +65,7 @@ class Scheduler: INSTANCE: dbos.DBOS = None # type: ignore AUTOMATION_WORKFLOW_QUEUE: dbos.Queue = None # type: ignore USER_ACTION_QUEUE: dbos.Queue = None # type: ignore + DBOS_CLEANUP_QUEUE: dbos.Queue = None # type: ignore @staticmethod def _wallet_filter_queue(queue_names: typing.Optional[list[str]]) -> octobot_node.enums.SchedulerQueues: @@ -122,7 +119,7 @@ def _get_dbos_workflow_id() -> typing.Optional[str]: if workflow_id := getattr(dbos.DBOS, "workflow_id", None): # group children workflows and parent workflows together # (a child workflow has the parent's workflow ID as a prefix) - return workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + return workflows_util.normalize_parent_automation_id(workflow_id) return None def is_enabled(self) -> bool: @@ -145,15 +142,23 @@ def start(self): self.logger.warning("Scheduler not initialized") def stop(self) -> None: - if self.INSTANCE: - self.INSTANCE.destroy() - self.logger.info("Scheduler stopped") - else: - self.logger.warning("Scheduler not initialized") + if not self.INSTANCE: + return + self.INSTANCE.destroy() + self.logger.info("Scheduler stopped") + Scheduler.INSTANCE = None + Scheduler.AUTOMATION_WORKFLOW_QUEUE = None + Scheduler.USER_ACTION_QUEUE = None + Scheduler.DBOS_CLEANUP_QUEUE = None def create_queues(self): self.AUTOMATION_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value) self.USER_ACTION_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value) + self.DBOS_CLEANUP_QUEUE = dbos.Queue( + name=octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, + # only one cleanup workflow can run at a time + concurrency=1, + ) async def get_periodic_tasks(self, user_id: typing.Optional[str] = None) -> list[octobot_node.models.Execution]: """DBOS scheduled workflows are not easily introspectable; return empty list.""" @@ -222,16 +227,16 @@ async def _get_parent_and_children_automation_workflows( user_id, statuses, [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], load_output ) parent_workflow_ids = set( - workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + workflows_util.normalize_parent_automation_id(workflow_id) for workflow_id in workflow_ids ) return [ workflow for workflow in all_workflows - if workflow.workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] in parent_workflow_ids + if workflows_util.normalize_parent_automation_id(workflow.workflow_id) in parent_workflow_ids ] - async def _get_parent_and_children_automation_workflow_ids( + async def get_parent_and_children_automation_workflow_ids( self, wallet_address: typing.Optional[str], workflow_ids: list[str], @@ -290,6 +295,64 @@ async def resolve_active_automation_workflow_ids_for_parent_id( latest_workflow = workflows_util.get_latest_child_workflow(matching_workflows) return [latest_workflow.workflow_id] + async def resolve_automation_owner_user_id( + self, + parent_id: str, + ) -> typing.Optional[str]: + """ + Return the Starfish ``user_id`` that owns the active automation for ``parent_id``. + + Unlike :meth:`resolve_active_automation_workflow_ids_for_parent_id`, this lookup is not + wallet-scoped so callers can resolve cross-wallet ownership after API-side authorization. + """ + matching_workflows = await self._get_parent_and_children_automation_workflows( + None, + [parent_id], + [ + dbos.WorkflowStatusString.ENQUEUED, + dbos.WorkflowStatusString.PENDING, + ], + load_output=False, + ) + if not matching_workflows: + return None + latest_workflow = workflows_util.get_latest_child_workflow(matching_workflows) + task = workflows_util.get_automation_input_task(latest_workflow) + if task is None: + return None + return task.user_id + + async def resolve_latest_terminal_automation_workflow_for_parent_id( + self, + user_id: typing.Optional[str], + parent_id: str, + ) -> typing.Optional[dbos.WorkflowStatus]: + """ + Return the latest terminal (SUCCESS/ERROR) child workflow for ``parent_id`` that has + parseable automation output state, or None when no prior execution exists. + """ + matching_workflows = await self._get_parent_and_children_automation_workflows( + user_id, + [parent_id], + [ + dbos.WorkflowStatusString.SUCCESS, + dbos.WorkflowStatusString.ERROR, + ], + load_output=True, + ) + if not matching_workflows: + return None + sorted_workflows = sorted( + matching_workflows, + key=workflows_util._automation_child_workflow_sort_key, + reverse=True, + ) + for workflow_status in sorted_workflows: + workflow_output = workflows_util.parse_automation_workflow_output(workflow_status) + if workflow_output is not None and workflow_output.state: + return workflow_status + return None + async def _get_latest_workflow_for_each_automation( self, user_id: typing.Optional[str], @@ -307,7 +370,7 @@ async def _get_latest_workflow_for_each_automation( async def cancel_workflows(self, workflow_ids: list[str]) -> list[str]: try: - to_cancel = await self._get_parent_and_children_automation_workflow_ids( + to_cancel = await self.get_parent_and_children_automation_workflow_ids( None, workflow_ids, [ @@ -322,37 +385,18 @@ async def cancel_workflows(self, workflow_ids: list[str]) -> list[str]: self.logger.exception(e, True, f"Failed to cancel workflows {workflow_ids}: {e}") return [] - async def _get_workflows_to_delete(self, workflow_ids: list[str]) -> list[str]: - automation_workflows = await self._get_parent_and_children_automation_workflow_ids( - None, - workflow_ids, - [ - dbos.WorkflowStatusString.SUCCESS, dbos.WorkflowStatusString.ERROR, - dbos.WorkflowStatusString.CANCELLED, dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED - ] - ) - user_action_workflows = await self._get_user_action_workflow_ids( - None, - workflow_ids, - [ - dbos.WorkflowStatusString.SUCCESS, dbos.WorkflowStatusString.ERROR, - dbos.WorkflowStatusString.CANCELLED, dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED - ], - load_output=True, - ) - return automation_workflows + user_action_workflows - async def delete_workflows(self, to_delete_workflow_ids: list[str]): - self.logger.info(f"Deleting {len(to_delete_workflow_ids)} workflows") - merged_to_delete_workflow_ids = await self._get_workflows_to_delete(to_delete_workflow_ids) + merged_to_delete_workflow_ids = await workflows_retention.get_workflows_to_delete( + self, + to_delete_workflow_ids, + ) self.logger.info( f"Including {len(merged_to_delete_workflow_ids) - len(to_delete_workflow_ids)} associated children workflows to delete" ) - await self.INSTANCE.delete_workflows_async(merged_to_delete_workflow_ids, delete_children=False) - self.logger.info(f"Vacuuming database") - with self.INSTANCE._sys_db.engine.begin() as conn: - conn.execute(sqlalchemy.text("VACUUM")) - self.logger.info(f"Database vacuum completed") + await workflows_retention.delete_workflows_and_vacuum( + self.INSTANCE, + merged_to_delete_workflow_ids, + ) async def get_scheduled_tasks(self, user_id: typing.Optional[str] = None) -> list[octobot_node.models.Execution]: """DBOS has no direct 'scheduled for later' queue; return empty list.""" @@ -553,16 +597,20 @@ def get_task_name(self, task_data: dict | octobot_node.models.Task | None, defau else: return default_value - async def get_automation_states(self, user_id: typing.Optional[str]) -> list[protocol_models.AutomationState]: + async def get_automation_states( + self, + user_id: typing.Optional[str], + statuses: typing.Optional[list[dbos.WorkflowStatusString]] = None, + ) -> list[protocol_models.AutomationState]: workflows = await self._get_latest_workflow_for_each_automation( - user_id, None, load_output=True + user_id, statuses, load_output=True ) sources: list[automations_protocol.AutomationStateSource] = [] for workflow in workflows: workflow_output = workflows_util.parse_automation_workflow_output(workflow) task = workflows_util.get_resolved_automation_task(workflow) if task: - task.id = workflow.workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + task.id = workflows_util.normalize_parent_automation_id(workflow.workflow_id) sources.append(automations_protocol.AutomationStateSource( task=task, workflow_status=workflow.status, diff --git a/packages/node/octobot_node/scheduler/schedules.py b/packages/node/octobot_node/scheduler/schedules.py new file mode 100644 index 0000000000..6246ffafe1 --- /dev/null +++ b/packages/node/octobot_node/scheduler/schedules.py @@ -0,0 +1,280 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import datetime +import typing + +import dbos +import dbos._croniter as dbos_croniter +import zoneinfo + +import octobot_commons.logging as logging +import octobot_node.constants as constants +import octobot_node.scheduler.scheduler as scheduler_module +import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow +import octobot_node.scheduler.workflows_retention as workflows_retention + + +_DATETIME_CLASS = datetime.datetime + + +def _workflow_status_string( + workflow_status: dbos.WorkflowStatus | dict[str, typing.Any], +) -> str: + if isinstance(workflow_status, dict): + return workflow_status["status"] + return workflow_status.status + + +class ScheduleWindowSlotClassification(typing.TypedDict): + missing: list[str] + terminal: list[tuple[str, str]] + in_progress: list[tuple[str, str]] + + +def get_backfill_schedule_default_anchor() -> datetime.datetime: + return ( + datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(days=constants.SCHEDULES_DEFAULT_BACKFILL_DAYS) + ) + + +def build_scheduled_workflow_id( + schedule_name: str, + trigger_time: datetime.datetime, +) -> str: + return f"sched-{schedule_name}-{trigger_time.isoformat()}" + + +def get_scheduled_workflow_trigger_time( + workflow_id: str, + schedule_name: str, +) -> str | None: + prefix = f"sched-{schedule_name}-" + if not workflow_id.startswith(prefix): + return None + return workflow_id[len(prefix):] + + +def _enumerate_schedule_workflow_ids_in_window( + schedule_name: str, + schedule_input: dbos.ScheduleInput, + start: datetime.datetime, + end: datetime.datetime, +) -> list[str]: + schedule_cron = schedule_input["schedule"] + cron_timezone = schedule_input.get("cron_timezone") + tz = zoneinfo.ZoneInfo(cron_timezone) if cron_timezone else datetime.timezone.utc + if start.tzinfo is None: + start = start.replace(tzinfo=datetime.timezone.utc) + if end.tzinfo is None: + end = end.replace(tzinfo=datetime.timezone.utc) + start_in_tz = start.astimezone(tz) + iterator = dbos_croniter.croniter(schedule_cron, start_in_tz, second_at_beginning=True) + workflow_ids: list[str] = [] + while True: + next_time = iterator.get_next(_DATETIME_CLASS) + if next_time >= end: + break + workflow_ids.append(build_scheduled_workflow_id(schedule_name, next_time)) + return workflow_ids + + +async def _classify_schedule_window_slots( + workflow_ids: list[str], +) -> ScheduleWindowSlotClassification: + missing: list[str] = [] + terminal: list[tuple[str, str]] = [] + in_progress: list[tuple[str, str]] = [] + for workflow_id in workflow_ids: + workflow_status = await dbos.DBOS.get_workflow_status_async(workflow_id) + if workflow_status is None: + missing.append(workflow_id) + elif workflows_retention.is_terminal_workflow(workflow_status): + terminal.append((workflow_id, _workflow_status_string(workflow_status))) + else: + in_progress.append((workflow_id, _workflow_status_string(workflow_status))) + return { + "missing": missing, + "terminal": terminal, + "in_progress": in_progress, + } + + +def _existing_schedule_matches_configured( + existing: dbos.WorkflowSchedule, + schedule_input: dbos.ScheduleInput, +) -> bool: + return ( + existing["schedule"] == schedule_input["schedule"] + and bool(existing.get("automatic_backfill")) + == schedule_input.get("automatic_backfill", False) + and existing.get("cron_timezone") == schedule_input.get("cron_timezone") + and existing.get("queue_name") == schedule_input.get("queue_name") + ) + + +async def _maybe_backfill_schedule_on_startup( + schedule_name: str, + schedule_input: dbos.ScheduleInput, +) -> None: + # DBOS automatic_backfill only runs when last_fired_at is already set (i.e. the + # cron fired at least once while the node was up). On first install or after long + # downtime with a NULL last_fired_at, missed executions are never enqueued unless + # we backfill explicitly here during register_schedules (after SCHEDULER.start()). + # + # last_fired_at stays NULL when cleanup only runs via startup backfill (not live + # cron), so we also check whether expected cron slots already have terminal + # workflows before calling backfill_schedule. + if not schedule_input.get("automatic_backfill", False): + return + existing_schedule = await dbos.DBOS.get_schedule_async(schedule_name) + if existing_schedule is None: + return + last_fired_at = existing_schedule.get("last_fired_at") + if last_fired_at: + # DBOS already backfills on launch when last_fired_at is known. + return + + logger = _get_logger() + start = get_backfill_schedule_default_anchor() + end = datetime.datetime.now(datetime.timezone.utc) + if start >= end: + return + + window_workflow_ids = _enumerate_schedule_workflow_ids_in_window( + schedule_name, + schedule_input, + start, + end, + ) + if not window_workflow_ids: + logger.info( + "Startup backfill not needed for schedule %s: no cron slots in window [%s, %s)", + schedule_name, + start.isoformat(), + end.isoformat(), + ) + return + + classifications = await _classify_schedule_window_slots(window_workflow_ids) + if not classifications["missing"] and not classifications["in_progress"]: + logger.info( + "Startup backfill not needed for schedule %s: last_fired_at unset but %s cron slot(s) in [%s, %s) already terminal", + schedule_name, + len(window_workflow_ids), + start.isoformat(), + end.isoformat(), + ) + for workflow_id, status in classifications["terminal"]: + logger.info( + "Schedule %s slot %s already %s", + schedule_name, + workflow_id, + status, + ) + return + + logger.info( + "Startup backfill for schedule %s: last_fired_at unset, checking missed cron slots in [%s, %s)", + schedule_name, + start.isoformat(), + end.isoformat(), + ) + workflow_statuses_before_backfill = await asyncio.gather( + *[ + dbos.DBOS.get_workflow_status_async(workflow_id) + for workflow_id in window_workflow_ids + ], + ) + statuses_before_backfill = dict( + zip(window_workflow_ids, workflow_statuses_before_backfill, strict=True), + ) + workflow_handles = await asyncio.to_thread( + dbos.DBOS.backfill_schedule, + schedule_name, + start, + end, + ) + enqueued_count = 0 + unchanged_count = 0 + for workflow_handle in workflow_handles: + workflow_id = workflow_handle.get_workflow_id() + status_before = statuses_before_backfill.get(workflow_id) + if status_before is None: + enqueued_count += 1 + logger.info( + "Startup backfill enqueued schedule %s workflow %s", + schedule_name, + workflow_id, + ) + else: + unchanged_count += 1 + logger.info( + "Startup backfill left schedule %s workflow %s unchanged (already %s)", + schedule_name, + workflow_id, + _workflow_status_string(status_before), + ) + logger.info( + "Startup backfill finished for schedule %s: %s enqueued, %s unchanged", + schedule_name, + enqueued_count, + unchanged_count, + ) + + +async def _ensure_schedule( + scheduler: scheduler_module.Scheduler, + schedule_input: dbos.ScheduleInput, +) -> None: + logger = _get_logger() + schedule_name = schedule_input["schedule_name"] + schedule_cron = schedule_input["schedule"] + existing_schedule = await dbos.DBOS.get_schedule_async(schedule_name) + if existing_schedule is None: + logger.info("Creating schedule %s (%s)", schedule_name, schedule_cron) + await dbos.DBOS.create_schedule_async( + schedule_name=schedule_name, + workflow_fn=schedule_input["workflow_fn"], + schedule=schedule_cron, + context=schedule_input.get("context"), + automatic_backfill=schedule_input.get("automatic_backfill", False), + cron_timezone=schedule_input.get("cron_timezone"), + queue_name=schedule_input.get("queue_name"), + ) + elif _existing_schedule_matches_configured(existing_schedule, schedule_input): + logger.info("Keeping existing schedule %s (%s)", schedule_name, schedule_cron) + else: + logger.info( + "Updating schedule %s (%s): configuration changed", + schedule_name, + schedule_cron, + ) + await scheduler.INSTANCE.apply_schedules_async([schedule_input]) + await _maybe_backfill_schedule_on_startup(schedule_name, schedule_input) + + +async def register_schedules(scheduler: scheduler_module.Scheduler) -> None: + schedule_inputs: list[dbos.ScheduleInput] = [ + dbos_cleanup_workflow.get_schedule_input(), + ] + for schedule_input in schedule_inputs: + await _ensure_schedule(scheduler, schedule_input) + + +def _get_logger() -> logging.BotLogger: + return logging.get_logger("schedules") diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py index 52216602d0..a3b01750ec 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py @@ -36,6 +36,7 @@ import octobot_node.scheduler.user_actions.user_actions_executor.account.refresh_accounts as user_actions_executor_refresh_accounts import octobot_node.scheduler.user_actions.user_actions_executor.automation.signal_automation as user_actions_executor_signal_automation import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as user_actions_executor_strategy_base +import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as user_actions_executor_restart_automation import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as user_actions_executor_stop_automation from octobot_node.scheduler.user_actions.user_action_post_actions import UserActionPostActions @@ -53,6 +54,7 @@ EditAutomationActionExecutor = user_actions_executor_edit_automation.EditAutomationActionExecutor SignalAutomationActionExecutor = user_actions_executor_signal_automation.SignalAutomationActionExecutor StopAutomationActionExecutor = user_actions_executor_stop_automation.StopAutomationActionExecutor +RestartAutomationActionExecutor = user_actions_executor_restart_automation.RestartAutomationActionExecutor CreateAccountActionExecutor = user_actions_executor_create_account.CreateAccountActionExecutor EditAccountActionExecutor = user_actions_executor_edit_account.EditAccountActionExecutor DeleteAccountActionExecutor = user_actions_executor_delete_account.DeleteAccountActionExecutor @@ -76,6 +78,7 @@ "EditAutomationActionExecutor", "SignalAutomationActionExecutor", "StopAutomationActionExecutor", + "RestartAutomationActionExecutor", "CreateAccountActionExecutor", "EditAccountActionExecutor", "DeleteAccountActionExecutor", diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py index d6a119a5fe..0ed2ec08cf 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/automation_user_action_executor.py @@ -71,6 +71,7 @@ def _get_error_message(self, exc: BaseException) -> protocol_models.AutomationAc node_errors.InvalidAutomationConfigurationError, node_errors.UnsupportedAutomationConfigurationTypeError, node_errors.UnsupportedUserActionConfigurationTypeError, + node_errors.UnrestartableAutomationError, ), ): return protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py index 300b77a1ce..9626fffe2f 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py @@ -155,26 +155,42 @@ def _create_automation_actions(self, user_action: protocol_models.UserAction) -> ) inner_configuration = _get_strategy_configuration_instance(stored_strategy) - account_id = _get_single_account_id(automation_configuration) + account_id = _resolve_automation_account_id( + automation_configuration, + inner_configuration, + ) - try: - protocol_account = collection_providers.AccountProvider.instance().get_item( - self._user_id, - account_id, - ) - except collection_errors.ItemNotFoundError as err: - raise node_errors.AccountNotFoundError( - f"Failed to load account {account_id!r} for address {self._user_id!r}: {err}" - ) from err + protocol_account = None + if account_id is not None: + try: + protocol_account = collection_providers.AccountProvider.instance().get_item( + self._user_id, + account_id, + ) + except collection_errors.ItemNotFoundError as err: + raise node_errors.AccountNotFoundError( + f"Failed to load account {account_id!r} for address {self._user_id!r}: {err}" + ) from err - init_action = action_details_factory.init_action_factory( - automation_id=automation_id, - protocol_account=protocol_account, - strategy_reference=automation_configuration.strategy, - stored_strategy=stored_strategy, - user_id=self._user_id, - reference_market=stored_strategy.reference_market, - ) + if protocol_account is not None: + init_action = action_details_factory.init_action_factory( + automation_id=automation_id, + protocol_account=protocol_account, + strategy_reference=automation_configuration.strategy, + stored_strategy=stored_strategy, + user_id=self._user_id, + reference_market=stored_strategy.reference_market, + ) + elif not _requires_account(inner_configuration): + # unlike others, generic process configuration does not require an account + init_action = action_details_factory.generic_process_metadata_init_action_factory( + automation_id=automation_id, + strategy_reference=automation_configuration.strategy, + ) + else: + raise node_errors.InvalidAutomationConfigurationError( + "Create automation requires AutomationConfiguration.accounts to contain exactly one account reference." + ) match inner_configuration: case protocol_models.TradingTentaclesConfiguration() as trading_configuration: @@ -202,6 +218,8 @@ def _create_automation_actions(self, user_action: protocol_models.UserAction) -> protocol_account, self._user_id, automation_id=automation_id, + strategy_id=stored_strategy.id, + octobot_name=automation_configuration.name, ), ] case protocol_models.CopyConfiguration() as copy_configuration: @@ -230,6 +248,7 @@ def _create_automation_actions(self, user_action: protocol_models.UserAction) -> stored_strategy.reference_market, stored_strategy, automation_id=automation_id, + octobot_name=automation_configuration.name, ), ] case _: @@ -254,8 +273,24 @@ def _get_create_automation_payload( return payload -def _get_single_account_id(automation_configuration: protocol_models.AutomationConfiguration) -> str: +def _requires_account(inner_configuration: typing.Any) -> bool: + return not isinstance(inner_configuration, protocol_models.GenericProcessConfiguration) + + +def _resolve_automation_account_id( + automation_configuration: protocol_models.AutomationConfiguration, + inner_configuration: typing.Any, +) -> str | None: accounts_list = list(automation_configuration.accounts or []) + if not _requires_account(inner_configuration): + if not accounts_list: + return None + if len(accounts_list) > 1: + raise node_errors.InvalidAutomationConfigurationError( + f"Create automation currently supports at most one account reference for " + f"generic_process strategies, got {len(accounts_list)}" + ) + return accounts_list[0].id if not accounts_list: raise node_errors.InvalidAutomationConfigurationError( "Create automation requires AutomationConfiguration.accounts to contain exactly one account reference." diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py new file mode 100644 index 0000000000..6847fcfab5 --- /dev/null +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py @@ -0,0 +1,172 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import json + +import octobot_flow.entities as flow_entities +import octobot_flow.enums as flow_enums +import octobot_protocol.models as protocol_models + +import octobot_node.errors as node_errors +import octobot_node.models as models +import octobot_node.scheduler as scheduler_module +import octobot_node.scheduler.task_context as task_context +import octobot_node.scheduler.user_actions.user_actions_executor.automation.automation_user_action_executor as automation_user_action_executor +import octobot_node.scheduler.user_actions.user_actions_executor.util.action_details_factory as action_details_factory +import octobot_node.scheduler.workflows_util as workflows_util + + +def _get_restart_automation_payload( + user_action: protocol_models.UserAction, +) -> protocol_models.RestartAutomationConfiguration: + wrapper = user_action.configuration + if wrapper is None or wrapper.actual_instance is None: + raise node_errors.InvalidUserActionPayloadError( + "UserAction.configuration must wrap a concrete restart-automation configuration." + ) + payload = wrapper.actual_instance + if not isinstance(payload, protocol_models.RestartAutomationConfiguration): + raise node_errors.InvalidUserActionPayloadError( + f"RestartAutomationActionExecutor expected RestartAutomationConfiguration, " + f"got {type(payload).__name__}" + ) + return payload + + +def _resolve_restart_reset_target_action_id( + automation_state: flow_entities.AutomationState, +) -> str: + actions_dag = automation_state.automation.actions_dag + for action in reversed(actions_dag.actions): + if action.id == action_details_factory._ACTION_ID_INIT: + continue + if isinstance(action, flow_entities.ConfiguredActionDetails): + if action.action == flow_enums.ActionType.APPLY_CONFIGURATION.value: + continue + if not action.can_be_reset(): + continue + elif not action.can_be_reset(): + continue + return action.id + raise node_errors.UnrestartableAutomationError( + "No resettable automation action found in the latest execution state." + ) + + +def prepare_automation_state_for_restart( + automation_state: flow_entities.AutomationState, +) -> flow_entities.AutomationState: + automation_state.automation.post_actions.stop_automation = False + automation_state.automation.execution.execution_error = None + reset_target_action_id = _resolve_restart_reset_target_action_id(automation_state) + automation_state.automation.actions_dag.reset_to(reset_target_action_id) + return automation_state + + +def _task_content_json_from_prepared_state( + automation_state: flow_entities.AutomationState, +) -> str: + return json.dumps({"state": automation_state.to_dict(include_default_values=False)}) + + +class RestartAutomationActionExecutor(automation_user_action_executor.AutomationUserActionExecutor): + async def _id_binds_to_user_action(self, restart_id: str) -> bool: + listed_user_actions = await scheduler_module.SCHEDULER.list_user_actions( + self._user_id, + active_only=False, + ) + user_action_ids = {user_action.id for user_action in listed_user_actions} + return restart_id in user_action_ids + + async def _assert_automation_not_running(self, parent_automation_id: str) -> None: + active_workflow_ids = await scheduler_module.SCHEDULER.resolve_active_automation_workflow_ids_for_parent_id( + self._user_id, + parent_automation_id, + ) + if active_workflow_ids: + raise node_errors.UnrestartableAutomationError( + f"Automation {parent_automation_id!r} is still running " + f"(active workflows: {active_workflow_ids!r})." + ) + + async def _build_restart_task(self, parent_automation_id: str) -> models.Task: + latest_workflow = ( + await scheduler_module.SCHEDULER.resolve_latest_terminal_automation_workflow_for_parent_id( + self._user_id, + parent_automation_id, + ) + ) + if latest_workflow is None: + raise node_errors.UnrestartableAutomationError( + f"No prior terminal execution found for automation {parent_automation_id!r}." + ) + workflow_output = workflows_util.parse_automation_workflow_output(latest_workflow) + if workflow_output is None or not workflow_output.state: + raise node_errors.UnrestartableAutomationError( + f"Latest execution for automation {parent_automation_id!r} has no usable output state." + ) + input_task = workflows_util.get_automation_input_task(latest_workflow) + task_name = input_task.name if input_task is not None else None + with task_context.encrypted_task( + models.Task( + content=workflow_output.state, + content_metadata=workflow_output.state_metadata, + ) + ): + automation_state_dict = workflows_util.get_automation_dict(workflow_output.state)[ + workflows_util.STATE_KEY + ] + automation_state = flow_entities.AutomationState.from_dict(automation_state_dict) + prepared_state = prepare_automation_state_for_restart(automation_state) + task_content = _task_content_json_from_prepared_state(prepared_state) + try: + next_workflow_id = workflows_util.build_next_child_automation_workflow_id( + latest_workflow.workflow_id + ) + except ValueError as error: + raise node_errors.UnrestartableAutomationError( + f"Cannot derive restart workflow id from latest execution " + f"{latest_workflow.workflow_id!r}: {error}" + ) from error + return models.Task( + id=next_workflow_id, + name=task_name, + content=task_content, + content_metadata=workflow_output.state_metadata, + type=models.TaskType.EXECUTE_ACTIONS.value, + user_id=self._user_id, + ) + + async def _do_execute( + self, + user_action: protocol_models.UserAction, + ) -> None: + if not scheduler_module.is_initialized(): + raise RuntimeError("Scheduler is not initialized") + + restart_payload = _get_restart_automation_payload(user_action) + parent_automation_id = workflows_util.normalize_parent_automation_id(restart_payload.id) + if await self._id_binds_to_user_action(restart_payload.id): + raise node_errors.UnrestartableAutomationError( + f"Restart id {restart_payload.id!r} binds to a user action and cannot be restarted." + ) + await self._assert_automation_not_running(parent_automation_id) + restart_task = await self._build_restart_task(parent_automation_id) + self.post_actions.to_create_automation_task = restart_task + self._mark_user_action_completed( + user_action, + created_automation_id=parent_automation_id, + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py index 0002d02b17..792fda9cc6 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/create_strategy.py @@ -19,6 +19,7 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor +import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation def _get_create_strategy_payload( @@ -46,6 +47,9 @@ async def _do_execute( user_action: protocol_models.UserAction, ) -> None: create_payload = _get_create_strategy_payload(user_action) + strategy_profile_validation.validate_profile_strategy_configuration( + create_payload.configuration + ) collection_providers.StrategyProvider.instance().create_item( self._user_id, create_payload.configuration, diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py index 4255d277c0..1e0e0765b2 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/edit_strategy.py @@ -19,6 +19,7 @@ import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as strategy_user_action_executor +import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_profile_validation as strategy_profile_validation def _get_edit_strategy_payload( @@ -54,6 +55,9 @@ async def _do_execute( raise node_errors.InvalidUserActionPayloadError( "EditStrategyConfiguration.id must match configuration.id." ) + strategy_profile_validation.validate_profile_strategy_configuration( + edit_payload.configuration + ) collection_providers.StrategyProvider.instance().update_item( self._user_id, edit_payload.configuration, diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/strategy_profile_validation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/strategy_profile_validation.py new file mode 100644 index 0000000000..8877e0d621 --- /dev/null +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/strategy/strategy_profile_validation.py @@ -0,0 +1,60 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models +import octobot_protocol.models.generic_process_configuration as generic_process_configuration + +import octobot_node.errors as node_errors + + +def validate_profile_strategy_configuration( + strategy: protocol_models.Strategy, +) -> None: + """ + Keep strategy.id and embedded profile_data.profile_details.id aligned for + generic-process strategies. + + GenericProcessConfiguration can carry OctoBot profile JSON in profile_data. + Downstream sync/runtime code uses profile_details.id for profile paths and + identity; the stored strategy's canonical id is strategy.id. Clients may omit + profile_details.id or send a stale id from another profile, so normalize or + reject before create/edit persistence. + """ + configuration = strategy.configuration + if configuration is None or configuration.actual_instance is None: + return + if not isinstance( + configuration.actual_instance, + generic_process_configuration.GenericProcessConfiguration, + ): + return + generic_configuration = configuration.actual_instance + if generic_configuration.profile_data is None: + return + profile_details = dict( + generic_configuration.profile_data.get("profile_details") or {} + ) + profile_id = profile_details.get("id") + if profile_id is None: + # Client omitted profile_details.id: default to the strategy id before storage. + profile_details["id"] = strategy.id + generic_configuration.profile_data["profile_details"] = profile_details + return + if profile_id != strategy.id: + # Conflicting ids would bind the wrong profile at runtime; reject the payload. + raise node_errors.InvalidUserActionPayloadError( + "profile_data.profile_details.id must match strategy.id." + ) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py index e340354337..29a32a3ff1 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py @@ -43,6 +43,8 @@ def user_action_executor_factory( return user_actions_executor_package.EditAutomationActionExecutor case protocol_models.StopAutomationConfiguration: return user_actions_executor_package.StopAutomationActionExecutor + case protocol_models.RestartAutomationConfiguration: + return user_actions_executor_package.RestartAutomationActionExecutor case protocol_models.SignalAutomationConfiguration: return user_actions_executor_package.SignalAutomationActionExecutor case protocol_models.CreateAccountConfiguration: diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py index edc2f7b086..2f4fb8723d 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py @@ -196,13 +196,16 @@ async def _check_exchange_account_state( ), ) tentacles_setup_config = tentacles_manager_api.get_full_tentacles_setup_config() - async with trading_exchanges.exchange_manager_from_exchange_data( - exchange_data, - profile_data, - tentacles_setup_config, - price_fallback=None, - ) as exchange_manager: - return await _check_exchange_manager_state(exchange_manager, account) + try: + async with trading_exchanges.exchange_manager_from_exchange_data( + exchange_data, + profile_data, + tentacles_setup_config, + price_fallback=None, + ) as exchange_manager: + return await _check_exchange_manager_state(exchange_manager, account) + except Exception as error: + return _account_check_failure_result(error) async def _check_exchange_manager_state( @@ -224,22 +227,30 @@ async def _check_exchange_manager_state( ), assets, ) - except trading_errors.RetriableFailedRequest: - raise - except trading_errors.InvalidAPIKeyIPWhitelistError: + except Exception as error: + return _account_check_failure_result(error, permissions=permissions) + + +def _account_check_failure_result( + error: BaseException, + *, + permissions: list[protocol_models.AccountPermission] | None = None, +) -> tuple[protocol_models.AccountState, None]: + if isinstance(error, trading_errors.RetriableFailedRequest): + raise error + if isinstance(error, trading_errors.InvalidAPIKeyIPWhitelistError): return _invalid_state(protocol_models.AccountStatusMessage.INVALID_API_IP_WHITELIST), None - except trading_errors.InvalidAPIKeyPermissionsError as permissions_error: + if isinstance(error, trading_errors.InvalidAPIKeyPermissionsError): return _invalid_state_from_permissions_error(permissions), None - except trading_errors.AuthenticationError: + if isinstance(error, trading_errors.AuthenticationError): return _invalid_state( protocol_models.AccountStatusMessage.INVALID_API_KEYS, permissions=[], ), None - except Exception: - return _invalid_state( - protocol_models.AccountStatusMessage.INTERNAL_SERVER_ERROR, - permissions=permissions, - ), None + return _invalid_state( + protocol_models.AccountStatusMessage.INTERNAL_SERVER_ERROR, + permissions=permissions, + ), None def _balance_currency_holdings(balance: dict) -> list[tuple[str, float, float]]: diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py index 4f8099b94c..190a8006e1 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/action_details_factory.py @@ -30,6 +30,14 @@ def _run_octobot_process_recall_kwarg_segments() -> list[str]: ] +def _run_octobot_process_octobot_name_kwarg_segment(octobot_name: str | None) -> str | None: + if octobot_name and str(octobot_name).strip(): + return ( + f"octobot_name={dsl_interpreter.format_parameter_value(str(octobot_name).strip())}" + ) + return None + + def _protocol_account_updated_at_unix_seconds(protocol_account: protocol_models.Account) -> float: moment = protocol_account.updated_at if moment.tzinfo is None: @@ -197,6 +205,32 @@ def init_action_factory( ) +def generic_process_metadata_init_action_factory( + *, + automation_id: str, + strategy_reference: protocol_models.StrategyReference, +) -> flow_entities.AbstractActionDetails: + """ + Build a minimal APPLY_CONFIGURATION action for account-less generic-process automations: + automation metadata only (no exchange_account_details or portfolio seeding). + """ + automation_metadata = flow_entities.AutomationMetadata( + automation_id=automation_id, + strategy_id=strategy_reference.id, + emit_signals=bool(strategy_reference.emit_signals), + strategy_version=strategy_reference.version, + ) + automation_details = flow_entities.AutomationDetails(metadata=automation_metadata) + init_config = { + "automation": automation_details.to_dict(include_default_values=False), + } + return flow_entities.ConfiguredActionDetails( + id=_ACTION_ID_INIT, + action=flow_enums.ActionType.APPLY_CONFIGURATION.value, + config=init_config, + ) + + def copy_action_factory( init_action: flow_entities.AbstractActionDetails, copy_configuration: protocol_models.CopyConfiguration, @@ -415,6 +449,7 @@ def market_making_action_factory( stored_strategy: protocol_models.Strategy, *, automation_id: str, + octobot_name: str | None = None, ) -> flow_entities.AbstractActionDetails: profile_data = market_making_profile_data_factory( protocol_account=protocol_account, @@ -430,12 +465,15 @@ def market_making_action_factory( user_id, ) exchange_auth_segment = dsl_interpreter.format_parameter_value(exchange_auth_data) - run_dsl = ( - "run_octobot_process(" - f"{automation_id!r}, {dsl_interpreter.format_parameter_value(profile_data_dict)}, " - f"{exchange_auth_segment}, " - f"{', '.join(_run_octobot_process_recall_kwarg_segments())})" - ) + dsl_arguments = [ + f"{automation_id!r}", + f"profile_data={dsl_interpreter.format_parameter_value(profile_data_dict)}", + f"exchange_auth_data={exchange_auth_segment}", + ] + if octobot_name_segment := _run_octobot_process_octobot_name_kwarg_segment(octobot_name): + dsl_arguments.append(octobot_name_segment) + dsl_arguments.extend(_run_octobot_process_recall_kwarg_segments()) + run_dsl = "run_octobot_process(" + ", ".join(dsl_arguments) + ")" return flow_entities.DSLScriptActionDetails( id=_action_id_from_configuration(market_making_configuration), dsl_script=run_dsl, @@ -446,24 +484,33 @@ def market_making_action_factory( def generic_process_action_factory( init_action: flow_entities.AbstractActionDetails, generic_process_configuration: protocol_models.GenericProcessConfiguration, - protocol_account: protocol_models.Account, + protocol_account: protocol_models.Account | None, user_id: str, *, automation_id: str, + strategy_id: str | None = None, + octobot_name: str | None = None, ) -> flow_entities.AbstractActionDetails: - exchange_auth_data = _exchange_auth_data_list_from_protocol_account( - protocol_account, - user_id, - ) - dsl_arguments = [f"{automation_id!r}"] - if generic_process_configuration.profile_data is not None: + exchange_auth_data = None + if protocol_account is not None: + exchange_auth_data = _exchange_auth_data_list_from_protocol_account( + protocol_account, + user_id, + ) + dsl_arguments = [f"{automation_id!r}", f"user_id={user_id!r}"] + if strategy_id is not None: + dsl_arguments.append(f"sync_profile_id={strategy_id!r}") + elif generic_process_configuration.profile_data is not None: dsl_arguments.append( - dsl_interpreter.format_parameter_value(generic_process_configuration.profile_data) + f"profile_data={dsl_interpreter.format_parameter_value(generic_process_configuration.profile_data)}" ) if exchange_auth_data is not None: dsl_arguments.append( f"exchange_auth_data={dsl_interpreter.format_parameter_value(exchange_auth_data)}" ) + octobot_name_segment = _run_octobot_process_octobot_name_kwarg_segment(octobot_name) + if octobot_name_segment is not None: + dsl_arguments.append(octobot_name_segment) dsl_arguments.extend(_run_octobot_process_recall_kwarg_segments()) run_dsl = "run_octobot_process(" + ", ".join(dsl_arguments) + ")" return flow_entities.DSLScriptActionDetails( diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/trading_tentacles_config.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/trading_tentacles_config.py index 65b055901f..319a864e56 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/trading_tentacles_config.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/trading_tentacles_config.py @@ -14,16 +14,21 @@ # You should have received a copy of the GNU General Public License along # with OctoBot. If not, see . +import collections.abc import functools import typing import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.str_util as str_util import octobot_protocol.models as protocol_models import octobot_node.errors as node_errors +_SIMPLE_SCALAR_TYPES = (bool, int, float, str) + + def normalize_tentacle_name(tentacle_name: str) -> str: if not tentacle_name: return tentacle_name @@ -131,23 +136,11 @@ def _non_strategy_evaluator_operators_by_tentacle_name() -> dict[str, type]: return evaluator_operators_by_name -def _is_value_castable_to_parameter_type(value: typing.Any, param_type: type) -> bool: - if isinstance(value, param_type): - return True - if param_type is int: +def _is_value_castable_to_parameter_type(value: typing.Any, param_type: str) -> bool: + if param_type == commons_enums.DslValueType.NUMBER.value: if isinstance(value, bool): return False - if isinstance(value, float) and value.is_integer(): - return True - if isinstance(value, str): - try: - int(value) - except ValueError: - return False - return True - return False - if param_type is float: - if isinstance(value, int): + if isinstance(value, (int, float)): return True if isinstance(value, str): try: @@ -156,18 +149,34 @@ def _is_value_castable_to_parameter_type(value: typing.Any, param_type: type) -> return False return True return False - if param_type is bool: + if param_type == commons_enums.DslValueType.BOOLEAN.value: return isinstance(value, bool) - if param_type is str: + if param_type in ( + commons_enums.DslValueType.TEXT.value, + commons_enums.DslValueType.TIME_FRAME.value, + ): return isinstance(value, str) - if param_type is list: - return isinstance(value, list) - if param_type is dict: + if param_type == commons_enums.DslValueType.DICT.value: return isinstance(value, dict) + if param_type == commons_enums.DslValueType.ANY.value: + return True + if param_type == commons_enums.DslValueType.SERIES.value: + if isinstance(value, (str, bytes, dict)): + return False + return isinstance(value, collections.abc.Iterable) + if param_type in ( + commons_enums.DslValueType.SIGNAL.value, + commons_enums.DslValueType.ORDER.value, + ): + if value is None or isinstance(value, dict): + return True + if isinstance(value, (list, tuple)) or isinstance(value, _SIMPLE_SCALAR_TYPES): + return False + return True return False -def _parameter_names_by_operator(operator_class: type) -> dict[str, type]: +def _parameter_names_by_operator(operator_class: type) -> dict[str, str]: return { operator_parameter.name: operator_parameter.type for operator_parameter in operator_class.get_parameters() @@ -198,7 +207,7 @@ def _collect_config_validation_issues( if not _is_value_castable_to_parameter_type(config_value, expected_type): configuration_issues.append( f"{parameter_json_path}: value {config_value!r} is not castable to " - f"{expected_type.__name__}; expected type: {expected_type.__name__}" + f"{expected_type}; expected type: {expected_type}" ) return configuration_issues diff --git a/packages/node/octobot_node/scheduler/workflows/__init__.py b/packages/node/octobot_node/scheduler/workflows/__init__.py index 7b4fb564d6..624a4184a5 100644 --- a/packages/node/octobot_node/scheduler/workflows/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/__init__.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. -def register_workflows(): + +def register_workflows() -> None: import octobot_node.scheduler.workflows.automation_workflow import octobot_node.scheduler.workflows.user_action_workflow + import octobot_node.scheduler.workflows.dbos_cleanup_workflow diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index ca2c04a230..a9d38e5a55 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -20,13 +20,13 @@ import octobot_commons.logging -import octobot.community.authentication as community_authentication import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_trading.errors import octobot_flow.entities import octobot_flow.enums import octobot_flow.errors +import octobot_flow.repositories.community.community_repository as community_repository import octobot_node.enums import octobot_node.models @@ -40,26 +40,7 @@ from octobot_node.scheduler import SCHEDULER # avoid circular import - -def _user_id_to_evm(user_id: typing.Optional[str]) -> typing.Optional[str]: - """Return the EVM wallet address for a Starfish *user_id*, or None if unresolvable. - - The community repository (used inside OctoBotActionsJob) requires the EVM wallet - address for the community sync client. We store Starfish user_id in Task.user_id - (the sync-core identity), so we must translate back to the EVM address at the - automation-job boundary. - """ - if user_id is None: - return None - try: - return community_authentication.CommunityAuthentication.instance().get_wallet_by_user_id( - user_id - ).address - except Exception as err: - octobot_commons.logging.get_logger("AutomationWorkflow").warning( - f"Could not resolve EVM address for user_id={user_id!r}: {err}" - ) - return None +WORKFLOW_NAME = "execute_automation" @SCHEDULER.INSTANCE.dbos_class() @@ -67,7 +48,7 @@ class AutomationWorkflow: # Always use dict as input to parse minimizable dataclasses and facilitate data format updates @staticmethod - @SCHEDULER.INSTANCE.workflow(name="execute_automation") + @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) async def execute_automation(inputs: dict) -> typing.Optional[str]: """ Automation workflow runner: @@ -174,8 +155,9 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) AutomationWorkflow._log_iteration_execution_intent( parsed_inputs, user_actions, trading_signals ) + action_job = None try: - await octobot_flow_client.OctoBotActionsJob( + action_job = octobot_flow_client.OctoBotActionsJob( parsed_inputs.task.content, user_actions, trading_signals, @@ -183,24 +165,65 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) # CommunityRepository (used inside the job) needs the EVM wallet # address, not the Starfish user_id — derive it from the task # identity so the community sync client resolves correctly. - wallet_address=_user_id_to_evm(parsed_inputs.task.user_id), - ).run() - except octobot_flow.errors.CommunityTradingSignalError as err: - execution_error = octobot_flow.enums.ActionErrorStatus.NO_TRADING_SIGNAL.value - execution_error_message = str(err) - except octobot_trading.errors.AuthenticationError as err: + wallet_address=community_repository.CommunityRepository.user_id_to_evm( + parsed_inputs.task.user_id + ), + ) + await action_job.run() + except octobot_trading.errors.RetriableFailedRequest as err: + # instantly retriable errors, retry immediately + AutomationWorkflow.get_logger(parsed_inputs).exception( + err, True, f"Retriable error while running automation job: {err}" + ) + raise + except octobot_flow.errors.PendingPriorityActionsSkippedError as err: + # don't retry, just skip the iteration AutomationWorkflow.get_logger(parsed_inputs).error( - f"Authentication error: {err} ({err.__class__.__name__})" + f"Pending priority actions were skipped: {err}" + ) + if action_job is None: + # should never happen, but just in case + raise + next_step_at = octobot_flow_client.OctoBotActionsJobDescription.get_next_execution_time( + action_job.description.state ) - execution_error = octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value - execution_error_message = str(err) postponed_iteration = True - next_step_at = time.time() + constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS has_next_actions_override = True next_iteration_description_override = parsed_inputs.task.content next_iteration_description_metadata_override = parsed_inputs.task.content_metadata + except ( + octobot_trading.errors.AuthenticationError, + octobot_trading.errors.PortfolioNegativeValueError, + octobot_trading.errors.FailedRequest, + octobot_trading.errors.MissingFunds, + octobot_trading.errors.MissingMinimalExchangeTradeVolume, + ) as err: + # postponing errors, retry after a delay + AutomationWorkflow.get_logger(parsed_inputs).error( + f"{err.__class__.__name__} error (postponed iteration): {err}" + ) + execution_error_status, postpone_delay_seconds = ( + AutomationWorkflow._get_postponed_iteration_error_status_and_delay(err) + ) + next_step_at = time.time() + postpone_delay_seconds + execution_error = execution_error_status.value + execution_error_message = str(err) + postponed_iteration = True + has_next_actions_override = True + next_iteration_description_override = workflows_util.patch_task_content_degraded_state( + parsed_inputs.task.content, + execution_error, + execution_error_message, + since=time.time(), + ) + next_iteration_description_metadata_override = parsed_inputs.task.content_metadata + except octobot_flow.errors.CommunityTradingSignalError as err: + # Stop cases: don't forward error, just stop the workflow + execution_error = octobot_flow.enums.ActionErrorStatus.NO_TRADING_SIGNAL.value + execution_error_message = str(err) except Exception as err: - # log propagated errors to also associate them to the automation's error tracking + # use retry policy & log propagated errors to also associate + # them to the automation's error tracking AutomationWorkflow.get_logger(parsed_inputs).exception( err, True, f"Error while running automation job: {err}" ) @@ -224,7 +247,9 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) if result.actions_dag: next_actions = result.actions_dag.get_executable_actions() remaining_steps = len(result.actions_dag.get_pending_actions()) - next_step_at = result.next_actions_description.get_next_execution_time() if result.next_actions_description else None + next_step_at = octobot_flow_client.OctoBotActionsJobDescription.get_next_execution_time( + result.next_actions_description.state + ) if result.next_actions_description else None next_step = AutomationWorkflow._get_actions_summary(next_actions, minimal=True) next_actions_str = f"next immediate actions: {next_actions}" if next_actions else "all actions completed" AutomationWorkflow.get_logger(parsed_inputs).info( @@ -235,9 +260,10 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) result, ) else: + retry_delay_seconds = max(0.0, (next_step_at or time.time()) - time.time()) AutomationWorkflow.get_logger(parsed_inputs).info( - f"Iteration postponed after authentication error, retry scheduled in " - f"{constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS:.0f} seconds" + f"Iteration postponed ({execution_error}: {execution_error_message}), " + f"retry scheduled in {retry_delay_seconds:.0f} seconds" ) #### End of decryped task context - no clear data after this point in encrypted context #### @@ -375,14 +401,14 @@ async def _process_pending_priority_actions_and_reschedule( AutomationWorkflow.get_logger(parsed_inputs).info( f"Stopping workflow, should stop: {latest_iteration_result.progress_status.should_stop}" ) - else: - # successful iteration and a new iteration is required, schedule next iteration, don't return anything - await AutomationWorkflow._schedule_next_iteration( - parsed_inputs, - latest_iteration_result.next_iteration_description, # type: ignore - latest_iteration_result.progress_status, - latest_iteration_result.next_iteration_description_metadata, - ) + return False, latest_iteration_result + # successful iteration and a new iteration is required, schedule next iteration, don't return anything + await AutomationWorkflow._schedule_next_iteration( + parsed_inputs, + latest_iteration_result.next_iteration_description, # type: ignore + latest_iteration_result.progress_status, + latest_iteration_result.next_iteration_description_metadata, + ) return True, latest_iteration_result @staticmethod @@ -413,14 +439,13 @@ async def _schedule_next_iteration( def _get_next_child_workflow_id() -> str: workflow_id = dbos.DBOS.workflow_id if workflow_id is None: - raise errors.WorkflowInputError("Missing current workflow ID while scheduling next iteration.") - parent_workflow_id = workflow_id[:constants.PARENT_WORKFLOW_ID_LENGTH] + raise errors.WorkflowInputError( + "Missing current workflow ID while scheduling next iteration." + ) try: - current_child_id = workflows_util.parse_automation_child_workflow_index(workflow_id) + return workflows_util.build_next_child_automation_workflow_id(workflow_id) except ValueError as error: raise errors.WorkflowInputError(str(error)) from error - next_child_id = current_child_id + 1 - return f"{parent_workflow_id}_{next_child_id}" @staticmethod def _create_next_iteration_inputs( @@ -477,3 +502,16 @@ def get_logger(parsed_inputs: params.AutomationWorkflowInputs) -> octobot_common return octobot_commons.logging.get_logger( parsed_inputs.task.name or AutomationWorkflow.__name__ ) + + @staticmethod + def _get_postponed_iteration_error_status_and_delay(error: Exception) -> tuple[ + octobot_flow.enums.ActionErrorStatus, float + ]: + if isinstance(error, octobot_trading.errors.AuthenticationError): + return octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR, constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS + if isinstance(error, octobot_trading.errors.MissingFunds): + return octobot_flow.enums.ActionErrorStatus.NOT_ENOUGH_FUNDS, constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS + if isinstance(error, octobot_trading.errors.MissingMinimalExchangeTradeVolume): + return octobot_flow.enums.ActionErrorStatus.INVALID_ORDER, constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS + # other errors, like PortfolioNegativeValueError and FailedRequest + return octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS diff --git a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py new file mode 100644 index 0000000000..e2b0a9ecd2 --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py @@ -0,0 +1,75 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import datetime +import typing + +import dbos + +import octobot_commons.logging as logging +import octobot_node.enums +import octobot_node.scheduler.workflows_retention as workflows_retention + +from octobot_node.scheduler import SCHEDULER # avoid circular import + +WORKFLOW_NAME = "dbos_cleanup" +SCHEDULE_NAME = "dbos_cleanup_daily" +SCHEDULE_CRON = "0 0 * * *" # daily at midnight UTC + + +@SCHEDULER.INSTANCE.dbos_class() +class DbosCleanupWorkflow: + @staticmethod + @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + async def dbos_cleanup( + scheduled_time: datetime.datetime, + context: typing.Any, + ) -> dict[str, typing.Any]: + return await DbosCleanupWorkflow._cleanup_outdated_automation_executions( + scheduled_time, + context, + ) + + @staticmethod + @SCHEDULER.INSTANCE.step(name="cleanup_outdated_automation_executions") + async def _cleanup_outdated_automation_executions( + scheduled_time: datetime.datetime, + context: typing.Any, + ) -> dict[str, typing.Any]: + logger = logging.get_logger(DbosCleanupWorkflow.__name__) + if workflows_retention.should_skip_retention_cleanup_on_this_node(): + logger.info("dbos_cleanup skipped: consumer-only node") + return dict(workflows_retention.EMPTY_CLEANUP_SUMMARY) + if await workflows_retention.should_skip_retention_cleanup_for_scheduled_time( + SCHEDULER, + scheduled_time, + ): + logger.info( + "dbos_cleanup skipped for scheduled_time %s: latest completed cleanup is newer", + scheduled_time.isoformat(), + ) + return dict(workflows_retention.EMPTY_CLEANUP_SUMMARY) + return await workflows_retention.cleanup_outdated_automation_executions(SCHEDULER) + + +def get_schedule_input() -> dbos.ScheduleInput: + return { + "schedule_name": SCHEDULE_NAME, + "workflow_fn": DbosCleanupWorkflow.dbos_cleanup, + "schedule": SCHEDULE_CRON, + "context": None, + "automatic_backfill": True, + "queue_name": octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, + } diff --git a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py index d22b5ed4b1..5d5d892b19 100644 --- a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py @@ -17,6 +17,7 @@ import octobot_protocol.models as protocol_models import octobot_trading.errors as trading_errors +import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_node.errors as errors import octobot_node.scheduler.workflows.params as params import octobot_node.constants as constants @@ -54,6 +55,7 @@ def _should_retry(error: BaseException) -> bool: errors.UserActionError, pydantic.ValidationError, trading_errors.AuthenticationError, # includes credential / IP-whitelist subclasses + collection_errors.DuplicateItemError, )) @staticmethod @@ -79,7 +81,7 @@ async def _execute_user_action(inputs: dict) -> dict: executor = executor_class(parsed_inputs.user_id) try: await executor.execute(parsed_user_action) - except errors.UserActionError: + except (errors.UserActionError, collection_errors.DuplicateItemError): if parsed_user_action.status != protocol_models.UserActionStatus.FAILED: raise return params.UserActionExecutionResult( diff --git a/packages/node/octobot_node/scheduler/workflows_retention.py b/packages/node/octobot_node/scheduler/workflows_retention.py new file mode 100644 index 0000000000..01886d0e6f --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows_retention.py @@ -0,0 +1,248 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import datetime +import os +import time +import typing + +import dbos +import sqlalchemy + +import octobot_commons.logging as logging +import octobot_node.config +import octobot_node.enums +import octobot_node.scheduler.workflows_util as workflows_util + +if typing.TYPE_CHECKING: + import octobot_node.scheduler.scheduler as scheduler_module + +AUTOMATION_EXECUTION_RETENTION_SECONDS = float( + os.getenv("AUTOMATION_EXECUTION_RETENTION_SECONDS", 60 * 60 * 24 * 2) +) # 2 days +AUTOMATION_EXECUTIONS_TO_KEEP = 2 + +EMPTY_CLEANUP_SUMMARY: dict[str, typing.Any] = { + "deleted_by_automation": {}, + "deleted_cleanup_executions": 0, + "total_deleted": 0, +} + +_TERMINAL_WORKFLOW_STATUS_VALUES = frozenset( + workflow_status.value + for workflow_status in workflows_util.get_user_action_terminal_workflow_statuses() +) + +def should_skip_retention_cleanup_on_this_node() -> bool: + return octobot_node.config.settings.CONSUMER_ONLY + + +def is_terminal_workflow(workflow_status: dbos.WorkflowStatus) -> bool: + status = ( + workflow_status["status"] + if isinstance(workflow_status, dict) + else workflow_status.status + ) + return status in _TERMINAL_WORKFLOW_STATUS_VALUES + + +def _retention_cutoff_ms(*, retention_seconds: float, now_ms: int) -> int: + return now_ms - int(retention_seconds * 1000) + + +def get_outdated_automation_execution_deletions( + workflows: list[dbos.WorkflowStatus], + *, + retention_seconds: float, + now_ms: int, + executions_to_keep: int = AUTOMATION_EXECUTIONS_TO_KEEP, +) -> dict[str, list[str]]: + deletions_by_automation: dict[str, list[str]] = {} + cutoff_ms = _retention_cutoff_ms(retention_seconds=retention_seconds, now_ms=now_ms) + by_parent = workflows_util.get_workflows_by_parent_id(workflows) + for parent_id, parent_workflows in by_parent.items(): + sorted_workflows = sorted( + parent_workflows, + key=workflows_util._automation_child_workflow_sort_key, + reverse=True, + ) + protected_ids = { + workflow_status.workflow_id + for workflow_status in sorted_workflows[:executions_to_keep] + } + parent_deletions: list[str] = [] + for workflow_status in sorted_workflows[executions_to_keep:]: + if workflow_status.workflow_id in protected_ids: + continue + if not is_terminal_workflow(workflow_status): + continue + if (workflow_status.updated_at or 0) >= cutoff_ms: + continue + parent_deletions.append(workflow_status.workflow_id) + if parent_deletions: + deletions_by_automation[parent_id] = parent_deletions + return deletions_by_automation + + +def get_outdated_dbos_cleanup_execution_workflow_ids( + cleanup_workflows: list[dbos.WorkflowStatus], + *, + retention_seconds: float, + now_ms: int, +) -> list[str]: + cutoff_ms = _retention_cutoff_ms(retention_seconds=retention_seconds, now_ms=now_ms) + return [ + workflow_status.workflow_id + for workflow_status in cleanup_workflows + if is_terminal_workflow(workflow_status) + and (workflow_status.updated_at or 0) < cutoff_ms + ] + + +_TERMINAL_DELETE_WORKFLOW_STATUSES = [ + dbos.WorkflowStatusString.SUCCESS, + dbos.WorkflowStatusString.ERROR, + dbos.WorkflowStatusString.CANCELLED, + dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, +] + + +async def get_workflows_to_delete( + scheduler: "scheduler_module.Scheduler", + workflow_ids: list[str], +) -> list[str]: + automation_workflows = await scheduler.get_parent_and_children_automation_workflow_ids( + None, + workflow_ids, + _TERMINAL_DELETE_WORKFLOW_STATUSES, + ) + user_action_workflows = await scheduler._get_user_action_workflow_ids( + None, + workflow_ids, + _TERMINAL_DELETE_WORKFLOW_STATUSES, + load_output=True, + ) + return automation_workflows + user_action_workflows + + +def vacuum_dbos_system_database(dbos_instance: dbos.DBOS) -> None: + logger = _get_logger() + logger.info("Vacuuming database") + with dbos_instance._sys_db.engine.begin() as conn: + conn.execute(sqlalchemy.text("VACUUM")) + logger.info("Database vacuum completed") + + +async def delete_workflows_and_vacuum( + dbos_instance: dbos.DBOS, + workflow_ids: list[str] +) -> None: + _get_logger().info("Deleting %s workflows", len(workflow_ids)) + await dbos_instance.delete_workflows_async(workflow_ids, delete_children=False) + vacuum_dbos_system_database(dbos_instance) + + +async def cleanup_outdated_automation_executions( + scheduler: "scheduler_module.Scheduler", +) -> dict[str, typing.Any]: + if not scheduler.INSTANCE: + _get_logger().warning("Scheduler not initialized, skipping cleanup") + return dict(EMPTY_CLEANUP_SUMMARY) + now_ms = int(time.time() * 1000) + retention_seconds = AUTOMATION_EXECUTION_RETENTION_SECONDS + import octobot_node.scheduler.workflows.automation_workflow as automation_workflow + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow + automation_workflows = await scheduler.INSTANCE.list_workflows_async( + name=automation_workflow.WORKFLOW_NAME, + queue_name=[octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], + load_input=False, + load_output=False, + ) + cleanup_workflows = await scheduler.INSTANCE.list_workflows_async( + name=dbos_cleanup_workflow.WORKFLOW_NAME, + load_input=False, + load_output=False, + ) + deletions_by_automation = get_outdated_automation_execution_deletions( + automation_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + cleanup_execution_ids = get_outdated_dbos_cleanup_execution_workflow_ids( + cleanup_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + automation_execution_ids = [ + workflow_id + for workflow_ids in deletions_by_automation.values() + for workflow_id in workflow_ids + ] + all_ids_to_delete = automation_execution_ids + cleanup_execution_ids + summary = { + "deleted_by_automation": { + parent_id: len(workflow_ids) + for parent_id, workflow_ids in deletions_by_automation.items() + }, + "deleted_cleanup_executions": len(cleanup_execution_ids), + "total_deleted": len(all_ids_to_delete), + } + if all_ids_to_delete: + _get_logger().info( + "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs", + len(all_ids_to_delete), + len(deletions_by_automation), + len(cleanup_execution_ids), + ) + await delete_workflows_and_vacuum( + scheduler.INSTANCE, + all_ids_to_delete, + ) + _get_logger().info("DBOS cleanup summary: %s", summary) + return summary + +def _get_latest_completed_cleanup_timestamp_ms( + cleanup_workflows: list[dbos.WorkflowStatus], +) -> int: + if not cleanup_workflows: + return 0 + workflow_status = cleanup_workflows[0] + return workflow_status.updated_at or workflow_status.created_at or 0 + + +async def should_skip_retention_cleanup_for_scheduled_time( + scheduler: "scheduler_module.Scheduler", + scheduled_time: datetime.datetime, +) -> bool: + if not scheduler.is_initialized(): + return True + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow + cleanup_workflows = await scheduler.INSTANCE.list_workflows_async( + name=dbos_cleanup_workflow.WORKFLOW_NAME, + status=[dbos.WorkflowStatusString.SUCCESS.value], + sort_desc=True, + limit=1, + load_input=False, + load_output=False, + ) + latest_timestamp_ms = _get_latest_completed_cleanup_timestamp_ms(cleanup_workflows) + if latest_timestamp_ms == 0: + return False + scheduled_timestamp_ms = int(scheduled_time.timestamp() * 1000) + return latest_timestamp_ms > scheduled_timestamp_ms + + +def _get_logger() -> logging.BotLogger: + return logging.get_logger("workflows_retention") diff --git a/packages/node/octobot_node/scheduler/workflows_util.py b/packages/node/octobot_node/scheduler/workflows_util.py index 79b2b5b620..46a1730faf 100644 --- a/packages/node/octobot_node/scheduler/workflows_util.py +++ b/packages/node/octobot_node/scheduler/workflows_util.py @@ -201,6 +201,16 @@ def filter_by_wallet( raise ValueError(f"Unsupported scheduler queue for wallet filter: {queue!r}") +def normalize_parent_automation_id(workflow_id: str) -> str: + return workflow_id[:octobot_node.constants.PARENT_WORKFLOW_ID_LENGTH] + + +def build_next_child_automation_workflow_id(current_workflow_id: str) -> str: + parent_id = normalize_parent_automation_id(current_workflow_id) + child_index = parse_automation_child_workflow_index(current_workflow_id) + return f"{parent_id}_{child_index + 1}" + + def parse_automation_child_workflow_index(workflow_id: str) -> int: """ Return the child iteration index encoded in a workflow ID. @@ -282,7 +292,12 @@ def parse_automation_workflow_output( if not workflow_status.output: return None try: - return params.AutomationWorkflowOutput.from_dict(json.loads(workflow_status.output)) + raw_output = workflow_status.output + if isinstance(raw_output, str): + raw_output = json.loads(raw_output) + if not isinstance(raw_output, dict): + raise TypeError(f"Unexpected workflow output type: {type(raw_output).__name__}") + return params.AutomationWorkflowOutput.from_dict(raw_output) except (json.JSONDecodeError, TypeError, ValueError) as error: logger.warning( "Failed to parse automation workflow output for %s: %s", @@ -363,6 +378,32 @@ def get_automation_dict(description: typing.Union[str, dict]) -> dict: raise ValueError("No automation state found in description") +def patch_task_content_degraded_state( + task_content: str, + error_status: str, + error_message: str, + *, + since: float, +) -> str: + if octobot_flow is None: + raise RuntimeError("octobot_flow is required to patch automation degraded state") + description = get_automation_dict(task_content) + automation_state = octobot_flow.entities.AutomationState.from_dict(description[STATE_KEY]) + existing_degraded_state = automation_state.automation.execution.degraded_state + degraded_since = ( + existing_degraded_state.since + if existing_degraded_state.since > 0 + else since + ) + automation_state.automation.execution.degraded_state = octobot_flow.entities.DegradedStateDetails( + since=degraded_since, + error=error_status, + reason=error_message, + ) + description[STATE_KEY] = automation_state.to_dict(include_default_values=False) + return json.dumps(description) + + async def get_automation_workflow_status(automation_id: str) -> dbos_lib.WorkflowStatus: for workflow_status in await dbos_lib.DBOS.list_workflows_async(status=[ dbos_lib.WorkflowStatusString.PENDING.value, dbos_lib.WorkflowStatusString.ENQUEUED.value diff --git a/packages/node/octobot_node/scheduler/workflows_version_migration.py b/packages/node/octobot_node/scheduler/workflows_version_migration.py new file mode 100644 index 0000000000..0b1a319364 --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows_version_migration.py @@ -0,0 +1,152 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import logging + +import dbos +import sqlalchemy + +import octobot_node.config +import octobot_node.constants + +_DBOS_SYSTEM_SCHEMA = "dbos" +_PENDING_WORKFLOW_STATUSES = ( + dbos.WorkflowStatusString.ENQUEUED.value, + dbos.WorkflowStatusString.PENDING.value, +) + + +def _resolve_system_database_url() -> str: + postgres_url = octobot_node.config.settings.SCHEDULER_POSTGRES_URL + if postgres_url: + return str(postgres_url) + return f"sqlite:///{octobot_node.config.settings.SCHEDULER_SQLITE_FILE}" + + +def _workflow_status_table_reference(database_url: str) -> str: + if database_url.startswith("sqlite"): + return "workflow_status" + return f'"{_DBOS_SYSTEM_SCHEMA}"."workflow_status"' + + +def _workflow_status_table_exists( + connection: sqlalchemy.Connection, + *, + database_url: str, +) -> bool: + inspector = sqlalchemy.inspect(connection) + if database_url.startswith("sqlite"): + return inspector.has_table("workflow_status") + return inspector.has_table("workflow_status", schema=_DBOS_SYSTEM_SCHEMA) + + +def migrate_stranded_workflow_versions( + *, + target_version: str | None = None, +) -> int: + """ + Re-tag ENQUEUED/PENDING workflows to ``target_version`` before DBOS launch. + + OctoBot releases used to set DBOS ``application_version`` to the OctoBot + version string, which stranded in-flight workflows after upgrades. This + migration retags them to the stable scheduler application version. + """ + resolved_target_version = target_version or octobot_node.constants.SCHEDULER_APPLICATION_VERSION + database_url = _resolve_system_database_url() + workflow_status_table = _workflow_status_table_reference(database_url) + logger = logging.getLogger(__name__) + + engine = sqlalchemy.create_engine(database_url) + try: + with engine.begin() as connection: + if not _workflow_status_table_exists(connection, database_url=database_url): + logger.info( + "Skipping DBOS workflow version migration: workflow_status table not found" + ) + return 0 + + status_placeholders = ", ".join( + f":status_{status_index}" + for status_index in range(len(_PENDING_WORKFLOW_STATUSES)) + ) + status_parameters = { + f"status_{status_index}": status_value + for status_index, status_value in enumerate(_PENDING_WORKFLOW_STATUSES) + } + select_previous_versions_query = sqlalchemy.text( + f""" + SELECT DISTINCT application_version + FROM {workflow_status_table} + WHERE status IN ({status_placeholders}) + AND ( + application_version IS NULL + OR application_version != :target_version + ) + """ + ) + previous_version_rows = connection.execute( + select_previous_versions_query, + { + **status_parameters, + "target_version": resolved_target_version, + }, + ).fetchall() + previous_versions = [ + row[0] if row[0] is not None else "" + for row in previous_version_rows + ] + + update_query = sqlalchemy.text( + f""" + UPDATE {workflow_status_table} + SET application_version = :target_version + WHERE status IN ({status_placeholders}) + AND ( + application_version IS NULL + OR application_version != :target_version + ) + """ + ) + update_result = connection.execute( + update_query, + { + **status_parameters, + "target_version": resolved_target_version, + }, + ) + updated_count = update_result.rowcount or 0 + except Exception as error: + raise RuntimeError( + "Failed to migrate stranded DBOS workflow application versions " + f"to {resolved_target_version!r}: {error}" + ) from error + finally: + engine.dispose() + + if updated_count: + logger.info( + "Migrated %s stranded DBOS workflow(s) to application version %s " + "(previous versions: %s)", + updated_count, + resolved_target_version, + ", ".join(previous_versions) if previous_versions else "none", + ) + else: + logger.info( + "No stranded DBOS workflows needed application version migration " + "(target version: %s)", + resolved_target_version, + ) + return updated_count diff --git a/packages/node/requirements.txt b/packages/node/requirements.txt index 45b87198b3..e495a6f636 100644 --- a/packages/node/requirements.txt +++ b/packages/node/requirements.txt @@ -2,4 +2,4 @@ fastapi[standard]==0.135.1 passlib[bcrypt]==1.7.4 pydantic -dbos==2.21.0 +dbos==2.27.0 diff --git a/packages/node/tests/conftest.py b/packages/node/tests/conftest.py index a39916a6a0..b9c2767756 100644 --- a/packages/node/tests/conftest.py +++ b/packages/node/tests/conftest.py @@ -5,11 +5,16 @@ import pytest _TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS = 2 +_TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS = 30.0 os.environ.setdefault( "RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS", str(_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS), ) +os.environ.setdefault( + "RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS", + str(_TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS), +) import octobot.community.local_authenticator as local_community_auth @@ -39,3 +44,15 @@ def _fast_run_octobot_process_recall(monkeypatch): "RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS", _TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS, ) + monkeypatch.setattr( + node_constants, + "RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS", + _TESTS_RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS, + ) + + +@pytest.fixture(autouse=True) +def _disable_auto_open_in_web_browser(): + import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops + with mock.patch.object(octobot_process_ops, "AUTO_OPEN_IN_WEB_BROWSER", False): + yield diff --git a/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py b/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py index 69b216f862..dabb42905b 100644 --- a/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py +++ b/packages/node/tests/functional_tests/test_accounts_CRUD_operations.py @@ -437,6 +437,40 @@ def build_delete_user_action(*, user_action_id: str, account_id: str) -> protoco assert len(account_provider.list_items(user_id)) == 1 _assert_functional_assets(persisted_created_account.assets) + # Step 2b — Create (duplicate identity): same remote_account_id, new Account.id; workflow fails; nothing new persisted. + duplicate_account = build_account( + account_id="functional-account-2", + account_name="Duplicate account", + ) + duplicate_create = build_create_user_action( + user_action_id="ua-account-create-duplicate", + account=duplicate_account, + ) + await _run_user_action_to_completion(user_id, duplicate_create) + + listed_after_duplicate = await scheduler_api.list_user_actions(user_id, active_only=True) + _assert_listed_user_actions_match_expected_id_status_pairs( + listed_after_duplicate, + [ + (doomed_create.id, protocol_models.UserActionStatus.FAILED), + (happy_create.id, protocol_models.UserActionStatus.COMPLETED), + (duplicate_create.id, protocol_models.UserActionStatus.FAILED), + ], + ) + duplicate_latest = next( + row for row in listed_after_duplicate if row.id == duplicate_create.id + ) + assert duplicate_latest.status == protocol_models.UserActionStatus.FAILED + assert duplicate_latest.result is not None + duplicate_inner = duplicate_latest.result.actual_instance + assert isinstance(duplicate_inner, protocol_models.AccountActionResult) + assert duplicate_inner.result_type == protocol_models.UserActionResultType.ACCOUNT + assert duplicate_inner.error_message == protocol_models.AccountActionResultErrorMessage.DUPLICATE_ITEM + assert duplicate_inner.error_details is not None + assert "functional-account-1" in duplicate_inner.error_details + assert len(account_provider.list_items(user_id)) == 1 + assert account_provider.get_item(user_id, "functional-account-1").name == "Functional account" + # Step 3 — Edit: enqueue workflow only first; poll listings mid retry before awaiting terminal output. edited_account = build_account( account_id="functional-account-1", @@ -468,6 +502,7 @@ def build_delete_user_action(*, user_action_id: str, account_id: str) -> protoco [ (doomed_create.id, protocol_models.UserActionStatus.FAILED), (happy_create.id, protocol_models.UserActionStatus.COMPLETED), + (duplicate_create.id, protocol_models.UserActionStatus.FAILED), (edit_action.id, protocol_models.UserActionStatus.PENDING), ], ) @@ -493,6 +528,7 @@ def build_delete_user_action(*, user_action_id: str, account_id: str) -> protoco [ (doomed_create.id, protocol_models.UserActionStatus.FAILED), (happy_create.id, protocol_models.UserActionStatus.COMPLETED), + (duplicate_create.id, protocol_models.UserActionStatus.FAILED), (edit_action.id, protocol_models.UserActionStatus.COMPLETED), ], ) @@ -527,6 +563,7 @@ def build_delete_user_action(*, user_action_id: str, account_id: str) -> protoco [ (doomed_create.id, protocol_models.UserActionStatus.FAILED), (happy_create.id, protocol_models.UserActionStatus.COMPLETED), + (duplicate_create.id, protocol_models.UserActionStatus.FAILED), (edit_action.id, protocol_models.UserActionStatus.COMPLETED), (refresh_action.id, protocol_models.UserActionStatus.COMPLETED), ], @@ -553,13 +590,14 @@ def build_delete_user_action(*, user_action_id: str, account_id: str) -> protoco ) await _run_user_action_to_completion(user_id, delete_action) - # Step 5 (continued) — Full listing is five terminal rows with expected id/status pairs. + # Step 5 (continued) — Full listing is six terminal rows with expected id/status pairs. listed_after_delete = await scheduler_api.list_user_actions(user_id, active_only=True) _assert_listed_user_actions_match_expected_id_status_pairs( listed_after_delete, [ (doomed_create.id, protocol_models.UserActionStatus.FAILED), (happy_create.id, protocol_models.UserActionStatus.COMPLETED), + (duplicate_create.id, protocol_models.UserActionStatus.FAILED), (edit_action.id, protocol_models.UserActionStatus.COMPLETED), (refresh_action.id, protocol_models.UserActionStatus.COMPLETED), (delete_action.id, protocol_models.UserActionStatus.COMPLETED), diff --git a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py index 8d0ff9fa1a..ec4fa29351 100644 --- a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py +++ b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py @@ -494,8 +494,8 @@ async def test_emit_and_copy_grid_master_forced_trigger_copies_signals( None, True, ) - # Persistent auth singleton so _user_id_to_evm() resolves correctly when - # automation_workflow.py translates the Starfish user_id → EVM address for + # Persistent auth singleton so CommunityRepository.user_id_to_evm() resolves correctly when + # translating the Starfish user_id → EVM address for # auth_details["wallet_address"] (needed by CommunityRepository inside the job). auth_config = local_authenticator_module.get_stateless_configuration() auth_instance_for_singleton = community_authentication_module.CommunityAuthentication( diff --git a/packages/node/tests/functional_tests/test_schedule_startup_backfill_workflow.py b/packages/node/tests/functional_tests/test_schedule_startup_backfill_workflow.py new file mode 100644 index 0000000000..a8d93342c7 --- /dev/null +++ b/packages/node/tests/functional_tests/test_schedule_startup_backfill_workflow.py @@ -0,0 +1,358 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import asyncio +import datetime +import tempfile +import time + +import dbos +import mock +import pytest + +import octobot_node.scheduler + +import tests.scheduler as scheduler_tests + +_BLANK_WORKFLOW_NAME = "blank_backfill_test" +_BLANK_SCHEDULE_NAME = "blank_backfill_test_daily" +_BLANK_QUEUE_NAME = "blank_backfill_test_queue" +_TEST_APP_VERSION = "functional-test-app-version" +_STALE_APP_VERSION = "stale-hash-version" +_WORKFLOW_RESULT_TIMEOUT_SECONDS = 30.0 +_STUCK_POLL_SECONDS = 5.0 + +blank_backfill_test_runs: list[datetime.datetime] = [] + +pytestmark = pytest.mark.xdist_group("schedule_backfill_functional") + + +def _register_blank_backfill_workflow(): + @octobot_node.scheduler.SCHEDULER.INSTANCE.workflow(name=_BLANK_WORKFLOW_NAME) + async def blank_backfill_test(scheduled_time, context): + blank_backfill_test_runs.append(scheduled_time) + return {"ran": True} + + return blank_backfill_test + + +def _blank_schedule_input(blank_workflow_fn) -> dbos.ScheduleInput: + return { + "schedule_name": _BLANK_SCHEDULE_NAME, + "workflow_fn": blank_workflow_fn, + "schedule": "0 * * * *", + "context": None, + "automatic_backfill": True, + "queue_name": _BLANK_QUEUE_NAME, + } + + +async def _run_startup_backfill( + schedule_input: dbos.ScheduleInput, + *, + backfill_anchor: datetime.datetime, + backfill_end: datetime.datetime, +) -> None: + import octobot_node.scheduler.schedules as schedules_module + + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = backfill_end + with mock.patch.object( + schedules_module, + "get_backfill_schedule_default_anchor", + return_value=backfill_anchor, + ), mock.patch.object( + schedules_module, + "datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ): + await schedules_module._maybe_backfill_schedule_on_startup( + schedule_input["schedule_name"], + schedule_input, + ) + + +async def _assert_workflow_stays_enqueued(workflow_id: str, poll_seconds: float) -> None: + poll_deadline = time.monotonic() + poll_seconds + while time.monotonic() < poll_deadline: + workflow_status = await dbos.DBOS.get_workflow_status_async(workflow_id) + assert workflow_status is not None + assert workflow_status.status == dbos.WorkflowStatusString.ENQUEUED.value + await asyncio.sleep(0.2) + + +def _seed_stale_latest_application_version(stale_version_name: str) -> None: + import dbos._dbos as dbos_internals + + dbos_internals._get_dbos_instance()._sys_db.create_application_version( + stale_version_name, + ) + + +def _get_or_create_registry_queue(queue_name: str, **queue_options) -> dbos.Queue: + # destroy_launched_dbos() keeps the global registry; reuse an existing Queue + # declaration when this fixture runs more than once on the same xdist worker. + import dbos._dbos as dbos_internals + + registry = dbos_internals._get_or_create_dbos_registry() + existing_queue = registry.queue_info_map.get(queue_name) + if existing_queue is not None: + return existing_queue + return dbos.Queue(name=queue_name, **queue_options) + + +@pytest.fixture +def temp_dbos_scheduler_backfill(): + blank_backfill_test_runs.clear() + with tempfile.NamedTemporaryFile() as temp_file: + scheduler_tests.destroy_launched_dbos() + dbos_runtime = scheduler_tests.init_scheduler_with_app_version( + temp_file.name, + _TEST_APP_VERSION, + ) + _get_or_create_registry_queue(_BLANK_QUEUE_NAME, concurrency=1) + blank_workflow_fn = _register_blank_backfill_workflow() + dbos_runtime.reset_system_database() + dbos_runtime.launch() + try: + yield blank_workflow_fn + finally: + scheduler_tests.destroy_launched_dbos() + + +@pytest.mark.asyncio +class TestScheduleBackfillFunctional: + async def test_blank_workflow_runs_after_startup_backfill( + self, + temp_dbos_scheduler_backfill, + ): + blank_workflow_fn = temp_dbos_scheduler_backfill + schedule_input = _blank_schedule_input(blank_workflow_fn) + backfill_anchor = datetime.datetime( + 2026, 7, 15, 11, 0, 0, tzinfo=datetime.timezone.utc, + ) + backfill_end = datetime.datetime( + 2026, 7, 15, 12, 30, 0, tzinfo=datetime.timezone.utc, + ) + expected_scheduled_time = datetime.datetime( + 2026, 7, 15, 12, 0, 0, tzinfo=datetime.timezone.utc, + ) + expected_workflow_id = ( + f"sched-{_BLANK_SCHEDULE_NAME}-{expected_scheduled_time.isoformat()}" + ) + + dbos.DBOS.create_schedule( + schedule_name=schedule_input["schedule_name"], + workflow_fn=schedule_input["workflow_fn"], + schedule=schedule_input["schedule"], + context=schedule_input.get("context"), + automatic_backfill=schedule_input.get("automatic_backfill", False), + queue_name=schedule_input.get("queue_name"), + ) + existing_schedule = dbos.DBOS.get_schedule(_BLANK_SCHEDULE_NAME) + assert existing_schedule is not None + assert existing_schedule.get("last_fired_at") is None + + await _run_startup_backfill( + schedule_input, + backfill_anchor=backfill_anchor, + backfill_end=backfill_end, + ) + + workflow_status = await dbos.DBOS.get_workflow_status_async(expected_workflow_id) + assert workflow_status is not None + assert workflow_status.app_version == dbos.DBOS.application_version + + workflow_handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async( + expected_workflow_id, + ) + workflow_result = await asyncio.wait_for( + workflow_handle.get_result(), + timeout=_WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + assert workflow_result == {"ran": True} + + final_workflow_status = await dbos.DBOS.get_workflow_status_async(expected_workflow_id) + assert final_workflow_status is not None + assert final_workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value + assert expected_scheduled_time in blank_backfill_test_runs + + async def test_startup_backfill_skipped_when_slot_already_success( + self, + temp_dbos_scheduler_backfill, + ): + blank_workflow_fn = temp_dbos_scheduler_backfill + schedule_input = _blank_schedule_input(blank_workflow_fn) + backfill_anchor = datetime.datetime( + 2026, 7, 15, 11, 0, 0, tzinfo=datetime.timezone.utc, + ) + backfill_end = datetime.datetime( + 2026, 7, 15, 12, 30, 0, tzinfo=datetime.timezone.utc, + ) + expected_scheduled_time = datetime.datetime( + 2026, 7, 15, 12, 0, 0, tzinfo=datetime.timezone.utc, + ) + expected_workflow_id = ( + f"sched-{_BLANK_SCHEDULE_NAME}-{expected_scheduled_time.isoformat()}" + ) + + dbos.DBOS.create_schedule( + schedule_name=schedule_input["schedule_name"], + workflow_fn=schedule_input["workflow_fn"], + schedule=schedule_input["schedule"], + context=schedule_input.get("context"), + automatic_backfill=schedule_input.get("automatic_backfill", False), + queue_name=schedule_input.get("queue_name"), + ) + + await _run_startup_backfill( + schedule_input, + backfill_anchor=backfill_anchor, + backfill_end=backfill_end, + ) + workflow_handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async( + expected_workflow_id, + ) + await asyncio.wait_for( + workflow_handle.get_result(), + timeout=_WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + assert len(blank_backfill_test_runs) == 1 + + await _run_startup_backfill( + schedule_input, + backfill_anchor=backfill_anchor, + backfill_end=backfill_end, + ) + + final_workflow_status = await dbos.DBOS.get_workflow_status_async(expected_workflow_id) + assert final_workflow_status is not None + assert final_workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value + assert len(blank_backfill_test_runs) == 1 + + async def test_backfill_stuck_when_db_latest_differs_from_runtime( + self, + temp_dbos_scheduler_backfill, + ): + blank_workflow_fn = temp_dbos_scheduler_backfill + schedule_input = _blank_schedule_input(blank_workflow_fn) + backfill_anchor = datetime.datetime( + 2026, 7, 15, 11, 0, 0, tzinfo=datetime.timezone.utc, + ) + backfill_end = datetime.datetime( + 2026, 7, 15, 12, 30, 0, tzinfo=datetime.timezone.utc, + ) + expected_scheduled_time = datetime.datetime( + 2026, 7, 15, 12, 0, 0, tzinfo=datetime.timezone.utc, + ) + expected_workflow_id = ( + f"sched-{_BLANK_SCHEDULE_NAME}-{expected_scheduled_time.isoformat()}" + ) + + dbos.DBOS.create_schedule( + schedule_name=schedule_input["schedule_name"], + workflow_fn=schedule_input["workflow_fn"], + schedule=schedule_input["schedule"], + context=schedule_input.get("context"), + automatic_backfill=schedule_input.get("automatic_backfill", False), + queue_name=schedule_input.get("queue_name"), + ) + + _seed_stale_latest_application_version(_STALE_APP_VERSION) + dbos.DBOS.set_latest_application_version(_STALE_APP_VERSION) + + await _run_startup_backfill( + schedule_input, + backfill_anchor=backfill_anchor, + backfill_end=backfill_end, + ) + + workflow_status = await dbos.DBOS.get_workflow_status_async(expected_workflow_id) + assert workflow_status is not None + assert workflow_status.app_version == _STALE_APP_VERSION + assert workflow_status.app_version != dbos.DBOS.application_version + + await _assert_workflow_stays_enqueued( + expected_workflow_id, + _STUCK_POLL_SECONDS, + ) + + async def test_backfill_resumes_after_application_version_migration( + self, + temp_dbos_scheduler_backfill, + ): + blank_workflow_fn = temp_dbos_scheduler_backfill + schedule_input = _blank_schedule_input(blank_workflow_fn) + backfill_anchor = datetime.datetime( + 2026, 7, 15, 11, 0, 0, tzinfo=datetime.timezone.utc, + ) + backfill_end = datetime.datetime( + 2026, 7, 15, 12, 30, 0, tzinfo=datetime.timezone.utc, + ) + expected_scheduled_time = datetime.datetime( + 2026, 7, 15, 12, 0, 0, tzinfo=datetime.timezone.utc, + ) + expected_workflow_id = ( + f"sched-{_BLANK_SCHEDULE_NAME}-{expected_scheduled_time.isoformat()}" + ) + + dbos.DBOS.create_schedule( + schedule_name=schedule_input["schedule_name"], + workflow_fn=schedule_input["workflow_fn"], + schedule=schedule_input["schedule"], + context=schedule_input.get("context"), + automatic_backfill=schedule_input.get("automatic_backfill", False), + queue_name=schedule_input.get("queue_name"), + ) + + _seed_stale_latest_application_version(_STALE_APP_VERSION) + dbos.DBOS.set_latest_application_version(_STALE_APP_VERSION) + + await _run_startup_backfill( + schedule_input, + backfill_anchor=backfill_anchor, + backfill_end=backfill_end, + ) + + workflow_status = await dbos.DBOS.get_workflow_status_async(expected_workflow_id) + assert workflow_status is not None + assert workflow_status.app_version == _STALE_APP_VERSION + assert workflow_status.app_version != dbos.DBOS.application_version + + import dbos._dbos as dbos_internals + import octobot_node.scheduler.workflows_version_migration as workflows_version_migration + + scheduler_database_url = dbos_internals._get_dbos_instance()._sys_db.engine.url + assert scheduler_database_url is not None + assert scheduler_database_url.database is not None + + with mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_POSTGRES_URL", + None, + ), mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_SQLITE_FILE", + scheduler_database_url.database, + ): + workflows_version_migration.migrate_stranded_workflow_versions( + target_version=dbos.DBOS.application_version, + ) + + workflow_handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async( + expected_workflow_id, + ) + workflow_result = await asyncio.wait_for( + workflow_handle.get_result(), + timeout=_WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + assert workflow_result == {"ran": True} + + final_workflow_status = await dbos.DBOS.get_workflow_status_async(expected_workflow_id) + assert final_workflow_status is not None + assert final_workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value + assert expected_scheduled_time in blank_backfill_test_runs diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py index 1f5bfb3e64..f010003c9d 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py @@ -3,16 +3,20 @@ import asyncio import os +import pathlib import shutil import time import typing import mock import pytest +import dbos import octobot.constants as octobot_constants_module import octobot_protocol.models as octobot_protocol_models +import octobot_commons.constants as commons_constants_module import octobot_commons.process_util as process_util_module +import octobot_commons.user_root_folder_provider as user_root_folder_provider_module from .util import authenticator_mocks as authenticator_mocks_module from .util import octobot_process_workflow as octobot_process_workflow_module @@ -21,18 +25,27 @@ import octobot.community.authentication as community_authentication_module import octobot_node.config +import octobot_node.constants as octobot_node_constants_module import octobot_node.scheduler +import octobot_node.scheduler.generic_process_octobot as generic_process_octobot_module import octobot_node.scheduler.workflows_util as workflows_util_module +import octobot_sync.sync.collection_providers as collection_providers_module +import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops from tests.scheduler import temp_dbos_scheduler _T_ENQUEUE_SECONDS = 30.0 _T_INIT_SECONDS = 60.0 _T_STOP_SEND_SECONDS = 30.0 -_T_STOP_COMPLETE_SECONDS = 45.0 +_T_RESTART_ENQUEUE_SECONDS = 30.0 +_T_RESTART_INIT_SECONDS = 60.0 _GENERIC_PROCESS_ACCOUNT_ID = "functional_generic_process_account" _GENERIC_PROCESS_AUTOMATION_NAME = "test_generic_process_default_config_automation" +_GENERIC_PROCESS_ACCOUNTLESS_AUTOMATION_NAME = "test_generic_process_accountless_automation" + +# Spawn real OctoBot child processes on fixed local ports; serialize under pytest-xdist. +pytestmark = pytest.mark.xdist_group("octobot_process_functional") class TestStartCheckAndStopDefaultConfigOctobotProcessWorkflow: @@ -54,6 +67,13 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule monkeypatch.setenv(octobot_constants_module.ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "5") user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + monkeypatch.setattr( + octobot_process_ops, + "_assert_sync_strategy_exists", + lambda sync_user_id, sync_profile_id: mock.Mock( + configuration=mock.Mock(actual_instance=mock.Mock(profile_data=None)) + ), + ) protocol_account = workflow_common_module.protocol_account_for_functional( account_id=_GENERIC_PROCESS_ACCOUNT_ID, usdc_total=1000.0, @@ -156,7 +176,9 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule stop_user_action = workflow_common_module.build_stop_user_action( automation_id=parent_automation_id, - user_action_id=f"ua-stop-{create_user_action.id}", + user_action_id=workflow_common_module.unique_stop_user_action_id( + f"ua-stop-{create_user_action.id}" + ), ) try: await asyncio.wait_for( @@ -170,7 +192,7 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule except TimeoutError as exc: raise AssertionError("execute_user_action timed out enqueueing automation stop") from exc - stop_assert_deadline = time.monotonic() + _T_STOP_COMPLETE_SECONDS + stop_assert_deadline = time.monotonic() + octobot_process_workflow_module.stop_workflow_complete_seconds() while time.monotonic() < stop_assert_deadline: listed_user_actions = await octobot_node.scheduler.SCHEDULER.list_user_actions(user_id) latest_by_id = user_action_assertions_module.merge_user_actions_latest_per_id(listed_user_actions) @@ -187,7 +209,7 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule await workflow_common_module.wait_for_stop_success_output( temp_dbos_scheduler, metadata_automation_id, - _T_STOP_COMPLETE_SECONDS, + octobot_process_workflow_module.stop_workflow_complete_seconds(), ) protocol_automation_after_stop = await workflow_common_module.load_protocol_automation_state_for_workflow( @@ -196,7 +218,7 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule ) assert protocol_automation_after_stop.status == octobot_protocol_models.WorkflowStatus.COMPLETED - stop_deadline = time.monotonic() + octobot_process_workflow_module.CHILD_STOP_WAIT_SEC + stop_deadline = time.monotonic() + octobot_process_workflow_module.child_stop_wait_seconds() while time.monotonic() < stop_deadline: if child_pid is not None and not process_util_module.pid_is_running(child_pid): break @@ -204,7 +226,229 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule else: pytest.fail(f"expected child pid {child_pid} to exit after AUTOMATION_STOP") + restart_user_action = workflow_common_module.build_restart_user_action( + automation_id=parent_automation_id, + user_action_id=f"ua-restart-{create_user_action.id}", + ) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + restart_user_action, + user_id, + ), + timeout=_T_RESTART_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing automation restart") from exc + + await user_action_assertions_module.assert_user_action_selector_completed_automation_restart( + user_id=user_id, + user_action_id=restart_user_action.id, + ) + + restarted_inner_state = await octobot_process_workflow_module.wait_for_init_state_ok( + temp_dbos_scheduler, + metadata_automation_id, + timeout_sec=_T_RESTART_INIT_SECONDS, + active_workflows_only=True, + ) + assert restarted_inner_state.get("pid") + restarted_child_pid = int(restarted_inner_state["pid"]) + assert process_util_module.pid_is_running(restarted_child_pid) + child_pid = restarted_child_pid + child_user_root = restarted_inner_state.get("user_root") or child_user_root + child_log_folder = restarted_inner_state.get("log_folder") or child_log_folder + + workflow_rows_after_restart = await temp_dbos_scheduler.INSTANCE.list_workflows_async() + workflow_row_after_restart: typing.Any = None + for workflow_row in workflow_rows_after_restart: + if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + continue + if workflow_row.status not in ( + dbos.WorkflowStatusString.PENDING.value, + dbos.WorkflowStatusString.ENQUEUED.value, + ): + continue + workflow_row_after_restart = workflow_row + break + assert workflow_row_after_restart is not None + + protocol_automation_after_restart = await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_after_restart, + ) + assert protocol_automation_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING + + if child_pid is not None and process_util_module.pid_is_running(child_pid): + process_util_module.request_graceful_stop_via_sigterm(child_pid) if child_user_root and os.path.isdir(child_user_root): shutil.rmtree(child_user_root, ignore_errors=True) if child_log_folder and os.path.isdir(child_log_folder): shutil.rmtree(child_log_folder, ignore_errors=True) + + @pytest.mark.asyncio + async def test_generic_process_accountless_default_config_lifecycle( + self, + temp_dbos_scheduler, + monkeypatch, + tmp_path: pathlib.Path, + ): + if not os.path.isfile(os.path.join(os.getcwd(), "start.py")): + pytest.skip("start.py missing: run pytest with cwd set to the OctoBot project root") + + non_trading_profile_json = os.path.join( + os.getcwd(), + "user", + "profiles", + "non-trading", + "profile.json", + ) + if not os.path.isfile(non_trading_profile_json): + pytest.skip("non-trading profile missing under OctoBot user/profiles") + + monkeypatch.setenv(octobot_constants_module.ENV_PROCESS_BOT_STATE_DUMP_INTERVAL_SECONDS, "5") + + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + user_root_provider = user_root_folder_provider_module.instance() + previous_user_root = user_root_provider.get_root() + test_user_root = tmp_path / "functional_generic_process_accountless_user_root" + user_root_provider.set_root(str(test_user_root)) + + child_user_root: str | None = None + child_log_folder: str | None = None + child_pid: int | None = None + + try: + with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), + ): + try: + parent_automation_id = await asyncio.wait_for( + generic_process_octobot_module.create_generic_process_bot( + user_id, + _GENERIC_PROCESS_ACCOUNTLESS_AUTOMATION_NAME, + ), + timeout=_T_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError( + "create_generic_process_bot timed out creating accountless automation" + ) from exc + + stored_strategy = collection_providers_module.StrategyProvider.instance().get_item( + user_id, + octobot_node_constants_module.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID, + ) + assert stored_strategy.reference_market == commons_constants_module.DEFAULT_REFERENCE_MARKET + strategy_configuration = stored_strategy.configuration + assert strategy_configuration is not None + generic_process_configuration = strategy_configuration.actual_instance + assert isinstance( + generic_process_configuration, + octobot_protocol_models.GenericProcessConfiguration, + ) + assert generic_process_configuration.profile_data is None + + metadata_automation_id = parent_automation_id + + inner_state = await octobot_process_workflow_module.wait_for_init_state_ok( + temp_dbos_scheduler, + metadata_automation_id, + timeout_sec=_T_INIT_SECONDS, + ) + assert inner_state.get("pid") + child_pid = int(inner_state["pid"]) + assert process_util_module.pid_is_running(child_pid) + child_user_root = inner_state.get("user_root") + child_log_folder = inner_state.get("log_folder") + + workflow_rows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() + workflow_row_matching: typing.Any = None + for workflow_row in workflow_rows: + if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + continue + state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + if state_reader is None: + continue + workflow_row_matching = workflow_row + break + assert workflow_row_matching is not None + + protocol_automation = await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_matching, + ) + assert protocol_automation.status == octobot_protocol_models.WorkflowStatus.RUNNING + assert protocol_automation.exchange_account_ids is None + + stop_user_action = workflow_common_module.build_stop_user_action( + automation_id=parent_automation_id, + user_action_id=workflow_common_module.unique_stop_user_action_id( + f"ua-stop-accountless-{parent_automation_id}" + ), + ) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + stop_user_action, + user_id, + ), + timeout=_T_STOP_SEND_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing accountless automation stop") from exc + + stop_assert_deadline = time.monotonic() + octobot_process_workflow_module.stop_workflow_complete_seconds() + while time.monotonic() < stop_assert_deadline: + listed_user_actions = await octobot_node.scheduler.SCHEDULER.list_user_actions(user_id) + latest_by_id = user_action_assertions_module.merge_user_actions_latest_per_id(listed_user_actions) + stop_row = latest_by_id.get(stop_user_action.id) + if stop_row is not None and stop_row.status == octobot_protocol_models.UserActionStatus.COMPLETED: + break + await asyncio.sleep(workflow_common_module.DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS) + else: + await user_action_assertions_module.assert_user_action_selector_completed_automation_stop( + user_id=user_id, + user_action_id=stop_user_action.id, + ) + + await workflow_common_module.wait_for_stop_success_output( + temp_dbos_scheduler, + metadata_automation_id, + octobot_process_workflow_module.stop_workflow_complete_seconds(), + ) + + protocol_automation_after_stop = await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_matching, + ) + assert protocol_automation_after_stop.status == octobot_protocol_models.WorkflowStatus.COMPLETED + + stop_deadline = time.monotonic() + octobot_process_workflow_module.child_stop_wait_seconds() + while time.monotonic() < stop_deadline: + if child_pid is not None and not process_util_module.pid_is_running(child_pid): + break + await asyncio.sleep(0.5) + else: + pytest.fail(f"expected child pid {child_pid} to exit after AUTOMATION_STOP") + finally: + if child_pid is not None and process_util_module.pid_is_running(child_pid): + process_util_module.request_graceful_stop_via_sigterm(child_pid) + user_root_provider.set_root(previous_user_root) + if child_user_root and os.path.isdir(child_user_root): + shutil.rmtree(child_user_root, ignore_errors=True) + if child_log_folder and os.path.isdir(child_log_folder): + shutil.rmtree(child_log_folder, ignore_errors=True) diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py index 2fd231c6c3..77ad2bab5c 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py @@ -45,6 +45,8 @@ _T_SIGNAL_SECONDS = 5.0 _T_STOP_SEND_SECONDS = 5.0 _T_STOP_COMPLETE_SECONDS = 10.0 +_T_RESTART_ENQUEUE_SECONDS = 10.0 +_T_RESTART_RUNNING_SECONDS = 30.0 # Fast poll after stop/signal send; protocol status may flip RUNNING→COMPLETED quickly on CI. _POST_STOP_PROTOCOL_POLL_SECONDS = 0.05 @@ -81,7 +83,7 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d automation_id=_GRID_AUTOMATION_CONFIGURATION_ID, ) - # Step 0 (continued) — Import test wallet so _user_id_to_evm() can resolve the EVM address + # Step 0 (continued) — Import test wallet so CommunityRepository.user_id_to_evm() can resolve the EVM address # from the Starfish user_id inside automation_workflow.py (needed for auth_details assertion). authentication_instance = authenticator_mocks_module.build_community_authentication( workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, @@ -388,7 +390,7 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d final_job = workflow_common_module.job_description_dict_from_output(parsed_final) # OctoBotActionsJobDescription serialises only non-default fields (empty params omitted). # The EVM address (SIMULATOR_GRID_TEST_COMMUNITY_WALLET_ADDRESS) is translated from the - # Starfish user_id by _user_id_to_evm() and merged into auth_details as wallet_address. + # Starfish user_id by CommunityRepository.user_id_to_evm() and merged into auth_details as wallet_address. assert set(final_job.keys()) == {"auth_details", "state"} final_auth_details = octobot_flow_entities.UserAuthentication.from_dict( final_job["auth_details"] @@ -429,3 +431,64 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d protocol_state_final, _GRID_AUTOMATION_DISPLAY_NAME, ) + + restart_user_action = workflow_common_module.build_restart_user_action( + automation_id=parent_automation_id, + user_action_id=f"ua-restart-{create_user_action.id}", + ) + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + restart_user_action, + user_id, + ), + timeout=_T_RESTART_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing automation restart") from exc + + await user_action_assertions_module.assert_user_action_selector_completed_automation_restart( + user_id=user_id, + user_action_id=restart_user_action.id, + ) + + restart_running_deadline = time.monotonic() + _T_RESTART_RUNNING_SECONDS + workflow_row_after_restart = None + protocol_state_after_restart = None + while time.monotonic() < restart_running_deadline: + for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): + if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + continue + if workflow_row.status not in ( + dbos.WorkflowStatusString.PENDING.value, + dbos.WorkflowStatusString.ENQUEUED.value, + ): + continue + workflow_row_after_restart = workflow_row + protocol_state_after_restart = ( + await workflow_common_module.load_protocol_automation_state_for_workflow( + user_id, + workflow_row_after_restart, + ) + ) + if protocol_state_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING: + break + if ( + workflow_row_after_restart is not None + and protocol_state_after_restart is not None + and protocol_state_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING + ): + break + await asyncio.sleep(workflow_common_module.DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS) + else: + pytest.fail( + f"Timed out waiting for restarted grid automation {metadata_automation_id!r} to reach RUNNING" + ) + + assert workflow_row_after_restart is not None + assert protocol_state_after_restart.status == octobot_protocol_models.WorkflowStatus.RUNNING + protocol_assertions_module.assert_protocol_automation_metadata_name( + protocol_state_after_restart, + _GRID_AUTOMATION_DISPLAY_NAME, + ) diff --git a/packages/node/tests/functional_tests/util/octobot_process_workflow.py b/packages/node/tests/functional_tests/util/octobot_process_workflow.py index 3e71ba4166..86aca02d30 100644 --- a/packages/node/tests/functional_tests/util/octobot_process_workflow.py +++ b/packages/node/tests/functional_tests/util/octobot_process_workflow.py @@ -10,8 +10,10 @@ import typing import uuid +import dbos import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_flow.entities as flow_entities +import octobot_node.constants as node_constants_module import octobot_protocol.models as protocol_models_module import pytest @@ -21,7 +23,19 @@ GENERIC_PROCESS_ACTION_ID = f"{protocol_models_module.ActionConfigurationType.GENERIC_PROCESS.value}_1" GLOBAL_INIT_TIMEOUT_SEC = 60.0 INIT_POLL_INTERVAL_SEC = 2.0 -CHILD_STOP_WAIT_SEC = 20.0 +# Graceful child STOP is bounded by ping_timeout; add recall + scheduler margin. + + +def stop_workflow_complete_seconds() -> float: + ping_timeout_seconds = node_constants_module.RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS + waiting_time_seconds = node_constants_module.RUN_OCTOBOT_PROCESS_WAITING_TIME_SECONDS + # Allow multiple recall cycles (each bounded by ping) + workflow teardown margin. + return max(ping_timeout_seconds * 4 + waiting_time_seconds * 4 + 30.0, 180.0) + + +def child_stop_wait_seconds() -> float: + ping_timeout_seconds = node_constants_module.RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS + return max(ping_timeout_seconds + 30.0, 60.0) def build_generic_process_configuration( @@ -41,13 +55,16 @@ def seeded_generic_process_strategy_for_functional_wallet( stored_strategy_id: str = GENERIC_PROCESS_DEFAULT_STRATEGY_ID, profile_data: dict[str, typing.Any] | None = None, ) -> protocol_models_module.Strategy: + resolved_profile_data = profile_data + if resolved_profile_data is None: + resolved_profile_data = {"profile_details": {"id": stored_strategy_id}} return protocol_models_module.Strategy( id=stored_strategy_id, version=workflow_common_module.SIMULATOR_FUNCTIONAL_STRATEGY_VERSION, name="Generic process automation strategy", reference_market="USDC", configuration=protocol_models_module.StrategyConfiguration( - build_generic_process_configuration(profile_data=profile_data), + build_generic_process_configuration(profile_data=resolved_profile_data), ), ) @@ -132,7 +149,7 @@ def build_create_account_user_action( def build_create_generic_process_user_action( *, - account_id: str, + account_id: str | None = None, name: str, strategy_id: str = GENERIC_PROCESS_DEFAULT_STRATEGY_ID, automation_id: str | None = None, @@ -141,11 +158,14 @@ def build_create_generic_process_user_action( id=strategy_id, version=workflow_common_module.SIMULATOR_FUNCTIONAL_STRATEGY_VERSION, ) + account_references: list[protocol_models_module.AccountReference] = [] + if account_id is not None: + account_references = [protocol_models_module.AccountReference(id=account_id)] automation_configuration_fields: dict[str, typing.Any] = { "name": name, "created_at": datetime.datetime(2026, 6, 1, 11, 0, 0, tzinfo=datetime.UTC), "strategy": strategy_reference, - "accounts": [protocol_models_module.AccountReference(id=account_id)], + "accounts": account_references, } if automation_id is not None: automation_configuration_fields["id"] = automation_id @@ -198,6 +218,7 @@ async def wait_for_init_state_ok( *, timeout_sec: float = GLOBAL_INIT_TIMEOUT_SEC, poll_interval_sec: float = INIT_POLL_INTERVAL_SEC, + active_workflows_only: bool = False, ) -> dict: deadline = time.monotonic() + timeout_sec while time.monotonic() < deadline: @@ -205,6 +226,11 @@ async def wait_for_init_state_ok( for workflow_row in workflow_rows: import octobot_node.scheduler.workflows_util as workflows_util_module + if active_workflows_only and workflow_row.status not in ( + dbos.WorkflowStatusString.PENDING.value, + dbos.WorkflowStatusString.ENQUEUED.value, + ): + continue if workflows_util_module.get_automation_id(workflow_row) != automation_id: continue state_reader = workflows_util_module.get_automation_state_reader(workflow_row) diff --git a/packages/node/tests/functional_tests/util/user_action_assertions.py b/packages/node/tests/functional_tests/util/user_action_assertions.py index c23aa18ffb..ee02d6f261 100644 --- a/packages/node/tests/functional_tests/util/user_action_assertions.py +++ b/packages/node/tests/functional_tests/util/user_action_assertions.py @@ -125,6 +125,25 @@ async def assert_user_action_selector_completed_automation_stop( assert inner.error_message is None +async def assert_user_action_selector_completed_automation_restart( + *, + user_action_id: str, + user_id: str, +) -> None: + listed = await octobot_node.scheduler.SCHEDULER.list_user_actions(user_id) + by_id = merge_user_actions_latest_per_id(listed) + assert user_action_id in by_id, f"expected {user_action_id!r} in user action workflows, got {sorted(by_id)!r}" + stored = by_id[user_action_id] + assert stored.status == protocol_models_module.UserActionStatus.COMPLETED + assert stored.result is not None + inner = stored.result.actual_instance + assert isinstance(inner, protocol_models_module.AutomationActionResult) + assert inner.result_type == protocol_models_module.UserActionResultType.AUTOMATION + assert inner.error_details is None + assert inner.error_message is None + assert inner.created_automation_id + + async def assert_user_action_selector_completed_automation_signal( *, user_action_id: str, diff --git a/packages/node/tests/functional_tests/util/workflow_common.py b/packages/node/tests/functional_tests/util/workflow_common.py index 9ba1534677..9531087b49 100644 --- a/packages/node/tests/functional_tests/util/workflow_common.py +++ b/packages/node/tests/functional_tests/util/workflow_common.py @@ -9,6 +9,7 @@ import json import time import typing +import uuid import dbos import pytest @@ -19,6 +20,7 @@ import octobot_trading.enums as trading_enums_module import octobot_node.constants as node_constants_module +import octobot_node.enums as node_enums_module import octobot_node.scheduler import octobot_node.scheduler.workflows import octobot_node.scheduler.api as scheduler_api_module @@ -123,6 +125,25 @@ def build_stop_user_action( ) +def unique_stop_user_action_id(prefix: str) -> str: + return f"{prefix}-{uuid.uuid4()}" + + +def build_restart_user_action( + *, + automation_id: str, + user_action_id: str, +) -> protocol_models_module.UserAction: + payload = protocol_models_module.RestartAutomationConfiguration( + action_type=protocol_models_module.UserActionType.AUTOMATION_RESTART, + id=automation_id, + ) + return protocol_models_module.UserAction( + id=user_action_id, + configuration=wrap_user_action_configuration(payload), + ) + + def build_forced_trigger_signal_user_action( *, automation_id: str, @@ -221,6 +242,105 @@ async def load_protocol_automation_state_for_workflow( return automation_state +def _workflow_output_row_to_json_text(workflow_row: dbos.WorkflowStatus) -> str | None: + # list_workflows_async populates workflow_status.output; production reads it directly + # instead of retrieve_workflow_async().get_result(), which can be empty while status is SUCCESS. + row_output = workflow_row.output + if isinstance(row_output, str) and row_output: + return row_output + if isinstance(row_output, dict) and row_output: + return json.dumps(row_output) + parsed_output = workflows_util_module.parse_automation_workflow_output(workflow_row) + if parsed_output is not None: + return json.dumps(parsed_output.to_dict()) + return None + + +async def _resolve_success_output_text( + scheduler: typing.Any, + workflow_row: dbos.WorkflowStatus, +) -> str | None: + result_text = _workflow_output_row_to_json_text(workflow_row) + if result_text: + return result_text + workflow_handle = await scheduler.INSTANCE.retrieve_workflow_async(workflow_row.workflow_id) + handle_result = await workflow_handle.get_result() + if not handle_result: + return None + if isinstance(handle_result, str): + return handle_result + return json.dumps(handle_result) + + +async def _list_matching_automation_workflow_rows( + scheduler: typing.Any, + automation_id: str, +) -> list[dbos.WorkflowStatus]: + workflow_rows = await scheduler._list_workflows( + None, + None, + [node_enums_module.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], + load_output=False, + ) + return [ + workflow_row + for workflow_row in workflow_rows + if workflows_util_module.get_automation_id(workflow_row) == automation_id + ] + + +async def _stop_success_output_from_workflow_row( + scheduler: typing.Any, + workflow_row: dbos.WorkflowStatus, +) -> str | None | typing.Literal["output_error", "missing_stop_automation"]: + result_text = await _resolve_success_output_text(scheduler, workflow_row) + if not result_text: + return None + parsed_output = parse_automation_workflow_output(result_text) + if parsed_output.error: + return "output_error" + job_dict = job_description_dict_from_output(parsed_output) + automation_payload = job_dict["state"]["automation"] + if automation_payload.get("post_actions", {}).get("stop_automation"): + return result_text + return "missing_stop_automation" + + +def _user_id_from_matching_workflow_rows( + matching_rows: list[dbos.WorkflowStatus], +) -> str | None: + sorted_rows = sorted( + matching_rows, + key=workflows_util_module._automation_child_workflow_sort_key, + reverse=True, + ) + for workflow_row in sorted_rows: + workflow_inputs = workflows_util_module.get_automation_workflow_inputs(workflow_row) + if workflow_inputs is not None: + return workflow_inputs.task.user_id + return None + + +async def _try_stop_output_from_terminal_workflow( + scheduler: typing.Any, + user_id: str | None, + parent_workflow_id: str, +) -> str | None: + terminal_workflow_row = await scheduler.resolve_latest_terminal_automation_workflow_for_parent_id( + user_id, + parent_workflow_id, + ) + if terminal_workflow_row is None: + return None + stop_output = await _stop_success_output_from_workflow_row( + scheduler, + terminal_workflow_row, + ) + if isinstance(stop_output, str): + return stop_output + return None + + async def wait_for_stop_success_output( scheduler, automation_id: str, @@ -229,26 +349,73 @@ async def wait_for_stop_success_output( poll_interval_seconds: float = DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS, ) -> str: stop_deadline = time.monotonic() + deadline_seconds + observed_statuses: set[str] = set() + success_without_stop_automation = False + success_with_output_error = False + latest_success_empty_output_checks = 0 + latest_workflow_id: str | None = None + latest_workflow_status: str | None = None + matching_child_count = 0 while time.monotonic() < stop_deadline: - workflow_rows = await scheduler.INSTANCE.list_workflows_async() - for workflow_row in workflow_rows: - if workflow_row.status != dbos.WorkflowStatusString.SUCCESS.value: - continue - if workflows_util_module.get_automation_id(workflow_row) != automation_id: - continue - workflow_handle = await scheduler.INSTANCE.retrieve_workflow_async(workflow_row.workflow_id) - result_text = await workflow_handle.get_result() - if not result_text: - continue - parsed_output = parse_automation_workflow_output(result_text) - if parsed_output.error: - continue - job_dict = job_description_dict_from_output(parsed_output) - automation_payload = job_dict["state"]["automation"] - if automation_payload.get("post_actions", {}).get("stop_automation"): - return result_text + matching_rows = await _list_matching_automation_workflow_rows(scheduler, automation_id) + matching_child_count = len(matching_rows) + for workflow_row in matching_rows: + observed_statuses.add(workflow_row.status) + if not matching_rows: + await asyncio.sleep(poll_interval_seconds) + continue + latest_workflow_row = max( + matching_rows, + key=workflows_util_module._automation_child_workflow_sort_key, + ) + latest_workflow_id = latest_workflow_row.workflow_id + latest_workflow_status = latest_workflow_row.status + parent_workflow_id = latest_workflow_row.workflow_id[ + : node_constants_module.PARENT_WORKFLOW_ID_LENGTH + ] + user_id = _user_id_from_matching_workflow_rows(matching_rows) + if latest_workflow_row.status == dbos.WorkflowStatusString.SUCCESS.value: + stop_output = await _stop_success_output_from_workflow_row( + scheduler, + latest_workflow_row, + ) + if isinstance(stop_output, str): + return stop_output + if stop_output == "output_error": + success_with_output_error = True + elif stop_output == "missing_stop_automation": + success_without_stop_automation = True + else: + latest_success_empty_output_checks += 1 + else: + terminal_stop_output = await _try_stop_output_from_terminal_workflow( + scheduler, + user_id, + parent_workflow_id, + ) + if terminal_stop_output is not None: + return terminal_stop_output await asyncio.sleep(poll_interval_seconds) - pytest.fail(f"Timed out waiting for stop completion for {automation_id}") + diagnostic_details = [ + f"observed workflow statuses: {sorted(observed_statuses) or ['none']}", + f"matching child workflows: {matching_child_count}", + ] + if latest_workflow_id is not None: + diagnostic_details.append( + f"latest workflow: {latest_workflow_id!r} status={latest_workflow_status!r}" + ) + if latest_success_empty_output_checks: + diagnostic_details.append( + f"latest SUCCESS child had no retrievable output ({latest_success_empty_output_checks} checks)" + ) + if success_without_stop_automation: + diagnostic_details.append("latest SUCCESS output without post_actions.stop_automation") + if success_with_output_error: + diagnostic_details.append("latest SUCCESS output with workflow error") + pytest.fail( + f"Timed out waiting for stop completion for {automation_id} " + f"within {deadline_seconds}s; {'; '.join(diagnostic_details)}" + ) async def enqueue_forced_trigger_and_await( diff --git a/packages/node/tests/protocol/test_accounts_trading.py b/packages/node/tests/protocol/test_accounts_trading.py index 8298f8691b..dac79c4d70 100644 --- a/packages/node/tests/protocol/test_accounts_trading.py +++ b/packages/node/tests/protocol/test_accounts_trading.py @@ -166,6 +166,20 @@ def _exchange_trade_dict(trade_id: str) -> dict: } +def _process_bot_trade_dict(local_trade_id: str, exchange_order_id: str) -> dict: + return { + _ORDER_COLUMNS.ID.value: local_trade_id, + _ORDER_COLUMNS.EXCHANGE_ID.value: exchange_order_id, + _ORDER_COLUMNS.SYMBOL.value: "BTC/USDT", + _ORDER_COLUMNS.TYPE.value: trading_enums.TradeOrderType.LIMIT.value, + _ORDER_COLUMNS.SIDE.value: trading_enums.TradeOrderSide.BUY.value, + _ORDER_COLUMNS.AMOUNT.value: 1.0, + _ORDER_COLUMNS.PRICE.value: 1.0, + _ORDER_COLUMNS.STATUS.value: trading_enums.OrderStatus.FILLED.value, + _ORDER_COLUMNS.TIMESTAMP.value: 1735689600.0, + } + + def _exchange_position_dict(position_id: str) -> dict: return { _POSITION_COLUMNS.ID.value: position_id, @@ -242,3 +256,35 @@ def test_replaces_orders_and_upserts_trades(self): assert saved_state.account_trading.trades is not None trade_ids = {trade.trade_id for trade in saved_state.account_trading.trades} assert trade_ids == {"trade-existing", "trade-new"} + + def test_repeated_update_dedupes_process_bot_trade_without_exchange_trade_id(self): + process_trade = _process_bot_trade_dict("local-trade-1", "exchange-order-9") + provider_stub = mock.Mock() + provider_stub.load_state = mock.Mock( + side_effect=collection_errors.CollectionNoDataError("missing trading state"), + ) + with mock.patch.object( + accounts_trading_module.trading_provider.AccountTradingProvider, + "instance", + return_value=provider_stub, + ): + accounts_trading_module.update_account_trading( + _TEST_WALLET_ADDRESS, + _TEST_ACCOUNT_ID, + [], + [process_trade], + [], + ) + first_saved_state = provider_stub.save_state.call_args[0][2] + provider_stub.load_state = mock.Mock(return_value=first_saved_state) + accounts_trading_module.update_account_trading( + _TEST_WALLET_ADDRESS, + _TEST_ACCOUNT_ID, + [], + [process_trade], + [], + ) + saved_state = provider_stub.save_state.call_args[0][2] + assert saved_state.account_trading.trades is not None + assert len(saved_state.account_trading.trades) == 1 + assert saved_state.account_trading.trades[0].trade_id == "local-trade-1" diff --git a/packages/node/tests/protocol/test_automations.py b/packages/node/tests/protocol/test_automations.py index dd9164b25b..c298ba3574 100644 --- a/packages/node/tests/protocol/test_automations.py +++ b/packages/node/tests/protocol/test_automations.py @@ -426,6 +426,66 @@ def test_running_when_triggered(self): filled = automations_protocol._fill_protocol_automation_state(_minimal_protocol_base(), flow_state) assert filled.status == protocol_models.WorkflowStatus.RUNNING + def test_running_with_degraded_state_exposes_error_fields(self): + trigger = flow_entities.TriggerDetails(scheduled_to=1, triggered_at=2) + execution = flow_entities.ExecutionDetails( + current_execution=trigger, + degraded_state=flow_entities.DegradedStateDetails( + since=100.0, + error=flow_enums.ActionErrorStatus.NOT_ENOUGH_FUNDS.value, + reason="Insufficient funds", + ), + ) + pending_action = flow_entities.DSLScriptActionDetails(id="a1", dsl_script="True") + flow_state = flow_entities.AutomationState( + automation=flow_entities.AutomationDetails( + metadata=flow_entities.AutomationMetadata(automation_id="automation_1"), + actions_dag=flow_entities.ActionsDAG(actions=[pending_action]), + execution=execution, + ), + ) + filled = automations_protocol._fill_protocol_automation_state(_minimal_protocol_base(), flow_state) + assert filled.status == protocol_models.WorkflowStatus.RUNNING + assert filled.error == flow_enums.ActionErrorStatus.NOT_ENOUGH_FUNDS.value + assert filled.error_message == "Insufficient funds" + + def test_active_workflow_preserves_degraded_error_fields(self): + state_dict = { + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": { + "actions": [ + { + "id": "a1", + "dsl_script": "True", + } + ] + }, + "execution": { + "previous_execution": {"triggered_at": 0}, + "current_execution": {"triggered_at": 2}, + "degraded_state": { + "since": 100.0, + "error": flow_enums.ActionErrorStatus.INVALID_ORDER.value, + "reason": "Order volume below exchange minimum", + }, + }, + }, + } + task = node_models.Task( + id="task-1", + name="automation", + content=json.dumps({"state": state_dict}), + type="execute_actions", + ) + state = automations_protocol._to_protocol_automation_state( + task, + workflow_status=dbos.WorkflowStatusString.PENDING.value, + ) + assert state.status == protocol_models.WorkflowStatus.RUNNING + assert state.error == flow_enums.ActionErrorStatus.INVALID_ORDER.value + assert state.error_message == "Order volume below exchange minimum" + def test_running_when_previous_execution_but_current_not_started(self): previous_trigger = flow_entities.TriggerDetails(triggered_at=1_600_000_000.0) execution = flow_entities.ExecutionDetails( @@ -678,3 +738,102 @@ def test_no_exchange_elements_yields_empty_protocol_lists(self): assert filled.orders is None assert filled.positions is None assert filled.trades is None + + +class TestFillProtocolAutomationStateChildOctobotProcess: + def test_child_octobot_process_from_run_octobot_process_recall(self): + import octobot_commons.dsl_interpreter as dsl_interpreter + + inner_recall = { + "http_base_url": "http://127.0.0.1:5002", + "web_port": 5002, + "node_port": 6002, + "user_root": "/tmp/user", + "user_folder": "automation-a", + "log_folder": "/tmp/log", + "profile_id": None, + "pid": 12345, + "init_state_ok": True, + "executor_id": "exec-1", + } + recall_wrapper = dsl_interpreter.ReCallingOperatorResult( + keyword="run_octobot_process", + reset_to_id="action-1", + last_execution_result=inner_recall, + ) + dsl_action = flow_entities.DSLScriptActionDetails( + id="action-1", + dsl_script='run_octobot_process("folder", profile_data={})', + previous_execution_result={ + dsl_interpreter.ReCallingOperatorResult.__name__: recall_wrapper.to_dict(), + }, + ) + flow_state = flow_entities.AutomationState( + automation=flow_entities.AutomationDetails( + metadata=flow_entities.AutomationMetadata(automation_id="automation_1"), + actions_dag=flow_entities.ActionsDAG(actions=[dsl_action]), + ), + ) + filled = automations_protocol._fill_protocol_automation_state(_minimal_protocol_base(), flow_state) + assert filled.child_octobot_process is not None + assert filled.child_octobot_process.http_base_url == "http://127.0.0.1:5002" + assert filled.child_octobot_process.web_port == 5002 + assert filled.child_octobot_process.init_state_ok is True + + +class TestOctobotProcessStateToChildProtocol: + def test_octobot_process_state_to_child_protocol_projects_slim_fields(self): + import octobot_flow.entities.automations.octobot_process_state as octobot_process_state_module + + full_state = octobot_process_state_module.OctobotProcessState( + http_base_url="http://127.0.0.1:5002", + web_port=5002, + node_port=6002, + user_root="/tmp/user", + user_folder="automation-a", + log_folder="/tmp/log", + profile_id=None, + pid=12345, + init_state_ok=True, + executor_id="exec-1", + ) + child_state = automations_protocol._octobot_process_state_to_child_protocol(full_state) + assert child_state == protocol_models.ChildOctoBotProcessState( + http_base_url="http://127.0.0.1:5002", + web_port=5002, + init_state_ok=True, + ) + + +class TestChildOctobotProcessFromFlowActions: + def test_child_octobot_process_from_flow_actions_finds_run_octobot_process_recall(self): + import octobot_commons.dsl_interpreter as dsl_interpreter + + inner_recall = { + "http_base_url": "http://127.0.0.1:5002", + "web_port": 5002, + "node_port": 6002, + "user_root": "/tmp/user", + "user_folder": "automation-a", + "log_folder": "/tmp/log", + "profile_id": None, + "pid": 12345, + "init_state_ok": True, + "executor_id": "exec-1", + } + recall_wrapper = dsl_interpreter.ReCallingOperatorResult( + keyword="run_octobot_process", + reset_to_id="action-1", + last_execution_result=inner_recall, + ) + dsl_action = flow_entities.DSLScriptActionDetails( + id="action-1", + dsl_script='run_octobot_process("folder", profile_data={})', + previous_execution_result={ + dsl_interpreter.ReCallingOperatorResult.__name__: recall_wrapper.to_dict(), + }, + ) + child_state = automations_protocol._child_octobot_process_from_flow_actions([dsl_action]) + assert child_state is not None + assert child_state.web_port == 5002 + assert child_state.init_state_ok is True diff --git a/packages/node/tests/protocol/test_dsl.py b/packages/node/tests/protocol/test_dsl.py new file mode 100644 index 0000000000..e431dad2ba --- /dev/null +++ b/packages/node/tests/protocol/test_dsl.py @@ -0,0 +1,271 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import pytest + +import octobot_commons.enums as commons_enums +import octobot_commons.dsl_interpreter.operator_docs as dsl_interpreter_operator_docs +import octobot_commons.dsl_interpreter.operator_parameter as dsl_interpreter_operator_parameter +import octobot_node.protocol.dsl as dsl_protocol +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + + +def _sample_parameter( + name: str = "period", + parameter_type: str = commons_enums.DslValueType.NUMBER.value, + default=dsl_interpreter_operator_parameter.UNSET_VALUE, + **parameter_kwargs, +) -> dsl_interpreter_operator_parameter.OperatorParameter: + return dsl_interpreter_operator_parameter.OperatorParameter( + name=name, + description="lookback", + required=True, + type=parameter_type, + default=default, + **parameter_kwargs, + ) + + +def _sample_docs(**overrides) -> dsl_interpreter_operator_docs.OperatorDocs: + data = { + "name": "rsi", + "description": "Relative Strength Index", + "type": "ta", + "example": "rsi(data, 14)", + "parameters": [ + _sample_parameter( + name="data", + parameter_type=commons_enums.DslValueType.SERIES.value, + ) + ], + "label": "RSI", + "category": commons_enums.DslKeywordCategory.SOURCE.value, + "return_values": [ + dsl_interpreter_operator_parameter.OperatorParameter( + name="result", + description="RSI series", + required=True, + type=commons_enums.DslValueType.SERIES.value, + ) + ], + } + data.update(overrides) + return dsl_interpreter_operator_docs.OperatorDocs(**data) + + +class TestListDslOperatorDocs: + def test_returns_non_empty_operator_docs(self): + operator_docs_list = dsl_protocol.list_dsl_operator_docs() + assert len(operator_docs_list) > 0 + assert all( + isinstance(operator_docs, dsl_interpreter_operator_docs.OperatorDocs) + for operator_docs in operator_docs_list + ) + + def test_uses_flow_operator_assembly(self): + operator_names = { + operator_docs.name + for operator_docs in dsl_protocol.list_dsl_operator_docs() + } + assert "fetch_order" in operator_names + assert "copy_exchange_account" in operator_names + assert "run_octobot_process" in operator_names + assert "blockchain_wallet_balance" in operator_names + assert "set_leverage" not in operator_names + assert len(operator_names) > 69 + + +class TestOperatorParameterToDslParameter: + def test_maps_authored_type_and_default(self): + operator_parameter = _sample_parameter(default=14) + dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + operator_parameter + ) + assert dsl_parameter.name == "period" + assert dsl_parameter.label == "period" + assert dsl_parameter.value_type is protocol_models.DslValueType.NUMBER + assert dsl_parameter.required is True + assert dsl_parameter.default_value is not None + assert dsl_parameter.default_value.actual_instance == 14 + + def test_skips_none_default(self): + operator_parameter = _sample_parameter(default=None) + dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + operator_parameter + ) + assert dsl_parameter.default_value is None + + def test_skips_structured_default(self): + operator_parameter = _sample_parameter( + default=[{"crypto-currency": "Bitcoin"}], + parameter_type=commons_enums.DslValueType.ANY.value, + ) + dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + operator_parameter + ) + assert dsl_parameter.default_value is None + + def test_raises_when_type_missing(self): + operator_parameter = dsl_interpreter_operator_parameter.OperatorParameter( + name="data", + description="series", + required=True, + type="", + ) + with pytest.raises(ValueError, match="missing type"): + dsl_protocol.operator_parameter_to_dsl_parameter(operator_parameter) + + def test_raises_when_type_unknown(self): + operator_parameter = _sample_parameter(parameter_type="not_a_type") + with pytest.raises(ValueError, match="unknown type"): + dsl_protocol.operator_parameter_to_dsl_parameter(operator_parameter) + + def test_skips_unset_default(self): + operator_parameter = _sample_parameter() + dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + operator_parameter + ) + assert dsl_parameter.default_value is None + + def test_maps_bool_and_str_defaults(self): + bool_parameter = _sample_parameter( + name="enabled", + parameter_type=commons_enums.DslValueType.BOOLEAN.value, + default=True, + ) + bool_dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + bool_parameter + ) + assert bool_dsl_parameter.default_value is not None + assert bool_dsl_parameter.default_value.actual_instance is True + + text_parameter = _sample_parameter( + name="symbol", + parameter_type=commons_enums.DslValueType.TEXT.value, + default="BTC/USDT", + ) + text_dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + text_parameter + ) + assert text_dsl_parameter.default_value is not None + assert text_dsl_parameter.default_value.actual_instance == "BTC/USDT" + + def test_maps_options(self): + operator_parameter = _sample_parameter( + name="time_frame", + parameter_type=commons_enums.DslValueType.TIME_FRAME.value, + options=[ + dsl_interpreter_operator_parameter.OperatorParameterOption( + value="1h", + label="1 hour", + ), + dsl_interpreter_operator_parameter.OperatorParameterOption( + value="4h", + label="4 hours", + ), + ], + ) + dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + operator_parameter + ) + assert dsl_parameter.options is not None + assert len(dsl_parameter.options) == 2 + assert dsl_parameter.options[0].value == "1h" + assert dsl_parameter.options[0].label == "1 hour" + assert dsl_parameter.options[1].value == "4h" + assert dsl_parameter.options[1].label == "4 hours" + + def test_maps_numeric_constraints(self): + operator_parameter = _sample_parameter( + minimum=1.0, + maximum=100.0, + step=0.5, + ) + dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + operator_parameter + ) + assert dsl_parameter.minimum == 1.0 + assert dsl_parameter.maximum == 100.0 + assert dsl_parameter.step == 0.5 + + def test_maps_multiple_and_primary(self): + operator_parameter = _sample_parameter(multiple=True, primary=True) + dsl_parameter = dsl_protocol.operator_parameter_to_dsl_parameter( + operator_parameter + ) + assert dsl_parameter.multiple is True + assert dsl_parameter.primary is True + + +class TestOperatorDocsToDslKeyword: + def test_maps_inputs_and_outputs(self): + keyword = dsl_protocol.operator_docs_to_dsl_keyword(_sample_docs()) + assert keyword.name == "rsi" + assert keyword.category is protocol_models.DslKeywordCategory.SOURCE + assert keyword.label == "RSI" + assert len(keyword.inputs) == 1 + assert keyword.inputs[0].value_type is protocol_models.DslValueType.SERIES + assert len(keyword.outputs) == 1 + assert keyword.outputs[0].value_type is protocol_models.DslValueType.SERIES + + def test_raises_when_category_missing(self): + with pytest.raises(ValueError, match="CATEGORY"): + dsl_protocol.operator_docs_to_dsl_keyword(_sample_docs(category="")) + + def test_raises_when_return_values_missing(self): + with pytest.raises(ValueError, match="return_values"): + dsl_protocol.operator_docs_to_dsl_keyword(_sample_docs(return_values=[])) + + def test_raises_when_category_unknown(self): + with pytest.raises(ValueError, match="unknown category"): + dsl_protocol.operator_docs_to_dsl_keyword( + _sample_docs(category="not_a_category") + ) + + def test_falls_back_label_to_name(self): + keyword = dsl_protocol.operator_docs_to_dsl_keyword(_sample_docs(label="")) + assert keyword.label == "rsi" + assert keyword.label == keyword.name + + def test_raises_when_parameter_type_invalid(self): + with pytest.raises(ValueError, match="unknown type"): + dsl_protocol.operator_docs_to_dsl_keyword( + _sample_docs( + parameters=[ + _sample_parameter( + name="data", + parameter_type="not_a_type", + ) + ] + ) + ) + + +class TestGetDslKeywordsState: + def test_returns_full_catalog(self): + dsl_keywords_state = dsl_protocol.get_dsl_keywords_state() + assert ( + dsl_keywords_state.version == sync_constants.DSL_KEYWORDS_STATE_VERSION + ) + assert len(dsl_keywords_state.keywords) == len( + dsl_protocol.list_dsl_operator_docs() + ) + assert len(dsl_keywords_state.keywords) > 69 + assert all( + isinstance(keyword, protocol_models.DslKeyword) + for keyword in dsl_keywords_state.keywords + ) diff --git a/packages/node/tests/protocol/util/test_privacy_filter.py b/packages/node/tests/protocol/util/test_privacy_filter.py index b0ca10b134..47f72a4a25 100644 --- a/packages/node/tests/protocol/util/test_privacy_filter.py +++ b/packages/node/tests/protocol/util/test_privacy_filter.py @@ -42,7 +42,7 @@ def test_leaves_non_config_actions_unchanged(self): id="dsl-1", action_type="dsl_script", status=protocol_models.WorkflowStatus.COMPLETED, - dsl='run_octobot_process("acc", {}, [{"api_key": "leak"}])', + dsl='run_octobot_process("acc", profile_data={}, exchange_auth_data=[{"api_key": "leak"}])', ) privatized_actions = privacy_filter.privatize_dag_actions([dsl_action]) returned_action = privatized_actions[0] diff --git a/packages/node/tests/scheduler/__init__.py b/packages/node/tests/scheduler/__init__.py index 06612c167d..9d08455d04 100644 --- a/packages/node/tests/scheduler/__init__.py +++ b/packages/node/tests/scheduler/__init__.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import asyncio import pytest import dbos import mock @@ -58,17 +59,65 @@ def __float__(self) -> float: return float(self._effective_seconds()) -def destroy_launched_dbos() -> None: +def _reset_asyncio_default_executor() -> None: + # DBOS async APIs call loop.set_default_executor(dbos._executor) on first use. + # destroy_launched_dbos() shuts down that executor but leaves the loop pointing at it, + # so later tests on the same pytest-xdist worker see "System database accessed before + # DBOS was launched". Clear the binding so the next test gets a fresh default pool. + try: + loop = asyncio.get_running_loop() + except RuntimeError: + try: + loop = asyncio.get_event_loop() + except RuntimeError: + return + if loop.is_closed(): + return + # Python 3.13 rejects None in set_default_executor(); assign directly so asyncio + # lazily creates a fresh ThreadPoolExecutor on the next run_in_executor call. + loop._default_executor = None + + +def _ensure_scheduler_queues() -> None: + # destroy_launched_dbos() clears Scheduler queue handles; when destroy_registry=False + # the global registry still owns the Queue declarations and re-creating them fails. + scheduler = octobot_node.scheduler.SCHEDULER + if scheduler.AUTOMATION_WORKFLOW_QUEUE is not None: + return + import dbos._dbos as dbos_internals + import octobot_node.enums as octobot_node_enums_module + registry = dbos_internals._get_or_create_dbos_registry() + queue_bindings = { + octobot_node_enums_module.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value: "AUTOMATION_WORKFLOW_QUEUE", + octobot_node_enums_module.SchedulerQueues.USER_ACTION_QUEUE.value: "USER_ACTION_QUEUE", + octobot_node_enums_module.SchedulerQueues.DBOS_CLEANUP_QUEUE.value: "DBOS_CLEANUP_QUEUE", + } + if all(queue_name in registry.queue_info_map for queue_name in queue_bindings): + for queue_name, scheduler_attribute in queue_bindings.items(): + setattr(scheduler, scheduler_attribute, registry.queue_info_map[queue_name]) + return + scheduler.create_queues() + + +def destroy_launched_dbos(*, destroy_registry: bool = False) -> None: """ Tear down the DBOS singleton so the next test can reset the system database. Required when a previous test failed after ``launch()`` or when pytest-xdist runs many scheduler tests on the same worker. """ - dbos.DBOS.destroy(workflow_completion_timeout_sec=0) - - -def init_scheduler(db_file_name: str): + # Clear executor binding before DBOS.destroy — see _reset_asyncio_default_executor. + _reset_asyncio_default_executor() + # destroy_registry=False keeps @DBOS.workflow decorators on the same registry; DBOS() + # rebinds registry.dbos on the next launch. + dbos.DBOS.destroy(workflow_completion_timeout_sec=0, destroy_registry=destroy_registry) + octobot_node.scheduler.SCHEDULER.INSTANCE = None + octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE = None + octobot_node.scheduler.SCHEDULER.USER_ACTION_QUEUE = None + octobot_node.scheduler.SCHEDULER.DBOS_CLEANUP_QUEUE = None + + +def init_scheduler(db_file_name: str, application_version: str | None = None): destroy_launched_dbos() baseline_seconds = float(octobot_node_constants_module.AUTOMATION_WORKFLOW_RETRY_INTERVAL_SECONDS) retry_interval_proxy = AutomationWorkflowIterationRetryInterval(baseline_seconds) @@ -81,14 +130,19 @@ def init_scheduler(db_file_name: str): "name": "scheduler_test", "system_database_url": f"sqlite:///{db_file_name}", } - if octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE is None: - octobot_node.scheduler.SCHEDULER.create_queues() + if application_version is not None: + config["application_version"] = application_version + _ensure_scheduler_queues() dbos.DBOS(config=config) octobot_node.scheduler.SCHEDULER.INSTANCE = dbos.DBOS octobot_node.scheduler.workflows.register_workflows() return dbos.DBOS +def init_scheduler_with_app_version(db_file_name: str, application_version: str): + return init_scheduler(db_file_name, application_version=application_version) + + @pytest.fixture() def temp_dbos_scheduler(): # from https://docs.dbos.dev/python/tutorials/testing diff --git a/packages/node/tests/scheduler/test_api.py b/packages/node/tests/scheduler/test_api.py index 4854d021a6..ce3a95eb6f 100644 --- a/packages/node/tests/scheduler/test_api.py +++ b/packages/node/tests/scheduler/test_api.py @@ -17,6 +17,8 @@ import json import pytest import mock +import dbos +import octobot_protocol.models as protocol_models from octobot_node.models import Execution, Task, TaskStatus from octobot_node.scheduler.api import ( get_node_status, @@ -24,6 +26,8 @@ get_all_tasks, get_task_result, get_tasks_export_results, + await_workflow_result_from_id, + retrieve_workflow_handle, ) from tests.scheduler import temp_dbos_scheduler @@ -488,6 +492,73 @@ async def test_get_all_tasks_list_does_not_decrypt(self) -> None: assert completed_exec.result_metadata == "" +class TestGetAllTasksChildOctobotProcessEnrichment: + @pytest.mark.asyncio + async def test_get_all_tasks_fetches_automation_states_only_for_active_tasks(self) -> None: + parent_id = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + pending_executions = [ + Execution(id=parent_id, status=TaskStatus.PENDING, name="active-task"), + ] + completed_parent_id = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + completed_executions = [ + Execution(id=completed_parent_id, status=TaskStatus.COMPLETED, name="done-task"), + ] + child_state = protocol_models.ChildOctoBotProcessState( + http_base_url="http://127.0.0.1:5002", + web_port=5002, + init_state_ok=True, + ) + automation_state = protocol_models.AutomationState( + id=parent_id, + status=protocol_models.WorkflowStatus.RUNNING, + metadata=protocol_models.AutomationMetadata(name="active-task", description=""), + child_octobot_process=child_state, + ) + + mock_scheduler = mock.Mock() + mock_scheduler.get_periodic_tasks = mock.AsyncMock(return_value=[]) + mock_scheduler.get_pending_tasks = mock.AsyncMock(return_value=pending_executions) + mock_scheduler.get_scheduled_tasks = mock.AsyncMock(return_value=[]) + mock_scheduler.get_results = mock.AsyncMock(return_value=completed_executions) + mock_scheduler.get_automation_states = mock.AsyncMock(return_value=[automation_state]) + + with mock.patch("octobot_node.scheduler.SCHEDULER", mock_scheduler): + tasks = await get_all_tasks() + + mock_scheduler.get_automation_states.assert_awaited_once() + call_args = mock_scheduler.get_automation_states.await_args + assert call_args.args[0] is None + assert call_args.kwargs["statuses"] == [ + dbos.WorkflowStatusString.ENQUEUED, + dbos.WorkflowStatusString.PENDING, + ] + active_task = next(task for task in tasks if task.id == parent_id) + completed_task = next(task for task in tasks if task.id == completed_parent_id) + assert active_task.metadata is not None + assert active_task.metadata.child_octobot_process == child_state + assert completed_task.metadata is None + + @pytest.mark.asyncio + async def test_get_all_tasks_skips_automation_states_when_all_tasks_completed(self) -> None: + completed_parent_id = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + completed_executions = [ + Execution(id=completed_parent_id, status=TaskStatus.COMPLETED, name="done-task"), + ] + + mock_scheduler = mock.Mock() + mock_scheduler.get_periodic_tasks = mock.AsyncMock(return_value=[]) + mock_scheduler.get_pending_tasks = mock.AsyncMock(return_value=[]) + mock_scheduler.get_scheduled_tasks = mock.AsyncMock(return_value=[]) + mock_scheduler.get_results = mock.AsyncMock(return_value=completed_executions) + mock_scheduler.get_automation_states = mock.AsyncMock() + + with mock.patch("octobot_node.scheduler.SCHEDULER", mock_scheduler): + tasks = await get_all_tasks() + + mock_scheduler.get_automation_states.assert_not_awaited() + assert tasks[0].metadata is None + + class TestGetTasksExportResults: """Tests for get_tasks_export_results function (batch decrypt-on-demand).""" @@ -673,3 +744,37 @@ async def test_get_task_result_running(self) -> None: result = await get_task_result(task_id) assert result["status"] == "pending or running" + + +class TestAwaitWorkflowResultFromId: + @pytest.mark.asyncio + async def test_await_workflow_result_from_id(self, temp_dbos_scheduler) -> None: + workflow_result = {"updated_user_action": {"id": "ua-1", "status": "completed"}} + mock_handle = mock.AsyncMock() + mock_handle.get_result = mock.AsyncMock(return_value=workflow_result) + + with mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "retrieve_workflow_async", + mock.AsyncMock(return_value=mock_handle), + ) as mock_retrieve: + result = await await_workflow_result_from_id("workflow-1") + + assert result == workflow_result + mock_retrieve.assert_awaited_once_with("workflow-1") + mock_handle.get_result.assert_awaited_once() + + +class TestRetrieveWorkflowHandle: + @pytest.mark.asyncio + async def test_retrieve_workflow_handle(self, temp_dbos_scheduler) -> None: + mock_handle = mock.AsyncMock() + with mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "retrieve_workflow_async", + mock.AsyncMock(return_value=mock_handle), + ) as mock_retrieve: + handle = await retrieve_workflow_handle("workflow-2") + + assert handle is mock_handle + mock_retrieve.assert_awaited_once_with("workflow-2") diff --git a/packages/node/tests/scheduler/test_generic_process_octobot.py b/packages/node/tests/scheduler/test_generic_process_octobot.py new file mode 100644 index 0000000000..4f9323639c --- /dev/null +++ b/packages/node/tests/scheduler/test_generic_process_octobot.py @@ -0,0 +1,159 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import mock +import pytest +import uuid + +from octobot_node.scheduler.generic_process_octobot import create_generic_process_bot + +from tests.scheduler import temp_dbos_scheduler + + +class TestCreateGenericProcessBotCreatesMissingStrategy: + @pytest.mark.asyncio + async def test_create_generic_process_bot_creates_missing_strategy( + self, + tmp_path, + temp_dbos_scheduler, + ) -> None: + import octobot_commons.constants as commons_constants_module + import octobot_commons.user_root_folder_provider as user_root_folder_provider_module + import octobot_node.constants as octobot_node_constants_module + import octobot_protocol.models as protocol_models_module + import octobot_sync.sync.collection_providers as collection_providers_module + import octobot.community.authentication as community_authentication_module + from tests.functional_tests.util import authenticator_mocks as authenticator_mocks_module + from tests.functional_tests.util import workflow_common as workflow_common_module + + user_root_provider = user_root_folder_provider_module.instance() + previous_user_root = user_root_provider.get_root() + test_user_root = tmp_path / "create_generic_process_bot_user_root" + user_root_provider.set_root(str(test_user_root)) + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + try: + with mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ): + automation_id = await create_generic_process_bot(user_id, "My manual OctoBot") + + stored_strategy = collection_providers_module.StrategyProvider.instance().get_item( + user_id, + octobot_node_constants_module.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID, + ) + assert stored_strategy.reference_market == commons_constants_module.DEFAULT_REFERENCE_MARKET + generic_process_configuration = stored_strategy.configuration.actual_instance + assert isinstance(generic_process_configuration, protocol_models_module.GenericProcessConfiguration) + assert generic_process_configuration.profile_data is None + assert automation_id + assert len(automation_id) == octobot_node_constants_module.PARENT_WORKFLOW_ID_LENGTH + finally: + user_root_provider.set_root(previous_user_root) + + +class TestCreateGenericProcessBotReusesExistingStrategy: + @pytest.mark.asyncio + async def test_create_generic_process_bot_reuses_existing_strategy( + self, + tmp_path, + temp_dbos_scheduler, + ) -> None: + import octobot_commons.user_root_folder_provider as user_root_folder_provider_module + import octobot_node.constants as octobot_node_constants_module + import octobot_sync.sync.collection_providers as collection_providers_module + import octobot.community.authentication as community_authentication_module + from tests.functional_tests.util import authenticator_mocks as authenticator_mocks_module + from tests.functional_tests.util import workflow_common as workflow_common_module + + user_root_provider = user_root_folder_provider_module.instance() + previous_user_root = user_root_provider.get_root() + test_user_root = tmp_path / "create_generic_process_bot_existing_strategy_user_root" + user_root_provider.set_root(str(test_user_root)) + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + try: + with mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ): + first_automation_id = await create_generic_process_bot(user_id, "First manual OctoBot") + strategy_after_first_create = collection_providers_module.StrategyProvider.instance().get_item( + user_id, + octobot_node_constants_module.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID, + ) + second_automation_id = await create_generic_process_bot(user_id, "Second manual OctoBot") + strategy_after_second_create = collection_providers_module.StrategyProvider.instance().get_item( + user_id, + octobot_node_constants_module.NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID, + ) + + assert first_automation_id != second_automation_id + assert strategy_after_second_create.id == strategy_after_first_create.id + assert strategy_after_second_create.version == strategy_after_first_create.version + finally: + user_root_provider.set_root(previous_user_root) + + +class TestCreateGenericProcessBotUsesProvidedAutomationId: + @pytest.mark.asyncio + async def test_create_generic_process_bot_uses_provided_automation_id( + self, + tmp_path, + temp_dbos_scheduler, + ) -> None: + import octobot_commons.user_root_folder_provider as user_root_folder_provider_module + import octobot.community.authentication as community_authentication_module + from tests.functional_tests.util import authenticator_mocks as authenticator_mocks_module + from tests.functional_tests.util import workflow_common as workflow_common_module + + user_root_provider = user_root_folder_provider_module.instance() + previous_user_root = user_root_provider.get_root() + test_user_root = tmp_path / "create_generic_process_bot_provided_automation_id_user_root" + user_root_provider.set_root(str(test_user_root)) + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + provided_automation_id = str(uuid.uuid4()) + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + try: + with mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ): + automation_id = await create_generic_process_bot( + user_id, + "Provided id OctoBot", + automation_id=provided_automation_id, + ) + + assert automation_id == provided_automation_id + finally: + user_root_provider.set_root(previous_user_root) diff --git a/packages/node/tests/scheduler/test_octobot_flow_client.py b/packages/node/tests/scheduler/test_octobot_flow_client.py index d01653a1fb..4cd1b350c3 100644 --- a/packages/node/tests/scheduler/test_octobot_flow_client.py +++ b/packages/node/tests/scheduler/test_octobot_flow_client.py @@ -1622,3 +1622,61 @@ async def test_run_multiple_actions_bundle_with_wait(self, multiple_action_bundl assert post_withdraw_portfolio["BTC"] == post_trade_portfolio["BTC"] assert "ETH" not in post_withdraw_portfolio assert result.has_next_actions is False # no more actions to execute + + +class TestOctoBotActionsJobRunLogging: + @pytest.mark.asyncio + async def test_logs_executed_actions_not_input_priority_actions(self): + if message := misses_required_octobot_flow_client_import(): + pytest.skip(reason=message) + + executed_action = octobot_flow.entities.ConfiguredActionDetails( + id="executed_stop", + action="stop_automation()", + ) + priority_user_actions = [ + { + "id": "action_stop_priority_input_priority", + "dsl_script": "stop_automation()", + } + ] + mock_automation_job = mock.AsyncMock() + mock_automation_job.run = mock.AsyncMock(return_value=[executed_action]) + mock_automation_job.automation_state = mock.Mock() + mock_automation_job.automation_state.automation = mock.Mock() + mock_automation_job.automation_state.automation.actions_dag = mock.Mock() + mock_automation_job.automation_state.automation.actions_dag.get_executable_actions = mock.Mock( + return_value=[] + ) + mock_automation_job.automation_state.automation.actions_dag.get_pending_actions = mock.Mock( + return_value=[] + ) + mock_automation_job.automation_state.automation.post_actions = mock.Mock(stop_automation=False) + mock_automation_job.dump = mock.Mock( + return_value={"automation": {"metadata": {"automation_id": "automation_1"}}} + ) + + mock_automation_job_context = mock.AsyncMock() + mock_automation_job_context.__aenter__ = mock.AsyncMock(return_value=mock_automation_job) + mock_automation_job_context.__aexit__ = mock.AsyncMock(return_value=False) + + mock_logger = mock.Mock() + job = octobot_flow_client.OctoBotActionsJob( + {"state": {"automation": {"metadata": {"automation_id": "automation_1"}}}}, + priority_user_actions, + [], + octobot_flow_client.OctoBotActionsJobResult(), + ) + + with mock.patch( + "octobot_flow.jobs.AutomationJob", + return_value=mock_automation_job_context, + ), mock.patch( + "octobot_commons.logging.get_logger", + return_value=mock_logger, + ): + await job.run() + + mock_logger.info.assert_called_once_with( + f"Running automation actions: {[executed_action]}" + ) diff --git a/packages/node/tests/scheduler/test_scheduler.py b/packages/node/tests/scheduler/test_scheduler.py index bdcb6ca306..0ca64e7378 100644 --- a/packages/node/tests/scheduler/test_scheduler.py +++ b/packages/node/tests/scheduler/test_scheduler.py @@ -29,6 +29,7 @@ import octobot_node.scheduler.encryption.task_inputs as task_inputs_encryption import octobot_node.scheduler.workflows.params as params import octobot_node.scheduler.workflows_util as workflows_util +import octobot_node.scheduler.workflows_retention as workflows_retention import octobot_node.scheduler.scheduler as scheduler_module PARENT_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" @@ -88,12 +89,7 @@ def _make_scheduler_with_mock_instance() -> tuple[scheduler_module.Scheduler, mo return sched, sched.INSTANCE -_DELETE_WORKFLOW_STATUSES = [ - dbos.WorkflowStatusString.SUCCESS, - dbos.WorkflowStatusString.ERROR, - dbos.WorkflowStatusString.CANCELLED, - dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED, -] +_DELETE_WORKFLOW_STATUSES = workflows_retention._TERMINAL_DELETE_WORKFLOW_STATUSES def _build_user_action_workflow_with_inputs( @@ -1037,32 +1033,65 @@ async def test_matches_terminal_workflow_from_output_when_load_output_true(self) ) assert result == ["wf-done"] + +class TestResolveAutomationOwnerUserId: @pytest.mark.asyncio - async def test_get_workflows_to_delete_merges_automation_and_user_action_ids(self): - automation_task = octobot_node.models.Task( - id=PARENT_ID, - name="automation-task", - content="encrypted_content", - content_metadata="meta", + async def test_returns_owner_user_id_from_latest_active_workflow(self): + parent_id = PARENT_ID + owner_user_id = "tenant-starfish-id" + task = octobot_node.models.Task( + id=parent_id, + name="tenant-task", + content=None, type="execute_actions", + user_id=owner_user_id, ) - automation_workflow = _build_mock_workflow_status( - automation_task, - "encrypted_state", - None, - workflow_id=PARENT_ID, + older_child = _build_mock_workflow_status_no_output(task, workflow_id=f"{parent_id}_1") + older_child.status = dbos.WorkflowStatusString.PENDING.value + older_child.updated_at = 10 + latest_child = _build_mock_workflow_status_no_output(task, workflow_id=f"{parent_id}_2") + latest_child.status = dbos.WorkflowStatusString.ENQUEUED.value + latest_child.updated_at = 20 + + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock( + return_value=[older_child, latest_child], ) - user_action_workflow = _build_user_action_workflow_with_output("ua-delete", "wf-ua-delete") - async def list_workflows_side_effect(**kwargs): - queue_name = kwargs.get("queue_name") - if queue_name == [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value]: - return [automation_workflow] - if queue_name == [octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value]: - return [user_action_workflow] - return [] + result = await sched.resolve_automation_owner_user_id(parent_id) + + assert result == owner_user_id + @pytest.mark.asyncio + async def test_returns_none_when_no_matching_workflow(self): sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(side_effect=list_workflows_side_effect) - result = await sched._get_workflows_to_delete([PARENT_ID, "ua-delete"]) - assert result == [PARENT_ID, "wf-ua-delete"] + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[]) + + result = await sched.resolve_automation_owner_user_id("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") + + assert result is None + + +class TestSchedulerDeleteWorkflows: + @pytest.mark.asyncio + async def test_delegates_to_workflows_retention_helpers(self): + sched, mock_instance = _make_scheduler_with_mock_instance() + input_workflow_ids = ["parent-id"] + merged_workflow_ids = ["wf-a", "wf-b"] + + with mock.patch.object( + workflows_retention, + "get_workflows_to_delete", + mock.AsyncMock(return_value=merged_workflow_ids), + ) as get_workflows_mock, mock.patch.object( + workflows_retention, + "delete_workflows_and_vacuum", + mock.AsyncMock(), + ) as delete_vacuum_mock: + await sched.delete_workflows(input_workflow_ids) + + get_workflows_mock.assert_awaited_once_with(sched, input_workflow_ids) + delete_vacuum_mock.assert_awaited_once_with( + mock_instance, + merged_workflow_ids, + ) diff --git a/packages/node/tests/scheduler/test_schedules.py b/packages/node/tests/scheduler/test_schedules.py new file mode 100644 index 0000000000..48834486eb --- /dev/null +++ b/packages/node/tests/scheduler/test_schedules.py @@ -0,0 +1,666 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import datetime + +import dbos +import mock +import pytest + +import octobot_node.constants +import octobot_node.enums + +from tests.scheduler import temp_dbos_scheduler + + +def _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) -> dict: + return dbos_cleanup_workflow_module.get_schedule_input() + + +def _matching_existing_schedule( + dbos_cleanup_workflow_module, + scheduler, + *, + automatic_backfill: bool = True, + queue_name: str | None = None, +) -> dict: + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + if queue_name is None: + queue_name = octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value + return { + "schedule_id": "existing-schedule-id", + "schedule_name": dbos_cleanup_workflow_module.SCHEDULE_NAME, + "workflow_name": "ignored-workflow-name", + "workflow_class_name": None, + "schedule": schedule_input["schedule"], + "status": "ACTIVE", + "context": None, + "last_fired_at": "2026-07-13T00:00:00+00:00", + "automatic_backfill": automatic_backfill, + "cron_timezone": schedule_input.get("cron_timezone"), + "queue_name": queue_name, + } + + +class TestExistingScheduleMatchesConfigured: + def test_returns_true_when_all_fields_match(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ) + + assert schedules_module._existing_schedule_matches_configured( + existing_schedule, + schedule_input, + ) is True + + def test_returns_false_when_cron_differs(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ) + existing_schedule["schedule"] = "0 1 * * *" + + assert schedules_module._existing_schedule_matches_configured( + existing_schedule, + schedule_input, + ) is False + + def test_returns_false_when_queue_name_differs(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + queue_name="other_queue", + ) + + assert schedules_module._existing_schedule_matches_configured( + existing_schedule, + schedule_input, + ) is False + + def test_returns_false_when_automatic_backfill_differs(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + automatic_backfill=False, + ) + + assert schedules_module._existing_schedule_matches_configured( + existing_schedule, + schedule_input, + ) is False + + +class TestGetBackfillScheduleDefaultAnchor: + def test_returns_utc_now_minus_one_day(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + expected_now = datetime.datetime.now(datetime.timezone.utc) + anchor = schedules_module.get_backfill_schedule_default_anchor() + expected_anchor = expected_now - datetime.timedelta( + days=octobot_node.constants.SCHEDULES_DEFAULT_BACKFILL_DAYS, + ) + + assert anchor.tzinfo == datetime.timezone.utc + assert abs((anchor - expected_anchor).total_seconds()) < 1 + + +class TestBuildScheduledWorkflowId: + def test_builds_expected_workflow_id(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + trigger_time = datetime.datetime(2026, 7, 15, 0, 0, 0, tzinfo=datetime.timezone.utc) + assert schedules_module.build_scheduled_workflow_id( + dbos_cleanup_workflow_module.SCHEDULE_NAME, + trigger_time, + ) == f"sched-{dbos_cleanup_workflow_module.SCHEDULE_NAME}-2026-07-15T00:00:00+00:00" + + +class TestEnumerateScheduleWorkflowIdsInWindow: + def test_daily_cron_yields_expected_workflow_id(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + start = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) + end = datetime.datetime(2026, 7, 15, 8, 30, 0, tzinfo=datetime.timezone.utc) + workflow_ids = schedules_module._enumerate_schedule_workflow_ids_in_window( + dbos_cleanup_workflow_module.SCHEDULE_NAME, + schedule_input, + start, + end, + ) + assert workflow_ids == [ + f"sched-{dbos_cleanup_workflow_module.SCHEDULE_NAME}-2026-07-15T00:00:00+00:00", + ] + + +class TestGetScheduledWorkflowTriggerTime: + def test_returns_trigger_time_from_workflow_id(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + workflow_id = ( + f"sched-{dbos_cleanup_workflow_module.SCHEDULE_NAME}-" + "2026-07-15T00:00:00+00:00" + ) + assert schedules_module.get_scheduled_workflow_trigger_time( + workflow_id, + dbos_cleanup_workflow_module.SCHEDULE_NAME, + ) == "2026-07-15T00:00:00+00:00" + + def test_returns_none_for_unexpected_workflow_id(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + assert schedules_module.get_scheduled_workflow_trigger_time( + "trigger-dbos_cleanup_daily-2026-07-15T00:00:00+00:00", + dbos_cleanup_workflow_module.SCHEDULE_NAME, + ) is None + + +class TestRegisterSchedules: + pytestmark = pytest.mark.asyncio + + @pytest.fixture + def dbos_cleanup_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module_loaded + + yield dbos_cleanup_workflow_module_loaded + + async def test_creates_schedule_when_missing(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + mock_logger = mock.Mock() + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=None, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ) as create_schedule_mock, mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock_logger, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock, mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + create_schedule_mock.assert_awaited_once_with( + schedule_name=schedule_input["schedule_name"], + workflow_fn=schedule_input["workflow_fn"], + schedule=schedule_input["schedule"], + context=schedule_input.get("context"), + automatic_backfill=schedule_input.get("automatic_backfill", False), + cron_timezone=schedule_input.get("cron_timezone"), + queue_name=schedule_input.get("queue_name"), + ) + apply_schedules_mock.assert_not_awaited() + backfill_to_thread_mock.assert_not_awaited() + mock_logger.info.assert_called_once_with( + "Creating schedule %s (%s)", + schedule_input["schedule_name"], + schedule_input["schedule"], + ) + + async def test_keeps_schedule_when_config_matches(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ) + mock_logger = mock.Mock() + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=existing_schedule, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ) as create_schedule_mock, mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock_logger, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock, mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + create_schedule_mock.assert_not_awaited() + apply_schedules_mock.assert_not_awaited() + backfill_to_thread_mock.assert_not_awaited() + mock_logger.info.assert_called_once_with( + "Keeping existing schedule %s (%s)", + schedule_input["schedule_name"], + schedule_input["schedule"], + ) + + async def test_recreates_schedule_when_config_differs(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + automatic_backfill=False, + ) + mock_logger = mock.Mock() + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=existing_schedule, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ) as create_schedule_mock, mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock_logger, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock, mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + create_schedule_mock.assert_not_awaited() + apply_schedules_mock.assert_awaited_once_with([schedule_input]) + backfill_to_thread_mock.assert_not_awaited() + mock_logger.info.assert_called_once_with( + "Updating schedule %s (%s): configuration changed", + schedule_input["schedule_name"], + schedule_input["schedule"], + ) + + async def test_backfills_when_last_fired_at_is_null( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ) + existing_schedule["last_fired_at"] = None + anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) + workflow_id = ( + f"sched-{schedule_input['schedule_name']}-2026-07-15T00:00:00+00:00" + ) + mock_handle = mock.Mock() + mock_handle.get_workflow_id.return_value = workflow_id + mock_logger = mock.Mock() + backfill_end = datetime.datetime(2026, 7, 15, 8, 30, 0, tzinfo=datetime.timezone.utc) + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = backfill_end + + async def backfill_to_thread_side_effect(func, *args, **kwargs): + return func(*args, **kwargs) + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=existing_schedule, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock_logger, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.get_backfill_schedule_default_anchor", + return_value=anchor, + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=None, + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + side_effect=backfill_to_thread_side_effect, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.backfill_schedule", + return_value=[mock_handle], + ) as backfill_schedule_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + backfill_schedule_mock.assert_called_once() + schedule_name, backfill_start, backfill_end_arg = backfill_schedule_mock.call_args[0] + assert schedule_name == schedule_input["schedule_name"] + assert backfill_start == anchor + assert backfill_end_arg == backfill_end + mock_logger.info.assert_any_call( + "Startup backfill for schedule %s: last_fired_at unset, checking missed cron slots in [%s, %s)", + schedule_input["schedule_name"], + anchor.isoformat(), + backfill_end.isoformat(), + ) + mock_logger.info.assert_any_call( + "Startup backfill enqueued schedule %s workflow %s", + schedule_input["schedule_name"], + workflow_id, + ) + mock_logger.info.assert_any_call( + "Startup backfill finished for schedule %s: %s enqueued, %s unchanged", + schedule_input["schedule_name"], + 1, + 0, + ) + + async def test_skips_backfill_when_all_window_slots_terminal( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ) + existing_schedule["last_fired_at"] = None + anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) + backfill_end = datetime.datetime(2026, 7, 15, 8, 30, 0, tzinfo=datetime.timezone.utc) + workflow_id = ( + f"sched-{schedule_input['schedule_name']}-2026-07-15T00:00:00+00:00" + ) + success_status = mock.Mock(spec=dbos.WorkflowStatus) + success_status.status = dbos.WorkflowStatusString.SUCCESS.value + mock_logger = mock.Mock() + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = backfill_end + + async def get_workflow_status_side_effect(requested_workflow_id: str): + if requested_workflow_id == workflow_id: + return success_status + return None + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=existing_schedule, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock_logger, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.get_backfill_schedule_default_anchor", + return_value=anchor, + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=get_workflow_status_side_effect, + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + backfill_to_thread_mock.assert_not_awaited() + mock_logger.info.assert_any_call( + "Startup backfill not needed for schedule %s: last_fired_at unset but %s cron slot(s) in [%s, %s) already terminal", + schedule_input["schedule_name"], + 1, + anchor.isoformat(), + backfill_end.isoformat(), + ) + mock_logger.info.assert_any_call( + "Schedule %s slot %s already %s", + schedule_input["schedule_name"], + workflow_id, + dbos.WorkflowStatusString.SUCCESS.value, + ) + + async def test_backfill_logs_unchanged_when_slot_already_terminal_in_mixed_window( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + schedule_input["schedule"] = "0 * * * *" + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ) + existing_schedule["last_fired_at"] = None + existing_schedule["schedule"] = "0 * * * *" + anchor = datetime.datetime(2026, 7, 15, 10, 0, 0, tzinfo=datetime.timezone.utc) + backfill_end = datetime.datetime(2026, 7, 15, 12, 30, 0, tzinfo=datetime.timezone.utc) + terminal_workflow_id = ( + f"sched-{schedule_input['schedule_name']}-2026-07-15T11:00:00+00:00" + ) + missing_workflow_id = ( + f"sched-{schedule_input['schedule_name']}-2026-07-15T12:00:00+00:00" + ) + success_status = mock.Mock(spec=dbos.WorkflowStatus) + success_status.status = dbos.WorkflowStatusString.SUCCESS.value + mock_logger = mock.Mock() + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = backfill_end + mock_handles = [] + for workflow_id in (terminal_workflow_id, missing_workflow_id): + mock_handle = mock.Mock() + mock_handle.get_workflow_id.return_value = workflow_id + mock_handles.append(mock_handle) + + async def get_workflow_status_side_effect(requested_workflow_id: str): + if requested_workflow_id == terminal_workflow_id: + return success_status + return None + + async def backfill_to_thread_side_effect(func, *args, **kwargs): + return func(*args, **kwargs) + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=existing_schedule, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock_logger, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.get_backfill_schedule_default_anchor", + return_value=anchor, + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=get_workflow_status_side_effect, + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + side_effect=backfill_to_thread_side_effect, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.backfill_schedule", + return_value=mock_handles, + ) as backfill_schedule_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + backfill_schedule_mock.assert_called_once() + mock_logger.info.assert_any_call( + "Startup backfill enqueued schedule %s workflow %s", + schedule_input["schedule_name"], + missing_workflow_id, + ) + mock_logger.info.assert_any_call( + "Startup backfill left schedule %s workflow %s unchanged (already %s)", + schedule_input["schedule_name"], + terminal_workflow_id, + dbos.WorkflowStatusString.SUCCESS.value, + ) + mock_logger.info.assert_any_call( + "Startup backfill finished for schedule %s: %s enqueued, %s unchanged", + schedule_input["schedule_name"], + 1, + 1, + ) + + async def test_skips_backfill_when_last_fired_at_set( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ) + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=existing_schedule, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock.Mock(), + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + backfill_to_thread_mock.assert_not_awaited() + + async def test_skips_backfill_when_automatic_backfill_false( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + schedule_input["automatic_backfill"] = False + existing_schedule = _matching_existing_schedule( + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + automatic_backfill=False, + ) + existing_schedule["last_fired_at"] = None + + with mock.patch( + "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", + return_value=schedule_input, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=existing_schedule, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules._get_logger", + return_value=mock.Mock(), + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module.register_schedules(temp_dbos_scheduler) + + backfill_to_thread_mock.assert_not_awaited() diff --git a/packages/node/tests/scheduler/test_shutdown.py b/packages/node/tests/scheduler/test_shutdown.py new file mode 100644 index 0000000000..2183bc23d3 --- /dev/null +++ b/packages/node/tests/scheduler/test_shutdown.py @@ -0,0 +1,79 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import pytest +import mock + +import octobot_node.scheduler as scheduler_module + + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture(autouse=True) +def reset_scheduler_shutdown_guard(): + yield + scheduler_module._shutdown_done = False + + +class TestShutdownSchedulerAndTradingSignalChannel: + async def test_second_call_is_no_op_after_shutdown(self): + with mock.patch.object(scheduler_module.SCHEDULER, "is_initialized", return_value=True): + with mock.patch.object(scheduler_module.SCHEDULER, "stop") as stop_mock: + await scheduler_module.shutdown_scheduler_and_trading_signal_channel() + await scheduler_module.shutdown_scheduler_and_trading_signal_channel() + stop_mock.assert_called_once() + + async def test_skips_when_scheduler_not_initialized(self): + with mock.patch.object(scheduler_module.SCHEDULER, "is_initialized", return_value=False): + with mock.patch.object(scheduler_module.SCHEDULER, "stop") as stop_mock: + await scheduler_module.shutdown_scheduler_and_trading_signal_channel() + stop_mock.assert_not_called() + + async def test_initialize_scheduler_resets_shutdown_guard(self): + init_call_order: list[str] = [] + + def track_start() -> None: + init_call_order.append("start") + + async def track_register_schedules(*args, **kwargs) -> None: + init_call_order.append("register_schedules") + + previous_instance = scheduler_module.SCHEDULER.INSTANCE + scheduler_module.SCHEDULER.INSTANCE = mock.Mock() + try: + import octobot_node.scheduler.schedules as schedules_module + + with mock.patch.object(scheduler_module.SCHEDULER, "create"): + with mock.patch.object( + scheduler_module.SCHEDULER, + "start", + side_effect=track_start, + ): + with mock.patch("octobot_node.scheduler.workflows.register_workflows"): + with mock.patch.object( + schedules_module, + "register_schedules", + side_effect=track_register_schedules, + ): + scheduler_module._shutdown_done = True + await scheduler_module.initialize_scheduler() + finally: + scheduler_module.SCHEDULER.INSTANCE = previous_instance + assert scheduler_module._shutdown_done is False + assert init_call_order == [ + "start", + "register_schedules", + ] diff --git a/packages/node/tests/scheduler/test_user_action_executor_factory.py b/packages/node/tests/scheduler/test_user_action_executor_factory.py index dbbae514e4..302c762ab0 100644 --- a/packages/node/tests/scheduler/test_user_action_executor_factory.py +++ b/packages/node/tests/scheduler/test_user_action_executor_factory.py @@ -98,6 +98,15 @@ def test_returns_stop_automation_executor_class(self): resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) assert resolved_executor_cls is user_actions_executor_package.StopAutomationActionExecutor + def test_returns_restart_automation_executor_class(self): + configuration_inner = protocol_models.RestartAutomationConfiguration( + id="auto-restart", + action_type=protocol_models.UserActionType.AUTOMATION_RESTART, + ) + user_action_model = self._user_action(action_identifier="ua-restart", configuration_inner=configuration_inner) + resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) + assert resolved_executor_cls is user_actions_executor_package.RestartAutomationActionExecutor + def test_returns_signal_automation_executor_class(self): configuration_inner = protocol_models.SignalAutomationConfiguration( action_type=protocol_models.UserActionType.AUTOMATION_SIGNAL, diff --git a/packages/node/tests/scheduler/test_workflows_retention.py b/packages/node/tests/scheduler/test_workflows_retention.py new file mode 100644 index 0000000000..e992ad0d06 --- /dev/null +++ b/packages/node/tests/scheduler/test_workflows_retention.py @@ -0,0 +1,595 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import datetime +import json +import mock +import pytest +import dbos + +import octobot_protocol.models as protocol_models +import octobot_node.enums +import octobot_node.models +import octobot_node.scheduler.scheduler as scheduler_module +import octobot_node.scheduler.workflows.params as params +import octobot_node.scheduler.workflows_retention as workflows_retention + +from tests.scheduler import temp_dbos_scheduler + +_AUTOMATION_WORKFLOW_NAME = "execute_automation" +_DBOS_CLEANUP_WORKFLOW_NAME = "dbos_cleanup" + +_PARENT_WORKFLOW_ID_A = "741ce171-dac9-40be-83dc-b443c0eaf0e2" +_PARENT_WORKFLOW_ID_B = "852df282-edb0-51cf-94ed-c554d1fbf1f3" +_PARENT_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + + +def _child_workflow_id(parent_id: str, child_index: int) -> str: + if child_index == 0: + return parent_id + return f"{parent_id}_{child_index}" + + +def _workflow_status_row( + *, + workflow_id: str, + updated_at: int = 0, + status: str = dbos.WorkflowStatusString.SUCCESS.value, + name: str = _AUTOMATION_WORKFLOW_NAME, +) -> mock.Mock: + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = workflow_id + workflow_status.updated_at = updated_at + workflow_status.status = status + workflow_status.name = name + return workflow_status + + +def _build_mock_workflow_status( + task: octobot_node.models.Task, + encrypted_state: str, + state_metadata: str, + workflow_id: str = _PARENT_ID, +) -> mock.Mock: + output = params.AutomationWorkflowOutput(state=encrypted_state, state_metadata=state_metadata) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = workflow_id + workflow_status.name = "test-task" + workflow_status.status = dbos.WorkflowStatusString.SUCCESS.value + workflow_status.output = json.dumps(output.to_dict()) + workflow_status.input = {"args": [inputs.to_dict()], "kwargs": {}} + workflow_status.created_at = None + workflow_status.updated_at = None + return workflow_status + + +def _build_user_action_workflow_with_output( + user_action_id: str, + workflow_id: str, + user_id: str = "0xw1", +) -> mock.Mock: + user_action = protocol_models.UserAction( + id=user_action_id, + status=protocol_models.UserActionStatus.COMPLETED, + configuration=None, + ) + output_payload = params.UserActionWorkflowOutput( + user_id=user_id, + updated_user_action=user_action, + ).to_dict(include_default_values=False) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = workflow_id + workflow_status.input = {"args": [], "kwargs": {}} + workflow_status.output = output_payload + return workflow_status + + +def _make_scheduler_with_mock_instance() -> tuple[scheduler_module.Scheduler, mock.AsyncMock]: + sched = scheduler_module.Scheduler() + sched.INSTANCE = mock.AsyncMock() + return sched, sched.INSTANCE + + +class TestIsTerminalWorkflow: + def test_returns_true_for_terminal_status(self): + workflow_status = _workflow_status_row( + workflow_id="wf-terminal", + status=dbos.WorkflowStatusString.SUCCESS.value, + ) + assert workflows_retention.is_terminal_workflow(workflow_status) is True + + def test_returns_false_for_non_terminal_status(self): + workflow_status = _workflow_status_row( + workflow_id="wf-pending", + status=dbos.WorkflowStatusString.PENDING.value, + ) + assert workflows_retention.is_terminal_workflow(workflow_status) is False + + +class TestGetOutdatedAutomationExecutionDeletions: + def test_keeps_latest_two_and_deletes_older_terminal_executions(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + cutoff_ms = now_ms - int(retention_seconds * 1000) + workflows = [ + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 0), + updated_at=cutoff_ms - 2, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 1), + updated_at=cutoff_ms - 1, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 2), + updated_at=cutoff_ms + 1, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 3), + updated_at=cutoff_ms + 2, + ), + ] + + deletions = workflows_retention.get_outdated_automation_execution_deletions( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deletions == { + _PARENT_WORKFLOW_ID_A: [ + _child_workflow_id(_PARENT_WORKFLOW_ID_A, 1), + _child_workflow_id(_PARENT_WORKFLOW_ID_A, 0), + ], + } + + def test_skips_non_terminal_executions(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + workflows = [ + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 0), + updated_at=old_updated_at, + status=dbos.WorkflowStatusString.PENDING.value, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 1), + updated_at=old_updated_at, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 2), + updated_at=old_updated_at, + ), + ] + + deletions = workflows_retention.get_outdated_automation_execution_deletions( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deletions == {} + + def test_does_not_delete_recent_terminal_executions_beyond_keep_count(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + recent_updated_at = now_ms - 1 + workflows = [ + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 0), + updated_at=recent_updated_at, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 1), + updated_at=recent_updated_at + 1, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 2), + updated_at=recent_updated_at + 2, + ), + ] + + deletions = workflows_retention.get_outdated_automation_execution_deletions( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deletions == {} + + def test_isolates_deletions_per_parent(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + workflows = [ + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 0), + updated_at=old_updated_at, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 1), + updated_at=old_updated_at + 1, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 2), + updated_at=old_updated_at + 2, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_B, 0), + updated_at=old_updated_at, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_B, 1), + updated_at=old_updated_at + 1, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_B, 2), + updated_at=old_updated_at + 2, + ), + ] + + deletions = workflows_retention.get_outdated_automation_execution_deletions( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert set(deletions.keys()) == {_PARENT_WORKFLOW_ID_A, _PARENT_WORKFLOW_ID_B} + assert len(deletions[_PARENT_WORKFLOW_ID_A]) == 1 + assert len(deletions[_PARENT_WORKFLOW_ID_B]) == 1 + + +class TestGetOutdatedDbosCleanupExecutionWorkflowIds: + def test_deletes_old_terminal_cleanup_runs(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + cleanup_workflows = [ + _workflow_status_row( + workflow_id="cleanup-run-1", + updated_at=old_updated_at, + name=_DBOS_CLEANUP_WORKFLOW_NAME, + ), + _workflow_status_row( + workflow_id="cleanup-run-2", + updated_at=now_ms - 1, + name=_DBOS_CLEANUP_WORKFLOW_NAME, + ), + ] + + deleted_ids = workflows_retention.get_outdated_dbos_cleanup_execution_workflow_ids( + cleanup_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deleted_ids == ["cleanup-run-1"] + + def test_skips_non_terminal_cleanup_runs(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + cleanup_workflows = [ + _workflow_status_row( + workflow_id="cleanup-run-pending", + updated_at=old_updated_at, + status=dbos.WorkflowStatusString.PENDING.value, + name=_DBOS_CLEANUP_WORKFLOW_NAME, + ), + ] + + deleted_ids = workflows_retention.get_outdated_dbos_cleanup_execution_workflow_ids( + cleanup_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deleted_ids == [] + + +class TestGetWorkflowsToDelete: + @pytest.mark.asyncio + async def test_merges_automation_and_user_action_ids(self): + automation_task = octobot_node.models.Task( + id=_PARENT_ID, + name="automation-task", + content="encrypted_content", + content_metadata="meta", + type="execute_actions", + ) + automation_workflow = _build_mock_workflow_status( + automation_task, + "encrypted_state", + None, + workflow_id=_PARENT_ID, + ) + user_action_workflow = _build_user_action_workflow_with_output("ua-delete", "wf-ua-delete") + + async def list_workflows_side_effect(**kwargs): + queue_name = kwargs.get("queue_name") + if queue_name == [octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value]: + return [automation_workflow] + if queue_name == [octobot_node.enums.SchedulerQueues.USER_ACTION_QUEUE.value]: + return [user_action_workflow] + return [] + + sched, mock_instance = _make_scheduler_with_mock_instance() + mock_instance.list_workflows_async = mock.AsyncMock(side_effect=list_workflows_side_effect) + result = await workflows_retention.get_workflows_to_delete(sched, [_PARENT_ID, "ua-delete"]) + assert result == [_PARENT_ID, "wf-ua-delete"] + + +class TestVacuumDbosSystemDatabase: + def test_executes_vacuum_on_system_database(self): + mock_instance = mock.Mock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + mock_logger = mock.Mock() + + with mock.patch( + "octobot_node.scheduler.workflows_retention._get_logger", + return_value=mock_logger, + ): + workflows_retention.vacuum_dbos_system_database(mock_instance) + + mock_connection.execute.assert_called_once() + assert mock_connection.execute.call_args[0][0].text == "VACUUM" + mock_logger.info.assert_any_call("Vacuuming database") + mock_logger.info.assert_any_call("Database vacuum completed") + + +class TestDeleteWorkflowsAndVacuum: + @pytest.mark.asyncio + async def test_deletes_workflows_then_vacuums(self): + mock_instance = mock.Mock() + mock_instance.delete_workflows_async = mock.AsyncMock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + mock_logger = mock.Mock() + workflow_ids = ["wf-a", "wf-b"] + + with mock.patch( + "octobot_node.scheduler.workflows_retention._get_logger", + return_value=mock_logger, + ): + await workflows_retention.delete_workflows_and_vacuum( + mock_instance, + workflow_ids, + ) + + mock_instance.delete_workflows_async.assert_awaited_once_with( + workflow_ids, + delete_children=False, + ) + mock_connection.execute.assert_called_once() + assert mock_connection.execute.call_args[0][0].text == "VACUUM" + mock_logger.info.assert_any_call("Deleting %s workflows", len(workflow_ids)) + mock_logger.info.assert_any_call("Vacuuming database") + mock_logger.info.assert_any_call("Database vacuum completed") + + +class TestCleanupOutdatedAutomationExecutions: + @pytest.mark.asyncio + async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_scheduler): + now_ms = 10_000_000 + retention_seconds = 100.0 + cutoff_ms = now_ms - int(retention_seconds * 1000) + automation_workflows = [ + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 0), + updated_at=cutoff_ms - 1, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 1), + updated_at=cutoff_ms + 1, + ), + _workflow_status_row( + workflow_id=_child_workflow_id(_PARENT_WORKFLOW_ID_A, 2), + updated_at=cutoff_ms + 2, + ), + ] + cleanup_workflows = [ + _workflow_status_row( + workflow_id="cleanup-run-old", + updated_at=cutoff_ms - 1, + name=_DBOS_CLEANUP_WORKFLOW_NAME, + ), + ] + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock( + side_effect=[automation_workflows, cleanup_workflows] + ) + mock_instance.delete_workflows_async = mock.AsyncMock() + mock_engine = mock.Mock() + mock_connection = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + sched.INSTANCE = mock_instance + mock_logger = mock.Mock() + + with mock.patch("octobot_node.scheduler.workflows_retention.time.time", return_value=now_ms / 1000), mock.patch.object( + workflows_retention, + "AUTOMATION_EXECUTION_RETENTION_SECONDS", + retention_seconds, + ), mock.patch( + "octobot_node.scheduler.workflows_retention._get_logger", + return_value=mock_logger, + ): + summary = await workflows_retention.cleanup_outdated_automation_executions(sched) + + assert summary == { + "deleted_by_automation": {_PARENT_WORKFLOW_ID_A: 1}, + "deleted_cleanup_executions": 1, + "total_deleted": 2, + } + mock_instance.delete_workflows_async.assert_awaited_once_with( + [ + _child_workflow_id(_PARENT_WORKFLOW_ID_A, 0), + "cleanup-run-old", + ], + delete_children=False, + ) + mock_connection.execute.assert_called_once() + mock_logger.info.assert_any_call( + "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs", + 2, + 1, + 1, + ) + mock_logger.info.assert_any_call("DBOS cleanup summary: %s", summary) + + @pytest.mark.asyncio + async def test_skips_delete_and_vacuum_when_nothing_to_delete(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock(side_effect=[[], []]) + mock_instance.delete_workflows_async = mock.AsyncMock() + mock_instance._sys_db.engine = mock.Mock() + sched.INSTANCE = mock_instance + mock_logger = mock.Mock() + + with mock.patch( + "octobot_node.scheduler.workflows_retention._get_logger", + return_value=mock_logger, + ): + summary = await workflows_retention.cleanup_outdated_automation_executions(sched) + + assert summary == { + "deleted_by_automation": {}, + "deleted_cleanup_executions": 0, + "total_deleted": 0, + } + mock_instance.delete_workflows_async.assert_not_called() + mock_instance._sys_db.engine.begin.assert_not_called() + mock_logger.info.assert_called_once_with("DBOS cleanup summary: %s", summary) + + +class TestShouldSkipRetentionCleanupForScheduledTime: + @pytest.mark.asyncio + async def test_returns_true_when_scheduler_not_initialized(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + sched.INSTANCE = None + + result = await workflows_retention.should_skip_retention_cleanup_for_scheduled_time( + sched, + datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc), + ) + + assert result is True + + @pytest.mark.asyncio + async def test_returns_false_when_cleanup_never_ran(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[]) + sched.INSTANCE = mock_instance + + result = await workflows_retention.should_skip_retention_cleanup_for_scheduled_time( + sched, + datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc), + ) + + assert result is False + mock_instance.list_workflows_async.assert_awaited_once_with( + name="dbos_cleanup", + status=[dbos.WorkflowStatusString.SUCCESS.value], + sort_desc=True, + limit=1, + load_input=False, + load_output=False, + ) + + @pytest.mark.asyncio + async def test_returns_true_when_latest_cleanup_is_newer_than_scheduled_time(self, temp_dbos_scheduler): + scheduled_time = datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc) + latest_cleanup = mock.Mock(spec=dbos.WorkflowStatus) + latest_cleanup.status = dbos.WorkflowStatusString.SUCCESS.value + latest_cleanup.updated_at = int(datetime.datetime(2025, 1, 2, tzinfo=datetime.timezone.utc).timestamp() * 1000) + latest_cleanup.created_at = 0 + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[latest_cleanup]) + sched.INSTANCE = mock_instance + + result = await workflows_retention.should_skip_retention_cleanup_for_scheduled_time( + sched, + scheduled_time, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_returns_false_when_latest_cleanup_is_older_than_scheduled_time(self, temp_dbos_scheduler): + scheduled_time = datetime.datetime(2025, 1, 2, tzinfo=datetime.timezone.utc) + latest_cleanup = mock.Mock(spec=dbos.WorkflowStatus) + latest_cleanup.status = dbos.WorkflowStatusString.SUCCESS.value + latest_cleanup.updated_at = int(datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc).timestamp() * 1000) + latest_cleanup.created_at = 0 + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[latest_cleanup]) + sched.INSTANCE = mock_instance + + result = await workflows_retention.should_skip_retention_cleanup_for_scheduled_time( + sched, + scheduled_time, + ) + + assert result is False + + @pytest.mark.asyncio + async def test_returns_false_when_latest_non_terminal_cleanup_is_newer_than_scheduled_time( + self, + temp_dbos_scheduler, + ): + scheduled_time = datetime.datetime(2026, 7, 15, 0, 0, 0, tzinfo=datetime.timezone.utc) + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[]) + sched.INSTANCE = mock_instance + + result = await workflows_retention.should_skip_retention_cleanup_for_scheduled_time( + sched, + scheduled_time, + ) + + assert result is False + + @pytest.mark.asyncio + async def test_returns_false_when_latest_terminal_cleanup_is_older_than_backfilled_slot( + self, + temp_dbos_scheduler, + ): + scheduled_time = datetime.datetime(2026, 7, 15, 0, 0, 0, tzinfo=datetime.timezone.utc) + latest_cleanup = mock.Mock(spec=dbos.WorkflowStatus) + latest_cleanup.status = dbos.WorkflowStatusString.SUCCESS.value + latest_cleanup.updated_at = int( + datetime.datetime(2026, 7, 7, 20, 16, 40, tzinfo=datetime.timezone.utc).timestamp() * 1000, + ) + latest_cleanup.created_at = 0 + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock(return_value=[latest_cleanup]) + sched.INSTANCE = mock_instance + + result = await workflows_retention.should_skip_retention_cleanup_for_scheduled_time( + sched, + scheduled_time, + ) + + assert result is False diff --git a/packages/node/tests/scheduler/test_workflows_util_automation_state.py b/packages/node/tests/scheduler/test_workflows_util_automation_state.py index 69e16f6c47..716471d521 100644 --- a/packages/node/tests/scheduler/test_workflows_util_automation_state.py +++ b/packages/node/tests/scheduler/test_workflows_util_automation_state.py @@ -78,6 +78,34 @@ def _workflow_status_with_automation_task( return workflow_status +class TestNormalizeParentAutomationId: + def test_parent_workflow_id_unchanged(self): + assert workflows_util.normalize_parent_automation_id(_PARENT_WORKFLOW_ID) == _PARENT_WORKFLOW_ID + + def test_child_workflow_id_truncated_to_parent(self): + child_id = _child_workflow_id(5) + assert workflows_util.normalize_parent_automation_id(child_id) == _PARENT_WORKFLOW_ID + + +class TestBuildNextChildAutomationWorkflowId: + def test_parent_workflow_id_maps_to_first_child(self): + assert ( + workflows_util.build_next_child_automation_workflow_id(_PARENT_WORKFLOW_ID) + == _child_workflow_id(1) + ) + + def test_child_workflow_id_increments_suffix(self): + assert ( + workflows_util.build_next_child_automation_workflow_id(_child_workflow_id(2)) + == _child_workflow_id(3) + ) + + def test_invalid_suffix_raises_value_error(self): + invalid_child_id = f"{_PARENT_WORKFLOW_ID}-4-4" + with pytest.raises(ValueError, match="Invalid child workflow suffix format"): + workflows_util.build_next_child_automation_workflow_id(invalid_child_id) + + class TestParseAutomationChildWorkflowIndex: def test_parent_workflow_id_maps_to_zero(self): assert workflows_util.parse_automation_child_workflow_index(_PARENT_WORKFLOW_ID) == 0 @@ -214,3 +242,45 @@ def test_success_workflow_state_dict_matches_output(self): assert state_dict is not None assert state_dict["automation"]["metadata"]["name"] == "from-output" + + +class TestPatchTaskContentDegradedState: + def test_persists_degraded_state_in_task_content(self): + task_content = _automation_task_content(automation_name="copy-grid") + + patched_content = workflows_util.patch_task_content_degraded_state( + task_content, + "not_enough_funds", + "Insufficient funds", + since=1234.5, + ) + + degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] + assert degraded_state == { + "since": 1234.5, + "error": "not_enough_funds", + "reason": "Insufficient funds", + } + + def test_preserves_existing_degraded_since_on_subsequent_patch(self): + task_content = _automation_task_content(automation_name="copy-grid") + task_content = workflows_util.patch_task_content_degraded_state( + task_content, + "not_enough_funds", + "Insufficient funds", + since=1000.0, + ) + + patched_content = workflows_util.patch_task_content_degraded_state( + task_content, + "invalid_order", + "Order volume below exchange minimum", + since=2000.0, + ) + + degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] + assert degraded_state == { + "since": 1000.0, + "error": "invalid_order", + "reason": "Order volume below exchange minimum", + } diff --git a/packages/node/tests/scheduler/test_workflows_version_migration.py b/packages/node/tests/scheduler/test_workflows_version_migration.py new file mode 100644 index 0000000000..565397febe --- /dev/null +++ b/packages/node/tests/scheduler/test_workflows_version_migration.py @@ -0,0 +1,223 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import tempfile +import time + +import dbos +import mock +import pytest + +import octobot_node.constants +import octobot_node.scheduler +import octobot_node.scheduler.workflows_version_migration as workflows_version_migration + +import tests.scheduler as scheduler_test_util + +_OLD_APPLICATION_VERSION = "3.0.0-beta0" +_TEST_QUEUE = dbos.Queue(name="version_migration_test_queue") +_WORKFLOW_SLEEP_SECONDS = 1.5 + + +class TestMigrateStrandedWorkflowVersions: + def test_returns_zero_when_workflow_status_table_missing(self): + with tempfile.TemporaryDirectory() as temp_directory: + sqlite_path = f"{temp_directory}/empty.db" + with mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_POSTGRES_URL", + None, + ), mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_SQLITE_FILE", + sqlite_path, + ): + updated_count = workflows_version_migration.migrate_stranded_workflow_versions( + target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + + assert updated_count == 0 + + def test_retags_enqueued_workflows_to_target_version(self): + with tempfile.NamedTemporaryFile() as temp_file: + scheduler_test_util.destroy_launched_dbos() + scheduler_test_util.init_scheduler( + temp_file.name, + application_version=_OLD_APPLICATION_VERSION, + ) + dbos.DBOS.reset_system_database() + dbos.DBOS.launch() + + @octobot_node.scheduler.SCHEDULER.INSTANCE.workflow() + async def stranded_workflow() -> str: + return "done" + + workflow_handle = _TEST_QUEUE.enqueue(stranded_workflow) + stranded_workflow_id = workflow_handle.get_workflow_id() + + octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() + octobot_node.scheduler.SCHEDULER.INSTANCE = None + + with mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_POSTGRES_URL", + None, + ), mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_SQLITE_FILE", + temp_file.name, + ): + updated_count = workflows_version_migration.migrate_stranded_workflow_versions( + target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + + assert updated_count == 1 + + scheduler_test_util.init_scheduler( + temp_file.name, + application_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + + migrated_status = dbos.DBOS.get_workflow_status(stranded_workflow_id) + assert migrated_status is not None + assert migrated_status.app_version == octobot_node.constants.SCHEDULER_APPLICATION_VERSION + assert migrated_status.status == dbos.WorkflowStatusString.ENQUEUED.value + + scheduler_test_util.destroy_launched_dbos() + + +class TestStrandedWorkflowRecoveryAfterVersionMigration: + @pytest.mark.asyncio + async def test_stranded_workflows_resume_after_application_version_migration(self): + with tempfile.NamedTemporaryFile() as temp_file: + try: + scheduler_test_util.destroy_launched_dbos() + scheduler_test_util.init_scheduler( + temp_file.name, + application_version=_OLD_APPLICATION_VERSION, + ) + + @octobot_node.scheduler.SCHEDULER.INSTANCE.workflow() + async def sleeper_workflow() -> str: + await dbos.DBOS.sleep_async(_WORKFLOW_SLEEP_SECONDS) + return "recovered" + + dbos.DBOS.reset_system_database() + dbos.DBOS.launch() + await _TEST_QUEUE.enqueue_async(sleeper_workflow) + + enqueued_workflows = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( + status=[dbos.WorkflowStatusString.ENQUEUED.value], + ) + assert len(enqueued_workflows) == 1 + stranded_workflow_id = enqueued_workflows[0].workflow_id + + octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() + octobot_node.scheduler.SCHEDULER.INSTANCE = None + + with mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_POSTGRES_URL", + None, + ), mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_SQLITE_FILE", + temp_file.name, + ): + workflows_version_migration.migrate_stranded_workflow_versions( + target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + + scheduler_test_util.init_scheduler( + temp_file.name, + application_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + + recovery_handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async( + stranded_workflow_id, + ) + workflow_result = await recovery_handle.get_result() + assert workflow_result == "recovered" + + final_status = await recovery_handle.get_status() + assert final_status.status == dbos.WorkflowStatusString.SUCCESS.value + assert final_status.app_version == octobot_node.constants.SCHEDULER_APPLICATION_VERSION + finally: + scheduler_test_util.destroy_launched_dbos() + + @pytest.mark.asyncio + async def test_pending_workflow_recovers_after_version_migration(self): + with tempfile.NamedTemporaryFile() as temp_file: + try: + scheduler_test_util.destroy_launched_dbos() + scheduler_test_util.init_scheduler( + temp_file.name, + application_version=_OLD_APPLICATION_VERSION, + ) + + @octobot_node.scheduler.SCHEDULER.INSTANCE.workflow() + async def sleeper_workflow() -> str: + await dbos.DBOS.sleep_async(_WORKFLOW_SLEEP_SECONDS) + return "pending-recovered" + + dbos.DBOS.reset_system_database() + dbos.DBOS.launch() + await _TEST_QUEUE.enqueue_async(sleeper_workflow) + + pending_workflows = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( + status=[ + dbos.WorkflowStatusString.ENQUEUED.value, + dbos.WorkflowStatusString.PENDING.value, + ], + ) + assert len(pending_workflows) == 1 + stranded_workflow_id = pending_workflows[0].workflow_id + + time.sleep(0.2) + octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() + octobot_node.scheduler.SCHEDULER.INSTANCE = None + + with mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_POSTGRES_URL", + None, + ), mock.patch.object( + workflows_version_migration.octobot_node.config.settings, + "SCHEDULER_SQLITE_FILE", + temp_file.name, + ): + workflows_version_migration.migrate_stranded_workflow_versions( + target_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + + scheduler_test_util.init_scheduler( + temp_file.name, + application_version=octobot_node.constants.SCHEDULER_APPLICATION_VERSION, + ) + octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + + recovery_handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async( + stranded_workflow_id, + ) + workflow_result = await recovery_handle.get_result() + assert workflow_result == "pending-recovered" + + final_status = await recovery_handle.get_status() + assert final_status.status == dbos.WorkflowStatusString.SUCCESS.value + assert final_status.app_version == octobot_node.constants.SCHEDULER_APPLICATION_VERSION + finally: + scheduler_test_util.destroy_launched_dbos() diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py index cc36e9e033..57d6ae3f4f 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py @@ -123,6 +123,43 @@ async def test_duplicate_account_error_propagates(self): expected_error_message=protocol_models.AccountActionResultErrorMessage.DUPLICATE_ITEM, ) + @pytest.mark.asyncio + async def test_duplicate_remote_identity_error_propagates(self): + account_model = account_executor_test_utils.minimal_exchange_account(account_id="dup-remote") + inner = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=account_model, + ) + user_action = protocol_models.UserAction( + id="ua-dup-remote", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + provider_mock = mock.Mock() + provider_mock.create_item.side_effect = collection_errors.DuplicateItemError( + "Exchange account identity already exists as account 'acc-1'" + ) + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + mock.patch.object( + account_state_updater_module, + "update_account_state", + new=mock.AsyncMock(return_value=account_model), + ), + ): + executor = create_account_executor.CreateAccountActionExecutor(account_executor_test_utils.WALLET_ADDRESS) + with pytest.raises(collection_errors.DuplicateItemError): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.DUPLICATE_ITEM, + ) + @pytest.mark.asyncio async def test_raises_for_unsupported_blockchain_account(self): account_model = account_executor_test_utils.minimal_blockchain_account(account_id="blockchain-acc") diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py index fdf5c6efc8..9d9a080250 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_create_automation.py @@ -59,7 +59,7 @@ def _automation_configuration( *, name: str, strategy_reference: protocol_models.StrategyReference, - account_id: str, + account_id: str | None = "acc-1", created_at: datetime.datetime | None = None, automation_id: str | None = None, ) -> protocol_models.AutomationConfiguration: @@ -71,7 +71,11 @@ def _automation_configuration( else datetime.datetime(2026, 5, 14, 11, 30, tzinfo=datetime.UTC) ), "strategy": strategy_reference, - "accounts": [protocol_models.AccountReference(id=account_id)], + "accounts": ( + [protocol_models.AccountReference(id=account_id)] + if account_id is not None + else [] + ), } if automation_id is not None: configuration_fields["id"] = automation_id @@ -525,8 +529,10 @@ def test_market_making_returns_init_and_run_octobot_process(self): expected_exchange_auth_segment = dsl_interpreter.format_parameter_value(expected_exchange_auth) expected_dsl = ( "run_octobot_process(" - f"{'ua-mm'!r}, {dsl_interpreter.format_parameter_value(expected_profile_dict)}, " - f"{expected_exchange_auth_segment}, " + f"{'ua-mm'!r}, " + f"profile_data={dsl_interpreter.format_parameter_value(expected_profile_dict)}, " + f"exchange_auth_data={expected_exchange_auth_segment}, " + f"octobot_name={dsl_interpreter.format_parameter_value('market-making-automation')}, " f"{', '.join(action_details_factory._run_octobot_process_recall_kwarg_segments())})" ) _assert_init_action_matches_minimal_account( @@ -857,7 +863,9 @@ def test_generic_process_returns_init_and_run_octobot_process(self): ) expected_dsl = ( "run_octobot_process(" - f"{'ua-generic-process'!r}, exchange_auth_data={dsl_interpreter.format_parameter_value(expected_exchange_auth)}, " + f"{'ua-generic-process'!r}, user_id={_TEST_WALLET_ADDRESS!r}, sync_profile_id={strat_ref.id!r}, " + f"exchange_auth_data={dsl_interpreter.format_parameter_value(expected_exchange_auth)}, " + f"octobot_name={dsl_interpreter.format_parameter_value('generic-process-automation')}, " f"{', '.join(action_details_factory._run_octobot_process_recall_kwarg_segments())})" ) _assert_init_action_matches_minimal_account( @@ -871,6 +879,70 @@ def test_generic_process_returns_init_and_run_octobot_process(self): assert main_action.dsl_script == expected_dsl assert "profile_data" not in main_action.dsl_script + def test_generic_process_with_empty_accounts_uses_metadata_init_without_exchange_auth(self): + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + ) + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="generic-process-no-account", + strategy_reference=strat_ref, + account_id=None, + ), + ) + user_action = _user_action_with_context( + action_id="ua-generic-process-no-account", + payload=create_payload, + ) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + stored = _stored_strategy_matching_reference(strat_ref, generic_process_configuration) + with mock.patch(_STRATEGY_PROVIDER_INSTANCE_PATCH) as strategy_mock: + strategy_mock.return_value.get_item.return_value = stored + actions = executor._create_automation_actions(user_action) + + assert len(actions) == 2 + init_action = actions[0] + assert init_action.id == "action_init" + init_config = init_action.config + metadata = init_config["automation"]["metadata"] + assert metadata["automation_id"] == "ua-generic-process-no-account" + assert metadata["strategy_id"] == strat_ref.id + assert "exchange_account_details" not in init_config + assert "exchange_account_elements" not in init_config["automation"] + + main_action = actions[1] + assert isinstance(main_action, flow_entities.DSLScriptActionDetails) + expected_dsl = ( + "run_octobot_process(" + f"{'ua-generic-process-no-account'!r}, user_id={_TEST_WALLET_ADDRESS!r}, " + f"sync_profile_id={strat_ref.id!r}, " + f"octobot_name={dsl_interpreter.format_parameter_value('generic-process-no-account')}, " + f"{', '.join(action_details_factory._run_octobot_process_recall_kwarg_segments())})" + ) + assert main_action.dsl_script == expected_dsl + assert "exchange_auth_data" not in main_action.dsl_script + + def test_non_generic_process_with_empty_accounts_raises(self): + dca_configuration = trading_tentacles_test_utils.functional_dca_trading_configuration() + strat_ref = _default_strategy_reference() + create_payload = protocol_models.CreateAutomationConfiguration( + action_type=protocol_models.UserActionType.AUTOMATION_CREATE, + configuration=_automation_configuration( + name="dca-no-account", + strategy_reference=strat_ref, + account_id=None, + ), + ) + user_action = _user_action_with_context(action_id="ua-dca-no-account", payload=create_payload) + executor = create_automation_executor.CreateAutomationActionExecutor(_TEST_WALLET_ADDRESS) + stored = _stored_strategy_matching_reference(strat_ref, dca_configuration) + with mock.patch(_STRATEGY_PROVIDER_INSTANCE_PATCH) as strategy_mock: + strategy_mock.return_value.get_item.return_value = stored + with pytest.raises(node_errors.InvalidAutomationConfigurationError): + executor._create_automation_actions(user_action) + def test_generic_process_run_octobot_process_uses_configuration_id_as_user_folder(self): configuration_automation_id = _DEFAULT_AUTOMATION_CONFIGURATION_ID generic_process_configuration = protocol_models.GenericProcessConfiguration( diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py new file mode 100644 index 0000000000..6dc2a13ed6 --- /dev/null +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_restart_automation.py @@ -0,0 +1,325 @@ +import json +import datetime +import mock +import pytest +import dbos + +import octobot_flow.entities as flow_entities +import octobot_flow.enums as flow_enums +import octobot_protocol.models as protocol_models + +import octobot_node.errors as node_errors +import octobot_node.models as models_module +import octobot_node.scheduler as scheduler_module +import octobot_node.scheduler.workflows.params as workflow_params_module +import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as restart_automation_executor + +from .. import provider_assertions + + +_TEST_WALLET_ADDRESS = "0xaaabbbcccddd" +_PARENT_AUTOMATION_ID = "00000000-0000-4000-8000-000000000001" + + +def _wrap(configuration_payload) -> protocol_models.UserActionConfiguration: + return protocol_models.UserActionConfiguration.from_json(configuration_payload.to_json()) + + +def _user_action_restart(*, user_action_id: str, automation_parent_id: str) -> protocol_models.UserAction: + restart_payload = protocol_models.RestartAutomationConfiguration( + id=automation_parent_id, + action_type=protocol_models.UserActionType.AUTOMATION_RESTART, + ) + return protocol_models.UserAction(id=user_action_id, configuration=_wrap(restart_payload)) + + +def _stopped_automation_state_dict(*, stop_automation: bool = True) -> dict: + return { + "automation": { + "metadata": {"automation_id": _PARENT_AUTOMATION_ID}, + "actions_dag": { + "actions": [ + { + "id": "action_init", + "action": flow_enums.ActionType.APPLY_CONFIGURATION.value, + "executed_at": 1.0, + }, + { + "id": "action_run", + "dsl_script": "run_octobot_process('auto')", + "dependencies": [{"action_id": "action_init"}], + "executed_at": 2.0, + "result": {"pid": 42}, + }, + ] + }, + "post_actions": {"stop_automation": stop_automation}, + "execution": { + "previous_execution": {"triggered_at": 1.0}, + "current_execution": {"triggered_at": 2.0}, + }, + }, + "exchange_account_details": { + "exchange_details": {"internal_name": "binanceus"}, + }, + } + + +def _terminal_workflow_with_output( + *, + parent_id: str = _PARENT_AUTOMATION_ID, + stop_automation: bool = True, +) -> mock.Mock: + state_dict = _stopped_automation_state_dict(stop_automation=stop_automation) + task_content = json.dumps({"state": state_dict}) + task = models_module.Task( + name="restart-test-automation", + content=task_content, + type=models_module.TaskType.EXECUTE_ACTIONS.value, + ) + encoded_inputs = workflow_params_module.AutomationWorkflowInputs(task=task).to_dict( + include_default_values=False + ) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = parent_id + workflow_status.updated_at = 100 + workflow_status.input = {"args": [encoded_inputs], "kwargs": {}} + workflow_status.output = json.dumps( + workflow_params_module.AutomationWorkflowOutput(state=task_content).to_dict( + include_default_values=False + ) + ) + return workflow_status + + +class TestPrepareAutomationStateForRestart: + def test_clears_stop_automation_and_resets_main_action(self): + automation_state = flow_entities.AutomationState.from_dict( + _stopped_automation_state_dict(stop_automation=True) + ) + prepared_state = restart_automation_executor.prepare_automation_state_for_restart(automation_state) + assert prepared_state.automation.post_actions.stop_automation is False + run_action = prepared_state.automation.actions_dag.get_actions_by_id()["action_run"] + assert run_action.executed_at is None + assert run_action.previous_execution_result == {"pid": 42} + init_action = prepared_state.automation.actions_dag.get_actions_by_id()["action_init"] + assert init_action.executed_at == 1.0 + + +class TestRestartAutomationActionExecutor: + @pytest.mark.asyncio + async def test_execute_enqueues_task_from_latest_output(self): + user_action = _user_action_restart( + user_action_id="ua-restart-1", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + terminal_workflow = _terminal_workflow_with_output() + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_latest_terminal_automation_workflow_for_parent_id", + new_callable=mock.AsyncMock, + return_value=terminal_workflow, + ), + ): + await executor.execute(user_action) + + scheduled_task = executor.post_actions.to_create_automation_task + assert scheduled_task is not None + assert scheduled_task.id == f"{_PARENT_AUTOMATION_ID}_1" + assert scheduled_task.name == "restart-test-automation" + assert scheduled_task.user_id == _TEST_WALLET_ADDRESS + task_payload = json.loads(scheduled_task.content) + assert task_payload["state"]["automation"]["post_actions"]["stop_automation"] is False + run_action = task_payload["state"]["automation"]["actions_dag"]["actions"][1] + assert run_action["executed_at"] is None + assert run_action["previous_execution_result"] == {"pid": 42} + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.COMPLETED, + result_channel="automation", + expect_error_details=False, + ) + inner = user_action.result.actual_instance + assert inner.created_automation_id == _PARENT_AUTOMATION_ID + + @pytest.mark.asyncio + async def test_raises_unrestartable_when_id_binds_to_user_action(self): + user_action = _user_action_restart( + user_action_id="ua-restart-2", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + bound_user_action = protocol_models.UserAction(id=_PARENT_AUTOMATION_ID) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[bound_user_action], + ), + pytest.raises(node_errors.UnrestartableAutomationError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + @pytest.mark.asyncio + async def test_raises_unrestartable_when_automation_still_running(self): + user_action = _user_action_restart( + user_action_id="ua-restart-3", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[f"{_PARENT_AUTOMATION_ID}_1"], + ), + pytest.raises(node_errors.UnrestartableAutomationError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + @pytest.mark.asyncio + async def test_raises_unrestartable_when_no_prior_execution(self): + user_action = _user_action_restart( + user_action_id="ua-restart-4", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "list_user_actions", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[], + ), + mock.patch.object( + scheduler_module.SCHEDULER, + "resolve_latest_terminal_automation_workflow_for_parent_id", + new_callable=mock.AsyncMock, + return_value=None, + ), + pytest.raises(node_errors.UnrestartableAutomationError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + @pytest.mark.asyncio + async def test_raises_when_scheduler_not_initialized(self): + user_action = _user_action_restart( + user_action_id="ua-restart-5", + automation_parent_id=_PARENT_AUTOMATION_ID, + ) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=False, + ): + with pytest.raises(RuntimeError, match="Scheduler is not initialized"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INTERNAL_ERROR, + ) + + @pytest.mark.asyncio + async def test_invalid_payload_raises_invalid_user_action_payload(self): + wrong = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=protocol_models.Account( + id="a", + name="n", + is_simulated=True, + created_at=datetime.datetime(2026, 6, 1, 12, 0, 0, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 6, 1, 13, 0, 0, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="r", + exchange_config_ids=["test-exchange-config-id"], + ) + ), + ), + ) + user_action = protocol_models.UserAction(id="ua-bad", configuration=_wrap(wrong)) + executor = restart_automation_executor.RestartAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation.scheduler_module.is_initialized", + return_value=True, + ), + pytest.raises(node_errors.InvalidUserActionPayloadError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py index e59c371fd0..1e09d8502d 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/automation/test_stop_automation.py @@ -1,172 +1,181 @@ -import mock -import pytest -import dbos -import datetime - -import octobot_protocol.models as protocol_models - -import octobot_node.errors as node_errors -import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as stop_automation_executor -import octobot_node.scheduler as scheduler_module - -from .. import provider_assertions - - -_TEST_WALLET_ADDRESS = "0xaaabbbcccddd" - - -def _wrap(configuration_payload) -> protocol_models.UserActionConfiguration: - return protocol_models.UserActionConfiguration.from_json(configuration_payload.to_json()) - - -def _user_action_stop(*, user_action_id: str, automation_parent_id: str) -> protocol_models.UserAction: - stop_payload = protocol_models.StopAutomationConfiguration( - id=automation_parent_id, - action_type=protocol_models.UserActionType.AUTOMATION_STOP, - ) - return protocol_models.UserAction(id=user_action_id, configuration=_wrap(stop_payload)) - - -class TestResolveActiveAutomationWorkflowIdsForParentId: - @pytest.mark.asyncio - async def test_returns_latest_pending_child_workflow_id(self): - parent_id = "741ce171-dac9-40be-83dc-b443c0eaf0e2" - older_child = mock.Mock(spec=dbos.WorkflowStatus) - older_child.workflow_id = f"{parent_id}_1" - older_child.updated_at = 10 - older_child.status = dbos.WorkflowStatusString.PENDING.value - latest_child = mock.Mock(spec=dbos.WorkflowStatus) - latest_child.workflow_id = f"{parent_id}_2" - latest_child.updated_at = 20 - latest_child.status = dbos.WorkflowStatusString.ENQUEUED.value - with mock.patch.object( - scheduler_module.SCHEDULER, - "_get_parent_and_children_automation_workflows", - new_callable=mock.AsyncMock, - return_value=[older_child, latest_child], - ) as inner_mock: - result = await scheduler_module.SCHEDULER.resolve_active_automation_workflow_ids_for_parent_id( - _TEST_WALLET_ADDRESS, - parent_id, - ) - assert result == [f"{parent_id}_2"] - inner_mock.assert_awaited_once_with( - _TEST_WALLET_ADDRESS, - [parent_id], - [ - dbos.WorkflowStatusString.ENQUEUED, - dbos.WorkflowStatusString.PENDING, - ], - load_output=False, - ) - - -class TestStopAutomationActionExecutor: - @pytest.mark.asyncio - async def test_execute_sends_stop_actions_to_active_automation(self): - user_action = _user_action_stop(user_action_id="ua-stop-1", automation_parent_id="00000000-0000-4000-8000-000000000001") - executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) - with ( - mock.patch("octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", return_value=True), - mock.patch( - "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_tasks.send_actions_to_active_automation", - new_callable=mock.AsyncMock, - ) as send_actions_mock, - ): - await executor.execute(user_action) - - send_actions_mock.assert_awaited_once_with( - "00000000-0000-4000-8000-000000000001", - _TEST_WALLET_ADDRESS, - [ - { - "id": "action_stop_priority_ua-stop-1", - "dsl_script": "stop_automation()", - } - ], - ) - provider_assertions.assert_user_action_terminal_state( - user_action=user_action, - expected_status=protocol_models.UserActionStatus.COMPLETED, - result_channel="automation", - expect_error_details=False - ) - - @pytest.mark.asyncio - async def test_execute_raises_active_automation_workflow_not_found_when_wrapper_returns_empty(self): - user_action = _user_action_stop(user_action_id="ua-stop-2", automation_parent_id="no-match") - executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) - with ( - mock.patch("octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", return_value=True), - mock.patch( - "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_tasks.send_actions_to_active_automation", - new_callable=mock.AsyncMock, - side_effect=node_errors.ActiveAutomationWorkflowNotFoundError("no-match"), - ), - ): - with pytest.raises(node_errors.ActiveAutomationWorkflowNotFoundError): - await executor.execute(user_action) - provider_assertions.assert_user_action_terminal_state( - user_action=user_action, - expected_status=protocol_models.UserActionStatus.FAILED, - result_channel="automation", - expect_error_details=True, - expected_error_message=protocol_models.AutomationActionResultErrorMessage.AUTOMATION_NOT_FOUND - ) - - @pytest.mark.asyncio - async def test_execute_raises_when_scheduler_not_initialized(self): - user_action = _user_action_stop(user_action_id="ua-stop-4", automation_parent_id="x") - executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) - with mock.patch( - "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", - return_value=False, - ): - with pytest.raises(RuntimeError, match="Scheduler is not initialized"): - await executor.execute(user_action) - provider_assertions.assert_user_action_terminal_state( - user_action=user_action, - expected_status=protocol_models.UserActionStatus.FAILED, - result_channel="automation", - expect_error_details=True, - expected_error_message=protocol_models.AutomationActionResultErrorMessage.INTERNAL_ERROR - ) - - @pytest.mark.asyncio - async def test_invalid_payload_raises_invalid_user_action_payload(self): - wrong = protocol_models.CreateAccountConfiguration( - action_type=protocol_models.UserActionType.ACCOUNT_CREATE, - configuration=protocol_models.Account( - id="a", - name="n", - is_simulated=True, - created_at=datetime.datetime(2026, 6, 1, 12, 0, 0, tzinfo=datetime.UTC), - updated_at=datetime.datetime(2026, 6, 1, 13, 0, 0, tzinfo=datetime.UTC), - specifics=protocol_models.AccountSpecifics( - actual_instance=protocol_models.ExchangeAccount( - account_type=protocol_models.AccountType.EXCHANGE, - remote_account_id="r", - exchange_config_ids=["test-exchange-config-id"], - ) - ), - ), - ) - user_action = protocol_models.UserAction(id="ua-bad", configuration=_wrap(wrong)) - executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) - - with ( - mock.patch( - "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", - return_value=True, - ), - pytest.raises(node_errors.InvalidUserActionPayloadError), - ): - await executor.execute(user_action) - provider_assertions.assert_user_action_terminal_state( - user_action=user_action, - expected_status=protocol_models.UserActionStatus.FAILED, - result_channel="automation", - expect_error_details=True, - expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION - ) - \ No newline at end of file +import datetime + +import dbos +import mock +import pytest + +import octobot_protocol.models as protocol_models + +import octobot_node.errors as node_errors +import octobot_node.scheduler as scheduler_module +import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as stop_automation_executor + +from .. import provider_assertions + + +_TEST_WALLET_ADDRESS = "0xaaabbbcccddd" + + +def _wrap(configuration_payload) -> protocol_models.UserActionConfiguration: + return protocol_models.UserActionConfiguration.from_json(configuration_payload.to_json()) + + +def _user_action_stop(*, user_action_id: str, automation_parent_id: str) -> protocol_models.UserAction: + stop_payload = protocol_models.StopAutomationConfiguration( + id=automation_parent_id, + action_type=protocol_models.UserActionType.AUTOMATION_STOP, + ) + return protocol_models.UserAction(id=user_action_id, configuration=_wrap(stop_payload)) + + +class TestResolveActiveAutomationWorkflowIdsForParentId: + @pytest.mark.asyncio + async def test_returns_latest_pending_child_workflow_id(self): + parent_id = "741ce171-dac9-40be-83dc-b443c0eaf0e2" + older_child = mock.Mock(spec=dbos.WorkflowStatus) + older_child.workflow_id = f"{parent_id}_1" + older_child.updated_at = 10 + older_child.status = dbos.WorkflowStatusString.PENDING.value + latest_child = mock.Mock(spec=dbos.WorkflowStatus) + latest_child.workflow_id = f"{parent_id}_2" + latest_child.updated_at = 20 + latest_child.status = dbos.WorkflowStatusString.ENQUEUED.value + with mock.patch.object( + scheduler_module.SCHEDULER, + "_get_parent_and_children_automation_workflows", + new_callable=mock.AsyncMock, + return_value=[older_child, latest_child], + ) as inner_mock: + result = await scheduler_module.SCHEDULER.resolve_active_automation_workflow_ids_for_parent_id( + _TEST_WALLET_ADDRESS, + parent_id, + ) + assert result == [f"{parent_id}_2"] + inner_mock.assert_awaited_once_with( + _TEST_WALLET_ADDRESS, + [parent_id], + [ + dbos.WorkflowStatusString.ENQUEUED, + dbos.WorkflowStatusString.PENDING, + ], + load_output=False, + ) + + +class TestStopAutomationActionExecutor: + @pytest.mark.asyncio + async def test_execute_sends_stop_actions_to_active_automation(self): + user_action = _user_action_stop( + user_action_id="ua-stop-1", + automation_parent_id="00000000-0000-4000-8000-000000000001", + ) + executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_tasks.send_actions_to_active_automation", + new_callable=mock.AsyncMock, + ) as send_actions_mock, + ): + await executor.execute(user_action) + + send_actions_mock.assert_awaited_once_with( + "00000000-0000-4000-8000-000000000001", + _TEST_WALLET_ADDRESS, + [ + { + "id": "action_stop_priority_ua-stop-1", + "dsl_script": "stop_automation()", + } + ], + ) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.COMPLETED, + result_channel="automation", + expect_error_details=False, + ) + + @pytest.mark.asyncio + async def test_execute_raises_active_automation_workflow_not_found_when_wrapper_returns_empty(self): + user_action = _user_action_stop(user_action_id="ua-stop-2", automation_parent_id="no-match") + executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_tasks.send_actions_to_active_automation", + new_callable=mock.AsyncMock, + side_effect=node_errors.ActiveAutomationWorkflowNotFoundError("no-match"), + ), + ): + with pytest.raises(node_errors.ActiveAutomationWorkflowNotFoundError): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.AUTOMATION_NOT_FOUND, + ) + + @pytest.mark.asyncio + async def test_execute_raises_when_scheduler_not_initialized(self): + user_action = _user_action_stop(user_action_id="ua-stop-4", automation_parent_id="x") + executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) + with mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", + return_value=False, + ): + with pytest.raises(RuntimeError, match="Scheduler is not initialized"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INTERNAL_ERROR, + ) + + @pytest.mark.asyncio + async def test_invalid_payload_raises_invalid_user_action_payload(self): + wrong = protocol_models.CreateAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_CREATE, + configuration=protocol_models.Account( + id="a", + name="n", + is_simulated=True, + created_at=datetime.datetime(2026, 6, 1, 12, 0, 0, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 6, 1, 13, 0, 0, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="r", + exchange_config_ids=["test-exchange-config-id"], + ) + ), + ), + ) + user_action = protocol_models.UserAction(id="ua-bad", configuration=_wrap(wrong)) + executor = stop_automation_executor.StopAutomationActionExecutor(_TEST_WALLET_ADDRESS) + + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation.scheduler_module.is_initialized", + return_value=True, + ), + pytest.raises(node_errors.InvalidUserActionPayloadError), + ): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="automation", + expect_error_details=True, + expected_error_message=protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION, + ) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py index 6469b09126..f0c94980a6 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/strategy/strategy_executor_test_utils.py @@ -35,7 +35,7 @@ def minimal_strategy( ) -> protocol_models.Strategy: configuration = protocol_models.GenericProcessConfiguration( configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, - profile_data={}, + profile_data={"profile_details": {"id": strategy_id}}, ) return protocol_models.Strategy( id=strategy_id, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py index 73e4e25313..ef04ea7881 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_channel_user_action_executor_get_error_message.py @@ -23,6 +23,7 @@ import octobot_node.scheduler.user_actions.user_actions_executor.exchange_config.create_exchange_config as create_exchange_config_executor import octobot_node.scheduler.user_actions.user_actions_executor.strategy.create_strategy as create_strategy_executor import octobot_node.scheduler.user_actions.user_actions_executor.account_auth.create_account_auth as create_account_auth_executor +import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as restart_automation_executor import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as stop_automation_executor _WALLET = "0xwallet" @@ -39,6 +40,11 @@ def test_invalid_user_action_payload(self): resolved = executor._get_error_message(node_errors.InvalidUserActionPayloadError("bad")) assert resolved == protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION + def test_unrestartable_automation_error(self): + executor = restart_automation_executor.RestartAutomationActionExecutor(_WALLET) + resolved = executor._get_error_message(node_errors.UnrestartableAutomationError("cannot restart")) + assert resolved == protocol_models.AutomationActionResultErrorMessage.INVALID_CONFIGURATION + def test_ambiguous_active_automation_workflow(self): executor = stop_automation_executor.StopAutomationActionExecutor(_WALLET) resolved = executor._get_error_message(node_errors.AmbiguousActiveAutomationWorkflowError("ambiguous")) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py index 667da15e77..8b67437d9c 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py @@ -319,6 +319,54 @@ async def fake_exchange_manager_from_exchange_data(*args, **kwargs): profile_data = captured_exchange_data_args["profile_data"] assert profile_data.exchanges[0].exchange_type == expected_exchange_type + @pytest.mark.asyncio + async def test_maps_authentication_error_on_exchange_manager_init_to_invalid_api_keys(self): + exchange_account = account_executor_test_utils.exchange_account_payload() + account = _sample_account() + + @contextlib.asynccontextmanager + async def failing_exchange_manager_from_exchange_data(*args, **kwargs): + raise trading_errors.AuthenticationError("Incorrect apiKey") + yield # pragma: no cover + + with ( + mock.patch.object( + account_state_updater_module.exchange_account_resolver, + "get_exchange_config", + return_value=account_executor_test_utils.exchange_config_payload(), + ), + mock.patch.object( + account_state_updater_module.account_authentication_resolver, + "get_exchange_authentication", + return_value=_authentication(), + ), + mock.patch.object( + account_state_updater_module.fields_utils, + "encrypt", + side_effect=lambda plain_text: ("enc:" + plain_text).encode(), + ), + mock.patch.object( + account_state_updater_module.trading_exchanges, + "exchange_manager_from_exchange_data", + failing_exchange_manager_from_exchange_data, + ), + mock.patch.object( + account_state_updater_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + ): + account_state, assets = await account_state_updater_module._check_exchange_account_state( + exchange_account, + account, + _WALLET_ADDRESS, + ) + + assert account_state.status == protocol_models.AccountStatus.INVALID + assert account_state.message == protocol_models.AccountStatusMessage.INVALID_API_KEYS + assert account_state.permissions == [] + assert assets is None + class TestAccountStateUpdaterCheckExchangeManagerState: @pytest.mark.asyncio diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py index 88d65ae8f3..6519f5cdea 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_action_details_factory.py @@ -290,3 +290,61 @@ def test_passes_different_reference_market_to_dsl(self): 'copy_exchange_account(strategy_id="copied-strategy", ' 'reference_market="USDC", reference_account=\'\', account_copy_settings=\'{}\')' ) + + +class TestGenericProcessMetadataInitActionFactory: + def test_builds_metadata_only_apply_configuration(self): + strategy_reference = protocol_models.StrategyReference( + id="strategy-1", + version="1.0.0", + emit_signals=True, + ) + init_action = action_details_factory_module.generic_process_metadata_init_action_factory( + automation_id="automation-1", + strategy_reference=strategy_reference, + ) + assert init_action.id == "action_init" + assert init_action.action == flow_enums.ActionType.APPLY_CONFIGURATION.value + metadata = init_action.config["automation"]["metadata"] + assert metadata["automation_id"] == "automation-1" + assert metadata["strategy_id"] == "strategy-1" + assert metadata["strategy_version"] == "1.0.0" + assert metadata["emit_signals"] is True + assert "exchange_account_details" not in init_action.config + + +class TestGenericProcessActionFactoryWithoutAccount: + def test_omits_exchange_auth_data_when_protocol_account_is_none(self): + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + ) + process_action = action_details_factory_module.generic_process_action_factory( + _init_action(), + generic_process_configuration, + None, + _WALLET_ADDRESS, + automation_id="automation-1", + strategy_id="strategy-1", + ) + assert "exchange_auth_data" not in process_action.dsl_script + assert "user_id=" in process_action.dsl_script + assert "sync_profile_id=" in process_action.dsl_script + + +class TestGenericProcessActionFactoryWithEmbeddedProfileData: + def test_uses_profile_data_keyword_when_strategy_id_is_omitted(self): + embedded_profile_data = {"profile_details": {"id": "embedded-profile"}} + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + profile_data=embedded_profile_data, + ) + process_action = action_details_factory_module.generic_process_action_factory( + _init_action(), + generic_process_configuration, + None, + _WALLET_ADDRESS, + automation_id="automation-1", + ) + assert "profile_data=" in process_action.dsl_script + assert "sync_profile_id=" not in process_action.dsl_script + assert process_action.dsl_script.startswith("run_octobot_process('automation-1', user_id=") diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_trading_tentacles_config.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_trading_tentacles_config.py index bfc3104bfa..4900937d32 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_trading_tentacles_config.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_trading_tentacles_config.py @@ -1,5 +1,6 @@ import pytest +import octobot_commons.enums as commons_enums import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.util.action_details_factory as action_details_factory_module import octobot_node.scheduler.user_actions.user_actions_executor.util.trading_tentacles_config as trading_tentacles_config_module @@ -12,6 +13,205 @@ from . import trading_tentacles_test_utils +class _StubStructuredPayload: + pass + + +class TestIsValueCastableToParameterType: + @pytest.mark.parametrize( + ("value", "expected"), + [ + (42, True), + (3.5, True), + ("42", True), + ("3.5", True), + (True, False), + (False, False), + ("not-a-number", False), + (None, False), + ([], False), + ({}, False), + ], + ) + def test_number(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.NUMBER.value, + ) + is expected + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [ + (True, True), + (False, True), + (0, False), + (1, False), + ("true", False), + (None, False), + ], + ) + def test_boolean(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.BOOLEAN.value, + ) + is expected + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [ + ("hello", True), + ("", True), + (1, False), + (True, False), + (None, False), + ([], False), + ], + ) + def test_text(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.TEXT.value, + ) + is expected + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [ + ("1h", True), + (1, False), + (None, False), + ([], False), + ], + ) + def test_time_frame(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.TIME_FRAME.value, + ) + is expected + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [ + ({}, True), + ({"key": "value"}, True), + ([], False), + ("dict", False), + (None, False), + ], + ) + def test_dict(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.DICT.value, + ) + is expected + ) + + @pytest.mark.parametrize( + "value", + [None, 1, "text", [], {}, True], + ) + def test_any_always_accepts(self, value): + assert trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.ANY.value, + ) is True + + @pytest.mark.parametrize( + ("value", "expected"), + [ + ([1, 2], True), + ((1, 2), True), + ([], True), + (range(3), True), + ("series", False), + (b"bytes", False), + (None, False), + (1, False), + ({}, False), + (True, False), + ], + ) + def test_series(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.SERIES.value, + ) + is expected + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [ + (None, True), + ({}, True), + ({"eval_note": 1}, True), + (_StubStructuredPayload(), True), + (True, False), + (1, False), + (1.5, False), + ("signal", False), + ([], False), + ((), False), + ], + ) + def test_signal(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.SIGNAL.value, + ) + is expected + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [ + (None, True), + ({}, True), + ({"order_id": "1"}, True), + (_StubStructuredPayload(), True), + (True, False), + (1, False), + (1.5, False), + ("order", False), + ([], False), + ((), False), + ], + ) + def test_order(self, value, expected: bool): + assert ( + trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + commons_enums.DslValueType.ORDER.value, + ) + is expected + ) + + @pytest.mark.parametrize( + "value", + [None, 1, "text", [], {}, True, _StubStructuredPayload()], + ) + def test_unknown_type_rejects(self, value): + assert trading_tentacles_config_module._is_value_castable_to_parameter_type( + value, + "not_a_type", + ) is False + + class TestNormalizeTentacleName: def test_camel_case_trading_mode(self): assert ( @@ -241,7 +441,7 @@ def test_invalid_config_value_type_reports_expected_type(self): ) error_message = str(error_info.value) assert dca_trading.DCATradingModeProducer.MAX_ASSET_HOLDING_PERCENT in error_message - assert "expected type: float" in error_message + assert "expected type: number" in error_message def test_collects_multiple_configuration_issues_in_one_error(self): trading_configuration = trading_tentacles_test_utils.trading_tentacles_configuration( @@ -396,7 +596,7 @@ def test_invalid_evaluator_config_value_type_reports_expected_type(self): ) error_message = str(error_info.value) assert ".evaluators[0].config.period_length" in error_message - assert "expected type: int" in error_message + assert "expected type: number" in error_message class TestValidateStrategyEvaluator: diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index c1c0f67202..f16b61146b 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -16,6 +16,7 @@ import asyncio import contextlib +import importlib import json import os import functools @@ -65,8 +66,15 @@ def import_automation_workflow(): if not AUTOMATION_WORKFLOW_IMPORTED: with tempfile.NamedTemporaryFile() as temp_file: init_and_destroy_scheduler(temp_file.name) - import octobot_node.scheduler.workflows.automation_workflow - AUTOMATION_WORKFLOW_IMPORTED = True + importlib.import_module("octobot_node.scheduler.workflows.automation_workflow") + AUTOMATION_WORKFLOW_IMPORTED = True + # init_and_destroy_scheduler() tears down DBOS after registering workflows, leaving + # INSTANCE/queues as None. Unit tests patch recv_async/enqueue_async on these objects; + # provide mocks so patch.object has a real target (re-apply after other fixtures tear down). + if octobot_node.scheduler.SCHEDULER.INSTANCE is None: + octobot_node.scheduler.SCHEDULER.INSTANCE = mock.Mock() + if octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE is None: + octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE = mock.Mock() def _automation_state_dict(actions: list[dict[str, typing.Any]]) -> dict[str, typing.Any]: @@ -79,6 +87,36 @@ def _automation_state_dict(actions: list[dict[str, typing.Any]]) -> dict[str, ty } +def _automation_state_dict_with_scheduled_to( + scheduled_to: float, + actions: list[dict[str, typing.Any]] | None = None, +) -> dict[str, typing.Any]: + state = _automation_state_dict(actions or []) + state["automation"]["execution"] = { + "current_execution": {"scheduled_to": scheduled_to}, + } + return state + + +def _octobot_actions_job_mock_class_pending_priority_skipped( + *, + automation_inner_state: dict[str, typing.Any], + skip_error: "octobot_flow.errors.PendingPriorityActionsSkippedError", +) -> mock.Mock: + async def run_raises(*args, **kwargs): + raise skip_error + + def mock_job_factory(*args, **kwargs): + job_mock = mock.Mock() + job_mock.description = octobot_flow_client.OctoBotActionsJobDescription( + state=automation_inner_state, + ) + job_mock.run = mock.AsyncMock(side_effect=run_raises) + return job_mock + + return mock.Mock(side_effect=mock_job_factory) + + def _parse_automation_workflow_output( workflow_output: str, ) -> params.AutomationWorkflowOutput: @@ -637,17 +675,92 @@ async def test_execute_iteration_logs_and_reraises_when_octobot_actions_job_fail @pytest.mark.asyncio @required_imports - async def test_execute_iteration_authentication_error_sets_postponed_iteration( + async def test_execute_iteration_logs_and_reraises_on_retriable_failed_request( self, import_automation_workflow, task ): - task_content = json.dumps({"params": {"ACTIONS": "trade", "EXCHANGE_FROM": "binance", + task.content = json.dumps({"params": {"ACTIONS": "trade", "EXCHANGE_FROM": "binance", "ORDER_SYMBOL": "ETH/BTC", "ORDER_AMOUNT": 1, "ORDER_TYPE": "market", "ORDER_SIDE": "BUY", "SIMULATED_PORTFOLIO": {"BTC": 1}}}) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + run_error = octobot_trading_errors.RetriableFailedRequest("transient exchange failure") + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_side_effect=run_error, + ) + mock_logger = mock.Mock() + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + automation_workflow, + "get_logger", + return_value=mock_logger, + ): + with pytest.raises(octobot_trading_errors.RetriableFailedRequest, match="transient exchange failure"): + await automation_workflow.execute_iteration(inputs, None) + + mock_logger.exception.assert_called_once_with( + run_error, + True, + f"Retriable error while running automation job: {run_error}", + ) + + @pytest.mark.asyncio + @required_imports + @pytest.mark.parametrize( + "run_side_effect,expected_error_status,expected_retry_delay_seconds", + [ + pytest.param( + octobot_trading_errors.AuthenticationError("Invalid API credentials"), + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value, + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, + id="authentication_error", + ), + pytest.param( + octobot_trading_errors.PortfolioNegativeValueError( + "Trying to update BTC with -0.00074 but quantity was 0.00068" + ), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="portfolio_negative_value_error", + ), + pytest.param( + octobot_trading_errors.FailedRequest("Exchange API request failed"), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="failed_request", + ), + pytest.param( + octobot_trading_errors.MissingFunds("Insufficient funds for order"), + octobot_flow.enums.ActionErrorStatus.NOT_ENOUGH_FUNDS.value, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="missing_funds", + ), + pytest.param( + octobot_trading_errors.MissingMinimalExchangeTradeVolume( + "Order volume below exchange minimum" + ), + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER.value, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="missing_minimal_exchange_trade_volume", + ), + ], + ) + async def test_execute_iteration_postponed_error_sets_postponed_iteration( + self, + import_automation_workflow, + task, + run_side_effect, + expected_error_status, + expected_retry_delay_seconds, + ): + task_content = json.dumps({"state": _automation_state_dict([])}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) - authentication_error_message = "Invalid API credentials" mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( - run_side_effect=octobot_trading_errors.AuthenticationError(authentication_error_message), + run_side_effect=run_side_effect, ) fixed_now = 1000.0 @@ -668,14 +781,18 @@ async def test_execute_iteration_authentication_error_sets_postponed_iteration( update_account_trading_mock.assert_not_called() parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) - assert parsed_progress_status.error == octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value - assert parsed_progress_status.error_message == authentication_error_message + assert parsed_progress_status.error == expected_error_status + assert parsed_progress_status.error_message == str(run_side_effect) assert parsed_progress_status.postponed_iteration is True assert parsed_progress_status.next_step_at == ( - fixed_now + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS + fixed_now + expected_retry_delay_seconds ) assert result["has_next_actions"] is True - assert result["next_iteration_description"] == task_content + next_iteration_description = json.loads(result["next_iteration_description"]) + degraded_state = next_iteration_description["state"]["automation"]["execution"]["degraded_state"] + assert degraded_state["error"] == expected_error_status + assert degraded_state["reason"] == str(run_side_effect) + assert degraded_state["since"] == fixed_now @pytest.mark.asyncio @required_imports @@ -852,18 +969,212 @@ async def test_execute_iteration_continues_when_trading_persistence_wallet_missi assert parsed_progress_status.error is None +class TestExecuteIterationPendingPriorityActionsSkippedError: + @pytest.mark.asyncio + @required_imports + async def test_postpones_iteration_at_scheduled_to_without_degraded_state( + self, import_automation_workflow, task + ): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + skip_error = octobot_flow.errors.PendingPriorityActionsSkippedError( + "Supplied priority actions were not pending and were skipped: ['action_stop_priority_ua-stop-1']" + ) + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=skip_error, + ) + fixed_now = 1000.0 + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, + "update_account_trading", + ) as update_account_trading_mock: + result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + update_account_trading_mock.assert_not_called() + parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) + assert parsed_progress_status.postponed_iteration is True + assert parsed_progress_status.next_step_at == scheduled_to + assert parsed_progress_status.error is None + assert parsed_progress_status.error_message is None + assert result["has_next_actions"] is True + assert result["next_iteration_description"] == task_content + next_iteration_description = json.loads(result["next_iteration_description"]) + execution = next_iteration_description["state"]["automation"].get("execution", {}) + assert "degraded_state" not in execution + + @pytest.mark.asyncio + @required_imports + async def test_logs_pending_priority_skipped_error(self, import_automation_workflow, task): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + skip_error = octobot_flow.errors.PendingPriorityActionsSkippedError("stale priority skipped") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=skip_error, + ) + mock_logger = mock.Mock() + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + automation_workflow, + "get_logger", + return_value=mock_logger, + ): + await automation_workflow.execute_iteration(inputs, None) + + mock_logger.error.assert_called_once_with( + f"Pending priority actions were skipped: {skip_error}" + ) + + @pytest.mark.asyncio + @required_imports + async def test_postponed_log_uses_none_error_fields(self, import_automation_workflow, task): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + skip_error = octobot_flow.errors.PendingPriorityActionsSkippedError("stale priority skipped") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=skip_error, + ) + mock_logger = mock.Mock() + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + fixed_now = 1000.0 + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + automation_workflow, + "get_logger", + return_value=mock_logger, + ), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ): + await automation_workflow.execute_iteration(inputs, None) + + mock_logger.info.assert_any_call( + f"Iteration postponed (None: None), retry scheduled in {scheduled_to - fixed_now:.0f} seconds" + ) + + class TestExecuteAutomationPostponedIteration: @pytest.mark.asyncio @required_imports + @pytest.mark.parametrize( + "error_status,error_message,retry_delay_seconds", + [ + pytest.param( + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value, + "Invalid API credentials", + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, + id="authentication_error", + ), + pytest.param( + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + "Trying to update BTC with -0.00074 but quantity was 0.00068", + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="portfolio_negative_value_error", + ), + ], + ) async def test_execute_automation_reschedules_on_postponed_iteration( - self, temp_dbos_scheduler, import_automation_workflow, parsed_inputs + self, + temp_dbos_scheduler, + import_automation_workflow, + parsed_inputs, + error_status, + error_message, + retry_delay_seconds, + ): + postponed_iteration_result = params.AutomationWorkflowIterationResult( + progress_status=params.ProgressStatus( + latest_step="no action executed", + next_step_at=time.time() + retry_delay_seconds, + error=error_status, + error_message=error_message, + postponed_iteration=True, + should_stop=False, + ), + next_iteration_description='{"state": {"automation": {}}}', + has_next_actions=True, + ) + inputs = parsed_inputs.to_dict(include_default_values=False) + mock_wait = mock.AsyncMock(return_value=None) + mock_iteration = mock.AsyncMock( + return_value=postponed_iteration_result.to_dict(include_default_values=False) + ) + mock_should_continue = mock.Mock(return_value=False) + mock_process = mock.AsyncMock(return_value=(True, postponed_iteration_result)) + + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + mock_wait, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "execute_iteration", + mock_iteration, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_should_continue_workflow", + mock_should_continue, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_process_pending_priority_actions_and_reschedule", + mock_process, + ): + handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_automation, + inputs=inputs, + ) + assert await handle.get_result() is None + + mock_should_continue.assert_not_called() + mock_process.assert_awaited_once_with(parsed_inputs, postponed_iteration_result) + + +class TestExecuteAutomationPendingPrioritySkipped: + @pytest.mark.asyncio + @required_imports + async def test_execute_automation_reschedules_on_pending_priority_skipped( + self, + temp_dbos_scheduler, + import_automation_workflow, + parsed_inputs, ): + scheduled_to = 5000.0 postponed_iteration_result = params.AutomationWorkflowIterationResult( progress_status=params.ProgressStatus( latest_step="no action executed", - next_step_at=time.time() + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, - error=octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR.value, - error_message="Invalid API credentials", + next_step_at=scheduled_to, + error=None, + error_message=None, postponed_iteration=True, should_stop=False, ), @@ -978,7 +1289,7 @@ async def test_process_pending_returns_false_when_should_stop(self, import_autom should_continue, _ = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._process_pending_priority_actions_and_reschedule( parsed_inputs, iteration_result ) - assert should_continue is True + assert should_continue is False @pytest.mark.asyncio async def test_process_pending_raises_when_no_next_iteration_after_priority_actions( @@ -1160,6 +1471,61 @@ def test_create_next_iteration_inputs_uses_zero_when_execution_time_none(self, i assert result.execution_time == 0 +class TestGetPostponedIterationErrorStatusAndDelay: + @pytest.mark.parametrize( + "error,expected_status,expected_delay", + [ + pytest.param( + octobot_trading_errors.AuthenticationError("Invalid API credentials"), + octobot_flow.enums.ActionErrorStatus.AUTHENTICATION_ERROR, + octobot_node.constants.INVALID_AUTHENTICATION_RETRY_DELAY_SECONDS, + id="authentication_error", + ), + pytest.param( + octobot_trading_errors.PortfolioNegativeValueError( + "Trying to update BTC with -0.00074 but quantity was 0.00068" + ), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="portfolio_negative_value_error", + ), + pytest.param( + octobot_trading_errors.FailedRequest("Exchange API request failed"), + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="failed_request", + ), + pytest.param( + octobot_trading_errors.MissingFunds("Insufficient funds for order"), + octobot_flow.enums.ActionErrorStatus.NOT_ENOUGH_FUNDS, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="missing_funds", + ), + pytest.param( + octobot_trading_errors.MissingMinimalExchangeTradeVolume( + "Order volume below exchange minimum" + ), + octobot_flow.enums.ActionErrorStatus.INVALID_ORDER, + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS, + id="missing_minimal_exchange_trade_volume", + ), + ], + ) + def test_returns_expected_status_and_delay( + self, + import_automation_workflow, + error, + expected_status, + expected_delay, + ): + resolved_status, resolved_delay = ( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + ._get_postponed_iteration_error_status_and_delay(error) + ) + assert resolved_status == expected_status + assert resolved_delay == expected_delay + + class TestShouldContinueWorkflow: def test_should_continue_returns_stop_on_error_when_error(self, import_automation_workflow, parsed_inputs): progress = params.ProgressStatus(error="some_error", should_stop=False) diff --git a/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py new file mode 100644 index 0000000000..61eb5832e2 --- /dev/null +++ b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py @@ -0,0 +1,125 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import datetime +import mock +import pytest + +import octobot_node.enums + +from tests.scheduler import temp_dbos_scheduler + +_PARENT_WORKFLOW_ID_A = "741ce171-dac9-40be-83dc-b443c0eaf0e2" + + +class TestDbosCleanupWorkflowDbosCleanup: + @pytest.fixture + def dbos_cleanup_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module_loaded + + yield dbos_cleanup_workflow_module_loaded + + @pytest.mark.asyncio + async def test_delegates_to_cleanup_outdated_automation_executions(self, dbos_cleanup_workflow_module): + expected_summary = { + "deleted_by_automation": {_PARENT_WORKFLOW_ID_A: 2}, + "deleted_cleanup_executions": 1, + "total_deleted": 3, + } + with mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.workflows_retention.should_skip_retention_cleanup_on_this_node", + return_value=False, + ), mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.workflows_retention.should_skip_retention_cleanup_for_scheduled_time", + mock.AsyncMock(return_value=False), + ), mock.patch( + "octobot_node.scheduler.workflows_retention.cleanup_outdated_automation_executions", + mock.AsyncMock(return_value=expected_summary), + ) as cleanup_mock: + result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( + datetime.datetime.now(datetime.timezone.utc), + None, + ) + + cleanup_mock.assert_awaited_once() + assert result == expected_summary + + @pytest.mark.asyncio + async def test_skips_cleanup_on_consumer_only(self, dbos_cleanup_workflow_module): + mock_logger = mock.Mock() + with mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.workflows_retention.should_skip_retention_cleanup_on_this_node", + return_value=True, + ), mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.logging.get_logger", + return_value=mock_logger, + ), mock.patch( + "octobot_node.scheduler.workflows_retention.cleanup_outdated_automation_executions", + mock.AsyncMock(), + ) as cleanup_mock, mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.workflows_retention.should_skip_retention_cleanup_for_scheduled_time", + mock.AsyncMock(), + ) as skip_for_scheduled_time_mock: + result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( + datetime.datetime.now(datetime.timezone.utc), + None, + ) + + cleanup_mock.assert_not_called() + skip_for_scheduled_time_mock.assert_not_called() + mock_logger.info.assert_called_once_with("dbos_cleanup skipped: consumer-only node") + assert result == { + "deleted_by_automation": {}, + "deleted_cleanup_executions": 0, + "total_deleted": 0, + } + + @pytest.mark.asyncio + async def test_skips_cleanup_when_newer_execution_already_ran(self, dbos_cleanup_workflow_module): + scheduled_time = datetime.datetime(2025, 1, 1, tzinfo=datetime.timezone.utc) + mock_logger = mock.Mock() + with mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.workflows_retention.should_skip_retention_cleanup_on_this_node", + return_value=False, + ), mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.logging.get_logger", + return_value=mock_logger, + ), mock.patch( + "octobot_node.scheduler.workflows.dbos_cleanup_workflow.workflows_retention.should_skip_retention_cleanup_for_scheduled_time", + mock.AsyncMock(return_value=True), + ), mock.patch( + "octobot_node.scheduler.workflows_retention.cleanup_outdated_automation_executions", + mock.AsyncMock(), + ) as cleanup_mock: + result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( + scheduled_time, + None, + ) + + cleanup_mock.assert_not_called() + mock_logger.info.assert_called_once_with( + "dbos_cleanup skipped for scheduled_time %s: latest completed cleanup is newer", + scheduled_time.isoformat(), + ) + assert result == { + "deleted_by_automation": {}, + "deleted_cleanup_executions": 0, + "total_deleted": 0, + } + + +class TestDbosCleanupWorkflowGetScheduleInput: + def test_returns_daily_schedule_input(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = dbos_cleanup_workflow_module.get_schedule_input() + + assert schedule_input == { + "schedule_name": dbos_cleanup_workflow_module.SCHEDULE_NAME, + "workflow_fn": dbos_cleanup_workflow_module.DbosCleanupWorkflow.dbos_cleanup, + "schedule": dbos_cleanup_workflow_module.SCHEDULE_CRON, + "context": None, + "automatic_backfill": True, + "queue_name": octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, + } + diff --git a/packages/protocol/.openapi-generator/FILES b/packages/protocol/.openapi-generator/FILES index f79681b486..c85c6a2073 100644 --- a/packages/protocol/.openapi-generator/FILES +++ b/packages/protocol/.openapi-generator/FILES @@ -29,6 +29,7 @@ docs/AutomationState.md docs/BlockchainAccount.md docs/CancelPolicy.md docs/CancelPolicyType.md +docs/ChildOctoBotProcessState.md docs/CopiedAccount.md docs/CopiedAsset.md docs/CopyConfiguration.md @@ -45,6 +46,13 @@ docs/DeleteExchangeConfigConfiguration.md docs/DeleteStrategyConfiguration.md docs/DetailedAsset.md docs/DetailedAssetsForTradingType.md +docs/DslKeyword.md +docs/DslKeywordCategory.md +docs/DslKeywordsState.md +docs/DslParameter.md +docs/DslParameterDefaultValue.md +docs/DslParameterOption.md +docs/DslValueType.md docs/EditAccountAuthConfiguration.md docs/EditAccountConfiguration.md docs/EditAutomationConfiguration.md @@ -82,6 +90,7 @@ docs/Position.md docs/PositionStatus.md docs/PositionSummary.md docs/RefreshAccountsConfiguration.md +docs/RestartAutomationConfiguration.md docs/Side.md docs/SignalAutomationConfiguration.md docs/SignalAutomationConfigurationSignalPayload.md @@ -96,6 +105,9 @@ docs/StrategyReference.md docs/TimeFrame.md docs/Trade.md docs/TradeSummary.md +docs/TradedPairVolume.md +docs/TradedPairsByExchange.md +docs/TradedPairsForExchange.md docs/TradingTentaclesConfiguration.md docs/TradingType.md docs/TrailingProfile.md @@ -143,6 +155,7 @@ octobot_protocol/models/automation_state.py octobot_protocol/models/blockchain_account.py octobot_protocol/models/cancel_policy.py octobot_protocol/models/cancel_policy_type.py +octobot_protocol/models/child_octo_bot_process_state.py octobot_protocol/models/copied_account.py octobot_protocol/models/copied_asset.py octobot_protocol/models/copy_configuration.py @@ -159,6 +172,13 @@ octobot_protocol/models/delete_exchange_config_configuration.py octobot_protocol/models/delete_strategy_configuration.py octobot_protocol/models/detailed_asset.py octobot_protocol/models/detailed_assets_for_trading_type.py +octobot_protocol/models/dsl_keyword.py +octobot_protocol/models/dsl_keyword_category.py +octobot_protocol/models/dsl_keywords_state.py +octobot_protocol/models/dsl_parameter.py +octobot_protocol/models/dsl_parameter_default_value.py +octobot_protocol/models/dsl_parameter_option.py +octobot_protocol/models/dsl_value_type.py octobot_protocol/models/edit_account_auth_configuration.py octobot_protocol/models/edit_account_configuration.py octobot_protocol/models/edit_automation_configuration.py @@ -196,6 +216,7 @@ octobot_protocol/models/position.py octobot_protocol/models/position_status.py octobot_protocol/models/position_summary.py octobot_protocol/models/refresh_accounts_configuration.py +octobot_protocol/models/restart_automation_configuration.py octobot_protocol/models/side.py octobot_protocol/models/signal_automation_configuration.py octobot_protocol/models/signal_automation_configuration_signal_payload.py @@ -210,6 +231,9 @@ octobot_protocol/models/strategy_reference.py octobot_protocol/models/time_frame.py octobot_protocol/models/trade.py octobot_protocol/models/trade_summary.py +octobot_protocol/models/traded_pair_volume.py +octobot_protocol/models/traded_pairs_by_exchange.py +octobot_protocol/models/traded_pairs_for_exchange.py octobot_protocol/models/trading_tentacles_configuration.py octobot_protocol/models/trading_type.py octobot_protocol/models/trailing_profile.py @@ -255,6 +279,7 @@ test/test_automation_state.py test/test_blockchain_account.py test/test_cancel_policy.py test/test_cancel_policy_type.py +test/test_child_octo_bot_process_state.py test/test_copied_account.py test/test_copied_asset.py test/test_copy_configuration.py @@ -271,6 +296,13 @@ test/test_delete_exchange_config_configuration.py test/test_delete_strategy_configuration.py test/test_detailed_asset.py test/test_detailed_assets_for_trading_type.py +test/test_dsl_keyword.py +test/test_dsl_keyword_category.py +test/test_dsl_keywords_state.py +test/test_dsl_parameter.py +test/test_dsl_parameter_default_value.py +test/test_dsl_parameter_option.py +test/test_dsl_value_type.py test/test_edit_account_auth_configuration.py test/test_edit_account_configuration.py test/test_edit_automation_configuration.py @@ -308,6 +340,7 @@ test/test_position.py test/test_position_status.py test/test_position_summary.py test/test_refresh_accounts_configuration.py +test/test_restart_automation_configuration.py test/test_side.py test/test_signal_automation_configuration.py test/test_signal_automation_configuration_signal_payload.py @@ -322,6 +355,9 @@ test/test_strategy_reference.py test/test_time_frame.py test/test_trade.py test/test_trade_summary.py +test/test_traded_pair_volume.py +test/test_traded_pairs_by_exchange.py +test/test_traded_pairs_for_exchange.py test/test_trading_tentacles_configuration.py test/test_trading_type.py test/test_trailing_profile.py diff --git a/packages/protocol/docs/AccountAuthenticationDetails.md b/packages/protocol/docs/AccountAuthenticationDetails.md deleted file mode 100644 index a15fbfd806..0000000000 --- a/packages/protocol/docs/AccountAuthenticationDetails.md +++ /dev/null @@ -1,35 +0,0 @@ -# AccountAuthenticationDetails - -AccountAuthenticationDetails - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**api_key** | **str** | | [optional] -**api_secret** | **str** | | [optional] -**api_passphrase** | **str** | | [optional] -**public_key** | **str** | | [optional] -**private_key** | **str** | | [optional] -**seed_phrase** | **str** | | [optional] - -## Example - -```python -from octobot_protocol.models.account_authentication_details import AccountAuthenticationDetails - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountAuthenticationDetails from a JSON string -account_authentication_details_instance = AccountAuthenticationDetails.from_json(json) -# print the JSON string representation of the object -print(AccountAuthenticationDetails.to_json()) - -# convert the object into a dict -account_authentication_details_dict = account_authentication_details_instance.to_dict() -# create an instance of AccountAuthenticationDetails from a dict -account_authentication_details_from_dict = AccountAuthenticationDetails.from_dict(account_authentication_details_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountAuthenticationSummary.md b/packages/protocol/docs/AccountAuthenticationSummary.md deleted file mode 100644 index c6a2bbe66d..0000000000 --- a/packages/protocol/docs/AccountAuthenticationSummary.md +++ /dev/null @@ -1,33 +0,0 @@ -# AccountAuthenticationSummary - -AccountAuthenticationSummary - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**id** | **str** | | -**updated_at** | **datetime** | | [optional] -**api_key** | **str** | | [optional] -**public_key** | **str** | | [optional] - -## Example - -```python -from octobot_protocol.models.account_authentication_summary import AccountAuthenticationSummary - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountAuthenticationSummary from a JSON string -account_authentication_summary_instance = AccountAuthenticationSummary.from_json(json) -# print the JSON string representation of the object -print(AccountAuthenticationSummary.to_json()) - -# convert the object into a dict -account_authentication_summary_dict = account_authentication_summary_instance.to_dict() -# create an instance of AccountAuthenticationSummary from a dict -account_authentication_summary_from_dict = AccountAuthenticationSummary.from_dict(account_authentication_summary_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountDetails.md b/packages/protocol/docs/AccountDetails.md deleted file mode 100644 index 5dbd41db4b..0000000000 --- a/packages/protocol/docs/AccountDetails.md +++ /dev/null @@ -1,34 +0,0 @@ -# AccountDetails - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**account_type** | [**AccountType**](AccountType.md) | generic | -**trading_type** | [**TradingType**](TradingType.md) | | -**exchange** | **str** | | -**remote_account_id** | **str** | | -**blockchain** | **str** | | -**network** | **str** | | [optional] - -## Example - -```python -from octobot_protocol.models.account_details import AccountDetails - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountDetails from a JSON string -account_details_instance = AccountDetails.from_json(json) -# print the JSON string representation of the object -print(AccountDetails.to_json()) - -# convert the object into a dict -account_details_dict = account_details_instance.to_dict() -# create an instance of AccountDetails from a dict -account_details_from_dict = AccountDetails.from_dict(account_details_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountTradingDetails.md b/packages/protocol/docs/AccountTradingDetails.md deleted file mode 100644 index 7a59e46de4..0000000000 --- a/packages/protocol/docs/AccountTradingDetails.md +++ /dev/null @@ -1,33 +0,0 @@ -# AccountTradingDetails - -AccountTradingDetails - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**updated_at** | **datetime** | | -**orders** | [**List[Order]**](Order.md) | | [optional] -**trades** | [**List[Trade]**](Trade.md) | | [optional] -**positions** | [**List[Position]**](Position.md) | | [optional] - -## Example - -```python -from octobot_protocol.models.account_trading_details import AccountTradingDetails - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountTradingDetails from a JSON string -account_trading_details_instance = AccountTradingDetails.from_json(json) -# print the JSON string representation of the object -print(AccountTradingDetails.to_json()) - -# convert the object into a dict -account_trading_details_dict = account_trading_details_instance.to_dict() -# create an instance of AccountTradingDetails from a dict -account_trading_details_from_dict = AccountTradingDetails.from_dict(account_trading_details_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountTradingDetailsState.md b/packages/protocol/docs/AccountTradingDetailsState.md deleted file mode 100644 index 363b0746f5..0000000000 --- a/packages/protocol/docs/AccountTradingDetailsState.md +++ /dev/null @@ -1,31 +0,0 @@ -# AccountTradingDetailsState - -AccountsTradingDetailsState - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**version** | **str** | | -**details** | [**List[AccountTradingDetails]**](AccountTradingDetails.md) | | - -## Example - -```python -from octobot_protocol.models.account_trading_details_state import AccountTradingDetailsState - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountTradingDetailsState from a JSON string -account_trading_details_state_instance = AccountTradingDetailsState.from_json(json) -# print the JSON string representation of the object -print(AccountTradingDetailsState.to_json()) - -# convert the object into a dict -account_trading_details_state_dict = account_trading_details_state_instance.to_dict() -# create an instance of AccountTradingDetailsState from a dict -account_trading_details_state_from_dict = AccountTradingDetailsState.from_dict(account_trading_details_state_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountTrailingDetailsState.md b/packages/protocol/docs/AccountTrailingDetailsState.md deleted file mode 100644 index 284a1f3c55..0000000000 --- a/packages/protocol/docs/AccountTrailingDetailsState.md +++ /dev/null @@ -1,32 +0,0 @@ -# AccountTrailingDetailsState - -AccountTrailingDetailsState - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**version** | **str** | | -**account_id** | **str** | | -**details** | [**List[AccountTradingDetails]**](AccountTradingDetails.md) | | - -## Example - -```python -from octobot_protocol.models.account_trailing_details_state import AccountTrailingDetailsState - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountTrailingDetailsState from a JSON string -account_trailing_details_state_instance = AccountTrailingDetailsState.from_json(json) -# print the JSON string representation of the object -print(AccountTrailingDetailsState.to_json()) - -# convert the object into a dict -account_trailing_details_state_dict = account_trailing_details_state_instance.to_dict() -# create an instance of AccountTrailingDetailsState from a dict -account_trailing_details_state_from_dict = AccountTrailingDetailsState.from_dict(account_trailing_details_state_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountsAuthenticationDetailsState.md b/packages/protocol/docs/AccountsAuthenticationDetailsState.md deleted file mode 100644 index eb01ed164d..0000000000 --- a/packages/protocol/docs/AccountsAuthenticationDetailsState.md +++ /dev/null @@ -1,31 +0,0 @@ -# AccountsAuthenticationDetailsState - -AccountsAuthenticationDetailsState - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**version** | **str** | | -**details** | [**List[AccountAuthenticationDetails]**](AccountAuthenticationDetails.md) | | - -## Example - -```python -from octobot_protocol.models.accounts_authentication_details_state import AccountsAuthenticationDetailsState - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountsAuthenticationDetailsState from a JSON string -accounts_authentication_details_state_instance = AccountsAuthenticationDetailsState.from_json(json) -# print the JSON string representation of the object -print(AccountsAuthenticationDetailsState.to_json()) - -# convert the object into a dict -accounts_authentication_details_state_dict = accounts_authentication_details_state_instance.to_dict() -# create an instance of AccountsAuthenticationDetailsState from a dict -accounts_authentication_details_state_from_dict = AccountsAuthenticationDetailsState.from_dict(accounts_authentication_details_state_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountsTradingSummary.md b/packages/protocol/docs/AccountsTradingSummary.md deleted file mode 100644 index b703705913..0000000000 --- a/packages/protocol/docs/AccountsTradingSummary.md +++ /dev/null @@ -1,33 +0,0 @@ -# AccountsTradingSummary - -AccountsTradingSummary - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**account_id** | **str** | | -**orders** | [**List[Order]**](Order.md) | | [optional] -**positions** | [**List[Position]**](Position.md) | | [optional] -**trades** | [**List[Trade]**](Trade.md) | | [optional] - -## Example - -```python -from octobot_protocol.models.accounts_trading_summary import AccountsTradingSummary - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountsTradingSummary from a JSON string -accounts_trading_summary_instance = AccountsTradingSummary.from_json(json) -# print the JSON string representation of the object -print(AccountsTradingSummary.to_json()) - -# convert the object into a dict -accounts_trading_summary_dict = accounts_trading_summary_instance.to_dict() -# create an instance of AccountsTradingSummary from a dict -accounts_trading_summary_from_dict = AccountsTradingSummary.from_dict(accounts_trading_summary_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AccountsTradingWithAccountId.md b/packages/protocol/docs/AccountsTradingWithAccountId.md deleted file mode 100644 index 4b4e58b0b8..0000000000 --- a/packages/protocol/docs/AccountsTradingWithAccountId.md +++ /dev/null @@ -1,31 +0,0 @@ -# AccountsTradingWithAccountId - -AccountsTradingWithAccountId - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**account_id** | **str** | | -**account_trading** | [**AccountTrading**](AccountTrading.md) | | [optional] - -## Example - -```python -from octobot_protocol.models.accounts_trading_with_account_id import AccountsTradingWithAccountId - -# TODO update the JSON string below -json = "{}" -# create an instance of AccountsTradingWithAccountId from a JSON string -accounts_trading_with_account_id_instance = AccountsTradingWithAccountId.from_json(json) -# print the JSON string representation of the object -print(AccountsTradingWithAccountId.to_json()) - -# convert the object into a dict -accounts_trading_with_account_id_dict = accounts_trading_with_account_id_instance.to_dict() -# create an instance of AccountsTradingWithAccountId from a dict -accounts_trading_with_account_id_from_dict = AccountsTradingWithAccountId.from_dict(accounts_trading_with_account_id_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/ActionType.md b/packages/protocol/docs/ActionType.md deleted file mode 100644 index 752f93288a..0000000000 --- a/packages/protocol/docs/ActionType.md +++ /dev/null @@ -1,23 +0,0 @@ -# ActionType - -ActionType - -## Enum - -* `AUTOMATION_CREATE` (value: `'automation_create'`) - -* `AUTOMATION_EDIT` (value: `'automation_edit'`) - -* `AUTOMATION_STOP` (value: `'automation_stop'`) - -* `ACCOUNT_CREATE` (value: `'account_create'`) - -* `ACCOUNT_EDIT` (value: `'account_edit'`) - -* `ACCOUNT_DELETE` (value: `'account_delete'`) - -* `ACCOUNTS_REFRESH` (value: `'accounts_refresh'`) - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/Asset.md b/packages/protocol/docs/Asset.md deleted file mode 100644 index 2fcf32d5a9..0000000000 --- a/packages/protocol/docs/Asset.md +++ /dev/null @@ -1,34 +0,0 @@ -# Asset - -Asset - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**symbol** | **str** | | -**total** | **float** | | -**available** | **float** | | -**value** | **float** | | [optional] -**unit** | **str** | | [optional] - -## Example - -```python -from octobot_protocol.models.asset import Asset - -# TODO update the JSON string below -json = "{}" -# create an instance of Asset from a JSON string -asset_instance = Asset.from_json(json) -# print the JSON string representation of the object -print(Asset.to_json()) - -# convert the object into a dict -asset_dict = asset_instance.to_dict() -# create an instance of Asset from a dict -asset_from_dict = Asset.from_dict(asset_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AuthenticationDetailsState.md b/packages/protocol/docs/AuthenticationDetailsState.md deleted file mode 100644 index fe3ce4d46b..0000000000 --- a/packages/protocol/docs/AuthenticationDetailsState.md +++ /dev/null @@ -1,31 +0,0 @@ -# AuthenticationDetailsState - -AuthenticationDetailsState - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**version** | **str** | | -**details** | [**List[AccountAuthenticationDetails]**](AccountAuthenticationDetails.md) | | - -## Example - -```python -from octobot_protocol.models.authentication_details_state import AuthenticationDetailsState - -# TODO update the JSON string below -json = "{}" -# create an instance of AuthenticationDetailsState from a JSON string -authentication_details_state_instance = AuthenticationDetailsState.from_json(json) -# print the JSON string representation of the object -print(AuthenticationDetailsState.to_json()) - -# convert the object into a dict -authentication_details_state_dict = authentication_details_state_instance.to_dict() -# create an instance of AuthenticationDetailsState from a dict -authentication_details_state_from_dict = AuthenticationDetailsState.from_dict(authentication_details_state_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AutomationConfiguration.md b/packages/protocol/docs/AutomationConfiguration.md index 03675c317a..678b95a127 100644 --- a/packages/protocol/docs/AutomationConfiguration.md +++ b/packages/protocol/docs/AutomationConfiguration.md @@ -12,7 +12,7 @@ Name | Type | Description | Notes **created_at** | **datetime** | | **updated_at** | **datetime** | | [optional] **strategy** | [**StrategyReference**](StrategyReference.md) | | -**accounts** | [**List[AccountReference]**](AccountReference.md) | | +**accounts** | [**List[AccountReference]**](AccountReference.md) | May be empty only for automations whose strategy configuration is generic_process; otherwise exactly one account reference is required at runtime. | ## Example diff --git a/packages/protocol/docs/AutomationConfigurationConfiguration.md b/packages/protocol/docs/AutomationConfigurationConfiguration.md deleted file mode 100644 index 465c5c2275..0000000000 --- a/packages/protocol/docs/AutomationConfigurationConfiguration.md +++ /dev/null @@ -1,54 +0,0 @@ -# AutomationConfigurationConfiguration - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**ActionConfigurationType**](ActionConfigurationType.md) | generic_workflow | -**pair_settings** | [**List[MarketMakingSymbolConfiguration]**](MarketMakingSymbolConfiguration.md) | | -**symbols** | **List[str]** | | -**buy_orders_count** | **float** | | -**percent_amount_per_buy_order** | **float** | | -**profit_target_percent** | **float** | | -**buy_order_price_discount_percent** | **float** | | -**enable_stop_loss** | **bool** | | [default to False] -**stop_loss_price_discount_percent** | **float** | | -**trigger_mode** | **str** | | -**use_init_entry_orders** | **bool** | | [default to True] -**time_frames** | [**List[TimeFrame]**](TimeFrame.md) | | -**evaluators** | [**List[EvaluatorConfiguration]**](EvaluatorConfiguration.md) | | -**coins** | [**List[IndexCoin]**](IndexCoin.md) | | -**rebalance_trigger_min_percent** | **float** | | -**symbol** | **str** | | -**spread** | **float** | Price difference between the closest buy and sell orders. Denominated in the quote currency (600 for a 600 USDT spread on BTC/USDT). | -**increment** | **float** | Price difference between two orders of the same side. Denominated in the quote currency (200 for a 200 USDT spread on BTC/USDT). | -**buy_count** | **float** | Number of initial buy orders to create. Make sure to have enough funds to create that many orders. | -**sell_count** | **float** | Number of initial sell orders to create. Make sure to have enough funds to create that many orders. | -**enable_trailing_up** | **bool** | | [default to True] -**enable_trailing_down** | **bool** | | [default to False] -**order_by_order_trailing** | **bool** | | [default to True] -**strategy_id** | **str** | | -**profile_data** | **object** | | -**actions** | [**List[Action]**](Action.md) | | - -## Example - -```python -from octobot_protocol.models.automation_configuration_configuration import AutomationConfigurationConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of AutomationConfigurationConfiguration from a JSON string -automation_configuration_configuration_instance = AutomationConfigurationConfiguration.from_json(json) -# print the JSON string representation of the object -print(AutomationConfigurationConfiguration.to_json()) - -# convert the object into a dict -automation_configuration_configuration_dict = automation_configuration_configuration_instance.to_dict() -# create an instance of AutomationConfigurationConfiguration from a dict -automation_configuration_configuration_from_dict = AutomationConfigurationConfiguration.from_dict(automation_configuration_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/AutomationState.md b/packages/protocol/docs/AutomationState.md index d474fa4b01..5f08168eae 100644 --- a/packages/protocol/docs/AutomationState.md +++ b/packages/protocol/docs/AutomationState.md @@ -19,6 +19,7 @@ Name | Type | Description | Notes **orders** | [**List[OrderSummary]**](OrderSummary.md) | | [optional] **trades** | [**List[TradeSummary]**](TradeSummary.md) | | [optional] **positions** | [**List[PositionSummary]**](PositionSummary.md) | | [optional] +**child_octobot_process** | [**ChildOctoBotProcessState**](ChildOctoBotProcessState.md) | | [optional] ## Example diff --git a/packages/protocol/docs/AutomationsState.md b/packages/protocol/docs/AutomationsState.md deleted file mode 100644 index e5eafd8541..0000000000 --- a/packages/protocol/docs/AutomationsState.md +++ /dev/null @@ -1,31 +0,0 @@ -# AutomationsState - -AutomationsState - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**version** | **str** | | -**automations** | [**List[AutomationState]**](AutomationState.md) | | [optional] - -## Example - -```python -from octobot_protocol.models.automations_state import AutomationsState - -# TODO update the JSON string below -json = "{}" -# create an instance of AutomationsState from a JSON string -automations_state_instance = AutomationsState.from_json(json) -# print the JSON string representation of the object -print(AutomationsState.to_json()) - -# convert the object into a dict -automations_state_dict = automations_state_instance.to_dict() -# create an instance of AutomationsState from a dict -automations_state_from_dict = AutomationsState.from_dict(automations_state_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/ChildOctoBotProcessState.md b/packages/protocol/docs/ChildOctoBotProcessState.md new file mode 100644 index 0000000000..d7d89c92d2 --- /dev/null +++ b/packages/protocol/docs/ChildOctoBotProcessState.md @@ -0,0 +1,32 @@ +# ChildOctoBotProcessState + +ChildOctoBotProcessState + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**http_base_url** | **str** | | +**web_port** | **int** | | +**init_state_ok** | **bool** | | + +## Example + +```python +from octobot_protocol.models.child_octo_bot_process_state import ChildOctoBotProcessState + +# TODO update the JSON string below +json = "{}" +# create an instance of ChildOctoBotProcessState from a JSON string +child_octo_bot_process_state_instance = ChildOctoBotProcessState.from_json(json) +# print the JSON string representation of the object +print(ChildOctoBotProcessState.to_json()) + +# convert the object into a dict +child_octo_bot_process_state_dict = child_octo_bot_process_state_instance.to_dict() +# create an instance of ChildOctoBotProcessState from a dict +child_octo_bot_process_state_from_dict = ChildOctoBotProcessState.from_dict(child_octo_bot_process_state_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/CopiedAccountContentValueValue.md b/packages/protocol/docs/CopiedAccountContentValueValue.md deleted file mode 100644 index 78255d07f0..0000000000 --- a/packages/protocol/docs/CopiedAccountContentValueValue.md +++ /dev/null @@ -1,28 +0,0 @@ -# CopiedAccountContentValueValue - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- - -## Example - -```python -from octobot_protocol.models.copied_account_content_value_value import CopiedAccountContentValueValue - -# TODO update the JSON string below -json = "{}" -# create an instance of CopiedAccountContentValueValue from a JSON string -copied_account_content_value_value_instance = CopiedAccountContentValueValue.from_json(json) -# print the JSON string representation of the object -print(CopiedAccountContentValueValue.to_json()) - -# convert the object into a dict -copied_account_content_value_value_dict = copied_account_content_value_value_instance.to_dict() -# create an instance of CopiedAccountContentValueValue from a dict -copied_account_content_value_value_from_dict = CopiedAccountContentValueValue.from_dict(copied_account_content_value_value_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/DCAConfiguration.md b/packages/protocol/docs/DCAConfiguration.md deleted file mode 100644 index 292a68c15f..0000000000 --- a/packages/protocol/docs/DCAConfiguration.md +++ /dev/null @@ -1,44 +0,0 @@ -# DCAConfiguration - -DCAConfiguration - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**ActionConfigurationType**](ActionConfigurationType.md) | dca | -**symbols** | **List[str]** | | -**entry_order_amount** | **str** | Amout to buy, can be in %t, %s, in q, in base, etc | -**exit_limit_orders_price_percent** | **float** | | -**entry_limit_orders_price_percent** | **float** | | -**secondary_entry_orders_count** | **float** | | [default to 0] -**secondary_entry_orders_amount** | **str** | Amout to buy, can be in %t, %s, in q, in base, etc | [default to '0%t'] -**secondary_entry_orders_price_percent** | **float** | | [default to 10] -**enable_stop_loss** | **bool** | | [optional] [default to False] -**stop_loss_price_discount_percent** | **float** | | [optional] [default to 10] -**trigger_mode** | **str** | | [optional] [default to 'Maximum evaluators signals based'] -**use_init_entry_orders** | **bool** | | [optional] [default to True] -**max_asset_holding_percent** | **float** | | [optional] [default to 50] -**strategies** | [**List[StrategyEvaluatorConfiguration]**](StrategyEvaluatorConfiguration.md) | | -**evaluators** | [**List[EvaluatorConfiguration]**](EvaluatorConfiguration.md) | | - -## Example - -```python -from octobot_protocol.models.dca_configuration import DCAConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of DCAConfiguration from a JSON string -dca_configuration_instance = DCAConfiguration.from_json(json) -# print the JSON string representation of the object -print(DCAConfiguration.to_json()) - -# convert the object into a dict -dca_configuration_dict = dca_configuration_instance.to_dict() -# create an instance of DCAConfiguration from a dict -dca_configuration_from_dict = DCAConfiguration.from_dict(dca_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/DEXConfig.md b/packages/protocol/docs/DEXConfig.md deleted file mode 100644 index dc10b1e6b6..0000000000 --- a/packages/protocol/docs/DEXConfig.md +++ /dev/null @@ -1,33 +0,0 @@ -# DEXConfig - -DEXConfig - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**chain_id** | **str** | | -**dex_id** | **str** | | -**base_token_addresses** | **List[str]** | | [optional] -**quote_token_addresses** | **List[str]** | | [optional] - -## Example - -```python -from octobot_protocol.models.dex_config import DEXConfig - -# TODO update the JSON string below -json = "{}" -# create an instance of DEXConfig from a JSON string -dex_config_instance = DEXConfig.from_json(json) -# print the JSON string representation of the object -print(DEXConfig.to_json()) - -# convert the object into a dict -dex_config_dict = dex_config_instance.to_dict() -# create an instance of DEXConfig from a dict -dex_config_from_dict = DEXConfig.from_dict(dex_config_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/DslKeyword.md b/packages/protocol/docs/DslKeyword.md new file mode 100644 index 0000000000..bba3672f63 --- /dev/null +++ b/packages/protocol/docs/DslKeyword.md @@ -0,0 +1,35 @@ +# DslKeyword + +Definition of one DSL keyword (signature and configuration). + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**name** | **str** | Stable DSL keyword id. | +**category** | [**DslKeywordCategory**](DslKeywordCategory.md) | | +**label** | **str** | | +**description** | **str** | | +**inputs** | [**List[DslParameter]**](DslParameter.md) | | +**outputs** | [**List[DslParameter]**](DslParameter.md) | | + +## Example + +```python +from octobot_protocol.models.dsl_keyword import DslKeyword + +# TODO update the JSON string below +json = "{}" +# create an instance of DslKeyword from a JSON string +dsl_keyword_instance = DslKeyword.from_json(json) +# print the JSON string representation of the object +print(DslKeyword.to_json()) + +# convert the object into a dict +dsl_keyword_dict = dsl_keyword_instance.to_dict() +# create an instance of DslKeyword from a dict +dsl_keyword_from_dict = DslKeyword.from_dict(dsl_keyword_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/DslKeywordCategory.md b/packages/protocol/docs/DslKeywordCategory.md new file mode 100644 index 0000000000..351e2b37be --- /dev/null +++ b/packages/protocol/docs/DslKeywordCategory.md @@ -0,0 +1,19 @@ +# DslKeywordCategory + +Composition role of a DSL keyword. + +## Enum + +* `TRIGGER` (value: `'trigger'`) + +* `SOURCE` (value: `'source'`) + +* `CONDITION` (value: `'condition'`) + +* `LOGIC` (value: `'logic'`) + +* `ACTION` (value: `'action'`) + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/DslKeywordsState.md b/packages/protocol/docs/DslKeywordsState.md new file mode 100644 index 0000000000..7a86016921 --- /dev/null +++ b/packages/protocol/docs/DslKeywordsState.md @@ -0,0 +1,31 @@ +# DslKeywordsState + +Versioned list of DSL keywords available on a node. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**version** | **str** | | +**keywords** | [**List[DslKeyword]**](DslKeyword.md) | Available keywords in intended display order. | + +## Example + +```python +from octobot_protocol.models.dsl_keywords_state import DslKeywordsState + +# TODO update the JSON string below +json = "{}" +# create an instance of DslKeywordsState from a JSON string +dsl_keywords_state_instance = DslKeywordsState.from_json(json) +# print the JSON string representation of the object +print(DslKeywordsState.to_json()) + +# convert the object into a dict +dsl_keywords_state_dict = dsl_keywords_state_instance.to_dict() +# create an instance of DslKeywordsState from a dict +dsl_keywords_state_from_dict = DslKeywordsState.from_dict(dsl_keywords_state_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/DslParameter.md b/packages/protocol/docs/DslParameter.md new file mode 100644 index 0000000000..20407dd13f --- /dev/null +++ b/packages/protocol/docs/DslParameter.md @@ -0,0 +1,41 @@ +# DslParameter + +Configurable parameter on a DSL keyword. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**name** | **str** | Unique within this keyword. | +**label** | **str** | | +**value_type** | [**DslValueType**](DslValueType.md) | | +**description** | **str** | | [optional] +**default_value** | [**DslParameterDefaultValue**](DslParameterDefaultValue.md) | | [optional] +**minimum** | **float** | Only for parameter_type=number. | [optional] +**maximum** | **float** | Only for parameter_type=number. | [optional] +**step** | **float** | Only for parameter_type=number. | [optional] +**options** | [**List[DslParameterOption]**](DslParameterOption.md) | Allowed choices for value_type=text or value_type=time_frame. | [optional] +**required** | **bool** | | [optional] +**multiple** | **bool** | Accepts more than one value of this parameter. | [optional] +**primary** | **bool** | Prefer in compact editors. | [optional] + +## Example + +```python +from octobot_protocol.models.dsl_parameter import DslParameter + +# TODO update the JSON string below +json = "{}" +# create an instance of DslParameter from a JSON string +dsl_parameter_instance = DslParameter.from_json(json) +# print the JSON string representation of the object +print(DslParameter.to_json()) + +# convert the object into a dict +dsl_parameter_dict = dsl_parameter_instance.to_dict() +# create an instance of DslParameter from a dict +dsl_parameter_from_dict = DslParameter.from_dict(dsl_parameter_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/DslParameterDefaultValue.md b/packages/protocol/docs/DslParameterDefaultValue.md new file mode 100644 index 0000000000..2c044468c8 --- /dev/null +++ b/packages/protocol/docs/DslParameterDefaultValue.md @@ -0,0 +1,29 @@ +# DslParameterDefaultValue + +Must match parameter_type; for select/time_frame must be one of options[].value. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- + +## Example + +```python +from octobot_protocol.models.dsl_parameter_default_value import DslParameterDefaultValue + +# TODO update the JSON string below +json = "{}" +# create an instance of DslParameterDefaultValue from a JSON string +dsl_parameter_default_value_instance = DslParameterDefaultValue.from_json(json) +# print the JSON string representation of the object +print(DslParameterDefaultValue.to_json()) + +# convert the object into a dict +dsl_parameter_default_value_dict = dsl_parameter_default_value_instance.to_dict() +# create an instance of DslParameterDefaultValue from a dict +dsl_parameter_default_value_from_dict = DslParameterDefaultValue.from_dict(dsl_parameter_default_value_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/DslParameterOption.md b/packages/protocol/docs/DslParameterOption.md new file mode 100644 index 0000000000..e9277a1367 --- /dev/null +++ b/packages/protocol/docs/DslParameterOption.md @@ -0,0 +1,31 @@ +# DslParameterOption + +Selectable value for a DSL parameter. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**value** | **str** | | +**label** | **str** | | + +## Example + +```python +from octobot_protocol.models.dsl_parameter_option import DslParameterOption + +# TODO update the JSON string below +json = "{}" +# create an instance of DslParameterOption from a JSON string +dsl_parameter_option_instance = DslParameterOption.from_json(json) +# print the JSON string representation of the object +print(DslParameterOption.to_json()) + +# convert the object into a dict +dsl_parameter_option_dict = dsl_parameter_option_instance.to_dict() +# create an instance of DslParameterOption from a dict +dsl_parameter_option_from_dict = DslParameterOption.from_dict(dsl_parameter_option_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/DslValueType.md b/packages/protocol/docs/DslValueType.md new file mode 100644 index 0000000000..d2f973c981 --- /dev/null +++ b/packages/protocol/docs/DslValueType.md @@ -0,0 +1,27 @@ +# DslValueType + +Typed value carried by a DSL keyword parameter. + +## Enum + +* `NUMBER` (value: `'number'`) + +* `TEXT` (value: `'text'`) + +* `SERIES` (value: `'series'`) + +* `SIGNAL` (value: `'signal'`) + +* `DICT` (value: `'dict'`) + +* `BOOLEAN` (value: `'boolean'`) + +* `ORDER` (value: `'order'`) + +* `TIME_FRAME` (value: `'time_frame'`) + +* `ANY` (value: `'any'`) + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/EMAMomentumEvaluatorConfiguration.md b/packages/protocol/docs/EMAMomentumEvaluatorConfiguration.md deleted file mode 100644 index e298018f62..0000000000 --- a/packages/protocol/docs/EMAMomentumEvaluatorConfiguration.md +++ /dev/null @@ -1,33 +0,0 @@ -# EMAMomentumEvaluatorConfiguration - -EMAMomentumEvaluatorConfiguration - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**EvaluatorType**](EvaluatorType.md) | EMAMomentumEvaluator | -**period_length** | **float** | | -**price_threshold_percent** | **float** | | -**reverse_signal** | **bool** | When true, emits a short signal when the current price is bellow the EMA. | [default to False] - -## Example - -```python -from octobot_protocol.models.ema_momentum_evaluator_configuration import EMAMomentumEvaluatorConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of EMAMomentumEvaluatorConfiguration from a JSON string -ema_momentum_evaluator_configuration_instance = EMAMomentumEvaluatorConfiguration.from_json(json) -# print the JSON string representation of the object -print(EMAMomentumEvaluatorConfiguration.to_json()) - -# convert the object into a dict -ema_momentum_evaluator_configuration_dict = ema_momentum_evaluator_configuration_instance.to_dict() -# create an instance of EMAMomentumEvaluatorConfiguration from a dict -ema_momentum_evaluator_configuration_from_dict = EMAMomentumEvaluatorConfiguration.from_dict(ema_momentum_evaluator_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/EvaluatorConfigurationConfiguration.md b/packages/protocol/docs/EvaluatorConfigurationConfiguration.md deleted file mode 100644 index 323fee45eb..0000000000 --- a/packages/protocol/docs/EvaluatorConfigurationConfiguration.md +++ /dev/null @@ -1,35 +0,0 @@ -# EvaluatorConfigurationConfiguration - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**EvaluatorType**](EvaluatorType.md) | EMAMomentumEvaluator | -**period_length** | **float** | | -**short_threshold** | **float** | | -**long_threshold** | **float** | | -**trend_change_identifier** | **bool** | | [optional] [default to False] -**price_threshold_percent** | **float** | | -**reverse_signal** | **bool** | When true, emits a short signal when the current price is bellow the EMA. | [default to False] - -## Example - -```python -from octobot_protocol.models.evaluator_configuration_configuration import EvaluatorConfigurationConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of EvaluatorConfigurationConfiguration from a JSON string -evaluator_configuration_configuration_instance = EvaluatorConfigurationConfiguration.from_json(json) -# print the JSON string representation of the object -print(EvaluatorConfigurationConfiguration.to_json()) - -# convert the object into a dict -evaluator_configuration_configuration_dict = evaluator_configuration_configuration_instance.to_dict() -# create an instance of EvaluatorConfigurationConfiguration from a dict -evaluator_configuration_configuration_from_dict = EvaluatorConfigurationConfiguration.from_dict(evaluator_configuration_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/EvaluatorType.md b/packages/protocol/docs/EvaluatorType.md deleted file mode 100644 index 6496c577a7..0000000000 --- a/packages/protocol/docs/EvaluatorType.md +++ /dev/null @@ -1,13 +0,0 @@ -# EvaluatorType - -EvaluatorType - -## Enum - -* `RSIMOMENTUMEVALUATOR` (value: `'RSIMomentumEvaluator'`) - -* `EMAMOMENTUMEVALUATOR` (value: `'EMAMomentumEvaluator'`) - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/ExchangeAccountsState.md b/packages/protocol/docs/ExchangeAccountsState.md deleted file mode 100644 index e2f8ba9221..0000000000 --- a/packages/protocol/docs/ExchangeAccountsState.md +++ /dev/null @@ -1,31 +0,0 @@ -# ExchangeAccountsState - -ExchangeAccountsState - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**version** | **str** | | -**accounts** | [**List[Account]**](Account.md) | | [optional] - -## Example - -```python -from octobot_protocol.models.exchange_accounts_state import ExchangeAccountsState - -# TODO update the JSON string below -json = "{}" -# create an instance of ExchangeAccountsState from a JSON string -exchange_accounts_state_instance = ExchangeAccountsState.from_json(json) -# print the JSON string representation of the object -print(ExchangeAccountsState.to_json()) - -# convert the object into a dict -exchange_accounts_state_dict = exchange_accounts_state_instance.to_dict() -# create an instance of ExchangeAccountsState from a dict -exchange_accounts_state_from_dict = ExchangeAccountsState.from_dict(exchange_accounts_state_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/GridConfiguration.md b/packages/protocol/docs/GridConfiguration.md deleted file mode 100644 index 85c8d4ea49..0000000000 --- a/packages/protocol/docs/GridConfiguration.md +++ /dev/null @@ -1,38 +0,0 @@ -# GridConfiguration - -GridConfiguration - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**ActionConfigurationType**](ActionConfigurationType.md) | grid | -**symbol** | **str** | | -**spread** | **float** | Price difference between the closest buy and sell orders. Denominated in the quote currency (600 for a 600 USDT spread on BTC/USDT). | -**increment** | **float** | Price difference between two orders of the same side. Denominated in the quote currency (200 for a 200 USDT spread on BTC/USDT). | -**buy_count** | **float** | Number of initial buy orders to create. Make sure to have enough funds to create that many orders. | -**sell_count** | **float** | Number of initial sell orders to create. Make sure to have enough funds to create that many orders. | -**enable_trailing_up** | **bool** | | [default to True] -**enable_trailing_down** | **bool** | | [default to False] -**order_by_order_trailing** | **bool** | | [default to True] - -## Example - -```python -from octobot_protocol.models.grid_configuration import GridConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of GridConfiguration from a JSON string -grid_configuration_instance = GridConfiguration.from_json(json) -# print the JSON string representation of the object -print(GridConfiguration.to_json()) - -# convert the object into a dict -grid_configuration_dict = grid_configuration_instance.to_dict() -# create an instance of GridConfiguration from a dict -grid_configuration_from_dict = GridConfiguration.from_dict(grid_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/IndexCoin.md b/packages/protocol/docs/IndexCoin.md deleted file mode 100644 index 26894da7e7..0000000000 --- a/packages/protocol/docs/IndexCoin.md +++ /dev/null @@ -1,31 +0,0 @@ -# IndexCoin - -IndexCoin - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**name** | **str** | | -**ratio** | **float** | | [default to 1] - -## Example - -```python -from octobot_protocol.models.index_coin import IndexCoin - -# TODO update the JSON string below -json = "{}" -# create an instance of IndexCoin from a JSON string -index_coin_instance = IndexCoin.from_json(json) -# print the JSON string representation of the object -print(IndexCoin.to_json()) - -# convert the object into a dict -index_coin_dict = index_coin_instance.to_dict() -# create an instance of IndexCoin from a dict -index_coin_from_dict = IndexCoin.from_dict(index_coin_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/IndexConfiguration.md b/packages/protocol/docs/IndexConfiguration.md deleted file mode 100644 index ea284ba202..0000000000 --- a/packages/protocol/docs/IndexConfiguration.md +++ /dev/null @@ -1,32 +0,0 @@ -# IndexConfiguration - -IndexConfiguration - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**ActionConfigurationType**](ActionConfigurationType.md) | index | -**coins** | [**List[IndexCoin]**](IndexCoin.md) | | -**rebalance_trigger_min_percent** | **float** | | - -## Example - -```python -from octobot_protocol.models.index_configuration import IndexConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of IndexConfiguration from a JSON string -index_configuration_instance = IndexConfiguration.from_json(json) -# print the JSON string representation of the object -print(IndexConfiguration.to_json()) - -# convert the object into a dict -index_configuration_dict = index_configuration_instance.to_dict() -# create an instance of IndexConfiguration from a dict -index_configuration_from_dict = IndexConfiguration.from_dict(index_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/NodeAccount.md b/packages/protocol/docs/NodeAccount.md deleted file mode 100644 index df3e61c26d..0000000000 --- a/packages/protocol/docs/NodeAccount.md +++ /dev/null @@ -1,39 +0,0 @@ -# NodeAccount - -octobot_node.models.Account (distinct name: NodeAccount) - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**id** | **str** | | -**type** | **str** | | -**name** | **str** | | -**is_simulated** | **bool** | | -**description** | **str** | | [optional] -**created_at** | **datetime** | | -**updated_at** | **datetime** | | [optional] -**exchange_account** | [**ExchangeAccount**](ExchangeAccount.md) | | [optional] -**blockchain_account** | [**BlockchainAccount**](BlockchainAccount.md) | | [optional] -**generic_account** | [**GenericAccount**](GenericAccount.md) | | [optional] - -## Example - -```python -from octobot_protocol.models.node_account import NodeAccount - -# TODO update the JSON string below -json = "{}" -# create an instance of NodeAccount from a JSON string -node_account_instance = NodeAccount.from_json(json) -# print the JSON string representation of the object -print(NodeAccount.to_json()) - -# convert the object into a dict -node_account_dict = node_account_instance.to_dict() -# create an instance of NodeAccount from a dict -node_account_from_dict = NodeAccount.from_dict(node_account_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/PortfolioContent.md b/packages/protocol/docs/PortfolioContent.md deleted file mode 100644 index f6b5bdd595..0000000000 --- a/packages/protocol/docs/PortfolioContent.md +++ /dev/null @@ -1,32 +0,0 @@ -# PortfolioContent - -PortfolioContent - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**total** | **float** | | -**unit** | **str** | | -**assets** | [**List[DetailedAsset]**](DetailedAsset.md) | | - -## Example - -```python -from octobot_protocol.models.portfolio_content import PortfolioContent - -# TODO update the JSON string below -json = "{}" -# create an instance of PortfolioContent from a JSON string -portfolio_content_instance = PortfolioContent.from_json(json) -# print the JSON string representation of the object -print(PortfolioContent.to_json()) - -# convert the object into a dict -portfolio_content_dict = portfolio_content_instance.to_dict() -# create an instance of PortfolioContent from a dict -portfolio_content_from_dict = PortfolioContent.from_dict(portfolio_content_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/RSIMomentumEvaluatorConfiguration.md b/packages/protocol/docs/RSIMomentumEvaluatorConfiguration.md deleted file mode 100644 index 3cb0d85fa1..0000000000 --- a/packages/protocol/docs/RSIMomentumEvaluatorConfiguration.md +++ /dev/null @@ -1,34 +0,0 @@ -# RSIMomentumEvaluatorConfiguration - -RSIMomentumEvaluatorConfiguration - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**EvaluatorType**](EvaluatorType.md) | RSIMomentumEvaluator | -**period_length** | **float** | | -**short_threshold** | **float** | | -**long_threshold** | **float** | | -**trend_change_identifier** | **bool** | | [optional] [default to False] - -## Example - -```python -from octobot_protocol.models.rsi_momentum_evaluator_configuration import RSIMomentumEvaluatorConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of RSIMomentumEvaluatorConfiguration from a JSON string -rsi_momentum_evaluator_configuration_instance = RSIMomentumEvaluatorConfiguration.from_json(json) -# print the JSON string representation of the object -print(RSIMomentumEvaluatorConfiguration.to_json()) - -# convert the object into a dict -rsi_momentum_evaluator_configuration_dict = rsi_momentum_evaluator_configuration_instance.to_dict() -# create an instance of RSIMomentumEvaluatorConfiguration from a dict -rsi_momentum_evaluator_configuration_from_dict = RSIMomentumEvaluatorConfiguration.from_dict(rsi_momentum_evaluator_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/RestartAutomationConfiguration.md b/packages/protocol/docs/RestartAutomationConfiguration.md new file mode 100644 index 0000000000..a012eb4114 --- /dev/null +++ b/packages/protocol/docs/RestartAutomationConfiguration.md @@ -0,0 +1,31 @@ +# RestartAutomationConfiguration + +RestartAutomationConfiguration + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**id** | **str** | | +**action_type** | [**UserActionType**](UserActionType.md) | automation_restart | + +## Example + +```python +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration + +# TODO update the JSON string below +json = "{}" +# create an instance of RestartAutomationConfiguration from a JSON string +restart_automation_configuration_instance = RestartAutomationConfiguration.from_json(json) +# print the JSON string representation of the object +print(RestartAutomationConfiguration.to_json()) + +# convert the object into a dict +restart_automation_configuration_dict = restart_automation_configuration_instance.to_dict() +# create an instance of RestartAutomationConfiguration from a dict +restart_automation_configuration_from_dict = RestartAutomationConfiguration.from_dict(restart_automation_configuration_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/SimpleStrategyEvaluatorConfiguration.md b/packages/protocol/docs/SimpleStrategyEvaluatorConfiguration.md deleted file mode 100644 index 89bdd399ae..0000000000 --- a/packages/protocol/docs/SimpleStrategyEvaluatorConfiguration.md +++ /dev/null @@ -1,30 +0,0 @@ -# SimpleStrategyEvaluatorConfiguration - -SimpleStrategyEvaluatorConfiguration - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**StrategyEvaluatorType**](StrategyEvaluatorType.md) | SimpleStrategyEvaluator | - -## Example - -```python -from octobot_protocol.models.simple_strategy_evaluator_configuration import SimpleStrategyEvaluatorConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of SimpleStrategyEvaluatorConfiguration from a JSON string -simple_strategy_evaluator_configuration_instance = SimpleStrategyEvaluatorConfiguration.from_json(json) -# print the JSON string representation of the object -print(SimpleStrategyEvaluatorConfiguration.to_json()) - -# convert the object into a dict -simple_strategy_evaluator_configuration_dict = simple_strategy_evaluator_configuration_instance.to_dict() -# create an instance of SimpleStrategyEvaluatorConfiguration from a dict -simple_strategy_evaluator_configuration_from_dict = SimpleStrategyEvaluatorConfiguration.from_dict(simple_strategy_evaluator_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/StrategyEvaluatorConfigurationConfiguration.md b/packages/protocol/docs/StrategyEvaluatorConfigurationConfiguration.md deleted file mode 100644 index 688688a814..0000000000 --- a/packages/protocol/docs/StrategyEvaluatorConfigurationConfiguration.md +++ /dev/null @@ -1,30 +0,0 @@ -# StrategyEvaluatorConfigurationConfiguration - - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**StrategyEvaluatorType**](StrategyEvaluatorType.md) | TechnicalAnalysisStrategyEvaluator | -**time_frames_to_weight** | [**List[TimeFrameAndWeight]**](TimeFrameAndWeight.md) | | - -## Example - -```python -from octobot_protocol.models.strategy_evaluator_configuration_configuration import StrategyEvaluatorConfigurationConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of StrategyEvaluatorConfigurationConfiguration from a JSON string -strategy_evaluator_configuration_configuration_instance = StrategyEvaluatorConfigurationConfiguration.from_json(json) -# print the JSON string representation of the object -print(StrategyEvaluatorConfigurationConfiguration.to_json()) - -# convert the object into a dict -strategy_evaluator_configuration_configuration_dict = strategy_evaluator_configuration_configuration_instance.to_dict() -# create an instance of StrategyEvaluatorConfigurationConfiguration from a dict -strategy_evaluator_configuration_configuration_from_dict = StrategyEvaluatorConfigurationConfiguration.from_dict(strategy_evaluator_configuration_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/StrategyEvaluatorType.md b/packages/protocol/docs/StrategyEvaluatorType.md deleted file mode 100644 index 02cc942975..0000000000 --- a/packages/protocol/docs/StrategyEvaluatorType.md +++ /dev/null @@ -1,13 +0,0 @@ -# StrategyEvaluatorType - -StrategyEvaluatorType - -## Enum - -* `SIMPLESTRATEGYEVALUATOR` (value: `'SimpleStrategyEvaluator'`) - -* `TECHNICALANALYSISSTRATEGYEVALUATOR` (value: `'TechnicalAnalysisStrategyEvaluator'`) - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/StrategyType.md b/packages/protocol/docs/StrategyType.md deleted file mode 100644 index a66c0a5ccb..0000000000 --- a/packages/protocol/docs/StrategyType.md +++ /dev/null @@ -1,11 +0,0 @@ -# StrategyType - -StrategyType - -## Enum - -* `SIMPLESTRATEGYEVALUATOR` (value: `'SimpleStrategyEvaluator'`) - -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/TechnicalAnalysisStrategyEvaluatorConfiguration.md b/packages/protocol/docs/TechnicalAnalysisStrategyEvaluatorConfiguration.md deleted file mode 100644 index 74c65f67ee..0000000000 --- a/packages/protocol/docs/TechnicalAnalysisStrategyEvaluatorConfiguration.md +++ /dev/null @@ -1,31 +0,0 @@ -# TechnicalAnalysisStrategyEvaluatorConfiguration - -TechnicalAnalysisStrategyEvaluatorConfiguration - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**configuration_type** | [**StrategyEvaluatorType**](StrategyEvaluatorType.md) | TechnicalAnalysisStrategyEvaluator | -**time_frames_to_weight** | [**List[TimeFrameAndWeight]**](TimeFrameAndWeight.md) | | - -## Example - -```python -from octobot_protocol.models.technical_analysis_strategy_evaluator_configuration import TechnicalAnalysisStrategyEvaluatorConfiguration - -# TODO update the JSON string below -json = "{}" -# create an instance of TechnicalAnalysisStrategyEvaluatorConfiguration from a JSON string -technical_analysis_strategy_evaluator_configuration_instance = TechnicalAnalysisStrategyEvaluatorConfiguration.from_json(json) -# print the JSON string representation of the object -print(TechnicalAnalysisStrategyEvaluatorConfiguration.to_json()) - -# convert the object into a dict -technical_analysis_strategy_evaluator_configuration_dict = technical_analysis_strategy_evaluator_configuration_instance.to_dict() -# create an instance of TechnicalAnalysisStrategyEvaluatorConfiguration from a dict -technical_analysis_strategy_evaluator_configuration_from_dict = TechnicalAnalysisStrategyEvaluatorConfiguration.from_dict(technical_analysis_strategy_evaluator_configuration_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/TimeFrameAndWeight.md b/packages/protocol/docs/TimeFrameAndWeight.md deleted file mode 100644 index 7f1dc2d486..0000000000 --- a/packages/protocol/docs/TimeFrameAndWeight.md +++ /dev/null @@ -1,31 +0,0 @@ -# TimeFrameAndWeight - -TimeFrameAndWeight - -## Properties - -Name | Type | Description | Notes ------------- | ------------- | ------------- | ------------- -**time_frame** | [**TimeFrame**](TimeFrame.md) | | -**weight** | **float** | | - -## Example - -```python -from octobot_protocol.models.time_frame_and_weight import TimeFrameAndWeight - -# TODO update the JSON string below -json = "{}" -# create an instance of TimeFrameAndWeight from a JSON string -time_frame_and_weight_instance = TimeFrameAndWeight.from_json(json) -# print the JSON string representation of the object -print(TimeFrameAndWeight.to_json()) - -# convert the object into a dict -time_frame_and_weight_dict = time_frame_and_weight_instance.to_dict() -# create an instance of TimeFrameAndWeight from a dict -time_frame_and_weight_from_dict = TimeFrameAndWeight.from_dict(time_frame_and_weight_dict) -``` -[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) - - diff --git a/packages/protocol/docs/TradedPairVolume.md b/packages/protocol/docs/TradedPairVolume.md new file mode 100644 index 0000000000..9f7d801b94 --- /dev/null +++ b/packages/protocol/docs/TradedPairVolume.md @@ -0,0 +1,31 @@ +# TradedPairVolume + +Optional 24h volume for a traded pair. Empty object when volume was not requested. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**base_volume** | **float** | 24h base-currency volume. | [optional] +**quote_volume** | **float** | 24h quote-currency volume. | [optional] + +## Example + +```python +from octobot_protocol.models.traded_pair_volume import TradedPairVolume + +# TODO update the JSON string below +json = "{}" +# create an instance of TradedPairVolume from a JSON string +traded_pair_volume_instance = TradedPairVolume.from_json(json) +# print the JSON string representation of the object +print(TradedPairVolume.to_json()) + +# convert the object into a dict +traded_pair_volume_dict = traded_pair_volume_instance.to_dict() +# create an instance of TradedPairVolume from a dict +traded_pair_volume_from_dict = TradedPairVolume.from_dict(traded_pair_volume_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/TradedPairsByExchange.md b/packages/protocol/docs/TradedPairsByExchange.md new file mode 100644 index 0000000000..76bc82b3a1 --- /dev/null +++ b/packages/protocol/docs/TradedPairsByExchange.md @@ -0,0 +1,29 @@ +# TradedPairsByExchange + +Map of exchange internal name to traded pairs (and optional volumes). + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- + +## Example + +```python +from octobot_protocol.models.traded_pairs_by_exchange import TradedPairsByExchange + +# TODO update the JSON string below +json = "{}" +# create an instance of TradedPairsByExchange from a JSON string +traded_pairs_by_exchange_instance = TradedPairsByExchange.from_json(json) +# print the JSON string representation of the object +print(TradedPairsByExchange.to_json()) + +# convert the object into a dict +traded_pairs_by_exchange_dict = traded_pairs_by_exchange_instance.to_dict() +# create an instance of TradedPairsByExchange from a dict +traded_pairs_by_exchange_from_dict = TradedPairsByExchange.from_dict(traded_pairs_by_exchange_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/TradedPairsForExchange.md b/packages/protocol/docs/TradedPairsForExchange.md new file mode 100644 index 0000000000..026ded2fe7 --- /dev/null +++ b/packages/protocol/docs/TradedPairsForExchange.md @@ -0,0 +1,29 @@ +# TradedPairsForExchange + +Map of trading pair symbol to optional volume fields. + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- + +## Example + +```python +from octobot_protocol.models.traded_pairs_for_exchange import TradedPairsForExchange + +# TODO update the JSON string below +json = "{}" +# create an instance of TradedPairsForExchange from a JSON string +traded_pairs_for_exchange_instance = TradedPairsForExchange.from_json(json) +# print the JSON string representation of the object +print(TradedPairsForExchange.to_json()) + +# convert the object into a dict +traded_pairs_for_exchange_dict = traded_pairs_for_exchange_instance.to_dict() +# create an instance of TradedPairsForExchange from a dict +traded_pairs_for_exchange_from_dict = TradedPairsForExchange.from_dict(traded_pairs_for_exchange_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/UserActionType.md b/packages/protocol/docs/UserActionType.md index 2d57fb2d1f..8d10bdec95 100644 --- a/packages/protocol/docs/UserActionType.md +++ b/packages/protocol/docs/UserActionType.md @@ -10,6 +10,8 @@ UserActionType * `AUTOMATION_STOP` (value: `'automation_stop'`) +* `AUTOMATION_RESTART` (value: `'automation_restart'`) + * `AUTOMATION_SIGNAL` (value: `'automation_signal'`) * `ACCOUNT_CREATE` (value: `'account_create'`) diff --git a/packages/protocol/octobot_protocol/models/__init__.py b/packages/protocol/octobot_protocol/models/__init__.py index 225894506f..6e387e173a 100644 --- a/packages/protocol/octobot_protocol/models/__init__.py +++ b/packages/protocol/octobot_protocol/models/__init__.py @@ -44,6 +44,7 @@ from octobot_protocol.models.blockchain_account import BlockchainAccount from octobot_protocol.models.cancel_policy import CancelPolicy from octobot_protocol.models.cancel_policy_type import CancelPolicyType +from octobot_protocol.models.child_octo_bot_process_state import ChildOctoBotProcessState from octobot_protocol.models.copied_account import CopiedAccount from octobot_protocol.models.copied_asset import CopiedAsset from octobot_protocol.models.copy_configuration import CopyConfiguration @@ -60,6 +61,13 @@ from octobot_protocol.models.delete_strategy_configuration import DeleteStrategyConfiguration from octobot_protocol.models.detailed_asset import DetailedAsset from octobot_protocol.models.detailed_assets_for_trading_type import DetailedAssetsForTradingType +from octobot_protocol.models.dsl_keyword import DslKeyword +from octobot_protocol.models.dsl_keyword_category import DslKeywordCategory +from octobot_protocol.models.dsl_keywords_state import DslKeywordsState +from octobot_protocol.models.dsl_parameter import DslParameter +from octobot_protocol.models.dsl_parameter_default_value import DslParameterDefaultValue +from octobot_protocol.models.dsl_parameter_option import DslParameterOption +from octobot_protocol.models.dsl_value_type import DslValueType from octobot_protocol.models.edit_account_auth_configuration import EditAccountAuthConfiguration from octobot_protocol.models.edit_account_configuration import EditAccountConfiguration from octobot_protocol.models.edit_automation_configuration import EditAutomationConfiguration @@ -97,6 +105,7 @@ from octobot_protocol.models.position_status import PositionStatus from octobot_protocol.models.position_summary import PositionSummary from octobot_protocol.models.refresh_accounts_configuration import RefreshAccountsConfiguration +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration from octobot_protocol.models.side import Side from octobot_protocol.models.signal_automation_configuration import SignalAutomationConfiguration from octobot_protocol.models.signal_automation_configuration_signal_payload import SignalAutomationConfigurationSignalPayload @@ -111,6 +120,9 @@ from octobot_protocol.models.time_frame import TimeFrame from octobot_protocol.models.trade import Trade from octobot_protocol.models.trade_summary import TradeSummary +from octobot_protocol.models.traded_pair_volume import TradedPairVolume +from octobot_protocol.models.traded_pairs_by_exchange import TradedPairsByExchange +from octobot_protocol.models.traded_pairs_for_exchange import TradedPairsForExchange from octobot_protocol.models.trading_tentacles_configuration import TradingTentaclesConfiguration from octobot_protocol.models.trading_type import TradingType from octobot_protocol.models.trailing_profile import TrailingProfile diff --git a/packages/protocol/octobot_protocol/models/automation_configuration.py b/packages/protocol/octobot_protocol/models/automation_configuration.py index 16745eea47..33872bb613 100644 --- a/packages/protocol/octobot_protocol/models/automation_configuration.py +++ b/packages/protocol/octobot_protocol/models/automation_configuration.py @@ -18,7 +18,7 @@ import json from datetime import datetime -from pydantic import BaseModel, ConfigDict, StrictStr +from pydantic import BaseModel, ConfigDict, Field, StrictStr from typing import Any, ClassVar, Dict, List, Optional from octobot_protocol.models.account_reference import AccountReference from octobot_protocol.models.strategy_reference import StrategyReference @@ -36,7 +36,7 @@ class AutomationConfiguration(BaseModel): created_at: datetime updated_at: Optional[datetime] = None strategy: StrategyReference - accounts: List[AccountReference] + accounts: List[AccountReference] = Field(description="May be empty only for automations whose strategy configuration is generic_process; otherwise exactly one account reference is required at runtime.") __properties: ClassVar[List[str]] = ["id", "name", "description", "created_at", "updated_at", "strategy", "accounts"] model_config = ConfigDict( diff --git a/packages/protocol/octobot_protocol/models/automation_state.py b/packages/protocol/octobot_protocol/models/automation_state.py index b35944b776..042d010ffb 100644 --- a/packages/protocol/octobot_protocol/models/automation_state.py +++ b/packages/protocol/octobot_protocol/models/automation_state.py @@ -21,6 +21,7 @@ from typing import Any, ClassVar, Dict, List, Optional from octobot_protocol.models.action import Action from octobot_protocol.models.automation_metadata import AutomationMetadata +from octobot_protocol.models.child_octo_bot_process_state import ChildOctoBotProcessState from octobot_protocol.models.detailed_assets_for_trading_type import DetailedAssetsForTradingType from octobot_protocol.models.order_summary import OrderSummary from octobot_protocol.models.position_summary import PositionSummary @@ -47,7 +48,8 @@ class AutomationState(BaseModel): orders: Optional[List[OrderSummary]] = None trades: Optional[List[TradeSummary]] = None positions: Optional[List[PositionSummary]] = None - __properties: ClassVar[List[str]] = ["id", "status", "error", "error_message", "metadata", "actions", "priority_actions", "exchanges", "exchange_account_ids", "assets", "orders", "trades", "positions"] + child_octobot_process: Optional[ChildOctoBotProcessState] = None + __properties: ClassVar[List[str]] = ["id", "status", "error", "error_message", "metadata", "actions", "priority_actions", "exchanges", "exchange_account_ids", "assets", "orders", "trades", "positions", "child_octobot_process"] model_config = ConfigDict( validate_by_name=True, @@ -133,6 +135,9 @@ def to_dict(self) -> Dict[str, Any]: if _item_positions: _items.append(_item_positions.to_dict()) _dict['positions'] = _items + # override the default output from pydantic by calling `to_dict()` of child_octobot_process + if self.child_octobot_process: + _dict['child_octobot_process'] = self.child_octobot_process.to_dict() return _dict @classmethod @@ -157,7 +162,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "assets": [DetailedAssetsForTradingType.from_dict(_item) for _item in obj["assets"]] if obj.get("assets") is not None else None, "orders": [OrderSummary.from_dict(_item) for _item in obj["orders"]] if obj.get("orders") is not None else None, "trades": [TradeSummary.from_dict(_item) for _item in obj["trades"]] if obj.get("trades") is not None else None, - "positions": [PositionSummary.from_dict(_item) for _item in obj["positions"]] if obj.get("positions") is not None else None + "positions": [PositionSummary.from_dict(_item) for _item in obj["positions"]] if obj.get("positions") is not None else None, + "child_octobot_process": ChildOctoBotProcessState.from_dict(obj["child_octobot_process"]) if obj.get("child_octobot_process") is not None else None }) return _obj diff --git a/packages/protocol/octobot_protocol/models/child_octo_bot_process_state.py b/packages/protocol/octobot_protocol/models/child_octo_bot_process_state.py new file mode 100644 index 0000000000..6f2e936366 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/child_octo_bot_process_state.py @@ -0,0 +1,92 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, StrictBool, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class ChildOctoBotProcessState(BaseModel): + """ + ChildOctoBotProcessState + """ # noqa: E501 + http_base_url: StrictStr + web_port: StrictInt + init_state_ok: StrictBool + __properties: ClassVar[List[str]] = ["http_base_url", "web_port", "init_state_ok"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of ChildOctoBotProcessState from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of ChildOctoBotProcessState from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "http_base_url": obj.get("http_base_url"), + "web_port": obj.get("web_port"), + "init_state_ok": obj.get("init_state_ok") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/dsl_keyword.py b/packages/protocol/octobot_protocol/models/dsl_keyword.py new file mode 100644 index 0000000000..c52cd7a99c --- /dev/null +++ b/packages/protocol/octobot_protocol/models/dsl_keyword.py @@ -0,0 +1,128 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictStr +from typing import Any, ClassVar, Dict, List +from typing_extensions import Annotated +from octobot_protocol.models.dsl_keyword_category import DslKeywordCategory +from octobot_protocol.models.dsl_parameter import DslParameter +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class DslKeyword(BaseModel): + """ + Definition of one DSL keyword (signature and configuration). + """ # noqa: E501 + name: Annotated[str, Field(min_length=1, strict=True)] = Field(description="Stable DSL keyword id.") + category: DslKeywordCategory + label: StrictStr + description: StrictStr + inputs: List[DslParameter] + outputs: List[DslParameter] + additional_properties: Dict[str, Any] = {} + __properties: ClassVar[List[str]] = ["name", "category", "label", "description", "inputs", "outputs"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of DslKeyword from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + * Fields in `self.additional_properties` are added to the output dict. + """ + excluded_fields: Set[str] = set([ + "additional_properties", + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in inputs (list) + _items = [] + if self.inputs: + for _item_inputs in self.inputs: + if _item_inputs: + _items.append(_item_inputs.to_dict()) + _dict['inputs'] = _items + # override the default output from pydantic by calling `to_dict()` of each item in outputs (list) + _items = [] + if self.outputs: + for _item_outputs in self.outputs: + if _item_outputs: + _items.append(_item_outputs.to_dict()) + _dict['outputs'] = _items + # puts key-value pairs in additional_properties in the top level + if self.additional_properties is not None: + for _key, _value in self.additional_properties.items(): + _dict[_key] = _value + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of DslKeyword from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "name": obj.get("name"), + "category": obj.get("category"), + "label": obj.get("label"), + "description": obj.get("description"), + "inputs": [DslParameter.from_dict(_item) for _item in obj["inputs"]] if obj.get("inputs") is not None else None, + "outputs": [DslParameter.from_dict(_item) for _item in obj["outputs"]] if obj.get("outputs") is not None else None + }) + # store additional fields in additional_properties + for _key in obj.keys(): + if _key not in cls.__properties: + _obj.additional_properties[_key] = obj.get(_key) + + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/dsl_keyword_category.py b/packages/protocol/octobot_protocol/models/dsl_keyword_category.py new file mode 100644 index 0000000000..2856892fff --- /dev/null +++ b/packages/protocol/octobot_protocol/models/dsl_keyword_category.py @@ -0,0 +1,40 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import json +from enum import Enum +from typing_extensions import Self + + +class DslKeywordCategory(str, Enum): + """ + Composition role of a DSL keyword. + """ + + """ + allowed enum values + """ + TRIGGER = 'trigger' + SOURCE = 'source' + CONDITION = 'condition' + LOGIC = 'logic' + ACTION = 'action' + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of DslKeywordCategory from a JSON string""" + return cls(json.loads(json_str)) + + diff --git a/packages/protocol/octobot_protocol/models/dsl_keywords_state.py b/packages/protocol/octobot_protocol/models/dsl_keywords_state.py new file mode 100644 index 0000000000..bcc983d5b4 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/dsl_keywords_state.py @@ -0,0 +1,111 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictStr +from typing import Any, ClassVar, Dict, List +from octobot_protocol.models.dsl_keyword import DslKeyword +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class DslKeywordsState(BaseModel): + """ + Versioned list of DSL keywords available on a node. + """ # noqa: E501 + version: StrictStr + keywords: List[DslKeyword] = Field(description="Available keywords in intended display order.") + additional_properties: Dict[str, Any] = {} + __properties: ClassVar[List[str]] = ["version", "keywords"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of DslKeywordsState from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + * Fields in `self.additional_properties` are added to the output dict. + """ + excluded_fields: Set[str] = set([ + "additional_properties", + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of each item in keywords (list) + _items = [] + if self.keywords: + for _item_keywords in self.keywords: + if _item_keywords: + _items.append(_item_keywords.to_dict()) + _dict['keywords'] = _items + # puts key-value pairs in additional_properties in the top level + if self.additional_properties is not None: + for _key, _value in self.additional_properties.items(): + _dict[_key] = _value + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of DslKeywordsState from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "version": obj.get("version"), + "keywords": [DslKeyword.from_dict(_item) for _item in obj["keywords"]] if obj.get("keywords") is not None else None + }) + # store additional fields in additional_properties + for _key in obj.keys(): + if _key not in cls.__properties: + _obj.additional_properties[_key] = obj.get(_key) + + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/dsl_parameter.py b/packages/protocol/octobot_protocol/models/dsl_parameter.py new file mode 100644 index 0000000000..7bf9349665 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/dsl_parameter.py @@ -0,0 +1,137 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictFloat, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List, Optional, Union +from typing_extensions import Annotated +from octobot_protocol.models.dsl_parameter_default_value import DslParameterDefaultValue +from octobot_protocol.models.dsl_parameter_option import DslParameterOption +from octobot_protocol.models.dsl_value_type import DslValueType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class DslParameter(BaseModel): + """ + Configurable parameter on a DSL keyword. + """ # noqa: E501 + name: Annotated[str, Field(min_length=1, strict=True)] = Field(description="Unique within this keyword.") + label: StrictStr + value_type: DslValueType + description: Optional[StrictStr] = None + default_value: Optional[DslParameterDefaultValue] = None + minimum: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Only for parameter_type=number.") + maximum: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Only for parameter_type=number.") + step: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="Only for parameter_type=number.") + options: Optional[List[DslParameterOption]] = Field(default=None, description="Allowed choices for value_type=text or value_type=time_frame.") + required: Optional[StrictBool] = None + multiple: Optional[StrictBool] = Field(default=None, description="Accepts more than one value of this parameter.") + primary: Optional[StrictBool] = Field(default=None, description="Prefer in compact editors.") + additional_properties: Dict[str, Any] = {} + __properties: ClassVar[List[str]] = ["name", "label", "value_type", "description", "default_value", "minimum", "maximum", "step", "options", "required", "multiple", "primary"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of DslParameter from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + * Fields in `self.additional_properties` are added to the output dict. + """ + excluded_fields: Set[str] = set([ + "additional_properties", + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # override the default output from pydantic by calling `to_dict()` of default_value + if self.default_value: + _dict['default_value'] = self.default_value.to_dict() + # override the default output from pydantic by calling `to_dict()` of each item in options (list) + _items = [] + if self.options: + for _item_options in self.options: + if _item_options: + _items.append(_item_options.to_dict()) + _dict['options'] = _items + # puts key-value pairs in additional_properties in the top level + if self.additional_properties is not None: + for _key, _value in self.additional_properties.items(): + _dict[_key] = _value + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of DslParameter from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "name": obj.get("name"), + "label": obj.get("label"), + "value_type": obj.get("value_type"), + "description": obj.get("description"), + "default_value": DslParameterDefaultValue.from_dict(obj["default_value"]) if obj.get("default_value") is not None else None, + "minimum": obj.get("minimum"), + "maximum": obj.get("maximum"), + "step": obj.get("step"), + "options": [DslParameterOption.from_dict(_item) for _item in obj["options"]] if obj.get("options") is not None else None, + "required": obj.get("required"), + "multiple": obj.get("multiple"), + "primary": obj.get("primary") + }) + # store additional fields in additional_properties + for _key in obj.keys(): + if _key not in cls.__properties: + _obj.additional_properties[_key] = obj.get(_key) + + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/dsl_parameter_default_value.py b/packages/protocol/octobot_protocol/models/dsl_parameter_default_value.py new file mode 100644 index 0000000000..213618f110 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/dsl_parameter_default_value.py @@ -0,0 +1,160 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import json +import pprint +from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictFloat, StrictInt, StrictStr, ValidationError, field_validator +from typing import Any, List, Optional, Union +from pydantic import StrictStr, Field +from typing import Union, List, Set, Optional, Dict +from typing_extensions import Literal, Self + +DSLPARAMETERDEFAULTVALUE_ONE_OF_SCHEMAS = ["bool", "float", "str"] + +class DslParameterDefaultValue(BaseModel): + """ + Must match parameter_type; for select/time_frame must be one of options[].value. + """ + # data type: str + oneof_schema_1_validator: Optional[StrictStr] = None + # data type: float + oneof_schema_2_validator: Optional[Union[StrictFloat, StrictInt]] = None + # data type: bool + oneof_schema_3_validator: Optional[StrictBool] = None + actual_instance: Optional[Union[bool, float, str]] = None + one_of_schemas: Set[str] = { "bool", "float", "str" } + + model_config = ConfigDict( + validate_assignment=True, + protected_namespaces=(), + ) + + + def __init__(self, *args, **kwargs) -> None: + if args: + if len(args) > 1: + raise ValueError("If a position argument is used, only 1 is allowed to set `actual_instance`") + if kwargs: + raise ValueError("If a position argument is used, keyword arguments cannot be used.") + super().__init__(actual_instance=args[0]) + else: + super().__init__(**kwargs) + + @field_validator('actual_instance') + def actual_instance_must_validate_oneof(cls, v): + instance = DslParameterDefaultValue.model_construct() + error_messages = [] + match = 0 + # validate data type: str + try: + instance.oneof_schema_1_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: float + try: + instance.oneof_schema_2_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # validate data type: bool + try: + instance.oneof_schema_3_validator = v + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + if match > 1: + # more than 1 match + raise ValueError("Multiple matches found when setting `actual_instance` in DslParameterDefaultValue with oneOf schemas: bool, float, str. Details: " + ", ".join(error_messages)) + elif match == 0: + # no match + raise ValueError("No match found when setting `actual_instance` in DslParameterDefaultValue with oneOf schemas: bool, float, str. Details: " + ", ".join(error_messages)) + else: + return v + + @classmethod + def from_dict(cls, obj: Union[str, Dict[str, Any]]) -> Self: + return cls.from_json(json.dumps(obj)) + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Returns the object represented by the json string""" + instance = cls.model_construct() + error_messages = [] + match = 0 + + # deserialize data into str + try: + # validation + instance.oneof_schema_1_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_1_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into float + try: + # validation + instance.oneof_schema_2_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_2_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into bool + try: + # validation + instance.oneof_schema_3_validator = json.loads(json_str) + # assign value to actual_instance + instance.actual_instance = instance.oneof_schema_3_validator + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + + if match > 1: + # more than 1 match + raise ValueError("Multiple matches found when deserializing the JSON string into DslParameterDefaultValue with oneOf schemas: bool, float, str. Details: " + ", ".join(error_messages)) + elif match == 0: + # no match + raise ValueError("No match found when deserializing the JSON string into DslParameterDefaultValue with oneOf schemas: bool, float, str. Details: " + ", ".join(error_messages)) + else: + return instance + + def to_json(self) -> str: + """Returns the JSON representation of the actual instance""" + if self.actual_instance is None: + return "null" + + if hasattr(self.actual_instance, "to_json") and callable(self.actual_instance.to_json): + return self.actual_instance.to_json() + else: + return json.dumps(self.actual_instance) + + def to_dict(self) -> Optional[Union[Dict[str, Any], bool, float, str]]: + """Returns the dict representation of the actual instance""" + if self.actual_instance is None: + return None + + if hasattr(self.actual_instance, "to_dict") and callable(self.actual_instance.to_dict): + return self.actual_instance.to_dict() + else: + # primitive type + return self.actual_instance + + def to_str(self) -> str: + """Returns the string representation of the actual instance""" + return pprint.pformat(self.model_dump()) + + diff --git a/packages/protocol/octobot_protocol/models/dsl_parameter_option.py b/packages/protocol/octobot_protocol/models/dsl_parameter_option.py new file mode 100644 index 0000000000..4c6c632e48 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/dsl_parameter_option.py @@ -0,0 +1,90 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, StrictStr +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class DslParameterOption(BaseModel): + """ + Selectable value for a DSL parameter. + """ # noqa: E501 + value: StrictStr + label: StrictStr + __properties: ClassVar[List[str]] = ["value", "label"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of DslParameterOption from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of DslParameterOption from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "value": obj.get("value"), + "label": obj.get("label") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/dsl_value_type.py b/packages/protocol/octobot_protocol/models/dsl_value_type.py new file mode 100644 index 0000000000..bf1922a1ea --- /dev/null +++ b/packages/protocol/octobot_protocol/models/dsl_value_type.py @@ -0,0 +1,44 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import json +from enum import Enum +from typing_extensions import Self + + +class DslValueType(str, Enum): + """ + Typed value carried by a DSL keyword parameter. + """ + + """ + allowed enum values + """ + NUMBER = 'number' + TEXT = 'text' + SERIES = 'series' + SIGNAL = 'signal' + DICT = 'dict' + BOOLEAN = 'boolean' + ORDER = 'order' + TIME_FRAME = 'time_frame' + ANY = 'any' + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of DslValueType from a JSON string""" + return cls(json.loads(json_str)) + + diff --git a/packages/protocol/octobot_protocol/models/restart_automation_configuration.py b/packages/protocol/octobot_protocol/models/restart_automation_configuration.py new file mode 100644 index 0000000000..344156f194 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/restart_automation_configuration.py @@ -0,0 +1,91 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictStr +from typing import Any, ClassVar, Dict, List +from octobot_protocol.models.user_action_type import UserActionType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class RestartAutomationConfiguration(BaseModel): + """ + RestartAutomationConfiguration + """ # noqa: E501 + id: StrictStr + action_type: UserActionType = Field(description="automation_restart") + __properties: ClassVar[List[str]] = ["id", "action_type"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of RestartAutomationConfiguration from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of RestartAutomationConfiguration from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "id": obj.get("id"), + "action_type": obj.get("action_type") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/traded_pair_volume.py b/packages/protocol/octobot_protocol/models/traded_pair_volume.py new file mode 100644 index 0000000000..09985fff02 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/traded_pair_volume.py @@ -0,0 +1,113 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt +from typing import Any, ClassVar, Dict, List, Optional, Union +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class TradedPairVolume(BaseModel): + """ + Optional 24h volume for a traded pair. Empty object when volume was not requested. + """ # noqa: E501 + base_volume: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="24h base-currency volume.", alias="baseVolume") + quote_volume: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="24h quote-currency volume.", alias="quoteVolume") + additional_properties: Dict[str, Any] = {} + __properties: ClassVar[List[str]] = ["baseVolume", "quoteVolume"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of TradedPairVolume from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + * Fields in `self.additional_properties` are added to the output dict. + """ + excluded_fields: Set[str] = set([ + "additional_properties", + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # puts key-value pairs in additional_properties in the top level + if self.additional_properties is not None: + for _key, _value in self.additional_properties.items(): + _dict[_key] = _value + + # set to None if base_volume (nullable) is None + # and model_fields_set contains the field + if self.base_volume is None and "base_volume" in self.model_fields_set: + _dict['baseVolume'] = None + + # set to None if quote_volume (nullable) is None + # and model_fields_set contains the field + if self.quote_volume is None and "quote_volume" in self.model_fields_set: + _dict['quoteVolume'] = None + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of TradedPairVolume from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "baseVolume": obj.get("baseVolume"), + "quoteVolume": obj.get("quoteVolume") + }) + # store additional fields in additional_properties + for _key in obj.keys(): + if _key not in cls.__properties: + _obj.additional_properties[_key] = obj.get(_key) + + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/traded_pairs_by_exchange.py b/packages/protocol/octobot_protocol/models/traded_pairs_by_exchange.py new file mode 100644 index 0000000000..1f4904fc37 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/traded_pairs_by_exchange.py @@ -0,0 +1,99 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class TradedPairsByExchange(BaseModel): + """ + Map of exchange internal name to traded pairs (and optional volumes). + """ # noqa: E501 + additional_properties: Dict[str, Any] = {} + __properties: ClassVar[List[str]] = [] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of TradedPairsByExchange from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + * Fields in `self.additional_properties` are added to the output dict. + """ + excluded_fields: Set[str] = set([ + "additional_properties", + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # puts key-value pairs in additional_properties in the top level + if self.additional_properties is not None: + for _key, _value in self.additional_properties.items(): + _dict[_key] = _value + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of TradedPairsByExchange from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + }) + # store additional fields in additional_properties + for _key in obj.keys(): + if _key not in cls.__properties: + _obj.additional_properties[_key] = obj.get(_key) + + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/traded_pairs_for_exchange.py b/packages/protocol/octobot_protocol/models/traded_pairs_for_exchange.py new file mode 100644 index 0000000000..dea8038d29 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/traded_pairs_for_exchange.py @@ -0,0 +1,99 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict +from typing import Any, ClassVar, Dict, List +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class TradedPairsForExchange(BaseModel): + """ + Map of trading pair symbol to optional volume fields. + """ # noqa: E501 + additional_properties: Dict[str, Any] = {} + __properties: ClassVar[List[str]] = [] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of TradedPairsForExchange from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + * Fields in `self.additional_properties` are added to the output dict. + """ + excluded_fields: Set[str] = set([ + "additional_properties", + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + # puts key-value pairs in additional_properties in the top level + if self.additional_properties is not None: + for _key, _value in self.additional_properties.items(): + _dict[_key] = _value + + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of TradedPairsForExchange from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + }) + # store additional fields in additional_properties + for _key in obj.keys(): + if _key not in cls.__properties: + _obj.additional_properties[_key] = obj.get(_key) + + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/user_action_configuration.py b/packages/protocol/octobot_protocol/models/user_action_configuration.py index b5a431f73e..f4f6866acc 100644 --- a/packages/protocol/octobot_protocol/models/user_action_configuration.py +++ b/packages/protocol/octobot_protocol/models/user_action_configuration.py @@ -32,13 +32,14 @@ from octobot_protocol.models.edit_exchange_config_configuration import EditExchangeConfigConfiguration from octobot_protocol.models.edit_strategy_configuration import EditStrategyConfiguration from octobot_protocol.models.refresh_accounts_configuration import RefreshAccountsConfiguration +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration from octobot_protocol.models.signal_automation_configuration import SignalAutomationConfiguration from octobot_protocol.models.stop_automation_configuration import StopAutomationConfiguration from pydantic import StrictStr, Field from typing import Union, List, Set, Optional, Dict from typing_extensions import Literal, Self -USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration"] +USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration"] class UserActionConfiguration(BaseModel): """ @@ -50,36 +51,38 @@ class UserActionConfiguration(BaseModel): oneof_schema_2_validator: Optional[EditAutomationConfiguration] = None # data type: StopAutomationConfiguration oneof_schema_3_validator: Optional[StopAutomationConfiguration] = None + # data type: RestartAutomationConfiguration + oneof_schema_4_validator: Optional[RestartAutomationConfiguration] = None # data type: SignalAutomationConfiguration - oneof_schema_4_validator: Optional[SignalAutomationConfiguration] = None + oneof_schema_5_validator: Optional[SignalAutomationConfiguration] = None # data type: CreateAccountConfiguration - oneof_schema_5_validator: Optional[CreateAccountConfiguration] = None + oneof_schema_6_validator: Optional[CreateAccountConfiguration] = None # data type: EditAccountConfiguration - oneof_schema_6_validator: Optional[EditAccountConfiguration] = None + oneof_schema_7_validator: Optional[EditAccountConfiguration] = None # data type: DeleteAccountConfiguration - oneof_schema_7_validator: Optional[DeleteAccountConfiguration] = None + oneof_schema_8_validator: Optional[DeleteAccountConfiguration] = None # data type: CreateExchangeConfigConfiguration - oneof_schema_8_validator: Optional[CreateExchangeConfigConfiguration] = None + oneof_schema_9_validator: Optional[CreateExchangeConfigConfiguration] = None # data type: EditExchangeConfigConfiguration - oneof_schema_9_validator: Optional[EditExchangeConfigConfiguration] = None + oneof_schema_10_validator: Optional[EditExchangeConfigConfiguration] = None # data type: DeleteExchangeConfigConfiguration - oneof_schema_10_validator: Optional[DeleteExchangeConfigConfiguration] = None + oneof_schema_11_validator: Optional[DeleteExchangeConfigConfiguration] = None # data type: RefreshAccountsConfiguration - oneof_schema_11_validator: Optional[RefreshAccountsConfiguration] = None + oneof_schema_12_validator: Optional[RefreshAccountsConfiguration] = None # data type: CreateStrategyConfiguration - oneof_schema_12_validator: Optional[CreateStrategyConfiguration] = None + oneof_schema_13_validator: Optional[CreateStrategyConfiguration] = None # data type: EditStrategyConfiguration - oneof_schema_13_validator: Optional[EditStrategyConfiguration] = None + oneof_schema_14_validator: Optional[EditStrategyConfiguration] = None # data type: DeleteStrategyConfiguration - oneof_schema_14_validator: Optional[DeleteStrategyConfiguration] = None + oneof_schema_15_validator: Optional[DeleteStrategyConfiguration] = None # data type: CreateAccountAuthConfiguration - oneof_schema_15_validator: Optional[CreateAccountAuthConfiguration] = None + oneof_schema_16_validator: Optional[CreateAccountAuthConfiguration] = None # data type: EditAccountAuthConfiguration - oneof_schema_16_validator: Optional[EditAccountAuthConfiguration] = None + oneof_schema_17_validator: Optional[EditAccountAuthConfiguration] = None # data type: DeleteAccountAuthConfiguration - oneof_schema_17_validator: Optional[DeleteAccountAuthConfiguration] = None - actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]] = None - one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration" } + oneof_schema_18_validator: Optional[DeleteAccountAuthConfiguration] = None + actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]] = None + one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration" } model_config = ConfigDict( validate_assignment=True, @@ -120,6 +123,11 @@ def actual_instance_must_validate_oneof(cls, v): error_messages.append(f"Error! Input type `{type(v)}` is not `StopAutomationConfiguration`") else: match += 1 + # validate data type: RestartAutomationConfiguration + if not isinstance(v, RestartAutomationConfiguration): + error_messages.append(f"Error! Input type `{type(v)}` is not `RestartAutomationConfiguration`") + else: + match += 1 # validate data type: SignalAutomationConfiguration if not isinstance(v, SignalAutomationConfiguration): error_messages.append(f"Error! Input type `{type(v)}` is not `SignalAutomationConfiguration`") @@ -192,10 +200,10 @@ def actual_instance_must_validate_oneof(cls, v): match += 1 if match > 1: # more than 1 match - raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) else: return v @@ -260,6 +268,11 @@ def from_json(cls, json_str: str) -> Self: instance.actual_instance = EditAutomationConfiguration.from_json(json_str) return instance + # check if data type is `RestartAutomationConfiguration` + if _data_type == "automation_restart": + instance.actual_instance = RestartAutomationConfiguration.from_json(json_str) + return instance + # check if data type is `SignalAutomationConfiguration` if _data_type == "automation_signal": instance.actual_instance = SignalAutomationConfiguration.from_json(json_str) @@ -318,6 +331,12 @@ def from_json(cls, json_str: str) -> Self: match += 1 except (ValidationError, ValueError) as e: error_messages.append(str(e)) + # deserialize data into RestartAutomationConfiguration + try: + instance.actual_instance = RestartAutomationConfiguration.from_json(json_str) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) # deserialize data into SignalAutomationConfiguration try: instance.actual_instance = SignalAutomationConfiguration.from_json(json_str) @@ -405,10 +424,10 @@ def from_json(cls, json_str: str) -> Self: if match > 1: # more than 1 match - raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) else: return instance @@ -422,7 +441,7 @@ def to_json(self) -> str: else: return json.dumps(self.actual_instance) - def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]]: + def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]]: """Returns the dict representation of the actual instance""" if self.actual_instance is None: return None diff --git a/packages/protocol/octobot_protocol/models/user_action_type.py b/packages/protocol/octobot_protocol/models/user_action_type.py index fbe31eec41..8836a794d0 100644 --- a/packages/protocol/octobot_protocol/models/user_action_type.py +++ b/packages/protocol/octobot_protocol/models/user_action_type.py @@ -29,6 +29,7 @@ class UserActionType(str, Enum): AUTOMATION_CREATE = 'automation_create' AUTOMATION_EDIT = 'automation_edit' AUTOMATION_STOP = 'automation_stop' + AUTOMATION_RESTART = 'automation_restart' AUTOMATION_SIGNAL = 'automation_signal' ACCOUNT_CREATE = 'account_create' ACCOUNT_EDIT = 'account_edit' diff --git a/packages/protocol/octobot_protocol_ts/models/AutomationConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/AutomationConfiguration.ts index 854ac5ed6d..37bf493432 100644 --- a/packages/protocol/octobot_protocol_ts/models/AutomationConfiguration.ts +++ b/packages/protocol/octobot_protocol_ts/models/AutomationConfiguration.ts @@ -23,6 +23,9 @@ export class AutomationConfiguration { 'created_at': string; 'updated_at'?: string; 'strategy': StrategyReference; + /** + * May be empty only for automations whose strategy configuration is generic_process; otherwise exactly one account reference is required at runtime. + */ 'accounts': Array; static readonly discriminator: string | undefined = undefined; diff --git a/packages/protocol/octobot_protocol_ts/models/AutomationState.ts b/packages/protocol/octobot_protocol_ts/models/AutomationState.ts index d25670bfc8..9f7ac084d6 100644 --- a/packages/protocol/octobot_protocol_ts/models/AutomationState.ts +++ b/packages/protocol/octobot_protocol_ts/models/AutomationState.ts @@ -12,6 +12,7 @@ import { Action } from '../models/Action'; import { AutomationMetadata } from '../models/AutomationMetadata'; +import { ChildOctoBotProcessState } from '../models/ChildOctoBotProcessState'; import { DetailedAssetsForTradingType } from '../models/DetailedAssetsForTradingType'; import { OrderSummary } from '../models/OrderSummary'; import { PositionSummary } from '../models/PositionSummary'; @@ -35,6 +36,7 @@ export class AutomationState { 'orders'?: Array; 'trades'?: Array; 'positions'?: Array; + 'child_octobot_process'?: ChildOctoBotProcessState; static readonly discriminator: string | undefined = undefined; @@ -118,6 +120,12 @@ export class AutomationState { "baseName": "positions", "type": "Array", "format": "" + }, + { + "name": "child_octobot_process", + "baseName": "child_octobot_process", + "type": "ChildOctoBotProcessState", + "format": "" } ]; static getAttributeTypeMap() { diff --git a/packages/protocol/octobot_protocol_ts/models/ChildOctoBotProcessState.ts b/packages/protocol/octobot_protocol_ts/models/ChildOctoBotProcessState.ts new file mode 100644 index 0000000000..b6af62f0bb --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/ChildOctoBotProcessState.ts @@ -0,0 +1,52 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* ChildOctoBotProcessState +*/ +export class ChildOctoBotProcessState { + 'http_base_url': string; + 'web_port': number; + 'init_state_ok': boolean; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "http_base_url", + "baseName": "http_base_url", + "type": "string", + "format": "" + }, + { + "name": "web_port", + "baseName": "web_port", + "type": "number", + "format": "" + }, + { + "name": "init_state_ok", + "baseName": "init_state_ok", + "type": "boolean", + "format": "" + } ]; + + static getAttributeTypeMap() { + return ChildOctoBotProcessState.attributeTypeMap; + } + + public constructor() { + } +} diff --git a/packages/protocol/octobot_protocol_ts/models/DslKeyword.ts b/packages/protocol/octobot_protocol_ts/models/DslKeyword.ts new file mode 100644 index 0000000000..b1016c140f --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/DslKeyword.ts @@ -0,0 +1,80 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { DslKeywordCategory } from '../models/DslKeywordCategory'; +import { DslParameter } from '../models/DslParameter'; + +/** +* Definition of one DSL keyword (signature and configuration). +*/ +export class DslKeyword { + /** + * Stable DSL keyword id. + */ + 'name': string; + 'category': DslKeywordCategory; + 'label': string; + 'description': string; + 'inputs': Array; + 'outputs': Array; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "name", + "baseName": "name", + "type": "string", + "format": "" + }, + { + "name": "category", + "baseName": "category", + "type": "DslKeywordCategory", + "format": "" + }, + { + "name": "label", + "baseName": "label", + "type": "string", + "format": "" + }, + { + "name": "description", + "baseName": "description", + "type": "string", + "format": "" + }, + { + "name": "inputs", + "baseName": "inputs", + "type": "Array", + "format": "" + }, + { + "name": "outputs", + "baseName": "outputs", + "type": "Array", + "format": "" + } ]; + + static getAttributeTypeMap() { + return DslKeyword.attributeTypeMap; + } + + public constructor() { + } +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/DslKeywordCategory.ts b/packages/protocol/octobot_protocol_ts/models/DslKeywordCategory.ts new file mode 100644 index 0000000000..624104a6b6 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/DslKeywordCategory.ts @@ -0,0 +1,17 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* Composition role of a DSL keyword. +*/ +export type DslKeywordCategory = 'trigger' | 'source' | 'condition' | 'logic' | 'action' diff --git a/packages/protocol/octobot_protocol_ts/models/DslKeywordsState.ts b/packages/protocol/octobot_protocol_ts/models/DslKeywordsState.ts new file mode 100644 index 0000000000..f3c5a213df --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/DslKeywordsState.ts @@ -0,0 +1,49 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { DslKeyword } from '../models/DslKeyword'; + +/** +* Versioned list of DSL keywords available on a node. +*/ +export class DslKeywordsState { + 'version': string; + /** + * Available keywords in intended display order. + */ + 'keywords': Array; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "version", + "baseName": "version", + "type": "string", + "format": "" + }, + { + "name": "keywords", + "baseName": "keywords", + "type": "Array", + "format": "" + } ]; + + static getAttributeTypeMap() { + return DslKeywordsState.attributeTypeMap; + } + + public constructor() { + } +} diff --git a/packages/protocol/octobot_protocol_ts/models/DslParameter.ts b/packages/protocol/octobot_protocol_ts/models/DslParameter.ts new file mode 100644 index 0000000000..a4103289f9 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/DslParameter.ts @@ -0,0 +1,141 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { DslParameterDefaultValue } from '../models/DslParameterDefaultValue'; +import { DslParameterOption } from '../models/DslParameterOption'; +import { DslValueType } from '../models/DslValueType'; + +/** +* Configurable parameter on a DSL keyword. +*/ +export class DslParameter { + /** + * Unique within this keyword. + */ + 'name': string; + 'label': string; + 'value_type': DslValueType; + 'description'?: string; + 'default_value'?: DslParameterDefaultValue; + /** + * Only for parameter_type=number. + */ + 'minimum'?: number; + /** + * Only for parameter_type=number. + */ + 'maximum'?: number; + /** + * Only for parameter_type=number. + */ + 'step'?: number; + /** + * Allowed choices for value_type=text or value_type=time_frame. + */ + 'options'?: Array; + 'required'?: boolean; + /** + * Accepts more than one value of this parameter. + */ + 'multiple'?: boolean; + /** + * Prefer in compact editors. + */ + 'primary'?: boolean; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "name", + "baseName": "name", + "type": "string", + "format": "" + }, + { + "name": "label", + "baseName": "label", + "type": "string", + "format": "" + }, + { + "name": "value_type", + "baseName": "value_type", + "type": "DslValueType", + "format": "" + }, + { + "name": "description", + "baseName": "description", + "type": "string", + "format": "" + }, + { + "name": "default_value", + "baseName": "default_value", + "type": "DslParameterDefaultValue", + "format": "" + }, + { + "name": "minimum", + "baseName": "minimum", + "type": "number", + "format": "" + }, + { + "name": "maximum", + "baseName": "maximum", + "type": "number", + "format": "" + }, + { + "name": "step", + "baseName": "step", + "type": "number", + "format": "" + }, + { + "name": "options", + "baseName": "options", + "type": "Array", + "format": "" + }, + { + "name": "required", + "baseName": "required", + "type": "boolean", + "format": "" + }, + { + "name": "multiple", + "baseName": "multiple", + "type": "boolean", + "format": "" + }, + { + "name": "primary", + "baseName": "primary", + "type": "boolean", + "format": "" + } ]; + + static getAttributeTypeMap() { + return DslParameter.attributeTypeMap; + } + + public constructor() { + } +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/DslParameterDefaultValue.ts b/packages/protocol/octobot_protocol_ts/models/DslParameterDefaultValue.ts new file mode 100644 index 0000000000..7f991e8318 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/DslParameterDefaultValue.ts @@ -0,0 +1,35 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* Must match parameter_type; for select/time_frame must be one of options[].value. +*/ +/** + * @type DslParameterDefaultValue + * Type + * @export + */ +export type DslParameterDefaultValue = boolean | number | string; + +/** +* @type DslParameterDefaultValueClass + * Must match parameter_type; for select/time_frame must be one of options[].value. +* @export +*/ +export class DslParameterDefaultValueClass { + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/DslParameterOption.ts b/packages/protocol/octobot_protocol_ts/models/DslParameterOption.ts new file mode 100644 index 0000000000..5a89b1c223 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/DslParameterOption.ts @@ -0,0 +1,45 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* Selectable value for a DSL parameter. +*/ +export class DslParameterOption { + 'value': string; + 'label': string; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "value", + "baseName": "value", + "type": "string", + "format": "" + }, + { + "name": "label", + "baseName": "label", + "type": "string", + "format": "" + } ]; + + static getAttributeTypeMap() { + return DslParameterOption.attributeTypeMap; + } + + public constructor() { + } +} diff --git a/packages/protocol/octobot_protocol_ts/models/DslValueType.ts b/packages/protocol/octobot_protocol_ts/models/DslValueType.ts new file mode 100644 index 0000000000..f2f2880ef6 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/DslValueType.ts @@ -0,0 +1,17 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* Typed value carried by a DSL keyword parameter. +*/ +export type DslValueType = 'number' | 'text' | 'series' | 'signal' | 'dict' | 'boolean' | 'order' | 'time_frame' | 'any' diff --git a/packages/protocol/octobot_protocol_ts/models/RestartAutomationConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/RestartAutomationConfiguration.ts new file mode 100644 index 0000000000..1e69d83d92 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/RestartAutomationConfiguration.ts @@ -0,0 +1,51 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { UserActionType } from '../models/UserActionType'; + +/** +* RestartAutomationConfiguration +*/ +export class RestartAutomationConfiguration { + 'id': string; + /** + * automation_restart + */ + 'action_type': 'automation_restart'; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "id", + "baseName": "id", + "type": "string", + "format": "" + }, + { + "name": "action_type", + "baseName": "action_type", + "type": "UserActionType", + "format": "" + } ]; + + static getAttributeTypeMap() { + return RestartAutomationConfiguration.attributeTypeMap; + } + + public constructor() { + } +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/TradedPairVolume.ts b/packages/protocol/octobot_protocol_ts/models/TradedPairVolume.ts new file mode 100644 index 0000000000..c5c0d2e8d1 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/TradedPairVolume.ts @@ -0,0 +1,51 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* Optional 24h volume for a traded pair. Empty object when volume was not requested. +*/ +export class TradedPairVolume { + /** + * 24h base-currency volume. + */ + 'baseVolume'?: number | null; + /** + * 24h quote-currency volume. + */ + 'quoteVolume'?: number | null; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "baseVolume", + "baseName": "baseVolume", + "type": "number", + "format": "" + }, + { + "name": "quoteVolume", + "baseName": "quoteVolume", + "type": "number", + "format": "" + } ]; + + static getAttributeTypeMap() { + return TradedPairVolume.attributeTypeMap; + } + + public constructor() { + } +} diff --git a/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts index 0422574164..2bc1cdbf1e 100644 --- a/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts +++ b/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts @@ -25,6 +25,7 @@ import { EditAutomationConfiguration } from '../models/EditAutomationConfigurati import { EditExchangeConfigConfiguration } from '../models/EditExchangeConfigConfiguration'; import { EditStrategyConfiguration } from '../models/EditStrategyConfiguration'; import { RefreshAccountsConfiguration } from '../models/RefreshAccountsConfiguration'; +import { RestartAutomationConfiguration } from '../models/RestartAutomationConfiguration'; import { SignalAutomationConfiguration } from '../models/SignalAutomationConfiguration'; import { StopAutomationConfiguration } from '../models/StopAutomationConfiguration'; @@ -33,7 +34,7 @@ import { StopAutomationConfiguration } from '../models/StopAutomationConfigurati * Type * @export */ -export type UserActionConfiguration = CreateAccountAuthConfiguration | CreateAccountConfiguration | CreateAutomationConfiguration | CreateExchangeConfigConfiguration | CreateStrategyConfiguration | DeleteAccountAuthConfiguration | DeleteAccountConfiguration | DeleteExchangeConfigConfiguration | DeleteStrategyConfiguration | EditAccountAuthConfiguration | EditAccountConfiguration | EditAutomationConfiguration | EditExchangeConfigConfiguration | EditStrategyConfiguration | RefreshAccountsConfiguration | SignalAutomationConfiguration | StopAutomationConfiguration; +export type UserActionConfiguration = CreateAccountAuthConfiguration | CreateAccountConfiguration | CreateAutomationConfiguration | CreateExchangeConfigConfiguration | CreateStrategyConfiguration | DeleteAccountAuthConfiguration | DeleteAccountConfiguration | DeleteExchangeConfigConfiguration | DeleteStrategyConfiguration | EditAccountAuthConfiguration | EditAccountConfiguration | EditAutomationConfiguration | EditExchangeConfigConfiguration | EditStrategyConfiguration | RefreshAccountsConfiguration | RestartAutomationConfiguration | SignalAutomationConfiguration | StopAutomationConfiguration; /** * @type UserActionConfigurationClass @@ -52,6 +53,7 @@ export class UserActionConfigurationClass { "accounts_refresh": "RefreshAccountsConfiguration", "automation_create": "CreateAutomationConfiguration", "automation_edit": "EditAutomationConfiguration", + "automation_restart": "RestartAutomationConfiguration", "automation_signal": "SignalAutomationConfiguration", "automation_stop": "StopAutomationConfiguration", "exchange_config_create": "CreateExchangeConfigConfiguration", @@ -78,3 +80,4 @@ export class UserActionConfigurationClass { + diff --git a/packages/protocol/octobot_protocol_ts/models/UserActionType.ts b/packages/protocol/octobot_protocol_ts/models/UserActionType.ts index 910a785bfe..d87710ca2f 100644 --- a/packages/protocol/octobot_protocol_ts/models/UserActionType.ts +++ b/packages/protocol/octobot_protocol_ts/models/UserActionType.ts @@ -14,4 +14,4 @@ /** * UserActionType */ -export type UserActionType = 'automation_create' | 'automation_edit' | 'automation_stop' | 'automation_signal' | 'account_create' | 'account_edit' | 'account_delete' | 'accounts_refresh' | 'exchange_config_create' | 'exchange_config_edit' | 'exchange_config_delete' | 'strategy_create' | 'strategy_edit' | 'strategy_delete' | 'account_auth_create' | 'account_auth_edit' | 'account_auth_delete' +export type UserActionType = 'automation_create' | 'automation_edit' | 'automation_stop' | 'automation_restart' | 'automation_signal' | 'account_create' | 'account_edit' | 'account_delete' | 'accounts_refresh' | 'exchange_config_create' | 'exchange_config_edit' | 'exchange_config_delete' | 'strategy_create' | 'strategy_edit' | 'strategy_delete' | 'account_auth_create' | 'account_auth_edit' | 'account_auth_delete' diff --git a/packages/protocol/octobot_protocol_ts/models/index.ts b/packages/protocol/octobot_protocol_ts/models/index.ts index db1897ead0..0a55b96c7f 100644 --- a/packages/protocol/octobot_protocol_ts/models/index.ts +++ b/packages/protocol/octobot_protocol_ts/models/index.ts @@ -29,6 +29,7 @@ export * from "./AutomationState"; export * from "./BlockchainAccount"; export * from "./CancelPolicy"; export * from "./CancelPolicyType"; +export * from "./ChildOctoBotProcessState"; export * from "./CopiedAccount"; export * from "./CopiedAsset"; export * from "./CopyConfiguration"; @@ -45,6 +46,13 @@ export * from "./DeleteExchangeConfigConfiguration"; export * from "./DeleteStrategyConfiguration"; export * from "./DetailedAsset"; export * from "./DetailedAssetsForTradingType"; +export * from "./DslKeyword"; +export * from "./DslKeywordCategory"; +export * from "./DslKeywordsState"; +export * from "./DslParameter"; +export * from "./DslParameterDefaultValue"; +export * from "./DslParameterOption"; +export * from "./DslValueType"; export * from "./EditAccountAuthConfiguration"; export * from "./EditAccountConfiguration"; export * from "./EditAutomationConfiguration"; @@ -82,6 +90,7 @@ export * from "./Position"; export * from "./PositionStatus"; export * from "./PositionSummary"; export * from "./RefreshAccountsConfiguration"; +export * from "./RestartAutomationConfiguration"; export * from "./Side"; export * from "./SignalAutomationConfiguration"; export * from "./SignalAutomationConfigurationSignalPayload"; @@ -96,6 +105,7 @@ export * from "./StrategyReference"; export * from "./TimeFrame"; export * from "./Trade"; export * from "./TradeSummary"; +export * from "./TradedPairVolume"; export * from "./TradingTentaclesConfiguration"; export * from "./TradingType"; export * from "./TrailingProfile"; diff --git a/packages/protocol/octobot_protocol_ts/package.json b/packages/protocol/octobot_protocol_ts/package.json index a03f247eb8..4c518c1107 100644 --- a/packages/protocol/octobot_protocol_ts/package.json +++ b/packages/protocol/octobot_protocol_ts/package.json @@ -1,6 +1,6 @@ { "name": "@drakkar.software/octobot-protocol", - "version": "0.5.0", + "version": "0.6.0", "description": "TypeScript types generated from OctoBot OpenAPI schema", "repository": { "type": "git", diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index 41b3451ab9..61cd8bc5df 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -504,6 +504,26 @@ } } }, + "ChildOctoBotProcessState": { + "description": "ChildOctoBotProcessState", + "type": "object", + "required": [ + "http_base_url", + "web_port", + "init_state_ok" + ], + "properties": { + "http_base_url": { + "type": "string" + }, + "web_port": { + "type": "integer" + }, + "init_state_ok": { + "type": "boolean" + } + } + }, "AutomationState": { "description": "AutomationState", "type": "object", @@ -575,6 +595,9 @@ "items": { "$ref": "#/components/schemas/PositionSummary" } + }, + "child_octobot_process": { + "$ref": "#/components/schemas/ChildOctoBotProcessState" } } }, @@ -1632,6 +1655,7 @@ }, "accounts": { "type": "array", + "description": "May be empty only for automations whose strategy configuration is generic_process; otherwise exactly one account reference is required at runtime.", "items": { "$ref": "#/components/schemas/AccountReference" } @@ -1693,6 +1717,23 @@ } } }, + "RestartAutomationConfiguration": { + "description": "RestartAutomationConfiguration", + "type": "object", + "required": [ + "id", + "action_type" + ], + "properties": { + "id": { + "type": "string" + }, + "action_type": { + "$ref": "#/components/schemas/UserActionType", + "description": "automation_restart" + } + } + }, "AutomationSignalType": { "description": "AutomationSignalType", "type": "string", @@ -2015,6 +2056,7 @@ "automation_create", "automation_edit", "automation_stop", + "automation_restart", "automation_signal", "account_create", "account_edit", @@ -2228,6 +2270,9 @@ { "$ref": "#/components/schemas/StopAutomationConfiguration" }, + { + "$ref": "#/components/schemas/RestartAutomationConfiguration" + }, { "$ref": "#/components/schemas/SignalAutomationConfiguration" }, @@ -2277,6 +2322,7 @@ "automation_create": "#/components/schemas/CreateAutomationConfiguration", "automation_edit": "#/components/schemas/EditAutomationConfiguration", "automation_stop": "#/components/schemas/StopAutomationConfiguration", + "automation_restart": "#/components/schemas/RestartAutomationConfiguration", "automation_signal": "#/components/schemas/SignalAutomationConfiguration", "account_create": "#/components/schemas/CreateAccountConfiguration", "account_edit": "#/components/schemas/EditAccountConfiguration", @@ -2473,6 +2519,210 @@ "$ref": "#/components/schemas/Debug" } } + }, + "DslKeywordCategory": { + "description": "Composition role of a DSL keyword.", + "type": "string", + "enum": [ + "trigger", + "source", + "condition", + "logic", + "action" + ] + }, + "DslValueType": { + "description": "Typed value carried by a DSL keyword parameter.", + "type": "string", + "enum": [ + "number", + "text", + "series", + "signal", + "dict", + "boolean", + "order", + "time_frame", + "any" + ] + }, + "DslParameterOption": { + "description": "Selectable value for a DSL parameter.", + "type": "object", + "required": [ + "value", + "label" + ], + "additionalProperties": false, + "properties": { + "value": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "DslParameter": { + "description": "Configurable parameter on a DSL keyword.", + "type": "object", + "required": [ + "name", + "label", + "value_type" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Unique within this keyword." + }, + "label": { + "type": "string" + }, + "value_type": { + "$ref": "#/components/schemas/DslValueType" + }, + "description": { + "type": "string" + }, + "default_value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + } + ], + "description": "Must match parameter_type; for select/time_frame must be one of options[].value." + }, + "minimum": { + "type": "number", + "description": "Only for parameter_type=number." + }, + "maximum": { + "type": "number", + "description": "Only for parameter_type=number." + }, + "step": { + "type": "number", + "description": "Only for parameter_type=number." + }, + "options": { + "type": "array", + "description": "Allowed choices for value_type=text or value_type=time_frame.", + "items": { + "$ref": "#/components/schemas/DslParameterOption" + } + }, + "required": { + "type": "boolean" + }, + "multiple": { + "type": "boolean", + "description": "Accepts more than one value of this parameter." + }, + "primary": { + "type": "boolean", + "description": "Prefer in compact editors." + } + } + }, + "DslKeyword": { + "description": "Definition of one DSL keyword (signature and configuration).", + "type": "object", + "required": [ + "name", + "category", + "label", + "description", + "inputs", + "outputs" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Stable DSL keyword id." + }, + "category": { + "$ref": "#/components/schemas/DslKeywordCategory" + }, + "label": { + "type": "string" + }, + "description": { + "type": "string" + }, + "inputs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DslParameter" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DslParameter" + } + } + } + }, + "DslKeywordsState": { + "description": "Versioned list of DSL keywords available on a node.", + "type": "object", + "required": [ + "version", + "keywords" + ], + "additionalProperties": false, + "properties": { + "version": { + "type": "string" + }, + "keywords": { + "type": "array", + "description": "Available keywords in intended display order.", + "items": { + "$ref": "#/components/schemas/DslKeyword" + } + } + } + }, + "TradedPairVolume": { + "description": "Optional 24h volume for a traded pair. Empty object when volume was not requested.", + "type": "object", + "additionalProperties": false, + "properties": { + "baseVolume": { + "description": "24h base-currency volume.", + "type": ["number", "null"] + }, + "quoteVolume": { + "description": "24h quote-currency volume.", + "type": ["number", "null"] + } + } + }, + "TradedPairsForExchange": { + "description": "Map of trading pair symbol to optional volume fields.", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/TradedPairVolume" + } + }, + "TradedPairsByExchange": { + "description": "Map of exchange internal name to traded pairs (and optional volumes).", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/TradedPairsForExchange" + } } } } diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 02ce9525c1..97e447b7d2 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -8,7 +8,7 @@ "test:postprocess": "node --test test/postprocess-typescript.test.mjs", "test:postprocess:codegen": "node --test test/postprocess-typescript.codegen.test.mjs", "test": "npm run test:postprocess && npm run test:postprocess:codegen && npm run test:models", - "generate:python": "node scripts/clean-protocol-codegen-output.mjs python && openapi-generator-cli generate -i openapi.json -g python -o . -t openapi_generator_templates/python --global-property models,modelTests=true,supportingFiles=__init__.py --additional-properties packageName=octobot_protocol,useOneOfDiscriminatorLookup=true && npm run test:models", + "generate:python": "node scripts/clean-protocol-codegen-output.mjs python && openapi-generator-cli generate -i openapi.json -g python -o . -t openapi_generator_templates/python --global-property models,modelTests=true,supportingFiles=__init__.py --additional-properties packageName=octobot_protocol,useOneOfDiscriminatorLookup=true,generateAliasAsModel=true && npm run test:models", "gen": "npm run generate:python", "generate:typescript": "node scripts/clean-protocol-codegen-output.mjs typescript && openapi-generator-cli generate -i openapi.json -g typescript -o octobot_protocol_ts --global-property models --additional-properties modelPropertyNaming=original,stringEnums=true,supportsES6=true,useOneOfDiscriminatorLookup=true && node scripts/postprocess-protocol-typescript.mjs", "generate:rust": "node scripts/clean-protocol-codegen-output.mjs rust && openapi-generator-cli generate -i openapi.json -g rust -o octobot_protocol_rs --global-property models --additional-properties packageName=octobot_protocol_rs,useOneOfDiscriminatorLookup=true", diff --git a/packages/protocol/scripts/clean-protocol-codegen-output.mjs b/packages/protocol/scripts/clean-protocol-codegen-output.mjs index f38caab686..b72a826827 100644 --- a/packages/protocol/scripts/clean-protocol-codegen-output.mjs +++ b/packages/protocol/scripts/clean-protocol-codegen-output.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node /** * Removes generated artifacts under octobot_protocol / octobot_protocol_ts / - * octobot_protocol_rs / test (Python model tests) + * octobot_protocol_rs / test (Python model tests) / docs * before openapi-generator runs so stale files do not linger. */ @@ -29,13 +29,17 @@ const targets = { keep: new Set(), keepPredicate: (name) => name.endsWith(".mjs"), }, + docs: { + dir: path.join(protocolDir, "docs"), + keep: new Set(), + }, }; function cleanTarget(key) { const config = targets[key]; if (!config) { console.error( - "Usage: clean-protocol-codegen-output.mjs ", + "Usage: clean-protocol-codegen-output.mjs ", ); process.exit(1); } @@ -57,9 +61,11 @@ if (mode === "all") { cleanTarget("typescript"); cleanTarget("rust"); cleanTarget("test"); + cleanTarget("docs"); } else if (mode === "python") { cleanTarget("python"); cleanTarget("test"); + cleanTarget("docs"); } else { cleanTarget(mode); } diff --git a/packages/protocol/test/test_automation_state.py b/packages/protocol/test/test_automation_state.py index 3760a82abc..63320c7d2a 100644 --- a/packages/protocol/test/test_automation_state.py +++ b/packages/protocol/test/test_automation_state.py @@ -96,7 +96,11 @@ def make_instance(self, include_optional) -> AutomationState: octobot_protocol.models.position_summary.PositionSummary( id = '', symbol = '', ) - ] + ], + child_octobot_process = octobot_protocol.models.child_octo_bot_process_state.ChildOctoBotProcessState( + http_base_url = '', + web_port = 56, + init_state_ok = True, ) ) else: return AutomationState( diff --git a/packages/protocol/test/test_child_octo_bot_process_state.py b/packages/protocol/test/test_child_octo_bot_process_state.py new file mode 100644 index 0000000000..b3ea15a507 --- /dev/null +++ b/packages/protocol/test/test_child_octo_bot_process_state.py @@ -0,0 +1,56 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.child_octo_bot_process_state import ChildOctoBotProcessState + +class TestChildOctoBotProcessState(unittest.TestCase): + """ChildOctoBotProcessState unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> ChildOctoBotProcessState: + """Test ChildOctoBotProcessState + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `ChildOctoBotProcessState` + """ + model = ChildOctoBotProcessState() + if include_optional: + return ChildOctoBotProcessState( + http_base_url = '', + web_port = 56, + init_state_ok = True + ) + else: + return ChildOctoBotProcessState( + http_base_url = '', + web_port = 56, + init_state_ok = True, + ) + """ + + def testChildOctoBotProcessState(self): + """Test ChildOctoBotProcessState""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_debug.py b/packages/protocol/test/test_debug.py index 920b15001f..714cd0887c 100644 --- a/packages/protocol/test/test_debug.py +++ b/packages/protocol/test/test_debug.py @@ -97,7 +97,11 @@ def make_instance(self, include_optional) -> Debug: octobot_protocol.models.position_summary.PositionSummary( id = '', symbol = '', ) - ], ) + ], + child_octobot_process = octobot_protocol.models.child_octo_bot_process_state.ChildOctoBotProcessState( + http_base_url = '', + web_port = 56, + init_state_ok = True, ), ) ], user_actions = [ octobot_protocol.models.user_action.UserAction( @@ -297,7 +301,11 @@ def make_instance(self, include_optional) -> Debug: octobot_protocol.models.position_summary.PositionSummary( id = '', symbol = '', ) - ], ) + ], + child_octobot_process = octobot_protocol.models.child_octo_bot_process_state.ChildOctoBotProcessState( + http_base_url = '', + web_port = 56, + init_state_ok = True, ), ) ], user_actions = [ octobot_protocol.models.user_action.UserAction( diff --git a/packages/protocol/test/test_debug_state.py b/packages/protocol/test/test_debug_state.py index 865fded888..7e9a2381b2 100644 --- a/packages/protocol/test/test_debug_state.py +++ b/packages/protocol/test/test_debug_state.py @@ -99,7 +99,11 @@ def make_instance(self, include_optional) -> DebugState: octobot_protocol.models.position_summary.PositionSummary( id = '', symbol = '', ) - ], ) + ], + child_octobot_process = octobot_protocol.models.child_octo_bot_process_state.ChildOctoBotProcessState( + http_base_url = '', + web_port = 56, + init_state_ok = True, ), ) ], user_actions = [ octobot_protocol.models.user_action.UserAction( diff --git a/packages/protocol/test/test_dsl_keyword.py b/packages/protocol/test/test_dsl_keyword.py new file mode 100644 index 0000000000..1452726a9e --- /dev/null +++ b/packages/protocol/test/test_dsl_keyword.py @@ -0,0 +1,134 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.dsl_keyword import DslKeyword + +class TestDslKeyword(unittest.TestCase): + """DslKeyword unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> DslKeyword: + """Test DslKeyword + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `DslKeyword` + """ + model = DslKeyword() + if include_optional: + return DslKeyword( + name = '0', + category = 'trigger', + label = '', + description = '', + inputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + default_value = null, + minimum = 1.337, + maximum = 1.337, + step = 1.337, + options = [ + octobot_protocol.models.dsl_parameter_option.DslParameterOption( + value = '', + label = '', ) + ], + required = True, + multiple = True, + primary = True, ) + ], + outputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + default_value = null, + minimum = 1.337, + maximum = 1.337, + step = 1.337, + options = [ + octobot_protocol.models.dsl_parameter_option.DslParameterOption( + value = '', + label = '', ) + ], + required = True, + multiple = True, + primary = True, ) + ] + ) + else: + return DslKeyword( + name = '0', + category = 'trigger', + label = '', + description = '', + inputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + default_value = null, + minimum = 1.337, + maximum = 1.337, + step = 1.337, + options = [ + octobot_protocol.models.dsl_parameter_option.DslParameterOption( + value = '', + label = '', ) + ], + required = True, + multiple = True, + primary = True, ) + ], + outputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + default_value = null, + minimum = 1.337, + maximum = 1.337, + step = 1.337, + options = [ + octobot_protocol.models.dsl_parameter_option.DslParameterOption( + value = '', + label = '', ) + ], + required = True, + multiple = True, + primary = True, ) + ], + ) + """ + + def testDslKeyword(self): + """Test DslKeyword""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_dsl_keyword_category.py b/packages/protocol/test/test_dsl_keyword_category.py new file mode 100644 index 0000000000..1b83029d45 --- /dev/null +++ b/packages/protocol/test/test_dsl_keyword_category.py @@ -0,0 +1,33 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.dsl_keyword_category import DslKeywordCategory + +class TestDslKeywordCategory(unittest.TestCase): + """DslKeywordCategory unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def testDslKeywordCategory(self): + """Test DslKeywordCategory""" + # inst = DslKeywordCategory() + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_dsl_keywords_state.py b/packages/protocol/test/test_dsl_keywords_state.py new file mode 100644 index 0000000000..41630fbb1a --- /dev/null +++ b/packages/protocol/test/test_dsl_keywords_state.py @@ -0,0 +1,130 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.dsl_keywords_state import DslKeywordsState + +class TestDslKeywordsState(unittest.TestCase): + """DslKeywordsState unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> DslKeywordsState: + """Test DslKeywordsState + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `DslKeywordsState` + """ + model = DslKeywordsState() + if include_optional: + return DslKeywordsState( + version = '', + keywords = [ + octobot_protocol.models.dsl_keyword.DslKeyword( + name = '0', + category = 'trigger', + label = '', + description = '', + inputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + default_value = null, + minimum = 1.337, + maximum = 1.337, + step = 1.337, + options = [ + octobot_protocol.models.dsl_parameter_option.DslParameterOption( + value = '', + label = '', ) + ], + required = True, + multiple = True, + primary = True, ) + ], + outputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + minimum = 1.337, + maximum = 1.337, + step = 1.337, + required = True, + multiple = True, + primary = True, ) + ], ) + ] + ) + else: + return DslKeywordsState( + version = '', + keywords = [ + octobot_protocol.models.dsl_keyword.DslKeyword( + name = '0', + category = 'trigger', + label = '', + description = '', + inputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + default_value = null, + minimum = 1.337, + maximum = 1.337, + step = 1.337, + options = [ + octobot_protocol.models.dsl_parameter_option.DslParameterOption( + value = '', + label = '', ) + ], + required = True, + multiple = True, + primary = True, ) + ], + outputs = [ + octobot_protocol.models.dsl_parameter.DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + minimum = 1.337, + maximum = 1.337, + step = 1.337, + required = True, + multiple = True, + primary = True, ) + ], ) + ], + ) + """ + + def testDslKeywordsState(self): + """Test DslKeywordsState""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_dsl_parameter.py b/packages/protocol/test/test_dsl_parameter.py new file mode 100644 index 0000000000..c916e70054 --- /dev/null +++ b/packages/protocol/test/test_dsl_parameter.py @@ -0,0 +1,69 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.dsl_parameter import DslParameter + +class TestDslParameter(unittest.TestCase): + """DslParameter unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> DslParameter: + """Test DslParameter + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `DslParameter` + """ + model = DslParameter() + if include_optional: + return DslParameter( + name = '0', + label = '', + value_type = 'number', + description = '', + default_value = None, + minimum = 1.337, + maximum = 1.337, + step = 1.337, + options = [ + octobot_protocol.models.dsl_parameter_option.DslParameterOption( + value = '', + label = '', ) + ], + required = True, + multiple = True, + primary = True + ) + else: + return DslParameter( + name = '0', + label = '', + value_type = 'number', + ) + """ + + def testDslParameter(self): + """Test DslParameter""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_dsl_parameter_default_value.py b/packages/protocol/test/test_dsl_parameter_default_value.py new file mode 100644 index 0000000000..566455d944 --- /dev/null +++ b/packages/protocol/test/test_dsl_parameter_default_value.py @@ -0,0 +1,50 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.dsl_parameter_default_value import DslParameterDefaultValue + +class TestDslParameterDefaultValue(unittest.TestCase): + """DslParameterDefaultValue unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> DslParameterDefaultValue: + """Test DslParameterDefaultValue + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `DslParameterDefaultValue` + """ + model = DslParameterDefaultValue() + if include_optional: + return DslParameterDefaultValue( + ) + else: + return DslParameterDefaultValue( + ) + """ + + def testDslParameterDefaultValue(self): + """Test DslParameterDefaultValue""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_dsl_parameter_option.py b/packages/protocol/test/test_dsl_parameter_option.py new file mode 100644 index 0000000000..8246e89155 --- /dev/null +++ b/packages/protocol/test/test_dsl_parameter_option.py @@ -0,0 +1,54 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.dsl_parameter_option import DslParameterOption + +class TestDslParameterOption(unittest.TestCase): + """DslParameterOption unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> DslParameterOption: + """Test DslParameterOption + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `DslParameterOption` + """ + model = DslParameterOption() + if include_optional: + return DslParameterOption( + value = '', + label = '' + ) + else: + return DslParameterOption( + value = '', + label = '', + ) + """ + + def testDslParameterOption(self): + """Test DslParameterOption""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_dsl_value_type.py b/packages/protocol/test/test_dsl_value_type.py new file mode 100644 index 0000000000..23ef4199d3 --- /dev/null +++ b/packages/protocol/test/test_dsl_value_type.py @@ -0,0 +1,33 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.dsl_value_type import DslValueType + +class TestDslValueType(unittest.TestCase): + """DslValueType unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def testDslValueType(self): + """Test DslValueType""" + # inst = DslValueType() + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_restart_automation_configuration.py b/packages/protocol/test/test_restart_automation_configuration.py new file mode 100644 index 0000000000..f75cbbc985 --- /dev/null +++ b/packages/protocol/test/test_restart_automation_configuration.py @@ -0,0 +1,54 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration + +class TestRestartAutomationConfiguration(unittest.TestCase): + """RestartAutomationConfiguration unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> RestartAutomationConfiguration: + """Test RestartAutomationConfiguration + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `RestartAutomationConfiguration` + """ + model = RestartAutomationConfiguration() + if include_optional: + return RestartAutomationConfiguration( + id = '', + action_type = 'automation_create' + ) + else: + return RestartAutomationConfiguration( + id = '', + action_type = 'automation_create', + ) + """ + + def testRestartAutomationConfiguration(self): + """Test RestartAutomationConfiguration""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_traded_pair_volume.py b/packages/protocol/test/test_traded_pair_volume.py new file mode 100644 index 0000000000..49e25fff40 --- /dev/null +++ b/packages/protocol/test/test_traded_pair_volume.py @@ -0,0 +1,52 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.traded_pair_volume import TradedPairVolume + +class TestTradedPairVolume(unittest.TestCase): + """TradedPairVolume unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> TradedPairVolume: + """Test TradedPairVolume + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `TradedPairVolume` + """ + model = TradedPairVolume() + if include_optional: + return TradedPairVolume( + base_volume = 1.337, + quote_volume = 1.337 + ) + else: + return TradedPairVolume( + ) + """ + + def testTradedPairVolume(self): + """Test TradedPairVolume""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_traded_pairs_by_exchange.py b/packages/protocol/test/test_traded_pairs_by_exchange.py new file mode 100644 index 0000000000..01fb0f2fbe --- /dev/null +++ b/packages/protocol/test/test_traded_pairs_by_exchange.py @@ -0,0 +1,50 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.traded_pairs_by_exchange import TradedPairsByExchange + +class TestTradedPairsByExchange(unittest.TestCase): + """TradedPairsByExchange unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> TradedPairsByExchange: + """Test TradedPairsByExchange + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `TradedPairsByExchange` + """ + model = TradedPairsByExchange() + if include_optional: + return TradedPairsByExchange( + ) + else: + return TradedPairsByExchange( + ) + """ + + def testTradedPairsByExchange(self): + """Test TradedPairsByExchange""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_traded_pairs_for_exchange.py b/packages/protocol/test/test_traded_pairs_for_exchange.py new file mode 100644 index 0000000000..e37ca8e716 --- /dev/null +++ b/packages/protocol/test/test_traded_pairs_for_exchange.py @@ -0,0 +1,50 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.traded_pairs_for_exchange import TradedPairsForExchange + +class TestTradedPairsForExchange(unittest.TestCase): + """TradedPairsForExchange unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> TradedPairsForExchange: + """Test TradedPairsForExchange + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `TradedPairsForExchange` + """ + model = TradedPairsForExchange() + if include_optional: + return TradedPairsForExchange( + ) + else: + return TradedPairsForExchange( + ) + """ + + def testTradedPairsForExchange(self): + """Test TradedPairsForExchange""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_user_data_state.py b/packages/protocol/test/test_user_data_state.py index 952d9c5d66..ff3b80e8ac 100644 --- a/packages/protocol/test/test_user_data_state.py +++ b/packages/protocol/test/test_user_data_state.py @@ -98,7 +98,11 @@ def make_instance(self, include_optional) -> UserDataState: octobot_protocol.models.position_summary.PositionSummary( id = '', symbol = '', ) - ], ) + ], + child_octobot_process = octobot_protocol.models.child_octo_bot_process_state.ChildOctoBotProcessState( + http_base_url = '', + web_port = 56, + init_state_ok = True, ), ) ], user_actions = [ octobot_protocol.models.user_action.UserAction( diff --git a/packages/services/octobot_services/constants.py b/packages/services/octobot_services/constants.py index e8c0ec9bee..ee1adbd2fe 100644 --- a/packages/services/octobot_services/constants.py +++ b/packages/services/octobot_services/constants.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import os # Config CONFIG_CATEGORY_SERVICES = "services" @@ -86,8 +87,10 @@ NODE_SQLITE_FILE = "node-sqlite-file" NODE_REDIS_URL = "node-redis-url" BACKEND_CORS_ALLOWED_ORIGINS = "backend-cors-allowed-origins" +NODE_EXTERNAL_HOST = "node-external-host" ENV_NODE_SQLITE_FILE = "ENV_NODE_SQLITE_FILE" ENV_NODE_POSTGRES_URL = "ENV_NODE_POSTGRES_URL" +ENV_NODE_EXTERNAL_HOST = "NODE_EXTERNAL_HOST" # Webhook CONFIG_WEBHOOK = "webhook" @@ -253,3 +256,5 @@ # external resources EXTERNAL_RESOURCE_CURRENT_USER_FORM = "current-user-feedback-form" EXTERNAL_RESOURCE_PUBLIC_ANNOUNCEMENTS = "public-announcements" + +SERVICE_STOP_TIMEOUT_SECONDS = float(os.getenv("OCTOBOT_SERVICE_STOP_TIMEOUT_SECONDS", "5.0")) diff --git a/packages/services/octobot_services/managers/service_manager.py b/packages/services/octobot_services/managers/service_manager.py index 73b5fafe69..4c32fb2d00 100644 --- a/packages/services/octobot_services/managers/service_manager.py +++ b/packages/services/octobot_services/managers/service_manager.py @@ -13,18 +13,33 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import asyncio + import octobot_commons.logging as logging +import octobot_services.constants as constants import octobot_services.services as services - async def stop_services(): + logger = logging.get_logger(__name__) for service_instance in _get_service_instances(): + service_name = service_instance.get_name() try: - logging.get_logger(__name__).debug(f"Stopping {service_instance.get_name()} ...") - await service_instance.stop() - logging.get_logger(__name__).debug(f"Stopped {service_instance.get_name()}") - except Exception as e: - raise e + logger.debug(f"Stopping {service_name} ...") + await asyncio.wait_for( + service_instance.stop(), + timeout=constants.SERVICE_STOP_TIMEOUT_SECONDS, + ) + logger.debug(f"Stopped {service_name}") + except asyncio.TimeoutError: + logger.warning( + f"Timed out stopping {service_name} after {constants.SERVICE_STOP_TIMEOUT_SECONDS}s, continuing shutdown" + ) + except Exception as error: + logger.exception( + error, + True, + f"Error when stopping {service_name}: {error}", + ) def _get_service_instances(): diff --git a/packages/services/tests/test_service_manager.py b/packages/services/tests/test_service_manager.py new file mode 100644 index 0000000000..7f570c841c --- /dev/null +++ b/packages/services/tests/test_service_manager.py @@ -0,0 +1,56 @@ +# Drakkar-Software OctoBot-Services +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio + +import mock +import pytest + +import octobot_services.constants as services_constants +import octobot_services.managers.service_manager as service_manager_module + + +pytestmark = pytest.mark.asyncio + + +class _SlowService: + def get_name(self): + return "SlowService" + + async def stop(self): + await asyncio.sleep(60) + + +class _FastService: + stopped = False + + def get_name(self): + return "FastService" + + async def stop(self): + _FastService.stopped = True + + +class TestStopServices: + async def test_continues_after_slow_service_timeout(self): + _FastService.stopped = False + with mock.patch.object( + service_manager_module, + "_get_service_instances", + return_value=[_SlowService(), _FastService()], + ): + with mock.patch.object(services_constants, "SERVICE_STOP_TIMEOUT_SECONDS", 0.01): + await service_manager_module.stop_services() + assert _FastService.stopped is True diff --git a/packages/sync/octobot_sync/app.py b/packages/sync/octobot_sync/app.py index 91ae4a16a0..27ef2e3b19 100644 --- a/packages/sync/octobot_sync/app.py +++ b/packages/sync/octobot_sync/app.py @@ -66,6 +66,38 @@ async def __call__(self, scope, receive, send): await self.app(scope, receive, send) +class HostNormalizeMiddleware: + """Rewrite the inbound ``Host`` header to a configured external host. + + The Starfish cap resolver binds each request signature to a host (the ``h`` + field of the canonical signing input) to stop a signed request from being + replayed against a different server. The client signs the host it dialed + (e.g. a tailnet hostname); when a reverse proxy in front of this app — such + as ``tailscale serve`` — terminates TLS and forwards to this process, the + ``Host`` header the ASGI server sees is the proxy's local target, not the + host the client signed, so every signature mismatches. When ``external_host`` + is set, force the header the resolver reads back to that value so it equals + what the client signed. No-op when unset (e.g. the community sync server, + which is not behind such a proxy). + """ + + def __init__(self, app, external_host: str | None): + self.app = app + self.external_host = external_host + + async def __call__(self, scope, receive, send): + if scope["type"] == "http" and self.external_host: + headers = [ + (name, value) + for name, value in scope.get("headers", []) + if name != b"host" + ] + headers.append((b"host", self.external_host.encode("latin-1"))) + scope = dict(scope) + scope["headers"] = headers + await self.app(scope, receive, send) + + def _build_role_resolver(is_allowed_user_id: Callable[[str], bool] | None): """Cap-cert role resolver (device caps), optionally gated by a userId allowlist. @@ -103,6 +135,7 @@ def create_app( is_allowed_user_id: Callable[[str], bool] | None = None, sync_config: SyncConfig | None = None, plugins: list[ServerPlugin] | None = None, + external_host: str | None = None, ): if sync_config is None: sync_config = sync.load_sync_config(collections_path) @@ -128,5 +161,8 @@ async def health(): # Always wrap: the cap resolver verifies the request signature against # request.url.path, which must equal the client-signed /v1/... path - # regardless of how this app is mounted (see SignedPathMiddleware). - return SignedPathMiddleware(app) + # regardless of how this app is mounted (see SignedPathMiddleware), and + # against request.url's host, which must equal the client-signed host even + # behind a reverse proxy that presents a different Host (see + # HostNormalizeMiddleware). + return HostNormalizeMiddleware(SignedPathMiddleware(app), external_host) diff --git a/packages/sync/octobot_sync/constants.py b/packages/sync/octobot_sync/constants.py index d4f2c967f4..ea4509d05a 100644 --- a/packages/sync/octobot_sync/constants.py +++ b/packages/sync/octobot_sync/constants.py @@ -46,3 +46,4 @@ USER_DATA_STATE_VERSION = "1.0.0" USER_ACTIONS_STATE_VERSION = "1.0.0" DEBUG_STATE_VERSION = "1.0.0" +DSL_KEYWORDS_STATE_VERSION = "1.0.0" diff --git a/packages/sync/octobot_sync/server.py b/packages/sync/octobot_sync/server.py index 7f38aa9ed2..48a41f2534 100644 --- a/packages/sync/octobot_sync/server.py +++ b/packages/sync/octobot_sync/server.py @@ -312,10 +312,12 @@ def build_object_store() -> AbstractObjectStore: def build_default_sync_app( is_allowed_user_id: Callable[[str], bool] | None = None, sync_config: SyncConfig | None = None, + external_host: str | None = None, ): return sync_app.create_app( build_object_store(), is_allowed_user_id=is_allowed_user_id, sync_config=sync_config, plugins=[user_actions_plugin], + external_host=external_host, ) diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py index 1028659d47..6b61185367 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_provider.py @@ -18,11 +18,10 @@ import abc import typing -import cachetools - import octobot_sync.sync.collection_backend.abstract_local_collection_provider as abstract_provider import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_backend.file_checksum_tracked_cache as file_checksum_tracked_cache import octobot_sync.sync.collection_backend.state_model as state_model import octobot_sync.sync.collection_backend.tolerant_state_loading as tolerant_state_loading @@ -64,9 +63,12 @@ def _create_storage( ) def _setup_caches(self) -> None: - self._cache: cachetools.TTLCache[str, S] = cachetools.TTLCache( - maxsize=self._CACHE_MAXSIZE, - ttl=self._CACHE_TTL_SECONDS, + self._cache: file_checksum_tracked_cache.FileChecksumTrackedCache[str, S] = ( + file_checksum_tracked_cache.FileChecksumTrackedCache( + self._storage, + maxsize=self._CACHE_MAXSIZE, + ttl=self._CACHE_TTL_SECONDS, + ) ) @abc.abstractmethod @@ -81,10 +83,10 @@ def _get_item_id_for_key(self, items_key: str, item: typing.Any) -> str: return self._get_item_id(item) def _get_cached_state(self, user_id: str) -> S | None: - return self._cache.get(user_id) + return self._cache.get_if_fresh(user_id, user_id) def _set_cached_state(self, user_id: str, state: S) -> None: - self._cache[user_id] = state + self._cache.set(user_id, user_id, state) def _empty_state(self) -> S: return typing.cast( diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py index aeaa8b6e3a..05370b1a2f 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py @@ -16,6 +16,7 @@ import datetime +import hashlib import json import os import pathlib @@ -32,6 +33,9 @@ import octobot_sync.sync.collection_backend.tolerant_state_loading as tolerant_state_loading +_MISSING_FILE_CHECKSUM = "" + + class BaseLocalCollectionStorage: """ Thread-safe, per-wallet-user_id encrypted collection storage. @@ -41,7 +45,7 @@ class BaseLocalCollectionStorage: """ def __init__(self, collection: str, base_folder: typing.Optional[str] = None) -> None: - root = base_folder or user_root_folder_provider.get_user_root_folder() + root = base_folder or user_root_folder_provider.get_sync_data_root() self.collection = collection self._root = pathlib.Path(root) / collection self._lock = threading.Lock() @@ -62,6 +66,15 @@ def _missing_data_error(self, storage_key: str) -> collection_errors.CollectionN f"{self.collection} file does not exist for user_id {storage_key}" ) + def get_file_checksum(self, storage_key: str) -> str: + """Return the SHA-256 hex digest of the raw on-disk collection file bytes.""" + path = self._file_path(storage_key) + if not path.exists(): + return _MISSING_FILE_CHECKSUM + with self._lock: + with open(path, "rb") as handle: + return hashlib.sha256(handle.read()).hexdigest() + def _payload_to_json_bytes(self, payload: state_model.StateModel) -> bytes: """Serialize a state dict to JSON bytes (handles datetime values from protocol models).""" diff --git a/packages/sync/octobot_sync/sync/collection_backend/file_checksum_tracked_cache.py b/packages/sync/octobot_sync/sync/collection_backend/file_checksum_tracked_cache.py new file mode 100644 index 0000000000..1819a9e434 --- /dev/null +++ b/packages/sync/octobot_sync/sync/collection_backend/file_checksum_tracked_cache.py @@ -0,0 +1,67 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + + +import dataclasses +import typing + +import cachetools + +import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage +import octobot_sync.sync.collection_backend.state_model as state_model + + +CacheKeyT = typing.TypeVar("CacheKeyT") +S = typing.TypeVar("S", bound=state_model.StateModel) + + +@dataclasses.dataclass(frozen=True, slots=True) +class CachedStateEnvelope(typing.Generic[S]): + state: S + file_checksum: str + + +class FileChecksumTrackedCache(typing.Generic[CacheKeyT, S]): + """TTL cache that invalidates entries when the backing collection file changes.""" + + def __init__( + self, + storage: base_storage.BaseLocalCollectionStorage, + *, + maxsize: int, + ttl: float, + ) -> None: + self._storage = storage + self._cache: cachetools.TTLCache[CacheKeyT, CachedStateEnvelope[S]] = cachetools.TTLCache( + maxsize=maxsize, + ttl=ttl, + ) + + def get_if_fresh(self, cache_key: CacheKeyT, storage_key: str) -> S | None: + envelope = self._cache.get(cache_key) + if envelope is None: + return None + current_checksum = self._storage.get_file_checksum(storage_key) + if current_checksum != envelope.file_checksum: + self._cache.pop(cache_key, None) + return None + return envelope.state + + def set(self, cache_key: CacheKeyT, storage_key: str, state: S) -> None: + self._cache[cache_key] = CachedStateEnvelope( + state=state, + file_checksum=self._storage.get_file_checksum(storage_key), + ) diff --git a/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py b/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py index e69ebb80e3..f78820419c 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py +++ b/packages/sync/octobot_sync/sync/collection_backend/single_item_local_collection_provider.py @@ -17,10 +17,9 @@ import typing -import cachetools - import octobot_sync.sync.collection_backend.abstract_local_collection_provider as abstract_provider import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage +import octobot_sync.sync.collection_backend.file_checksum_tracked_cache as file_checksum_tracked_cache import octobot_sync.sync.collection_backend.single_item_local_collection_storage as single_item_storage import octobot_sync.sync.collection_backend.state_model as state_model @@ -47,9 +46,12 @@ def _create_storage( ) def _setup_caches(self) -> None: - self._state_cache: cachetools.TTLCache[tuple[str, str], S] = cachetools.TTLCache( - maxsize=self._CACHE_MAXSIZE, - ttl=self._CACHE_TTL_SECONDS, + self._state_cache: file_checksum_tracked_cache.FileChecksumTrackedCache[tuple[str, str], S] = ( + file_checksum_tracked_cache.FileChecksumTrackedCache( + self._storage, + maxsize=self._CACHE_MAXSIZE, + ttl=self._CACHE_TTL_SECONDS, + ) ) def _build_identifier(self, user_id: str, account_id: str) -> str: @@ -58,10 +60,12 @@ def _build_identifier(self, user_id: str, account_id: str) -> str: ) def _get_cached_state(self, user_id: str, account_id: str) -> S | None: - return self._state_cache.get((user_id, account_id)) + identifier = self._build_identifier(user_id, account_id) + return self._state_cache.get_if_fresh((user_id, account_id), identifier) def _set_cached_state(self, user_id: str, account_id: str, state: S) -> None: - self._state_cache[(user_id, account_id)] = state + identifier = self._build_identifier(user_id, account_id) + self._state_cache.set((user_id, account_id), identifier, state) def load_state(self, user_id: str, account_id: str) -> S: cached_state = self._get_cached_state(user_id, account_id) diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py index e2fd2e11b4..c7d731ae10 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py @@ -24,6 +24,7 @@ import octobot_sync.sync.collection_backend.base_local_collection_provider as base_provider import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers.validation.exchange_account_identity as exchange_account_identity class AccountProvider( @@ -54,6 +55,37 @@ def _get_item_id_for_key(self, items_key: str, item: typing.Any) -> str: f"Unsupported items key {items_key!r} for {self.__class__.__name__}" ) + def _empty_state(self) -> protocol_models.AccountsState: + return protocol_models.AccountsState( + version=self.STATE_VERSION, + accounts=[], + exchange_configs=[], + ) + + def _assert_unique_exchange_account_identity( + self, + user_id: str, + account: protocol_models.Account, + *, + exclude_account_id: str | None = None, + ) -> None: + state = self._load_state(user_id) + exchange_account_identity.assert_unique_exchange_account_identity( + user_id, + account, + self._items_from_state(state, self.ITEMS_KEY), + self._items_from_state(state, self.EXCHANGE_CONFIGS_KEY), + exclude_account_id=exclude_account_id, + ) + + def create_item(self, user_id: str, item: protocol_models.Account) -> protocol_models.Account: + self._assert_unique_exchange_account_identity(user_id, item) + return super().create_item(user_id, item) + + def update_item(self, user_id: str, item: protocol_models.Account) -> protocol_models.Account: + self._assert_unique_exchange_account_identity(user_id, item, exclude_account_id=item.id) + return super().update_item(user_id, item) + def list_accounts(self, address: str) -> list[protocol_models.Account]: return self.list_items(address) diff --git a/packages/sync/octobot_sync/sync/collection_providers/validation/__init__.py b/packages/sync/octobot_sync/sync/collection_providers/validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/sync/octobot_sync/sync/collection_providers/validation/exchange_account_identity.py b/packages/sync/octobot_sync/sync/collection_providers/validation/exchange_account_identity.py new file mode 100644 index 0000000000..ecc240418c --- /dev/null +++ b/packages/sync/octobot_sync/sync/collection_providers/validation/exchange_account_identity.py @@ -0,0 +1,127 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + + +import typing + +import octobot_protocol.models as protocol_models + +import octobot_sync.sync.collection_backend.errors as collection_errors + + +class ExchangeAccountIdentity(typing.NamedTuple): + remote_account_id: str + exchange: str + url: str | None + sandboxed: bool + + +def _normalize_exchange_url(url: str | None) -> str | None: + if url is None or url == "": + return None + return url + + +def _resolved_remote_account_id( + account: protocol_models.Account, + exchange_account: protocol_models.ExchangeAccount, +) -> str: + remote_account_id = exchange_account.remote_account_id + if remote_account_id: + return remote_account_id + return account.id + + +def _exchange_configs_by_id( + exchange_configs: list[protocol_models.ExchangeConfig], +) -> dict[str, protocol_models.ExchangeConfig]: + return { + exchange_config.id: exchange_config + for exchange_config in exchange_configs + } + + +def _resolve_exchange_account_identity( + account: protocol_models.Account, + exchange_configs_by_id: dict[str, protocol_models.ExchangeConfig], +) -> ExchangeAccountIdentity | None: + if account.is_simulated: + return None + account_specifics = account.specifics + if account_specifics is None or account_specifics.actual_instance is None: + return None + if not isinstance(account_specifics.actual_instance, protocol_models.ExchangeAccount): + return None + + exchange_account = account_specifics.actual_instance + exchange_config_ids = exchange_account.exchange_config_ids + if not exchange_config_ids: + return None + + exchange_config = exchange_configs_by_id.get(exchange_config_ids[0]) + if exchange_config is None: + return None + + return ExchangeAccountIdentity( + remote_account_id=_resolved_remote_account_id(account, exchange_account), + exchange=exchange_config.exchange, + url=_normalize_exchange_url(exchange_config.url), + sandboxed=exchange_config.sandboxed, + ) + + +def _find_conflicting_account( + accounts: list[protocol_models.Account], + exchange_configs_by_id: dict[str, protocol_models.ExchangeConfig], + candidate_identity: ExchangeAccountIdentity, + *, + exclude_account_id: str | None = None, +) -> protocol_models.Account | None: + for existing_account in accounts: + if exclude_account_id is not None and existing_account.id == exclude_account_id: + continue + existing_identity = _resolve_exchange_account_identity( + existing_account, + exchange_configs_by_id, + ) + if existing_identity == candidate_identity: + return existing_account + return None + + +def assert_unique_exchange_account_identity( + user_id: str, + account: protocol_models.Account, + accounts: list[protocol_models.Account], + exchange_configs: list[protocol_models.ExchangeConfig], + *, + exclude_account_id: str | None = None, +) -> None: + exchange_configs_by_id = _exchange_configs_by_id(exchange_configs) + identity = _resolve_exchange_account_identity(account, exchange_configs_by_id) + if identity is None: + return + conflict = _find_conflicting_account( + accounts, + exchange_configs_by_id, + identity, + exclude_account_id=exclude_account_id, + ) + if conflict is not None: + raise collection_errors.DuplicateItemError( + f"Exchange account identity {identity!r} already exists for user_id {user_id!r} " + f"as account {conflict.id!r}" + ) diff --git a/packages/sync/octobot_sync/sync/collections.py b/packages/sync/octobot_sync/sync/collections.py index cb27e0a500..a42b911b21 100644 --- a/packages/sync/octobot_sync/sync/collections.py +++ b/packages/sync/octobot_sync/sync/collections.py @@ -150,12 +150,10 @@ def load_sync_config( path = collections_path or os.path.join( commons_constants.USER_FOLDER, constants.COLLECTIONS_FILE ) - if not os.path.isfile(path): - logger.warning( - f"Collections file not found at {path}, using default config" - ) - return DEFAULT_SYNC_CONFIG - return load_config_file(path) + if os.path.isfile(path): + logger.warning(f"Using custom collections file at {path}") + return load_config_file(path) + return DEFAULT_SYNC_CONFIG def is_replicable_collection(col: CollectionConfig) -> bool: diff --git a/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py b/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py index ca991ba570..9383bafc22 100644 --- a/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py +++ b/packages/sync/tests/sync/collection_backend/test_abstract_local_collection_provider.py @@ -16,13 +16,13 @@ import typing -import cachetools import mock import pydantic import octobot.community.authentication as community_authentication import octobot_sync.sync.collection_backend.abstract_local_collection_provider as abstract_provider_module import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage_module +import octobot_sync.sync.collection_backend.file_checksum_tracked_cache as file_checksum_tracked_cache_module import octobot_sync.sync.collection_backend.single_item_local_collection_storage as single_item_storage_module _TEST_ADDRESS = "0xaaabbbcccddd" @@ -49,7 +49,8 @@ def _create_storage( ) def _setup_caches(self) -> None: - self._state_cache: cachetools.TTLCache[tuple[str, str], _TestState] = cachetools.TTLCache( + self._state_cache = file_checksum_tracked_cache_module.FileChecksumTrackedCache( + self._storage, maxsize=self._CACHE_MAXSIZE, ttl=self._CACHE_TTL_SECONDS, ) @@ -83,7 +84,10 @@ def test_calls_setup_caches(self, tmp_path): provider = _make_provider(tmp_path) assert hasattr(provider, "_state_cache") - assert isinstance(provider._state_cache, cachetools.TTLCache) + assert isinstance( + provider._state_cache, + file_checksum_tracked_cache_module.FileChecksumTrackedCache, + ) class TestAbstractLocalCollectionProviderGetWalletPrivateKey: diff --git a/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py b/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py index da6fa26fbe..2098057418 100644 --- a/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py +++ b/packages/sync/tests/sync/collection_backend/test_base_local_collection_provider.py @@ -385,3 +385,25 @@ def test_unknown_items_key_raises(self, tmp_path): with pytest.raises(collection_errors.UnsupportedItemsKeyError): provider._get_item_id_for_key("unknown_key", _item("item-1")) + +class TestBaseLocalCollectionProviderCacheInvalidation: + def test_reloads_from_disk_when_file_changed_externally(self, tmp_path): + provider = _make_provider(tmp_path) + with _patch_wallet(): + provider.create_item(_TEST_ADDRESS, _item("item-1", label="Cached")) + provider.list_items(_TEST_ADDRESS) + + external_state = _TestState( + version="1.0.0", + items=[_TestItem(id="external", label="From disk")], + ) + provider._storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, external_state) + + with _patch_wallet(): + listed = provider.list_items(_TEST_ADDRESS) + + assert len(listed) == 1 + assert listed[0].id == "external" + assert listed[0].label == "From disk" + + diff --git a/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py b/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py index fbb57bf612..ee15c1d556 100644 --- a/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py +++ b/packages/sync/tests/sync/collection_backend/test_base_local_collection_storage.py @@ -248,3 +248,37 @@ def test_different_collections_are_isolated(self, tmp_path): assert storage_a.load_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, TestStateModel) == state assert storage_b.load_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, TestStateModel) == TestStateModel(version="1.0.0", items=[TestItemModel(id="b1")]) + + +class TestBaseLocalCollectionStorageGetFileChecksum: + def test_returns_empty_string_when_file_absent(self, tmp_path): + storage = _make_storage(tmp_path) + + checksum = storage.get_file_checksum(_TEST_ADDRESS) + + assert checksum == "" + + def test_returns_stable_checksum_for_same_file_bytes(self, tmp_path): + storage = _make_storage(tmp_path) + storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, _SAMPLE_STATE) + + first_checksum = storage.get_file_checksum(_TEST_ADDRESS) + second_checksum = storage.get_file_checksum(_TEST_ADDRESS) + + assert first_checksum == second_checksum + assert len(first_checksum) == 64 + + def test_returns_different_checksum_after_file_rewrite(self, tmp_path): + storage = _make_storage(tmp_path) + storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, _SAMPLE_STATE) + first_checksum = storage.get_file_checksum(_TEST_ADDRESS) + + storage.save_state( + _TEST_ADDRESS, + _TEST_PRIVATE_KEY, + TestStateModel(version="1.0.0", items=[TestItemModel(id="item-3")]), + ) + second_checksum = storage.get_file_checksum(_TEST_ADDRESS) + + assert first_checksum != second_checksum + diff --git a/packages/sync/tests/sync/collection_backend/test_file_checksum_tracked_cache.py b/packages/sync/tests/sync/collection_backend/test_file_checksum_tracked_cache.py new file mode 100644 index 0000000000..6d01ebd0fb --- /dev/null +++ b/packages/sync/tests/sync/collection_backend/test_file_checksum_tracked_cache.py @@ -0,0 +1,199 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import typing + +import mock +import pydantic + +import octobot_sync.sync.collection_backend.base_local_collection_storage as base_storage_module +import octobot_sync.sync.collection_backend.file_checksum_tracked_cache as file_checksum_tracked_cache_module + +_CACHE_KEY = "user-1" +_STORAGE_KEY = "user-1" +_TEST_ADDRESS = "0xaaabbbcccddd" +_TEST_PRIVATE_KEY = "private-key" +_CHECKSUM_V1 = "checksum-v1" +_CHECKSUM_V2 = "checksum-v2" + + +class _TestItem(pydantic.BaseModel): + id: str + label: typing.Optional[str] = None + + def to_dict(self) -> dict[str, typing.Any]: + return self.model_dump(exclude_none=True) + + def to_json(self) -> str: + return self.model_dump_json() + + @classmethod + def from_json(cls, json_str: str) -> typing.Optional["_TestItem"]: + return cls.model_validate_json(json_str) + + @classmethod + def from_dict(cls, raw: dict[str, typing.Any]) -> typing.Optional["_TestItem"]: + return cls.model_validate(raw) + + +class _TestState(pydantic.BaseModel): + version: str + items: typing.Optional[list[_TestItem]] = None + + def to_dict(self) -> dict[str, typing.Any]: + result = {"version": self.version} + if self.items is not None: + result["items"] = [item.to_dict() for item in self.items] + return result + + def to_json(self) -> str: + return self.model_dump_json() + + @classmethod + def from_json(cls, json_str: str) -> typing.Optional["_TestState"]: + return cls.model_validate_json(json_str) + + @classmethod + def from_dict(cls, raw: dict[str, typing.Any]) -> typing.Optional["_TestState"]: + return cls.model_validate(raw) + + +_SAMPLE_STATE = _TestState( + version="1.0.0", + items=[_TestItem(id="item-1", label="First")], +) + + +def _make_storage_mock(*, checksum: str = _CHECKSUM_V1) -> mock.Mock: + storage = mock.Mock() + storage.get_file_checksum = mock.Mock(return_value=checksum) + return storage + + +def _make_real_storage(tmp_path) -> base_storage_module.BaseLocalCollectionStorage: + return base_storage_module.BaseLocalCollectionStorage( + collection="test-checksum-cache", + base_folder=str(tmp_path), + ) + + +def _make_cache( + storage: base_storage_module.BaseLocalCollectionStorage, + *, + maxsize: int = 10, + ttl: float = 60.0, +) -> file_checksum_tracked_cache_module.FileChecksumTrackedCache[str, _TestState]: + return file_checksum_tracked_cache_module.FileChecksumTrackedCache( + storage, + maxsize=maxsize, + ttl=ttl, + ) + + +class TestFileChecksumTrackedCacheGetIfFresh: + def test_returns_none_on_cache_miss(self): + storage = _make_storage_mock() + cache = _make_cache(storage) + + result = cache.get_if_fresh(_CACHE_KEY, _STORAGE_KEY) + + assert result is None + storage.get_file_checksum.assert_not_called() + + def test_returns_state_when_checksum_matches(self): + storage = _make_storage_mock(checksum=_CHECKSUM_V1) + cache = _make_cache(storage) + cache.set(_CACHE_KEY, _STORAGE_KEY, _SAMPLE_STATE) + storage.get_file_checksum.reset_mock() + + result = cache.get_if_fresh(_CACHE_KEY, _STORAGE_KEY) + + assert result == _SAMPLE_STATE + storage.get_file_checksum.assert_called_once_with(_STORAGE_KEY) + + def test_returns_none_and_evicts_when_checksum_changes(self): + storage = _make_storage_mock(checksum=_CHECKSUM_V1) + cache = _make_cache(storage) + cache.set(_CACHE_KEY, _STORAGE_KEY, _SAMPLE_STATE) + storage.get_file_checksum.return_value = _CHECKSUM_V2 + + result = cache.get_if_fresh(_CACHE_KEY, _STORAGE_KEY) + + assert result is None + assert _CACHE_KEY not in cache._cache + + def test_uses_storage_key_for_checksum_lookup(self): + cache_key = ("user", "acc") + storage_key = "user/acc" + storage = _make_storage_mock(checksum=_CHECKSUM_V1) + cache = _make_cache(storage) + cache.set(cache_key, storage_key, _SAMPLE_STATE) + storage.get_file_checksum.reset_mock() + + result = cache.get_if_fresh(cache_key, storage_key) + + assert result == _SAMPLE_STATE + storage.get_file_checksum.assert_called_once_with(storage_key) + + +class TestFileChecksumTrackedCacheSet: + def test_records_current_checksum_from_storage(self): + storage = _make_storage_mock(checksum=_CHECKSUM_V1) + cache = _make_cache(storage) + + cache.set(_CACHE_KEY, _STORAGE_KEY, _SAMPLE_STATE) + + envelope = cache._cache[_CACHE_KEY] + assert envelope.state == _SAMPLE_STATE + assert envelope.file_checksum == _CHECKSUM_V1 + storage.get_file_checksum.assert_called_once_with(_STORAGE_KEY) + + def test_subsequent_get_if_fresh_hits_without_recalling_set(self): + storage = _make_storage_mock(checksum=_CHECKSUM_V1) + cache = _make_cache(storage) + cache.set(_CACHE_KEY, _STORAGE_KEY, _SAMPLE_STATE) + storage.get_file_checksum.reset_mock() + + result = cache.get_if_fresh(_CACHE_KEY, _STORAGE_KEY) + + assert result == _SAMPLE_STATE + storage.get_file_checksum.assert_called_once_with(_STORAGE_KEY) + + +class TestFileChecksumTrackedCacheRealStorage: + def test_invalidates_when_backing_file_changes_on_disk(self, tmp_path): + storage = _make_real_storage(tmp_path) + cache = _make_cache(storage) + storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, _SAMPLE_STATE) + + cache.set(_CACHE_KEY, _TEST_ADDRESS, _SAMPLE_STATE) + + envelope = cache._cache[_CACHE_KEY] + assert len(envelope.file_checksum) == 64 + + fresh_result = cache.get_if_fresh(_CACHE_KEY, _TEST_ADDRESS) + assert fresh_result == _SAMPLE_STATE + + external_state = _TestState( + version="1.0.0", + items=[_TestItem(id="external", label="From disk")], + ) + storage.save_state(_TEST_ADDRESS, _TEST_PRIVATE_KEY, external_state) + + stale_result = cache.get_if_fresh(_CACHE_KEY, _TEST_ADDRESS) + + assert stale_result is None + assert _CACHE_KEY not in cache._cache diff --git a/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py b/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py index 44f469dcea..1d7ac91274 100644 --- a/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py +++ b/packages/sync/tests/sync/collection_backend/test_single_item_local_collection_provider.py @@ -132,6 +132,29 @@ def test_persists_and_updates_cache(self, tmp_path): assert persisted_state == _SAMPLE_STATE +class TestSingleItemLocalCollectionProviderCacheInvalidation: + def test_reloads_from_disk_when_file_changed_externally(self, tmp_path): + provider = _make_provider(tmp_path) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, _SAMPLE_STATE) + provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + + external_state = _TestState( + version="1.0.0", + items=[_TestItem(id="external", label="From disk")], + ) + identifier = provider._build_identifier(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + provider._storage.save_state(identifier, _TEST_PRIVATE_KEY, external_state) + + with _patch_wallet(): + loaded_state = provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + + assert loaded_state.items is not None + assert len(loaded_state.items) == 1 + assert loaded_state.items[0].id == "external" + assert loaded_state.items[0].label == "From disk" + + class TestSingleItemLocalCollectionProviderLoadStateEncrypted: def test_reads_encrypted_blob_for_account_id(self, tmp_path): provider = _make_provider(tmp_path) diff --git a/packages/sync/tests/sync/collection_providers/validation/__init__.py b/packages/sync/tests/sync/collection_providers/validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/sync/tests/sync/collection_providers/validation/test_exchange_account_identity.py b/packages/sync/tests/sync/collection_providers/validation/test_exchange_account_identity.py new file mode 100644 index 0000000000..8047b19741 --- /dev/null +++ b/packages/sync/tests/sync/collection_providers/validation/test_exchange_account_identity.py @@ -0,0 +1,428 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import datetime + +import mock +import pytest + +import octobot.community.authentication as community_authentication +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers.validation.exchange_account_identity as exchange_account_identity_module +import octobot_sync.sync.collection_providers.user_account_provider as account_provider_module +import octobot_protocol.models as protocol_models + +_TEST_ADDRESS = "0xaaabbbcccddd" +_TEST_PRIVATE_KEY = "private-key" +_DEFAULT_EXCHANGE_CONFIG_ID = "cfg-1" + + +def _patch_wallet(private_key: str = _TEST_PRIVATE_KEY): + wallet = mock.Mock() + wallet.private_key = private_key + auth = mock.Mock() + auth.get_wallet_by_user_id.return_value = wallet + return mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ) + + +def _make_provider(tmp_path): + return account_provider_module.AccountProvider(base_folder=str(tmp_path)) + + +def _fixture_time() -> datetime.datetime: + return datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + + +def _sample_exchange_config( + *, + config_id: str = _DEFAULT_EXCHANGE_CONFIG_ID, + exchange: str = "binanceus", + url: str | None = None, + sandboxed: bool = False, +) -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id=config_id, + name="binance-main", + exchange=exchange, + sandboxed=sandboxed, + url=url, + ) + + +def _exchange_account_specifics( + *, + remote_account_id: str, + exchange_config_ids: list[str] | None = None, +) -> protocol_models.AccountSpecifics: + return protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=remote_account_id, + exchange_config_ids=exchange_config_ids or [_DEFAULT_EXCHANGE_CONFIG_ID], + ), + ) + + +def _live_exchange_account( + *, + account_id: str, + remote_account_id: str, + is_simulated: bool = False, + exchange_config_ids: list[str] | None = None, + name: str = "Test account", +) -> protocol_models.Account: + fixture_time = _fixture_time() + return protocol_models.Account( + id=account_id, + name=name, + is_simulated=is_simulated, + created_at=fixture_time, + updated_at=fixture_time, + specifics=_exchange_account_specifics( + remote_account_id=remote_account_id, + exchange_config_ids=exchange_config_ids, + ), + ) + + +class TestNormalizeExchangeUrl: + def test_none_stays_none(self): + assert exchange_account_identity_module._normalize_exchange_url(None) is None + + def test_empty_string_becomes_none(self): + assert exchange_account_identity_module._normalize_exchange_url("") is None + + def test_non_empty_url_preserved(self): + assert exchange_account_identity_module._normalize_exchange_url("https://api.binance.com") == ( + "https://api.binance.com" + ) + + +class TestResolveExchangeAccountIdentity: + def test_returns_none_for_simulated_account(self): + account = _live_exchange_account( + account_id="acc-1", + remote_account_id="remote-1", + is_simulated=True, + ) + exchange_configs_by_id = {_DEFAULT_EXCHANGE_CONFIG_ID: _sample_exchange_config()} + assert exchange_account_identity_module._resolve_exchange_account_identity( + account, + exchange_configs_by_id, + ) is None + + def test_returns_none_for_non_exchange_specifics(self): + fixture_time = _fixture_time() + account = protocol_models.Account( + id="acc-1", + name="Blockchain", + is_simulated=False, + created_at=fixture_time, + updated_at=fixture_time, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.BlockchainAccount( + account_type=protocol_models.AccountType.BLOCKCHAIN, + blockchain="ethereum", + public_key="0x1234567890123456789012345678901234567890", + ), + ), + ) + exchange_configs_by_id = {_DEFAULT_EXCHANGE_CONFIG_ID: _sample_exchange_config()} + assert exchange_account_identity_module._resolve_exchange_account_identity( + account, + exchange_configs_by_id, + ) is None + + def test_resolves_identity_from_exchange_config(self): + account = _live_exchange_account(account_id="acc-1", remote_account_id="remote-1") + exchange_configs_by_id = { + _DEFAULT_EXCHANGE_CONFIG_ID: _sample_exchange_config( + exchange="binanceus", + url="https://api.binance.us", + sandboxed=True, + ), + } + identity = exchange_account_identity_module._resolve_exchange_account_identity( + account, + exchange_configs_by_id, + ) + assert identity == exchange_account_identity_module.ExchangeAccountIdentity( + remote_account_id="remote-1", + exchange="binanceus", + url="https://api.binance.us", + sandboxed=True, + ) + + def test_falls_back_to_account_id_when_remote_account_id_empty(self): + account = _live_exchange_account(account_id="acc-1", remote_account_id="") + exchange_configs_by_id = {_DEFAULT_EXCHANGE_CONFIG_ID: _sample_exchange_config()} + identity = exchange_account_identity_module._resolve_exchange_account_identity( + account, + exchange_configs_by_id, + ) + assert identity is not None + assert identity.remote_account_id == "acc-1" + + +class TestFindConflictingAccount: + def test_finds_matching_account(self): + first_account = _live_exchange_account(account_id="acc-1", remote_account_id="remote-1") + second_account = _live_exchange_account(account_id="acc-2", remote_account_id="remote-1") + exchange_configs_by_id = {_DEFAULT_EXCHANGE_CONFIG_ID: _sample_exchange_config()} + candidate_identity = exchange_account_identity_module._resolve_exchange_account_identity( + second_account, + exchange_configs_by_id, + ) + assert candidate_identity is not None + conflict = exchange_account_identity_module._find_conflicting_account( + [first_account], + exchange_configs_by_id, + candidate_identity, + ) + assert conflict is first_account + + def test_excludes_account_id_when_requested(self): + account = _live_exchange_account(account_id="acc-1", remote_account_id="remote-1") + exchange_configs_by_id = {_DEFAULT_EXCHANGE_CONFIG_ID: _sample_exchange_config()} + candidate_identity = exchange_account_identity_module._resolve_exchange_account_identity( + account, + exchange_configs_by_id, + ) + assert candidate_identity is not None + conflict = exchange_account_identity_module._find_conflicting_account( + [account], + exchange_configs_by_id, + candidate_identity, + exclude_account_id="acc-1", + ) + assert conflict is None + + +class TestAssertUniqueExchangeAccountIdentity: + def test_no_op_for_simulated_account(self): + account = _live_exchange_account( + account_id="acc-1", + remote_account_id="remote-1", + is_simulated=True, + ) + exchange_account_identity_module.assert_unique_exchange_account_identity( + _TEST_ADDRESS, + account, + [], + [_sample_exchange_config()], + ) + + def test_no_op_for_non_exchange_specifics(self): + fixture_time = _fixture_time() + account = protocol_models.Account( + id="acc-1", + name="Blockchain", + is_simulated=False, + created_at=fixture_time, + updated_at=fixture_time, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.BlockchainAccount( + account_type=protocol_models.AccountType.BLOCKCHAIN, + blockchain="ethereum", + public_key="0x1234567890123456789012345678901234567890", + ), + ), + ) + exchange_account_identity_module.assert_unique_exchange_account_identity( + _TEST_ADDRESS, + account, + [], + [_sample_exchange_config()], + ) + + def test_raises_when_identity_collides(self): + first_account = _live_exchange_account(account_id="acc-1", remote_account_id="remote-1") + second_account = _live_exchange_account(account_id="acc-2", remote_account_id="remote-1") + exchange_configs = [_sample_exchange_config()] + with pytest.raises(collection_errors.DuplicateItemError) as raised: + exchange_account_identity_module.assert_unique_exchange_account_identity( + _TEST_ADDRESS, + second_account, + [first_account], + exchange_configs, + ) + assert _TEST_ADDRESS in str(raised.value) + assert "acc-1" in str(raised.value) + + def test_passes_when_exclude_account_id_matches_self(self): + account = _live_exchange_account(account_id="acc-1", remote_account_id="remote-1") + exchange_configs = [_sample_exchange_config()] + exchange_account_identity_module.assert_unique_exchange_account_identity( + _TEST_ADDRESS, + account, + [account], + exchange_configs, + exclude_account_id="acc-1", + ) + + +class TestAccountProviderCreateItemExchangeAccountIdentity: + def _seed_exchange_config(self, provider, exchange_config: protocol_models.ExchangeConfig | None = None): + config = exchange_config or _sample_exchange_config() + with _patch_wallet(): + provider.create_exchange_config(_TEST_ADDRESS, config) + return config + + def test_duplicate_remote_identity_raises(self, tmp_path): + provider = _make_provider(tmp_path) + self._seed_exchange_config(provider) + first_account = _live_exchange_account(account_id="acc-1", remote_account_id="remote-1") + second_account = _live_exchange_account(account_id="acc-2", remote_account_id="remote-1") + with _patch_wallet(): + provider.create_item(_TEST_ADDRESS, first_account) + with pytest.raises(collection_errors.DuplicateItemError, match="remote-1"): + with _patch_wallet(): + provider.create_item(_TEST_ADDRESS, second_account) + + def test_different_remote_id_allowed(self, tmp_path): + provider = _make_provider(tmp_path) + self._seed_exchange_config(provider) + with _patch_wallet(): + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account(account_id="acc-1", remote_account_id="remote-1"), + ) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account(account_id="acc-2", remote_account_id="remote-2"), + ) + assert len(provider.list_items(_TEST_ADDRESS)) == 2 + + def test_different_url_allowed(self, tmp_path): + provider = _make_provider(tmp_path) + self._seed_exchange_config(provider, _sample_exchange_config(config_id="cfg-live", url=None)) + with _patch_wallet(): + provider.create_exchange_config( + _TEST_ADDRESS, + _sample_exchange_config(config_id="cfg-custom", url="https://custom.example"), + ) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account( + account_id="acc-1", + remote_account_id="remote-1", + exchange_config_ids=["cfg-live"], + ), + ) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account( + account_id="acc-2", + remote_account_id="remote-1", + exchange_config_ids=["cfg-custom"], + ), + ) + assert len(provider.list_items(_TEST_ADDRESS)) == 2 + + def test_sandboxed_differs_allowed(self, tmp_path): + provider = _make_provider(tmp_path) + with _patch_wallet(): + provider.create_exchange_config( + _TEST_ADDRESS, + _sample_exchange_config(config_id="cfg-live", sandboxed=False), + ) + provider.create_exchange_config( + _TEST_ADDRESS, + _sample_exchange_config(config_id="cfg-sandbox", sandboxed=True), + ) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account( + account_id="acc-1", + remote_account_id="remote-1", + exchange_config_ids=["cfg-live"], + ), + ) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account( + account_id="acc-2", + remote_account_id="remote-1", + exchange_config_ids=["cfg-sandbox"], + ), + ) + assert len(provider.list_items(_TEST_ADDRESS)) == 2 + + def test_simulated_accounts_skipped(self, tmp_path): + provider = _make_provider(tmp_path) + self._seed_exchange_config(provider) + with _patch_wallet(): + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account( + account_id="acc-1", + remote_account_id="remote-1", + is_simulated=True, + ), + ) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account( + account_id="acc-2", + remote_account_id="remote-1", + is_simulated=True, + ), + ) + assert len(provider.list_items(_TEST_ADDRESS)) == 2 + + +class TestAccountProviderUpdateItemExchangeAccountIdentity: + def test_edit_same_identity_allowed(self, tmp_path): + provider = _make_provider(tmp_path) + with _patch_wallet(): + provider.create_exchange_config(_TEST_ADDRESS, _sample_exchange_config()) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account(account_id="acc-1", remote_account_id="remote-1"), + ) + provider.update_item( + _TEST_ADDRESS, + _live_exchange_account( + account_id="acc-1", + remote_account_id="remote-1", + name="Renamed account", + ), + ) + updated = provider.get_item(_TEST_ADDRESS, "acc-1") + assert updated.name == "Renamed account" + + def test_edit_to_conflicting_identity_raises(self, tmp_path): + provider = _make_provider(tmp_path) + with _patch_wallet(): + provider.create_exchange_config(_TEST_ADDRESS, _sample_exchange_config()) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account(account_id="acc-1", remote_account_id="remote-1"), + ) + provider.create_item( + _TEST_ADDRESS, + _live_exchange_account(account_id="acc-2", remote_account_id="remote-2"), + ) + with pytest.raises(collection_errors.DuplicateItemError, match="remote-1"): + with _patch_wallet(): + provider.update_item( + _TEST_ADDRESS, + _live_exchange_account(account_id="acc-2", remote_account_id="remote-1"), + ) diff --git a/packages/sync/tests/test_app_helpers.py b/packages/sync/tests/test_app_helpers.py index 4b803210d1..6f95d0ffa9 100644 --- a/packages/sync/tests/test_app_helpers.py +++ b/packages/sync/tests/test_app_helpers.py @@ -37,18 +37,48 @@ async def test_health_endpoint(app): async def test_create_app_returns_signed_path_middleware(app): - assert isinstance(app, sync_app.SignedPathMiddleware) + assert isinstance(app, sync_app.HostNormalizeMiddleware) + assert isinstance(app.app, sync_app.SignedPathMiddleware) async def test_create_app_with_custom_collections_path(): store = MemoryObjectStore() # Should not raise even with a non-existent collections path (falls back to default) created_app = sync_app.create_app(store, collections_path="/nonexistent/path.json") - assert isinstance(created_app, sync_app.SignedPathMiddleware) + assert isinstance(created_app, sync_app.HostNormalizeMiddleware) + assert isinstance(created_app.app, sync_app.SignedPathMiddleware) async def test_create_app_with_allowlist(): store = MemoryObjectStore() # is_allowed_user_id callable accepted without error created_app = sync_app.create_app(store, is_allowed_user_id=lambda uid: True) - assert isinstance(created_app, sync_app.SignedPathMiddleware) + assert isinstance(created_app, sync_app.HostNormalizeMiddleware) + assert isinstance(created_app.app, sync_app.SignedPathMiddleware) + + +async def _echo_host_app(scope, receive, send): + headers = dict(scope["headers"]) + await send({ + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"text/plain")], + }) + await send({ + "type": "http.response.body", + "body": headers.get(b"host", b""), + }) + + +async def test_host_normalize_middleware_rewrites_host(): + wrapped = sync_app.HostNormalizeMiddleware(_echo_host_app, "signed-host.example") + async with AsyncClient(transport=ASGITransport(app=wrapped), base_url="http://proxy-local:8000") as client: + resp = await client.get("/health") + assert resp.text == "signed-host.example" + + +async def test_host_normalize_middleware_noop_when_unset(): + wrapped = sync_app.HostNormalizeMiddleware(_echo_host_app, None) + async with AsyncClient(transport=ASGITransport(app=wrapped), base_url="http://proxy-local:8000") as client: + resp = await client.get("/health") + assert resp.text == "proxy-local:8000" diff --git a/packages/sync/tests/test_exception_handlers.py b/packages/sync/tests/test_exception_handlers.py index 2909248d5f..86983b01c6 100644 --- a/packages/sync/tests/test_exception_handlers.py +++ b/packages/sync/tests/test_exception_handlers.py @@ -27,7 +27,7 @@ class TestSyncAppUnhandledExceptionHandler: async def test_create_app_registers_unhandled_exception_handler(self): store = mock.Mock() wrapped_app = sync_app.create_app(store) - inner_app = wrapped_app.app + inner_app = wrapped_app.app.app assert Exception in inner_app.exception_handlers def test_mounted_sync_unhandled_exception_is_logged(self): diff --git a/packages/tentacles/Evaluator/Social/forum_evaluator/forum.py b/packages/tentacles/Evaluator/Social/forum_evaluator/forum.py index 3bbf4f5b5a..0518407fc1 100644 --- a/packages/tentacles/Evaluator/Social/forum_evaluator/forum.py +++ b/packages/tentacles/Evaluator/Social/forum_evaluator/forum.py @@ -52,8 +52,12 @@ def init_user_inputs(self, inputs: dict) -> None: item_title="Crypto currency", title="Crypto currencies to watch." ) - # init one user input to generate user input schema and default values - cryptocurrencies.append(self._init_cryptocurrencies(inputs, "Bitcoin", ["Bitcoin"])) + # Register nested OBJECT_ARRAY item schema/defaults. + # Commons ensures a placeholder at [0] when the array is empty (no pre-append). + # Nested user_input prefers saved values over these def_vals, so existing + # user config is not replaced. Do not assign to [0] / append the init result + # (append after ensure would duplicate the row). + self._init_cryptocurrencies(inputs, "Bitcoin", ["Bitcoin"]) # remove other symbols data to avoid unnecessary entries self.subreddits_by_cryptocurrency = self._get_config_elements(config_cryptocurrencies, CONFIG_REDDIT_SUBREDDITS) self.feed_config[services_constants.CONFIG_REDDIT_SUBREDDITS] = self.subreddits_by_cryptocurrency diff --git a/packages/tentacles/Evaluator/Social/forum_evaluator/tests/test_reddit_forum_evaluator_init_user_inputs.py b/packages/tentacles/Evaluator/Social/forum_evaluator/tests/test_reddit_forum_evaluator_init_user_inputs.py new file mode 100644 index 0000000000..6e597ddccd --- /dev/null +++ b/packages/tentacles/Evaluator/Social/forum_evaluator/tests/test_reddit_forum_evaluator_init_user_inputs.py @@ -0,0 +1,63 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import copy + +import octobot_commons.constants as commons_constants +import tentacles.Evaluator.Social as Social +import tentacles.Evaluator.Social.forum_evaluator.forum as forum_module +import tests.test_utils.config as test_utils_config + + +def _create_forum_evaluator(specific_config: dict) -> Social.RedditForumEvaluator: + evaluator = Social.RedditForumEvaluator(test_utils_config.load_test_tentacles_config()) + evaluator.specific_config = specific_config + return evaluator + + +class TestRedditForumEvaluatorInitUserInputs: + def test_seeds_default_cryptocurrency_when_config_empty(self): + evaluator = _create_forum_evaluator({}) + evaluator.init_user_inputs({}) + + cryptocurrencies = evaluator.specific_config[commons_constants.CONFIG_CRYPTO_CURRENCIES] + assert len(cryptocurrencies) == 1 + assert cryptocurrencies[0][commons_constants.CONFIG_CRYPTO_CURRENCY] == "Bitcoin" + assert cryptocurrencies[0][forum_module.CONFIG_REDDIT_SUBREDDITS] == ["Bitcoin"] + + def test_preserves_existing_user_cryptocurrencies(self): + user_cryptocurrencies = [ + { + commons_constants.CONFIG_CRYPTO_CURRENCY: "Ethereum", + forum_module.CONFIG_REDDIT_SUBREDDITS: ["ethereum"], + }, + { + commons_constants.CONFIG_CRYPTO_CURRENCY: "NEO", + forum_module.CONFIG_REDDIT_SUBREDDITS: ["NEO"], + }, + ] + evaluator = _create_forum_evaluator( + { + commons_constants.CONFIG_CRYPTO_CURRENCIES: copy.deepcopy(user_cryptocurrencies), + } + ) + evaluator.init_user_inputs({}) + + cryptocurrencies = evaluator.specific_config[commons_constants.CONFIG_CRYPTO_CURRENCIES] + assert len(cryptocurrencies) == 2 + assert cryptocurrencies[0][commons_constants.CONFIG_CRYPTO_CURRENCY] == "Ethereum" + assert cryptocurrencies[0][forum_module.CONFIG_REDDIT_SUBREDDITS] == ["ethereum"] + assert cryptocurrencies[1][commons_constants.CONFIG_CRYPTO_CURRENCY] == "NEO" + assert cryptocurrencies[1][forum_module.CONFIG_REDDIT_SUBREDDITS] == ["NEO"] diff --git a/packages/tentacles/Evaluator/Social/news_evaluator/news.py b/packages/tentacles/Evaluator/Social/news_evaluator/news.py index 5f137a8144..e2ca4a9754 100644 --- a/packages/tentacles/Evaluator/Social/news_evaluator/news.py +++ b/packages/tentacles/Evaluator/Social/news_evaluator/news.py @@ -53,8 +53,14 @@ def init_user_inputs(self, inputs: dict) -> None: item_title="Crypto currency", title="Crypto currencies to watch." ) - # init one user input to generate user input schema and default values - cryptocurrencies.append(self._init_cryptocurrencies(inputs, "Bitcoin", ["BTCFoundation"], [])) + # Register nested OBJECT_ARRAY item schema/defaults. + # Commons ensures a placeholder at [0] when the array is empty (no pre-append). + # Nested user_input prefers saved values over these def_vals, so existing + # user config is not replaced. Do not assign to [0] / append the init result + # (append after ensure would duplicate the row). + self._init_cryptocurrencies( + inputs, "Bitcoin", ["BTCFoundation"], [] + ) # remove other symbols data to avoid unnecessary entries self.accounts_by_cryptocurrency = self._get_config_elements(config_cryptocurrencies, services_constants.CONFIG_TWITTERS_ACCOUNTS) diff --git a/packages/tentacles/Evaluator/Social/signal_evaluator/signal.py b/packages/tentacles/Evaluator/Social/signal_evaluator/signal.py index f870917145..934a995fe9 100644 --- a/packages/tentacles/Evaluator/Social/signal_evaluator/signal.py +++ b/packages/tentacles/Evaluator/Social/signal_evaluator/signal.py @@ -125,8 +125,14 @@ def init_user_inputs(self, inputs: dict) -> None: channels, inputs, item_title="Channel", other_schema_values={"minItems": 1, "uniqueItems": True}, title="Channels to watch") - channels.append(self._init_channel_config(inputs, "Test-Channel", "Pair: (.*)$", - "Side: (BUY)$", "Side: (SELL)$")) + # Register nested OBJECT_ARRAY item schema/defaults. + # Commons ensures a placeholder at [0] when the array is empty (no pre-append). + # Nested user_input prefers saved values over these def_vals, so existing + # user config is not replaced. Do not assign to [0] / append the init result + # (append after ensure would duplicate the row). + self._init_channel_config( + inputs, "Test-Channel", "Pair: (.*)$", "Side: (BUY)$", "Side: (SELL)$" + ) self.channels_config_by_channel_name = { channel[self.SIGNAL_CHANNEL_NAME_KEY]: channel for channel in config_channels diff --git a/packages/tentacles/Evaluator/Social/signal_evaluator/tests/test_telegram_channel_signal_evaluator_init_user_inputs.py b/packages/tentacles/Evaluator/Social/signal_evaluator/tests/test_telegram_channel_signal_evaluator_init_user_inputs.py new file mode 100644 index 0000000000..54a87cf58d --- /dev/null +++ b/packages/tentacles/Evaluator/Social/signal_evaluator/tests/test_telegram_channel_signal_evaluator_init_user_inputs.py @@ -0,0 +1,88 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import copy + +import octobot_services.constants as services_constants +import tentacles.Evaluator.Social as Social +import tests.test_utils.config as test_utils_config + + +def _create_signal_evaluator(specific_config: dict) -> Social.TelegramChannelSignalEvaluator: + evaluator = Social.TelegramChannelSignalEvaluator( + test_utils_config.load_test_tentacles_config() + ) + evaluator.specific_config = specific_config + return evaluator + + +class TestTelegramChannelSignalEvaluatorInitUserInputs: + def test_seeds_default_channel_when_config_empty(self): + evaluator = _create_signal_evaluator({}) + evaluator.init_user_inputs({}) + + channels = evaluator.specific_config[services_constants.CONFIG_TELEGRAM_CHANNEL] + assert len(channels) == 1 + channel = channels[0] + assert channel[evaluator.SIGNAL_CHANNEL_NAME_KEY] == "Test-Channel" + assert channel[evaluator.SIGNAL_PAIR_KEY] == "Pair: (.*)$" + assert channel[evaluator.SIGNAL_PATTERN_KEY] == { + evaluator.SIGNAL_PATTERN_MARKET_BUY_KEY: "Side: (BUY)$", + evaluator.SIGNAL_PATTERN_MARKET_SELL_KEY: "Side: (SELL)$", + } + assert list(evaluator.channels_config_by_channel_name) == ["Test-Channel"] + + def test_preserves_existing_user_channels(self): + channel_name_key = Social.TelegramChannelSignalEvaluator.SIGNAL_CHANNEL_NAME_KEY + signal_pair_key = Social.TelegramChannelSignalEvaluator.SIGNAL_PAIR_KEY + signal_pattern_key = Social.TelegramChannelSignalEvaluator.SIGNAL_PATTERN_KEY + buy_key = Social.TelegramChannelSignalEvaluator.SIGNAL_PATTERN_MARKET_BUY_KEY + sell_key = Social.TelegramChannelSignalEvaluator.SIGNAL_PATTERN_MARKET_SELL_KEY + user_channels = [ + { + channel_name_key: "My-Chan", + signal_pair_key: "X:(.*)", + signal_pattern_key: { + buy_key: "buy-regex", + sell_key: "sell-regex", + }, + }, + { + channel_name_key: "Other-Chan", + signal_pair_key: "Y:(.*)", + signal_pattern_key: { + buy_key: "other-buy", + sell_key: "other-sell", + }, + }, + ] + evaluator = _create_signal_evaluator( + { + services_constants.CONFIG_TELEGRAM_CHANNEL: copy.deepcopy(user_channels), + } + ) + evaluator.init_user_inputs({}) + + channels = evaluator.specific_config[services_constants.CONFIG_TELEGRAM_CHANNEL] + assert len(channels) == 2 + assert channels[0][channel_name_key] == "My-Chan" + assert channels[0][signal_pair_key] == "X:(.*)" + assert channels[0][signal_pattern_key] == { + buy_key: "buy-regex", + sell_key: "sell-regex", + } + assert channels[1][channel_name_key] == "Other-Chan" + assert "Test-Channel" not in evaluator.channels_config_by_channel_name + assert set(evaluator.channels_config_by_channel_name) == {"My-Chan", "Other-Chan"} diff --git a/packages/tentacles/Evaluator/Social/trends_evaluator/trends.py b/packages/tentacles/Evaluator/Social/trends_evaluator/trends.py index 869dd67f19..efe5fb4b66 100644 --- a/packages/tentacles/Evaluator/Social/trends_evaluator/trends.py +++ b/packages/tentacles/Evaluator/Social/trends_evaluator/trends.py @@ -44,7 +44,8 @@ def init_user_inputs(self, inputs: dict) -> None: self.relevant_history_months, inputs, min_val=3, max_val=3, title="Number of months to look into to compute the trend " "evaluation (for now works only with 3).") - self.feed_config[services_constants.CONFIG_TREND_TOPICS] = self._build_trend_topics() + if self.cryptocurrency_name: + self.feed_config[services_constants.CONFIG_TREND_TOPICS] = self._build_trend_topics() @classmethod def get_is_cryptocurrencies_wildcard(cls) -> bool: diff --git a/packages/tentacles/Evaluator/Strategies/mixed_strategies_evaluator/mixed_strategies.py b/packages/tentacles/Evaluator/Strategies/mixed_strategies_evaluator/mixed_strategies.py index f2503e6c65..b56bacb45d 100644 --- a/packages/tentacles/Evaluator/Strategies/mixed_strategies_evaluator/mixed_strategies.py +++ b/packages/tentacles/Evaluator/Strategies/mixed_strategies_evaluator/mixed_strategies.py @@ -18,7 +18,6 @@ import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums import octobot_commons.evaluators_util as evaluators_util -import octobot_commons.time_frame_manager as time_frame_manager import octobot_evaluators.api as evaluators_api import octobot_evaluators.evaluators.channel as evaluators_channel import octobot_evaluators.matrix as matrix @@ -27,7 +26,6 @@ import octobot_evaluators.errors as errors import octobot_evaluators.evaluators as evaluators import octobot_tentacles_manager.api.configurator as tentacles_manager_api -import octobot_tentacles_manager.configuration as tm_configuration import octobot_trading.api as trading_api @@ -256,11 +254,15 @@ def __init__(self, tentacles_setup_config): super().__init__(tentacles_setup_config) self.allowed_evaluator_types = [evaluators_enums.EvaluatorMatrixTypes.TA.value, evaluators_enums.EvaluatorMatrixTypes.REAL_TIME.value] - config = tentacles_manager_api.get_tentacle_config(self.tentacles_setup_config, self.__class__) - if config: - self.weight_by_time_frames = TechnicalAnalysisStrategyEvaluator._get_weight_by_time_frames( - config[TechnicalAnalysisStrategyEvaluator.TIME_FRAMES_TO_WEIGHT] + self.weight_by_time_frames = {} + if self.tentacles_setup_config is not None: + config = tentacles_manager_api.get_tentacle_config( + self.tentacles_setup_config, self.__class__ ) + if config and self.TIME_FRAMES_TO_WEIGHT in config: + self.weight_by_time_frames = TechnicalAnalysisStrategyEvaluator._get_weight_by_time_frames( + config[TechnicalAnalysisStrategyEvaluator.TIME_FRAMES_TO_WEIGHT] + ) def init_user_inputs(self, inputs: dict) -> None: """ @@ -275,8 +277,14 @@ def init_user_inputs(self, inputs: dict) -> None: item_title="Time frame", title="Analysed time frames and their associated weight." ) - # init one user input to generate user input schema and default values - time_frames_and_weight.append(self._init_tf_and_weight(inputs, commons_enums.TimeFrames.THIRTY_MINUTES, 30)) + # Register nested OBJECT_ARRAY item schema/defaults. + # Commons ensures a placeholder at [0] when the array is empty (no pre-append). + # Nested user_input prefers saved values over these def_vals, so existing + # user config is not replaced. Do not assign to [0] / append the init result + # (append after ensure would duplicate the row). + self._init_tf_and_weight( + inputs, commons_enums.TimeFrames.THIRTY_MINUTES, 30 + ) self.weight_by_time_frames = TechnicalAnalysisStrategyEvaluator._get_weight_by_time_frames( config_time_frames_and_weight ) diff --git a/packages/tentacles/Evaluator/Strategies/mixed_strategies_evaluator/tests/test_technical_analysis_strategy_init_user_inputs.py b/packages/tentacles/Evaluator/Strategies/mixed_strategies_evaluator/tests/test_technical_analysis_strategy_init_user_inputs.py new file mode 100644 index 0000000000..d12bdcc17d --- /dev/null +++ b/packages/tentacles/Evaluator/Strategies/mixed_strategies_evaluator/tests/test_technical_analysis_strategy_init_user_inputs.py @@ -0,0 +1,75 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import copy + +import octobot_commons.enums as commons_enums +import tentacles.Evaluator.Strategies as Strategies +import tests.test_utils.config as test_utils_config + + +def _create_tas_evaluator( + specific_config: dict, +) -> Strategies.TechnicalAnalysisStrategyEvaluator: + evaluator = Strategies.TechnicalAnalysisStrategyEvaluator( + test_utils_config.load_test_tentacles_config() + ) + evaluator.specific_config = specific_config + return evaluator + + +class TestTechnicalAnalysisStrategyEvaluatorInitUserInputs: + def test_seeds_default_time_frame_weight_when_config_empty(self): + evaluator = _create_tas_evaluator({}) + evaluator.init_user_inputs({}) + + time_frames = evaluator.specific_config[evaluator.TIME_FRAMES_TO_WEIGHT] + assert len(time_frames) == 1 + assert time_frames[0][evaluator.TIME_FRAME] == commons_enums.TimeFrames.THIRTY_MINUTES.value + assert time_frames[0][evaluator.WEIGHT] == 30 + assert evaluator.weight_by_time_frames == { + commons_enums.TimeFrames.THIRTY_MINUTES.value: 30 + } + + def test_preserves_existing_user_time_frame_weights(self): + user_time_frames = [ + { + Strategies.TechnicalAnalysisStrategyEvaluator.TIME_FRAME: commons_enums.TimeFrames.ONE_HOUR.value, + Strategies.TechnicalAnalysisStrategyEvaluator.WEIGHT: 70, + }, + { + Strategies.TechnicalAnalysisStrategyEvaluator.TIME_FRAME: commons_enums.TimeFrames.FOUR_HOURS.value, + Strategies.TechnicalAnalysisStrategyEvaluator.WEIGHT: 20, + }, + ] + evaluator = _create_tas_evaluator( + { + Strategies.TechnicalAnalysisStrategyEvaluator.TIME_FRAMES_TO_WEIGHT: copy.deepcopy( + user_time_frames + ) + } + ) + evaluator.init_user_inputs({}) + + time_frames = evaluator.specific_config[evaluator.TIME_FRAMES_TO_WEIGHT] + assert len(time_frames) == 2 + assert time_frames[0][evaluator.TIME_FRAME] == commons_enums.TimeFrames.ONE_HOUR.value + assert time_frames[0][evaluator.WEIGHT] == 70 + assert time_frames[1][evaluator.TIME_FRAME] == commons_enums.TimeFrames.FOUR_HOURS.value + assert time_frames[1][evaluator.WEIGHT] == 20 + assert evaluator.weight_by_time_frames == { + commons_enums.TimeFrames.ONE_HOUR.value: 70, + commons_enums.TimeFrames.FOUR_HOURS.value: 20, + } diff --git a/packages/tentacles/Evaluator/TA/momentum_evaluator/momentum.py b/packages/tentacles/Evaluator/TA/momentum_evaluator/momentum.py index 843b240be2..fdb1ea3b4a 100644 --- a/packages/tentacles/Evaluator/TA/momentum_evaluator/momentum.py +++ b/packages/tentacles/Evaluator/TA/momentum_evaluator/momentum.py @@ -232,15 +232,22 @@ def init_user_inputs(self, inputs: dict) -> None: title="Number of recent RSI values to consider to get the current fast " "moving market sentiment.") weights = [] - self.weights = sorted( - self.UI.user_input(self.RSI_TO_WEIGHTS, enums.UserInputTypes.OBJECT_ARRAY, weights, inputs, - item_title="Slow RSI interpretation", - other_schema_values={"minItems": 1, "uniqueItems": True}, - title="RSI values and interpretations."), - key=lambda a: a[self.SLOW_THRESHOLD] + config_weights = self.UI.user_input( + self.RSI_TO_WEIGHTS, enums.UserInputTypes.OBJECT_ARRAY, weights, inputs, + item_title="Slow RSI interpretation", + other_schema_values={"minItems": 1, "uniqueItems": True}, + title="RSI values and interpretations.", ) - # init one user input to generate user input schema and default values - weights.append(self._init_RSI_to_weight(inputs, 30, [[20, 2, 2]])) + # Register nested OBJECT_ARRAY item schema/defaults. + # Commons ensures a placeholder at [0] when the array is empty (no pre-append). + # Nested user_input prefers saved values over these def_vals, so existing + # user config is not replaced. For this multi-level nested array, assign the + # full init tree only when config was empty (do not append after ensure). + config_was_empty = not config_weights + seeded_weight = self._init_RSI_to_weight(inputs, 30, [[20, 2, 2]]) + if config_was_empty: + config_weights[0] = seeded_weight + self.weights = sorted(config_weights, key=lambda a: a[self.SLOW_THRESHOLD]) for i, fast_threshold in enumerate(self.weights): fast_threshold[self.FAST_THRESHOLDS] = sorted(fast_threshold[self.FAST_THRESHOLDS], diff --git a/packages/tentacles/Evaluator/TA/momentum_evaluator/tests/test_rsi_weight_momentum_evaluator_init_user_inputs.py b/packages/tentacles/Evaluator/TA/momentum_evaluator/tests/test_rsi_weight_momentum_evaluator_init_user_inputs.py new file mode 100644 index 0000000000..a9bacd45f5 --- /dev/null +++ b/packages/tentacles/Evaluator/TA/momentum_evaluator/tests/test_rsi_weight_momentum_evaluator_init_user_inputs.py @@ -0,0 +1,81 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import copy + +import tentacles.Evaluator.TA as TA +import tests.test_utils.config as test_utils_config + + +def _create_rsi_weight_evaluator( + specific_config: dict, +) -> TA.RSIWeightMomentumEvaluator: + evaluator = TA.RSIWeightMomentumEvaluator(test_utils_config.load_test_tentacles_config()) + evaluator.specific_config = specific_config + return evaluator + + +class TestRSIWeightMomentumEvaluatorInitUserInputs: + def test_seeds_default_weight_when_config_empty(self): + evaluator = _create_rsi_weight_evaluator({}) + evaluator.init_user_inputs({}) + + weights = evaluator.specific_config[evaluator.RSI_TO_WEIGHTS] + assert len(weights) == 1 + assert weights[0][evaluator.SLOW_THRESHOLD] == 30 + assert len(weights[0][evaluator.FAST_THRESHOLDS]) == 1 + assert weights[0][evaluator.FAST_THRESHOLDS][0][evaluator.FAST_THRESHOLD] == 20 + assert weights[0][evaluator.FAST_THRESHOLDS][0][evaluator.WEIGHTS] == { + evaluator.PRICE: 2, + evaluator.VOLUME: 2, + } + assert evaluator.weights == weights + + def test_preserves_existing_user_weights(self): + user_weights = [ + { + "slow_threshold": 35, + "fast_thresholds": [ + { + "fast_threshold": 25, + "weights": {"price": 3, "volume": 1}, + } + ], + }, + { + "slow_threshold": 40, + "fast_thresholds": [ + { + "fast_threshold": 15, + "weights": {"price": 1, "volume": 2}, + } + ], + }, + ] + evaluator = _create_rsi_weight_evaluator( + {TA.RSIWeightMomentumEvaluator.RSI_TO_WEIGHTS: copy.deepcopy(user_weights)} + ) + evaluator.init_user_inputs({}) + + weights = evaluator.specific_config[evaluator.RSI_TO_WEIGHTS] + assert len(weights) == 2 + assert weights[0][evaluator.SLOW_THRESHOLD] == 35 + assert weights[0][evaluator.FAST_THRESHOLDS][0][evaluator.FAST_THRESHOLD] == 25 + assert weights[0][evaluator.FAST_THRESHOLDS][0][evaluator.WEIGHTS] == { + evaluator.PRICE: 3, + evaluator.VOLUME: 1, + } + assert weights[1][evaluator.SLOW_THRESHOLD] == 40 + assert [weight[evaluator.SLOW_THRESHOLD] for weight in evaluator.weights] == [35, 40] diff --git a/packages/tentacles/Meta/DSL_operators/automation_operators/automation_management.py b/packages/tentacles/Meta/DSL_operators/automation_operators/automation_management.py index 00b4881e77..1a7e9be0ae 100644 --- a/packages/tentacles/Meta/DSL_operators/automation_operators/automation_management.py +++ b/packages/tentacles/Meta/DSL_operators/automation_operators/automation_management.py @@ -13,8 +13,8 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter as dsl_interpreter -import octobot_commons.dsl_interpreter.operator_parameter as operator_parameter import octobot_flow.entities @@ -24,11 +24,19 @@ class StopAutomationOperator(dsl_interpreter.CallOperator): MAX_PARAMS = 0 DESCRIPTION = "Signals the automation to stop." EXAMPLE = "stop_automation()" + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value @staticmethod def get_name() -> str: return "stop_automation" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Automation stop signal", + ) + def compute(self) -> dict: return { octobot_flow.entities.PostIterationActionsDetails.__name__: @@ -47,15 +55,16 @@ class UpdateAutomationConfigurationOperator(dsl_interpreter.CallOperator): "executable DSL action can be retargeted." ) EXAMPLE = 'update_automation_configuration("your_dsl_call(...)")' + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value @staticmethod def get_name() -> str: return "update_automation_configuration" @classmethod - def get_parameters(cls) -> list[operator_parameter.OperatorParameter]: + def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - operator_parameter.OperatorParameter( + dsl_interpreter.OperatorParameter( name="configuration_update", description=( "Full replacement DSL for the single currently executable DAG script action " @@ -64,11 +73,18 @@ def get_parameters(cls) -> list[operator_parameter.OperatorParameter]: "exchange calls, etc.)." ), required=True, - type=str, + type=commons_enums.DslValueType.TEXT.value, default=None, ), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Automation configuration update signal", + ) + def compute(self) -> dict: configuration_update = self.get_computed_value_by_parameter()["configuration_update"] return { diff --git a/packages/tentacles/Meta/DSL_operators/automation_operators/tests/test_automation_management.py b/packages/tentacles/Meta/DSL_operators/automation_operators/tests/test_automation_management.py index 192102161c..885fbe7296 100644 --- a/packages/tentacles/Meta/DSL_operators/automation_operators/tests/test_automation_management.py +++ b/packages/tentacles/Meta/DSL_operators/automation_operators/tests/test_automation_management.py @@ -40,7 +40,7 @@ def _assert_stop_automation_result(result): assert details.stop_automation is True -_SAMPLE_CONFIGURATION_UPDATE_DSL = 'run_octobot_process("u", {})' +_SAMPLE_CONFIGURATION_UPDATE_DSL = 'run_octobot_process("u", profile_data={})' def _assert_update_automation_configuration_result(result, expected_configuration_update: str): diff --git a/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py b/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py index 4ce581af7c..a51b81293d 100644 --- a/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py +++ b/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py @@ -17,6 +17,7 @@ import dataclasses import decimal +import octobot_commons.enums as commons_enums import octobot_commons.dataclasses import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter @@ -51,6 +52,8 @@ class TransferFundsParams(octobot_commons.dataclasses.FlexibleDataclass): class BlockchainWalletOperator(dsl_interpreter.PreComputingCallOperator): + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value + @staticmethod def get_library() -> str: # this is a contextual operator, so it should not be included by default in the get_all_operators function return values @@ -59,8 +62,16 @@ def get_library() -> str: @classmethod def get_blockchain_wallet_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="blockchain_descriptor", description="descriptor of the blockchain to use as in octobot_trading.blockchain_wallets.BlockchainDescriptor", required=True, type=dict), - dsl_interpreter.OperatorParameter(name="wallet_descriptor", description="descriptor of the wallet to use as in octobot_trading.blockchain_wallets.WalletDescriptor", required=True, type=dict), + dsl_interpreter.OperatorParameter( + name="blockchain_descriptor", + description="descriptor of the blockchain to use as in octobot_trading.blockchain_wallets.BlockchainDescriptor", + required=True, + type=commons_enums.DslValueType.DICT.value), + dsl_interpreter.OperatorParameter( + name="wallet_descriptor", + description="descriptor of the wallet to use as in octobot_trading.blockchain_wallets.WalletDescriptor", + required=True, + type=commons_enums.DslValueType.DICT.value), ] @@ -71,6 +82,7 @@ def create_blockchain_wallet_operators( class _BlockchainWalletBalanceOperator(BlockchainWalletOperator): DESCRIPTION = "Returns the balance of the asset in the blockchain wallet" EXAMPLE = "blockchain_wallet_balance({blockchain_descriptor}, {wallet_descriptor}, 'BTC')" + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value @staticmethod def get_name() -> str: @@ -80,9 +92,20 @@ def get_name() -> str: def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ *cls.get_blockchain_wallet_parameters(), - dsl_interpreter.OperatorParameter(name="asset", description="the asset to get the balance for", required=True, type=str), + dsl_interpreter.OperatorParameter( + name="asset", + description="the asset to get the balance for", + required=True, + type=commons_enums.DslValueType.TEXT.value), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Free wallet balance for the asset", + ) + async def pre_compute(self) -> None: param_by_name = self.get_computed_value_by_parameter() blockchain_wallet_balance_params = BlockchainWalletBalanceParams.from_dict(param_by_name) @@ -112,12 +135,37 @@ def get_name() -> str: def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ *cls.get_blockchain_wallet_parameters(), - dsl_interpreter.OperatorParameter(name="asset", description="the asset to transfer", required=True, type=str), - dsl_interpreter.OperatorParameter(name="amount", description="the amount to transfer", required=True, type=float), - dsl_interpreter.OperatorParameter(name="address", description="the address to transfer to", required=False, type=str, default=None), - dsl_interpreter.OperatorParameter(name="destination_exchange", description="the exchange to transfer to", required=False, type=str, default=None), + dsl_interpreter.OperatorParameter( + name="asset", + description="the asset to transfer", + required=True, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="amount", + description="the amount to transfer", + required=True, + type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter( + name="address", + description="the address to transfer to", + required=False, + type=commons_enums.DslValueType.TEXT.value, + default=None), + dsl_interpreter.OperatorParameter( + name="destination_exchange", + description="the exchange to transfer to", + required=False, + type=commons_enums.DslValueType.TEXT.value, + default=None), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Created transfer transactions", + ) + async def pre_compute(self) -> None: await super().pre_compute() param_by_name = self.get_computed_value_by_parameter() @@ -160,6 +208,13 @@ def get_name() -> str: def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return cls.get_blockchain_wallet_parameters() + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Open wallet details", + ) + async def pre_compute(self) -> None: param_by_name = self.get_computed_value_by_parameter() async with octobot_trading.api.blockchain_wallet_context( diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/cancel_order_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/cancel_order_operators.py index e8334b86c4..92fa1b55b7 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/cancel_order_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/cancel_order_operators.py @@ -15,7 +15,8 @@ # License along with this library. import typing -import octobot_commons.constants +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.signals @@ -41,6 +42,7 @@ def create_cancel_order_operators( class _CancelOrderOperator(exchange_operator.ExchangeOperator): DESCRIPTION = "Cancels one or many orders" EXAMPLE = "cancel_order('BTC/USDT', side='buy')" + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value @staticmethod def get_name() -> str: @@ -49,17 +51,44 @@ def get_name() -> str: @staticmethod def get_library() -> str: # this is a contextual operator, so it should not be included by default in the get_all_operators function return values - return octobot_commons.constants.CONTEXTUAL_OPERATORS_LIBRARY + return commons_constants.CONTEXTUAL_OPERATORS_LIBRARY @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="symbol", description="the symbol of the orders to cancel", required=True, type=str, default=None), - dsl_interpreter.OperatorParameter(name="side", description="the side of the orders to cancel", required=False, type=str, default=None), - dsl_interpreter.OperatorParameter(name="tag", description="the tag of the orders to cancel", required=False, type=str, default=None), - dsl_interpreter.OperatorParameter(name="exchange_order_ids", description="the exchange id of the orders to cancel", required=False, type=list[str], default=None), + dsl_interpreter.OperatorParameter( + name="symbol", + description="the symbol of the orders to cancel", + required=True, + type=commons_enums.DslValueType.TEXT.value, + default=None), + dsl_interpreter.OperatorParameter( + name="side", + description="the side of the orders to cancel", + required=False, + type=commons_enums.DslValueType.TEXT.value, + default=None), + dsl_interpreter.OperatorParameter( + name="tag", + description="the tag of the orders to cancel", + required=False, + type=commons_enums.DslValueType.TEXT.value, + default=None), + dsl_interpreter.OperatorParameter( + name="exchange_order_ids", + description="the exchange id of the orders to cancel", + required=False, + type=commons_enums.DslValueType.ANY.value, + default=None), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Cancelled orders result", + ) + def get_dependencies(self) -> typing.List[dsl_interpreter.InterpreterDependency]: local_dependencies = [] if symbol := self.get_input_value_by_parameter().get("symbol"): diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py index 15e2541964..ff0dfd4496 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py @@ -20,6 +20,7 @@ import enum import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors as commons_errors @@ -62,6 +63,7 @@ class _CopyExchangeAccountOperator(dsl_interpreter.PreComputingCallOperator, dsl r"""reference_account='{"version":"1.0.0","updated_at":1710000000,"copied_assets":[{"name":"BTC","total":1,"available":1,"ratio":1}]}', """ r"""account_copy_settings='{"reference_market_ratio":"1","allow_skip_asset":false}')""" ) + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value @staticmethod def get_library() -> str: @@ -80,14 +82,12 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "Identifier of the copied community strategy." ), required=True, - type=str, - ), + type=commons_enums.DslValueType.TEXT.value), dsl_interpreter.OperatorParameter( name="reference_market", description="Quote asset symbol for rebalance (e.g. USDT).", required=True, - type=str, - ), + type=commons_enums.DslValueType.TEXT.value), dsl_interpreter.OperatorParameter( name="reference_account", description=( @@ -95,7 +95,7 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "(list of {name, total, available, ratio}); optional orders, positions, historical_snapshots." ), required=True, - type=str, + type=commons_enums.DslValueType.TEXT.value, ), dsl_interpreter.OperatorParameter( name="account_copy_settings", @@ -107,10 +107,17 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "(omit keys to use defaults)." ), required=False, - type=str, + type=commons_enums.DslValueType.TEXT.value, ), ] + super().get_re_callable_parameters() + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Re-callable copy result including created orders", + ) + def _parse_reference_account(self, raw: typing.Any) -> protocol_models.CopiedAccount: if raw is None: raise commons_errors.InvalidParameterFormatError("reference_account is required") diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/create_order_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/create_order_operators.py index 88df5176d0..097c8ec24d 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/create_order_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/create_order_operators.py @@ -17,7 +17,8 @@ import asyncio import json -import octobot_commons.constants +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.signals as commons_signals import octobot_commons.dsl_interpreter as dsl_interpreter @@ -35,7 +36,6 @@ CREATED_ORDERS_KEY = "created_orders" - _CANCEL_POLICIES_CACHE = {} def _parse_cancel_policy(kwargs: dict) -> typing.Optional[octobot_trading.personal_data.OrderCancelPolicy]: if policy := kwargs.get("cancel_policy"): @@ -58,6 +58,8 @@ def _parse_cancel_policy(kwargs: dict) -> typing.Optional[octobot_trading.person class CreateOrderOperator(exchange_operator.ExchangeOperator): + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value + def __init__(self, *parameters: dsl_interpreter.OperatorParameterType, **kwargs: typing.Any): super().__init__(*parameters, **kwargs) self.param_by_name: dict[str, dsl_interpreter.ComputedOperatorParameterType] = dsl_interpreter.UNINITIALIZED_VALUE # type: ignore @@ -65,7 +67,7 @@ def __init__(self, *parameters: dsl_interpreter.OperatorParameterType, **kwargs: @staticmethod def get_library() -> str: # this is a contextual operator, so it should not be included by default in the get_all_operators function return values - return octobot_commons.constants.CONTEXTUAL_OPERATORS_LIBRARY + return commons_constants.CONTEXTUAL_OPERATORS_LIBRARY @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: @@ -78,9 +80,22 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: @classmethod def get_first_required_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="side", description="the side of the order", required=True, type=str), - dsl_interpreter.OperatorParameter(name="symbol", description="the symbol of the order", required=True, type=str), - dsl_interpreter.OperatorParameter(name="amount", description="the amount of the order", required=True, type=float), + dsl_interpreter.OperatorParameter( + name="side", + description="the side of the order", + required=True, + type=commons_enums.DslValueType.TEXT.value, + options=octobot_trading.enums.TRADE_ORDER_SIDE_OPERATOR_PARAMETER_OPTIONS), + dsl_interpreter.OperatorParameter( + name="symbol", + description="the symbol of the order", + required=True, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="amount", + description="the amount of the order", + required=True, + type=commons_enums.DslValueType.TEXT.value), ] @classmethod @@ -90,20 +105,75 @@ def get_second_required_parameters(cls) -> list[dsl_interpreter.OperatorParamete @classmethod def get_last_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="reduce_only", description="whether the order is reduce only", required=False, type=bool), - dsl_interpreter.OperatorParameter(name="tag", description="the tag of the order", required=False, type=str), - dsl_interpreter.OperatorParameter(name="take_profit_prices", description="the price or price offset of the take profit order(s)", required=False, type=list[str]), - dsl_interpreter.OperatorParameter(name="take_profit_volume_percents", description="% volume of the entry for each take profit", required=False, type=list[float]), - dsl_interpreter.OperatorParameter(name="stop_loss_price", description="the stop loss price or price offset of the order", required=False, type=str), - dsl_interpreter.OperatorParameter(name="trailing_profile", description="the trailing profile of the order", required=False, type=dict), - dsl_interpreter.OperatorParameter(name="cancel_policy", description="the cancel policy of the order", required=False, type=str), - dsl_interpreter.OperatorParameter(name="cancel_policy_params", description="the cancel policy params of the order", required=False, type=dict), - dsl_interpreter.OperatorParameter(name="active_order_swap_strategy", description="the type of the active order swap strategy", required=False, type=str), - dsl_interpreter.OperatorParameter(name="active_order_swap_strategy_params", description="the params of the active order swap strategy", required=False, type=dict), - dsl_interpreter.OperatorParameter(name="params", description="additional params for the order", required=False, type=dict), - dsl_interpreter.OperatorParameter(name="allow_holdings_adaptation", description="allow reducing the order amount to account for available holdings", required=False, type=bool), + dsl_interpreter.OperatorParameter( + name="reduce_only", + description="whether the order is reduce only", + required=False, + type=commons_enums.DslValueType.BOOLEAN.value), + dsl_interpreter.OperatorParameter( + name="tag", + description="the tag of the order", + required=False, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="take_profit_prices", + description="the price or price offset of the take profit order(s)", + required=False, + type=commons_enums.DslValueType.ANY.value), + dsl_interpreter.OperatorParameter( + name="take_profit_volume_percents", + description="% volume of the entry for each take profit", + required=False, + type=commons_enums.DslValueType.ANY.value), + dsl_interpreter.OperatorParameter( + name="stop_loss_price", + description="the stop loss price or price offset of the order", + required=False, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="trailing_profile", + description="the trailing profile of the order", + required=False, + type=commons_enums.DslValueType.ANY.value), + dsl_interpreter.OperatorParameter( + name="cancel_policy", + description="the cancel policy of the order", + required=False, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="cancel_policy_params", + description="the cancel policy params of the order", + required=False, + type=commons_enums.DslValueType.DICT.value), + dsl_interpreter.OperatorParameter( + name="active_order_swap_strategy", + description="the type of the active order swap strategy", + required=False, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="active_order_swap_strategy_params", + description="the params of the active order swap strategy", + required=False, + type=commons_enums.DslValueType.DICT.value), + dsl_interpreter.OperatorParameter( + name="params", + description="additional params for the order", + required=False, + type=commons_enums.DslValueType.DICT.value), + dsl_interpreter.OperatorParameter( + name="allow_holdings_adaptation", + description="allow reducing the order amount to account for available holdings", + required=False, + type=commons_enums.DslValueType.BOOLEAN.value), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ORDER.value, + description="Created order result", + ) + def get_dependencies(self) -> typing.List[dsl_interpreter.InterpreterDependency]: local_dependencies = [] if symbol := self.get_input_value_by_parameter().get("symbol"): @@ -216,7 +286,11 @@ def get_name() -> str: @classmethod def get_second_required_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="price", description="the limit price of the order: a flat or offset price", required=True, type=str), + dsl_interpreter.OperatorParameter( + name="price", + description="the limit price of the order: a flat or offset price", + required=True, + type=commons_enums.DslValueType.TEXT.value), ] def get_order_type(self) -> octobot_trading.enums.TraderOrderType: @@ -243,7 +317,11 @@ async def pre_compute(self) -> None: @classmethod def get_second_required_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="price", description="the trigger price of the order: a flat or offset price", required=True, type=str), + dsl_interpreter.OperatorParameter( + name="price", + description="the trigger price of the order: a flat or offset price", + required=True, + type=commons_enums.DslValueType.TEXT.value), ] def get_order_type(self) -> octobot_trading.enums.TraderOrderType: diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/fetch_order_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/fetch_order_operators.py index 9cc40e8ee0..3f60e130f6 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/fetch_order_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/fetch_order_operators.py @@ -16,6 +16,7 @@ import typing import octobot_commons.constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_trading.constants as trading_constants @@ -118,6 +119,7 @@ class _FetchOrderOperator(exchange_operator.ExchangeOperator): "fetch_order('BTC/USDT', exchange_order_id='12345') " "or fetch_order('BTC/USDT', exchange_order_id='12345', raise_if_not_found=True)" ) + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value @staticmethod def get_name() -> str: @@ -131,26 +133,34 @@ def get_library() -> str: def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ dsl_interpreter.OperatorParameter( - name="symbol", description="the symbol of the order", required=True, type=str, default=None - ), + name="symbol", + description="the symbol of the order", + required=True, + type=commons_enums.DslValueType.TEXT.value, + default=None), dsl_interpreter.OperatorParameter( name="exchange_order_id", description="the exchange id of the order", required=True, - type=str, - default=None, - ), + type=commons_enums.DslValueType.TEXT.value, + default=None), dsl_interpreter.OperatorParameter( name="raise_if_not_found", description=( "if True, raise when the order cannot be resolved; if False, return None." ), required=False, - type=bool, - default=False, - ), + type=commons_enums.DslValueType.BOOLEAN.value, + default=False), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ORDER.value, + description="Fetched order, or None when not found", + ) + def get_dependencies(self) -> typing.List[dsl_interpreter.InterpreterDependency]: local_dependencies = [] if symbol := self.get_input_value_by_parameter().get("symbol"): diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/futures_contracts_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/futures_contracts_operators.py index a484b72858..c4fe756771 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/futures_contracts_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/futures_contracts_operators.py @@ -16,7 +16,8 @@ import typing import decimal -import octobot_commons.constants +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_trading.exchanges @@ -25,11 +26,20 @@ class FuturesContractsOperator(exchange_operator.ExchangeOperator): + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value + @staticmethod def get_library() -> str: # this is a contextual operator, so it should not be included by default in the get_all_operators function return values - return octobot_commons.constants.CONTEXTUAL_OPERATORS_LIBRARY - + return commons_constants.CONTEXTUAL_OPERATORS_LIBRARY + + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Futures contract action result", + ) + def create_futures_contracts_operators( exchange_manager: typing.Optional[octobot_trading.exchanges.ExchangeManager], @@ -46,8 +56,16 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="symbol", description="the symbol of the futures contract", required=True, type=str), - dsl_interpreter.OperatorParameter(name="leverage", description="the leverage to set", required=True, type=float), + dsl_interpreter.OperatorParameter( + name="symbol", + description="the symbol of the futures contract", + required=True, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="leverage", + description="the leverage to set", + required=True, + type=commons_enums.DslValueType.NUMBER.value), ] async def pre_compute(self) -> None: diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/portfolio_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/portfolio_operators.py index 7464a4e48b..6c452a7c98 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/portfolio_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/portfolio_operators.py @@ -18,7 +18,8 @@ import decimal import octobot_commons.dataclasses -import octobot_commons.constants +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_trading.personal_data @@ -42,17 +43,30 @@ class WithdrawFundsParams(octobot_commons.dataclasses.FlexibleDataclass): class PortfolioOperator(exchange_operator.ExchangeOperator): + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value + @staticmethod def get_library() -> str: # this is a contextual operator, so it should not be included by default in the get_all_operators function return values - return octobot_commons.constants.CONTEXTUAL_OPERATORS_LIBRARY + return commons_constants.CONTEXTUAL_OPERATORS_LIBRARY @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="asset", description="the asset to get the value for", required=False, type=str), + dsl_interpreter.OperatorParameter( + name="asset", + description="the asset to get the value for", + required=False, + type=commons_enums.DslValueType.TEXT.value), ] - + + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Portfolio asset amount", + ) + def create_portfolio_operators( exchange_manager: typing.Optional[octobot_trading.exchanges.ExchangeManager], @@ -94,6 +108,7 @@ async def pre_compute(self) -> None: class _WithdrawOperator(PortfolioOperator): DESCRIPTION = "Withdraws an asset from the exchange's portfolio. requires ALLOW_FUNDS_TRANSFER env to be True (disabled by default to protect funds)" EXAMPLE = "withdraw('BTC', 'ethereum', '0x1234567890abcdef1234567890abcdef12345678', 0.1)" + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value @staticmethod def get_name() -> str: @@ -102,14 +117,46 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="asset", description="the asset to withdraw", required=True, type=str), - dsl_interpreter.OperatorParameter(name="network", description="the network to withdraw to", required=True, type=str), - dsl_interpreter.OperatorParameter(name="address", description="the address to withdraw to", required=True, type=str), - dsl_interpreter.OperatorParameter(name="amount", description="the amount to withdraw", required=False, type=float, default=None), - dsl_interpreter.OperatorParameter(name="tag", description="a tag to associate with the withdrawal", required=False, type=str, default=None), - dsl_interpreter.OperatorParameter(name="params", description="extra parameters specific to the exchange API endpoint", required=False, type=dict), + dsl_interpreter.OperatorParameter( + name="asset", + description="the asset to withdraw", + required=True, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="network", + description="the network to withdraw to", + required=True, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="address", + description="the address to withdraw to", + required=True, + type=commons_enums.DslValueType.TEXT.value), + dsl_interpreter.OperatorParameter( + name="amount", + description="the amount to withdraw", + required=False, + type=commons_enums.DslValueType.NUMBER.value, + default=None), + dsl_interpreter.OperatorParameter( + name="tag", + description="a tag to associate with the withdrawal", + required=False, + type=commons_enums.DslValueType.TEXT.value, + default=None), + dsl_interpreter.OperatorParameter( + name="params", + description="extra parameters specific to the exchange API endpoint", + required=False, + type=commons_enums.DslValueType.DICT.value), ] - + + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Created withdrawal result", + ) async def pre_compute(self) -> None: await super().pre_compute() diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ohlcv_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ohlcv_operators.py index bcc9872bd7..53f382cb3e 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ohlcv_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ohlcv_operators.py @@ -18,7 +18,7 @@ import dataclasses import numpy as np -import octobot_commons.constants +import octobot_commons.constants as commons_constants import octobot_commons.errors import octobot_commons.logging import octobot_commons.enums as commons_enums @@ -42,16 +42,29 @@ def __hash__(self) -> int: class OHLCVOperator(exchange_operator.ExchangeOperator): + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value + @staticmethod def get_library() -> str: # this is a contextual operator, so it should not be included by default in the get_all_operators function return values - return octobot_commons.constants.CONTEXTUAL_OPERATORS_LIBRARY + return commons_constants.CONTEXTUAL_OPERATORS_LIBRARY @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="symbol", description="the symbol to get the OHLCV data for", required=False, type=str), - dsl_interpreter.OperatorParameter(name="time_frame", description="the time frame to get the OHLCV data for", required=False, type=str), + dsl_interpreter.OperatorParameter( + name="symbol", + description="the symbol to get the OHLCV data for", + required=False, + type=commons_enums.DslValueType.TEXT.value, + ), + dsl_interpreter.OperatorParameter( + name="time_frame", + description="the time frame to get the OHLCV data for", + required=False, + type=commons_enums.DslValueType.TIME_FRAME.value, + options=dsl_interpreter.TIME_FRAME_OPERATOR_PARAMETER_OPTIONS, + ), ] def get_symbol_and_time_frame(self) -> typing.Tuple[typing.Optional[str], typing.Optional[str]]: @@ -114,7 +127,7 @@ def _get_candles_values_with_latest_kline_if_available( # kline is an update of the last candle return _adapt_last_candle_value(candles_manager, value_type, candles_values, kline) else: - tf_seconds = commons_enums.TimeFramesMinutes[commons_enums.TimeFrames(_time_frame)] * octobot_commons.constants.MINUTE_TO_SECONDS + tf_seconds = commons_enums.TimeFramesMinutes[commons_enums.TimeFrames(_time_frame)] * commons_constants.MINUTE_TO_SECONDS if kline_time == last_candle_time + tf_seconds: # kline is a new candle kline_value = kline[value_type.value] @@ -138,6 +151,13 @@ def _static_get_dependencies() -> typing.List[ExchangeDataDependency]: class _LocalOHLCVOperator(OHLCVOperator): PRICE_INDEX: commons_enums.PriceIndexes = None # type: ignore + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.SERIES.value, + description="OHLCV candle series", + ) + def get_dependencies(self) -> typing.List[dsl_interpreter.InterpreterDependency]: local_dependencies = _static_get_dependencies() param_by_name = self.get_input_value_by_parameter() diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/price_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/price_operators.py index 0a2a619fbc..4f59e88786 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/price_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/price_operators.py @@ -17,7 +17,8 @@ import typing import decimal -import octobot_commons.constants +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_trading.exchanges @@ -29,10 +30,12 @@ class PriceOperator(exchange_operator.ExchangeOperator): + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value + @staticmethod def get_library() -> str: # this is a contextual operator, so it should not be included by default in the get_all_operators function return values - return octobot_commons.constants.CONTEXTUAL_OPERATORS_LIBRARY + return commons_constants.CONTEXTUAL_OPERATORS_LIBRARY @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: @@ -41,10 +44,16 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: name="symbol", description="the symbol to get the latest mark price for", required=False, - type=str, - ), + type=commons_enums.DslValueType.TEXT.value), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Latest mark price", + ) + def get_symbol(self) -> typing.Optional[str]: if parameters := self.get_computed_parameters(): symbol = parameters[0] if len(parameters) > 0 else None diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py index e1a6ba6bb9..b39b93c9bc 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/symbol_operators.py @@ -17,6 +17,7 @@ import typing import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_trading.enums as trading_enums @@ -72,7 +73,14 @@ class _MarketExpiryOperator(exchange_operator.ExchangeOperator): @classmethod def get_parameters(cls) -> list: - return [dsl_interpreter.OperatorParameter("symbol", "The market symbol", True, str)] + return [ + dsl_interpreter.OperatorParameter( + "symbol", + "The market symbol", + True, + commons_enums.DslValueType.TEXT.value, + ) + ] @staticmethod def get_name() -> str: diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ticker_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ticker_operators.py index 8ec2dc6908..4348090309 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ticker_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_public_data_operators/ticker_operators.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import octobot_commons.constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter @@ -45,7 +46,7 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ dsl_interpreter.OperatorParameter( name="symbol", description="The symbol to get the ticker value for", - required=True, type=str, + required=True, type=commons_enums.DslValueType.TEXT.value, ), ] diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py index bf39a5dfe2..bcf24f1df8 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py @@ -13,17 +13,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # pylint: disable=missing-class-docstring,missing-function-docstring -import asyncio import json import os +import pathlib import shutil import sys +import threading import time import types import typing import uuid import aiofiles -import pydantic import octobot_commons.constants as commons_constants import octobot_commons.dsl_interpreter as dsl_interpreter @@ -35,51 +35,38 @@ import octobot_commons.profiles.profile_data as profile_data_module import octobot_commons.profiles.profile_data_import as profile_data_import import octobot_commons.profiles.exchange_auth_data as exchange_auth_data_module -import octobot_commons.profiles.profile as profiles_profile_module import octobot_commons.profiles.tentacles_profile_data_translator as tentacles_profile_data_translator import octobot_commons.enums as commons_enums import octobot_commons.configuration import octobot.constants as octobot_constants +import octobot.community.activity_analysis.activity_metrics as activity_metrics import octobot.community.supabase_backend.enums as community_enums import octobot_flow.entities as octobot_flow_entities import octobot_flow.entities.accounts.process_bot_state as process_bot_state_import +import octobot_flow.entities.automations.octobot_process_state as octobot_process_state_import import octobot_node.constants as octobot_node_constants import octobot_services.constants as services_constants +import octobot_protocol.models.generic_process_configuration as generic_process_configuration +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers -# Written only after a successful full init so re-runs can detect an existing per-bot tree. -DSL_PREPARED_MARKER = ".octobot_dsl_prepared" DEFAULT_PING_WAITING_TIME = 2.0 DEFAULT_ENSURE_TIMEOUT = 120.0 +DEFAULT_FORCE_KILL_EXIT_WAIT_SECONDS = 5.0 DEFAULT_DSL_PROFILE_ID = "non-trading" +RUN_OCTOBOT_PROCESS_OPERATOR_NAME = "run_octobot_process" +AUTO_OPEN_IN_WEB_BROWSER = True # run_octobot_process uses two state layers: -# - Recall state (`EnsureOctobotProcessState` in DSL `last_execution_result`): master-side +# - Recall state (`OctobotProcessState` in DSL `last_execution_result`): master-side # snapshot (ports, paths, stored pid, init_state_ok, executor_id). Persisted across # re-calls until STOP, UPDATE_CONFIG, or respawn. # - Child dump (`process_bot_state.json` → `ProcessBotState`): written by the child; used for # timestamp-fresh checks and metadata.pid when the stored recall pid is stale. # executor_id ties recall to the current DBOS scheduler worker. On mismatch # with all child PIDs dead, respawn is forced immediately (grace is bypassed). -class EnsureOctobotProcessState(pydantic.BaseModel): - model_config = pydantic.ConfigDict(validate_assignment=True, extra="ignore") - http_base_url: str - web_port: int - node_port: int - user_root: str - user_folder: str - log_folder: str - profile_id: str | None - # Last known child PID on the master; may lag after a child self-restart until adoption. - pid: int - state_file_path: str = "" - # Wall-clock when the first spawn began; used only while init_state_ok is False (ping_timeout). - started_waiting_at: float = 0.0 - # True once the child reached confirmed-alive; switches from init ping_timeout to recall/grace rules. - init_state_ok: bool = False - # Required scheduler executor id at emit time; compared on recall to detect worker restart. - executor_id: str # Keys on `last_result` that `create_re_callable_result_dict` takes as top-level args (not state). @@ -93,7 +80,7 @@ class EnsureOctobotProcessState(pydantic.BaseModel): _DEFAULT_ENCRYPTED_VALUE = octobot_commons.configuration.encrypt("").decode() -def _resolve_state_file_path(recall_state: EnsureOctobotProcessState) -> str: +def _resolve_state_file_path(recall_state: octobot_process_state_import.OctobotProcessState) -> str: if recall_state.state_file_path: return recall_state.state_file_path return os.path.normpath( @@ -103,6 +90,16 @@ def _resolve_state_file_path(recall_state: EnsureOctobotProcessState) -> str: ) ) +def _report_child_octobot_first_start_if_needed(init_info: dict[str, typing.Any]) -> None: + try: + if init_info.get("already_prepared"): + return + activity_metrics.ActivityMetrics.report_child_octobot_first_start() + except Exception as err: + _get_logger().exception( + err, True, f"Failed to report child OctoBot first start {err}" + ) + # --- Liveness and routing (recall state + child dump) --- @@ -141,16 +138,6 @@ def _is_state_timestamp_fresh( return _is_process_state_alive(loaded_state) -def _parse_ensure_recall_state(raw: dict) -> typing.Optional[EnsureOctobotProcessState]: - """Parse recall payload; empty or invalid dict → None.""" - if not raw: - return None - try: - return EnsureOctobotProcessState.model_validate(raw) - except pydantic.ValidationError: - return None - - def _metadata_pid_is_running( loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], ) -> bool: @@ -175,7 +162,7 @@ def _is_child_confirmed_alive( def _any_child_pid_running( - recall_state: EnsureOctobotProcessState, + recall_state: octobot_process_state_import.OctobotProcessState, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], ) -> bool: """True when either recall pid or child dump metadata.pid is running.""" @@ -185,7 +172,7 @@ def _any_child_pid_running( def _executor_restarted_requires_respawn( - recall_state: EnsureOctobotProcessState, + recall_state: octobot_process_state_import.OctobotProcessState, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], *, current_executor_id: str, @@ -196,7 +183,7 @@ def _executor_restarted_requires_respawn( return recall_state.executor_id != current_executor_id -def _stored_pid_is_running(recall_state: EnsureOctobotProcessState) -> bool: +def _stored_pid_is_running(recall_state: octobot_process_state_import.OctobotProcessState) -> bool: """Fast path: recall pid still running.""" if recall_state.pid <= 0: return False @@ -204,7 +191,7 @@ def _stored_pid_is_running(recall_state: EnsureOctobotProcessState) -> bool: def _in_restart_grace_period( - recall_state: EnsureOctobotProcessState, + recall_state: octobot_process_state_import.OctobotProcessState, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], *, now: float, @@ -223,7 +210,7 @@ def _in_restart_grace_period( def _should_use_recall_path( - recall_state: EnsureOctobotProcessState, + recall_state: octobot_process_state_import.OctobotProcessState, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], *, stored_pid_running: bool, @@ -247,28 +234,27 @@ def _should_use_recall_path( def _resolve_bound_pid( - recall_state: EnsureOctobotProcessState, + recall_state: octobot_process_state_import.OctobotProcessState, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], ) -> typing.Optional[int]: - """Bind operating PID from recall or fresh dump; None if metadata.pid dead (no raise).""" + """Bind operating PID from recall or fresh dump; None if no running pid (no raise).""" + if loaded_state is not None and _is_state_timestamp_fresh(loaded_state): + state_pid = loaded_state.metadata.pid + if state_pid <= 0: + raise commons_errors.DSLInterpreterError( + "process_bot_state.json is live but metadata.pid is missing or invalid." + ) + if process_util.pid_is_running(state_pid): + return state_pid if _stored_pid_is_running(recall_state): return recall_state.pid - if loaded_state is None or not _is_state_timestamp_fresh(loaded_state): - return None - state_pid = loaded_state.metadata.pid - if state_pid <= 0: - raise commons_errors.DSLInterpreterError( - "process_bot_state.json is live but metadata.pid is missing or invalid." - ) - if process_util.pid_is_running(state_pid): - return state_pid return None def _apply_resolved_pid_to_state( - recall_state: EnsureOctobotProcessState, + recall_state: octobot_process_state_import.OctobotProcessState, resolved_pid: typing.Optional[int], -) -> EnsureOctobotProcessState: +) -> octobot_process_state_import.OctobotProcessState: if resolved_pid is None or resolved_pid == recall_state.pid: return recall_state return recall_state.model_copy(update={"pid": resolved_pid}) @@ -328,26 +314,194 @@ async def _convert_profile_data_to_profile_directory( ) +def _path_segments(relative_path: str) -> tuple[str, ...]: + return tuple( + segment + for segment in str(relative_path).replace("\\", "/").split("/") + if segment + ) + + +def _assert_automation_child_config_path(config_path: str) -> None: + """ + Reject writes outside ``user/automations//config.json`` (master ``user/config.json`` is forbidden). + """ + normalized_config_path = os.path.normpath(config_path) + if os.path.basename(normalized_config_path) != commons_constants.CONFIG_FILE: + raise commons_errors.DSLInterpreterError( + f"Process child config must be named {commons_constants.CONFIG_FILE!r}, not {config_path!r}." + ) + path_segments = pathlib.PurePath(normalized_config_path).parts + automation_prefix = ( + commons_constants.USER_FOLDER, + commons_constants.AUTOMATIONS_FOLDER, + ) + prefix_length = len(automation_prefix) + for segment_index in range(len(path_segments) - prefix_length): + if path_segments[segment_index : segment_index + prefix_length] != automation_prefix: + continue + leaf_segments = path_segments[segment_index + prefix_length : -1] + if not leaf_segments: + raise commons_errors.DSLInterpreterError( + f"Process child config must live under " + f"{commons_constants.USER_AUTOMATIONS_FOLDER}//{commons_constants.CONFIG_FILE}, " + f"not {config_path!r}." + ) + if ".." in leaf_segments: + raise commons_errors.DSLInterpreterError( + f"Process child config path must not contain parent segments: {config_path!r}." + ) + return + raise commons_errors.DSLInterpreterError( + f"Process child config must be under {commons_constants.USER_AUTOMATIONS_FOLDER}//, " + f"not {config_path!r}." + ) + + +def _assert_automation_rel_folder( + rel_folder: str, + expected_prefix: tuple[str, ...], + *, + cli_flag_label: str, + expected_folder_path: str, +) -> None: + path_segments = _path_segments(rel_folder) + if len(path_segments) < len(expected_prefix) + 1: + raise commons_errors.DSLInterpreterError( + f"Process child {cli_flag_label} must be under {expected_folder_path}//, " + f"got {rel_folder!r}." + ) + if path_segments[: len(expected_prefix)] != expected_prefix: + raise commons_errors.DSLInterpreterError( + f"Process child {cli_flag_label} must start with {expected_folder_path}/, " + f"got {rel_folder!r}." + ) + if ".." in path_segments: + raise commons_errors.DSLInterpreterError( + f"Process child {cli_flag_label} must not contain parent segments: {rel_folder!r}." + ) + + +def _assert_automation_rel_user_folder(rel_user_folder: str) -> None: + _assert_automation_rel_folder( + rel_user_folder, + ( + commons_constants.USER_FOLDER, + commons_constants.AUTOMATIONS_FOLDER, + ), + cli_flag_label="--user-folder", + expected_folder_path=commons_constants.USER_AUTOMATIONS_FOLDER, + ) + + +def _assert_automation_rel_log_folder(rel_log_folder: str) -> None: + _assert_automation_rel_folder( + rel_log_folder, + tuple(octobot_node_constants.AUTOMATION_LOGS_FOLDER.split("/")), + cli_flag_label="--log-folder", + expected_folder_path=octobot_node_constants.AUTOMATION_LOGS_FOLDER, + ) + + +def _assert_spawn_cmd_isolation(cmd: list[str], rel_user: str, rel_log: str) -> None: + if "--user-folder" not in cmd or "--log-folder" not in cmd: + raise commons_errors.DSLInterpreterError( + "Process child spawn command must include --user-folder and --log-folder." + ) + _assert_automation_rel_user_folder(rel_user) + _assert_automation_rel_log_folder(rel_log) + + +def _master_user_config_path(working_directory: str) -> str: + return os.path.join( + working_directory, + commons_constants.USER_FOLDER, + commons_constants.CONFIG_FILE, + ) + + +def _load_master_exchange_auth_data( + working_directory: str, +) -> dict[str, exchange_auth_data_module.ExchangeAuthData]: + master_config_path = _master_user_config_path(working_directory) + if not os.path.isfile(master_config_path): + return {} + try: + master_config = json_util.read_file(master_config_path) + except Exception as error: + raise commons_errors.DSLInterpreterError( + f"Failed to read master user config at {master_config_path!r}: {error}" + ) from error + exchanges_config = master_config.get(commons_constants.CONFIG_EXCHANGES) or {} + return { + internal_name: exchange_auth_data_module.ExchangeAuthData( + internal_name=internal_name, + api_key=exchange_config.get(commons_constants.CONFIG_EXCHANGE_KEY, ""), + api_secret=exchange_config.get(commons_constants.CONFIG_EXCHANGE_SECRET, ""), + api_password=exchange_config.get(commons_constants.CONFIG_EXCHANGE_PASSWORD, ""), + exchange_type=exchange_config.get( + commons_constants.CONFIG_EXCHANGE_TYPE, + commons_constants.DEFAULT_EXCHANGE_TYPE, + ), + sandboxed=exchange_config.get(commons_constants.CONFIG_EXCHANGE_SANDBOXED, False), + ) + for internal_name, exchange_config in exchanges_config.items() + if isinstance(exchange_config, dict) + } + + +def _resolved_exchange_auth_data( + working_directory: str, + override_dicts: list[dict] | None, +) -> list[exchange_auth_data_module.ExchangeAuthData] | None: + merged_auth = _load_master_exchange_auth_data(working_directory) + if override_dicts: + for override_dict in override_dicts: + if not isinstance(override_dict, dict): + continue + internal_name = override_dict.get("internal_name") + if not internal_name: + continue + merged_auth[internal_name] = exchange_auth_data_module.ExchangeAuthData.from_dict( + override_dict + ) + if not merged_auth: + return None + return list(merged_auth.values()) + + def _write_user_root_config_json( config_path: str, profile_id: str, profile_data: typing.Optional[profile_data_module.ProfileData] = None, - exchange_auth_data: typing.Optional[ - list[exchange_auth_data_module.ExchangeAuthData] - ] = None, + exchange_auth_overrides: list[dict] | None = None, + working_directory: str = "", + readonly_profiles_path: str | None = None, + readonly_reference_tentacles_path: str | None = None, + octobot_name: str | None = None, ) -> None: """ - Writes user-root ``config.json``: selected profile, disabled web auto-open for DSL-spawned - processes, optional exchange stubs from ``profile_data``, then credentials from - ``exchange_auth_data`` (merged into ``exchanges``). + Writes user-root ``config.json``: selected profile, web auto-open from + ``AUTO_OPEN_IN_WEB_BROWSER``, optional exchange stubs from ``profile_data``, then credentials + from the executor master ``user/config.json`` (forwarded for all exchanges), with optional + ``exchange_auth_overrides`` fully replacing matching entries by ``internal_name``. """ - # Load packaged defaults; pin profile and disable browser auto-open for headless DSL children. + _assert_automation_child_config_path(config_path) + # Load packaged defaults; pin profile and apply browser auto-open from module constant. default_cfg = json_util.read_file(octobot_constants.DEFAULT_CONFIG_FILE) default_cfg[commons_constants.CONFIG_PROFILE] = profile_id - default_cfg[commons_constants.CONFIG_ACCEPTED_TERMS] = True + default_cfg[commons_constants.CONFIG_ACCEPTED_TERMS] = False + if octobot_name and str(octobot_name).strip(): + default_cfg[commons_constants.CONFIG_OCTOBOT_NAME] = str(octobot_name).strip() + if readonly_profiles_path: + default_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] = readonly_profiles_path + if readonly_reference_tentacles_path: + default_cfg[commons_constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH] = ( + readonly_reference_tentacles_path + ) services_cfg = default_cfg.setdefault(services_constants.CONFIG_CATEGORY_SERVICES, {}) web_cfg = services_cfg.setdefault(services_constants.CONFIG_WEB, {}) - web_cfg[services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER] = False + web_cfg[services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER] = AUTO_OPEN_IN_WEB_BROWSER # Seed top-level exchanges so partially-managed merge targets exist before applying secrets. if profile_data is not None: exchanges_cfg = default_cfg.setdefault(commons_constants.CONFIG_EXCHANGES, {}) @@ -361,11 +515,28 @@ def _write_user_root_config_json( commons_constants.CONFIG_EXCHANGE_TYPE, exchange_details.exchange_type or commons_constants.DEFAULT_EXCHANGE_TYPE, ) - # Overlay credentials onto matching exchange entries (adds exchange if missing). - if exchange_auth_data: + # Overlay master credentials and optional overrides onto matching exchange entries. + resolved_auth = _resolved_exchange_auth_data(working_directory, exchange_auth_overrides) + if resolved_auth: exchange_config_holder = types.SimpleNamespace(config=default_cfg) - for auth_element in exchange_auth_data: + for auth_element in resolved_auth: auth_element.apply_to_exchange_config(exchange_config_holder) + if profile_data is not None: + profile_exchange_names = { + exchange_details.internal_name + for exchange_details in profile_data.exchanges + if exchange_details.internal_name + } + exchanges_cfg = default_cfg.get(commons_constants.CONFIG_EXCHANGES) or {} + for exchange_name, exchange_cfg in exchanges_cfg.items(): + if exchange_name not in profile_exchange_names and isinstance(exchange_cfg, dict): + # don't inherit exchange activation from master config + exchange_cfg[commons_constants.CONFIG_ENABLED_OPTION] = False + # Re-enable profile exchanges after master auth overlay and non-profile disable guard. + for profile_exchange_name in profile_exchange_names: + profile_exchange_cfg = exchanges_cfg.get(profile_exchange_name) + if isinstance(profile_exchange_cfg, dict): + profile_exchange_cfg[commons_constants.CONFIG_ENABLED_OPTION] = True exchanges_cfg = default_cfg.get(commons_constants.CONFIG_EXCHANGES) or {} for exchange_cfg in exchanges_cfg.values(): if isinstance(exchange_cfg, dict): @@ -374,98 +545,97 @@ def _write_user_root_config_json( json_util.safe_dump(default_cfg, config_path) -def _executor_non_trading_profile_source(working_directory: str) -> str: +def _executor_profiles_directory(working_directory: str) -> str: return os.path.normpath( os.path.join( working_directory, commons_constants.USER_FOLDER, commons_constants.PROFILES_FOLDER, - DEFAULT_DSL_PROFILE_ID, ) ) -def _executor_profiles_directory(working_directory: str) -> str: +def _executor_reference_tentacles_directory(working_directory: str) -> str: return os.path.normpath( os.path.join( working_directory, commons_constants.USER_FOLDER, - commons_constants.PROFILES_FOLDER, + "reference_tentacles_config", ) ) -async def _copy_read_only_profiles_to_user_root( +def _child_master_profile_config_kwargs( working_directory: str, - user_root: str, - *, - active_profile_id: str, -) -> None: - """ - Copy read-only profiles from the master OctoBot into a generic process child layout. +) -> dict[str, typing.Any]: + return { + "readonly_profiles_path": _executor_profiles_directory(working_directory), + "readonly_reference_tentacles_path": _executor_reference_tentacles_directory( + working_directory + ), + } - Generic process bots start on the default non-trading profile but should still see - the same read-only strategy profiles as the master (community/imported templates). - Editable profiles are intentionally omitted so each child keeps its own user edits. - """ - profiles_src = _executor_profiles_directory(working_directory) - if not os.path.isdir(profiles_src): - return - for profile in profiles_profile_module.Profile.get_all_profiles(profiles_src): - if not profile.read_only: - continue - # Active profile was already copied by _copy_non_trading_profile_to_user_root. - if profile.profile_id == active_profile_id: - continue - destination_profile_path = os.path.join( - user_root, - commons_constants.PROFILES_FOLDER, - profile.profile_id, - ) - if os.path.exists(destination_profile_path): - shutil.rmtree(destination_profile_path) - shutil.copytree(profile.path, destination_profile_path) +def _get_sync_strategy(sync_user_id: str, strategy_id: str) -> typing.Any: + return collection_providers.StrategyProvider.instance().get_item( + sync_user_id, + strategy_id, + ) -async def _copy_non_trading_profile_to_user_root( - working_directory: str, - user_root: str, -) -> str: - source_profile_path = _executor_non_trading_profile_source(working_directory) - if not os.path.isdir(source_profile_path): + +def _sync_strategy_has_profile_data(strategy: typing.Any) -> bool: + configuration = strategy.configuration + if configuration is None or configuration.actual_instance is None: + return False + if not isinstance( + configuration.actual_instance, + generic_process_configuration.GenericProcessConfiguration, + ): + return False + return configuration.actual_instance.profile_data is not None + + +def _assert_sync_strategy_exists(sync_user_id: str, sync_profile_id: str) -> typing.Any: + try: + return _get_sync_strategy(sync_user_id, sync_profile_id) + except collection_errors.ItemNotFoundError as err: raise commons_errors.DSLInterpreterError( - f"Default profile not found at {source_profile_path!r}; expected " - f"{DEFAULT_DSL_PROFILE_ID!r} under the OctoBot user profiles folder." - ) - destination_profile_path = os.path.join( - user_root, - commons_constants.PROFILES_FOLDER, - DEFAULT_DSL_PROFILE_ID, - ) - if os.path.exists(destination_profile_path): - shutil.rmtree(destination_profile_path) - shutil.copytree(source_profile_path, destination_profile_path) - return DEFAULT_DSL_PROFILE_ID + f"sync strategy {sync_profile_id!r} not found for sync user {sync_user_id!r}." + ) from err + + +def _materialize_dsl_parameter_value(value: typing.Any) -> typing.Any: + if isinstance(value, dsl_interpreter.Operator): + return _materialize_dsl_parameter_value(value.compute()) + if isinstance(value, dict): + return { + key: _materialize_dsl_parameter_value(nested_value) + for key, nested_value in value.items() + } + if isinstance(value, list): + return [ + _materialize_dsl_parameter_value(nested_value) for nested_value in value + ] + return value async def ensure_user_profile_and_layout( user_folder: str, working_directory: str, - profile_data_dict: dict | None, - source_reference_tentacles_config: str | None, - exchange_auth_data: typing.Optional[ - list[exchange_auth_data_module.ExchangeAuthData] - ] = None, + profile_data_dict: dict | None = None, + exchange_auth_overrides: list[dict] | None = None, + *, + sync_profile_id: str | None = None, + user_id: str | None = None, + octobot_name: str | None = None, ) -> dict[str, typing.Any]: """ One-time layout under user_root (/user/automations//): - profile tree, top-level config.json, reference_tentacles_config copy. - Idempotent when config.json + marker both exist. + profile tree and top-level config.json (with master readonly overlays). + Idempotent when config.json already exists. """ dsl_interpreter.ProcessBoundOperatorMixin.reject_user_path_segment(user_folder) - user_folder_leaf_segments = [ - segment for segment in str(user_folder).replace("\\", "/").split("/") if segment - ] + user_folder_leaf_segments = _path_segments(user_folder) user_root = os.path.normpath( os.path.join( working_directory, @@ -474,9 +644,8 @@ async def ensure_user_profile_and_layout( ) ) config_path = os.path.join(user_root, commons_constants.CONFIG_FILE) - marker_path = os.path.join(user_root, DSL_PREPARED_MARKER) # Already prepared: do not rewrite files (host may have re-used this folder). - if os.path.isfile(config_path) and os.path.isfile(marker_path): + if os.path.isfile(config_path): profile_id = _read_top_level_profile_id(config_path) return { "user_root": user_root, @@ -484,26 +653,17 @@ async def ensure_user_profile_and_layout( "already_prepared": True, } + master_reference_tentacles_path = _executor_reference_tentacles_directory(working_directory) + if not os.path.isdir(master_reference_tentacles_path): + raise commons_errors.DSLInterpreterError( + f"Master reference tentacles config not found at {master_reference_tentacles_path!r}. " + "Install tentacles on the executor (master OctoBot) before spawning process children." + ) + os.makedirs(user_root, exist_ok=True) - if profile_data_dict is None: - # Generic process: default non-trading profile plus master's read-only profiles. - profile_id = await _copy_non_trading_profile_to_user_root( - working_directory, - user_root, - ) - await _copy_read_only_profiles_to_user_root( - working_directory, - user_root, - active_profile_id=profile_id, - ) - _write_user_root_config_json( - config_path, - profile_id, - None, - exchange_auth_data, - ) - else: + if profile_data_dict is not None: + profile_data_dict = _materialize_dsl_parameter_value(profile_data_dict) # Import writes to a throwaway folder first: the real profile id is assigned during import (see rename below). temp_profile_path = os.path.join( user_root, @@ -529,24 +689,46 @@ async def ensure_user_profile_and_layout( shutil.rmtree(final_profile_path) os.replace(temp_profile_path, final_profile_path) - _write_user_root_config_json(config_path, profile_id, profile_data, exchange_auth_data) - - # Mirror default reference tentacles layout expected by the child. - ref_src = source_reference_tentacles_config or os.path.join( - working_directory, commons_constants.USER_FOLDER, "reference_tentacles_config" - ) - ref_src = os.path.normpath(ref_src) - ref_dst = os.path.join(user_root, "reference_tentacles_config") - if os.path.isdir(ref_src): - if os.path.exists(ref_dst): - shutil.rmtree(ref_dst) - shutil.copytree(ref_src, ref_dst) + _write_user_root_config_json( + config_path, + profile_id, + profile_data, + exchange_auth_overrides, + working_directory, + octobot_name=octobot_name, + **_child_master_profile_config_kwargs(working_directory), + ) + elif sync_profile_id is not None: + if not user_id or not str(user_id).strip(): + raise commons_errors.DSLInterpreterError( + f"sync_profile_id={sync_profile_id!r} requires user_id." + ) + strategy = _assert_sync_strategy_exists(str(user_id), sync_profile_id) + if _sync_strategy_has_profile_data(strategy): + profile_id = sync_profile_id + else: + profile_id = DEFAULT_DSL_PROFILE_ID + _write_user_root_config_json( + config_path, + profile_id, + None, + exchange_auth_overrides, + working_directory, + octobot_name=octobot_name, + **_child_master_profile_config_kwargs(working_directory), + ) else: - os.makedirs(ref_dst, exist_ok=True) - - # Marker last: if anything above failed, a partial tree will not look "prepared". - with open(marker_path, "w", encoding="utf-8") as marker_file: - marker_file.write("1") + # Generic process: master profiles via overlay config. + profile_id = DEFAULT_DSL_PROFILE_ID + _write_user_root_config_json( + config_path, + profile_id, + None, + exchange_auth_overrides, + working_directory, + octobot_name=octobot_name, + **_child_master_profile_config_kwargs(working_directory), + ) return { "user_root": user_root, @@ -568,7 +750,7 @@ def _read_top_level_profile_id(config_path: str) -> str | None: def _ensure_log_folder_path(working_directory: str, user_folder: str) -> str: """Absolute log directory for this `user_folder` (matches ensure_state.log_folder).""" - log_folder_param_segments = [segment for segment in str(user_folder).replace("\\", "/").split("/") if segment] + log_folder_param_segments = _path_segments(user_folder) return os.path.normpath( os.path.join( working_directory, @@ -578,32 +760,57 @@ def _ensure_log_folder_path(working_directory: str, user_folder: str) -> str: ) -def _ensure_child_environ(web_port: int, node_port: int, bind_host: str) -> dict: - """Environment passed to the OctoBot child (ports and bind addresses).""" +def _ensure_child_environ( + web_port: int, + node_port: int, + bind_host: str, + sync_user_id: str, + working_directory: str, +) -> dict: + """Environment passed to the OctoBot child (ports, bind addresses, sync user id).""" child_env = os.environ.copy() child_env[services_constants.ENV_WEB_PORT] = str(web_port) child_env[services_constants.ENV_WEB_ADDRESS] = bind_host child_env[services_constants.ENV_NODE_API_PORT] = str(node_port) child_env[services_constants.ENV_NODE_API_ADDRESS] = bind_host child_env[commons_constants.ENV_USE_MINIMAL_LIBS] = "false" + child_env["DISTRIBUTION"] = commons_constants.DEFAULT_DISTRIBUTION + child_env[services_constants.ENV_ENABLE_NODE_API] = "false" + child_env[octobot_constants.ENV_PROCESS_BOT_SYNC_USER_ID] = sync_user_id + child_env[commons_constants.ENV_OCTOBOT_SYNC_DATA_ROOT] = os.path.normpath( + os.path.join(working_directory, commons_constants.USER_FOLDER) + ) + if os_util.is_frozen_binary_octobot(): + child_env.update(os_util.PYINSTALLER_RESET_ENVIRONMENT_VARS) return child_env +def _octobot_spawn_argv_prefix(working_directory: str) -> list[str]: + if os_util.is_frozen_binary_octobot(): + return [sys.executable] + start_script = os.path.join(working_directory, "start.py") + if not os.path.isfile(start_script): + raise commons_errors.DSLInterpreterError( + f"start.py not found at {start_script} (current working directory must be the OctoBot project root)." + ) + return [sys.executable, start_script] + + def _ensure_start_cmd( - start_script: str, + argv_prefix: list[str], rel_user: str, rel_log: str, no_telegram: bool, state_file_path: str, ) -> list[str]: - """Argv for `python start.py --user-folder … --log-folder …` (+ optional -nt, --dump-state).""" + """Argv for OctoBot child: prefix + --user-folder … --standalone (+ optional -nt, --dump-state).""" cmd: list[str] = [ - sys.executable, - start_script, + *argv_prefix, "--user-folder", rel_user, "--log-folder", rel_log, + "--standalone", ] if no_telegram: cmd.append("-nt") @@ -611,28 +818,84 @@ def _ensure_start_cmd( return cmd +_child_listen_ports_reserved: dict[int, str] = {} +_child_listen_ports_lock = threading.Lock() + + +def _reserve_child_listen_ports(web_port: int, node_port: int, user_folder: str) -> None: + with _child_listen_ports_lock: + _child_listen_ports_reserved[web_port] = user_folder + _child_listen_ports_reserved[node_port] = user_folder + + +def _release_child_listen_ports(web_port: int, node_port: int, user_folder: str) -> None: + with _child_listen_ports_lock: + for listen_port in (web_port, node_port): + if _child_listen_ports_reserved.get(listen_port) == user_folder: + _child_listen_ports_reserved.pop(listen_port, None) + + def _listen_port_pair_with_shared_scan_offset( probe_host: str, primary_listen_port_base: int, secondary_listen_port_base: int, *, max_offset: int = 256, + extra_blocklist: set[int] | frozenset[int] | None = None, ) -> tuple[int, int]: """Delegates to ``find_first_free_listen_port_after_base`` paired scan (one loop).""" + primary_blocklist = list(extra_blocklist) if extra_blocklist else None primary_listen_port = os_util.find_first_free_listen_port_after_base( probe_host, primary_listen_port_base, max_offset=max_offset, + blocklist=primary_blocklist, ) + secondary_blocklist = set(extra_blocklist or ()) + secondary_blocklist.add(primary_listen_port) secondary_listen_port = os_util.find_first_free_listen_port_after_base( probe_host, secondary_listen_port_base, max_offset=max_offset, - blocklist=[primary_listen_port], + blocklist=list(secondary_blocklist), ) return primary_listen_port, secondary_listen_port +def _allocate_child_listen_port_pair( + probe_host: str, + primary_listen_port_base: int, + secondary_listen_port_base: int, + user_folder: str, + *, + max_offset: int = 256, +) -> tuple[int, int]: + with _child_listen_ports_lock: + reserved_ports = frozenset(_child_listen_ports_reserved) + web_port, node_port = _listen_port_pair_with_shared_scan_offset( + probe_host, + primary_listen_port_base, + secondary_listen_port_base, + max_offset=max_offset, + extra_blocklist=reserved_ports, + ) + _child_listen_ports_reserved[web_port] = user_folder + _child_listen_ports_reserved[node_port] = user_folder + return web_port, node_port + + +def _release_recall_state_listen_ports( + recall_state: typing.Optional[octobot_process_state_import.OctobotProcessState], +) -> None: + if recall_state is None: + return + _release_child_listen_ports( + recall_state.web_port, + recall_state.node_port, + recall_state.user_folder, + ) + + def create_octobot_process_operators( signals: typing.Optional[dsl_interpreter.OperatorSignals] = None, executor_id: str = "", @@ -645,16 +908,17 @@ class EnsureOctobotProcessOperator( dsl_interpreter.ProcessBoundOperatorMixin, ): DESCRIPTION = ( - "Prepares a per-bot user directory (profile + config + reference_tentacles_config), " + "Prepares a per-bot user directory (profile + config with master readonly overlays), " "spawns an OctoBot child with unique WEB/NODE ports and --dump-state for process_bot_state.json. " "Always re-callable: each fresh state file (updated_at within twice the dump interval) schedules the next check (see waiting_time). " "If the state file never becomes live before ping_timeout from the first spawn, the keyword fails and the child is killed." ) EXAMPLE = ( - "run_octobot_process(user_folder='bots/b1', " + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME}(user_folder='bots/b1', " "exchange_auth_data=[{'internal_name': 'binance', 'api_key': '...', 'api_secret': '...'}], " "last_execution_result=None)" ) + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value def __init__(self, *args, **kwargs): dsl_interpreter.PreComputingCallOperator.__init__(self, *args, **kwargs) @@ -664,7 +928,7 @@ def __init__(self, *args, **kwargs): def _read_executor_id(self) -> str: if not executor_id: raise commons_errors.DSLInterpreterError( - "executor_id is required for run_octobot_process" + f"executor_id is required for {RUN_OCTOBOT_PROCESS_OPERATOR_NAME}" ) return executor_id @@ -674,7 +938,7 @@ def get_library() -> str: @staticmethod def get_name() -> str: - return "run_octobot_process" + return RUN_OCTOBOT_PROCESS_OPERATOR_NAME @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: @@ -685,17 +949,49 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "Path segment(s) under /user/automations/ for this bot." ), required=True, - type=str, + type=commons_enums.DslValueType.TEXT.value, ), + dsl_interpreter.OperatorParameter( + name="octobot_name", + description=( + "Optional display name for this child OctoBot instance. " + "Written to config.json and shown in the web interface title and navbar." + ), + required=False, + type=commons_enums.DslValueType.TEXT.value, + default=None), dsl_interpreter.OperatorParameter( name="profile_data", description=( "Optional object compatible with octobot_commons.profiles.profile_data.ProfileData. " - "When omitted, the child uses the packaged default config and copies the " - f"{DEFAULT_DSL_PROFILE_ID!r} profile from the executor user profiles folder." + "When omitted, the child uses the packaged default config and selects the " + f"{DEFAULT_DSL_PROFILE_ID!r} profile from the executor user profiles folder " + "via master profile overlay." ), required=False, - type=dict, + type=commons_enums.DslValueType.DICT.value, + default=None), + dsl_interpreter.OperatorParameter( + name="sync_profile_id", + description=( + "Optional sync profile id (strategy id). Used only when profile_data is omitted. " + "Validates the strategy exists in sync for user_id. When the strategy embeds " + "profile_data, selects it in child config.json without a local profiles/ tree; " + f"otherwise the child uses {DEFAULT_DSL_PROFILE_ID!r} from the executor profiles overlay." + ), + required=False, + type=commons_enums.DslValueType.TEXT.value, + default=None, + ), + dsl_interpreter.OperatorParameter( + name="user_id", + description=( + "Sync wallet user id (same as node task user_id). Required to spawn the " + "process child (passed via environment, not config.json). Also required " + "when sync_profile_id is set for strategy validation." + ), + required=False, + type=commons_enums.DslValueType.TEXT.value, default=None, ), dsl_interpreter.OperatorParameter( @@ -703,47 +999,44 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: description=( "Optional list of dicts compatible with " "octobot_commons.profiles.exchange_auth_data.ExchangeAuthData " - "(e.g. internal_name, api_key, api_secret, api_password, exchange_type, sandboxed)." + "(e.g. internal_name, api_key, api_secret, api_password, exchange_type, sandboxed). " + "Exchanges not listed inherit credentials from the executor master user/config.json; " + "each listed entry fully replaces the master entry for that internal_name." ), required=False, - type=list[dict], + type=commons_enums.DslValueType.ANY.value, default=None, ), dsl_interpreter.OperatorParameter( name="web_port_base", description="Base port for the web interface (uses base+offset; default from services constants).", required=False, - type=int, - default=services_constants.DEFAULT_SERVER_PORT, - ), + type=commons_enums.DslValueType.NUMBER.value, + default=services_constants.DEFAULT_SERVER_PORT), dsl_interpreter.OperatorParameter( name="node_port_base", description="Base port for the node API (uses base+offset).", required=False, - type=int, - default=services_constants.DEFAULT_NODE_API_PORT, - ), + type=commons_enums.DslValueType.NUMBER.value, + default=services_constants.DEFAULT_NODE_API_PORT), dsl_interpreter.OperatorParameter( name="bind_host", description="Host used for free-port checks and WEB_ADDRESS / NODE_API_ADDRESS for the child.", required=False, - type=str, - default="127.0.0.1", - ), + type=commons_enums.DslValueType.TEXT.value, + default="127.0.0.1"), dsl_interpreter.OperatorParameter( name="http_scheme", description="Scheme for http_base_url (default http).", required=False, - type=str, - default="http", - ), + type=commons_enums.DslValueType.TEXT.value, + default="http"), dsl_interpreter.OperatorParameter( name="no_telegram", description="If true, spawns with -nt (default true).", required=False, - type=bool, - default=True, - ), + type=commons_enums.DslValueType.BOOLEAN.value, + default=True), dsl_interpreter.OperatorParameter( name="ping_timeout", description=( @@ -752,7 +1045,7 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "Does not cap liveness re-calls once up." ), required=False, - type=float, + type=commons_enums.DslValueType.NUMBER.value, default=DEFAULT_ENSURE_TIMEOUT, ), dsl_interpreter.OperatorParameter( @@ -761,11 +1054,18 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "Fixed interval in seconds before each re-call (init polling and ongoing liveness while the state file is live)." ), required=False, - type=float, + type=commons_enums.DslValueType.NUMBER.value, default=DEFAULT_PING_WAITING_TIME, ), ] + super().get_re_callable_parameters() + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.DICT.value, + description="Re-callable process state or stop outcome", + ) + @classmethod def _re_calling_result_dispatches_this_ensure( cls, @@ -788,7 +1088,7 @@ def _re_calling_result_dispatches_this_ensure( if not isinstance(inner, dict): return False try: - return _parse_ensure_recall_state(inner) is not None + return octobot_process_state_import.parse_octobot_process_state(inner) is not None except commons_errors.DSLInterpreterError: return False @@ -808,7 +1108,7 @@ def should_dispatch_operator_signal_for_result( def _emit_ensure_recall( self, *, - state: EnsureOctobotProcessState, + state: octobot_process_state_import.OctobotProcessState, last_result: dict, start_time: float, recall_interval: float, @@ -838,7 +1138,7 @@ def _emit_ensure_recall( async def _pre_compute_recall_path( self, - recall_state: EnsureOctobotProcessState, + recall_state: octobot_process_state_import.OctobotProcessState, last_result: dict, *, start_time: float, @@ -859,7 +1159,14 @@ async def _pre_compute_recall_path( ): resolved_pid = _resolve_bound_pid(recall_state, loaded_state) if resolved_pid is not None: - self.pid = resolved_pid + if resolved_pid != recall_state.pid: + self.bind_authoritative_child_pid( + resolved_pid, + spawn_pid=recall_state.pid, + ) + else: + self.pid = resolved_pid + _release_recall_state_listen_ports(recall_state) self.value = self.request_graceful_stop(logger=_get_logger()) raise commons_errors.DSLInterpreterError( "Timed out waiting for OctoBot process_bot_state.json during init (see ping_timeout).", @@ -884,7 +1191,12 @@ async def _pre_compute_recall_path( resolved_pid, recall_state.pid, ) - self.pid = resolved_pid + self.bind_authoritative_child_pid( + resolved_pid, + spawn_pid=recall_state.pid, + ) + else: + self.pid = resolved_pid recall_state = _apply_resolved_pid_to_state(recall_state, resolved_pid) _get_logger().info("process state path (re-call path): %s", state_path) # Running: stored recall pid or child-confirmed-alive → init_state_ok, optional EAE. @@ -897,6 +1209,8 @@ async def _pre_compute_recall_path( recall_state.http_base_url, logged_pid, ) + if not recall_state.init_state_ok: + _release_recall_state_listen_ports(recall_state) updated = recall_state.model_copy( update={"init_state_ok": True, "state_file_path": state_path} ) @@ -937,23 +1251,15 @@ async def _pre_compute_first_spawn( recall_interval: float, ) -> None: # One-time (or re-) materialization, free ports, env, and `Popen` at project root. - raw_exchange_auth = params.get("exchange_auth_data") - exchange_auth: typing.Optional[ - list[exchange_auth_data_module.ExchangeAuthData] - ] = None - if raw_exchange_auth: - exchange_auth = [ - exchange_auth_data_module.ExchangeAuthData.from_dict(entry) - if isinstance(entry, dict) - else entry - for entry in raw_exchange_auth - ] + exchange_auth_overrides = params.get("exchange_auth_data") init_info = await ensure_user_profile_and_layout( user_folder, working_directory, params.get("profile_data"), - None, - exchange_auth, + exchange_auth_overrides, + sync_profile_id=params.get("sync_profile_id"), + user_id=params.get("user_id"), + octobot_name=params.get("octobot_name"), ) user_root = init_info["user_root"] log_folder = _ensure_log_folder_path(working_directory, user_folder) @@ -962,38 +1268,51 @@ async def _pre_compute_first_spawn( params ) ) + prior_recall_state = octobot_process_state_import.parse_octobot_process_state(last_result) + _release_recall_state_listen_ports(prior_recall_state) web_b = int(params.get("web_port_base") or services_constants.DEFAULT_SERVER_PORT) node_b = int(params.get("node_port_base") or services_constants.DEFAULT_NODE_API_PORT) - web_port, node_port = _listen_port_pair_with_shared_scan_offset( - probe_host, web_b, node_b + web_port, node_port = _allocate_child_listen_port_pair( + probe_host, web_b, node_b, user_folder ) - start_script = os.path.join(working_directory, "start.py") - if not os.path.isfile(start_script): + argv_prefix = _octobot_spawn_argv_prefix(working_directory) + process_sync_user_id = params.get("user_id") + if not process_sync_user_id or not str(process_sync_user_id).strip(): raise commons_errors.DSLInterpreterError( - f"start.py not found at {start_script} (current working directory must be the OctoBot project root)." + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME} requires user_id to spawn a process child." ) - child_env = _ensure_child_environ(web_port, node_port, bind_host) + child_env = _ensure_child_environ( + web_port, + node_port, + bind_host, + str(process_sync_user_id), + working_directory, + ) rel_user = os.path.relpath(user_root, working_directory) rel_log = os.path.relpath(log_folder, working_directory) state_file_path = os.path.normpath( os.path.join(user_root, octobot_constants.PROCESS_BOT_STATE_FILE_NAME) ) cmd = _ensure_start_cmd( - start_script, + argv_prefix, rel_user, rel_log, bool(params.get("no_telegram", True)), state_file_path, ) + _assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + _get_logger().info("Spawning OctoBot process child: cmd=%r", cmd) self.spawn_subprocess( cmd, working_directory=working_directory, environment=child_env, hide_console_window=True, ) + _report_child_octobot_first_start_if_needed(init_info) + spawn_pid = self.pid or 0 scheme = str(params.get("http_scheme") or "http").rstrip(":/") http_base_url = f"{scheme}://{bind_host}:{web_port}" - state = EnsureOctobotProcessState( + state = octobot_process_state_import.OctobotProcessState( http_base_url=http_base_url, web_port=web_port, node_port=node_port, @@ -1016,7 +1335,10 @@ async def _pre_compute_first_spawn( "process_bot_state.json is live but metadata.pid is missing or invalid." ) if process_util.pid_is_running(state_pid): - self.pid = state_pid + if state_pid != spawn_pid: + self.bind_authoritative_child_pid(state_pid, spawn_pid=spawn_pid) + else: + self.pid = state_pid state = state.model_copy(update={"pid": state_pid}) _get_logger().info( "OctoBot is running (first-spawn path): user_folder=%r base_url=%r pid=%s", @@ -1025,6 +1347,7 @@ async def _pre_compute_first_spawn( state_pid, ) ready = state.model_copy(update={"init_state_ok": True}) + _release_child_listen_ports(web_port, node_port, user_folder) self._emit_ensure_recall( state=ready, last_result=last_result, @@ -1051,6 +1374,43 @@ async def _pre_compute_first_spawn( parsed_process_bot_state=loaded, ) + async def _stop_bound_child_and_wait_for_exit( + self, + pid: int, + *, + ping_timeout: float, + logger: typing.Any, + ) -> dict[str, typing.Any]: + # 1. Ask the child to shut down gracefully, then wait up to ping_timeout. + stop_outcome = self.request_graceful_stop(logger=logger) + try: + await self.wait_until_pid_stopped( + pid, + logger=logger, + timeout_seconds=ping_timeout, + ) + return stop_outcome + except commons_errors.DSLInterpreterError as graceful_wait_error: + logger.warning( + "Graceful stop timed out after %ss for pid=%s; attempting force kill: %s", + ping_timeout, + pid, + graceful_wait_error, + ) + # 2. Escalate to force kill when graceful shutdown does not complete in time. + process_util.request_force_kill(pid, logger=logger) + try: + await self.wait_until_pid_stopped( + pid, + logger=logger, + timeout_seconds=DEFAULT_FORCE_KILL_EXIT_WAIT_SECONDS, + ) + except commons_errors.DSLInterpreterError as force_wait_error: + raise commons_errors.DSLInterpreterError( + f"Child pid={pid} did not exit after graceful stop and force kill." + ) from force_wait_error + return {"status": "force_killed"} + async def _pre_compute_update_config_refresh( self, last_result: dict, @@ -1063,11 +1423,11 @@ async def _pre_compute_update_config_refresh( ping_timeout: float, ) -> None: # Resolve prior child layout from re-call payload; required for stop, wait, and paths to remove. - recall_state = self._try_parse_ensure_recall_state(last_result) + recall_state = self._try_parse_octobot_process_state(last_result) if recall_state is None: raise commons_errors.DSLInterpreterError( - "run_octobot_process(UPDATE_CONFIG) requires last_execution_result from a prior " - "run_octobot_process call.", + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME}(UPDATE_CONFIG) requires last_execution_result from a prior " + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME} call.", ) process_logger = _get_logger() state_path = _resolve_state_file_path(recall_state) @@ -1075,7 +1435,7 @@ async def _pre_compute_update_config_refresh( resolved_pid = _resolve_bound_pid(recall_state, loaded_state) if resolved_pid is None: raise commons_errors.DSLInterpreterError( - "run_octobot_process(UPDATE_CONFIG) cannot resolve a running child pid to stop." + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME}(UPDATE_CONFIG) cannot resolve a running child pid to stop." ) self.pid = resolved_pid process_logger.info( @@ -1085,13 +1445,13 @@ async def _pre_compute_update_config_refresh( recall_state.log_folder, resolved_pid, ) - stop_outcome = self.request_graceful_stop(logger=process_logger) - process_logger.info("configuration update: graceful stop outcome: %s", stop_outcome) - await self.wait_until_pid_stopped( + stop_outcome = await self._stop_bound_child_and_wait_for_exit( resolved_pid, + ping_timeout=ping_timeout, logger=process_logger, - timeout_seconds=ping_timeout, ) + process_logger.info("configuration update: stop outcome: %s", stop_outcome) + _release_recall_state_listen_ports(recall_state) process_logger.info("configuration update: removing automation user and log directories") _remove_path_for_fresh_start(recall_state.user_root, logger=process_logger) _remove_path_for_fresh_start(recall_state.log_folder, logger=process_logger) @@ -1111,10 +1471,10 @@ async def pre_compute(self) -> None: params = self.get_computed_value_by_parameter() if self.matches_operator_signal(dsl_interpreter.OperatorSignal.STOP.value): last_result = self.get_last_execution_result(params) or {} - recall_state = self._try_parse_ensure_recall_state(last_result) + recall_state = self._try_parse_octobot_process_state(last_result) if recall_state is None: raise commons_errors.DSLInterpreterError( - "run_octobot_process(execution_stop) requires last_execution_result from a prior run_octobot_process call.", + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME}(execution_stop) requires last_execution_result from a prior {RUN_OCTOBOT_PROCESS_OPERATOR_NAME} call.", ) state_path = _resolve_state_file_path(recall_state) loaded_state = await _load_process_bot_state(state_path) @@ -1122,7 +1482,14 @@ async def pre_compute(self) -> None: resolved_pid = _resolve_bound_pid(recall_state, loaded_state) if resolved_pid is not None: self.pid = resolved_pid - self.value = self.request_graceful_stop(logger=_get_logger()) + process_logger = _get_logger() + ping_timeout = float(params.get("ping_timeout") or DEFAULT_ENSURE_TIMEOUT) + self.value = await self._stop_bound_child_and_wait_for_exit( + resolved_pid, + ping_timeout=ping_timeout, + logger=process_logger, + ) + _release_recall_state_listen_ports(recall_state) return # Grace with dead metadata pid: child restarting; no SIGTERM, report already_stopped. if _in_restart_grace_period( @@ -1133,9 +1500,10 @@ async def pre_compute(self) -> None: stored_pid_running=stored_pid_running, ): _get_logger().info( - "run_octobot_process(STOP): child in restart grace; treating as already_stopped" + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME}(STOP): child in restart grace; treating as already_stopped" ) self.value = {"status": "already_stopped", "reason": "not_running"} + _release_recall_state_listen_ports(recall_state) return working_directory = os.path.normpath(os.getcwd()) user_folder = params["user_folder"] @@ -1157,7 +1525,7 @@ async def pre_compute(self) -> None: ping_timeout=ping_timeout, ) return - recall_state = self._try_parse_ensure_recall_state(last_result) + recall_state = self._try_parse_octobot_process_state(last_result) if recall_state is not None: # 1. Load child dump alongside strict recall state. state_path = _resolve_state_file_path(recall_state) @@ -1208,7 +1576,7 @@ async def pre_compute(self) -> None: loaded_state.metadata.updated_at if loaded_state is not None else None ) _get_logger().info( - "run_octobot_process: respawning child (recall path declined): user_folder=%r " + f"{RUN_OCTOBOT_PROCESS_OPERATOR_NAME}: respawning child (recall path declined): user_folder=%r " "stored_pid=%s stored_pid_running=%s state_timestamp_fresh=%s init_state_ok=%s " "last_state_updated_at=%s ping_timeout=%s", recall_state.user_folder, @@ -1229,8 +1597,8 @@ async def pre_compute(self) -> None: ) - def _try_parse_ensure_recall_state(self, raw: dict) -> typing.Optional[EnsureOctobotProcessState]: - if state := _parse_ensure_recall_state(raw): + def _try_parse_octobot_process_state(self, raw: dict) -> typing.Optional[octobot_process_state_import.OctobotProcessState]: + if state := octobot_process_state_import.parse_octobot_process_state(raw): if state.pid: self.pid = state.pid return state @@ -1239,6 +1607,5 @@ def _try_parse_ensure_recall_state(self, raw: dict) -> typing.Optional[EnsureOct return [EnsureOctobotProcessOperator] - def _get_logger(): return commons_logging.get_logger("OctoBotProcessOperators") diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py index b07143d73d..3b38cb4ee1 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py @@ -29,6 +29,7 @@ import octobot.constants as octobot_constants import octobot_commons.constants as commons_constants import octobot_commons.dsl_interpreter as dsl_interpreter +import octobot_commons.enums as commons_enums import octobot_commons.errors as commons_errors import octobot_commons.os_util as os_util import octobot_commons.process_util as process_util @@ -36,10 +37,13 @@ import octobot_services.constants as services_constants import octobot_commons.profiles.profile_data as profile_data_module -import octobot_commons.profiles.exchange_auth_data as exchange_auth_data_module +import octobot_commons.profiles.profile_storage as profile_storage_module import octobot_flow.entities as octobot_flow_entities import octobot_flow.entities.accounts.process_bot_state as process_bot_state_import +import octobot_flow.entities.automations.octobot_process_state as octobot_process_state_import import octobot_tentacles_manager.constants as tentacles_manager_constants +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_backend.errors as collection_errors import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops import tentacles.Trading.Mode.grid_trading_mode.grid_trading as grid_trading_module @@ -47,6 +51,7 @@ # Nested class from factory (not exposed on ``octobot_process_ops``). TEST_EXECUTOR_ID = "test-executor" +_PROCESS_TEST_USER_ID = "wallet-user" EnsureOctobotProcessOperator = octobot_process_ops.create_octobot_process_operators( None, TEST_EXECUTOR_ID )[0] @@ -55,6 +60,12 @@ pytestmark = pytest.mark.asyncio +@pytest.fixture(autouse=True) +def _disable_auto_open_in_web_browser(): + with mock.patch.object(octobot_process_ops, "AUTO_OPEN_IN_WEB_BROWSER", False): + yield + + async def _async_return_none_mock(*_unused): return None @@ -120,7 +131,7 @@ def _healthy_recall_inner( "executor_id": executor_id, } def _stop_test_ensure_state_dict(http_base_url: str) -> dict: - return octobot_process_ops.EnsureOctobotProcessState( + return octobot_process_state_import.OctobotProcessState( http_base_url=http_base_url, web_port=1, node_port=1, @@ -215,7 +226,44 @@ def _require_octobot_project_root_for_subprocess_tests() -> str: return project_root +def _seed_executor_user_config(working_directory: pathlib.Path) -> None: + user_directory = working_directory / commons_constants.USER_FOLDER + user_directory.mkdir(parents=True, exist_ok=True) + config_path = user_directory / commons_constants.CONFIG_FILE + config_path.write_text("{}", encoding="utf-8") + + +def _seed_executor_reference_tentacles_config(working_directory: pathlib.Path) -> None: + reference_directory = working_directory.joinpath( + commons_constants.USER_FOLDER, + "reference_tentacles_config", + ) + reference_directory.mkdir(parents=True, exist_ok=True) + tentacles_config_path = reference_directory / commons_constants.CONFIG_TENTACLES_FILE + if not tentacles_config_path.is_file(): + tentacles_config_path.write_text("{}", encoding="utf-8") + + +def _generic_process_sync_strategy( + strategy_id: str, + *, + profile_data: dict | None = None, +) -> protocol_models.Strategy: + generic_process_configuration = protocol_models.GenericProcessConfiguration( + configuration_type=protocol_models.ActionConfigurationType.GENERIC_PROCESS, + profile_data=profile_data, + ) + return protocol_models.Strategy( + id=strategy_id, + version="1.0.0", + name="test-sync-strategy", + reference_market="USDT", + configuration=protocol_models.StrategyConfiguration(generic_process_configuration), + ) + + def _seed_executor_non_trading_profile(working_directory: pathlib.Path) -> None: + _seed_executor_reference_tentacles_config(working_directory) source_profile_path = _octobot_project_root_from_test_file().joinpath( commons_constants.USER_FOLDER, commons_constants.PROFILES_FOLDER, @@ -242,7 +290,9 @@ def _seed_executor_non_trading_profile(working_directory: pathlib.Path) -> None: commons_constants.CONFIG_PROFILE: { commons_constants.CONFIG_ID: octobot_process_ops.DEFAULT_DSL_PROFILE_ID, commons_constants.CONFIG_NAME: octobot_process_ops.DEFAULT_DSL_PROFILE_ID, - } + commons_constants.CONFIG_READ_ONLY: True, + }, + commons_constants.PROFILE_CONFIG: {}, } (minimal_profile_path / commons_constants.PROFILE_CONFIG_FILE).write_text( json.dumps(profile_payload), @@ -294,7 +344,8 @@ async def _poll_dsl_until_init_state_ok( timeout_sec: float = 60.0, ) -> dict: base_arguments = ( - f"{user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"{user_folder!r}, user_id={_PROCESS_TEST_USER_ID!r}, " + f"exchange_auth_data={repr(exchange_auth_list)}, " f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0" ) deadline = time.monotonic() + timeout_sec @@ -329,9 +380,43 @@ def _fresh_default_like_cfg_template(): } +def _automation_child_config_path(tmp_path, automation_id: str = "test-automation") -> str: + automation_root = ( + tmp_path + / commons_constants.USER_FOLDER + / commons_constants.AUTOMATIONS_FOLDER + / automation_id + ) + automation_root.mkdir(parents=True, exist_ok=True) + return str(automation_root / commons_constants.CONFIG_FILE) + + +def _seed_master_user_config(tmp_path, exchanges: dict) -> pathlib.Path: + master_config_path = tmp_path / commons_constants.USER_FOLDER / commons_constants.CONFIG_FILE + master_config_path.parent.mkdir(parents=True, exist_ok=True) + master_config_path.write_text( + json.dumps({commons_constants.CONFIG_EXCHANGES: exchanges}), + encoding="utf-8", + ) + return master_config_path + + +def _default_config_read_side_effect(tmp_path, master_config: dict | None = None): + default_template = _fresh_default_like_cfg_template() + master_config_path = tmp_path / commons_constants.USER_FOLDER / commons_constants.CONFIG_FILE + + def _read_file_side_effect(path, *_unused, **_kwargs): + normalized_path = os.path.normpath(str(path)) + if master_config is not None and normalized_path == os.path.normpath(str(master_config_path)): + return master_config + return default_template + + return _read_file_side_effect + + class TestWriteUserRootConfigJson: - def test_sets_profile_and_disables_browser_auto_open(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + def test_sets_profile_and_writes_browser_auto_open_from_constant(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) profile_id = "dsl_profile_abc" with mock.patch.object( octobot_process_ops.json_util, @@ -339,17 +424,55 @@ def test_sets_profile_and_disables_browser_auto_open(self, tmp_path): side_effect=lambda *_unused: _fresh_default_like_cfg_template(), ): octobot_process_ops._write_user_root_config_json( - config_path, profile_id, None, None + config_path, profile_id, None, None, str(tmp_path) ) written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) assert written[commons_constants.CONFIG_PROFILE] == profile_id assert written[services_constants.CONFIG_CATEGORY_SERVICES][services_constants.CONFIG_WEB][ services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER - ] is False + ] is octobot_process_ops.AUTO_OPEN_IN_WEB_BROWSER assert written[commons_constants.CONFIG_EXCHANGES] == {} + def test_writes_octobot_name_when_provided(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + profile_id = "dsl_profile_abc" + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + profile_id, + None, + None, + str(tmp_path), + octobot_name="My Bot", + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert written[commons_constants.CONFIG_OCTOBOT_NAME] == "My Bot" + + def test_omits_octobot_name_when_blank(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + profile_id = "dsl_profile_abc" + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + profile_id, + None, + None, + str(tmp_path), + octobot_name=" ", + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert commons_constants.CONFIG_OCTOBOT_NAME not in written + def test_seeds_exchanges_from_profile_data(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) profile_dict = { **_MINIMAL_PROFILE_DATA, "exchanges": [ @@ -364,7 +487,7 @@ def test_seeds_exchanges_from_profile_data(self, tmp_path): side_effect=lambda *_unused: _fresh_default_like_cfg_template(), ): octobot_process_ops._write_user_root_config_json( - config_path, "p1", profile_data, None + config_path, "p1", profile_data, None, str(tmp_path) ) written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) exchanges_cfg = written[commons_constants.CONFIG_EXCHANGES] @@ -375,11 +498,43 @@ def test_seeds_exchanges_from_profile_data(self, tmp_path): assert seeded[commons_constants.CONFIG_EXCHANGE_KEY] == octobot_process_ops._DEFAULT_ENCRYPTED_VALUE assert seeded[commons_constants.CONFIG_EXCHANGE_SECRET] == octobot_process_ops._DEFAULT_ENCRYPTED_VALUE + def test_profile_exchange_stays_enabled_when_default_template_disables_it(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + profile_exchange_internal_name = "binanceus" + template = _fresh_default_like_cfg_template() + template[commons_constants.CONFIG_EXCHANGES] = { + profile_exchange_internal_name: { + commons_constants.CONFIG_ENABLED_OPTION: False, + commons_constants.CONFIG_EXCHANGE_TYPE: commons_constants.CONFIG_EXCHANGE_SPOT, + } + } + profile_dict = { + **_MINIMAL_PROFILE_DATA, + "exchanges": [ + { + "internal_name": profile_exchange_internal_name, + "exchange_type": commons_constants.CONFIG_EXCHANGE_SPOT, + } + ], + } + profile_data = profile_data_module.ProfileData.from_dict(profile_dict) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: template, + ): + octobot_process_ops._write_user_root_config_json( + config_path, "grid-profile", profile_data, None, str(tmp_path) + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + profile_exchange_cfg = written[commons_constants.CONFIG_EXCHANGES][profile_exchange_internal_name] + assert profile_exchange_cfg[commons_constants.CONFIG_ENABLED_OPTION] is True + def test_presets_encrypted_empty_credentials_when_default_config_exchange_has_no_api_fields( self, tmp_path ): """Mirrors packaged ``default_config.json`` rows that omit api-key/api-secret until setdefault.""" - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) template = _fresh_default_like_cfg_template() template[commons_constants.CONFIG_EXCHANGES] = { "prefilled_exchange": { @@ -392,30 +547,34 @@ def test_presets_encrypted_empty_credentials_when_default_config_exchange_has_no "read_file", side_effect=lambda *_unused: template, ): - octobot_process_ops._write_user_root_config_json(config_path, "p0", None, None) + octobot_process_ops._write_user_root_config_json( + config_path, "p0", None, None, str(tmp_path) + ) written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) exch = written[commons_constants.CONFIG_EXCHANGES]["prefilled_exchange"] assert exch[commons_constants.CONFIG_EXCHANGE_KEY] == octobot_process_ops._DEFAULT_ENCRYPTED_VALUE assert exch[commons_constants.CONFIG_EXCHANGE_SECRET] == octobot_process_ops._DEFAULT_ENCRYPTED_VALUE def test_applies_exchange_auth_credentials(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) - auth_list = [ - exchange_auth_data_module.ExchangeAuthData( - internal_name="binance_test", - api_key="key-a", - api_secret="secret-b", - api_password="pwd-c", - exchange_type="spot", - sandboxed=True, - ) + config_path = _automation_child_config_path(tmp_path) + auth_overrides = [ + { + "internal_name": "binance_test", + "api_key": "key-a", + "api_secret": "secret-b", + "api_password": "pwd-c", + "exchange_type": "spot", + "sandboxed": True, + } ] with mock.patch.object( octobot_process_ops.json_util, "read_file", side_effect=lambda *_unused: _fresh_default_like_cfg_template(), ): - octobot_process_ops._write_user_root_config_json(config_path, "p2", None, auth_list) + octobot_process_ops._write_user_root_config_json( + config_path, "p2", None, auth_overrides, str(tmp_path) + ) written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) exch = written[commons_constants.CONFIG_EXCHANGES]["binance_test"] assert exch[commons_constants.CONFIG_EXCHANGE_KEY] == "key-a" @@ -425,20 +584,20 @@ def test_applies_exchange_auth_credentials(self, tmp_path): assert exch[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is True def test_profile_seed_then_auth_overlay(self, tmp_path): - config_path = str(tmp_path / commons_constants.CONFIG_FILE) + config_path = _automation_child_config_path(tmp_path) exchange_internal_name = "overlay_exchange" profile_dict = { **_MINIMAL_PROFILE_DATA, "exchanges": [{"internal_name": exchange_internal_name, "exchange_type": "spot"}], } profile_data = profile_data_module.ProfileData.from_dict(profile_dict) - auth_list = [ - exchange_auth_data_module.ExchangeAuthData( - internal_name=exchange_internal_name, - api_key="overlay-key", - api_secret="overlay-secret", - exchange_type="spot", - ) + auth_overrides = [ + { + "internal_name": exchange_internal_name, + "api_key": "overlay-key", + "api_secret": "overlay-secret", + "exchange_type": "spot", + } ] with mock.patch.object( octobot_process_ops.json_util, @@ -446,7 +605,7 @@ def test_profile_seed_then_auth_overlay(self, tmp_path): side_effect=lambda *_unused: _fresh_default_like_cfg_template(), ): octobot_process_ops._write_user_root_config_json( - config_path, "p3", profile_data, auth_list + config_path, "p3", profile_data, auth_overrides, str(tmp_path) ) written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) exch = written[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] @@ -455,9 +614,312 @@ def test_profile_seed_then_auth_overlay(self, tmp_path): assert exch[commons_constants.CONFIG_EXCHANGE_KEY] == "overlay-key" assert exch[commons_constants.CONFIG_EXCHANGE_SECRET] == "overlay-secret" + def test_profile_seed_disables_non_profile_master_exchanges_but_forwards_creds(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + profile_exchange_internal_name = "binanceus" + master_exchange_internal_name = "binance" + master_api_key = "master-binance-key" + master_api_secret = "master-binance-secret" + master_api_password = "master-binance-password" + master_config = { + commons_constants.CONFIG_EXCHANGES: { + master_exchange_internal_name: { + commons_constants.CONFIG_EXCHANGE_KEY: master_api_key, + commons_constants.CONFIG_EXCHANGE_SECRET: master_api_secret, + commons_constants.CONFIG_EXCHANGE_PASSWORD: master_api_password, + commons_constants.CONFIG_EXCHANGE_TYPE: commons_constants.CONFIG_EXCHANGE_SPOT, + commons_constants.CONFIG_EXCHANGE_SANDBOXED: True, + } + } + } + _seed_master_user_config(tmp_path, master_config[commons_constants.CONFIG_EXCHANGES]) + profile_dict = { + **_MINIMAL_PROFILE_DATA, + "exchanges": [ + { + "internal_name": profile_exchange_internal_name, + "exchange_type": commons_constants.CONFIG_EXCHANGE_SPOT, + } + ], + } + profile_data = profile_data_module.ProfileData.from_dict(profile_dict) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=_default_config_read_side_effect(tmp_path, master_config), + ): + octobot_process_ops._write_user_root_config_json( + config_path, "p-profile-master", profile_data, None, str(tmp_path) + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + exchanges_cfg = written[commons_constants.CONFIG_EXCHANGES] + profile_exchange_cfg = exchanges_cfg[profile_exchange_internal_name] + master_exchange_cfg = exchanges_cfg[master_exchange_internal_name] + assert profile_exchange_cfg[commons_constants.CONFIG_ENABLED_OPTION] is True + assert profile_exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == ( + octobot_process_ops._DEFAULT_ENCRYPTED_VALUE + ) + assert profile_exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == ( + octobot_process_ops._DEFAULT_ENCRYPTED_VALUE + ) + assert master_exchange_cfg[commons_constants.CONFIG_ENABLED_OPTION] is False + assert master_exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == master_api_key + assert master_exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == master_api_secret + assert master_exchange_cfg[commons_constants.CONFIG_EXCHANGE_PASSWORD] == master_api_password + assert master_exchange_cfg[commons_constants.CONFIG_EXCHANGE_TYPE] == commons_constants.CONFIG_EXCHANGE_SPOT + assert master_exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is True + + def test_does_not_persist_sync_user_id_in_config(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + "sync-profile-id", + None, + None, + str(tmp_path), + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert written[commons_constants.CONFIG_PROFILE] == "sync-profile-id" + assert "sync_user_id" not in written + + def test_writes_master_overlay_config_when_provided(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + master_profiles_path = str(tmp_path / "master" / commons_constants.PROFILES_FOLDER) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + None, + None, + str(tmp_path), + readonly_profiles_path=master_profiles_path, + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert written[commons_constants.CONFIG_READONLY_PROFILES_PATH] == master_profiles_path + + def test_writes_master_reference_tentacles_overlay_config_when_provided(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + master_reference_path = str(tmp_path / "master" / "reference_tentacles_config") + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + None, + None, + str(tmp_path), + readonly_reference_tentacles_path=master_reference_path, + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert ( + written[commons_constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH] + == master_reference_path + ) + + def test_forwards_master_exchange_auth_when_no_overrides(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + master_exchange_internal_name = "master_binance" + master_api_key = "master-forwarded-key" + master_api_secret = "master-forwarded-secret" + master_api_password = "master-forwarded-password" + master_config = { + commons_constants.CONFIG_EXCHANGES: { + master_exchange_internal_name: { + commons_constants.CONFIG_EXCHANGE_KEY: master_api_key, + commons_constants.CONFIG_EXCHANGE_SECRET: master_api_secret, + commons_constants.CONFIG_EXCHANGE_PASSWORD: master_api_password, + commons_constants.CONFIG_EXCHANGE_TYPE: commons_constants.CONFIG_EXCHANGE_SPOT, + commons_constants.CONFIG_EXCHANGE_SANDBOXED: True, + } + } + } + _seed_master_user_config( + tmp_path, + master_config[commons_constants.CONFIG_EXCHANGES], + ) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=_default_config_read_side_effect(tmp_path, master_config), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + "p-master-forward", + None, + None, + str(tmp_path), + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + exchange_cfg = written[commons_constants.CONFIG_EXCHANGES][master_exchange_internal_name] + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == master_api_key + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == master_api_secret + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_PASSWORD] == master_api_password + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_TYPE] == commons_constants.CONFIG_EXCHANGE_SPOT + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is True + + def test_exchange_auth_override_fully_replaces_master_entry(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + exchange_internal_name = "binance" + master_config = { + commons_constants.CONFIG_EXCHANGES: { + exchange_internal_name: { + commons_constants.CONFIG_EXCHANGE_KEY: "master-key", + commons_constants.CONFIG_EXCHANGE_SECRET: "master-secret", + commons_constants.CONFIG_EXCHANGE_TYPE: commons_constants.CONFIG_EXCHANGE_SPOT, + commons_constants.CONFIG_EXCHANGE_SANDBOXED: False, + } + } + } + _seed_master_user_config( + tmp_path, + master_config[commons_constants.CONFIG_EXCHANGES], + ) + auth_overrides = [ + { + "internal_name": exchange_internal_name, + "sandboxed": True, + } + ] + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=_default_config_read_side_effect(tmp_path, master_config), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + "p-master-override", + None, + auth_overrides, + str(tmp_path), + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + exchange_cfg = written[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == "" + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == "" + assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is True + + def test_master_missing_config_preserves_current_behavior(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + octobot_process_ops._write_user_root_config_json( + config_path, + "p-no-master", + None, + None, + str(tmp_path), + ) + written = json.loads(pathlib.Path(config_path).read_text(encoding="utf-8")) + assert written[commons_constants.CONFIG_EXCHANGES] == {} + + def test_unreadable_master_config_raises(self, tmp_path): + config_path = _automation_child_config_path(tmp_path) + master_config_path = _seed_master_user_config(tmp_path, {}) + default_template = _fresh_default_like_cfg_template() + + def read_file_side_effect(path, *_unused, **_kwargs): + if os.path.normpath(str(path)) == os.path.normpath(str(master_config_path)): + raise OSError("permission denied") + return default_template + + with mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=read_file_side_effect, + ), pytest.raises(commons_errors.DSLInterpreterError, match="Failed to read master user config"): + octobot_process_ops._write_user_root_config_json( + config_path, + "p-bad-master", + None, + None, + str(tmp_path), + ) + + +class TestAutomationChildPathGuards: + def test_write_user_root_config_json_rejects_master_config_path(self, tmp_path): + master_config_path = str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.CONFIG_FILE + ) + with pytest.raises(commons_errors.DSLInterpreterError, match="user/automations"): + octobot_process_ops._write_user_root_config_json( + master_config_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + None, + None, + str(tmp_path), + ) + + def test_write_user_root_config_json_rejects_automations_root_without_leaf(self, tmp_path): + automations_config_path = str( + tmp_path + / commons_constants.USER_FOLDER + / commons_constants.AUTOMATIONS_FOLDER + / commons_constants.CONFIG_FILE + ) + with pytest.raises(commons_errors.DSLInterpreterError, match="automation_id"): + octobot_process_ops._write_user_root_config_json( + automations_config_path, + octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + None, + None, + str(tmp_path), + ) + + def test_assert_spawn_cmd_isolation_rejects_master_user_folder(self): + rel_user = commons_constants.USER_FOLDER + rel_log = os.path.join(octobot_node_constants.AUTOMATION_LOGS_FOLDER, "bot-1") + cmd = [ + sys.executable, + "start.py", + "--user-folder", + rel_user, + "--log-folder", + rel_log, + ] + with pytest.raises(commons_errors.DSLInterpreterError, match="user/automations"): + octobot_process_ops._assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + + def test_assert_spawn_cmd_isolation_rejects_missing_log_folder_flag(self): + rel_user = os.path.join(commons_constants.USER_AUTOMATIONS_FOLDER, "bot-1") + rel_log = os.path.join(octobot_node_constants.AUTOMATION_LOGS_FOLDER, "bot-1") + cmd = [sys.executable, "start.py", "--user-folder", rel_user] + with pytest.raises(commons_errors.DSLInterpreterError, match="--log-folder"): + octobot_process_ops._assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + + def test_assert_spawn_cmd_isolation_accepts_valid_cmd(self): + rel_user = os.path.join(commons_constants.USER_AUTOMATIONS_FOLDER, "bot-1") + rel_log = os.path.join(octobot_node_constants.AUTOMATION_LOGS_FOLDER, "bot-1") + cmd = [ + sys.executable, + "start.py", + "--user-folder", + rel_user, + "--log-folder", + rel_log, + "--dump-state", + "/tmp/state.json", + ] + octobot_process_ops._assert_spawn_cmd_isolation(cmd, rel_user, rel_log) + class TestEnsureUserProfileAndLayout: - async def test_marked_prepared_is_skipped(self, tmp_path): + async def test_prepared_layout_is_skipped_when_config_exists(self, tmp_path): user = tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "u1" user.mkdir(parents=True) config_path = user / commons_constants.CONFIG_FILE @@ -465,16 +927,21 @@ async def test_marked_prepared_is_skipped(self, tmp_path): json.dumps({commons_constants.CONFIG_PROFILE: "p1"}), encoding="utf-8", ) - (user / octobot_process_ops.DSL_PREPARED_MARKER).write_text("1", encoding="utf-8") res = await octobot_process_ops.ensure_user_profile_and_layout( "u1", str(tmp_path), _MINIMAL_PROFILE_DATA, - None, ) assert res["already_prepared"] is True assert res["profile_id"] == "p1" + async def test_raises_when_master_reference_missing(self, tmp_path): + with pytest.raises(commons_errors.DSLInterpreterError, match="Master reference tentacles"): + await octobot_process_ops.ensure_user_profile_and_layout( + "missing_ref_user", + str(tmp_path), + ) + class TestEnsureOctobotProcessOperatorProfileDataOptional: def test_declares_optional_profile_data_parameter(self): @@ -488,78 +955,169 @@ def test_declares_optional_profile_data_parameter(self): assert profile_parameter.default is None -class TestCopyReadOnlyProfilesToUserRoot: - async def test_copies_read_only_profiles_and_skips_editable(self, tmp_path): - _seed_executor_non_trading_profile(tmp_path) - readonly_profile_id = "readonly_strategy" - editable_profile_id = "editable_strategy" - _seed_executor_profile(tmp_path, readonly_profile_id, read_only=True) - _seed_executor_profile(tmp_path, editable_profile_id, read_only=False) - user_root = tmp_path / "child_user_root" - user_root.mkdir() - await octobot_process_ops._copy_read_only_profiles_to_user_root( - str(tmp_path), - str(user_root), - active_profile_id=octobot_process_ops.DEFAULT_DSL_PROFILE_ID, +class TestEnsureOctobotProcessOperatorSyncProfileIdOptional: + def test_declares_optional_sync_profile_id_parameter(self): + params = EnsureOctobotProcessOperator.get_parameters() + sync_profile_parameter = next( + (parameter for parameter in params if parameter.name == "sync_profile_id"), + None, ) - profiles_root = user_root / commons_constants.PROFILES_FOLDER - readonly_profile_json = ( - profiles_root / readonly_profile_id / commons_constants.PROFILE_CONFIG_FILE + assert sync_profile_parameter is not None + assert sync_profile_parameter.required is False + assert sync_profile_parameter.default is None + + def test_declares_optional_user_id_parameter(self): + params = EnsureOctobotProcessOperator.get_parameters() + user_id_parameter = next( + (parameter for parameter in params if parameter.name == "user_id"), + None, ) - editable_profile_json = ( - profiles_root / editable_profile_id / commons_constants.PROFILE_CONFIG_FILE + assert user_id_parameter is not None + assert user_id_parameter.required is False + assert user_id_parameter.default is None + + +class TestAssertSyncStrategyExists: + def test_accepts_existing_strategy(self): + stored_strategy = _generic_process_sync_strategy("strategy-1") + strategy_provider_mock = mock.Mock() + strategy_provider_mock.get_item.return_value = stored_strategy + with mock.patch.object( + octobot_process_ops.collection_providers, + "StrategyProvider", + ) as strategy_provider_class: + strategy_provider_class.instance.return_value = strategy_provider_mock + result = octobot_process_ops._assert_sync_strategy_exists( + "wallet-user", + "strategy-1", + ) + assert result is stored_strategy + strategy_provider_mock.get_item.assert_called_once_with("wallet-user", "strategy-1") + + def test_raises_when_strategy_not_found(self): + strategy_provider_mock = mock.Mock() + strategy_provider_mock.get_item.side_effect = collection_errors.ItemNotFoundError( + "missing" ) - non_trading_profile_json = ( - profiles_root - / octobot_process_ops.DEFAULT_DSL_PROFILE_ID - / commons_constants.PROFILE_CONFIG_FILE + with mock.patch.object( + octobot_process_ops.collection_providers, + "StrategyProvider", + ) as strategy_provider_class: + strategy_provider_class.instance.return_value = strategy_provider_mock + with pytest.raises(commons_errors.DSLInterpreterError, match="not found"): + octobot_process_ops._assert_sync_strategy_exists( + "wallet-user", + "strategy-1", + ) + + +class TestEnsureUserProfileAndLayoutSyncProfileId: + async def test_raises_when_user_id_missing_with_sync_profile_id(self, tmp_path): + _seed_executor_reference_tentacles_config(tmp_path) + with pytest.raises(commons_errors.DSLInterpreterError, match="requires user_id"): + await octobot_process_ops.ensure_user_profile_and_layout( + "sync_user_folder", + str(tmp_path), + sync_profile_id="sync-strategy-1", + ) + + async def test_bare_generic_process_strategy_uses_non_trading_profile(self, tmp_path): + sync_profile_id = "sync-strategy-1" + executor_sync_user_id = "wallet-user" + bare_strategy = _generic_process_sync_strategy(sync_profile_id) + _seed_executor_reference_tentacles_config(tmp_path) + with mock.patch.object( + octobot_process_ops, + "_assert_sync_strategy_exists", + return_value=bare_strategy, + ), mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + result = await octobot_process_ops.ensure_user_profile_and_layout( + "sync_user_folder", + str(tmp_path), + sync_profile_id=sync_profile_id, + user_id=executor_sync_user_id, + ) + user_root = pathlib.Path(result["user_root"]) + assert result["profile_id"] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert not (user_root / commons_constants.PROFILES_FOLDER / sync_profile_id).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert "sync_user_id" not in root_cfg + assert root_cfg[commons_constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH] == ( + octobot_process_ops._executor_reference_tentacles_directory(str(tmp_path)) ) - assert readonly_profile_json.is_file() - assert not editable_profile_json.exists() - assert not non_trading_profile_json.exists() - async def test_skips_active_profile_id(self, tmp_path): - _seed_executor_profile( - tmp_path, - octobot_process_ops.DEFAULT_DSL_PROFILE_ID, - read_only=True, + async def test_strategy_with_profile_data_uses_sync_profile_id(self, tmp_path): + sync_profile_id = "sync-strategy-1" + executor_sync_user_id = "wallet-user" + strategy_with_profile_data = _generic_process_sync_strategy( + sync_profile_id, + profile_data={"profile_details": {"id": sync_profile_id}}, ) - user_root = tmp_path / "child_user_root" - user_root.mkdir() - destination_profile_path = ( - user_root - / commons_constants.PROFILES_FOLDER - / octobot_process_ops.DEFAULT_DSL_PROFILE_ID + _seed_executor_reference_tentacles_config(tmp_path) + with mock.patch.object( + octobot_process_ops, + "_assert_sync_strategy_exists", + return_value=strategy_with_profile_data, + ), mock.patch.object( + octobot_process_ops.json_util, + "read_file", + side_effect=lambda *_unused: _fresh_default_like_cfg_template(), + ): + result = await octobot_process_ops.ensure_user_profile_and_layout( + "sync_user_folder", + str(tmp_path), + sync_profile_id=sync_profile_id, + user_id=executor_sync_user_id, + ) + user_root = pathlib.Path(result["user_root"]) + assert result["profile_id"] == sync_profile_id + assert not (user_root / commons_constants.PROFILES_FOLDER / sync_profile_id).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_PROFILE] == sync_profile_id + assert "sync_user_id" not in root_cfg + assert root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(str(tmp_path)) ) - destination_profile_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree( - tmp_path.joinpath( - commons_constants.USER_FOLDER, - commons_constants.PROFILES_FOLDER, - octobot_process_ops.DEFAULT_DSL_PROFILE_ID, - ), - destination_profile_path, - ) - profile_json_path = destination_profile_path / commons_constants.PROFILE_CONFIG_FILE - original_mtime = profile_json_path.stat().st_mtime - await octobot_process_ops._copy_read_only_profiles_to_user_root( + + +class TestEnsureUserProfileAndLayoutProfileDataPriority: + async def test_profile_data_wins_over_sync_profile_id(self, tmp_path): + sync_profile_id = "ignored-sync-profile" + _seed_executor_reference_tentacles_config(tmp_path) + result = await octobot_process_ops.ensure_user_profile_and_layout( + "priority_user_folder", str(tmp_path), - str(user_root), - active_profile_id=octobot_process_ops.DEFAULT_DSL_PROFILE_ID, + _MINIMAL_PROFILE_DATA, + sync_profile_id=sync_profile_id, ) - assert profile_json_path.stat().st_mtime == original_mtime + user_root = pathlib.Path(result["user_root"]) + profile_id = result["profile_id"] + assert profile_id + assert profile_id != sync_profile_id + assert ( + user_root + / commons_constants.PROFILES_FOLDER + / profile_id + / commons_constants.PROFILE_CONFIG_FILE + ).is_file() + assert not (user_root / commons_constants.PROFILES_FOLDER / sync_profile_id).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_PROFILE] == profile_id + assert "sync_user_id" not in root_cfg class TestEnsureUserProfileAndLayoutDefaultProfile: - async def test_copies_non_trading_profile_and_writes_default_config(self, tmp_path): + async def test_pins_non_trading_profile_via_overlay_and_writes_default_config(self, tmp_path): _seed_executor_non_trading_profile(tmp_path) user_leaf = "default_profile_layout_user" result = await octobot_process_ops.ensure_user_profile_and_layout( user_leaf, str(tmp_path), - None, - None, - None, ) assert result["already_prepared"] is False assert result["profile_id"] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID @@ -570,19 +1128,51 @@ async def test_copies_non_trading_profile_and_writes_default_config(self, tmp_pa / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not profile_json_path.exists() root_config_path = user_root / commons_constants.CONFIG_FILE root_cfg = json.loads(root_config_path.read_text(encoding="utf-8")) assert root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(str(tmp_path)) + ) assert ( root_cfg[services_constants.CONFIG_CATEGORY_SERVICES][services_constants.CONFIG_WEB][ services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER ] - is False + is octobot_process_ops.AUTO_OPEN_IN_WEB_BROWSER + ) + assert root_cfg[commons_constants.CONFIG_ACCEPTED_TERMS] is False + + async def test_resolves_non_trading_profile_from_master_overlay(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + result = await octobot_process_ops.ensure_user_profile_and_layout( + "default_overlay_resolution_user", + str(tmp_path), + ) + user_root = pathlib.Path(result["user_root"]) + child_profiles_path = user_root / commons_constants.PROFILES_FOLDER + child_profiles_path.mkdir(parents=True, exist_ok=True) + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + profile_schema_path = str( + _octobot_project_root_from_test_file() / octobot_constants.PROFILE_FILE_SCHEMA + ) + profile_storage = profile_storage_module.ProfileStorage( + str(child_profiles_path), + profile_schema_path, + ) + profile_storage.configure_readonly_profiles_path( + root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH], + ) + loaded_profiles = profile_storage.load_all_profiles() + resolved_profile = loaded_profiles.get(commons_constants.DEFAULT_PROFILE) + if resolved_profile is None: + resolved_profile = loaded_profiles.get(octobot_process_ops.DEFAULT_DSL_PROFILE_ID) + assert resolved_profile is not None + assert octobot_process_ops.DEFAULT_DSL_PROFILE_ID in resolved_profile.path.replace( + "\\", "/" ) - assert root_cfg[commons_constants.CONFIG_ACCEPTED_TERMS] is True - async def test_copies_read_only_profiles_on_default_layout(self, tmp_path): + async def test_default_layout_does_not_copy_read_only_profiles(self, tmp_path): _seed_executor_non_trading_profile(tmp_path) readonly_profile_id = "readonly_strategy" _seed_executor_profile(tmp_path, readonly_profile_id, read_only=True) @@ -590,39 +1180,46 @@ async def test_copies_read_only_profiles_on_default_layout(self, tmp_path): result = await octobot_process_ops.ensure_user_profile_and_layout( user_leaf, str(tmp_path), - None, - None, - None, ) user_root = pathlib.Path(result["user_root"]) profiles_root = user_root / commons_constants.PROFILES_FOLDER - assert ( + assert not ( profiles_root / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE - ).is_file() - assert ( + ).exists() + assert not ( profiles_root / readonly_profile_id / commons_constants.PROFILE_CONFIG_FILE - ).is_file() + ).exists() + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert commons_constants.CONFIG_READONLY_PROFILES_PATH in root_cfg + + async def test_default_layout_omits_sync_user_id_without_user_id_arg(self, tmp_path): + _seed_executor_non_trading_profile(tmp_path) + result = await octobot_process_ops.ensure_user_profile_and_layout( + "default_layout_without_sync_user", + str(tmp_path), + ) + user_root = pathlib.Path(result["user_root"]) + root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert "sync_user_id" not in root_cfg async def test_applies_exchange_auth_without_profile_data(self, tmp_path): _seed_executor_non_trading_profile(tmp_path) exchange_internal_name = "default_layout_exchange" - exchange_auth_list = [ - exchange_auth_data_module.ExchangeAuthData( - internal_name=exchange_internal_name, - api_key="layout-key", - api_secret="layout-secret", - exchange_type=commons_constants.CONFIG_EXCHANGE_SPOT, - sandboxed=True, - ) + exchange_auth_overrides = [ + { + "internal_name": exchange_internal_name, + "api_key": "layout-key", + "api_secret": "layout-secret", + "exchange_type": commons_constants.CONFIG_EXCHANGE_SPOT, + "sandboxed": True, + } ] result = await octobot_process_ops.ensure_user_profile_and_layout( "default_exchange_user", str(tmp_path), - None, - None, - exchange_auth_list, + exchange_auth_overrides=exchange_auth_overrides, ) user_root = pathlib.Path(result["user_root"]) root_cfg = json.loads((user_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) @@ -649,23 +1246,23 @@ async def test_writes_profile_tree_top_level_config_and_exchange_credentials(sel } ], } - exchange_auth_list = [ - exchange_auth_data_module.ExchangeAuthData( - internal_name=exchange_internal_name, - api_key=fake_api_key, - api_secret=fake_api_secret, - api_password=fake_api_password, - exchange_type=commons_constants.CONFIG_EXCHANGE_SPOT, - sandboxed=True, - ) + exchange_auth_overrides = [ + { + "internal_name": exchange_internal_name, + "api_key": fake_api_key, + "api_secret": fake_api_secret, + "api_password": fake_api_password, + "exchange_type": commons_constants.CONFIG_EXCHANGE_SPOT, + "sandboxed": True, + } ] + _seed_executor_reference_tentacles_config(tmp_path) result = await octobot_process_ops.ensure_user_profile_and_layout( user_leaf, str(tmp_path), profile_dict, - None, - exchange_auth_list, + exchange_auth_overrides, ) assert result["already_prepared"] is False @@ -676,26 +1273,26 @@ async def test_writes_profile_tree_top_level_config_and_exchange_credentials(sel tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / user_leaf ) - marker_path = user_root / octobot_process_ops.DSL_PREPARED_MARKER root_config_path = user_root / commons_constants.CONFIG_FILE profile_dir = user_root / commons_constants.PROFILES_FOLDER / profile_id profile_json_path = profile_dir / commons_constants.PROFILE_CONFIG_FILE tentacles_setup_path = profile_dir / commons_constants.CONFIG_TENTACLES_FILE - assert marker_path.is_file() assert root_config_path.is_file() assert profile_json_path.is_file() assert tentacles_setup_path.is_file() - reference_layout = user_root / "reference_tentacles_config" - assert reference_layout.is_dir() + assert not (user_root / "reference_tentacles_config").exists() root_cfg = json.loads(root_config_path.read_text(encoding="utf-8")) assert root_cfg[commons_constants.CONFIG_PROFILE] == profile_id + assert root_cfg[commons_constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH] == ( + octobot_process_ops._executor_reference_tentacles_directory(str(tmp_path)) + ) assert ( root_cfg[services_constants.CONFIG_CATEGORY_SERVICES][services_constants.CONFIG_WEB][ services_constants.CONFIG_AUTO_OPEN_IN_WEB_BROWSER ] - is False + is octobot_process_ops.AUTO_OPEN_IN_WEB_BROWSER ) exchange_root = root_cfg[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] assert exchange_root[commons_constants.CONFIG_ENABLED_OPTION] is True @@ -885,6 +1482,129 @@ def test_skips_port_occupied_on_host(self): assert web_port != occupied_port +class TestChildListenPortReservation: + @pytest.fixture(autouse=True) + def _clear_reserved_ports(self): + octobot_process_ops._child_listen_ports_reserved.clear() + yield + octobot_process_ops._child_listen_ports_reserved.clear() + + def test_parallel_allocations_get_distinct_web_ports(self): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + first_web_port, first_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-a", max_offset=100 + ) + second_web_port, second_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-b", max_offset=100 + ) + try: + assert first_web_port != second_web_port + assert first_node_port != second_node_port + finally: + octobot_process_ops._release_child_listen_ports(first_web_port, first_node_port, "automation-a") + octobot_process_ops._release_child_listen_ports(second_web_port, second_node_port, "automation-b") + + def test_release_allows_reuse_when_no_listener(self): + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port, node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-a", max_offset=100 + ) + octobot_process_ops._release_child_listen_ports(web_port, node_port, "automation-a") + reused_web_port, reused_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 20000, 30000, "automation-a", max_offset=100 + ) + try: + assert reused_web_port == web_port + assert reused_node_port == node_port + finally: + octobot_process_ops._release_child_listen_ports(reused_web_port, reused_node_port, "automation-a") + + def test_second_allocation_skips_reserved_port(self): + octobot_process_ops._reserve_child_listen_ports(5002, 5999, "automation-a") + try: + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port, _node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, "automation-b", max_offset=10 + ) + assert web_port != 5002 + finally: + octobot_process_ops._release_child_listen_ports(5002, 5999, "automation-a") + + def test_stale_prior_recall_release_does_not_free_other_automation_ports(self): + user_folder_a = "03e38366-99ea-4c47-84d5-4329c7aa00df" + user_folder_b = "7ae4e140-1dc0-4d7c-a4cf-38c121a80f72" + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port_a, node_port_a = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder_a, max_offset=10 + ) + assert web_port_a == 5002 + stale_recall_b = octobot_process_state_import.OctobotProcessState( + http_base_url="http://127.0.0.1:5002", + web_port=5002, + node_port=node_port_a, + user_root="/x/b", + user_folder=user_folder_b, + log_folder="/x/logs/b", + profile_id=None, + pid=0, + executor_id=TEST_EXECUTOR_ID, + ) + octobot_process_ops._release_recall_state_listen_ports(stale_recall_b) + assert octobot_process_ops._child_listen_ports_reserved.get(5002) == user_folder_a + try: + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port_b, node_port_b = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder_b, max_offset=10 + ) + assert web_port_b != 5002 + finally: + octobot_process_ops._release_child_listen_ports(web_port_a, node_port_a, user_folder_a) + octobot_process_ops._release_child_listen_ports(web_port_b, node_port_b, user_folder_b) + + def test_same_automation_recall_release_frees_ports_for_reuse(self): + user_folder = "automation-a" + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + web_port, node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder, max_offset=10 + ) + recall_state = octobot_process_state_import.OctobotProcessState( + http_base_url=f"http://127.0.0.1:{web_port}", + web_port=web_port, + node_port=node_port, + user_root="/x/a", + user_folder=user_folder, + log_folder="/x/logs/a", + profile_id=None, + pid=0, + executor_id=TEST_EXECUTOR_ID, + ) + octobot_process_ops._release_recall_state_listen_ports(recall_state) + assert web_port not in octobot_process_ops._child_listen_ports_reserved + with mock.patch.object(os_util, "tcp_port_has_listener_on_host", return_value=False), mock.patch.object( + os_util, "tcp_port_is_free", return_value=True + ): + reused_web_port, reused_node_port = octobot_process_ops._allocate_child_listen_port_pair( + "127.0.0.1", 5002, 6000, user_folder, max_offset=10 + ) + try: + assert reused_web_port == web_port + assert reused_node_port == node_port + finally: + octobot_process_ops._release_child_listen_ports(reused_web_port, reused_node_port, user_folder) + + class TestEnsureOctobotProcessOperatorExchangeAuthData: def test_declares_optional_exchange_auth_parameter(self): params = EnsureOctobotProcessOperator.get_parameters() @@ -895,7 +1615,7 @@ def test_declares_optional_exchange_auth_parameter(self): assert auth_parameter is not None assert auth_parameter.required is False assert auth_parameter.default is None - assert auth_parameter.type == list[dict] + assert auth_parameter.type == commons_enums.DslValueType.ANY.value async def test_pre_compute_passes_dict_exchange_auth_into_ensure_layout(self, tmp_path): exchange_auth_dicts = [ @@ -919,6 +1639,7 @@ async def test_pre_compute_passes_dict_exchange_auth_into_ensure_layout(self, tm start_script.write_text("#", encoding="utf-8") operator_instance = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, exchange_auth_data=exchange_auth_dicts, last_execution_result=None, @@ -944,23 +1665,254 @@ async def test_pre_compute_passes_dict_exchange_auth_into_ensure_layout(self, tm ensure_layout_mock.assert_awaited_once() await_arguments = ensure_layout_mock.await_args.args - assert len(await_arguments) >= 5 - parsed_exchange_auth = await_arguments[4] - assert parsed_exchange_auth is not None - assert len(parsed_exchange_auth) == 1 - assert isinstance(parsed_exchange_auth[0], exchange_auth_data_module.ExchangeAuthData) - assert parsed_exchange_auth[0].internal_name == "dsl_exchange_okx" - assert parsed_exchange_auth[0].api_key == "dsl-precompute-key" - assert parsed_exchange_auth[0].api_secret == "dsl-precompute-secret" - assert parsed_exchange_auth[0].exchange_type == commons_constants.CONFIG_EXCHANGE_SPOT + assert len(await_arguments) >= 4 + passed_exchange_auth_overrides = await_arguments[3] + assert passed_exchange_auth_overrides is not None + assert len(passed_exchange_auth_overrides) == 1 + assert isinstance(passed_exchange_auth_overrides[0], dict) + assert passed_exchange_auth_overrides[0]["internal_name"] == "dsl_exchange_okx" + assert passed_exchange_auth_overrides[0]["api_key"] == "dsl-precompute-key" + assert passed_exchange_auth_overrides[0]["api_secret"] == "dsl-precompute-secret" + assert passed_exchange_auth_overrides[0]["exchange_type"] == commons_constants.CONFIG_EXCHANGE_SPOT + + +class TestEnsureChildEnviron: + def test_sets_process_bot_sync_user_id_env(self, tmp_path): + working_directory = str(tmp_path) + child_env = octobot_process_ops._ensure_child_environ( + 20050, + 30050, + "127.0.0.1", + "wallet-user", + working_directory, + ) + assert child_env[octobot_constants.ENV_PROCESS_BOT_SYNC_USER_ID] == "wallet-user" + assert child_env[commons_constants.ENV_OCTOBOT_SYNC_DATA_ROOT] == os.path.normpath( + os.path.join(working_directory, commons_constants.USER_FOLDER) + ) + assert child_env["DISTRIBUTION"] == commons_constants.DEFAULT_DISTRIBUTION + assert child_env[services_constants.ENV_ENABLE_NODE_API] == "false" + + def test_sets_pyinstaller_reset_environment_in_binary_mode(self, tmp_path): + with mock.patch.object( + octobot_process_ops.os_util, + "is_frozen_binary_octobot", + return_value=True, + ): + child_env = octobot_process_ops._ensure_child_environ( + 20050, + 30050, + "127.0.0.1", + "wallet-user", + str(tmp_path), + ) + assert child_env["PYINSTALLER_RESET_ENVIRONMENT"] == "1" + + def test_omits_pyinstaller_reset_environment_in_python_mode(self, tmp_path): + with mock.patch.object( + octobot_process_ops.os_util, + "is_frozen_binary_octobot", + return_value=False, + ): + child_env = octobot_process_ops._ensure_child_environ( + 20050, + 30050, + "127.0.0.1", + "wallet-user", + str(tmp_path), + ) + assert "PYINSTALLER_RESET_ENVIRONMENT" not in child_env + + +class TestOctobotSpawnArgvPrefix: + def test_binary_mode_returns_executable_only(self): + with mock.patch.object( + octobot_process_ops.os_util, + "is_frozen_binary_octobot", + return_value=True, + ): + argv_prefix = octobot_process_ops._octobot_spawn_argv_prefix("/any/cwd") + assert argv_prefix == [sys.executable] + + def test_python_mode_requires_start_py(self, tmp_path): + with mock.patch.object( + octobot_process_ops.os_util, + "is_frozen_binary_octobot", + return_value=False, + ): + with pytest.raises(commons_errors.DSLInterpreterError, match="start.py not found"): + octobot_process_ops._octobot_spawn_argv_prefix(str(tmp_path)) + + def test_python_mode_includes_start_script(self, tmp_path): + start_script = tmp_path / "start.py" + start_script.write_text("#", encoding="utf-8") + with mock.patch.object( + octobot_process_ops.os_util, + "is_frozen_binary_octobot", + return_value=False, + ): + argv_prefix = octobot_process_ops._octobot_spawn_argv_prefix(str(tmp_path)) + assert argv_prefix == [sys.executable, str(start_script)] + + +class TestEnsureStartCmd: + def test_includes_standalone_flag(self): + cmd = octobot_process_ops._ensure_start_cmd( + [sys.executable, "start.py"], + "user/automations/bot-1", + "logs/automations/bot-1", + no_telegram=False, + state_file_path="/tmp/process_bot_state.json", + ) + assert "--standalone" in cmd + assert cmd.index("--standalone") < cmd.index("--dump-state") + + def test_binary_prefix_omits_start_script(self): + cmd = octobot_process_ops._ensure_start_cmd( + [sys.executable], + "user/automations/bot-1", + "logs/automations/bot-1", + no_telegram=True, + state_file_path="/tmp/process_bot_state.json", + ) + assert cmd[0] == sys.executable + assert "start.py" not in cmd + assert cmd[1:3] == ["--user-folder", "user/automations/bot-1"] class TestEnsureOctobotProcessOperatorPrecompute: + async def test_raises_when_user_id_missing_at_spawn(self, tmp_path): + start_script = tmp_path / "start.py" + start_script.write_text("#", encoding="utf-8") + op = EnsureOctobotProcessOperator( + user_folder="ub", + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=None, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ), + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ): + with pytest.raises(commons_errors.DSLInterpreterError, match="requires user_id"): + await op.pre_compute() + + async def test_spawns_child_in_binary_mode_without_start_py(self, tmp_path): + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=None, + ) + with mock.patch.object( + octobot_process_ops.os_util, + "is_frozen_binary_octobot", + return_value=True, + ), mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ), + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock, mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_return_none_mock), + ): + spawn_mock.return_value.pid = 99999 + await op.pre_compute() + spawn_argv = spawn_mock.call_args.args[0] + assert spawn_argv[0] == sys.executable + assert "start.py" not in spawn_argv + assert spawn_argv[1] == "--user-folder" + assert spawn_mock.call_args.kwargs["environment"]["PYINSTALLER_RESET_ENVIRONMENT"] == "1" + async def test_returns_recallable_when_process_bot_state_not_live(self, tmp_path): start_script = tmp_path / "start.py" start_script.write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=None, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ), + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock, mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_return_none_mock), + ): + spawn_mock.return_value.pid = 99999 + await op.pre_compute() + assert isinstance(op.value, dict) + assert dsl_interpreter.ReCallingOperatorResult.__name__ in op.value + rec = op.value[dsl_interpreter.ReCallingOperatorResult.__name__] + le = rec["last_execution_result"] + assert le.get("init_state_ok") is False + + +class TestEnsureOctobotProcessPrecomputeWhenProcessStateLiveAfterFirstSpawn: + async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp_path): + start_script = tmp_path / "start.py" + start_script.write_text("#", encoding="utf-8") + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, ) @@ -977,7 +1929,110 @@ async def test_returns_recallable_when_process_bot_state_not_live(self, tmp_path tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" ), "profile_id": "x", - "already_prepared": True, + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock, mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == 10001, + ), mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), + ): + spawn_mock.return_value.pid = 10001 + await op.pre_compute() + assert isinstance(op.value, dict) + assert dsl_interpreter.ReCallingOperatorResult.__name__ in op.value + le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] + assert isinstance(le, dict) + assert le.get("init_state_ok") is True + assert le.get("http_base_url", "").startswith("http://") + assert le.get("pid") == 10001 + assert le.get("waiting_time") == octobot_process_ops.DEFAULT_PING_WAITING_TIME + assert octobot_flow_entities.PostIterationActionsDetails.__name__ in le + post = octobot_flow_entities.PostIterationActionsDetails.from_dict( + le[octobot_flow_entities.PostIterationActionsDetails.__name__] + ) + assert post.updated_exchange_account_elements is not None + + +class TestReportChildOctobotFirstStartIfNeeded: + def test_skips_when_layout_already_prepared(self): + with mock.patch.object( + octobot_process_ops.activity_metrics.ActivityMetrics, + "report_child_octobot_first_start", + ) as report_mock: + octobot_process_ops._report_child_octobot_first_start_if_needed( + {"already_prepared": True}, + ) + report_mock.assert_not_called() + + def test_reports_first_child_start_when_layout_was_created(self): + with mock.patch.object( + octobot_process_ops.activity_metrics.ActivityMetrics, + "report_child_octobot_first_start", + ) as report_mock: + octobot_process_ops._report_child_octobot_first_start_if_needed( + {"already_prepared": False}, + ) + report_mock.assert_called_once_with() + + def test_logs_exception_when_report_fails(self): + report_error = RuntimeError("sentry unavailable") + logger_mock = mock.Mock() + with mock.patch.object( + octobot_process_ops.activity_metrics.ActivityMetrics, + "report_child_octobot_first_start", + side_effect=report_error, + ), mock.patch.object( + octobot_process_ops, + "_get_logger", + return_value=logger_mock, + ): + octobot_process_ops._report_child_octobot_first_start_if_needed( + {"already_prepared": False}, + ) + logger_mock.exception.assert_called_once_with( + report_error, + True, + f"Failed to report child OctoBot first start {report_error}", + ) + + +class TestEnsureOctobotProcessPrecomputeReportsChildFirstStart: + async def test_reports_child_first_start_on_first_spawn_when_layout_is_new(self, tmp_path): + start_script = tmp_path / "start.py" + start_script.write_text("#", encoding="utf-8") + user_root = str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ) + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=None, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": user_root, + "profile_id": "x", + "already_prepared": False, } ), ), mock.patch.object( @@ -988,25 +2043,36 @@ async def test_returns_recallable_when_process_bot_state_not_live(self, tmp_path process_util, "spawn_managed_subprocess", ) as spawn_mock, mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == 10001, + ), mock.patch.object( octobot_process_ops, "_load_process_bot_state", - new=mock.AsyncMock(side_effect=_async_return_none_mock), - ): - spawn_mock.return_value.pid = 99999 + new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), + ), mock.patch.object( + octobot_process_ops, + "_report_child_octobot_first_start_if_needed", + ) as report_child_mock: + spawn_mock.return_value.pid = 10001 await op.pre_compute() - assert isinstance(op.value, dict) - assert dsl_interpreter.ReCallingOperatorResult.__name__ in op.value - rec = op.value[dsl_interpreter.ReCallingOperatorResult.__name__] - le = rec["last_execution_result"] - assert le.get("init_state_ok") is False - + report_child_mock.assert_called_once_with( + { + "user_root": user_root, + "profile_id": "x", + "already_prepared": False, + }, + ) -class TestEnsureOctobotProcessPrecomputeWhenProcessStateLiveAfterFirstSpawn: - async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp_path): + async def test_reports_child_first_start_on_spawn_when_child_not_yet_live(self, tmp_path): start_script = tmp_path / "start.py" start_script.write_text("#", encoding="utf-8") + user_root = str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, ) @@ -1019,11 +2085,9 @@ async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp "ensure_user_profile_and_layout", new=mock.AsyncMock( return_value={ - "user_root": str( - tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" - ), + "user_root": user_root, "profile_id": "x", - "already_prepared": True, + "already_prepared": False, } ), ), mock.patch.object( @@ -1034,29 +2098,24 @@ async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp process_util, "spawn_managed_subprocess", ) as spawn_mock, mock.patch.object( - process_util, - "pid_is_running", - side_effect=lambda process_id: process_id == 10001, - ), mock.patch.object( octobot_process_ops, "_load_process_bot_state", - new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), - ): + new=mock.AsyncMock(side_effect=_async_return_none_mock), + ), mock.patch.object( + octobot_process_ops, + "_report_child_octobot_first_start_if_needed", + ) as report_child_mock: spawn_mock.return_value.pid = 10001 await op.pre_compute() - assert isinstance(op.value, dict) - assert dsl_interpreter.ReCallingOperatorResult.__name__ in op.value - le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] - assert isinstance(le, dict) - assert le.get("init_state_ok") is True - assert le.get("http_base_url", "").startswith("http://") - assert le.get("pid") == 10001 - assert le.get("waiting_time") == octobot_process_ops.DEFAULT_PING_WAITING_TIME - assert octobot_flow_entities.PostIterationActionsDetails.__name__ in le - post = octobot_flow_entities.PostIterationActionsDetails.from_dict( - le[octobot_flow_entities.PostIterationActionsDetails.__name__] + report_child_mock.assert_called_once_with( + { + "user_root": user_root, + "profile_id": "x", + "already_prepared": False, + }, ) - assert post.updated_exchange_account_elements is not None + first_le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] + assert first_le.get("init_state_ok") is False class TestEnsureOctobotProcessPrecomputeRecallPathWhenProcessStateLive: @@ -1065,6 +2124,7 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa start_script.write_text("#", encoding="utf-8") op1 = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, ) @@ -1095,9 +2155,13 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa octobot_process_ops, "_load_process_bot_state", new=mock.AsyncMock(side_effect=_async_return_none_mock), - ): + ), mock.patch.object( + octobot_process_ops.activity_metrics.ActivityMetrics, + "report_child_octobot_first_start", + ) as report_child_mock: spawn_mock.return_value.pid = 10002 await op1.pre_compute() + report_child_mock.assert_not_called() first_value = op1.value assert isinstance(first_value, dict) first_le = first_value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] @@ -1105,6 +2169,7 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa anchor = first_le["started_waiting_at"] op2 = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=first_value, ) @@ -1120,8 +2185,12 @@ async def test_returns_recallable_with_init_state_ok_on_recall_path(self, tmp_pa octobot_process_ops, "_load_process_bot_state", new=mock.AsyncMock(side_effect=_async_live_process_bot_state_mock), - ): + ), mock.patch.object( + octobot_process_ops.activity_metrics.ActivityMetrics, + "report_child_octobot_first_start", + ) as report_child_mock: await op2.pre_compute() + report_child_mock.assert_not_called() assert isinstance(op2.value, dict) assert dsl_interpreter.ReCallingOperatorResult.__name__ in op2.value le2 = op2.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] @@ -1160,6 +2229,7 @@ async def test_init_timeout_kills_and_raises_dsl_error(self, tmp_path): } op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1218,6 +2288,7 @@ async def test_does_not_apply_init_timeout_after_init_state_ok(self, tmp_path): } op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1252,6 +2323,7 @@ async def test_waiting_time_uses_parameter_for_recall_emissions(self, tmp_path): start_script.write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=None, waiting_time=7.0, @@ -1304,8 +2376,12 @@ async def test_run_octobot_process_via_dsl(self, tmp_path, monkeypatch): # Minimal OctoBot project: `getcwd` must resolve `start.py` where `pre_compute` expects it. monkeypatch.chdir(tmp_path) (tmp_path / "start.py").write_text("#", encoding="utf-8") + _seed_executor_reference_tentacles_config(tmp_path) user_folder = "integration_dsl_bot" - expression = f"run_octobot_process({user_folder!r}, {repr(_MINIMAL_PROFILE_DATA_DSL_LITERAL)})" + expression = ( + f"run_octobot_process({user_folder!r}, profile_data={repr(_MINIMAL_PROFILE_DATA_DSL_LITERAL)}, " + f"user_id={_PROCESS_TEST_USER_ID!r})" + ) # Contextual operator is excluded from get_all_operators(); append it explicitly. interpreter = dsl_interpreter.Interpreter( dsl_interpreter.get_all_operators() @@ -1313,6 +2389,10 @@ async def test_run_octobot_process_via_dsl(self, tmp_path, monkeypatch): ) try: with mock.patch.object( + octobot_process_ops.os_util, + "is_frozen_binary_octobot", + return_value=False, + ), mock.patch.object( octobot_process_ops, "_load_process_bot_state", new=mock.AsyncMock(side_effect=_async_return_none_mock), @@ -1333,7 +2413,10 @@ async def test_run_octobot_process_via_dsl(self, tmp_path, monkeypatch): / user_folder ) assert (user_data_root / commons_constants.CONFIG_FILE).is_file() - assert (user_data_root / octobot_process_ops.DSL_PREPARED_MARKER).is_file() + root_cfg = json.loads((user_data_root / commons_constants.CONFIG_FILE).read_text(encoding="utf-8")) + assert root_cfg[commons_constants.CONFIG_READONLY_REFERENCE_TENTACLES_PATH] == ( + octobot_process_ops._executor_reference_tentacles_directory(str(tmp_path)) + ) # Same normpath as ensure uses for the computed absolute log path (dir may not exist until the child runs). expected_log_folder = os.path.normpath( os.path.join( @@ -1371,6 +2454,7 @@ async def test_run_octobot_process_via_dsl(self, tmp_path, monkeypatch): rel_user, "--log-folder", rel_log, + "--standalone", "-nt", "--dump-state", expected_state_path, @@ -1380,6 +2464,9 @@ async def test_run_octobot_process_via_dsl(self, tmp_path, monkeypatch): assert child_env[services_constants.ENV_WEB_ADDRESS] == "127.0.0.1" assert child_env[services_constants.ENV_NODE_API_PORT] == str(last_execution["node_port"]) assert child_env[services_constants.ENV_NODE_API_ADDRESS] == "127.0.0.1" + assert child_env["DISTRIBUTION"] == commons_constants.DEFAULT_DISTRIBUTION + assert child_env[services_constants.ENV_ENABLE_NODE_API] == "false" + assert child_env[octobot_constants.ENV_PROCESS_BOT_SYNC_USER_ID] == _PROCESS_TEST_USER_ID assert spawn_kwargs.get("hide_console_window") is True finally: # Redundant with pytest’s tmp_path teardown; makes intent obvious if the test is copied elsewhere. @@ -1391,12 +2478,13 @@ async def test_run_octobot_process_via_dsl_writes_exchange_auth_into_user_config self, tmp_path, monkeypatch ): """ - Same pipeline as ``test_run_octobot_process_via_dsl``, plus positional + Same pipeline as ``test_run_octobot_process_via_dsl``, plus keyword ``exchange_auth_data`` (list of dicts). Verifies API fields land under ``exchanges`` in the user-root ``config.json`` written during layout. """ monkeypatch.chdir(tmp_path) (tmp_path / "start.py").write_text("#", encoding="utf-8") + _seed_executor_reference_tentacles_config(tmp_path) user_folder = "integration_dsl_exchange_auth_bot" exchange_internal_name = "dsl_integration_cred_exchange" fake_api_key = "dsl-integration-api-key" @@ -1413,8 +2501,8 @@ async def test_run_octobot_process_via_dsl_writes_exchange_auth_into_user_config } ] expression = ( - f"run_octobot_process({user_folder!r}, {repr(_MINIMAL_PROFILE_DATA_DSL_LITERAL)}, " - f"{repr(exchange_auth_list)})" + f"run_octobot_process({user_folder!r}, profile_data={repr(_MINIMAL_PROFILE_DATA_DSL_LITERAL)}, " + f"exchange_auth_data={repr(exchange_auth_list)}, user_id={_PROCESS_TEST_USER_ID!r})" ) interpreter = dsl_interpreter.Interpreter( dsl_interpreter.get_all_operators() @@ -1448,7 +2536,8 @@ async def test_run_octobot_process_via_dsl_writes_exchange_auth_into_user_config assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_PASSWORD] == fake_api_password assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_TYPE] == commons_constants.CONFIG_EXCHANGE_SPOT assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is True - assert exchange_cfg[commons_constants.CONFIG_ENABLED_OPTION] is True + # Profile has no exchanges: auth-only entries are disabled after non-profile guard. + assert exchange_cfg[commons_constants.CONFIG_ENABLED_OPTION] is False finally: shutil.rmtree(tmp_path / commons_constants.USER_FOLDER, ignore_errors=True) if (tmp_path / "logs").exists(): @@ -1472,7 +2561,8 @@ async def test_run_octobot_process_via_dsl_without_profile_data_accepts_exchange } ] expression = ( - f"run_octobot_process({user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"run_octobot_process({user_folder!r}, user_id={_PROCESS_TEST_USER_ID!r}, " + f"exchange_auth_data={repr(exchange_auth_list)}, " f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0)" ) interpreter = dsl_interpreter.Interpreter( @@ -1502,13 +2592,16 @@ async def test_run_octobot_process_via_dsl_without_profile_data_accepts_exchange assert root_config_path.is_file() written_root_cfg = json.loads(root_config_path.read_text(encoding="utf-8")) assert written_root_cfg[commons_constants.CONFIG_PROFILE] == octobot_process_ops.DEFAULT_DSL_PROFILE_ID + assert written_root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(str(tmp_path)) + ) profile_json_path = ( user_data_root / commons_constants.PROFILES_FOLDER / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not profile_json_path.exists() exchange_cfg = written_root_cfg[commons_constants.CONFIG_EXCHANGES][exchange_internal_name] assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_KEY] == "no-profile-key" assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SECRET] == "no-profile-secret" @@ -1575,15 +2668,19 @@ async def _run_default_config_lifecycle( else: assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_SANDBOXED] is exchange_auth_list[0]["sandboxed"] assert exchange_cfg[commons_constants.CONFIG_EXCHANGE_TYPE] == exchange_auth_list[0]["exchange_type"] + assert root_cfg[commons_constants.CONFIG_READONLY_PROFILES_PATH] == ( + octobot_process_ops._executor_profiles_directory(project_root) + ) profile_json_path = ( user_root / commons_constants.PROFILES_FOLDER / octobot_process_ops.DEFAULT_DSL_PROFILE_ID / commons_constants.PROFILE_CONFIG_FILE ) - assert profile_json_path.is_file() + assert not profile_json_path.exists() stop_expression = ( - f"run_octobot_process({user_folder!r}, exchange_auth_data={repr(exchange_auth_list)}, " + f"run_octobot_process({user_folder!r}, user_id={_PROCESS_TEST_USER_ID!r}, " + f"exchange_auth_data={repr(exchange_auth_list)}, " f"waiting_time={_TESTS_RUN_OCTOBOT_PROCESS_WAITING_TIME_SEC}, ping_timeout=30.0, " f"last_execution_result={repr(_re_calling_ensure_value(inner))})" ) @@ -1665,6 +2762,7 @@ async def test_execution_stop_dead_child_is_already_stopped(self): }) op = operator_under_test( user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1690,6 +2788,7 @@ async def test_execution_stop_short_circuits_without_sigterm_when_not_running(se }) op = operator_under_test( user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1722,6 +2821,7 @@ async def test_execution_stop_os_kill_failure_raises(self): }) op = operator_under_test( user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1743,6 +2843,146 @@ def _kill_failed(_pid, _sig): with pytest.raises(commons_errors.DSLInterpreterError, match="simulated"): await op.pre_compute() + async def test_execution_stop_waits_for_child_exit_after_graceful_stop(self): + inner = _stop_test_ensure_state_dict("http://127.0.0.1:7") + operator_signals_holder = dsl_interpreter.OperatorSignals() + operator_under_test = octobot_process_ops.create_octobot_process_operators( + operator_signals_holder, + TEST_EXECUTOR_ID, + )[0] + operator_signals_holder.sync({ + operator_under_test.get_name(): dsl_interpreter.OperatorSignal.STOP.value, + }) + stop_operator = operator_under_test( + user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + ) + graceful_stop_mock = mock.Mock(return_value={"status": "stopped", "signal": "sigterm"}) + wait_mock = mock.AsyncMock() + with ( + mock.patch.object( + process_util, + "pid_is_running", + return_value=True, + ), + mock.patch.object( + stop_operator, + "request_graceful_stop", + new=graceful_stop_mock, + ), + mock.patch.object( + stop_operator, + "wait_until_pid_stopped", + new=wait_mock, + ), + ): + await stop_operator.pre_compute() + graceful_stop_mock.assert_called_once() + wait_mock.assert_awaited_once() + assert stop_operator.value == {"status": "stopped", "signal": "sigterm"} + + async def test_execution_stop_force_kills_when_graceful_wait_times_out(self): + inner = _stop_test_ensure_state_dict("http://127.0.0.1:7") + operator_signals_holder = dsl_interpreter.OperatorSignals() + operator_under_test = octobot_process_ops.create_octobot_process_operators( + operator_signals_holder, + TEST_EXECUTOR_ID, + )[0] + operator_signals_holder.sync({ + operator_under_test.get_name(): dsl_interpreter.OperatorSignal.STOP.value, + }) + stop_operator = operator_under_test( + user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + ) + graceful_stop_mock = mock.Mock(return_value={"status": "stopped", "signal": "sigterm"}) + wait_mock = mock.AsyncMock( + side_effect=[ + commons_errors.DSLInterpreterError("timed out waiting for pid=1 to exit"), + None, + ], + ) + force_kill_mock = mock.Mock(return_value={"status": "force_killed"}) + with ( + mock.patch.object( + process_util, + "pid_is_running", + return_value=True, + ), + mock.patch.object( + stop_operator, + "request_graceful_stop", + new=graceful_stop_mock, + ), + mock.patch.object( + stop_operator, + "wait_until_pid_stopped", + new=wait_mock, + ), + mock.patch.object( + process_util, + "request_force_kill", + new=force_kill_mock, + ), + ): + await stop_operator.pre_compute() + graceful_stop_mock.assert_called_once() + force_kill_mock.assert_called_once() + assert force_kill_mock.call_args.args[0] == 1 + assert wait_mock.await_count == 2 + assert stop_operator.value == {"status": "force_killed"} + + async def test_execution_stop_raises_when_force_kill_insufficient(self): + inner = _stop_test_ensure_state_dict("http://127.0.0.1:7") + operator_signals_holder = dsl_interpreter.OperatorSignals() + operator_under_test = octobot_process_ops.create_octobot_process_operators( + operator_signals_holder, + TEST_EXECUTOR_ID, + )[0] + operator_signals_holder.sync({ + operator_under_test.get_name(): dsl_interpreter.OperatorSignal.STOP.value, + }) + stop_operator = operator_under_test( + user_folder="u1", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + ) + wait_mock = mock.AsyncMock( + side_effect=commons_errors.DSLInterpreterError("still running"), + ) + with ( + mock.patch.object( + process_util, + "pid_is_running", + return_value=True, + ), + mock.patch.object( + stop_operator, + "request_graceful_stop", + return_value={"status": "stopped", "signal": "sigterm"}, + ), + mock.patch.object( + stop_operator, + "wait_until_pid_stopped", + new=wait_mock, + ), + mock.patch.object( + process_util, + "request_force_kill", + return_value={"status": "force_killed"}, + ), + pytest.raises( + commons_errors.DSLInterpreterError, + match="did not exit after graceful stop and force kill", + ), + ): + await stop_operator.pre_compute() + class TestEnsureOctobotProcessOperatorSignalDispatch: def test_should_dispatch_stop_and_update_config_for_valid_ensure_payload(self): @@ -1778,6 +3018,7 @@ class TestEnsureOctobotProcessOperatorUpdateConfig: async def test_update_config_triggers_respawn_and_recallable_result(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "start.py").write_text("#", encoding="utf-8") + _seed_executor_reference_tentacles_config(tmp_path) user_automation = ( tmp_path / commons_constants.USER_FOLDER @@ -1793,12 +3034,13 @@ async def test_update_config_triggers_respawn_and_recallable_result(self, tmp_pa ) log_dir.mkdir(parents=True) (user_automation / "stale_marker.txt").write_text("x", encoding="utf-8") - inner = octobot_process_ops.EnsureOctobotProcessState( + inner = octobot_process_state_import.OctobotProcessState( http_base_url="http://127.0.0.1:5001", web_port=5001, node_port=5002, user_root=str(user_automation), user_folder="nested/upd_bot", + user_id=_PROCESS_TEST_USER_ID, log_folder=str(log_dir), profile_id="p1", pid=4242, @@ -1819,6 +3061,7 @@ async def test_update_config_triggers_respawn_and_recallable_result(self, tmp_pa }) op = operator_under_test( user_folder="nested/upd_bot", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1827,14 +3070,9 @@ async def test_update_config_triggers_respawn_and_recallable_result(self, tmp_pa with ( mock.patch.object( operator_under_test, - "wait_until_pid_stopped", - new=mock.AsyncMock(), - ) as wait_mock, - mock.patch.object( - dsl_interpreter.ProcessBoundOperatorMixin, - "request_graceful_stop", - return_value={"status": "stopped", "signal": "sigterm"}, - ) as stop_mock, + "_stop_bound_child_and_wait_for_exit", + new=mock.AsyncMock(return_value={"status": "stopped", "signal": "sigterm"}), + ) as stop_child_mock, mock.patch.object( process_util, "pid_is_running", @@ -1852,8 +3090,11 @@ async def test_update_config_triggers_respawn_and_recallable_result(self, tmp_pa ): spawn_mock.return_value = mock.Mock(spec=["pid"], pid=5151) await op.pre_compute() - stop_mock.assert_called_once() - wait_mock.assert_awaited_once() + stop_child_mock.assert_awaited_once_with( + 4242, + ping_timeout=octobot_process_ops.DEFAULT_ENSURE_TIMEOUT, + logger=mock.ANY, + ) spawn_mock.assert_called_once() assert dsl_interpreter.ReCallingOperatorResult.__name__ in op.value finally: @@ -1880,6 +3121,7 @@ async def test_adopts_pid_from_live_state_without_spawn(self, tmp_path): inner = _healthy_recall_inner(pid=10002, tmp_path=tmp_path) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1912,6 +3154,7 @@ async def test_recall_without_spawn_during_init(self, tmp_path): inner["started_waiting_at"] = octobot_process_ops.time.time() op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1941,6 +3184,7 @@ async def test_recall_when_pid_running_without_state_file(self, tmp_path): inner = _healthy_recall_inner(pid=10002, tmp_path=tmp_path) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -1974,6 +3218,7 @@ async def test_recall_during_restart_grace(self, tmp_path): ) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2006,6 +3251,7 @@ async def test_respawns_when_grace_expired(self, tmp_path): stale_state = _stale_process_bot_state_for_grace(age_seconds=200.0, metadata_pid=10002) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2051,6 +3297,7 @@ async def test_respawns_when_no_state_file_and_pid_dead(self, tmp_path): inner = _healthy_recall_inner(pid=10002, tmp_path=tmp_path) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2102,10 +3349,12 @@ async def test_stop_signals_adopted_pid(self): }) op = operator_under_test( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) graceful_stop_mock = mock.Mock(return_value={"status": "stopped", "signal": "sigterm"}) + wait_mock = mock.AsyncMock() with ( mock.patch.object( process_util, @@ -2122,9 +3371,15 @@ async def test_stop_signals_adopted_pid(self): "request_graceful_stop", new=graceful_stop_mock, ), + mock.patch.object( + operator_under_test, + "wait_until_pid_stopped", + new=wait_mock, + ), ): await op.pre_compute() graceful_stop_mock.assert_called_once() + wait_mock.assert_awaited_once() assert op.pid == 20002 assert op.value == {"status": "stopped", "signal": "sigterm"} @@ -2142,6 +3397,7 @@ async def test_marker_mismatch_forces_first_spawn(self, tmp_path): None, TEST_EXECUTOR_ID )[0]( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2191,6 +3447,7 @@ async def test_marker_mismatch_but_metadata_pid_running_recalls(self, tmp_path): None, TEST_EXECUTOR_ID )[0]( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2230,6 +3487,7 @@ async def test_recall_during_grace_without_spawn(self, tmp_path): ) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2264,6 +3522,7 @@ async def test_recall_during_grace_when_marker_matches(self, tmp_path): ) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2296,6 +3555,7 @@ async def test_first_spawn_when_grace_expired(self, tmp_path): stale_state = _stale_process_bot_state_for_grace(age_seconds=200.0, metadata_pid=10002) op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ping_timeout=120.0, @@ -2335,12 +3595,13 @@ async def test_first_spawn_when_grace_expired(self, tmp_path): spawn_mock.assert_called_once() -class TestEnsureOctobotProcessStateEmitsExecutorId: +class TestOctobotProcessStateEmitsExecutorId: async def test_first_spawn_emits_executor_id(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "start.py").write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( user_folder="emit_master_bot", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, ) with mock.patch.object( @@ -2383,6 +3644,7 @@ async def test_missing_executor_id_falls_through_to_first_spawn(self, tmp_path): del inner["executor_id"] op = EnsureOctobotProcessOperator( user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, profile_data=_MINIMAL_PROFILE_DATA, last_execution_result=_re_calling_ensure_value(inner), ) @@ -2415,3 +3677,173 @@ async def test_missing_executor_id_falls_through_to_first_spawn(self, tmp_path): spawn_mock.return_value = mock.Mock(spec=["pid"], pid=30007) await op.pre_compute() spawn_mock.assert_called_once() + + +class TestResolveBoundPid: + pytestmark = [] + + def test_prefers_metadata_pid_when_both_running_but_differ(self): + recall_state = octobot_process_state_import.OctobotProcessState( + http_base_url="http://127.0.0.1:20050", + web_port=20050, + node_port=30050, + user_root="/x/ub", + user_folder="ub", + log_folder="/x/logs/ub", + profile_id="p", + pid=20520, + state_file_path="/x/ub/process_bot_state.json", + executor_id=TEST_EXECUTOR_ID, + ) + loaded_state = process_bot_state_import.ProcessBotState( + metadata=process_bot_state_import.Metadata( + updated_at=octobot_process_ops.time.time() - 0.1, + next_updated_at=octobot_process_ops.time.time() + 1.0, + pid=25044, + ), + exchange_account_elements=octobot_flow_entities.ExchangeAccountElements(), + ) + with mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id in {20520, 25044}, + ): + resolved_pid = octobot_process_ops._resolve_bound_pid(recall_state, loaded_state) + assert resolved_pid == 25044 + + def test_returns_stored_pid_when_metadata_missing(self): + recall_state = octobot_process_state_import.OctobotProcessState( + http_base_url="http://127.0.0.1:20050", + web_port=20050, + node_port=30050, + user_root="/x/ub", + user_folder="ub", + log_folder="/x/logs/ub", + profile_id="p", + pid=10002, + state_file_path="/x/ub/process_bot_state.json", + executor_id=TEST_EXECUTOR_ID, + ) + with mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == 10002, + ): + resolved_pid = octobot_process_ops._resolve_bound_pid(recall_state, None) + assert resolved_pid == 10002 + + def test_returns_none_when_both_dead(self): + recall_state = octobot_process_state_import.OctobotProcessState( + http_base_url="http://127.0.0.1:20050", + web_port=20050, + node_port=30050, + user_root="/x/ub", + user_folder="ub", + log_folder="/x/logs/ub", + profile_id="p", + pid=10002, + state_file_path="/x/ub/process_bot_state.json", + executor_id=TEST_EXECUTOR_ID, + ) + loaded_state = process_bot_state_import.ProcessBotState( + metadata=process_bot_state_import.Metadata( + updated_at=octobot_process_ops.time.time() - 0.1, + next_updated_at=octobot_process_ops.time.time() + 1.0, + pid=20002, + ), + exchange_account_elements=octobot_flow_entities.ExchangeAccountElements(), + ) + with mock.patch.object(process_util, "pid_is_running", return_value=False): + resolved_pid = octobot_process_ops._resolve_bound_pid(recall_state, loaded_state) + assert resolved_pid is None + + +class TestRecallPathRebindsManagedChildRegistry: + @pytest.fixture(autouse=True) + def reset_managed_child_process_registry(self): + import octobot_commons.managed_child_process_registry as managed_child_process_registry + import octobot_commons.singleton.singleton_class as singleton_class + + singleton_class.Singleton._instances.pop( + managed_child_process_registry.ManagedChildProcessRegistry, + None, + ) + yield + singleton_class.Singleton._instances.pop( + managed_child_process_registry.ManagedChildProcessRegistry, + None, + ) + + async def test_recall_rebinds_registry_when_metadata_pid_differs(self, tmp_path): + import octobot_commons.managed_child_process_registry as managed_child_process_registry + + registry = managed_child_process_registry.ManagedChildProcessRegistry.instance() + inner = _healthy_recall_inner(pid=20520, tmp_path=tmp_path) + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + ) + + async def live_state_with_authoritative_pid(*_unused): + return await _async_live_process_bot_state_mock(metadata_pid=25044) + + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id in {20520, 25044}, + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock, mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=live_state_with_authoritative_pid), + ): + registry.register(20520) + await op.pre_compute() + spawn_mock.assert_not_called() + with mock.patch.object(process_util, "pid_is_running", return_value=True): + assert registry.snapshot_running_pids() == frozenset({25044}) + assert op.pid == 25044 + + async def test_recall_does_not_rebind_when_authoritative_pid_unchanged(self, tmp_path): + inner = _healthy_recall_inner(pid=25044, tmp_path=tmp_path) + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + ) + + async def live_state_with_same_pid(*_unused): + return await _async_live_process_bot_state_mock(metadata_pid=25044) + + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == 25044, + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock, mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=live_state_with_same_pid), + ), mock.patch.object( + process_util, + "rebind_managed_child_pid", + ) as rebind_mock: + await op.pre_compute() + spawn_mock.assert_not_called() + rebind_mock.assert_not_called() + assert op.pid == 25044 diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_call_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_call_operators.py index e1a5eec335..2f3d4de9d3 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_call_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_call_operators.py @@ -17,7 +17,8 @@ import math import time -import octobot_commons.constants as octobot_commons_constants +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.json_util as json_util @@ -29,11 +30,19 @@ class MinOperator(dsl_interpreter.CallOperator): NAME = "min" DESCRIPTION = "Returns the minimum value from the given operands." EXAMPLE = "min(1, 2, 3)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "min" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Minimum of the operands", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() return min(operands) @@ -44,11 +53,19 @@ class MaxOperator(dsl_interpreter.CallOperator): NAME = "max" DESCRIPTION = "Returns the maximum value from the given operands." EXAMPLE = "max(1, 2, 3)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "max" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Maximum of the operands", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() return max(operands) @@ -59,11 +76,19 @@ class MeanOperator(dsl_interpreter.CallOperator): NAME = "mean" DESCRIPTION = "Returns the arithmetic mean (average) of the given numeric operands." EXAMPLE = "mean(1, 2, 3, 4)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "mean" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Arithmetic mean of the operands", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() # Ensure all operands are numeric @@ -84,11 +109,19 @@ class SqrtOperator(dsl_interpreter.CallOperator): NAME = "sqrt" DESCRIPTION = "Returns the square root of the given numeric operand." EXAMPLE = "sqrt(16)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "sqrt" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Square root of the operand", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() operand = computed_parameters[0] @@ -105,11 +138,19 @@ class AbsOperator(dsl_interpreter.CallOperator): NAME = "abs" DESCRIPTION = "Returns the absolute value of the given operand." EXAMPLE = "abs(-5)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "abs" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Absolute value of the operand", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() operand = computed_parameters[0] @@ -120,6 +161,7 @@ class RoundOperator(dsl_interpreter.CallOperator): NAME = "round" DESCRIPTION = "Rounds the given numeric value to the specified number of decimal digits. If digits is not provided, rounds to the nearest integer." EXAMPLE = "round(3.14159, 2)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: @@ -128,10 +170,25 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="value", description="the value to round", required=True, type=list), - dsl_interpreter.OperatorParameter(name="digits", description="the number of digits to round to", required=False, type=int), + dsl_interpreter.OperatorParameter( + name="value", + description="the value to round", + required=True, + type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter( + name="digits", + description="the number of digits to round to", + required=False, + type=commons_enums.DslValueType.NUMBER.value), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Rounded numeric value", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() operand = computed_parameters[0] @@ -149,11 +206,19 @@ class FloorOperator(dsl_interpreter.CallOperator): NAME = "floor" DESCRIPTION = "Returns the floor of the given numeric operand (largest integer less than or equal to the value)." EXAMPLE = "floor(3.7)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "floor" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Floor of the operand", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() operand = computed_parameters[0] @@ -170,11 +235,19 @@ class CeilOperator(dsl_interpreter.CallOperator): NAME = "ceil" DESCRIPTION = "Returns the ceiling of the given numeric operand (smallest integer greater than or equal to the value)." EXAMPLE = "ceil(3.2)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "ceil" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Ceiling of the operand", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() operand = computed_parameters[0] @@ -191,11 +264,19 @@ class SinOperator(dsl_interpreter.CallOperator): NAME = "sin" DESCRIPTION = "Returns the sine of the given numeric operand (in radians)." EXAMPLE = "sin(1.23)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "sin" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Sine of the operand", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() operand = computed_parameters[0] @@ -212,11 +293,19 @@ class CosOperator(dsl_interpreter.CallOperator): NAME = "cos" DESCRIPTION = "Returns the cosine of the given numeric operand (in radians)." EXAMPLE = "cos(1.23)" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "cos" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Cosine of the operand", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() operand = computed_parameters[0] @@ -233,11 +322,19 @@ class OscillatorOperator(dsl_interpreter.CallOperator): NAME = "oscillate" DESCRIPTION = "Returns the base value with a time-based oscillating component added. The oscillation uses a sine wave with the specified maximum percentage of the base value and period in minutes." EXAMPLE = "oscillate(100, 10, 60)" + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value @staticmethod def get_name() -> str: return "oscillate" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Base value with oscillation applied", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() base_value = computed_parameters[0] @@ -276,11 +373,19 @@ class GetOperator(dsl_interpreter.CallOperator): "On TypeError or AttributeError, returns default." ) EXAMPLE = 'get(dict_var, "color", "blue")' + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "get" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Value for key, or default", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: computed_parameters = self.get_computed_parameters() element = computed_parameters[0] @@ -304,6 +409,7 @@ class ValueIfOperator( "Otherwise the inner expression is repr(value) followed by the condition string (suffix mode)." ) EXAMPLE = "value_if(15, ' > 12') or value_if(order, \"get(LOCAL_VALUE_PLACEHOLDER, 'status', 'x') == 'open'\")" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: @@ -319,7 +425,7 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "LOCAL_VALUE_PLACEHOLDER with repr(sanitize(value)) when present" ), required=True, - type=object, + type=commons_enums.DslValueType.ANY.value, ), dsl_interpreter.OperatorParameter( name="condition", @@ -328,10 +434,17 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: "where LOCAL_VALUE_PLACEHOLDER is replaced by repr(sanitize(value))" ), required=True, - type=str, + type=commons_enums.DslValueType.TEXT.value, ), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Value when condition is truthy, otherwise False", + ) + async def pre_compute(self) -> None: await super().pre_compute() param_by_name = self.get_computed_value_by_parameter() @@ -341,11 +454,11 @@ async def pre_compute(self) -> None: raise octobot_commons.errors.InvalidParametersError( f"value_if() requires condition to be a str, got {type(condition_script).__name__}" ) - if octobot_commons_constants.LOCAL_VALUE_PLACEHOLDER not in condition_script: + if commons_constants.LOCAL_VALUE_PLACEHOLDER not in condition_script: inner_expression = repr(computed_value) + condition_script else: inner_expression = condition_script.replace( - octobot_commons_constants.LOCAL_VALUE_PLACEHOLDER, + commons_constants.LOCAL_VALUE_PLACEHOLDER, repr(json_util.sanitize(computed_value)), ) condition_result = await self.interprete_in_nested_interpreter(inner_expression) @@ -363,6 +476,7 @@ class IfErrorOperator( "and returns that result. Pass on_error as a string literal so it is not evaluated eagerly." ) EXAMPLE = "if_error(sqrt(-1), \"'fallback'\")" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: @@ -375,18 +489,24 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: name="value", description="base DSL expression; its computed value is returned when evaluation succeeds", required=True, - type=object, - ), + type=commons_enums.DslValueType.ANY.value), dsl_interpreter.OperatorParameter( name="on_error", description=( "DSL source string (use a string literal): interpreted only when the base expression raises" ), required=True, - type=str, + type=commons_enums.DslValueType.TEXT.value, ), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Base value, or on_error expression result when base raises", + ) + async def pre_compute(self) -> None: param_by_name = self.get_input_value_by_parameter() value_input = param_by_name["value"] @@ -409,11 +529,19 @@ class ErrorOperator(dsl_interpreter.CallOperator): NAME = "error" DESCRIPTION = "Raises a ErrorStatementEncountered exception with the given parameters." EXAMPLE = "error('123-error')" + CATEGORY = commons_enums.DslKeywordCategory.ACTION.value @staticmethod def get_name() -> str: return "error" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Does not return; raises ErrorStatementEncountered", + ) + def compute(self): params = self.get_computed_parameters() raise octobot_commons.errors.ErrorStatementEncountered(*params) diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_expression_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_expression_operators.py index ada3bf5590..488ca9e848 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_expression_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_expression_operators.py @@ -16,6 +16,9 @@ # License along with this library. import ast +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums +import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.dsl_interpreter.operators.expression_operator as dsl_interpreter_expression_operator import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator @@ -28,6 +31,7 @@ class IfExpOperator(dsl_interpreter_expression_operator.ExpressionOperator): NAME = "if ... else" DESCRIPTION = "Conditional expression operator. Returns the body expression if the test condition is True, otherwise returns the orelse expression." EXAMPLE = "5 if True else 3" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value def __init__( self, @@ -44,6 +48,13 @@ def __init__( def get_name() -> str: return ast.IfExp.__name__ + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Body or orelse expression result", + ) + def compute(self) -> dsl_interpreter_operator.ComputedOperatorParameterType: # Compute the test condition test_value = ( diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_iterable_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_iterable_operators.py index eb25ead05f..6cfb9f5199 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_iterable_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_iterable_operators.py @@ -16,6 +16,9 @@ # License along with this library. import ast +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums +import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.dsl_interpreter.operators.iterable_operator as dsl_interpreter_iterable_operator import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator @@ -28,11 +31,19 @@ class ListOperator(dsl_interpreter_iterable_operator.IterableOperator): NAME = "[...]" DESCRIPTION = "List constructor operator. Creates a list from the given operands." EXAMPLE = "[1, 2, 3]" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return ast.List.__name__ + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="List of computed operands", + ) + def compute(self) -> dsl_interpreter_operator.ComputedOperatorParameterType: # Compute the test condition return list(self.get_computed_parameters()) diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_name_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_name_operators.py index 758cff7ad5..8557e391a7 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_name_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_name_operators.py @@ -16,6 +16,9 @@ # License along with this library. import math +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums +import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.dsl_interpreter.operators.name_operator as dsl_interpreter_name_operator import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator @@ -25,11 +28,19 @@ class PiOperator(dsl_interpreter_name_operator.NameOperator): NAME = "pi" DESCRIPTION = "Mathematical constant pi (π), approximately 3.14159." EXAMPLE = "pi" + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value @staticmethod def get_name() -> str: return "pi" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Mathematical constant pi", + ) + def compute(self) -> dsl_interpreter_operator.ComputedOperatorParameterType: return math.pi @@ -39,10 +50,18 @@ class NaNOperator(dsl_interpreter_name_operator.NameOperator): NAME = "nan" DESCRIPTION = "Not a Number constant. Represents an undefined or unrepresentable numeric value." EXAMPLE = "nan" + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value @staticmethod def get_name() -> str: return "nan" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Not a Number (NaN)", + ) + def compute(self) -> dsl_interpreter_operator.ComputedOperatorParameterType: return float("nan") diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py index 64d67e278d..fb5bcea187 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_resetting_operators.py @@ -18,7 +18,10 @@ import time import asyncio import random +import dataclasses +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors import octobot_commons.dsl_interpreter as dsl_interpreter @@ -33,6 +36,7 @@ class WaitOperator(dsl_interpreter.PreComputingCallOperator, dsl_interpreter.ReC NAME = "wait" DESCRIPTION = "Pauses execution for the specified number of seconds. If return_remaining_time is True, instantly returns the remaining time to wait." EXAMPLE = "wait(5)" + CATEGORY = commons_enums.DslKeywordCategory.TRIGGER.value @staticmethod def get_name() -> str: @@ -41,10 +45,36 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="min_seconds", description="minimum number of seconds to wait", required=True, type=float), - dsl_interpreter.OperatorParameter(name="max_seconds", description="maximum number of seconds to wait", required=False, type=float, default=None), - dsl_interpreter.OperatorParameter(name="return_remaining_time", description="if True, instantly returns the remaining time to wait", required=False, type=bool, default=False), - ] + cls.get_re_callable_parameters() + dsl_interpreter.OperatorParameter( + name="min_seconds", + description="minimum number of seconds to wait", + required=True, + type=commons_enums.DslValueType.NUMBER.value), + dsl_interpreter.OperatorParameter( + name="max_seconds", + description="maximum number of seconds to wait", + required=False, + type=commons_enums.DslValueType.NUMBER.value, + default=None), + dsl_interpreter.OperatorParameter( + name="return_remaining_time", + description="if True, instantly returns the remaining time to wait", + required=False, + type=commons_enums.DslValueType.BOOLEAN.value, + default=False), + ] + [ + dataclasses.replace( + parameter, + ) + for parameter in cls.get_re_callable_parameters() + ] + + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="None after waiting, or remaining-time re-callable result", + ) async def pre_compute(self) -> None: await super().pre_compute() @@ -101,6 +131,7 @@ class LoopUntilOperator(dsl_interpreter.PreComputingCallOperator, dsl_interprete "Returns the condition value when it becomes true." ) EXAMPLE = "loop_until(x > 0, 1, timeout=30, max_attempts=10)" + CATEGORY = commons_enums.DslKeywordCategory.TRIGGER.value LOOP_START_TIME_KEY = "loop_until_start_time" ATTEMPT_COUNT_KEY = "loop_until_attempt_count" @@ -116,36 +147,43 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: name="condition", description="expression that must become true", required=True, - type=bool, - ), + type=commons_enums.DslValueType.BOOLEAN.value), dsl_interpreter.OperatorParameter( name="retry_interval", description="seconds to wait between condition checks", required=True, - type=float, - ), + type=commons_enums.DslValueType.NUMBER.value), dsl_interpreter.OperatorParameter( name="timeout", description="if set, maximum total seconds; if still false, raises ErrorStatementEncountered", required=False, - type=float, - default=None, - ), + type=commons_enums.DslValueType.NUMBER.value, + default=None), dsl_interpreter.OperatorParameter( name="max_attempts", description="if set, maximum condition evaluations; if still false, raises ErrorStatementEncountered", required=False, - type=int, - default=None, - ), + type=commons_enums.DslValueType.NUMBER.value, + default=None), dsl_interpreter.OperatorParameter( name="return_remaining_time", description="if True, instantly returns the remaining time until the next check", required=False, - type=bool, - default=False, - ), - ] + cls.get_re_callable_parameters() + type=commons_enums.DslValueType.BOOLEAN.value, + default=False), + ] + [ + dataclasses.replace( + parameter, + ) + for parameter in cls.get_re_callable_parameters() + ] + + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Condition value when true, or remaining-time re-callable result", + ) async def pre_compute(self) -> None: await super().pre_compute() diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_subscripting_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_subscripting_operators.py index 7883a6c9d1..2a66d49d06 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_subscripting_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_subscripting_operators.py @@ -18,7 +18,10 @@ import numpy as np import typing +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors +import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.dsl_interpreter.operators.subscripting_operator as dsl_interpreter_subscripting_operator import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator @@ -31,6 +34,7 @@ class SubscriptOperator(dsl_interpreter_subscripting_operator.SubscriptingOperat NAME = "[...]" DESCRIPTION = "Subscripting operator. Accesses an element from a list or array using an index." EXAMPLE = "my_list[0]" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value def __init__( self, @@ -65,6 +69,13 @@ def get_computed_array_or_list_and_index_or_slice_and_context_parameters( def get_name() -> str: return ast.Subscript.__name__ + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Element at the given index", + ) + def compute(self) -> dsl_interpreter_operator.ComputedOperatorParameterType: # Compute the test condition array_or_list, index, context = self.get_computed_array_or_list_and_index_or_slice_and_context_parameters() @@ -81,11 +92,19 @@ class SliceOperator(dsl_interpreter_subscripting_operator.SubscriptingOperator): NAME = "[start:stop:step]" DESCRIPTION = "Slice operator. Creates a slice object for array/list slicing with optional start, stop, and step parameters." EXAMPLE = "my_list[1:5:2]" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return ast.Slice.__name__ + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.ANY.value, + description="Python slice object", + ) + def get_computed_lower_and_upper_and_step_parameters( self, ) -> typing.Tuple[ diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_time_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_time_operators.py index f9d5171508..74fd7f7fba 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_time_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_time_operators.py @@ -16,6 +16,8 @@ # License along with this library. import time +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter as dsl_interpreter @@ -25,10 +27,18 @@ class NowMsOperator(dsl_interpreter.CallOperator): NAME = "now_ms" DESCRIPTION = "Returns the current time in milliseconds since epoch." EXAMPLE = "now_ms()" + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value @staticmethod def get_name() -> str: return "now_ms" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.NUMBER.value, + description="Current Unix time in milliseconds", + ) + def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: return int(time.time() * 1000) diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_unary_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_unary_operators.py index 28ff077c07..c4028a23c6 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/base_unary_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/base_unary_operators.py @@ -16,6 +16,9 @@ # License along with this library. import ast +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums +import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.dsl_interpreter.operators.unary_operator as dsl_interpreter_unary_operator import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator @@ -57,6 +60,13 @@ class NotOperator(dsl_interpreter_unary_operator.UnaryOperator): def get_name() -> str: return ast.Not.__name__ + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.BOOLEAN.value, + description="Logical NOT of the operand", + ) + def compute(self) -> dsl_interpreter_operator.ComputedOperatorParameterType: operand = self.get_computed_operand() return not operand @@ -71,6 +81,13 @@ class InvertOperator(dsl_interpreter_unary_operator.UnaryOperator): def get_name() -> str: return ast.Invert.__name__ + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.BOOLEAN.value, + description="Logical NOT of the operand (bitwise invert behaves as not)", + ) + def compute(self) -> dsl_interpreter_operator.ComputedOperatorParameterType: operand = self.get_computed_operand() return not operand # ~operand has been deprecated in favor of "not" diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py index d97fd45893..df7d650b6b 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py @@ -22,6 +22,7 @@ import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import tentacles.Meta.DSL_operators.python_std_operators.base_call_operators as base_call_operators @@ -30,11 +31,19 @@ class RaisesInPreComputeTestOperator(dsl_interpreter.CallOperator): """Fails in pre_compute only; used to assert if_error runs the on_error DSL.""" NAME = "raises_in_precompute_test_op" + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "raises_in_precompute_test_op" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.TEXT.value, + description="Test result", + ) + async def pre_compute(self) -> None: raise ValueError("pre_compute failed for test") diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py index 88f3e4a280..70d5184c5f 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_resetting_operators.py @@ -22,16 +22,25 @@ import tentacles.Meta.DSL_operators.python_std_operators.base_resetting_operators as base_resetting_operators import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors +import octobot_commons.enums as commons_enums class _LoopUntilSingleEvalTestConditionOperator(dsl_interpreter.CallOperator): MIN_PARAMS = 0 MAX_PARAMS = 0 + CATEGORY = commons_enums.DslKeywordCategory.LOGIC.value @staticmethod def get_name() -> str: return "test_loop_until_single_eval_condition" + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.BOOLEAN.value, + description="Test condition result", + ) + def compute(self): return True diff --git a/packages/tentacles/Meta/DSL_operators/ta_operators/ta_operator.py b/packages/tentacles/Meta/DSL_operators/ta_operators/ta_operator.py index b6ac54bc5c..a48da044b7 100644 --- a/packages/tentacles/Meta/DSL_operators/ta_operators/ta_operator.py +++ b/packages/tentacles/Meta/DSL_operators/ta_operators/ta_operator.py @@ -14,6 +14,8 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter.operators.call_operator as dsl_interpreter_call_operator @@ -21,6 +23,7 @@ class TAOperator(dsl_interpreter_call_operator.CallOperator): + CATEGORY = commons_enums.DslKeywordCategory.SOURCE.value @staticmethod def get_library() -> str: diff --git a/packages/tentacles/Meta/DSL_operators/ta_operators/tulipy_technical_analysis_operators.py b/packages/tentacles/Meta/DSL_operators/ta_operators/tulipy_technical_analysis_operators.py index 78f7652bc0..bbb26be884 100644 --- a/packages/tentacles/Meta/DSL_operators/ta_operators/tulipy_technical_analysis_operators.py +++ b/packages/tentacles/Meta/DSL_operators/ta_operators/tulipy_technical_analysis_operators.py @@ -17,9 +17,11 @@ import tulipy import numpy as np +import octobot_commons.constants as commons_constants +import octobot_commons.enums as commons_enums import octobot_commons.errors -import tentacles.Meta.DSL_operators.ta_operators.ta_operator as ta_operator import octobot_commons.dsl_interpreter as dsl_interpreter +import tentacles.Meta.DSL_operators.ta_operators.ta_operator as ta_operator def _to_numpy_array(data): @@ -66,10 +68,26 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="data", description="the data to compute the RSI on", required=True, type=list), - dsl_interpreter.OperatorParameter(name="period", description="the period to use for the RSI", required=True, type=int), + dsl_interpreter.OperatorParameter( + name="data", + description="the data to compute the RSI on", + required=True, + type=commons_enums.DslValueType.SERIES.value), + dsl_interpreter.OperatorParameter( + name="period", + description="the period to use for the RSI", + required=True, + type=commons_enums.DslValueType.NUMBER.value, + minimum=1), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.SERIES.value, + description="RSI series", + ) + @converted_tulipy_error def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() @@ -87,12 +105,38 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="data", description="the data to compute the MACD on", required=True, type=list), - dsl_interpreter.OperatorParameter(name="short_period", description="the short period to use for the MACD", required=True, type=int), - dsl_interpreter.OperatorParameter(name="long_period", description="the long period to use for the MACD", required=True, type=int), - dsl_interpreter.OperatorParameter(name="signal_period", description="the signal period to use for the MACD", required=True, type=int), + dsl_interpreter.OperatorParameter( + name="data", + description="the data to compute the MACD on", + required=True, + type=commons_enums.DslValueType.SERIES.value), + dsl_interpreter.OperatorParameter( + name="short_period", + description="the short period to use for the MACD", + required=True, + type=commons_enums.DslValueType.NUMBER.value, + minimum=1), + dsl_interpreter.OperatorParameter( + name="long_period", + description="the long period to use for the MACD", + required=True, + type=commons_enums.DslValueType.NUMBER.value, + minimum=1), + dsl_interpreter.OperatorParameter( + name="signal_period", + description="the signal period to use for the MACD", + required=True, + type=commons_enums.DslValueType.NUMBER.value, + minimum=1), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.SERIES.value, + description="MACD histogram series", + ) + @converted_tulipy_error def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() @@ -113,10 +157,26 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="data", description="the data to compute the moving average on", required=True, type=list), - dsl_interpreter.OperatorParameter(name="period", description="the period to use for the moving average", required=True, type=int), + dsl_interpreter.OperatorParameter( + name="data", + description="the data to compute the moving average on", + required=True, + type=commons_enums.DslValueType.SERIES.value), + dsl_interpreter.OperatorParameter( + name="period", + description="the period to use for the moving average", + required=True, + type=commons_enums.DslValueType.NUMBER.value, + minimum=1), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.SERIES.value, + description="Moving average series", + ) + @converted_tulipy_error def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() @@ -134,10 +194,26 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="data", description="the data to compute the exponential moving average on", required=True, type=list), - dsl_interpreter.OperatorParameter(name="period", description="the period to use for the exponential moving average", required=True, type=int), + dsl_interpreter.OperatorParameter( + name="data", + description="the data to compute the exponential moving average on", + required=True, + type=commons_enums.DslValueType.SERIES.value), + dsl_interpreter.OperatorParameter( + name="period", + description="the period to use for the exponential moving average", + required=True, + type=commons_enums.DslValueType.NUMBER.value, + minimum=1), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.SERIES.value, + description="Exponential moving average series", + ) + @converted_tulipy_error def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() @@ -155,11 +231,31 @@ def get_name() -> str: @classmethod def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: return [ - dsl_interpreter.OperatorParameter(name="data", description="the data to compute the volume weighted moving average on", required=True, type=list), - dsl_interpreter.OperatorParameter(name="volume", description="the volume data to use for the volume weighted moving average", required=True, type=list), - dsl_interpreter.OperatorParameter(name="period", description="the period to use for the volume weighted moving average", required=True, type=int), + dsl_interpreter.OperatorParameter( + name="data", + description="the data to compute the volume weighted moving average on", + required=True, + type=commons_enums.DslValueType.SERIES.value), + dsl_interpreter.OperatorParameter( + name="volume", + description="the volume data to use for the volume weighted moving average", + required=True, + type=commons_enums.DslValueType.SERIES.value), + dsl_interpreter.OperatorParameter( + name="period", + description="the period to use for the volume weighted moving average", + required=True, + type=commons_enums.DslValueType.NUMBER.value, + minimum=1), ] + @classmethod + def get_return_values(cls) -> list[dsl_interpreter.OperatorParameter]: + return cls.result_return_value( + commons_enums.DslValueType.SERIES.value, + description="Volume weighted moving average series", + ) + @converted_tulipy_error def compute(self) -> dsl_interpreter.ComputedOperatorParameterType: operands = self.get_computed_parameters() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py index d856596857..7f34fad531 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py @@ -33,10 +33,12 @@ debug, config, logs, + octobots, + dsl, ) except ImportError: from api.route_provider import register_all_provider_routes # type: ignore[no-redef] - from api.routes import login, nodes, users, tasks, setup, exchanges, wallets, debug, config, logs # type: ignore[no-redef] + from api.routes import login, nodes, users, tasks, setup, exchanges, wallets, debug, config, logs, octobots, dsl # type: ignore[no-redef] def build_api_router() -> APIRouter: @@ -48,8 +50,10 @@ def build_api_router() -> APIRouter: api_router.include_router(users.router, prefix="/users") api_router.include_router(wallets.router, prefix="/wallets") api_router.include_router(tasks.router, prefix="/tasks") + api_router.include_router(octobots.router, prefix="/octobots") api_router.include_router(nodes.router, prefix="/nodes") api_router.include_router(debug.router, prefix="/debug") api_router.include_router(config.router, prefix="/config") api_router.include_router(logs.router, prefix="/logs") + api_router.include_router(dsl.router, prefix="/dsl") return api_router diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py index 16329befa3..5125cfc0fa 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py @@ -26,13 +26,13 @@ import octobot_node.protocol.user_actions as user_actions_protocol import octobot_node.scheduler import octobot_protocol.models as protocol_models -import octobot_sync.server as _sync_server -import octobot.community.authentication as _community_auth try: from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser # type: ignore[no-redef] + from tentacles.Services.Interfaces.node_api_interface.api.user_id import evm_to_user_id # type: ignore[no-redef] except ImportError: from api.deps import CurrentUser # type: ignore[no-redef] + from api.user_id import evm_to_user_id # type: ignore[no-redef] router = APIRouter(tags=["debug"]) @@ -94,8 +94,69 @@ def _resolve_user_id( internal protocol and scheduler calls use the Starfish user_id. """ evm_address = _resolve_wallet_address(current_user, wallet_address) - wallet = _community_auth.CommunityAuthentication.instance().get_wallet(evm_address) - return _sync_server.derive_user_id(wallet.private_key) + return evm_to_user_id(evm_address) + + +def _extract_automation_parent_id( + user_action: protocol_models.UserAction, +) -> typing.Optional[str]: + wrapper = user_action.configuration + if wrapper is None or wrapper.actual_instance is None: + return None + payload = wrapper.actual_instance + if isinstance(payload, protocol_models.StopAutomationConfiguration): + return payload.id + if isinstance(payload, protocol_models.RestartAutomationConfiguration): + return payload.id + if isinstance(payload, protocol_models.SignalAutomationConfiguration): + return payload.automation_id + return None + + +async def _resolve_execution_user_id( + current_user: octobot_node.models.User, + wallet_address: typing.Optional[str], + user_action: protocol_models.UserAction, +) -> str: + """Pick the Starfish user_id passed to the scheduler for this user action. + + For stop/signal/restart, the executor resolves the active DBOS workflow using a + wallet-scoped lookup. Admins may act on another wallet's automation without passing + ``wallet_address``, but only after API-side authorization and owner resolution here. + """ + # Explicit wallet override: admin-gated in _resolve_wallet_address. + if wallet_address is not None: + return _resolve_user_id(current_user, wallet_address) + + caller_user_id = _resolve_user_id(current_user, None) + parent_automation_id = _extract_automation_parent_id(user_action) + if parent_automation_id is None: + return caller_user_id + + scheduler = octobot_node.scheduler.SCHEDULER + # Caller-owned automation: keep the authenticated wallet's user_id. + active_workflow_ids = await scheduler.resolve_active_automation_workflow_ids_for_parent_id( + caller_user_id, + parent_automation_id, + ) + if active_workflow_ids: + return caller_user_id + + # Cross-wallet: only superusers may resolve the automation owner without wallet filter. + if current_user.is_superuser: + owner_user_id = await scheduler.resolve_automation_owner_user_id(parent_automation_id) + if owner_user_id is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Automation not found", + ) + return owner_user_id + + # Non-superuser and automation not under caller's wallet. + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Automation not found", + ) def _ensure_debug_routes_enabled() -> None: @@ -145,7 +206,7 @@ async def execute_user_action( _ensure_debug_routes_enabled() _ensure_scheduler_initialized() user_action = _parse_user_action_payload(payload) - resolved_user_id = _resolve_user_id(current_user, wallet_address) + resolved_user_id = await _resolve_execution_user_id(current_user, wallet_address, user_action) try: await user_actions_protocol.execute_user_action(user_action, resolved_user_id) except RuntimeError as error: diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/dsl.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/dsl.py new file mode 100644 index 0000000000..533d7c57ad --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/dsl.py @@ -0,0 +1,37 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import json + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +import octobot_node.protocol.dsl as dsl_protocol +import octobot_protocol.models as protocol_models + +try: + from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser +except ImportError: + from api.deps import CurrentUser # type: ignore[no-redef] + +router = APIRouter(tags=["dsl"]) + + +@router.get("/keywords", response_model=protocol_models.DslKeywordsState) +def get_dsl_keywords(current_user: CurrentUser) -> JSONResponse: + """Return the versioned DSL keywords state for this node.""" + dsl_keywords_state = dsl_protocol.get_dsl_keywords_state() + return JSONResponse(content=json.loads(dsl_keywords_state.to_json())) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/exchanges.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/exchanges.py index e86b712603..9f524cb993 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/exchanges.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/exchanges.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . +import json import typing import octobot_commons.json_util as json_util @@ -46,19 +47,26 @@ def _exchange_config_from_query( ) -@router.get("/traded-pairs") +@router.get("/traded-pairs", response_model=protocol_models.TradedPairsByExchange) async def get_traded_pairs( exchange_config: typing.Annotated[protocol_models.ExchangeConfig, Depends(_exchange_config_from_query)], trading_type: typing.Annotated[protocol_models.TradingType, Query()] = protocol_models.TradingType.SPOT, + with_volume: typing.Annotated[bool, Query()] = False, ) -> JSONResponse: - pairs_and_tf_by_exchange = await exchange_core.get_traded_pairs_and_timeframes_by_exchange( - exchange_config, - trading_type=trading_type, + try: + content = await exchange_core.get_traded_pairs_by_exchange( + exchange_config, + trading_type=trading_type, + with_volume=with_volume, + ) + except trading_errors.NotSupported as err: + return JSONResponse(status_code=501, content={"error": str(err)}) + traded_pairs = protocol_models.TradedPairsByExchange.from_dict( + json_util.sanitize(content) ) - return JSONResponse(content={ - exchange: pairs_and_tf[exchange_core.ExchangeInfo.PAIRS.value] - for exchange, pairs_and_tf in pairs_and_tf_by_exchange.items() - }) + if traded_pairs is None: + raise RuntimeError("TradedPairsByExchange.from_dict returned None for non-null content") + return JSONResponse(content=json.loads(traded_pairs.to_json())) @router.get("/traded-pairs-and-timeframes") diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/logs.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/logs.py index d21eee7a03..341cf055c9 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/logs.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/logs.py @@ -36,10 +36,13 @@ _SAFE_TASK_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$") # Top-level log files in BASE_LOGS_FOLDER (no subdirectories), including RotatingFileHandler backups. _SAFE_MAIN_LOG_NAME_RE = re.compile(r"^[a-zA-Z0-9_.-]+\.log(\.\d+)?$") +# Active log file (no ``.log.N`` rotation suffix). +_ACTIVE_LOG_NAME_RE = re.compile(r"^[a-zA-Z0-9_.-]+\.log$") class ExportLogsRequest(BaseModel): task_ids: list[str] | None = None + latest_only: bool = False def _build_zip_from_log_files(log_entries: list[tuple[str, str]]) -> bytes | None: @@ -107,17 +110,54 @@ def _collect_task_log_entries(task_id: str) -> list[tuple[str, str]]: return entries -def build_logs_zip(task_ids: list[str]) -> bytes | None: +def _collect_latest_task_log_entries(task_id: str) -> list[tuple[str, str]]: + """Active ``.log`` files for one automation (flat file and/or folder tree).""" + logs_root = octobot_node.constants.AUTOMATION_LOGS_FOLDER + entries: list[tuple[str, str]] = [] + + log_file_path = os.path.join(logs_root, f"{task_id}.log") + if os.path.isfile(log_file_path): + entries.append((log_file_path, f"{task_id}.log")) + + log_dir_path = os.path.join(logs_root, task_id) + if os.path.isdir(log_dir_path): + log_dir_real = os.path.realpath(log_dir_path) + for dirpath, _dirnames, filenames in os.walk(log_dir_path): + for filename in filenames: + if not _ACTIVE_LOG_NAME_RE.match(filename): + continue + if not _SAFE_MAIN_LOG_NAME_RE.match(filename): + continue + file_path = os.path.join(dirpath, filename) + if not os.path.isfile(file_path): + continue + file_real = os.path.realpath(file_path) + if not _is_path_under_root(file_real, log_dir_real): + continue + rel_path = os.path.relpath(file_real, logs_root) + arcname = rel_path.replace(os.sep, "/") + entries.append((file_real, arcname)) + + return entries + + +def build_logs_zip(task_ids: list[str], *, latest_only: bool = False) -> bytes | None: """Zip per-automation logs for the given task ids. - Includes ``AUTOMATION_LOGS_FOLDER/.log`` when present and, for process - automations, all files under ``AUTOMATION_LOGS_FOLDER//``. + When ``latest_only`` is False, includes ``AUTOMATION_LOGS_FOLDER/.log`` when + present and, for process automations, all files under ``AUTOMATION_LOGS_FOLDER//``. + + When ``latest_only`` is True, includes only active ``.log`` files (no ``.log.N`` + backups) from the flat path and/or automation folder for each id. Missing files or folders are skipped. Returns None when nothing was found. """ log_entries: list[tuple[str, str]] = [] + collector = ( + _collect_latest_task_log_entries if latest_only else _collect_task_log_entries + ) for task_id in task_ids: - log_entries.extend(_collect_task_log_entries(task_id)) + log_entries.extend(collector(task_id)) return _build_zip_from_log_files(log_entries) @@ -136,7 +176,12 @@ def export_logs(body: ExportLogsRequest, current_user: CurrentUser) -> fastapi.R raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid task id" ) - archive = build_logs_zip(body.task_ids) + if body.latest_only and not body.task_ids: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="task_ids required when latest_only is true", + ) + archive = build_logs_zip(body.task_ids, latest_only=body.latest_only) if archive is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/nodes.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/nodes.py index f84ab54599..fbb7151aa9 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/nodes.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/nodes.py @@ -15,11 +15,13 @@ # License along with OctoBot. If not, see . import logging +import os import threading import time import typing import pydantic +import octobot_services.constants as services_constants import octobot_services.interfaces.util as interfaces_util from fastapi import APIRouter, status @@ -39,12 +41,19 @@ except ImportError: context_based_file_handler = None +# Service_bases is only needed at runtime, not for build (see node_api.py). +try: + import tentacles.Services.Services_bases.node_api_service.node_api as node_api_service_module +except ImportError: + node_api_service_module = None + router = APIRouter(tags=["nodes"]) class NodeConfigUpdate(pydantic.BaseModel): node_type: typing.Optional[typing.Literal["standalone", "master"]] = None use_dedicated_log_file_per_automation: typing.Optional[bool] = None + external_host: typing.Optional[str] = None @router.get("/me", response_model=octobot_node.models.Node) @@ -60,6 +69,11 @@ def get_node_config(current_user: CurrentUser) -> typing.Any: "use_dedicated_log_file_per_automation": octobot_node.config.settings.USE_DEDICATED_LOG_FILE_PER_AUTOMATION, "tasks_encryption_enabled": octobot_node.config.settings.tasks_encryption_enabled, "server_encryption_env_vars": octobot_node.constants.TASKS_ENCRYPTION_ENV_VARS, + "external_host": ( + node_api_service_module.NodeApiService.instance().get_node_external_host() + if node_api_service_module else None + ), + "external_host_env_override": bool(os.getenv(services_constants.ENV_NODE_EXTERNAL_HOST)), } @@ -86,6 +100,8 @@ def update_node_config(config: NodeConfigUpdate, current_user: CurrentUser) -> t octobot_node.scheduler.scheduler.Scheduler._setup_workflow_logging() else: _remove_context_based_file_handlers() + if config.external_host is not None and node_api_service_module is not None: + node_api_service_module.NodeApiService.instance().set_node_external_host(config.external_host) return get_node_config(current_user) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/octobots.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/octobots.py new file mode 100644 index 0000000000..a2a0fe1ae7 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/octobots.py @@ -0,0 +1,75 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import pydantic +from fastapi import APIRouter, HTTPException, status + +import octobot_node.errors as node_errors +import octobot_node.scheduler +import octobot_node.scheduler.generic_process_octobot as generic_process_octobot_module + +try: + from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser # type: ignore[no-redef] + from tentacles.Services.Interfaces.node_api_interface.api.user_id import evm_to_user_id # type: ignore[no-redef] +except ImportError: + from api.deps import CurrentUser # type: ignore[no-redef] + from api.user_id import evm_to_user_id # type: ignore[no-redef] + +router = APIRouter(tags=["octobots"]) + + +class CreateGenericProcessBotRequest(pydantic.BaseModel): + name: str + + +class CreateGenericProcessBotResponse(pydantic.BaseModel): + automation_id: str + + +@router.post( + "/generic-process", + response_model=CreateGenericProcessBotResponse, + status_code=status.HTTP_201_CREATED, +) +async def create_generic_process_bot( + body: CreateGenericProcessBotRequest, + current_user: CurrentUser, +) -> CreateGenericProcessBotResponse: + if not octobot_node.scheduler.is_initialized(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Scheduler not initialized", + ) + bot_name = body.name.strip() + if not bot_name: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="name must not be empty", + ) + user_id = evm_to_user_id(current_user.email) + try: + automation_id = await generic_process_octobot_module.create_generic_process_bot(user_id, bot_name) + except node_errors.UserActionError as error: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(error), + ) from error + except TimeoutError as error: + raise HTTPException( + status_code=status.HTTP_504_GATEWAY_TIMEOUT, + detail="Timed out waiting for bot creation workflow", + ) from error + return CreateGenericProcessBotResponse(automation_id=automation_id) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py index 844203be1c..facaa6bfd1 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/setup.py @@ -26,8 +26,10 @@ try: from api.deps import CurrentUser, security_basic # type: ignore[no-redef] + from core import network except ImportError: from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser, security_basic + from tentacles.Services.Interfaces.node_api_interface.core import network router = APIRouter(tags=["setup"]) @@ -53,6 +55,14 @@ class WalletExport(pydantic.BaseModel): seed: typing.Optional[str] = None +class LocalNetworkAddress(pydantic.BaseModel): + local_network_ip: typing.Optional[str] = None + + +class VPNNetworkAddress(pydantic.BaseModel): + vpn_network_ip: typing.Optional[str] = None + + @router.get("/setup/status", response_model=SetupStatus) def get_setup_status() -> SetupStatus: auth = community_auth.CommunityAuthentication.instance() @@ -60,6 +70,16 @@ def get_setup_status() -> SetupStatus: return SetupStatus(configured=configured) +@router.get("/setup/local-network-address", response_model=LocalNetworkAddress) +def get_local_network_address() -> LocalNetworkAddress: + return LocalNetworkAddress(local_network_ip=network.get_local_network_ip()) + + +@router.get("/setup/vpn-network-address", response_model=VPNNetworkAddress) +def get_vpn_network_address() -> VPNNetworkAddress: + return VPNNetworkAddress(vpn_network_ip=network.get_vpn_network_ip()) + + @router.post("/setup/init", response_model=SetupResult) def init_setup(body: SetupInit) -> SetupResult: auth = community_auth.CommunityAuthentication.instance() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py index 4a9ad82c64..f53718f753 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/tasks.py @@ -25,13 +25,13 @@ import octobot_node.scheduler import octobot_node.scheduler.api import octobot_node.scheduler.tasks -import octobot_sync.server as _sync_server -import octobot.community.authentication as _community_auth try: from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser # type: ignore[no-redef] + from tentacles.Services.Interfaces.node_api_interface.api.user_id import evm_to_user_id # type: ignore[no-redef] except ImportError: from api.deps import CurrentUser # type: ignore[no-redef] + from api.user_id import evm_to_user_id # type: ignore[no-redef] router = APIRouter(tags=["tasks"]) logger = logging.get_logger(__name__) @@ -39,17 +39,6 @@ _MAX_PAGE_LIMIT = 500 -def _evm_to_user_id(evm_address: str) -> str: - """Translate an EVM wallet address (HTTP Basic login username) to the Starfish - user_id used throughout the sync-core and scheduler. - - The HTTP API front uses EVM addresses for login; every internal scheduler API - uses the Starfish user_id. This helper bridges the boundary. - """ - wallet = _community_auth.CommunityAuthentication.instance().get_wallet(evm_address) - return _sync_server.derive_user_id(wallet.private_key) - - @router.post("/", response_model=tuple[int, int]) async def create_tasks( tasks: list[octobot_node.models.Task], @@ -60,7 +49,7 @@ async def create_tasks( success_count = 0 error_count = 0 for task in tasks: - task.user_id = _evm_to_user_id(current_user.email) + task.user_id = evm_to_user_id(current_user.email) is_scheduled = await octobot_node.scheduler.tasks.trigger_task(task) if is_scheduled: success_count += 1 @@ -84,7 +73,7 @@ def get_server_public_keys(current_user: CurrentUser) -> dict: @router.get("/metrics") async def get_metrics(current_user: CurrentUser) -> typing.Any: - user_id_filter = None if current_user.is_superuser else _evm_to_user_id(current_user.email) + user_id_filter = None if current_user.is_superuser else evm_to_user_id(current_user.email) return await octobot_node.scheduler.api.get_task_metrics(user_id=user_id_filter) @@ -95,7 +84,7 @@ async def get_tasks( limit: int = 100, ) -> typing.Any: limit = max(1, min(limit, _MAX_PAGE_LIMIT)) - user_id_filter = None if current_user.is_superuser else _evm_to_user_id(current_user.email) + user_id_filter = None if current_user.is_superuser else evm_to_user_id(current_user.email) tasks_data = await octobot_node.scheduler.api.get_all_tasks(user_id=user_id_filter) start_idx = (page - 1) * limit @@ -116,7 +105,7 @@ class ExportResultsBody(BaseModel): @router.post("/export-results", response_model=dict[str, dict[str, str]]) async def export_results(body: ExportResultsBody, current_user: CurrentUser) -> dict[str, dict[str, str]]: """Batch-decrypt completed task results for export. One round-trip for all selected tasks.""" - user_id_filter = None if current_user.is_superuser else _evm_to_user_id(current_user.email) + user_id_filter = None if current_user.is_superuser else evm_to_user_id(current_user.email) return await octobot_node.scheduler.api.get_tasks_export_results( body.task_ids, user_id_filter, user_rsa_public_key=body.user_rsa_public_key ) @@ -130,7 +119,7 @@ async def delete_tasks( requested_ids = [str(t) for t in taskIds] if not current_user.is_superuser: # Ownership check: only allow deleting own tasks - owned_tasks = await octobot_node.scheduler.api.get_all_tasks(user_id=_evm_to_user_id(current_user.email)) + owned_tasks = await octobot_node.scheduler.api.get_all_tasks(user_id=evm_to_user_id(current_user.email)) owned_ids = {t.id for t in owned_tasks if t.id is not None} unauthorized = [tid for tid in requested_ids if tid not in owned_ids] if unauthorized: @@ -155,7 +144,7 @@ async def cancel_tasks( current_user: CurrentUser, ) -> list[str]: if not current_user.is_superuser: - owned_tasks = await octobot_node.scheduler.api.get_all_tasks(user_id=_evm_to_user_id(current_user.email)) + owned_tasks = await octobot_node.scheduler.api.get_all_tasks(user_id=evm_to_user_id(current_user.email)) owned_ids = {t.id for t in owned_tasks if t.id is not None} unauthorized = [tid for tid in body.task_ids if tid not in owned_ids] if unauthorized: diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py index 5482fd44e3..cd32c8dd4f 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/wallets.py @@ -110,6 +110,11 @@ def create_wallet(body: CreateWalletBody, current_user: CurrentUser) -> WalletIn passphrase=body.passphrase, is_admin=False, ) + except wallet_backend.WalletAlreadyExistsError as err: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(err), + ) from err except wallet_backend.WalletError as err: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/user_id.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/user_id.py new file mode 100644 index 0000000000..587eb060a7 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/user_id.py @@ -0,0 +1,29 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_sync.server as sync_server +import octobot.community.authentication as community_auth + + +def evm_to_user_id(evm_address: str) -> str: + """Translate an EVM wallet address (HTTP Basic login username) to the Starfish + user_id used throughout the sync-core and scheduler. + + The HTTP API front uses EVM addresses for login; every internal scheduler API + uses the Starfish user_id. This helper bridges the boundary. + """ + wallet = community_auth.CommunityAuthentication.instance().get_wallet(evm_address) + return sync_server.derive_user_id(wallet.private_key) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/core/exchanges.py b/packages/tentacles/Services/Interfaces/node_api_interface/core/exchanges.py index 81b76cb2aa..793f71cffd 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/core/exchanges.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/core/exchanges.py @@ -1,4 +1,5 @@ import enum +import typing import octobot_commons.constants as commons_constants import octobot_commons.profiles.profile_data @@ -48,6 +49,71 @@ async def get_traded_pairs_and_timeframes_by_exchange( return traded_pairs_and_tf_by_exchange +def _empty_pair_volume() -> dict: + return {} + + +def _pair_volume_from_ticker(ticker: typing.Optional[dict]) -> dict: + ticker_columns = octobot_trading.enums.ExchangeConstantsTickersColumns + if ticker is None: + return { + ticker_columns.BASE_VOLUME.value: None, + ticker_columns.QUOTE_VOLUME.value: None, + } + return { + ticker_columns.BASE_VOLUME.value: ticker.get(ticker_columns.BASE_VOLUME.value), + ticker_columns.QUOTE_VOLUME.value: ticker.get(ticker_columns.QUOTE_VOLUME.value), + } + + +def _ticker_symbols_for_all_tickers_fetch( + exchange, + pairs: list[str], +) -> typing.Optional[list[str]]: + if exchange.get_option_value( + octobot_trading.enums.ExchangeClientOptions.REQUIRES_SYMBOLS_PARAM_TO_FETCH_TICKERS + ): + return pairs + return None + + +async def get_traded_pairs_by_exchange( + exchange_config: protocol_models.ExchangeConfig, + trading_type: protocol_models.TradingType = protocol_models.TradingType.SPOT, + with_volume: bool = False, +) -> dict[str, dict[str, dict]]: + traded_pairs_by_exchange = {} + tentacles_setup_config = octobot_tentacles_manager.api.get_full_tentacles_setup_config() + profile_data = _get_exchange_profile_data(exchange_config, trading_type=trading_type) + for exchange in profile_data.exchanges: + internal_name = exchange.internal_name + local_exchange_type = octobot_trading.enums.ExchangeTypes(exchange.exchange_type) + exchange_data = octobot_trading.exchanges.exchange_data_factory( + internal_name, + exchange_type=local_exchange_type.value + ) + async with octobot_trading.exchanges.exchange_manager_from_exchange_data( + exchange_data, profile_data, tentacles_setup_config, None + ) as exchange_manager: + pairs = list( + octobot_trading.api.get_all_available_symbols(exchange_manager, exchange_type=local_exchange_type) + ) + if not with_volume: + traded_pairs_by_exchange[internal_name] = { + pair: _empty_pair_volume() + for pair in pairs + } + continue + tickers = await exchange_manager.exchange.get_all_currencies_price_ticker( + symbols=_ticker_symbols_for_all_tickers_fetch(exchange_manager.exchange, pairs) + ) or {} + traded_pairs_by_exchange[internal_name] = { + pair: _pair_volume_from_ticker(tickers.get(pair)) + for pair in pairs + } + return traded_pairs_by_exchange + + def _dex_pair_matches_input_trading_pair( dex_pair: dict, requested, diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/core/network.py b/packages/tentacles/Services/Interfaces/node_api_interface/core/network.py new file mode 100644 index 0000000000..12f481b773 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/core/network.py @@ -0,0 +1,33 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_commons.network as commons_network + +TAILSCALE_INTERFACE_NAME_SUBSTRING = "tailscale" +TAILSCALE_IPV4_PREFIX = "100." # tailscale ip range is 100.x.x.x/16 + + +def get_vpn_network_ip() -> str | None: + interface_ipv4 = commons_network.get_interface_ipv4_by_name_substring( + TAILSCALE_INTERFACE_NAME_SUBSTRING, + ) + if interface_ipv4 is not None and interface_ipv4.startswith(TAILSCALE_IPV4_PREFIX): + return interface_ipv4 + return commons_network.get_interface_ipv4_by_prefix(TAILSCALE_IPV4_PREFIX) + + +def get_local_network_ip() -> str | None: + return commons_network.get_local_network_ip() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py b/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py index e293726382..92f4d39765 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/node_api.py @@ -29,7 +29,9 @@ import octobot_services.interfaces as services_interfaces import octobot_services.interfaces.util.web as web_util import octobot_commons.logging as octobot_commons_logging +import octobot_commons.network as network_module import octobot_node.config as node_config +import octobot_node.constants as node_constants import octobot_node.scheduler as scheduler # noqa: F401 import octobot_sync.server as sync_server @@ -99,10 +101,10 @@ async def _async_run(self) -> bool: self.logger.warning( "Scheduler not initialized by NodeApiService.prepare(); initializing now" ) - scheduler.initialize_scheduler() + await scheduler.initialize_scheduler() host = self.host port = self.port - self.app = self.create_app() + self.app = self.create_app(external_host=self.node_api_service.get_node_external_host()) # Set CORS from service config cors_origins_str = self.node_api_service.get_backend_cors_origins() cors_origins = [i.strip() for i in cors_origins_str.split(",") if i.strip()] if cors_origins_str else [] @@ -130,8 +132,22 @@ async def _async_run(self) -> bool: return True async def stop(self): - if self.server is not None: - self.server.should_exit = True + if self.server is None: + return + self.server.should_exit = True + serve_finished = self._serve_finished + if serve_finished is None: + return + try: + await asyncio.wait_for( + asyncio.to_thread(serve_finished.wait), + timeout=node_constants.NODE_API_STOP_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError: + self.logger.warning( + "Timed out waiting for Node API server to stop after %ss", + node_constants.NODE_API_STOP_TIMEOUT_SECONDS, + ) def _should_open_node_ui_in_browser(self) -> bool: try: @@ -143,7 +159,7 @@ def _should_open_node_ui_in_browser(self) -> bool: def _open_node_ui_on_browser(self): try: web_util.open_in_background_browser( - f"http://{node_api_service.LOCAL_HOST_IP}:{self.port}/app" + f"http://{network_module.LOCAL_HOST_IP}:{self.port}/app" ) except Exception as err: self.logger.warning( @@ -151,7 +167,7 @@ def _open_node_ui_on_browser(self): ) @classmethod - def create_app(cls) -> FastAPI: + def create_app(cls, external_host: str | None = None) -> FastAPI: @asynccontextmanager async def lifespan(app: FastAPI): yield @@ -182,6 +198,7 @@ async def lifespan(app: FastAPI): sync_server.derive_user_id(w.private_key) == user_id for w in community_auth.CommunityAuthentication.instance().list_wallet_entries() ), + external_host=external_host, ), ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_exchanges.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_exchanges.py index f2f15a52a9..c950504bdd 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_exchanges.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_exchanges.py @@ -146,6 +146,100 @@ async def test_futures_trading_type_uses_future_exchange_type( assert profile_data.exchanges[0].exchange_type == commons_constants.CONFIG_EXCHANGE_FUTURE +class TestGetTradedPairsByExchange: + pytestmark = pytest.mark.asyncio + + async def test_without_volume_returns_empty_volume_objects( + self, + ) -> None: + config = _spot_exchange_config() + exchange_mock = mock.Mock() + exchange_mock.get_all_currencies_price_ticker = mock.AsyncMock() + exchange_manager_mock = mock.Mock() + exchange_manager_mock.exchange = exchange_mock + context_manager_mock = mock.MagicMock() + context_manager_mock.__aenter__ = mock.AsyncMock(return_value=exchange_manager_mock) + context_manager_mock.__aexit__ = mock.AsyncMock(return_value=False) + with ( + mock.patch( + "octobot_trading.exchanges.exchange_manager_from_exchange_data", + return_value=context_manager_mock, + ), + mock.patch( + "octobot_trading.api.get_all_available_symbols", + return_value=["BTC/USDT", "ETH/USDC"], + ), + ): + result = await exchanges.get_traded_pairs_by_exchange(config, with_volume=False) + exchange_mock.get_all_currencies_price_ticker.assert_not_called() + assert result == { + config.exchange: { + "BTC/USDT": {}, + "ETH/USDC": {}, + } + } + + async def test_with_volume_fetches_all_tickers_and_maps_volumes( + self, + ) -> None: + config = _spot_exchange_config() + ticker_columns = trading_enums.ExchangeConstantsTickersColumns + exchange_mock = mock.Mock() + exchange_mock.get_option_value = mock.Mock(return_value=False) + exchange_mock.get_all_currencies_price_ticker = mock.AsyncMock(return_value={ + "BTC/USDT": { + ticker_columns.BASE_VOLUME.value: 12345.6, + ticker_columns.QUOTE_VOLUME.value: 987654321.0, + }, + "ETH/USDC": { + ticker_columns.BASE_VOLUME.value: 5000.0, + ticker_columns.QUOTE_VOLUME.value: 10000000.0, + }, + }) + exchange_manager_mock = mock.Mock() + exchange_manager_mock.exchange = exchange_mock + context_manager_mock = mock.MagicMock() + context_manager_mock.__aenter__ = mock.AsyncMock(return_value=exchange_manager_mock) + context_manager_mock.__aexit__ = mock.AsyncMock(return_value=False) + with ( + mock.patch( + "octobot_trading.exchanges.exchange_manager_from_exchange_data", + return_value=context_manager_mock, + ), + mock.patch( + "octobot_trading.api.get_all_available_symbols", + return_value=["BTC/USDT", "ETH/USDC"], + ), + ): + result = await exchanges.get_traded_pairs_by_exchange(config, with_volume=True) + exchange_mock.get_all_currencies_price_ticker.assert_awaited_once_with(symbols=None) + assert result == { + config.exchange: { + "BTC/USDT": { + ticker_columns.BASE_VOLUME.value: 12345.6, + ticker_columns.QUOTE_VOLUME.value: 987654321.0, + }, + "ETH/USDC": { + ticker_columns.BASE_VOLUME.value: 5000.0, + ticker_columns.QUOTE_VOLUME.value: 10000000.0, + }, + } + } + + async def test_with_volume_on_public_exchange_returns_btc_usdt_volumes( + self, + ) -> None: + public_name = _public_exchange_name_for_test() + config = _spot_exchange_config() + result = await exchanges.get_traded_pairs_by_exchange(config, with_volume=True) + assert public_name in result + assert LIQUID_TEST_SYMBOL in result[public_name] + btc_usdt_volume = result[public_name][LIQUID_TEST_SYMBOL] + ticker_columns = trading_enums.ExchangeConstantsTickersColumns + assert btc_usdt_volume[ticker_columns.BASE_VOLUME.value] > 0 + assert btc_usdt_volume[ticker_columns.QUOTE_VOLUME.value] > 0 + + class TestDexPairsForInputSymbol: def test_plain_trading_pair_returns_all_matching_dex_pairs(self) -> None: dex_pairs = _mock_btcb_usdt_dex_pairs() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_network.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_network.py new file mode 100644 index 0000000000..8182b41dc3 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/core/test_network.py @@ -0,0 +1,84 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import mock + +try: + import tentacles.Services.Interfaces.node_api_interface.core.network as network +except ImportError: + from core import network + + +NETWORK_MODULE = "tentacles.Services.Interfaces.node_api_interface.core.network" + + +class TestGetVpnNetworkIp: + def test_get_vpn_network_ip_delegates_with_tailscale_substring(self): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_name_substring", + return_value="100.64.0.1", + ) as get_interface_ipv4: + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_prefix", + ) as get_interface_ipv4_by_prefix: + assert network.get_vpn_network_ip() == "100.64.0.1" + get_interface_ipv4.assert_called_once_with(network.TAILSCALE_INTERFACE_NAME_SUBSTRING) + get_interface_ipv4_by_prefix.assert_not_called() + + def test_get_vpn_network_ip_falls_back_to_prefix_when_name_scan_returns_none(self): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_name_substring", + return_value=None, + ): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_prefix", + return_value="100.64.0.9", + ) as get_interface_ipv4_by_prefix: + assert network.get_vpn_network_ip() == "100.64.0.9" + get_interface_ipv4_by_prefix.assert_called_once_with(network.TAILSCALE_IPV4_PREFIX) + + def test_get_vpn_network_ip_returns_none_when_name_and_prefix_scans_find_nothing(self): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_name_substring", + return_value=None, + ): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_prefix", + return_value=None, + ): + assert network.get_vpn_network_ip() is None + + def test_get_vpn_network_ip_falls_back_to_prefix_when_name_scan_returns_non_tailscale_ip(self): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_name_substring", + return_value="192.168.0.5", + ): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_interface_ipv4_by_prefix", + return_value="100.64.0.2", + ) as get_interface_ipv4_by_prefix: + assert network.get_vpn_network_ip() == "100.64.0.2" + get_interface_ipv4_by_prefix.assert_called_once_with(network.TAILSCALE_IPV4_PREFIX) + + +class TestGetLocalNetworkIp: + def test_get_local_network_ip_delegates_to_commons(self): + with mock.patch( + f"{NETWORK_MODULE}.commons_network.get_local_network_ip", + return_value="192.168.0.10", + ) as get_local_network_ip: + assert network.get_local_network_ip() == "192.168.0.10" + get_local_network_ip.assert_called_once_with() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_exchanges_api.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_exchanges_api.py index 22655c2a8f..0d7e260e05 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_exchanges_api.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_exchanges_api.py @@ -16,6 +16,7 @@ import mock import typing +import pytest import octobot_commons import octobot_commons.symbols.symbol_util as commons_symbols import octobot_protocol.models as protocol_models @@ -183,12 +184,17 @@ def test_returns_pairs_map_only( self, client: typing.Any, ) -> None: - raw = _mock_pairs_timeframes_payload() + expected = { + "binance": { + "BTC/USDT": {}, + "ETH/USDC": {}, + } + } with mock.patch.object( node_exchanges_core, - "get_traded_pairs_and_timeframes_by_exchange", - mock.AsyncMock(return_value=raw), - ) as get_pairs_tf_mock: + "get_traded_pairs_by_exchange", + mock.AsyncMock(return_value=expected), + ) as get_traded_pairs_mock: response = client.get( _TRADED_PAIRS, params={ @@ -199,14 +205,143 @@ def test_returns_pairs_map_only( "trading_type": "spot", }, ) - get_pairs_tf_mock.assert_awaited_once() + get_traded_pairs_mock.assert_awaited_once_with( + mock.ANY, + trading_type=protocol_models.TradingType.SPOT, + with_volume=False, + ) + assert response.status_code == 200 + assert response.json() == expected + assert_response_headers(response) + + def test_returns_volumes_when_with_volume_true( + self, + client: typing.Any, + ) -> None: + expected = { + "binance": { + "BTC/USDT": {"baseVolume": 12345.6, "quoteVolume": 987654321.0}, + "ETH/USDC": {"baseVolume": 5000.0, "quoteVolume": 10000000.0}, + } + } + with mock.patch.object( + node_exchanges_core, + "get_traded_pairs_by_exchange", + mock.AsyncMock(return_value=expected), + ) as get_traded_pairs_mock: + response = client.get( + _TRADED_PAIRS, + params={ + "id": "test-exchange-config", + "name": "binance-test", + "exchange": "binance", + "sandboxed": False, + "trading_type": "spot", + "with_volume": True, + }, + ) + get_traded_pairs_mock.assert_awaited_once_with( + mock.ANY, + trading_type=protocol_models.TradingType.SPOT, + with_volume=True, + ) assert response.status_code == 200 - pair_key = node_exchanges_core.ExchangeInfo.PAIRS.value - assert response.json() == { - "binance": raw["binance"][pair_key], + assert response.json() == expected + assert_response_headers(response) + + def test_returns_501_when_tickers_not_supported( + self, + client: typing.Any, + ) -> None: + not_supported_error = trading_errors.NotSupported( + "This exchange doesn't support fetchTickers" + ) + with mock.patch.object( + node_exchanges_core, + "get_traded_pairs_by_exchange", + mock.AsyncMock(side_effect=not_supported_error), + ): + response = client.get( + _TRADED_PAIRS, + params={ + "id": "binance-config", + "name": "binance-test", + "exchange": "binance", + "sandboxed": False, + "trading_type": "spot", + "with_volume": True, + }, + ) + assert response.status_code == 501 + assert response.json() == {"error": str(not_supported_error)} + + def test_returns_null_volumes_when_ticker_missing( + self, + client: typing.Any, + ) -> None: + expected = { + "binance": { + "BTC/USDT": {"baseVolume": None, "quoteVolume": None}, } + } + with mock.patch.object( + node_exchanges_core, + "get_traded_pairs_by_exchange", + mock.AsyncMock(return_value=expected), + ) as get_traded_pairs_mock: + response = client.get( + _TRADED_PAIRS, + params={ + "id": "test-exchange-config", + "name": "binance-test", + "exchange": "binance", + "sandboxed": False, + "trading_type": "spot", + "with_volume": True, + }, + ) + get_traded_pairs_mock.assert_awaited_once_with( + mock.ANY, + trading_type=protocol_models.TradingType.SPOT, + with_volume=True, + ) + assert response.status_code == 200 + assert response.json() == expected assert_response_headers(response) + def test_raises_when_protocol_from_dict_returns_none( + self, + client: typing.Any, + ) -> None: + core_payload = { + "binance": { + "BTC/USDT": {}, + } + } + with mock.patch.object( + node_exchanges_core, + "get_traded_pairs_by_exchange", + mock.AsyncMock(return_value=core_payload), + ), mock.patch.object( + protocol_models.TradedPairsByExchange, + "from_dict", + return_value=None, + ): + with pytest.raises( + RuntimeError, + match="TradedPairsByExchange.from_dict returned None for non-null content", + ): + client.get( + _TRADED_PAIRS, + params={ + "id": "test-exchange-config", + "name": "binance-test", + "exchange": "binance", + "sandboxed": False, + "trading_type": "spot", + }, + ) + class TestExchangesGetTradedPairsAndTimeframes: def test_returns_full_payload_from_core( diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_node_api_stop.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_node_api_stop.py new file mode 100644 index 0000000000..c05ac248ba --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_node_api_stop.py @@ -0,0 +1,51 @@ +# Drakkar-Software OctoBot-Interfaces +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import threading + +import mock +import pytest + +import tentacles.Services.Interfaces.node_api_interface.node_api as node_api_module + + +pytestmark = pytest.mark.asyncio + + +class TestNodeApiInterfaceStop: + async def test_stop_sets_should_exit_and_waits_for_serve_finished(self): + interface = node_api_module.NodeApiInterface({}) + interface.logger = mock.Mock() + interface.server = mock.Mock() + interface.server.should_exit = False + serve_finished = threading.Event() + serve_finished.set() + interface._serve_finished = serve_finished + + await interface.stop() + + assert interface.server.should_exit is True + + async def test_stop_logs_warning_when_serve_does_not_finish_in_time(self): + interface = node_api_module.NodeApiInterface({}) + interface.logger = mock.Mock() + interface.server = mock.Mock() + interface._serve_finished = threading.Event() + + with mock.patch("asyncio.wait_for", side_effect=asyncio.TimeoutError): + await interface.stop() + + interface.logger.warning.assert_called_once() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py index ce78110662..0515232921 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_debug.py @@ -15,15 +15,18 @@ # License along with OctoBot. If not, see . import base64 +import contextlib import mock +import pytest import octobot_node.config +import octobot_node.scheduler import octobot_protocol.models as protocol_models import octobot_sync.constants as sync_constants from datetime import datetime, timezone -from .conftest import ADMIN_ADDRESS, ADMIN_PASSPHRASE, TENANT_ADDRESS, TENANT_USER_ID +from .conftest import ADMIN_ADDRESS, ADMIN_PASSPHRASE, ADMIN_USER_ID, TENANT_ADDRESS, TENANT_USER_ID def _sample_debug_state() -> protocol_models.DebugState: @@ -80,6 +83,62 @@ def _stop_automation_user_action_payload() -> dict: } +def _restart_automation_user_action_payload() -> dict: + return { + "id": "ua-restart-api-test", + "configuration": { + "action_type": "automation_restart", + "id": "00000000-0000-4000-8000-000000000099", + }, + } + + +_AUTOMATION_PARENT_ID = "00000000-0000-4000-8000-000000000099" + + +@contextlib.contextmanager +def _automation_owned_by_caller(): + with mock.patch.object( + octobot_node.scheduler.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[f"{_AUTOMATION_PARENT_ID}_1"], + ): + yield + + +@contextlib.contextmanager +def _automation_not_owned_by_caller(*, owner_user_id: str): + with mock.patch.object( + octobot_node.scheduler.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[], + ), mock.patch.object( + octobot_node.scheduler.SCHEDULER, + "resolve_automation_owner_user_id", + new_callable=mock.AsyncMock, + return_value=owner_user_id, + ): + yield + + +@contextlib.contextmanager +def _automation_not_found(): + with mock.patch.object( + octobot_node.scheduler.SCHEDULER, + "resolve_active_automation_workflow_ids_for_parent_id", + new_callable=mock.AsyncMock, + return_value=[], + ), mock.patch.object( + octobot_node.scheduler.SCHEDULER, + "resolve_automation_owner_user_id", + new_callable=mock.AsyncMock, + return_value=None, + ): + yield + + def _auth_header(address: str, passphrase: str) -> dict: token = base64.b64encode(f"{address}:{passphrase}".encode()).decode() return {"Authorization": f"Basic {token}"} @@ -217,10 +276,11 @@ def test_parses_flat_signal_configuration(self, tenant_client, mock_auth): new=mock_execute_user_action, ): with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): - response = tenant_client.post( - "/api/v1/debug/", - json=_signal_user_action_payload(), - ) + with _automation_owned_by_caller(): + response = tenant_client.post( + "/api/v1/debug/", + json=_signal_user_action_payload(), + ) assert response.status_code == 204 assert response.content == b"" user_action_argument = mock_execute_user_action.await_args[0][0] @@ -239,10 +299,11 @@ def test_parses_flat_stop_configuration(self, tenant_client, mock_auth): new=mock_execute_user_action, ): with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): - response = tenant_client.post( - "/api/v1/debug/", - json=_stop_automation_user_action_payload(), - ) + with _automation_owned_by_caller(): + response = tenant_client.post( + "/api/v1/debug/", + json=_stop_automation_user_action_payload(), + ) assert response.status_code == 204 user_action_argument = mock_execute_user_action.await_args[0][0] configuration = user_action_argument.configuration.actual_instance @@ -320,3 +381,81 @@ def test_when_encryption_enabled_returns_404(self, tenant_client, mock_auth): response = tenant_client.post("/api/v1/debug/", json=_minimal_user_action_payload()) assert response.status_code == 404 assert response.json()["detail"] == "Debug routes are disabled when node-side encryption is enabled" + + +class TestExecuteUserActionCrossWalletAutomation: + @pytest.mark.parametrize( + "payload_factory", + [ + _stop_automation_user_action_payload, + _signal_user_action_payload, + _restart_automation_user_action_payload, + ], + ) + def test_admin_without_wallet_address_uses_owner_user_id( + self, + admin_client, + mock_auth, + payload_factory, + ): + mock_execute_user_action = mock.AsyncMock(return_value=None) + with mock.patch( + "octobot_node.protocol.user_actions.execute_user_action", + new=mock_execute_user_action, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + with _automation_not_owned_by_caller(owner_user_id=TENANT_USER_ID): + response = admin_client.post( + "/api/v1/debug/", + json=payload_factory(), + ) + assert response.status_code == 204 + assert mock_execute_user_action.await_args[0][1] == TENANT_USER_ID + + @pytest.mark.parametrize( + "payload_factory", + [ + _stop_automation_user_action_payload, + _signal_user_action_payload, + _restart_automation_user_action_payload, + ], + ) + def test_non_admin_without_wallet_address_returns_404_when_not_owned( + self, + tenant_client, + mock_auth, + payload_factory, + ): + mock_execute_user_action = mock.AsyncMock(return_value=None) + with mock.patch( + "octobot_node.protocol.user_actions.execute_user_action", + new=mock_execute_user_action, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + with _automation_not_found(): + response = tenant_client.post( + "/api/v1/debug/", + json=payload_factory(), + ) + assert response.status_code == 404 + assert response.json()["detail"] == "Automation not found" + mock_execute_user_action.assert_not_awaited() + + def test_admin_without_wallet_address_uses_own_user_id_when_automation_is_owned( + self, + admin_client, + mock_auth, + ): + mock_execute_user_action = mock.AsyncMock(return_value=None) + with mock.patch( + "octobot_node.protocol.user_actions.execute_user_action", + new=mock_execute_user_action, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + with _automation_owned_by_caller(): + response = admin_client.post( + "/api/v1/debug/", + json=_stop_automation_user_action_payload(), + ) + assert response.status_code == 204 + assert mock_execute_user_action.await_args[0][1] == ADMIN_USER_ID diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_dsl.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_dsl.py new file mode 100644 index 0000000000..e5bde29c0e --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_dsl.py @@ -0,0 +1,36 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import octobot_sync.constants as sync_constants + + +class TestGetDslKeywords: + def test_without_auth_returns_401(self, client, mock_auth): + response = client.get("/api/v1/dsl/keywords") + assert response.status_code == 401, response.text + + def test_returns_keywords_state(self, admin_client): + response = admin_client.get("/api/v1/dsl/keywords") + assert response.status_code == 200, response.text + body = response.json() + assert body["version"] == sync_constants.DSL_KEYWORDS_STATE_VERSION + assert isinstance(body["keywords"], list) + assert len(body["keywords"]) > 0 + first_keyword = body["keywords"][0] + assert "name" in first_keyword + assert "category" in first_keyword + assert "inputs" in first_keyword + assert "outputs" in first_keyword diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py index 4153c4462c..70ceb2de6d 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_logs.py @@ -57,6 +57,45 @@ def test_zips_existing_log_files_and_skips_missing(self, tmp_path): assert zip_file.namelist() == ["task-a.log"] assert zip_file.read("task-a.log") == b"hello a" + def test_latest_only_zips_flat_log(self, tmp_path): + (tmp_path / "task-a.log").write_text("hello a") + with mock.patch.object(octobot_node.constants, "AUTOMATION_LOGS_FOLDER", str(tmp_path)): + archive = build_logs_zip(["task-a"], latest_only=True) + assert archive is not None + with zipfile.ZipFile(io.BytesIO(archive)) as zip_file: + assert zip_file.namelist() == ["task-a.log"] + + def test_latest_only_picks_active_log_in_folder(self, tmp_path): + task_dir = tmp_path / "task-a" + task_dir.mkdir() + (task_dir / "OctoBot.log").write_text("current") + (task_dir / "OctoBot.log.1").write_text("backup") + with mock.patch.object(octobot_node.constants, "AUTOMATION_LOGS_FOLDER", str(tmp_path)): + archive = build_logs_zip(["task-a"], latest_only=True) + assert archive is not None + with zipfile.ZipFile(io.BytesIO(archive)) as zip_file: + assert zip_file.namelist() == ["task-a/OctoBot.log"] + assert zip_file.read("task-a/OctoBot.log") == b"current" + + def test_latest_only_includes_flat_and_folder(self, tmp_path): + (tmp_path / "task-a.log").write_text("flat") + task_dir = tmp_path / "task-a" + task_dir.mkdir() + (task_dir / "OctoBot.log").write_text("process") + with mock.patch.object(octobot_node.constants, "AUTOMATION_LOGS_FOLDER", str(tmp_path)): + archive = build_logs_zip(["task-a"], latest_only=True) + assert archive is not None + with zipfile.ZipFile(io.BytesIO(archive)) as zip_file: + assert sorted(zip_file.namelist()) == ["task-a.log", "task-a/OctoBot.log"] + + def test_latest_only_skips_missing_tasks(self, tmp_path): + (tmp_path / "task-a.log").write_text("hello a") + with mock.patch.object(octobot_node.constants, "AUTOMATION_LOGS_FOLDER", str(tmp_path)): + archive = build_logs_zip(["task-a", "task-missing"], latest_only=True) + assert archive is not None + with zipfile.ZipFile(io.BytesIO(archive)) as zip_file: + assert zip_file.namelist() == ["task-a.log"] + class TestBuildMainLogsZip: def test_returns_none_when_folder_missing(self, tmp_path): @@ -182,3 +221,18 @@ def test_returns_zip_with_selected_logs(self, admin_client, tmp_path): with zipfile.ZipFile(io.BytesIO(response.content)) as zip_file: assert zip_file.namelist() == ["task-a.log"] assert zip_file.read("task-a.log") == b"log line a" + + def test_export_latest_only_route(self, admin_client, tmp_path): + task_dir = tmp_path / "task-a" + task_dir.mkdir() + (task_dir / "OctoBot.log").write_text("process log") + (task_dir / "OctoBot.log.1").write_text("rotated") + with mock.patch.object(octobot_node.constants, "AUTOMATION_LOGS_FOLDER", str(tmp_path)): + response = admin_client.post( + "/api/v1/logs/export", + json={"task_ids": ["task-a"], "latest_only": True}, + ) + assert response.status_code == 200 + with zipfile.ZipFile(io.BytesIO(response.content)) as zip_file: + assert zip_file.namelist() == ["task-a/OctoBot.log"] + assert zip_file.read("task-a/OctoBot.log") == b"process log" diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_nodes.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_nodes.py index bfa44f1b21..589618f9ab 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_nodes.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_nodes.py @@ -19,6 +19,9 @@ import time import mock +import pytest + +import octobot_node.config as octobot_node_config from .conftest import ADMIN_ADDRESS, ADMIN_PASSPHRASE, TENANT_ADDRESS, TENANT_PASSPHRASE @@ -78,3 +81,73 @@ def test_tenant_with_basic_auth_returns_403(self, client, mock_auth): headers=_auth_header(TENANT_ADDRESS, TENANT_PASSPHRASE), ) assert response.status_code == 403 + + +class TestNodeConfigExternalHost: + @pytest.fixture(autouse=True) + def _configured_node_settings(self, admin_client, monkeypatch): + # admin_client's unit_app fixture patches octobot_node.config.settings with a bare + # mock.Mock(); explicitly set the attributes get_node_config() reads so the response + # can be JSON-serialized. + monkeypatch.setattr(octobot_node_config.settings, "IS_MASTER_MODE", False) + monkeypatch.setattr(octobot_node_config.settings, "USE_DEDICATED_LOG_FILE_PER_AUTOMATION", False) + monkeypatch.setattr(octobot_node_config.settings, "tasks_encryption_enabled", False) + + def test_get_config_reflects_stored_and_env_override(self, admin_client, monkeypatch): + node_api_service = mock.Mock() + node_api_service.get_node_external_host.return_value = "node.example.com" + monkeypatch.setenv("NODE_EXTERNAL_HOST", "node.example.com") + with mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.nodes.node_api_service_module" + ".NodeApiService.instance", + return_value=node_api_service, + ): + response = admin_client.get("/api/v1/nodes/config") + assert response.status_code == 200 + body = response.json() + assert body["external_host"] == "node.example.com" + assert body["external_host_env_override"] is True + + def test_get_config_without_env_override(self, admin_client, monkeypatch): + node_api_service = mock.Mock() + node_api_service.get_node_external_host.return_value = "stored.example.com" + monkeypatch.delenv("NODE_EXTERNAL_HOST", raising=False) + with mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.nodes.node_api_service_module" + ".NodeApiService.instance", + return_value=node_api_service, + ): + response = admin_client.get("/api/v1/nodes/config") + assert response.status_code == 200 + body = response.json() + assert body["external_host"] == "stored.example.com" + assert body["external_host_env_override"] is False + + def test_patch_config_persists_external_host(self, admin_client, monkeypatch): + node_api_service = mock.Mock() + node_api_service.get_node_external_host.return_value = "new-host.example.com" + monkeypatch.delenv("NODE_EXTERNAL_HOST", raising=False) + with mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.nodes.node_api_service_module" + ".NodeApiService.instance", + return_value=node_api_service, + ): + response = admin_client.patch( + "/api/v1/nodes/config", json={"external_host": "new-host.example.com"} + ) + assert response.status_code == 200 + node_api_service.set_node_external_host.assert_called_once_with("new-host.example.com") + assert response.json()["external_host"] == "new-host.example.com" + + def test_patch_config_without_external_host_does_not_set(self, admin_client, monkeypatch): + node_api_service = mock.Mock() + node_api_service.get_node_external_host.return_value = None + monkeypatch.delenv("NODE_EXTERNAL_HOST", raising=False) + with mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.nodes.node_api_service_module" + ".NodeApiService.instance", + return_value=node_api_service, + ): + response = admin_client.patch("/api/v1/nodes/config", json={"node_type": "standalone"}) + assert response.status_code == 200 + node_api_service.set_node_external_host.assert_not_called() diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py index c74269137d..5952f8f90c 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_setup.py @@ -129,3 +129,23 @@ def test_wallet_export_success(admin_client, mock_auth): data = resp.json() assert data["address"] == ADMIN_ADDRESS assert data["private_key"] == "0xdeadbeef" + + +def test_setup_local_network_address(client): + with mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.network.get_local_network_ip", + return_value="192.168.0.10", + ): + resp = client.get("/api/v1/setup/local-network-address") + assert resp.status_code == 200 + assert resp.json() == {"local_network_ip": "192.168.0.10"} + + +def test_setup_vpn_network_address(client): + with mock.patch( + "tentacles.Services.Interfaces.node_api_interface.api.routes.setup.network.get_vpn_network_ip", + return_value="100.64.0.1", + ): + resp = client.get("/api/v1/setup/vpn-network-address") + assert resp.status_code == 200 + assert resp.json() == {"vpn_network_ip": "100.64.0.1"} diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py index b7da0e2625..38d35b711d 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_wallets.py @@ -112,10 +112,10 @@ def test_create_wallet_with_private_key(admin_client, mock_auth): ) -def test_create_wallet_duplicate_raises_422(admin_client, mock_auth): +def test_create_wallet_duplicate_raises_409(admin_client, mock_auth): mock_auth.create_wallet.side_effect = wallet_backend.WalletAlreadyExistsError("Wallet already exists") resp = admin_client.post("/api/v1/wallets/", json={"passphrase": "newpass123"}) - assert resp.status_code == 422 + assert resp.status_code == 409 def test_create_wallet_service_unavailable(admin_client): diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/ConfirmWalletSecretCopyDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/ConfirmWalletSecretCopyDialog.tsx new file mode 100644 index 0000000000..894857d8ff --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/ConfirmWalletSecretCopyDialog.tsx @@ -0,0 +1,129 @@ +import { TriangleAlert } from "lucide-react" + +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { + OCTOBOT_PLAY_STORE_URL, + OCTOBOT_TESTFLIGHT_URL, + OCTOBOT_WEB_INTERFACE_URL, +} from "@/lib/external-links" +import type { WalletSecretType } from "@/lib/use-confirm-wallet-secret-copy" + +type ConfirmWalletSecretCopyDialogProps = { + open: boolean + onOpenChange: (open: boolean) => void + secretType: WalletSecretType + onConfirm: () => void +} + +const DIALOG_COPY: Record< + WalletSecretType, + { title: string; confirmLabel: string } +> = { + private_key: { + title: "Confirm private key copy", + confirmLabel: "I confirm I want to copy my key", + }, + seed_phrase: { + title: "Confirm seed phrase copy", + confirmLabel: "I confirm I want to copy my seed phrase", + }, +} + +export function ConfirmWalletSecretCopyDialog({ + open, + onOpenChange, + secretType, + onConfirm, +}: ConfirmWalletSecretCopyDialogProps) { + const { title, confirmLabel } = DIALOG_COPY[secretType] + + const handleConfirm = () => { + onConfirm() + onOpenChange(false) + } + + return ( + + + + {title} + +
+

+ Only paste this secret into the official OctoBot web interface or + mobile app. +

+

+ Pasting it anywhere else is extremely dangerous. In 99% of + cases, it is a scam attempt to steal your account. +

+

+ Never share your private key or seed phrase with anyone. Even the + OctoBot support team will never ask you for your keys. +

+
+ + +
+ + + Do not paste your secret into websites, chats, forms, or tools other + than the official OctoBot interfaces listed below. + +
+ + + + + + + + + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/CreateGenericProcessBotDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/CreateGenericProcessBotDialog.tsx new file mode 100644 index 0000000000..e29aabfda1 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/CreateGenericProcessBotDialog.tsx @@ -0,0 +1,177 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query" +import { useNavigate } from "@tanstack/react-router" +import { TriangleAlert } from "lucide-react" +import { useState } from "react" + +import { OctobotsService } from "@/client" +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { LoadingButton } from "@/components/ui/loading-button" +import { + buildCreateGenericProcessBotRequestBody, + formatCreateGenericProcessBotError, + validateCreateGenericProcessBotName, +} from "@/lib/octobots/create-generic-process-bot" + +type DialogStep = "form" | "success" + +interface CreateGenericProcessBotDialogProps { + open: boolean + onOpenChange: (open: boolean) => void +} + +export function CreateGenericProcessBotDialog({ + open, + onOpenChange, +}: CreateGenericProcessBotDialogProps) { + const navigate = useNavigate() + const queryClient = useQueryClient() + const [step, setStep] = useState("form") + const [name, setName] = useState("") + const [createdBotName, setCreatedBotName] = useState("") + const [inlineError, setInlineError] = useState(null) + const [nameValidationMessage, setNameValidationMessage] = useState< + string | null + >(null) + + const createMutation = useMutation({ + mutationFn: (trimmedName: string) => + OctobotsService.createGenericProcessBot({ + requestBody: buildCreateGenericProcessBotRequestBody(trimmedName), + }), + onSuccess: (_result, trimmedName) => { + setCreatedBotName(trimmedName) + setStep("success") + setInlineError(null) + queryClient.invalidateQueries({ queryKey: ["tasks"] }) + }, + onError: (error) => { + setInlineError(formatCreateGenericProcessBotError(error)) + }, + }) + + const resetDialog = () => { + setStep("form") + setName("") + setCreatedBotName("") + setInlineError(null) + setNameValidationMessage(null) + createMutation.reset() + } + + const handleOpenChange = (nextOpen: boolean) => { + if (!nextOpen) { + resetDialog() + } + onOpenChange(nextOpen) + } + + const handleNameChange = (value: string) => { + setName(value) + setInlineError(null) + if (nameValidationMessage !== null) { + setNameValidationMessage(null) + } + } + + const handleCreate = () => { + const validation = validateCreateGenericProcessBotName(name) + if (!validation.valid) { + setNameValidationMessage(validation.message) + return + } + setNameValidationMessage(null) + createMutation.mutate(validation.trimmedName) + } + + const handleBackToOctoBots = () => { + handleOpenChange(false) + navigate({ to: "/" }) + } + + return ( + + + {step === "form" ? ( + <> + + Name your OctoBot + + This starts an OctoBot you can configure manually with its + dedicated interface. Best for backtesting and in-depth analysis. + + +
+
+ + handleNameChange(event.target.value)} + placeholder="My manual OctoBot" + maxLength={200} + autoFocus + /> + {nameValidationMessage && ( +

{nameValidationMessage}

+ )} +
+ {inlineError && ( +
+ +

+ {inlineError} +

+
+ )} +
+ + + + + + Create OctoBot + + + + ) : ( + <> + + OctoBot created + + + {createdBotName} + {" "} + is starting. Your OctoBot web interface will automatically open to let you select your profile once the bot will be ready. +
+ If the interface doesn't open automatically, click on the link of the OctoBot's automation. + + + + + + + )} + + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/NewBotCards.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/NewBotCards.tsx index 419cbc010c..7bb1e4c886 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/NewBotCards.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/NewBotCards.tsx @@ -1,6 +1,9 @@ import { Link } from "@tanstack/react-router" -import { Star, Upload } from "lucide-react" +import { Upload } from "lucide-react" +import { useState } from "react" +import { CreateGenericProcessBotDialog } from "@/components/Common/CreateGenericProcessBotDialog" +import { StartAutomationDialog } from "@/components/Setup/StartAutomationDialog" import { Button } from "@/components/ui/button" import { Card, @@ -9,21 +12,31 @@ import { CardHeader, CardTitle, } from "@/components/ui/card" -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip" import { getAssetPath } from "@/lib/utils" -export function NewBotCards() { +type NewBotCardsProps = { + onFinishSetup?: () => void + onSkip?: () => void +} + +function ComplexityBadge({ label }: { label: string }) { + return ( + + {label} + + ) +} + +export function NewBotCards({ onFinishSetup, onSkip }: NewBotCardsProps) { + const [genericProcessDialogOpen, setGenericProcessDialogOpen] = useState(false) + const [automationDialogOpen, setAutomationDialogOpen] = useState(false) const launchImage = getAssetPath("images/octobot_launching_512.png") const designImage = getAssetPath("images/octobot_design_512.png") const labImage = getAssetPath("images/octobot_lab_512.png") return (
- +
- Pre-configured setup + Pre-configured automation - Start fast with curated presets. Available soon on octobot.cloud and - from the mobile app. + Start configurable strategies from the OctoBot interface. Includes + Baskets, DCA, grid trading and more. - - - - - - - Easy to setup - - + + @@ -65,22 +73,14 @@ export function NewBotCards() { Your own rules - Build with your own rules. Available soon on octobot.cloud and from - the mobile app. + Build with your own rules, with your own logic, indicators and + conditions. Coming soon. - - - - - - - - Easy to medium setup - + @@ -94,29 +94,32 @@ export function NewBotCards() { />
- Custom configuration + Manual configuration - Full control with advanced options. You'll configure everything - after start, including each parameter. + Configure an OctoBot manually with its dedicated interface for + backtesting and advanced control. - - - - - - - - - Advanced setup - - + +
Already have a saved configuration? Restore from a file + {onSkip && ( + + )}
) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx index aad772b86c..6ecfc2e5a8 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx @@ -21,6 +21,7 @@ import { buildAccountEditUserActionJson, buildAutomationCreateUserActionJsonForAccount, buildAutomationCreateUserActionJsonForStrategy, + buildAutomationRestartUserActionJson, buildAutomationSignalUserActionJson, buildAutomationStopUserActionJson, buildExchangeConfigEditUserActionJson, @@ -166,6 +167,12 @@ function DebugTabsPanelComponent({ jsonText: buildAutomationStopUserActionJson(automation.id), }) } + onRestart={(automation) => + onOpenExecuteAction({ + actionType: "automation_restart", + jsonText: buildAutomationRestartUserActionJson(automation.id), + }) + } /> diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugToolbar.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugToolbar.tsx index f5eab48e42..68c975c236 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugToolbar.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugToolbar.tsx @@ -2,11 +2,18 @@ import { Download, Play, RefreshCw, Upload } from "lucide-react" import type { WalletInfo } from "@/client" import { Button } from "@/components/ui/button" -import { truncateAddress } from "@/lib/wallet-utils" +import { Skeleton } from "@/components/ui/skeleton" +import { + DEBUG_WALLET_SELECTOR_LAYOUT_CLASS, + formatWalletSelectOptionLabel, + getDebugWalletSelectorWidthStyle, +} from "@/lib/wallet-utils" +import { cn } from "@/lib/utils" type DebugToolbarProps = { isImportedMode: boolean isSuperuser: boolean + isWalletsLoading?: boolean wallets: WalletInfo[] walletAddress: string onWalletAddressChange: (address: string) => void @@ -14,6 +21,7 @@ type DebugToolbarProps = { onReturnToLive: () => void onExport: () => void canExportSnapshot: boolean + isRefreshPending?: boolean onRefresh: () => void onExecute: () => void } @@ -21,6 +29,7 @@ type DebugToolbarProps = { export function DebugToolbar({ isImportedMode, isSuperuser, + isWalletsLoading = false, wallets, walletAddress, onWalletAddressChange, @@ -28,6 +37,7 @@ export function DebugToolbar({ onReturnToLive, onExport, canExportSnapshot, + isRefreshPending = false, onRefresh, onExecute, }: DebugToolbarProps) { @@ -45,24 +55,36 @@ export function DebugToolbar({ ) } + const walletSelectorWidthStyle = getDebugWalletSelectorWidthStyle() + return ( <> - {isSuperuser && ( - - )} + {isSuperuser && + (isWalletsLoading ? ( + + ) : ( + + ))} - + {importedSummary && ( diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/AutomationTradingCountCell.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/AutomationTradingCountCell.tsx index 64f957b4bc..a48a0068d6 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/AutomationTradingCountCell.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/AutomationTradingCountCell.tsx @@ -1,27 +1,39 @@ +import { useState } from "react" + import { ScrollableTooltipContent } from "@/components/Common/Tables/ScrollableTooltipContent" import { Tooltip, TooltipTrigger } from "@/components/ui/tooltip" type AutomationTradingCountCellProps = { count: number - tooltip: string | null + getTooltip: () => string | null } export function AutomationTradingCountCell({ count, - tooltip, + getTooltip, }: AutomationTradingCountCellProps) { + const [tooltip, setTooltip] = useState(null) + if (count <= 0) { return <>0 } - if (!tooltip) { - return <>{count} - } + return ( - + { + if (isOpen) { + setTooltip(getTooltip()) + } else { + setTooltip(null) + } + }} + > {count} - {tooltip} + {tooltip ? ( + {tooltip} + ) : null} ) } diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/DebugStatusCell.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/DebugStatusCell.tsx index d48e35edc8..3630c3464d 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/DebugStatusCell.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/cells/DebugStatusCell.tsx @@ -4,6 +4,7 @@ import { TooltipTrigger, } from "@/components/ui/tooltip" import { + type DebugStatusDisplay, formatDebugStatusTooltip, getDebugStatusDisplay, } from "@/lib/debug/display-utils" @@ -11,16 +12,18 @@ import { cn } from "@/lib/utils" type DebugStatusCellProps = { status: string | null | undefined + display?: DebugStatusDisplay extraTooltipLines?: string[] pulseWhenRunning?: boolean } export function DebugStatusCell({ status, + display, extraTooltipLines, pulseWhenRunning = false, }: DebugStatusCellProps) { - const { emoji, label } = getDebugStatusDisplay(status) + const { emoji, label } = display ?? getDebugStatusDisplay(status) const tooltip = formatDebugStatusTooltip(status, extraTooltipLines) const isLive = pulseWhenRunning && status?.toLowerCase() === "running" diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/DownloadLogsDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/DownloadLogsDialog.tsx index 5e91847a1e..2de8cb9775 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/DownloadLogsDialog.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/DownloadLogsDialog.tsx @@ -24,13 +24,15 @@ import { ID_DISPLAY_LENGTH, } from "@/lib/debug/constants" import { getDebugStatusDisplay } from "@/lib/debug/display-utils" +import { isRunningAutomation } from "@/lib/debug/automation" import { sortAutomations } from "@/lib/debug/table-automations" import { downloadAutomationLogsArchive, downloadNodeLogsArchive, + downloadRunningAutomationsLatestLogsArchive, } from "@/lib/logs-export" -type LogsTarget = "node" | "automation" +type LogsTarget = "node" | "automation" | "running-latest" type DownloadLogsDialogProps = { open: boolean @@ -61,6 +63,16 @@ export function DownloadLogsDialog({ [automations], ) + const runningAutomations = useMemo( + () => automations.filter(isRunningAutomation), + [automations], + ) + + const runningCountLabel = + runningAutomations.length === 1 + ? "1 running automation" + : `${runningAutomations.length} running automations` + useEffect(() => { if (!open) return setTarget("node") @@ -73,6 +85,10 @@ export function DownloadLogsDialog({ try { if (target === "node") { await downloadNodeLogsArchive() + } else if (target === "running-latest") { + await downloadRunningAutomationsLatestLogsArchive( + runningAutomations.map((automation) => automation.id), + ) } else { const automation = sortedAutomations.find( (row) => row.id === selectedAutomationId, @@ -102,7 +118,8 @@ export function DownloadLogsDialog({ Download logs - Download the node log files or logs for a single automation. + Download node logs, one automation's logs, or the latest active + log file from each running automation. @@ -129,8 +146,26 @@ export function DownloadLogsDialog({ /> Automation logs + + {target === "running-latest" && ( +

+ {runningAutomations.length === 0 + ? "No running automations" + : runningCountLabel} +

+ )} + {target === "automation" && (

@@ -166,7 +201,10 @@ export function DownloadLogsDialog({ void handleDownload()} > Download diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx index 6c9531ef6c..faeaf035b8 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx @@ -247,10 +247,9 @@ export function AccountsTable({ > + getAccountOrdersTooltipContent(row.id, accountTradings) + } /> + getAccountTradesTooltipContent(row.id, accountTradings) + } /> diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx index 0b9a58d32d..74b04438ac 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AutomationsTable.tsx @@ -1,4 +1,4 @@ -import { Eye, X, Zap } from "lucide-react" +import { Eye, RotateCcw, X, Zap } from "lucide-react" import { useMemo, useState } from "react" import type { AccountTradingWithAccountId, AutomationState } from "@/client" @@ -30,8 +30,10 @@ import { } from "@/components/ui/tooltip" import { formatActionProgress, + getAutomationDebugStatusDisplay, getAutomationErrorTooltipLines, getAutomationUpdatedAt, + isRestartableAutomation, isRunningAutomation, } from "@/lib/debug/automation" import { @@ -64,6 +66,7 @@ type AutomationsTableProps = { accountTradings: AccountTradingWithAccountId[] onSuccess?: () => void onStop?: (automation: AutomationState) => void + onRestart?: (automation: AutomationState) => void onSignal?: (automation: AutomationState) => void readOnly?: boolean selectionMode?: boolean @@ -78,6 +81,7 @@ export function AutomationsTable({ accountTradings, onSuccess, onStop, + onRestart, onSignal, readOnly = false, selectionMode = false, @@ -240,9 +244,9 @@ export function AutomationsTable({ const canSignal = readOnly ? Boolean(onSignal) : isRunningAutomation(row) - const canStop = readOnly - ? Boolean(onStop) - : isRunningAutomation(row) + const canStop = Boolean(onStop) && isRunningAutomation(row) + const canRestart = + Boolean(onRestart) && isRestartableAutomation(row) const signalButton = ( - ) : ( - - - - - - - - {readOnly - ? "Stop action unavailable" - : "Only running automations can be stopped"} - - - )} + ) : null} + {canRestart ? ( + + ) : null}

diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/StrategiesTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/StrategiesTable.tsx index c95866b4b6..3aa1350bfd 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/StrategiesTable.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/StrategiesTable.tsx @@ -3,6 +3,7 @@ import { useMemo, useState } from "react" import type { Strategy } from "@/client" import { CenteredCellContent } from "@/components/Common/Tables/CenteredCellContent" +import { TruncatedTextWithTooltip } from "@/components/Common/Tables/TruncatedTextWithTooltip" import { ClearTableFiltersButton } from "@/components/Common/Tables/ClearTableFiltersButton" import { ColumnFilterInput } from "@/components/Common/Tables/ColumnFilterInput" import { CopyableIdCell } from "@/components/Common/Tables/CopyableIdCell" @@ -16,6 +17,11 @@ import { TableHeader, TableRow, } from "@/components/ui/table" +import { + STRATEGY_DESCRIPTION_DISPLAY_LENGTH, + STRATEGY_NAME_DISPLAY_LENGTH, + STRATEGY_TABLE_DEFAULT_SORT, +} from "@/lib/debug/constants" import { debugTableCellClass, getStrategyConfigurationType, @@ -38,10 +44,9 @@ export function StrategiesTable({ onStartAutomation, }: StrategiesTableProps) { const [detail, setDetail] = useState(null) - const [sort, setSort] = useState>({ - key: "id", - dir: "asc", - }) + const [sort, setSort] = useState>( + STRATEGY_TABLE_DEFAULT_SORT, + ) const [filters, setFilters] = useState>({}) const displayRows = useMemo( @@ -109,7 +114,7 @@ export function StrategiesTable({ onSort={(key) => setSort((current) => toggleSort(current, key))} /> setSort((current) => toggleSort(current, key))} @@ -157,7 +162,10 @@ export function StrategiesTable({ - {row.name ?? "—"} + - {row.description ?? "—"} + {row.reference_market} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/DslKeywords/DslKeywordsTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/DslKeywords/DslKeywordsTable.tsx new file mode 100644 index 0000000000..e83f3c575c --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/DslKeywords/DslKeywordsTable.tsx @@ -0,0 +1,160 @@ +import { useMemo, useState } from "react" + +import type { DslKeyword } from "@/client" +import { TruncatedTextWithTooltip } from "@/components/Common/Tables/TruncatedTextWithTooltip" +import { SortableTableHead } from "@/components/Common/Tables/SortableTableHead" +import { JsonDetailDialog } from "@/components/Debug/dialogs/JsonDetailDialog" +import { Input } from "@/components/ui/input" +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table" +import { + DSL_KEYWORD_CATEGORY_DISPLAY_LENGTH, + DSL_KEYWORD_DESCRIPTION_DISPLAY_LENGTH, + DSL_KEYWORD_LABEL_DISPLAY_LENGTH, + DSL_KEYWORD_PARAMS_DISPLAY_LENGTH, + DSL_KEYWORD_TABLE_DEFAULT_SORT, + type DslKeywordSortKey, +} from "@/lib/dsl-keywords/constants" +import { formatDslParametersList } from "@/lib/dsl-keywords/format" +import { + filterKeywordsBySearch, + sortKeywords, +} from "@/lib/dsl-keywords/table-keywords" +import { debugTableCellClass } from "@/lib/debug/display-utils" +import { toggleSort } from "@/lib/table" +import type { SortState } from "@/lib/table-types" + +type DslKeywordsTableProps = { + rows: DslKeyword[] +} + +export function DslKeywordsTable({ rows }: DslKeywordsTableProps) { + const [detail, setDetail] = useState(null) + const [search, setSearch] = useState("") + const [sort, setSort] = useState>( + DSL_KEYWORD_TABLE_DEFAULT_SORT, + ) + + const displayRows = useMemo( + () => sortKeywords(filterKeywordsBySearch(rows, search), sort), + [rows, search, sort], + ) + + const columnCount = 5 + + if (rows.length === 0) { + return ( +

+ No DSL keywords. +

+ ) + } + + return ( + <> +
+ setSearch(event.target.value)} + placeholder="Search name, label, description, category…" + aria-label="Search DSL keywords" + /> + + {displayRows.length} keywords + +
+ + + + setSort((current) => toggleSort(current, key))} + /> + setSort((current) => toggleSort(current, key))} + /> + setSort((current) => toggleSort(current, key))} + /> + Inputs + Outputs + + + + {displayRows.length === 0 ? ( + + + No keywords match search. + + + ) : ( + displayRows.map((row) => ( + setDetail(row)} + > + + + + + + + + + + + + + + + + + )) + )} + +
+ { + if (!open) setDetail(null) + }} + /> + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/DslKeywords/DslKeywordsView.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/DslKeywords/DslKeywordsView.tsx new file mode 100644 index 0000000000..f4a6c11b74 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/DslKeywords/DslKeywordsView.tsx @@ -0,0 +1,76 @@ +import { useQuery } from "@tanstack/react-query" +import { BookOpen } from "lucide-react" +import { useEffect } from "react" + +import { type ApiError, DslService } from "@/client" +import { DslKeywordsTable } from "@/components/DslKeywords/DslKeywordsTable" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" +import useCustomToast from "@/hooks/useCustomToast" +import { handleError } from "@/utils" + +export function DslKeywordsView() { + const { showErrorToast } = useCustomToast() + + const keywordsQuery = useQuery({ + queryKey: ["dsl-keywords"], + queryFn: () => DslService.getDslKeywords(), + }) + + useEffect(() => { + if (keywordsQuery.isError && keywordsQuery.error) { + handleError.bind(showErrorToast)(keywordsQuery.error as ApiError) + } + }, [keywordsQuery.isError, keywordsQuery.error, showErrorToast]) + + const keywords = keywordsQuery.data?.keywords ?? [] + const version = keywordsQuery.data?.version + + return ( +
+
+ +
+

DSL keywords

+

+ Keywords available on this node + {version ? ( + <> + {" "} + · catalog version {version} + + ) : null} +

+
+
+ + + + Catalog + + Search and sort the full DSL operator catalog. Open a row for the + complete keyword JSON. + + + + {keywordsQuery.isPending ? ( +

+ Loading keywords… +

+ ) : keywordsQuery.isError ? ( +

+ Could not load DSL keywords. +

+ ) : ( + + )} + + +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/BotCard.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/BotCard.tsx index c6bdaaec58..e5a1090856 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/BotCard.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/BotCard.tsx @@ -1,8 +1,13 @@ -import { Check, Clock, Layers, Lock, TriangleAlert } from "lucide-react" +import { ArrowUpRight, Check, Clock, Layers, Lock, TriangleAlert } from "lucide-react" import { memo } from "react" -import type { Task_Output as Task, TaskStatus } from "@/client" +import type { + ChildOctoBotProcessState, + Task_Output as Task, + TaskStatus, +} from "@/client" import { Badge } from "@/components/ui/badge" +import { Button } from "@/components/ui/button" import { Card, CardContent, CardHeader } from "@/components/ui/card" import { Tooltip, @@ -15,6 +20,7 @@ import { getStatusGroup, hasStartedExecution, } from "@/utils/executions" +import { formatProcessAddress } from "@/utils/process-address" import { resolveTaskError, type TaskErrorInfo } from "@/utils/task-errors" import { formatDate, @@ -83,6 +89,85 @@ function ErrorPanel({ status, message }: TaskErrorInfo) { ) } +function ProcessAddressRow({ + childProcess, +}: { + childProcess: ChildOctoBotProcessState +}) { + const processAddress = formatProcessAddress(childProcess) + + return ( + + ) +} + +function RunsCounterRow({ + task, + isRunning, +}: { + task: Task + isRunning: boolean +}) { + const activeExec = getActiveExecution(task.executions) + const runCount = task.executions?.length ?? 0 + const completedSteps = + task.executions?.filter( + (execution) => + execution.status === "completed" || execution.status === "failed", + ).length ?? 0 + const runningExec = task.executions?.find( + (execution) => execution.status === "running", + ) + const elapsedFrom = runningExec?.scheduled_at ?? activeExec?.scheduled_at + + return ( +
+ {runCount > 0 && ( + + + {runCount} run{runCount !== 1 ? "s" : ""} + + )} + {completedSteps > 0 && ( + + {completedSteps} done + + )} + {isRunning && elapsedFrom && ( + + Running {formatElapsed(elapsedFrom)} + + )} +
+ ) +} + function BotCardBody({ task, isRunning, @@ -97,12 +182,8 @@ function BotCardBody({ const activeExec = getActiveExecution(task.executions) const group = getStatusGroup(activeExec?.status) const date = getDisplayDate(task) - const runCount = task.executions?.length ?? 0 - const completedSteps = - task.executions?.filter( - (e) => e.status === "completed" || e.status === "failed", - ).length ?? 0 const actionCount = parseActionCount(activeExec?.actions) + const childProcess = task.metadata?.child_octobot_process ?? null const errorPanel = errorInfo && (errorInfo.status || errorInfo.message) ? ( @@ -111,29 +192,14 @@ function BotCardBody({ if (group === "active") { if (isRunning) { - const runningExec = task.executions?.find((e) => e.status === "running") - const elapsedFrom = runningExec?.scheduled_at ?? activeExec?.scheduled_at return ( {errorPanel} -
- {runCount > 0 && ( - - - {runCount} run{runCount !== 1 ? "s" : ""} - - )} - {completedSteps > 0 && ( - - {completedSteps} done - - )} - {elapsedFrom && ( - - Running {formatElapsed(elapsedFrom)} - - )} -
+ {childProcess ? ( + + ) : ( + + )} ) @@ -142,13 +208,17 @@ function BotCardBody({ return ( {errorPanel} -
- {actionCount != null && ( - - {actionCount} action{actionCount !== 1 ? "s" : ""} queued - - )} -
+ {childProcess ? ( + + ) : ( +
+ {actionCount != null && ( + + {actionCount} action{actionCount !== 1 ? "s" : ""} queued + + )} +
+ )} {isScheduled && activeExec?.scheduled_at ? ( @@ -173,15 +243,21 @@ function BotCardBody({ return ( {errorPanel} -
- {runCount > 0 && ( - - - {runCount} run{runCount !== 1 ? "s" : ""} - - )} - -
+ {childProcess ? ( + + ) : ( +
+ {(task.executions?.length ?? 0) > 0 && ( + + + {task.executions?.length} run + {(task.executions?.length ?? 0) !== 1 ? "s" : ""} + + )} + +
+ )} + {childProcess && } ) } @@ -198,7 +274,9 @@ function areTaskPropsEqual( prev.task.error_message === next.task.error_message && prev.task.executions?.length === next.task.executions?.length && JSON.stringify(prev.task.executions) === - JSON.stringify(next.task.executions) + JSON.stringify(next.task.executions) && + JSON.stringify(prev.task.metadata?.child_octobot_process) === + JSON.stringify(next.task.metadata?.child_octobot_process) ) } @@ -245,7 +323,6 @@ export const BotCard = memo(function BotCard({ const isRunning = displayLabel === "Running" const isScheduled = displayLabel === "Scheduled" - return (
- ID: {task.id?.slice(0, 12) || "—"} + ID: {task.id?.slice(0, 12) || "-"}
diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/SelectionToolbar.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/SelectionToolbar.tsx index 61536be567..dfe14b8dd1 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/SelectionToolbar.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/SelectionToolbar.tsx @@ -1,10 +1,11 @@ -import { useMutation, useQueryClient } from "@tanstack/react-query" +import { useMutation, useQueryClient } from "@tanstack/react-query" import { Link, useNavigate } from "@tanstack/react-router" -import { Ban, ScrollText, Trash2 } from "lucide-react" +import { Ban, ScrollText, Square, Trash2 } from "lucide-react" import { useMemo, useState } from "react" import type { Task_Output as Task } from "@/client" import { TasksService } from "@/client" +import { StopAutomationDialog } from "@/components/OctoBots/StopAutomationDialog" import { Button } from "@/components/ui/button" import { Dialog, @@ -17,6 +18,7 @@ import { } from "@/components/ui/dialog" import { LoadingButton } from "@/components/ui/loading-button" import useCustomToast from "@/hooks/useCustomToast" +import { canStopOctoBot } from "@/lib/octobots/stop-automation" import { shareWorkflowLogs } from "@/lib/support-share" import { getTaskFilterGroup } from "@/utils/task-status" @@ -52,6 +54,8 @@ export function SelectionToolbar({ onDeleted: () => void }) { const [deleteOpen, setDeleteOpen] = useState(false) + const [cancelOpen, setCancelOpen] = useState(false) + const [stopOpen, setStopOpen] = useState(false) const [exportLoading, setExportLoading] = useState(false) const [logsModal, setLogsModal] = useState< null | "none" | "pending" | "disabled" @@ -63,17 +67,26 @@ export function SelectionToolbar({ const activeTasks = useMemo( () => allTasks.filter( - (t) => - t.id && selectedIds.has(t.id) && getTaskFilterGroup(t) === "active", + (task) => + task.id && + selectedIds.has(task.id) && + getTaskFilterGroup(task) === "active", ), [allTasks, selectedIds], ) + const stoppableActiveTasks = useMemo( + () => activeTasks.filter(canStopOctoBot), + [activeTasks], + ) + const inactiveTasks = useMemo( () => allTasks.filter( - (t) => - t.id && selectedIds.has(t.id) && getTaskFilterGroup(t) !== "active", + (task) => + task.id && + selectedIds.has(task.id) && + getTaskFilterGroup(task) !== "active", ), [allTasks, selectedIds], ) @@ -81,7 +94,7 @@ export function SelectionToolbar({ const deleteMutation = useMutation({ mutationFn: () => TasksService.deleteTasks({ - taskIds: inactiveTasks.map((t) => t.id as string), + taskIds: inactiveTasks.map((task) => task.id as string), }), onSuccess: () => { showSuccessToast( @@ -99,12 +112,13 @@ export function SelectionToolbar({ const cancelMutation = useMutation({ mutationFn: () => TasksService.cancelTasks({ - requestBody: { task_ids: activeTasks.map((t) => t.id as string) }, + requestBody: { task_ids: activeTasks.map((task) => task.id as string) }, }), onSuccess: () => { showSuccessToast( `Cancelled ${activeTasks.length} OctoBot${activeTasks.length !== 1 ? "s" : ""}`, ) + setCancelOpen(false) queryClient.invalidateQueries({ queryKey: ["tasks"] }) }, onError: () => { @@ -136,14 +150,14 @@ export function SelectionToolbar({ } setExportLoading(true) const taskIds = exportableTasks - .map((t) => t.id) + .map((task) => task.id) .filter(Boolean) .join(",") navigate({ to: "/octobots/export", search: { tasks: taskIds } }) } const allFilteredSelected = filteredTasks.every( - (t) => t.id && selectedIds.has(t.id), + (task) => task.id && selectedIds.has(task.id), ) return ( @@ -179,15 +193,24 @@ export function SelectionToolbar({ Share logs {activeTasks.length > 0 && ( - cancelMutation.mutate()} + onClick={() => setCancelOpen(true)} > Cancel - + + )} + {stoppableActiveTasks.length > 0 && ( + )} {inactiveTasks.length > 0 && ( + + cancelMutation.mutate()} + > + Cancel + + + + + + + diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/StopAutomationDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/StopAutomationDialog.tsx new file mode 100644 index 0000000000..a0ce2007ca --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/OctoBots/StopAutomationDialog.tsx @@ -0,0 +1,167 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query" +import { TriangleAlert } from "lucide-react" +import { useEffect, useMemo, useState } from "react" + +import type { Task_Output as Task } from "@/client" +import { DebugService } from "@/client" +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { LoadingButton } from "@/components/ui/loading-button" +import useCustomToast from "@/hooks/useCustomToast" +import { + buildStopAutomationUserAction, + formatStopAutomationError, + getOctoBotDisplayName, +} from "@/lib/octobots/stop-automation" + +type StopAutomationDialogProps = { + open: boolean + onOpenChange: (open: boolean) => void + tasks: Task[] + onSuccess?: () => void +} + +function buildStopDescription(tasks: Task[]): string { + if (tasks.length === 1) { + const displayName = getOctoBotDisplayName(tasks[0]) + return `Stop ${displayName}? This will stop the automation and end its current run.` + } + const previewNames = tasks + .slice(0, 3) + .map((task) => getOctoBotDisplayName(task)) + .join(", ") + const suffix = + tasks.length > 3 ? ` and ${tasks.length - 3} more` : "" + return `Stop ${tasks.length} OctoBots (${previewNames}${suffix})? This will stop each automation and end its current run.` +} + +async function stopAutomationsSequentially( + automationIds: string[], +): Promise<{ failed: { automationId: string; message: string }[] }> { + const failed: { automationId: string; message: string }[] = [] + for (const automationId of automationIds) { + try { + await DebugService.executeUserAction({ + requestBody: buildStopAutomationUserAction(automationId), + walletAddress: null, + }) + } catch (error) { + failed.push({ + automationId, + message: formatStopAutomationError(error), + }) + } + } + return { failed } +} + +function formatBulkStopError( + totalCount: number, + failed: { automationId: string; message: string }[], +): string { + if (failed.length === 0) return "" + const header = `${failed.length} of ${totalCount} stop request${totalCount !== 1 ? "s" : ""} failed.` + const details = failed + .map((entry) => `${entry.automationId.slice(0, 8)}: ${entry.message}`) + .join("\n") + return `${header}\n${details}` +} + +export function StopAutomationDialog({ + open, + onOpenChange, + tasks, + onSuccess, +}: StopAutomationDialogProps) { + const queryClient = useQueryClient() + const { showSuccessToast } = useCustomToast() + const [submitError, setSubmitError] = useState(null) + + const stoppableTasks = useMemo( + () => tasks.filter((task): task is Task & { id: string } => Boolean(task.id)), + [tasks], + ) + + useEffect(() => { + if (open) { + setSubmitError(null) + } + }, [open, stoppableTasks.map((task) => task.id).join(",")]) + + const stopMutation = useMutation({ + mutationFn: (automationIds: string[]) => + stopAutomationsSequentially(automationIds), + onSuccess: (result, automationIds) => { + if (result.failed.length > 0) { + setSubmitError(formatBulkStopError(automationIds.length, result.failed)) + queryClient.invalidateQueries({ queryKey: ["tasks"] }) + return + } + const count = automationIds.length + showSuccessToast( + count === 1 + ? "OctoBot stop requested" + : `Stop requested for ${count} OctoBots`, + ) + queryClient.invalidateQueries({ queryKey: ["tasks"] }) + onOpenChange(false) + onSuccess?.() + }, + onError: (error) => { + setSubmitError(formatStopAutomationError(error)) + }, + }) + + const handleConfirm = () => { + if (stoppableTasks.length === 0 || stopMutation.isPending) return + setSubmitError(null) + stopMutation.mutate(stoppableTasks.map((task) => task.id)) + } + + const title = + stoppableTasks.length === 1 ? "Stop OctoBot" : `Stop ${stoppableTasks.length} OctoBots` + + return ( + + + + {title} + + {buildStopDescription(stoppableTasks)} + + + {submitError && ( +
+ +

+ {submitError} +

+
+ )} + + + + + + Stop + + + + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Settings/CardCornerButton.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Settings/CardCornerButton.tsx new file mode 100644 index 0000000000..2dc2f11b82 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Settings/CardCornerButton.tsx @@ -0,0 +1,40 @@ +import type { LucideIcon } from "lucide-react" +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip" + +export function CardCornerButton({ + icon: Icon, + label, + onClick, + variant = "default", +}: { + icon: LucideIcon + label: string + onClick: () => void + variant?: "default" | "destructive" +}) { + return ( +
+ + + + + {label} + +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Settings/ClientEncryptionKeysCard.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Settings/ClientEncryptionKeysCard.tsx new file mode 100644 index 0000000000..3ae9f964f7 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Settings/ClientEncryptionKeysCard.tsx @@ -0,0 +1,279 @@ +import { Check, KeyRound, ShieldCheck, TriangleAlert, X } from "lucide-react" +import { useEffect, useRef, useState } from "react" +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip" +import type { ClientKeys } from "@/lib/client-encryption" +import { + areClientKeysConfigured, + CLIENT_KEY_LABELS, + CLIENT_KEY_NAMES, + emptyKeys, +} from "@/lib/client-encryption" +import { + clearClientKeys, + hasStoredClientKeys, + loadClientKeys, + saveClientKeys, +} from "@/lib/device-key" +import { fetchNodeConfig } from "@/lib/node-config" + +function StatusIndicator({ enabled }: { enabled: boolean | null }) { + if (enabled === null) return null + return ( + + + {enabled ? ( + + + + ) : ( + + + + )} + + + {enabled ? "Enabled" : "Disabled"} + + + ) +} + +export function ClientEncryptionKeysCard() { + const [keys, setKeys] = useState(emptyKeys) + const [status, setStatus] = useState<"loading" | "ready" | "saved" | "error">( + "loading", + ) + const [hasStored, setHasStored] = useState(false) + const [editing, setEditing] = useState(false) + const [error, setError] = useState("") + const timerRef = useRef | null>(null) + const configured = areClientKeysConfigured(keys) + const [serverEnabled, setServerEnabled] = useState(null) + const [serverEnvVars, setServerEnvVars] = useState([]) + + useEffect(() => { + void (async () => { + try { + const data = await fetchNodeConfig() + setServerEnabled(data.tasks_encryption_enabled ?? false) + setServerEnvVars(data.server_encryption_env_vars ?? []) + } catch { + setServerEnabled(false) + } + })() + }, []) + + useEffect(() => { + ;(async () => { + const stored = await hasStoredClientKeys() + setHasStored(stored) + if (!stored) { + setStatus("ready") + return + } + try { + const loaded = await loadClientKeys() + if (loaded) setKeys(loaded as ClientKeys) + setStatus("ready") + } catch { + setStatus("error") + setError("Failed to decrypt stored keys.") + } + })() + return () => { + if (timerRef.current) clearTimeout(timerRef.current) + } + }, []) + + const handleSave = async () => { + try { + await saveClientKeys(keys) + setHasStored(true) + setStatus("saved") + setEditing(false) + setError("") + if (timerRef.current) clearTimeout(timerRef.current) + timerRef.current = setTimeout(() => setStatus("ready"), 2000) + } catch (e) { + setStatus("error") + setError(e instanceof Error ? e.message : "Encryption failed") + } + } + + const handleClear = async () => { + await clearClientKeys() + setHasStored(false) + setKeys(emptyKeys()) + setStatus("ready") + setError("") + } + + return ( + +
+ +
+ + + + Encryption keys + + + Server-side and browser-stored client keys for end-to-end task + encryption. + + + +
+ + Server keys + + {serverEnabled === null ? ( +

Loading…

+ ) : serverEnabled ? ( + + All server encryption keys + are configured. + + ) : ( +
+ + Set these environment variables to enable: + +
    + {serverEnvVars.map((v) => ( +
  • {v}
    • + ))} +
+
+ )} +
+
+
+ + Client keys + + {status === "error" ? ( +
+
+ + {error} +
+ +
+ ) : status === "loading" ? ( +

Decrypting…

+ ) : hasStored && !editing ? ( + <> +
+ {CLIENT_KEY_NAMES.map((k) => ( +
+ + {CLIENT_KEY_LABELS[k]} + +
+ {"•".repeat(24)} +
+
+ ))} +
+
+ {status === "saved" ? ( + + Saved + + ) : ( + + )} + +
+ + ) : ( + <> +
+ {CLIENT_KEY_NAMES.map((k) => ( +
+ +