From 2eca355e7a12a5e6c828428e974795728607f410 Mon Sep 17 00:00:00 2001 From: FerTV Date: Wed, 2 Jul 2025 13:09:36 +0200 Subject: [PATCH 01/26] postgres docker created --- nebula/controller/controller.py | 3 +++ nebula/controller/database.py | 1 + nebula/controller/scenarios.py | 21 +++++++++++++++++++++ nebula/frontend/app.py | 2 ++ 4 files changed, 27 insertions(+) diff --git a/nebula/controller/controller.py b/nebula/controller/controller.py index 7e59ae7ec..3c0fe1d22 100755 --- a/nebula/controller/controller.py +++ b/nebula/controller/controller.py @@ -326,10 +326,13 @@ async def run_scenario( # Manager for the actual scenario scenarioManagement = ScenarioManagement(scenario_data, user) + logging.info(f"[FER] scenario run_scenario {scenario_data}") + await update_scenario( scenario_name=scenarioManagement.scenario_name, start_time=scenarioManagement.start_date_scenario, end_time="", + scenario=db_scenario, scenario=scenario_data, status="running", role=role, diff --git a/nebula/controller/database.py b/nebula/controller/database.py index 407ce3908..a5c0632e8 100755 --- a/nebula/controller/database.py +++ b/nebula/controller/database.py @@ -82,6 +82,7 @@ async def list_users(all_info=False): result = await conn.fetch("SELECT * FROM users") if not all_info: + # In PostgreSQL, you can access columns by key from DictCursor result = [user["user"] for user in result] return result diff --git a/nebula/controller/scenarios.py b/nebula/controller/scenarios.py index 0d8f2286e..dd6d5c770 100644 --- a/nebula/controller/scenarios.py +++ b/nebula/controller/scenarios.py @@ -548,6 +548,26 @@ def from_dict(cls, data): scenario = cls(**scenario_data) return scenario + + @staticmethod + def to_json(scenario_obj): + """ + Converts a Scenario object to a JSON string. + + Args: + scenario_obj (Scenario): An instance of the Scenario class. + + Returns: + str: A JSON string representation of the Scenario object. + """ + if not isinstance(scenario_obj, Scenario): + raise TypeError("Input must be an instance of the Scenario class.") + + # Get all attributes of the Scenario object + scenario_dict = scenario_obj.__dict__ + + # Convert the dictionary to a JSON string + return json.dumps(scenario_dict, indent=2) # Using indent for pretty-printing @staticmethod def to_json(scenario_obj): @@ -592,6 +612,7 @@ class ScenarioManagement: def __init__(self, scenario, user=None): # Current scenario self.scenario = Scenario.from_dict(scenario) + logging.info(f"[FER] scenario from scenarios.py {Scenario.to_json(self.scenario)}") # Uid of the user self.user = user # Scenario management settings diff --git a/nebula/frontend/app.py b/nebula/frontend/app.py index 42f5b3a68..185a87139 100755 --- a/nebula/frontend/app.py +++ b/nebula/frontend/app.py @@ -580,6 +580,7 @@ async def deploy_scenario(scenario_data, role, user): HTTPException: If the underlying HTTP POST request fails. """ url = f"http://{settings.controller_host}:{settings.controller_port}/scenarios/run" + logging.info(f"[FER] scenario {scenario_data}") data = {"scenario_data": scenario_data, "role": role, "user": user} return await controller_post(url, data) @@ -1530,6 +1531,7 @@ async def nebula_dashboard(request: Request, session: dict = Depends(get_session scenario_running = None bool_completed = False + logging.info(f"[FER] scenarios {scenarios} scenario_running {scenario_running}") if scenario_running: bool_completed = scenario_running["status"] == "completed" if scenarios: From a2f290b876b0e4ea60523fb63732690b42b74342 Mon Sep 17 00:00:00 2001 From: FerTV Date: Mon, 7 Jul 2025 17:12:07 +0200 Subject: [PATCH 02/26] fix postgres db endpoints --- nebula/controller/controller.py | 3 --- nebula/controller/database.py | 6 ++++-- nebula/controller/scenarios.py | 1 - nebula/frontend/app.py | 2 -- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/nebula/controller/controller.py b/nebula/controller/controller.py index 3c0fe1d22..7e59ae7ec 100755 --- a/nebula/controller/controller.py +++ b/nebula/controller/controller.py @@ -326,13 +326,10 @@ async def run_scenario( # Manager for the actual scenario scenarioManagement = ScenarioManagement(scenario_data, user) - logging.info(f"[FER] scenario run_scenario {scenario_data}") - await update_scenario( scenario_name=scenarioManagement.scenario_name, start_time=scenarioManagement.start_date_scenario, end_time="", - scenario=db_scenario, scenario=scenario_data, status="running", role=role, diff --git a/nebula/controller/database.py b/nebula/controller/database.py index a5c0632e8..ed4ead816 100755 --- a/nebula/controller/database.py +++ b/nebula/controller/database.py @@ -82,7 +82,6 @@ async def list_users(all_info=False): result = await conn.fetch("SELECT * FROM users") if not all_info: - # In PostgreSQL, you can access columns by key from DictCursor result = [user["user"] for user in result] return result @@ -195,6 +194,9 @@ async def list_nodes_by_scenario_name(scenario_name): except Exception as e: logging.error(f"Error occurred while listing nodes by scenario name: {e}") return None + finally: + if conn: + await conn.close() async def update_node_record( @@ -322,7 +324,7 @@ async def get_all_scenarios_and_check_completed(username, role, sort_by="start_t if sort_by not in allowed_sort_fields: sort_by = "start_time" # Safe default value - # Building the ORDER BY clause + # Building the ORDER BY clause (same as get_all_scenarios) if sort_by == "start_time": order_by_clause = """ ORDER BY diff --git a/nebula/controller/scenarios.py b/nebula/controller/scenarios.py index dd6d5c770..65ea731af 100644 --- a/nebula/controller/scenarios.py +++ b/nebula/controller/scenarios.py @@ -612,7 +612,6 @@ class ScenarioManagement: def __init__(self, scenario, user=None): # Current scenario self.scenario = Scenario.from_dict(scenario) - logging.info(f"[FER] scenario from scenarios.py {Scenario.to_json(self.scenario)}") # Uid of the user self.user = user # Scenario management settings diff --git a/nebula/frontend/app.py b/nebula/frontend/app.py index 185a87139..42f5b3a68 100755 --- a/nebula/frontend/app.py +++ b/nebula/frontend/app.py @@ -580,7 +580,6 @@ async def deploy_scenario(scenario_data, role, user): HTTPException: If the underlying HTTP POST request fails. """ url = f"http://{settings.controller_host}:{settings.controller_port}/scenarios/run" - logging.info(f"[FER] scenario {scenario_data}") data = {"scenario_data": scenario_data, "role": role, "user": user} return await controller_post(url, data) @@ -1531,7 +1530,6 @@ async def nebula_dashboard(request: Request, session: dict = Depends(get_session scenario_running = None bool_completed = False - logging.info(f"[FER] scenarios {scenarios} scenario_running {scenario_running}") if scenario_running: bool_completed = scenario_running["status"] == "completed" if scenarios: From a5d0a0f1c5e23a09cdfd27dac2513315817f0b29 Mon Sep 17 00:00:00 2001 From: FerTV Date: Mon, 7 Jul 2025 17:12:59 +0200 Subject: [PATCH 03/26] redis docker added --- nebula/controller/scenarios.py | 2 +- nebula/database/redis/Dockerfile | 1 + nebula/database/rediscommander/Dockerfile | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 nebula/database/redis/Dockerfile create mode 100644 nebula/database/rediscommander/Dockerfile diff --git a/nebula/controller/scenarios.py b/nebula/controller/scenarios.py index 65ea731af..20783250a 100644 --- a/nebula/controller/scenarios.py +++ b/nebula/controller/scenarios.py @@ -548,7 +548,7 @@ def from_dict(cls, data): scenario = cls(**scenario_data) return scenario - + @staticmethod def to_json(scenario_obj): """ diff --git a/nebula/database/redis/Dockerfile b/nebula/database/redis/Dockerfile new file mode 100644 index 000000000..31ff6af02 --- /dev/null +++ b/nebula/database/redis/Dockerfile @@ -0,0 +1 @@ +FROM redis:latest \ No newline at end of file diff --git a/nebula/database/rediscommander/Dockerfile b/nebula/database/rediscommander/Dockerfile new file mode 100644 index 000000000..0eb1e1ead --- /dev/null +++ b/nebula/database/rediscommander/Dockerfile @@ -0,0 +1 @@ +FROM rediscommander/redis-commander:latest \ No newline at end of file From acc11b121465e1af3eab542a92c3b9667f2f82ff Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 10 Jul 2025 13:19:30 +0200 Subject: [PATCH 04/26] feature federation controller class en API endpoints --- nebula/controller/federation/__init__.py | 0 .../docker_federation_controller.py | 4 + .../federation/federation_controller.py | 122 ++++++++++++++++++ .../physicall_federation_controller.py | 4 + .../processes_federation_controller.py | 4 + nebula/utils.py | 61 +++++++++ 6 files changed, 195 insertions(+) create mode 100755 nebula/controller/federation/__init__.py create mode 100644 nebula/controller/federation/docker_federation_controller.py create mode 100644 nebula/controller/federation/federation_controller.py create mode 100644 nebula/controller/federation/physicall_federation_controller.py create mode 100644 nebula/controller/federation/processes_federation_controller.py diff --git a/nebula/controller/federation/__init__.py b/nebula/controller/federation/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/nebula/controller/federation/docker_federation_controller.py b/nebula/controller/federation/docker_federation_controller.py new file mode 100644 index 000000000..499ead1cb --- /dev/null +++ b/nebula/controller/federation/docker_federation_controller.py @@ -0,0 +1,4 @@ +from nebula.controller.federation.federation_controller import FederationController + +class DockerFederationController(FederationController): + pass \ No newline at end of file diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py new file mode 100644 index 000000000..5bb24ab94 --- /dev/null +++ b/nebula/controller/federation/federation_controller.py @@ -0,0 +1,122 @@ +import os +import logging +from abc import ABC, abstractmethod +from fastapi import FastAPI, Body, Path, Request +from fastapi.concurrency import asynccontextmanager +from typing import Dict +from typing import Annotated +from functools import wraps +from fastapi import HTTPException +from nebula.utils import LoggerUtils + + +class FederationController(ABC): + + def __init__(self, logger): + self._logger = logger + + @abstractmethod + async def run_scenario(self, scenario_data: Dict, role: str, user: str): + pass + + @abstractmethod + async def stop_scenario(self, scenario_name: str, username: str, all: bool): + pass + + @abstractmethod + async def remove_scenario(self, scenario_name: str): + pass + + @abstractmethod + async def update_nodes(self, scenario_name: str, request: Request): + pass + +def federation_controller_factory(mode: str, logger) -> FederationController: + from nebula.controller.federation.docker_federation_controller import DockerFederationController + from nebula.controller.federation.processes_federation_controller import ProcessesFederationController + from nebula.controller.federation.physicall_federation_controller import PhysicalFederationController + + if mode == "docker": + return DockerFederationController(logger) + elif mode == "physical": + return PhysicalFederationController(logger) + elif mode == "processes": + return ProcessesFederationController(logger) + else: + raise ValueError("Unknown federation mode") + +def require_initialized_controller(func): + @wraps(func) + async def wrapper(*args, **kwargs): + if fed_controller is None: + raise HTTPException(status_code=400, detail="FederationController not initialized") + return await func(*args, **kwargs) + return wrapper + +@asynccontextmanager +async def lifespan(app: FastAPI): + log_path = os.path.join("app", "logs", "federation.log") + + # Configure and register the logger under the name "controller" + LoggerUtils.configure_logger(name="Federation-Controller", log_file=log_path) + + # Retrieve the logger by name + logger = logging.getLogger("Federation-Controller") + logger.info("Logger initialized for FederationController") + + yield + +app = FastAPI(lifespan=lifespan) +fed_controller: FederationController = None + +@app.post("/init") +async def init_federation_experiment(payload: dict = Body(...)): + global fed_controller + + experiment_type = payload["type"] + logger = logging.getLogger("Federation-Controller") + fed_controller = federation_controller_factory(experiment_type, logger) + + return {"message": f"{experiment_type} controller initialized"} + +@app.post("/scenarios/run") +@require_initialized_controller +async def run_scenario( + scenario_data: dict = Body(..., embed=True), + role: str = Body(..., embed=True), + user: str = Body(..., embed=True), +): + global fed_controller + return await fed_controller.run_scenario(scenario_data, role, user) + +@app.post("/scenarios/stop") +@require_initialized_controller +async def stop_scenario( + scenario_name: str = Body(..., embed=True), + username: str = Body(..., embed=True), + all: bool = Body(False, embed=True), +): + global fed_controller + return await fed_controller.stop_scenario(scenario_name, username, all) + +@app.post("/scenarios/remove") +@require_initialized_controller +async def remove_scenario( + scenario_name: str = Body(..., embed=True), +): + global fed_controller + return await fed_controller.remove_scenario(scenario_name) + +@app.post("/nodes/{scenario_name}/update") +@require_initialized_controller +async def update_nodes( + scenario_name: Annotated[ + str, + Path(regex="^[a-zA-Z0-9_-]+$", min_length=1, max_length=50, description="Valid scenario name"), + ], + request: Request, +): + global fed_controller + return await fed_controller.update_nodes(scenario_name, request) + + diff --git a/nebula/controller/federation/physicall_federation_controller.py b/nebula/controller/federation/physicall_federation_controller.py new file mode 100644 index 000000000..f60c2a8e1 --- /dev/null +++ b/nebula/controller/federation/physicall_federation_controller.py @@ -0,0 +1,4 @@ +from nebula.controller.federation.federation_controller import FederationController + +class PhysicalFederationController(FederationController): + pass \ No newline at end of file diff --git a/nebula/controller/federation/processes_federation_controller.py b/nebula/controller/federation/processes_federation_controller.py new file mode 100644 index 000000000..1fbf8672e --- /dev/null +++ b/nebula/controller/federation/processes_federation_controller.py @@ -0,0 +1,4 @@ +from nebula.controller.federation.federation_controller import FederationController + +class ProcessesFederationController(FederationController): + pass \ No newline at end of file diff --git a/nebula/utils.py b/nebula/utils.py index 60819ed1a..e247087e4 100644 --- a/nebula/utils.py +++ b/nebula/utils.py @@ -4,6 +4,9 @@ import docker +import re +from typing import Optional + class FileUtils: """ @@ -202,3 +205,61 @@ def check_docker_by_prefix(cls, prefix): logging.exception("Error interacting with Docker") except Exception: logging.exception("Unexpected error") + + +class LoggerUtils: + + @staticmethod + def configure_logger( + name: Optional[str] = None, + log_file: Optional[str] = None, + level: int = logging.INFO, + console: bool = True, + strip_ansi: bool = True, + file_mode: str = "a", + log_format: str = "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s", + date_format: str = "%Y-%m-%d %H:%M:%S", + ) -> logging.Logger: + """ + Configure and return a logger with optional console and file output. + + Args: + name (str): Logger name. If None, the root logger is used. + log_file (str): Path to the log file. + level (int): Logging level (DEBUG, INFO, etc). + console (bool): If True, output is also printed to the console. + strip_ansi (bool): Placeholder for future ANSI stripping support. + file_mode (str): File mode for the log file ('a' for append, 'w' for overwrite). + log_format (str): Format for log messages. + date_format (str): Format for timestamps. + + Returns: + logging.Logger: Configured logger instance. + """ + logger = logging.getLogger(name) + logger.setLevel(level) + + # Prevent duplicate handler setup + if getattr(logger, "_is_configured", False): + return logger + + formatter = logging.Formatter(fmt=log_format, datefmt=date_format) + + if log_file: + os.makedirs(os.path.dirname(log_file), exist_ok=True) + fh = logging.FileHandler(log_file, mode=file_mode) + fh.setLevel(level) + fh.setFormatter(formatter) + logger.addHandler(fh) + + if console: + ch = logging.StreamHandler() + ch.setLevel(level) + ch.setFormatter(formatter) + logger.addHandler(ch) + + # Mark this logger as configured to avoid re-adding handlers + logger._is_configured = True + logger.propagate = False + + return logger \ No newline at end of file From b92c113726d399a978844737d2d2edb77f5389b4 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 10 Jul 2025 16:38:41 +0200 Subject: [PATCH 05/26] feature federation controller api running --- app/deployer.py | 38 +++++++++++++------ app/main.py | 8 ++++ .../federation/federation_controller.py | 25 +++++++++++- nebula/controller/start_services.sh | 12 +++++- .../{controller.py => web_app_controller.py} | 0 5 files changed, 67 insertions(+), 16 deletions(-) rename nebula/controller/{controller.py => web_app_controller.py} (100%) diff --git a/app/deployer.py b/app/deployer.py index 95bffda7e..7b624358e 100644 --- a/app/deployer.py +++ b/app/deployer.py @@ -17,7 +17,7 @@ from watchdog.observers import Observer from nebula.addons.env import check_environment -from nebula.controller.controller import TermEscapeCodeFormatter +from nebula.controller.web_app_controller import TermEscapeCodeFormatter from nebula.controller.scenarios import ScenarioManagement from nebula.utils import DockerUtils, FileUtils, SocketUtils @@ -637,6 +637,16 @@ def __init__(self, args): logging.exception(warning_msg) sys.exit(1) + self.controller_port = int(args.controllerport) if hasattr(args, "controllerport") else 5050 + self.federation_controller_port = int(args.federationcontrollerport) if hasattr(args, "federationcontrollerport") else 5051 + self.waf_port = int(args.wafport) if hasattr(args, "wafport") else 6000 + self.frontend_port = int(args.webport) if hasattr(args, "webport") else 6060 + self.grafana_port = int(args.grafanaport) if hasattr(args, "grafanaport") else 6040 + self.loki_port = int(args.lokiport) if hasattr(args, "lokiport") else 6010 + self.statistics_port = int(args.statsport) if hasattr(args, "statsport") else 8080 + self.production = args.production if hasattr(args, "production") else False + self.dev = args.developement if hasattr(args, "developement") else False + self.advanced_analytics = args.advanced_analytics if hasattr(args, "advanced_analytics") else False self.databases_dir = args.databases if hasattr(args, "databases") else "/nebula/app/databases" self.config_dir = args.config self.log_dir = args.logs @@ -843,6 +853,9 @@ def start(self): # Check ports available if not SocketUtils.is_port_open(self.controller_port): self.controller_port = SocketUtils.find_free_port(start_port=self.controller_port) + + if not SocketUtils.is_port_open(self.federation_controller_port): + self.federation_controller_port = SocketUtils.find_free_port(start_port=self.federation_controller_port) if not SocketUtils.is_port_open(self.frontend_port): self.frontend_port = SocketUtils.find_free_port(start_port=self.frontend_port) @@ -1110,11 +1123,13 @@ def run_controller(self): "NEBULA_ROOT_HOST": self.root_path, "NEBULA_DATABASES_DIR": "/nebula/app/databases", "NEBULA_CONTROLLER_LOG": "/nebula/app/logs/controller.log", + "NEBULA_FEDERATION_CONTROLLER_LOG": "/nebula/app/logs/federation.log", "NEBULA_CONFIG_DIR": "/nebula/app/config/", "NEBULA_LOGS_DIR": "/nebula/app/logs/", "NEBULA_CERTS_DIR": "/nebula/app/certs/", "NEBULA_HOST_PLATFORM": self.host_platform, "NEBULA_CONTROLLER_PORT": self.controller_port, + "NEBULA_FEDERATION_CONTROLLER_PORT" : self.federation_controller_port, "NEBULA_CONTROLLER_HOST": self.controller_host, "NEBULA_FRONTEND_PORT": self.frontend_port, "DB_HOST": self.get_container_name("nebula-database"), @@ -1126,7 +1141,7 @@ def run_controller(self): volumes = ["/nebula", "/var/run/docker.sock"] - ports = [self.controller_port] + ports = [self.controller_port, self.federation_controller_port] host_config = client.api.create_host_config( binds=[ @@ -1135,16 +1150,15 @@ def run_controller(self): f"{self.databases_dir}:/nebula/app/databases", ], extra_hosts={"host.docker.internal": "host-gateway"}, - port_bindings={self.controller_port: self.controller_port}, - device_requests=[ - { - "Driver": "nvidia", - "Count": -1, - "Capabilities": [["gpu"]], - } - ] - if self.gpu_available - else None, + port_bindings={ + self.controller_port: self.controller_port, + self.federation_controller_port: self.federation_controller_port + }, + device_requests=[{ + "Driver": "nvidia", + "Count": -1, + "Capabilities": [["gpu"]], + }] if self.gpu_available else None, ) networking_config = client.api.create_networking_config({ diff --git a/app/main.py b/app/main.py index e13d1641e..e822bfb4d 100755 --- a/app/main.py +++ b/app/main.py @@ -17,6 +17,14 @@ help="Controller port (default: 5050)", ) +argparser.add_argument( + "-fcp", + "--federationcontrollerport", + dest="federationcontrollerport", + default=5051, + help="federation controller port port (default: 5051)", +) + argparser.add_argument( "--grafanaport", dest="grafanaport", diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index 5bb24ab94..9788275f8 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -1,3 +1,4 @@ +import argparse import os import logging from abc import ABC, abstractmethod @@ -55,20 +56,32 @@ async def wrapper(*args, **kwargs): @asynccontextmanager async def lifespan(app: FastAPI): - log_path = os.path.join("app", "logs", "federation.log") + log_path = os.environ.get("NEBULA_FEDERATION_CONTROLLER_LOG") # Configure and register the logger under the name "controller" LoggerUtils.configure_logger(name="Federation-Controller", log_file=log_path) # Retrieve the logger by name logger = logging.getLogger("Federation-Controller") - logger.info("Logger initialized for FederationController") + logger.info("Logger initialized for Federation Controller") yield app = FastAPI(lifespan=lifespan) fed_controller: FederationController = None +@app.get("/") +async def read_root(): + """ + Root endpoint of the NEBULA Controller API. + + Returns: + dict: A welcome message indicating the API is accessible. + """ + logger = logging.getLogger("Federation-Controller") + logger.info("Test curl succesfull") + return {"message": "Welcome to the NEBULA Federation Controller API"} + @app.post("/init") async def init_federation_experiment(payload: dict = Body(...)): global fed_controller @@ -120,3 +133,11 @@ async def update_nodes( return await fed_controller.update_nodes(scenario_name, request) +if __name__ == "__main__": + # Parse args from command line + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=5051, help="Port to run the Federation controller on.") + args = parser.parse_args() + + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=args.port) \ No newline at end of file diff --git a/nebula/controller/start_services.sh b/nebula/controller/start_services.sh index fa0eaf031..e2d4c8777 100644 --- a/nebula/controller/start_services.sh +++ b/nebula/controller/start_services.sh @@ -11,7 +11,15 @@ echo "path $(pwd)" # Start Gunicorn NEBULA_SOCK=nebula.sock -echo "Starting Gunicorn..." -uvicorn nebula.controller.controller:app --host 0.0.0.0 --port $NEBULA_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & +echo "NEBULA_PRODUCTION: $NEBULA_PRODUCTION" +if [ "$NEBULA_PRODUCTION" = "False" ]; then + echo "Starting Gunicorn in dev mode..." + uvicorn nebula.controller.web_app_controller:app --host 0.0.0.0 --port $NEBULA_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & + uvicorn nebula.controller.federation.federation_controller:app --host 0.0.0.0 --port $NEBULA_FEDERATION_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & +else + echo "Starting Gunicorn in production mode..." + uvicorn nebula.controller.web_app_controller:app --host 0.0.0.0 --port $NEBULA_CONTROLLER_PORT --log-level info --proxy-headers --forwarded-allow-ips "*" & + uvicorn nebula.controller.federation.federation_controller:app --host 0.0.0.0 --port $NEBULA_FEDERATION_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & +fi tail -f /dev/null diff --git a/nebula/controller/controller.py b/nebula/controller/web_app_controller.py similarity index 100% rename from nebula/controller/controller.py rename to nebula/controller/web_app_controller.py From 646c5a9a07cb205769003074794407a69d9457ce Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 17 Jul 2025 14:47:34 +0200 Subject: [PATCH 06/26] feature federation API: - Scenario Builder - Dataset factories --- .../docker_federation_controller.py | 175 +++- .../controller/federation/federation_api.py | 113 +++ .../federation/federation_controller.py | 133 +-- .../controller/federation/scenario_builder.py | 759 ++++++++++++++++++ nebula/controller/scenarios.py | 2 + nebula/controller/start_services.sh | 4 +- nebula/core/datasets/nebuladataset.py | 14 + nebula/utils.py | 62 +- 8 files changed, 1144 insertions(+), 118 deletions(-) create mode 100644 nebula/controller/federation/federation_api.py create mode 100644 nebula/controller/federation/scenario_builder.py diff --git a/nebula/controller/federation/docker_federation_controller.py b/nebula/controller/federation/docker_federation_controller.py index 499ead1cb..480f5e9be 100644 --- a/nebula/controller/federation/docker_federation_controller.py +++ b/nebula/controller/federation/docker_federation_controller.py @@ -1,4 +1,177 @@ +import glob +import json +import os +import shutil from nebula.controller.federation.federation_controller import FederationController +from typing import Dict +from fastapi import Request +from nebula.config.config import Config +from nebula.core.utils.certificate import generate_ca_certificate, generate_certificate + class DockerFederationController(FederationController): - pass \ No newline at end of file + + def __init__(self, wa_controller_url, logger): + super.__init__(wa_controller_url, logger) + self.root_path = "" + self.host_platform = "" + self.config_dir = "" + self.log_dir = "" + self.cert_dir = "" + self.advanced_analytics = "" + self.controller = "" + self.config = Config(entity="scenarioManagement") + + #TODO remove unnecesary parameters role and user + async def run_scenario(self, scenario_data: Dict, role: str, user: str): + #TODO maintain files on memory, not read them again + await self._initialize_scenario(scenario_data) + generate_ca_certificate(dir_path=self.cert_dir) + await self._load_configuration_and_start_nodes() + + async def stop_scenario(self, scenario_name: str, username: str, all: bool): + pass + + async def remove_scenario(self, scenario_name: str): + pass + + async def update_nodes(self, scenario_name: str, request: Request): + pass + + async def _initialize_scenario(self, scenario_data): + # Initialize Scenario builder using scenario_data from user + self.sb.set_scenario_data(scenario_data) + scenario_name = self.sb.get_scenario_name() + + self.root_path = os.environ.get("NEBULA_ROOT_HOST") + self.host_platform = os.environ.get("NEBULA_HOST_PLATFORM") + self.config_dir = os.path.join(os.environ.get("NEBULA_CONFIG_DIR"), scenario_name) + self.log_dir = os.environ.get("NEBULA_LOGS_DIR") + self.cert_dir = os.environ.get("NEBULA_CERTS_DIR") + self.advanced_analytics = os.environ.get("NEBULA_ADVANCED_ANALYTICS", "False") == "True" + self.config = Config(entity="scenarioManagement") + + self.controller = f"{os.environ.get('NEBULA_CONTROLLER_HOST')}:{os.environ.get('NEBULA_CONTROLLER_PORT')}" + + # Create Scenario management dirs + os.makedirs(self.config_dir, exist_ok=True) + os.makedirs(os.path.join(self.log_dir, scenario_name), exist_ok=True) + os.makedirs(self.cert_dir, exist_ok=True) + + # Give permissions to the directories + os.chmod(self.config_dir, 0o777) + os.chmod(os.path.join(self.log_dir, scenario_name), 0o777) + os.chmod(self.cert_dir, 0o777) + + # Save the scenario configuration + scenario_file = os.path.join(self.config_dir, "scenario.json") + with open(scenario_file, "w") as f: + json.dump(scenario_data, f, sort_keys=False, indent=2) + + os.chmod(scenario_file, 0o777) + + # Save management settings + settings = { + "scenario_name": scenario_name, + "root_path": self.root_path, + "config_dir": self.config_dir, + "log_dir": self.log_dir, + "cert_dir": self.cert_dir, + "env": None, + } + + settings_file = os.path.join(self.config_dir, "settings.json") + with open(settings_file, "w") as f: + json.dump(settings, f, sort_keys=False, indent=2) + + os.chmod(settings_file, 0o777) + + # Attacks assigment and mobility + self.sb.build_general_configuration() + + # Create participant configs and .json + for index, node in enumerate(self.sb.get_federation_nodes().keys()): + node_config = node + participant_file = os.path.join(self.config_dir, f"participant_{node_config['id']}.json") + os.makedirs(os.path.dirname(participant_file), exist_ok=True) + os.chmod(participant_file, 0o777) + + participant_config = self.sb.build_scenario_config_for_node(index, node) + with open(participant_file, "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + + async def _load_configuration_and_start_nodes(self): + # Get participants configurations + participant_files = glob.glob(f"{self.config_dir}/participant_*.json") + participant_files.sort() + if len(participant_files) == 0: + raise ValueError("No participant files found in config folder") + + self.config.set_participants_config(participant_files) + self.n_nodes = len(participant_files) + self.logger.info(f"Number of nodes: {self.n_nodes}") + + self.sb.create_topology_manager(self.config) + + # Update participants configuration + is_start_node = False + config_participants = [] + + additional_participants = self.sb.get_additional_nodes() + additional_nodes = len(additional_participants) if additional_participants else 0 + self.logger.info(f"######## nodes: {self.n_nodes} + additionals: {additional_nodes} ######") + + participant_files.sort(key=lambda x: int(x.split("_")[-1].split(".")[0])) + + # Initial participants + for i in range(self.n_nodes): + with open(f"{self.config_dir}/participant_" + str(i) + ".json") as f: + participant_config = json.load(f) + + self.sb.build_preload_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) + + with open(f"{self.config_dir}/participant_" + str(i) + ".json", "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + + config_participants.append(( + participant_config["network_args"]["ip"], + participant_config["network_args"]["port"], + participant_config["device_args"]["role"], + )) + + if not is_start_node: + raise ValueError("No start node found") + self.config.set_participants_config(participant_files) + + # Add role to the topology (visualization purposes) + self.sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) + + # Additional participants + additional_participants_files = [] + if additional_participants: + last_participant_file = participant_files[-1] + last_participant_index = len(participant_files) + + for i, _ in enumerate(additional_participants): + additional_participant_file = f"{self.config_dir}/participant_{last_participant_index + i}.json" + shutil.copy(last_participant_file, additional_participant_file) + + with open(additional_participant_file) as f: + participant_config = json.load(f) + + self.logger.info(f"Configuration | additional nodes | participant: {self.n_nodes + i + 1}") + self.sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) + + with open(additional_participant_file, "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + + additional_participants_files.append(additional_participant_file) + + if additional_participants_files: + self.config.add_participants_config(additional_participants_files) + + if additional_participants: + self.n_nodes += len(additional_participants) + + # Build dataset + dataset = self.sb.configure_dataset() \ No newline at end of file diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py new file mode 100644 index 000000000..7be0bcc6c --- /dev/null +++ b/nebula/controller/federation/federation_api.py @@ -0,0 +1,113 @@ +import argparse +import os +import logging +from fastapi import FastAPI, Body, Path, Request +from fastapi.concurrency import asynccontextmanager +from typing import Dict +from typing import Annotated +from functools import wraps +from fastapi import HTTPException +from nebula.utils import LoggerUtils +from nebula.controller.federation.federation_controller import FederationController, federation_controller_factory + +def require_initialized_controller(func): + @wraps(func) + async def wrapper(*args, **kwargs): + if fed_controller is None: + raise HTTPException(status_code=400, detail="FederationController not initialized") + return await func(*args, **kwargs) + return wrapper + +@asynccontextmanager +async def lifespan(app: FastAPI): + log_path = os.environ.get("NEBULA_FEDERATION_CONTROLLER_LOG") + + # Configure and register the logger under the name "controller" + LoggerUtils.configure_logger(name="Federation-Controller", log_file=log_path) + + # Retrieve the logger by name + logger = logging.getLogger("Federation-Controller") + logger.info("Logger initialized for Federation Controller") + + yield + +app = FastAPI(lifespan=lifespan) +fed_controller: FederationController = None + +@app.get("/") +async def read_root(): + """ + Root endpoint of the NEBULA Controller API. + + Returns: + dict: A welcome message indicating the API is accessible. + """ + logger = logging.getLogger("Federation-Controller") + logger.info("Test curl succesfull") + return {"message": "Welcome to the NEBULA Federation Controller API"} + +@app.post("/init") +async def init_federation_experiment(payload: dict = Body(...)): + global fed_controller + + experiment_type = payload["type"] + logger = logging.getLogger("Federation-Controller") + + # Modify when deploying controllers on differents systems + web_app_controller_url = os.environ.get("NEBULA_CONTROLLER_PORT") + controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") + + controller_url = f"http://{controller_host}:{web_app_controller_url}" + fed_controller = federation_controller_factory(experiment_type, controller_url, logger) + + return {"message": f"{experiment_type} controller initialized"} + +@app.post("/scenarios/run") +@require_initialized_controller +async def run_scenario( + scenario_data: dict = Body(..., embed=True), + role: str = Body(..., embed=True), + user: str = Body(..., embed=True), +): + global fed_controller + return await fed_controller.run_scenario(scenario_data, role, user) + +@app.post("/scenarios/stop") +@require_initialized_controller +async def stop_scenario( + scenario_name: str = Body(..., embed=True), + username: str = Body(..., embed=True), + all: bool = Body(False, embed=True), +): + global fed_controller + return await fed_controller.stop_scenario(scenario_name, username, all) + +@app.post("/scenarios/remove") +@require_initialized_controller +async def remove_scenario( + scenario_name: str = Body(..., embed=True), +): + global fed_controller + return await fed_controller.remove_scenario(scenario_name) + +@app.post("/nodes/{scenario_name}/update") +@require_initialized_controller +async def update_nodes( + scenario_name: Annotated[ + str, + Path(regex="^[a-zA-Z0-9_-]+$", min_length=1, max_length=50, description="Valid scenario name"), + ], + request: Request, +): + global fed_controller + return await fed_controller.update_nodes(scenario_name, request) + + +if __name__ == "__main__": + # Parse args from command line + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=5051, help="Port to run the Federation controller on.") + args = parser.parse_args() + + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=args.port) \ No newline at end of file diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index 9788275f8..c7f55a5d8 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -1,20 +1,23 @@ -import argparse -import os -import logging from abc import ABC, abstractmethod -from fastapi import FastAPI, Body, Path, Request -from fastapi.concurrency import asynccontextmanager +from fastapi import Request from typing import Dict -from typing import Annotated -from functools import wraps -from fastapi import HTTPException -from nebula.utils import LoggerUtils - +from nebula.controller.federation.scenario_builder import ScenarioBuilder +import logging class FederationController(ABC): - def __init__(self, logger): - self._logger = logger + def __init__(self, wa_controller_url, logger): + self._logger: logging.Logger = logger + self._wa_url = wa_controller_url + self._scenario_builder = ScenarioBuilder() + + @property + def sb(self): + return self._scenario_builder + + @property + def logger(self): + return self._logger @abstractmethod async def run_scenario(self, scenario_data: Dict, role: str, user: str): @@ -32,112 +35,16 @@ async def remove_scenario(self, scenario_name: str): async def update_nodes(self, scenario_name: str, request: Request): pass -def federation_controller_factory(mode: str, logger) -> FederationController: +def federation_controller_factory(mode: str, wa_controller_url: str, logger) -> FederationController: from nebula.controller.federation.docker_federation_controller import DockerFederationController from nebula.controller.federation.processes_federation_controller import ProcessesFederationController from nebula.controller.federation.physicall_federation_controller import PhysicalFederationController if mode == "docker": - return DockerFederationController(logger) + return DockerFederationController(wa_controller_url, logger) elif mode == "physical": - return PhysicalFederationController(logger) + return PhysicalFederationController(wa_controller_url, logger) elif mode == "processes": - return ProcessesFederationController(logger) + return ProcessesFederationController(wa_controller_url, logger) else: - raise ValueError("Unknown federation mode") - -def require_initialized_controller(func): - @wraps(func) - async def wrapper(*args, **kwargs): - if fed_controller is None: - raise HTTPException(status_code=400, detail="FederationController not initialized") - return await func(*args, **kwargs) - return wrapper - -@asynccontextmanager -async def lifespan(app: FastAPI): - log_path = os.environ.get("NEBULA_FEDERATION_CONTROLLER_LOG") - - # Configure and register the logger under the name "controller" - LoggerUtils.configure_logger(name="Federation-Controller", log_file=log_path) - - # Retrieve the logger by name - logger = logging.getLogger("Federation-Controller") - logger.info("Logger initialized for Federation Controller") - - yield - -app = FastAPI(lifespan=lifespan) -fed_controller: FederationController = None - -@app.get("/") -async def read_root(): - """ - Root endpoint of the NEBULA Controller API. - - Returns: - dict: A welcome message indicating the API is accessible. - """ - logger = logging.getLogger("Federation-Controller") - logger.info("Test curl succesfull") - return {"message": "Welcome to the NEBULA Federation Controller API"} - -@app.post("/init") -async def init_federation_experiment(payload: dict = Body(...)): - global fed_controller - - experiment_type = payload["type"] - logger = logging.getLogger("Federation-Controller") - fed_controller = federation_controller_factory(experiment_type, logger) - - return {"message": f"{experiment_type} controller initialized"} - -@app.post("/scenarios/run") -@require_initialized_controller -async def run_scenario( - scenario_data: dict = Body(..., embed=True), - role: str = Body(..., embed=True), - user: str = Body(..., embed=True), -): - global fed_controller - return await fed_controller.run_scenario(scenario_data, role, user) - -@app.post("/scenarios/stop") -@require_initialized_controller -async def stop_scenario( - scenario_name: str = Body(..., embed=True), - username: str = Body(..., embed=True), - all: bool = Body(False, embed=True), -): - global fed_controller - return await fed_controller.stop_scenario(scenario_name, username, all) - -@app.post("/scenarios/remove") -@require_initialized_controller -async def remove_scenario( - scenario_name: str = Body(..., embed=True), -): - global fed_controller - return await fed_controller.remove_scenario(scenario_name) - -@app.post("/nodes/{scenario_name}/update") -@require_initialized_controller -async def update_nodes( - scenario_name: Annotated[ - str, - Path(regex="^[a-zA-Z0-9_-]+$", min_length=1, max_length=50, description="Valid scenario name"), - ], - request: Request, -): - global fed_controller - return await fed_controller.update_nodes(scenario_name, request) - - -if __name__ == "__main__": - # Parse args from command line - parser = argparse.ArgumentParser() - parser.add_argument("--port", type=int, default=5051, help="Port to run the Federation controller on.") - args = parser.parse_args() - - import uvicorn - uvicorn.run(app, host="0.0.0.0", port=args.port) \ No newline at end of file + raise ValueError("Unknown federation mode") \ No newline at end of file diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py new file mode 100644 index 000000000..956c7e761 --- /dev/null +++ b/nebula/controller/federation/scenario_builder.py @@ -0,0 +1,759 @@ +import logging +from datetime import datetime +import hashlib +from nebula.addons.topologymanager import TopologyManager +from nebula.config.config import Config +from nebula.core.utils.certificate import generate_certificate +from nebula.core.datasets.nebuladataset import NebulaDataset, factory_nebuladataset, factory_dataset_setup + +#TODO set_scenario_data -> set_config_setup -> build_scenario_config_for_node -> build_preload_configuration + +class ScenarioBuilder(): + def __init__(self, ): + self._scenario_data = None + self._config_setup = None + self.logger = logging.getLogger("Federation-Controller") + self._topology_manager: TopologyManager = None + self._scenario_name = "" + + @property + def sd(self): + """Scenario data dict""" + return self._scenario_data + + @property + def tm(self): + """Topology Manager""" + return self._topology_manager + + def get_scenario_name(self): + return self._scenario_name + + def set_scenario_data(self, scenario_data: dict): + self._scenario_data = scenario_data + self._scenario_name = f"nebula_{self.sd["federation"]}_{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}" + + def set_config_setup(self, setup: dict): + self._config_setup = setup + + def get_federation_nodes(self) -> dict: + return self.sd["nodes"] + + def get_additional_nodes(self): + return self.sd["additional_participants"] + + def get_dataset_name(self) -> str: + return self.sd["dataset"] + + """ ############################### + # SCENARIO CONFIG NODE # + ############################### + """ + def build_general_configuration(self): + self.sd["nodes"] = self._configure_nodes_attacks() + + if self.sd.get("mobility", None): + mobile_participants_percent = int(self.sd["mobile_participants_percent"]) + self.sd["nodes"] = self._mobility_assign(self.sd["nodes"], mobile_participants_percent) + else: + self.sd["nodes"] = self._mobility_assign(self.sd["nodes"], 0) + + """ ############################### + # SCENARIO CONFIG NODE # + ############################### + """ + + def build_scenario_config_for_node(self, index, node) -> dict: + self.logger.info("Start building the scenario configuration") + participant_config = {"addons": set()} + + # General configuration + participant_config["scenario_args"]["name"] = self._scenario_name + participant_config["scenario_args"]["start_time"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + + node_config = self.sd["nodes"][node] + participant_config["network_args"]["ip"] = node_config["ip"] + if self.sd["deployment"] == "physical": + participant_config["network_args"]["port"] = 8000 + else: + participant_config["network_args"]["port"] = int(node_config["port"]) + + participant_config["network_args"]["simulation"] = self.sd["network_simulation"] + participant_config["device_args"]["idx"] = node_config["id"] + participant_config["device_args"]["start"] = node_config["start"] + participant_config["device_args"]["role"] = node_config["role"] + participant_config["device_args"]["proxy"] = node_config["proxy"] + participant_config["device_args"]["malicious"] = node_config["malicious"] + participant_config["scenario_args"]["rounds"] = int(self.sd["rounds"]) + participant_config["data_args"]["dataset"] = self.sd["dataset"] + participant_config["data_args"]["iid"] = self.sd["iid"] + participant_config["data_args"]["partition_selection"] = self.sd["partition_selection"] + participant_config["data_args"]["partition_parameter"] = self.sd["partition_parameter"] + participant_config["model_args"]["model"] = self.sd["model"] + participant_config["training_args"]["epochs"] = int(self.sd["epochs"]) + participant_config["device_args"]["accelerator"] = self.sd["accelerator"] + participant_config["device_args"]["gpu_id"] = self.sd["gpu_id"] + participant_config["device_args"]["logging"] = self.sd["logginglevel"] + participant_config["aggregator_args"]["algorithm"] = self.sd["agg_algorithm"] + + # Addons configuration + + # Trustworthiness + if self.sd.get("with_trustworthiness", None): + participant_config["trust_args"] = self._configure_trustworthiness() + participant_config["addons"].add("trustworthiness") + + # Reputation + if self.sd.get("reputation", None): + participant_config["defense_args"]["reputation"] = self._configure_reputation() + participant_config["addons"].add("reputation") + + # Network simulation + if dict(self.sd.get("network_args"))["enabled"]: + participant_config["network_args"]["network_simulation"] = self._configure_network_simulation() + participant_config["addons"].add("network_simulation") + + # Attacks + self._configure_role(participant_config, node_config) + + # Mobility + if self.sd.get("mobility", None): + participant_config["addons"].add("mobility") + + # Situational awareness module + if self._situational_awareness_needed(): + participant_config["situational_awareness"] = self._configure_situational_awareness(index) + participant_config["addons"].add("situational_awareness") + + return participant_config + + def _configure_nodes_attacks(self): + poisoned_node_percent = self.sd["attack_params"].get("poisoned_node_percent", 0) + poisoned_sample_percent = self.sd["attack_params"].get("poisoned_sample_percent", 0) + poisoned_noise_percent = self.sd["attack_params"].get("poisoned_noise_percent", 0) + + nodes = self.attack_node_assign( + self.sd.get("nodes"), + self.sd.get("federation"), + int(poisoned_node_percent), + int(poisoned_sample_percent), + int(poisoned_noise_percent), + self.sd.get("attack_params"), + ) + + return nodes + + def attack_node_assign( + self, + nodes, + federation, + poisoned_node_percent, + poisoned_sample_percent, + poisoned_noise_percent, + attack_params, + ): + """ + Assign and configure attack parameters to nodes within a federated learning network. + + This method: + - Validates input attack parameters and percentages. + - Determines which nodes will be marked as malicious based on the specified + poisoned node percentage and attack type. + - Assigns attack roles and parameters to selected nodes. + - Supports multiple attack types such as Label Flipping, Sample Poisoning, + Model Poisoning, GLL Neuron Inversion, Swapping Weights, Delayer, and Flooding. + - Ensures proper validation and setting of attack-specific parameters, including + targeting, noise types, delays, intervals, and attack rounds. + - Updates nodes' malicious status, reputation, and attack parameters accordingly. + + Args: + nodes (dict): Dictionary of nodes with their current attributes. + federation (str): Type of federated learning framework (e.g., "DFL"). + poisoned_node_percent (float): Percentage of nodes to be poisoned (0-100). + poisoned_sample_percent (float): Percentage of samples to be poisoned (0-100). + poisoned_noise_percent (float): Percentage of noise to apply in poisoning (0-100). + attack_params (dict): Dictionary containing attack type and associated parameters. + + Returns: + dict: Updated nodes dictionary with assigned malicious roles and attack parameters. + + Raises: + ValueError: If any input parameter is invalid or attack type is unrecognized. + """ + import logging + import math + import random + + # Validate input parameters + def validate_percentage(value, name): + """ + Validate that a given value is a float percentage between 0 and 100. + + Args: + value: The value to validate, expected to be convertible to float. + name (str): Name of the parameter, used for error messages. + + Returns: + float: The validated percentage value. + + Raises: + ValueError: If the value is not a float or not within the range [0, 100]. + """ + try: + value = float(value) + if not 0 <= value <= 100: + raise ValueError(f"{name} must be between 0 and 100") + return value + except (TypeError, ValueError) as e: + raise ValueError(f"Invalid {name}: {e!s}") + + def validate_positive_int(value, name): + """ + Validate that a given value is a positive integer (including zero). + + Args: + value: The value to validate, expected to be convertible to int. + name (str): Name of the parameter, used for error messages. + + Returns: + int: The validated positive integer value. + + Raises: + ValueError: If the value is not an integer or is negative. + """ + try: + value = int(value) + if value < 0: + raise ValueError(f"{name} must be positive") + return value + except (TypeError, ValueError) as e: + raise ValueError(f"Invalid {name}: {e!s}") + + # Validate attack type + valid_attacks = { + "No Attack", + "Label Flipping", + "Sample Poisoning", + "Model Poisoning", + "GLL Neuron Inversion", + "Swapping Weights", + "Delayer", + "Flooding", + } + + # Get attack type from attack_params + if attack_params and "attacks" in attack_params: + attack = attack_params["attacks"] + + # Handle attack parameter which can be either a string or None + if attack is None: + attack = "No Attack" + elif not isinstance(attack, str): + raise ValueError(f"Invalid attack type: {attack}. Expected string or None.") + + if attack not in valid_attacks: + raise ValueError(f"Invalid attack type: {attack}. Must be one of {valid_attacks}") + + # Get attack parameters from attack_params + poisoned_node_percent = attack_params.get("poisoned_node_percent", poisoned_node_percent) + poisoned_sample_percent = attack_params.get("poisoned_sample_percent", poisoned_sample_percent) + poisoned_noise_percent = attack_params.get("poisoned_noise_percent", poisoned_noise_percent) + + # Validate percentage parameters + poisoned_node_percent = validate_percentage(poisoned_node_percent, "poisoned_node_percent") + poisoned_sample_percent = validate_percentage(poisoned_sample_percent, "poisoned_sample_percent") + poisoned_noise_percent = validate_percentage(poisoned_noise_percent, "poisoned_noise_percent") + + nodes_index = [] + # Get the nodes index + if federation == "DFL": + nodes_index = list(nodes.keys()) + else: + for node in nodes: + if nodes[node]["role"] != "server": + nodes_index.append(node) + + logging.info(f"Nodes index: {nodes_index}") + logging.info(f"Attack type: {attack}") + logging.info(f"Poisoned node percent: {poisoned_node_percent}") + + mal_nodes_defined = any(nodes[node]["malicious"] for node in nodes) + logging.info(f"Malicious nodes already defined: {mal_nodes_defined}") + + attacked_nodes = [] + + if not mal_nodes_defined and attack != "No Attack": + n_nodes = len(nodes_index) + # Number of attacked nodes, round up + num_attacked = int(math.ceil(poisoned_node_percent / 100 * n_nodes)) + if num_attacked > n_nodes: + num_attacked = n_nodes + + # Get the index of attacked nodes + attacked_nodes = random.sample(nodes_index, num_attacked) + logging.info(f"Number of nodes to attack: {num_attacked}") + logging.info(f"Attacked nodes: {attacked_nodes}") + + # Assign the role of each node + for node in nodes: + node_att = "No Attack" + malicious = False + node_reputation = self.reputation.copy() if self.reputation else None + + if node in attacked_nodes or nodes[node]["malicious"]: + malicious = True + node_reputation = None + node_att = attack + logging.info(f"Node {node} marked as malicious with attack {attack}") + + # Initialize attack parameters with defaults + node_attack_params = attack_params.copy() if attack_params else {} + + # Set attack-specific parameters + if attack == "Label Flipping": + node_attack_params["poisoned_node_percent"] = poisoned_node_percent + node_attack_params["poisoned_sample_percent"] = poisoned_sample_percent + node_attack_params["targeted"] = attack_params.get("targeted", False) + if node_attack_params["targeted"]: + node_attack_params["target_label"] = validate_positive_int( + attack_params.get("target_label", 4), "target_label" + ) + node_attack_params["target_changed_label"] = validate_positive_int( + attack_params.get("target_changed_label", 7), "target_changed_label" + ) + + elif attack == "Sample Poisoning": + node_attack_params["poisoned_node_percent"] = poisoned_node_percent + node_attack_params["poisoned_sample_percent"] = poisoned_sample_percent + node_attack_params["poisoned_noise_percent"] = poisoned_noise_percent + node_attack_params["noise_type"] = attack_params.get("noise_type", "Gaussian") + node_attack_params["targeted"] = attack_params.get("targeted", False) + if node_attack_params["targeted"]: + node_attack_params["target_label"] = validate_positive_int( + attack_params.get("target_label", 4), "target_label" + ) + + elif attack == "Model Poisoning": + node_attack_params["poisoned_node_percent"] = poisoned_node_percent + node_attack_params["poisoned_noise_percent"] = poisoned_noise_percent + node_attack_params["noise_type"] = attack_params.get("noise_type", "Gaussian") + + elif attack == "GLL Neuron Inversion": + node_attack_params["poisoned_node_percent"] = poisoned_node_percent + + elif attack == "Swapping Weights": + node_attack_params["poisoned_node_percent"] = poisoned_node_percent + node_attack_params["layer_idx"] = validate_positive_int( + attack_params.get("layer_idx", 0), "layer_idx" + ) + + elif attack == "Delayer": + node_attack_params["poisoned_node_percent"] = poisoned_node_percent + node_attack_params["delay"] = validate_positive_int(attack_params.get("delay", 10), "delay") + node_attack_params["target_percentage"] = validate_percentage( + attack_params.get("target_percentage", 100), "target_percentage" + ) + node_attack_params["selection_interval"] = validate_positive_int( + attack_params.get("selection_interval", 1), "selection_interval" + ) + + elif attack == "Flooding": + node_attack_params["poisoned_node_percent"] = poisoned_node_percent + node_attack_params["flooding_factor"] = validate_positive_int( + attack_params.get("flooding_factor", 100), "flooding_factor" + ) + node_attack_params["target_percentage"] = validate_percentage( + attack_params.get("target_percentage", 100), "target_percentage" + ) + node_attack_params["selection_interval"] = validate_positive_int( + attack_params.get("selection_interval", 1), "selection_interval" + ) + + # Add common attack parameters + node_attack_params["round_start_attack"] = validate_positive_int( + attack_params.get("round_start_attack", 1), "round_start_attack" + ) + node_attack_params["round_stop_attack"] = validate_positive_int( + attack_params.get("round_stop_attack", 10), "round_stop_attack" + ) + node_attack_params["attack_interval"] = validate_positive_int( + attack_params.get("attack_interval", 1), "attack_interval" + ) + + # Validate round parameters + if node_attack_params["round_start_attack"] >= node_attack_params["round_stop_attack"]: + raise ValueError("round_start_attack must be less than round_stop_attack") + + node_attack_params["attacks"] = node_att + nodes[node]["malicious"] = True + nodes[node]["attack_params"] = node_attack_params + nodes[node]["fake_behavior"] = nodes[node]["role"] + nodes[node]["role"] = "malicious" + else: + nodes[node]["attack_params"] = {"attacks": "No Attack"} + + nodes[node]["reputation"] = node_reputation + + logging.info( + f"Node {node} final configuration - malicious: {nodes[node]['malicious']}, attack: {nodes[node]['attack_params']['attacks']}" + ) + + return nodes + + def _configure_role(self, participant_config, node_config: dict): + if node_config["role"] == "malicious": + participant_config["adversarial_args"]["fake_behavior"] = node_config["fake_behavior"] + participant_config["adversarial_args"]["attack_params"] = node_config["attack_params"] + + def _mobility_assign(self, nodes, mobile_participants_percent): + """ + Assign mobility status to a subset of nodes based on a specified percentage. + + This method: + - Calculates the number of mobile nodes by applying the given percentage. + - Randomly selects nodes to be marked as mobile. + - Updates each node's "mobility" attribute to True or False accordingly. + + Args: + nodes (dict): Dictionary of nodes with their current attributes. + mobile_participants_percent (float): Percentage of nodes to be assigned mobility (0-100). + + Returns: + dict: Updated nodes dictionary with mobility status assigned. + """ + import random + + # Number of mobile nodes, round down + num_mobile = math.floor(mobile_participants_percent / 100 * len(nodes)) + if num_mobile > len(nodes): + num_mobile = len(nodes) + + # Get the index of mobile nodes + mobile_nodes = random.sample(list(nodes.keys()), num_mobile) + + # Assign the role of each node + for node in nodes: + node_mob = False + if node in mobile_nodes: + node_mob = True + nodes[node]["mobility"] = node_mob + return nodes + + def _configure_trustworthiness(self) -> dict: + trust_config = { + "robustness_pillar": self.sd["robustness_pillar"], + "resilience_to_attacks": self.sd["resilience_to_attacks"], + "algorithm_robustness": self.sd["algorithm_robustness"], + "client_reliability": self.sd["client_reliability"], + "privacy_pillar": self.sd["privacy_pillar"], + "technique": self.sd["technique"], + "uncertainty": self.sd["uncertainty"], + "indistinguishability": self.sd["indistinguishability"], + "fairness_pillar": self.sd["fairness_pillar"], + "selection_fairness": self.sd["selection_fairness"], + "performance_fairness": self.sd["performance_fairness"], + "class_distribution": self.sd["class_distribution"], + "explainability_pillar": self.sd["explainability_pillar"], + "interpretability": self.sd["interpretability"], + "post_hoc_methods": self.sd["post_hoc_methods"], + "accountability_pillar": self.sd["accountability_pillar"], + "factsheet_completeness": self.sd["factsheet_completeness"], + "architectural_soundness_pillar": self.sd["sd.architectural_soundness_pillar"], + "client_management": self.sd["client_management"], + "optimization": self.sd["optimization"], + "sustainability_pillar": self.sd["sustainability_pillar"], + "energy_source": self.sd["energy_source"], + "hardware_efficiency": self.sd["hardware_efficiency"], + "federation_complexity": self.sd["federation_complexity"], + "scenario": self.sd, + } + return trust_config + + def _configure_reputation(self) -> dict: + return self.sd.get("reputation") + + def _configure_network_simulation(self) -> dict: + network_parameters = {} + network_generation = dict(self.sd["network_args"]).pop("network_type") + enabled = dict(self.sd["network_args"]).pop("enabled") + type = dict(self.sd["network_args"]).pop("type") + addrs = "" + + for node in self.sd["nodes"]: + ip = self.sd["nodes"][node]["ip"] + port = self.sd["nodes"][node]["port"] + addrs = addrs + " " + f"{ip}:{port}" + + network_configuration = { + "interface": "eth0", + "verbose": False, + "preset": network_generation, + "federation": addrs + } + + network_parameters = { + "enabled": enabled, + "type": type, + "network_config": network_configuration + } + + return network_parameters + + def _situational_awareness_needed(self): + enabled = False + arrivals_dep = self.sd.get("arrivals_departures_args", None) + if arrivals_dep: + enabled = arrivals_dep["enabled"] + with_sa = self.sd.get("with_sa", None) + additionals = self.sd.get("additional_participants", None) + mob = self.sd.get("mobility", None) + + return with_sa or enabled or arrivals_dep or additionals or mob + + def _configure_situational_awareness(self, index) -> dict: + scheduled_isolation = self._configure_arrivals_departures(index) + snp = self.sd.get("sar_neighbor_policy", None) + topology_management = snp if (snp != "") else self.sd["topology"] + + situational_awareness_config = { + "strict_topology": self.sd["strict_topology"], + "sa_discovery": { + "candidate_selector": topology_management, + "model_handler": self.sd["sad_model_handler"], + "verbose": True, + }, + "sa_reasoner": { + "arbitration_policy": self.sd["sar_arbitration_policy"], + "verbose": True, + "sar_components": { + "sa_network": True, + "sa_training": self.sd["sar_training"] + }, + "sa_network": { + "neighbor_policy": topology_management, + "scheduled_isolation" : scheduled_isolation, + "verbose": True + }, + "sa_training": { + "training_policy": self.sd["sar_training_policy"], + "verbose": True + }, + }, + } + return situational_awareness_config + + def _configure_arrivals_departures(self, index) -> dict: + if not self.sd["arrivals_departures_args"]["enabled"]: + return {"enabled": False} + + config = {"enabled": True} + departures: list = self.sd["arrivals_departures_args"]["departures"] + index_departure_config: dict = departures[index] + if index_departure_config["round_start"] != "": + config["round_start"] = index_departure_config["round_start"] + config["duration"] = index_departure_config["duration"] if index_departure_config["duration"] != "" else None + else: + config = {"enabled": False} + + return config + + """ ############################### + # PRELOAD CONFIG # + ############################### + """ + + def build_preload_configuration(self, index, participant_config, log_dir, config_dir, cert_dir, advanced_analytics): + participant_config["scenario_args"]["federation"] = self.sd["federation"] + n_nodes = len(self.sd["nodes"].keys()) + n_additionals = len(self.sd["additional_participants"]) + participant_config["scenario_args"]["n_nodes"] = n_nodes + n_additionals + + participant_config["network_args"]["neighbors"] = self.tm.get_neighbors_string(index) + + participant_config["device_args"]["idx"] = index + participant_config["device_args"]["uid"] = hashlib.sha1( + ( + str(participant_config["network_args"]["ip"]) + + str(participant_config["network_args"]["port"]) + + str(participant_config["scenario_args"]["name"]) + ).encode() + ).hexdigest() + + if participant_config["mobility_args"]["random_geo"]: + ( + participant_config["mobility_args"]["latitude"], + participant_config["mobility_args"]["longitude"], + ) = TopologyManager.get_coordinates(random_geo=True) + else: + participant_config["mobility_args"]["latitude"] = self.sd["latitude"] + participant_config["mobility_args"]["longitude"] = self.sd["scenario.longitude"] + + # If not, use the given coordinates in the frontend + participant_config["tracking_args"]["local_tracking"] = "advanced" if advanced_analytics else "basic" + participant_config["tracking_args"]["log_dir"] = log_dir + participant_config["tracking_args"]["config_dir"] = config_dir + # Generate node certificate + keyfile_path, certificate_path = generate_certificate( + dir_path=cert_dir, + node_id=f"participant_{index}", + ip=participant_config["network_args"]["ip"], + ) + participant_config["security_args"]["certfile"] = certificate_path + participant_config["security_args"]["keyfile"] = keyfile_path + if participant_config["device_args"]["start"]: + if not is_start_node: + is_start_node = True + else: + raise ValueError("Only one node can be start node") + + def build_preload_additional_node_configuration(self, last_participant_index, index, participant_config): + n_nodes = len(self.sd["nodes"].keys()) + n_additionals = len(self.sd["additional_participants"]) + last_ip = participant_config["network_args"]["ip"] + self.logger.info(f"Valores de la ultima ip: ({last_ip})") + participant_config["scenario_args"]["n_nodes"] = n_nodes + n_additionals # self.n_nodes + i + 1 + participant_config["device_args"]["idx"] = last_participant_index + index + participant_config["network_args"]["neighbors"] = "" + participant_config["network_args"]["ip"] = ( + participant_config["network_args"]["ip"].rsplit(".", 1)[0] + + "." + + str(int(participant_config["network_args"]["ip"].rsplit(".", 1)[1]) + index + 1) + ) + participant_config["device_args"]["uid"] = hashlib.sha1( + ( + str(participant_config["network_args"]["ip"]) + + str(participant_config["network_args"]["port"]) + + str(self._scenario_name) + ).encode() + ).hexdigest() + participant_config["mobility_args"]["additional_node"]["status"] = True + + # used for late creation nodes + participant_config["mobility_args"]["late_creation"] = True + + """ ############################### + # TOPOLOGY MANAGER # + ############################### + """ + + def create_topology_manager(self, config: Config): + self._topology_manager = ( + self._create_topology(config, matrix=self.sd["matrix"]) if self.sd["matrix"] else self._create_topology(config) + ) + + def _create_topology(self, config: Config, matrix=None): + """ + Create and return a network topology manager based on the scenario's topology settings or a given adjacency matrix. + + Supports multiple topology types: + - Random: Generates an Erdős-Rényi random graph with specified connection probability. + - Matrix: Uses a provided adjacency matrix to define the topology. + - Fully: Creates a fully connected network. + - Ring: Creates a ring-structured network with partial connectivity. + - Star: Creates a centralized star topology (only for CFL federation). + + The method assigns IP and port information to nodes and returns the configured TopologyManager instance. + + Args: + matrix (optional): Adjacency matrix to define custom topology. If provided, overrides scenario topology. + + Raises: + ValueError: If an unknown topology type is specified in the scenario. + + Returns: + TopologyManager: Configured topology manager with nodes assigned. + """ + import numpy as np + + n_nodes = len(self.sd["nodes"].keys()) + if self.sd["topology"] == "Random": + # Create network topology using topology manager (random) + probability = float(self.sd["random_topology_probability"]) + logging.info( + f"Creating random network topology using erdos_renyi_graph: nodes={n_nodes}, probability={probability}" + ) + topologymanager = TopologyManager( + scenario_name=self._scenario_name, + n_nodes=n_nodes, + b_symmetric=True, + undirected_neighbor_num=3, + ) + topologymanager.generate_random_topology(probability) + elif matrix is not None: + if n_nodes > 2: + topologymanager = TopologyManager( + topology=np.array(matrix), + scenario_name=self._scenario_name, + n_nodes=n_nodes, + b_symmetric=True, + undirected_neighbor_num=n_nodes - 1, + ) + else: + topologymanager = TopologyManager( + topology=np.array(matrix), + scenario_name=self._scenario_name, + n_nodes=n_nodes, + b_symmetric=True, + undirected_neighbor_num=2, + ) + elif self.sd["topology"] == "Fully": + # Create a fully connected network + topologymanager = TopologyManager( + scenario_name=self._scenario_name, + n_nodes=n_nodes, + b_symmetric=True, + undirected_neighbor_num=n_nodes - 1, + ) + topologymanager.generate_topology() + elif self.sd["topology"] == "Ring": + # Create a partially connected network (ring-structured network) + topologymanager = TopologyManager(scenario_name=self._scenario_name, n_nodes=n_nodes, b_symmetric=True) + topologymanager.generate_ring_topology(increase_convergence=True) + elif self.sd["topology"] == "Star" and self.sd["federation"] == "CFL": + # Create a centralized network + topologymanager = TopologyManager(scenario_name=self._scenario_name, n_nodes=n_nodes, b_symmetric=True) + topologymanager.generate_server_topology() + else: + raise ValueError(f"Unknown topology type: {self.sd["topology"]}") + + # Assign nodes to topology + nodes_ip_port = [] + config.participants.sort(key=lambda x: int(x["device_args"]["idx"])) + for i, node in enumerate(config.participants): + nodes_ip_port.append(( + node["network_args"]["ip"], + node["network_args"]["port"], + "undefined", + )) + + topologymanager.add_nodes(nodes_ip_port) + return topologymanager + + def visualize_topology(self, config_participants, path, plot): + self.tm.update_nodes(config_participants) + self.tm.draw_graph(path, plot) + + """ ############################### + # DATASET CONFIGURATION # + ############################### + """ + + def configure_dataset(self, config_dir) -> NebulaDataset: + dataset_name = self.get_dataset_name() + dataset = factory_nebuladataset( + dataset_name, + self._configure_dataset_config(dataset_name, config_dir) + ) + return dataset + + def _configure_dataset_config(self, dataset_name, config_dir): + num_classes = factory_dataset_setup(dataset_name) + return { + "num_classes": num_classes, + "partitions_number": n_nodes, + "iid": scenario.iid, + "partition": scenario.partition_selection, + "partition_parameter": scenario.partition_parameter, + "seed": 42, + "config_dir": config_dir, + } \ No newline at end of file diff --git a/nebula/controller/scenarios.py b/nebula/controller/scenarios.py index 20783250a..64c5d84a2 100644 --- a/nebula/controller/scenarios.py +++ b/nebula/controller/scenarios.py @@ -1013,6 +1013,8 @@ async def load_configurations_and_start_nodes( with open(additional_participant_file) as f: participant_config = json.load(f) + + logging.info(f"Configuration | additional nodes | participant: {self.n_nodes + i + 1}") last_ip = participant_config["network_args"]["ip"] diff --git a/nebula/controller/start_services.sh b/nebula/controller/start_services.sh index e2d4c8777..c0ecb234b 100644 --- a/nebula/controller/start_services.sh +++ b/nebula/controller/start_services.sh @@ -15,11 +15,11 @@ echo "NEBULA_PRODUCTION: $NEBULA_PRODUCTION" if [ "$NEBULA_PRODUCTION" = "False" ]; then echo "Starting Gunicorn in dev mode..." uvicorn nebula.controller.web_app_controller:app --host 0.0.0.0 --port $NEBULA_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & - uvicorn nebula.controller.federation.federation_controller:app --host 0.0.0.0 --port $NEBULA_FEDERATION_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & + uvicorn nebula.controller.federation.federation_api:app --host 0.0.0.0 --port $NEBULA_FEDERATION_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & else echo "Starting Gunicorn in production mode..." uvicorn nebula.controller.web_app_controller:app --host 0.0.0.0 --port $NEBULA_CONTROLLER_PORT --log-level info --proxy-headers --forwarded-allow-ips "*" & - uvicorn nebula.controller.federation.federation_controller:app --host 0.0.0.0 --port $NEBULA_FEDERATION_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & + uvicorn nebula.controller.federation.federation_api:app --host 0.0.0.0 --port $NEBULA_FEDERATION_CONTROLLER_PORT --log-level debug --proxy-headers --forwarded-allow-ips "*" & fi tail -f /dev/null diff --git a/nebula/core/datasets/nebuladataset.py b/nebula/core/datasets/nebuladataset.py index 0c2e03d8a..a3d1cbe92 100755 --- a/nebula/core/datasets/nebuladataset.py +++ b/nebula/core/datasets/nebuladataset.py @@ -1299,3 +1299,17 @@ def factory_nebuladataset(dataset, **config) -> NebulaDataset: if not cs: raise ValueError(f"Dataset {dataset} not supported") return cs(**config) + +def factory_dataset_setup(dataset) -> dict: + options = { + "MNIST": 10, + "FashionMNIST": 10, + "EMNIST": 47, + "CIFAR10": 10, + "CIFAR100": 100, + } + + num_classes = options.get(dataset, None) + if not num_classes: + raise ValueError(f"Dataset {dataset} not supported") + return num_classes diff --git a/nebula/utils.py b/nebula/utils.py index e247087e4..6cf5b8c89 100644 --- a/nebula/utils.py +++ b/nebula/utils.py @@ -2,11 +2,16 @@ import os import socket +import aiohttp import docker import re from typing import Optional +from fastapi import HTTPException + +from nebula.frontend.app import retry_with_backoff + class FileUtils: """ @@ -99,7 +104,6 @@ def find_free_port(cls, start_port=49152, end_port=65535): return port return None - class DockerUtils: """ Utility class for Docker operations such as creating networks, @@ -262,4 +266,58 @@ def configure_logger( logger._is_configured = True logger.propagate = False - return logger \ No newline at end of file + return logger + +class APIUtils(): + + @staticmethod + async def get(url): + """ + Fetch JSON data from a remote controller endpoint via asynchronous HTTP GET. + + Parameters: + url (str): The full URL of the controller API endpoint. + + Returns: + Any: Parsed JSON response when the HTTP status code is 200. + + Raises: + HTTPException: If the response status is not 200, raises with the response status code and an error detail. + """ + + async def _get(): + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + if response.status == 200: + return await response.json() + else: + raise HTTPException(status_code=response.status, detail="Error fetching data") + + return await retry_with_backoff(_get) + + @staticmethod + async def post(url, data=None): + """ + Asynchronously send a JSON payload via HTTP POST to a controller endpoint and parse the response. + + Parameters: + url (str): The full URL of the controller API endpoint. + data (Any, optional): JSON-serializable payload to include in the POST request (default: None). + + Returns: + Any: Parsed JSON response when the HTTP status code is 200. + + Raises: + HTTPException: If the response status is not 200, with the status code and an error detail. + """ + + async def _post(): + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as response: + if response.status == 200: + return await response.json() + else: + detail = await response.text() + raise HTTPException(status_code=response.status, detail=detail) + + return await retry_with_backoff(_post) \ No newline at end of file From 6d8313ff652c58c803eeb6e0cd7827715c8229ab Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Fri, 18 Jul 2025 11:13:21 +0200 Subject: [PATCH 07/26] feature scenario building --- .../docker_federation_controller.py | 152 +++++++++++++++++- .../controller/federation/scenario_builder.py | 13 +- nebula/controller/web_app_controller.py | 10 +- 3 files changed, 163 insertions(+), 12 deletions(-) diff --git a/nebula/controller/federation/docker_federation_controller.py b/nebula/controller/federation/docker_federation_controller.py index 480f5e9be..abe86474b 100644 --- a/nebula/controller/federation/docker_federation_controller.py +++ b/nebula/controller/federation/docker_federation_controller.py @@ -2,6 +2,8 @@ import json import os import shutil +from nebula.utils import DockerUtils, FileUtils +import docker from nebula.controller.federation.federation_controller import FederationController from typing import Dict from fastapi import Request @@ -13,6 +15,7 @@ class DockerFederationController(FederationController): def __init__(self, wa_controller_url, logger): super.__init__(wa_controller_url, logger) + self._user = "" self.root_path = "" self.host_platform = "" self.config_dir = "" @@ -21,13 +24,19 @@ def __init__(self, wa_controller_url, logger): self.advanced_analytics = "" self.controller = "" self.config = Config(entity="scenarioManagement") - - #TODO remove unnecesary parameters role and user + + """ ############################### + # ENDPOINT CALLBACKS # + ############################### + """ + async def run_scenario(self, scenario_data: Dict, role: str, user: str): #TODO maintain files on memory, not read them again + self._user = user await self._initialize_scenario(scenario_data) generate_ca_certificate(dir_path=self.cert_dir) await self._load_configuration_and_start_nodes() + #self._start_nodes() async def stop_scenario(self, scenario_name: str, username: str, all: bool): pass @@ -37,9 +46,15 @@ async def remove_scenario(self, scenario_name: str): async def update_nodes(self, scenario_name: str, request: Request): pass - + + """ ############################### + # FUNCTIONALITIES # + ############################### + """ + async def _initialize_scenario(self, scenario_data): # Initialize Scenario builder using scenario_data from user + self.logger.info("Initializing Scenario Builder using scenario data") self.sb.set_scenario_data(scenario_data) scenario_name = self.sb.get_scenario_name() @@ -87,7 +102,9 @@ async def _initialize_scenario(self, scenario_data): os.chmod(settings_file, 0o777) # Attacks assigment and mobility + self.logger.info("Building general configuration") self.sb.build_general_configuration() + self.logger.info("Building general configuration done") # Create participant configs and .json for index, node in enumerate(self.sb.get_federation_nodes().keys()): @@ -99,8 +116,11 @@ async def _initialize_scenario(self, scenario_data): participant_config = self.sb.build_scenario_config_for_node(index, node) with open(participant_file, "w") as f: json.dump(participant_config, f, sort_keys=False, indent=2) + + self.logger.info("Initializing Scenario Builder done") async def _load_configuration_and_start_nodes(self): + self.logger.info("Loading Scenario configuration...") # Get participants configurations participant_files = glob.glob(f"{self.config_dir}/participant_*.json") participant_files.sort() @@ -124,6 +144,7 @@ async def _load_configuration_and_start_nodes(self): participant_files.sort(key=lambda x: int(x.split("_")[-1].split(".")[0])) # Initial participants + self.logger.info("Building preload configuration for initial nodes...") for i in range(self.n_nodes): with open(f"{self.config_dir}/participant_" + str(i) + ".json") as f: participant_config = json.load(f) @@ -138,7 +159,8 @@ async def _load_configuration_and_start_nodes(self): participant_config["network_args"]["port"], participant_config["device_args"]["role"], )) - + + self.logger.info("Building preload configuration for initial nodes done") if not is_start_node: raise ValueError("No start node found") self.config.set_participants_config(participant_files) @@ -147,6 +169,7 @@ async def _load_configuration_and_start_nodes(self): self.sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) # Additional participants + self.logger.info("Building preload configuration for additional nodes...") additional_participants_files = [] if additional_participants: last_participant_file = participant_files[-1] @@ -172,6 +195,125 @@ async def _load_configuration_and_start_nodes(self): if additional_participants: self.n_nodes += len(additional_participants) + + self.logger.info("Building preload configuration for additional nodes done") # Build dataset - dataset = self.sb.configure_dataset() \ No newline at end of file + dataset = self.sb.configure_dataset(self.config_dir) + self.logger.info(f"Splitting {self.sb.get_dataset_name()} dataset...") + dataset.initialize_dataset() + self.logger.info(f"Splitting {self.sb.get_dataset_name()} dataset... Done") + + #TODO delay additionals deployment until conditions + def _start_nodes(self): + """ + Starts participant nodes as Docker containers using Docker SDK. + + This method performs the following steps: + - Logs the beginning of the Docker container startup process. + - Creates a Docker network specific to the current user and scenario. + - Sorts participant nodes by their index. + - For each participant node: + - Sets up environment variables and host configuration, + enabling GPU support if required. + - Prepares Docker volume bindings and static network IP assignment. + - Updates the node configuration, replacing IP addresses as needed, + and writes the configuration to a JSON file. + - Creates and starts the Docker container for the node. + - Logs any exceptions encountered during container creation or startup. + + Raises: + docker.errors.DockerException: If there are issues communicating with the Docker daemon. + OSError: If there are issues accessing file system paths for volume binding. + Exception: For any other unexpected errors during container creation or startup. + + Note: + - The method assumes Docker and NVIDIA runtime are properly installed and configured. + - IP addresses in node configurations are replaced with network base dynamically. + """ + self.logger.info("Starting nodes using Docker Compose...") + + network_name = f"{os.environ.get('NEBULA_CONTROLLER_NAME')}_{str(self._user).lower()}-nebula-net-scenario" + + # Create the Docker network + base = DockerUtils.create_docker_network(network_name) + + client = docker.from_env() + + self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) + i = 2 + container_ids = [] + for idx, node in enumerate(self.config.participants): + image = "nebula-core" + name = f"{os.environ.get('NEBULA_CONTROLLER_NAME')}_{self._user}-participant{node['device_args']['idx']}" + + if node["device_args"]["accelerator"] == "gpu": + environment = { + "NVIDIA_DISABLE_REQUIRE": True, + "NEBULA_LOGS_DIR": "/nebula/app/logs/", + "NEBULA_CONFIG_DIR": "/nebula/app/config/", + } + host_config = client.api.create_host_config( + binds=[f"{self.root_path}:/nebula", "/var/run/docker.sock:/var/run/docker.sock"], + privileged=True, + device_requests=[docker.types.DeviceRequest(driver="nvidia", count=-1, capabilities=[["gpu"]])], + extra_hosts={"host.docker.internal": "host-gateway"}, + ) + else: + environment = {"NEBULA_LOGS_DIR": "/nebula/app/logs/", "NEBULA_CONFIG_DIR": "/nebula/app/config/"} + host_config = client.api.create_host_config( + binds=[f"{self.root_path}:/nebula", "/var/run/docker.sock:/var/run/docker.sock"], + privileged=True, + device_requests=[], + extra_hosts={"host.docker.internal": "host-gateway"}, + ) + + volumes = ["/nebula", "/var/run/docker.sock"] + + start_command = "sleep 10" if node["device_args"]["start"] else "sleep 0" + command = [ + "/bin/bash", + "-c", + f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{self.scenario_name}/participant_{node['device_args']['idx']}.json", + ] + + networking_config = client.api.create_networking_config({ + f"{network_name}": client.api.create_endpoint_config( + ipv4_address=f"{base}.{i}", + ), + f"{os.environ.get('NEBULA_CONTROLLER_NAME')}_nebula-net-base": client.api.create_endpoint_config(), + }) + + node["tracking_args"]["log_dir"] = "/nebula/app/logs" + node["tracking_args"]["config_dir"] = f"/nebula/app/config/{self.sb.get_scenario_name()}" + node["scenario_args"]["controller"] = self.controller + node["scenario_args"]["deployment"] = "docker" + node["security_args"]["certfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_cert.pem" + node["security_args"]["keyfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_key.pem" + node["security_args"]["cafile"] = "/nebula/app/certs/ca_cert.pem" + node = json.loads(json.dumps(node).replace("192.168.50.", f"{base}.")) # TODO change this + + # Write the config file in config directory + with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f: + json.dump(node, f, indent=4) + + try: + container_id = client.api.create_container( + image=image, + name=name, + detach=True, + volumes=volumes, + environment=environment, + command=command, + host_config=host_config, + networking_config=networking_config, + ) + except Exception as e: + self.logger.exception(f"Creating container {name}: {e}") + + try: + client.api.start(container_id) + container_ids.append(container_id) + except Exception as e: + self.logger.exception(f"Starting participant {name} error: {e}") + i += 1 \ No newline at end of file diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index 956c7e761..1010e08fc 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -6,8 +6,6 @@ from nebula.core.utils.certificate import generate_certificate from nebula.core.datasets.nebuladataset import NebulaDataset, factory_nebuladataset, factory_dataset_setup -#TODO set_scenario_data -> set_config_setup -> build_scenario_config_for_node -> build_preload_configuration - class ScenarioBuilder(): def __init__(self, ): self._scenario_data = None @@ -44,6 +42,7 @@ def get_additional_nodes(self): def get_dataset_name(self) -> str: return self.sd["dataset"] + """ ############################### # SCENARIO CONFIG NODE # @@ -742,18 +741,20 @@ def configure_dataset(self, config_dir) -> NebulaDataset: dataset_name = self.get_dataset_name() dataset = factory_nebuladataset( dataset_name, - self._configure_dataset_config(dataset_name, config_dir) + **self._configure_dataset_config(dataset_name, config_dir) ) return dataset def _configure_dataset_config(self, dataset_name, config_dir): num_classes = factory_dataset_setup(dataset_name) + n_nodes = len(self.sd["nodes"].keys()) + n_nodes += len(self.sd["additional_participants"]) return { "num_classes": num_classes, "partitions_number": n_nodes, - "iid": scenario.iid, - "partition": scenario.partition_selection, - "partition_parameter": scenario.partition_parameter, + "iid": self.sd["iid"], + "partition": self.sd["partition_selection"], + "partition_parameter": self.sd["partition_parameter"], "seed": 42, "config_dir": config_dir, } \ No newline at end of file diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index 7e59ae7ec..33577e4e4 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -24,7 +24,7 @@ scenario_set_status_to_finished, ) from nebula.controller.http_helpers import remote_get, remote_post_form -from nebula.utils import DockerUtils +from nebula.utils import DockerUtils, APIUtils # Setup controller logger @@ -319,6 +319,14 @@ async def run_scenario( from nebula.controller.scenarios import ScenarioManagement + fed_controller_port = os.environ.get("NEBULA_FEDERATION_CONTROLLER_PORT") + fed_controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") + url = f"http://{fed_controller_host}:{fed_controller_port}/init" + data = {"type": "docker"} + data2 = {"scenario_data": scenario_data, "role": role, "user": user} + APIUtils.post(url, data) + APIUtils.post(url, data2) + validate_physical_fields(scenario_data) db_scenario = copy.deepcopy(scenario_data) From 430f08e402f65ef5717c3377056af7d316789acb Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Sun, 20 Jul 2025 11:24:15 +0200 Subject: [PATCH 08/26] fix attack assigment on ScenarioBuilder --- .../docker_federation_controller.py | 30 ++- .../controller/federation/federation_api.py | 5 +- .../controller/federation/scenario_builder.py | 211 ++++++++++-------- nebula/controller/web_app_controller.py | 61 ++--- nebula/utils.py | 47 +++- 5 files changed, 209 insertions(+), 145 deletions(-) diff --git a/nebula/controller/federation/docker_federation_controller.py b/nebula/controller/federation/docker_federation_controller.py index abe86474b..ff0bd1319 100644 --- a/nebula/controller/federation/docker_federation_controller.py +++ b/nebula/controller/federation/docker_federation_controller.py @@ -14,7 +14,7 @@ class DockerFederationController(FederationController): def __init__(self, wa_controller_url, logger): - super.__init__(wa_controller_url, logger) + super().__init__(wa_controller_url, logger) self._user = "" self.root_path = "" self.host_platform = "" @@ -37,6 +37,8 @@ async def run_scenario(self, scenario_data: Dict, role: str, user: str): generate_ca_certificate(dir_path=self.cert_dir) await self._load_configuration_and_start_nodes() #self._start_nodes() + + return self.sb.get_scenario_name() async def stop_scenario(self, scenario_name: str, username: str, all: bool): pass @@ -79,7 +81,7 @@ async def _initialize_scenario(self, scenario_data): os.chmod(self.cert_dir, 0o777) # Save the scenario configuration - scenario_file = os.path.join(self.config_dir, "scenario.json") + scenario_file = os.path.join(self.config_dir, "scenario2.json") with open(scenario_file, "w") as f: json.dump(scenario_data, f, sort_keys=False, indent=2) @@ -107,15 +109,23 @@ async def _initialize_scenario(self, scenario_data): self.logger.info("Building general configuration done") # Create participant configs and .json - for index, node in enumerate(self.sb.get_federation_nodes().keys()): + for index, (_, node) in enumerate(self.sb.get_federation_nodes().items()): + self.logger.info(f"Creating .json file for participant: {index}, Configuration: {node}") node_config = node - participant_file = os.path.join(self.config_dir, f"participant_{node_config['id']}.json") - os.makedirs(os.path.dirname(participant_file), exist_ok=True) - os.chmod(participant_file, 0o777) - - participant_config = self.sb.build_scenario_config_for_node(index, node) - with open(participant_file, "w") as f: - json.dump(participant_config, f, sort_keys=False, indent=2) + try: + participant_file = os.path.join(self.config_dir, f"participant_{node_config['id']}_xxx.json") + self.logger.info(f"{participant_file}") + os.makedirs(os.path.dirname(participant_file), exist_ok=True) + except Exception as e: + self.logger.info(f"ERROR while creating files: {e}") + + try: + participant_config = self.sb.build_scenario_config_for_node(index, node) + with open(participant_file, "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + os.chmod(participant_file, 0o777) + except Exception as e: + self.logger.info(f"ERROR while dumping configuration into files: {e}") self.logger.info("Initializing Scenario Builder done") diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index 7be0bcc6c..dd19b8a2f 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -52,13 +52,16 @@ async def init_federation_experiment(payload: dict = Body(...)): experiment_type = payload["type"] logger = logging.getLogger("Federation-Controller") + logger.info(f"Experiment type received: {experiment_type}") # Modify when deploying controllers on differents systems web_app_controller_url = os.environ.get("NEBULA_CONTROLLER_PORT") controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") controller_url = f"http://{controller_host}:{web_app_controller_url}" - fed_controller = federation_controller_factory(experiment_type, controller_url, logger) + logger.info(f"Docker Hub URL => {controller_url}") + fed_controller = federation_controller_factory(str(experiment_type), controller_url, logger) + logger.info("Federation controller created.") return {"message": f"{experiment_type} controller initialized"} diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index 1010e08fc..4d1efcafc 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -1,6 +1,8 @@ import logging from datetime import datetime import hashlib +import math +from collections import defaultdict from nebula.addons.topologymanager import TopologyManager from nebula.config.config import Config from nebula.core.utils.certificate import generate_certificate @@ -29,7 +31,8 @@ def get_scenario_name(self): def set_scenario_data(self, scenario_data: dict): self._scenario_data = scenario_data - self._scenario_name = f"nebula_{self.sd["federation"]}_{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}" + federation_name = self.sd["federation"] + self._scenario_name = f"nebula_{federation_name}_{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}" def set_config_setup(self, setup: dict): self._config_setup = setup @@ -49,84 +52,19 @@ def get_dataset_name(self) -> str: ############################### """ def build_general_configuration(self): - self.sd["nodes"] = self._configure_nodes_attacks() - - if self.sd.get("mobility", None): - mobile_participants_percent = int(self.sd["mobile_participants_percent"]) - self.sd["nodes"] = self._mobility_assign(self.sd["nodes"], mobile_participants_percent) - else: - self.sd["nodes"] = self._mobility_assign(self.sd["nodes"], 0) - - """ ############################### - # SCENARIO CONFIG NODE # - ############################### - """ - - def build_scenario_config_for_node(self, index, node) -> dict: - self.logger.info("Start building the scenario configuration") - participant_config = {"addons": set()} - - # General configuration - participant_config["scenario_args"]["name"] = self._scenario_name - participant_config["scenario_args"]["start_time"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - - node_config = self.sd["nodes"][node] - participant_config["network_args"]["ip"] = node_config["ip"] - if self.sd["deployment"] == "physical": - participant_config["network_args"]["port"] = 8000 - else: - participant_config["network_args"]["port"] = int(node_config["port"]) + try: + self.sd["nodes"] = self._configure_nodes_attacks() - participant_config["network_args"]["simulation"] = self.sd["network_simulation"] - participant_config["device_args"]["idx"] = node_config["id"] - participant_config["device_args"]["start"] = node_config["start"] - participant_config["device_args"]["role"] = node_config["role"] - participant_config["device_args"]["proxy"] = node_config["proxy"] - participant_config["device_args"]["malicious"] = node_config["malicious"] - participant_config["scenario_args"]["rounds"] = int(self.sd["rounds"]) - participant_config["data_args"]["dataset"] = self.sd["dataset"] - participant_config["data_args"]["iid"] = self.sd["iid"] - participant_config["data_args"]["partition_selection"] = self.sd["partition_selection"] - participant_config["data_args"]["partition_parameter"] = self.sd["partition_parameter"] - participant_config["model_args"]["model"] = self.sd["model"] - participant_config["training_args"]["epochs"] = int(self.sd["epochs"]) - participant_config["device_args"]["accelerator"] = self.sd["accelerator"] - participant_config["device_args"]["gpu_id"] = self.sd["gpu_id"] - participant_config["device_args"]["logging"] = self.sd["logginglevel"] - participant_config["aggregator_args"]["algorithm"] = self.sd["agg_algorithm"] - - # Addons configuration - - # Trustworthiness - if self.sd.get("with_trustworthiness", None): - participant_config["trust_args"] = self._configure_trustworthiness() - participant_config["addons"].add("trustworthiness") - - # Reputation - if self.sd.get("reputation", None): - participant_config["defense_args"]["reputation"] = self._configure_reputation() - participant_config["addons"].add("reputation") + if self.sd.get("mobility", None): + mobile_participants_percent = int(self.sd["mobile_participants_percent"]) + self.sd["nodes"] = self._mobility_assign(self.sd["nodes"], mobile_participants_percent) + else: + self.sd["nodes"] = self._mobility_assign(self.sd["nodes"], 0) + except Exception as e: + self.logger.info(f"ERROR: {e}") - # Network simulation - if dict(self.sd.get("network_args"))["enabled"]: - participant_config["network_args"]["network_simulation"] = self._configure_network_simulation() - participant_config["addons"].add("network_simulation") - - # Attacks - self._configure_role(participant_config, node_config) - - # Mobility - if self.sd.get("mobility", None): - participant_config["addons"].add("mobility") - - # Situational awareness module - if self._situational_awareness_needed(): - participant_config["situational_awareness"] = self._configure_situational_awareness(index) - participant_config["addons"].add("situational_awareness") - - return participant_config - def _configure_nodes_attacks(self): + self.logger.info("Configurating node attacks...") poisoned_node_percent = self.sd["attack_params"].get("poisoned_node_percent", 0) poisoned_sample_percent = self.sd["attack_params"].get("poisoned_sample_percent", 0) poisoned_noise_percent = self.sd["attack_params"].get("poisoned_noise_percent", 0) @@ -140,6 +78,7 @@ def _configure_nodes_attacks(self): self.sd.get("attack_params"), ) + self.logger.info("Configurating node attacks done") return nodes def attack_node_assign( @@ -179,8 +118,6 @@ def attack_node_assign( Raises: ValueError: If any input parameter is invalid or attack type is unrecognized. """ - import logging - import math import random # Validate input parameters @@ -272,12 +209,12 @@ def validate_positive_int(value, name): if nodes[node]["role"] != "server": nodes_index.append(node) - logging.info(f"Nodes index: {nodes_index}") - logging.info(f"Attack type: {attack}") - logging.info(f"Poisoned node percent: {poisoned_node_percent}") + self.logger.info(f"Nodes index: {nodes_index}") + self.logger.info(f"Attack type: {attack}") + self.logger.info(f"Poisoned node percent: {poisoned_node_percent}") mal_nodes_defined = any(nodes[node]["malicious"] for node in nodes) - logging.info(f"Malicious nodes already defined: {mal_nodes_defined}") + self.logger.info(f"Malicious nodes already defined: {mal_nodes_defined}") attacked_nodes = [] @@ -290,20 +227,20 @@ def validate_positive_int(value, name): # Get the index of attacked nodes attacked_nodes = random.sample(nodes_index, num_attacked) - logging.info(f"Number of nodes to attack: {num_attacked}") - logging.info(f"Attacked nodes: {attacked_nodes}") + self.logger.info(f"Number of nodes to attack: {num_attacked}") + self.logger.info(f"Attacked nodes: {attacked_nodes}") # Assign the role of each node for node in nodes: node_att = "No Attack" malicious = False - node_reputation = self.reputation.copy() if self.reputation else None + #node_reputation = self.reputation.copy() if self.reputation else None if node in attacked_nodes or nodes[node]["malicious"]: malicious = True node_reputation = None node_att = attack - logging.info(f"Node {node} marked as malicious with attack {attack}") + self.logger.info(f"Node {node} marked as malicious with attack {attack}") # Initialize attack parameters with defaults node_attack_params = attack_params.copy() if attack_params else {} @@ -388,22 +325,20 @@ def validate_positive_int(value, name): nodes[node]["attack_params"] = node_attack_params nodes[node]["fake_behavior"] = nodes[node]["role"] nodes[node]["role"] = "malicious" - else: - nodes[node]["attack_params"] = {"attacks": "No Attack"} - - nodes[node]["reputation"] = node_reputation + # else: + # nodes[node]["attack_params"] = {"attacks": "No Attack"} - logging.info( - f"Node {node} final configuration - malicious: {nodes[node]['malicious']}, attack: {nodes[node]['attack_params']['attacks']}" - ) + if nodes[node].get("attack_params", None): + self.logger.info( + f"Node {node} final configuration - malicious: {nodes[node]['malicious']}, attack: {nodes[node]['attack_params']['attacks']}" + ) + else: + self.logger.info( + f"Node {node} final configuration - malicious: {nodes[node]['malicious']}" + ) return nodes - def _configure_role(self, participant_config, node_config: dict): - if node_config["role"] == "malicious": - participant_config["adversarial_args"]["fake_behavior"] = node_config["fake_behavior"] - participant_config["adversarial_args"]["attack_params"] = node_config["attack_params"] - def _mobility_assign(self, nodes, mobile_participants_percent): """ Assign mobility status to a subset of nodes based on a specified percentage. @@ -437,6 +372,83 @@ def _mobility_assign(self, nodes, mobile_participants_percent): node_mob = True nodes[node]["mobility"] = node_mob return nodes + + + """ ############################### + # SCENARIO CONFIG NODE # + ############################### + """ + + def build_scenario_config_for_node(self, index, node) -> dict: + self.logger.info(f"Start building the scenario configuration for participant {index}") + participant_config = defaultdict() + participant_config["addons"] = list() + + # General configuration + participant_config["scenario_args"]["name"] = self._scenario_name + participant_config["scenario_args"]["start_time"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + + node_config = self.sd["nodes"][node] + participant_config["network_args"]["ip"] = node_config["ip"] + if self.sd["deployment"] == "physical": + participant_config["network_args"]["port"] = 8000 + else: + participant_config["network_args"]["port"] = int(node_config["port"]) + + participant_config["network_args"]["simulation"] = self.sd["network_simulation"] + participant_config["device_args"]["idx"] = node_config["id"] + participant_config["device_args"]["start"] = node_config["start"] + participant_config["device_args"]["role"] = node_config["role"] + participant_config["device_args"]["proxy"] = node_config["proxy"] + participant_config["device_args"]["malicious"] = node_config["malicious"] + participant_config["scenario_args"]["rounds"] = int(self.sd["rounds"]) + participant_config["data_args"]["dataset"] = self.sd["dataset"] + participant_config["data_args"]["iid"] = self.sd["iid"] + participant_config["data_args"]["partition_selection"] = self.sd["partition_selection"] + participant_config["data_args"]["partition_parameter"] = self.sd["partition_parameter"] + participant_config["model_args"]["model"] = self.sd["model"] + participant_config["training_args"]["epochs"] = int(self.sd["epochs"]) + participant_config["device_args"]["accelerator"] = self.sd["accelerator"] + participant_config["device_args"]["gpu_id"] = self.sd["gpu_id"] + participant_config["device_args"]["logging"] = self.sd["logginglevel"] + participant_config["aggregator_args"]["algorithm"] = self.sd["agg_algorithm"] + + # Addons configuration + + # Trustworthiness + if self.sd.get("with_trustworthiness", None): + participant_config["trust_args"] = self._configure_trustworthiness() + participant_config["addons"].append("trustworthiness") + + # Reputation + if self.sd.get("reputation", None): + participant_config["defense_args"]["reputation"] = self._configure_reputation() + participant_config["addons"].append("reputation") + + # Network simulation + #TODO revisar + if dict(self.sd.get("network_args"))["enabled"]: + participant_config["network_args"]["network_simulation"] = self._configure_network_simulation() + participant_config["addons"].append("network_simulation") + + # Attacks + self._configure_role(participant_config, node_config) + + # Mobility + if self.sd.get("mobility", None): + participant_config["addons"].append("mobility") + + # Situational awareness module + if self._situational_awareness_needed(): + participant_config["situational_awareness"] = self._configure_situational_awareness(index) + participant_config["addons"].append("situational_awareness") + + return participant_config + + def _configure_role(self, participant_config, node_config: dict): + if node_config["role"] == "malicious": + participant_config["adversarial_args"]["fake_behavior"] = node_config["fake_behavior"] + participant_config["adversarial_args"]["attack_params"] = node_config["attack_params"] def _configure_trustworthiness(self) -> dict: trust_config = { @@ -713,7 +725,8 @@ def _create_topology(self, config: Config, matrix=None): topologymanager = TopologyManager(scenario_name=self._scenario_name, n_nodes=n_nodes, b_symmetric=True) topologymanager.generate_server_topology() else: - raise ValueError(f"Unknown topology type: {self.sd["topology"]}") + top = self.sd["topology"] + raise ValueError(f"Unknown topology type: {top}") # Assign nodes to topology nodes_ip_port = [] diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index 33577e4e4..091e2b16c 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -318,14 +318,17 @@ async def run_scenario( import subprocess from nebula.controller.scenarios import ScenarioManagement - - fed_controller_port = os.environ.get("NEBULA_FEDERATION_CONTROLLER_PORT") - fed_controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") - url = f"http://{fed_controller_host}:{fed_controller_port}/init" - data = {"type": "docker"} - data2 = {"scenario_data": scenario_data, "role": role, "user": user} - APIUtils.post(url, data) - APIUtils.post(url, data2) + try: + fed_controller_port = os.environ.get("NEBULA_FEDERATION_CONTROLLER_PORT") + fed_controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") + url = f"http://{fed_controller_host}:{fed_controller_port}/init" + url2 = f"http://{fed_controller_host}:{fed_controller_port}/scenarios/run" + data = {"type": "docker"} + data2 = {"scenario_data": scenario_data, "role": role, "user": user} + await APIUtils.post(url, data) + await APIUtils.post(url2, data2) + except Exception as e: + logging.info(e) validate_physical_fields(scenario_data) @@ -334,29 +337,29 @@ async def run_scenario( # Manager for the actual scenario scenarioManagement = ScenarioManagement(scenario_data, user) - await update_scenario( - scenario_name=scenarioManagement.scenario_name, - start_time=scenarioManagement.start_date_scenario, - end_time="", - scenario=scenario_data, - status="running", - role=role, - username=user, - ) + # await update_scenario( + # scenario_name=scenarioManagement.scenario_name, + # start_time=scenarioManagement.start_date_scenario, + # end_time="", + # scenario=scenario_data, + # status="running", + # role=role, + # username=user, + # ) # Run the actual scenario - try: - if scenarioManagement.scenario.mobility: - additional_participants = scenario_data["additional_participants"] - schema_additional_participants = scenario_data["schema_additional_participants"] - await scenarioManagement.load_configurations_and_start_nodes( - additional_participants, schema_additional_participants - ) - else: - await scenarioManagement.load_configurations_and_start_nodes() - except subprocess.CalledProcessError as e: - logging.exception(f"Error docker-compose up: {e}") - return + # try: + # if scenarioManagement.scenario.mobility: + # additional_participants = scenario_data["additional_participants"] + # schema_additional_participants = scenario_data["schema_additional_participants"] + # await scenarioManagement.load_configurations_and_start_nodes( + # additional_participants, schema_additional_participants + # ) + # else: + # await scenarioManagement.load_configurations_and_start_nodes() + # except subprocess.CalledProcessError as e: + # logging.exception(f"Error docker-compose up: {e}") + # return return scenarioManagement.scenario_name diff --git a/nebula/utils.py b/nebula/utils.py index 6cf5b8c89..34b0c9621 100644 --- a/nebula/utils.py +++ b/nebula/utils.py @@ -9,9 +9,9 @@ from typing import Optional from fastapi import HTTPException - -from nebula.frontend.app import retry_with_backoff - +from aiohttp import ClientConnectorError +from aiohttp.client_exceptions import ClientError +import asyncio class FileUtils: """ @@ -220,7 +220,7 @@ def configure_logger( level: int = logging.INFO, console: bool = True, strip_ansi: bool = True, - file_mode: str = "a", + file_mode: str = "w", log_format: str = "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s", date_format: str = "%Y-%m-%d %H:%M:%S", ) -> logging.Logger: @@ -270,6 +270,39 @@ def configure_logger( class APIUtils(): + @staticmethod + async def retry_with_backoff(func, *args, max_retries=5, initial_delay=1): + """ + Retry a function with exponential backoff. + + Args: + func: The async function to retry + *args: Arguments to pass to the function + max_retries: Maximum number of retry attempts + initial_delay: Initial delay between retries in seconds + + Returns: + The result of the function if successful + + Raises: + The last exception if all retries fail + """ + delay = initial_delay + last_exception = None + + for attempt in range(max_retries): + try: + return await func(*args) + except (ClientConnectorError, ClientError) as e: + last_exception = e + if attempt < max_retries - 1: + logging.warning(f"Connection attempt {attempt + 1} failed: {str(e)}. Retrying in {delay} seconds...") + await asyncio.sleep(delay) + delay *= 2 # Exponential backoff + else: + logging.error(f"All {max_retries} connection attempts failed") + raise last_exception + @staticmethod async def get(url): """ @@ -293,7 +326,7 @@ async def _get(): else: raise HTTPException(status_code=response.status, detail="Error fetching data") - return await retry_with_backoff(_get) + return await APIUtils.retry_with_backoff(_get) @staticmethod async def post(url, data=None): @@ -320,4 +353,6 @@ async def _post(): detail = await response.text() raise HTTPException(status_code=response.status, detail=detail) - return await retry_with_backoff(_post) \ No newline at end of file + return await APIUtils.retry_with_backoff(_post) + + \ No newline at end of file From 5538aa155631e2cd96e5949bb37ea083a6b6ca3b Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Mon, 21 Jul 2025 15:27:48 +0200 Subject: [PATCH 09/26] feature scenario building complete on federation controller --- .../docker_federation_controller.py | 65 +++-- .../controller/federation/scenario_builder.py | 224 +++++++++++------- nebula/controller/web_app_controller.py | 4 +- 3 files changed, 187 insertions(+), 106 deletions(-) diff --git a/nebula/controller/federation/docker_federation_controller.py b/nebula/controller/federation/docker_federation_controller.py index ff0bd1319..03f276d32 100644 --- a/nebula/controller/federation/docker_federation_controller.py +++ b/nebula/controller/federation/docker_federation_controller.py @@ -56,7 +56,7 @@ async def update_nodes(self, scenario_name: str, request: Request): async def _initialize_scenario(self, scenario_data): # Initialize Scenario builder using scenario_data from user - self.logger.info("Initializing Scenario Builder using scenario data") + self.logger.info("🔧 Initializing Scenario Builder using scenario data") self.sb.set_scenario_data(scenario_data) scenario_name = self.sb.get_scenario_name() @@ -81,7 +81,7 @@ async def _initialize_scenario(self, scenario_data): os.chmod(self.cert_dir, 0o777) # Save the scenario configuration - scenario_file = os.path.join(self.config_dir, "scenario2.json") + scenario_file = os.path.join(self.config_dir, "scenario.json") with open(scenario_file, "w") as f: json.dump(scenario_data, f, sort_keys=False, indent=2) @@ -104,33 +104,38 @@ async def _initialize_scenario(self, scenario_data): os.chmod(settings_file, 0o777) # Attacks assigment and mobility - self.logger.info("Building general configuration") + self.logger.info("🔧 Building general configuration") self.sb.build_general_configuration() - self.logger.info("Building general configuration done") + self.logger.info("✅ Building general configuration done") # Create participant configs and .json for index, (_, node) in enumerate(self.sb.get_federation_nodes().items()): self.logger.info(f"Creating .json file for participant: {index}, Configuration: {node}") node_config = node try: - participant_file = os.path.join(self.config_dir, f"participant_{node_config['id']}_xxx.json") - self.logger.info(f"{participant_file}") + participant_file = os.path.join(self.config_dir, f"participant_{node_config['id']}.json") + self.logger.info(f"Filename: {participant_file}") os.makedirs(os.path.dirname(participant_file), exist_ok=True) except Exception as e: self.logger.info(f"ERROR while creating files: {e}") try: participant_config = self.sb.build_scenario_config_for_node(index, node) + #self.logger.info(f"dictionary: {participant_config}") + except Exception as e: + self.logger.info(f"ERROR while building configuration for node: {e}") + + try: with open(participant_file, "w") as f: json.dump(participant_config, f, sort_keys=False, indent=2) os.chmod(participant_file, 0o777) except Exception as e: self.logger.info(f"ERROR while dumping configuration into files: {e}") - self.logger.info("Initializing Scenario Builder done") + self.logger.info("✅ Initializing Scenario Builder done") async def _load_configuration_and_start_nodes(self): - self.logger.info("Loading Scenario configuration...") + self.logger.info("🔧 Loading Scenario configuration...") # Get participants configurations participant_files = glob.glob(f"{self.config_dir}/participant_*.json") participant_files.sort() @@ -154,32 +159,46 @@ async def _load_configuration_and_start_nodes(self): participant_files.sort(key=lambda x: int(x.split("_")[-1].split(".")[0])) # Initial participants - self.logger.info("Building preload configuration for initial nodes...") + self.logger.info("🔧 Building preload configuration for initial nodes...") for i in range(self.n_nodes): - with open(f"{self.config_dir}/participant_" + str(i) + ".json") as f: - participant_config = json.load(f) - - self.sb.build_preload_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) - - with open(f"{self.config_dir}/participant_" + str(i) + ".json", "w") as f: - json.dump(participant_config, f, sort_keys=False, indent=2) + try: + with open(f"{self.config_dir}/participant_" + str(i) + ".json") as f: + participant_config = json.load(f) + except Exception as e: + self.logger.info(f"ERROR: open/load participant .json") + + self.logger.info(f"Building preload conf for participant {i}") + try: + self.sb.build_preload_initial_node_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) + except Exception as e: + self.logger.info(f"ERROR: cannot build preload configuration") + + try: + with open(f"{self.config_dir}/participant_" + str(i) + ".json", "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + except Exception as e: + self.logger.info(f"ERROR: cannot dump preload configuration into participant .json file") config_participants.append(( participant_config["network_args"]["ip"], participant_config["network_args"]["port"], participant_config["device_args"]["role"], )) + if participant_config["device_args"]["start"]: + if not is_start_node: + is_start_node = True + else: + raise ValueError("Only one node can be start node") - self.logger.info("Building preload configuration for initial nodes done") - if not is_start_node: - raise ValueError("No start node found") + self.logger.info("✅ Building preload configuration for initial nodes done") + self.config.set_participants_config(participant_files) # Add role to the topology (visualization purposes) self.sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) # Additional participants - self.logger.info("Building preload configuration for additional nodes...") + self.logger.info("🔧 Building preload configuration for additional nodes...") additional_participants_files = [] if additional_participants: last_participant_file = participant_files[-1] @@ -206,13 +225,13 @@ async def _load_configuration_and_start_nodes(self): if additional_participants: self.n_nodes += len(additional_participants) - self.logger.info("Building preload configuration for additional nodes done") + self.logger.info("✅ Building preload configuration for additional nodes done") # Build dataset dataset = self.sb.configure_dataset(self.config_dir) - self.logger.info(f"Splitting {self.sb.get_dataset_name()} dataset...") + self.logger.info(f"🔧 Splitting {self.sb.get_dataset_name()} dataset...") dataset.initialize_dataset() - self.logger.info(f"Splitting {self.sb.get_dataset_name()} dataset... Done") + self.logger.info(f"✅ Splitting {self.sb.get_dataset_name()} dataset... Done") #TODO delay additionals deployment until conditions def _start_nodes(self): diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index 4d1efcafc..76e48e074 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -45,7 +45,6 @@ def get_additional_nodes(self): def get_dataset_name(self) -> str: return self.sd["dataset"] - """ ############################### # SCENARIO CONFIG NODE # @@ -381,14 +380,25 @@ def _mobility_assign(self, nodes, mobile_participants_percent): def build_scenario_config_for_node(self, index, node) -> dict: self.logger.info(f"Start building the scenario configuration for participant {index}") - participant_config = defaultdict() - participant_config["addons"] = list() + + def recursive_defaultdict(): + return defaultdict(recursive_defaultdict) + + def dictify(d): + if isinstance(d, defaultdict): + return {k: dictify(v) for k, v in d.items()} + return d + + participant_config = recursive_defaultdict() + + addons_config = defaultdict() + #participant_config["addons"] = dict() # General configuration participant_config["scenario_args"]["name"] = self._scenario_name participant_config["scenario_args"]["start_time"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - node_config = self.sd["nodes"][node] + node_config = node #self.sd["nodes"][index] participant_config["network_args"]["ip"] = node_config["ip"] if self.sd["deployment"] == "physical": participant_config["network_args"]["port"] = 8000 @@ -416,39 +426,69 @@ def build_scenario_config_for_node(self, index, node) -> dict: # Addons configuration # Trustworthiness - if self.sd.get("with_trustworthiness", None): - participant_config["trust_args"] = self._configure_trustworthiness() - participant_config["addons"].append("trustworthiness") + try: + if self.sd.get("with_trustworthiness", None): + #participant_config["trust_args"] = + addons_config["trustworthiness"] = self._configure_trustworthiness() + except Exception as e: + self.logger.info(f"ERROR: Cannot build trustworthiness configuration - {e}") # Reputation - if self.sd.get("reputation", None): - participant_config["defense_args"]["reputation"] = self._configure_reputation() - participant_config["addons"].append("reputation") + try: + if self.sd.get("reputation", None) and self.sd["reputation"]["enabled"]: + #participant_config["defense_args"]["reputation"] = self._configure_reputation() + addons_config["reputation"] = self._configure_reputation() + except Exception as e: + self.logger.info(f"ERROR: Cannot build reputation configuration - {e}") # Network simulation - #TODO revisar - if dict(self.sd.get("network_args"))["enabled"]: - participant_config["network_args"]["network_simulation"] = self._configure_network_simulation() - participant_config["addons"].append("network_simulation") + try: + network_args: dict = (self.sd.get("network_args"), None) + if network_args and isinstance(network_args, dict) and network_args.get("enabled", None): + #participant_config["network_args"]["network_simulation"] = self._configure_network_simulation() + addons_config["network_simulation"] = self._configure_network_simulation() + except Exception as e: + self.logger.info(f"ERROR: Cannot build network simulation configuration - {e}") # Attacks - self._configure_role(participant_config, node_config) + try: + #TODO moverlo a addons-adversarial_args + if node_config["role"] == "malicious": + addons_config["adversarial_args"] = self._configure_malicious_role(node_config) + except Exception as e: + self.logger.info(f"ERROR: Cannot build role configuration - {e}") # Mobility - if self.sd.get("mobility", None): - participant_config["addons"].append("mobility") + try: + if self.sd.get("mobility", None): + #participant_config["addons"].append("mobility") + addons_config["mobility"] = {"enabled": True} + except Exception as e: + self.logger.info(f"ERROR: Cannot build mobility configuration - {e}") # Situational awareness module - if self._situational_awareness_needed(): - participant_config["situational_awareness"] = self._configure_situational_awareness(index) - participant_config["addons"].append("situational_awareness") - - return participant_config + try: + if self._situational_awareness_needed(): + #participant_config["situational_awareness"] = self._configure_situational_awareness(index) + addons_config["situational_awareness"] = self._configure_situational_awareness(index) + except Exception as e: + self.logger.info(f"ERROR: Cannot build situational awareness configuration - {e}") + + # Addon addition to the configuration + participant_config["addons"] = addons_config + + try: + config = dictify(participant_config) + except Exception as e: + self.logger.info(f"ERROR: Translating into dictionary - {e}") + + return config - def _configure_role(self, participant_config, node_config: dict): - if node_config["role"] == "malicious": - participant_config["adversarial_args"]["fake_behavior"] = node_config["fake_behavior"] - participant_config["adversarial_args"]["attack_params"] = node_config["attack_params"] + def _configure_malicious_role(self, node_config: dict): + return { + "fake_behavior": node_config["fake_behavior"], + "attack_params": node_config["attack_params"] + } def _configure_trustworthiness(self) -> dict: trust_config = { @@ -469,7 +509,7 @@ def _configure_trustworthiness(self) -> dict: "post_hoc_methods": self.sd["post_hoc_methods"], "accountability_pillar": self.sd["accountability_pillar"], "factsheet_completeness": self.sd["factsheet_completeness"], - "architectural_soundness_pillar": self.sd["sd.architectural_soundness_pillar"], + "architectural_soundness_pillar": self.sd["architectural_soundness_pillar"], "client_management": self.sd["client_management"], "optimization": self.sd["optimization"], "sustainability_pillar": self.sd["sustainability_pillar"], @@ -522,7 +562,11 @@ def _situational_awareness_needed(self): return with_sa or enabled or arrivals_dep or additionals or mob def _configure_situational_awareness(self, index) -> dict: - scheduled_isolation = self._configure_arrivals_departures(index) + try: + scheduled_isolation = self._configure_arrivals_departures(index) + except Exception as e: + self.logger.info(f"ERROR: cannot configure arrival departures section - {e}") + snp = self.sd.get("sar_neighbor_policy", None) topology_management = snp if (snp != "") else self.sd["topology"] @@ -554,9 +598,10 @@ def _configure_situational_awareness(self, index) -> dict: return situational_awareness_config def _configure_arrivals_departures(self, index) -> dict: - if not self.sd["arrivals_departures_args"]["enabled"]: + arrival_dep_section = self.sd.get("arrivals_departures_args", None) + if not arrival_dep_section or (arrival_dep_section and not self.sd["arrivals_departures_args"]["enabled"]): return {"enabled": False} - + config = {"enabled": True} departures: list = self.sd["arrivals_departures_args"]["departures"] index_departure_config: dict = departures[index] @@ -573,49 +618,57 @@ def _configure_arrivals_departures(self, index) -> dict: ############################### """ - def build_preload_configuration(self, index, participant_config, log_dir, config_dir, cert_dir, advanced_analytics): - participant_config["scenario_args"]["federation"] = self.sd["federation"] - n_nodes = len(self.sd["nodes"].keys()) - n_additionals = len(self.sd["additional_participants"]) - participant_config["scenario_args"]["n_nodes"] = n_nodes + n_additionals - - participant_config["network_args"]["neighbors"] = self.tm.get_neighbors_string(index) - - participant_config["device_args"]["idx"] = index - participant_config["device_args"]["uid"] = hashlib.sha1( - ( - str(participant_config["network_args"]["ip"]) - + str(participant_config["network_args"]["port"]) - + str(participant_config["scenario_args"]["name"]) - ).encode() - ).hexdigest() + def build_preload_initial_node_configuration(self, index, participant_config: dict, log_dir, config_dir, cert_dir, advanced_analytics): + try: + participant_config["scenario_args"]["federation"] = self.sd["federation"] + n_nodes = len(self.sd["nodes"].keys()) + n_additionals = len(self.sd["additional_participants"]) + participant_config["scenario_args"]["n_nodes"] = n_nodes + n_additionals + + participant_config["network_args"]["neighbors"] = self.tm.get_neighbors_string(index) + + participant_config["device_args"]["idx"] = index + participant_config["device_args"]["uid"] = hashlib.sha1( + ( + str(participant_config["network_args"]["ip"]) + + str(participant_config["network_args"]["port"]) + + str(participant_config["scenario_args"]["name"]) + ).encode() + ).hexdigest() + except Exception as e: + self.logger.info(f"ERROR while setting up general stuff") - if participant_config["mobility_args"]["random_geo"]: - ( - participant_config["mobility_args"]["latitude"], - participant_config["mobility_args"]["longitude"], - ) = TopologyManager.get_coordinates(random_geo=True) - else: - participant_config["mobility_args"]["latitude"] = self.sd["latitude"] - participant_config["mobility_args"]["longitude"] = self.sd["scenario.longitude"] + try: + if participant_config.get("addons", None) and participant_config["addons"].get("mobility", None): + if participant_config["addons"]["mobility"].get("random_geo", None): + ( + participant_config["addons"]["mobility"]["latitude"], + participant_config["addons"]["mobility"]["longitude"], + ) = TopologyManager.get_coordinates(random_geo=True) + else: + participant_config["addons"]["mobility"]["latitude"] = self.sd["latitude"] + participant_config["addons"]["mobility"]["longitude"] = self.sd["longitude"] + except Exception as e: + self.logger.info(f"ERROR while setting up mobility parameters - {e}") - # If not, use the given coordinates in the frontend - participant_config["tracking_args"]["local_tracking"] = "advanced" if advanced_analytics else "basic" - participant_config["tracking_args"]["log_dir"] = log_dir - participant_config["tracking_args"]["config_dir"] = config_dir - # Generate node certificate - keyfile_path, certificate_path = generate_certificate( - dir_path=cert_dir, - node_id=f"participant_{index}", - ip=participant_config["network_args"]["ip"], - ) - participant_config["security_args"]["certfile"] = certificate_path - participant_config["security_args"]["keyfile"] = keyfile_path - if participant_config["device_args"]["start"]: - if not is_start_node: - is_start_node = True - else: - raise ValueError("Only one node can be start node") + try: + participant_config["tracking_args"] = {} + participant_config["security_args"] = {} + + # If not, use the given coordinates in the frontend + participant_config["tracking_args"]["local_tracking"] = "advanced" if advanced_analytics else "basic" + participant_config["tracking_args"]["log_dir"] = log_dir + participant_config["tracking_args"]["config_dir"] = config_dir + # Generate node certificate + keyfile_path, certificate_path = generate_certificate( + dir_path=cert_dir, + node_id=f"participant_{index}", + ip=participant_config["network_args"]["ip"], + ) + participant_config["security_args"]["certfile"] = certificate_path + participant_config["security_args"]["keyfile"] = keyfile_path + except Exception as e: + self.logger.info(f"ERROR while setting up tracking args and certificates") def build_preload_additional_node_configuration(self, last_participant_index, index, participant_config): n_nodes = len(self.sd["nodes"].keys()) @@ -648,9 +701,12 @@ def build_preload_additional_node_configuration(self, last_participant_index, in """ def create_topology_manager(self, config: Config): - self._topology_manager = ( - self._create_topology(config, matrix=self.sd["matrix"]) if self.sd["matrix"] else self._create_topology(config) - ) + try: + self._topology_manager = ( + self._create_topology(config, matrix=self.sd["matrix"]) if self.sd["matrix"] else self._create_topology(config) + ) + except Exception as e: + self.logger.info(f"ERROR: cannot create topology manager - {e}") def _create_topology(self, config: Config, matrix=None): """ @@ -742,8 +798,11 @@ def _create_topology(self, config: Config, matrix=None): return topologymanager def visualize_topology(self, config_participants, path, plot): - self.tm.update_nodes(config_participants) - self.tm.draw_graph(path, plot) + try: + self.tm.update_nodes(config_participants) + self.tm.draw_graph(path, plot) + except Exception as e: + self.logger.info(f"ERROR: cannot visualize topology - {e}") """ ############################### # DATASET CONFIGURATION # @@ -751,11 +810,14 @@ def visualize_topology(self, config_participants, path, plot): """ def configure_dataset(self, config_dir) -> NebulaDataset: - dataset_name = self.get_dataset_name() - dataset = factory_nebuladataset( - dataset_name, - **self._configure_dataset_config(dataset_name, config_dir) - ) + try: + dataset_name = self.get_dataset_name() + dataset = factory_nebuladataset( + dataset_name, + **self._configure_dataset_config(dataset_name, config_dir) + ) + except Exception as e: + self.logger.info(f"ERROR: cannot configure dataset - {e}") return dataset def _configure_dataset_config(self, dataset_name, config_dir): diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index 091e2b16c..91d5475bb 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -335,7 +335,7 @@ async def run_scenario( db_scenario = copy.deepcopy(scenario_data) # Manager for the actual scenario - scenarioManagement = ScenarioManagement(scenario_data, user) + #scenarioManagement = ScenarioManagement(scenario_data, user) # await update_scenario( # scenario_name=scenarioManagement.scenario_name, @@ -361,7 +361,7 @@ async def run_scenario( # logging.exception(f"Error docker-compose up: {e}") # return - return scenarioManagement.scenario_name + return ""#scenarioManagement.scenario_name @app.post("/scenarios/stop") From 9f0f891fd56c65b31ea681cc79963eaefb1bb9f2 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Tue, 22 Jul 2025 10:26:45 +0200 Subject: [PATCH 10/26] feature scenario building fix and complete. Fix references on backend --- False.png | Bin 0 -> 27701 bytes nebula/addons/gps/nebulagps.py | 4 +- nebula/addons/mobility.py | 14 ++-- nebula/addons/reputation/reputation.py | 9 ++- .../docker_federation_controller.py | 35 ++++++++-- nebula/core/addonmanager.py | 65 +++++++++++++----- nebula/core/engine.py | 10 +-- nebula/core/node.py | 9 +-- .../awareness/sareasoner.py | 10 +-- .../situationalawareness.py | 14 ++-- 10 files changed, 113 insertions(+), 57 deletions(-) create mode 100644 False.png diff --git a/False.png b/False.png new file mode 100644 index 0000000000000000000000000000000000000000..7bfbf233a10b097dbcf91a63dd36c91cba616c38 GIT binary patch literal 27701 zcmd?Ri9eNX_ddK4%_M{pwM)uO=0XEyu9V1JLYb!unX0=84U|lwGG$hhAtXd)DpSfl zhbSR4?{RkD-)H##e((DaywB%zKljky*S@avT<2QHajauqH;-x^Ub|}JDhh?NR#inw zn?j-0BY&-6z<(*S2y4aPWL=c?T#nmYxLi8zWKPjM?c!i#?_zWQ%$AGhPR{4;?RE>x z3X2PFIp^Zy;4CL1V*9TT2-`bZifq-Q`+|$CbWl0vOrbEHCV$a9RZKijp;$jrRoZ{T z_2Ed{CD#+??Mq`*ak}fbgltgpq0}9_=67|~({*>-ZNDYw`lk%%=Ubmz_|jDI>-&>O z3c`AKsau46&g@^O=SUy7Q^o$uU(x&A^Sr&c%SPU%(Cm;6e?M@+$l3gZXN`yDlm4E( z+L?C7;w@DCPpgOCp|hA~s3FPhk`Vb1!XonWg*#0fla3#zQCCt@DpTG2U3ee)hqj%j zH~swlWWs45;BS8S+`bt3;O{D$$B!P}XKZYIJ;Uwt<(|PoxAqaXOngDfpVLf5Rkf=j z>G;5A_Kh3WPMlz6^o=d~{Li0;{FUP8w}O_I_5=M+vo+!2;asv#%Y43l`?hK-)`u6j za^TYoQ`OvLlh=!~6*F^lbPm1mnDL+a9|||9j*@#djdEsNx6a|sGhVZr3(*h$+?UF( z0lAE!Gv)v5W_SJPy{qgh53A(<=SPnDUi7J|s#<+pi1YaK>Msq;$x|=;IWe(aLZZQn z>i+mp?BmCe2MopkUXM6EJcbY3^P2dxNe`VJDLaIOZpmF3lF3qF#{WVu$OgzKCMFKA zAZzsQ%U*f;+D%V{mH6@k@bCXxy8pTxzkC0G_}ARLX!*CV!gE-IR4SboprJ4bYMwBV`de<^s*HqUY@ zDH*D|yXW6Fz1gtgld5RoX$m8q(G{)Y;_j`+#iCnMrX7`%s<)}Pb;N|N@MY9FpGT#RD$%Hy z85RCoRWSd2Qj}e9wGC~_6|dq|TNb$@BkP!ck6v#t?CO2bzf0%uN<9M66GbIgj`-@Lo&QymoyxGjq`R@#6K4(lH*idoCI_>=4m*VYL=nr~aw& zFKe13o!ME-6~{H=OC$YMY~y`+yR`zZTx!qzRxNkbMd76VmQ6BRpC-E2cRt8kZvSF&8}7Z;yrZVYi2w-=bx#qu*i{K`103pb)!u7fZTF)+5)wollYHHV^)AJ+tF}vM1XW zJLEgmmCD}Ytmoc)v}MXe1Q)@vwzKT}5Hc~+68iI}0~h0RM&E0fIy`f#!U``>Iej*| z#LU8SEhdI%Y;3IH_g8Dy4IA{$PNwOeZ_nzKoj#k-=!Hw%yr#=GNuWyn!{Q z$h8`CzejH6@w8w{;wN1HWp(+$H!|GmoE9L zhi*LHu(mRPa%Z~i28{zrEuQh^4{2CT_SW!d1a!~&xb>$laEk5_`?Ra$_)Q#8=hC&( zdMz{-gLUyu!y$ zz2r&9apcI6Ebk>Rvp}&d%UUV?N!Ckjdeqp@pQ1jTqNc@e*6(U-rLlnG|ZZ5x!ZkuqP*00+C5+F z$`v+-)Fd|g`0kcXZ_T(&r8X!|>5VzJn`+mzTXZW83kl50Yeva6CcD)}%bOSZ>Y8{q zQY@x_cCoL~8>mk=+CvwXZuBampkVL4=xDnLDO)Q_{cCUUl%ilyO~FO3)$GDAq9gZQ z8dY9kR932a@Zf=kgz`?iM6ZR}>AAU>@yWV+AB4HN*9KPBOph5S$_>WNYCqZw)1F1i zU7*Uya8OK4I>wti2M1MMTqfwPqa>|2P@X+|*6KAkvW$VDF8_k{4$X+aaJ(p=K7Ep~ zZ|CwEh`xNgCTh=a*Po^Z1qB?^c3e5_?!(@fIJnm2bQpJCmRI}KU?S+iqkQ}-?}H6X zHZ(6piWK9fExCVc4sNDrc;I%YaweM6lwnf4yLRuvrNsqKMX!1K3SVDe7FJfJM6D>- zrG=@q4ofE|@knWV2Ge368do*CeZJlAq+PwbkT9JciA;5~jX zWMQ+v-wEVjtEeK+x%DSJu3kKFfOhkqi@jHtb7vYK(b7tHcy&Z^;XaBFa>!Ks<&5K2 zGb1@26um-sNrc;0aq(c=mX~6uOO~~JjB7sn#J zmqu}(EWbo4%!|6-U(jvgIk)03y30OGbm=tW{w9h?=XXD8kP3<){UK7A<=C@jd+)=T zLBo#YDk>|Gp~@;MdPYXp?z6Difq>c}C>Xfwc)~;H7=n+^eSCCoa4^PuF?{aF>zKr7 z_vCN8xnz!7Je-uJ1}LbWJh|cU9qu%ThrYhcaR`)SE_jC@5uiG$vS!9L-Mn?n{L6Fm zR)^BHuCA^Q-#%^~%^w!VgZfNXyncb-a6n0>`#C$DQJ;8(C$S;eq;=?#aiRP7*1UU2 zaGz3iMJVlFufDO4my{gfcI~8TN;9ZH#wbC~36sApg4*Ef_Wo`*(@Kfm ze(r_Mm*Bg1*I@~Cjoo;Ux@?cn%)Dz~G;`x^06U$-K-8u6931aHK0cUqJdyFJ#H3D5 z$0EbXSDKGEpCs+yneXo4l60E4!fGS?(q8SMY*LG5xPHD_&cVE0TnsF~waFlZN7gB>)zjPC8#!5k zOIJ_t-B@>dl4gWNo@pDE%KpTMH@hxbCuQf(os>@G@4buDJ(Rz$Uw5q6l%}~G8Clu< z%*<^tK6J&(m8aha^EQ17uWR?3v(?YGQcgLUCY*meI98`6$DU7KezVz{;ifdYEyBW# z$k<4Nsbh)@6Za$+CkE?ryFO2ns)ObFg-+3`KX8b1G;3d`73yAlDt=U2Xwgl5takIh z_u8MF1H9eoq-i?V^3=5Kxp;prHDn>WzdF*=*1&yZ6UV_rhZs0HInP>J8q``^THXr@ zp)bBlzgq9*_t)yjV&pD-kkjQy!9|_kC}pG4(k;b?1hrjJF$PgL-l%JGsv{e4F{jlj zAUh_{IY6PW`)xqll<-2n-@Cq#I)8Pa?5~Nk?(Xi^(9j6HV6Nxtnu`*?8z)7ooWuDo zn{L?0k4H&TWLdY))M&qqWeu0LqR8v{=h>d6H`Z3ZVBnB(cqEw~r%7XJ)U|<~-4d4y zjERY9JoC6W@7i*vjb?#g3NPn0I&Iy%x3#RiTz_tMy64Ul-Ic7jw=ZAD{iq9XC{hnA%oPK@yqN1;tT z-(HZna>WYK$>Gn^ZTS~cGHaU%`pLgAY%xDSuW|I~-NCk3IY5d$m4P_%lC=tRspr4G zNE-Y7TWak_2~n0E8lr=PgP{`uJG+F0gsNry=^k8A@b$}0EHv@1Zf!5*T;5Rlu@iXu z&GN_I-m5k<-eBUf(@c~+U-wtG_2>0$Y&wm+fX#6e6D|ecg%2ntR~$~!9hdIRD89d( z=d6pz-uYjv&z|+kk(RJoVU%CosLS3a(of?tKUVR*CFf3z*X+%Oxhdxs+ni@8n_#sz zvl3LQWF#RRwB;P#Jw0EZnGq<8Pt`v?78~%jqN~(bRYfIpR%PvGx%)s>r!@7ImEu&u zQqrlcNo&1k2l+03Z>HIYLN6j`xc}JUnYg&^DXPN{Wu0C7!z~}`C;IulI$FFwi9SIy z(xQ!Lw?!52k3`S8-{!aV3+I1TdQ5luh8jfs`L%pvee4%mSzDWUy7UHN*}#a}$B(aE zyS6aZQ|qBj&`Opa6y>NNp+W|S$dA3&V`i3qwCg~SerKe8(KERRd^>g+U(BU-#Rc8I zJ@%!%tE@Ep+4;t%qiEszlmN9`m^D+WwH145)yA{uVru>Gxh4bqQ0 zaWV&e)jIwc)lP8RwrlU+sfdeJTaK`IsfTLBo z%fv!oKfjbew{MW^uobCf-!TJv%Nqhd;M{lb-&Z<&b~{BtH@&xLZZvB{%?9SAG;%9? zgM)7>^W1-WoH=tw58w0h@}iJJQt<18fX**tDmDE8xtSqkN~Obx@5RI%i9K|y86h1a zMIW`>k`c)vi`)D67fW;Q0KbL}5mZ)3vl!3coHH{slt6Cz{11E5nXqS}CSG6MJt{U| z%Dg!8mEZFSb5MjD2%V;EO5V4(uD{;%dCdj`-DYH&BY(K^|w**TT!v zj*gC$7q%@Asw1U!TP_du_t&A;o=nv{UAQ#Vm@?VH!JH%sbk?1aa3sdA?UKchln>y& zU4YAp$^jcFSgh;J+Yaf~hft~6S3r{@#mevJxjQsGJ=SP^5EMeMz*TH=sFAJ{Xt*=z zF!!-J23(OWP!%Z77gXA#4bxew_jbH&0RCL9ur%*nP*_M4CUJf}3!m1D_zlcKLkXaK z7Rj+wI_AhH*Oum9FJ*Bn&VGv(G%jc^_g7L%_SJG?cicu#PcLED8dmDd1h}H9rsglH zX_<(ChzK=yU+MGc(H1Z)WguglbTy?ot8d#cF-vFRoG|hztl9V4Yr#qh(baXGJvAZL zvgYAdB%$QvN@bfNjN~%!aW~Pq0Sv?AIl;VgG|>6nM#wzGtN3xYE2>*V27HCyk1+ivdE#9m$ozC zvuaagPkJxSbV zZi<|%Q+Bwu}Twlsoe2q!r9iSBLJ})l?+}*Q{ zad=QU(0S>j_aXv}1Et$tuK};qI1DxHTF=5Fd+~c%wa!gHLj`<dOKPV{)-$B>}L@IdZU+}I969XCBl#=KA z=FOY!yu8e8Z1lyz8M(l|*`hD}awW+1EdqDd8@hnfN}Y5b1jntyI{uuRTJFCwp=i20 zKuVXF>NJQDGi(NynF0Q$$py(KsTy~3Lm;Cd7@)dAj&=&GDDK>^@n;z%K<9<4bdK05 zhGOByhEg*0@B9EBU0nF#-J%!d_b&bz4}Mz*LZgPpn)mPD1KG%(Ok{dy_Q_<35=tNGs5a2-H=^%p$4{jt65xafIj$m*g zXLBkwgzWT_R$KQ>31puBs_=sz9(yUT7iR@XI*3T}_iJz=+qM78moG|(4h2U?YaKnh zE-5K#*|KHLMi1)o!vTN&wd^*JB4hEwR7WkKnNd31b-xB-@~vY5>>@h=xqP-Bype03 zOY0X9VAfvXHa$H}`PqK?$JlEet=}7XsoE>aTD*%&TwZ&z?ZW3%dgwOPoBuQ{(Ph^IO>rXeR)*Y*tLtELQ6+CvpDx-up#Ll ziH!S+h)#A1pdTk^XBrM!r?Y7*bSd6*Egc+^R;(0c)rlV+9r^y+N^uFtAO(rVCoZqJ zz1iA~x3RLzLDc*MYkFS)MoRF#dxIUX6>&hH<%u%~{UW>k4kz){vorJ*a21!))-V13 z$)O;B;{&xh50-t+JWU5m9slT2aeIl9QVo!6_W?xWX;P~+(rk3sF*Cm*h;IB4yP#h2 zJwOO@n9Rwszgt&-4LI)Qv-Ov=&)46Ob?(<2h$&p09WGc}oFh^8%Y-b{neqN=oAv@u zACTVl0C3LAzlYjhN&hGqzdbqqW;stUipk4oROX;1Z9Eur_GItx@jP6??fbI|pC@lK zkdi(C%Gn|+g{mx01`ZtmS54`OKGj@ z>tsTyJ9qjag@1aqpN`U$da5+nu}8GL*)ZI%;UxAqsmsa9$BJ~KCHut?CM<+9kZOxRa+MCY%AB|&BS6APL zZ20gY7f2>fPC3-2(dSls&DJpO{pr)_ApsW3<2C)bX^N30Xy`F;!bZE%*1X={UW%ww zugdilH76Oaf`#8L6GM%weGmXaVPWSqvq85d=>>x}YSdo~t!ZhA{E}r^54nH>(pL^% zg{+x{uu57>Zt-bv@7V_;CXD;Ay?H~)DK_cvCLtF+cO5&EV&5#?;Zcet`SkZ%s&?O> zk8|#}(A#f{4AvCl=;Rn7X5P0&&=azcD#g$k-VR|bsH}~P)z#?~}f`WoBhW1HelXC|XB2)WgH_N%0HYV#Vqo@CZ7|30Xuu8f@;sVkHzjfJI-JVYZ3i9+?-)%}tN@QHVt!r;@zZj|`01a0B?7MX-Gv|=rtv{zX(lbD( z0QAwmRuw6I2cM@L3gmd~pmS^Q-o0T`wi}J|U1&@@iaghF$=pTpj}%~G)`fbkeNZD( zsyaVo@XZiA*m}wJ)#n?YGE)@if6_q#mcD-dx|Hq?YPt%IdK`m{$FyjN*PIYhuUtmH zHun2lB|?M9Jokx;^hpy2VXIbt>LJOW0>~XnG!``v&u`wvDeo@9=v#_UXWZVrhg7h= zd;J&3QTjxFft~zW>y0GL_vtb<3EsSUvtXt_vZ?3tkHXLp0o|`CX1Vxd=153Kx1bu+ ze2iCT2Q@c$R;Cq&l$0`f&e)h675dCTO%zFU=?8!)E#%yFcFS@xF3!|?6O2d!Xx|Um zlzJ$WN*#z*(4$I;mwzm9bH4ba{T86oawhKWa&lispT`mTD!uTxzSm42JB9R4fLU+e zxpRl)Bh7g7#W`$bGr;K47d8PUI@K)e*B{WDm|DWEvXI0mq#Q~#e*U=fT^Ya0Y4Bk)+j0}f@MtpIK5Wdx=HZRZLah3=mAL` z^kvLO&?Mc&{+@}25bgNAS&YK4hU1Y#A1$`|4bbX2`6A>GJnfU*NjqbqB&ME`ea{wOTO>nji+yR2our?*NkyCrhxSdL4w2N$Mia zxzFjF!JX^84=XSGh;P_J>OTU#jDpYV<~oTL6%~=VgN{L$={9bGTUYh+D*CeukFg4L zlI#~e!BqKn?YfCvK%40@sLg%(yV##qAk3ms5IPlt>8PHb3S{Rm+18=inVGrKKZ@pV zw-;VEIGnLV>*2eDO!8-uCC>2SXf~VTxZ@bmin>pnyDq@~Im_}05uJy}h^Q9Kt8SC3 zqGUz85NZ?`k(BpzE^lu1l^Yla%}znVvt@qFHeX-7C+AFyl#fGuE`EQ6g{8QBZw`W( z)@&FgqmIIcr?W(&rVymg<2@e)9*>g)jE@vll(f#5Eq16i z^(4zM;*2+zahTBy`j);yLbOR&c__4Hi_i7d>~*=0W)!davBRSU<66TH_b?~PBP%Dr zzoWSL4$3c0Zr6=X=bAGgSF@?=psZ1`Y+wWwKrAB0*tU^LwOaH^OCj z)hFn1*P*t3Mfy6y7u74Qz4q7m`1$6{t%q-Oz4`n(w4vd|fddD=3#%tLx7fd~E@e_& zN(X$fq1vuw3zaHw9VT77`6KZxx=%c(~ahTmNHRZSbx3^^83Xw zr~%D=(r5c@CFhPtJ%|ZfPFXpk%#I`-@{4=Ux(pX-*@b4!Oiznm8a32^agL@KT3m@A z^EQ;896Y@v1No$B;IoJTIW9(Mq{xceER8tB&|r^e7|T-#dE$2KWzM!==SW9x#1vvt zQ}9k88F}c!FY~BLNx1Nv0=My>zkf53a;EW6<`Z_W*9ke~1Wi<{FnbTSX8^b$^AC|l zk^F+JY$!~y-1GT)50DxSHMLa=3JTcWqfqc)Rjz0MIZ=OPyiQ$&W*zDZ$ob&M$JbCA z-o4f4!7cwpUm!R*n3hmk*7UeK0>Y8W5hEx7TnZk|<96Fxr|5zGRGkG10X=;xd9naTU$Fddq9a_00{-;0*7Ncfp=lMEpMNs zUFXP5*uZljNyq*w&PV4|S+5J^vEti=zHxD_DaJau4kaZLv?6>1RFCDn`_YH-s6AED z@|!>lDM;Sg;<_9CRjLsn%zWUI9O&Eh4h+<1o<8{dM@Po4&3QKH_4NZwFDtIB48V>B zZ?Z4=v!otGoKOOLefdC^Q_|{wNr=qb>9HQS-k`%iPd01;cRdZhOv07@$!iK#>&f8$ zB!2sXF$VB9x=w)GiQ&(8kkY^4gg+!ql;!)EEMM$5oddMnr?PP}Ay9uIYh^Cx5xwN! z#I@HDr}d%f?8IQxf|yx3D4k8}q1Iy#4x#}Ygs5_I8+~+APCEW5oU%H7E@&e}nO<>D zvkw|fx4GrNmRprYNg@Mc)w+KCU@CrbzTS20EjytA{9q_f;a-a~mM`tvR_&V_ZNvUO z%cDA40&sL8S7qN0R+#&q1)y0^;pe;-m=zz(1u5?To8SCzr~#T=zLX8$L91sxHt$AKa`Ax-XNlL1?L+*dGk z^w-WxZkBhS*okHA!7BdVu!HIYO+z@q-Q;Ez6dd$m7{SLVHf{MK`0~BbP)iWpC&&Qm z_oJdnX@uOiZ29tg5fN5KZM@o1IKj&3epTGzS>B2IX!)!6y%jt448e?sH=v3jx6yEj zpYE^lHZNWYGXI;r($Y>7^4gl0g%kFN zJ?AMo_g6jB+Rly6NJ&To$G@m6TtlF)F<=X*y-A)33gIge0cx6k;P~R^t3De#ucd(E zJ@H;z#DOYD9kYTjA}HVmQAyGbSMCD}_(oVV+s*rZ0|OOc4Uj5!Q8r205H`We%GzuY zrXobbB1kP`PRgQ+!)TR=q&6Hv;PWqQGK`=udM}R0AbR3{LyOH*0S-s=%hrsSI`BC` za}$!WS=6@a@=f-H<|EQoMjqj6%u=>r%N__BP`)*#Z2|(7(%tUl1%HACGFDSh$I{}I z$7G`}qDOlV(uU!($)feTE*+jzW6frFXp~pqXWAqcNMw0LVQb_pGMhH)yO@|5#Ibb{ znV_p_=;?*EncqC2wtqhjS{W}R?jZXU-gX79E0ldl#i47s53Is_d-K!BPWAB9v!(*V%df8SX zx7cpxvXgi%24wIH4BRd)9f}79>Kps@>p6bE(jxHm{PY+IJhaa&5P0-@$W3RH0=*u6 zBy~GGVG3xKcJBI|F2s+FBNZFM(20x&mB{H=uPRwJPaJvnNBvK_YjDtr6nL&JUy_0} zXZG#mV+1uJaMtkC$lk%CKR@$!6@}EJ5MAT*&dSsp*x)`u@L0*jw7e6T#T++e)v?8i z6M|`=DS%NZZ@thjr{B3h!N-AK&*$=LUZ0G|DFOa<@DA z{QkokaWCD`QAq83^!)krZ_s#v{Rm_J`~(bv4G=R9?TLT%OvcS}Fv$HyLMsvyf&1i; zxg^iX$H#OyN{}_)ku#=%S}IEtIk8JnkfmcCc(ZTA4FfvA4IULA)M|07hMmA7WD6t_ z;wSbMW+-ievVc+#4nn&RBn0>|7bzf@0sH!p7aIz|sujIgb>BV;WVeV7LHZ-;Pjo#n zu-mGr$m4@P65y-HN1O>OVSMF;DY1rVZ>3w#cp^7g|3^Zk-5F%IrXI9Ae)ff$fQ+eB zgY-LXFp}S?-Cj7=UKz7E!AHSeaVU7m7`(K3md76yp$?=K9~U1FJyiMPMX9c?uI+q$ zP`Pa{ouWz-y@=Gg#rYXZSy>qx0s2s)M}#!uZXmATog$h#4g?K{*qOgdLR5CaTLFrC zOV}u%l8GEd!Wc`AMTqN%^0rI4{u*`QKdCFoa32oM6L@FH!frtFI)jAPjo_h2-}W}P zI8tDJQus!Rb7aM?T)9H=K;L5TRI4-gS33(U%Ni-`yrZ1s(YJ3QrCphM)S!qIP_U|p zvMfw@o+xe;DVnAN^FJfGmJ8AKw!!d3Z#N?T7jyy0u!U2`$Pwp(4kh7kTakHT_@tCg zGo=_It%^XiO;;(C)Cd!OVsy)nU<_9+dxEY)XcOO2~IC0`WqAU0RgSwtjM1au{fdp)d=5K-I9WNT* zTR|;w>M3Xk?IEs&;Sw~S4#DIhugT;ua~NFgRHi8nX$L|633r@j=?eup;PRoN zW7uhUQ|8Uju2MHZiL{EG4zE3mi_@{lN+%TQ1(O=pjxa$kXex(lgwDb*_R-uPMhXH0 zwjs*g)Q=nl+7Lv%r0oCTt{EmVU~JZomdtz~-oJl*Bt`+r3#Iy*RU`jGA@nMIz7ZVt zFqrF6iTKA3yxoRit{zaT{upo2R@x0c4y4Hz@inx|R5dL=Ww~;)F>r&bn%Wb)wtR}9 zQU3i3?#op;Gs)2jE8?c$zCp zORphSgr%hCukyiRhZBo}B7=u6%OgK4RxWD$Rf2*r1POeB1MrpaAKn_L%0fZ$UUEPW zPj5ikKbH=Br2{fl3H(BnJ668w{KrRE01g_o09gd}U-s610voFz6RNZ2)ZgNehBggci6vP zHnKor4IoMgEN#t(A@AbrRY$c`h}Vxx-ktTI@M>7cBtLPNEff36DSmbZEF2_iD#;xj zdjEdp(cu1|i2iEnj$ZUR=!r3epzEFrBLCrntdAR3k=7dIx2h_Qp{6wTzd6puP+nxW z#6)6vgq3wo@rBPu6)=@u&&bGt;982i38!?Y*hfT`hFvZ;{1*K=~dM=~IJ7@d8{e_+nceCq$vhjr-O2c(tKxUuXam}|}phpq!P4`u0! z|BkHI6QCxls;ZzSbVQYp_V}fWCD;Y{ZD$F;SCc2Qcy10F2I5G?@Yd5z$1Kgn5dPW> zJ*4|2QtTlYmlqeS~%-y8<3qPLaKFBDvf_l<++ZcLSzn=3%@2N&H@5KY@yjXYhQxS5*nBg zdy;OkW|rBK4o!Z>y%uGw#LiCB4MO@#C_&+f=i@J%4EA1Rrs0q{w~n;5(<+eab1p233 zD?W;n1oxKLbk}lpCXjMl!x35rdPN?xzDzs}kL){&R_RjDILdHoL`d9(aKP+?u3m6h z*gL@V??M`_(KtH=b$mI)L;IO!BO%`*32aqR;Py!~$Pu?`7X9_>7tm2_I6^Gm+}xZQ zAvRc_NKAOUcI|>?wFP0`uwzcwc^c=QZ#f=B;MTBz@h4q z@|Ry;2p&T6XG>R#t0TV~()5~?OTaig}Whm;JB;2|JN(*^DU;OzIH`x{xCxdIWYQI~5EU6W6d6NqRRS0j}vnLWJC8!=NSFIwJQ@HI(DwEDWUCMyr6fH<}6gbcX;ae{N z)D=JvA@@Fr3oO@+#hq`ik!@Ck73C)BOM3o>9C%$^=!{YgMu-e=q;s2U3E1t!VFlZv#-w_JVSdmhZ%>WS3#r43phVgjb1&-&6SB1 zxEDe4m^d_sgM_>S-kiu?A7s_@-|Us(07!UW4Z|DGBF5K~7{B=I_wW9!T7_6Nb13t_ zTkO}s&xEl4b4}po0{nL7ANKmvMm1E}2x`BQRV>_;kU3ZUH+yB%E)L#kPsr;6WH5c* z2|vFRIsX^(8nhnhpeLkA(EX2lgi56a2{pqMLsJjYnh9)0L6Z!keb8_YD!K)@A_a{( z9o-Q>m0O}+F;O6-HxM z_=b@ZzQci+^z4DK(FbHZ3UR6;${Sqa{@I4ze93(zO!NxKTS>WpnHH8|TS$?6W-tF{ zn9xR{H1*x&i>5q!^vDMSV3v912Gr3#1g$@W$^wRp610^{y$az>XjG6Y9E&9a>c?7` z!7=yhq7XD>v`QOMRkgQON9_p()Bn2#i(K~>c>9RLOqss)1+-vWFSgj8H8+Ruae@!% z*%1H?nFACZPTn|_RtkyMJwFQW!6=UC8bUFbR6`TIGE`*wBe(H>(q|_C$m#q)9CrWy z{S?U4k(M!@P5sGn`8%<|;%4P*0>$=y`Xu06dK!8LH8lM{IBbqc;lz5_!-A@HR4*Jw zG!h!!Z3msF0(tAa{36^z6>tF|Athf!cySO@kK)q&ZVG913F*Jc%-O=s%oi?t`f8(# zgowx;2x3`g6^sB1$#J;d8@G*b`n`gwVICXW3Aur$6UK9LtYDLaYvTkP3A4vo!{%l^ zNc6r?t+BvEoZF~&uvtfu9BHNSHTEzQ6u%pPcVBlWG2upl6;5LfDR4@tcYzyf|I--4 z;qYM-I7w=m0sEBS9C@hycqnZHA~K*N!4ylGMgwLB0vy^`-8RS`Eq(p{>!GRMWZ{D& ziw}7Vh=Sy;ByD0|+;%81lNrj0&ga71KKVC)GG z+IBcNCdS^`+({O7=j6X}m|rs)T7eBQ>|ZTd&xvE$vP!gQUlN(U!()8yzq|WNO5f{& z-H9MGtg=WNhS)>%RKwL?Ogs+YS`--fSuV9Y`d%1KZa`Z?dxStT(D${)aKESk6q9Fa&;X4^ zZ|Ib|A34;o7bi1N`9rW^jaqy;0e&Zp0i=lYfty>|`?-ndq)viHIBbwkTlM)5C+iRr zyHSEv(dn5Pkd0*T-!R#Aq50L1b`}F^5B;0Oo={TnQ^|&i#tADK9hu4j5Tu4od_2f> zibep01-jW!Yy;Bx!vNX`Cj^i{{S5Qr?YRf=PD?5w4zT6Ik(36(2n+ zkhRfN}Z31J8F z$DigaL24ZfqnG~Z*J{U$gROQ$Di|(GSgx!jY(#pmFU~P9E-q3$&}%P%ry9!@55YKD z*~*F^Yw-wYJ}is_>uhV}B#5+Rf&O|^upDK#WesV|qo9#2eh8h|=lM)`xaIY6%F9*) ziI${f6j{j^jo0j%nR>kh_m%bm@4h(wZ-Bu;$v|XB$j}OfScFDiwlt%&3!fKOsSp!N zKhaa6{rJr6aC&a{DyVPrhqNSp=yxq-CYEwx#pj?YiHY~jy|5HX~=+$q%_6-U5QA7LCpUWN&FgC)$Sc%Lxf5S1kP0bY=3aBuLdWC z9z#BiFIt?nzRe{|yYI}IC#aArlI~<;A|V#GMxxtev?($r8HR+o4`gwf=b$nYZ3bhU ztM;J|!H>_kZCfeUVL8fl$h&v%h!q$l_`tz~;G&<@eo(1W|LG(|f@rs0Z?C4R-KIpDQDMO3e z1)UZ?-s?hndA*oNwklr!ZJ7ieBi)x`zuTL`^E_n#y76^r~=4UMZ?)bWm zCnq|PF(Zmw9^EbwVY5b4AQGum|C$xZ$i#v@?XK{>lFq70&nW0sv-)X6iN*0m_yQhDGF3tq zPsAK4JMnN)V6lzG^ce{_YP_4mqKwh;<7pLn#rmc!>li2KPEuBev~?A@{Y+Gp%~Sgf zX^|pLj}8)GcQjm+XkGyOO**es;%oL9gPWnC zJocDl1oXoP_wTb(W`B>QAiHVbvuQV!Ur2Ux!dQLEqNkw`?E@L*!Fw%@b1>fDfM7jz zIYY=G`A4CZD7|2w4H~FbQG3{ZU^O7wmOv}*^6=e_Q^uT}_&>=o+6RU&34ue%G&rqgk`(R6l0@AHso?uN!|RO)Y8r`{Etr0oQqTkEtMnYwT#AiQ3_%PtB@q3D*94b^@ICrlk?FkoQBc|oJQ9c^)OGx$y_ zjV`q*!x;$O9`lgnNcG3`toHW;MTp_tppUm=&6+rz4Kf&wTyK(j1H6wY>@e0{houSG z5%%DdGEx{mAu|a;Vco;S>YyK_i3Xj1o12>(nWiAcM<6|y7PEJd@_>Oc2b`BKR+iYv%M@K&}11y2mg^eP+Q~)Im!I@Za4|asOan0icnZcGx7UZ zotIzAAa=N6a)tO9aoo$^z8!-(^`41U+FR_37;+JM?*6Y|zn+ZB(0lg8a0Kt8sP;4T z$HW1r|H)ILq=nYj=jZpZo;2z)f2)Dqk+x)~8NK(5`vNlU4otRDcA#$ued-OGzZ;A9 z{(85qDHl__HIyAxY6--KcGt_74dx;xt()AeW!!$n(P+RCuL%X4Mgs<{JVzMq`jb=A z#eK*mG{h+>SV+o&{eV?Tksg+>y)^#ompE)El4ghQNDXe?TJCdU_}JbZ|HqiO_MBHj zgUs*-7BkNqK!D+p+lfcGKF`^b1p~ddd(gX7Td0A$uL(8tgGT~XBK-^zlcH8ic|}D9 z8AH!ofN0Z{ODfFQA_&O|utYcg_z7SOhL6^s2CDEHQ#5ZiF%SA@bUP1L!!*w}w}B`I z(OVlO|J)}mo|N+X`g$n(_?m{ke#Eij;7X^F?9cj3AMKhm-;6rJ&<^p%YzHtJY%7T+ zK9eZ z&v>v;2>unW6Dwjfg;DeT^^_0Gb`>{dm~@a?dH4A{-t(mZ%1cw#F&@uLd?d4yd-nq80j-pmzTEEb+>5qn()JZ)Ea!GD zrdU8^A6Tm=7Js$v3lIiz3eZ&D-Fh%ktQ@UPgDXFP<}g6H8!oM3c}SAD#r~M3`NYXq zwK4X$2x$^a*l7w-B>V;4xKo&prr3Uc0g}P^ht}DlkAsfd(!-3< zXod;2N5-?jO8Y89>EJ&*3em{+)<{DV3uaCzsQS@446r3ZT!VRRn~>02pSD+*oB`*s z`x;QYIAxu-Lq{^oi%>&?!0~!Qgb%*UQasl{Z77?6_}q|DH?fh)_4W9MTQ z4)2PX4WJ*5J*SpgR@WUP6wq9bZW6Y>f(WPiN8E&^w-G z!YcCPxKn-rIh$U*J?eK};Yj7q4yB|GWWWV6>^1nqfJR;T3sOwl9K!ldmqWAOEzGqJ zw&aKq4vb(XAn#8ba}wtIRXKAopj5B7(} z$E?cyH8PcE1{Kfm&K*7lg?4vglrp?Y!WYfUF=TI@>B9RPn3?w@rGr8eeE{f5EYJ1) zBdAXIIW0YJzk6~7Gv15hJ;f& z!C3rVSXC)byGBy^T%Z_uwU~6P!!^u{iJ@WN4?!fm=GjaPx&=;ivt()usc?;{vDb__ znPMe+@pv&EQ6zg1jQr3D zhSpb+`JEi|B0gF5giOE2x&~ei?s{f zuL{y~S_R4Gs8UjXm_?4-S$(7j}kGBvW0K0sTscBjb{Ff%bjtkn%wpy zA;R{dxBvTr9FmmN45V2$Z?>iU8M3aoynYY*z{XRO7 zenuGL$BUy_Sj@nTH823Pa?KhE?0?f`hGomb_@Z6AAGR=ZikGdDFXzZYOWe8zwbG!6 zWE4t0#u@RN4zTwO8Zyw0SAB?@6h|5y&^ycve5Yw>_z;5>FJv*f1~-Tp z!hFZ}?QwO7<8~?}vvP2p!`yBa<1FJG_2Vk(D#CmOQZNw9Pw^7_S;+`;X z5Vf?8W@ct-o}PH9@~|fLLhRf*JNBg+-+>%ap z9C+oQS(cZuLS26c-ntMDz0aUym#M=1&x6*jq(vfclCOCXB~?{RORAi8!FIU4@YUo( zwpHVYe1B)~xuznN69sD%xcNXU04-k%)8WIUd`Ebt#Seuaw#1er2$ZK|# zl$FDwrsyw(1t!UuM!^C(KbhezscYmGpEU5pem>w@(>+tIhmsduS5!rrmh7~u{`jG^ zbRJ#1^4|XFsjMQLt&ObW7ES#L@ zi6yKEh0bXXqdF0!K?PAy5(7yV7@)I2;xaFXohlqItT2>H4m7;$VKo_tmoB1JJ0^R3 zgO1BEUeA-wbJ($%-}(C0cQb*Tq{)NQ5+48=+b1-I4Si*y9G5EW%?Fo$1$F>UqKC-2 zd^2r>7sFmtIxi$uFie(3VTsFYcBQuVha&FHYjurRf_(q%a)`djPK%1Q)AYUa`x*~ z5TTvKU{b0^aZFo>(aas>1!(B{3+JYlb`e3p@3tE3)q02H%XlH($HO)l17=6Nj)%v@ zLdS(UsVz_pZ%oK&J(dLQqGL6OCWX`C_^)`AM*6EfKLOxuweUX!$rz}cRa6Pm=ij~i zo{+)I`?CD_xdidbmk*DiXn%Nd03?YHwW_rApg_*e%jAo@oz2ujmhlKVBRqNi0Ey8ZoZmA48woBaj7+E*pxza z(H`Y5g3N!yY7PU1o%3 zf{CR*IOY}foc5Df8z9k9T0O?AA)iVD5LWQx`f>JcGazN52671)E7X@e)GNNr^t~J0 z{c-&mrm%$M+E>y549=hJ`Mypp-X6pQ1S!#9fIZ0*p)>IR(v6BcDFbONe4UQdGgb2k zx5p+XQXK}vV7tuLV@_g#3_Gs>{On5fuxL#`KH85v(jf{c_Az7+e>MOco@3)iePI| zU!eUngenl^{0?<)gM_fG9EnmCUx92-8ylNE(MFtTq0QU6yC-yy^T)vv_d&cIj+Qs8 zLT`(e8#>JS`1s%-KYhRAce#%B@qK5QUGM^xt711mg@|H3$ifp3bj9M((}Q z2fPI1c)EN zs7W+_0P%-qg0K8(_Xp_o#8dVhBX}t{yV3V$OYaTFj8-NVPJF)=;_e}miFQjxdHDkz zUTpp5koC-=(DHmRiqp+$hE~fyECpVPmfBE<*RNe^suSZqoJgLOxHne)-j0M4tTZQdc5TiGg?7lLKjC{PmMlpAQdm$6I|&7eG&rPdXR$| z17-Ny;5VJ>dk-FzBh`N33i*mwTs#uY&!10`U?7ZH4e2VyMGS7A(ogP<`;E>*C#n|C zeS&ZDF?g);!nIvE_XxL;uSKYDCw=(O`#y>{gEGz#)oUEEfIjHUrYo;7p3It_d=!&c zun-7eV(UO99lbWJ>MNYarm<#2|ht zM=3Hf)&t|lx4?>5+nKlzhSd#dT)%PSq~{nF?-8I+)mPDFIw{DMF={asU7=}7d zmYYBNm=Qm?2a3Wb_GFpnvu`H<#bGc&cBe_{`@-J6BJB_PopKJ z37zaaf8mF|zCN8?7Y!>bE99iWK*7(9`w_K%@2&&xZ`!$1?@trLZ_B_t=9j(w!ACxE z{j@S&|C9OS)9Jjzw%Rn({VZuFVFLPzRPEOW!RPh8`_UjbG;?9 zGus8*+6$F#y&1sOVJZJdPmo%8sPz105n*|OZ6GEN2$UCud?T`__=3e3LhB3MQ@>Vk^{L~3#pKNt z=1~4)Zb~+r#8hL`A0Hb_8#PH~kzo{FMn>j=Rr5J}t)S*ox`FeJMLXqk;TaB@n9k;7 z(Xo~LA}JvPGd#LHeVh?1vMfK|jvL(e5dotZs7BI~pJUW0NKZyKRwzy{0In|Uom;mS zfipg0txsVEiex=i(~euusdxRm_Mq;dq4dV(&DB*4(a$SCSE-Qq)QC}uL%~Ts9bXQ; ztJ!V||F_mffMB*#slzGnu+0}I;13NOm7rC=%G+nk702-#2Y<|to9;$jQlT4s8by`o z#%pjQ9(X^uxjz)@T(sSL6qEWB)zU4b7=2T%%B|#hEmyFtA*d@;N8h^oLf-s+N%=+R zlWXCY!iAPC;nOXZOowVIcM{oxvG5Eekv4}OLAB?I6J=uZqfwL`stQ7r+Mg;pZ%Fkv zwsN*CTCKZ$R`zjr@7jy-nI79=s_+~b!!)hg_gDy1!E}=f)BW4ckOvN)gAMq zN4#Ej>|E*zPW|=GL$*tSxaJQMv$Gx0%%CS8!7zeXe_D2|>APup95V-8KnvM0R}I4k zo$$Nz&ZH#|BRI8;)RTi|*3IW+MMcXa!>j7;MdB|831ul3eEf>CIi)={>uK68g`}ze z^!{sCizh$e8(81oj|rAd9g`SQ%JJ&kDI;_4C%k~Er{`jWBQ&H-$eWV!VT^G0Vw)x4 z9sntTm%l5HeKJ%q(h!S>jGP<~7AORTg#u-Myl@KWrVtceJK*xb3QMR2Og$NTQES|% z5h?Ww%IBM4K+wqHejEb^cgcOPs%i+d#f0JoPSS(@0uq1Xz;kxTZEpiueTr30%bh|%KuDkZTi6LyVeY&6)ZeZ^zyOMruvz z5vNyYY;A1JVV$ayaD(SZVNpvFs1#62!;CXf*m*THA}(fP1$nxvbi@VIhLESkHxZ0f zLBmglg->vh3`4jEJ(9L*1$k#o*AEBx;~rw|a7$Dn{x3q}eD5>RM`C!OpmtFM4v=q? zi#YS!L=8X=Ahrkd6DLk&=shmr>V+oDU8f1sFaTx1a2hEj_V@XJAS`giK5K^1XRuxk z#^6j$hNA9?uej!-(CM!n1PR2Z>~LZgC=-4fagU8d*5#O_!XPo6z5bYA6zZO6 z_$zFt*)oMvn(M8#8I%+5v;J3v^hWvRM&6vs%TN00=Uwv;V7$KvWv&W+Ajstsxc?j&V~hnmDOd?u;$bzAM8RgY-tH zYTsLnEXW@4o1jfx$0~OEH;j7UGqfQ@87(y=Cyb>3wp(mP!mLzO)VhD8YnWKMheX}g zT(lTuGKxo|dw16bpFQo3!E>X408jI?XG`^Jz(b(6ifnN{`ST6#1K!?B#Poyt5%=H* z;Fp)9jy-xhnws!rWOB^_p6>#Q#Um)0n4T_zBR>R}#;&?26fQWX&P^}M8`(xxP`HnHHG&qd+=7>@*p5~QR?P&h7Xy$47HK=h+Kt}$-&&UYzL98@+= zUoA2Ph=S)7)kG17=UXsg_67~qW4c)yW z-rf;`$^%#j&Pfs&mTl7xW}2rNR(as1f>`Ce5qLt3cYvfh!62>7jRd^zz64z%++@VF zdDn%m4A@e+sU(rEj>xO8bLI*?DWkq_f-_OgY6rfWo#0J~`YNUGPJsHo4+X&vg#sz> z8i^4DK=3B$>mMTMGOUv+hWrxLk&HX}pth0KccZ+#+$_7FiY@XYduAEZGJoLU^F6kQ zg(Bm)cpoQL>KgkUBeGGlN4?ySAXJQK3Az(&h?vE3A!Z;T*f?U3fZ0Sr0SGS``vTI8 zU|H|yE`j0?XdHuIofRD)&r3j4)bdtqR?yA7@h6tUH_7N{3Av$<4*Z{bdX)l9BCctg5MDo_) z#lE8R_WF}?`AuRd;h?rQAVmZg!c9~OQ{*EJL#KIsmfgm|0t<`CD^*Z~c=Wlb7Lln# zBivGv8v@7wYtn8COLYGhMv~&Z1N7=_7@y#N%K|1(*qsUFuIzzhKA@A?RzEDDu=>@H zRqm_MEtTEB|2mPhV#2-#ln8+}A)QQUQfx$hM9WFiA%yEBJ!BY(K)eZ@Ef(qn8xCNz z14TGeknSq8V@Is)p){Itg;_Yt<0(YTV-v3((=JiQE1sxQ;zAdQ{+%Z|8o^okj6hXw;)G z7pnVy+H9dd`!?`dS8?FPxm-19@BUx|5V@7Z{PD-~WMzmOn8jraUzVsa`W^(h#6v)V z9>qa}TM~kSN>{DG@sz;(c5o5l;1JXw^`5gF_8o|p$!`R$VCv%0a73n|;nP+#wCH2D z%7};zo4)!C(!?jKfq&2v&ZHp?pR4Jyk41S=1EMb{C&#EAS0o9$ZEYDi#M|5xYuE{j z=ANE+Ntq}9@%z$Q@F7&?5*t}xBk<<0ow=az|3%EPyECTpzVE+~16~zYb1ZD3sQ7PQ zH1vDbY>|}ofj8yf*nuh`&;AFcpp=wqTbcd+*tS|evwdq;hf0(d+R10GxF_jQmA)sx z+`Z-E<|hTtdzUYs0!?+S`Fh>-J~_pStlX;ayWJJSg^qsS zwQ5s@XJf$D{zGkdWk4nmI*aP05|P(M<`PM0-3S<55jYPfuUW zK}H5cL$8T!S=Gb#v49}nAY2f$yb}|_4=KRgoQ!v%<{6^UFGGGgGdGuq^766Q+JDUf z`oH%Bq}?RdWdP;Xw?&i8f6%~`ASDxD^p`abB_U~8>`lzg!TA&Nx(l2h=`Z+s0@R~I zHDQZh5cPxNO~T;F3x1xwqXI<9SxdzYKrq)pqjnb^5=mZO@!Ralt-Yy)=Y7H>Zi-^G zR?LBfPNo<4`?Dq+cgxDQr0g6Vb!^eHyMij?PlB{1B|E{BffOL`nb2`V?J|*iiv>@V z(!WE59W(}v*`Y_kSMCH)zbK(qTma%h8`6C@ww+Qq!>v|bH8cB_D7?E0$S@{HlHO-*c-oE+bQ zz&+=+H;dWbk+_JrXgSh|hmTu{9=N=x#Kn> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{self.scenario_name}/participant_{node['device_args']['idx']}.json", + f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json", ] networking_config = client.api.create_networking_config({ diff --git a/nebula/core/addonmanager.py b/nebula/core/addonmanager.py index 46fbfd60a..f3bf3a505 100644 --- a/nebula/core/addonmanager.py +++ b/nebula/core/addonmanager.py @@ -1,5 +1,6 @@ import logging from typing import TYPE_CHECKING +from abc import ABC, abstractmethod from nebula.addons.functions import print_msg_box from nebula.addons.gps.gpsmodule import factory_gpsmodule @@ -10,6 +11,15 @@ if TYPE_CHECKING: from nebula.core.engine import Engine +class NebulaAddon(ABC): + @abstractmethod + async def start(): + raise NotImplementedError + + @abstractmethod + async def stop(): + raise NotImplementedError + class AddondManager: """ @@ -51,24 +61,43 @@ async def deploy_additional_services(self): - Services are only launched if the corresponding configuration flags are set. """ print_msg_box(msg="Deploying Additional Services", indent=2, title="Addons Manager") - if self._config.participant["trustworthiness"]: - from nebula.addons.trustworthiness.trustworthiness import Trustworthiness - - trustworthiness = Trustworthiness(self._engine, self._config) - self._addons.append(trustworthiness) - - if self._config.participant["mobility_args"]["mobility"]: - mobility = Mobility(self._config, verbose=False) - self._addons.append(mobility) - - update_interval = 5 - gps = factory_gpsmodule("nebula", self._config, self._engine.addr, update_interval, verbose=False) - self._addons.append(gps) - - if self._config.participant["network_args"]["simulation"]: - refresh_conditions_interval = 5 - network_simulation = factory_network_simulator("nebula", refresh_conditions_interval, "eth0", verbose=False) - self._addons.append(network_simulation) + for addon, addon_config in self._config.participant["addons"].items(): + if addon == "mobility": + mobility = Mobility(self._config, verbose=False) + self._addons.append(mobility) + update_interval = 5 + gps = factory_gpsmodule("nebula", self._config, self._engine.addr, update_interval, verbose=False) + self._addons.append(gps) + elif addon == "trustworthiness": + from nebula.addons.trustworthiness.trustworthiness import Trustworthiness + trustworthiness = Trustworthiness(self._engine, self._config) + self._addons.append(trustworthiness) + elif addon == "network_simulation": + #TODO review parameters for network simulation + type_of_network = self._config.participant["network_args"]["network_simulation"]["type"] + network_config = self._config.participant["network_args"]["network_simulation"]["network_config"] + network_simulation = factory_network_simulator(type_of_network, network_config) + self._addons.append(network_simulation) + #TODO update config access + + # if self._config.participant["trustworthiness"]: + # from nebula.addons.trustworthiness.trustworthiness import Trustworthiness + + # trustworthiness = Trustworthiness(self._engine, self._config) + # self._addons.append(trustworthiness) + + # if self._config.participant["mobility_args"]["mobility"]: + # mobility = Mobility(self._config, verbose=False) + # self._addons.append(mobility) + + # update_interval = 5 + # gps = factory_gpsmodule("nebula", self._config, self._engine.addr, update_interval, verbose=False) + # self._addons.append(gps) + + # if self._config.participant["network_args"]["simulation"]: + # refresh_conditions_interval = 5 + # network_simulation = factory_network_simulator("nebula", refresh_conditions_interval, "eth0", verbose=False) + # self._addons.append(network_simulation) for add in self._addons: await add.start() diff --git a/nebula/core/engine.py b/nebula/core/engine.py index fc2fef4f6..5d420816b 100644 --- a/nebula/core/engine.py +++ b/nebula/core/engine.py @@ -160,12 +160,12 @@ def __init__( self._addon_manager = AddondManager(self, self.config) # Additional Components - if "situational_awareness" in self.config.participant: + if "situational_awareness" in self.config.participant["addons"]: self._situational_awareness = SituationalAwareness(self.config, self) else: self._situational_awareness = None - if self.config.participant["defense_args"]["reputation"]["enabled"]: + if self.config.participant["addons"]["reputation"]["enabled"]: self._reputation = Reputation(engine=self, config=self.config) @property @@ -620,9 +620,9 @@ async def deploy_components(self): """ await self.aggregator.init() if "situational_awareness" in self.config.participant: - await self.sa.init() - if self.config.participant["defense_args"]["reputation"]["enabled"]: - await self._reputation.setup() + await self.sa.start() + if self.config.participant["addons"]["reputation"]["enabled"]: + await self._reputation.start() await self._reporter.start() await self._addon_manager.deploy_additional_services() diff --git a/nebula/core/node.py b/nebula/core/node.py index adc83fe1d..8fbeeb28f 100755 --- a/nebula/core/node.py +++ b/nebula/core/node.py @@ -77,7 +77,7 @@ async def main(config: Config): model_name = config.participant["model_args"]["model"] idx = config.participant["device_args"]["idx"] - additional_node_status = config.participant["mobility_args"]["additional_node"]["status"] + additional_node_status = config.participant["deployment_args"]["additional"] # Adjust the total number of nodes and the index of the current node for CFL, as it doesn't require a specific partition for the server (not used for training) if config.participant["scenario_args"]["federation"] == "CFL": @@ -209,9 +209,10 @@ def randomize_value(value, variability): await node.deploy_federation() if additional_node_status: - time = config.participant["mobility_args"]["additional_node"]["time_start"] - logging.info(f"Waiting time to start finding federation: {time}") - await asyncio.sleep(int(config.participant["mobility_args"]["additional_node"]["time_start"])) + # time = config.participant["addons"]["mobility"]["additional_node"]["time_start"] + # logging.info(f"Waiting time to start finding federation: {time}") + # await asyncio.sleep(int(config.participant["addons"]["mobility"]["additional_node"]["time_start"])) + await asyncio.sleep(120) #TODO REMOVE await node._aditional_node_start() if node.cm is not None: diff --git a/nebula/core/situationalawareness/awareness/sareasoner.py b/nebula/core/situationalawareness/awareness/sareasoner.py index 40e6de94d..743187b14 100644 --- a/nebula/core/situationalawareness/awareness/sareasoner.py +++ b/nebula/core/situationalawareness/awareness/sareasoner.py @@ -85,9 +85,9 @@ def __init__( title="SA Reasoner", ) logging.info("🌐 Initializing SAReasoner") - self._config = copy.deepcopy(config.participant) + self._config = copy.deepcopy(config.participant["addons"]) self._addr = config.participant["network_args"]["addr"] - self._topology = config.participant["mobility_args"]["topology_type"] + self._topology = config.participant["addons"]["mobility"]["topology_type"] self._situational_awareness_network: SANetwork | None = None self._situational_awareness_training = None self._restructure_process_lock = Locker(name="restructure_process_lock", async_lock=True) @@ -96,11 +96,11 @@ def __init__( self._suggestion_buffer = SuggestionBuffer(self._arbitrator_notification, verbose=True) self._communciation_manager = CommunicationsManager.get_instance() self._sys_monitor = SystemMonitor() - arb_pol = config.participant["situational_awareness"]["sa_reasoner"]["arbitration_policy"] + arb_pol = config.participant["addons"]["situational_awareness"]["sa_reasoner"]["arbitration_policy"] self._arbitatrion_policy = factory_arbitration_policy(arb_pol, True) self._sa_components: dict[str, SAMComponent] = {} self._sa_discovery: ISADiscovery | None = None - self._verbose = config.participant["situational_awareness"]["sa_reasoner"]["verbose"] + self._verbose = config.participant["addons"]["situational_awareness"]["sa_reasoner"]["verbose"] @property def san(self) -> SANetwork | None: @@ -146,7 +146,7 @@ def is_additional_participant(self): Returns: bool: True if the node is marked as an additional participant, False otherwise. """ - return self._config["mobility_args"]["additional_node"]["status"] + return self._config["addons"]["mobility"]["additional_node"]["status"] """ ############################### # REESTRUCTURE TOPOLOGY # diff --git a/nebula/core/situationalawareness/situationalawareness.py b/nebula/core/situationalawareness/situationalawareness.py index 6a5dbcbd6..5cdeec03d 100644 --- a/nebula/core/situationalawareness/situationalawareness.py +++ b/nebula/core/situationalawareness/situationalawareness.py @@ -1,6 +1,6 @@ import asyncio from abc import ABC, abstractmethod - +from nebula.core.addonmanager import NebulaAddon from nebula.addons.functions import print_msg_box @@ -156,7 +156,7 @@ def factory_sa_reasoner(sa_reasoner, config) -> ISAReasoner: raise Exception(f"SA Reasoner service {sa_reasoner} not found.") -class SituationalAwareness: +class SituationalAwareness(NebulaAddon): """ High-level coordinator for Situational Awareness in the DFL federation. @@ -178,16 +178,16 @@ def __init__(self, config, engine): title="Situational Awareness module", ) self._config = config - selector = self._config.participant["situational_awareness"]["sa_discovery"]["candidate_selector"] + selector = self._config.participant["addons"]["situational_awareness"]["sa_discovery"]["candidate_selector"] selector = selector.lower() - model_handler = config.participant["situational_awareness"]["sa_discovery"]["model_handler"] + model_handler = config.participant["addons"]["situational_awareness"]["sa_discovery"]["model_handler"] self._sad = factory_sa_discovery( "nebula", - self._config.participant["mobility_args"]["additional_node"]["status"], + self._config.participant["addons"]["mobility"]["additional_node"]["status"], selector, model_handler, engine=engine, - verbose=config.participant["situational_awareness"]["sa_discovery"]["verbose"], + verbose=config.participant["addons"]["situational_awareness"]["sa_discovery"]["verbose"], ) self._sareasoner = factory_sa_reasoner( "nebula", @@ -214,7 +214,7 @@ def sar(self): """ return self._sareasoner - async def init(self): + async def start(self): """ Initialize both discovery and reasoner components, linking them together. """ From 778bfacb9fa5ea596d8eb66d00885c8aabc05ea4 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Tue, 22 Jul 2025 12:11:45 +0200 Subject: [PATCH 11/26] feature additionals late deployment logic --- .../docker_federation_controller.py | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/nebula/controller/federation/docker_federation_controller.py b/nebula/controller/federation/docker_federation_controller.py index 25eb6af43..091c91c6f 100644 --- a/nebula/controller/federation/docker_federation_controller.py +++ b/nebula/controller/federation/docker_federation_controller.py @@ -1,3 +1,4 @@ +import datetime import glob import json import os @@ -25,7 +26,7 @@ def __init__(self, wa_controller_url, logger): self.controller = "" self.config = Config(entity="scenarioManagement") self.round_per_node = {} - self.additionals: list[tuple[str, int]] = [] + self.additionals: dict[str, int] = {} """ ############################### # ENDPOINT CALLBACKS # @@ -38,7 +39,7 @@ async def run_scenario(self, scenario_data: Dict, role: str, user: str): await self._initialize_scenario(scenario_data) generate_ca_certificate(dir_path=self.cert_dir) await self._load_configuration_and_start_nodes() - self._start_nodes() + self._start_initial_nodes() return self.sb.get_scenario_name() @@ -49,7 +50,24 @@ async def remove_scenario(self, scenario_name: str): pass async def update_nodes(self, scenario_name: str, request: Request): - pass + config = await request.json() + participant_idx = str(config["device_args"]["idx"]) + participant_round = int(config["federation_args"]["round"]) + + self.round_per_node[participant_idx] = participant_round + federation_round = min(self.round_per_node.values()) + + additionals_deployables = [ + idx + for idx, round in self.additionals.items() + if federation_round >= round + ] + + #TODO deploy additionals deployables + # update self.round_per_node -> add additional nodes that are going to + # be deployed + + #TODO get the others parameters """ ############################### # FUNCTIONALITIES # @@ -236,8 +254,7 @@ async def _load_configuration_and_start_nodes(self): dataset.initialize_dataset() self.logger.info(f"✅ Splitting {self.sb.get_dataset_name()} dataset... Done") - #TODO delay additionals deployment until conditions - def _start_nodes(self): + def _start_initial_nodes(self): self.logger.info("Starting nodes using Docker Compose...") self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) @@ -245,16 +262,15 @@ def _start_nodes(self): container_ids = [] for idx, node in enumerate(self.config.participants): if node["deployment_args"]["additional"]: - self.additionals.append((idx, int(node["deployment_args"]["deployment_round"]))) + self.additionals[idx] = int(node["deployment_args"]["deployment_round"]) continue # deploy initial node + self.round_per_node[idx] = 0 # Ordered list for additional participants deployment - ordered = sorted(self.additionals, key=lambda t: t[1]) - self.additionals = ordered - for an in self.additionals: - self.logger.info(f"Additional node: {an}") + for an, anr in self.additionals.items(): + self.logger.info(f"Additional node: {an}:{anr}") def _start_node(self): """ From 98add5f8a98ef9e7f42730b02271037dcb521cf0 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Wed, 23 Jul 2025 13:51:57 +0200 Subject: [PATCH 12/26] feature federation API request parameters using pydantic. Directories changed --- nebula/controller/federation/api_requests.py | 14 ++ .../docker_federation_controller.py | 144 ++++++++++++++++-- .../physicall_federation_controller.py | 0 .../processes_federation_controller.py | 0 .../factory_federation_controller.py | 15 ++ .../controller/federation/federation_api.py | 33 ++-- .../federation/federation_controller.py | 21 +-- .../controller/federation/scenario_builder.py | 3 + nebula/controller/web_app_controller.py | 2 +- nebula/database/redis/Dockerfile | 1 - nebula/database/rediscommander/Dockerfile | 1 - 11 files changed, 174 insertions(+), 60 deletions(-) create mode 100644 nebula/controller/federation/api_requests.py rename nebula/controller/federation/{ => controllers}/docker_federation_controller.py (73%) rename nebula/controller/federation/{ => controllers}/physicall_federation_controller.py (100%) rename nebula/controller/federation/{ => controllers}/processes_federation_controller.py (100%) create mode 100644 nebula/controller/federation/factory_federation_controller.py delete mode 100644 nebula/database/redis/Dockerfile delete mode 100644 nebula/database/rediscommander/Dockerfile diff --git a/nebula/controller/federation/api_requests.py b/nebula/controller/federation/api_requests.py new file mode 100644 index 000000000..55f393b7f --- /dev/null +++ b/nebula/controller/federation/api_requests.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel +from typing import Dict, Any + +class InitFederationRequest(BaseModel): + experiment_type: str + +class RunScenarioRequest(BaseModel): + scenario_data: Dict[str, Any] + user: str + federation_id: str + +class StopScenarioRequest(BaseModel): + federation_id: str + \ No newline at end of file diff --git a/nebula/controller/federation/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py similarity index 73% rename from nebula/controller/federation/docker_federation_controller.py rename to nebula/controller/federation/controllers/docker_federation_controller.py index 091c91c6f..5e96f2d9f 100644 --- a/nebula/controller/federation/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -33,7 +33,7 @@ def __init__(self, wa_controller_url, logger): ############################### """ - async def run_scenario(self, scenario_data: Dict, role: str, user: str): + async def run_scenario(self, id: str, scenario_data: Dict, user: str): #TODO maintain files on memory, not read them again self._user = user await self._initialize_scenario(scenario_data) @@ -43,11 +43,89 @@ async def run_scenario(self, scenario_data: Dict, role: str, user: str): return self.sb.get_scenario_name() - async def stop_scenario(self, scenario_name: str, username: str, all: bool): - pass + async def stop_scenario(self, id: str): + """ + Remove all participant containers and the scenario network. + Reads ALL scenario.metadata and removes all listed containers and the network, then deletes the metadata file. + Also forcibly stops and removes any containers still attached to the network before removing it. + """ + # Try multiple possible config directory locations. This depends on where the user called the function from. + possible_config_dirs = [ + os.environ.get("NEBULA_CONFIG_DIR"), + "/nebula/app/config", + "./app/config", + os.path.join(os.getcwd(), "app", "config"), + os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "app", "config"), + ] - async def remove_scenario(self, scenario_name: str): - pass + config_dir = None + for dir_path in possible_config_dirs: + if dir_path and os.path.exists(dir_path): + config_dir = dir_path + break + + if not config_dir: + self.logger.warning("No valid config directory found, skipping cleanup") + return + + scenario_dirs = [] + self.logger.info(f"Config directory: {config_dir}") + if os.path.exists(config_dir): + for item in os.listdir(config_dir): + scenario_path = os.path.join(config_dir, item) + if os.path.isdir(scenario_path): + metadata_file = os.path.join(scenario_path, "scenario.metadata") + if os.path.exists(metadata_file): + scenario_dirs.append(scenario_path) + + self.logger.info(f"Removing scenario containers for {scenario_dirs}") + if not scenario_dirs: + self.logger.info("No active scenarios found to clean up") + return + + client = docker.from_env() + + for scenario_dir in scenario_dirs: + metadata_path = os.path.join(scenario_dir, "scenario.metadata") + if not os.path.exists(metadata_path): + self.logger.info(f"Skipping {scenario_dir} - no scenario.metadata found") + continue + + with open(metadata_path) as f: + meta = json.load(f) + + # Remove containers listed in metadata + for name in meta.get("containers", []): + try: + container = client.containers.get(name) + container.remove(force=True) + self.logger.info(f"Removed scenario container {name}") + except Exception as e: + self.logger.warning(f"Could not remove scenario container {name}: {e}") + + # Remove network, but first forcibly remove any containers still attached + network_name = meta.get("network") + if network_name: + try: + network = client.networks.get(network_name) + attached_containers = network.attrs.get("Containers") or {} + for container_id in attached_containers: + try: + c = client.containers.get(container_id) + c.remove(force=True) + self.logger.info(f"Force-removed container {c.name} attached to {network_name}") + except Exception as e: + self.logger.warning(f"Could not force-remove container {container_id}: {e}") + network.remove() + self.logger.info(f"Removed scenario network {network_name}") + except Exception as e: + self.logger.warning(f"Could not remove scenario network {network_name}: {e}") + + # Remove metadata file + try: + os.remove(metadata_path) + except Exception as e: + self.logger.warning(f"Could not remove scenario.metadata: {e}") async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() @@ -87,6 +165,9 @@ async def _initialize_scenario(self, scenario_data): self.cert_dir = os.environ.get("NEBULA_CERTS_DIR") self.advanced_analytics = os.environ.get("NEBULA_ADVANCED_ANALYTICS", "False") == "True" self.config = Config(entity="scenarioManagement") + self.env_tag = os.environ.get("NEBULA_ENV_TAG", "dev") + self.prefix_tag = os.environ.get("NEBULA_PREFIX_TAG", "dev") + self.user_tag = os.environ.get("NEBULA_USER_TAG", os.environ.get("USER", "unknown")) self.controller = f"{os.environ.get('NEBULA_CONTROLLER_HOST')}:{os.environ.get('NEBULA_CONTROLLER_PORT')}" @@ -254,6 +335,26 @@ async def _load_configuration_and_start_nodes(self): dataset.initialize_dataset() self.logger.info(f"✅ Splitting {self.sb.get_dataset_name()} dataset... Done") + def get_network_name(self, suffix: str) -> str: + """ + Generate a standardized network name using tags. + Args: + suffix (str): Suffix for the network (default: 'net-base'). + Returns: + str: The composed network name. + """ + return f"{self.env_tag}_{self.prefix_tag}_{self.user_tag}_{suffix}" + + def get_participant_container_name(self, idx: int) -> str: + """ + Generate a standardized container name for a participant using tags. + Args: + idx (int): The participant index. + Returns: + str: The composed container name. + """ + return f"{self.env_tag}_{self.prefix_tag}_{self.user_tag}_{self.sb.get_scenario_name()}_participant{idx}" + def _start_initial_nodes(self): self.logger.info("Starting nodes using Docker Compose...") @@ -271,7 +372,7 @@ def _start_initial_nodes(self): # Ordered list for additional participants deployment for an, anr in self.additionals.items(): self.logger.info(f"Additional node: {an}:{anr}") - + def _start_node(self): """ Starts participant nodes as Docker containers using Docker SDK. @@ -297,10 +398,11 @@ def _start_node(self): Note: - The method assumes Docker and NVIDIA runtime are properly installed and configured. - IP addresses in node configurations are replaced with network base dynamically. - """ + """ self.logger.info("Starting nodes using Docker Compose...") - network_name = f"{os.environ.get('NEBULA_CONTROLLER_NAME')}_{str(self._user).lower()}-nebula-net-scenario" + network_name = self.get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") + base_network_name = self.get_network_name("net-base") # Create the Docker network base = DockerUtils.create_docker_network(network_name) @@ -310,10 +412,10 @@ def _start_node(self): self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) i = 2 container_ids = [] - for idx, node in enumerate(self.config.participants): - self.logger.info(f"Deploying participant {idx}...") + container_names = [] # Track names for metadata + for idx, node in enumerate(self.config.participants): image = "nebula-core" - name = f"{os.environ.get('NEBULA_CONTROLLER_NAME')}_{self._user}-participant{node['device_args']['idx']}" + name = self.get_participant_container_name(node["device_args"]["idx"]) if node["device_args"]["accelerator"] == "gpu": environment = { @@ -346,21 +448,27 @@ def _start_node(self): ] networking_config = client.api.create_networking_config({ - f"{network_name}": client.api.create_endpoint_config( + network_name: client.api.create_endpoint_config( ipv4_address=f"{base}.{i}", ), - f"{os.environ.get('NEBULA_CONTROLLER_NAME')}_nebula-net-base": client.api.create_endpoint_config(), + base_network_name: client.api.create_endpoint_config(), }) node["tracking_args"]["log_dir"] = "/nebula/app/logs" node["tracking_args"]["config_dir"] = f"/nebula/app/config/{self.sb.get_scenario_name()}" node["scenario_args"]["controller"] = self.controller - node["scenario_args"]["deployment"] = "docker" + node["scenario_args"]["deployment"] = self.sb.get_deployment() node["security_args"]["certfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_cert.pem" node["security_args"]["keyfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_key.pem" node["security_args"]["cafile"] = "/nebula/app/certs/ca_cert.pem" node = json.loads(json.dumps(node).replace("192.168.50.", f"{base}.")) # TODO change this + try: + existing = client.containers.get(name) + self.logger.warning(f"Container {name} already exists. Deployment may fail or cause conflicts.") + except docker.errors.NotFound: + pass # No conflict, safe to proceed + # Write the config file in config directory with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f: json.dump(node, f, indent=4) @@ -382,6 +490,12 @@ def _start_node(self): try: client.api.start(container_id) container_ids.append(container_id) + container_names.append(name) except Exception as e: self.logger.exception(f"Starting participant {name} error: {e}") - i += 1 \ No newline at end of file + i += 1 + + # Write scenario-level metadata for cleanup + scenario_metadata = {"containers": container_names, "network": network_name} + with open(os.path.join(self.config_dir, "scenario.metadata"), "w") as f: + json.dump(scenario_metadata, f, indent=2) \ No newline at end of file diff --git a/nebula/controller/federation/physicall_federation_controller.py b/nebula/controller/federation/controllers/physicall_federation_controller.py similarity index 100% rename from nebula/controller/federation/physicall_federation_controller.py rename to nebula/controller/federation/controllers/physicall_federation_controller.py diff --git a/nebula/controller/federation/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py similarity index 100% rename from nebula/controller/federation/processes_federation_controller.py rename to nebula/controller/federation/controllers/processes_federation_controller.py diff --git a/nebula/controller/federation/factory_federation_controller.py b/nebula/controller/federation/factory_federation_controller.py new file mode 100644 index 000000000..11882c4db --- /dev/null +++ b/nebula/controller/federation/factory_federation_controller.py @@ -0,0 +1,15 @@ +from nebula.controller.federation.federation_controller import FederationController + +def federation_controller_factory(mode: str, wa_controller_url: str, logger) -> FederationController: + from nebula.controller.federation.controllers.docker_federation_controller import DockerFederationController + from nebula.controller.federation.controllers.processes_federation_controller import ProcessesFederationController + from nebula.controller.federation.controllers.physicall_federation_controller import PhysicalFederationController + + if mode == "docker": + return DockerFederationController(wa_controller_url, logger) + elif mode == "physical": + return PhysicalFederationController(wa_controller_url, logger) + elif mode == "processes": + return ProcessesFederationController(wa_controller_url, logger) + else: + raise ValueError("Unknown federation mode") \ No newline at end of file diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index dd19b8a2f..e8ec30ee3 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -8,7 +8,9 @@ from functools import wraps from fastapi import HTTPException from nebula.utils import LoggerUtils -from nebula.controller.federation.federation_controller import FederationController, federation_controller_factory +from nebula.controller.federation.federation_controller import FederationController +from nebula.controller.federation.factory_federation_controller import federation_controller_factory +from nebula.controller.federation.api_requests import InitFederationRequest, RunScenarioRequest, StopScenarioRequest def require_initialized_controller(func): @wraps(func) @@ -46,11 +48,12 @@ async def read_root(): logger.info("Test curl succesfull") return {"message": "Welcome to the NEBULA Federation Controller API"} +#TODO modificar para q reciba str en vez de dict @app.post("/init") -async def init_federation_experiment(payload: dict = Body(...)): +async def init_federation_experiment(ifr: InitFederationRequest): global fed_controller - experiment_type = payload["type"] + experiment_type = ifr.experiment_type logger = logging.getLogger("Federation-Controller") logger.info(f"Experiment type received: {experiment_type}") @@ -67,31 +70,15 @@ async def init_federation_experiment(payload: dict = Body(...)): @app.post("/scenarios/run") @require_initialized_controller -async def run_scenario( - scenario_data: dict = Body(..., embed=True), - role: str = Body(..., embed=True), - user: str = Body(..., embed=True), -): +async def run_scenario(run_scenario_request: RunScenarioRequest): global fed_controller - return await fed_controller.run_scenario(scenario_data, role, user) + return await fed_controller.run_scenario(run_scenario_request.federation_id, run_scenario_request.scenario_data, run_scenario_request.user) @app.post("/scenarios/stop") @require_initialized_controller -async def stop_scenario( - scenario_name: str = Body(..., embed=True), - username: str = Body(..., embed=True), - all: bool = Body(False, embed=True), -): - global fed_controller - return await fed_controller.stop_scenario(scenario_name, username, all) - -@app.post("/scenarios/remove") -@require_initialized_controller -async def remove_scenario( - scenario_name: str = Body(..., embed=True), -): +async def stop_scenario(stop_scenario_request: StopScenarioRequest): global fed_controller - return await fed_controller.remove_scenario(scenario_name) + return await fed_controller.stop_scenario(stop_scenario_request.federation_id) @app.post("/nodes/{scenario_name}/update") @require_initialized_controller diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index c7f55a5d8..087d6eb9a 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -20,31 +20,14 @@ def logger(self): return self._logger @abstractmethod - async def run_scenario(self, scenario_data: Dict, role: str, user: str): + async def run_scenario(self, id: str, scenario_data: Dict, user: str): pass @abstractmethod - async def stop_scenario(self, scenario_name: str, username: str, all: bool): - pass - - @abstractmethod - async def remove_scenario(self, scenario_name: str): + async def stop_scenario(self, id: str): pass @abstractmethod async def update_nodes(self, scenario_name: str, request: Request): pass -def federation_controller_factory(mode: str, wa_controller_url: str, logger) -> FederationController: - from nebula.controller.federation.docker_federation_controller import DockerFederationController - from nebula.controller.federation.processes_federation_controller import ProcessesFederationController - from nebula.controller.federation.physicall_federation_controller import PhysicalFederationController - - if mode == "docker": - return DockerFederationController(wa_controller_url, logger) - elif mode == "physical": - return PhysicalFederationController(wa_controller_url, logger) - elif mode == "processes": - return ProcessesFederationController(wa_controller_url, logger) - else: - raise ValueError("Unknown federation mode") \ No newline at end of file diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index 76e48e074..0f5942743 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -45,6 +45,9 @@ def get_additional_nodes(self): def get_dataset_name(self) -> str: return self.sd["dataset"] + + def get_deployment(self) -> str: + return self.sd["deployment"] """ ############################### # SCENARIO CONFIG NODE # diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index 91d5475bb..c675e1a18 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -324,7 +324,7 @@ async def run_scenario( url = f"http://{fed_controller_host}:{fed_controller_port}/init" url2 = f"http://{fed_controller_host}:{fed_controller_port}/scenarios/run" data = {"type": "docker"} - data2 = {"scenario_data": scenario_data, "role": role, "user": user} + data2 = {"scenario_data": scenario_data, "federation_id": "id_nebula", "user": user} await APIUtils.post(url, data) await APIUtils.post(url2, data2) except Exception as e: diff --git a/nebula/database/redis/Dockerfile b/nebula/database/redis/Dockerfile deleted file mode 100644 index 31ff6af02..000000000 --- a/nebula/database/redis/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM redis:latest \ No newline at end of file diff --git a/nebula/database/rediscommander/Dockerfile b/nebula/database/rediscommander/Dockerfile deleted file mode 100644 index 0eb1e1ead..000000000 --- a/nebula/database/rediscommander/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM rediscommander/redis-commander:latest \ No newline at end of file From 347d0f045e5edeb1506e379e0f7941d85627ab4e Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 24 Jul 2025 13:47:32 +0200 Subject: [PATCH 13/26] fix participants deployment --- nebula/addons/gps/nebulagps.py | 4 +- nebula/addons/mobility.py | 14 +- .../nebulanetworksimulator.py | 2 +- nebula/config/config.py | 16 +- .../docker_federation_controller.py | 211 ++++++++---------- .../controller/federation/federation_api.py | 8 +- .../federation/federation_controller.py | 4 +- .../controller/federation/scenario_builder.py | 64 +++++- nebula/controller/web_app_controller.py | 2 +- nebula/core/engine.py | 8 +- nebula/core/network/communications.py | 2 +- nebula/core/node.py | 5 - nebula/core/noderole.py | 4 +- .../awareness/sareasoner.py | 4 +- .../awareness/suggestionbuffer.py | 2 +- .../situationalawareness.py | 2 +- 16 files changed, 185 insertions(+), 167 deletions(-) diff --git a/nebula/addons/gps/nebulagps.py b/nebula/addons/gps/nebulagps.py index c67a3b5d4..b1025cf22 100644 --- a/nebula/addons/gps/nebulagps.py +++ b/nebula/addons/gps/nebulagps.py @@ -74,8 +74,8 @@ async def is_running(self): return self._running.is_set() async def get_geoloc(self): - latitude = self._config.participant["addons"]["mobility_args"]["latitude"] - longitude = self._config.participant["addons"]["mobility_args"]["longitude"] + latitude = self._config.participant["addons"]["mobility"]["latitude"] + longitude = self._config.participant["addons"]["mobility"]["longitude"] return (latitude, longitude) async def calculate_distance(self, self_lat, self_long, other_lat, other_long): diff --git a/nebula/addons/mobility.py b/nebula/addons/mobility.py index d7e8d0f43..fdf4265a4 100755 --- a/nebula/addons/mobility.py +++ b/nebula/addons/mobility.py @@ -57,7 +57,7 @@ def __init__(self, config, verbose=False): self._mobility_task = None # Track the background task # Mobility configuration - self.mobility = self.config.participant["addons"]["mobility"]["mobility"] + self.mobility = self.config.participant["addons"]["mobility"]["enabled"] self.mobility_type = self.config.participant["addons"]["mobility"]["mobility_type"] self.grace_time = self.config.participant["addons"]["mobility"]["grace_time_mobility"] self.period = self.config.participant["addons"]["mobility"]["change_geo_interval"] @@ -267,11 +267,11 @@ async def set_geo_location(self, latitude, longitude): if latitude < -90 or latitude > 90 or longitude < -180 or longitude > 180: # If the new location is out of bounds, we keep the old location - latitude = self.config.participant["mobility_args"]["latitude"] - longitude = self.config.participant["mobility_args"]["longitude"] + latitude = self.config.participant["addons"]["mobility"]["latitude"] + longitude = self.config.participant["addons"]["mobility"]["longitude"] - self.config.participant["mobility_args"]["latitude"] = latitude - self.config.participant["mobility_args"]["longitude"] = longitude + self.config.participant["addons"]["mobility"]["latitude"] = latitude + self.config.participant["addons"]["mobility"]["longitude"] = longitude if self._verbose: logging.info(f"📍 New geo location: {latitude}, {longitude}") cle = ChangeLocationEvent(latitude, longitude) @@ -301,8 +301,8 @@ async def change_geo_location(self): """ if self.mobility and (self.mobility_type == "topology" or self.mobility_type == "both"): random.seed(time.time() + self.config.participant["device_args"]["idx"]) - latitude = float(self.config.participant["mobility_args"]["latitude"]) - longitude = float(self.config.participant["mobility_args"]["longitude"]) + latitude = float(self.config.participant["addons"]["mobility"]["latitude"]) + longitude = float(self.config.participant["addons"]["mobility"]["longitude"]) if True: # Get neighbor closer to me async with self._nodes_distances_lock: diff --git a/nebula/addons/networksimulation/nebulanetworksimulator.py b/nebula/addons/networksimulation/nebulanetworksimulator.py index 9af1f768e..85efbb2a2 100644 --- a/nebula/addons/networksimulation/nebulanetworksimulator.py +++ b/nebula/addons/networksimulation/nebulanetworksimulator.py @@ -35,7 +35,7 @@ def cm(self): async def start(self): logging.info("🌐 Nebula Network Simulator starting...") self._running.set() - grace_time = self.cm.config.participant["mobility_args"]["grace_time_mobility"] + grace_time = self.cm.config.participant["addons"]["mobility"]["grace_time_mobility"] # if self._verbose: logging.info(f"Waiting {grace_time}s to start applying network conditions based on distances between devices") # await asyncio.sleep(grace_time) await EventManager.get_instance().subscribe_addonevent( diff --git a/nebula/config/config.py b/nebula/config/config.py index d5f3a813f..1e0a5eaa7 100755 --- a/nebula/config/config.py +++ b/nebula/config/config.py @@ -208,14 +208,14 @@ def add_neighbor_from_config(self, addr): if not neighbors: self.participant["network_args"]["neighbors"] = [addr] - self.participant["mobility_args"]["neighbors_distance"][addr] = None + self.participant["addons"]["mobility"]["neighbors_distance"][addr] = None else: if addr not in neighbors: self.participant["network_args"]["neighbors"].append(addr) - self.participant["mobility_args"]["neighbors_distance"][addr] = None + self.participant["addons"]["mobility"]["neighbors_distance"][addr] = None def update_nodes_distance(self, distances: dict): - self.participant["mobility_args"]["neighbors_distance"] = {node: dist for node, (dist, _) in distances.items()} + self.participant["addons"]["mobility"]["neighbors_distance"] = {node: dist for node, (dist, _) in distances.items()} def update_neighbors_from_config(self, current_connections, dest_addr): final_neighbors = [n for n in current_connections if n != dest_addr] @@ -224,10 +224,10 @@ def update_neighbors_from_config(self, current_connections, dest_addr): self.participant["network_args"]["neighbors"] = final_neighbors # Update neighbors location - self.participant["mobility_args"]["neighbors_distance"] = { - n: self.participant["mobility_args"]["neighbors_distance"][n] + self.participant["addons"]["mobility"]["neighbors_distance"] = { + n: self.participant["addons"]["mobility"]["neighbors_distance"][n] for n in final_neighbors - if n in self.participant["mobility_args"]["neighbors_distance"] + if n in self.participant["addons"]["mobility"]["neighbors_distance"] } logging.info(f"Final neighbors: {final_neighbors} (config updated)") @@ -241,8 +241,8 @@ def remove_neighbor_from_config(self, addr): neighbors.remove(addr) self.participant["network_args"]["neighbors"] = neighbors - if addr in self.participant["mobility_args"]["neighbors_distance"]: - del self.participant["mobility_args"]["neighbors_distance"][addr] + if addr in self.participant["addons"]["mobility"]["neighbors_distance"]: + del self.participant["addons"]["mobility"]["neighbors_distance"][addr] def reload_config_file(self): diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index 5e96f2d9f..225f5926a 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -14,8 +14,8 @@ class DockerFederationController(FederationController): - def __init__(self, wa_controller_url, logger): - super().__init__(wa_controller_url, logger) + def __init__(self, hub_url, logger): + super().__init__(hub_url, logger) self._user = "" self.root_path = "" self.host_platform = "" @@ -23,10 +23,14 @@ def __init__(self, wa_controller_url, logger): self.log_dir = "" self.cert_dir = "" self.advanced_analytics = "" - self.controller = "" + self.url = "" self.config = Config(entity="scenarioManagement") self.round_per_node = {} self.additionals: dict[str, int] = {} + self._ip_last_index = 0 + self._network_name = "" + self._base_network_name = "" + self._base = "" """ ############################### # ENDPOINT CALLBACKS # @@ -169,7 +173,7 @@ async def _initialize_scenario(self, scenario_data): self.prefix_tag = os.environ.get("NEBULA_PREFIX_TAG", "dev") self.user_tag = os.environ.get("NEBULA_USER_TAG", os.environ.get("USER", "unknown")) - self.controller = f"{os.environ.get('NEBULA_CONTROLLER_HOST')}:{os.environ.get('NEBULA_CONTROLLER_PORT')}" + self.url = f"{os.environ.get('NEBULA_CONTROLLER_HOST')}:{os.environ.get('NEBULA_FEDERATION_CONTROLLER_PORT')}" # Create Scenario management dirs os.makedirs(self.config_dir, exist_ok=True) @@ -357,143 +361,108 @@ def get_participant_container_name(self, idx: int) -> str: def _start_initial_nodes(self): self.logger.info("Starting nodes using Docker Compose...") + network_name = self.get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") + base_network_name = self.get_network_name("net-base") + + # Create the Docker network + base = DockerUtils.create_docker_network(network_name) self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) - i = 2 + self._ip_last_index = 2 container_ids = [] for idx, node in enumerate(self.config.participants): + self.logger.info(f"Deployment starting for participant {idx}") if node["deployment_args"]["additional"]: self.additionals[idx] = int(node["deployment_args"]["deployment_round"]) + self.logger.info(f"Participant {idx} is additional. Round of deployment: {int(node['deployment_args']['deployment_round'])}") continue - # deploy initial node + # deploy initial nodes self.round_per_node[idx] = 0 + self._start_node(node, network_name, base_network_name, base, self._ip_last_index) + self._ip_last_index += 1 # Ordered list for additional participants deployment for an, anr in self.additionals.items(): self.logger.info(f"Additional node: {an}:{anr}") - def _start_node(self): - """ - Starts participant nodes as Docker containers using Docker SDK. - - This method performs the following steps: - - Logs the beginning of the Docker container startup process. - - Creates a Docker network specific to the current user and scenario. - - Sorts participant nodes by their index. - - For each participant node: - - Sets up environment variables and host configuration, - enabling GPU support if required. - - Prepares Docker volume bindings and static network IP assignment. - - Updates the node configuration, replacing IP addresses as needed, - and writes the configuration to a JSON file. - - Creates and starts the Docker container for the node. - - Logs any exceptions encountered during container creation or startup. - - Raises: - docker.errors.DockerException: If there are issues communicating with the Docker daemon. - OSError: If there are issues accessing file system paths for volume binding. - Exception: For any other unexpected errors during container creation or startup. - - Note: - - The method assumes Docker and NVIDIA runtime are properly installed and configured. - - IP addresses in node configurations are replaced with network base dynamically. - """ - self.logger.info("Starting nodes using Docker Compose...") - - network_name = self.get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") - base_network_name = self.get_network_name("net-base") - - # Create the Docker network - base = DockerUtils.create_docker_network(network_name) - + def _start_node(self, node, network_name, base_network_name, base, i): client = docker.from_env() self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) - i = 2 container_ids = [] container_names = [] # Track names for metadata - for idx, node in enumerate(self.config.participants): - image = "nebula-core" - name = self.get_participant_container_name(node["device_args"]["idx"]) - - if node["device_args"]["accelerator"] == "gpu": - environment = { - "NVIDIA_DISABLE_REQUIRE": True, - "NEBULA_LOGS_DIR": "/nebula/app/logs/", - "NEBULA_CONFIG_DIR": "/nebula/app/config/", - } - host_config = client.api.create_host_config( - binds=[f"{self.root_path}:/nebula", "/var/run/docker.sock:/var/run/docker.sock"], - privileged=True, - device_requests=[docker.types.DeviceRequest(driver="nvidia", count=-1, capabilities=[["gpu"]])], - extra_hosts={"host.docker.internal": "host-gateway"}, - ) - else: - environment = {"NEBULA_LOGS_DIR": "/nebula/app/logs/", "NEBULA_CONFIG_DIR": "/nebula/app/config/"} - host_config = client.api.create_host_config( - binds=[f"{self.root_path}:/nebula", "/var/run/docker.sock:/var/run/docker.sock"], - privileged=True, - device_requests=[], - extra_hosts={"host.docker.internal": "host-gateway"}, - ) - - volumes = ["/nebula", "/var/run/docker.sock"] - - start_command = "sleep 10" if node["device_args"]["start"] else "sleep 0" - command = [ - "/bin/bash", - "-c", - f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json", - ] - - networking_config = client.api.create_networking_config({ - network_name: client.api.create_endpoint_config( - ipv4_address=f"{base}.{i}", - ), - base_network_name: client.api.create_endpoint_config(), - }) - - node["tracking_args"]["log_dir"] = "/nebula/app/logs" - node["tracking_args"]["config_dir"] = f"/nebula/app/config/{self.sb.get_scenario_name()}" - node["scenario_args"]["controller"] = self.controller - node["scenario_args"]["deployment"] = self.sb.get_deployment() - node["security_args"]["certfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_cert.pem" - node["security_args"]["keyfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_key.pem" - node["security_args"]["cafile"] = "/nebula/app/certs/ca_cert.pem" - node = json.loads(json.dumps(node).replace("192.168.50.", f"{base}.")) # TODO change this - - try: - existing = client.containers.get(name) - self.logger.warning(f"Container {name} already exists. Deployment may fail or cause conflicts.") - except docker.errors.NotFound: - pass # No conflict, safe to proceed - - # Write the config file in config directory - with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f: - json.dump(node, f, indent=4) - try: - container_id = client.api.create_container( - image=image, - name=name, - detach=True, - volumes=volumes, - environment=environment, - command=command, - host_config=host_config, - networking_config=networking_config, - ) - except Exception as e: - self.logger.exception(f"Creating container {name}: {e}") - - try: - client.api.start(container_id) - container_ids.append(container_id) - container_names.append(name) - except Exception as e: - self.logger.exception(f"Starting participant {name} error: {e}") - i += 1 + image = "nebula-core" + name = self.get_participant_container_name(node["device_args"]["idx"]) + if node["device_args"]["accelerator"] == "gpu": + environment = { + "NVIDIA_DISABLE_REQUIRE": True, + "NEBULA_LOGS_DIR": "/nebula/app/logs/", + "NEBULA_CONFIG_DIR": "/nebula/app/config/", + } + host_config = client.api.create_host_config( + binds=[f"{self.root_path}:/nebula", "/var/run/docker.sock:/var/run/docker.sock"], + privileged=True, + device_requests=[docker.types.DeviceRequest(driver="nvidia", count=-1, capabilities=[["gpu"]])], + extra_hosts={"host.docker.internal": "host-gateway"}, + ) + else: + environment = {"NEBULA_LOGS_DIR": "/nebula/app/logs/", "NEBULA_CONFIG_DIR": "/nebula/app/config/"} + host_config = client.api.create_host_config( + binds=[f"{self.root_path}:/nebula", "/var/run/docker.sock:/var/run/docker.sock"], + privileged=True, + device_requests=[], + extra_hosts={"host.docker.internal": "host-gateway"}, + ) + volumes = ["/nebula", "/var/run/docker.sock"] + start_command = "sleep 10" if node["device_args"]["start"] else "sleep 0" + command = [ + "/bin/bash", + "-c", + f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json", + ] + networking_config = client.api.create_networking_config({ + network_name: client.api.create_endpoint_config( + ipv4_address=f"{base}.{i}", + ), + base_network_name: client.api.create_endpoint_config(), + }) + node["tracking_args"]["log_dir"] = "/nebula/app/logs" + node["tracking_args"]["config_dir"] = f"/nebula/app/config/{self.sb.get_scenario_name()}" + node["scenario_args"]["controller"] = self.url + node["scenario_args"]["deployment"] = self.sb.get_deployment() + node["security_args"]["certfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_cert.pem" + node["security_args"]["keyfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_key.pem" + node["security_args"]["cafile"] = "/nebula/app/certs/ca_cert.pem" + node = json.loads(json.dumps(node).replace("192.168.50.", f"{base}.")) # TODO change this + try: + existing = client.containers.get(name) + self.logger.warning(f"Container {name} already exists. Deployment may fail or cause conflicts.") + except docker.errors.NotFound: + pass # No conflict, safe to proceed + # Write the config file in config directory + with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f: + json.dump(node, f, indent=4) + try: + container_id = client.api.create_container( + image=image, + name=name, + detach=True, + volumes=volumes, + environment=environment, + command=command, + host_config=host_config, + networking_config=networking_config, + ) + except Exception as e: + self.logger.exception(f"Creating container {name}: {e}") + try: + client.api.start(container_id) + container_ids.append(container_id) + container_names.append(name) + except Exception as e: + self.logger.exception(f"Starting participant {name} error: {e}") # Write scenario-level metadata for cleanup scenario_metadata = {"containers": container_names, "network": network_name} diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index e8ec30ee3..74feb6b07 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -58,12 +58,12 @@ async def init_federation_experiment(ifr: InitFederationRequest): logger.info(f"Experiment type received: {experiment_type}") # Modify when deploying controllers on differents systems - web_app_controller_url = os.environ.get("NEBULA_CONTROLLER_PORT") + hub_port = os.environ.get("NEBULA_CONTROLLER_PORT") controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") - controller_url = f"http://{controller_host}:{web_app_controller_url}" - logger.info(f"Docker Hub URL => {controller_url}") - fed_controller = federation_controller_factory(str(experiment_type), controller_url, logger) + hub_url = f"http://{controller_host}:{hub_port}" + logger.info(f"Docker Hub URL => {hub_url}") + fed_controller = federation_controller_factory(str(experiment_type), hub_url, logger) logger.info("Federation controller created.") return {"message": f"{experiment_type} controller initialized"} diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index 087d6eb9a..daaf85bb9 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -6,9 +6,9 @@ class FederationController(ABC): - def __init__(self, wa_controller_url, logger): + def __init__(self, hub_url, logger): self._logger: logging.Logger = logger - self._wa_url = wa_controller_url + self._hub_url = hub_url self._scenario_builder = ScenarioBuilder() @property diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index 0f5942743..ee891c454 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -399,7 +399,8 @@ def dictify(d): # General configuration participant_config["scenario_args"]["name"] = self._scenario_name - participant_config["scenario_args"]["start_time"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + participant_config["scenario_args"]["start_time"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + participant_config["deployment_args"]["additional"] = False node_config = node #self.sd["nodes"][index] participant_config["network_args"]["ip"] = node_config["ip"] @@ -415,17 +416,26 @@ def dictify(d): participant_config["device_args"]["proxy"] = node_config["proxy"] participant_config["device_args"]["malicious"] = node_config["malicious"] participant_config["scenario_args"]["rounds"] = int(self.sd["rounds"]) + participant_config["scenario_args"]["random_seed"] = 42 participant_config["data_args"]["dataset"] = self.sd["dataset"] participant_config["data_args"]["iid"] = self.sd["iid"] + participant_config["data_args"]["num_workers"] = 0 participant_config["data_args"]["partition_selection"] = self.sd["partition_selection"] participant_config["data_args"]["partition_parameter"] = self.sd["partition_parameter"] participant_config["model_args"]["model"] = self.sd["model"] participant_config["training_args"]["epochs"] = int(self.sd["epochs"]) + participant_config["training_args"]["trainer"] = "lightning" participant_config["device_args"]["accelerator"] = self.sd["accelerator"] participant_config["device_args"]["gpu_id"] = self.sd["gpu_id"] participant_config["device_args"]["logging"] = self.sd["logginglevel"] participant_config["aggregator_args"]["algorithm"] = self.sd["agg_algorithm"] + participant_config["message_args"]= self._configure_message_args() + participant_config["reporter_args"]= self._configure_reporter_args() + participant_config["forwarder_args"]= self._configure_forwarder_args() + participant_config["propagator_args"]= self._configure_propagator_args() + participant_config["misc_args"]= self._configure_misc_args() + # Addons configuration # Trustworthiness @@ -455,7 +465,6 @@ def dictify(d): # Attacks try: - #TODO moverlo a addons-adversarial_args if node_config["role"] == "malicious": addons_config["adversarial_args"] = self._configure_malicious_role(node_config) except Exception as e: @@ -465,7 +474,7 @@ def dictify(d): try: if self.sd.get("mobility", None): #participant_config["addons"].append("mobility") - addons_config["mobility"] = {"enabled": True} + addons_config["mobility"] = self._configure_mobility_args() except Exception as e: self.logger.info(f"ERROR: Cannot build mobility configuration - {e}") @@ -487,6 +496,52 @@ def dictify(d): return config + def _configure_message_args(self): + return { + "max_local_messages": 10000, + "compression": "zlib" + } + + def _configure_reporter_args(self): + return { + "grace_time_reporter": 10, + "report_frequency": 5, + "report_status_data_queue": True + } + + def _configure_forwarder_args(self): + return { + "forwarder_interval": 1, + "forward_messages_interval": 0, + "number_forwarded_messages": 100 + } + + def _configure_propagator_args(self): + return { + "propagate_interval": 3, + "propagate_model_interval": 0, + "propagation_early_stop": 3, + "history_size": 20 + } + + def _configure_misc_args(self): + return { + "grace_time_connection": 10, + "grace_time_start_federation": 10 + } + + def _configure_mobility_args(self): + return { + "enabled": True, + "mobility_type": self.sd["mobility_type"], + "topology_type": self.sd["topology"], + "radius_federation": self.sd["radius_federation"], + "scheme_mobility": self.sd["scheme_mobility"], + "round_frequency": self.sd["round_frequency"], + "grace_time_mobility": 60, + "change_geo_interval": 5 + } + def _configure_malicious_role(self, node_config: dict): return { "fake_behavior": node_config["fake_behavior"], @@ -693,10 +748,9 @@ def build_preload_additional_node_configuration(self, last_participant_index, in + str(self._scenario_name) ).encode() ).hexdigest() - participant_config["mobility_args"]["additional_node"]["status"] = True + participant_config["deployment_args"]["additional"] = True # used for late creation nodes - participant_config["mobility_args"]["late_creation"] = True """ ############################### # TOPOLOGY MANAGER # diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index c675e1a18..249ee9e38 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -323,7 +323,7 @@ async def run_scenario( fed_controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") url = f"http://{fed_controller_host}:{fed_controller_port}/init" url2 = f"http://{fed_controller_host}:{fed_controller_port}/scenarios/run" - data = {"type": "docker"} + data = {"experiment_type": "docker"} data2 = {"scenario_data": scenario_data, "federation_id": "id_nebula", "user": user} await APIUtils.post(url, data) await APIUtils.post(url2, data2) diff --git a/nebula/core/engine.py b/nebula/core/engine.py index 5d420816b..6195f6926 100644 --- a/nebula/core/engine.py +++ b/nebula/core/engine.py @@ -115,7 +115,6 @@ def __init__( self._aggregator = create_aggregator(config=self.config, engine=self) self._secure_neighbors = [] - self._is_malicious = self.config.participant["adversarial_args"]["attack_params"]["attacks"] != "No Attack" role = config.participant["device_args"]["role"] self._role_behavior: RoleBehavior = factory_role_behavior(role, self, config) @@ -132,7 +131,6 @@ def __init__( msg += f"\nIID: {self.config.participant['data_args']['iid']}" msg += f"\nModel: {model.__class__.__name__}" msg += f"\nAggregation algorithm: {self._aggregator.__class__.__name__}" - msg += f"\nNode behavior: {'malicious' if self._is_malicious else 'benign'}" print_msg_box(msg=msg, indent=2, title="Scenario information") print_msg_box( msg=f"Logging type: {self._trainer.logger.__class__.__name__}", @@ -165,7 +163,7 @@ def __init__( else: self._situational_awareness = None - if self.config.participant["addons"]["reputation"]["enabled"]: + if dict(self.config.participant["addons"]).get("reputation", None): self._reputation = Reputation(engine=self, config=self.config) @property @@ -619,9 +617,9 @@ async def deploy_components(self): the federated learning process starts. """ await self.aggregator.init() - if "situational_awareness" in self.config.participant: + if "situational_awareness" in self.config.participant["addons"]: await self.sa.start() - if self.config.participant["addons"]["reputation"]["enabled"]: + if "reputation" in self.config.participant["addons"]: await self._reputation.start() await self._reporter.start() await self._addon_manager.deploy_additional_services() diff --git a/nebula/core/network/communications.py b/nebula/core/network/communications.py index e0b1c17a5..18191d266 100755 --- a/nebula/core/network/communications.py +++ b/nebula/core/network/communications.py @@ -88,7 +88,7 @@ def __init__(self, engine: "Engine"): ) self.receive_messages_lock = Locker(name="receive_messages_lock", async_lock=True) - self._discoverer = Discoverer(addr=self.addr, config=self.config) + #self._discoverer = Discoverer(addr=self.addr, config=self.config) # self._health = Health(addr=self.addr, config=self.config) self._health = None self._forwarder = Forwarder(config=self.config) diff --git a/nebula/core/node.py b/nebula/core/node.py index 8fbeeb28f..954e425f0 100755 --- a/nebula/core/node.py +++ b/nebula/core/node.py @@ -179,11 +179,6 @@ def randomize_value(value, variability): config_keys = [ ["reporter_args", "report_frequency"], - ["discoverer_args", "discovery_frequency"], - ["health_args", "health_interval"], - ["health_args", "grace_time_health"], - ["health_args", "check_alive_interval"], - ["health_args", "send_alive_interval"], ["forwarder_args", "forwarder_interval"], ["forwarder_args", "forward_messages_interval"], ] diff --git a/nebula/core/noderole.py b/nebula/core/noderole.py index 9bd258fef..d1b227e6e 100644 --- a/nebula/core/noderole.py +++ b/nebula/core/noderole.py @@ -190,7 +190,7 @@ def __init__(self, engine: Engine, config: Config): self.attack = create_attack(self._engine) logging.info("Attack behavior created") self.aggregator_bening = self._engine._aggregator - benign_role = self._config.participant["adversarial_args"]["fake_behavior"] + benign_role = self._config.participant["addons"]["adversarial_args"]["fake_behavior"] self._fake_role_behavior = factory_role_behavior(benign_role, self._engine, self._config) self._role = factory_node_role("malicious") @@ -206,7 +206,7 @@ async def extended_learning_cycle(self): try: await self.attack.attack() except Exception: - attack_name = self._config.participant["adversarial_args"]["attacks"] + attack_name = self._config.participant["addons"]["adversarial_args"]["attacks"] logging.exception(f"Attack {attack_name} failed") await self._fake_role_behavior.extended_learning_cycle() diff --git a/nebula/core/situationalawareness/awareness/sareasoner.py b/nebula/core/situationalawareness/awareness/sareasoner.py index 743187b14..b7f76e25c 100644 --- a/nebula/core/situationalawareness/awareness/sareasoner.py +++ b/nebula/core/situationalawareness/awareness/sareasoner.py @@ -85,6 +85,7 @@ def __init__( title="SA Reasoner", ) logging.info("🌐 Initializing SAReasoner") + self._is_additional_node = config.participant["deployment_args"]["additional"] self._config = copy.deepcopy(config.participant["addons"]) self._addr = config.participant["network_args"]["addr"] self._topology = config.participant["addons"]["mobility"]["topology_type"] @@ -146,7 +147,8 @@ def is_additional_participant(self): Returns: bool: True if the node is marked as an additional participant, False otherwise. """ - return self._config["addons"]["mobility"]["additional_node"]["status"] + return self._is_additional_node + """ ############################### # REESTRUCTURE TOPOLOGY # diff --git a/nebula/core/situationalawareness/awareness/suggestionbuffer.py b/nebula/core/situationalawareness/awareness/suggestionbuffer.py index 98cae49b2..c1d354133 100644 --- a/nebula/core/situationalawareness/awareness/suggestionbuffer.py +++ b/nebula/core/situationalawareness/awareness/suggestionbuffer.py @@ -5,7 +5,7 @@ from nebula.core.situationalawareness.awareness.sautils.sacommand import SACommand from nebula.core.situationalawareness.awareness.sautils.samoduleagent import SAModuleAgent from nebula.core.utils.locker import Locker -from nebula.utils import logging +import logging class SuggestionBuffer: diff --git a/nebula/core/situationalawareness/situationalawareness.py b/nebula/core/situationalawareness/situationalawareness.py index 5cdeec03d..2fee2d7de 100644 --- a/nebula/core/situationalawareness/situationalawareness.py +++ b/nebula/core/situationalawareness/situationalawareness.py @@ -183,7 +183,7 @@ def __init__(self, config, engine): model_handler = config.participant["addons"]["situational_awareness"]["sa_discovery"]["model_handler"] self._sad = factory_sa_discovery( "nebula", - self._config.participant["addons"]["mobility"]["additional_node"]["status"], + self._config.participant["deployment_args"]["additional"], selector, model_handler, engine=engine, From b17752fa054fe2bb04d184a0a4c6463d447e65b2 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Mon, 28 Jul 2025 10:47:24 +0200 Subject: [PATCH 14/26] feature eputation and attacks configuration --- nebula/addons/attacks/attacks.py | 2 +- nebula/addons/reputation/reputation.py | 6 +-- .../docker_federation_controller.py | 41 +++++++++---------- .../controller/federation/scenario_builder.py | 14 +++++-- nebula/core/network/communications.py | 1 + nebula/core/node.py | 2 +- 6 files changed, 36 insertions(+), 30 deletions(-) diff --git a/nebula/addons/attacks/attacks.py b/nebula/addons/attacks/attacks.py index f5587f365..3d7bd71a7 100755 --- a/nebula/addons/attacks/attacks.py +++ b/nebula/addons/attacks/attacks.py @@ -129,7 +129,7 @@ def create_attack(engine) -> Attack: } # Get attack name and parameters from the engine configuration - attack_params = engine.config.participant["adversarial_args"].get("attack_params", {}) + attack_params = engine.config.participant["addons"]["adversarial_args"].get("attack_params", {}) attack_name = attack_params.get("attacks", None) if attack_name is None: raise AttackException("No attack specified") diff --git a/nebula/addons/reputation/reputation.py b/nebula/addons/reputation/reputation.py index 60602a02c..9377b2d42 100644 --- a/nebula/addons/reputation/reputation.py +++ b/nebula/addons/reputation/reputation.py @@ -114,7 +114,7 @@ def __init__(self, engine: "Engine", config: "Config"): def _configure_constants(self): """Configure system constants from config or use defaults.""" - reputation_config = self._config.participant.get("defense_args", {}).get("reputation", {}) + reputation_config = self._config.participant.get("addons", {}).get("reputation", {}) constants_config = reputation_config.get("constants", {}) self.REPUTATION_THRESHOLD = constants_config.get("reputation_threshold", self.REPUTATION_THRESHOLD) @@ -168,7 +168,7 @@ def _initialize_data_structures(self): def _load_configuration(self): """Load and validate reputation configuration.""" - reputation_config = self._config.participant["defense_args"]["reputation"] + reputation_config = self._config.participant["addons"]["reputation"] self._enabled = reputation_config["enabled"] self._metrics = reputation_config["metrics"] self._initial_reputation = float(reputation_config["initial_reputation"]) @@ -1966,7 +1966,7 @@ async def recollect_similarity(self, ure: UpdateReceivedEvent): if not (self._enabled and self._is_metric_enabled("model_similarity")): return - if not self._engine.config.participant["adaptive_args"]["model_similarity"]: + if not self._engine.config.participant["addons"]["reputation"]["adaptive_args"] and not self._engine.config.participant["addons"]["reputation"]["adaptive_args"]["model_similarity"]: return if nei == self._addr: diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index 225f5926a..cc07a3443 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -1,15 +1,14 @@ -import datetime import glob import json import os import shutil -from nebula.utils import DockerUtils, FileUtils +from nebula.utils import DockerUtils import docker from nebula.controller.federation.federation_controller import FederationController from typing import Dict from fastapi import Request from nebula.config.config import Config -from nebula.core.utils.certificate import generate_ca_certificate, generate_certificate +from nebula.core.utils.certificate import generate_ca_certificate class DockerFederationController(FederationController): @@ -26,7 +25,7 @@ def __init__(self, hub_url, logger): self.url = "" self.config = Config(entity="scenarioManagement") self.round_per_node = {} - self.additionals: dict[str, int] = {} + self.additionals: dict = {} self._ip_last_index = 0 self._network_name = "" self._base_network_name = "" @@ -133,9 +132,9 @@ async def stop_scenario(self, id: str): async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() - participant_idx = str(config["device_args"]["idx"]) + participant_idx = int(config["device_args"]["idx"]) participant_round = int(config["federation_args"]["round"]) - + self.logger.info self.round_per_node[participant_idx] = participant_round federation_round = min(self.round_per_node.values()) @@ -148,6 +147,12 @@ async def update_nodes(self, scenario_name: str, request: Request): #TODO deploy additionals deployables # update self.round_per_node -> add additional nodes that are going to # be deployed + for index in additionals_deployables: + for idx, node in enumerate(self.config.participants): + if index == idx: + self.logger.info(f"Deploying additional participant: {index}") + self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index) + self._ip_last_index += 1 #TODO get the others parameters @@ -361,31 +366,25 @@ def get_participant_container_name(self, idx: int) -> str: def _start_initial_nodes(self): self.logger.info("Starting nodes using Docker Compose...") - network_name = self.get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") - base_network_name = self.get_network_name("net-base") + self._network_name = self.get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") + self._base_network_name = self.get_network_name("net-base") # Create the Docker network - base = DockerUtils.create_docker_network(network_name) + self._base = DockerUtils.create_docker_network(self._network_name) self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) self._ip_last_index = 2 - container_ids = [] for idx, node in enumerate(self.config.participants): self.logger.info(f"Deployment starting for participant {idx}") if node["deployment_args"]["additional"]: self.additionals[idx] = int(node["deployment_args"]["deployment_round"]) self.logger.info(f"Participant {idx} is additional. Round of deployment: {int(node['deployment_args']['deployment_round'])}") - continue - - # deploy initial nodes - self.round_per_node[idx] = 0 - self._start_node(node, network_name, base_network_name, base, self._ip_last_index) - self._ip_last_index += 1 - - # Ordered list for additional participants deployment - for an, anr in self.additionals.items(): - self.logger.info(f"Additional node: {an}:{anr}") - + else: + # deploy initial nodes + self.round_per_node[idx] = 0 + self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index) + self._ip_last_index += 1 + def _start_node(self, node, network_name, base_network_name, base, i): client = docker.from_env() diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index ee891c454..d5a66665b 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -417,6 +417,7 @@ def dictify(d): participant_config["device_args"]["malicious"] = node_config["malicious"] participant_config["scenario_args"]["rounds"] = int(self.sd["rounds"]) participant_config["scenario_args"]["random_seed"] = 42 + participant_config["federation_args"]["round"] = 0 participant_config["data_args"]["dataset"] = self.sd["dataset"] participant_config["data_args"]["iid"] = self.sd["iid"] participant_config["data_args"]["num_workers"] = 0 @@ -429,6 +430,7 @@ def dictify(d): participant_config["device_args"]["gpu_id"] = self.sd["gpu_id"] participant_config["device_args"]["logging"] = self.sd["logginglevel"] participant_config["aggregator_args"]["algorithm"] = self.sd["agg_algorithm"] + participant_config["aggregator_args"]["aggregation_timeout"] = 60 participant_config["message_args"]= self._configure_message_args() participant_config["reporter_args"]= self._configure_reporter_args() @@ -448,7 +450,7 @@ def dictify(d): # Reputation try: - if self.sd.get("reputation", None) and self.sd["reputation"]["enabled"]: + if self.sd.get("reputation", None) and self.sd["reputation"]["enabled"] and not node_config["role"] == "malicious": #participant_config["defense_args"]["reputation"] = self._configure_reputation() addons_config["reputation"] = self._configure_reputation() except Exception as e: @@ -579,7 +581,9 @@ def _configure_trustworthiness(self) -> dict: return trust_config def _configure_reputation(self) -> dict: - return self.sd.get("reputation") + rep = self.sd.get("reputation") + rep["adaptive_args"] = True + return rep def _configure_network_simulation(self) -> dict: network_parameters = {} @@ -714,7 +718,7 @@ def build_preload_initial_node_configuration(self, index, participant_config: di participant_config["security_args"] = {} # If not, use the given coordinates in the frontend - participant_config["tracking_args"]["local_tracking"] = "advanced" if advanced_analytics else "basic" + participant_config["tracking_args"]["local_tracking"] = "default" participant_config["tracking_args"]["log_dir"] = log_dir participant_config["tracking_args"]["config_dir"] = config_dir # Generate node certificate @@ -732,7 +736,6 @@ def build_preload_additional_node_configuration(self, last_participant_index, in n_nodes = len(self.sd["nodes"].keys()) n_additionals = len(self.sd["additional_participants"]) last_ip = participant_config["network_args"]["ip"] - self.logger.info(f"Valores de la ultima ip: ({last_ip})") participant_config["scenario_args"]["n_nodes"] = n_nodes + n_additionals # self.n_nodes + i + 1 participant_config["device_args"]["idx"] = last_participant_index + index participant_config["network_args"]["neighbors"] = "" @@ -750,6 +753,9 @@ def build_preload_additional_node_configuration(self, last_participant_index, in ).hexdigest() participant_config["deployment_args"]["additional"] = True + deployment_round = self.sd["additional_participants"][index]["time_start"] + participant_config["deployment_args"]["deployment_round"] = deployment_round + # used for late creation nodes """ ############################### diff --git a/nebula/core/network/communications.py b/nebula/core/network/communications.py index 18191d266..5b022b61e 100755 --- a/nebula/core/network/communications.py +++ b/nebula/core/network/communications.py @@ -89,6 +89,7 @@ def __init__(self, engine: "Engine"): self.receive_messages_lock = Locker(name="receive_messages_lock", async_lock=True) #self._discoverer = Discoverer(addr=self.addr, config=self.config) + self._discoverer = None # self._health = Health(addr=self.addr, config=self.config) self._health = None self._forwarder = Forwarder(config=self.config) diff --git a/nebula/core/node.py b/nebula/core/node.py index 954e425f0..a8df622c2 100755 --- a/nebula/core/node.py +++ b/nebula/core/node.py @@ -207,7 +207,7 @@ def randomize_value(value, variability): # time = config.participant["addons"]["mobility"]["additional_node"]["time_start"] # logging.info(f"Waiting time to start finding federation: {time}") # await asyncio.sleep(int(config.participant["addons"]["mobility"]["additional_node"]["time_start"])) - await asyncio.sleep(120) #TODO REMOVE + #await asyncio.sleep(120) #TODO REMOVE await node._aditional_node_start() if node.cm is not None: From 47e70a3b4b529bb09267c9dee3c5008952934208 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Mon, 28 Jul 2025 11:15:11 +0200 Subject: [PATCH 15/26] feature additionals participant deployment --- .../docker_federation_controller.py | 58 ++++++++++++------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index cc07a3443..a886d0d9f 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -6,9 +6,10 @@ import docker from nebula.controller.federation.federation_controller import FederationController from typing import Dict -from fastapi import Request +from fastapi import Request, Response from nebula.config.config import Config from nebula.core.utils.certificate import generate_ca_certificate +from nebula.core.utils.locker import Locker class DockerFederationController(FederationController): @@ -30,6 +31,8 @@ def __init__(self, hub_url, logger): self._network_name = "" self._base_network_name = "" self._base = "" + self._deployment_lock = Locker("deployment_lock", async_lock=True) + self._federation_round = 0 """ ############################### # ENDPOINT CALLBACKS # @@ -134,27 +137,37 @@ async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() participant_idx = int(config["device_args"]["idx"]) participant_round = int(config["federation_args"]["round"]) - self.logger.info + self.logger.info(f"Update received from participant: {participant_idx}, round: {participant_round}") + self.logger.info(f"Update: {self.round_per_node.items()}") self.round_per_node[participant_idx] = participant_round - federation_round = min(self.round_per_node.values()) + last_fed_round = self._federation_round + self._federation_round = min(self.round_per_node.values()) additionals_deployables = [ idx for idx, round in self.additionals.items() - if federation_round >= round + if self._federation_round >= round ] - #TODO deploy additionals deployables - # update self.round_per_node -> add additional nodes that are going to - # be deployed - for index in additionals_deployables: - for idx, node in enumerate(self.config.participants): - if index == idx: - self.logger.info(f"Deploying additional participant: {index}") - self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index) - self._ip_last_index += 1 + adds_deployed = set() + # Only verify when federation round is updated + if self._federation_round != last_fed_round: + self.logger.info(f"Federation Round updates, current value: {self._federation_round}") + # Ensure concurrency + for index in additionals_deployables: + if index in adds_deployed: + continue + + for idx, node in enumerate(self.config.participants): + if index == idx: + async with self._deployment_lock: + self.logger.info(f"Deploying additional participant: {index}") + self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index, additional=True) + self._ip_last_index += 1 + adds_deployed.add(index) - #TODO get the others parameters + #TODO return the others parameters + return Response(content=json.dumps({"message": "Node updated successfully"}), status_code=200) """ ############################### # FUNCTIONALITIES # @@ -321,7 +334,7 @@ async def _load_configuration_and_start_nodes(self): with open(additional_participant_file) as f: participant_config = json.load(f) - self.logger.info(f"Configuration | additional nodes | participant: {self.n_nodes + i + 1}") + self.logger.info(f"Configuration | additional nodes | participant: {self.n_nodes + i}") self.sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) with open(additional_participant_file, "w") as f: @@ -375,17 +388,18 @@ def _start_initial_nodes(self): self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) self._ip_last_index = 2 for idx, node in enumerate(self.config.participants): - self.logger.info(f"Deployment starting for participant {idx}") + if node["deployment_args"]["additional"]: self.additionals[idx] = int(node["deployment_args"]["deployment_round"]) self.logger.info(f"Participant {idx} is additional. Round of deployment: {int(node['deployment_args']['deployment_round'])}") else: # deploy initial nodes + self.logger.info(f"Deployment starting for participant {idx}") self.round_per_node[idx] = 0 self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index) self._ip_last_index += 1 - - def _start_node(self, node, network_name, base_network_name, base, i): + + def _start_node(self, node, network_name, base_network_name, base, i, additional=False): client = docker.from_env() self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) @@ -465,5 +479,9 @@ def _start_node(self, node, network_name, base_network_name, base, i): # Write scenario-level metadata for cleanup scenario_metadata = {"containers": container_names, "network": network_name} - with open(os.path.join(self.config_dir, "scenario.metadata"), "w") as f: - json.dump(scenario_metadata, f, indent=2) \ No newline at end of file + if not additional: + with open(os.path.join(self.config_dir, "scenario.metadata"), "w") as f: + json.dump(scenario_metadata, f, indent=2) + else: + with open(os.path.join(self.config_dir, "scenario.metadata"), "a") as f: + json.dump(scenario_metadata, f, indent=2) \ No newline at end of file From 2f24706c5e08a425a1299043e211cc5159e49372 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Tue, 29 Jul 2025 13:16:37 +0200 Subject: [PATCH 16/26] feature docker federation controller integration --- nebula/controller/federation/api_requests.py | 14 - .../docker_federation_controller.py | 57 ++- .../processes_federation_controller.py | 437 +++++++++++++++++- .../controller/federation/federation_api.py | 14 +- .../federation/federation_controller.py | 5 +- .../controller/federation/utils_requests.py | 29 ++ nebula/controller/web_app_controller.py | 66 +-- 7 files changed, 560 insertions(+), 62 deletions(-) delete mode 100644 nebula/controller/federation/api_requests.py create mode 100644 nebula/controller/federation/utils_requests.py diff --git a/nebula/controller/federation/api_requests.py b/nebula/controller/federation/api_requests.py deleted file mode 100644 index 55f393b7f..000000000 --- a/nebula/controller/federation/api_requests.py +++ /dev/null @@ -1,14 +0,0 @@ -from pydantic import BaseModel -from typing import Dict, Any - -class InitFederationRequest(BaseModel): - experiment_type: str - -class RunScenarioRequest(BaseModel): - scenario_data: Dict[str, Any] - user: str - federation_id: str - -class StopScenarioRequest(BaseModel): - federation_id: str - \ No newline at end of file diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index a886d0d9f..2f9efbded 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -1,12 +1,14 @@ +import asyncio import glob import json import os import shutil -from nebula.utils import DockerUtils +from nebula.utils import DockerUtils, APIUtils import docker from nebula.controller.federation.federation_controller import FederationController +from nebula.controller.federation.utils_requests import factory_requests_path from typing import Dict -from fastapi import Request, Response +from fastapi import Request from nebula.config.config import Config from nebula.core.utils.certificate import generate_ca_certificate from nebula.core.utils.locker import Locker @@ -24,7 +26,7 @@ def __init__(self, hub_url, logger): self.cert_dir = "" self.advanced_analytics = "" self.url = "" - self.config = Config(entity="scenarioManagement") + self.config = Config(entity="FederationController") self.round_per_node = {} self.additionals: dict = {} self._ip_last_index = 0 @@ -137,8 +139,8 @@ async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() participant_idx = int(config["device_args"]["idx"]) participant_round = int(config["federation_args"]["round"]) - self.logger.info(f"Update received from participant: {participant_idx}, round: {participant_round}") - self.logger.info(f"Update: {self.round_per_node.items()}") + #self.logger.info(f"Update received from participant: {participant_idx}, round: {participant_round}") + #self.logger.info(f"Update: {self.round_per_node.items()}") self.round_per_node[participant_idx] = participant_round last_fed_round = self._federation_round self._federation_round = min(self.round_per_node.values()) @@ -152,7 +154,7 @@ async def update_nodes(self, scenario_name: str, request: Request): adds_deployed = set() # Only verify when federation round is updated if self._federation_round != last_fed_round: - self.logger.info(f"Federation Round updates, current value: {self._federation_round}") + self.logger.info(f"Federation Round updating, current value: {self._federation_round}") # Ensure concurrency for index in additionals_deployables: if index in adds_deployed: @@ -161,18 +163,39 @@ async def update_nodes(self, scenario_name: str, request: Request): for idx, node in enumerate(self.config.participants): if index == idx: async with self._deployment_lock: - self.logger.info(f"Deploying additional participant: {index}") - self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index, additional=True) - self._ip_last_index += 1 - adds_deployed.add(index) + if index in self.additionals.keys(): + self.logger.info(f"Deploying additional participant: {index}") + self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index, additional=True) + self._ip_last_index += 1 + self.additionals.pop(index) + adds_deployed.add(index) + + request_body = await request.json() + payload = {"scenario_name": scenario_name, "data": request_body} + + asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) - #TODO return the others parameters - return Response(content=json.dumps({"message": "Node updated successfully"}), status_code=200) + return {"message": "Node updated successfully in Federation Controller"} + + async def node_done(self, scenario_name: str, request: Request): + request_body = await request.json() + payload = {"scenario_name": scenario_name, "data": request_body} + asyncio.create_task(self._send_to_hub("done", payload, scenario_name)) + return {"message": "Nodes done"} """ ############################### # FUNCTIONALITIES # ############################### """ + + async def _send_to_hub(self, path, payload, scenario_name=""): + try: + url_request = self._hub_url + factory_requests_path(path, scenario_name) + # self.logger.info(f"Seding to hub, url: {url_request}") + # self.logger.info(f"payload sent to hub, data: {payload}") + await APIUtils.post(url_request, payload) + except Exception as e: + self.logger.info(f"Failed to send update to Hub: {e}") async def _initialize_scenario(self, scenario_data): # Initialize Scenario builder using scenario_data from user @@ -357,7 +380,7 @@ async def _load_configuration_and_start_nodes(self): dataset.initialize_dataset() self.logger.info(f"✅ Splitting {self.sb.get_dataset_name()} dataset... Done") - def get_network_name(self, suffix: str) -> str: + def _get_network_name(self, suffix: str) -> str: """ Generate a standardized network name using tags. Args: @@ -367,7 +390,7 @@ def get_network_name(self, suffix: str) -> str: """ return f"{self.env_tag}_{self.prefix_tag}_{self.user_tag}_{suffix}" - def get_participant_container_name(self, idx: int) -> str: + def _get_participant_container_name(self, idx: int) -> str: """ Generate a standardized container name for a participant using tags. Args: @@ -379,8 +402,8 @@ def get_participant_container_name(self, idx: int) -> str: def _start_initial_nodes(self): self.logger.info("Starting nodes using Docker Compose...") - self._network_name = self.get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") - self._base_network_name = self.get_network_name("net-base") + self._network_name = self._get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") + self._base_network_name = self._get_network_name("net-base") # Create the Docker network self._base = DockerUtils.create_docker_network(self._network_name) @@ -407,7 +430,7 @@ def _start_node(self, node, network_name, base_network_name, base, i, additional container_names = [] # Track names for metadata image = "nebula-core" - name = self.get_participant_container_name(node["device_args"]["idx"]) + name = self._get_participant_container_name(node["device_args"]["idx"]) if node["device_args"]["accelerator"] == "gpu": environment = { "NVIDIA_DISABLE_REQUIRE": True, diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index 1fbf8672e..ee4240bb7 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -1,4 +1,439 @@ +import asyncio +import glob +import json +import os +import shutil +from nebula.utils import APIUtils +import docker from nebula.controller.federation.federation_controller import FederationController +from nebula.controller.federation.utils_requests import factory_requests_path +from typing import Dict +from fastapi import Request +from nebula.config.config import Config +from nebula.core.utils.certificate import generate_ca_certificate +from nebula.core.utils.locker import Locker class ProcessesFederationController(FederationController): - pass \ No newline at end of file + def __init__(self, hub_url, logger): + super().__init__(hub_url, logger) + self._user = "" + self.root_path = "" + self.host_platform = "" + self.config_dir = "" + self.log_dir = "" + self.cert_dir = "" + self.advanced_analytics = "" + self.url = "" + self.config = Config(entity="FederationController") + self.round_per_node = {} + self.additionals: dict = {} + self._last_file_index = 0 + self._deployment_lock = Locker("deployment_lock", async_lock=True) + self._federation_round = 0 + + """ ############################### + # ENDPOINT CALLBACKS # + ############################### + """ + + async def run_scenario(self, id: str, scenario_data: Dict, user: str): + #TODO maintain files on memory, not read them again + self._user = user + await self._initialize_scenario(scenario_data) + generate_ca_certificate(dir_path=self.cert_dir) + await self._load_configuration_and_start_nodes() + self._start_initial_nodes() + + return self.sb.get_scenario_name() + + async def stop_scenario(self, id: str = ""): + """ + Stop running participant nodes by removing the scenario command files. + + This method deletes the 'current_scenario_commands.sh' (or '.ps1' on Windows) + file associated with a scenario. Removing this file signals the nodes to stop + by terminating their processes. + + Args: + scenario_name (str, optional): The name of the scenario to stop. If None, + all scenarios' command files will be removed. + + Notes: + - If the environment variable NEBULA_CONFIG_DIR is not set, a default + configuration directory path is used. + - Supports both Linux/macOS ('.sh') and Windows ('.ps1') script files. + - Any errors during file removal are logged with the traceback. + """ + # When stopping the nodes, we need to remove the current_scenario_commands.sh file -> it will cause the nodes to stop using PIDs + try: + nebula_config_dir = os.environ.get("NEBULA_CONFIG_DIR") + if not nebula_config_dir: + current_dir = os.path.dirname(__file__) + nebula_base_dir = os.path.abspath(os.path.join(current_dir, "..", "..")) + nebula_config_dir = os.path.join(nebula_base_dir, "app", "config") + self.logger.info(f"NEBULA_CONFIG_DIR not found. Using default path: {nebula_config_dir}") + + if id: + if os.environ.get("NEBULA_HOST_PLATFORM") == "windows": + scenario_commands_file = os.path.join( + nebula_config_dir, self.sb.get_scenario_name(), "current_scenario_commands.ps1" + ) + else: + scenario_commands_file = os.path.join( + nebula_config_dir, self.sb.get_scenario_name(), "current_scenario_commands.sh" + ) + if os.path.exists(scenario_commands_file): + os.remove(scenario_commands_file) + else: + if os.environ.get("NEBULA_HOST_PLATFORM") == "windows": + files = glob.glob( + os.path.join(nebula_config_dir, "**/current_scenario_commands.ps1"), recursive=True + ) + else: + files = glob.glob( + os.path.join(nebula_config_dir, "**/current_scenario_commands.sh"), recursive=True + ) + for file in files: + os.remove(file) + except Exception as e: + self.logger.exception(f"Error while removing current_scenario_commands.sh file: {e}") + + async def update_nodes(self, scenario_name: str, request: Request): + config = await request.json() + participant_idx = int(config["device_args"]["idx"]) + participant_round = int(config["federation_args"]["round"]) + self.round_per_node[participant_idx] = participant_round + last_fed_round = self._federation_round + self._federation_round = min(self.round_per_node.values()) + + additionals_deployables = [ + idx + for idx, round in self.additionals.items() + if self._federation_round >= round + ] + + adds_deployed = set() + # Only verify when federation round is updated + if self._federation_round != last_fed_round: + self.logger.info(f"Federation Round updating, current value: {self._federation_round}") + # Ensure concurrency + for index in additionals_deployables: + if index in adds_deployed: + continue + + for idx, node in enumerate(self.config.participants): + if index == idx: + async with self._deployment_lock: + if index in self.additionals.keys(): + self.logger.info(f"Deploying additional participant: {index}") + self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index, additional=True) + self._ip_last_index += 1 + self.additionals.pop(index) + adds_deployed.add(index) + + request_body = await request.json() + payload = {"scenario_name": scenario_name, "data": request_body} + + asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) + + return {"message": "Node updated successfully in Federation Controller"} + + async def node_done(self, scenario_name: str, request: Request): + request_body = await request.json() + payload = {"scenario_name": scenario_name, "data": request_body} + asyncio.create_task(self._send_to_hub("done", payload, scenario_name)) + return {"message": "Nodes done"} + + """ ############################### + # FUNCTIONALITIES # + ############################### + """ + + async def _send_to_hub(self, path, payload, scenario_name=""): + try: + url_request = self._hub_url + factory_requests_path(path, scenario_name) + await APIUtils.post(url_request, payload) + except Exception as e: + self.logger.info(f"Failed to send update to Hub: {e}") + + async def _initialize_scenario(self, scenario_data): + # Initialize Scenario builder using scenario_data from user + self.logger.info("🔧 Initializing Scenario Builder using scenario data") + self.sb.set_scenario_data(scenario_data) + scenario_name = self.sb.get_scenario_name() + + self.root_path = os.environ.get("NEBULA_ROOT_HOST") + self.host_platform = os.environ.get("NEBULA_HOST_PLATFORM") + self.config_dir = os.path.join(os.environ.get("NEBULA_CONFIG_DIR"), scenario_name) + self.log_dir = os.environ.get("NEBULA_LOGS_DIR") + self.cert_dir = os.environ.get("NEBULA_CERTS_DIR") + self.advanced_analytics = os.environ.get("NEBULA_ADVANCED_ANALYTICS", "False") == "True" + self.config = Config(entity="scenarioManagement") + self.env_tag = os.environ.get("NEBULA_ENV_TAG", "dev") + self.prefix_tag = os.environ.get("NEBULA_PREFIX_TAG", "dev") + self.user_tag = os.environ.get("NEBULA_USER_TAG", os.environ.get("USER", "unknown")) + + self.url = f"127.0.0.1:{os.environ.get('NEBULA_FEDERATION_CONTROLLER_PORT')}" + + # Create Scenario management dirs + os.makedirs(self.config_dir, exist_ok=True) + os.makedirs(os.path.join(self.log_dir, scenario_name), exist_ok=True) + os.makedirs(self.cert_dir, exist_ok=True) + + # Give permissions to the directories + os.chmod(self.config_dir, 0o777) + os.chmod(os.path.join(self.log_dir, scenario_name), 0o777) + os.chmod(self.cert_dir, 0o777) + + # Save the scenario configuration + scenario_file = os.path.join(self.config_dir, "scenario.json") + with open(scenario_file, "w") as f: + json.dump(scenario_data, f, sort_keys=False, indent=2) + + os.chmod(scenario_file, 0o777) + + # Save management settings + settings = { + "scenario_name": scenario_name, + "root_path": self.root_path, + "config_dir": self.config_dir, + "log_dir": self.log_dir, + "cert_dir": self.cert_dir, + "env": None, + } + + settings_file = os.path.join(self.config_dir, "settings.json") + with open(settings_file, "w") as f: + json.dump(settings, f, sort_keys=False, indent=2) + + os.chmod(settings_file, 0o777) + + # Attacks assigment and mobility + self.logger.info("🔧 Building general configuration") + self.sb.build_general_configuration() + self.logger.info("✅ Building general configuration done") + + # Create participant configs and .json + for index, (_, node) in enumerate(self.sb.get_federation_nodes().items()): + self.logger.info(f"Creating .json file for participant: {index}, Configuration: {node}") + node_config = node + try: + participant_file = os.path.join(self.config_dir, f"participant_{node_config['id']}.json") + self.logger.info(f"Filename: {participant_file}") + os.makedirs(os.path.dirname(participant_file), exist_ok=True) + except Exception as e: + self.logger.info(f"ERROR while creating files: {e}") + + try: + participant_config = self.sb.build_scenario_config_for_node(index, node) + #self.logger.info(f"dictionary: {participant_config}") + except Exception as e: + self.logger.info(f"ERROR while building configuration for node: {e}") + + try: + with open(participant_file, "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + os.chmod(participant_file, 0o777) + except Exception as e: + self.logger.info(f"ERROR while dumping configuration into files: {e}") + + self.logger.info("✅ Initializing Scenario Builder done") + + async def _load_configuration_and_start_nodes(self): + self.logger.info("🔧 Loading Scenario configuration...") + # Get participants configurations + participant_files = glob.glob(f"{self.config_dir}/participant_*.json") + participant_files.sort() + if len(participant_files) == 0: + raise ValueError("No participant files found in config folder") + + self.config.set_participants_config(participant_files) + self.n_nodes = len(participant_files) + self.logger.info(f"Number of nodes: {self.n_nodes}") + + self.sb.create_topology_manager(self.config) + + # Update participants configuration + is_start_node = False + config_participants = [] + + additional_participants = self.sb.get_additional_nodes() + additional_nodes = len(additional_participants) if additional_participants else 0 + self.logger.info(f"######## nodes: {self.n_nodes} + additionals: {additional_nodes} ######") + + participant_files.sort(key=lambda x: int(x.split("_")[-1].split(".")[0])) + + # Initial participants + self.logger.info("🔧 Building preload configuration for initial nodes...") + for i in range(self.n_nodes): + try: + with open(f"{self.config_dir}/participant_" + str(i) + ".json") as f: + participant_config = json.load(f) + except Exception as e: + self.logger.info(f"ERROR: open/load participant .json") + + self.logger.info(f"Building preload conf for participant {i}") + try: + self.sb.build_preload_initial_node_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) + except Exception as e: + self.logger.info(f"ERROR: cannot build preload configuration") + + try: + with open(f"{self.config_dir}/participant_" + str(i) + ".json", "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + except Exception as e: + self.logger.info(f"ERROR: cannot dump preload configuration into participant .json file") + + config_participants.append(( + participant_config["network_args"]["ip"], + participant_config["network_args"]["port"], + participant_config["device_args"]["role"], + )) + if participant_config["device_args"]["start"]: + if not is_start_node: + is_start_node = True + else: + raise ValueError("Only one node can be start node") + + self.logger.info("✅ Building preload configuration for initial nodes done") + + self.config.set_participants_config(participant_files) + + # Add role to the topology (visualization purposes) + self.sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) + + # Additional participants + self.logger.info("🔧 Building preload configuration for additional nodes...") + additional_participants_files = [] + if additional_participants: + last_participant_file = participant_files[-1] + last_participant_index = len(participant_files) + + for i, _ in enumerate(additional_participants): + additional_participant_file = f"{self.config_dir}/participant_{last_participant_index + i}.json" + shutil.copy(last_participant_file, additional_participant_file) + + with open(additional_participant_file) as f: + participant_config = json.load(f) + + self.logger.info(f"Configuration | additional nodes | participant: {self.n_nodes + i}") + self.sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) + + with open(additional_participant_file, "w") as f: + json.dump(participant_config, f, sort_keys=False, indent=2) + + additional_participants_files.append(additional_participant_file) + + if additional_participants_files: + self.config.add_participants_config(additional_participants_files) + + if additional_participants: + self.n_nodes += len(additional_participants) + + self.logger.info("✅ Building preload configuration for additional nodes done") + self.logger.info("✅ Loading Scenario configuration done") + + # Build dataset + dataset = self.sb.configure_dataset(self.config_dir) + self.logger.info(f"🔧 Splitting {self.sb.get_dataset_name()} dataset...") + dataset.initialize_dataset() + self.logger.info(f"✅ Splitting {self.sb.get_dataset_name()} dataset... Done") + + def _start_initial_nodes(self): + self.logger.info("Starting nodes as processes...") + + self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) + self._ip_last_index = 2 + for idx, node in enumerate(self.config.participants): + + if node["deployment_args"]["additional"]: + self.additionals[idx] = int(node["deployment_args"]["deployment_round"]) + self.logger.info(f"Participant {idx} is additional. Round of deployment: {int(node['deployment_args']['deployment_round'])}") + else: + # deploy initial nodes + self.logger.info(f"Deployment starting for participant {idx}") + self.round_per_node[idx] = 0 + self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index) + self._ip_last_index += 1 + + def _start_node(self, node, network_name, base_network_name, base, i, additional=False): + self.processes_root_path = os.path.join(os.path.dirname(__file__), "..", "..") + + self.logger.info(f"env path: {self.env_path}") + + # Include additional config to the participants + for idx, node in enumerate(self.config.participants): + node["tracking_args"]["log_dir"] = os.path.join(self.root_path, "app", "logs") + node["tracking_args"]["config_dir"] = os.path.join(self.root_path, "app", "config", self.sb.get_scenario_name()) + node["scenario_args"]["controller"] = self.url + node["scenario_args"]["deployment"] = self.sb.get_deployment() + node["security_args"]["certfile"] = os.path.join( + self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_cert.pem" + ) + node["security_args"]["keyfile"] = os.path.join( + self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_key.pem" + ) + node["security_args"]["cafile"] = os.path.join(self.root_path, "app", "certs", "ca_cert.pem") + + # Write the config file in config directory + with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f: + json.dump(node, f, indent=4) + + try: + if self.host_platform == "windows": + commands = """ + $ParentDir = Split-Path -Parent $PSScriptRoot + $PID_FILE = "$PSScriptRoot\\current_scenario_pids.txt" + New-Item -Path $PID_FILE -Force -ItemType File + + """ + sorted_participants = sorted( + self.config.participants, + key=lambda node: node["device_args"]["idx"], + reverse=True, + ) + for node in sorted_participants: + if node["device_args"]["start"]: + commands += "Start-Sleep -Seconds 10\n" + else: + commands += "Start-Sleep -Seconds 2\n" + + commands += f'Write-Host "Running node {node["device_args"]["idx"]}..."\n' + commands += f'$OUT_FILE = "{self.root_path}\\app\\logs\\{self.sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.out"\n' + commands += f'$ERROR_FILE = "{self.root_path}\\app\\logs\\{self.sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.err"\n' + + # Use Start-Process for executing Python in background and capture PID + commands += f"""$process = Start-Process -FilePath "python" -ArgumentList "{self.root_path}\\nebula\\core\\node.py {self.root_path}\\app\\config\\{self.sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.json" -PassThru -NoNewWindow -RedirectStandardOutput $OUT_FILE -RedirectStandardError $ERROR_FILE + Add-Content -Path $PID_FILE -Value $process.Id + """ + + commands += 'Write-Host "All nodes started. PIDs stored in $PID_FILE"\n' + + with open(f"{self.config_dir}/current_scenario_commands.ps1", "w") as f: + f.write(commands) + os.chmod(f"{self.config_dir}/current_scenario_commands.ps1", 0o755) + else: + commands = '#!/bin/bash\n\nPID_FILE="$(dirname "$0")/current_scenario_pids.txt"\n\n> $PID_FILE\n\n' + sorted_participants = sorted( + self.config.participants, + key=lambda node: node["device_args"]["idx"], + reverse=True, + ) + for node in sorted_participants: + if node["device_args"]["start"]: + commands += "sleep 10\n" + else: + commands += "sleep 2\n" + commands += f'echo "Running node {node["device_args"]["idx"]}..."\n' + commands += f"OUT_FILE={self.root_path}/app/logs/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.out\n" + commands += f"python {self.root_path}/nebula/core/node.py {self.root_path}/app/config/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json &\n" + commands += "echo $! >> $PID_FILE\n\n" + + commands += 'echo "All nodes started. PIDs stored in $PID_FILE"\n' + + with open(f"{self.config_dir}/current_scenario_commands.sh", "w") as f: + f.write(commands) + os.chmod(f"{self.config_dir}/current_scenario_commands.sh", 0o755) + + except Exception as e: + raise Exception(f"Error starting nodes as processes: {e}") \ No newline at end of file diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index 74feb6b07..acf9c2aa6 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -10,7 +10,7 @@ from nebula.utils import LoggerUtils from nebula.controller.federation.federation_controller import FederationController from nebula.controller.federation.factory_federation_controller import federation_controller_factory -from nebula.controller.federation.api_requests import InitFederationRequest, RunScenarioRequest, StopScenarioRequest +from nebula.controller.federation.utils_requests import InitFederationRequest, RunScenarioRequest, StopScenarioRequest def require_initialized_controller(func): @wraps(func) @@ -92,6 +92,18 @@ async def update_nodes( global fed_controller return await fed_controller.update_nodes(scenario_name, request) +@app.post("/nodes/{scenario_name}/done") +@require_initialized_controller +async def update_nodes( + scenario_name: Annotated[ + str, + Path(regex="^[a-zA-Z0-9_-]+$", min_length=1, max_length=50, description="Valid scenario name"), + ], + request: Request, +): + global fed_controller + return await fed_controller.node_done(scenario_name, request) + if __name__ == "__main__": # Parse args from command line diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index daaf85bb9..b5c82cd89 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -30,4 +30,7 @@ async def stop_scenario(self, id: str): @abstractmethod async def update_nodes(self, scenario_name: str, request: Request): pass - + + abstractmethod + async def node_done(self, scenario_name: str, request: Request): + pass diff --git a/nebula/controller/federation/utils_requests.py b/nebula/controller/federation/utils_requests.py new file mode 100644 index 000000000..64e478003 --- /dev/null +++ b/nebula/controller/federation/utils_requests.py @@ -0,0 +1,29 @@ +from pydantic import BaseModel +from typing import Dict, Any + +class InitFederationRequest(BaseModel): + experiment_type: str + +class RunScenarioRequest(BaseModel): + scenario_data: Dict[str, Any] + user: str + federation_id: str + +class StopScenarioRequest(BaseModel): + federation_id: str + +def factory_requests_path(resource: str, scenario_name: str = "") -> str: + if resource == "init": + return "/init" + elif resource == "run": + return "/scenarios/run" + elif resource == "stop": + return "/scenarios/stop" + elif resource == "update": + return f"/nodes/{scenario_name}/update" + elif resource == "done": + return f"/nodes/{scenario_name}/done" + else: + raise Exception(f"resource not found: {resource}") + + \ No newline at end of file diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index 249ee9e38..bcd11b963 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -25,6 +25,7 @@ ) from nebula.controller.http_helpers import remote_get, remote_post_form from nebula.utils import DockerUtils, APIUtils +from nebula.controller.federation.utils_requests import RunScenarioRequest, InitFederationRequest, StopScenarioRequest, factory_requests_path # Setup controller logger @@ -321,12 +322,12 @@ async def run_scenario( try: fed_controller_port = os.environ.get("NEBULA_FEDERATION_CONTROLLER_PORT") fed_controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") - url = f"http://{fed_controller_host}:{fed_controller_port}/init" - url2 = f"http://{fed_controller_host}:{fed_controller_port}/scenarios/run" - data = {"experiment_type": "docker"} - data2 = {"scenario_data": scenario_data, "federation_id": "id_nebula", "user": user} - await APIUtils.post(url, data) - await APIUtils.post(url2, data2) + url_init_fed_controller = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("init") + url_run_scenario = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("run") + init_fed_req = InitFederationRequest(experiment_type="docker") + run_scenario_req = RunScenarioRequest(scenario_data=scenario_data, federation_id="id_nebula", user=user) + await APIUtils.post(url_init_fed_controller, init_fed_req.model_dump()) + await APIUtils.post(url_run_scenario, run_scenario_req.model_dump()) except Exception as e: logging.info(e) @@ -389,17 +390,26 @@ async def stop_scenario( Note: This function does not currently trigger statistics generation. """ - from nebula.controller.scenarios import ScenarioManagement - - ScenarioManagement.cleanup_scenario_containers() + fed_controller_port = os.environ.get("NEBULA_FEDERATION_CONTROLLER_PORT") + fed_controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") + url_stop_scenario = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("stop") + stop_scenario_req = StopScenarioRequest(federation_id="id_nebula") try: - if all: - await scenario_set_all_status_to_finished() - else: - await scenario_set_status_to_finished(scenario_name) + await APIUtils.post(url_stop_scenario, stop_scenario_req.model_dump()) except Exception as e: - logging.exception(f"Error setting scenario {scenario_name} to finished: {e}") - raise HTTPException(status_code=500, detail="Internal server error") + logging.info(f"ERROR: sending stop scenario to federation Controller: {e}") + + # from nebula.controller.scenarios import ScenarioManagement + + # ScenarioManagement.cleanup_scenario_containers() + # try: + # if all: + # await scenario_set_all_status_to_finished() + # else: + # await scenario_set_status_to_finished(scenario_name) + # except Exception as e: + # logging.exception(f"Error setting scenario {scenario_name} to finished: {e}") + # raise HTTPException(status_code=500, detail="Internal server error") @app.post("/scenarios/remove") @@ -649,20 +659,20 @@ async def update_nodes( timestamp = datetime.datetime.now() # Update the node in database await update_node_record( - str(config["device_args"]["uid"]), - str(config["device_args"]["idx"]), - str(config["network_args"]["ip"]), - str(config["network_args"]["port"]), - str(config["device_args"]["role"]), - config["network_args"]["neighbors"], - str(config["mobility_args"]["latitude"]), - str(config["mobility_args"]["longitude"]), + str(config["data"]["device_args"]["uid"]), + str(config["data"]["device_args"]["idx"]), + str(config["data"]["network_args"]["ip"]), + str(config["data"]["network_args"]["port"]), + str(config["data"]["device_args"]["role"]), + config["data"]["network_args"]["neighbors"], + str(config["data"]["mobility_args"]["latitude"]), + str(config["data"]["mobility_args"]["longitude"]), str(timestamp), - str(config["scenario_args"]["federation"]), - str(config["federation_args"]["round"]), - str(config["scenario_args"]["name"]), - str(config["tracking_args"]["run_hash"]), - str(config["device_args"]["malicious"]), + str(config["data"]["data"]["scenario_args"]["federation"]), + str(config["data"]["federation_args"]["round"]), + str(config["data"]["scenario_args"]["name"]), + str(config["data"]["tracking_args"]["run_hash"]), + str(config["data"]["device_args"]["malicious"]), ) except Exception as e: logging.exception(f"Error updating nodes: {e}") From b1876fb71b6ae451e760f2eda953b830bef77c7e Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Tue, 2 Sep 2025 12:24:42 +0200 Subject: [PATCH 17/26] feature federationID on docker controller --- .../docker_federation_controller.py | 250 +++++++++++------- .../controller/federation/federation_api.py | 7 +- .../federation/federation_controller.py | 10 +- .../controller/federation/scenario_builder.py | 4 +- nebula/controller/web_app_controller.py | 2 +- 5 files changed, 163 insertions(+), 110 deletions(-) diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index 2f9efbded..95b507029 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -5,7 +5,8 @@ import shutil from nebula.utils import DockerUtils, APIUtils import docker -from nebula.controller.federation.federation_controller import FederationController +from nebula.controller.federation.federation_controller import FederationController, NebulaFederation +from nebula.controller.federation.scenario_builder import ScenarioBuilder from nebula.controller.federation.utils_requests import factory_requests_path from typing import Dict from fastapi import Request @@ -13,7 +14,41 @@ from nebula.core.utils.certificate import generate_ca_certificate from nebula.core.utils.locker import Locker - +class NebulaFederationDocker(NebulaFederation): + def __init__(self): + self.participants = [] + self.round_per_participant = {} + self.additionals_participants = {} + self.additionals_deployables = [] + self.config = Config(entity="FederationController") + self.network_name = "" + self.base_network_name = "" + self.base = "" + self.last_index_deployed: int = 0 + self.federation_round: int = 0 + self.federation_deployment_lock = Locker("federation_deployment_lock", async_lock=True) + + async def get_additionals_to_be_deployed(self, config): + async with self.federation_deployment_lock: + if not self.additionals_participants: + return False + + participant_idx = int(config["device_args"]["idx"]) + participant_round = int(config["federation_args"]["round"]) + self.round_per_participant[participant_idx] = participant_round + self.federation_round = min(self.round_per_participant.values()) + + self.additionals_deployables = [ + idx + for idx, round in self.additionals_participants.items() + if self.federation_round >= round + ] + + additionals_deployables = self.additionals_deployables.copy() + for idx in additionals_deployables: + self.additionals_participants.pop(idx) + return additionals_deployables + class DockerFederationController(FederationController): def __init__(self, hub_url, logger): @@ -26,30 +61,32 @@ def __init__(self, hub_url, logger): self.cert_dir = "" self.advanced_analytics = "" self.url = "" - self.config = Config(entity="FederationController") - self.round_per_node = {} - self.additionals: dict = {} - self._ip_last_index = 0 - self._network_name = "" - self._base_network_name = "" - self._base = "" - self._deployment_lock = Locker("deployment_lock", async_lock=True) - self._federation_round = 0 + self._nebula_federations_pool: dict[tuple[str,str], NebulaFederationDocker] = {} + self._federations_dict_lock = Locker("federations_dict_lock", async_lock=True) + + @property + def nfp(self): + """Nebula Federations Pool""" + return self._nebula_federations_pool """ ############################### # ENDPOINT CALLBACKS # ############################### """ - async def run_scenario(self, id: str, scenario_data: Dict, user: str): + async def run_scenario(self, federation_id: str, scenario_data: Dict, user: str): #TODO maintain files on memory, not read them again self._user = user - await self._initialize_scenario(scenario_data) - generate_ca_certificate(dir_path=self.cert_dir) - await self._load_configuration_and_start_nodes() - self._start_initial_nodes() - - return self.sb.get_scenario_name() + federation = await self._add_nebula_federation_to_pool(federation_id, user) + id = "" + if federation: + scenario_builder = ScenarioBuilder(federation_id) + await self._initialize_scenario(scenario_builder, scenario_data, federation) + generate_ca_certificate(dir_path=self.cert_dir) + await self._load_configuration_and_start_nodes(scenario_builder, federation) + self._start_initial_nodes(scenario_builder, federation) + id = scenario_builder.get_scenario_name() + return id async def stop_scenario(self, id: str): """ @@ -137,45 +174,36 @@ async def stop_scenario(self, id: str): async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() - participant_idx = int(config["device_args"]["idx"]) - participant_round = int(config["federation_args"]["round"]) - #self.logger.info(f"Update received from participant: {participant_idx}, round: {participant_round}") - #self.logger.info(f"Update: {self.round_per_node.items()}") - self.round_per_node[participant_idx] = participant_round - last_fed_round = self._federation_round - self._federation_round = min(self.round_per_node.values()) - - additionals_deployables = [ - idx - for idx, round in self.additionals.items() - if self._federation_round >= round - ] - - adds_deployed = set() - # Only verify when federation round is updated - if self._federation_round != last_fed_round: - self.logger.info(f"Federation Round updating, current value: {self._federation_round}") - # Ensure concurrency - for index in additionals_deployables: - if index in adds_deployed: - continue - - for idx, node in enumerate(self.config.participants): - if index == idx: - async with self._deployment_lock: - if index in self.additionals.keys(): - self.logger.info(f"Deploying additional participant: {index}") - self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index, additional=True) - self._ip_last_index += 1 - self.additionals.pop(index) - adds_deployed.add(index) - - request_body = await request.json() - payload = {"scenario_name": scenario_name, "data": request_body} - - asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) - - return {"message": "Node updated successfully in Federation Controller"} + fed_id = config["scenario_args"]["federation_id"] + + try: + nebula_federation = self.nfp[fed_id] + last_fed_round = nebula_federation.federation_round + additionals = nebula_federation.get_additionals_to_be_deployed(config) # It modifies if neccesary the federation round + if additionals: + current_fed_round = nebula_federation.federation_round + adds_deployed = set() + if current_fed_round != last_fed_round: + self.logger.info(f"Federation Round updating for ID: ({fed_id}), current value: {current_fed_round}") + for index in additionals: + if index in adds_deployed: + continue + + for idx, node in enumerate(nebula_federation.config.participants): + if index == idx: + if index in additionals: + self.logger.info(f"Deploying additional participant: {index}") + self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation, additional=True) + nebula_federation.last_index_deployed += 1 + self.additionals.pop(index) + adds_deployed.add(index) + request_body = await request.json() + payload = {"scenario_name": scenario_name, "data": request_body} + asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) + return {"message": "Node updated successfully in Federation Controller"} + except Exception as e: + self.logger.info(f"ERROR: federation ID: ({fed_id}) not found on pool..") + return {"message": "Node updated failed in Federation Controller, ID not found.."} async def node_done(self, scenario_name: str, request: Request): request_body = await request.json() @@ -188,6 +216,28 @@ async def node_done(self, scenario_name: str, request: Request): ############################### """ + async def _add_nebula_federation_to_pool(self, federation_id: str, user: str): + fed = None + async with self._federations_dict_lock: + if not federation_id in self.nfp: + fed = NebulaFederation() + self.nfp[federation_id] = fed + self.logger.info(f"SUCCESS: new ID: ({federation_id}) added to the pool") + else: + self.logger.info(f"ERROR: trying to add ({federation_id}) to federations pool..") + return fed + + async def _update_federation_on_pool(self, federation_id: str, user: str, nf: NebulaFederation): + updated = False + async with self._federations_dict_lock: + if not federation_id in self.nfp: + self.nfp[federation_id] = nf + updated = True + self.logger.info(f"UPDATED: federation: ({federation_id}) successfully updated") + else: + self.logger.info(f"ERROR: trying to update ({federation_id}) on federations pool..") + return updated + async def _send_to_hub(self, path, payload, scenario_name=""): try: url_request = self._hub_url + factory_requests_path(path, scenario_name) @@ -197,11 +247,11 @@ async def _send_to_hub(self, path, payload, scenario_name=""): except Exception as e: self.logger.info(f"Failed to send update to Hub: {e}") - async def _initialize_scenario(self, scenario_data): + async def _initialize_scenario(self, sb: ScenarioBuilder, scenario_data, federation: NebulaFederationDocker): # Initialize Scenario builder using scenario_data from user self.logger.info("🔧 Initializing Scenario Builder using scenario data") - self.sb.set_scenario_data(scenario_data) - scenario_name = self.sb.get_scenario_name() + sb.set_scenario_data(scenario_data) + scenario_name = sb.get_scenario_name() self.root_path = os.environ.get("NEBULA_ROOT_HOST") self.host_platform = os.environ.get("NEBULA_HOST_PLATFORM") @@ -209,7 +259,7 @@ async def _initialize_scenario(self, scenario_data): self.log_dir = os.environ.get("NEBULA_LOGS_DIR") self.cert_dir = os.environ.get("NEBULA_CERTS_DIR") self.advanced_analytics = os.environ.get("NEBULA_ADVANCED_ANALYTICS", "False") == "True" - self.config = Config(entity="scenarioManagement") + #self.config = Config(entity="FederationController") self.env_tag = os.environ.get("NEBULA_ENV_TAG", "dev") self.prefix_tag = os.environ.get("NEBULA_PREFIX_TAG", "dev") self.user_tag = os.environ.get("NEBULA_USER_TAG", os.environ.get("USER", "unknown")) @@ -251,11 +301,11 @@ async def _initialize_scenario(self, scenario_data): # Attacks assigment and mobility self.logger.info("🔧 Building general configuration") - self.sb.build_general_configuration() + sb.build_general_configuration() self.logger.info("✅ Building general configuration done") # Create participant configs and .json - for index, (_, node) in enumerate(self.sb.get_federation_nodes().items()): + for index, (_, node) in enumerate(sb.get_federation_nodes().items()): self.logger.info(f"Creating .json file for participant: {index}, Configuration: {node}") node_config = node try: @@ -266,7 +316,7 @@ async def _initialize_scenario(self, scenario_data): self.logger.info(f"ERROR while creating files: {e}") try: - participant_config = self.sb.build_scenario_config_for_node(index, node) + participant_config = sb.build_scenario_config_for_node(index, node) #self.logger.info(f"dictionary: {participant_config}") except Exception as e: self.logger.info(f"ERROR while building configuration for node: {e}") @@ -280,7 +330,7 @@ async def _initialize_scenario(self, scenario_data): self.logger.info("✅ Initializing Scenario Builder done") - async def _load_configuration_and_start_nodes(self): + async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federation: NebulaFederationDocker): self.logger.info("🔧 Loading Scenario configuration...") # Get participants configurations participant_files = glob.glob(f"{self.config_dir}/participant_*.json") @@ -288,25 +338,25 @@ async def _load_configuration_and_start_nodes(self): if len(participant_files) == 0: raise ValueError("No participant files found in config folder") - self.config.set_participants_config(participant_files) - self.n_nodes = len(participant_files) - self.logger.info(f"Number of nodes: {self.n_nodes}") + federation.config.set_participants_config(participant_files) + n_nodes = len(participant_files) + #self.logger.info(f"Number of nodes: {n_nodes}") - self.sb.create_topology_manager(self.config) + sb.create_topology_manager(federation.config) # Update participants configuration is_start_node = False config_participants = [] - additional_participants = self.sb.get_additional_nodes() + additional_participants = sb.get_additional_nodes() additional_nodes = len(additional_participants) if additional_participants else 0 - self.logger.info(f"######## nodes: {self.n_nodes} + additionals: {additional_nodes} ######") + #self.logger.info(f"######## nodes: {n_nodes} + additionals: {additional_nodes} ######") participant_files.sort(key=lambda x: int(x.split("_")[-1].split(".")[0])) # Initial participants self.logger.info("🔧 Building preload configuration for initial nodes...") - for i in range(self.n_nodes): + for i in range(n_nodes): try: with open(f"{self.config_dir}/participant_" + str(i) + ".json") as f: participant_config = json.load(f) @@ -315,7 +365,7 @@ async def _load_configuration_and_start_nodes(self): self.logger.info(f"Building preload conf for participant {i}") try: - self.sb.build_preload_initial_node_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) + sb.build_preload_initial_node_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) except Exception as e: self.logger.info(f"ERROR: cannot build preload configuration") @@ -338,10 +388,10 @@ async def _load_configuration_and_start_nodes(self): self.logger.info("✅ Building preload configuration for initial nodes done") - self.config.set_participants_config(participant_files) + federation.config.set_participants_config(participant_files) # Add role to the topology (visualization purposes) - self.sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) + sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) # Additional participants self.logger.info("🔧 Building preload configuration for additional nodes...") @@ -357,8 +407,8 @@ async def _load_configuration_and_start_nodes(self): with open(additional_participant_file) as f: participant_config = json.load(f) - self.logger.info(f"Configuration | additional nodes | participant: {self.n_nodes + i}") - self.sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) + self.logger.info(f"Configuration | additional nodes | participant: {n_nodes + i}") + sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) with open(additional_participant_file, "w") as f: json.dump(participant_config, f, sort_keys=False, indent=2) @@ -366,19 +416,19 @@ async def _load_configuration_and_start_nodes(self): additional_participants_files.append(additional_participant_file) if additional_participants_files: - self.config.add_participants_config(additional_participants_files) + federation.config.add_participants_config(additional_participants_files) if additional_participants: - self.n_nodes += len(additional_participants) + n_nodes += len(additional_participants) self.logger.info("✅ Building preload configuration for additional nodes done") self.logger.info("✅ Loading Scenario configuration done") # Build dataset - dataset = self.sb.configure_dataset(self.config_dir) - self.logger.info(f"🔧 Splitting {self.sb.get_dataset_name()} dataset...") + dataset = sb.configure_dataset(self.config_dir) + self.logger.info(f"🔧 Splitting {sb.get_dataset_name()} dataset...") dataset.initialize_dataset() - self.logger.info(f"✅ Splitting {self.sb.get_dataset_name()} dataset... Done") + self.logger.info(f"✅ Splitting {sb.get_dataset_name()} dataset... Done") def _get_network_name(self, suffix: str) -> str: """ @@ -390,7 +440,7 @@ def _get_network_name(self, suffix: str) -> str: """ return f"{self.env_tag}_{self.prefix_tag}_{self.user_tag}_{suffix}" - def _get_participant_container_name(self, idx: int) -> str: + def _get_participant_container_name(self, scenario_name, idx: int) -> str: """ Generate a standardized container name for a participant using tags. Args: @@ -398,39 +448,39 @@ def _get_participant_container_name(self, idx: int) -> str: Returns: str: The composed container name. """ - return f"{self.env_tag}_{self.prefix_tag}_{self.user_tag}_{self.sb.get_scenario_name()}_participant{idx}" + return f"{self.env_tag}_{self.prefix_tag}_{self.user_tag}_{scenario_name}_participant{idx}" - def _start_initial_nodes(self): + def _start_initial_nodes(self, sb: ScenarioBuilder, federation: NebulaFederationDocker): self.logger.info("Starting nodes using Docker Compose...") - self._network_name = self._get_network_name(f"{self.sb.get_scenario_name()}-net-scenario") - self._base_network_name = self._get_network_name("net-base") + federation.network_name = self._get_network_name(f"{sb.get_scenario_name()}-net-scenario") + federation.base_network_name = self._get_network_name("net-base") # Create the Docker network - self._base = DockerUtils.create_docker_network(self._network_name) + federation.base = DockerUtils.create_docker_network(federation.network_name) - self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) - self._ip_last_index = 2 + federation.config.participants.sort(key=lambda x: x["device_args"]["idx"]) + federation.last_index_deployed = 2 for idx, node in enumerate(self.config.participants): if node["deployment_args"]["additional"]: - self.additionals[idx] = int(node["deployment_args"]["deployment_round"]) + federation.additionals_participants[idx] = int(node["deployment_args"]["deployment_round"]) self.logger.info(f"Participant {idx} is additional. Round of deployment: {int(node['deployment_args']['deployment_round'])}") else: # deploy initial nodes self.logger.info(f"Deployment starting for participant {idx}") - self.round_per_node[idx] = 0 - self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index) - self._ip_last_index += 1 + federation.round_per_participant[idx] = 0 + self._start_node(sb, node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed) + federation.last_index_deployed += 1 - def _start_node(self, node, network_name, base_network_name, base, i, additional=False): + def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name, base, i, federation: NebulaFederationDocker, additional=False): client = docker.from_env() - self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) + federation.config.participants.sort(key=lambda x: x["device_args"]["idx"]) container_ids = [] container_names = [] # Track names for metadata image = "nebula-core" - name = self._get_participant_container_name(node["device_args"]["idx"]) + name = self._get_participant_container_name(sb.get_scenario_name(), node["device_args"]["idx"]) if node["device_args"]["accelerator"] == "gpu": environment = { "NVIDIA_DISABLE_REQUIRE": True, @@ -456,7 +506,7 @@ def _start_node(self, node, network_name, base_network_name, base, i, additional command = [ "/bin/bash", "-c", - f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json", + f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json", ] networking_config = client.api.create_networking_config({ network_name: client.api.create_endpoint_config( @@ -465,9 +515,9 @@ def _start_node(self, node, network_name, base_network_name, base, i, additional base_network_name: client.api.create_endpoint_config(), }) node["tracking_args"]["log_dir"] = "/nebula/app/logs" - node["tracking_args"]["config_dir"] = f"/nebula/app/config/{self.sb.get_scenario_name()}" + node["tracking_args"]["config_dir"] = f"/nebula/app/config/{sb.get_scenario_name()}" node["scenario_args"]["controller"] = self.url - node["scenario_args"]["deployment"] = self.sb.get_deployment() + node["scenario_args"]["deployment"] = sb.get_deployment() node["security_args"]["certfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_cert.pem" node["security_args"]["keyfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_key.pem" node["security_args"]["cafile"] = "/nebula/app/certs/ca_cert.pem" diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index acf9c2aa6..d828f6ab3 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -12,6 +12,9 @@ from nebula.controller.federation.factory_federation_controller import federation_controller_factory from nebula.controller.federation.utils_requests import InitFederationRequest, RunScenarioRequest, StopScenarioRequest +#TODO we need all 3 controllers instanciated. When /init received we put the request on the right +# controller + def require_initialized_controller(func): @wraps(func) async def wrapper(*args, **kwargs): @@ -48,7 +51,6 @@ async def read_root(): logger.info("Test curl succesfull") return {"message": "Welcome to the NEBULA Federation Controller API"} -#TODO modificar para q reciba str en vez de dict @app.post("/init") async def init_federation_experiment(ifr: InitFederationRequest): global fed_controller @@ -57,7 +59,7 @@ async def init_federation_experiment(ifr: InitFederationRequest): logger = logging.getLogger("Federation-Controller") logger.info(f"Experiment type received: {experiment_type}") - # Modify when deploying controllers on differents systems + #TODO Modify when deploying controllers on differents systems hub_port = os.environ.get("NEBULA_CONTROLLER_PORT") controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") @@ -68,6 +70,7 @@ async def init_federation_experiment(ifr: InitFederationRequest): return {"message": f"{experiment_type} controller initialized"} +#ADVICE: return ID if sucess otherwise empty string @app.post("/scenarios/run") @require_initialized_controller async def run_scenario(run_scenario_request: RunScenarioRequest): diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index b5c82cd89..688927468 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -4,17 +4,15 @@ from nebula.controller.federation.scenario_builder import ScenarioBuilder import logging +class NebulaFederation(ABC): + pass + class FederationController(ABC): def __init__(self, hub_url, logger): self._logger: logging.Logger = logger self._hub_url = hub_url - self._scenario_builder = ScenarioBuilder() - - @property - def sb(self): - return self._scenario_builder - + @property def logger(self): return self._logger diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index d5a66665b..6779b4703 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -9,12 +9,13 @@ from nebula.core.datasets.nebuladataset import NebulaDataset, factory_nebuladataset, factory_dataset_setup class ScenarioBuilder(): - def __init__(self, ): + def __init__(self, federation_id): self._scenario_data = None self._config_setup = None self.logger = logging.getLogger("Federation-Controller") self._topology_manager: TopologyManager = None self._scenario_name = "" + self._federation_id = federation_id @property def sd(self): @@ -400,6 +401,7 @@ def dictify(d): # General configuration participant_config["scenario_args"]["name"] = self._scenario_name participant_config["scenario_args"]["start_time"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + participant_config["scenario_args"]["federation_id"] = self._federation_id participant_config["deployment_args"]["additional"] = False node_config = node #self.sd["nodes"][index] diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index bcd11b963..b1ff9673b 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -325,7 +325,7 @@ async def run_scenario( url_init_fed_controller = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("init") url_run_scenario = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("run") init_fed_req = InitFederationRequest(experiment_type="docker") - run_scenario_req = RunScenarioRequest(scenario_data=scenario_data, federation_id="id_nebula", user=user) + run_scenario_req = RunScenarioRequest(scenario_data=scenario_data, federation_id="id_nebula", user=user) #TODO ID per experiment await APIUtils.post(url_init_fed_controller, init_fed_req.model_dump()) await APIUtils.post(url_run_scenario, run_scenario_req.model_dump()) except Exception as e: From 9ae3cb5c36a3df2d85d2643d3e3cc327259cf7a9 Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 4 Sep 2025 12:27:33 +0200 Subject: [PATCH 18/26] feature process deployment and completion of comunication hub-controller --- nebula/addons/reporter.py | 4 +- .../docker_federation_controller.py | 60 +++-- .../processes_federation_controller.py | 254 ++++++++++++------ .../controller/federation/federation_api.py | 69 +++-- .../federation/federation_controller.py | 2 +- .../controller/federation/utils_requests.py | 4 +- nebula/controller/web_app_controller.py | 8 +- 7 files changed, 263 insertions(+), 138 deletions(-) diff --git a/nebula/addons/reporter.py b/nebula/addons/reporter.py index 376f6a208..4ca82b91d 100755 --- a/nebula/addons/reporter.py +++ b/nebula/addons/reporter.py @@ -171,7 +171,9 @@ async def report_scenario_finished(self): - Logs exceptions if the connection attempt to the controller fails. """ url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant['scenario_args']['name']}/done" - data = json.dumps({"idx": self.config.participant["device_args"]["idx"]}) + data = json.dumps({"idx": self.config.participant["device_args"]["idx"], + "deployment": self.config.participant["scenario_args"]["deployment"], + "federation_id": self.config.participant["scenario_args"]["federation_id"]}) headers = { "Content-Type": "application/json", "User-Agent": f"NEBULA Participant {self.config.participant['device_args']['idx']}", diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index 95b507029..c4a35d99e 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -5,7 +5,7 @@ import shutil from nebula.utils import DockerUtils, APIUtils import docker -from nebula.controller.federation.federation_controller import FederationController, NebulaFederation +from nebula.controller.federation.federation_controller import FederationController from nebula.controller.federation.scenario_builder import ScenarioBuilder from nebula.controller.federation.utils_requests import factory_requests_path from typing import Dict @@ -14,9 +14,9 @@ from nebula.core.utils.certificate import generate_ca_certificate from nebula.core.utils.locker import Locker -class NebulaFederationDocker(NebulaFederation): +class NebulaFederationDocker(): def __init__(self): - self.participants = [] + self.participants_alive = 0 self.round_per_participant = {} self.additionals_participants = {} self.additionals_deployables = [] @@ -27,8 +27,9 @@ def __init__(self): self.last_index_deployed: int = 0 self.federation_round: int = 0 self.federation_deployment_lock = Locker("federation_deployment_lock", async_lock=True) + self.participants_alive_lock = Locker("participants_alive_lock", async_lock=True) - async def get_additionals_to_be_deployed(self, config): + async def get_additionals_to_be_deployed(self, config) -> list: async with self.federation_deployment_lock: if not self.additionals_participants: return False @@ -48,6 +49,16 @@ async def get_additionals_to_be_deployed(self, config): for idx in additionals_deployables: self.additionals_participants.pop(idx) return additionals_deployables + + async def is_experiment_finish(self): + async with self.participants_alive_lock: + self.participants_alive -= 1 + if self.participants_alive <= 0: + return True + else: + return False + + class DockerFederationController(FederationController): @@ -178,8 +189,9 @@ async def update_nodes(self, scenario_name: str, request: Request): try: nebula_federation = self.nfp[fed_id] + self.logger.info(f"Update received from node on federation ID: ({fed_id})") last_fed_round = nebula_federation.federation_round - additionals = nebula_federation.get_additionals_to_be_deployed(config) # It modifies if neccesary the federation round + additionals = await nebula_federation.get_additionals_to_be_deployed(config) # It modifies if neccesary the federation round if additionals: current_fed_round = nebula_federation.federation_round adds_deployed = set() @@ -195,7 +207,7 @@ async def update_nodes(self, scenario_name: str, request: Request): self.logger.info(f"Deploying additional participant: {index}") self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation, additional=True) nebula_federation.last_index_deployed += 1 - self.additionals.pop(index) + additionals.remove(index) adds_deployed.add(index) request_body = await request.json() payload = {"scenario_name": scenario_name, "data": request_body} @@ -207,9 +219,16 @@ async def update_nodes(self, scenario_name: str, request: Request): async def node_done(self, scenario_name: str, request: Request): request_body = await request.json() + federation_id = request_body["federation_id"] + nebula_federation = self.nfp[federation_id] + + if await nebula_federation.is_experiment_finish(): + payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} + asyncio.create_task(self._send_to_hub("finish", payload, scenario_name)) + payload = {"scenario_name": scenario_name, "data": request_body} asyncio.create_task(self._send_to_hub("done", payload, scenario_name)) - return {"message": "Nodes done"} + return {"message": "Nodes done received successfully"} """ ############################### # FUNCTIONALITIES # @@ -220,14 +239,14 @@ async def _add_nebula_federation_to_pool(self, federation_id: str, user: str): fed = None async with self._federations_dict_lock: if not federation_id in self.nfp: - fed = NebulaFederation() + fed = NebulaFederationDocker() self.nfp[federation_id] = fed self.logger.info(f"SUCCESS: new ID: ({federation_id}) added to the pool") else: self.logger.info(f"ERROR: trying to add ({federation_id}) to federations pool..") return fed - async def _update_federation_on_pool(self, federation_id: str, user: str, nf: NebulaFederation): + async def _update_federation_on_pool(self, federation_id: str, user: str, nf: NebulaFederationDocker): updated = False async with self._federations_dict_lock: if not federation_id in self.nfp: @@ -238,9 +257,9 @@ async def _update_federation_on_pool(self, federation_id: str, user: str, nf: Ne self.logger.info(f"ERROR: trying to update ({federation_id}) on federations pool..") return updated - async def _send_to_hub(self, path, payload, scenario_name=""): + async def _send_to_hub(self, path, payload, scenario_name="", federation_id="" ): try: - url_request = self._hub_url + factory_requests_path(path, scenario_name) + url_request = self._hub_url + factory_requests_path(path, scenario_name, federation_id) # self.logger.info(f"Seding to hub, url: {url_request}") # self.logger.info(f"payload sent to hub, data: {payload}") await APIUtils.post(url_request, payload) @@ -460,19 +479,23 @@ def _start_initial_nodes(self, sb: ScenarioBuilder, federation: NebulaFederation federation.config.participants.sort(key=lambda x: x["device_args"]["idx"]) federation.last_index_deployed = 2 - for idx, node in enumerate(self.config.participants): + for idx, node in enumerate(federation.config.participants): if node["deployment_args"]["additional"]: federation.additionals_participants[idx] = int(node["deployment_args"]["deployment_round"]) + federation.participants_alive += 1 self.logger.info(f"Participant {idx} is additional. Round of deployment: {int(node['deployment_args']['deployment_round'])}") else: # deploy initial nodes self.logger.info(f"Deployment starting for participant {idx}") federation.round_per_participant[idx] = 0 - self._start_node(sb, node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed) - federation.last_index_deployed += 1 + deployed_successfully = self._start_node(sb, node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed, federation) + if deployed_successfully: + federation.last_index_deployed += 1 + federation.participants_alive += 1 def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name, base, i, federation: NebulaFederationDocker, additional=False): + success = True client = docker.from_env() federation.config.participants.sort(key=lambda x: x["device_args"]["idx"]) @@ -525,6 +548,7 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name try: existing = client.containers.get(name) self.logger.warning(f"Container {name} already exists. Deployment may fail or cause conflicts.") + success = False except docker.errors.NotFound: pass # No conflict, safe to proceed # Write the config file in config directory @@ -542,14 +566,16 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name networking_config=networking_config, ) except Exception as e: + success = False self.logger.exception(f"Creating container {name}: {e}") try: client.api.start(container_id) container_ids.append(container_id) container_names.append(name) except Exception as e: + success = False self.logger.exception(f"Starting participant {name} error: {e}") - + # Write scenario-level metadata for cleanup scenario_metadata = {"containers": container_names, "network": network_name} if not additional: @@ -557,4 +583,6 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name json.dump(scenario_metadata, f, indent=2) else: with open(os.path.join(self.config_dir, "scenario.metadata"), "a") as f: - json.dump(scenario_metadata, f, indent=2) \ No newline at end of file + json.dump(scenario_metadata, f, indent=2) + + return success \ No newline at end of file diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index ee4240bb7..853d8e6e5 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -6,6 +6,7 @@ from nebula.utils import APIUtils import docker from nebula.controller.federation.federation_controller import FederationController +from nebula.controller.federation.scenario_builder import ScenarioBuilder from nebula.controller.federation.utils_requests import factory_requests_path from typing import Dict from fastapi import Request @@ -13,6 +14,52 @@ from nebula.core.utils.certificate import generate_ca_certificate from nebula.core.utils.locker import Locker +#TODO save participants when deployed +class NebulaFederationProcesses(): + def __init__(self): + self.scenario_name = "" + self.participants_alive = 0 + self.round_per_participant = {} + self.additionals_participants = {} + self.additionals_deployables = [] + self.config = Config(entity="FederationController") + self.network_name = "" + self.base_network_name = "" + self.base = "" + self.last_index_deployed: int = 0 + self.federation_round: int = 0 + self.federation_deployment_lock = Locker("federation_deployment_lock", async_lock=True) + self.participants_alive_lock = Locker("participants_alive_lock", async_lock=True) + + async def get_additionals_to_be_deployed(self, config) -> list: + async with self.federation_deployment_lock: + if not self.additionals_participants: + return False + + participant_idx = int(config["device_args"]["idx"]) + participant_round = int(config["federation_args"]["round"]) + self.round_per_participant[participant_idx] = participant_round + self.federation_round = min(self.round_per_participant.values()) + + self.additionals_deployables = [ + idx + for idx, round in self.additionals_participants.items() + if self.federation_round >= round + ] + + additionals_deployables = self.additionals_deployables.copy() + for idx in additionals_deployables: + self.additionals_participants.pop(idx) + return additionals_deployables + + async def is_experiment_finish(self): + async with self.participants_alive_lock: + self.participants_alive -= 1 + if self.participants_alive <= 0: + return True + else: + return False + class ProcessesFederationController(FederationController): def __init__(self, hub_url, logger): super().__init__(hub_url, logger) @@ -24,29 +71,41 @@ def __init__(self, hub_url, logger): self.cert_dir = "" self.advanced_analytics = "" self.url = "" - self.config = Config(entity="FederationController") - self.round_per_node = {} - self.additionals: dict = {} - self._last_file_index = 0 - self._deployment_lock = Locker("deployment_lock", async_lock=True) - self._federation_round = 0 + + self._nebula_federations_pool: dict[tuple[str,str], NebulaFederationProcesses] = {} + self._federations_dict_lock = Locker("federations_dict_lock", async_lock=True) + + @property + def nfp(self): + """Nebula Federations Pool""" + return self._nebula_federations_pool """ ############################### # ENDPOINT CALLBACKS # ############################### """ - async def run_scenario(self, id: str, scenario_data: Dict, user: str): + async def run_scenario(self, federation_id: str, scenario_data: Dict, user: str): #TODO maintain files on memory, not read them again self._user = user - await self._initialize_scenario(scenario_data) - generate_ca_certificate(dir_path=self.cert_dir) - await self._load_configuration_and_start_nodes() - self._start_initial_nodes() - - return self.sb.get_scenario_name() + federation = await self._add_nebula_federation_to_pool(federation_id, user) + id = "" + if federation: + scenario_builder = ScenarioBuilder(federation_id) + await self._initialize_scenario(scenario_builder, scenario_data, federation) + generate_ca_certificate(dir_path=self.cert_dir) + await self._load_configuration_and_start_nodes(scenario_builder, federation) + self._start_initial_nodes(scenario_builder, federation) + id = scenario_builder.get_scenario_name() + try: + nebula_federation = self.nfp[federation_id] + nebula_federation.scenario_name = id + except Exception as e: + self.logger.info(f"ERROR: federation ID: ({federation_id}) not found on pool..") + return None + return id - async def stop_scenario(self, id: str = ""): + async def stop_scenario(self, federation_id: str = ""): """ Stop running participant nodes by removing the scenario command files. @@ -72,15 +131,15 @@ async def stop_scenario(self, id: str = ""): nebula_base_dir = os.path.abspath(os.path.join(current_dir, "..", "..")) nebula_config_dir = os.path.join(nebula_base_dir, "app", "config") self.logger.info(f"NEBULA_CONFIG_DIR not found. Using default path: {nebula_config_dir}") - + nebula_federation = self.nfp[federation_id] if id: if os.environ.get("NEBULA_HOST_PLATFORM") == "windows": scenario_commands_file = os.path.join( - nebula_config_dir, self.sb.get_scenario_name(), "current_scenario_commands.ps1" + nebula_config_dir, nebula_federation.scenario_name, "current_scenario_commands.ps1" ) else: scenario_commands_file = os.path.join( - nebula_config_dir, self.sb.get_scenario_name(), "current_scenario_commands.sh" + nebula_config_dir, nebula_federation.scenario_name, "current_scenario_commands.sh" ) if os.path.exists(scenario_commands_file): os.remove(scenario_commands_file) @@ -100,67 +159,81 @@ async def stop_scenario(self, id: str = ""): async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() - participant_idx = int(config["device_args"]["idx"]) - participant_round = int(config["federation_args"]["round"]) - self.round_per_node[participant_idx] = participant_round - last_fed_round = self._federation_round - self._federation_round = min(self.round_per_node.values()) - - additionals_deployables = [ - idx - for idx, round in self.additionals.items() - if self._federation_round >= round - ] - - adds_deployed = set() - # Only verify when federation round is updated - if self._federation_round != last_fed_round: - self.logger.info(f"Federation Round updating, current value: {self._federation_round}") - # Ensure concurrency - for index in additionals_deployables: - if index in adds_deployed: - continue - - for idx, node in enumerate(self.config.participants): - if index == idx: - async with self._deployment_lock: - if index in self.additionals.keys(): - self.logger.info(f"Deploying additional participant: {index}") - self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index, additional=True) - self._ip_last_index += 1 - self.additionals.pop(index) - adds_deployed.add(index) - - request_body = await request.json() - payload = {"scenario_name": scenario_name, "data": request_body} - - asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) - - return {"message": "Node updated successfully in Federation Controller"} + fed_id = config["scenario_args"]["federation_id"] + + try: + nebula_federation = self.nfp[fed_id] + self.logger.info(f"Update received from node on federation ID: ({fed_id})") + last_fed_round = nebula_federation.federation_round + additionals = await nebula_federation.get_additionals_to_be_deployed(config) # It modifies if neccesary the federation round + if additionals: + current_fed_round = nebula_federation.federation_round + adds_deployed = set() + if current_fed_round != last_fed_round: + self.logger.info(f"Federation Round updating for ID: ({fed_id}), current value: {current_fed_round}") + for index in additionals: + if index in adds_deployed: + continue + + for idx, node in enumerate(nebula_federation.config.participants): + if index == idx: + if index in additionals: + self.logger.info(f"Deploying additional participant: {index}") + self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation, additional=True) + nebula_federation.last_index_deployed += 1 + additionals.remove(index) + adds_deployed.add(index) + request_body = await request.json() + payload = {"scenario_name": scenario_name, "data": request_body} + asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) + return {"message": "Node updated successfully in Federation Controller"} + except Exception as e: + self.logger.info(f"ERROR: federation ID: ({fed_id}) not found on pool..") + return {"message": "Node updated failed in Federation Controller, ID not found.."} async def node_done(self, scenario_name: str, request: Request): request_body = await request.json() + federation_id = request_body["federation_id"] + nebula_federation = self.nfp[federation_id] + + if await nebula_federation.is_experiment_finish(): + payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} + asyncio.create_task(self._send_to_hub("finish", payload, scenario_name)) + payload = {"scenario_name": scenario_name, "data": request_body} asyncio.create_task(self._send_to_hub("done", payload, scenario_name)) - return {"message": "Nodes done"} + return {"message": "Nodes done received successfully"} """ ############################### # FUNCTIONALITIES # ############################### """ - async def _send_to_hub(self, path, payload, scenario_name=""): + async def _add_nebula_federation_to_pool(self, federation_id: str, user: str): + fed = None + async with self._federations_dict_lock: + if not federation_id in self.nfp: + fed = NebulaFederationProcesses() + self.nfp[federation_id] = fed + self.logger.info(f"SUCCESS: new ID: ({federation_id}) added to the pool") + else: + self.logger.info(f"ERROR: trying to add ({federation_id}) to federations pool..") + return fed + + async def _send_to_hub(self, path, payload, scenario_name="", federation_id="" ): try: - url_request = self._hub_url + factory_requests_path(path, scenario_name) + url_request = self._hub_url + factory_requests_path(path, scenario_name, federation_id) + # self.logger.info(f"Seding to hub, url: {url_request}") + # self.logger.info(f"payload sent to hub, data: {payload}") await APIUtils.post(url_request, payload) except Exception as e: self.logger.info(f"Failed to send update to Hub: {e}") - async def _initialize_scenario(self, scenario_data): + async def _initialize_scenario(self, sb: ScenarioBuilder, scenario_data, federation: NebulaFederationProcesses): # Initialize Scenario builder using scenario_data from user self.logger.info("🔧 Initializing Scenario Builder using scenario data") - self.sb.set_scenario_data(scenario_data) - scenario_name = self.sb.get_scenario_name() + sb.set_scenario_data(scenario_data) + scenario_name = sb.get_scenario_name() self.root_path = os.environ.get("NEBULA_ROOT_HOST") self.host_platform = os.environ.get("NEBULA_HOST_PLATFORM") @@ -210,11 +283,11 @@ async def _initialize_scenario(self, scenario_data): # Attacks assigment and mobility self.logger.info("🔧 Building general configuration") - self.sb.build_general_configuration() + sb.build_general_configuration() self.logger.info("✅ Building general configuration done") # Create participant configs and .json - for index, (_, node) in enumerate(self.sb.get_federation_nodes().items()): + for index, (_, node) in enumerate(sb.get_federation_nodes().items()): self.logger.info(f"Creating .json file for participant: {index}, Configuration: {node}") node_config = node try: @@ -225,7 +298,7 @@ async def _initialize_scenario(self, scenario_data): self.logger.info(f"ERROR while creating files: {e}") try: - participant_config = self.sb.build_scenario_config_for_node(index, node) + participant_config = sb.build_scenario_config_for_node(index, node) #self.logger.info(f"dictionary: {participant_config}") except Exception as e: self.logger.info(f"ERROR while building configuration for node: {e}") @@ -239,7 +312,7 @@ async def _initialize_scenario(self, scenario_data): self.logger.info("✅ Initializing Scenario Builder done") - async def _load_configuration_and_start_nodes(self): + async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federation: NebulaFederationProcesses): self.logger.info("🔧 Loading Scenario configuration...") # Get participants configurations participant_files = glob.glob(f"{self.config_dir}/participant_*.json") @@ -251,13 +324,13 @@ async def _load_configuration_and_start_nodes(self): self.n_nodes = len(participant_files) self.logger.info(f"Number of nodes: {self.n_nodes}") - self.sb.create_topology_manager(self.config) + sb.create_topology_manager(self.config) # Update participants configuration is_start_node = False config_participants = [] - additional_participants = self.sb.get_additional_nodes() + additional_participants = sb.get_additional_nodes() additional_nodes = len(additional_participants) if additional_participants else 0 self.logger.info(f"######## nodes: {self.n_nodes} + additionals: {additional_nodes} ######") @@ -274,7 +347,7 @@ async def _load_configuration_and_start_nodes(self): self.logger.info(f"Building preload conf for participant {i}") try: - self.sb.build_preload_initial_node_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) + sb.build_preload_initial_node_configuration(i, participant_config, self.log_dir, self.config_dir, self.cert_dir, self.advanced_analytics) except Exception as e: self.logger.info(f"ERROR: cannot build preload configuration") @@ -300,7 +373,7 @@ async def _load_configuration_and_start_nodes(self): self.config.set_participants_config(participant_files) # Add role to the topology (visualization purposes) - self.sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) + sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) # Additional participants self.logger.info("🔧 Building preload configuration for additional nodes...") @@ -317,7 +390,7 @@ async def _load_configuration_and_start_nodes(self): participant_config = json.load(f) self.logger.info(f"Configuration | additional nodes | participant: {self.n_nodes + i}") - self.sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) + sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) with open(additional_participant_file, "w") as f: json.dump(participant_config, f, sort_keys=False, indent=2) @@ -334,39 +407,42 @@ async def _load_configuration_and_start_nodes(self): self.logger.info("✅ Loading Scenario configuration done") # Build dataset - dataset = self.sb.configure_dataset(self.config_dir) - self.logger.info(f"🔧 Splitting {self.sb.get_dataset_name()} dataset...") + dataset = sb.configure_dataset(self.config_dir) + self.logger.info(f"🔧 Splitting {sb.get_dataset_name()} dataset...") dataset.initialize_dataset() - self.logger.info(f"✅ Splitting {self.sb.get_dataset_name()} dataset... Done") + self.logger.info(f"✅ Splitting {sb.get_dataset_name()} dataset... Done") - def _start_initial_nodes(self): + def _start_initial_nodes(self, sb: ScenarioBuilder, federation: NebulaFederationProcesses): self.logger.info("Starting nodes as processes...") - self.config.participants.sort(key=lambda x: x["device_args"]["idx"]) - self._ip_last_index = 2 - for idx, node in enumerate(self.config.participants): + federation.config.participants.sort(key=lambda x: x["device_args"]["idx"]) + federation.last_index_deployed = 2 + for idx, node in enumerate(federation.config.participants): if node["deployment_args"]["additional"]: - self.additionals[idx] = int(node["deployment_args"]["deployment_round"]) + federation.additionals_participants[idx] = int(node["deployment_args"]["deployment_round"]) + federation.participants_alive += 1 self.logger.info(f"Participant {idx} is additional. Round of deployment: {int(node['deployment_args']['deployment_round'])}") else: # deploy initial nodes self.logger.info(f"Deployment starting for participant {idx}") - self.round_per_node[idx] = 0 - self._start_node(node, self._network_name, self._base_network_name, self._base, self._ip_last_index) - self._ip_last_index += 1 + self.logger.info(f"Deployment starting for participant {idx}") + federation.round_per_participant[idx] = 0 + deployed_successfully = self._start_node(sb, node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed, federation) + if deployed_successfully: + federation.last_index_deployed += 1 + federation.participants_alive += 1 - def _start_node(self, node, network_name, base_network_name, base, i, additional=False): + def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name, base, i, federation: NebulaFederationProcesses, additional=False): self.processes_root_path = os.path.join(os.path.dirname(__file__), "..", "..") - self.logger.info(f"env path: {self.env_path}") # Include additional config to the participants for idx, node in enumerate(self.config.participants): node["tracking_args"]["log_dir"] = os.path.join(self.root_path, "app", "logs") - node["tracking_args"]["config_dir"] = os.path.join(self.root_path, "app", "config", self.sb.get_scenario_name()) + node["tracking_args"]["config_dir"] = os.path.join(self.root_path, "app", "config", sb.get_scenario_name()) node["scenario_args"]["controller"] = self.url - node["scenario_args"]["deployment"] = self.sb.get_deployment() + node["scenario_args"]["deployment"] = sb.get_deployment() node["security_args"]["certfile"] = os.path.join( self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_cert.pem" ) @@ -399,11 +475,11 @@ def _start_node(self, node, network_name, base_network_name, base, i, additional commands += "Start-Sleep -Seconds 2\n" commands += f'Write-Host "Running node {node["device_args"]["idx"]}..."\n' - commands += f'$OUT_FILE = "{self.root_path}\\app\\logs\\{self.sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.out"\n' - commands += f'$ERROR_FILE = "{self.root_path}\\app\\logs\\{self.sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.err"\n' + commands += f'$OUT_FILE = "{self.root_path}\\app\\logs\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.out"\n' + commands += f'$ERROR_FILE = "{self.root_path}\\app\\logs\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.err"\n' # Use Start-Process for executing Python in background and capture PID - commands += f"""$process = Start-Process -FilePath "python" -ArgumentList "{self.root_path}\\nebula\\core\\node.py {self.root_path}\\app\\config\\{self.sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.json" -PassThru -NoNewWindow -RedirectStandardOutput $OUT_FILE -RedirectStandardError $ERROR_FILE + commands += f"""$process = Start-Process -FilePath "python" -ArgumentList "{self.root_path}\\nebula\\core\\node.py {self.root_path}\\app\\config\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.json" -PassThru -NoNewWindow -RedirectStandardOutput $OUT_FILE -RedirectStandardError $ERROR_FILE Add-Content -Path $PID_FILE -Value $process.Id """ @@ -425,8 +501,8 @@ def _start_node(self, node, network_name, base_network_name, base, i, additional else: commands += "sleep 2\n" commands += f'echo "Running node {node["device_args"]["idx"]}..."\n' - commands += f"OUT_FILE={self.root_path}/app/logs/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.out\n" - commands += f"python {self.root_path}/nebula/core/node.py {self.root_path}/app/config/{self.sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json &\n" + commands += f"OUT_FILE={self.root_path}/app/logs/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.out\n" + commands += f"python {self.root_path}/nebula/core/node.py {self.root_path}/app/config/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json &\n" commands += "echo $! >> $PID_FILE\n\n" commands += 'echo "All nodes started. PIDs stored in $PID_FILE"\n' diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index d828f6ab3..5fba08045 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -12,17 +12,11 @@ from nebula.controller.federation.factory_federation_controller import federation_controller_factory from nebula.controller.federation.utils_requests import InitFederationRequest, RunScenarioRequest, StopScenarioRequest -#TODO we need all 3 controllers instanciated. When /init received we put the request on the right -# controller - -def require_initialized_controller(func): - @wraps(func) - async def wrapper(*args, **kwargs): - if fed_controller is None: - raise HTTPException(status_code=400, detail="FederationController not initialized") - return await func(*args, **kwargs) - return wrapper - +#TODO Route the request to the right controller + +#fed_controller: FederationController = None +fed_controllers: Dict[str, FederationController] = {} + @asynccontextmanager async def lifespan(app: FastAPI): log_path = os.environ.get("NEBULA_FEDERATION_CONTROLLER_LOG") @@ -34,10 +28,19 @@ async def lifespan(app: FastAPI): logger = logging.getLogger("Federation-Controller") logger.info("Logger initialized for Federation Controller") + # Create all controller types + hub_port = os.environ.get("NEBULA_CONTROLLER_PORT") + controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") + hub_url = f"http://{controller_host}:{hub_port}" + + #["docker", "processes", "physical"] + for exp_type in ["docker"]: + fed_controllers[exp_type] = federation_controller_factory(exp_type, hub_url, logger) + logger.info(f"{exp_type} Federation controller created.") + yield app = FastAPI(lifespan=lifespan) -fed_controller: FederationController = None @app.get("/") async def read_root(): @@ -70,21 +73,23 @@ async def init_federation_experiment(ifr: InitFederationRequest): return {"message": f"{experiment_type} controller initialized"} -#ADVICE: return ID if sucess otherwise empty string @app.post("/scenarios/run") -@require_initialized_controller async def run_scenario(run_scenario_request: RunScenarioRequest): - global fed_controller - return await fed_controller.run_scenario(run_scenario_request.federation_id, run_scenario_request.scenario_data, run_scenario_request.user) - + global fed_controllers + experiment_type = run_scenario_request.scenario_data["deployment"] + controller = fed_controllers.get(experiment_type, None) + if controller: + return await controller.run_scenario(run_scenario_request.federation_id, run_scenario_request.scenario_data, run_scenario_request.user) + else: + return {"message": "Experyment type not allowed"} + +#TODO need to use fedID? @app.post("/scenarios/stop") -@require_initialized_controller async def stop_scenario(stop_scenario_request: StopScenarioRequest): global fed_controller return await fed_controller.stop_scenario(stop_scenario_request.federation_id) @app.post("/nodes/{scenario_name}/update") -@require_initialized_controller async def update_nodes( scenario_name: Annotated[ str, @@ -92,11 +97,16 @@ async def update_nodes( ], request: Request, ): - global fed_controller - return await fed_controller.update_nodes(scenario_name, request) + global fed_controllers + config = await request.json() + experiment_type = config["scenario_args"]["deployment"] + controller = fed_controllers.get(experiment_type, None) + if controller: + return await controller.update_nodes(scenario_name, request) + else: + return {"message": "Experyment type not allowed on response for update message.."} @app.post("/nodes/{scenario_name}/done") -@require_initialized_controller async def update_nodes( scenario_name: Annotated[ str, @@ -104,9 +114,14 @@ async def update_nodes( ], request: Request, ): - global fed_controller - return await fed_controller.node_done(scenario_name, request) - + global fed_controllers + config = await request.json() + experiment_type = config["deployment"] + controller = fed_controllers.get(experiment_type, None) + if controller: + return await controller.node_done(scenario_name, request) + else: + return {"message": "Experyment type not allowed on responde for Node done message.."} if __name__ == "__main__": # Parse args from command line @@ -115,4 +130,6 @@ async def update_nodes( args = parser.parse_args() import uvicorn - uvicorn.run(app, host="0.0.0.0", port=args.port) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=args.port) + + \ No newline at end of file diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index 688927468..2c6fb824d 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -31,4 +31,4 @@ async def update_nodes(self, scenario_name: str, request: Request): abstractmethod async def node_done(self, scenario_name: str, request: Request): - pass + pass \ No newline at end of file diff --git a/nebula/controller/federation/utils_requests.py b/nebula/controller/federation/utils_requests.py index 64e478003..4af0d6d07 100644 --- a/nebula/controller/federation/utils_requests.py +++ b/nebula/controller/federation/utils_requests.py @@ -12,7 +12,7 @@ class RunScenarioRequest(BaseModel): class StopScenarioRequest(BaseModel): federation_id: str -def factory_requests_path(resource: str, scenario_name: str = "") -> str: +def factory_requests_path(resource: str, scenario_name: str = "", federation_id: str = "") -> str: if resource == "init": return "/init" elif resource == "run": @@ -23,6 +23,8 @@ def factory_requests_path(resource: str, scenario_name: str = "") -> str: return f"/nodes/{scenario_name}/update" elif resource == "done": return f"/nodes/{scenario_name}/done" + elif resource == "finish": + return f"/scenarios/{federation_id}/finish" else: raise Exception(f"resource not found: {resource}") diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index b1ff9673b..5cfd8564d 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -324,9 +324,9 @@ async def run_scenario( fed_controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") url_init_fed_controller = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("init") url_run_scenario = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("run") - init_fed_req = InitFederationRequest(experiment_type="docker") + #init_fed_req = InitFederationRequest(experiment_type="docker") run_scenario_req = RunScenarioRequest(scenario_data=scenario_data, federation_id="id_nebula", user=user) #TODO ID per experiment - await APIUtils.post(url_init_fed_controller, init_fed_req.model_dump()) + #await APIUtils.post(url_init_fed_controller, init_fed_req.model_dump()) await APIUtils.post(url_run_scenario, run_scenario_req.model_dump()) except Exception as e: logging.info(e) @@ -665,8 +665,8 @@ async def update_nodes( str(config["data"]["network_args"]["port"]), str(config["data"]["device_args"]["role"]), config["data"]["network_args"]["neighbors"], - str(config["data"]["mobility_args"]["latitude"]), - str(config["data"]["mobility_args"]["longitude"]), + str(config["data"]["addons"]["mobility"]["latitude"]), + str(config["data"]["addons"]["mobility"]["longitude"]), str(timestamp), str(config["data"]["data"]["scenario_args"]["federation"]), str(config["data"]["federation_args"]["round"]), From 2f1dcdd31e103855f065f93e06008d176c1c506c Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Mon, 8 Sep 2025 12:48:27 +0200 Subject: [PATCH 19/26] fix remove docker containers --- .../docker_federation_controller.py | 58 +++++++++++++++---- .../processes_federation_controller.py | 26 ++++++++- .../factory_federation_controller.py | 2 +- .../controller/federation/federation_api.py | 43 ++++++++------ .../federation/federation_controller.py | 4 +- .../controller/federation/utils_requests.py | 1 + 6 files changed, 101 insertions(+), 33 deletions(-) diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index c4a35d99e..5d7e062b0 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -16,6 +16,7 @@ class NebulaFederationDocker(): def __init__(self): + self.scenario_name = "" self.participants_alive = 0 self.round_per_participant = {} self.additionals_participants = {} @@ -72,7 +73,7 @@ def __init__(self, hub_url, logger): self.cert_dir = "" self.advanced_analytics = "" self.url = "" - self._nebula_federations_pool: dict[tuple[str,str], NebulaFederationDocker] = {} + self._nebula_federations_pool: dict[str, NebulaFederationDocker] = {} self._federations_dict_lock = Locker("federations_dict_lock", async_lock=True) @property @@ -97,14 +98,26 @@ async def run_scenario(self, federation_id: str, scenario_data: Dict, user: str) await self._load_configuration_and_start_nodes(scenario_builder, federation) self._start_initial_nodes(scenario_builder, federation) id = scenario_builder.get_scenario_name() + try: + nebula_federation = self.nfp[federation_id] + nebula_federation.scenario_name = id + except Exception as e: + self.logger.info(f"ERROR: federation ID: ({federation_id}) not found on pool..") + return None + else: + self.logger.info(f"ERROR: federation ID: ({federation_id}) already exists..") return id - async def stop_scenario(self, id: str): + async def stop_scenario(self, federation_id: str): """ Remove all participant containers and the scenario network. Reads ALL scenario.metadata and removes all listed containers and the network, then deletes the metadata file. Also forcibly stops and removes any containers still attached to the network before removing it. """ + federation_scenario_name = await self._remove_nebula_federation_from_pool(federation_id) + if not federation_scenario_name: + return False + # Try multiple possible config directory locations. This depends on where the user called the function from. possible_config_dirs = [ os.environ.get("NEBULA_CONFIG_DIR"), @@ -142,6 +155,9 @@ async def stop_scenario(self, id: str): client = docker.from_env() for scenario_dir in scenario_dirs: + if scenario_dir != federation_scenario_name: + continue + metadata_path = os.path.join(scenario_dir, "scenario.metadata") if not os.path.exists(metadata_path): self.logger.info(f"Skipping {scenario_dir} - no scenario.metadata found") @@ -182,6 +198,11 @@ async def stop_scenario(self, id: str): os.remove(metadata_path) except Exception as e: self.logger.warning(f"Could not remove scenario.metadata: {e}") + + if scenario_dir == federation_scenario_name: + break + + return True #TODO care about cases async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() @@ -205,7 +226,7 @@ async def update_nodes(self, scenario_name: str, request: Request): if index == idx: if index in additionals: self.logger.info(f"Deploying additional participant: {index}") - self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation, additional=True) + self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation) nebula_federation.last_index_deployed += 1 additionals.remove(index) adds_deployed.add(index) @@ -221,9 +242,12 @@ async def node_done(self, scenario_name: str, request: Request): request_body = await request.json() federation_id = request_body["federation_id"] nebula_federation = self.nfp[federation_id] + self.logger.info(f"Node-Done received from node on federation ID: ({federation_id})") if await nebula_federation.is_experiment_finish(): payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} + self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") + await self._remove_nebula_federation_from_pool(federation_id) asyncio.create_task(self._send_to_hub("finish", payload, scenario_name)) payload = {"scenario_name": scenario_name, "data": request_body} @@ -245,6 +269,16 @@ async def _add_nebula_federation_to_pool(self, federation_id: str, user: str): else: self.logger.info(f"ERROR: trying to add ({federation_id}) to federations pool..") return fed + + async def _remove_nebula_federation_from_pool(self, federation_id: str): + async with self._federations_dict_lock: + if federation_id in self.nfp: + federation = self.nfp.pop(federation_id) + self.logger.info(f"SUCCESS: Federation ID: ({federation_id}) removed from pool") + return federation.scenario_name + else: + self.logger.info(f"ERROR: trying to remove ({federation_id}) from federations pool..") + return "" async def _update_federation_on_pool(self, federation_id: str, user: str, nf: NebulaFederationDocker): updated = False @@ -494,7 +528,7 @@ def _start_initial_nodes(self, sb: ScenarioBuilder, federation: NebulaFederation federation.last_index_deployed += 1 federation.participants_alive += 1 - def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name, base, i, federation: NebulaFederationDocker, additional=False): + def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name, base, i, federation: NebulaFederationDocker): success = True client = docker.from_env() @@ -572,17 +606,21 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name client.api.start(container_id) container_ids.append(container_id) container_names.append(name) + self.logger.info(f"Adding name: {name} for metadata") except Exception as e: success = False self.logger.exception(f"Starting participant {name} error: {e}") # Write scenario-level metadata for cleanup scenario_metadata = {"containers": container_names, "network": network_name} - if not additional: - with open(os.path.join(self.config_dir, "scenario.metadata"), "w") as f: - json.dump(scenario_metadata, f, indent=2) - else: - with open(os.path.join(self.config_dir, "scenario.metadata"), "a") as f: + with open(os.path.join(self.config_dir, "scenario.metadata"), "a") as f: + if i == 2: json.dump(scenario_metadata, f, indent=2) - + else: + with open(os.path.join(self.config_dir, "scenario.metadata"), "r") as f: + metadata = json.load(f) + metadata["containers"].extend(container_names) + with open(os.path.join(self.config_dir, "scenario.metadata"), "w") as f: + json.dump(metadata, f, indent=2) + return success \ No newline at end of file diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index 853d8e6e5..ebb71ee3d 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -103,6 +103,8 @@ async def run_scenario(self, federation_id: str, scenario_data: Dict, user: str) except Exception as e: self.logger.info(f"ERROR: federation ID: ({federation_id}) not found on pool..") return None + else: + self.logger.info(f"ERROR: federation ID: ({federation_id}) already exists..") return id async def stop_scenario(self, federation_id: str = ""): @@ -123,6 +125,10 @@ async def stop_scenario(self, federation_id: str = ""): - Supports both Linux/macOS ('.sh') and Windows ('.ps1') script files. - Any errors during file removal are logged with the traceback. """ + federation_name = await self._remove_nebula_federation_from_pool(federation_id) + if not federation_name: + return False + # When stopping the nodes, we need to remove the current_scenario_commands.sh file -> it will cause the nodes to stop using PIDs try: nebula_config_dir = os.environ.get("NEBULA_CONFIG_DIR") @@ -131,15 +137,14 @@ async def stop_scenario(self, federation_id: str = ""): nebula_base_dir = os.path.abspath(os.path.join(current_dir, "..", "..")) nebula_config_dir = os.path.join(nebula_base_dir, "app", "config") self.logger.info(f"NEBULA_CONFIG_DIR not found. Using default path: {nebula_config_dir}") - nebula_federation = self.nfp[federation_id] if id: if os.environ.get("NEBULA_HOST_PLATFORM") == "windows": scenario_commands_file = os.path.join( - nebula_config_dir, nebula_federation.scenario_name, "current_scenario_commands.ps1" + nebula_config_dir, federation_name, "current_scenario_commands.ps1" ) else: scenario_commands_file = os.path.join( - nebula_config_dir, nebula_federation.scenario_name, "current_scenario_commands.sh" + nebula_config_dir, federation_name, "current_scenario_commands.sh" ) if os.path.exists(scenario_commands_file): os.remove(scenario_commands_file) @@ -154,8 +159,10 @@ async def stop_scenario(self, federation_id: str = ""): ) for file in files: os.remove(file) + return True except Exception as e: self.logger.exception(f"Error while removing current_scenario_commands.sh file: {e}") + return False async def update_nodes(self, scenario_name: str, request: Request): config = await request.json() @@ -195,9 +202,12 @@ async def node_done(self, scenario_name: str, request: Request): request_body = await request.json() federation_id = request_body["federation_id"] nebula_federation = self.nfp[federation_id] + self.logger.info(f"Node-Done received from node on federation ID: ({federation_id})") if await nebula_federation.is_experiment_finish(): payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} + self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") + await self._remove_nebula_federation_from_pool(federation_id) asyncio.create_task(self._send_to_hub("finish", payload, scenario_name)) payload = {"scenario_name": scenario_name, "data": request_body} @@ -220,6 +230,16 @@ async def _add_nebula_federation_to_pool(self, federation_id: str, user: str): self.logger.info(f"ERROR: trying to add ({federation_id}) to federations pool..") return fed + async def _remove_nebula_federation_from_pool(self, federation_id: str): + async with self._federations_dict_lock: + if federation_id in self.nfp: + federation = self.nfp.pop(federation_id) + self.logger.info(f"SUCCESS: Federation ID: ({federation_id}) removed from pool") + return federation.scenario_name + else: + self.logger.info(f"ERROR: trying to remove ({federation_id}) from federations pool..") + return "" + async def _send_to_hub(self, path, payload, scenario_name="", federation_id="" ): try: url_request = self._hub_url + factory_requests_path(path, scenario_name, federation_id) diff --git a/nebula/controller/federation/factory_federation_controller.py b/nebula/controller/federation/factory_federation_controller.py index 11882c4db..1cdfc86b2 100644 --- a/nebula/controller/federation/factory_federation_controller.py +++ b/nebula/controller/federation/factory_federation_controller.py @@ -9,7 +9,7 @@ def federation_controller_factory(mode: str, wa_controller_url: str, logger) -> return DockerFederationController(wa_controller_url, logger) elif mode == "physical": return PhysicalFederationController(wa_controller_url, logger) - elif mode == "processes": + elif mode == "process": return ProcessesFederationController(wa_controller_url, logger) else: raise ValueError("Unknown federation mode") \ No newline at end of file diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index 5fba08045..6053c1904 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -34,7 +34,7 @@ async def lifespan(app: FastAPI): hub_url = f"http://{controller_host}:{hub_port}" #["docker", "processes", "physical"] - for exp_type in ["docker"]: + for exp_type in ["docker", "process"]: fed_controllers[exp_type] = federation_controller_factory(exp_type, hub_url, logger) logger.info(f"{exp_type} Federation controller created.") @@ -54,29 +54,31 @@ async def read_root(): logger.info("Test curl succesfull") return {"message": "Welcome to the NEBULA Federation Controller API"} -@app.post("/init") -async def init_federation_experiment(ifr: InitFederationRequest): - global fed_controller +# @app.post("/init") +# async def init_federation_experiment(ifr: InitFederationRequest): +# global fed_controller - experiment_type = ifr.experiment_type - logger = logging.getLogger("Federation-Controller") - logger.info(f"Experiment type received: {experiment_type}") +# experiment_type = ifr.experiment_type +# logger = logging.getLogger("Federation-Controller") +# logger.info(f"Experiment type received: {experiment_type}") - #TODO Modify when deploying controllers on differents systems - hub_port = os.environ.get("NEBULA_CONTROLLER_PORT") - controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") +# #TODO Modify when deploying controllers on differents systems +# hub_port = os.environ.get("NEBULA_CONTROLLER_PORT") +# controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") - hub_url = f"http://{controller_host}:{hub_port}" - logger.info(f"Docker Hub URL => {hub_url}") - fed_controller = federation_controller_factory(str(experiment_type), hub_url, logger) - logger.info("Federation controller created.") +# hub_url = f"http://{controller_host}:{hub_port}" +# logger.info(f"Docker Hub URL => {hub_url}") +# fed_controller = federation_controller_factory(str(experiment_type), hub_url, logger) +# logger.info("Federation controller created.") - return {"message": f"{experiment_type} controller initialized"} +# return {"message": f"{experiment_type} controller initialized"} @app.post("/scenarios/run") async def run_scenario(run_scenario_request: RunScenarioRequest): global fed_controllers experiment_type = run_scenario_request.scenario_data["deployment"] + logger = logging.getLogger("Federation-Controller") + logger.info(f"[API]: run experiment request for deployment type: {experiment_type}") controller = fed_controllers.get(experiment_type, None) if controller: return await controller.run_scenario(run_scenario_request.federation_id, run_scenario_request.scenario_data, run_scenario_request.user) @@ -86,8 +88,15 @@ async def run_scenario(run_scenario_request: RunScenarioRequest): #TODO need to use fedID? @app.post("/scenarios/stop") async def stop_scenario(stop_scenario_request: StopScenarioRequest): - global fed_controller - return await fed_controller.stop_scenario(stop_scenario_request.federation_id) + global fed_controllers + experiment_type = stop_scenario_request.experiment_type + controller = fed_controllers.get(experiment_type, None) + logger = logging.getLogger("Federation-Controller") + logger.info(f"[API]: stop experiment request for federation ID: {stop_scenario_request.federation_id}") + if controller: + return await controller.stop_scenario(stop_scenario_request.federation_id) + else: + return {"message": "Experyment type not allowed"} @app.post("/nodes/{scenario_name}/update") async def update_nodes( diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index 2c6fb824d..e75d4575c 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -18,11 +18,11 @@ def logger(self): return self._logger @abstractmethod - async def run_scenario(self, id: str, scenario_data: Dict, user: str): + async def run_scenario(self, federation_id: str, scenario_data: Dict, user: str): pass @abstractmethod - async def stop_scenario(self, id: str): + async def stop_scenario(self, federation_id: str): pass @abstractmethod diff --git a/nebula/controller/federation/utils_requests.py b/nebula/controller/federation/utils_requests.py index 4af0d6d07..259814986 100644 --- a/nebula/controller/federation/utils_requests.py +++ b/nebula/controller/federation/utils_requests.py @@ -10,6 +10,7 @@ class RunScenarioRequest(BaseModel): federation_id: str class StopScenarioRequest(BaseModel): + experiment_type: str federation_id: str def factory_requests_path(resource: str, scenario_name: str = "", federation_id: str = "") -> str: From 7cfd6b86e6275869c758e3e273aa02ab2fdfea3f Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Tue, 9 Sep 2025 11:28:53 +0200 Subject: [PATCH 20/26] feature process deployment --- .../docker_federation_controller.py | 31 ++-- .../processes_federation_controller.py | 166 +++++++++--------- 2 files changed, 102 insertions(+), 95 deletions(-) diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index 5d7e062b0..ef8f80979 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -59,13 +59,10 @@ async def is_experiment_finish(self): else: return False - - class DockerFederationController(FederationController): def __init__(self, hub_url, logger): super().__init__(hub_url, logger) - self._user = "" self.root_path = "" self.host_platform = "" self.config_dir = "" @@ -88,7 +85,6 @@ def nfp(self): async def run_scenario(self, federation_id: str, scenario_data: Dict, user: str): #TODO maintain files on memory, not read them again - self._user = user federation = await self._add_nebula_federation_to_pool(federation_id, user) id = "" if federation: @@ -134,7 +130,7 @@ async def stop_scenario(self, federation_id: str): break if not config_dir: - self.logger.warning("No valid config directory found, skipping cleanup") + self.logger.info("No valid config directory found, skipping cleanup") return scenario_dirs = [] @@ -173,7 +169,7 @@ async def stop_scenario(self, federation_id: str): container.remove(force=True) self.logger.info(f"Removed scenario container {name}") except Exception as e: - self.logger.warning(f"Could not remove scenario container {name}: {e}") + self.logger.info(f"Could not remove scenario container {name}: {e}") # Remove network, but first forcibly remove any containers still attached network_name = meta.get("network") @@ -187,17 +183,17 @@ async def stop_scenario(self, federation_id: str): c.remove(force=True) self.logger.info(f"Force-removed container {c.name} attached to {network_name}") except Exception as e: - self.logger.warning(f"Could not force-remove container {container_id}: {e}") + self.logger.info(f"Could not force-remove container {container_id}: {e}") network.remove() self.logger.info(f"Removed scenario network {network_name}") except Exception as e: - self.logger.warning(f"Could not remove scenario network {network_name}: {e}") + self.logger.info(f"Could not remove scenario network {network_name}: {e}") # Remove metadata file try: os.remove(metadata_path) except Exception as e: - self.logger.warning(f"Could not remove scenario.metadata: {e}") + self.logger.info(f"Could not remove scenario.metadata: {e}") if scenario_dir == federation_scenario_name: break @@ -226,17 +222,20 @@ async def update_nodes(self, scenario_name: str, request: Request): if index == idx: if index in additionals: self.logger.info(f"Deploying additional participant: {index}") - self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation) + deployed_successfully = self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation) + if deployed_successfully: + self.logger.info(f"Deployment successfully for additional participant: {index}") + nebula_federation.participants_alive += 1 nebula_federation.last_index_deployed += 1 - additionals.remove(index) + #additionals.remove(index) adds_deployed.add(index) request_body = await request.json() payload = {"scenario_name": scenario_name, "data": request_body} asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) return {"message": "Node updated successfully in Federation Controller"} except Exception as e: - self.logger.info(f"ERROR: federation ID: ({fed_id}) not found on pool..") - return {"message": "Node updated failed in Federation Controller, ID not found.."} + self.logger.info(f"ERROR: federation ID: ({fed_id}), {e}") + return {"message": "Node updated failed in Federation Controller"} async def node_done(self, scenario_name: str, request: Request): request_body = await request.json() @@ -581,7 +580,7 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name node = json.loads(json.dumps(node).replace("192.168.50.", f"{base}.")) # TODO change this try: existing = client.containers.get(name) - self.logger.warning(f"Container {name} already exists. Deployment may fail or cause conflicts.") + self.logger.info(f"Container {name} already exists. Deployment may fail or cause conflicts.") success = False except docker.errors.NotFound: pass # No conflict, safe to proceed @@ -601,7 +600,7 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name ) except Exception as e: success = False - self.logger.exception(f"Creating container {name}: {e}") + self.logger.info(f"Creating container {name}: {e}") try: client.api.start(container_id) container_ids.append(container_id) @@ -609,7 +608,7 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name self.logger.info(f"Adding name: {name} for metadata") except Exception as e: success = False - self.logger.exception(f"Starting participant {name} error: {e}") + self.logger.info(f"Starting participant {name} error: {e}") # Write scenario-level metadata for cleanup scenario_metadata = {"containers": container_names, "network": network_name} diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index ebb71ee3d..2a45f5f1c 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -14,7 +14,6 @@ from nebula.core.utils.certificate import generate_ca_certificate from nebula.core.utils.locker import Locker -#TODO save participants when deployed class NebulaFederationProcesses(): def __init__(self): self.scenario_name = "" @@ -63,7 +62,6 @@ async def is_experiment_finish(self): class ProcessesFederationController(FederationController): def __init__(self, hub_url, logger): super().__init__(hub_url, logger) - self._user = "" self.root_path = "" self.host_platform = "" self.config_dir = "" @@ -87,7 +85,6 @@ def nfp(self): async def run_scenario(self, federation_id: str, scenario_data: Dict, user: str): #TODO maintain files on memory, not read them again - self._user = user federation = await self._add_nebula_federation_to_pool(federation_id, user) id = "" if federation: @@ -186,6 +183,7 @@ async def update_nodes(self, scenario_name: str, request: Request): if index == idx: if index in additionals: self.logger.info(f"Deploying additional participant: {index}") + #TODO additionals not working self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation, additional=True) nebula_federation.last_index_deployed += 1 additionals.remove(index) @@ -261,7 +259,7 @@ async def _initialize_scenario(self, sb: ScenarioBuilder, scenario_data, federat self.log_dir = os.environ.get("NEBULA_LOGS_DIR") self.cert_dir = os.environ.get("NEBULA_CERTS_DIR") self.advanced_analytics = os.environ.get("NEBULA_ADVANCED_ANALYTICS", "False") == "True" - self.config = Config(entity="scenarioManagement") + # self.config = Config(entity="scenarioManagement") self.env_tag = os.environ.get("NEBULA_ENV_TAG", "dev") self.prefix_tag = os.environ.get("NEBULA_PREFIX_TAG", "dev") self.user_tag = os.environ.get("NEBULA_USER_TAG", os.environ.get("USER", "unknown")) @@ -340,11 +338,11 @@ async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federat if len(participant_files) == 0: raise ValueError("No participant files found in config folder") - self.config.set_participants_config(participant_files) - self.n_nodes = len(participant_files) - self.logger.info(f"Number of nodes: {self.n_nodes}") + federation.config.set_participants_config(participant_files) + n_nodes = len(participant_files) + self.logger.info(f"Number of nodes: {n_nodes}") - sb.create_topology_manager(self.config) + sb.create_topology_manager(federation.config) # Update participants configuration is_start_node = False @@ -352,13 +350,12 @@ async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federat additional_participants = sb.get_additional_nodes() additional_nodes = len(additional_participants) if additional_participants else 0 - self.logger.info(f"######## nodes: {self.n_nodes} + additionals: {additional_nodes} ######") participant_files.sort(key=lambda x: int(x.split("_")[-1].split(".")[0])) # Initial participants self.logger.info("🔧 Building preload configuration for initial nodes...") - for i in range(self.n_nodes): + for i in range(n_nodes): try: with open(f"{self.config_dir}/participant_" + str(i) + ".json") as f: participant_config = json.load(f) @@ -390,7 +387,7 @@ async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federat self.logger.info("✅ Building preload configuration for initial nodes done") - self.config.set_participants_config(participant_files) + federation.config.set_participants_config(participant_files) # Add role to the topology (visualization purposes) sb.visualize_topology(config_participants, path=f"{self.config_dir}/topology.png", plot=False) @@ -409,7 +406,7 @@ async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federat with open(additional_participant_file) as f: participant_config = json.load(f) - self.logger.info(f"Configuration | additional nodes | participant: {self.n_nodes + i}") + self.logger.info(f"Configuration | additional nodes | participant: {n_nodes + i}") sb.build_preload_additional_node_configuration(last_participant_index, i, participant_config) with open(additional_participant_file, "w") as f: @@ -418,10 +415,10 @@ async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federat additional_participants_files.append(additional_participant_file) if additional_participants_files: - self.config.add_participants_config(additional_participants_files) + federation.config.add_participants_config(additional_participants_files) if additional_participants: - self.n_nodes += len(additional_participants) + n_nodes += len(additional_participants) self.logger.info("✅ Building preload configuration for additional nodes done") self.logger.info("✅ Loading Scenario configuration done") @@ -434,11 +431,17 @@ async def _load_configuration_and_start_nodes(self, sb: ScenarioBuilder, federat def _start_initial_nodes(self, sb: ScenarioBuilder, federation: NebulaFederationProcesses): self.logger.info("Starting nodes as processes...") - + self.logger.info(f"Number of participants: {len(federation.config.participants)}") federation.config.participants.sort(key=lambda x: x["device_args"]["idx"]) federation.last_index_deployed = 2 - for idx, node in enumerate(federation.config.participants): - + + commands = "" + commands = self._build_initial_commands() + if not commands: + self.logger.info("ERROR: Cannot create commands file, abort..") + return + + for idx, node in enumerate(federation.config.participants): if node["deployment_args"]["additional"]: federation.additionals_participants[idx] = int(node["deployment_args"]["deployment_round"]) federation.participants_alive += 1 @@ -446,35 +449,68 @@ def _start_initial_nodes(self, sb: ScenarioBuilder, federation: NebulaFederation else: # deploy initial nodes self.logger.info(f"Deployment starting for participant {idx}") - self.logger.info(f"Deployment starting for participant {idx}") federation.round_per_participant[idx] = 0 - deployed_successfully = self._start_node(sb, node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed, federation) - if deployed_successfully: + node_command = self._start_node(sb, node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed, federation) + commands += node_command + if node_command: federation.last_index_deployed += 1 federation.participants_alive += 1 - + + if federation.config.participants and commands: + self._write_commands_on_file(commands) + else: + self.logger.info("ERROR: No commands on a proccesses deployment..") + def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name, base, i, federation: NebulaFederationProcesses, additional=False): self.processes_root_path = os.path.join(os.path.dirname(__file__), "..", "..") - - + node_idx = node['device_args']['idx'] # Include additional config to the participants - for idx, node in enumerate(self.config.participants): - node["tracking_args"]["log_dir"] = os.path.join(self.root_path, "app", "logs") - node["tracking_args"]["config_dir"] = os.path.join(self.root_path, "app", "config", sb.get_scenario_name()) - node["scenario_args"]["controller"] = self.url - node["scenario_args"]["deployment"] = sb.get_deployment() - node["security_args"]["certfile"] = os.path.join( - self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_cert.pem" - ) - node["security_args"]["keyfile"] = os.path.join( - self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_key.pem" - ) - node["security_args"]["cafile"] = os.path.join(self.root_path, "app", "certs", "ca_cert.pem") - - # Write the config file in config directory - with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f: - json.dump(node, f, indent=4) - + node["tracking_args"]["log_dir"] = os.path.join(self.root_path, "app", "logs") + node["tracking_args"]["config_dir"] = os.path.join(self.root_path, "app", "config", sb.get_scenario_name()) + node["scenario_args"]["controller"] = self.url + node["scenario_args"]["deployment"] = sb.get_deployment() + node["security_args"]["certfile"] = os.path.join( + self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_cert.pem" + ) + node["security_args"]["keyfile"] = os.path.join( + self.root_path, "app", "certs", f"participant_{node['device_args']['idx']}_key.pem" + ) + node["security_args"]["cafile"] = os.path.join(self.root_path, "app", "certs", "ca_cert.pem") + # Write the config file in config directory + with open(f"{self.config_dir}/participant_{node['device_args']['idx']}.json", "w") as f: + json.dump(node, f, indent=4) + + self.logger.info(f"Configuration file created successfully: {node_idx}") + commands = "" + try: + if self.host_platform == "windows": + if node["device_args"]["start"]: + commands += "Start-Sleep -Seconds 10\n" + else: + commands += "Start-Sleep -Seconds 2\n" + commands += f'Write-Host "Running node {node["device_args"]["idx"]}..."\n' + commands += f'$OUT_FILE = "{self.root_path}\\app\\logs\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.out"\n' + commands += f'$ERROR_FILE = "{self.root_path}\\app\\logs\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.err"\n' + # Use Start-Process for executing Python in background and capture PID + commands += f"""$process = Start-Process -FilePath "python" -ArgumentList "{self.root_path}\\nebula\\core\\node.py {self.root_path}\\app\\config\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.json" -PassThru -NoNewWindow -RedirectStandardOutput $OUT_FILE -RedirectStandardError $ERROR_FILE + Add-Content -Path $PID_FILE -Value $process.Id + """ + else: + if node["device_args"]["start"]: + commands += "sleep 10\n" + else: + commands += "sleep 2\n" + commands += f'echo "Running node {node["device_args"]["idx"]}..."\n' + commands += f"OUT_FILE={self.root_path}/app/logs/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.out\n" + commands += f"python {self.root_path}/nebula/core/node.py {self.root_path}/app/config/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json &\n" + commands += "echo $! >> $PID_FILE\n\n" + except Exception as e: + raise Exception(f"Error starting nodes as processes: {e}") + + return commands + + def _build_initial_commands(self): + commands = "" try: if self.host_platform == "windows": commands = """ @@ -483,53 +519,25 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name New-Item -Path $PID_FILE -Force -ItemType File """ - sorted_participants = sorted( - self.config.participants, - key=lambda node: node["device_args"]["idx"], - reverse=True, - ) - for node in sorted_participants: - if node["device_args"]["start"]: - commands += "Start-Sleep -Seconds 10\n" - else: - commands += "Start-Sleep -Seconds 2\n" - - commands += f'Write-Host "Running node {node["device_args"]["idx"]}..."\n' - commands += f'$OUT_FILE = "{self.root_path}\\app\\logs\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.out"\n' - commands += f'$ERROR_FILE = "{self.root_path}\\app\\logs\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.err"\n' - - # Use Start-Process for executing Python in background and capture PID - commands += f"""$process = Start-Process -FilePath "python" -ArgumentList "{self.root_path}\\nebula\\core\\node.py {self.root_path}\\app\\config\\{sb.get_scenario_name()}\\participant_{node["device_args"]["idx"]}.json" -PassThru -NoNewWindow -RedirectStandardOutput $OUT_FILE -RedirectStandardError $ERROR_FILE - Add-Content -Path $PID_FILE -Value $process.Id - """ - + else: + commands = '#!/bin/bash\n\nPID_FILE="$(dirname "$0")/current_scenario_pids.txt"\n\n> $PID_FILE\n\n' + except Exception as e: + raise Exception(f"Error starting nodes as processes: {e}") + return commands + + def _write_commands_on_file(self, commands: str): + try: + if self.host_platform == "windows": commands += 'Write-Host "All nodes started. PIDs stored in $PID_FILE"\n' - with open(f"{self.config_dir}/current_scenario_commands.ps1", "w") as f: + #self.logger.info(f"Process commands: {commands}") f.write(commands) os.chmod(f"{self.config_dir}/current_scenario_commands.ps1", 0o755) else: - commands = '#!/bin/bash\n\nPID_FILE="$(dirname "$0")/current_scenario_pids.txt"\n\n> $PID_FILE\n\n' - sorted_participants = sorted( - self.config.participants, - key=lambda node: node["device_args"]["idx"], - reverse=True, - ) - for node in sorted_participants: - if node["device_args"]["start"]: - commands += "sleep 10\n" - else: - commands += "sleep 2\n" - commands += f'echo "Running node {node["device_args"]["idx"]}..."\n' - commands += f"OUT_FILE={self.root_path}/app/logs/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.out\n" - commands += f"python {self.root_path}/nebula/core/node.py {self.root_path}/app/config/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json &\n" - commands += "echo $! >> $PID_FILE\n\n" - commands += 'echo "All nodes started. PIDs stored in $PID_FILE"\n' - with open(f"{self.config_dir}/current_scenario_commands.sh", "w") as f: + #self.logger.info(f"Process commands: {commands}") f.write(commands) os.chmod(f"{self.config_dir}/current_scenario_commands.sh", 0o755) - except Exception as e: raise Exception(f"Error starting nodes as processes: {e}") \ No newline at end of file From cdbd28ba5d92c0e6cefe3430f32d70754585063d Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Tue, 9 Sep 2025 12:38:54 +0200 Subject: [PATCH 21/26] fix minor details --- .../docker_federation_controller.py | 15 +++++++------ .../processes_federation_controller.py | 2 +- .../controller/federation/federation_api.py | 21 +------------------ .../controller/federation/utils_requests.py | 3 --- nebula/controller/web_app_controller.py | 8 ++++--- 5 files changed, 14 insertions(+), 35 deletions(-) diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index ef8f80979..ff7814909 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -222,10 +222,9 @@ async def update_nodes(self, scenario_name: str, request: Request): if index == idx: if index in additionals: self.logger.info(f"Deploying additional participant: {index}") - deployed_successfully = self._start_node(node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation) + deployed_successfully = self._start_node(nebula_federation.scenario_name, node, nebula_federation.network_name, nebula_federation.base_network_name, nebula_federation.base, nebula_federation.last_index_deployed, nebula_federation) if deployed_successfully: self.logger.info(f"Deployment successfully for additional participant: {index}") - nebula_federation.participants_alive += 1 nebula_federation.last_index_deployed += 1 #additionals.remove(index) adds_deployed.add(index) @@ -522,12 +521,12 @@ def _start_initial_nodes(self, sb: ScenarioBuilder, federation: NebulaFederation # deploy initial nodes self.logger.info(f"Deployment starting for participant {idx}") federation.round_per_participant[idx] = 0 - deployed_successfully = self._start_node(sb, node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed, federation) + deployed_successfully = self._start_node(sb.get_scenario_name(), node, federation.network_name, federation.base_network_name, federation.base, federation.last_index_deployed, federation) if deployed_successfully: federation.last_index_deployed += 1 federation.participants_alive += 1 - def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name, base, i, federation: NebulaFederationDocker): + def _start_node(self, scenario_name, node, network_name, base_network_name, base, i, federation: NebulaFederationDocker): success = True client = docker.from_env() @@ -536,7 +535,7 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name container_names = [] # Track names for metadata image = "nebula-core" - name = self._get_participant_container_name(sb.get_scenario_name(), node["device_args"]["idx"]) + name = self._get_participant_container_name(scenario_name, node["device_args"]["idx"]) if node["device_args"]["accelerator"] == "gpu": environment = { "NVIDIA_DISABLE_REQUIRE": True, @@ -562,7 +561,7 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name command = [ "/bin/bash", "-c", - f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{sb.get_scenario_name()}/participant_{node['device_args']['idx']}.json", + f"{start_command} && ifconfig && echo '{base}.1 host.docker.internal' >> /etc/hosts && python /nebula/nebula/core/node.py /nebula/app/config/{scenario_name}/participant_{node['device_args']['idx']}.json", ] networking_config = client.api.create_networking_config({ network_name: client.api.create_endpoint_config( @@ -571,9 +570,9 @@ def _start_node(self, sb: ScenarioBuilder, node, network_name, base_network_name base_network_name: client.api.create_endpoint_config(), }) node["tracking_args"]["log_dir"] = "/nebula/app/logs" - node["tracking_args"]["config_dir"] = f"/nebula/app/config/{sb.get_scenario_name()}" + node["tracking_args"]["config_dir"] = f"/nebula/app/config/{scenario_name}" node["scenario_args"]["controller"] = self.url - node["scenario_args"]["deployment"] = sb.get_deployment() + node["scenario_args"]["deployment"] = "docker" node["security_args"]["certfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_cert.pem" node["security_args"]["keyfile"] = f"/nebula/app/certs/participant_{node['device_args']['idx']}_key.pem" node["security_args"]["cafile"] = "/nebula/app/certs/ca_cert.pem" diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index 2a45f5f1c..a97f7c96e 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -134,7 +134,7 @@ async def stop_scenario(self, federation_id: str = ""): nebula_base_dir = os.path.abspath(os.path.join(current_dir, "..", "..")) nebula_config_dir = os.path.join(nebula_base_dir, "app", "config") self.logger.info(f"NEBULA_CONFIG_DIR not found. Using default path: {nebula_config_dir}") - if id: + if federation_id: if os.environ.get("NEBULA_HOST_PLATFORM") == "windows": scenario_commands_file = os.path.join( nebula_config_dir, federation_name, "current_scenario_commands.ps1" diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index 6053c1904..e439c3785 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -10,7 +10,7 @@ from nebula.utils import LoggerUtils from nebula.controller.federation.federation_controller import FederationController from nebula.controller.federation.factory_federation_controller import federation_controller_factory -from nebula.controller.federation.utils_requests import InitFederationRequest, RunScenarioRequest, StopScenarioRequest +from nebula.controller.federation.utils_requests import RunScenarioRequest, StopScenarioRequest #TODO Route the request to the right controller @@ -54,25 +54,6 @@ async def read_root(): logger.info("Test curl succesfull") return {"message": "Welcome to the NEBULA Federation Controller API"} -# @app.post("/init") -# async def init_federation_experiment(ifr: InitFederationRequest): -# global fed_controller - -# experiment_type = ifr.experiment_type -# logger = logging.getLogger("Federation-Controller") -# logger.info(f"Experiment type received: {experiment_type}") - -# #TODO Modify when deploying controllers on differents systems -# hub_port = os.environ.get("NEBULA_CONTROLLER_PORT") -# controller_host = os.environ.get("NEBULA_CONTROLLER_HOST") - -# hub_url = f"http://{controller_host}:{hub_port}" -# logger.info(f"Docker Hub URL => {hub_url}") -# fed_controller = federation_controller_factory(str(experiment_type), hub_url, logger) -# logger.info("Federation controller created.") - -# return {"message": f"{experiment_type} controller initialized"} - @app.post("/scenarios/run") async def run_scenario(run_scenario_request: RunScenarioRequest): global fed_controllers diff --git a/nebula/controller/federation/utils_requests.py b/nebula/controller/federation/utils_requests.py index 259814986..894acc492 100644 --- a/nebula/controller/federation/utils_requests.py +++ b/nebula/controller/federation/utils_requests.py @@ -1,9 +1,6 @@ from pydantic import BaseModel from typing import Dict, Any -class InitFederationRequest(BaseModel): - experiment_type: str - class RunScenarioRequest(BaseModel): scenario_data: Dict[str, Any] user: str diff --git a/nebula/controller/web_app_controller.py b/nebula/controller/web_app_controller.py index 5cfd8564d..7835f2a1a 100755 --- a/nebula/controller/web_app_controller.py +++ b/nebula/controller/web_app_controller.py @@ -25,7 +25,7 @@ ) from nebula.controller.http_helpers import remote_get, remote_post_form from nebula.utils import DockerUtils, APIUtils -from nebula.controller.federation.utils_requests import RunScenarioRequest, InitFederationRequest, StopScenarioRequest, factory_requests_path +from nebula.controller.federation.utils_requests import RunScenarioRequest, StopScenarioRequest, factory_requests_path # Setup controller logger @@ -107,6 +107,7 @@ def configure_logger(controller_log): handler.setFormatter(logging.Formatter("[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s")) logger.addHandler(handler) +id_counter = 1 @asynccontextmanager async def lifespan(app: FastAPI): @@ -317,7 +318,7 @@ async def run_scenario( """ import subprocess - + global id_counter from nebula.controller.scenarios import ScenarioManagement try: fed_controller_port = os.environ.get("NEBULA_FEDERATION_CONTROLLER_PORT") @@ -325,7 +326,8 @@ async def run_scenario( url_init_fed_controller = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("init") url_run_scenario = f"http://{fed_controller_host}:{fed_controller_port}" + factory_requests_path("run") #init_fed_req = InitFederationRequest(experiment_type="docker") - run_scenario_req = RunScenarioRequest(scenario_data=scenario_data, federation_id="id_nebula", user=user) #TODO ID per experiment + run_scenario_req = RunScenarioRequest(scenario_data=scenario_data, federation_id=f"id_nebula_{id_counter}", user=user) #TODO ID per experiment + id_counter += 1 #await APIUtils.post(url_init_fed_controller, init_fed_req.model_dump()) await APIUtils.post(url_run_scenario, run_scenario_req.model_dump()) except Exception as e: From f64bbb47e556e90694908bb1a33f43c725d054ea Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 11 Sep 2025 13:04:05 +0200 Subject: [PATCH 22/26] fix draw graph --- False.png | Bin 27701 -> 0 bytes .../controller/federation/scenario_builder.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 False.png diff --git a/False.png b/False.png deleted file mode 100644 index 7bfbf233a10b097dbcf91a63dd36c91cba616c38..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27701 zcmd?Ri9eNX_ddK4%_M{pwM)uO=0XEyu9V1JLYb!unX0=84U|lwGG$hhAtXd)DpSfl zhbSR4?{RkD-)H##e((DaywB%zKljky*S@avT<2QHajauqH;-x^Ub|}JDhh?NR#inw zn?j-0BY&-6z<(*S2y4aPWL=c?T#nmYxLi8zWKPjM?c!i#?_zWQ%$AGhPR{4;?RE>x z3X2PFIp^Zy;4CL1V*9TT2-`bZifq-Q`+|$CbWl0vOrbEHCV$a9RZKijp;$jrRoZ{T z_2Ed{CD#+??Mq`*ak}fbgltgpq0}9_=67|~({*>-ZNDYw`lk%%=Ubmz_|jDI>-&>O z3c`AKsau46&g@^O=SUy7Q^o$uU(x&A^Sr&c%SPU%(Cm;6e?M@+$l3gZXN`yDlm4E( z+L?C7;w@DCPpgOCp|hA~s3FPhk`Vb1!XonWg*#0fla3#zQCCt@DpTG2U3ee)hqj%j zH~swlWWs45;BS8S+`bt3;O{D$$B!P}XKZYIJ;Uwt<(|PoxAqaXOngDfpVLf5Rkf=j z>G;5A_Kh3WPMlz6^o=d~{Li0;{FUP8w}O_I_5=M+vo+!2;asv#%Y43l`?hK-)`u6j za^TYoQ`OvLlh=!~6*F^lbPm1mnDL+a9|||9j*@#djdEsNx6a|sGhVZr3(*h$+?UF( z0lAE!Gv)v5W_SJPy{qgh53A(<=SPnDUi7J|s#<+pi1YaK>Msq;$x|=;IWe(aLZZQn z>i+mp?BmCe2MopkUXM6EJcbY3^P2dxNe`VJDLaIOZpmF3lF3qF#{WVu$OgzKCMFKA zAZzsQ%U*f;+D%V{mH6@k@bCXxy8pTxzkC0G_}ARLX!*CV!gE-IR4SboprJ4bYMwBV`de<^s*HqUY@ zDH*D|yXW6Fz1gtgld5RoX$m8q(G{)Y;_j`+#iCnMrX7`%s<)}Pb;N|N@MY9FpGT#RD$%Hy z85RCoRWSd2Qj}e9wGC~_6|dq|TNb$@BkP!ck6v#t?CO2bzf0%uN<9M66GbIgj`-@Lo&QymoyxGjq`R@#6K4(lH*idoCI_>=4m*VYL=nr~aw& zFKe13o!ME-6~{H=OC$YMY~y`+yR`zZTx!qzRxNkbMd76VmQ6BRpC-E2cRt8kZvSF&8}7Z;yrZVYi2w-=bx#qu*i{K`103pb)!u7fZTF)+5)wollYHHV^)AJ+tF}vM1XW zJLEgmmCD}Ytmoc)v}MXe1Q)@vwzKT}5Hc~+68iI}0~h0RM&E0fIy`f#!U``>Iej*| z#LU8SEhdI%Y;3IH_g8Dy4IA{$PNwOeZ_nzKoj#k-=!Hw%yr#=GNuWyn!{Q z$h8`CzejH6@w8w{;wN1HWp(+$H!|GmoE9L zhi*LHu(mRPa%Z~i28{zrEuQh^4{2CT_SW!d1a!~&xb>$laEk5_`?Ra$_)Q#8=hC&( zdMz{-gLUyu!y$ zz2r&9apcI6Ebk>Rvp}&d%UUV?N!Ckjdeqp@pQ1jTqNc@e*6(U-rLlnG|ZZ5x!ZkuqP*00+C5+F z$`v+-)Fd|g`0kcXZ_T(&r8X!|>5VzJn`+mzTXZW83kl50Yeva6CcD)}%bOSZ>Y8{q zQY@x_cCoL~8>mk=+CvwXZuBampkVL4=xDnLDO)Q_{cCUUl%ilyO~FO3)$GDAq9gZQ z8dY9kR932a@Zf=kgz`?iM6ZR}>AAU>@yWV+AB4HN*9KPBOph5S$_>WNYCqZw)1F1i zU7*Uya8OK4I>wti2M1MMTqfwPqa>|2P@X+|*6KAkvW$VDF8_k{4$X+aaJ(p=K7Ep~ zZ|CwEh`xNgCTh=a*Po^Z1qB?^c3e5_?!(@fIJnm2bQpJCmRI}KU?S+iqkQ}-?}H6X zHZ(6piWK9fExCVc4sNDrc;I%YaweM6lwnf4yLRuvrNsqKMX!1K3SVDe7FJfJM6D>- zrG=@q4ofE|@knWV2Ge368do*CeZJlAq+PwbkT9JciA;5~jX zWMQ+v-wEVjtEeK+x%DSJu3kKFfOhkqi@jHtb7vYK(b7tHcy&Z^;XaBFa>!Ks<&5K2 zGb1@26um-sNrc;0aq(c=mX~6uOO~~JjB7sn#J zmqu}(EWbo4%!|6-U(jvgIk)03y30OGbm=tW{w9h?=XXD8kP3<){UK7A<=C@jd+)=T zLBo#YDk>|Gp~@;MdPYXp?z6Difq>c}C>Xfwc)~;H7=n+^eSCCoa4^PuF?{aF>zKr7 z_vCN8xnz!7Je-uJ1}LbWJh|cU9qu%ThrYhcaR`)SE_jC@5uiG$vS!9L-Mn?n{L6Fm zR)^BHuCA^Q-#%^~%^w!VgZfNXyncb-a6n0>`#C$DQJ;8(C$S;eq;=?#aiRP7*1UU2 zaGz3iMJVlFufDO4my{gfcI~8TN;9ZH#wbC~36sApg4*Ef_Wo`*(@Kfm ze(r_Mm*Bg1*I@~Cjoo;Ux@?cn%)Dz~G;`x^06U$-K-8u6931aHK0cUqJdyFJ#H3D5 z$0EbXSDKGEpCs+yneXo4l60E4!fGS?(q8SMY*LG5xPHD_&cVE0TnsF~waFlZN7gB>)zjPC8#!5k zOIJ_t-B@>dl4gWNo@pDE%KpTMH@hxbCuQf(os>@G@4buDJ(Rz$Uw5q6l%}~G8Clu< z%*<^tK6J&(m8aha^EQ17uWR?3v(?YGQcgLUCY*meI98`6$DU7KezVz{;ifdYEyBW# z$k<4Nsbh)@6Za$+CkE?ryFO2ns)ObFg-+3`KX8b1G;3d`73yAlDt=U2Xwgl5takIh z_u8MF1H9eoq-i?V^3=5Kxp;prHDn>WzdF*=*1&yZ6UV_rhZs0HInP>J8q``^THXr@ zp)bBlzgq9*_t)yjV&pD-kkjQy!9|_kC}pG4(k;b?1hrjJF$PgL-l%JGsv{e4F{jlj zAUh_{IY6PW`)xqll<-2n-@Cq#I)8Pa?5~Nk?(Xi^(9j6HV6Nxtnu`*?8z)7ooWuDo zn{L?0k4H&TWLdY))M&qqWeu0LqR8v{=h>d6H`Z3ZVBnB(cqEw~r%7XJ)U|<~-4d4y zjERY9JoC6W@7i*vjb?#g3NPn0I&Iy%x3#RiTz_tMy64Ul-Ic7jw=ZAD{iq9XC{hnA%oPK@yqN1;tT z-(HZna>WYK$>Gn^ZTS~cGHaU%`pLgAY%xDSuW|I~-NCk3IY5d$m4P_%lC=tRspr4G zNE-Y7TWak_2~n0E8lr=PgP{`uJG+F0gsNry=^k8A@b$}0EHv@1Zf!5*T;5Rlu@iXu z&GN_I-m5k<-eBUf(@c~+U-wtG_2>0$Y&wm+fX#6e6D|ecg%2ntR~$~!9hdIRD89d( z=d6pz-uYjv&z|+kk(RJoVU%CosLS3a(of?tKUVR*CFf3z*X+%Oxhdxs+ni@8n_#sz zvl3LQWF#RRwB;P#Jw0EZnGq<8Pt`v?78~%jqN~(bRYfIpR%PvGx%)s>r!@7ImEu&u zQqrlcNo&1k2l+03Z>HIYLN6j`xc}JUnYg&^DXPN{Wu0C7!z~}`C;IulI$FFwi9SIy z(xQ!Lw?!52k3`S8-{!aV3+I1TdQ5luh8jfs`L%pvee4%mSzDWUy7UHN*}#a}$B(aE zyS6aZQ|qBj&`Opa6y>NNp+W|S$dA3&V`i3qwCg~SerKe8(KERRd^>g+U(BU-#Rc8I zJ@%!%tE@Ep+4;t%qiEszlmN9`m^D+WwH145)yA{uVru>Gxh4bqQ0 zaWV&e)jIwc)lP8RwrlU+sfdeJTaK`IsfTLBo z%fv!oKfjbew{MW^uobCf-!TJv%Nqhd;M{lb-&Z<&b~{BtH@&xLZZvB{%?9SAG;%9? zgM)7>^W1-WoH=tw58w0h@}iJJQt<18fX**tDmDE8xtSqkN~Obx@5RI%i9K|y86h1a zMIW`>k`c)vi`)D67fW;Q0KbL}5mZ)3vl!3coHH{slt6Cz{11E5nXqS}CSG6MJt{U| z%Dg!8mEZFSb5MjD2%V;EO5V4(uD{;%dCdj`-DYH&BY(K^|w**TT!v zj*gC$7q%@Asw1U!TP_du_t&A;o=nv{UAQ#Vm@?VH!JH%sbk?1aa3sdA?UKchln>y& zU4YAp$^jcFSgh;J+Yaf~hft~6S3r{@#mevJxjQsGJ=SP^5EMeMz*TH=sFAJ{Xt*=z zF!!-J23(OWP!%Z77gXA#4bxew_jbH&0RCL9ur%*nP*_M4CUJf}3!m1D_zlcKLkXaK z7Rj+wI_AhH*Oum9FJ*Bn&VGv(G%jc^_g7L%_SJG?cicu#PcLED8dmDd1h}H9rsglH zX_<(ChzK=yU+MGc(H1Z)WguglbTy?ot8d#cF-vFRoG|hztl9V4Yr#qh(baXGJvAZL zvgYAdB%$QvN@bfNjN~%!aW~Pq0Sv?AIl;VgG|>6nM#wzGtN3xYE2>*V27HCyk1+ivdE#9m$ozC zvuaagPkJxSbV zZi<|%Q+Bwu}Twlsoe2q!r9iSBLJ})l?+}*Q{ zad=QU(0S>j_aXv}1Et$tuK};qI1DxHTF=5Fd+~c%wa!gHLj`<dOKPV{)-$B>}L@IdZU+}I969XCBl#=KA z=FOY!yu8e8Z1lyz8M(l|*`hD}awW+1EdqDd8@hnfN}Y5b1jntyI{uuRTJFCwp=i20 zKuVXF>NJQDGi(NynF0Q$$py(KsTy~3Lm;Cd7@)dAj&=&GDDK>^@n;z%K<9<4bdK05 zhGOByhEg*0@B9EBU0nF#-J%!d_b&bz4}Mz*LZgPpn)mPD1KG%(Ok{dy_Q_<35=tNGs5a2-H=^%p$4{jt65xafIj$m*g zXLBkwgzWT_R$KQ>31puBs_=sz9(yUT7iR@XI*3T}_iJz=+qM78moG|(4h2U?YaKnh zE-5K#*|KHLMi1)o!vTN&wd^*JB4hEwR7WkKnNd31b-xB-@~vY5>>@h=xqP-Bype03 zOY0X9VAfvXHa$H}`PqK?$JlEet=}7XsoE>aTD*%&TwZ&z?ZW3%dgwOPoBuQ{(Ph^IO>rXeR)*Y*tLtELQ6+CvpDx-up#Ll ziH!S+h)#A1pdTk^XBrM!r?Y7*bSd6*Egc+^R;(0c)rlV+9r^y+N^uFtAO(rVCoZqJ zz1iA~x3RLzLDc*MYkFS)MoRF#dxIUX6>&hH<%u%~{UW>k4kz){vorJ*a21!))-V13 z$)O;B;{&xh50-t+JWU5m9slT2aeIl9QVo!6_W?xWX;P~+(rk3sF*Cm*h;IB4yP#h2 zJwOO@n9Rwszgt&-4LI)Qv-Ov=&)46Ob?(<2h$&p09WGc}oFh^8%Y-b{neqN=oAv@u zACTVl0C3LAzlYjhN&hGqzdbqqW;stUipk4oROX;1Z9Eur_GItx@jP6??fbI|pC@lK zkdi(C%Gn|+g{mx01`ZtmS54`OKGj@ z>tsTyJ9qjag@1aqpN`U$da5+nu}8GL*)ZI%;UxAqsmsa9$BJ~KCHut?CM<+9kZOxRa+MCY%AB|&BS6APL zZ20gY7f2>fPC3-2(dSls&DJpO{pr)_ApsW3<2C)bX^N30Xy`F;!bZE%*1X={UW%ww zugdilH76Oaf`#8L6GM%weGmXaVPWSqvq85d=>>x}YSdo~t!ZhA{E}r^54nH>(pL^% zg{+x{uu57>Zt-bv@7V_;CXD;Ay?H~)DK_cvCLtF+cO5&EV&5#?;Zcet`SkZ%s&?O> zk8|#}(A#f{4AvCl=;Rn7X5P0&&=azcD#g$k-VR|bsH}~P)z#?~}f`WoBhW1HelXC|XB2)WgH_N%0HYV#Vqo@CZ7|30Xuu8f@;sVkHzjfJI-JVYZ3i9+?-)%}tN@QHVt!r;@zZj|`01a0B?7MX-Gv|=rtv{zX(lbD( z0QAwmRuw6I2cM@L3gmd~pmS^Q-o0T`wi}J|U1&@@iaghF$=pTpj}%~G)`fbkeNZD( zsyaVo@XZiA*m}wJ)#n?YGE)@if6_q#mcD-dx|Hq?YPt%IdK`m{$FyjN*PIYhuUtmH zHun2lB|?M9Jokx;^hpy2VXIbt>LJOW0>~XnG!``v&u`wvDeo@9=v#_UXWZVrhg7h= zd;J&3QTjxFft~zW>y0GL_vtb<3EsSUvtXt_vZ?3tkHXLp0o|`CX1Vxd=153Kx1bu+ ze2iCT2Q@c$R;Cq&l$0`f&e)h675dCTO%zFU=?8!)E#%yFcFS@xF3!|?6O2d!Xx|Um zlzJ$WN*#z*(4$I;mwzm9bH4ba{T86oawhKWa&lispT`mTD!uTxzSm42JB9R4fLU+e zxpRl)Bh7g7#W`$bGr;K47d8PUI@K)e*B{WDm|DWEvXI0mq#Q~#e*U=fT^Ya0Y4Bk)+j0}f@MtpIK5Wdx=HZRZLah3=mAL` z^kvLO&?Mc&{+@}25bgNAS&YK4hU1Y#A1$`|4bbX2`6A>GJnfU*NjqbqB&ME`ea{wOTO>nji+yR2our?*NkyCrhxSdL4w2N$Mia zxzFjF!JX^84=XSGh;P_J>OTU#jDpYV<~oTL6%~=VgN{L$={9bGTUYh+D*CeukFg4L zlI#~e!BqKn?YfCvK%40@sLg%(yV##qAk3ms5IPlt>8PHb3S{Rm+18=inVGrKKZ@pV zw-;VEIGnLV>*2eDO!8-uCC>2SXf~VTxZ@bmin>pnyDq@~Im_}05uJy}h^Q9Kt8SC3 zqGUz85NZ?`k(BpzE^lu1l^Yla%}znVvt@qFHeX-7C+AFyl#fGuE`EQ6g{8QBZw`W( z)@&FgqmIIcr?W(&rVymg<2@e)9*>g)jE@vll(f#5Eq16i z^(4zM;*2+zahTBy`j);yLbOR&c__4Hi_i7d>~*=0W)!davBRSU<66TH_b?~PBP%Dr zzoWSL4$3c0Zr6=X=bAGgSF@?=psZ1`Y+wWwKrAB0*tU^LwOaH^OCj z)hFn1*P*t3Mfy6y7u74Qz4q7m`1$6{t%q-Oz4`n(w4vd|fddD=3#%tLx7fd~E@e_& zN(X$fq1vuw3zaHw9VT77`6KZxx=%c(~ahTmNHRZSbx3^^83Xw zr~%D=(r5c@CFhPtJ%|ZfPFXpk%#I`-@{4=Ux(pX-*@b4!Oiznm8a32^agL@KT3m@A z^EQ;896Y@v1No$B;IoJTIW9(Mq{xceER8tB&|r^e7|T-#dE$2KWzM!==SW9x#1vvt zQ}9k88F}c!FY~BLNx1Nv0=My>zkf53a;EW6<`Z_W*9ke~1Wi<{FnbTSX8^b$^AC|l zk^F+JY$!~y-1GT)50DxSHMLa=3JTcWqfqc)Rjz0MIZ=OPyiQ$&W*zDZ$ob&M$JbCA z-o4f4!7cwpUm!R*n3hmk*7UeK0>Y8W5hEx7TnZk|<96Fxr|5zGRGkG10X=;xd9naTU$Fddq9a_00{-;0*7Ncfp=lMEpMNs zUFXP5*uZljNyq*w&PV4|S+5J^vEti=zHxD_DaJau4kaZLv?6>1RFCDn`_YH-s6AED z@|!>lDM;Sg;<_9CRjLsn%zWUI9O&Eh4h+<1o<8{dM@Po4&3QKH_4NZwFDtIB48V>B zZ?Z4=v!otGoKOOLefdC^Q_|{wNr=qb>9HQS-k`%iPd01;cRdZhOv07@$!iK#>&f8$ zB!2sXF$VB9x=w)GiQ&(8kkY^4gg+!ql;!)EEMM$5oddMnr?PP}Ay9uIYh^Cx5xwN! z#I@HDr}d%f?8IQxf|yx3D4k8}q1Iy#4x#}Ygs5_I8+~+APCEW5oU%H7E@&e}nO<>D zvkw|fx4GrNmRprYNg@Mc)w+KCU@CrbzTS20EjytA{9q_f;a-a~mM`tvR_&V_ZNvUO z%cDA40&sL8S7qN0R+#&q1)y0^;pe;-m=zz(1u5?To8SCzr~#T=zLX8$L91sxHt$AKa`Ax-XNlL1?L+*dGk z^w-WxZkBhS*okHA!7BdVu!HIYO+z@q-Q;Ez6dd$m7{SLVHf{MK`0~BbP)iWpC&&Qm z_oJdnX@uOiZ29tg5fN5KZM@o1IKj&3epTGzS>B2IX!)!6y%jt448e?sH=v3jx6yEj zpYE^lHZNWYGXI;r($Y>7^4gl0g%kFN zJ?AMo_g6jB+Rly6NJ&To$G@m6TtlF)F<=X*y-A)33gIge0cx6k;P~R^t3De#ucd(E zJ@H;z#DOYD9kYTjA}HVmQAyGbSMCD}_(oVV+s*rZ0|OOc4Uj5!Q8r205H`We%GzuY zrXobbB1kP`PRgQ+!)TR=q&6Hv;PWqQGK`=udM}R0AbR3{LyOH*0S-s=%hrsSI`BC` za}$!WS=6@a@=f-H<|EQoMjqj6%u=>r%N__BP`)*#Z2|(7(%tUl1%HACGFDSh$I{}I z$7G`}qDOlV(uU!($)feTE*+jzW6frFXp~pqXWAqcNMw0LVQb_pGMhH)yO@|5#Ibb{ znV_p_=;?*EncqC2wtqhjS{W}R?jZXU-gX79E0ldl#i47s53Is_d-K!BPWAB9v!(*V%df8SX zx7cpxvXgi%24wIH4BRd)9f}79>Kps@>p6bE(jxHm{PY+IJhaa&5P0-@$W3RH0=*u6 zBy~GGVG3xKcJBI|F2s+FBNZFM(20x&mB{H=uPRwJPaJvnNBvK_YjDtr6nL&JUy_0} zXZG#mV+1uJaMtkC$lk%CKR@$!6@}EJ5MAT*&dSsp*x)`u@L0*jw7e6T#T++e)v?8i z6M|`=DS%NZZ@thjr{B3h!N-AK&*$=LUZ0G|DFOa<@DA z{QkokaWCD`QAq83^!)krZ_s#v{Rm_J`~(bv4G=R9?TLT%OvcS}Fv$HyLMsvyf&1i; zxg^iX$H#OyN{}_)ku#=%S}IEtIk8JnkfmcCc(ZTA4FfvA4IULA)M|07hMmA7WD6t_ z;wSbMW+-ievVc+#4nn&RBn0>|7bzf@0sH!p7aIz|sujIgb>BV;WVeV7LHZ-;Pjo#n zu-mGr$m4@P65y-HN1O>OVSMF;DY1rVZ>3w#cp^7g|3^Zk-5F%IrXI9Ae)ff$fQ+eB zgY-LXFp}S?-Cj7=UKz7E!AHSeaVU7m7`(K3md76yp$?=K9~U1FJyiMPMX9c?uI+q$ zP`Pa{ouWz-y@=Gg#rYXZSy>qx0s2s)M}#!uZXmATog$h#4g?K{*qOgdLR5CaTLFrC zOV}u%l8GEd!Wc`AMTqN%^0rI4{u*`QKdCFoa32oM6L@FH!frtFI)jAPjo_h2-}W}P zI8tDJQus!Rb7aM?T)9H=K;L5TRI4-gS33(U%Ni-`yrZ1s(YJ3QrCphM)S!qIP_U|p zvMfw@o+xe;DVnAN^FJfGmJ8AKw!!d3Z#N?T7jyy0u!U2`$Pwp(4kh7kTakHT_@tCg zGo=_It%^XiO;;(C)Cd!OVsy)nU<_9+dxEY)XcOO2~IC0`WqAU0RgSwtjM1au{fdp)d=5K-I9WNT* zTR|;w>M3Xk?IEs&;Sw~S4#DIhugT;ua~NFgRHi8nX$L|633r@j=?eup;PRoN zW7uhUQ|8Uju2MHZiL{EG4zE3mi_@{lN+%TQ1(O=pjxa$kXex(lgwDb*_R-uPMhXH0 zwjs*g)Q=nl+7Lv%r0oCTt{EmVU~JZomdtz~-oJl*Bt`+r3#Iy*RU`jGA@nMIz7ZVt zFqrF6iTKA3yxoRit{zaT{upo2R@x0c4y4Hz@inx|R5dL=Ww~;)F>r&bn%Wb)wtR}9 zQU3i3?#op;Gs)2jE8?c$zCp zORphSgr%hCukyiRhZBo}B7=u6%OgK4RxWD$Rf2*r1POeB1MrpaAKn_L%0fZ$UUEPW zPj5ikKbH=Br2{fl3H(BnJ668w{KrRE01g_o09gd}U-s610voFz6RNZ2)ZgNehBggci6vP zHnKor4IoMgEN#t(A@AbrRY$c`h}Vxx-ktTI@M>7cBtLPNEff36DSmbZEF2_iD#;xj zdjEdp(cu1|i2iEnj$ZUR=!r3epzEFrBLCrntdAR3k=7dIx2h_Qp{6wTzd6puP+nxW z#6)6vgq3wo@rBPu6)=@u&&bGt;982i38!?Y*hfT`hFvZ;{1*K=~dM=~IJ7@d8{e_+nceCq$vhjr-O2c(tKxUuXam}|}phpq!P4`u0! z|BkHI6QCxls;ZzSbVQYp_V}fWCD;Y{ZD$F;SCc2Qcy10F2I5G?@Yd5z$1Kgn5dPW> zJ*4|2QtTlYmlqeS~%-y8<3qPLaKFBDvf_l<++ZcLSzn=3%@2N&H@5KY@yjXYhQxS5*nBg zdy;OkW|rBK4o!Z>y%uGw#LiCB4MO@#C_&+f=i@J%4EA1Rrs0q{w~n;5(<+eab1p233 zD?W;n1oxKLbk}lpCXjMl!x35rdPN?xzDzs}kL){&R_RjDILdHoL`d9(aKP+?u3m6h z*gL@V??M`_(KtH=b$mI)L;IO!BO%`*32aqR;Py!~$Pu?`7X9_>7tm2_I6^Gm+}xZQ zAvRc_NKAOUcI|>?wFP0`uwzcwc^c=QZ#f=B;MTBz@h4q z@|Ry;2p&T6XG>R#t0TV~()5~?OTaig}Whm;JB;2|JN(*^DU;OzIH`x{xCxdIWYQI~5EU6W6d6NqRRS0j}vnLWJC8!=NSFIwJQ@HI(DwEDWUCMyr6fH<}6gbcX;ae{N z)D=JvA@@Fr3oO@+#hq`ik!@Ck73C)BOM3o>9C%$^=!{YgMu-e=q;s2U3E1t!VFlZv#-w_JVSdmhZ%>WS3#r43phVgjb1&-&6SB1 zxEDe4m^d_sgM_>S-kiu?A7s_@-|Us(07!UW4Z|DGBF5K~7{B=I_wW9!T7_6Nb13t_ zTkO}s&xEl4b4}po0{nL7ANKmvMm1E}2x`BQRV>_;kU3ZUH+yB%E)L#kPsr;6WH5c* z2|vFRIsX^(8nhnhpeLkA(EX2lgi56a2{pqMLsJjYnh9)0L6Z!keb8_YD!K)@A_a{( z9o-Q>m0O}+F;O6-HxM z_=b@ZzQci+^z4DK(FbHZ3UR6;${Sqa{@I4ze93(zO!NxKTS>WpnHH8|TS$?6W-tF{ zn9xR{H1*x&i>5q!^vDMSV3v912Gr3#1g$@W$^wRp610^{y$az>XjG6Y9E&9a>c?7` z!7=yhq7XD>v`QOMRkgQON9_p()Bn2#i(K~>c>9RLOqss)1+-vWFSgj8H8+Ruae@!% z*%1H?nFACZPTn|_RtkyMJwFQW!6=UC8bUFbR6`TIGE`*wBe(H>(q|_C$m#q)9CrWy z{S?U4k(M!@P5sGn`8%<|;%4P*0>$=y`Xu06dK!8LH8lM{IBbqc;lz5_!-A@HR4*Jw zG!h!!Z3msF0(tAa{36^z6>tF|Athf!cySO@kK)q&ZVG913F*Jc%-O=s%oi?t`f8(# zgowx;2x3`g6^sB1$#J;d8@G*b`n`gwVICXW3Aur$6UK9LtYDLaYvTkP3A4vo!{%l^ zNc6r?t+BvEoZF~&uvtfu9BHNSHTEzQ6u%pPcVBlWG2upl6;5LfDR4@tcYzyf|I--4 z;qYM-I7w=m0sEBS9C@hycqnZHA~K*N!4ylGMgwLB0vy^`-8RS`Eq(p{>!GRMWZ{D& ziw}7Vh=Sy;ByD0|+;%81lNrj0&ga71KKVC)GG z+IBcNCdS^`+({O7=j6X}m|rs)T7eBQ>|ZTd&xvE$vP!gQUlN(U!()8yzq|WNO5f{& z-H9MGtg=WNhS)>%RKwL?Ogs+YS`--fSuV9Y`d%1KZa`Z?dxStT(D${)aKESk6q9Fa&;X4^ zZ|Ib|A34;o7bi1N`9rW^jaqy;0e&Zp0i=lYfty>|`?-ndq)viHIBbwkTlM)5C+iRr zyHSEv(dn5Pkd0*T-!R#Aq50L1b`}F^5B;0Oo={TnQ^|&i#tADK9hu4j5Tu4od_2f> zibep01-jW!Yy;Bx!vNX`Cj^i{{S5Qr?YRf=PD?5w4zT6Ik(36(2n+ zkhRfN}Z31J8F z$DigaL24ZfqnG~Z*J{U$gROQ$Di|(GSgx!jY(#pmFU~P9E-q3$&}%P%ry9!@55YKD z*~*F^Yw-wYJ}is_>uhV}B#5+Rf&O|^upDK#WesV|qo9#2eh8h|=lM)`xaIY6%F9*) ziI${f6j{j^jo0j%nR>kh_m%bm@4h(wZ-Bu;$v|XB$j}OfScFDiwlt%&3!fKOsSp!N zKhaa6{rJr6aC&a{DyVPrhqNSp=yxq-CYEwx#pj?YiHY~jy|5HX~=+$q%_6-U5QA7LCpUWN&FgC)$Sc%Lxf5S1kP0bY=3aBuLdWC z9z#BiFIt?nzRe{|yYI}IC#aArlI~<;A|V#GMxxtev?($r8HR+o4`gwf=b$nYZ3bhU ztM;J|!H>_kZCfeUVL8fl$h&v%h!q$l_`tz~;G&<@eo(1W|LG(|f@rs0Z?C4R-KIpDQDMO3e z1)UZ?-s?hndA*oNwklr!ZJ7ieBi)x`zuTL`^E_n#y76^r~=4UMZ?)bWm zCnq|PF(Zmw9^EbwVY5b4AQGum|C$xZ$i#v@?XK{>lFq70&nW0sv-)X6iN*0m_yQhDGF3tq zPsAK4JMnN)V6lzG^ce{_YP_4mqKwh;<7pLn#rmc!>li2KPEuBev~?A@{Y+Gp%~Sgf zX^|pLj}8)GcQjm+XkGyOO**es;%oL9gPWnC zJocDl1oXoP_wTb(W`B>QAiHVbvuQV!Ur2Ux!dQLEqNkw`?E@L*!Fw%@b1>fDfM7jz zIYY=G`A4CZD7|2w4H~FbQG3{ZU^O7wmOv}*^6=e_Q^uT}_&>=o+6RU&34ue%G&rqgk`(R6l0@AHso?uN!|RO)Y8r`{Etr0oQqTkEtMnYwT#AiQ3_%PtB@q3D*94b^@ICrlk?FkoQBc|oJQ9c^)OGx$y_ zjV`q*!x;$O9`lgnNcG3`toHW;MTp_tppUm=&6+rz4Kf&wTyK(j1H6wY>@e0{houSG z5%%DdGEx{mAu|a;Vco;S>YyK_i3Xj1o12>(nWiAcM<6|y7PEJd@_>Oc2b`BKR+iYv%M@K&}11y2mg^eP+Q~)Im!I@Za4|asOan0icnZcGx7UZ zotIzAAa=N6a)tO9aoo$^z8!-(^`41U+FR_37;+JM?*6Y|zn+ZB(0lg8a0Kt8sP;4T z$HW1r|H)ILq=nYj=jZpZo;2z)f2)Dqk+x)~8NK(5`vNlU4otRDcA#$ued-OGzZ;A9 z{(85qDHl__HIyAxY6--KcGt_74dx;xt()AeW!!$n(P+RCuL%X4Mgs<{JVzMq`jb=A z#eK*mG{h+>SV+o&{eV?Tksg+>y)^#ompE)El4ghQNDXe?TJCdU_}JbZ|HqiO_MBHj zgUs*-7BkNqK!D+p+lfcGKF`^b1p~ddd(gX7Td0A$uL(8tgGT~XBK-^zlcH8ic|}D9 z8AH!ofN0Z{ODfFQA_&O|utYcg_z7SOhL6^s2CDEHQ#5ZiF%SA@bUP1L!!*w}w}B`I z(OVlO|J)}mo|N+X`g$n(_?m{ke#Eij;7X^F?9cj3AMKhm-;6rJ&<^p%YzHtJY%7T+ zK9eZ z&v>v;2>unW6Dwjfg;DeT^^_0Gb`>{dm~@a?dH4A{-t(mZ%1cw#F&@uLd?d4yd-nq80j-pmzTEEb+>5qn()JZ)Ea!GD zrdU8^A6Tm=7Js$v3lIiz3eZ&D-Fh%ktQ@UPgDXFP<}g6H8!oM3c}SAD#r~M3`NYXq zwK4X$2x$^a*l7w-B>V;4xKo&prr3Uc0g}P^ht}DlkAsfd(!-3< zXod;2N5-?jO8Y89>EJ&*3em{+)<{DV3uaCzsQS@446r3ZT!VRRn~>02pSD+*oB`*s z`x;QYIAxu-Lq{^oi%>&?!0~!Qgb%*UQasl{Z77?6_}q|DH?fh)_4W9MTQ z4)2PX4WJ*5J*SpgR@WUP6wq9bZW6Y>f(WPiN8E&^w-G z!YcCPxKn-rIh$U*J?eK};Yj7q4yB|GWWWV6>^1nqfJR;T3sOwl9K!ldmqWAOEzGqJ zw&aKq4vb(XAn#8ba}wtIRXKAopj5B7(} z$E?cyH8PcE1{Kfm&K*7lg?4vglrp?Y!WYfUF=TI@>B9RPn3?w@rGr8eeE{f5EYJ1) zBdAXIIW0YJzk6~7Gv15hJ;f& z!C3rVSXC)byGBy^T%Z_uwU~6P!!^u{iJ@WN4?!fm=GjaPx&=;ivt()usc?;{vDb__ znPMe+@pv&EQ6zg1jQr3D zhSpb+`JEi|B0gF5giOE2x&~ei?s{f zuL{y~S_R4Gs8UjXm_?4-S$(7j}kGBvW0K0sTscBjb{Ff%bjtkn%wpy zA;R{dxBvTr9FmmN45V2$Z?>iU8M3aoynYY*z{XRO7 zenuGL$BUy_Sj@nTH823Pa?KhE?0?f`hGomb_@Z6AAGR=ZikGdDFXzZYOWe8zwbG!6 zWE4t0#u@RN4zTwO8Zyw0SAB?@6h|5y&^ycve5Yw>_z;5>FJv*f1~-Tp z!hFZ}?QwO7<8~?}vvP2p!`yBa<1FJG_2Vk(D#CmOQZNw9Pw^7_S;+`;X z5Vf?8W@ct-o}PH9@~|fLLhRf*JNBg+-+>%ap z9C+oQS(cZuLS26c-ntMDz0aUym#M=1&x6*jq(vfclCOCXB~?{RORAi8!FIU4@YUo( zwpHVYe1B)~xuznN69sD%xcNXU04-k%)8WIUd`Ebt#Seuaw#1er2$ZK|# zl$FDwrsyw(1t!UuM!^C(KbhezscYmGpEU5pem>w@(>+tIhmsduS5!rrmh7~u{`jG^ zbRJ#1^4|XFsjMQLt&ObW7ES#L@ zi6yKEh0bXXqdF0!K?PAy5(7yV7@)I2;xaFXohlqItT2>H4m7;$VKo_tmoB1JJ0^R3 zgO1BEUeA-wbJ($%-}(C0cQb*Tq{)NQ5+48=+b1-I4Si*y9G5EW%?Fo$1$F>UqKC-2 zd^2r>7sFmtIxi$uFie(3VTsFYcBQuVha&FHYjurRf_(q%a)`djPK%1Q)AYUa`x*~ z5TTvKU{b0^aZFo>(aas>1!(B{3+JYlb`e3p@3tE3)q02H%XlH($HO)l17=6Nj)%v@ zLdS(UsVz_pZ%oK&J(dLQqGL6OCWX`C_^)`AM*6EfKLOxuweUX!$rz}cRa6Pm=ij~i zo{+)I`?CD_xdidbmk*DiXn%Nd03?YHwW_rApg_*e%jAo@oz2ujmhlKVBRqNi0Ey8ZoZmA48woBaj7+E*pxza z(H`Y5g3N!yY7PU1o%3 zf{CR*IOY}foc5Df8z9k9T0O?AA)iVD5LWQx`f>JcGazN52671)E7X@e)GNNr^t~J0 z{c-&mrm%$M+E>y549=hJ`Mypp-X6pQ1S!#9fIZ0*p)>IR(v6BcDFbONe4UQdGgb2k zx5p+XQXK}vV7tuLV@_g#3_Gs>{On5fuxL#`KH85v(jf{c_Az7+e>MOco@3)iePI| zU!eUngenl^{0?<)gM_fG9EnmCUx92-8ylNE(MFtTq0QU6yC-yy^T)vv_d&cIj+Qs8 zLT`(e8#>JS`1s%-KYhRAce#%B@qK5QUGM^xt711mg@|H3$ifp3bj9M((}Q z2fPI1c)EN zs7W+_0P%-qg0K8(_Xp_o#8dVhBX}t{yV3V$OYaTFj8-NVPJF)=;_e}miFQjxdHDkz zUTpp5koC-=(DHmRiqp+$hE~fyECpVPmfBE<*RNe^suSZqoJgLOxHne)-j0M4tTZQdc5TiGg?7lLKjC{PmMlpAQdm$6I|&7eG&rPdXR$| z17-Ny;5VJ>dk-FzBh`N33i*mwTs#uY&!10`U?7ZH4e2VyMGS7A(ogP<`;E>*C#n|C zeS&ZDF?g);!nIvE_XxL;uSKYDCw=(O`#y>{gEGz#)oUEEfIjHUrYo;7p3It_d=!&c zun-7eV(UO99lbWJ>MNYarm<#2|ht zM=3Hf)&t|lx4?>5+nKlzhSd#dT)%PSq~{nF?-8I+)mPDFIw{DMF={asU7=}7d zmYYBNm=Qm?2a3Wb_GFpnvu`H<#bGc&cBe_{`@-J6BJB_PopKJ z37zaaf8mF|zCN8?7Y!>bE99iWK*7(9`w_K%@2&&xZ`!$1?@trLZ_B_t=9j(w!ACxE z{j@S&|C9OS)9Jjzw%Rn({VZuFVFLPzRPEOW!RPh8`_UjbG;?9 zGus8*+6$F#y&1sOVJZJdPmo%8sPz105n*|OZ6GEN2$UCud?T`__=3e3LhB3MQ@>Vk^{L~3#pKNt z=1~4)Zb~+r#8hL`A0Hb_8#PH~kzo{FMn>j=Rr5J}t)S*ox`FeJMLXqk;TaB@n9k;7 z(Xo~LA}JvPGd#LHeVh?1vMfK|jvL(e5dotZs7BI~pJUW0NKZyKRwzy{0In|Uom;mS zfipg0txsVEiex=i(~euusdxRm_Mq;dq4dV(&DB*4(a$SCSE-Qq)QC}uL%~Ts9bXQ; ztJ!V||F_mffMB*#slzGnu+0}I;13NOm7rC=%G+nk702-#2Y<|to9;$jQlT4s8by`o z#%pjQ9(X^uxjz)@T(sSL6qEWB)zU4b7=2T%%B|#hEmyFtA*d@;N8h^oLf-s+N%=+R zlWXCY!iAPC;nOXZOowVIcM{oxvG5Eekv4}OLAB?I6J=uZqfwL`stQ7r+Mg;pZ%Fkv zwsN*CTCKZ$R`zjr@7jy-nI79=s_+~b!!)hg_gDy1!E}=f)BW4ckOvN)gAMq zN4#Ej>|E*zPW|=GL$*tSxaJQMv$Gx0%%CS8!7zeXe_D2|>APup95V-8KnvM0R}I4k zo$$Nz&ZH#|BRI8;)RTi|*3IW+MMcXa!>j7;MdB|831ul3eEf>CIi)={>uK68g`}ze z^!{sCizh$e8(81oj|rAd9g`SQ%JJ&kDI;_4C%k~Er{`jWBQ&H-$eWV!VT^G0Vw)x4 z9sntTm%l5HeKJ%q(h!S>jGP<~7AORTg#u-Myl@KWrVtceJK*xb3QMR2Og$NTQES|% z5h?Ww%IBM4K+wqHejEb^cgcOPs%i+d#f0JoPSS(@0uq1Xz;kxTZEpiueTr30%bh|%KuDkZTi6LyVeY&6)ZeZ^zyOMruvz z5vNyYY;A1JVV$ayaD(SZVNpvFs1#62!;CXf*m*THA}(fP1$nxvbi@VIhLESkHxZ0f zLBmglg->vh3`4jEJ(9L*1$k#o*AEBx;~rw|a7$Dn{x3q}eD5>RM`C!OpmtFM4v=q? zi#YS!L=8X=Ahrkd6DLk&=shmr>V+oDU8f1sFaTx1a2hEj_V@XJAS`giK5K^1XRuxk z#^6j$hNA9?uej!-(CM!n1PR2Z>~LZgC=-4fagU8d*5#O_!XPo6z5bYA6zZO6 z_$zFt*)oMvn(M8#8I%+5v;J3v^hWvRM&6vs%TN00=Uwv;V7$KvWv&W+Ajstsxc?j&V~hnmDOd?u;$bzAM8RgY-tH zYTsLnEXW@4o1jfx$0~OEH;j7UGqfQ@87(y=Cyb>3wp(mP!mLzO)VhD8YnWKMheX}g zT(lTuGKxo|dw16bpFQo3!E>X408jI?XG`^Jz(b(6ifnN{`ST6#1K!?B#Poyt5%=H* z;Fp)9jy-xhnws!rWOB^_p6>#Q#Um)0n4T_zBR>R}#;&?26fQWX&P^}M8`(xxP`HnHHG&qd+=7>@*p5~QR?P&h7Xy$47HK=h+Kt}$-&&UYzL98@+= zUoA2Ph=S)7)kG17=UXsg_67~qW4c)yW z-rf;`$^%#j&Pfs&mTl7xW}2rNR(as1f>`Ce5qLt3cYvfh!62>7jRd^zz64z%++@VF zdDn%m4A@e+sU(rEj>xO8bLI*?DWkq_f-_OgY6rfWo#0J~`YNUGPJsHo4+X&vg#sz> z8i^4DK=3B$>mMTMGOUv+hWrxLk&HX}pth0KccZ+#+$_7FiY@XYduAEZGJoLU^F6kQ zg(Bm)cpoQL>KgkUBeGGlN4?ySAXJQK3Az(&h?vE3A!Z;T*f?U3fZ0Sr0SGS``vTI8 zU|H|yE`j0?XdHuIofRD)&r3j4)bdtqR?yA7@h6tUH_7N{3Av$<4*Z{bdX)l9BCctg5MDo_) z#lE8R_WF}?`AuRd;h?rQAVmZg!c9~OQ{*EJL#KIsmfgm|0t<`CD^*Z~c=Wlb7Lln# zBivGv8v@7wYtn8COLYGhMv~&Z1N7=_7@y#N%K|1(*qsUFuIzzhKA@A?RzEDDu=>@H zRqm_MEtTEB|2mPhV#2-#ln8+}A)QQUQfx$hM9WFiA%yEBJ!BY(K)eZ@Ef(qn8xCNz z14TGeknSq8V@Is)p){Itg;_Yt<0(YTV-v3((=JiQE1sxQ;zAdQ{+%Z|8o^okj6hXw;)G z7pnVy+H9dd`!?`dS8?FPxm-19@BUx|5V@7Z{PD-~WMzmOn8jraUzVsa`W^(h#6v)V z9>qa}TM~kSN>{DG@sz;(c5o5l;1JXw^`5gF_8o|p$!`R$VCv%0a73n|;nP+#wCH2D z%7};zo4)!C(!?jKfq&2v&ZHp?pR4Jyk41S=1EMb{C&#EAS0o9$ZEYDi#M|5xYuE{j z=ANE+Ntq}9@%z$Q@F7&?5*t}xBk<<0ow=az|3%EPyECTpzVE+~16~zYb1ZD3sQ7PQ zH1vDbY>|}ofj8yf*nuh`&;AFcpp=wqTbcd+*tS|evwdq;hf0(d+R10GxF_jQmA)sx z+`Z-E<|hTtdzUYs0!?+S`Fh>-J~_pStlX;ayWJJSg^qsS zwQ5s@XJf$D{zGkdWk4nmI*aP05|P(M<`PM0-3S<55jYPfuUW zK}H5cL$8T!S=Gb#v49}nAY2f$yb}|_4=KRgoQ!v%<{6^UFGGGgGdGuq^766Q+JDUf z`oH%Bq}?RdWdP;Xw?&i8f6%~`ASDxD^p`abB_U~8>`lzg!TA&Nx(l2h=`Z+s0@R~I zHDQZh5cPxNO~T;F3x1xwqXI<9SxdzYKrq)pqjnb^5=mZO@!Ralt-Yy)=Y7H>Zi-^G zR?LBfPNo<4`?Dq+cgxDQr0g6Vb!^eHyMij?PlB{1B|E{BffOL`nb2`V?J|*iiv>@V z(!WE59W(}v*`Y_kSMCH)zbK(qTma%h8`6C@ww+Qq!>v|bH8cB_D7?E0$S@{HlHO-*c-oE+bQ zz&+=+H;dWbk+_JrXgSh|hmTu{9=N=x#Kn Date: Thu, 11 Sep 2025 13:47:10 +0200 Subject: [PATCH 23/26] remove comments --- nebula/controller/federation/controllers/__init__.py | 0 nebula/controller/federation/federation_api.py | 4 ---- nebula/controller/federation/scenario_builder.py | 5 ----- 3 files changed, 9 deletions(-) create mode 100644 nebula/controller/federation/controllers/__init__.py diff --git a/nebula/controller/federation/controllers/__init__.py b/nebula/controller/federation/controllers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index e439c3785..f2f7f8e25 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -12,9 +12,6 @@ from nebula.controller.federation.factory_federation_controller import federation_controller_factory from nebula.controller.federation.utils_requests import RunScenarioRequest, StopScenarioRequest -#TODO Route the request to the right controller - -#fed_controller: FederationController = None fed_controllers: Dict[str, FederationController] = {} @asynccontextmanager @@ -66,7 +63,6 @@ async def run_scenario(run_scenario_request: RunScenarioRequest): else: return {"message": "Experyment type not allowed"} -#TODO need to use fedID? @app.post("/scenarios/stop") async def stop_scenario(stop_scenario_request: StopScenarioRequest): global fed_controllers diff --git a/nebula/controller/federation/scenario_builder.py b/nebula/controller/federation/scenario_builder.py index 5b5092675..4e7780563 100644 --- a/nebula/controller/federation/scenario_builder.py +++ b/nebula/controller/federation/scenario_builder.py @@ -445,7 +445,6 @@ def dictify(d): # Trustworthiness try: if self.sd.get("with_trustworthiness", None): - #participant_config["trust_args"] = addons_config["trustworthiness"] = self._configure_trustworthiness() except Exception as e: self.logger.info(f"ERROR: Cannot build trustworthiness configuration - {e}") @@ -453,7 +452,6 @@ def dictify(d): # Reputation try: if self.sd.get("reputation", None) and self.sd["reputation"]["enabled"] and not node_config["role"] == "malicious": - #participant_config["defense_args"]["reputation"] = self._configure_reputation() addons_config["reputation"] = self._configure_reputation() except Exception as e: self.logger.info(f"ERROR: Cannot build reputation configuration - {e}") @@ -462,7 +460,6 @@ def dictify(d): try: network_args: dict = (self.sd.get("network_args"), None) if network_args and isinstance(network_args, dict) and network_args.get("enabled", None): - #participant_config["network_args"]["network_simulation"] = self._configure_network_simulation() addons_config["network_simulation"] = self._configure_network_simulation() except Exception as e: self.logger.info(f"ERROR: Cannot build network simulation configuration - {e}") @@ -477,7 +474,6 @@ def dictify(d): # Mobility try: if self.sd.get("mobility", None): - #participant_config["addons"].append("mobility") addons_config["mobility"] = self._configure_mobility_args() except Exception as e: self.logger.info(f"ERROR: Cannot build mobility configuration - {e}") @@ -485,7 +481,6 @@ def dictify(d): # Situational awareness module try: if self._situational_awareness_needed(): - #participant_config["situational_awareness"] = self._configure_situational_awareness(index) addons_config["situational_awareness"] = self._configure_situational_awareness(index) except Exception as e: self.logger.info(f"ERROR: Cannot build situational awareness configuration - {e}") From 3809fb9a9934cc340cba6359492b4c36ddfeb03f Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Wed, 17 Sep 2025 11:58:33 +0200 Subject: [PATCH 24/26] feature federation ID on all API requests --- nebula/addons/reporter.py | 4 ++-- .../docker_federation_controller.py | 6 +++--- .../processes_federation_controller.py | 6 +++--- nebula/controller/federation/federation_api.py | 10 +++++----- nebula/controller/federation/utils_requests.py | 18 +++++++++++++----- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/nebula/addons/reporter.py b/nebula/addons/reporter.py index 4ca82b91d..6e61abb8a 100755 --- a/nebula/addons/reporter.py +++ b/nebula/addons/reporter.py @@ -54,7 +54,7 @@ def __init__(self, config, trainer): self.frequency = self.config.participant["reporter_args"]["report_frequency"] self.grace_time = self.config.participant["reporter_args"]["grace_time_reporter"] self.data_queue = asyncio.Queue() - self.url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant['scenario_args']['name']}/update" + self.url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant["scenario_args"]["federation_id"]}/update" self.counter = 0 self.first_net_metrics = True @@ -170,7 +170,7 @@ async def report_scenario_finished(self): might be temporarily overloaded. - Logs exceptions if the connection attempt to the controller fails. """ - url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant['scenario_args']['name']}/done" + url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant["scenario_args"]["federation_id"]}/done" data = json.dumps({"idx": self.config.participant["device_args"]["idx"], "deployment": self.config.participant["scenario_args"]["deployment"], "federation_id": self.config.participant["scenario_args"]["federation_id"]}) diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index ff7814909..9d1553276 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -230,7 +230,7 @@ async def update_nodes(self, scenario_name: str, request: Request): adds_deployed.add(index) request_body = await request.json() payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) + asyncio.create_task(self._send_to_hub("update", payload, fed_id)) return {"message": "Node updated successfully in Federation Controller"} except Exception as e: self.logger.info(f"ERROR: federation ID: ({fed_id}), {e}") @@ -246,10 +246,10 @@ async def node_done(self, scenario_name: str, request: Request): payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") await self._remove_nebula_federation_from_pool(federation_id) - asyncio.create_task(self._send_to_hub("finish", payload, scenario_name)) + asyncio.create_task(self._send_to_hub("finish", payload, federation_id)) payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("done", payload, scenario_name)) + asyncio.create_task(self._send_to_hub("done", payload, federation_id)) return {"message": "Nodes done received successfully"} """ ############################### diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index a97f7c96e..9fd9f1729 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -190,7 +190,7 @@ async def update_nodes(self, scenario_name: str, request: Request): adds_deployed.add(index) request_body = await request.json() payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("update", payload, scenario_name)) + asyncio.create_task(self._send_to_hub("update", payload, fed_id)) return {"message": "Node updated successfully in Federation Controller"} except Exception as e: self.logger.info(f"ERROR: federation ID: ({fed_id}) not found on pool..") @@ -206,10 +206,10 @@ async def node_done(self, scenario_name: str, request: Request): payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") await self._remove_nebula_federation_from_pool(federation_id) - asyncio.create_task(self._send_to_hub("finish", payload, scenario_name)) + asyncio.create_task(self._send_to_hub("finish", payload, federation_id)) payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("done", payload, scenario_name)) + asyncio.create_task(self._send_to_hub("done", payload, federation_id)) return {"message": "Nodes done received successfully"} """ ############################### diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index f2f7f8e25..ff68707ed 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -10,7 +10,7 @@ from nebula.utils import LoggerUtils from nebula.controller.federation.federation_controller import FederationController from nebula.controller.federation.factory_federation_controller import federation_controller_factory -from nebula.controller.federation.utils_requests import RunScenarioRequest, StopScenarioRequest +from nebula.controller.federation.utils_requests import RunScenarioRequest, StopScenarioRequest, Routes fed_controllers: Dict[str, FederationController] = {} @@ -51,7 +51,7 @@ async def read_root(): logger.info("Test curl succesfull") return {"message": "Welcome to the NEBULA Federation Controller API"} -@app.post("/scenarios/run") +@app.post(Routes.RUN) async def run_scenario(run_scenario_request: RunScenarioRequest): global fed_controllers experiment_type = run_scenario_request.scenario_data["deployment"] @@ -63,7 +63,7 @@ async def run_scenario(run_scenario_request: RunScenarioRequest): else: return {"message": "Experyment type not allowed"} -@app.post("/scenarios/stop") +@app.post(Routes.STOP) async def stop_scenario(stop_scenario_request: StopScenarioRequest): global fed_controllers experiment_type = stop_scenario_request.experiment_type @@ -75,7 +75,7 @@ async def stop_scenario(stop_scenario_request: StopScenarioRequest): else: return {"message": "Experyment type not allowed"} -@app.post("/nodes/{scenario_name}/update") +@app.post(Routes.UPDATE) async def update_nodes( scenario_name: Annotated[ str, @@ -92,7 +92,7 @@ async def update_nodes( else: return {"message": "Experyment type not allowed on response for update message.."} -@app.post("/nodes/{scenario_name}/done") +@app.post(Routes.DONE) async def update_nodes( scenario_name: Annotated[ str, diff --git a/nebula/controller/federation/utils_requests.py b/nebula/controller/federation/utils_requests.py index 894acc492..db861a57a 100644 --- a/nebula/controller/federation/utils_requests.py +++ b/nebula/controller/federation/utils_requests.py @@ -10,19 +10,27 @@ class StopScenarioRequest(BaseModel): experiment_type: str federation_id: str +class Routes: + INIT = "/init" + RUN = "/scenarios/run" + STOP = "/scenarios/stop" + UPDATE = "/nodes/{federation_id}/update" + DONE = "/nodes/{federation_id}/done" + FINISH = "/scenarios/{federation_id}/finish" + def factory_requests_path(resource: str, scenario_name: str = "", federation_id: str = "") -> str: if resource == "init": return "/init" elif resource == "run": - return "/scenarios/run" + return Routes.RUN elif resource == "stop": - return "/scenarios/stop" + return Routes.STOP elif resource == "update": - return f"/nodes/{scenario_name}/update" + return Routes.UPDATE.format(federation_id=federation_id) elif resource == "done": - return f"/nodes/{scenario_name}/done" + return Routes.DONE.format(federation_id=federation_id) elif resource == "finish": - return f"/scenarios/{federation_id}/finish" + return Routes.FINISH.format(federation_id=federation_id) else: raise Exception(f"resource not found: {resource}") From 248ed1fcbce7bf83ee38510c2733ec13872546db Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 18 Sep 2025 11:03:40 +0200 Subject: [PATCH 25/26] update API requests --- nebula/addons/reporter.py | 5 +++-- .../controllers/docker_federation_controller.py | 15 ++++++++------- .../processes_federation_controller.py | 13 +++++++------ nebula/controller/federation/federation_api.py | 14 ++++---------- .../federation/federation_controller.py | 4 ++-- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/nebula/addons/reporter.py b/nebula/addons/reporter.py index 6e61abb8a..5db64f16f 100755 --- a/nebula/addons/reporter.py +++ b/nebula/addons/reporter.py @@ -54,7 +54,7 @@ def __init__(self, config, trainer): self.frequency = self.config.participant["reporter_args"]["report_frequency"] self.grace_time = self.config.participant["reporter_args"]["grace_time_reporter"] self.data_queue = asyncio.Queue() - self.url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant["scenario_args"]["federation_id"]}/update" + self.url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant['scenario_args']['federation_id']}/update" self.counter = 0 self.first_net_metrics = True @@ -170,9 +170,10 @@ async def report_scenario_finished(self): might be temporarily overloaded. - Logs exceptions if the connection attempt to the controller fails. """ - url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant["scenario_args"]["federation_id"]}/done" + url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant['scenario_args']['federation_id']}/done" data = json.dumps({"idx": self.config.participant["device_args"]["idx"], "deployment": self.config.participant["scenario_args"]["deployment"], + "name": self.config.participant["scenario_args"]["name"], "federation_id": self.config.participant["scenario_args"]["federation_id"]}) headers = { "Content-Type": "application/json", diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index 9d1553276..5be7934d8 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -200,8 +200,9 @@ async def stop_scenario(self, federation_id: str): return True #TODO care about cases - async def update_nodes(self, scenario_name: str, request: Request): + async def update_nodes(self, federation_id: str, request: Request): config = await request.json() + scenario_name = config["scenario_args"]["name"] fed_id = config["scenario_args"]["federation_id"] try: @@ -230,15 +231,15 @@ async def update_nodes(self, scenario_name: str, request: Request): adds_deployed.add(index) request_body = await request.json() payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("update", payload, fed_id)) + asyncio.create_task(self._send_to_hub("update", payload, federation_id=fed_id)) return {"message": "Node updated successfully in Federation Controller"} except Exception as e: self.logger.info(f"ERROR: federation ID: ({fed_id}), {e}") return {"message": "Node updated failed in Federation Controller"} - async def node_done(self, scenario_name: str, request: Request): + async def node_done(self, federation_id: str, request: Request): request_body = await request.json() - federation_id = request_body["federation_id"] + scenario_name = request_body["scenario_args"]["name"] nebula_federation = self.nfp[federation_id] self.logger.info(f"Node-Done received from node on federation ID: ({federation_id})") @@ -246,10 +247,10 @@ async def node_done(self, scenario_name: str, request: Request): payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") await self._remove_nebula_federation_from_pool(federation_id) - asyncio.create_task(self._send_to_hub("finish", payload, federation_id)) + asyncio.create_task(self._send_to_hub("finish", payload, federation_id=federation_id)) payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("done", payload, federation_id)) + asyncio.create_task(self._send_to_hub("done", payload, federation_id=federation_id)) return {"message": "Nodes done received successfully"} """ ############################### @@ -292,7 +293,7 @@ async def _update_federation_on_pool(self, federation_id: str, user: str, nf: Ne async def _send_to_hub(self, path, payload, scenario_name="", federation_id="" ): try: url_request = self._hub_url + factory_requests_path(path, scenario_name, federation_id) - # self.logger.info(f"Seding to hub, url: {url_request}") + # self.logger.info(f"Sending to hub, url: {url_request}") # self.logger.info(f"payload sent to hub, data: {payload}") await APIUtils.post(url_request, payload) except Exception as e: diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index 9fd9f1729..7a3c35d87 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -161,9 +161,10 @@ async def stop_scenario(self, federation_id: str = ""): self.logger.exception(f"Error while removing current_scenario_commands.sh file: {e}") return False - async def update_nodes(self, scenario_name: str, request: Request): + async def update_nodes(self, federation_id: str, request: Request): config = await request.json() fed_id = config["scenario_args"]["federation_id"] + scenario_name = config["scenario_args"]["name"] try: nebula_federation = self.nfp[fed_id] @@ -190,15 +191,15 @@ async def update_nodes(self, scenario_name: str, request: Request): adds_deployed.add(index) request_body = await request.json() payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("update", payload, fed_id)) + asyncio.create_task(self._send_to_hub("update", payload, federation_id=fed_id)) return {"message": "Node updated successfully in Federation Controller"} except Exception as e: self.logger.info(f"ERROR: federation ID: ({fed_id}) not found on pool..") return {"message": "Node updated failed in Federation Controller, ID not found.."} - async def node_done(self, scenario_name: str, request: Request): + async def node_done(self, federation_id: str, request: Request): request_body = await request.json() - federation_id = request_body["federation_id"] + scenario_name = request_body["scenario_args"]["name"] nebula_federation = self.nfp[federation_id] self.logger.info(f"Node-Done received from node on federation ID: ({federation_id})") @@ -206,10 +207,10 @@ async def node_done(self, scenario_name: str, request: Request): payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") await self._remove_nebula_federation_from_pool(federation_id) - asyncio.create_task(self._send_to_hub("finish", payload, federation_id)) + asyncio.create_task(self._send_to_hub("finish", payload, federation_id=federation_id)) payload = {"scenario_name": scenario_name, "data": request_body} - asyncio.create_task(self._send_to_hub("done", payload, federation_id)) + asyncio.create_task(self._send_to_hub("done", payload, federation_id=federation_id)) return {"message": "Nodes done received successfully"} """ ############################### diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index ff68707ed..7bbea89e5 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -77,10 +77,7 @@ async def stop_scenario(stop_scenario_request: StopScenarioRequest): @app.post(Routes.UPDATE) async def update_nodes( - scenario_name: Annotated[ - str, - Path(regex="^[a-zA-Z0-9_-]+$", min_length=1, max_length=50, description="Valid scenario name"), - ], + federation_id: str, request: Request, ): global fed_controllers @@ -88,16 +85,13 @@ async def update_nodes( experiment_type = config["scenario_args"]["deployment"] controller = fed_controllers.get(experiment_type, None) if controller: - return await controller.update_nodes(scenario_name, request) + return await controller.update_nodes(federation_id, request) else: return {"message": "Experyment type not allowed on response for update message.."} @app.post(Routes.DONE) async def update_nodes( - scenario_name: Annotated[ - str, - Path(regex="^[a-zA-Z0-9_-]+$", min_length=1, max_length=50, description="Valid scenario name"), - ], + federation_id: str, request: Request, ): global fed_controllers @@ -105,7 +99,7 @@ async def update_nodes( experiment_type = config["deployment"] controller = fed_controllers.get(experiment_type, None) if controller: - return await controller.node_done(scenario_name, request) + return await controller.node_done(federation_id, request) else: return {"message": "Experyment type not allowed on responde for Node done message.."} diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index e75d4575c..317f2061b 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -26,9 +26,9 @@ async def stop_scenario(self, federation_id: str): pass @abstractmethod - async def update_nodes(self, scenario_name: str, request: Request): + async def update_nodes(self, federation_id: str, request: Request): pass abstractmethod - async def node_done(self, scenario_name: str, request: Request): + async def node_done(self, federation_id: str, request: Request): pass \ No newline at end of file From e9d27602fbe3dd7eb622abc6e0e08df088bc08bf Mon Sep 17 00:00:00 2001 From: "Alejandro.A.S" Date: Thu, 18 Sep 2025 12:43:20 +0200 Subject: [PATCH 26/26] feature node update-done format --- nebula/addons/reporter.py | 20 ++++++++++---- .../docker_federation_controller.py | 16 +++++------- .../processes_federation_controller.py | 16 +++++------- .../controller/federation/federation_api.py | 26 +++++++++---------- .../federation/federation_controller.py | 5 ++-- .../controller/federation/utils_requests.py | 9 +++++++ 6 files changed, 53 insertions(+), 39 deletions(-) diff --git a/nebula/addons/reporter.py b/nebula/addons/reporter.py index 5db64f16f..5b3952519 100755 --- a/nebula/addons/reporter.py +++ b/nebula/addons/reporter.py @@ -4,6 +4,7 @@ import logging import os import sys +from nebula.controller.federation.utils_requests import NodeUpdateRequest, NodeDoneRequest from typing import TYPE_CHECKING import aiohttp @@ -171,10 +172,17 @@ async def report_scenario_finished(self): - Logs exceptions if the connection attempt to the controller fails. """ url = f"http://{self.config.participant['scenario_args']['controller']}/nodes/{self.config.participant['scenario_args']['federation_id']}/done" - data = json.dumps({"idx": self.config.participant["device_args"]["idx"], - "deployment": self.config.participant["scenario_args"]["deployment"], - "name": self.config.participant["scenario_args"]["name"], - "federation_id": self.config.participant["scenario_args"]["federation_id"]}) + node_done_req = NodeDoneRequest(idx=self.config.participant["device_args"]["idx"], + deployment=self.config.participant["scenario_args"]["deployment"], + name=self.config.participant["scenario_args"]["name"], + federation_id=self.config.participant["scenario_args"]["federation_id"] + ) + payload = node_done_req.model_dump() + data = json.dumps(payload) + # data = json.dumps({"idx": self.config.participant["device_args"]["idx"], + # "deployment": self.config.participant["scenario_args"]["deployment"], + # "name": self.config.participant["scenario_args"]["name"], + # "federation_id": self.config.participant["scenario_args"]["federation_id"]}) headers = { "Content-Type": "application/json", "User-Agent": f"NEBULA Participant {self.config.participant['device_args']['idx']}", @@ -266,11 +274,13 @@ async def __report_status_to_controller(self): - Delays for 5 seconds upon general exceptions to avoid rapid retry loops. """ try: + node_updt_req = NodeUpdateRequest(config=self.config.participant) + payload = node_updt_req.model_dump() async with ( aiohttp.ClientSession() as session, session.post( self.url, - data=json.dumps(self.config.participant), + data=json.dumps(payload), headers={ "Content-Type": "application/json", "User-Agent": f"NEBULA Participant {self.config.participant['device_args']['idx']}", diff --git a/nebula/controller/federation/controllers/docker_federation_controller.py b/nebula/controller/federation/controllers/docker_federation_controller.py index 5be7934d8..64ea123a4 100644 --- a/nebula/controller/federation/controllers/docker_federation_controller.py +++ b/nebula/controller/federation/controllers/docker_federation_controller.py @@ -8,6 +8,7 @@ from nebula.controller.federation.federation_controller import FederationController from nebula.controller.federation.scenario_builder import ScenarioBuilder from nebula.controller.federation.utils_requests import factory_requests_path +from nebula.controller.federation.utils_requests import NodeUpdateRequest, NodeDoneRequest from typing import Dict from fastapi import Request from nebula.config.config import Config @@ -200,8 +201,8 @@ async def stop_scenario(self, federation_id: str): return True #TODO care about cases - async def update_nodes(self, federation_id: str, request: Request): - config = await request.json() + async def update_nodes(self, federation_id: str, node_update_request: NodeUpdateRequest): + config = node_update_request.config scenario_name = config["scenario_args"]["name"] fed_id = config["scenario_args"]["federation_id"] @@ -229,27 +230,24 @@ async def update_nodes(self, federation_id: str, request: Request): nebula_federation.last_index_deployed += 1 #additionals.remove(index) adds_deployed.add(index) - request_body = await request.json() - payload = {"scenario_name": scenario_name, "data": request_body} + payload = node_update_request.model_dump() asyncio.create_task(self._send_to_hub("update", payload, federation_id=fed_id)) return {"message": "Node updated successfully in Federation Controller"} except Exception as e: self.logger.info(f"ERROR: federation ID: ({fed_id}), {e}") return {"message": "Node updated failed in Federation Controller"} - async def node_done(self, federation_id: str, request: Request): - request_body = await request.json() - scenario_name = request_body["scenario_args"]["name"] + async def node_done(self, federation_id: str, node_done_request: NodeDoneRequest): nebula_federation = self.nfp[federation_id] self.logger.info(f"Node-Done received from node on federation ID: ({federation_id})") if await nebula_federation.is_experiment_finish(): - payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} + payload = node_done_request.model_dump() self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") await self._remove_nebula_federation_from_pool(federation_id) asyncio.create_task(self._send_to_hub("finish", payload, federation_id=federation_id)) - payload = {"scenario_name": scenario_name, "data": request_body} + payload = node_done_request.model_dump() asyncio.create_task(self._send_to_hub("done", payload, federation_id=federation_id)) return {"message": "Nodes done received successfully"} diff --git a/nebula/controller/federation/controllers/processes_federation_controller.py b/nebula/controller/federation/controllers/processes_federation_controller.py index 7a3c35d87..cd5c36121 100644 --- a/nebula/controller/federation/controllers/processes_federation_controller.py +++ b/nebula/controller/federation/controllers/processes_federation_controller.py @@ -8,6 +8,7 @@ from nebula.controller.federation.federation_controller import FederationController from nebula.controller.federation.scenario_builder import ScenarioBuilder from nebula.controller.federation.utils_requests import factory_requests_path +from nebula.controller.federation.utils_requests import NodeUpdateRequest, NodeDoneRequest from typing import Dict from fastapi import Request from nebula.config.config import Config @@ -161,8 +162,8 @@ async def stop_scenario(self, federation_id: str = ""): self.logger.exception(f"Error while removing current_scenario_commands.sh file: {e}") return False - async def update_nodes(self, federation_id: str, request: Request): - config = await request.json() + async def update_nodes(self, federation_id: str, node_update_request: NodeUpdateRequest): + config = node_update_request.config fed_id = config["scenario_args"]["federation_id"] scenario_name = config["scenario_args"]["name"] @@ -189,27 +190,24 @@ async def update_nodes(self, federation_id: str, request: Request): nebula_federation.last_index_deployed += 1 additionals.remove(index) adds_deployed.add(index) - request_body = await request.json() - payload = {"scenario_name": scenario_name, "data": request_body} + payload = node_update_request.model_dump() asyncio.create_task(self._send_to_hub("update", payload, federation_id=fed_id)) return {"message": "Node updated successfully in Federation Controller"} except Exception as e: self.logger.info(f"ERROR: federation ID: ({fed_id}) not found on pool..") return {"message": "Node updated failed in Federation Controller, ID not found.."} - async def node_done(self, federation_id: str, request: Request): - request_body = await request.json() - scenario_name = request_body["scenario_args"]["name"] + async def node_done(self, federation_id: str, node_done_request: NodeDoneRequest): nebula_federation = self.nfp[federation_id] self.logger.info(f"Node-Done received from node on federation ID: ({federation_id})") if await nebula_federation.is_experiment_finish(): - payload = {"federation_id": federation_id, "scenario_name": scenario_name, "data": request_body} + payload = node_done_request.model_dump() self.logger.info(f"All nodes have finished on federation ID: ({federation_id}), reporting to hub..") await self._remove_nebula_federation_from_pool(federation_id) asyncio.create_task(self._send_to_hub("finish", payload, federation_id=federation_id)) - payload = {"scenario_name": scenario_name, "data": request_body} + payload = node_done_request.model_dump() asyncio.create_task(self._send_to_hub("done", payload, federation_id=federation_id)) return {"message": "Nodes done received successfully"} diff --git a/nebula/controller/federation/federation_api.py b/nebula/controller/federation/federation_api.py index 7bbea89e5..746696c4c 100644 --- a/nebula/controller/federation/federation_api.py +++ b/nebula/controller/federation/federation_api.py @@ -10,7 +10,7 @@ from nebula.utils import LoggerUtils from nebula.controller.federation.federation_controller import FederationController from nebula.controller.federation.factory_federation_controller import federation_controller_factory -from nebula.controller.federation.utils_requests import RunScenarioRequest, StopScenarioRequest, Routes +from nebula.controller.federation.utils_requests import RunScenarioRequest, StopScenarioRequest, NodeUpdateRequest, NodeDoneRequest, Routes fed_controllers: Dict[str, FederationController] = {} @@ -61,7 +61,7 @@ async def run_scenario(run_scenario_request: RunScenarioRequest): if controller: return await controller.run_scenario(run_scenario_request.federation_id, run_scenario_request.scenario_data, run_scenario_request.user) else: - return {"message": "Experyment type not allowed"} + return {"message": "Experiment type not allowed"} @app.post(Routes.STOP) async def stop_scenario(stop_scenario_request: StopScenarioRequest): @@ -73,35 +73,33 @@ async def stop_scenario(stop_scenario_request: StopScenarioRequest): if controller: return await controller.stop_scenario(stop_scenario_request.federation_id) else: - return {"message": "Experyment type not allowed"} + return {"message": "Experiment type not allowed"} @app.post(Routes.UPDATE) async def update_nodes( federation_id: str, - request: Request, + node_update_request: NodeUpdateRequest, ): global fed_controllers - config = await request.json() - experiment_type = config["scenario_args"]["deployment"] + experiment_type = node_update_request.config["scenario_args"]["deployment"] controller = fed_controllers.get(experiment_type, None) if controller: - return await controller.update_nodes(federation_id, request) + return await controller.update_nodes(federation_id, node_update_request) else: - return {"message": "Experyment type not allowed on response for update message.."} + return {"message": "Experiment type not allowed on response for update message.."} @app.post(Routes.DONE) -async def update_nodes( +async def node_done( federation_id: str, - request: Request, + node_done_request: NodeDoneRequest, ): global fed_controllers - config = await request.json() - experiment_type = config["deployment"] + experiment_type = node_done_request.deployment controller = fed_controllers.get(experiment_type, None) if controller: - return await controller.node_done(federation_id, request) + return await controller.node_done(federation_id, node_done_request) else: - return {"message": "Experyment type not allowed on responde for Node done message.."} + return {"message": "Experiment type not allowed on responde for Node done message.."} if __name__ == "__main__": # Parse args from command line diff --git a/nebula/controller/federation/federation_controller.py b/nebula/controller/federation/federation_controller.py index 317f2061b..99bdda0bc 100644 --- a/nebula/controller/federation/federation_controller.py +++ b/nebula/controller/federation/federation_controller.py @@ -2,6 +2,7 @@ from fastapi import Request from typing import Dict from nebula.controller.federation.scenario_builder import ScenarioBuilder +from nebula.controller.federation.utils_requests import NodeUpdateRequest, NodeDoneRequest import logging class NebulaFederation(ABC): @@ -26,9 +27,9 @@ async def stop_scenario(self, federation_id: str): pass @abstractmethod - async def update_nodes(self, federation_id: str, request: Request): + async def update_nodes(self, federation_id: str, node_update_request: NodeUpdateRequest): pass abstractmethod - async def node_done(self, federation_id: str, request: Request): + async def node_done(self, federation_id: str, node_done_request: NodeDoneRequest): pass \ No newline at end of file diff --git a/nebula/controller/federation/utils_requests.py b/nebula/controller/federation/utils_requests.py index db861a57a..e501b3377 100644 --- a/nebula/controller/federation/utils_requests.py +++ b/nebula/controller/federation/utils_requests.py @@ -10,6 +10,15 @@ class StopScenarioRequest(BaseModel): experiment_type: str federation_id: str +class NodeUpdateRequest(BaseModel): + config: Dict[str, Any] = {} + +class NodeDoneRequest(BaseModel): + idx: int + deployment: str + name: str + federation_id: str + class Routes: INIT = "/init" RUN = "/scenarios/run"