From 33cb982f996ee0a1e3f63732029bf105bf5f5646 Mon Sep 17 00:00:00 2001 From: zTz01 <1773266173@qq.com> Date: Fri, 7 Aug 2026 19:19:26 +0800 Subject: [PATCH 1/3] fix: add fluxon_py quick start entry --- fluxon_py/quick_start.py | 622 +++++++++++++++++++++++++ fluxon_py/runtime/process_runner.py | 18 + fluxon_py/tests/test_process_runner.py | 9 + fluxon_py/tests/test_quick_start.py | 99 ++++ 4 files changed, 748 insertions(+) create mode 100644 fluxon_py/quick_start.py create mode 100644 fluxon_py/tests/test_quick_start.py diff --git a/fluxon_py/quick_start.py b/fluxon_py/quick_start.py new file mode 100644 index 0000000..2606089 --- /dev/null +++ b/fluxon_py/quick_start.py @@ -0,0 +1,622 @@ +"""Compatibility quick-start helpers for installed `fluxon_py` packages.""" + +from __future__ import annotations + +from collections.abc import Mapping +import copy +import os +import re +import socket +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from fluxon_py.config import _to_plain_yaml_obj +from fluxon_py.runtime import ( + start_fs_agent_process, + start_fs_master_process, + start_kv_master_process, + start_owner_kvclient_process, +) +from fluxon_py.runtime.process_runner import ManagedSubprocess, wait_subproc_or_ctrlc + +__all__ = ["serve_s3_single_node"] + +_DEFAULT_PANEL_PORT = 26180 +_DEFAULT_EXPORT_NAME = "quick-start-export" +_DEFAULT_CACHE_MAX_BYTES = 1024 * 1024 * 1024 +_DEFAULT_CLUSTER_NAME = "fluxon_s3" +_DEFAULT_FS_MASTER_INSTANCE_KEY = "fluxon_s3_fs_master" +_DEFAULT_FS_AGENT_INSTANCE_KEY = "fluxon_s3_fs_agent" +_DEFAULT_KV_MASTER_INSTANCE_KEY = "fluxon_s3_master" +_DEFAULT_KV_OWNER_INSTANCE_KEY = "fluxon_s3_owner" +_DEFAULT_SHARE_MEM_DIRNAME = "sharemem" +_DEFAULT_FS_MASTER_LOG_DIRNAME = "kv-master" +_DEFAULT_FS_OWNER_LOG_DIRNAME = "kv-owner" +_DEFAULT_ACCESS_DB_RELATIVE_PATH = Path("fs_master") / "access.db" +_DEFAULT_PY_REACTOR_MODE = "event_driven" +_EXPORT_NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])?$") + + +@dataclass(frozen=True) +class _S3SingleNodeBundle: + data_root: Path + state_root: Path + kv_master_config: dict[str, Any] + kv_owner_config: dict[str, Any] + fs_master_config: dict[str, Any] + fs_agent_config: dict[str, Any] + kv_master_config_path: Path + kv_owner_config_path: Path + fs_master_config_path: Path + fs_agent_config_path: Path + kv_master_workdir: Path + kv_owner_workdir: Path + fs_master_workdir: Path + fs_agent_workdir: Path + share_mem_path: Path + access_db_path: Path + panel_port: int + panel_public_base_url: str + export_name: str + + @property + def s3_endpoint(self) -> str: + return f"{self.panel_public_base_url}/fs_s3" + + @property + def s3_ui_url(self) -> str: + return f"{self.s3_endpoint}/ui/" + + +def serve_s3_single_node( + data_dir: str | os.PathLike[str], + state_dir: str | os.PathLike[str], + *, + kv_master_config: Mapping[str, Any], + kv_owner_config: Mapping[str, Any], + export_name: str = _DEFAULT_EXPORT_NAME, + start_middleware: bool = False, + greptime_base_url: str | None = None, + panel_port: int = _DEFAULT_PANEL_PORT, + panel_listen_host: str = "0.0.0.0", + bootstrap_username: str = "admin", + bootstrap_password: str = "admin", + export_cache_max_bytes: int = _DEFAULT_CACHE_MAX_BYTES, +) -> None: + if start_middleware: + raise NotImplementedError("quick_start only supports start_middleware=False") + + bundle = _build_s3_single_node_bundle( + data_dir=data_dir, + state_dir=state_dir, + kv_master_config=kv_master_config, + kv_owner_config=kv_owner_config, + export_name=export_name, + greptime_base_url=greptime_base_url, + panel_port=panel_port, + panel_listen_host=panel_listen_host, + bootstrap_username=bootstrap_username, + bootstrap_password=bootstrap_password, + export_cache_max_bytes=export_cache_max_bytes, + ) + + _prepare_runtime_dirs(bundle) + + children: list[ManagedSubprocess] = [] + started = False + try: + print("[fluxon_quick_start] starting kv master...") + kv_master_proc = start_kv_master_process( + workdir=bundle.kv_master_workdir, + config_path=bundle.kv_master_config_path, + log_path=bundle.state_root / "log" / "kv_master.log", + ) + children.append(ManagedSubprocess(label="kv_master", proc=kv_master_proc)) + _wait_for_process_alive(kv_master_proc, label="kv_master", seconds=10, log_path=bundle.state_root / "log" / "kv_master.log") + + print("[fluxon_quick_start] starting owner kvclient...") + _clear_stale_shared_json(bundle.share_mem_path, bundle.kv_owner_config["fluxonkv_spec"]["cluster_name"]) + owner_proc = start_owner_kvclient_process( + workdir=bundle.kv_owner_workdir, + config_path=bundle.kv_owner_config_path, + log_path=bundle.state_root / "log" / "kv_owner.log", + ) + children.append(ManagedSubprocess(label="kv_owner", proc=owner_proc)) + _wait_for_shared_json( + share_mem_path=bundle.share_mem_path, + cluster_name=bundle.kv_owner_config["fluxonkv_spec"]["cluster_name"], + proc=owner_proc, + label="kv_owner", + log_path=bundle.state_root / "log" / "kv_owner.log", + ) + + print("[fluxon_quick_start] starting fluxon_fs master...") + fs_master_proc = start_fs_master_process( + workdir=bundle.fs_master_workdir, + config_path=bundle.fs_master_config_path, + log_path=bundle.state_root / "log" / "fs_master.log", + ) + children.append(ManagedSubprocess(label="fs_master", proc=fs_master_proc)) + _wait_for_tcp_ready( + fs_master_proc, + label="fs_master", + host="127.0.0.1", + port=bundle.panel_port, + timeout=30, + log_path=bundle.state_root / "log" / "fs_master.log", + ) + + print("[fluxon_quick_start] starting fluxon_fs agent...") + fs_agent_proc = start_fs_agent_process( + workdir=bundle.fs_agent_workdir, + config_path=bundle.fs_agent_config_path, + log_path=bundle.state_root / "log" / "fs_agent.log", + ) + children.append(ManagedSubprocess(label="fs_agent", proc=fs_agent_proc)) + _wait_for_log_text( + bundle.state_root / "log" / "fs_agent.log", + "fluxon_fs agent ready", + proc=fs_agent_proc, + label="fs_agent", + ) + + print() + print(f"S3 endpoint: {bundle.s3_endpoint}") + print(f"Web UI: {bundle.s3_ui_url}") + print(f"bucket: {bundle.export_name}") + print(f"Basic Auth: {bootstrap_username} / {bootstrap_password}") + print(f"data dir: {bundle.data_root}") + print(f"state dir: {bundle.state_root}") + + started = True + wait_subproc_or_ctrlc(children, stop_timeout_seconds=5.0) + finally: + if not started: + _terminate_children(children) + + +def _build_s3_single_node_bundle( + *, + data_dir: str | os.PathLike[str], + state_dir: str | os.PathLike[str], + kv_master_config: Mapping[str, Any], + kv_owner_config: Mapping[str, Any], + export_name: str, + greptime_base_url: str | None, + panel_port: int, + panel_listen_host: str, + bootstrap_username: str, + bootstrap_password: str, + export_cache_max_bytes: int, +) -> _S3SingleNodeBundle: + if panel_port <= 0: + raise ValueError("panel_port must be > 0") + if export_cache_max_bytes <= 0: + raise ValueError("export_cache_max_bytes must be > 0") + if not bootstrap_username.strip(): + raise ValueError("bootstrap_username must be non-empty") + if not bootstrap_password.strip(): + raise ValueError("bootstrap_password must be non-empty") + if not panel_listen_host.strip(): + raise ValueError("panel_listen_host must be non-empty") + _validate_export_name(export_name) + + data_root = Path(data_dir).expanduser().resolve() + state_root = Path(state_dir).expanduser().resolve() + data_root.mkdir(parents=True, exist_ok=True) + state_root.mkdir(parents=True, exist_ok=True) + + kv_master = _plain_mapping_copy(kv_master_config, "kv_master_config") + kv_owner = _plain_mapping_copy(kv_owner_config, "kv_owner_config") + + master_cluster_name = _require_str( + kv_master.get("cluster_name") or _DEFAULT_CLUSTER_NAME, + "kv_master_config.cluster_name", + ) + _ensure_master_defaults(kv_master, state_root=state_root, greptime_base_url=greptime_base_url) + + owner_spec = _require_mapping(kv_owner.get("fluxonkv_spec"), "kv_owner_config.fluxonkv_spec") + owner_cluster_name = owner_spec.get("cluster_name") + if owner_cluster_name is None: + owner_spec["cluster_name"] = master_cluster_name + else: + owner_cluster_name = _require_str(owner_cluster_name, "kv_owner_config.fluxonkv_spec.cluster_name") + if owner_cluster_name != master_cluster_name: + raise ValueError( + "kv_owner_config.fluxonkv_spec.cluster_name must match kv_master_config.cluster_name" + ) + + etcd_endpoints = kv_master.get("etcd_endpoints") + if not isinstance(etcd_endpoints, list) or not etcd_endpoints: + raise ValueError("kv_master_config.etcd_endpoints must be a non-empty list") + normalized_etcd_endpoints = [_require_str(endpoint, "kv_master_config.etcd_endpoints[]") for endpoint in etcd_endpoints] + + share_mem_path = Path( + owner_spec.get("share_mem_path") + or (state_root / _DEFAULT_SHARE_MEM_DIRNAME) + ).expanduser().resolve() + owner_spec["share_mem_path"] = str(share_mem_path) + owner_spec.setdefault("sub_cluster", "default") + owner_spec.setdefault("large_file_paths", [str(state_root / "kv-owner" / "large")]) + owner_spec.setdefault("etcd_addresses", list(normalized_etcd_endpoints)) + _ensure_owner_defaults(kv_owner, master_cluster_name=master_cluster_name) + + panel_public_base_url = f"http://127.0.0.1:{panel_port}" + prometheus_base_url = _resolve_prometheus_base_url( + greptime_base_url=greptime_base_url, + kv_master_config=kv_master, + ) + access_db_path = (state_root / _DEFAULT_ACCESS_DB_RELATIVE_PATH).resolve() + + fs_master_instance_key = str( + kv_master.get("fs_master_instance_key") + or f"{master_cluster_name}_fs_master" + or _DEFAULT_FS_MASTER_INSTANCE_KEY + ) + fs_agent_instance_key = str( + kv_master.get("fs_agent_instance_key") + or f"{master_cluster_name}_fs_agent" + or _DEFAULT_FS_AGENT_INSTANCE_KEY + ) + + fs_master_config = { + "kvclient": _build_external_kvclient_config( + instance_key=fs_master_instance_key, + cluster_name=master_cluster_name, + share_mem_path=share_mem_path, + ), + "fluxon_fs": { + "master": { + "instance_key": fs_master_instance_key, + "pull_interval_ms": 1000, + }, + "master_panel": { + "listen_addr": f"{panel_listen_host}:{panel_port}", + "public_base_url": panel_public_base_url, + "prometheus_base_url": prometheus_base_url, + "auto_refresh_interval_secs": 2, + "access_db_path": str(access_db_path), + "bootstrap_access_model": { + "users": [ + { + "username": bootstrap_username, + "password": bootstrap_password, + "can_manage_users": True, + } + ], + "scope_access": [], + }, + "s3_gateway": { + "get_object_inflight_pieces": 8, + "kv_miss_policy": "remote_read", + }, + }, + "cache": { + "stale_window_ms": 1000, + "rules": [], + "exports": { + export_name: { + "remote_root_dir_abs": str(data_root), + "cache_max_bytes": export_cache_max_bytes, + } + }, + }, + }, + } + + fs_agent_config = { + "kvclient": _build_external_kvclient_config( + instance_key=fs_agent_instance_key, + cluster_name=master_cluster_name, + share_mem_path=share_mem_path, + ), + "fluxon_fs": { + "master": { + "instance_key": fs_master_instance_key, + }, + "cache": { + "stale_window_ms": 1000, + "rules": [], + "exports": { + export_name: { + "remote_root_dir_abs": str(data_root), + "cache_max_bytes": export_cache_max_bytes, + } + }, + }, + }, + } + + kv_master_config = dict(kv_master) + kv_owner_config = dict(kv_owner) + + kv_master_workdir = state_root / _DEFAULT_FS_MASTER_LOG_DIRNAME + kv_owner_workdir = state_root / _DEFAULT_FS_OWNER_LOG_DIRNAME + fs_master_workdir = state_root / "fs_master_runtime" + fs_agent_workdir = state_root / "fs_agent_runtime" + + kv_master_config_path = kv_master_workdir / "config.yaml" + kv_owner_config_path = kv_owner_workdir / "config.yaml" + fs_master_config_path = fs_master_workdir / "config.yaml" + fs_agent_config_path = fs_agent_workdir / "config.yaml" + + return _S3SingleNodeBundle( + data_root=data_root, + state_root=state_root, + kv_master_config=kv_master_config, + kv_owner_config=kv_owner_config, + fs_master_config=fs_master_config, + fs_agent_config=fs_agent_config, + kv_master_config_path=kv_master_config_path, + kv_owner_config_path=kv_owner_config_path, + fs_master_config_path=fs_master_config_path, + fs_agent_config_path=fs_agent_config_path, + kv_master_workdir=kv_master_workdir, + kv_owner_workdir=kv_owner_workdir, + fs_master_workdir=fs_master_workdir, + fs_agent_workdir=fs_agent_workdir, + share_mem_path=share_mem_path, + access_db_path=access_db_path, + panel_port=panel_port, + panel_public_base_url=panel_public_base_url, + export_name=export_name, + ) + + +def _ensure_master_defaults( + kv_master: dict[str, Any], + *, + state_root: Path, + greptime_base_url: str | None, +) -> None: + kv_master.setdefault("cluster_name", _DEFAULT_CLUSTER_NAME) + kv_master.setdefault("instance_key", _DEFAULT_KV_MASTER_INSTANCE_KEY) + kv_master.setdefault("port", 25100) + kv_master.setdefault("log_dir", str(state_root / "kv-master" / "log")) + kv_master.setdefault("network", {"tcp_reactor_mode": _DEFAULT_PY_REACTOR_MODE}) + if "monitoring" not in kv_master: + kv_master["monitoring"] = _build_monitoring_block(greptime_base_url) + + +def _ensure_owner_defaults(kv_owner: dict[str, Any], *, master_cluster_name: str) -> None: + kv_owner.setdefault("instance_key", _DEFAULT_KV_OWNER_INSTANCE_KEY) + kv_owner.setdefault("contribute_to_cluster_pool_size", {"dram": 1024 * 1024 * 1024, "vram": {}}) + kv_owner.setdefault("network", {"tcp_reactor_mode": _DEFAULT_PY_REACTOR_MODE}) + owner_spec = _require_mapping(kv_owner.get("fluxonkv_spec"), "kv_owner_config.fluxonkv_spec") + owner_spec.setdefault("cluster_name", master_cluster_name) + + +def _build_external_kvclient_config( + *, + instance_key: str, + cluster_name: str, + share_mem_path: Path, +) -> dict[str, Any]: + return { + "instance_key": instance_key, + "network": {"tcp_reactor_mode": _DEFAULT_PY_REACTOR_MODE}, + "fluxonkv_spec": { + "cluster_name": cluster_name, + "share_mem_path": str(share_mem_path), + }, + } + + +def _build_monitoring_block(greptime_base_url: str | None) -> dict[str, Any]: + base_url = _resolve_greptime_base_url(greptime_base_url) + return { + "prometheus_base_url": f"{base_url}/v1/prometheus", + "prom_remote_write_url": [f"{base_url}/v1/prometheus/write"], + "otlp_log_api": { + "otlp_endpoint": f"{base_url}/v1/otlp/v1/logs", + "db_name": "public", + "table_name": "fluxon_logs", + }, + } + + +def _resolve_prometheus_base_url(*, greptime_base_url: str | None, kv_master_config: dict[str, Any]) -> str: + if greptime_base_url: + return f"{greptime_base_url.rstrip('/')}/v1/prometheus" + monitoring = kv_master_config.get("monitoring") + if isinstance(monitoring, Mapping): + prometheus_base_url = monitoring.get("prometheus_base_url") + if isinstance(prometheus_base_url, str) and prometheus_base_url.strip(): + return prometheus_base_url + return "http://127.0.0.1:24000/v1/prometheus" + + +def _resolve_greptime_base_url(greptime_base_url: str | None) -> str: + if greptime_base_url: + return greptime_base_url.rstrip("/") + return "http://127.0.0.1:24000" + + +def _plain_mapping_copy(value: Mapping[str, Any], name: str) -> dict[str, Any]: + plain = _to_plain_yaml_obj(value, name) + if not isinstance(plain, dict): + raise TypeError(f"{name} must decode to a mapping") + return copy.deepcopy(plain) + + +def _require_mapping(value: Any, name: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise TypeError(f"{name} must be a mapping") + return value + + +def _require_str(value: Any, name: str) -> str: + if not isinstance(value, str): + raise TypeError(f"{name} must be a string") + stripped = value.strip() + if not stripped: + raise ValueError(f"{name} must be non-empty") + return stripped + + +def _validate_export_name(export_name: str) -> None: + if not _EXPORT_NAME_RE.fullmatch(export_name): + raise ValueError( + "export_name must match ^[a-z0-9](?:[a-z0-9-]{1,61}[a-z0-9])?$" + ) + + +def _prepare_runtime_dirs(bundle: _S3SingleNodeBundle) -> None: + for path in ( + bundle.state_root / "log", + bundle.kv_master_workdir, + bundle.kv_master_workdir / "log", + bundle.kv_owner_workdir, + bundle.fs_master_workdir, + bundle.fs_agent_workdir, + bundle.share_mem_path, + bundle.access_db_path.parent, + bundle.state_root / "kv-owner" / "large", + ): + path.mkdir(parents=True, exist_ok=True) + _write_yaml(bundle.kv_master_config_path, bundle.kv_master_config) + _write_yaml(bundle.kv_owner_config_path, bundle.kv_owner_config) + _write_yaml(bundle.fs_master_config_path, bundle.fs_master_config) + _write_yaml(bundle.fs_agent_config_path, bundle.fs_agent_config) + + +def _write_yaml(path: Path, value: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + plain = _to_plain_yaml_obj(value, str(path)) + path.write_text(yaml.safe_dump(plain, sort_keys=False), encoding="utf-8") + + +def _clear_stale_shared_json(share_mem_path: Path, cluster_name: str) -> None: + target = share_mem_path / cluster_name / "shared.json" + if target.exists(): + target.unlink() + + +def _wait_for_shared_json( + *, + share_mem_path: Path, + cluster_name: str, + timeout: int = 180, + proc: subprocess.Popen[bytes] | None = None, + label: str = "owner", + log_path: Path | None = None, +) -> None: + target = share_mem_path / cluster_name / "shared.json" + deadline = time.time() + timeout + while time.time() < deadline: + _raise_if_process_exited(proc, label=label, log_path=log_path) + if target.exists(): + return + time.sleep(0.5) + _raise_if_process_exited(proc, label=label, log_path=log_path) + raise RuntimeError(f"{label} did not create shared.json under {target.parent} within {timeout}s") + + +def _wait_for_tcp_ready( + proc: subprocess.Popen[bytes], + *, + label: str, + host: str, + port: int, + timeout: int, + log_path: Path | None = None, +) -> None: + probe_host = "127.0.0.1" if host in {"0.0.0.0", "::", "[::]"} else host + deadline = time.time() + timeout + while time.time() < deadline: + _raise_if_process_exited(proc, label=label, log_path=log_path) + try: + with socket.create_connection((probe_host, port), timeout=1): + return + except OSError: + time.sleep(0.5) + _raise_if_process_exited(proc, label=label, log_path=log_path) + raise RuntimeError(f"{label} did not open {probe_host}:{port} within {timeout}s") + + +def _wait_for_log_text( + log_path: Path, + needle: str, + *, + proc: subprocess.Popen[bytes] | None = None, + label: str = "process", + timeout: int = 60, +) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + _raise_if_process_exited(proc, label=label, log_path=log_path) + if log_path.exists(): + try: + text = log_path.read_text(encoding="utf-8", errors="replace") + except OSError: + text = "" + if needle in text: + return + time.sleep(0.5) + _raise_if_process_exited(proc, label=label, log_path=log_path) + raise RuntimeError(f"{label} log did not contain {needle!r} within {timeout}s: {log_path}") + + +def _wait_for_process_alive( + proc: subprocess.Popen[bytes], + *, + label: str, + seconds: int, + log_path: Path | None = None, +) -> None: + deadline = time.time() + seconds + while time.time() < deadline: + _raise_if_process_exited(proc, label=label, log_path=log_path) + time.sleep(0.5) + _raise_if_process_exited(proc, label=label, log_path=log_path) + + +def _raise_if_process_exited( + proc: subprocess.Popen[bytes] | None, + *, + label: str, + log_path: Path | None = None, +) -> None: + if proc is None: + return + rc = proc.poll() + if rc is None: + return + detail = f"{label} exited unexpectedly with rc={rc}" + if log_path is not None and log_path.exists(): + detail += f"; log tail:\n{_tail_text(log_path)}" + raise RuntimeError(detail) + + +def _tail_text(path: Path, limit: int = 4000) -> str: + try: + data = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + if len(data) <= limit: + return data + return data[-limit:] + + +def _terminate_children(children: list[ManagedSubprocess], timeout_seconds: float = 5.0) -> None: + for child in reversed(children): + if child.proc.poll() is None: + try: + child.proc.terminate() + except Exception: + pass + deadline = time.time() + timeout_seconds + while time.time() < deadline: + if all(child.proc.poll() is not None for child in children): + return + time.sleep(0.2) + for child in reversed(children): + if child.proc.poll() is None: + try: + child.proc.kill() + except Exception: + pass diff --git a/fluxon_py/runtime/process_runner.py b/fluxon_py/runtime/process_runner.py index 0421fff..2cefa78 100644 --- a/fluxon_py/runtime/process_runner.py +++ b/fluxon_py/runtime/process_runner.py @@ -22,6 +22,7 @@ RuntimeConfigInput = Path | Mapping[str, Any] FORCE_KILL_WAIT_SECONDS = 10.0 +_PACKAGE_ROOT = Path(__file__).resolve().parents[2] @dataclass(frozen=True) @@ -517,6 +518,7 @@ def _start_runtime_process( ) -> subprocess.Popen[bytes]: popen_kwargs: dict[str, Any] = { "preexec_fn": build_parent_death_sigterm_preexec(expected_parent_pid=os.getpid()), + "env": _build_runtime_subprocess_env(), } if cwd is not None: popen_kwargs["cwd"] = str(cwd) @@ -538,6 +540,22 @@ def _start_runtime_process( return proc +def _build_runtime_subprocess_env() -> dict[str, str]: + env = os.environ.copy() + existing_pythonpath = env.get("PYTHONPATH", "") + package_root = str(_PACKAGE_ROOT) + pythonpath_parts: list[str] = [package_root] + if existing_pythonpath: + for entry in existing_pythonpath.split(os.pathsep): + stripped = entry.strip() + if not stripped or stripped == package_root: + continue + pythonpath_parts.append(stripped) + env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts) + env.setdefault("PYTHONUNBUFFERED", "1") + return env + + def _set_parent_death_sigterm(*, expected_parent_pid: int) -> None: # Keep this even in the attached parent/child model: # - A plain attached child does not die automatically when the parent is diff --git a/fluxon_py/tests/test_process_runner.py b/fluxon_py/tests/test_process_runner.py index ae9dbef..7c01c83 100644 --- a/fluxon_py/tests/test_process_runner.py +++ b/fluxon_py/tests/test_process_runner.py @@ -10,6 +10,7 @@ import threading import time import unittest +from unittest import mock from pathlib import Path @@ -22,6 +23,7 @@ def main() -> None: from fluxon_py.runtime.process_runner import ( # noqa: E402 + _build_runtime_subprocess_env, build_runtime_singleton_spec, register_ctrlc_callback, _stop_existing_processes_if_running, @@ -64,6 +66,13 @@ def setUp(self) -> None: self._tmp = _new_test_dir("process_runner") self.addCleanup(lambda: shutil.rmtree(self._tmp, ignore_errors=False)) + def test_build_runtime_subprocess_env_prepends_repo_root(self) -> None: + with mock.patch.dict(os.environ, {"PYTHONPATH": "/tmp/custom"}, clear=True): + env = _build_runtime_subprocess_env() + + self.assertEqual(env["PYTHONPATH"], f"{REPO_ROOT}:/tmp/custom") + self.assertEqual(env["PYTHONUNBUFFERED"], "1") + def test_register_ctrlc_callback_runs_outside_signal_frame(self) -> None: script_path = self._tmp / "ctrlc_callback.py" marker_path = self._tmp / "ctrlc_callback.txt" diff --git a/fluxon_py/tests/test_quick_start.py b/fluxon_py/tests/test_quick_start.py new file mode 100644 index 0000000..7cc9ba0 --- /dev/null +++ b/fluxon_py/tests/test_quick_start.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import importlib +import tempfile +from pathlib import Path +import unittest + + +class QuickStartCompatTest(unittest.TestCase): + def test_import_quick_start_module(self) -> None: + module = importlib.import_module("fluxon_py.quick_start") + self.assertTrue(hasattr(module, "serve_s3_single_node")) + + def test_build_s3_single_node_bundle_uses_expected_paths(self) -> None: + module = importlib.import_module("fluxon_py.quick_start") + + with tempfile.TemporaryDirectory() as td: + root = Path(td) + data_dir = root / "data" + state_dir = root / "state" + data_dir.mkdir() + state_dir.mkdir() + + kv_master_config = { + "etcd_endpoints": ["127.0.0.1:22379"], + "cluster_name": "fluxon_s3", + "instance_key": "fluxon_s3_master", + "port": 25100, + "log_dir": "/tmp/unused", + "monitoring": { + "prometheus_base_url": "http://127.0.0.1:24000/v1/prometheus", + "prom_remote_write_url": ["http://127.0.0.1:24000/v1/prometheus/write"], + "otlp_log_api": { + "otlp_endpoint": "http://127.0.0.1:24000/v1/otlp/v1/logs", + "db_name": "public", + "table_name": "fluxon_logs", + }, + }, + } + kv_owner_config = { + "instance_key": "fluxon_s3_owner", + "contribute_to_cluster_pool_size": {"dram": 1024 * 1024 * 1024, "vram": {}}, + "fluxonkv_spec": { + "etcd_addresses": ["127.0.0.1:22379"], + "cluster_name": "fluxon_s3", + "share_mem_path": str(state_dir / "sharemem"), + "sub_cluster": "default", + "large_file_paths": [str(state_dir / "large" / "owner")], + }, + } + + bundle = module._build_s3_single_node_bundle( + data_dir=data_dir, + state_dir=state_dir, + kv_master_config=kv_master_config, + kv_owner_config=kv_owner_config, + export_name="quick-start-export", + greptime_base_url="http://127.0.0.1:24000", + panel_port=26180, + panel_listen_host="0.0.0.0", + bootstrap_username="admin", + bootstrap_password="admin", + export_cache_max_bytes=1024 * 1024 * 1024, + ) + + self.assertEqual(bundle.s3_endpoint, "http://127.0.0.1:26180/fs_s3") + self.assertEqual(bundle.s3_ui_url, "http://127.0.0.1:26180/fs_s3/ui/") + self.assertEqual( + bundle.fs_master_config["fluxon_fs"]["cache"]["exports"]["quick-start-export"]["remote_root_dir_abs"], + str(data_dir.resolve()), + ) + self.assertEqual( + bundle.fs_master_config["fluxon_fs"]["master_panel"]["access_db_path"], + str((state_dir / "fs_master" / "access.db").resolve()), + ) + self.assertEqual( + bundle.fs_master_config["fluxon_fs"]["master_panel"]["bootstrap_access_model"]["users"][0]["username"], + "admin", + ) + self.assertEqual( + bundle.fs_agent_config["fluxon_fs"]["cache"]["exports"]["quick-start-export"]["remote_root_dir_abs"], + str(data_dir.resolve()), + ) + self.assertEqual( + bundle.kv_owner_config["fluxonkv_spec"]["share_mem_path"], + str((state_dir / "sharemem").resolve()), + ) + + def test_start_middleware_true_is_rejected(self) -> None: + module = importlib.import_module("fluxon_py.quick_start") + with self.assertRaises(NotImplementedError): + module.serve_s3_single_node( + "/tmp/data", + "/tmp/state", + kv_master_config={}, + kv_owner_config={}, + start_middleware=True, + ) + From 7f598c3f6df93f2fcbc8c921735467215ac33785 Mon Sep 17 00:00:00 2001 From: zTz01 <1773266173@qq.com> Date: Fri, 7 Aug 2026 21:17:06 +0800 Subject: [PATCH 2/3] release: bump version to 0.2.3 --- README.md | 8 ++-- README_CN.md | 8 ++-- examples/fluxon_quick_start/README.md | 8 ++-- examples/fluxon_quick_start/build_image.py | 2 +- ...23\345\255\230\351\223\276\350\267\257.md" | 2 +- fluxon_py/__init__.py | 2 +- fluxon_release/release_notes/v0.2.3.md | 46 +++++++++++++++++++ fluxon_rs/Cargo.toml | 2 +- fluxon_rs/fluxon_cli/Cargo.toml | 2 +- fluxon_rs/fluxon_commu/Cargo.toml | 2 +- .../Cargo.toml | 2 +- fluxon_rs/fluxon_framework/Cargo.toml | 2 +- .../fluxon_framework_compiled/Cargo.toml | 2 +- fluxon_rs/fluxon_fs_s3_gateway/Cargo.toml | 2 +- fluxon_rs/fluxon_kv/Cargo.toml | 2 +- fluxon_rs/fluxon_mq/Cargo.toml | 2 +- fluxon_rs/fluxon_observability/Cargo.toml | 2 +- fluxon_rs/fluxon_ops/Cargo.toml | 2 +- fluxon_rs/fluxon_pyo3/Cargo.toml | 2 +- fluxon_rs/fluxon_util/Cargo.toml | 2 +- fluxon_rs/limit_thirdparty/Cargo.toml | 2 +- fluxon_rs/setup.py | 2 +- .../utils/docker_build_runtime_utils.py | 2 +- 23 files changed, 77 insertions(+), 31 deletions(-) create mode 100644 fluxon_release/release_notes/v0.2.3.md diff --git a/README.md b/README.md index ffc9dcb..2a1e54f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ An AI-native distributed data plane that supports high performance RPC, KV Cache [![Linux Only](https://img.shields.io/badge/Linux-Only-2ea44f)](#runtime-requirements) [![Python](https://img.shields.io/badge/Python-%3E%3D3.10-3776AB)](#runtime-requirements) [![Rust](https://img.shields.io/badge/Rust-1.93.0-000000)](./fluxon_rs/rust-toolchain.toml) -[![Latest](https://img.shields.io/badge/Latest-v0.2.2-f28500)](./fluxon_release) +[![Latest](https://img.shields.io/badge/Latest-v0.2.3-f28500)](./fluxon_release) [![Interfaces](https://img.shields.io/badge/Interfaces-KV%2FRPC%20%7C%20MQ%20%7C%20FS-1f6feb)](#interface-capabilities)
@@ -201,7 +201,7 @@ The distribution installs the `fluxon_py` import package. Service-plane runtimes ```bash docker run --rm -it --network host \ - hanbaoaaa/fluxon_quick_start:0.2.2 \ + hanbaoaaa/fluxon_quick_start:0.2.3 \ --mode kv \ --etcd-client-port 12379 \ --master-p2p-port 31000 \ @@ -234,7 +234,7 @@ Related interface docs: ```bash docker run --rm -it --network host \ - hanbaoaaa/fluxon_quick_start:0.2.2 \ + hanbaoaaa/fluxon_quick_start:0.2.3 \ --mode mq \ --etcd-client-port 37379 \ --kv-master-port 34200 \ @@ -265,7 +265,7 @@ Related interface docs: ```bash docker run --rm -it --network host \ - hanbaoaaa/fluxon_quick_start:0.2.2 \ + hanbaoaaa/fluxon_quick_start:0.2.3 \ --mode fs \ --etcd-client-port 36379 \ --kv-master-port 34100 \ diff --git a/README_CN.md b/README_CN.md index e5d01b8..58b82a8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -9,7 +9,7 @@ [![Linux Only](https://img.shields.io/badge/Linux-Only-2ea44f)](#运行要求) [![Python](https://img.shields.io/badge/Python-%3E%3D3.10-3776AB)](#运行要求) [![Rust](https://img.shields.io/badge/Rust-1.93.0-000000)](./fluxon_rs/rust-toolchain.toml) -[![Latest](https://img.shields.io/badge/Latest-v0.2.2-f28500)](./fluxon_release) +[![Latest](https://img.shields.io/badge/Latest-v0.2.3-f28500)](./fluxon_release) [![Interfaces](https://img.shields.io/badge/Interfaces-KV%2FRPC%20%7C%20MQ%20%7C%20FS-1f6feb)](#接口能力)
@@ -198,7 +198,7 @@ python3 -m pip install fluxon-py ```bash docker run --rm -it --network host \ - hanbaoaaa/fluxon_quick_start:0.2.2 \ + hanbaoaaa/fluxon_quick_start:0.2.3 \ --mode kv \ --etcd-client-port 12379 \ --master-p2p-port 31000 \ @@ -231,7 +231,7 @@ del demo:hello ```bash docker run --rm -it --network host \ - hanbaoaaa/fluxon_quick_start:0.2.2 \ + hanbaoaaa/fluxon_quick_start:0.2.3 \ --mode mq \ --etcd-client-port 37379 \ --kv-master-port 34200 \ @@ -262,7 +262,7 @@ exit ```bash docker run --rm -it --network host \ - hanbaoaaa/fluxon_quick_start:0.2.2 \ + hanbaoaaa/fluxon_quick_start:0.2.3 \ --mode fs \ --etcd-client-port 36379 \ --kv-master-port 34100 \ diff --git a/examples/fluxon_quick_start/README.md b/examples/fluxon_quick_start/README.md index 1c1a3bd..352c4ad 100644 --- a/examples/fluxon_quick_start/README.md +++ b/examples/fluxon_quick_start/README.md @@ -11,7 +11,7 @@ It does not replace the formal service-plane, KV, MQ, or FS interface docs. - unified quick-start entrypoint - `build_image.py` - quick-start image build entrypoint -- `fluxon_quick_start:0.2.2` +- `fluxon_quick_start:0.2.3` - quick-start Docker image ## Runtime Modes @@ -86,7 +86,7 @@ Python environment can already import both `fluxon_py` and `fluxon_pyo3`. ```bash docker run --rm -it --network host \ - fluxon_quick_start:0.2.2 \ + fluxon_quick_start:0.2.3 \ --mode kv \ --etcd-client-port 12379 \ --master-p2p-port 31000 \ @@ -119,7 +119,7 @@ del demo:hello ```bash docker run --rm -it --network host \ - fluxon_quick_start:0.2.2 \ + fluxon_quick_start:0.2.3 \ --mode mq \ --etcd-client-port 37379 \ --kv-master-port 34200 \ @@ -155,7 +155,7 @@ The background consumer keeps printing received messages. ```bash docker run --rm -it --network host \ - fluxon_quick_start:0.2.2 \ + fluxon_quick_start:0.2.3 \ --mode fs \ --etcd-client-port 36379 \ --kv-master-port 34100 \ diff --git a/examples/fluxon_quick_start/build_image.py b/examples/fluxon_quick_start/build_image.py index 6c14a64..4a6762b 100644 --- a/examples/fluxon_quick_start/build_image.py +++ b/examples/fluxon_quick_start/build_image.py @@ -22,7 +22,7 @@ SCRIPTS_DIR = REPO_ROOT / "setup_and_pack" DOCKERFILE_PATH = SCRIPT_DIR / "Dockerfile" IMAGE_NAME = "fluxon_quick_start" -IMAGE_TAG = "0.2.2" +IMAGE_TAG = "0.2.3" # Binaries to copy from ext_images into quick_start bin/. EXT_BINARIES = ("etcd/etcd", "etcd/etcdctl", "greptime/greptime") diff --git "a/fluxon_doc_cn/blog/blog_3_\346\212\212\344\270\200\344\270\252\346\234\254\345\234\260\346\226\207\344\273\266\345\244\271\345\217\230\346\210\220 S3 \346\234\215\345\212\241\357\274\232FluxonFS \347\232\204\347\233\256\345\275\225\343\200\201\345\257\271\350\261\241\344\270\216\347\274\223\345\255\230\351\223\276\350\267\257.md" "b/fluxon_doc_cn/blog/blog_3_\346\212\212\344\270\200\344\270\252\346\234\254\345\234\260\346\226\207\344\273\266\345\244\271\345\217\230\346\210\220 S3 \346\234\215\345\212\241\357\274\232FluxonFS \347\232\204\347\233\256\345\275\225\343\200\201\345\257\271\350\261\241\344\270\216\347\274\223\345\255\230\351\223\276\350\267\257.md" index 4126b3c..55f1acc 100644 --- "a/fluxon_doc_cn/blog/blog_3_\346\212\212\344\270\200\344\270\252\346\234\254\345\234\260\346\226\207\344\273\266\345\244\271\345\217\230\346\210\220 S3 \346\234\215\345\212\241\357\274\232FluxonFS \347\232\204\347\233\256\345\275\225\343\200\201\345\257\271\350\261\241\344\270\216\347\274\223\345\255\230\351\223\276\350\267\257.md" +++ "b/fluxon_doc_cn/blog/blog_3_\346\212\212\344\270\200\344\270\252\346\234\254\345\234\260\346\226\207\344\273\266\345\244\271\345\217\230\346\210\220 S3 \346\234\215\345\212\241\357\274\232FluxonFS \347\232\204\347\233\256\345\275\225\343\200\201\345\257\271\350\261\241\344\270\216\347\274\223\345\255\230\351\223\276\350\267\257.md" @@ -207,7 +207,7 @@ docker run -d --name fluxon-s3 ` --mount "type=bind,src=C:\fluxon-s3\data,dst=/data" ` --mount "type=bind,src=C:\fluxon-s3\state,dst=/state" ` --entrypoint python3 ` - "hanbaoaaa/fluxon_quick_start:0.2.2" ` + "hanbaoaaa/fluxon_quick_start:0.2.3" ` -c " from fluxon_py.quick_start import serve_s3_single_node diff --git a/fluxon_py/__init__.py b/fluxon_py/__init__.py index b5b6e45..391acee 100644 --- a/fluxon_py/__init__.py +++ b/fluxon_py/__init__.py @@ -58,7 +58,7 @@ from typing import Any -__version__ = "0.2.2" +__version__ = "0.2.3" __all__ = [ # Core API "KvClient", diff --git a/fluxon_release/release_notes/v0.2.3.md b/fluxon_release/release_notes/v0.2.3.md new file mode 100644 index 0000000..17955d3 --- /dev/null +++ b/fluxon_release/release_notes/v0.2.3.md @@ -0,0 +1,46 @@ +# Fluxon v0.2.3 + +`v0.2.3` is a packaging and quick-start release for the public `fluxon-py` +distribution. It keeps the closed communication SDK and open-surface contract at +their existing versions while making the installed wheel usable for the S3 +single-node quick-start flow. + +## Highlights + +- Added the installed-package entrypoint + `fluxon_py.quick_start.serve_s3_single_node(...)` for exposing a local + directory through FluxonFS S3. +- Kept the public PyPI distribution name as `fluxon-py`, with the import package + remaining `fluxon_py`. +- Aligned runtime subprocess imports with the installed `fluxon_py` package so + child service processes use the same package tree as the caller. +- Verified the Linux pip flow with a locally built release wheel, including + service startup, first-credential update, rclone bucket listing, upload, + local read, local append, S3 readback, and S3 delete. + +## Version and SDK Contract + +- Public Python package, Rust workspace, and Quick Start version: `0.2.3`. +- Closed communication SDK version: `0.2.1`. +- Closed SDK required open-surface contract version: `0.2.1`. + +The SDK version and open-surface contract version are independent from the +public release version. This release does not change the closed SDK manifest, +closed ABI/schema constants, or `FLUXON_COMMU_OPEN_SURFACE_VERSION`. + +## Release Artifacts + +- PyPI: `fluxon-py==0.2.3`. +- Docker Hub: `hanbaoaaa/fluxon_quick_start:0.2.3`. +- GitHub Release: `fluxon_release.tar.gz`. +- GitHub Release: `fluxon_quick_start_0.2.3_docker_image.tar.gz`. + +The Docker publication does not update `latest`. + +## Runtime Requirements and Known Limits + +- Linux only; Python `>=3.10`. +- The S3 pip quick start reuses external etcd and GreptimeDB when + `start_middleware=False`. +- Dynamic bucket creation is not part of the quick-start contract; configure the + exported bucket name up front. diff --git a/fluxon_rs/Cargo.toml b/fluxon_rs/Cargo.toml index 6cb3c08..5fbb1eb 100644 --- a/fluxon_rs/Cargo.toml +++ b/fluxon_rs/Cargo.toml @@ -22,7 +22,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.2.2" +version = "0.2.3" edition = "2024" license = "" authors = ["teleai_infra"] diff --git a/fluxon_rs/fluxon_cli/Cargo.toml b/fluxon_rs/fluxon_cli/Cargo.toml index 88be1e4..668faa8 100644 --- a/fluxon_rs/fluxon_cli/Cargo.toml +++ b/fluxon_rs/fluxon_cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_cli" -version = "0.2.2" +version = "0.2.3" edition = "2024" [dependencies] diff --git a/fluxon_rs/fluxon_commu/Cargo.toml b/fluxon_rs/fluxon_commu/Cargo.toml index 0373ca2..af67440 100644 --- a/fluxon_rs/fluxon_commu/Cargo.toml +++ b/fluxon_rs/fluxon_commu/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_commu" -version = "0.2.2" +version = "0.2.3" edition = "2024" build = "build.rs" diff --git a/fluxon_rs/fluxon_commu_closed_sdk_consumer/Cargo.toml b/fluxon_rs/fluxon_commu_closed_sdk_consumer/Cargo.toml index a6ca631..437ab27 100644 --- a/fluxon_rs/fluxon_commu_closed_sdk_consumer/Cargo.toml +++ b/fluxon_rs/fluxon_commu_closed_sdk_consumer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_commu_closed_sdk_consumer" -version = "0.2.2" +version = "0.2.3" edition = "2024" build = "build.rs" diff --git a/fluxon_rs/fluxon_framework/Cargo.toml b/fluxon_rs/fluxon_framework/Cargo.toml index b3a3d86..d579a24 100644 --- a/fluxon_rs/fluxon_framework/Cargo.toml +++ b/fluxon_rs/fluxon_framework/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_framework" -version = "0.2.2" +version = "0.2.3" edition = "2024" [dependencies] diff --git a/fluxon_rs/fluxon_framework_compiled/Cargo.toml b/fluxon_rs/fluxon_framework_compiled/Cargo.toml index 5a8c95c..715bd11 100644 --- a/fluxon_rs/fluxon_framework_compiled/Cargo.toml +++ b/fluxon_rs/fluxon_framework_compiled/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_framework_compiled" -version = "0.2.2" +version = "0.2.3" edition = "2024" [dependencies] diff --git a/fluxon_rs/fluxon_fs_s3_gateway/Cargo.toml b/fluxon_rs/fluxon_fs_s3_gateway/Cargo.toml index e6caba2..d2bbb34 100644 --- a/fluxon_rs/fluxon_fs_s3_gateway/Cargo.toml +++ b/fluxon_rs/fluxon_fs_s3_gateway/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_fs_s3_gateway" -version = "0.2.2" +version = "0.2.3" edition = "2024" [dependencies] diff --git a/fluxon_rs/fluxon_kv/Cargo.toml b/fluxon_rs/fluxon_kv/Cargo.toml index 9ddeae3..c49b502 100644 --- a/fluxon_rs/fluxon_kv/Cargo.toml +++ b/fluxon_rs/fluxon_kv/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_kv" -version = "0.2.2" +version = "0.2.3" edition = "2024" [features] diff --git a/fluxon_rs/fluxon_mq/Cargo.toml b/fluxon_rs/fluxon_mq/Cargo.toml index 8a0324e..69dd113 100644 --- a/fluxon_rs/fluxon_mq/Cargo.toml +++ b/fluxon_rs/fluxon_mq/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_mq" -version = "0.2.2" +version = "0.2.3" edition = "2021" [lib] diff --git a/fluxon_rs/fluxon_observability/Cargo.toml b/fluxon_rs/fluxon_observability/Cargo.toml index f8c1572..00dbf2c 100644 --- a/fluxon_rs/fluxon_observability/Cargo.toml +++ b/fluxon_rs/fluxon_observability/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_observability" -version = "0.2.2" +version = "0.2.3" edition = "2024" [dependencies] diff --git a/fluxon_rs/fluxon_ops/Cargo.toml b/fluxon_rs/fluxon_ops/Cargo.toml index 5cf2e0e..c1637db 100644 --- a/fluxon_rs/fluxon_ops/Cargo.toml +++ b/fluxon_rs/fluxon_ops/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_ops" -version = "0.2.2" +version = "0.2.3" edition = "2024" [dependencies] diff --git a/fluxon_rs/fluxon_pyo3/Cargo.toml b/fluxon_rs/fluxon_pyo3/Cargo.toml index a74496b..4d0438a 100644 --- a/fluxon_rs/fluxon_pyo3/Cargo.toml +++ b/fluxon_rs/fluxon_pyo3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_pyo3" -version = "0.2.2" +version = "0.2.3" edition = "2024" [lib] diff --git a/fluxon_rs/fluxon_util/Cargo.toml b/fluxon_rs/fluxon_util/Cargo.toml index cb7ef9b..305f50c 100644 --- a/fluxon_rs/fluxon_util/Cargo.toml +++ b/fluxon_rs/fluxon_util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fluxon_util" -version = "0.2.2" +version = "0.2.3" edition = "2024" authors = ["Your Name "] description = "Utility crate with macros and helper functions" diff --git a/fluxon_rs/limit_thirdparty/Cargo.toml b/fluxon_rs/limit_thirdparty/Cargo.toml index f91e685..b946566 100644 --- a/fluxon_rs/limit_thirdparty/Cargo.toml +++ b/fluxon_rs/limit_thirdparty/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "limit_thirdparty" -version = "0.2.2" +version = "0.2.3" edition = "2024" [dependencies] diff --git a/fluxon_rs/setup.py b/fluxon_rs/setup.py index 0bc10e5..b87a0c5 100644 --- a/fluxon_rs/setup.py +++ b/fluxon_rs/setup.py @@ -36,7 +36,7 @@ def find_libs(): setup( name="fluxon_pyo3", - version="0.2.2", + version="0.2.3", description="for export fluxonkv core to python layer", long_description=open("README.md").read() if os.path.exists("README.md") else "", long_description_content_type="text/markdown", diff --git a/setup_and_pack/utils/docker_build_runtime_utils.py b/setup_and_pack/utils/docker_build_runtime_utils.py index 47ec139..c40b23a 100644 --- a/setup_and_pack/utils/docker_build_runtime_utils.py +++ b/setup_and_pack/utils/docker_build_runtime_utils.py @@ -550,7 +550,7 @@ def build_docker_run_cmd( """Build a `docker run` command (without executing it). Args: - image: Image name (with tag), e.g. "fluxon_quick_start:0.2.2". + image: Image name (with tag), e.g. "fluxon_quick_start:0.2.3". name: Container name (`--name`). remove: Auto-remove on exit (`--rm`). detach: Run detached (`-d`). From 6e594dca792cc25faa94e7cc7b4ac0d189eb2f25 Mon Sep 17 00:00:00 2001 From: zTz01 <1773266173@qq.com> Date: Fri, 7 Aug 2026 21:51:39 +0800 Subject: [PATCH 3/3] docs: update v0.2.3 release notes --- fluxon_release/release_notes/v0.2.3.md | 124 +++++++++++++++++++------ 1 file changed, 98 insertions(+), 26 deletions(-) diff --git a/fluxon_release/release_notes/v0.2.3.md b/fluxon_release/release_notes/v0.2.3.md index 17955d3..a546cc4 100644 --- a/fluxon_release/release_notes/v0.2.3.md +++ b/fluxon_release/release_notes/v0.2.3.md @@ -1,34 +1,92 @@ -# Fluxon v0.2.3 +# 🚀 Fluxon v0.2.3 -`v0.2.3` is a packaging and quick-start release for the public `fluxon-py` -distribution. It keeps the closed communication SDK and open-surface contract at -their existing versions while making the installed wheel usable for the S3 -single-node quick-start flow. +`v0.2.3` rolls up the mainline work merged from July 14 through August 5, 2026, together with the new release pipeline in the tagged revision. The largest changes are a distributed SSD backing tier for Fluxon KV, hybrid S3 object writes, communication ABI 9, event-driven TCP reactors, stronger MQ and framework lifecycle handling, and expanded release-grade CI. -## Highlights +## ✨ Highlights -- Added the installed-package entrypoint - `fluxon_py.quick_start.serve_s3_single_node(...)` for exposing a local - directory through FluxonFS S3. -- Kept the public PyPI distribution name as `fluxon-py`, with the import package - remaining `fluxon_py`. -- Aligned runtime subprocess imports with the installed `fluxon_py` package so - child service processes use the same package tree as the caller. -- Verified the Linux pip flow with a locally built release wheel, including - service startup, first-credential update, rclone bucket listing, upload, - local read, local append, S3 readback, and S3 delete. +- Added an owner-local SSD backing tier behind the existing Fluxon KV `put` / `get` / `delete` contract. +- Added size-aware S3 writes, KV-backed write sessions, shared lease keepalive, and retryable temporary-key cleanup. +- Added `FluxonFsVideoReader` and a pooled reader API for cached random access to video data. +- Upgraded the closed communication boundary to ABI 9 and added event-driven TCP reactor support. +- Strengthened KV member cleanup, MQ close semantics, framework shutdown barriers, and background-task ownership. +- Added rclone S3 integration coverage and a resource-bounded large-scale MPMC MQ CI scenario. +- Unified GitHub Release, PyPI, and Docker Hub publication behind one parameter-free GitHub Actions entrypoint. -## Version and SDK Contract +## 🗄️ Fluxon KV: DRAM + SSD Backing Tier + +Fluxon KV can now use each owner's local SSD as a runtime backing layer for DRAM replicas without adding a second public storage API. + +- `put` completes when the memory replica is published; SSD persistence proceeds asynchronously. +- `get` remains memory-first and enters SSD refill only when no readable memory replica is available. +- Local refill can write directly into the requester target when alignment permits; remote refill pipelines SSD reads and chunked transfer to the requester. +- Owner-local `O_DIRECT` / `io_uring` I/O, bounded queues, a fixed-capacity ring, read pins, and route-commit pins protect in-flight data from overwrite. +- Key-version checks reject stale SSD commits and stale routes; eviction notifications and bounded retry converge control-plane state. +- Capacity and usage are reported separately for memory segments and KV SSD storage. + +The public API still returns `MemHolder`. SSD is a runtime cache: it is rebuilt empty after owner restart, does not provide cold-start recovery, and does not stripe one value across multiple devices. + +In the documented single-node H100 SSD-pressure experiment, measured through CUDA event completion, Fluxon's c16 hit-payload throughput for 4 / 8 / 16 MiB values was 3.83× / 5.61× / 6.73× that of the faster of the two measured Mooncake topologies. These figures apply only to the documented dataset, capacity, concurrency, topology, and counting boundary. + +## 🪣 FluxonFS and S3 Data Path + +S3-compatible object I/O now chooses between a small-object fast path and the general write-session pipeline: + +- Objects below `4 MiB` use one `put_small_object` RPC that creates missing parents and writes the complete object. +- Objects at or above `4 MiB` use a bounded write session; the same policy applies to `PutObject`, multipart part upload, and final multipart assembly. +- Write-session batches use KV references for the normal payload path and retain Raw RPC as a correctness fallback. +- A controller-owned cleanup actor retains final responsibility for temporary KV keys when put completion is uncertain or eager deletion fails. +- FS and MQ reuse a bounded shared lease-keepalive actor instead of creating one keepalive loop per payload. +- KV cache hits on S3 reads can produce holder-backed `Bytes` without materializing the complete encoded `FlatDict`; `TCP_NODELAY` reduces avoidable small-response latency. +- FS-before-KV shutdown barriers keep holders, sessions, cleanup actors, and registered tasks alive until their dependents quiesce. + +The documented single-node `rclone v1.60.1` comparison reports that FluxonFS led Alluxio S3 Proxy in all 18 persisted-PUT and cold-read object-size/concurrency combinations. Hot-read gains were strongest for 4 KiB objects, while sequential medium- and large-object hot-read throughput was generally close. This result is scoped to that published setup. + +## 🎬 Cached Video Reading + +- Added `FluxonFsPatcher.open_video_reader(...) -> FluxonFsVideoReader` for random byte reads through FluxonFS export, permission, and cache paths. +- Added `open_video_reader_pool(...)` to reuse readers in dataloader-style workloads. +- Added a benchmark and analysis harness that compares the Fluxon-backed path with the original `decord.VideoReader` path under an explicit dataset and sampling plan. + +## 🌐 Communication and KV Lifecycle + +- Updated the closed communication boundary to ABI 9 while retaining runtime ABI, open-surface, boundary-mode, and provider-anchor checks. +- Added event-driven TCP reactor mode alongside the existing busy-poll path, plus bounded test controls for reactor shards and control/bulk lanes. +- Added explicit KV member lifecycle indexes and cleanup paths for member departure, in-flight requests, replicas, holders, and allocation ownership. +- Tightened configuration validation and separated stable network configuration from developer-only `test_spec_config` switches. + +## 📬 MQ Reliability and Scale Coverage + +- Strengthened MPSC/MPMC producer and consumer close behavior, lazy producer binding, ready-state publication, and lease-backed membership cleanup. +- Public callers close every producer or consumer and consume its `Result` before closing the backing KV store. +- Endpoint-local shutdown remains a strong contract; leased-key deletion is best effort and falls back to backend TTL after keepalive release. +- Added a direct-process large-scale MPMC scenario with resource limits, readiness checks, complete worker-result validation, and orderly process cleanup in GitHub Actions. + +## 🧪 CI, Packaging, and Release Safety + +- Added rclone-based S3 end-to-end coverage to the virtual-node CI path. +- Added validated publication of the release-built `fluxon-py` wheel, including wheel-tag, Python-version, checksum, size, and `twine check` gates. +- Added Codex-assisted CI failure analysis with bounded, read-only evidence collection. +- Added tag provenance that distinguishes an actual tag ref from a same-named branch and binds release artifacts to the tested commit. +- Added deterministic artifact checksums and a read-only Codex release-readiness report. +- The parameter-free `create_release_tag` Action derives the tag and release notes from the repository; no release version or body is entered in the GitHub UI. +- After common validation, GitHub Release, PyPI, and Docker Hub jobs become eligible in parallel and wait on their own protected environments. + +## 📚 Documentation and Project Presentation + +- Reworked the README around Fluxon as an AI-native distributed data plane spanning KV/RPC, MQ, and FS/S3 interfaces. +- Added scoped benchmark explanations for KV SSD and S3, together with architecture, lifecycle, documentation-review, event-subscription, index-design, and test-extension guidance. +- Added practical deep dives for KV SSD storage and serving a local directory through FluxonFS S3. +- Added the project WeChat contact entry and refreshed the public overview and background narrative. + +## 🔐 Version and SDK Contract - Public Python package, Rust workspace, and Quick Start version: `0.2.3`. - Closed communication SDK version: `0.2.1`. - Closed SDK required open-surface contract version: `0.2.1`. -The SDK version and open-surface contract version are independent from the -public release version. This release does not change the closed SDK manifest, -closed ABI/schema constants, or `FLUXON_COMMU_OPEN_SURFACE_VERSION`. +The SDK version and open-surface contract version are independent from the public release version. The tagged runtime must still pass ABI, open-surface, boundary-mode, and provider-anchor validation. -## Release Artifacts +## 📦 Release Artifacts - PyPI: `fluxon-py==0.2.3`. - Docker Hub: `hanbaoaaa/fluxon_quick_start:0.2.3`. @@ -37,10 +95,24 @@ closed ABI/schema constants, or `FLUXON_COMMU_OPEN_SURFACE_VERSION`. The Docker publication does not update `latest`. -## Runtime Requirements and Known Limits +## ⚠️ Runtime Requirements and Known Limits - Linux only; Python `>=3.10`. -- The S3 pip quick start reuses external etcd and GreptimeDB when - `start_middleware=False`. -- Dynamic bucket creation is not part of the quick-start contract; configure the - exported bucket name up front. +- Building and running the full stack requires the external services and native dependencies documented by each interface. +- KV SSD is a runtime backing cache and does not recover old shard contents after restart. +- The published KV SSD and S3 benchmark conclusions are limited to their documented hardware, topology, workload, capacity, and measurement boundaries. +- Release CI covers the configured virtual-node, large-scale MQ, rclone S3, packaging, SDK contract, wheel, and Quick Start checks. It does not establish compatibility for untested platforms or production environments. + +## 🧾 Included Mainline Changes + +- #43 — add the WeChat contact entry to the README. +- #34 — run large-scale MQ coverage in GitHub Actions with bounded resources. +- #37 — add distributed owner SSD backing storage for Fluxon KV, VideoReader, benchmarks, and supporting lifecycle work. +- #46 — add rclone S3 integration coverage. +- #45 — publish the validated `fluxon-py` wheel through PyPI trusted publishing; this path is now folded into the unified release workflow. +- #48, #49, #51, #52 — expand and refine the public Fluxon overview and AI-native distributed-data-plane positioning. +- #47 — support communication ABI 9 and event-driven TCP reactors. +- #50 — add hybrid S3 writes, shared lease keepalive, cleanup ownership, and shutdown barriers. +- #58 — restore the public PyPI distribution name to `fluxon-py` while keeping `fluxon_py` as the Python import package. +- #59 — add the installed-package `fluxon_py.quick_start.serve_s3_single_node(...)` entrypoint for starting the local-directory S3 Quick Start flow. +- The tagged `v0.2.3` revision also contains the unified GitHub Release / PyPI / Docker Hub release workflow and its deterministic readiness gates.