From 02f86e1ba5aa50b5639e23553d2fc4d678932c2e Mon Sep 17 00:00:00 2001 From: kmce2019 <50456552+kmce2019@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:26:36 -0500 Subject: [PATCH 1/7] fix: restore Starlark API routes --- .../web_interface/test_starlark_api_routes.py | 77 ++ web_interface/blueprints/api_v3.py | 907 +++++++++++++++++- 2 files changed, 982 insertions(+), 2 deletions(-) create mode 100644 test/web_interface/test_starlark_api_routes.py diff --git a/test/web_interface/test_starlark_api_routes.py b/test/web_interface/test_starlark_api_routes.py new file mode 100644 index 00000000..5a75b94c --- /dev/null +++ b/test/web_interface/test_starlark_api_routes.py @@ -0,0 +1,77 @@ +"""Regression tests for the Starlark API routes used by the v3 plugin UI.""" + +import pytest +from flask import Flask + +from web_interface.blueprints import api_v3 as mod + + +@pytest.fixture +def client(monkeypatch, tmp_path): + apps_dir = tmp_path / "starlark-apps" + monkeypatch.setattr(mod, "_STARLARK_APPS_DIR", apps_dir) + monkeypatch.setattr(mod, "_STARLARK_MANIFEST_FILE", apps_dir / "manifest.json") + monkeypatch.setattr(mod.api_v3, "plugin_manager", None, raising=False) + + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(mod.api_v3, url_prefix="/api/v3") + with app.test_client() as test_client: + yield test_client + + +def test_all_documented_starlark_routes_are_registered(): + app = Flask(__name__) + app.register_blueprint(mod.api_v3, url_prefix="/api/v3") + rules = {(rule.rule, tuple(sorted(rule.methods - {"HEAD", "OPTIONS"}))) for rule in app.url_map.iter_rules()} + + expected = { + ("/api/v3/starlark/status", ("GET",)), + ("/api/v3/starlark/install-pixlet", ("POST",)), + ("/api/v3/starlark/apps", ("GET",)), + ("/api/v3/starlark/apps/", ("GET",)), + ("/api/v3/starlark/apps/", ("DELETE",)), + ("/api/v3/starlark/apps//config", ("GET",)), + ("/api/v3/starlark/apps//config", ("PUT",)), + ("/api/v3/starlark/apps//render", ("POST",)), + ("/api/v3/starlark/apps//toggle", ("POST",)), + ("/api/v3/starlark/repository/categories", ("GET",)), + ("/api/v3/starlark/repository/browse", ("GET",)), + ("/api/v3/starlark/repository/install", ("POST",)), + ("/api/v3/starlark/upload", ("POST",)), + } + assert expected <= rules + + +def test_status_works_when_display_plugin_is_not_loaded(client): + response = client.get("/api/v3/starlark/status") + + assert response.status_code == 200 + body = response.get_json() + assert body["status"] == "success" + assert body["installed_apps"] == 0 + assert body["enabled_apps"] == 0 + assert body["plugin_enabled"] is True + assert "pixlet_available" in body + + +def test_apps_falls_back_to_the_standalone_manifest(client): + assert mod._write_starlark_manifest({ + "apps": {"clock": {"name": "Clock", "enabled": True}} + }) + + response = client.get("/api/v3/starlark/apps") + + assert response.status_code == 200 + apps = response.get_json()["apps"] + assert len(apps) == 1 + assert apps[0]["id"] == "clock" + assert apps[0]["name"] == "Clock" + assert apps[0]["enabled"] is True + + +def test_app_path_traversal_is_rejected(client): + response = client.get("/api/v3/starlark/apps/..%5Csecret") + + assert response.status_code == 400 + assert "invalid app_id" in response.get_json()["message"].lower() diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 984637e8..09135c53 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -13,7 +13,7 @@ import logging from datetime import datetime from pathlib import Path -from typing import Dict, Any, Optional +from typing import Dict, Any, Optional, Tuple, Type from urllib.parse import urlparse, urlunparse logger = logging.getLogger(__name__) @@ -8472,4 +8472,907 @@ def backup_delete(filename): except OSError as e: logger.error("backup_delete failed: %s", e, exc_info=True) return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 - return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 \ No newline at end of file + return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 + + +# ─── Starlark Apps API ────────────────────────────────────────────────────── + +def _get_tronbyte_repository_class() -> Type[Any]: + """Import TronbyteRepository from plugin-repos directory.""" + import importlib.util + import importlib + + module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'tronbyte_repository.py' + if not module_path.exists(): + raise ImportError(f"TronbyteRepository module not found at {module_path}") + + # If already imported, reload to pick up code changes + if "tronbyte_repository" in sys.modules: + importlib.reload(sys.modules["tronbyte_repository"]) + return sys.modules["tronbyte_repository"].TronbyteRepository + + spec = importlib.util.spec_from_file_location("tronbyte_repository", str(module_path)) + if spec is None: + raise ImportError(f"Failed to create module spec for tronbyte_repository at {module_path}") + + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError("Failed to create module from spec for tronbyte_repository") + + sys.modules["tronbyte_repository"] = module + spec.loader.exec_module(module) + return module.TronbyteRepository + + +def _get_pixlet_renderer_class() -> Type[Any]: + """Import PixletRenderer from plugin-repos directory.""" + import importlib.util + import importlib + + module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'pixlet_renderer.py' + if not module_path.exists(): + raise ImportError(f"PixletRenderer module not found at {module_path}") + + # If already imported, reload to pick up code changes + if "pixlet_renderer" in sys.modules: + importlib.reload(sys.modules["pixlet_renderer"]) + return sys.modules["pixlet_renderer"].PixletRenderer + + spec = importlib.util.spec_from_file_location("pixlet_renderer", str(module_path)) + if spec is None: + raise ImportError(f"Failed to create module spec for pixlet_renderer at {module_path}") + + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError("Failed to create module from spec for pixlet_renderer") + + sys.modules["pixlet_renderer"] = module + spec.loader.exec_module(module) + return module.PixletRenderer + + +def _validate_and_sanitize_app_id(app_id: Optional[str], fallback_source: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]: + """Validate and sanitize app_id to a safe slug.""" + if not app_id and fallback_source: + app_id = fallback_source + if not app_id: + return None, "app_id is required" + if '..' in app_id or '/' in app_id or '\\' in app_id: + return None, "app_id contains invalid characters" + + sanitized = re.sub(r'[^a-z0-9_]', '_', app_id.lower()).strip('_') + if not sanitized: + sanitized = f"app_{hashlib.sha256(app_id.encode()).hexdigest()[:12]}" + if sanitized[0].isdigit(): + sanitized = f"app_{sanitized}" + return sanitized, None + + +def _validate_timing_value(value: Any, field_name: str, min_val: int = 1, max_val: int = 86400) -> Tuple[Optional[int], Optional[str]]: + """Validate and coerce timing values.""" + if value is None: + return None, None + try: + int_value = int(value) + except (ValueError, TypeError): + return None, f"{field_name} must be an integer" + if int_value < min_val: + return None, f"{field_name} must be at least {min_val}" + if int_value > max_val: + return None, f"{field_name} must be at most {max_val}" + return int_value, None + + +def _get_starlark_plugin() -> Optional[Any]: + """Get the starlark-apps plugin instance, or None.""" + if not api_v3.plugin_manager: + return None + return api_v3.plugin_manager.get_plugin('starlark-apps') + + +def _validate_starlark_app_path(app_id: str) -> Tuple[bool, Optional[str]]: + """ + Validate app_id for path traversal attacks before filesystem access. + + Args: + app_id: App identifier from user input + + Returns: + Tuple of (is_valid, error_message) + """ + # Check for path traversal characters + if '..' in app_id or '/' in app_id or '\\' in app_id: + return False, f"Invalid app_id: contains path traversal characters" + + # Construct and resolve the path + try: + app_path = (_STARLARK_APPS_DIR / app_id).resolve() + base_path = _STARLARK_APPS_DIR.resolve() + + # Verify the resolved path is within the base directory + try: + app_path.relative_to(base_path) + return True, None + except ValueError: + return False, f"Invalid app_id: path traversal attempt" + except Exception as e: + logger.warning(f"Path validation error for app_id '{app_id}': {e}") + return False, f"Invalid app_id" + + +# Starlark standalone helpers for web service (plugin not loaded) +_STARLARK_APPS_DIR = PROJECT_ROOT / 'starlark-apps' +_STARLARK_MANIFEST_FILE = _STARLARK_APPS_DIR / 'manifest.json' + + +def _read_starlark_manifest() -> Dict[str, Any]: + """Read the starlark-apps manifest.json directly from disk.""" + try: + if _STARLARK_MANIFEST_FILE.exists(): + with open(_STARLARK_MANIFEST_FILE, 'r') as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + logger.error(f"Error reading starlark manifest: {e}") + return {'apps': {}} + + +def _write_starlark_manifest(manifest: Dict[str, Any]) -> bool: + """Write the starlark-apps manifest.json to disk with atomic write.""" + temp_file = None + try: + _STARLARK_APPS_DIR.mkdir(parents=True, exist_ok=True) + + # Atomic write pattern: write to temp file, then rename + temp_file = _STARLARK_MANIFEST_FILE.with_suffix('.tmp') + with open(temp_file, 'w') as f: + json.dump(manifest, f, indent=2) + f.flush() + os.fsync(f.fileno()) # Ensure data is written to disk + + # Atomic rename (overwrites destination) + temp_file.replace(_STARLARK_MANIFEST_FILE) + return True + except OSError as e: + logger.error(f"Error writing starlark manifest: {e}") + # Clean up temp file if it exists + if temp_file and temp_file.exists(): + try: + temp_file.unlink() + except Exception: + pass + return False + + +def _install_star_file(app_id: str, star_file_path: str, metadata: Dict[str, Any], assets_dir: Optional[str] = None) -> bool: + """Install a .star file and update the manifest (standalone, no plugin needed).""" + import shutil + import json + app_dir = _STARLARK_APPS_DIR / app_id + app_dir.mkdir(parents=True, exist_ok=True) + dest = app_dir / f"{app_id}.star" + shutil.copy2(star_file_path, str(dest)) + + # Copy asset directories if provided (images/, sources/, etc.) + if assets_dir and Path(assets_dir).exists(): + assets_path = Path(assets_dir) + for item in assets_path.iterdir(): + if item.is_dir(): + # Copy entire directory (e.g., images/, sources/) + dest_dir = app_dir / item.name + if dest_dir.exists(): + shutil.rmtree(dest_dir) + shutil.copytree(item, dest_dir) + logger.debug(f"Copied assets directory: {item.name}") + logger.info(f"Installed assets for {app_id}") + + # Try to extract schema using PixletRenderer + schema = None + try: + PixletRenderer = _get_pixlet_renderer_class() + pixlet = PixletRenderer() + if pixlet.is_available(): + _, schema, _ = pixlet.extract_schema(str(dest)) + if schema: + schema_path = app_dir / "schema.json" + with open(schema_path, 'w') as f: + json.dump(schema, f, indent=2) + logger.info(f"Extracted schema for {app_id}") + except Exception as e: + logger.warning(f"Failed to extract schema for {app_id}: {e}") + + # Create default config — pre-populate with schema defaults + default_config = {} + if schema: + fields = schema.get('fields') or schema.get('schema') or [] + for field in fields: + if isinstance(field, dict) and 'id' in field and 'default' in field: + default_config[field['id']] = field['default'] + + # Create config.json file + config_path = app_dir / "config.json" + with open(config_path, 'w') as f: + json.dump(default_config, f, indent=2) + + manifest = _read_starlark_manifest() + manifest.setdefault('apps', {})[app_id] = { + 'name': metadata.get('name', app_id), + 'enabled': True, + 'render_interval': metadata.get('render_interval', 300), + 'display_duration': metadata.get('display_duration', 15), + 'config': metadata.get('config', {}), + 'star_file': str(dest), + } + return _write_starlark_manifest(manifest) + + +@api_v3.route('/starlark/status', methods=['GET']) +def get_starlark_status(): + """Get Starlark plugin status and Pixlet availability.""" + try: + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + info = starlark_plugin.get_info() + magnify_info = starlark_plugin.get_magnify_recommendation() + return jsonify({ + 'status': 'success', + 'pixlet_available': info.get('pixlet_available', False), + 'pixlet_version': info.get('pixlet_version'), + 'installed_apps': info.get('installed_apps', 0), + 'enabled_apps': info.get('enabled_apps', 0), + 'current_app': info.get('current_app'), + 'plugin_enabled': starlark_plugin.enabled, + 'display_info': magnify_info + }) + + # Plugin not loaded - check Pixlet availability directly + import shutil + import platform + + system = platform.system().lower() + machine = platform.machine().lower() + bin_dir = PROJECT_ROOT / 'bin' / 'pixlet' + + pixlet_binary = None + if system == "linux": + if "aarch64" in machine or "arm64" in machine: + pixlet_binary = bin_dir / "pixlet-linux-arm64" + elif "x86_64" in machine or "amd64" in machine: + pixlet_binary = bin_dir / "pixlet-linux-amd64" + elif system == "darwin": + pixlet_binary = bin_dir / ("pixlet-darwin-arm64" if "arm64" in machine else "pixlet-darwin-amd64") + + pixlet_available = (pixlet_binary and pixlet_binary.exists()) or shutil.which('pixlet') is not None + + # Read app counts from manifest + manifest = _read_starlark_manifest() + apps = manifest.get('apps', {}) + installed_count = len(apps) + enabled_count = sum(1 for a in apps.values() if a.get('enabled', True)) + + return jsonify({ + 'status': 'success', + 'pixlet_available': pixlet_available, + 'pixlet_version': None, + 'installed_apps': installed_count, + 'enabled_apps': enabled_count, + 'plugin_enabled': True, + 'plugin_loaded': False, + 'display_info': {} + }) + + except Exception as e: + logger.error(f"Error getting starlark status: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/apps', methods=['GET']) +def get_starlark_apps(): + """List all installed Starlark apps.""" + try: + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + apps_list = [] + for app_id, app_instance in starlark_plugin.apps.items(): + apps_list.append({ + 'id': app_id, + 'name': app_instance.manifest.get('name', app_id), + 'enabled': app_instance.is_enabled(), + 'has_frames': app_instance.frames is not None, + 'render_interval': app_instance.get_render_interval(), + 'display_duration': app_instance.get_display_duration(), + 'config': app_instance.config, + 'has_schema': app_instance.schema is not None, + 'last_render_time': app_instance.last_render_time + }) + return jsonify({'status': 'success', 'apps': apps_list, 'count': len(apps_list)}) + + # Standalone: read manifest from disk + manifest = _read_starlark_manifest() + apps_list = [] + for app_id, app_data in manifest.get('apps', {}).items(): + apps_list.append({ + 'id': app_id, + 'name': app_data.get('name', app_id), + 'enabled': app_data.get('enabled', True), + 'has_frames': False, + 'render_interval': app_data.get('render_interval', 300), + 'display_duration': app_data.get('display_duration', 15), + 'config': app_data.get('config', {}), + 'has_schema': False, + 'last_render_time': None + }) + return jsonify({'status': 'success', 'apps': apps_list, 'count': len(apps_list)}) + + except Exception as e: + logger.error(f"Error getting starlark apps: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/apps/', methods=['GET']) +def get_starlark_app(app_id): + """Get details for a specific Starlark app.""" + try: + # Validate app_id before any filesystem access + is_valid, error_msg = _validate_starlark_app_path(app_id) + if not is_valid: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + return jsonify({ + 'status': 'success', + 'app': { + 'id': app_id, + 'name': app.manifest.get('name', app_id), + 'enabled': app.is_enabled(), + 'config': app.config, + 'schema': app.schema, + 'render_interval': app.get_render_interval(), + 'display_duration': app.get_display_duration(), + 'has_frames': app.frames is not None, + 'frame_count': len(app.frames) if app.frames else 0, + 'last_render_time': app.last_render_time, + } + }) + + # Standalone: read from manifest + manifest = _read_starlark_manifest() + app_data = manifest.get('apps', {}).get(app_id) + if not app_data: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + # Load schema from schema.json if it exists (path already validated above) + schema = None + schema_file = _STARLARK_APPS_DIR / app_id / 'schema.json' + if schema_file.exists(): + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + except (OSError, json.JSONDecodeError) as e: + logger.warning(f"Failed to load schema for {app_id}: {e}") + + return jsonify({ + 'status': 'success', + 'app': { + 'id': app_id, + 'name': app_data.get('name', app_id), + 'enabled': app_data.get('enabled', True), + 'config': app_data.get('config', {}), + 'schema': schema, + 'render_interval': app_data.get('render_interval', 300), + 'display_duration': app_data.get('display_duration', 15), + 'has_frames': False, + 'frame_count': 0, + 'last_render_time': None, + } + }) + + except Exception as e: + logger.error(f"Error getting starlark app {app_id}: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/upload', methods=['POST']) +def upload_starlark_app(): + """Upload and install a new Starlark app.""" + try: + if 'file' not in request.files: + return jsonify({'status': 'error', 'message': 'No file uploaded'}), 400 + + file = request.files['file'] + if not file.filename or not file.filename.endswith('.star'): + return jsonify({'status': 'error', 'message': 'File must have .star extension'}), 400 + + # Check file size (limit to 5MB for .star files) + file.seek(0, 2) # Seek to end + file_size = file.tell() + file.seek(0) # Reset to beginning + MAX_STAR_SIZE = 5 * 1024 * 1024 # 5MB + if file_size > MAX_STAR_SIZE: + return jsonify({'status': 'error', 'message': f'File too large (max 5MB, got {file_size/1024/1024:.1f}MB)'}), 400 + + app_name = request.form.get('name') + app_id_input = request.form.get('app_id') + filename_base = file.filename.replace('.star', '') if file.filename else None + app_id, app_id_error = _validate_and_sanitize_app_id(app_id_input, fallback_source=filename_base) + if app_id_error: + return jsonify({'status': 'error', 'message': f'Invalid app_id: {app_id_error}'}), 400 + + render_interval_input = request.form.get('render_interval') + render_interval = 300 + if render_interval_input is not None: + render_interval, err = _validate_timing_value(render_interval_input, 'render_interval') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + render_interval = render_interval or 300 + + display_duration_input = request.form.get('display_duration') + display_duration = 15 + if display_duration_input is not None: + display_duration, err = _validate_timing_value(display_duration_input, 'display_duration') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + display_duration = display_duration or 15 + + import tempfile + with tempfile.NamedTemporaryFile(delete=False, suffix='.star') as tmp: + file.save(tmp.name) + temp_path = tmp.name + + try: + metadata = {'name': app_name or app_id, 'render_interval': render_interval, 'display_duration': display_duration} + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + success = starlark_plugin.install_app(app_id, temp_path, metadata) + else: + success = _install_star_file(app_id, temp_path, metadata) + if success: + return jsonify({'status': 'success', 'message': f'App installed: {app_id}', 'app_id': app_id}) + else: + return jsonify({'status': 'error', 'message': 'Failed to install app'}), 500 + finally: + try: + os.unlink(temp_path) + except OSError: + pass + + except (ValueError, OSError, IOError) as e: + logger.exception("[Starlark] Error uploading starlark app") + return jsonify({'status': 'error', 'message': 'Failed to upload app'}), 500 + + +@api_v3.route('/starlark/apps/', methods=['DELETE']) +def uninstall_starlark_app(app_id): + """Uninstall a Starlark app.""" + try: + # Validate app_id before any filesystem access + is_valid, error_msg = _validate_starlark_app_path(app_id) + if not is_valid: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + success = starlark_plugin.uninstall_app(app_id) + else: + # Standalone: remove app dir and manifest entry (path already validated) + import shutil + app_dir = _STARLARK_APPS_DIR / app_id + + if app_dir.exists(): + shutil.rmtree(app_dir) + manifest = _read_starlark_manifest() + manifest.get('apps', {}).pop(app_id, None) + success = _write_starlark_manifest(manifest) + + if success: + return jsonify({'status': 'success', 'message': f'App uninstalled: {app_id}'}) + else: + return jsonify({'status': 'error', 'message': 'Failed to uninstall app'}), 500 + + except Exception as e: + logger.error(f"Error uninstalling starlark app {app_id}: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/apps//config', methods=['GET']) +def get_starlark_app_config(app_id): + """Get configuration for a Starlark app.""" + try: + # Validate app_id before any filesystem access + is_valid, error_msg = _validate_starlark_app_path(app_id) + if not is_valid: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + return jsonify({'status': 'success', 'config': app.config, 'schema': app.schema}) + + # Standalone: read from config.json file (path already validated) + app_dir = _STARLARK_APPS_DIR / app_id + config_file = app_dir / "config.json" + + if not app_dir.exists(): + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + config = {} + if config_file.exists(): + try: + with open(config_file, 'r') as f: + config = json.load(f) + except (OSError, json.JSONDecodeError) as e: + logger.warning(f"Failed to load config for {app_id}: {e}") + + # Load schema from schema.json + schema = None + schema_file = app_dir / "schema.json" + if schema_file.exists(): + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + except Exception as e: + logger.warning(f"Failed to load schema for {app_id}: {e}") + + return jsonify({'status': 'success', 'config': config, 'schema': schema}) + + except Exception as e: + logger.error(f"Error getting config for {app_id}: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/apps//config', methods=['PUT']) +def update_starlark_app_config(app_id): + """Update configuration for a Starlark app.""" + try: + # Validate app_id before any filesystem access + is_valid, error_msg = _validate_starlark_app_path(app_id) + if not is_valid: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + data = request.get_json() + if not data: + return jsonify({'status': 'error', 'message': 'No configuration provided'}), 400 + + if 'render_interval' in data: + val, err = _validate_timing_value(data['render_interval'], 'render_interval') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + data['render_interval'] = val + + if 'display_duration' in data: + val, err = _validate_timing_value(data['display_duration'], 'display_duration') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + data['display_duration'] = val + + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + # Extract timing keys from data before updating config (they belong in manifest, not config) + render_interval = data.pop('render_interval', None) + display_duration = data.pop('display_duration', None) + + # Update config with non-timing fields only + app.config.update(data) + + # Update manifest with timing fields + timing_changed = False + if render_interval is not None: + app.manifest['render_interval'] = render_interval + timing_changed = True + if display_duration is not None: + app.manifest['display_duration'] = display_duration + timing_changed = True + if app.save_config(): + # Persist manifest if timing changed (same pattern as toggle endpoint) + if timing_changed: + try: + # Use safe manifest update to prevent race conditions + timing_updates = {} + if render_interval is not None: + timing_updates['render_interval'] = render_interval + if display_duration is not None: + timing_updates['display_duration'] = display_duration + + def update_fn(manifest): + manifest['apps'][app_id].update(timing_updates) + starlark_plugin._update_manifest_safe(update_fn) + except Exception as e: + logger.warning(f"Failed to persist timing to manifest for {app_id}: {e}") + starlark_plugin._render_app(app, force=True) + return jsonify({'status': 'success', 'message': 'Configuration updated', 'config': app.config}) + else: + return jsonify({'status': 'error', 'message': 'Failed to save configuration'}), 500 + + # Standalone: update both config.json and manifest + manifest = _read_starlark_manifest() + app_data = manifest.get('apps', {}).get(app_id) + if not app_data: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + # Extract timing keys (they go in manifest, not config.json) + render_interval = data.pop('render_interval', None) + display_duration = data.pop('display_duration', None) + + # Update manifest with timing values + if render_interval is not None: + app_data['render_interval'] = render_interval + if display_duration is not None: + app_data['display_duration'] = display_duration + + # Load current config from config.json + app_dir = _STARLARK_APPS_DIR / app_id + config_file = app_dir / "config.json" + current_config = {} + if config_file.exists(): + try: + with open(config_file, 'r') as f: + current_config = json.load(f) + except Exception as e: + logger.warning(f"Failed to load config for {app_id}: {e}") + + # Update config with new values (excluding timing keys) + current_config.update(data) + + # Write updated config to config.json + try: + with open(config_file, 'w') as f: + json.dump(current_config, f, indent=2) + except Exception as e: + logger.error(f"Failed to save config.json for {app_id}: {e}") + return jsonify({'status': 'error', 'message': f'Failed to save configuration: {e}'}), 500 + + # Also update manifest for backward compatibility + app_data.setdefault('config', {}).update(data) + + if _write_starlark_manifest(manifest): + return jsonify({'status': 'success', 'message': 'Configuration updated', 'config': current_config}) + else: + return jsonify({'status': 'error', 'message': 'Failed to save manifest'}), 500 + + except Exception as e: + logger.error(f"Error updating config for {app_id}: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/apps//toggle', methods=['POST']) +def toggle_starlark_app(app_id): + """Enable or disable a Starlark app.""" + try: + data = request.get_json() or {} + + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + enabled = data.get('enabled') + if enabled is None: + enabled = not app.is_enabled() + app.manifest['enabled'] = enabled + # Use safe manifest update to prevent race conditions + def update_fn(manifest): + manifest['apps'][app_id]['enabled'] = enabled + starlark_plugin._update_manifest_safe(update_fn) + return jsonify({'status': 'success', 'message': f"App {'enabled' if enabled else 'disabled'}", 'enabled': enabled}) + + # Standalone: update manifest directly + manifest = _read_starlark_manifest() + app_data = manifest.get('apps', {}).get(app_id) + if not app_data: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + enabled = data.get('enabled') + if enabled is None: + enabled = not app_data.get('enabled', True) + app_data['enabled'] = enabled + if _write_starlark_manifest(manifest): + return jsonify({'status': 'success', 'message': f"App {'enabled' if enabled else 'disabled'}", 'enabled': enabled}) + else: + return jsonify({'status': 'error', 'message': 'Failed to save'}), 500 + + except Exception as e: + logger.error(f"Error toggling app {app_id}: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/apps//render', methods=['POST']) +def render_starlark_app(app_id): + """Force render a Starlark app.""" + try: + starlark_plugin = _get_starlark_plugin() + if not starlark_plugin: + return jsonify({'status': 'error', 'message': 'Rendering requires the main LEDMatrix service (plugin not loaded in web service)'}), 503 + + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + success = starlark_plugin._render_app(app, force=True) + if success: + return jsonify({'status': 'success', 'message': 'App rendered', 'frame_count': len(app.frames) if app.frames else 0}) + else: + return jsonify({'status': 'error', 'message': 'Failed to render app'}), 500 + + except Exception as e: + logger.error(f"Error rendering app {app_id}: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/repository/browse', methods=['GET']) +def browse_tronbyte_repository(): + """Browse all apps in the Tronbyte repository (bulk cached fetch). + + Returns ALL apps with metadata, categories, and authors. + Filtering/sorting/pagination is handled client-side. + Results are cached server-side for 2 hours. + """ + try: + TronbyteRepository = _get_tronbyte_repository_class() + + config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + github_token = config.get('github_token') + repo = TronbyteRepository(github_token=github_token) + + result = repo.list_all_apps_cached() + + rate_limit = repo.get_rate_limit_info() + + return jsonify({ + 'status': 'success', + 'apps': result['apps'], + 'categories': result['categories'], + 'authors': result['authors'], + 'count': result['count'], + 'cached': result['cached'], + 'rate_limit': rate_limit, + }) + + except Exception as e: + logger.error(f"Error browsing repository: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/repository/install', methods=['POST']) +def install_from_tronbyte_repository(): + """Install an app from the Tronbyte repository.""" + try: + data = request.get_json() + if not data or 'app_id' not in data: + return jsonify({'status': 'error', 'message': 'app_id is required'}), 400 + + app_id, app_id_error = _validate_and_sanitize_app_id(data['app_id']) + if app_id_error: + return jsonify({'status': 'error', 'message': f'Invalid app_id: {app_id_error}'}), 400 + + TronbyteRepository = _get_tronbyte_repository_class() + import tempfile + + config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + github_token = config.get('github_token') + repo = TronbyteRepository(github_token=github_token) + + success, metadata, error = repo.get_app_metadata(data['app_id']) + if not success: + return jsonify({'status': 'error', 'message': f'Failed to fetch app metadata: {error}'}), 404 + + with tempfile.NamedTemporaryFile(delete=False, suffix='.star') as tmp: + temp_path = tmp.name + + try: + # Pass filename from metadata (e.g., "analog_clock.star" for analogclock app) + # Note: manifest uses 'fileName' (camelCase), not 'filename' + filename = metadata.get('fileName') if metadata else None + success, error = repo.download_star_file(data['app_id'], Path(temp_path), filename=filename) + if not success: + return jsonify({'status': 'error', 'message': f'Failed to download app: {error}'}), 500 + + # Download assets (images, sources, etc.) to a temp directory + import tempfile + temp_assets_dir = tempfile.mkdtemp() + try: + success_assets, error_assets = repo.download_app_assets(data['app_id'], Path(temp_assets_dir)) + # Asset download is non-critical - log warning but continue if it fails + if not success_assets: + logger.warning(f"Failed to download assets for {data['app_id']}: {error_assets}") + + render_interval = data.get('render_interval', 300) + ri, err = _validate_timing_value(render_interval, 'render_interval') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + render_interval = ri or 300 + + display_duration = data.get('display_duration', 15) + dd, err = _validate_timing_value(display_duration, 'display_duration') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + display_duration = dd or 15 + + install_metadata = { + 'name': metadata.get('name', app_id) if metadata else app_id, + 'render_interval': render_interval, + 'display_duration': display_duration + } + + starlark_plugin = _get_starlark_plugin() + if starlark_plugin: + success = starlark_plugin.install_app(app_id, temp_path, install_metadata, assets_dir=temp_assets_dir) + else: + success = _install_star_file(app_id, temp_path, install_metadata, assets_dir=temp_assets_dir) + finally: + # Clean up temp assets directory + import shutil + try: + shutil.rmtree(temp_assets_dir) + except OSError: + pass + + if success: + return jsonify({'status': 'success', 'message': f'App installed: {metadata.get("name", app_id) if metadata else app_id}', 'app_id': app_id}) + else: + return jsonify({'status': 'error', 'message': 'Failed to install app'}), 500 + finally: + try: + os.unlink(temp_path) + except OSError: + pass + + except Exception as e: + logger.error(f"Error installing from repository: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/repository/categories', methods=['GET']) +def get_tronbyte_categories(): + """Get list of available app categories (uses bulk cache).""" + try: + TronbyteRepository = _get_tronbyte_repository_class() + config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + repo = TronbyteRepository(github_token=config.get('github_token')) + + result = repo.list_all_apps_cached() + + return jsonify({'status': 'success', 'categories': result['categories']}) + + except Exception as e: + logger.error(f"Error fetching categories: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 + + +@api_v3.route('/starlark/install-pixlet', methods=['POST']) +def install_pixlet(): + """Download and install Pixlet binary.""" + try: + script_path = PROJECT_ROOT / 'scripts' / 'download_pixlet.sh' + if not script_path.exists(): + return jsonify({'status': 'error', 'message': 'Installation script not found'}), 404 + + os.chmod(script_path, 0o755) + + result = subprocess.run( + [str(script_path)], + cwd=str(PROJECT_ROOT), + capture_output=True, + text=True, + timeout=300 + ) + + if result.returncode == 0: + logger.info("Pixlet downloaded successfully") + return jsonify({'status': 'success', 'message': 'Pixlet installed successfully!', 'output': result.stdout}) + else: + return jsonify({'status': 'error', 'message': f'Failed to download Pixlet: {result.stderr}'}), 500 + + except subprocess.TimeoutExpired: + return jsonify({'status': 'error', 'message': 'Download timed out'}), 500 + except Exception as e: + logger.error(f"Error installing Pixlet: {e}") + return jsonify({'status': 'error', 'message': str(e)}), 500 From 5c231cd0213057bc7967c0b918c6fc22e58ba0a1 Mon Sep 17 00:00:00 2001 From: kmce2019 <50456552+kmce2019@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:36:05 -0500 Subject: [PATCH 2/7] fix: cache dynamically loaded Starlark modules --- .../web_interface/test_starlark_api_routes.py | 17 +++++ web_interface/blueprints/api_v3.py | 72 +++++++++---------- 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/test/web_interface/test_starlark_api_routes.py b/test/web_interface/test_starlark_api_routes.py index 5a75b94c..6c97b0bb 100644 --- a/test/web_interface/test_starlark_api_routes.py +++ b/test/web_interface/test_starlark_api_routes.py @@ -1,5 +1,7 @@ """Regression tests for the Starlark API routes used by the v3 plugin UI.""" +import sys + import pytest from flask import Flask @@ -75,3 +77,18 @@ def test_app_path_traversal_is_rejected(client): assert response.status_code == 400 assert "invalid app_id" in response.get_json()["message"].lower() + + +@pytest.mark.parametrize(("module_name", "helper", "class_name"), [ + ("tronbyte_repository", mod._get_tronbyte_repository_class, "TronbyteRepository"), + ("pixlet_renderer", mod._get_pixlet_renderer_class, "PixletRenderer"), +]) +def test_dynamic_starlark_loaders_can_be_called_repeatedly( + module_name, helper, class_name): + sys.modules.pop(module_name, None) + + first = helper() + second = helper() + + assert first is second + assert first is getattr(sys.modules[module_name], class_name) diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 09135c53..96de2dad 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -8477,58 +8477,50 @@ def backup_delete(filename): # ─── Starlark Apps API ────────────────────────────────────────────────────── -def _get_tronbyte_repository_class() -> Type[Any]: - """Import TronbyteRepository from plugin-repos directory.""" +def _load_starlark_class(module_name: str, filename: str, class_name: str) -> Type[Any]: + """Load and cache a class from the bundled Starlark plugin. + + Modules created with ``spec_from_file_location`` are not reliably + reloadable because their parent directory is not an importable package. + Keep the successfully executed module in ``sys.modules`` and reuse it on + subsequent requests instead. + """ import importlib.util - import importlib - module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'tronbyte_repository.py' + module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / filename if not module_path.exists(): - raise ImportError(f"TronbyteRepository module not found at {module_path}") + raise ImportError(f"Starlark module not found at {module_path}") - # If already imported, reload to pick up code changes - if "tronbyte_repository" in sys.modules: - importlib.reload(sys.modules["tronbyte_repository"]) - return sys.modules["tronbyte_repository"].TronbyteRepository + cached_module = sys.modules.get(module_name) + if cached_module is not None: + return getattr(cached_module, class_name) - spec = importlib.util.spec_from_file_location("tronbyte_repository", str(module_path)) - if spec is None: - raise ImportError(f"Failed to create module spec for tronbyte_repository at {module_path}") + spec = importlib.util.spec_from_file_location(module_name, str(module_path)) + if spec is None or spec.loader is None: + raise ImportError(f"Failed to create module spec for {module_name} at {module_path}") module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError("Failed to create module from spec for tronbyte_repository") + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + return getattr(module, class_name) + - sys.modules["tronbyte_repository"] = module - spec.loader.exec_module(module) - return module.TronbyteRepository +def _get_tronbyte_repository_class() -> Type[Any]: + """Import TronbyteRepository from plugin-repos directory.""" + return _load_starlark_class( + 'tronbyte_repository', 'tronbyte_repository.py', 'TronbyteRepository' + ) def _get_pixlet_renderer_class() -> Type[Any]: """Import PixletRenderer from plugin-repos directory.""" - import importlib.util - import importlib - - module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'pixlet_renderer.py' - if not module_path.exists(): - raise ImportError(f"PixletRenderer module not found at {module_path}") - - # If already imported, reload to pick up code changes - if "pixlet_renderer" in sys.modules: - importlib.reload(sys.modules["pixlet_renderer"]) - return sys.modules["pixlet_renderer"].PixletRenderer - - spec = importlib.util.spec_from_file_location("pixlet_renderer", str(module_path)) - if spec is None: - raise ImportError(f"Failed to create module spec for pixlet_renderer at {module_path}") - - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError("Failed to create module from spec for pixlet_renderer") - - sys.modules["pixlet_renderer"] = module - spec.loader.exec_module(module) - return module.PixletRenderer + return _load_starlark_class( + 'pixlet_renderer', 'pixlet_renderer.py', 'PixletRenderer' + ) def _validate_and_sanitize_app_id(app_id: Optional[str], fallback_source: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]: From 8ae26ff630d2c789e249efb08da76b0129d3eabd Mon Sep 17 00:00:00 2001 From: kmce2019 <50456552+kmce2019@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:47:42 -0500 Subject: [PATCH 3/7] fix: expose installed Starlark app configuration --- .../web_interface/test_starlark_api_routes.py | 29 +++++ web_interface/blueprints/api_v3.py | 106 ++++++++++++++++-- 2 files changed, 127 insertions(+), 8 deletions(-) diff --git a/test/web_interface/test_starlark_api_routes.py b/test/web_interface/test_starlark_api_routes.py index 6c97b0bb..80947717 100644 --- a/test/web_interface/test_starlark_api_routes.py +++ b/test/web_interface/test_starlark_api_routes.py @@ -72,6 +72,19 @@ def test_apps_falls_back_to_the_standalone_manifest(client): assert apps[0]["enabled"] is True +def test_installed_plugin_entries_include_configurable_starlark_apps(client): + assert mod._write_starlark_manifest({ + "apps": {"clock": {"name": "Clock", "enabled": True}} + }) + + entries = mod._get_starlark_plugin_entries() + + assert len(entries) == 1 + assert entries[0]["id"] == "starlark:clock" + assert entries[0]["is_starlark_app"] is True + assert entries[0]["enabled"] is True + + def test_app_path_traversal_is_rejected(client): response = client.get("/api/v3/starlark/apps/..%5Csecret") @@ -92,3 +105,19 @@ def test_dynamic_starlark_loaders_can_be_called_repeatedly( assert first is second assert first is getattr(sys.modules[module_name], class_name) + + +@pytest.mark.parametrize(("config", "expected"), [ + ({"github": {"api_token": " nested-token "}}, "nested-token"), + ({"github_token": "legacy-token"}, "legacy-token"), + ({"github": {"api_token": "YOUR_GITHUB_PERSONAL_ACCESS_TOKEN"}}, None), +]) +def test_starlark_repository_uses_configured_github_secret( + monkeypatch, config, expected): + class ConfigManager: + def load_config(self): + return config + + monkeypatch.setattr(mod.api_v3, "config_manager", ConfigManager(), raising=False) + + assert mod._get_starlark_github_token() == expected diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 96de2dad..54772a48 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -2688,6 +2688,7 @@ def _build_plugin_entry_inner(plugin_info, plugin_id): with ThreadPoolExecutor(max_workers=8) as executor: results = list(executor.map(_build_plugin_entry, all_plugin_info)) plugins = [r for r in results if r is not None] + plugins.extend(_get_starlark_plugin_entries()) return jsonify({'status': 'success', 'data': {'plugins': plugins}}) except Exception as e: @@ -2994,6 +2995,39 @@ def toggle_plugin(): current_enabled = config.get(plugin_id, {}).get('enabled', False) enabled = not current_enabled + # Starlark apps are presented as virtual plugins in the installed + # list, but their enabled state lives in the Starlark manifest rather + # than the main plugin configuration. + if plugin_id.startswith('starlark:'): + starlark_app_id = plugin_id[len('starlark:'):] + starlark_plugin = _get_starlark_plugin() + if starlark_plugin and starlark_app_id in starlark_plugin.apps: + app = starlark_plugin.apps[starlark_app_id] + app.manifest['enabled'] = enabled + + def update_fn(manifest): + manifest['apps'][starlark_app_id]['enabled'] = enabled + + if not starlark_plugin._update_manifest_safe(update_fn): + return jsonify({'status': 'error', 'message': 'Failed to save manifest'}), 500 + else: + manifest = _read_starlark_manifest() + app_data = manifest.get('apps', {}).get(starlark_app_id) + if not app_data: + return jsonify({ + 'status': 'error', + 'message': f'Starlark app not found: {starlark_app_id}', + }), 404 + app_data['enabled'] = enabled + if not _write_starlark_manifest(manifest): + return jsonify({'status': 'error', 'message': 'Failed to save manifest'}), 500 + + return jsonify({ + 'status': 'success', + 'message': f"Starlark app {'enabled' if enabled else 'disabled'}", + 'enabled': enabled, + }) + # Check if plugin exists in manifests (discovered but may not be loaded) if plugin_id not in api_v3.plugin_manager.plugin_manifests: return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 @@ -8523,6 +8557,23 @@ def _get_pixlet_renderer_class() -> Type[Any]: ) +def _get_starlark_github_token() -> Optional[str]: + """Return the configured GitHub token for Starlark repository requests.""" + if not api_v3.config_manager: + return None + + config = api_v3.config_manager.load_config() + github = config.get('github', {}) + token = github.get('api_token') if isinstance(github, dict) else None + token = token or config.get('github_token') # Legacy configuration key. + if not isinstance(token, str): + return None + token = token.strip() + if not token or token == 'YOUR_GITHUB_PERSONAL_ACCESS_TOKEN': + return None + return token + + def _validate_and_sanitize_app_id(app_id: Optional[str], fallback_source: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]: """Validate and sanitize app_id to a safe slug.""" if not app_id and fallback_source: @@ -8562,6 +8613,50 @@ def _get_starlark_plugin() -> Optional[Any]: return api_v3.plugin_manager.get_plugin('starlark-apps') +def _get_starlark_plugin_entries() -> list[Dict[str, Any]]: + """Build virtual installed-plugin entries for each Starlark app.""" + entries = [] + starlark_plugin = _get_starlark_plugin() + if starlark_plugin and hasattr(starlark_plugin, 'apps'): + apps = ( + (app_id, app.manifest, app.is_enabled(), True) + for app_id, app in starlark_plugin.apps.items() + ) + else: + manifest = _read_starlark_manifest() + apps = ( + (app_id, app_data, app_data.get('enabled', True), False) + for app_id, app_data in manifest.get('apps', {}).items() + ) + + for app_id, app_data, enabled, loaded in apps: + entries.append({ + 'id': f'starlark:{app_id}', + 'name': app_data.get('name', app_id), + 'version': 'starlark', + 'latest_version': '', + 'update_available': False, + 'author': app_data.get('author', 'Tronbyte Community'), + 'category': 'Starlark App', + 'description': app_data.get('summary', 'Starlark app'), + 'tags': ['starlark'], + 'enabled': enabled, + 'verified': False, + 'loaded': loaded, + 'state': None, + 'error_info': None, + 'last_updated': None, + 'last_commit': None, + 'last_commit_message': None, + 'branch': None, + 'web_ui_actions': [], + 'vegas_mode': 'fixed', + 'vegas_content_type': 'multi', + 'is_starlark_app': True, + }) + return entries + + def _validate_starlark_app_path(app_id: str) -> Tuple[bool, Optional[str]]: """ Validate app_id for path traversal attacks before filesystem access. @@ -9210,9 +9305,7 @@ def browse_tronbyte_repository(): try: TronbyteRepository = _get_tronbyte_repository_class() - config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - github_token = config.get('github_token') - repo = TronbyteRepository(github_token=github_token) + repo = TronbyteRepository(github_token=_get_starlark_github_token()) result = repo.list_all_apps_cached() @@ -9248,9 +9341,7 @@ def install_from_tronbyte_repository(): TronbyteRepository = _get_tronbyte_repository_class() import tempfile - config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - github_token = config.get('github_token') - repo = TronbyteRepository(github_token=github_token) + repo = TronbyteRepository(github_token=_get_starlark_github_token()) success, metadata, error = repo.get_app_metadata(data['app_id']) if not success: @@ -9327,8 +9418,7 @@ def get_tronbyte_categories(): """Get list of available app categories (uses bulk cache).""" try: TronbyteRepository = _get_tronbyte_repository_class() - config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - repo = TronbyteRepository(github_token=config.get('github_token')) + repo = TronbyteRepository(github_token=_get_starlark_github_token()) result = repo.list_all_apps_cached() From 0d1387d2724c00ab9ffe511cfd06b26770994099 Mon Sep 17 00:00:00 2001 From: kmce2019 <50456552+kmce2019@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:08:58 -0500 Subject: [PATCH 4/7] fix: resolve static Starlark schema values --- plugin-repos/starlark-apps/pixlet_renderer.py | 258 +++++++++++++----- .../starlark-apps/tools/rebuild_schemas.py | 167 ++++++++++++ test/test_starlark_schema_parser.py | 150 ++++++++++ 3 files changed, 507 insertions(+), 68 deletions(-) create mode 100644 plugin-repos/starlark-apps/tools/rebuild_schemas.py create mode 100644 test/test_starlark_schema_parser.py diff --git a/plugin-repos/starlark-apps/pixlet_renderer.py b/plugin-repos/starlark-apps/pixlet_renderer.py index 40f8f59d..f6c45f70 100644 --- a/plugin-repos/starlark-apps/pixlet_renderer.py +++ b/plugin-repos/starlark-apps/pixlet_renderer.py @@ -5,6 +5,7 @@ Supports bundled binaries and system-installed Pixlet. """ +import ast import json import logging import os @@ -17,6 +18,8 @@ logger = logging.getLogger(__name__) +_UNRESOLVED = object() + class PixletRenderer: """ @@ -373,7 +376,7 @@ def _parse_schema_from_source(self, content: str, file_path: str) -> Optional[Di Returns: Schema dict with format {"version": "1", "schema": [...]}, or None """ - # Extract variable definitions (for dropdown options) + # Extract statically resolvable constants and option variables. var_table = self._extract_variable_definitions(content) # Extract get_schema() function body @@ -449,32 +452,181 @@ def _parse_schema_from_source(self, content: str, file_path: str) -> Optional[Di "schema": schema_fields } - def _extract_variable_definitions(self, content: str) -> Dict[str, List[Dict]]: + def _extract_variable_definitions(self, content: str) -> Dict[str, Any]: """ - Extract top-level variable assignments (for dropdown options). + Extract safe top-level constants and get_schema() local variables. Args: content: .star file content Returns: - Dict mapping variable names to their option lists + Dict mapping variable names to statically resolved values """ - var_table = {} - - # Find variable definitions like: variableName = [schema.Option(...), ...] - var_pattern = r'^(\w+)\s*=\s*\[(.*?schema\.Option.*?)\]' - matches = re.finditer(var_pattern, content, re.MULTILINE | re.DOTALL) - - for match in matches: - var_name = match.group(1) - options_text = match.group(2) + values: Dict[str, Any] = {} + try: + tree = ast.parse(content) + except SyntaxError as e: + logger.warning("Could not statically parse Starlark schema variables: %s", e) + return values + + def collect_assignment(node: ast.stmt) -> None: + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + return + target = node.targets[0] + if not isinstance(target, ast.Name): + return + value = self._safe_eval_node(node.value, values) + if value is _UNRESOLVED: + logger.debug("Could not statically resolve Starlark variable %s", target.id) + return + values[target.id] = value + + # Assignment order matters because constants commonly reference an + # earlier constant (DEFAULT_ANIMATION = PACMAN_ANIMATION). + for node in tree.body: + collect_assignment(node) + + # Option variables are generally local to get_schema(). Resolve its + # straight-line assignments using the top-level constants above. + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == 'get_schema': + for statement in node.body: + collect_assignment(statement) + break - # Parse schema.Option entries - options = self._parse_schema_options(options_text, {}) - if options: - var_table[var_name] = options + return values + + def _safe_eval_node(self, node: ast.AST, values: Dict[str, Any]) -> Any: + """Resolve the small, data-only expression subset used by schemas.""" + if isinstance(node, ast.Constant): + if isinstance(node.value, (str, int, float, bool)) or node.value is None: + return node.value + return _UNRESOLVED + + if isinstance(node, ast.Name): + return values.get(node.id, _UNRESOLVED) + + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + operand = self._safe_eval_node(node.operand, values) + if isinstance(operand, (int, float)) and not isinstance(operand, bool): + return -operand if isinstance(node.op, ast.USub) else operand + return _UNRESOLVED + + if isinstance(node, ast.Dict): + result = {} + for key_node, value_node in zip(node.keys, node.values): + key = self._safe_eval_node(key_node, values) + value = self._safe_eval_node(value_node, values) + if key is _UNRESOLVED or value is _UNRESOLVED: + return _UNRESOLVED + if not isinstance(key, (str, int, float, bool)): + return _UNRESOLVED + result[key] = value + return result + + if isinstance(node, (ast.List, ast.Tuple)): + result = [] + for item_node in node.elts: + item = self._safe_eval_node(item_node, values) + if item is _UNRESOLVED: + return _UNRESOLVED + result.append(item) + return result + + if isinstance(node, ast.Call) and self._is_schema_option_call(node): + if node.args: + return _UNRESOLVED + kwargs = {keyword.arg: self._safe_eval_node(keyword.value, values) + for keyword in node.keywords if keyword.arg} + if set(kwargs) != {'display', 'value'}: + return _UNRESOLVED + if any(value is _UNRESOLVED for value in kwargs.values()): + return _UNRESOLVED + return {'display': kwargs['display'], 'value': kwargs['value']} + + if isinstance(node, ast.ListComp): + return self._safe_eval_option_comprehension(node, values) + + return _UNRESOLVED + + @staticmethod + def _is_schema_option_call(node: ast.AST) -> bool: + return (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == 'schema' + and node.func.attr == 'Option') + + def _safe_eval_option_comprehension( + self, node: ast.ListComp, values: Dict[str, Any]) -> Any: + """Resolve ``schema.Option(...) for key, value in DICT.items()``.""" + if len(node.generators) != 1 or not self._is_schema_option_call(node.elt): + return _UNRESOLVED + generator = node.generators[0] + if generator.ifs or generator.is_async: + return _UNRESOLVED + if not (isinstance(generator.target, (ast.Tuple, ast.List)) + and len(generator.target.elts) == 2 + and all(isinstance(item, ast.Name) for item in generator.target.elts)): + return _UNRESOLVED + iterator = generator.iter + if not (isinstance(iterator, ast.Call) and not iterator.args and not iterator.keywords + and isinstance(iterator.func, ast.Attribute) + and iterator.func.attr == 'items'): + return _UNRESOLVED + mapping = self._safe_eval_node(iterator.func.value, values) + if not isinstance(mapping, dict): + return _UNRESOLVED + + key_name, value_name = (item.id for item in generator.target.elts) + options = [] + for key, value in mapping.items(): + local_values = dict(values) + local_values[key_name] = key + local_values[value_name] = value + option = self._safe_eval_node(node.elt, local_values) + if option is _UNRESOLVED: + return _UNRESOLVED + options.append(option) + return options - return var_table + def _resolve_expression(self, expression: str, values: Dict[str, Any]) -> Any: + try: + node = ast.parse(expression.strip(), mode='eval').body + except SyntaxError: + return _UNRESOLVED + return self._safe_eval_node(node, values) + + @staticmethod + def _extract_keyword_expression(params_text: str, keyword: str) -> Optional[str]: + """Extract a keyword value while respecting nested brackets and strings.""" + match = re.search(rf'\b{re.escape(keyword)}\s*=\s*', params_text) + if not match: + return None + start = match.end() + stack = [] + quote = None + escaped = False + for index in range(start, len(params_text)): + char = params_text[index] + if quote: + if escaped: + escaped = False + elif char == '\\': + escaped = True + elif char == quote: + quote = None + continue + if char in ('"', "'"): + quote = char + elif char in '([{': + stack.append(char) + elif char in ')]}': + if stack: + stack.pop() + elif char == ',' and not stack: + return params_text[start:index].strip() + return params_text[start:].strip() def _extract_get_schema_body(self, content: str) -> Optional[str]: """ @@ -586,50 +738,27 @@ def _parse_schema_field(self, field_type: str, params_text: str, var_table: Dict if icon_match: field_dict['icon'] = icon_match.group(1) - # default (can be string, bool, or variable reference) - # First try to match quoted strings (which may contain commas) - default_match = re.search(r'default\s*=\s*"([^"]*)"', params_text) - if not default_match: - # Try single quotes - default_match = re.search(r"default\s*=\s*'([^']*)'", params_text) - if not default_match: - # Fall back to unquoted value (stop at comma or closing paren) - default_match = re.search(r'default\s*=\s*([^,\)]+)', params_text) - - if default_match: - default_value = default_match.group(1).strip() - # Handle boolean - if default_value in ('True', 'False'): - field_dict['default'] = default_value.lower() - # Handle string literal from first two patterns (already extracted without quotes) - elif re.search(r'default\s*=\s*["\']', params_text): - # This was a quoted string, use the captured content directly + default_expression = self._extract_keyword_expression(params_text, 'default') + if default_expression is not None: + default_value = self._resolve_expression(default_expression, var_table) + if default_value is _UNRESOLVED: + logger.warning("Could not statically resolve schema default for %s: %s", + field_dict['id'], default_expression) + elif isinstance(default_value, (str, int, float, bool)) or default_value is None: field_dict['default'] = default_value - # Handle variable reference (can't resolve, use as-is) - else: - # Try to extract just the value if it's like options[0].value - if '.' in default_value or '[' in default_value: - # Complex expression, skip default - pass - else: - field_dict['default'] = default_value # For dropdown, extract options if type_of == 'dropdown': - options_match = re.search(r'options\s*=\s*([^,\)]+)', params_text) - if options_match: - options_ref = options_match.group(1).strip() - # Check if it's a variable reference - if options_ref in var_table: - field_dict['options'] = var_table[options_ref] - # Or inline options - elif options_ref.startswith('['): - # Find the full options array (handle nested brackets) - # This is tricky, for now try to extract inline options - inline_match = re.search(r'options\s*=\s*(\[.*?\])', params_text, re.DOTALL) - if inline_match: - options_text = inline_match.group(1) - field_dict['options'] = self._parse_schema_options(options_text, var_table) + options_expression = self._extract_keyword_expression(params_text, 'options') + if options_expression is not None: + options = self._resolve_expression(options_expression, var_table) + if (isinstance(options, list) + and all(isinstance(option, dict) + and set(option) == {'display', 'value'} for option in options)): + field_dict['options'] = options + else: + logger.warning("Could not statically resolve dropdown options for %s: %s", + field_dict['id'], options_expression) return field_dict @@ -646,14 +775,7 @@ def _parse_schema_options(self, options_text: str, var_table: Dict) -> List[Dict """ options = [] - # Match schema.Option(display = "...", value = "...") - option_pattern = r'schema\.Option\s*\(\s*display\s*=\s*"([^"]+)"\s*,\s*value\s*=\s*"([^"]+)"\s*\)' - matches = re.finditer(option_pattern, options_text) - - for match in matches: - options.append({ - "display": match.group(1), - "value": match.group(2) - }) - + resolved = self._resolve_expression(options_text, var_table) + if isinstance(resolved, list): + return resolved return options diff --git a/plugin-repos/starlark-apps/tools/rebuild_schemas.py b/plugin-repos/starlark-apps/tools/rebuild_schemas.py new file mode 100644 index 00000000..f69e48f0 --- /dev/null +++ b/plugin-repos/starlark-apps/tools/rebuild_schemas.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Regenerate schemas and safely repair defaults for installed Starlark apps.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import re +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_APPS_DIR = PROJECT_ROOT / "starlark-apps" +RENDERER_PATH = Path(__file__).resolve().parents[1] / "pixlet_renderer.py" +SYMBOLIC_DEFAULT = re.compile(r"^DEFAULT_[A-Z0-9_]+$") + + +@dataclass +class Summary: + apps_scanned: int = 0 + schemas_regenerated: int = 0 + configs_repaired: int = 0 + unresolved_fields: int = 0 + errors: int = 0 + + +def _load_renderer_class(): + spec = importlib.util.spec_from_file_location("starlark_schema_rebuild_renderer", RENDERER_PATH) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load schema renderer from {RENDERER_PATH}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module.PixletRenderer + + +def _atomic_write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + mode = path.stat().st_mode & 0o777 if path.exists() else 0o664 + fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temp_path = Path(temp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temp_path, mode) + os.replace(temp_path, path) + finally: + if temp_path.exists(): + temp_path.unlink() + + +def _read_json(path: Path, fallback: Any) -> Any: + if not path.exists(): + return fallback + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _find_star_file(app_dir: Path, manifest_entry: dict[str, Any]) -> Optional[Path]: + configured = manifest_entry.get("star_file") + if configured: + configured_path = Path(configured) + candidate = configured_path if configured_path.is_absolute() else app_dir / configured_path + if candidate.is_file(): + return candidate + conventional = app_dir / f"{app_dir.name}.star" + if conventional.is_file(): + return conventional + candidates = sorted(app_dir.glob("*.star")) + return candidates[0] if len(candidates) == 1 else None + + +def rebuild_schemas(apps_dir: Path = DEFAULT_APPS_DIR) -> Summary: + summary = Summary() + renderer = _load_renderer_class()() + manifest = _read_json(apps_dir / "manifest.json", {"apps": {}}) + manifest_apps = manifest.get("apps", {}) if isinstance(manifest, dict) else {} + + if not apps_dir.is_dir(): + print(f"Apps directory does not exist: {apps_dir}", file=sys.stderr) + summary.errors += 1 + return summary + + for app_dir in sorted(path for path in apps_dir.iterdir() if path.is_dir()): + summary.apps_scanned += 1 + try: + star_file = _find_star_file(app_dir, manifest_apps.get(app_dir.name, {})) + if star_file is None: + raise FileNotFoundError("could not identify a single .star file") + + success, schema, error = renderer.extract_schema(str(star_file)) + if not success: + raise RuntimeError(error or "schema extraction failed") + if schema is None: + print(f"{app_dir.name}: no schema") + continue + + _atomic_write_json(app_dir / "schema.json", schema) + summary.schemas_regenerated += 1 + + fields = schema.get("fields") or schema.get("schema") or [] + defaults = { + field["id"]: field["default"] + for field in fields + if isinstance(field, dict) and "id" in field and "default" in field + } + for field in fields: + if (isinstance(field, dict) and field.get("typeOf") == "dropdown" + and not field.get("options")): + summary.unresolved_fields += 1 + + config_path = app_dir / "config.json" + config = _read_json(config_path, {}) + if not isinstance(config, dict): + raise ValueError("config.json must contain an object") + repaired = False + for field_id, default in defaults.items(): + if field_id not in config: + config[field_id] = default + repaired = True + elif (isinstance(config[field_id], str) + and SYMBOLIC_DEFAULT.fullmatch(config[field_id])): + config[field_id] = default + repaired = True + + for value in config.values(): + if isinstance(value, str) and SYMBOLIC_DEFAULT.fullmatch(value): + summary.unresolved_fields += 1 + + if repaired: + _atomic_write_json(config_path, config) + summary.configs_repaired += 1 + print(f"{app_dir.name}: schema regenerated" + + (", config repaired" if repaired else "")) + except Exception as exc: + summary.errors += 1 + print(f"{app_dir.name}: ERROR: {exc}", file=sys.stderr) + + return summary + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apps-dir", type=Path, default=DEFAULT_APPS_DIR, + help=f"installed apps directory (default: {DEFAULT_APPS_DIR})") + args = parser.parse_args() + summary = rebuild_schemas(args.apps_dir.resolve()) + print("\nSummary") + print(f"apps scanned: {summary.apps_scanned}") + print(f"schemas regenerated: {summary.schemas_regenerated}") + print(f"configs repaired: {summary.configs_repaired}") + print(f"unresolved fields: {summary.unresolved_fields}") + print(f"errors: {summary.errors}") + return 1 if summary.errors else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/test_starlark_schema_parser.py b/test/test_starlark_schema_parser.py new file mode 100644 index 00000000..2fa4074d --- /dev/null +++ b/test/test_starlark_schema_parser.py @@ -0,0 +1,150 @@ +"""Static Starlark schema extraction and installed-app migration tests.""" + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] + + +def _load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +renderer_module = _load_module( + "test_starlark_pixlet_renderer", + ROOT / "plugin-repos/starlark-apps/pixlet_renderer.py", +) +rebuild_module = _load_module( + "test_starlark_rebuild_schemas", + ROOT / "plugin-repos/starlark-apps/tools/rebuild_schemas.py", +) + + +@pytest.fixture +def renderer(): + return renderer_module.PixletRenderer() + + +def _extract(renderer, tmp_path, source): + star_file = tmp_path / "app.star" + star_file.write_text(source, encoding="utf-8") + success, schema, error = renderer.extract_schema(str(star_file)) + assert success, error + return {field["id"]: field for field in schema["schema"]} + + +def test_resolves_defaults_and_all_supported_dropdown_option_forms(renderer, tmp_path): + fields = _extract(renderer, tmp_path, ''' +DEFAULT_NAME = "two" +DEFAULT_COUNT = 7 +DEFAULT_RATIO = 1.5 +DEFAULT_ENABLED = True +DEFAULT_COLOR = "#A1B2C3" +CHOICES = {"One": "1", "Two": "2"} +literal_options = [ + schema.Option(display = "One", value = "1"), + schema.Option(display = "Two", value = "2"), +] +def get_schema(): + generated_options = [ + schema.Option(display = key, value = value) + for key, value in CHOICES.items() + ] + return schema.Schema(version = "1", fields = [ + schema.Dropdown(id = "inline", default = "1", options = [ + schema.Option(display = "One", value = "1"), + schema.Option(display = "Two", value = "2"), + ]), + schema.Dropdown(id = "variable", default = DEFAULT_NAME, options = literal_options), + schema.Dropdown(id = "generated", default = DEFAULT_NAME, options = generated_options), + schema.Text(id = "count", default = DEFAULT_COUNT), + schema.Text(id = "ratio", default = DEFAULT_RATIO), + schema.Toggle(id = "enabled", default = DEFAULT_ENABLED), + schema.Color(id = "color", default = DEFAULT_COLOR), + ]) +''') + + expected_options = [ + {"display": "One", "value": "1"}, + {"display": "Two", "value": "2"}, + ] + assert fields["inline"]["options"] == expected_options + assert fields["variable"]["options"] == expected_options + assert fields["generated"]["options"] == expected_options + assert fields["variable"]["default"] == "two" + assert fields["count"]["default"] == 7 + assert fields["ratio"]["default"] == 1.5 + assert fields["enabled"]["default"] is True + assert fields["color"]["default"] == "#A1B2C3" + + +def test_unresolved_expressions_are_omitted_without_crashing(renderer, tmp_path, caplog): + fields = _extract(renderer, tmp_path, ''' +UNRELATED = [item + 1 for item in [1, 2]] +def dynamic_value(): + return "runtime" +def get_schema(): + return schema.Schema(version = "1", fields = [ + schema.Dropdown(id = "choice", default = dynamic_value(), options = make_options()), + ]) +''') + + assert "default" not in fields["choice"] + assert "options" not in fields["choice"] + assert "Could not statically resolve" in caplog.text + + +def test_rebuild_repairs_only_missing_and_symbolic_defaults(tmp_path): + apps_dir = tmp_path / "starlark-apps" + app_dir = apps_dir / "sample" + app_dir.mkdir(parents=True) + (apps_dir / "manifest.json").write_text( + json.dumps({"apps": {"sample": {"star_file": "sample.star"}}}), encoding="utf-8") + (app_dir / "sample.star").write_text(''' +DEFAULT_SPEED = "2" +DEFAULT_ENABLED = True +def get_schema(): + return schema.Schema(version = "1", fields = [ + schema.Text(id = "speed", default = DEFAULT_SPEED), + schema.Toggle(id = "enabled", default = DEFAULT_ENABLED), + schema.Color(id = "color", default = "#FFFFFF"), + ]) +''', encoding="utf-8") + (app_dir / "config.json").write_text(json.dumps({ + "speed": "DEFAULT_SPEED", + "color": "#123456", + "custom": "keep-me", + }), encoding="utf-8") + + first = rebuild_module.rebuild_schemas(apps_dir) + config = json.loads((app_dir / "config.json").read_text(encoding="utf-8")) + schema = json.loads((app_dir / "schema.json").read_text(encoding="utf-8")) + + assert first.apps_scanned == 1 + assert first.schemas_regenerated == 1 + assert first.configs_repaired == 1 + assert first.unresolved_fields == 0 + assert first.errors == 0 + assert config == { + "speed": "2", + "enabled": True, + "color": "#123456", + "custom": "keep-me", + } + assert {field["id"]: field["default"] for field in schema["schema"]} == { + "speed": "2", "enabled": True, "color": "#FFFFFF", + } + + second = rebuild_module.rebuild_schemas(apps_dir) + assert second.schemas_regenerated == 1 + assert second.configs_repaired == 0 + assert second.errors == 0 From fe037808c13e087f83970c9a73c091f877a4ea9c Mon Sep 17 00:00:00 2001 From: kmce2019 <50456552+kmce2019@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:29:10 -0500 Subject: [PATCH 5/7] fix: rotate Starlark apps by per-mode duration --- plugin-repos/starlark-apps/manager.py | 31 +++++++-- src/display_controller.py | 2 + test/test_starlark_display_duration.py | 96 ++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 test/test_starlark_display_duration.py diff --git a/plugin-repos/starlark-apps/manager.py b/plugin-repos/starlark-apps/manager.py index 3da8c24d..54ca8c32 100644 --- a/plugin-repos/starlark-apps/manager.py +++ b/plugin-repos/starlark-apps/manager.py @@ -681,7 +681,7 @@ def update(self) -> None: if app.is_enabled() and app.should_render(current_time): self._render_app(app, force=False) - def display(self, force_clear: bool = False) -> None: + def display(self, display_mode: Optional[str] = None, force_clear: bool = False) -> bool: """ Display current Starlark app. @@ -692,27 +692,41 @@ def display(self, force_clear: bool = False) -> None: if force_clear: self.display_manager.clear() - # If no current app, try to select one - if not self.current_app: + # Every installed app is registered as its own DisplayController + # mode. Select that exact app when the controller dispatches a + # mode, including manual switches. Do not reset its frame index or + # frame timestamp: WebP animation timing is independent from the + # amount of time this mode remains in the central rotation. + if display_mode is not None: + requested_app = self.apps.get(display_mode) + if requested_app is None or not requested_app.is_enabled(): + self.logger.warning("Starlark display mode is unavailable: %s", display_mode) + return False + if self.current_app is not requested_app: + self.current_app = requested_app + self.logger.debug("Selected Starlark app for mode: %s", display_mode) + elif not self.current_app: self._select_next_app() if not self.current_app: # No apps available self.logger.debug("No Starlark apps to display") - return + return False # Render app if needed if not self.current_app.frames: success = self._render_app(self.current_app, force=True) if not success: self.logger.error(f"Failed to render app: {self.current_app.app_id}") - return + return False # Display current frame self._display_frame() + return True except Exception as e: self.logger.error(f"Error displaying Starlark app: {e}") + return False def _select_next_app(self) -> None: """Select the next enabled app for display.""" @@ -1010,6 +1024,13 @@ def get_display_duration(self) -> float: return float(self.current_app.get_display_duration()) return self.config.get('display_duration', 15.0) + def get_mode_display_duration(self, display_mode: str) -> float: + """Return the effective central-rotation duration for one app mode.""" + app = self.apps.get(display_mode) + if app is not None and app.is_enabled(): + return float(app.get_display_duration()) + return float(self.config.get('display_duration', 15.0)) + # ─── Vegas Mode Integration ────────────────────────────────────── def get_vegas_content(self) -> Optional[List[Image.Image]]: diff --git a/src/display_controller.py b/src/display_controller.py index e739e08e..50d8cd5a 100644 --- a/src/display_controller.py +++ b/src/display_controller.py @@ -1079,6 +1079,8 @@ def _get_display_duration(self, mode_key): # Check plugin-specific duration first if mode_key in self.plugin_modes: plugin_instance = self.plugin_modes[mode_key] + if hasattr(plugin_instance, 'get_mode_display_duration'): + return plugin_instance.get_mode_display_duration(mode_key) if hasattr(plugin_instance, 'get_display_duration'): return plugin_instance.get_display_duration() diff --git a/test/test_starlark_display_duration.py b/test/test_starlark_display_duration.py new file mode 100644 index 00000000..c3958e68 --- /dev/null +++ b/test/test_starlark_display_duration.py @@ -0,0 +1,96 @@ +"""Starlark per-app rotation duration and animation timing regression tests.""" + +import importlib.util +import sys +from pathlib import Path +from unittest.mock import MagicMock + + +ROOT = Path(__file__).resolve().parents[1] +PLUGIN_DIR = ROOT / "plugin-repos/starlark-apps" +sys.path.insert(0, str(PLUGIN_DIR)) +spec = importlib.util.spec_from_file_location( + "test_starlark_duration_manager", PLUGIN_DIR / "manager.py") +manager_module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = manager_module +spec.loader.exec_module(manager_module) + + +def _app(app_id, duration, frames=None): + app = manager_module.StarlarkApp.__new__(manager_module.StarlarkApp) + app.app_id = app_id + app.manifest = {"enabled": True, "display_duration": duration} + app.frames = frames or [(f"{app_id}-frame", 100)] + app.current_frame_index = 0 + app.last_frame_time = 0.0 + return app + + +def _manager(*apps): + manager = manager_module.StarlarkAppsPlugin.__new__(manager_module.StarlarkAppsPlugin) + manager.apps = {app.app_id: app for app in apps} + manager.current_app = None + manager.config = {"display_duration": 15} + manager.display_manager = MagicMock() + manager.logger = MagicMock() + return manager + + +def test_aquarium_advances_at_its_15_second_rotation_boundary(test_display_controller): + manager = _manager(_app("aquarium", 15), _app("clock", 30)) + test_display_controller.plugin_modes = {"aquarium": manager, "clock": manager} + + duration = test_display_controller._get_display_duration("aquarium") + + assert duration == 15 + assert (14.999 >= duration) is False + assert 15.0 >= duration + + +def test_second_starlark_app_advances_at_30_seconds(test_display_controller): + manager = _manager(_app("aquarium", 15), _app("clock", 30)) + test_display_controller.plugin_modes = {"aquarium": manager, "clock": manager} + + duration = test_display_controller._get_display_duration("clock") + + assert duration == 30 + assert (29.999 >= duration) is False + assert 30.0 >= duration + + +def test_non_starlark_plugin_duration_contract_is_unchanged(test_display_controller): + plugin = MagicMock(spec=["get_display_duration"]) + plugin.get_display_duration.return_value = 42 + test_display_controller.plugin_modes = {"legacy-mode": plugin} + + assert test_display_controller._get_display_duration("legacy-mode") == 42 + plugin.get_display_duration.assert_called_once_with() + + +def test_webp_frames_continue_advancing_within_mode_duration(monkeypatch): + aquarium = _app("aquarium", 15, [("frame-1", 100), ("frame-2", 100)]) + manager = _manager(aquarium) + times = iter((0.05, 0.11, 0.22)) + monkeypatch.setattr(manager_module.time, "time", lambda: next(times)) + + assert manager.display(display_mode="aquarium") is True + assert aquarium.current_frame_index == 0 + assert manager.display(display_mode="aquarium") is True + assert aquarium.current_frame_index == 1 + assert manager.display(display_mode="aquarium") is True + assert aquarium.current_frame_index == 0 + assert manager.get_mode_display_duration("aquarium") == 15 + + +def test_manual_mode_switch_selects_requested_app_immediately(): + aquarium = _app("aquarium", 15) + clock = _app("clock", 30) + manager = _manager(aquarium, clock) + + assert manager.display(display_mode="aquarium") is True + assert manager.current_app is aquarium + + assert manager.display(display_mode="clock", force_clear=True) is True + assert manager.current_app is clock + manager.display_manager.clear.assert_called_once_with() + assert manager.display_manager.image == "clock-frame" From 198ed397dbbe3167eb8ecdf1780d171d5962805f Mon Sep 17 00:00:00 2001 From: kmce2019 <50456552+kmce2019@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:40:48 -0500 Subject: [PATCH 6/7] fix: honor Starlark animation frame timing --- plugin-repos/starlark-apps/frame_extractor.py | 20 ++++--- plugin-repos/starlark-apps/manager.py | 45 ++++++++++++++-- test/test_starlark_display_duration.py | 27 +++++++++- test/test_starlark_frame_extractor.py | 52 +++++++++++++++++++ 4 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 test/test_starlark_frame_extractor.py diff --git a/plugin-repos/starlark-apps/frame_extractor.py b/plugin-repos/starlark-apps/frame_extractor.py index bb52d9e1..bff3d831 100644 --- a/plugin-repos/starlark-apps/frame_extractor.py +++ b/plugin-repos/starlark-apps/frame_extractor.py @@ -66,17 +66,25 @@ def load_webp(self, webp_path: str) -> Tuple[bool, Optional[List[Tuple[Image.Ima try: img.seek(frame_index) + # Pillow populates animated WebP timestamp/duration + # metadata while decoding the sought frame. Reading + # img.info before load() returns the previous frame's + # duration (and no duration for frame zero). + img.load() + # Get frame duration (in milliseconds) # WebP stores duration in milliseconds - duration = img.info.get("duration", self.default_frame_delay) - - # Ensure minimum frame delay (prevent too-fast animations) - if duration < 16: # Less than ~60fps - duration = 16 + embedded_duration = img.info.get("duration") + if (isinstance(embedded_duration, (int, float)) + and not isinstance(embedded_duration, bool) + and embedded_duration > 0): + duration = embedded_duration + else: + duration = self.default_frame_delay # Convert frame to RGB (LED matrix needs RGB) frame = img.convert("RGB") - frames.append((frame.copy(), duration)) + frames.append((frame.copy(), int(duration))) except EOFError: logger.warning(f"Reached end of frames at index {frame_index}") diff --git a/plugin-repos/starlark-apps/manager.py b/plugin-repos/starlark-apps/manager.py index 54ca8c32..91b7dfeb 100644 --- a/plugin-repos/starlark-apps/manager.py +++ b/plugin-repos/starlark-apps/manager.py @@ -301,6 +301,20 @@ def validate_config(self) -> bool: return True + @property + def needs_high_fps(self) -> bool: + """Request frame-rate dispatch only for an active animated app. + + The controller evaluates this after the first display() call, by which + point the selected app's cached WebP has been rendered and extracted. + Static Starlark apps remain on the normal low-frequency path. + """ + return bool( + self.current_app + and self.current_app.frames + and len(self.current_app.frames) > 1 + ) + def _calculate_optimal_magnify(self) -> int: """ Calculate optimal magnification factor based on display dimensions. @@ -862,13 +876,34 @@ def _display_frame(self) -> None: self.display_manager.image = frame self.display_manager.update_display() - # Check if it's time to advance to next frame - delay_seconds = delay_ms / 1000.0 - if (current_time - self.current_app.last_frame_time) >= delay_seconds: + # Advance against the WebP's wall-clock timeline. The controller + # may call us less frequently than a very short encoded delay + # (Arcade Classics contains 5ms frames), so advance through as + # many elapsed frames as necessary instead of stretching every + # frame to one controller callback. This never triggers a Pixlet + # rerender; it only changes the in-memory frame index. + if self.current_app.last_frame_time <= 0: + self.current_app.last_frame_time = current_time + return + + elapsed_ms = (current_time - self.current_app.last_frame_time) * 1000.0 + frames_advanced = 0 + frame_count = len(self.current_app.frames) + total_duration_ms = sum(frame_delay for _, frame_delay in self.current_app.frames) + if total_duration_ms > 0 and elapsed_ms >= total_duration_ms: + # Whole animation loops return to the same frame index. + elapsed_ms %= total_duration_ms + while elapsed_ms >= delay_ms and frames_advanced < frame_count: + elapsed_ms -= delay_ms self.current_app.current_frame_index = ( - (self.current_app.current_frame_index + 1) % len(self.current_app.frames) + (self.current_app.current_frame_index + 1) % frame_count ) - self.current_app.last_frame_time = current_time + frames_advanced += 1 + _, delay_ms = self.current_app.frames[self.current_app.current_frame_index] + + # Preserve sub-frame remainder so callback jitter does not + # accumulate into progressively slower playback. + self.current_app.last_frame_time = current_time - (elapsed_ms / 1000.0) except Exception as e: self.logger.error(f"Error displaying frame: {e}") diff --git a/test/test_starlark_display_duration.py b/test/test_starlark_display_duration.py index c3958e68..b919f287 100644 --- a/test/test_starlark_display_duration.py +++ b/test/test_starlark_display_duration.py @@ -70,11 +70,12 @@ def test_non_starlark_plugin_duration_contract_is_unchanged(test_display_control def test_webp_frames_continue_advancing_within_mode_duration(monkeypatch): aquarium = _app("aquarium", 15, [("frame-1", 100), ("frame-2", 100)]) manager = _manager(aquarium) - times = iter((0.05, 0.11, 0.22)) + times = iter((0.05, 0.16, 0.27)) monkeypatch.setattr(manager_module.time, "time", lambda: next(times)) assert manager.display(display_mode="aquarium") is True assert aquarium.current_frame_index == 0 + assert manager.needs_high_fps is True assert manager.display(display_mode="aquarium") is True assert aquarium.current_frame_index == 1 assert manager.display(display_mode="aquarium") is True @@ -82,6 +83,30 @@ def test_webp_frames_continue_advancing_within_mode_duration(monkeypatch): assert manager.get_mode_display_duration("aquarium") == 15 +def test_short_frames_catch_up_to_webp_wall_clock(monkeypatch): + arcade = _app("arcade", 15, [("f0", 5), ("f1", 5), ("f2", 5), ("f3", 5)]) + manager = _manager(arcade) + times = iter((1.000, 1.008, 1.016)) + monkeypatch.setattr(manager_module.time, "time", lambda: next(times)) + + manager.display(display_mode="arcade") + assert arcade.current_frame_index == 0 + + manager.display(display_mode="arcade") + assert arcade.current_frame_index == 1 + + manager.display(display_mode="arcade") + assert arcade.current_frame_index == 3 + + +def test_static_starlark_app_does_not_request_high_fps(): + static_app = _app("weather", 15, [("still", 50)]) + manager = _manager(static_app) + + assert manager.display(display_mode="weather") is True + assert manager.needs_high_fps is False + + def test_manual_mode_switch_selects_requested_app_immediately(): aquarium = _app("aquarium", 15) clock = _app("clock", 30) diff --git a/test/test_starlark_frame_extractor.py b/test/test_starlark_frame_extractor.py new file mode 100644 index 00000000..0f86d06c --- /dev/null +++ b/test/test_starlark_frame_extractor.py @@ -0,0 +1,52 @@ +"""Animated WebP timing regression tests for Starlark playback.""" + +import importlib.util +import sys +from pathlib import Path + +from PIL import Image + + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "plugin-repos/starlark-apps/frame_extractor.py" +spec = importlib.util.spec_from_file_location("test_starlark_frame_extractor_module", MODULE_PATH) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + + +def _write_animated_webp(path, durations): + frames = [Image.new("RGB", (2, 2), (index * 40, 0, 0)) + for index in range(len(durations))] + frames[0].save( + path, + format="WEBP", + save_all=True, + append_images=frames[1:], + duration=durations, + loop=0, + lossless=True, + ) + + +def test_animated_webp_preserves_each_embedded_frame_duration(tmp_path): + webp = tmp_path / "timed.webp" + expected = [40, 120, 250] + _write_animated_webp(webp, expected) + + success, frames, error = module.FrameExtractor(default_frame_delay=999).load_webp(str(webp)) + + assert success, error + assert [duration for _, duration in frames] == expected + + +def test_default_delay_is_used_only_without_usable_embedded_duration(tmp_path): + static_webp = tmp_path / "static.webp" + Image.new("RGB", (2, 2), "red").save(static_webp, format="WEBP", lossless=True) + + success, frames, error = module.FrameExtractor(default_frame_delay=73).load_webp( + str(static_webp)) + + assert success, error + assert len(frames) == 1 + assert frames[0][1] == 73 From 9f6fe59982c4ea901dd4a5177187b3a18ea83e1b Mon Sep 17 00:00:00 2001 From: kmce2019 <50456552+kmce2019@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:04:48 -0500 Subject: [PATCH 7/7] fix: dispatch Starlark animations at high FPS --- plugin-repos/starlark-apps/manager.py | 85 ++++++++++++------- plugin-repos/starlark-apps/pixlet_renderer.py | 18 ++++ test/test_starlark_display_duration.py | 18 +++- test/test_starlark_schema_parser.py | 24 ++++++ 4 files changed, 110 insertions(+), 35 deletions(-) diff --git a/plugin-repos/starlark-apps/manager.py b/plugin-repos/starlark-apps/manager.py index 91b7dfeb..2dc1c9f6 100644 --- a/plugin-repos/starlark-apps/manager.py +++ b/plugin-repos/starlark-apps/manager.py @@ -56,6 +56,8 @@ def __init__(self, app_id: str, app_dir: Path, manifest: Dict[str, Any]): self.current_frame_index = 0 self.last_frame_time = 0 self.last_render_time = 0 + self.last_presented_frame_index: Optional[int] = None + self.animation_total_duration_ms = 0 def _load_config(self) -> Dict[str, Any]: """Load app configuration from config.json.""" @@ -303,17 +305,15 @@ def validate_config(self) -> bool: @property def needs_high_fps(self) -> bool: - """Request frame-rate dispatch only for an active animated app. + """Request frame-rate dispatch before the controller selects a mode. - The controller evaluates this after the first display() call, by which - point the selected app's cached WebP has been rendered and extracted. - Static Starlark apps remain on the normal low-frequency path. + DisplayController reads this property before its first display() call, + so ``current_app`` is not available yet. Starlark owns both static and + animated modes; it must enter the fast dispatch path up front so an + animated WebP can honor its embedded timings. _display_frame avoids + redundant matrix writes when the selected frame has not changed. """ - return bool( - self.current_app - and self.current_app.frames - and len(self.current_app.frames) > 1 - ) + return True def _calculate_optimal_magnify(self) -> int: """ @@ -718,6 +718,7 @@ def display(self, display_mode: Optional[str] = None, force_clear: bool = False) return False if self.current_app is not requested_app: self.current_app = requested_app + self.current_app.last_presented_frame_index = None self.logger.debug("Selected Starlark app for mode: %s", display_mode) elif not self.current_app: self._select_next_app() @@ -735,6 +736,8 @@ def display(self, display_mode: Optional[str] = None, force_clear: bool = False) return False # Display current frame + if force_clear: + self.current_app.last_presented_frame_index = None self._display_frame() return True @@ -855,6 +858,8 @@ def _load_frames_from_cache(self, app: StarlarkApp) -> bool: app.frames = frames app.current_frame_index = 0 app.last_frame_time = time.time() + app.last_presented_frame_index = None + app.animation_total_duration_ms = sum(delay for _, delay in frames) self.logger.debug(f"Loaded {len(frames)} frames for {app.app_id}") return True @@ -870,11 +875,8 @@ def _display_frame(self) -> None: try: current_time = time.time() - frame, delay_ms = self.current_app.frames[self.current_app.current_frame_index] - - # Set frame on display manager - self.display_manager.image = frame - self.display_manager.update_display() + frame_count = len(self.current_app.frames) + _, delay_ms = self.current_app.frames[self.current_app.current_frame_index] # Advance against the WebP's wall-clock timeline. The controller # may call us less frequently than a very short encoded delay @@ -884,26 +886,43 @@ def _display_frame(self) -> None: # rerender; it only changes the in-memory frame index. if self.current_app.last_frame_time <= 0: self.current_app.last_frame_time = current_time - return - - elapsed_ms = (current_time - self.current_app.last_frame_time) * 1000.0 - frames_advanced = 0 - frame_count = len(self.current_app.frames) - total_duration_ms = sum(frame_delay for _, frame_delay in self.current_app.frames) - if total_duration_ms > 0 and elapsed_ms >= total_duration_ms: - # Whole animation loops return to the same frame index. - elapsed_ms %= total_duration_ms - while elapsed_ms >= delay_ms and frames_advanced < frame_count: - elapsed_ms -= delay_ms - self.current_app.current_frame_index = ( - (self.current_app.current_frame_index + 1) % frame_count + else: + elapsed_ms = (current_time - self.current_app.last_frame_time) * 1000.0 + frames_advanced = 0 + total_duration_ms = getattr( + self.current_app, "animation_total_duration_ms", 0 ) - frames_advanced += 1 - _, delay_ms = self.current_app.frames[self.current_app.current_frame_index] - - # Preserve sub-frame remainder so callback jitter does not - # accumulate into progressively slower playback. - self.current_app.last_frame_time = current_time - (elapsed_ms / 1000.0) + if total_duration_ms <= 0: + total_duration_ms = sum( + frame_delay for _, frame_delay in self.current_app.frames + ) + self.current_app.animation_total_duration_ms = total_duration_ms + if total_duration_ms > 0 and elapsed_ms >= total_duration_ms: + # Whole animation loops return to the same frame index. + elapsed_ms %= total_duration_ms + while elapsed_ms >= delay_ms and frames_advanced < frame_count: + elapsed_ms -= delay_ms + self.current_app.current_frame_index = ( + (self.current_app.current_frame_index + 1) % frame_count + ) + frames_advanced += 1 + _, delay_ms = self.current_app.frames[ + self.current_app.current_frame_index + ] + + # Preserve sub-frame remainder so callback jitter does not + # accumulate into progressively slower playback. + self.current_app.last_frame_time = current_time - (elapsed_ms / 1000.0) + + # The controller dispatches Starlark at high FPS before it knows + # whether the selected mode is animated. Only touch the matrix + # when frame progression (or a mode switch/clear) requires it. + frame_index = self.current_app.current_frame_index + if getattr(self.current_app, "last_presented_frame_index", None) != frame_index: + frame, _ = self.current_app.frames[frame_index] + self.display_manager.image = frame + self.display_manager.update_display() + self.current_app.last_presented_frame_index = frame_index except Exception as e: self.logger.error(f"Error displaying frame: {e}") diff --git a/plugin-repos/starlark-apps/pixlet_renderer.py b/plugin-repos/starlark-apps/pixlet_renderer.py index f6c45f70..01cb0195 100644 --- a/plugin-repos/starlark-apps/pixlet_renderer.py +++ b/plugin-repos/starlark-apps/pixlet_renderer.py @@ -506,6 +506,24 @@ def _safe_eval_node(self, node: ast.AST, values: Dict[str, Any]) -> Any: if isinstance(node, ast.Name): return values.get(node.id, _UNRESOLVED) + # Static constant lookup, e.g. + # DEFAULT_FRAME_DELAY = FRAME_DELAYS["Normal"]. Resolve only a + # previously evaluated data container and a primitive literal key; + # never invoke user code or Python's general eval machinery. + if isinstance(node, ast.Subscript): + container = self._safe_eval_node(node.value, values) + key = self._safe_eval_node(node.slice, values) + if container is _UNRESOLVED or key is _UNRESOLVED: + return _UNRESOLVED + if isinstance(container, dict) and isinstance( + key, (str, int, float, bool)): + return container.get(key, _UNRESOLVED) + if (isinstance(container, list) + and isinstance(key, int) and not isinstance(key, bool) + and -len(container) <= key < len(container)): + return container[key] + return _UNRESOLVED + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): operand = self._safe_eval_node(node.operand, values) if isinstance(operand, (int, float)) and not isinstance(operand, bool): diff --git a/test/test_starlark_display_duration.py b/test/test_starlark_display_duration.py index b919f287..cedfdae4 100644 --- a/test/test_starlark_display_duration.py +++ b/test/test_starlark_display_duration.py @@ -23,6 +23,8 @@ def _app(app_id, duration, frames=None): app.frames = frames or [(f"{app_id}-frame", 100)] app.current_frame_index = 0 app.last_frame_time = 0.0 + app.last_presented_frame_index = None + app.animation_total_duration_ms = sum(delay for _, delay in app.frames) return app @@ -99,12 +101,24 @@ def test_short_frames_catch_up_to_webp_wall_clock(monkeypatch): assert arcade.current_frame_index == 3 -def test_static_starlark_app_does_not_request_high_fps(): +def test_starlark_requests_high_fps_before_controller_selects_mode(): + manager = _manager(_app("animated", 15, [("f0", 50), ("f1", 50)])) + + assert manager.current_app is None + assert manager.needs_high_fps is True + + +def test_static_starlark_app_avoids_redundant_matrix_writes(monkeypatch): static_app = _app("weather", 15, [("still", 50)]) manager = _manager(static_app) + times = iter((1.0, 1.1, 1.2)) + monkeypatch.setattr(manager_module.time, "time", lambda: next(times)) assert manager.display(display_mode="weather") is True - assert manager.needs_high_fps is False + assert manager.display(display_mode="weather") is True + assert manager.display(display_mode="weather") is True + assert manager.needs_high_fps is True + assert manager.display_manager.update_display.call_count == 1 def test_manual_mode_switch_selects_requested_app_immediately(): diff --git a/test/test_starlark_schema_parser.py b/test/test_starlark_schema_parser.py index 2fa4074d..9642f0ba 100644 --- a/test/test_starlark_schema_parser.py +++ b/test/test_starlark_schema_parser.py @@ -103,6 +103,30 @@ def get_schema(): assert "Could not statically resolve" in caplog.text +def test_resolves_default_from_static_dictionary_lookup(renderer, tmp_path): + fields = _extract(renderer, tmp_path, ''' +FRAME_DELAYS = {"Normal": "100", "Fast": "60"} +DEFAULT_FRAME_DELAY = FRAME_DELAYS["Normal"] +def get_schema(): + return schema.Schema(version = "1", fields = [ + schema.Dropdown( + id = "frame_delay", + default = DEFAULT_FRAME_DELAY, + options = [ + schema.Option(display = "Normal", value = "100"), + schema.Option(display = "Fast", value = "60"), + ], + ), + ]) +''') + + assert fields["frame_delay"]["default"] == "100" + assert fields["frame_delay"]["options"] == [ + {"display": "Normal", "value": "100"}, + {"display": "Fast", "value": "60"}, + ] + + def test_rebuild_repairs_only_missing_and_symbolic_defaults(tmp_path): apps_dir = tmp_path / "starlark-apps" app_dir = apps_dir / "sample"