ENG-9148: feat(compiler): incremental compile cache & warm hot-reload daemon (REFLEX_COMPILE_CACHE)

docs/app/reflex_docs/pages/docs/component.py

@@ -905,16 +905,9 @@ def multi_docs(
     ll_component_list: list | None = None,
     source: str | None = None,
 ):
-    components = [
-        component_docs(component_tuple, previews)
-        for component_tuple in component_list[1:]
-    ]
     ll_actual_path = actual_path.replace(".md", "-ll.md")
     ll_doc_exists = os.path.exists(ll_actual_path)
     ll_list = ll_component_list if ll_component_list is not None else component_list
-    ll_components = [
-        component_docs(component_tuple, previews) for component_tuple in ll_list[1:]
-    ]
 
     active_class_name = "font-small bg-secondary-2 p-2 text-secondary-11 rounded-xl shadow-large w-28 cursor-default border border-secondary-4 text-center"
 
@@ -966,6 +959,11 @@ def links(current_page, ll_doc_exists, path):
 
     @docpage(set_path=path, t=title, description=description, image=image)
     def out():
+        # Build prop docs during page eval so imports stay cheap.
+        components = [
+            component_docs(component_tuple, previews)
+            for component_tuple in component_list[1:]
+        ]
         toc = get_docgen_toc(actual_path)
         # Reuse the source already read by the caller to avoid a second read.
         doc_content = (
@@ -1014,6 +1012,9 @@ def out():
         image=image,
     )
     def ll():
+        ll_components = [
+            component_docs(component_tuple, previews) for component_tuple in ll_list[1:]
+        ]
         ll_virtual = virtual_path.replace(".md", "-ll.md")
         toc = get_docgen_toc(ll_actual_path)
         doc_content = Path(ll_actual_path).read_text(encoding="utf-8")

news/6688.feature.md

@@ -0,0 +1 @@
+Added an experimental disk-persisted incremental compile cache, enabled by the `REFLEX_COMPILE_CACHE` environment variable. When on, a fresh compile reuses the previous build already on disk in `.web` and recompiles only the pages whose source changed, tracked via a per-page dependency graph (Python import closure, files read during page evaluation, component modules, and referenced state). App-wide inputs (Reflex version, config/lockfiles, and the app entrypoint's config modules such as theme/app-wraps/stylesheets) gate the whole cache, falling back to a full compile when they change. `reflex run` dev additionally gains a warm fork-per-compile daemon so hot reloads skip the cold reimport and rebuild only what changed. Off by default — the compile path is unchanged when the flag is unset.

packages/reflex-base/news/6688.misc.md

@@ -0,0 +1 @@
+Added an optional per-page source-read recorder hook (`page_source_recorder` in the compiler plugin) used by the incremental compile cache to track the exact files each page reads during evaluation, and made auto-generated unique ref names reproducible across in-process compiles so memo content hashes stay stable. No behavior change unless the compile cache is enabled.

packages/reflex-base/src/reflex_base/environment.py

@@ -642,6 +642,12 @@ class EnvironmentVariables:
     # If this env var is set to "yes", App.compile will be a no-op
     REFLEX_SKIP_COMPILE: EnvVar[bool] = env_var(False, internal=True)
 
+    # Experimental: incremental compile cache. A fresh compile process (e.g. a
+    # reflex-run hot-reload worker) reuses each page's compiled output from an
+    # on-disk manifest and recompiles only the pages whose source changed.
+    # See reflex/compiler/disk_cache.py and reflex/compiler/page_cache.py.
+    REFLEX_COMPILE_CACHE: EnvVar[bool] = env_var(False)
+
     # Inherited by uvicorn/granian reload workers so the backend can distinguish
     # dev reload-capable worker boots from other backend starts. Never set in prod.
     REFLEX_DEV_BACKEND_RELOAD_ACTIVE: EnvVar[bool] = env_var(False, internal=True)

packages/reflex-base/src/reflex_base/plugins/compiler.py

@@ -6,6 +6,7 @@
 import dataclasses
 import inspect
 from collections.abc import Callable, Sequence
+from contextlib import AbstractContextManager
 from contextvars import ContextVar, Token
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, TypeVar, cast
@@ -35,6 +36,10 @@
 _BaseComponentT = TypeVar("_BaseComponentT", bound=BaseComponent)
 
 
+#: Optional recorder for source files read during each page evaluation.
+page_source_recorder: Callable[[], AbstractContextManager[set[str]]] | None = None
+
+
 class PageDefinition(Protocol):
     """Protocol for page-like objects compiled by :class:`CompileContext`."""
 
@@ -690,6 +695,12 @@ class PageContext(BaseContext):
     output_path: str | None = None
     output_code: str | None = None
     source_module: str | None = None
+    # Source files read while evaluating this page, when a recorder is installed.
+    source_files: set[str] = dataclasses.field(default_factory=set)
+    # Auto-memo components first registered while compiling this page.
+    memo_contributions: dict[tuple[str, str | None], Any] = dataclasses.field(
+        default_factory=dict
+    )
     # Stack of ``id(component)`` for components whose subtree is
     # memoize-suppressed. Populated by ``MemoizeStatefulPlugin`` when it
     # encounters a ``MemoizationLeaf``-style snapshot boundary and popped on
@@ -762,7 +773,9 @@ class CompileContext(BaseContext):
     app_wrap_components: dict[tuple[int, str], Component] = dataclasses.field(
         default_factory=dict
     )
-    stateful_routes: dict[str, None] = dataclasses.field(default_factory=dict)
+    # Routes whose evaluation defined new state classes, mapped to the full
+    # names of the states each page defined.
+    stateful_routes: dict[str, list[str]] = dataclasses.field(default_factory=dict)
     # Auto-memoize wrapper tags seen during the tree walk (populated by
     # ``MemoizeStatefulPlugin``).
     memoize_wrappers: dict[str, None] = dataclasses.field(default_factory=dict)
@@ -794,6 +807,7 @@ def compile(
         """
         from reflex.compiler import compiler
         from reflex.state import all_base_state_classes
+        from reflex_base.vars.base import reset_unique_variable_names
 
         self.ensure_context_attached()
         self.compiled_pages.clear()
@@ -803,15 +817,30 @@ def compile(
         self.memoize_wrappers.clear()
         self.auto_memo_components.clear()
 
+        # Keep generated ref names stable across in-process compiles.
+        reset_unique_variable_names()
+
+        recorder = page_source_recorder
         for page in self.pages:
             page_fn = page.component
             n_states_before = len(all_base_state_classes)
-            page_ctx = self.hooks.eval_page(
-                page_fn,
-                page=page,
-                compile_context=self,
-                **kwargs,
-            )
+            if recorder is not None:
+                with recorder() as read_set:
+                    page_ctx = self.hooks.eval_page(
+                        page_fn,
+                        page=page,
+                        compile_context=self,
+                        **kwargs,
+                    )
+                if page_ctx is not None:
+                    page_ctx.source_files = read_set
+            else:
+                page_ctx = self.hooks.eval_page(
+                    page_fn,
+                    page=page,
+                    compile_context=self,
+                    **kwargs,
+                )
             if page_ctx is None:
                 page_name = getattr(page_fn, "__name__", repr(page_fn))
                 msg = (
@@ -824,7 +853,12 @@ def compile(
                 raise RuntimeError(msg)
 
             if len(all_base_state_classes) > n_states_before:
-                self.stateful_routes[page.route] = None
+                # Record which states this page defined (registration order is
+                # insertion order), so the compile cache can fingerprint the
+                # page's contribution to the contexts file.
+                self.stateful_routes[page.route] = list(all_base_state_classes)[
+                    n_states_before:
+                ]
 
             self.compiled_pages[page_ctx.route] = page_ctx
 
@@ -836,6 +870,7 @@ def compile(
             self.compiled_pages.values(),
             strict=True,
         ):
+            memo_before = set(self.auto_memo_components)
             with page_ctx:
                 page_ctx.root_component = self.hooks.compile_component(
                     page_ctx.root_component,
@@ -848,6 +883,12 @@ def compile(
                     compile_context=self,
                     **kwargs,
                 )
+            # Attribute newly-registered auto-memo components to this page.
+            page_ctx.memo_contributions = {
+                key: value
+                for key, value in self.auto_memo_components.items()
+                if key not in memo_before
+            }
 
             page_ctx.frontend_imports = page_ctx.merged_imports(collapse=True)
             self.all_imports = merge_imports(

packages/reflex-base/src/reflex_base/utils/imports.py

@@ -91,18 +91,18 @@ def collapse_imports(
 ) -> ParsedImportDict:
     """Remove all duplicate ImportVar within an ImportDict.
 
+    Deduplication preserves first-occurrence order: compiled import statements
+    follow this order, and a hash-seed-dependent order would rewrite every
+    page/memo module on each dev reload, defeating granular HMR.
+
     Args:
         imports: The import dict to collapse.
 
     Returns:
         The collapsed import dict.
     """
     return {
-        lib: (
-            list(set(import_vars))
-            if isinstance(import_vars, list)
-            else list(import_vars)
-        )
+        lib: list(dict.fromkeys(import_vars))
         for lib, import_vars in (
             imports if isinstance(imports, tuple) else imports.items()
         )

packages/reflex-base/src/reflex_base/utils/serializers.py

@@ -292,13 +292,20 @@ def serialize_base_model(model: BaseModel) -> dict:
 def serialize_set(value: set) -> list:
     """Serialize a set to a JSON serializable list.
 
+    Sets have no meaningful order and their iteration order varies with the
+    per-process hash seed, so sort when possible to keep serialized output
+    (compiled JSX, config fingerprints) stable across processes.
+
     Args:
         value: The set to serialize.
 
     Returns:
         The serialized list.
     """
-    return list(value)
+    try:
+        return sorted(value)
+    except TypeError:
+        return list(value)
 
 
 @serializer

packages/reflex-base/src/reflex_base/vars/base.py

@@ -10,6 +10,7 @@
 import functools
 import inspect
 import json
+import random
 import re
 import string
 import uuid
@@ -45,7 +46,6 @@
 from reflex_base.constants.state import FIELD_MARKER
 from reflex_base.utils import console, exceptions, imports, serializers, types
 from reflex_base.utils.compat import annotations_from_namespace
-from reflex_base.utils.decorator import once
 from reflex_base.utils.exceptions import (
     ComputedVarSignatureError,
     UntypedComputedVarError,
@@ -3296,12 +3296,17 @@ def get_uuid_string_var() -> Var:
 # Set of unique variable names.
 USED_VARIABLES = set()
 
+_UNIQUE_NAME_RNG = random.Random(42)
 
-@once
-def _rng():
-    import random
 
-    return random.Random(42)
+def reset_unique_variable_names() -> None:
+    """Reset the deterministic unique-name generator to its initial state.
+
+    Names only need to be unique within one compile, so resetting before each
+    compile makes auto-generated ref names reproducible.
+    """
+    USED_VARIABLES.clear()
+    _UNIQUE_NAME_RNG.seed(42)
 
 
 def get_unique_variable_name() -> str:
@@ -3310,7 +3315,7 @@ def get_unique_variable_name() -> str:
     Returns:
         The unique variable name.
     """
-    name = "".join([_rng().choice(string.ascii_lowercase) for _ in range(8)])
+    name = "".join([_UNIQUE_NAME_RNG.choice(string.ascii_lowercase) for _ in range(8)])
     if name not in USED_VARIABLES:
         USED_VARIABLES.add(name)
         return name

reflex/app.py

@@ -1681,8 +1681,12 @@ def _write_stateful_pages_marker(self):
                 prerequisites.get_backend_dir() / constants.Dirs.STATEFUL_PAGES
             )
             stateful_pages_marker.parent.mkdir(parents=True, exist_ok=True)
-            with stateful_pages_marker.open("w") as f:
-                json.dump(list(self._stateful_pages), f)
+            content = json.dumps(list(self._stateful_pages))
+            if (
+                not stateful_pages_marker.exists()
+                or stateful_pages_marker.read_text() != content
+            ):
+                stateful_pages_marker.write_text(content)
 
     def add_all_routes_endpoint(self):
         """Add an endpoint to the app that returns all the routes."""