Path-local thoroughness: N+1 hops, indexes, tests-to-fields, React

README.md

@@ -147,6 +147,7 @@ AST is enough for review. If you need live `_meta` (db_table, resolved relations
 | `no_queryset_in_serializer` | Serializers must not run querysets |
 | `celery_tasks_must_be_idempotent_on_model_pk` | Celery and Dramatiq tasks take a model pk |
 | `queryset_nplusone` | Loops over querysets that touch related objects need `select_related` / `prefetch_related` |
+| `queryset_missing_index` | `.filter()` / `.order_by()` on a field that has no `db_index` / `unique` |
 | `cascade_crosses_context` | `on_delete=CASCADE` must not blast into another bounded context |
 | `migration_blast_radius` | `RemoveField` / `DeleteModel` still referenced by the typed graph |
 

fixtures/demo_monorepo/loadpath.yml

@@ -26,6 +26,7 @@ rules:
   - no_queryset_in_serializer
   - celery_tasks_must_be_idempotent_on_model_pk
   - queryset_nplusone
+  - queryset_missing_index
   - cascade_crosses_context
   - migration_blast_radius
 django_root: backend

loadpath.yml.example

@@ -25,6 +25,7 @@ rules:
   - no_queryset_in_serializer
   - celery_tasks_must_be_idempotent_on_model_pk
   - queryset_nplusone
+  - queryset_missing_index
   - cascade_crosses_context
   - migration_blast_radius
 django_root: backend

src/loadpath/architecture/rules.py

@@ -16,6 +16,7 @@
     "celery_tasks_must_be_idempotent_on_model_pk": "Celery and Dramatiq tasks must take a model pk/id, not a full object payload.",
     "async_tasks_must_be_idempotent_on_model_pk": "Celery and Dramatiq tasks must take a model pk/id, not a full object payload.",
     "queryset_nplusone": "Querysets iterated in a loop must select_related/prefetch_related related objects they touch.",
+    "queryset_missing_index": "filter/order_by on a field should match db_index/unique on that field.",
     "cascade_crosses_context": "on_delete=CASCADE must not blast into another bounded context.",
     "migration_blast_radius": "Destructive migrations must not drop fields/models still referenced on the load path.",
 }
@@ -68,6 +69,8 @@ def evaluate(store: GraphStore, config: LoadpathConfig, changed_ids: set[str] |
         findings.extend(_task_idempotency(store, changed_ids))
     if "queryset_nplusone" in enabled:
         findings.extend(_nplusone(store))
+    if "queryset_missing_index" in enabled:
+        findings.extend(_missing_index(store))
     if "cascade_crosses_context" in enabled:
         findings.extend(_cascade_crosses_context(store, config))
     if "migration_blast_radius" in enabled:
@@ -316,6 +319,51 @@ def _nplusone(store: GraphStore) -> list[Finding]:
     return out
 
 
+def _missing_index(store: GraphStore) -> list[Finding]:
+    out: list[Finding] = []
+    fields_by_name: dict[str, list[dict]] = {}
+    for field in store.nodes([NodeType.FIELD]):
+        fields_by_name.setdefault(field["name"], []).append(field)
+    indexed_types = {"ForeignKey", "OneToOneField", "ManyToManyField"}
+    for node in store.nodes():
+        lookups = (node.get("extra") or {}).get("lookups") or []
+        owner_app = (node.get("extra") or {}).get("app")
+        for hit in lookups:
+            for fname in hit.get("fields") or []:
+                matches = fields_by_name.get(fname) or []
+                if owner_app:
+                    scoped = [f for f in matches if (f.get("extra") or {}).get("app") == owner_app]
+                    if scoped:
+                        matches = scoped
+                if not matches:
+                    continue
+                uncovered = [
+                    f
+                    for f in matches
+                    if not (f.get("extra") or {}).get("db_index")
+                    and not (f.get("extra") or {}).get("unique")
+                    and (f.get("extra") or {}).get("field_type") not in indexed_types
+                ]
+                if not uncovered:
+                    continue
+                sample = uncovered[0]
+                out.append(
+                    Finding(
+                        rule="queryset_missing_index",
+                        severity=RuleSeverity.WARNING,
+                        message=(
+                            f"{node['name']} {hit.get('kind')}s `{fname}` "
+                            f"({node.get('file_path')}:{hit.get('line')}) but "
+                            f"{sample.get('qualified_name')} has no db_index"
+                        ),
+                        node_id=node["id"],
+                        file_path=node.get("file_path"),
+                        extra={"field": fname, "kind": hit.get("kind"), "line": hit.get("line")},
+                    )
+                )
+    return out
+
+
 def _cascade_crosses_context(store: GraphStore, config: LoadpathConfig) -> list[Finding]:
     out: list[Finding] = []
     fields = {n["id"]: n for n in store.nodes([NodeType.FIELD])}

src/loadpath/config.py

@@ -13,6 +13,7 @@
     "no_queryset_in_serializer",
     "celery_tasks_must_be_idempotent_on_model_pk",
     "queryset_nplusone",
+    "queryset_missing_index",
     "cascade_crosses_context",
     "migration_blast_radius",
 ]

src/loadpath/extractors/django.py

@@ -268,6 +268,7 @@ def visit_Call(self, node: ast.Call) -> None:
             self._enqueue(node, fname, broker="celery")
         elif short in CELERY_CANVAS and self._looks_like_celery(fname):
             self.graph.residuals.append(f"Celery canvas {fname}() at {self.rel_path}:{node.lineno}")
+            self._enqueue_from_canvas(node)
         elif short == "send_task":
             self._send_task(node)
         elif short in DRAMATIQ_ENQUEUE and self._looks_like_dramatiq_send(fname):
@@ -468,6 +469,20 @@ def _view(self, node: ast.ClassDef) -> None:
                 Node(id=pid, type=NodeType.PERMISSION, name=perm, qualified_name=perm, extra={"from_view": qname})
             )
             self.add_edge(view.id, pid, EdgeType.HAS_PERMISSION)
+        for throttle in extra.get("throttles") or []:
+            tid = node_id(NodeType.THROTTLE, throttle)
+            self.graph.nodes.append(
+                Node(
+                    id=tid,
+                    type=NodeType.THROTTLE,
+                    name=throttle,
+                    qualified_name=throttle,
+                    extra={"from_view": qname},
+                )
+            )
+            self.add_edge(view.id, tid, EdgeType.HAS_PERMISSION)
+        if extra.get("pagination"):
+            extra["pagination_sink"] = True
         if queryset_model:
             self.add_edge(
                 view.id,
@@ -633,6 +648,18 @@ def _task_qname(self, fname: str) -> tuple[str, str, str]:
                     break
         return app, short, f"{app}.{short}"
 
+    def _enqueue_from_canvas(self, node: ast.Call) -> None:
+        for arg in list(node.args) + [kw.value for kw in node.keywords]:
+            for child in ast.walk(arg):
+                if not isinstance(child, ast.Call):
+                    continue
+                fname = _name(child.func) or ""
+                short = fname.split(".")[-1]
+                if short in CELERY_ENQUEUE or short in CELERY_SIGNATURE:
+                    self._enqueue(child, fname, broker="celery")
+                elif short in DRAMATIQ_ENQUEUE:
+                    self._enqueue(child, fname, broker="dramatiq")
+
     def _enqueue_from_on_commit(self, node: ast.Call) -> None:
         for arg in list(node.args) + [kw.value for kw in node.keywords]:
             for child in ast.walk(arg):
@@ -795,6 +822,13 @@ def _maybe_test(self, node: ast.FunctionDef) -> None:
             return
         qname = f"{self.app}.{node.name}"
         extra = {"app": self.app, "nodeid": f"{self.rel_path}::{node.name}"}
+        mentions: set[str] = set()
+        for child in ast.walk(node):
+            if isinstance(child, ast.Constant) and isinstance(child.value, str):
+                mentions.add(child.value)
+            elif isinstance(child, ast.Attribute):
+                mentions.add(child.attr)
+        extra["mentions"] = sorted(mentions)
         test = self.add_node(NodeType.TEST, node.name, qname, node.lineno, extra)
         # crude: referenced class names in the test become tested_by
         for child in ast.walk(node):
@@ -993,6 +1027,9 @@ def extract_django_file(rel_path: str, source: str, config: LoadpathConfig) -> E
     from loadpath.orm.nplusone import apply_nplusone
 
     apply_nplusone(extractor.graph, tree)
+    from loadpath.orm.lookups import apply_lookups
+
+    apply_lookups(extractor.graph, tree)
     return extractor.graph
 
 

src/loadpath/extractors/react.py

@@ -58,7 +58,12 @@
     r"""path\s*:\s*['"]([^'"]+)['"][^}]*?(?:element|Component)\s*:\s*<?\s*([A-Z][A-Za-z0-9_]*)""",
     re.S,
 )
-SWR_RE = re.compile(r"""useSWR\s*\(\s*(['"`][^'"`]+['"`]|[A-Za-z0-9_\.\[\]\s,"'`]+)""")
+INVALIDATE_RE = re.compile(
+    r"""invalidateQueries\s*\(\s*\{[^}]*queryKey\s*:\s*(\[[^\]]*\])""",
+    re.S,
+)
+DEFAULT_VALUE_RE = re.compile(r"""(?:defaultValue|name)\s*=\s*(?:\{[^}]*\.(\w+)|['"](\w+)['"])""")
+BOUNDARY_RE = re.compile(r"""\b(ErrorBoundary|Suspense)\b""")
 RTK_RE = re.compile(r"""createApi\s*\(\s*\{""")
 FEATURE_FOLDER_RE = re.compile(r"""features/([^/]+)""")
 
@@ -157,7 +162,13 @@ def edge(src: str, dst: str, etype: EdgeType, confidence: float = 1.0, extra: di
         line = source[: m.start()].count("\n") + 1
         is_page = name.endswith("Page") or "pages/" in rel or name.endswith("Screen")
         ntype = NodeType.PAGE if is_page else NodeType.COMPONENT
-        n = add(ntype, name, f"{feature or 'app'}.{name}", line, {"feature": feature})
+        extra: dict = {"feature": feature}
+        if is_page:
+            extra["has_error_boundary"] = bool(BOUNDARY_RE.search(source))
+        defaults = [a or b for a, b in DEFAULT_VALUE_RE.findall(source)]
+        if defaults:
+            extra["form_fields"] = sorted(set(defaults))
+        n = add(ntype, name, f"{feature or 'app'}.{name}", line, extra)
         components.append(n)
         if feature:
             edge(n.id, node_id(NodeType.FEATURE_MODULE, f"features.{feature}"), EdgeType.BELONGS_TO)
@@ -199,6 +210,23 @@ def edge(src: str, dst: str, etype: EdgeType, confidence: float = 1.0, extra: di
             )
             for h in hooks or components:
                 edge(h.id, qn.id, EdgeType.USES_QUERY_KEY)
+            if m.group(0).startswith("useMutation"):
+                for h in hooks:
+                    h.extra["mutation"] = True
+
+    for m in INVALIDATE_RE.finditer(source):
+        key_raw = m.group(1)
+        line = source[: m.start()].count("\n") + 1
+        key_name = re.sub(r"""\s+""", "", key_raw)
+        qn = add(
+            NodeType.QUERY_KEY,
+            key_name,
+            f"{feature or 'app'}.queryKey.{key_name}",
+            line,
+            {"raw": key_raw, "feature": feature, "invalidation": True},
+        )
+        for h in hooks or components:
+            edge(h.id, qn.id, EdgeType.USES_QUERY_KEY, extra={"invalidates": True})
 
     for m in TEMPLATE_FETCH_RE.finditer(source):
         url = m.group("turl") or m.group(3) or m.group(4)
@@ -279,7 +307,13 @@ def edge(src: str, dst: str, etype: EdgeType, confidence: float = 1.0, extra: di
 
     if is_test:
         line = 1
-        tn = add(NodeType.REACT_TEST, stem, f"test.{rel}", line, {"file": rel})
+        tn = add(
+            NodeType.REACT_TEST,
+            stem,
+            f"test.{rel}",
+            line,
+            {"file": rel, "mentions": sorted(set(re.findall(r"""['"](\w+)['"]""", source)))},
+        )
         for m in RTL_RENDER_RE.finditer(source):
             name = m.group(1)
             edge(node_id(NodeType.PAGE, f"{feature or 'app'}.{name}"), tn.id, EdgeType.TESTED_BY)

src/loadpath/orm/lookups.py

@@ -0,0 +1,87 @@
+"""Path-local missing-index hints from .filter() / .order_by() vs field extras."""
+
+from __future__ import annotations
+
+import ast
+
+from loadpath.types import ExtractedGraph, NodeType
+
+PREFERRED_OWNERS = {
+    NodeType.VIEW,
+    NodeType.SERVICE,
+    NodeType.MANAGEMENT_COMMAND,
+    NodeType.TASK,
+}
+
+SKIP_LOOKUPS = {"pk", "id", "pk__in", "id__in"}
+
+
+def apply_lookups(graph: ExtractedGraph, tree: ast.AST) -> None:
+    hits = scan_lookups(tree)
+    if not hits:
+        return
+    by_owner: dict[str, list[dict]] = {}
+    for item in hits:
+        by_owner.setdefault(item["owner"], []).append(item)
+    for owner_name, items in by_owner.items():
+        owner = _owner_node(graph, owner_name)
+        if owner is None:
+            continue
+        bucket = list(owner.extra.get("lookups") or [])
+        bucket.extend(items)
+        owner.extra["lookups"] = bucket
+
+
+def _owner_node(graph: ExtractedGraph, name: str | None):
+    if not name:
+        return None
+    candidates = [n for n in graph.nodes if n.name == name]
+    return next((n for n in candidates if n.type in PREFERRED_OWNERS), None) or (
+        candidates[0] if candidates else None
+    )
+
+
+def scan_lookups(tree: ast.AST) -> list[dict]:
+    out: list[dict] = []
+
+    def visit_function(fn: ast.FunctionDef | ast.AsyncFunctionDef, owner: str) -> None:
+        for node in ast.walk(fn):
+            if not isinstance(node, ast.Call):
+                continue
+            short = node.func.attr if isinstance(node.func, ast.Attribute) else ""
+            if short not in {"filter", "exclude", "order_by", "get"}:
+                continue
+            fields: list[str] = []
+            if short == "order_by":
+                for arg in node.args:
+                    if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
+                        fields.append(arg.value.lstrip("-").split("__")[0])
+            else:
+                for kw in node.keywords:
+                    if kw.arg:
+                        fields.append(kw.arg.split("__")[0])
+            fields = [f for f in fields if f and f not in SKIP_LOOKUPS and not f.startswith("_")]
+            if not fields:
+                continue
+            out.append(
+                {
+                    "owner": owner,
+                    "kind": short,
+                    "fields": fields,
+                    "line": getattr(node, "lineno", 0),
+                }
+            )
+        for stmt in fn.body:
+            if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                visit_function(stmt, stmt.name)
+
+    if not isinstance(tree, ast.Module):
+        return out
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef):
+            for stmt in node.body:
+                if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    visit_function(stmt, node.name)
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            visit_function(node, node.name)
+    return out