From 72a4e8e28864078f54ecd2bbf96a5b6e99549734 Mon Sep 17 00:00:00 2001 From: anonymous Date: Sun, 19 Jul 2026 23:05:10 -0500 Subject: [PATCH 1/5] feat: polyglot multilang ingest/parse with cross-language relink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add languages=[] / language="multi" for Maven dual-root discovery and merge+relink of import/extends/implements across Java↔Kotlin FQN space (roadmap #18 MVP). Wire MCP and CI analyze inputs; keep first entity on cross-language FQN collisions. Co-authored-by: Cursor --- .../architecture-analysis-reusable.yml | 17 +- README.md | 2 +- ROADMAP.md | 6 +- actions/analyze/action.yml | 20 +- src/arcade_agent/ci/run_self_analysis.py | 37 +++- src/arcade_agent/parsers/graph.py | 1 + src/arcade_agent/parsers/multilang.py | 139 +++++++++++++ src/arcade_agent/tools/adapters/mcp.py | 20 +- src/arcade_agent/tools/ingest.py | 193 +++++++++++++----- src/arcade_agent/tools/parse.py | 147 +++++++++---- .../com/example/mixed/JavaBaseService.java | 7 + .../com/example/mixed/KotlinService.kt | 5 + .../com/example/mixed/SharedContract.java | 5 + .../main/java/com/example/JavaGreeter.java | 7 + .../main/kotlin/com/example/KotlinGreeter.kt | 3 + tests/test_parsers/test_graph_merge.py | 135 ++++++++++++ tests/test_tools/test_ingest_multilang.py | 44 ++++ tests/test_tools/test_parse_multilang.py | 61 ++++++ 18 files changed, 742 insertions(+), 107 deletions(-) create mode 100644 src/arcade_agent/parsers/multilang.py create mode 100644 tests/fixtures/java_kotlin_mixed/com/example/mixed/JavaBaseService.java create mode 100644 tests/fixtures/java_kotlin_mixed/com/example/mixed/KotlinService.kt create mode 100644 tests/fixtures/java_kotlin_mixed/com/example/mixed/SharedContract.java create mode 100644 tests/fixtures/maven_java_kotlin/src/main/java/com/example/JavaGreeter.java create mode 100644 tests/fixtures/maven_java_kotlin/src/main/kotlin/com/example/KotlinGreeter.kt create mode 100644 tests/test_parsers/test_graph_merge.py create mode 100644 tests/test_tools/test_ingest_multilang.py create mode 100644 tests/test_tools/test_parse_multilang.py diff --git a/.github/workflows/architecture-analysis-reusable.yml b/.github/workflows/architecture-analysis-reusable.yml index 57ce166..0697615 100644 --- a/.github/workflows/architecture-analysis-reusable.yml +++ b/.github/workflows/architecture-analysis-reusable.yml @@ -24,7 +24,12 @@ on: type: string default: "." language: - description: Optional language override (java, python, typescript, c, go, kotlin). + description: Optional language override (java, python, typescript, c, go, kotlin, multi). + required: false + type: string + default: "" + languages: + description: Optional comma-separated polyglot languages (e.g. java,kotlin). required: false type: string default: "" @@ -142,12 +147,15 @@ jobs: env: SOURCE_PATH: ${{ inputs.source-path }} LANGUAGE: ${{ inputs.language }} + LANGUAGES: ${{ inputs.languages }} REPO_NAME: ${{ inputs.repo-name }} PRIMARY_ALGORITHM: ${{ inputs.primary-algorithm }} FILTER_NON_ARCHITECTURAL_HELPERS: ${{ inputs.filter-non-architectural-helpers }} run: | ARGS=(--source "target-repo/${SOURCE_PATH}") - if [ -n "${LANGUAGE}" ]; then + if [ -n "${LANGUAGES}" ]; then + ARGS+=(--languages "${LANGUAGES}") + elif [ -n "${LANGUAGE}" ]; then ARGS+=(--language "${LANGUAGE}") fi if [ -n "${REPO_NAME}" ]; then @@ -167,12 +175,15 @@ jobs: env: SOURCE_PATH: ${{ inputs.source-path }} LANGUAGE: ${{ inputs.language }} + LANGUAGES: ${{ inputs.languages }} REPO_NAME: ${{ inputs.repo-name }} WCA_NUM_CLUSTERS: ${{ inputs.wca-num-clusters }} FILTER_NON_ARCHITECTURAL_HELPERS: ${{ inputs.filter-non-architectural-helpers }} run: | ARGS=(--source "target-repo/${SOURCE_PATH}") - if [ -n "${LANGUAGE}" ]; then + if [ -n "${LANGUAGES}" ]; then + ARGS+=(--languages "${LANGUAGES}") + elif [ -n "${LANGUAGE}" ]; then ARGS+=(--language "${LANGUAGE}") fi if [ -n "${REPO_NAME}" ]; then diff --git a/README.md b/README.md index 939a1a3..15a44a9 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,7 @@ arcade-agent ports and extends the capabilities of the original [ARCADE](https:/ | 6 quality metrics | Done | RCI, TurboMQ, BasicMQ, IntraConnectivity, InterConnectivity, TwoWayPairRatio | | Balanced architecture score | Done | Derived reporting score combining core metrics, principle signals, and smell burden | | A2A architecture comparison | Done | Hungarian algorithm on Jaccard similarity | -| Multi-language parsing | Done | Java, Python, C/C++, TypeScript/JavaScript, Go (full); Kotlin (structural) | +| Multi-language parsing | Done | Java, Python, C/C++, TypeScript/JavaScript, Go (full); Kotlin (structural); polyglot merge+relink via `languages=[...]` / `language="multi"` | | 5 export formats | Done | HTML, DOT, JSON, RSF, Mermaid | | LLM concern extraction | Done | Claude CLI for semantic BCO/SPF detection | | MCP server | Done | Expose tools to AI agents via Model Context Protocol with session store | diff --git a/ROADMAP.md b/ROADMAP.md index 469311f..b79cedb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -45,7 +45,7 @@ Handle real-world polyglot monorepos. - [x] **16a2. Kotlin parser** — Shipped (`parsers/kotlin.py`) for JVM/Kotlin-first repos (e.g. embabel-agent). - [ ] **16b. Rust parser** — Still open. High-demand language for agent-assisted development. - [x] **17. Incremental parsing** — Content-hash extract cache shipped in #9 (`incremental.py`), wired for the Python parser only; extending to the other two-pass parsers is follow-up. -- [ ] **18. Cross-language dependency tracking** — Java↔Python via gRPC, TS frontend↔Java backend, etc. +- [x] **18. Cross-language dependency tracking** — MVP: multi-language ingest/parse (`languages=[...]` / `language="multi"`) merges per-language graphs and relinks import/extends/implements across FQN space (Java↔Kotlin first; same mechanism for other same-FQN or import-linked pairs). Broader RPC/IDL bridges (gRPC stubs, OpenAPI) remain follow-up. ## Phase 6 — Agent Protocol Integration @@ -60,7 +60,7 @@ Work everywhere agents work. | Priority | Items | Rationale | |----------|-------|-----------| -| **Done** | 1–9, 12, 13, 14, 15, 16a, 17 | Phases 1–2 + TS/JS & Go parsers, incremental parsing (Python), `diff_impact`, `context_for_task`, `api_surface` | +| **Done** | 1–9, 12, 13, 14, 15, 16a, 16a2, 17, 18 (MVP) | Phases 1–2 + TS/JS & Go & Kotlin parsers, incremental parsing (Python), `diff_impact`, `context_for_task`, `api_surface`, polyglot merge+relink | | **Now** | 10 | Architectural changelog | | **Next** | 11, 16b | Component ownership, Rust parser | -| **Then** | 18–22 | Cross-language tracking, ecosystem breadth | +| **Then** | 19–22 | Ecosystem breadth (OpenAI / LangChain / Claude SDK / IDE) | diff --git a/actions/analyze/action.yml b/actions/analyze/action.yml index 70a2107..24912a3 100644 --- a/actions/analyze/action.yml +++ b/actions/analyze/action.yml @@ -23,7 +23,15 @@ inputs: required: false default: "." language: - description: Optional language override (java, python, typescript, c, go, kotlin). + description: > + Optional language override (java, python, typescript, c, go, kotlin, or multi + for every detected language with cross-language edge relinking). + required: false + default: "" + languages: + description: > + Optional comma-separated polyglot languages (e.g. java,kotlin). Mutually + exclusive with language when both would be set by the caller. required: false default: "" repo-name: @@ -135,12 +143,15 @@ runs: env: SOURCE_PATH: ${{ inputs.source-path }} LANGUAGE: ${{ inputs.language }} + LANGUAGES: ${{ inputs.languages }} REPO_NAME: ${{ inputs.repo-name }} PRIMARY_ALGORITHM: ${{ inputs.primary-algorithm }} FILTER_NON_ARCHITECTURAL_HELPERS: ${{ inputs.filter-non-architectural-helpers }} run: | ARGS=(--source "${SOURCE_PATH}") - if [ -n "${LANGUAGE}" ]; then + if [ -n "${LANGUAGES}" ]; then + ARGS+=(--languages "${LANGUAGES}") + elif [ -n "${LANGUAGE}" ]; then ARGS+=(--language "${LANGUAGE}") fi if [ -n "${REPO_NAME}" ]; then @@ -161,12 +172,15 @@ runs: env: SOURCE_PATH: ${{ inputs.source-path }} LANGUAGE: ${{ inputs.language }} + LANGUAGES: ${{ inputs.languages }} REPO_NAME: ${{ inputs.repo-name }} WCA_NUM_CLUSTERS: ${{ inputs.wca-num-clusters }} FILTER_NON_ARCHITECTURAL_HELPERS: ${{ inputs.filter-non-architectural-helpers }} run: | ARGS=(--source "${SOURCE_PATH}") - if [ -n "${LANGUAGE}" ]; then + if [ -n "${LANGUAGES}" ]; then + ARGS+=(--languages "${LANGUAGES}") + elif [ -n "${LANGUAGE}" ]; then ARGS+=(--language "${LANGUAGE}") fi if [ -n "${REPO_NAME}" ]; then diff --git a/src/arcade_agent/ci/run_self_analysis.py b/src/arcade_agent/ci/run_self_analysis.py index 3e137f3..91b15b2 100644 --- a/src/arcade_agent/ci/run_self_analysis.py +++ b/src/arcade_agent/ci/run_self_analysis.py @@ -100,7 +100,15 @@ def main() -> None: parser.add_argument( "--language", default="", - help="Optional language override (java, python, typescript, c, go, kotlin)", + help=( + "Optional language override (java, python, typescript, c, go, kotlin, " + "or multi for every detected language)" + ), + ) + parser.add_argument( + "--languages", + default="", + help="Comma-separated polyglot languages (e.g. java,kotlin)", ) parser.add_argument( "--repo-name", @@ -140,21 +148,33 @@ def main() -> None: source = str(Path(args.source).resolve()) language = args.language or None + languages = [part.strip() for part in args.languages.split(",") if part.strip()] or None print(f"[1/5] Ingesting {source}...") - repo = ingest(source, language=language) - print(f" Found {len(repo.source_files)} source files") + repo = ingest(source, language=language, languages=languages) + print( + f" Found {len(repo.source_files)} source files " + f"(languages={repo.languages or [repo.language]})" + ) if not repo.source_files: print(" No source files found. Exiting.") sys.exit(1) print("[2/5] Parsing dependencies...") - raw_graph = parse( - str(repo.path), - language=repo.language or language or "python", - files=[str(f) for f in repo.source_files], - ) + parse_files = [str(f) for f in repo.source_files] + if len(repo.languages) > 1: + raw_graph = parse( + str(repo.path), + languages=repo.languages, + files=parse_files, + ) + else: + raw_graph = parse( + str(repo.path), + language=repo.language or language or "python", + files=parse_files, + ) graph = ( _filter_non_architectural_entities(raw_graph) if args.filter_non_architectural_helpers @@ -182,6 +202,7 @@ def main() -> None: "timestamp": datetime.now(timezone.utc).isoformat(), "repo_name": args.repo_name or repo.name, "language": repo.language or language, + "languages": repo.languages, "commit_sha": os.environ.get("GITHUB_SHA", "local"), "ref": os.environ.get("GITHUB_REF", "local"), "algorithm": args.algorithm, diff --git a/src/arcade_agent/parsers/graph.py b/src/arcade_agent/parsers/graph.py index 115615c..1d9c3a8 100644 --- a/src/arcade_agent/parsers/graph.py +++ b/src/arcade_agent/parsers/graph.py @@ -65,4 +65,5 @@ def merge(self, other: "DependencyGraph") -> "DependencyGraph": packages.setdefault(pkg, []).extend(fqns) for pkg, fqns in other.packages.items(): packages.setdefault(pkg, []).extend(fqns) + packages = {pkg: list(dict.fromkeys(fqns)) for pkg, fqns in packages.items()} return DependencyGraph(entities=entities, edges=edges, packages=packages) diff --git a/src/arcade_agent/parsers/multilang.py b/src/arcade_agent/parsers/multilang.py new file mode 100644 index 0000000..36d67e0 --- /dev/null +++ b/src/arcade_agent/parsers/multilang.py @@ -0,0 +1,139 @@ +"""Cross-language graph merge and edge relinking (roadmap #18).""" + +from __future__ import annotations + +import logging + +from arcade_agent.parsers.graph import DependencyGraph, Edge, Entity + +logger = logging.getLogger(__name__) + + +def resolve_name( + simple_name: str, + source_entity: Entity, + fqn_index: dict[str, str], + entities: dict[str, Entity], + aliases: dict[str, str] | None = None, +) -> str | None: + """Resolve a simple or qualified type name to an entity FQN.""" + if simple_name in entities: + return simple_name + + if aliases and simple_name in aliases: + aliased = aliases[simple_name] + if aliased in entities: + return aliased + + if "." in simple_name and simple_name in entities: + return simple_name + + for imp in source_entity.imports: + if imp.endswith(f".{simple_name}") and imp in entities: + return imp + + if source_entity.package: + same_pkg_fqn = f"{source_entity.package}.{simple_name}" + if same_pkg_fqn in entities: + return same_pkg_fqn + + leaf = simple_name.split(".")[-1] + if leaf in fqn_index: + return fqn_index[leaf] + + return None + + +def _aliases_for(entity: Entity) -> dict[str, str]: + raw = entity.properties.get("import_aliases") + if isinstance(raw, dict): + return {str(k): str(v) for k, v in raw.items()} + return {} + + +def _build_fqn_index(entities: dict[str, Entity]) -> dict[str, str]: + index: dict[str, str] = {} + for entity in entities.values(): + index[entity.name] = entity.fqn + return index + + +def relink_edges(graph: DependencyGraph) -> DependencyGraph: + """Add import/extends/implements edges resolvable against the full entity set. + + Language parsers only resolve against their own entities. After merging + Java+Kotlin (or other polyglot) graphs, re-run resolution so same-package + and imported cross-language types become real edges. + """ + entities = graph.entities + fqn_index = _build_fqn_index(entities) + seen = {(e.source, e.target, e.relation) for e in graph.edges} + new_edges: list[Edge] = list(graph.edges) + + def add(source: str, target: str, relation: str) -> None: + key = (source, target, relation) + if key in seen or source == target: + return + seen.add(key) + new_edges.append(Edge(source=source, target=target, relation=relation)) + + for entity in entities.values(): + aliases = _aliases_for(entity) + for imp in entity.imports: + if imp in entities: + add(entity.fqn, imp, "import") + else: + simple = imp.split(".")[-1] + resolved = fqn_index.get(simple) + if resolved and resolved != entity.fqn: + add(entity.fqn, resolved, "import") + + if entity.superclass: + target = resolve_name( + entity.superclass, entity, fqn_index, entities, aliases + ) + if target: + add(entity.fqn, target, "extends") + + for iface in entity.interfaces: + target = resolve_name(iface, entity, fqn_index, entities, aliases) + if target: + add(entity.fqn, target, "implements") + + packages: dict[str, list[str]] = { + pkg: list(dict.fromkeys(fqns)) for pkg, fqns in graph.packages.items() + } + return DependencyGraph(entities=entities, edges=new_edges, packages=packages) + + +def merge_and_relink(*graphs: DependencyGraph) -> DependencyGraph: + """Union graphs then relink edges across the combined entity set.""" + if not graphs: + return DependencyGraph() + + entities: dict[str, Entity] = {} + edges: list[Edge] = [] + packages: dict[str, list[str]] = {} + + for graph in graphs: + for fqn, entity in graph.entities.items(): + if fqn in entities: + existing = entities[fqn] + if existing.language != entity.language: + logger.warning( + "FQN collision across languages at %s (%s vs %s); " + "keeping first", + fqn, + existing.language, + entity.language, + ) + continue + entities[fqn] = entity + edges.extend(graph.edges) + for pkg, fqns in graph.packages.items(): + packages.setdefault(pkg, []).extend(fqns) + + packages = {pkg: list(dict.fromkeys(fqns)) for pkg, fqns in packages.items()} + return relink_edges( + DependencyGraph(entities=entities, edges=edges, packages=packages) + ) diff --git a/src/arcade_agent/tools/adapters/mcp.py b/src/arcade_agent/tools/adapters/mcp.py index 52b8d90..76d85ef 100644 --- a/src/arcade_agent/tools/adapters/mcp.py +++ b/src/arcade_agent/tools/adapters/mcp.py @@ -75,6 +75,10 @@ def _make_summary(obj: Any, label: str) -> dict: summary["num_files"] = len(obj.source_files) if hasattr(obj, "language"): summary["language"] = obj.language + if hasattr(obj, "languages"): + langs = getattr(obj, "languages") + if langs: + summary["languages"] = list(langs) if hasattr(obj, "name") and isinstance(getattr(obj, "name", None), str): summary["name"] = obj.name if hasattr(obj, "version"): @@ -134,6 +138,7 @@ def _build_server(): # type: ignore[no-untyped-def] def ingest( source: str, language: str | None = None, + languages: list[str] | None = None, work_dir: str | None = None, exclude_tests: bool = True, source_root: str | None = None, @@ -146,7 +151,10 @@ def ingest( Args: source: Git repo URL or local directory path. - language: Override language detection (java, python, c, typescript, go, kotlin). + language: Override language detection (java, python, c, typescript, + go, kotlin, or "multi" for every detected language). + languages: Explicit polyglot language list (e.g. ["java", "kotlin"]). + Mutually exclusive with language. work_dir: Directory to clone into. Uses temp dir if None. exclude_tests: Exclude test/vendor/build directories (default True). source_root: Override source root (e.g. 'src/main/java'). @@ -157,6 +165,7 @@ def ingest( result = _ingest( source=source, language=language, + languages=languages, work_dir=work_dir, exclude_tests=exclude_tests, source_root=source_root, @@ -170,6 +179,7 @@ def ingest( def parse( source_path: str, language: str | None = None, + languages: list[str] | None = None, files: list[str] | None = None, use_cache: bool = True, max_tokens: int | None = None, @@ -181,8 +191,11 @@ def parse( Args: source_path: Root directory of the project. - language: Language to parse (java, python, c, typescript, go, kotlin). - Auto-detected if None. + language: Language to parse (java, python, c, typescript, go, kotlin), + or "multi" to parse every detected language and relink + cross-language edges. + languages: Explicit polyglot language list (e.g. ["java", "kotlin"]). + Mutually exclusive with language. files: Specific files to parse. Discovers all if None. use_cache: Return cached results when source files haven't changed. max_tokens: Optional token budget for the response. @@ -192,6 +205,7 @@ def parse( graph = _parse( source_path=source_path, language=language, + languages=languages, files=files, use_cache=use_cache, ) diff --git a/src/arcade_agent/tools/ingest.py b/src/arcade_agent/tools/ingest.py index 6280cec..56bdb72 100644 --- a/src/arcade_agent/tools/ingest.py +++ b/src/arcade_agent/tools/ingest.py @@ -24,6 +24,7 @@ class IngestedRepo: is_temp: bool = False source_files: list[Path] = field(default_factory=list) language: str | None = None + languages: list[str] = field(default_factory=list) versions: list[str] = field(default_factory=list) def cleanup(self) -> None: @@ -48,20 +49,11 @@ def cleanup(self) -> None: for ext in exts: _EXT_TO_LANG[ext] = lang - -def _detect_language(path: Path) -> str | None: - """Auto-detect the primary language from file extensions.""" - ext_counts: dict[str, int] = {} - for f in path.rglob("*"): - if f.is_file() and f.suffix in _EXT_TO_LANG: - ext_counts[f.suffix] = ext_counts.get(f.suffix, 0) + 1 - - if not ext_counts: - return None - - best_ext = max(ext_counts, key=ext_counts.get) # type: ignore[arg-type] - return _EXT_TO_LANG.get(best_ext) - +_LANG_PREFERRED_ROOTS: dict[str, str] = { + "java": "src/main/java", + "kotlin": "src/main/kotlin", + "scala": "src/main/scala", +} # Well-known source root directories (tried in order) _SOURCE_ROOTS = [ @@ -99,12 +91,40 @@ def _detect_language(path: Path) -> str | None: } +def _detect_language(path: Path) -> str | None: + """Auto-detect the primary language from file extensions.""" + ext_counts: dict[str, int] = {} + for f in path.rglob("*"): + if f.is_file() and f.suffix in _EXT_TO_LANG: + ext_counts[f.suffix] = ext_counts.get(f.suffix, 0) + 1 + + if not ext_counts: + return None + + best_ext = max(ext_counts, key=ext_counts.get) # type: ignore[arg-type] + return _EXT_TO_LANG.get(best_ext) + + +def _detect_languages(path: Path) -> list[str]: + """Detect all languages present under path (sorted).""" + found: set[str] = set() + for f in path.rglob("*"): + if f.is_file() and f.suffix in _EXT_TO_LANG: + found.add(_EXT_TO_LANG[f.suffix]) + return sorted(found) + + def _detect_source_root(path: Path, language: str | None = None) -> Path: """Detect the main source root directory. - Checks for well-known source root patterns (e.g., src/main/java for Maven). - Falls back to the project root. + Prefers a language-specific Maven/Gradle root when *language* is set. + Falls back to well-known roots, then the project root. """ + if language: + preferred = _LANG_PREFERRED_ROOTS.get(language) + if preferred and (path / preferred).is_dir(): + return path / preferred + for candidate in _SOURCE_ROOTS: root = path / candidate if root.is_dir(): @@ -124,7 +144,6 @@ def _should_exclude(file_path: Path, root: Path) -> bool: subpath = "/".join(parts[: i + 1]) if subpath in _EXCLUDE_DIRS: return True - # Also check just the directory name if parts[i] in _EXCLUDE_DIRS: return True return False @@ -160,6 +179,28 @@ def _discover_files( return files +def _resolve_languages( + path: Path, + language: str | None, + languages: list[str] | None, +) -> list[str]: + if language is not None and languages is not None: + raise ValueError("Pass only one of language and languages") + if languages is not None: + if not languages: + raise ValueError("languages must be non-empty") + return list(languages) + if language == "multi": + detected = _detect_languages(path) + if not detected: + raise ValueError(f"Could not detect languages in {path}") + return detected + if language: + return [language] + primary = _detect_language(path) + return [primary] if primary else [] + + def _detect_version(repo: "Repo") -> str: """Detect the latest version tag from a repo.""" try: @@ -196,6 +237,7 @@ def _repo_name_from_url(url: str) -> str: def ingest( source: str, language: str | None = None, + languages: list[str] | None = None, work_dir: str | None = None, exclude_tests: bool = True, source_root: str | None = None, @@ -204,7 +246,10 @@ def ingest( Args: source: Git repo URL or local directory path. - language: Override language detection (java, python, typescript, c, go, kotlin). + language: Override language detection (java, python, typescript, c, go, + kotlin, or "multi" to ingest every detected language). + languages: Explicit language list for polyglot ingest (e.g. ["java", "kotlin"]). + Mutually exclusive with *language*. work_dir: Directory to clone into. Uses temp dir if None. exclude_tests: Exclude test/vendor/build directories (default: True). source_root: Override source root (e.g., 'src/main/java'). Auto-detected if None. @@ -215,15 +260,21 @@ def ingest( source_path = Path(source) sr = Path(source_root) if source_root else None if source_path.is_dir(): - return _ingest_local(source_path, language, exclude_tests, sr) + return _ingest_local(source_path, language, languages, exclude_tests, sr) return _clone_and_ingest( - source, language, Path(work_dir) if work_dir else None, exclude_tests, sr, + source, + language, + languages, + Path(work_dir) if work_dir else None, + exclude_tests, + sr, ) def _ingest_local( path: Path, language: str | None = None, + languages: list[str] | None = None, exclude_tests: bool = True, source_root: Path | None = None, ) -> IngestedRepo: @@ -240,32 +291,23 @@ def _ingest_local( except Exception: pass - if not language: - language = _detect_language(path) - - # Auto-detect source root if not provided - effective_root = source_root - if effective_root is None and exclude_tests: - detected = _detect_source_root(path, language) - if detected != path: - effective_root = detected - - source_files = _discover_files(path, language, exclude_tests, effective_root) - - return IngestedRepo( - path=effective_root if effective_root else path, + resolved = _resolve_languages(path, language, languages) + return _build_ingested_repo( + project_root=path, name=name, version=version, - is_temp=False, - source_files=source_files, - language=language, versions=versions, + is_temp=False, + languages=resolved, + exclude_tests=exclude_tests, + source_root=source_root, ) def _clone_and_ingest( url: str, language: str | None = None, + languages: list[str] | None = None, work_dir: Path | None = None, exclude_tests: bool = True, source_root: Path | None = None, @@ -289,23 +331,80 @@ def _clone_and_ingest( except GitCommandError: pass - if not language: - language = _detect_language(clone_path) + resolved = _resolve_languages(clone_path, language, languages) + return _build_ingested_repo( + project_root=clone_path, + name=name, + version=version, + versions=versions, + is_temp=True, + languages=resolved, + exclude_tests=exclude_tests, + source_root=source_root, + ) + - effective_root = source_root - if effective_root is None and exclude_tests: - detected = _detect_source_root(clone_path, language) - if detected != clone_path: +def _build_ingested_repo( + *, + project_root: Path, + name: str, + version: str, + versions: list[str], + is_temp: bool, + languages: list[str], + exclude_tests: bool, + source_root: Path | None, +) -> IngestedRepo: + multilang = len(languages) > 1 + + if source_root is not None: + effective_root = source_root + search_root = source_root + result_path = source_root + elif multilang: + # Keep the project root so every language-specific tree stays visible. + effective_root = None + search_root = None + result_path = project_root + elif exclude_tests and languages: + detected = _detect_source_root(project_root, languages[0]) + if detected != project_root: effective_root = detected + search_root = detected + result_path = detected + else: + effective_root = None + search_root = None + result_path = project_root + else: + effective_root = None + search_root = None + result_path = project_root + + source_files: list[Path] = [] + if languages: + for lang in languages: + source_files.extend( + _discover_files(project_root, lang, exclude_tests, search_root) + ) + # Preserve stable order while dropping duplicates across languages. + source_files = list(dict.fromkeys(source_files)) + else: + source_files = _discover_files( + project_root, None, exclude_tests, effective_root + ) - source_files = _discover_files(clone_path, language, exclude_tests, effective_root) + primary = languages[0] if len(languages) == 1 else ( + "multi" if languages else None + ) return IngestedRepo( - path=effective_root if effective_root else clone_path, + path=result_path, name=name, version=version, - is_temp=True, + is_temp=is_temp, source_files=source_files, - language=language, + language=primary, + languages=languages, versions=versions, ) diff --git a/src/arcade_agent/tools/parse.py b/src/arcade_agent/tools/parse.py index f9b3d23..a3a3ff6 100644 --- a/src/arcade_agent/tools/parse.py +++ b/src/arcade_agent/tools/parse.py @@ -3,14 +3,98 @@ import logging from pathlib import Path +import arcade_agent.parsers # noqa: F401 — register language parsers from arcade_agent.cache import cache_key, get_cached_graph, put_cached_graph from arcade_agent.parsers.base import detect_language, get_parser from arcade_agent.parsers.graph import DependencyGraph +from arcade_agent.parsers.multilang import merge_and_relink from arcade_agent.tools.registry import tool logger = logging.getLogger(__name__) +def _cache_language_key( + language: str | None, + languages: list[str] | None, +) -> str | None: + if languages: + return ",".join(sorted(languages)) + return language + + +def _resolve_languages( + root: Path, + language: str | None, + languages: list[str] | None, + file_paths: list[Path] | None, +) -> list[str]: + if language is not None and languages is not None: + raise ValueError("Pass only one of language and languages") + if languages is not None: + if not languages: + raise ValueError("languages must be non-empty") + return list(languages) + if language == "multi": + discover = file_paths if file_paths is not None else list(root.rglob("*")) + detected = detect_languages_from_files(discover) + if not detected: + raise ValueError(f"Could not detect languages in {root}") + return detected + if language: + return [language] + discover = file_paths if file_paths is not None else [ + f for f in root.rglob("*") if f.is_file() + ] + detected = detect_language(discover) + if not detected: + raise ValueError(f"Could not detect language in {root}") + return [detected] + + +def detect_languages_from_files(files: list[Path]) -> list[str]: + """Return sorted language names present among *files*.""" + found: set[str] = set() + for path in files: + if not path.is_file(): + continue + try: + parser = get_parser(path.suffix.lower()) + except KeyError: + continue + found.add(parser.language) + return sorted(found) + + +def _files_for_language(files: list[Path], language: str) -> list[Path]: + parser = get_parser(language) + exts = set(parser.file_extensions) + return [f for f in files if f.suffix in exts] + + +def _parse_one( + language: str, + file_paths: list[Path], + root: Path, + use_cache: bool, +) -> DependencyGraph: + parser = get_parser(language) + if not file_paths: + return DependencyGraph() + if use_cache and hasattr(parser, "parse_incremental"): + from arcade_agent.incremental import ExtractCache + return parser.parse_incremental(file_paths, root, ExtractCache(root)) + return parser.parse(file_paths, root) + + +def _discover_files(root: Path, languages: list[str]) -> list[Path]: + file_paths: list[Path] = [] + for language in languages: + parser = get_parser(language) + for ext in parser.file_extensions: + file_paths.extend(sorted(root.rglob(f"*{ext}"))) + return list(dict.fromkeys(file_paths)) + + @tool( name="parse", description=( @@ -21,6 +105,7 @@ def parse( source_path: str, language: str | None = None, + languages: list[str] | None = None, files: list[str] | None = None, use_cache: bool = True, ) -> DependencyGraph: @@ -28,7 +113,10 @@ def parse( Args: source_path: Root directory of the project. - language: Language to parse (java, python, etc.). Auto-detected if None. + language: Language to parse (java, python, etc.), or "multi" to parse + every detected language and merge+relink cross-language edges. + languages: Explicit language list for polyglot parse + (e.g. ["java", "kotlin"]). Mutually exclusive with *language*. files: Specific files to parse. If None, discovers all files. use_cache: If True, return cached results when source files haven't changed. @@ -36,55 +124,36 @@ def parse( DependencyGraph with entities, edges, and package info. """ root = Path(source_path) + provided_files = [Path(f) for f in files] if files else None + resolved = _resolve_languages(root, language, languages, provided_files) + cache_lang = _cache_language_key( + language if language != "multi" else "multi", + resolved if len(resolved) > 1 else None, + ) - # Check cache before doing expensive parsing if use_cache: - key = cache_key(source_path, language, files) + key = cache_key(source_path, cache_lang, files) cached = get_cached_graph(source_path, key) if cached is not None: return cached - if files: - file_paths = [Path(f) for f in files] + if provided_files is not None: + file_paths = provided_files else: - # Discover files - if language: - parser = get_parser(language) - file_paths = [] - for ext in parser.file_extensions: - file_paths.extend(sorted(root.rglob(f"*{ext}"))) - else: - # Try to detect language from files - all_files = list(root.rglob("*")) - source_files = [f for f in all_files if f.is_file()] - detected = detect_language(source_files) - if not detected: - raise ValueError(f"Could not detect language in {source_path}") - language = detected - parser = get_parser(language) - file_paths = [] - for ext in parser.file_extensions: - file_paths.extend(sorted(root.rglob(f"*{ext}"))) - - if not language: - raise ValueError("No language specified and auto-detection failed") - - parser = get_parser(language) + file_paths = _discover_files(root, resolved) - # Two cache layers: the whole-graph cache above returns instantly when NOTHING - # changed; when some files changed we fall here and parse incrementally — - # re-extracting only the changed files (by content hash) and re-linking. Edges - # are recomputed every link, so the incremental graph is identical to a full - # parse. Parsers that don't support it (or use_cache=False) take the full path. - if use_cache and hasattr(parser, "parse_incremental"): - from arcade_agent.incremental import ExtractCache - graph = parser.parse_incremental(file_paths, root, ExtractCache(root)) + if len(resolved) == 1: + graph = _parse_one(resolved[0], file_paths, root, use_cache) else: - graph = parser.parse(file_paths, root) + graphs = [ + _parse_one(lang, _files_for_language(file_paths, lang), root, use_cache) + for lang in resolved + ] + graphs = [g for g in graphs if g.num_entities or g.num_edges] + graph = merge_and_relink(*graphs) if graphs else DependencyGraph() - # Store in cache for next time if use_cache: - key = cache_key(source_path, language, files) + key = cache_key(source_path, cache_lang, files) put_cached_graph(source_path, key, graph) return graph diff --git a/tests/fixtures/java_kotlin_mixed/com/example/mixed/JavaBaseService.java b/tests/fixtures/java_kotlin_mixed/com/example/mixed/JavaBaseService.java new file mode 100644 index 0000000..e860f85 --- /dev/null +++ b/tests/fixtures/java_kotlin_mixed/com/example/mixed/JavaBaseService.java @@ -0,0 +1,7 @@ +package com.example.mixed; + +public class JavaBaseService { + public String name() { + return "java"; + } +} diff --git a/tests/fixtures/java_kotlin_mixed/com/example/mixed/KotlinService.kt b/tests/fixtures/java_kotlin_mixed/com/example/mixed/KotlinService.kt new file mode 100644 index 0000000..51a1d64 --- /dev/null +++ b/tests/fixtures/java_kotlin_mixed/com/example/mixed/KotlinService.kt @@ -0,0 +1,5 @@ +package com.example.mixed + +class KotlinService : JavaBaseService(), SharedContract { + override fun run() {} +} diff --git a/tests/fixtures/java_kotlin_mixed/com/example/mixed/SharedContract.java b/tests/fixtures/java_kotlin_mixed/com/example/mixed/SharedContract.java new file mode 100644 index 0000000..ff5df8d --- /dev/null +++ b/tests/fixtures/java_kotlin_mixed/com/example/mixed/SharedContract.java @@ -0,0 +1,5 @@ +package com.example.mixed; + +public interface SharedContract { + void run(); +} diff --git a/tests/fixtures/maven_java_kotlin/src/main/java/com/example/JavaGreeter.java b/tests/fixtures/maven_java_kotlin/src/main/java/com/example/JavaGreeter.java new file mode 100644 index 0000000..d44ffe1 --- /dev/null +++ b/tests/fixtures/maven_java_kotlin/src/main/java/com/example/JavaGreeter.java @@ -0,0 +1,7 @@ +package com.example; + +public class JavaGreeter { + public String greet() { + return "hello"; + } +} diff --git a/tests/fixtures/maven_java_kotlin/src/main/kotlin/com/example/KotlinGreeter.kt b/tests/fixtures/maven_java_kotlin/src/main/kotlin/com/example/KotlinGreeter.kt new file mode 100644 index 0000000..bf88cdd --- /dev/null +++ b/tests/fixtures/maven_java_kotlin/src/main/kotlin/com/example/KotlinGreeter.kt @@ -0,0 +1,3 @@ +package com.example + +class KotlinGreeter : JavaGreeter() diff --git a/tests/test_parsers/test_graph_merge.py b/tests/test_parsers/test_graph_merge.py new file mode 100644 index 0000000..5b92158 --- /dev/null +++ b/tests/test_parsers/test_graph_merge.py @@ -0,0 +1,135 @@ +"""Tests for DependencyGraph.merge and cross-language relink (roadmap #18).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_kotlin") +pytest.importorskip("tree_sitter_java") + +from arcade_agent.parsers.graph import DependencyGraph, Edge, Entity # noqa: E402 +from arcade_agent.parsers.java import JavaParser # noqa: E402 +from arcade_agent.parsers.kotlin import KotlinParser # noqa: E402 +from arcade_agent.parsers.multilang import merge_and_relink, relink_edges # noqa: E402 + + +def test_merge_unions_entities_and_edges(): + left = DependencyGraph( + entities={ + "a.A": Entity( + fqn="a.A", + name="A", + package="a", + file_path="A.java", + kind="class", + language="java", + ) + }, + edges=[Edge(source="a.A", target="a.B", relation="import")], + packages={"a": ["a.A"]}, + ) + right = DependencyGraph( + entities={ + "a.B": Entity( + fqn="a.B", + name="B", + package="a", + file_path="B.kt", + kind="class", + language="kotlin", + ) + }, + edges=[], + packages={"a": ["a.B"]}, + ) + + merged = left.merge(right) + assert set(merged.entities) == {"a.A", "a.B"} + assert merged.num_edges == 1 + assert sorted(merged.packages["a"]) == ["a.A", "a.B"] + + +def test_naive_merge_misses_cross_language_extends(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + java_files = sorted(root.rglob("*.java")) + kotlin_files = sorted(root.rglob("*.kt")) + + java_graph = JavaParser().parse(java_files, root) + kotlin_graph = KotlinParser().parse(kotlin_files, root) + naive = java_graph.merge(kotlin_graph) + + edge_tuples = set(naive.to_edge_tuples()) + assert ( + "com.example.mixed.KotlinService", + "com.example.mixed.JavaBaseService", + "extends", + ) not in edge_tuples + + +def test_relink_adds_cross_language_extends_and_implements(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + java_files = sorted(root.rglob("*.java")) + kotlin_files = sorted(root.rglob("*.kt")) + + java_graph = JavaParser().parse(java_files, root) + kotlin_graph = KotlinParser().parse(kotlin_files, root) + linked = merge_and_relink(java_graph, kotlin_graph) + + edge_tuples = set(linked.to_edge_tuples()) + assert ( + "com.example.mixed.KotlinService", + "com.example.mixed.JavaBaseService", + "extends", + ) in edge_tuples + assert ( + "com.example.mixed.KotlinService", + "com.example.mixed.SharedContract", + "implements", + ) in edge_tuples + + +def test_relink_is_idempotent(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + java_files = sorted(root.rglob("*.java")) + kotlin_files = sorted(root.rglob("*.kt")) + linked = merge_and_relink( + JavaParser().parse(java_files, root), + KotlinParser().parse(kotlin_files, root), + ) + again = relink_edges(linked) + assert set(again.to_edge_tuples()) == set(linked.to_edge_tuples()) + + +def test_merge_and_relink_keeps_first_on_cross_language_fqn_collision(): + left = DependencyGraph( + entities={ + "a.Shared": Entity( + fqn="a.Shared", + name="Shared", + package="a", + file_path="Shared.java", + kind="class", + language="java", + ) + }, + packages={"a": ["a.Shared"]}, + ) + right = DependencyGraph( + entities={ + "a.Shared": Entity( + fqn="a.Shared", + name="Shared", + package="a", + file_path="Shared.kt", + kind="class", + language="kotlin", + ) + }, + packages={"a": ["a.Shared"]}, + ) + + merged = merge_and_relink(left, right) + assert merged.entities["a.Shared"].language == "java" + assert merged.entities["a.Shared"].file_path == "Shared.java" diff --git a/tests/test_tools/test_ingest_multilang.py b/tests/test_tools/test_ingest_multilang.py new file mode 100644 index 0000000..9f1f532 --- /dev/null +++ b/tests/test_tools/test_ingest_multilang.py @@ -0,0 +1,44 @@ +"""Multi-language ingest discovery (roadmap #18).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from arcade_agent.tools.ingest import ingest + + +def test_ingest_languages_discovers_both_maven_roots(fixtures_dir: Path): + root = fixtures_dir / "maven_java_kotlin" + repo = ingest(str(root), languages=["java", "kotlin"]) + + suffixes = {p.suffix for p in repo.source_files} + assert ".java" in suffixes + assert ".kt" in suffixes + assert sorted(repo.languages) == ["java", "kotlin"] + # Project root kept so both Maven source trees remain visible. + assert repo.path.resolve() == root.resolve() + + +def test_ingest_language_multi_finds_java_and_kotlin(fixtures_dir: Path): + root = fixtures_dir / "maven_java_kotlin" + repo = ingest(str(root), language="multi") + assert "java" in repo.languages + assert "kotlin" in repo.languages + assert any(p.name == "JavaGreeter.java" for p in repo.source_files) + assert any(p.name == "KotlinGreeter.kt" for p in repo.source_files) + + +def test_ingest_single_language_still_narrows_to_matching_root(fixtures_dir: Path): + root = fixtures_dir / "maven_java_kotlin" + java_repo = ingest(str(root), language="java") + assert all(p.suffix == ".java" for p in java_repo.source_files) + assert java_repo.language == "java" + assert java_repo.languages == ["java"] + + +def test_ingest_rejects_language_and_languages_together(fixtures_dir: Path): + root = fixtures_dir / "maven_java_kotlin" + with pytest.raises(ValueError, match="language and languages"): + ingest(str(root), language="java", languages=["kotlin"]) diff --git a/tests/test_tools/test_parse_multilang.py b/tests/test_tools/test_parse_multilang.py new file mode 100644 index 0000000..8f4b39b --- /dev/null +++ b/tests/test_tools/test_parse_multilang.py @@ -0,0 +1,61 @@ +"""End-to-end multi-language parse (roadmap #18).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_kotlin") +pytest.importorskip("tree_sitter_java") + +from arcade_agent.tools.parse import parse # noqa: E402 + + +def test_parse_languages_java_kotlin_merges_and_relinks(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + graph = parse(str(root), languages=["java", "kotlin"], use_cache=False) + + assert "com.example.mixed.JavaBaseService" in graph.entities + assert "com.example.mixed.KotlinService" in graph.entities + assert graph.entities["com.example.mixed.JavaBaseService"].language == "java" + assert graph.entities["com.example.mixed.KotlinService"].language == "kotlin" + + edge_tuples = set(graph.to_edge_tuples()) + assert ( + "com.example.mixed.KotlinService", + "com.example.mixed.JavaBaseService", + "extends", + ) in edge_tuples + assert ( + "com.example.mixed.KotlinService", + "com.example.mixed.SharedContract", + "implements", + ) in edge_tuples + + +def test_parse_language_multi_auto_detects_present_languages(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + graph = parse(str(root), language="multi", use_cache=False) + + languages = {e.language for e in graph.entities.values()} + assert languages == {"java", "kotlin"} + assert any( + e.relation == "extends" + and e.source.endswith("KotlinService") + and e.target.endswith("JavaBaseService") + for e in graph.edges + ) + + +def test_parse_single_language_unchanged(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + java_only = parse(str(root), language="java", use_cache=False) + assert all(e.language == "java" for e in java_only.entities.values()) + assert "com.example.mixed.KotlinService" not in java_only.entities + + +def test_parse_rejects_language_and_languages_together(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + with pytest.raises(ValueError, match="language and languages"): + parse(str(root), language="java", languages=["kotlin"], use_cache=False) From 12d3496c4c476e0cdfea953c76ecc576e1dc294e Mon Sep 17 00:00:00 2001 From: anonymous Date: Sun, 19 Jul 2026 23:13:25 -0500 Subject: [PATCH 2/5] test: add Java+Kotlin polyglot E2E coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover ingest→parse→recover, MCP session pipeline, and CLI --languages self-analysis against existing mixed fixtures. Co-authored-by: Cursor --- tests/test_mcp_multilang_e2e.py | 121 ++++++++++++++++++ .../test_tools/test_pipeline_multilang_e2e.py | 74 +++++++++++ .../test_self_analysis_multilang.py | 48 +++++++ 3 files changed, 243 insertions(+) create mode 100644 tests/test_mcp_multilang_e2e.py create mode 100644 tests/test_tools/test_pipeline_multilang_e2e.py create mode 100644 tests/test_tools/test_self_analysis_multilang.py diff --git a/tests/test_mcp_multilang_e2e.py b/tests/test_mcp_multilang_e2e.py new file mode 100644 index 0000000..a68e34a --- /dev/null +++ b/tests/test_mcp_multilang_e2e.py @@ -0,0 +1,121 @@ +"""MCP end-to-end tests for polyglot Java+Kotlin ingest/parse/recover.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_kotlin") +pytest.importorskip("tree_sitter_java") + +_FIXTURES = Path(__file__).parent / "fixtures" +_MIXED = str(_FIXTURES / "java_kotlin_mixed") +_MAVEN = str(_FIXTURES / "maven_java_kotlin") + + +def _call(server, tool: str, args: dict) -> dict: + """Invoke a FastMCP tool (async) and return the parsed JSON response.""" + + async def _run(): + result = await server.call_tool(tool, args) + if isinstance(result, tuple): + content_list, _is_error = result + text = content_list[0].text if content_list else "" + elif isinstance(result, list): + text = result[0].text + elif isinstance(result, str): + text = result + else: + raise TypeError(f"Unexpected call_tool result type: {type(result)}") + return json.loads(text) + + return asyncio.run(_run()) + + +@pytest.fixture(scope="module") +def server(): + """Return the FastMCP server singleton (requires mcp extra).""" + pytest.importorskip("mcp", reason="mcp extra not installed") + from arcade_agent.tools.adapters.mcp import _session, get_server + + _session.clear() + return get_server() + + +class TestMcpMultilangE2E: + """MCP ingest/parse/recover against Java+Kotlin fixtures.""" + + def test_parse_languages_java_kotlin_then_recover(self, server): + parse_result = _call( + server, + "parse", + { + "source_path": _MIXED, + "languages": ["java", "kotlin"], + "use_cache": False, + }, + ) + assert parse_result.get("type") == "DependencyGraph" + assert parse_result["num_entities"] >= 2 + assert "session_id" in parse_result + + full = _call(server, "get_full_result", {"session_id": parse_result["session_id"]}) + entities = full["data"]["entities"] + languages = {e["language"] for e in entities.values()} + assert "java" in languages + assert "kotlin" in languages + + edges = full["data"]["edges"] + assert any( + edge["relation"] in {"extends", "implements", "import"} + and entities[edge["source"]]["language"] != entities[edge["target"]]["language"] + for edge in edges + if edge["source"] in entities and edge["target"] in entities + ) + + recover_result = _call( + server, + "recover", + {"dep_graph": parse_result["session_id"], "algorithm": "pkg"}, + ) + assert recover_result.get("type") == "Architecture" + assert recover_result["num_components"] > 0 + + def test_parse_language_multi_on_mixed_fixture(self, server): + result = _call( + server, + "parse", + {"source_path": _MIXED, "language": "multi", "use_cache": False}, + ) + assert result["num_entities"] >= 2 + full = _call(server, "get_full_result", {"session_id": result["session_id"]}) + languages = {e["language"] for e in full["data"]["entities"].values()} + assert languages == {"java", "kotlin"} + + def test_ingest_languages_then_parse_maven_fixture(self, server): + ingest_result = _call( + server, + "ingest", + {"source": _MAVEN, "languages": ["java", "kotlin"]}, + ) + assert ingest_result.get("type") == "IngestedRepo" + assert sorted(ingest_result.get("languages", [])) == ["java", "kotlin"] + assert ingest_result["num_files"] >= 2 + + parse_result = _call( + server, + "parse", + { + "source_path": _MAVEN, + "languages": ["java", "kotlin"], + "use_cache": False, + }, + ) + assert parse_result["num_entities"] >= 2 + full = _call(server, "get_full_result", {"session_id": parse_result["session_id"]}) + entity_fqns = set(full["data"]["entities"]) + assert "com.example.JavaGreeter" in entity_fqns + assert "com.example.KotlinGreeter" in entity_fqns diff --git a/tests/test_tools/test_pipeline_multilang_e2e.py b/tests/test_tools/test_pipeline_multilang_e2e.py new file mode 100644 index 0000000..63f6691 --- /dev/null +++ b/tests/test_tools/test_pipeline_multilang_e2e.py @@ -0,0 +1,74 @@ +"""Full ingest → parse → recover E2E for Java+Kotlin polyglot projects.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_kotlin") +pytest.importorskip("tree_sitter_java") + +from arcade_agent.tools.ingest import ingest # noqa: E402 +from arcade_agent.tools.parse import parse # noqa: E402 +from arcade_agent.tools.recover import recover # noqa: E402 + +_CROSS_LANG_RELATIONS = frozenset({"extends", "implements", "import"}) + + +def _assert_java_kotlin_graph(graph) -> None: + languages = {e.language for e in graph.entities.values()} + assert "java" in languages + assert "kotlin" in languages + assert any( + e.relation in _CROSS_LANG_RELATIONS + and graph.entities[e.source].language != graph.entities[e.target].language + for e in graph.edges + if e.source in graph.entities and e.target in graph.entities + ) + + +def test_pipeline_java_kotlin_mixed_ingest_parse_recover(fixtures_dir: Path): + root = fixtures_dir / "java_kotlin_mixed" + repo = ingest(str(root), languages=["java", "kotlin"]) + assert sorted(repo.languages) == ["java", "kotlin"] + assert any(p.suffix == ".java" for p in repo.source_files) + assert any(p.suffix == ".kt" for p in repo.source_files) + + graph = parse( + str(repo.path), + languages=repo.languages, + files=[str(f) for f in repo.source_files], + use_cache=False, + ) + _assert_java_kotlin_graph(graph) + assert "com.example.mixed.JavaBaseService" in graph.entities + assert "com.example.mixed.KotlinService" in graph.entities + + arch = recover(graph, algorithm="pkg") + assert len(arch.components) > 0 + assert any(c.entities for c in arch.components) + + +def test_pipeline_maven_java_kotlin_ingest_parse_recover(fixtures_dir: Path): + root = fixtures_dir / "maven_java_kotlin" + repo = ingest(str(root), languages=["java", "kotlin"]) + assert sorted(repo.languages) == ["java", "kotlin"] + + graph = parse( + str(repo.path), + languages=repo.languages, + files=[str(f) for f in repo.source_files], + use_cache=False, + ) + _assert_java_kotlin_graph(graph) + assert "com.example.JavaGreeter" in graph.entities + assert "com.example.KotlinGreeter" in graph.entities + assert ( + "com.example.KotlinGreeter", + "com.example.JavaGreeter", + "extends", + ) in set(graph.to_edge_tuples()) + + arch = recover(graph, algorithm="pkg") + assert len(arch.components) > 0 diff --git a/tests/test_tools/test_self_analysis_multilang.py b/tests/test_tools/test_self_analysis_multilang.py new file mode 100644 index 0000000..1b169eb --- /dev/null +++ b/tests/test_tools/test_self_analysis_multilang.py @@ -0,0 +1,48 @@ +"""Thin CLI E2E for arcade-self-analysis --languages java,kotlin.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_kotlin") +pytest.importorskip("tree_sitter_java") + +from scripts.run_self_analysis import main as run_self_analysis_main # noqa: E402 + + +def test_self_analysis_languages_java_kotlin_on_mixed_fixture( + fixtures_dir: Path, tmp_path, monkeypatch +): + root = fixtures_dir / "java_kotlin_mixed" + output_json = tmp_path / "results.json" + output_html = tmp_path / "report.html" + monkeypatch.setattr( + sys, + "argv", + [ + "run_self_analysis.py", + "--source", + str(root), + "--languages", + "java,kotlin", + "--algorithm", + "pkg", + "--output-json", + str(output_json), + "--output-html", + str(output_html), + ], + ) + + run_self_analysis_main() + + payload = json.loads(output_json.read_text()) + assert output_html.exists() + assert sorted(payload["languages"]) == ["java", "kotlin"] + assert payload["num_entities"] >= 2 + assert payload["num_components"] > 0 + assert payload["num_edges"] >= 1 From 09c1ceeb41e485f58178ec47ed1ebb73eabdc1e7 Mon Sep 17 00:00:00 2001 From: anonymous Date: Sun, 19 Jul 2026 23:47:46 -0500 Subject: [PATCH 3/5] fix: address Copilot multilang parse/ingest review Detect multi-language parse from path suffixes without requiring files on disk, and fail fast on unknown ingest language overrides. Co-authored-by: Cursor --- src/arcade_agent/tools/ingest.py | 15 ++++++++++-- src/arcade_agent/tools/parse.py | 7 +++--- tests/test_tools/test_ingest_multilang.py | 12 +++++++++ tests/test_tools/test_parse_multilang.py | 30 +++++++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/arcade_agent/tools/ingest.py b/src/arcade_agent/tools/ingest.py index 56bdb72..701f4f7 100644 --- a/src/arcade_agent/tools/ingest.py +++ b/src/arcade_agent/tools/ingest.py @@ -179,6 +179,17 @@ def _discover_files( return files +def _validate_known_languages(languages: list[str]) -> list[str]: + """Reject unknown language names before discovery expands to all extensions.""" + unknown = [lang for lang in languages if lang not in _LANG_EXTENSIONS] + if unknown: + available = ", ".join(sorted(_LANG_EXTENSIONS)) + raise ValueError( + f"Unknown language(s): {', '.join(unknown)}. Supported: {available}" + ) + return languages + + def _resolve_languages( path: Path, language: str | None, @@ -189,14 +200,14 @@ def _resolve_languages( if languages is not None: if not languages: raise ValueError("languages must be non-empty") - return list(languages) + return _validate_known_languages(list(languages)) if language == "multi": detected = _detect_languages(path) if not detected: raise ValueError(f"Could not detect languages in {path}") return detected if language: - return [language] + return _validate_known_languages([language]) primary = _detect_language(path) return [primary] if primary else [] diff --git a/src/arcade_agent/tools/parse.py b/src/arcade_agent/tools/parse.py index a3a3ff6..77e087f 100644 --- a/src/arcade_agent/tools/parse.py +++ b/src/arcade_agent/tools/parse.py @@ -52,13 +52,14 @@ def _resolve_languages( def detect_languages_from_files(files: list[Path]) -> list[str]: - """Return sorted language names present among *files*.""" + """Return sorted language names present among *files* (by suffix).""" found: set[str] = set() for path in files: - if not path.is_file(): + ext = path.suffix.lower() + if not ext: continue try: - parser = get_parser(path.suffix.lower()) + parser = get_parser(ext) except KeyError: continue found.add(parser.language) diff --git a/tests/test_tools/test_ingest_multilang.py b/tests/test_tools/test_ingest_multilang.py index 9f1f532..b3decf4 100644 --- a/tests/test_tools/test_ingest_multilang.py +++ b/tests/test_tools/test_ingest_multilang.py @@ -42,3 +42,15 @@ def test_ingest_rejects_language_and_languages_together(fixtures_dir: Path): root = fixtures_dir / "maven_java_kotlin" with pytest.raises(ValueError, match="language and languages"): ingest(str(root), language="java", languages=["kotlin"]) + + +def test_ingest_rejects_unknown_languages(fixtures_dir: Path): + root = fixtures_dir / "maven_java_kotlin" + with pytest.raises(ValueError, match="Unknown language"): + ingest(str(root), languages=["java", "cobol"]) + + +def test_ingest_rejects_unknown_single_language(fixtures_dir: Path): + root = fixtures_dir / "maven_java_kotlin" + with pytest.raises(ValueError, match="Unknown language"): + ingest(str(root), language="cobol") diff --git a/tests/test_tools/test_parse_multilang.py b/tests/test_tools/test_parse_multilang.py index 8f4b39b..eb5b838 100644 --- a/tests/test_tools/test_parse_multilang.py +++ b/tests/test_tools/test_parse_multilang.py @@ -59,3 +59,33 @@ def test_parse_rejects_language_and_languages_together(fixtures_dir: Path): root = fixtures_dir / "java_kotlin_mixed" with pytest.raises(ValueError, match="language and languages"): parse(str(root), language="java", languages=["kotlin"], use_cache=False) + + +def test_detect_languages_from_files_uses_suffix_without_existing_file(): + """Relative/missing paths still detect languages from suffixes.""" + from arcade_agent.tools.parse import detect_languages_from_files + + files = [ + Path("does/not/exist/Foo.java"), + Path("relative/Bar.kt"), + Path("no_suffix"), + Path("src/main/kotlin"), + ] + assert detect_languages_from_files(files) == ["java", "kotlin"] + + +def test_parse_multi_with_relative_file_list_detects_languages(fixtures_dir: Path): + """language='multi' + files=[...] must not require paths to exist on disk.""" + from arcade_agent.tools.parse import _resolve_languages + + root = fixtures_dir / "java_kotlin_mixed" + resolved = _resolve_languages( + root, + language="multi", + languages=None, + file_paths=[ + Path("src/main/java/com/example/mixed/JavaBaseService.java"), + Path("src/main/kotlin/com/example/mixed/KotlinService.kt"), + ], + ) + assert resolved == ["java", "kotlin"] From 43c0dd22d53100fd4c39c47979342401aacded70 Mon Sep 17 00:00:00 2001 From: anonymous Date: Mon, 20 Jul 2026 09:02:16 -0500 Subject: [PATCH 4/5] fix: complete polyglot MCP pipeline --- README.md | 18 ++++++- src/arcade_agent/parsers/multilang.py | 21 ++++---- src/arcade_agent/tools/adapters/mcp.py | 48 ++++++++++++++++-- src/arcade_agent/tools/parse.py | 12 ++--- tests/test_mcp_multilang_e2e.py | 23 +++++++-- tests/test_parsers/test_graph_merge.py | 69 ++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 15a44a9..37e2a0f 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ Provides composable tools for parsing source code, recovering architecture, dete ```bash pip install -e ".[dev]" -# With MCP server support (for AI agent integration) -pip install -e ".[mcp,dev]" +# With MCP and all optional language parsers (for polyglot AI agent integration) +pip install -e ".[mcp,languages,dev]" ``` ## Quick Start @@ -206,6 +206,20 @@ Agent: call get_full_result(session_id="a1b2c3", max_tokens=2000) → full graph data, truncated to fit token budget ``` +For a polyglot repository, let `ingest` preserve its file selection and pass +the returned session directly to `parse`: + +``` +Agent: call ingest(source="/path/to/project", languages=["java", "kotlin"]) + → {session_id: "p1q2r3", languages: ["java", "kotlin"], ...} + +Agent: call parse(source_path="p1q2r3") + → {session_id: "a1b2c3", num_entities: 420, num_edges: 960, ...} +``` + +Use `language="multi"` instead when every detected supported language should +be parsed automatically. + ## LLM-Powered Analysis Pass `--use-llm` to enable Claude-powered concern detection. Requires the `claude` CLI installed and authenticated. diff --git a/src/arcade_agent/parsers/multilang.py b/src/arcade_agent/parsers/multilang.py index 36d67e0..015cdb5 100644 --- a/src/arcade_agent/parsers/multilang.py +++ b/src/arcade_agent/parsers/multilang.py @@ -25,9 +25,6 @@ def resolve_name( if aliased in entities: return aliased - if "." in simple_name and simple_name in entities: - return simple_name - for imp in source_entity.imports: if imp.endswith(f".{simple_name}") and imp in entities: return imp @@ -37,6 +34,12 @@ def resolve_name( if same_pkg_fqn in entities: return same_pkg_fqn + # A qualified name is already explicit. Falling back to its leaf could link + # an unavailable external type (e.g. external.Base) to an unrelated local + # Base entity, creating a false cross-language dependency. + if "." in simple_name: + return None + leaf = simple_name.split(".")[-1] if leaf in fqn_index: return fqn_index[leaf] @@ -52,10 +55,11 @@ def _aliases_for(entity: Entity) -> dict[str, str]: def _build_fqn_index(entities: dict[str, Entity]) -> dict[str, str]: - index: dict[str, str] = {} + candidates: dict[str, list[str]] = {} for entity in entities.values(): - index[entity.name] = entity.fqn - return index + candidates.setdefault(entity.name, []).append(entity.fqn) + # Unqualified fallback is only safe when the leaf name is globally unique. + return {name: fqns[0] for name, fqns in candidates.items() if len(fqns) == 1} def relink_edges(graph: DependencyGraph) -> DependencyGraph: @@ -82,9 +86,8 @@ def add(source: str, target: str, relation: str) -> None: for imp in entity.imports: if imp in entities: add(entity.fqn, imp, "import") - else: - simple = imp.split(".")[-1] - resolved = fqn_index.get(simple) + elif "." not in imp: + resolved = fqn_index.get(imp) if resolved and resolved != entity.fqn: add(entity.fqn, resolved, "import") diff --git a/src/arcade_agent/tools/adapters/mcp.py b/src/arcade_agent/tools/adapters/mcp.py index 76d85ef..e8765f0 100644 --- a/src/arcade_agent/tools/adapters/mcp.py +++ b/src/arcade_agent/tools/adapters/mcp.py @@ -105,6 +105,41 @@ def _apply_budget(data: Any, max_tokens: int | None) -> Any: return enforce_budget(data, max_tokens) +def _resolve_parse_source( + source_path: str, + language: str | None, + languages: list[str] | None, + files: list[str] | None, +) -> tuple[str, str | None, list[str] | None, list[str] | None]: + """Resolve an ingest session into the concrete inputs required by parse. + + MCP clients should not need to discover a temporary clone path or repeat the + language/file selection already made by ``ingest``. Explicit parse arguments + still take precedence when a caller intentionally wants a narrower parse. + """ + if source_path not in _session: + return source_path, language, languages, files + + from arcade_agent.tools.ingest import IngestedRepo + + ingested = _session[source_path]["value"] + if not isinstance(ingested, IngestedRepo): + label = _session[source_path]["label"] + raise ValueError( + f"source_path session {source_path!r} contains {label}, not IngestedRepo" + ) + + if language is None and languages is None: + if ingested.languages: + languages = list(ingested.languages) + elif ingested.language: + language = ingested.language + if files is None: + files = [str(path) for path in ingested.source_files] + + return str(ingested.path), language, languages, files + + def _build_server(): # type: ignore[no-untyped-def] """Build and return the FastMCP server instance. @@ -127,8 +162,10 @@ def _build_server(): # type: ignore[no-untyped-def] "Use 'analyze' for a one-call end-to-end pipeline (offloaded from the event loop), " "or compose individual tools. " "Use session IDs from previous tool outputs as inputs to subsequent tools. " - "For example: call 'parse' to get a session_id, then pass that session_id " - "as dep_graph to 'recover'." + "For example: call 'ingest' and pass its session_id as source_path to " + "'parse', then pass the parse session_id as dep_graph to 'recover'. " + "For polyglot repositories, pass languages such as ['java', 'kotlin'] " + "to ingest; parse inherits that selection from the ingest session." ), ) @@ -190,7 +227,9 @@ def parse( session_id as the dep_graph argument to recover, detect_smells, etc. Args: - source_path: Root directory of the project. + source_path: Root directory of the project, or a session ID returned by + ingest. An ingest session carries its selected files and languages + into this parse call unless explicitly overridden. language: Language to parse (java, python, c, typescript, go, kotlin), or "multi" to parse every detected language and relink cross-language edges. @@ -202,6 +241,9 @@ def parse( """ from arcade_agent.tools.parse import parse as _parse + source_path, language, languages, files = _resolve_parse_source( + source_path, language, languages, files + ) graph = _parse( source_path=source_path, language=language, diff --git a/src/arcade_agent/tools/parse.py b/src/arcade_agent/tools/parse.py index 77e087f..c48ef07 100644 --- a/src/arcade_agent/tools/parse.py +++ b/src/arcade_agent/tools/parse.py @@ -36,19 +36,19 @@ def _resolve_languages( return list(languages) if language == "multi": discover = file_paths if file_paths is not None else list(root.rglob("*")) - detected = detect_languages_from_files(discover) - if not detected: + detected_languages = detect_languages_from_files(discover) + if not detected_languages: raise ValueError(f"Could not detect languages in {root}") - return detected + return detected_languages if language: return [language] discover = file_paths if file_paths is not None else [ f for f in root.rglob("*") if f.is_file() ] - detected = detect_language(discover) - if not detected: + detected_language = detect_language(discover) + if not detected_language: raise ValueError(f"Could not detect language in {root}") - return [detected] + return [detected_language] def detect_languages_from_files(files: list[Path]) -> list[str]: diff --git a/tests/test_mcp_multilang_e2e.py b/tests/test_mcp_multilang_e2e.py index a68e34a..d4487ff 100644 --- a/tests/test_mcp_multilang_e2e.py +++ b/tests/test_mcp_multilang_e2e.py @@ -95,7 +95,7 @@ def test_parse_language_multi_on_mixed_fixture(self, server): languages = {e["language"] for e in full["data"]["entities"].values()} assert languages == {"java", "kotlin"} - def test_ingest_languages_then_parse_maven_fixture(self, server): + def test_ingest_session_chains_languages_and_files_into_parse(self, server): ingest_result = _call( server, "ingest", @@ -109,8 +109,7 @@ def test_ingest_languages_then_parse_maven_fixture(self, server): server, "parse", { - "source_path": _MAVEN, - "languages": ["java", "kotlin"], + "source_path": ingest_result["session_id"], "use_cache": False, }, ) @@ -119,3 +118,21 @@ def test_ingest_languages_then_parse_maven_fixture(self, server): entity_fqns = set(full["data"]["entities"]) assert "com.example.JavaGreeter" in entity_fqns assert "com.example.KotlinGreeter" in entity_fqns + + def test_parse_rejects_non_ingest_source_session(self, server): + from mcp.server.fastmcp.exceptions import ToolError + + parsed = _call( + server, + "parse", + {"source_path": _MIXED, "language": "multi", "use_cache": False}, + ) + + async def _run(): + return await server.call_tool( + "parse", + {"source_path": parsed["session_id"], "use_cache": False}, + ) + + with pytest.raises(ToolError, match="not IngestedRepo"): + asyncio.run(_run()) diff --git a/tests/test_parsers/test_graph_merge.py b/tests/test_parsers/test_graph_merge.py index 5b92158..d9a071e 100644 --- a/tests/test_parsers/test_graph_merge.py +++ b/tests/test_parsers/test_graph_merge.py @@ -133,3 +133,72 @@ def test_merge_and_relink_keeps_first_on_cross_language_fqn_collision(): merged = merge_and_relink(left, right) assert merged.entities["a.Shared"].language == "java" assert merged.entities["a.Shared"].file_path == "Shared.java" + + +def test_relink_does_not_resolve_qualified_external_name_to_local_leaf(): + graph = DependencyGraph( + entities={ + "app.Consumer": Entity( + fqn="app.Consumer", + name="Consumer", + package="app", + file_path="Consumer.kt", + kind="class", + language="kotlin", + superclass="external.Base", + imports=["external.Contract"], + ), + "app.Base": Entity( + fqn="app.Base", + name="Base", + package="app", + file_path="Base.java", + kind="class", + language="java", + ), + "app.Contract": Entity( + fqn="app.Contract", + name="Contract", + package="app", + file_path="Contract.java", + kind="interface", + language="java", + ), + } + ) + + assert relink_edges(graph).edges == [] + + +def test_relink_does_not_guess_when_simple_name_is_ambiguous(): + graph = DependencyGraph( + entities={ + "consumer.Child": Entity( + fqn="consumer.Child", + name="Child", + package="consumer", + file_path="Child.kt", + kind="class", + language="kotlin", + superclass="Base", + ), + "one.Base": Entity( + fqn="one.Base", + name="Base", + package="one", + file_path="Base.java", + kind="class", + language="java", + ), + "two.Base": Entity( + fqn="two.Base", + name="Base", + package="two", + file_path="Base.kt", + kind="class", + language="kotlin", + ), + } + ) + + assert relink_edges(graph).edges == [] From 590caa2a0bdb8ab1d4601b50b5eff0b3d9246386 Mon Sep 17 00:00:00 2001 From: Duc Le Date: Wed, 22 Jul 2026 13:05:30 +0700 Subject: [PATCH 5/5] fix(multilang): scope cross-language merge and relink by language family The merge+relink introduced for roadmap #18 was language-blind: all three resolution paths (unique-leaf fallback in resolve_name, leaf import fallback and exact-FQN import match in relink_edges) could bind an entity to one written in an unrelated language. On a Python+Java fixture this fabricated `app.service.PaymentHandler -[extends]-> com.example.core.Base` and an import edge from a Python module to the Java class with the same FQN, which then propagated into pkg recovery and WCA clustering. Introduce language families (jvm = java+kotlin; every other language its own family) and gate every resolution path plus the merge step on family compatibility. The unique-leaf index is now built per family, so in-family resolution is unaffected. Cross-family FQN collisions no longer drop an entity silently: the later one is re-keyed as `#` with its edges and package listings remapped, and collision counts are reported in the new DependencyGraph.metadata (also surfaced in MCP summaries, since agents never see log lines). Single-language output is unchanged byte-for-byte (verified against origin/main across six fixture/language combinations; metadata is omitted from serialization when empty). Also from the review: - document the supported polyglot pairs (module docstring, parse/MCP docstrings, README table, ROADMAP) - drop the dead leaf-split in resolve_name - remove the scala entry from _LANG_PREFERRED_ROOTS (no Scala parser) - filter provided files by language in the single-language branch too, with a warning for skipped files - count only files (not directories) when resolving language="multi" - sort/deduplicate an explicit languages list so ordering cannot change collision winners Co-Authored-By: Claude Fable 5 --- README.md | 18 +- ROADMAP.md | 2 +- src/arcade_agent/parsers/graph.py | 11 +- src/arcade_agent/parsers/multilang.py | 253 ++++++++++++++-- src/arcade_agent/serialization.py | 18 +- src/arcade_agent/tools/adapters/mcp.py | 16 +- src/arcade_agent/tools/ingest.py | 3 +- src/arcade_agent/tools/parse.py | 44 ++- .../fixtures/python_java_mixed/app/service.py | 10 + .../python_java_mixed/com/auth/service.py | 5 + .../src/main/java/com/auth/service.java | 5 + .../src/main/java/com/example/core/Base.java | 5 + .../test_multilang_family_scoping.py | 272 ++++++++++++++++++ 13 files changed, 615 insertions(+), 47 deletions(-) create mode 100644 tests/fixtures/python_java_mixed/app/service.py create mode 100644 tests/fixtures/python_java_mixed/com/auth/service.py create mode 100644 tests/fixtures/python_java_mixed/src/main/java/com/auth/service.java create mode 100644 tests/fixtures/python_java_mixed/src/main/java/com/example/core/Base.java create mode 100644 tests/test_parsers/test_multilang_family_scoping.py diff --git a/README.md b/README.md index 37e2a0f..4a58382 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,22 @@ Agent: call parse(source_path="p1q2r3") Use `language="multi"` instead when every detected supported language should be parsed automatically. +**Supported polyglot pairs (MVP).** Every requested language is parsed, but +cross-language edges are only created *within a language family*: + +| Family | Languages | Cross-language relinking | +|--------|-----------|--------------------------| +| `jvm` | `java`, `kotlin` | Yes — validated pair (shared FQN space, extends/implements across languages) | +| every other language | `python`, `go`, `typescript`, `c`, ... | No — parsed and merged into one graph, but never linked to another family | + +The relink heuristics are dotted-name tuned (packages, unique leaf names, +`import a.b.C`) and only the JVM pair is covered by tests, so a Python +`com.auth.service` module and a Java `com.auth.service` class are never +confused for each other. When such an FQN coincidence happens across families +both entities are kept — the later one re-keyed as `#` — and the +counts appear in the graph's `metadata` (`fqn_collisions`, +`fqn_collisions_cross_family`, …), so nothing is silently dropped. + ## LLM-Powered Analysis Pass `--use-llm` to enable Claude-powered concern detection. Requires the `claude` CLI installed and authenticated. @@ -332,7 +348,7 @@ arcade-agent ports and extends the capabilities of the original [ARCADE](https:/ | 6 quality metrics | Done | RCI, TurboMQ, BasicMQ, IntraConnectivity, InterConnectivity, TwoWayPairRatio | | Balanced architecture score | Done | Derived reporting score combining core metrics, principle signals, and smell burden | | A2A architecture comparison | Done | Hungarian algorithm on Jaccard similarity | -| Multi-language parsing | Done | Java, Python, C/C++, TypeScript/JavaScript, Go (full); Kotlin (structural); polyglot merge+relink via `languages=[...]` / `language="multi"` | +| Multi-language parsing | Done | Java, Python, C/C++, TypeScript/JavaScript, Go (full); Kotlin (structural); polyglot merge+relink via `languages=[...]` / `language="multi"` (cross-language edges within the JVM family only) | | 5 export formats | Done | HTML, DOT, JSON, RSF, Mermaid | | LLM concern extraction | Done | Claude CLI for semantic BCO/SPF detection | | MCP server | Done | Expose tools to AI agents via Model Context Protocol with session store | diff --git a/ROADMAP.md b/ROADMAP.md index b79cedb..b826e30 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -45,7 +45,7 @@ Handle real-world polyglot monorepos. - [x] **16a2. Kotlin parser** — Shipped (`parsers/kotlin.py`) for JVM/Kotlin-first repos (e.g. embabel-agent). - [ ] **16b. Rust parser** — Still open. High-demand language for agent-assisted development. - [x] **17. Incremental parsing** — Content-hash extract cache shipped in #9 (`incremental.py`), wired for the Python parser only; extending to the other two-pass parsers is follow-up. -- [x] **18. Cross-language dependency tracking** — MVP: multi-language ingest/parse (`languages=[...]` / `language="multi"`) merges per-language graphs and relinks import/extends/implements across FQN space (Java↔Kotlin first; same mechanism for other same-FQN or import-linked pairs). Broader RPC/IDL bridges (gRPC stubs, OpenAPI) remain follow-up. +- [x] **18. Cross-language dependency tracking** — MVP: multi-language ingest/parse (`languages=[...]` / `language="multi"`) merges per-language graphs and relinks import/extends/implements across FQN space. Relinking is **family-scoped**: the `jvm` family (Java↔Kotlin) is the supported and validated pair; every other language is its own family and is merged without cross-language edges. Extending relinking to further families (and broader RPC/IDL bridges — gRPC stubs, OpenAPI) remains follow-up. ## Phase 6 — Agent Protocol Integration diff --git a/src/arcade_agent/parsers/graph.py b/src/arcade_agent/parsers/graph.py index 1d9c3a8..93c24e6 100644 --- a/src/arcade_agent/parsers/graph.py +++ b/src/arcade_agent/parsers/graph.py @@ -1,6 +1,7 @@ """Dependency graph data models.""" from dataclasses import dataclass, field +from typing import Any @dataclass @@ -35,6 +36,9 @@ class DependencyGraph: entities: dict[str, Entity] = field(default_factory=dict) edges: list[Edge] = field(default_factory=list) packages: dict[str, list[str]] = field(default_factory=dict) + # Parse-time notes for consumers (e.g. cross-language FQN collision counts + # from multilang.merge_and_relink). Empty for single-language parses. + metadata: dict[str, Any] = field(default_factory=dict) @property def num_entities(self) -> int: @@ -66,4 +70,9 @@ def merge(self, other: "DependencyGraph") -> "DependencyGraph": for pkg, fqns in other.packages.items(): packages.setdefault(pkg, []).extend(fqns) packages = {pkg: list(dict.fromkeys(fqns)) for pkg, fqns in packages.items()} - return DependencyGraph(entities=entities, edges=edges, packages=packages) + return DependencyGraph( + entities=entities, + edges=edges, + packages=packages, + metadata={**self.metadata, **other.metadata}, + ) diff --git a/src/arcade_agent/parsers/multilang.py b/src/arcade_agent/parsers/multilang.py index 015cdb5..87413ea 100644 --- a/src/arcade_agent/parsers/multilang.py +++ b/src/arcade_agent/parsers/multilang.py @@ -1,37 +1,108 @@ -"""Cross-language graph merge and edge relinking (roadmap #18).""" +"""Cross-language graph merge and edge relinking (roadmap #18). + +Scope — supported polyglot pairs +-------------------------------- +Merging and relinking only ever happens *within a language family*. A family is +a set of languages that genuinely share one fully-qualified-name space, so that +a name resolved in one of them may legitimately denote an entity written in +another: + +- ``jvm``: ``java`` + ``kotlin`` — the validated MVP pair. Both compile to the + same JVM namespace, use dotted package FQNs, and routinely extend/implement + each other's types. +- every other language is its own family (``python``, ``go``, ``typescript``, + ``c``, ...). Their graphs are still merged into one ``DependencyGraph``, but + no edge is ever fabricated between two different families, because a + ``com.auth.service`` Python module and a ``com.auth.service`` Java class are + unrelated things that merely spell alike. + +The resolution heuristics here are dotted-name tuned (packages, leaf names, +``import a.b.C``) and only the JVM pair has test coverage. ``language="multi"`` +on a repository containing, say, Go and TypeScript will therefore parse each +language correctly but will not attempt cross-language linking between them. +""" from __future__ import annotations import logging +from dataclasses import replace +from typing import Any from arcade_agent.parsers.graph import DependencyGraph, Edge, Entity logger = logging.getLogger(__name__) +# Languages that share a single FQN space. Anything not listed forms its own +# single-member family (see language_family). Only add a language here when its +# names really do resolve against the other members at compile/runtime. +_LANGUAGE_FAMILIES: dict[str, str] = { + "java": "jvm", + "kotlin": "jvm", +} + +#: Language pairs whose cross-language relinking is exercised by tests. +SUPPORTED_POLYGLOT_PAIRS: tuple[tuple[str, str], ...] = (("java", "kotlin"),) + + +def language_family(language: str | None) -> str: + """Return the FQN-space family of *language* (its own name when unknown). + + Args: + language: Entity language such as "java", "kotlin" or "python". + + Returns: + Family name; "jvm" for Java/Kotlin, the language itself otherwise. + """ + if not language: + return "unknown" + lang = language.lower() + return _LANGUAGE_FAMILIES.get(lang, lang) + def resolve_name( simple_name: str, source_entity: Entity, - fqn_index: dict[str, str], + fqn_index: dict[str, dict[str, str]], entities: dict[str, Entity], aliases: dict[str, str] | None = None, ) -> str | None: - """Resolve a simple or qualified type name to an entity FQN.""" - if simple_name in entities: + """Resolve a simple or qualified type name to an entity FQN. + + Resolution never crosses a language family boundary: a Python class named + ``Base`` can only bind to another Python entity, never to a Java one that + happens to share the leaf name. + + Args: + simple_name: Simple or dotted type name as written in the source. + source_entity: Entity that referenced *simple_name*. + fqn_index: Family-scoped unique-leaf index from ``_build_fqn_index``. + entities: All entities of the merged graph, keyed by FQN. + aliases: Optional import aliases declared by *source_entity*. + + Returns: + The resolved FQN, or None when no family-compatible entity matches. + """ + family = language_family(source_entity.language) + + def compatible(fqn: str) -> bool: + target = entities.get(fqn) + return target is not None and language_family(target.language) == family + + if compatible(simple_name): return simple_name if aliases and simple_name in aliases: aliased = aliases[simple_name] - if aliased in entities: + if compatible(aliased): return aliased for imp in source_entity.imports: - if imp.endswith(f".{simple_name}") and imp in entities: + if imp.endswith(f".{simple_name}") and compatible(imp): return imp if source_entity.package: same_pkg_fqn = f"{source_entity.package}.{simple_name}" - if same_pkg_fqn in entities: + if compatible(same_pkg_fqn): return same_pkg_fqn # A qualified name is already explicit. Falling back to its leaf could link @@ -40,11 +111,9 @@ def resolve_name( if "." in simple_name: return None - leaf = simple_name.split(".")[-1] - if leaf in fqn_index: - return fqn_index[leaf] - - return None + # Unqualified fallback: only same-family entities are candidates, and the + # leaf name must be unique inside that family. + return fqn_index.get(family, {}).get(simple_name) def _aliases_for(entity: Entity) -> dict[str, str]: @@ -54,20 +123,44 @@ def _aliases_for(entity: Entity) -> dict[str, str]: return {} -def _build_fqn_index(entities: dict[str, Entity]) -> dict[str, str]: - candidates: dict[str, list[str]] = {} +def _build_fqn_index(entities: dict[str, Entity]) -> dict[str, dict[str, str]]: + """Index leaf name -> FQN, scoped per language family. + + Uniqueness is evaluated inside a family: a Java ``Base`` and a Python + ``Base`` do not make each other ambiguous, and neither can resolve to the + other. + + Args: + entities: All entities of the merged graph, keyed by FQN. + + Returns: + Mapping family -> {leaf name: FQN} for leaves unique in that family. + """ + candidates: dict[str, dict[str, list[str]]] = {} for entity in entities.values(): - candidates.setdefault(entity.name, []).append(entity.fqn) - # Unqualified fallback is only safe when the leaf name is globally unique. - return {name: fqns[0] for name, fqns in candidates.items() if len(fqns) == 1} + family = language_family(entity.language) + candidates.setdefault(family, {}).setdefault(entity.name, []).append(entity.fqn) + # Unqualified fallback is only safe when the leaf name is unique in-family. + return { + family: {name: fqns[0] for name, fqns in names.items() if len(fqns) == 1} + for family, names in candidates.items() + } def relink_edges(graph: DependencyGraph) -> DependencyGraph: """Add import/extends/implements edges resolvable against the full entity set. Language parsers only resolve against their own entities. After merging - Java+Kotlin (or other polyglot) graphs, re-run resolution so same-package - and imported cross-language types become real edges. + Java+Kotlin (the supported polyglot pair) re-run resolution so same-package + and imported cross-language types become real edges. Every resolution path + is gated on language-family compatibility, so no edge is ever created + between entities of unrelated languages. + + Args: + graph: Merged graph whose entities may come from several languages. + + Returns: + New DependencyGraph with the additional resolvable edges. """ entities = graph.entities fqn_index = _build_fqn_index(entities) @@ -82,12 +175,18 @@ def add(source: str, target: str, relation: str) -> None: new_edges.append(Edge(source=source, target=target, relation=relation)) for entity in entities.values(): + family = language_family(entity.language) aliases = _aliases_for(entity) for imp in entity.imports: - if imp in entities: - add(entity.fqn, imp, "import") + imported = entities.get(imp) + if imported is not None: + # An FQN that merely coincides across families (a Python module + # com.auth.service vs a Java class of the same name) is not an + # import edge. + if language_family(imported.language) == family: + add(entity.fqn, imp, "import") elif "." not in imp: - resolved = fqn_index.get(imp) + resolved = fqn_index.get(family, {}).get(imp) if resolved and resolved != entity.fqn: add(entity.fqn, resolved, "import") @@ -106,37 +205,129 @@ def add(source: str, target: str, relation: str) -> None: packages: dict[str, list[str]] = { pkg: list(dict.fromkeys(fqns)) for pkg, fqns in graph.packages.items() } - return DependencyGraph(entities=entities, edges=new_edges, packages=packages) + return DependencyGraph( + entities=entities, + edges=new_edges, + packages=packages, + metadata=dict(graph.metadata), + ) + + +def _disambiguate_fqn(fqn: str, language: str, taken: dict[str, Entity]) -> str: + """Return an unused graph key for a cross-family collision on *fqn*.""" + candidate = f"{fqn}#{language}" + suffix = 2 + while candidate in taken: + candidate = f"{fqn}#{language}{suffix}" + suffix += 1 + return candidate def merge_and_relink(*graphs: DependencyGraph) -> DependencyGraph: - """Union graphs then relink edges across the combined entity set.""" + """Union graphs then relink edges across the combined entity set. + + Entities are merged inside language families only. When two *different* + families produce the same FQN (e.g. Python ``com/auth/service.py::login`` + and Java ``com.auth.service.login``) neither entity is dropped: the later + one is re-keyed as ``#`` and its own edges and package + listings are remapped, so a cross-family name coincidence no longer causes + silent data loss. Within a family the first entity still wins. + + Collision counts land in ``graph.metadata`` (``fqn_collisions``, + ``fqn_collisions_same_family``, ``fqn_collisions_cross_family`` and + ``fqn_collision_details``) so agents see them without reading log output. + + Args: + *graphs: Per-language graphs to union, in priority order. + + Returns: + Merged DependencyGraph with relinked edges and collision metadata. + """ if not graphs: return DependencyGraph() entities: dict[str, Entity] = {} edges: list[Edge] = [] packages: dict[str, list[str]] = {} + collision_details: list[dict[str, str]] = [] + same_family_collisions = 0 + cross_family_collisions = 0 for graph in graphs: + renamed: dict[str, str] = {} for fqn, entity in graph.entities.items(): - if fqn in entities: - existing = entities[fqn] + existing = entities.get(fqn) + if existing is None: + entities[fqn] = entity + continue + if language_family(existing.language) == language_family(entity.language): if existing.language != entity.language: + same_family_collisions += 1 + collision_details.append( + { + "fqn": fqn, + "kept": existing.language, + "other": entity.language, + "resolution": "kept_first", + } + ) logger.warning( - "FQN collision across languages at %s (%s vs %s); " + "FQN collision within language family at %s (%s vs %s); " "keeping first", fqn, existing.language, entity.language, ) continue - entities[fqn] = entity - edges.extend(graph.edges) + new_fqn = _disambiguate_fqn(fqn, entity.language, entities) + renamed[fqn] = new_fqn + entities[new_fqn] = replace(entity, fqn=new_fqn) + cross_family_collisions += 1 + collision_details.append( + { + "fqn": fqn, + "kept": existing.language, + "other": entity.language, + "resolution": "renamed", + "renamed_to": new_fqn, + } + ) + logger.warning( + "FQN collision across language families at %s (%s vs %s); " + "keeping both, re-keyed the %s entity as %s", + fqn, + existing.language, + entity.language, + entity.language, + new_fqn, + ) + if renamed: + edges.extend( + Edge( + source=renamed.get(edge.source, edge.source), + target=renamed.get(edge.target, edge.target), + relation=edge.relation, + ) + for edge in graph.edges + ) + else: + edges.extend(graph.edges) for pkg, fqns in graph.packages.items(): - packages.setdefault(pkg, []).extend(fqns) + packages.setdefault(pkg, []).extend(renamed.get(f, f) for f in fqns) packages = {pkg: list(dict.fromkeys(fqns)) for pkg, fqns in packages.items()} + metadata: dict[str, Any] = { + "fqn_collisions": same_family_collisions + cross_family_collisions, + "fqn_collisions_same_family": same_family_collisions, + "fqn_collisions_cross_family": cross_family_collisions, + } + if collision_details: + metadata["fqn_collision_details"] = collision_details return relink_edges( - DependencyGraph(entities=entities, edges=edges, packages=packages) + DependencyGraph( + entities=entities, + edges=edges, + packages=packages, + metadata=metadata, + ) ) diff --git a/src/arcade_agent/serialization.py b/src/arcade_agent/serialization.py index 9e1ea49..01b42ed 100644 --- a/src/arcade_agent/serialization.py +++ b/src/arcade_agent/serialization.py @@ -98,8 +98,12 @@ def load_graph(path: Path) -> DependencyGraph: def graph_to_dict(graph: DependencyGraph) -> dict: - """Convert a DependencyGraph to a JSON-serializable dict.""" - return { + """Convert a DependencyGraph to a JSON-serializable dict. + + ``metadata`` is emitted only when non-empty so single-language parse output + stays byte-identical to previous releases. + """ + data = { "entities": { fqn: { "fqn": e.fqn, @@ -121,6 +125,9 @@ def graph_to_dict(graph: DependencyGraph) -> dict: ], "packages": graph.packages, } + if graph.metadata: + data["metadata"] = graph.metadata + return data def dict_to_graph(data: dict) -> DependencyGraph: @@ -145,7 +152,12 @@ def dict_to_graph(data: dict) -> DependencyGraph: for e in data.get("edges", []) ] packages: dict[str, list[str]] = data.get("packages", {}) - return DependencyGraph(entities=entities, edges=edges, packages=packages) + return DependencyGraph( + entities=entities, + edges=edges, + packages=packages, + metadata=data.get("metadata", {}), + ) def architecture_to_dict(arch: Architecture) -> dict: diff --git a/src/arcade_agent/tools/adapters/mcp.py b/src/arcade_agent/tools/adapters/mcp.py index e8765f0..2f65b0f 100644 --- a/src/arcade_agent/tools/adapters/mcp.py +++ b/src/arcade_agent/tools/adapters/mcp.py @@ -79,6 +79,11 @@ def _make_summary(obj: Any, label: str) -> dict: langs = getattr(obj, "languages") if langs: summary["languages"] = list(langs) + metadata = getattr(obj, "metadata", None) if hasattr(obj, "num_entities") else None + if isinstance(metadata, dict) and metadata: + # Agents never see log lines; surface parse notes such as polyglot + # FQN-collision counts directly in the summary. + summary["metadata"] = metadata if hasattr(obj, "name") and isinstance(getattr(obj, "name", None), str): summary["name"] = obj.name if hasattr(obj, "version"): @@ -165,7 +170,10 @@ def _build_server(): # type: ignore[no-untyped-def] "For example: call 'ingest' and pass its session_id as source_path to " "'parse', then pass the parse session_id as dep_graph to 'recover'. " "For polyglot repositories, pass languages such as ['java', 'kotlin'] " - "to ingest; parse inherits that selection from the ingest session." + "to ingest; parse inherits that selection from the ingest session. " + "Cross-language edges are only linked within a language family " + "(java+kotlin today); other language pairs are parsed and merged " + "but never linked to each other." ), ) @@ -226,6 +234,12 @@ def parse( Returns a session_id referencing the parsed DependencyGraph. Pass this session_id as the dep_graph argument to recover, detect_smells, etc. + Cross-language relinking is family-scoped: java+kotlin is the supported + (and validated) pair. Any other combination is parsed and merged into + one graph without edges between the two languages. FQN collisions + across families keep both entities (the later one re-keyed as + "#") and are counted in the summary's metadata. + Args: source_path: Root directory of the project, or a session ID returned by ingest. An ingest session carries its selected files and languages diff --git a/src/arcade_agent/tools/ingest.py b/src/arcade_agent/tools/ingest.py index 701f4f7..0978ca0 100644 --- a/src/arcade_agent/tools/ingest.py +++ b/src/arcade_agent/tools/ingest.py @@ -49,10 +49,11 @@ def cleanup(self) -> None: for ext in exts: _EXT_TO_LANG[ext] = lang +# Per-language source roots. Only languages with a parser belong here; generic +# roots (including src/main/scala) are still probed via _SOURCE_ROOTS. _LANG_PREFERRED_ROOTS: dict[str, str] = { "java": "src/main/java", "kotlin": "src/main/kotlin", - "scala": "src/main/scala", } # Well-known source root directories (tried in order) diff --git a/src/arcade_agent/tools/parse.py b/src/arcade_agent/tools/parse.py index c48ef07..fe0d1f4 100644 --- a/src/arcade_agent/tools/parse.py +++ b/src/arcade_agent/tools/parse.py @@ -33,9 +33,13 @@ def _resolve_languages( if languages is not None: if not languages: raise ValueError("languages must be non-empty") - return list(languages) + # Sorted + de-duplicated so ["kotlin", "java"] and ["java", "kotlin"] + # produce the same graph (merge order decides collision winners). + return sorted(dict.fromkeys(languages)) if language == "multi": - discover = file_paths if file_paths is not None else list(root.rglob("*")) + discover = file_paths if file_paths is not None else [ + f for f in root.rglob("*") if f.is_file() + ] detected_languages = detect_languages_from_files(discover) if not detected_languages: raise ValueError(f"Could not detect languages in {root}") @@ -112,17 +116,28 @@ def parse( ) -> DependencyGraph: """Parse source code and extract a dependency graph. + Polyglot support (MVP): every requested language is parsed, but + cross-language edges are only linked *within a language family* — currently + just the JVM family (``java`` + ``kotlin``), the one validated pair. Other + languages each form their own family, so a Python and a Java graph are + unioned without inventing edges between them (see + ``arcade_agent.parsers.multilang``). Cross-family FQN collisions are kept + (re-keyed as ``#``) and counted in ``graph.metadata``. + Args: source_path: Root directory of the project. language: Language to parse (java, python, etc.), or "multi" to parse every detected language and merge+relink cross-language edges. languages: Explicit language list for polyglot parse - (e.g. ["java", "kotlin"]). Mutually exclusive with *language*. - files: Specific files to parse. If None, discovers all files. + (e.g. ["java", "kotlin"]). Sorted internally, so ordering does not + change the result. Mutually exclusive with *language*. + files: Specific files to parse. If None, discovers all files. Files not + matching any resolved language are skipped (with a warning). use_cache: If True, return cached results when source files haven't changed. Returns: - DependencyGraph with entities, edges, and package info. + DependencyGraph with entities, edges, package info and, for polyglot + parses, FQN-collision counts in ``metadata``. """ root = Path(source_path) provided_files = [Path(f) for f in files] if files else None @@ -143,12 +158,25 @@ def parse( else: file_paths = _discover_files(root, resolved) + per_language = {lang: _files_for_language(file_paths, lang) for lang in resolved} + selected = {f for files_ in per_language.values() for f in files_} + dropped = [f for f in file_paths if f not in selected] + if dropped: + # Same filtering rule for one or many languages: a file whose extension + # no resolved parser claims is not parseable and is reported, not + # silently handed to an arbitrary parser. + logger.warning( + "Skipping %d file(s) not matching languages %s (e.g. %s)", + len(dropped), + ",".join(resolved), + dropped[0], + ) + if len(resolved) == 1: - graph = _parse_one(resolved[0], file_paths, root, use_cache) + graph = _parse_one(resolved[0], per_language[resolved[0]], root, use_cache) else: graphs = [ - _parse_one(lang, _files_for_language(file_paths, lang), root, use_cache) - for lang in resolved + _parse_one(lang, per_language[lang], root, use_cache) for lang in resolved ] graphs = [g for g in graphs if g.num_entities or g.num_edges] graph = merge_and_relink(*graphs) if graphs else DependencyGraph() diff --git a/tests/fixtures/python_java_mixed/app/service.py b/tests/fixtures/python_java_mixed/app/service.py new file mode 100644 index 0000000..74d0700 --- /dev/null +++ b/tests/fixtures/python_java_mixed/app/service.py @@ -0,0 +1,10 @@ +"""Python service that must not be linked to the Java entities.""" + +import com.auth.service + + +class PaymentHandler(Base): # noqa: F821 — unresolvable on purpose (fixture) + """Extends a Base that does not exist in Python land.""" + + def login(self): + return com.auth.service diff --git a/tests/fixtures/python_java_mixed/com/auth/service.py b/tests/fixtures/python_java_mixed/com/auth/service.py new file mode 100644 index 0000000..36666aa --- /dev/null +++ b/tests/fixtures/python_java_mixed/com/auth/service.py @@ -0,0 +1,5 @@ +"""Python module whose FQN coincides with the Java com.auth.service class.""" + + +def login(user): + return user diff --git a/tests/fixtures/python_java_mixed/src/main/java/com/auth/service.java b/tests/fixtures/python_java_mixed/src/main/java/com/auth/service.java new file mode 100644 index 0000000..df2d43b --- /dev/null +++ b/tests/fixtures/python_java_mixed/src/main/java/com/auth/service.java @@ -0,0 +1,5 @@ +package com.auth; + +public class service { + public void login() {} +} diff --git a/tests/fixtures/python_java_mixed/src/main/java/com/example/core/Base.java b/tests/fixtures/python_java_mixed/src/main/java/com/example/core/Base.java new file mode 100644 index 0000000..1601401 --- /dev/null +++ b/tests/fixtures/python_java_mixed/src/main/java/com/example/core/Base.java @@ -0,0 +1,5 @@ +package com.example.core; + +public class Base { + public void handle() {} +} diff --git a/tests/test_parsers/test_multilang_family_scoping.py b/tests/test_parsers/test_multilang_family_scoping.py new file mode 100644 index 0000000..4cfc890 --- /dev/null +++ b/tests/test_parsers/test_multilang_family_scoping.py @@ -0,0 +1,272 @@ +"""Language-family scoping of merge+relink (review findings 1 and 2). + +Before family scoping, three resolution paths in ``multilang`` were +language-blind and fabricated cross-language edges on a Python+Java fixture: + +- a Python ``class PaymentHandler(Base)`` with no local ``Base`` linked to the + globally unique Java ``com.example.core.Base`` via the leaf fallback; +- a Python ``import com.auth.service`` (its own module) linked to the Java class + with the coinciding FQN; +- the same coincidence at method level silently dropped one entity on merge. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from arcade_agent.parsers.graph import DependencyGraph, Edge, Entity +from arcade_agent.parsers.multilang import ( + language_family, + merge_and_relink, + relink_edges, +) +from arcade_agent.serialization import graph_to_dict +from arcade_agent.tools.parse import parse + + +def _entity(fqn: str, language: str, **kwargs) -> Entity: + name = fqn.split(".")[-1] + package = fqn.rsplit(".", 1)[0] if "." in fqn else "" + return Entity( + fqn=fqn, + name=name, + package=package, + file_path=f"{fqn}.src", + kind=kwargs.pop("kind", "class"), + language=language, + **kwargs, + ) + + +def _cross_family_edges(graph: DependencyGraph) -> list[tuple[str, str, str]]: + bad = [] + for edge in graph.edges: + source = graph.entities.get(edge.source) + target = graph.entities.get(edge.target) + if source is None or target is None: + continue + if language_family(source.language) != language_family(target.language): + bad.append((edge.source, edge.target, edge.relation)) + return bad + + +def test_language_family_groups_jvm_only(): + assert language_family("java") == language_family("kotlin") == "jvm" + assert language_family("python") == "python" + assert language_family("go") != language_family("typescript") + assert language_family(None) == "unknown" + + +def test_unique_leaf_fallback_does_not_cross_family(): + """Python subclass of an unresolved Base must not link to a Java Base.""" + graph = DependencyGraph( + entities={ + "app.service.PaymentHandler": _entity( + "app.service.PaymentHandler", "python", superclass="Base" + ), + "com.example.core.Base": _entity("com.example.core.Base", "java"), + } + ) + + linked = relink_edges(graph) + assert linked.to_edge_tuples() == [] + assert _cross_family_edges(linked) == [] + + +def test_unique_leaf_fallback_still_works_inside_a_family(): + """The same fallback remains available for Kotlin -> Java (the MVP pair).""" + graph = DependencyGraph( + entities={ + "app.KotlinChild": _entity("app.KotlinChild", "kotlin", superclass="Base"), + "com.example.core.Base": _entity("com.example.core.Base", "java"), + } + ) + + assert relink_edges(graph).to_edge_tuples() == [ + ("app.KotlinChild", "com.example.core.Base", "extends") + ] + + +def test_leaf_import_fallback_does_not_cross_family(): + graph = DependencyGraph( + entities={ + "app.service.Handler": _entity( + "app.service.Handler", "python", imports=["Base"] + ), + "com.example.core.Base": _entity("com.example.core.Base", "java"), + } + ) + + assert relink_edges(graph).to_edge_tuples() == [] + + +def test_exact_fqn_import_match_does_not_cross_family(): + """Python `import com.auth.service` must not bind to a Java class.""" + graph = DependencyGraph( + entities={ + "app.service.PaymentHandler": _entity( + "app.service.PaymentHandler", "python", imports=["com.auth.service"] + ), + "com.auth.service": _entity("com.auth.service", "java"), + } + ) + + assert relink_edges(graph).to_edge_tuples() == [] + + +def test_exact_fqn_import_match_still_links_inside_a_family(): + graph = DependencyGraph( + entities={ + "app.KotlinCaller": _entity( + "app.KotlinCaller", "kotlin", imports=["com.auth.Service"] + ), + "com.auth.Service": _entity("com.auth.Service", "java"), + } + ) + + assert relink_edges(graph).to_edge_tuples() == [ + ("app.KotlinCaller", "com.auth.Service", "import") + ] + + +def test_same_package_fallback_does_not_cross_family(): + graph = DependencyGraph( + entities={ + "com.auth.PyHandler": _entity( + "com.auth.PyHandler", "python", superclass="Service" + ), + "com.auth.Service": _entity("com.auth.Service", "java"), + } + ) + + assert relink_edges(graph).to_edge_tuples() == [] + + +def test_cross_family_fqn_collision_keeps_both_entities(): + java = DependencyGraph( + entities={"com.auth.service.login": _entity("com.auth.service.login", "java")}, + packages={"com.auth.service": ["com.auth.service.login"]}, + ) + python = DependencyGraph( + entities={ + "com.auth.service.login": _entity("com.auth.service.login", "python") + }, + edges=[ + Edge( + source="com.auth.service.login", + target="com.auth.service.login", + relation="calls", + ) + ], + packages={"com.auth.service": ["com.auth.service.login"]}, + ) + + merged = merge_and_relink(java, python) + + assert merged.entities["com.auth.service.login"].language == "java" + renamed = merged.entities["com.auth.service.login#python"] + assert renamed.language == "python" + assert renamed.fqn == "com.auth.service.login#python" + assert merged.metadata["fqn_collisions"] == 1 + assert merged.metadata["fqn_collisions_cross_family"] == 1 + assert merged.metadata["fqn_collisions_same_family"] == 0 + assert merged.metadata["fqn_collision_details"] == [ + { + "fqn": "com.auth.service.login", + "kept": "java", + "other": "python", + "resolution": "renamed", + "renamed_to": "com.auth.service.login#python", + } + ] + # The Python graph's own edges follow the renamed entity. + assert ( + "com.auth.service.login#python", + "com.auth.service.login#python", + "calls", + ) in merged.to_edge_tuples() + assert sorted(merged.packages["com.auth.service"]) == [ + "com.auth.service.login", + "com.auth.service.login#python", + ] + + +def test_same_family_fqn_collision_keeps_first_and_is_counted(): + java = DependencyGraph( + entities={"a.Shared": _entity("a.Shared", "java")}, + packages={"a": ["a.Shared"]}, + ) + kotlin = DependencyGraph( + entities={"a.Shared": _entity("a.Shared", "kotlin")}, + packages={"a": ["a.Shared"]}, + ) + + merged = merge_and_relink(java, kotlin) + + assert merged.entities["a.Shared"].language == "java" + assert merged.metadata["fqn_collisions"] == 1 + assert merged.metadata["fqn_collisions_same_family"] == 1 + assert merged.metadata["fqn_collisions_cross_family"] == 0 + + +def test_merge_reports_zero_collisions_when_there_are_none(): + merged = merge_and_relink( + DependencyGraph(entities={"a.A": _entity("a.A", "java")}), + DependencyGraph(entities={"b.B": _entity("b.B", "kotlin")}), + ) + assert merged.metadata["fqn_collisions"] == 0 + assert "fqn_collision_details" not in merged.metadata + + +def test_polyglot_fixture_has_no_fabricated_cross_language_edges(fixtures_dir: Path): + """End-to-end reproduction of the review's Python+Java fabrication.""" + root = fixtures_dir / "python_java_mixed" + graph = parse(str(root), languages=["java", "python"], use_cache=False) + + assert _cross_family_edges(graph) == [] + tuples = set(graph.to_edge_tuples()) + assert ( + "app.service.PaymentHandler", + "com.example.core.Base", + "extends", + ) not in tuples + assert ("app.service.PaymentHandler", "com.auth.service", "import") not in tuples + # No entity is silently dropped by the com.auth.service.login coincidence. + assert "com.auth.service.login" in graph.entities + assert "com.auth.service.login#python" in graph.entities + assert graph.metadata["fqn_collisions_cross_family"] == 1 + + +def test_language_order_does_not_change_the_result(fixtures_dir: Path): + root = fixtures_dir / "python_java_mixed" + a = parse(str(root), languages=["java", "python"], use_cache=False) + b = parse(str(root), languages=["python", "java"], use_cache=False) + assert graph_to_dict(a) == graph_to_dict(b) + + +def test_single_language_parse_carries_no_metadata(fixtures_dir: Path): + """Single-language output must stay byte-identical: no metadata key.""" + root = fixtures_dir / "python_java_mixed" + graph = parse(str(root), language="python", use_cache=False) + assert graph.metadata == {} + assert "metadata" not in graph_to_dict(graph) + + +def test_single_language_parse_matches_direct_parser_output(fixtures_dir: Path): + from arcade_agent.parsers.java import JavaParser + + root = fixtures_dir / "python_java_mixed" + files = sorted(root.rglob("*.java")) + direct = JavaParser().parse(files, root) + viaparse = parse(str(root), language="java", use_cache=False) + assert graph_to_dict(viaparse) == graph_to_dict(direct) + + +@pytest.mark.parametrize("languages", [["java", "java"], ["java"]]) +def test_duplicate_language_list_is_deduplicated(fixtures_dir: Path, languages): + root = fixtures_dir / "python_java_mixed" + graph = parse(str(root), languages=languages, use_cache=False) + assert graph.metadata == {} + assert all(e.language == "java" for e in graph.entities.values())