From 4a4003b8dea5f992e49eaf8e89aaa5bce480d1fa Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 7 Apr 2026 11:24:32 +0800 Subject: [PATCH 1/5] Update test directory paths and requirements syncing --- eng/scripts/sync_from_typespec.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/scripts/sync_from_typespec.py b/eng/scripts/sync_from_typespec.py index 2beb28a5ac..42ca708a7f 100644 --- a/eng/scripts/sync_from_typespec.py +++ b/eng/scripts/sync_from_typespec.py @@ -38,8 +38,8 @@ TYPESPEC_COMMON_TS = Path("packages/http-client-python/eng/scripts/ci/regenerate-common.ts") AUTOREST_COMMON_TS = Path("packages/typespec-python/scripts/eng/regenerate-common.ts") -TYPESPEC_TEST_DIR = Path("packages/http-client-python/generator/test") -AUTOREST_TEST_DIR = Path("packages/typespec-python/test") +TYPESPEC_TEST_DIR = Path("packages/http-client-python/tests") +AUTOREST_TEST_DIR = Path("packages/typespec-python/tests") # --- Marker patterns for requirements.txt sync --- @@ -51,16 +51,16 @@ _SKIP_DIRS: Set[str] = {"__pycache__", "generated", ".venv", "node_modules", ".tox"} _TEST_SUBDIRS = [ - "generic_mock_api_tests", - os.path.join("azure", "mock_api_tests"), - os.path.join("unbranded", "mock_api_tests"), + os.path.join("mock_api", "shared"), + os.path.join("mock_api", "azure"), + os.path.join("mock_api", "unbranded"), ] # Files that remain repo-specific (different relative paths between repo layouts) _SKIP_FILES: Set[str] = { - os.path.join("generic_mock_api_tests", "conftest.py"), - os.path.join("azure", "mock_api_tests", "conftest.py"), - os.path.join("unbranded", "mock_api_tests", "conftest.py"), + os.path.join("mock_api", "shared", "conftest.py"), + os.path.join("mock_api", "azure", "conftest.py"), + os.path.join("mock_api", "unbranded", "conftest.py"), } _SKIP_EXTENSIONS: Set[str] = {".pyc"} @@ -220,14 +220,14 @@ def main() -> int: shutil.copy2(src_ts, dst_ts) print(f"Synced regenerate-common.ts") - # 2. Sync requirements.txt marker sections + # 2. Sync requirements marker sections for flavor in ("azure", "unbranded"): - src_req = typespec_repo / TYPESPEC_TEST_DIR / flavor / "requirements.txt" - dst_req = autorest_repo / AUTOREST_TEST_DIR / flavor / "requirements.txt" + src_req = typespec_repo / TYPESPEC_TEST_DIR / "requirements" / f"{flavor}.txt" + dst_req = autorest_repo / AUTOREST_TEST_DIR / "requirements" / f"{flavor}.txt" if src_req.is_file() and dst_req.is_file(): sync_requirements(src_req, dst_req) else: - print(f" WARNING: requirements.txt not found for {flavor}, skipping") + print(f" WARNING: requirements/{flavor}.txt not found, skipping") # 3. Sync test files print("Syncing test files...") From f58a25ca874d9a4830a96361fb246be9923ea47f Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 7 Apr 2026 11:48:53 +0800 Subject: [PATCH 2/5] Update sync_from_typespec.py --- eng/scripts/sync_from_typespec.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/eng/scripts/sync_from_typespec.py b/eng/scripts/sync_from_typespec.py index 42ca708a7f..c42b72436b 100644 --- a/eng/scripts/sync_from_typespec.py +++ b/eng/scripts/sync_from_typespec.py @@ -9,16 +9,8 @@ The typespec repo is the source of truth for: 1. regenerate-common.ts — shared regeneration logic - 2. requirements.txt — common test dependencies (delimited by marker comments) - 3. Test files — mock API tests and test data - -Marker convention in requirements.txt: - # === common azure dependencies across repos === - ... - # === end common azure dependencies across repos === - # === common test dependencies across repos === - ... - # === end common test dependencies across repos === + 2. requirements — test dependency files (azure.txt, unbranded.txt under tests/requirements/) + 3. Test files — mock API tests under tests/mock_api/{shared,azure,unbranded} Usage: python sync_from_typespec.py @@ -36,12 +28,12 @@ # --- Path configuration (relative to each repo root) --- TYPESPEC_COMMON_TS = Path("packages/http-client-python/eng/scripts/ci/regenerate-common.ts") -AUTOREST_COMMON_TS = Path("packages/typespec-python/scripts/eng/regenerate-common.ts") +AUTOREST_COMMON_TS = Path("packages/typespec-python/eng/scripts/regenerate-common.ts") TYPESPEC_TEST_DIR = Path("packages/http-client-python/tests") AUTOREST_TEST_DIR = Path("packages/typespec-python/tests") -# --- Marker patterns for requirements.txt sync --- +# --- Marker patterns for requirements sync (legacy, kept for forward compatibility) --- _MARKER_PATTERN = re.compile(r"^# === (common .+ across repos) ===$") _END_MARKER_PATTERN = re.compile(r"^# === end (common .+ across repos) ===$") @@ -56,7 +48,7 @@ os.path.join("mock_api", "unbranded"), ] -# Files that remain repo-specific (different relative paths between repo layouts) +# Files that remain repo-specific (e.g. conftest.py differs between repos) _SKIP_FILES: Set[str] = { os.path.join("mock_api", "shared", "conftest.py"), os.path.join("mock_api", "azure", "conftest.py"), @@ -68,7 +60,7 @@ # --------------------------------------------------------------------------- -# Requirements.txt marker-based sync +# Requirements marker-based sync (syncs marker-delimited sections if present) # --------------------------------------------------------------------------- @@ -220,7 +212,7 @@ def main() -> int: shutil.copy2(src_ts, dst_ts) print(f"Synced regenerate-common.ts") - # 2. Sync requirements marker sections + # 2. Sync requirements (marker sections if present) for flavor in ("azure", "unbranded"): src_req = typespec_repo / TYPESPEC_TEST_DIR / "requirements" / f"{flavor}.txt" dst_req = autorest_repo / AUTOREST_TEST_DIR / "requirements" / f"{flavor}.txt" From 9b056fab7c1d478d81ecbd46be3390a3269786a2 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 7 Apr 2026 13:28:09 +0800 Subject: [PATCH 3/5] Update sync_from_typespec.py --- eng/scripts/sync_from_typespec.py | 63 +++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/eng/scripts/sync_from_typespec.py b/eng/scripts/sync_from_typespec.py index c42b72436b..679ad7d1d1 100644 --- a/eng/scripts/sync_from_typespec.py +++ b/eng/scripts/sync_from_typespec.py @@ -33,12 +33,23 @@ TYPESPEC_TEST_DIR = Path("packages/http-client-python/tests") AUTOREST_TEST_DIR = Path("packages/typespec-python/tests") -# --- Marker patterns for requirements sync (legacy, kept for forward compatibility) --- +# --- Marker patterns for requirements sync --- +# +# Convention in requirements files (e.g. azure.txt, unbranded.txt): +# # === common azure dependencies across repos === +# azure-core>=1.37.0 +# ... +# # === end common azure dependencies across repos === _MARKER_PATTERN = re.compile(r"^# === (common .+ across repos) ===$") _END_MARKER_PATTERN = re.compile(r"^# === end (common .+ across repos) ===$") -# --- Test file sync configuration --- +_REQUIREMENTS_FILES = ["azure.txt", "unbranded.txt"] + + +# --------------------------------------------------------------------------- +# Test file sync +# --------------------------------------------------------------------------- _SKIP_DIRS: Set[str] = {"__pycache__", "generated", ".venv", "node_modules", ".tox"} @@ -60,7 +71,7 @@ # --------------------------------------------------------------------------- -# Requirements marker-based sync (syncs marker-delimited sections if present) +# Requirements sync # --------------------------------------------------------------------------- @@ -117,14 +128,30 @@ def _replace_marker_sections(filepath: Path, source_sections: Dict[str, List[str filepath.write_text("\n".join(result) + "\n", encoding="utf-8", newline="\n") -def sync_requirements(source: Path, target: Path) -> None: - """Sync common marker sections from source to target requirements.txt.""" - source_sections = _extract_marker_sections(source) - if not source_sections: - print(f" WARNING: no marker sections found in {source}, skipping") - return - _replace_marker_sections(target, source_sections) - print(f" Synced requirements: {source.name} ({source.parent.name}/)") +def sync_requirements(source_dir: Path, target_dir: Path) -> None: + """Copy requirements files from typespec to autorest. + + If marker sections are present, only the marker-delimited sections are + replaced in the target (preserving repo-specific dependencies outside + markers). Otherwise the file is copied directly. + """ + for filename in _REQUIREMENTS_FILES: + src = source_dir / filename + dst = target_dir / filename + if not src.is_file(): + print(f" WARNING: {src} not found, skipping") + continue + + source_sections = _extract_marker_sections(src) + if source_sections and dst.is_file(): + _replace_marker_sections(dst, source_sections) + print(f" Synced markers: requirements/{filename}") + else: + if dst.is_file() and src.read_bytes() == dst.read_bytes(): + continue + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + print(f" Copied: requirements/{filename}") # --------------------------------------------------------------------------- @@ -212,14 +239,12 @@ def main() -> int: shutil.copy2(src_ts, dst_ts) print(f"Synced regenerate-common.ts") - # 2. Sync requirements (marker sections if present) - for flavor in ("azure", "unbranded"): - src_req = typespec_repo / TYPESPEC_TEST_DIR / "requirements" / f"{flavor}.txt" - dst_req = autorest_repo / AUTOREST_TEST_DIR / "requirements" / f"{flavor}.txt" - if src_req.is_file() and dst_req.is_file(): - sync_requirements(src_req, dst_req) - else: - print(f" WARNING: requirements/{flavor}.txt not found, skipping") + # 2. Sync requirements files + print("Syncing requirements...") + sync_requirements( + typespec_repo / TYPESPEC_TEST_DIR / "requirements", + autorest_repo / AUTOREST_TEST_DIR / "requirements", + ) # 3. Sync test files print("Syncing test files...") From df72e79a13926b21e93259c6bd6f7d3691a67cf0 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 7 Apr 2026 14:31:22 +0800 Subject: [PATCH 4/5] add docs --- .../typespec-python/docs/test_framework.md | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 packages/typespec-python/docs/test_framework.md diff --git a/packages/typespec-python/docs/test_framework.md b/packages/typespec-python/docs/test_framework.md new file mode 100644 index 0000000000..aeebd1735f --- /dev/null +++ b/packages/typespec-python/docs/test_framework.md @@ -0,0 +1,180 @@ +# Test Framework for typespec-python + +This document describes the test framework used in the `typespec-python` package +and how it relates to the upstream +[`http-client-python`](https://github.com/microsoft/typespec/tree/main/packages/http-client-python) +package in the typespec repository. + +## Overview + +The test framework is a **dual-flavor testing system** (Azure and Unbranded) built +on **pytest** and **tox**. Tests run against a mock API server +([tsp-spector](https://github.com/microsoft/typespec)) that serves TypeSpec-defined +HTTP endpoints on `localhost:3000`. + +## Folder Structure + +``` +packages/typespec-python/ +└── tests/ + ├── conftest.py # Root fixtures (server lifecycle, core_library, credentials, image data) + ├── install_packages.py # Installs generated SDK packages before test runs + ├── pytest.ini # Pytest config (asyncio_mode = auto) + ├── tox.ini # Test environments (test, lint, mypy, pyright, docs, ci) + │ + ├── data/ # Static test data (image.png, image.jpg) + │ + ├── requirements/ # Dependency files + │ ├── base.txt # Common: pytest, pytest-asyncio, tox, coverage, etc. + │ ├── azure.txt # Azure flavor: azure-core, azure-mgmt-core, geojson + │ ├── unbranded.txt # Unbranded flavor: corehttp + │ ├── lint.txt # Linting: pylint, black + │ ├── typecheck.txt # Type checking: pyright, mypy + │ └── docs.txt # Documentation: sphinx, myst_parser + │ + ├── generated/ # Auto-generated SDK packages from TypeSpec specs + │ ├── azure/ # ~116 Azure-flavored packages + │ └── unbranded/ # ~64 Unbranded packages + │ + └── mock_api/ # Hand-written integration tests + ├── azure/ # Azure-specific tests + │ ├── conftest.py # Azure fixtures (credentials, LRO polling, header validation) + │ ├── asynctests/ # Async test variants + │ ├── data/ # Test image data + │ └── test_*.py # Sync test files + ├── shared/ # Tests that run for both flavors + │ ├── conftest.py # Shared fixtures + │ ├── asynctests/ # Async test variants + │ ├── unittests/ # Unit tests (e.g. pyproject parsing) + │ ├── data/ # Test image data + │ └── test_*.py # Sync test files + └── unbranded/ # Unbranded-specific tests + ├── conftest.py # Unbranded fixtures + ├── asynctests/ # Async test variants + ├── data/ # Test image data + └── test_*.py # Sync test files +``` + +## Test Flavors + +| Flavor | Core library | Credential class | What it tests | +|--------|-------------|-----------------|---------------| +| **azure** | `azure.core` | `AzureKeyCredential` | Azure SDK conventions, ARM resources, LRO, paging | +| **unbranded** | `corehttp` | `ServiceKeyCredential` | Non-Azure SDK generation without Azure branding | + +When tests run: +- **Azure**: `pytest mock_api/azure mock_api/shared` with `FLAVOR=azure` +- **Unbranded**: `pytest mock_api/unbranded mock_api/shared` with `FLAVOR=unbranded` + +The `shared/` tests run for **both** flavors. Root `conftest.py` uses `core_library()` +to dynamically import the appropriate core library. + +## Running Tests + +```bash +cd packages/typespec-python/tests + +# Run Azure flavor tests +tox -e test-azure + +# Run Unbranded flavor tests +tox -e test-unbranded + +# Run all CI checks for a flavor (tests + lint + type checking) +tox -e ci-azure +tox -e ci-unbranded +``` + +### Available tox Environments + +| Environment | Description | +|------------|-------------| +| `test-azure` / `test-unbranded` | Run pytest integration tests | +| `lint-azure` / `lint-unbranded` | Run pylint | +| `mypy-azure` / `mypy-unbranded` | Run mypy type checking | +| `pyright-azure` / `pyright-unbranded` | Run pyright type checking | +| `docs-azure` / `docs-unbranded` | Build API docs with Sphinx | +| `ci-azure` / `ci-unbranded` | All checks combined | + +## Key Components + +### Mock API Server + +Tests rely on `tsp-spector` to serve TypeSpec-defined mock endpoints. The root +`conftest.py` starts the server automatically at session start and tears it down +after all tests complete. The server runs on `localhost:3000`. + +### Generated Packages + +Each test spec produces a generated SDK package under `tests/generated/{flavor}/`. +Before tests run, `install_packages.py` installs all generated packages into the +test environment using `uv pip install --no-deps`. + +### Async Tests + +Every `test_*.py` in the sync directory has a corresponding `test_*_async.py` in +`asynctests/`. Async fixtures use `@pytest_asyncio.fixture` and `pytest.ini` +configures `asyncio_mode = auto`. + +## Folder Mapping to the typespec Repository + +The test files are shared with the upstream +[`http-client-python`](https://github.com/microsoft/typespec/tree/main/packages/http-client-python) +package. The **typespec repo is the source of truth** for shared test files. + +### Path Mapping + +| typespec repo | autorest.python repo | +|--------------|---------------------| +| `packages/http-client-python/tests/mock_api/azure/` | `packages/typespec-python/tests/mock_api/azure/` | +| `packages/http-client-python/tests/mock_api/shared/` | `packages/typespec-python/tests/mock_api/shared/` | +| `packages/http-client-python/tests/mock_api/unbranded/` | `packages/typespec-python/tests/mock_api/unbranded/` | +| `packages/http-client-python/tests/requirements/` | `packages/typespec-python/tests/requirements/` | +| `packages/http-client-python/eng/scripts/ci/regenerate-common.ts` | `packages/typespec-python/eng/scripts/regenerate-common.ts` | + +### What Is Synced + +The script `eng/scripts/sync_from_typespec.py` copies from typespec → autorest.python: + +1. **`regenerate-common.ts`** — shared regeneration logic +2. **Requirements files** — `azure.txt` and `unbranded.txt` (marker-delimited + common sections are synced; repo-specific deps like `geojson` are preserved) +3. **Test files** — all files under `mock_api/{shared,azure,unbranded}` except: + - `conftest.py` (each repo has its own) + - `tox.ini`, `requirements.txt`, `dev_requirements.txt` + - `.pyc` files + +### What Is NOT Synced + +| Item | Reason | +|------|--------| +| `conftest.py` files | Different server startup and fixture logic per repo | +| `tests/mock_api/shared/unittests/` | Repo-specific unit tests | +| `tests/generated/` | Regenerated independently in each repo | +| `tests/unit/` (typespec only) | Internal to http-client-python | +| `pytest.ini`, `tox.ini` | Different CI configurations per repo | + +### Requirements Marker Convention + +Requirements files (`azure.txt`, `unbranded.txt`) use markers to delimit the +common section synced between repos: + +``` +# === common azure dependencies across repos === +# Azure SDK dependencies +-r base.txt +azure-core>=1.37.0 +azure-mgmt-core==1.6.0 +# === end common azure dependencies across repos === +geojson>=3.0.0 # <-- autorest.python-only dependency, outside markers +``` + +Dependencies outside the markers are preserved during sync. + +### Sync Workflow + +The sync is run automatically as part of [pipeline](https://dev.azure.com/azure-sdk/internal/_build?definitionId=7257), or manually: + +```bash +python eng/scripts/sync_from_typespec.py +``` From db5d1879475bfea8da9e905e070b35e7d085abc1 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Tue, 7 Apr 2026 16:09:20 +0800 Subject: [PATCH 5/5] skip apiview until eng issue is fixed --- eng/pipelines/generated-code-checks-template.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/generated-code-checks-template.yml b/eng/pipelines/generated-code-checks-template.yml index 33608b95b3..3f0c63e5dd 100644 --- a/eng/pipelines/generated-code-checks-template.yml +++ b/eng/pipelines/generated-code-checks-template.yml @@ -23,7 +23,7 @@ steps: condition: and(eq(variables['PythonVersion'], '3.10'), or(contains('${{ parameters.folderName }}', 'version-tolerant'), eq('${{parameters.package}}', 'typespec-python'))) # docs/apiview only for typespec-python (removed from autorest.python to reduce CI time) - - script: npm run test -- --env=docs --flavor=${{ parameters.folderName }} - displayName: ApiView ${{ parameters.folderName }} - Python $(PythonVersion) - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.package}} - condition: and(eq(variables['PythonVersion'], '3.11'), eq('${{parameters.package}}', 'typespec-python')) + # - script: npm run test -- --env=docs --flavor=${{ parameters.folderName }} + # displayName: ApiView ${{ parameters.folderName }} - Python $(PythonVersion) + # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.package}} + # condition: and(eq(variables['PythonVersion'], '3.11'), eq('${{parameters.package}}', 'typespec-python'))