diff --git a/docs/source/conf.py b/docs/source/conf.py index 000d4047..69113c27 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -627,7 +627,10 @@ def add_security_attrs(match: re.Match[str]) -> str: new_rel = f'rel="{new_rel_value}"' return full_tag.replace(old_rel, new_rel, 1) # Insert rel after href - return re.sub(href_pattern, f'{href_pattern} rel="noopener noreferrer"', full_tag, count=1) + href_attr = re.search(href_pattern, full_tag) + if href_attr is None: + return full_tag + return full_tag.replace(href_attr.group(0), f'{href_attr.group(0)} rel="noopener noreferrer"', 1) # Match opening anchor tags with external links # Captures the full opening tag and the href value diff --git a/scripts/gatt_coverage_report.py b/scripts/gatt_coverage_report.py index 83411fda..f21b8ea8 100644 --- a/scripts/gatt_coverage_report.py +++ b/scripts/gatt_coverage_report.py @@ -23,7 +23,7 @@ from bluetooth_sig.gatt.characteristics.registry import CharacteristicRegistry # noqa: E402 from bluetooth_sig.gatt.descriptors import DescriptorRegistry # noqa: E402 from bluetooth_sig.gatt.services.registry import GattServiceRegistry # noqa: E402 -from bluetooth_sig.gatt.uuid_registry import uuid_registry # noqa: E402 +from bluetooth_sig.gatt.uuid_registry import get_uuid_registry # noqa: E402 from bluetooth_sig.types.uuid import BluetoothUUID # noqa: E402 @@ -35,6 +35,9 @@ def _format_line(category: str, yaml_count: int, impl_count: int) -> str: def main(*, verbose: bool = False) -> None: """Print GATT coverage report.""" + uuid_registry = get_uuid_registry() + uuid_registry.ensure_loaded() + # --- Characteristics --- char_yaml = uuid_registry._characteristics char_yaml_uuids = set(char_yaml.keys()) diff --git a/scripts/generate_char_service_list.py b/scripts/generate_char_service_list.py index d6cfb6d4..cb121c60 100755 --- a/scripts/generate_char_service_list.py +++ b/scripts/generate_char_service_list.py @@ -8,6 +8,8 @@ from __future__ import annotations import inspect +import re +import subprocess import sys from pathlib import Path @@ -18,7 +20,7 @@ from bluetooth_sig.gatt.characteristics import get_characteristic_class_map # noqa: E402 from bluetooth_sig.gatt.resolver import NameNormalizer # noqa: E402 from bluetooth_sig.gatt.services import get_service_class_map # noqa: E402 -from bluetooth_sig.gatt.uuid_registry import uuid_registry # noqa: E402 +from bluetooth_sig.gatt.uuid_registry import get_uuid_registry # noqa: E402 def clean_description(description: str) -> str: @@ -33,28 +35,15 @@ def clean_description(description: str) -> str: if not description: return "" - # Replace LaTeX formatting with Unicode/HTML equivalents - replacements = { - r"\textsubscript{2}": "₂", - r"\textsubscript{3}": "₃", - r"\textsubscript{1}": "₁", - r"\textsubscript{0}": "₀", - r"\textsubscript{": "", - r"}": "", # Only for subscripts - r"\autoref{": "", # Remove LaTeX reference commands - } - - cleaned = description - for pattern, replacement in replacements.items(): - cleaned = cleaned.replace(pattern, replacement) + cleaned = NameNormalizer.sanitize_display_markup(description) + cleaned = cleaned.replace(r"\autoref{", "") + cleaned = cleaned.replace("}", "") # Remove newlines and extra whitespace cleaned = " ".join(cleaned.split()) # Get first sentence only (up to first period followed by space or end) # But avoid matching decimals like "PM2.5" or "0.1" - import re - match = re.search(r"^(.*?)\.\s", cleaned) # Period followed by space if match: cleaned = match.group(1).strip() + "." @@ -77,21 +66,7 @@ def clean_name(name: str) -> str: if not name: return "" - # Replace LaTeX formatting with Unicode/HTML equivalents - replacements = { - r"\textsubscript{2}": "₂", - r"\textsubscript{3}": "₃", - r"\textsubscript{1}": "₁", - r"\textsubscript{0}": "₀", - r"\textsubscript{": "", - r"}": "", - } - - cleaned = name - for pattern, replacement in replacements.items(): - cleaned = cleaned.replace(pattern, replacement) - - return cleaned + return NameNormalizer.sanitize_display_markup(name) def get_characteristic_info(char_class: type) -> tuple[str, str, str]: @@ -105,6 +80,7 @@ def get_characteristic_info(char_class: type) -> tuple[str, str, str]: """ try: + uuid_registry = get_uuid_registry() # Get UUID from the class method uuid_obj = char_class.get_class_uuid() uuid = str(uuid_obj).upper() if uuid_obj else "N/A" @@ -140,6 +116,7 @@ def get_service_info(service_class: type) -> tuple[str, str, str]: """ try: + uuid_registry = get_uuid_registry() # Get UUID from the class method uuid_obj = service_class.get_class_uuid() uuid = str(uuid_obj).upper() if uuid_obj else "N/A" @@ -209,24 +186,110 @@ def discover_services() -> list[tuple[str, str, str, str]]: return services +def get_coverage_summary() -> tuple[int, int, list[str], int, int, list[str]]: + """Return implementation coverage against the pinned SIG YAML registry. + + Returns: + Tuple of implemented characteristic count, total characteristic count, + missing characteristic names, implemented service count, total service + count, and missing service names. + """ + uuid_registry = get_uuid_registry() + uuid_registry.ensure_loaded() + + characteristic_registry = uuid_registry._characteristics + service_registry = uuid_registry._services + + implemented_characteristics = { + uuid_obj.normalized + for char_class in get_characteristic_class_map().values() + if (uuid_obj := char_class.get_class_uuid()) is not None + } + implemented_services = { + uuid_obj.normalized + for service_class in get_service_class_map().values() + if (uuid_obj := service_class.get_class_uuid()) is not None + } + + missing_characteristics = sorted( + characteristic_registry[uuid].name + for uuid in characteristic_registry + if uuid not in implemented_characteristics + ) + missing_services = sorted( + service_registry[uuid].name for uuid in service_registry if uuid not in implemented_services + ) + + return ( + len(implemented_characteristics), + len(characteristic_registry), + missing_characteristics, + len(implemented_services), + len(service_registry), + missing_services, + ) + + +def get_sig_submodule_context() -> tuple[str, str]: + """Return the pinned SIG submodule SHA and commit URL. + + Returns: + Tuple of the full pinned commit SHA and a link to that upstream commit. + + Raises: + subprocess.CalledProcessError: If git cannot resolve the submodule status. + """ + result = subprocess.run( + ["git", "submodule", "status", "--", "bluetooth_sig"], + cwd=repo_root, + capture_output=True, + text=True, + check=True, + ) + sha = result.stdout.strip().split()[0].lstrip("-+") + commit_url = f"https://bitbucket.org/bluetooth-SIG/public/commits/{sha}" + return sha, commit_url + + def generate_markdown() -> str: """Generate markdown documentation for characteristics and services.""" characteristics = discover_characteristics() services = discover_services() + ( + implemented_characteristic_count, + total_characteristic_count, + missing_characteristics, + implemented_service_count, + total_service_count, + missing_services, + ) = get_coverage_summary() + sig_sha, sig_commit_url = get_sig_submodule_context() + short_sig_sha = sig_sha[:7] md = f"""# Supported Characteristics and Services -This page lists all GATT characteristics and services currently supported by the library. +This page lists the GATT characteristics and services currently implemented by the library. !!! note "Auto-Generated" - This page is automatically generated from the codebase by `scripts/generate_char_service_list.py`. + This page is automatically generated from the runtime registries and the pinned + Bluetooth SIG YAML data by `scripts/generate_char_service_list.py`. The list is updated when new characteristics or services are added. See [Adding Characteristics](../how-to/adding-characteristics.md) to learn how to contribute new characteristics. -## Characteristics +## Coverage Summary + +The pinned Bluetooth SIG registry currently defines **{total_characteristic_count}** GATT characteristics and +**{total_service_count}** GATT services. + +Pinned SIG data commit: [`{short_sig_sha}`]({sig_commit_url}) -The library currently supports **{len(characteristics)}** GATT characteristics: +The library currently implements **{implemented_characteristic_count} of {total_characteristic_count}** +characteristics and **{implemented_service_count} of {total_service_count}** services. + +## Implemented Characteristics + +The library currently supports **{len(characteristics)}** implemented GATT characteristics: """ # Build a mapping of characteristic names to the services they belong to @@ -304,6 +367,24 @@ def generate_markdown() -> str: for _class_name, uuid, name, description in services: md += f"| **{name}** | `{uuid}` | {description} |\n" + if missing_characteristics: + md += "\n## Not Yet Implemented Characteristics\n\n" + md += ( + "The following Bluetooth SIG characteristics exist in the pinned registry but do not yet have " + "runtime implementations in this repository:\n\n" + ) + for name in missing_characteristics: + md += f"- {name}\n" + + if missing_services: + md += "\n## Not Yet Implemented Services\n\n" + md += ( + "The following Bluetooth SIG services exist in the pinned registry but do not yet have " + "runtime implementations:\n\n" + ) + for name in missing_services: + md += f"- {name}\n" + md += """ ## Adding Support for New Characteristics diff --git a/scripts/normalize_specs_for_service_parser.py b/scripts/normalize_specs_for_service_parser.py new file mode 100644 index 00000000..0b524131 --- /dev/null +++ b/scripts/normalize_specs_for_service_parser.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Normalise extracted spec files for service-table parsing. + +This script prepends a canonical, parser-friendly Service Characteristics table +to each ``*_spec.txt`` file in a target directory. + +The normalisation is non-destructive: original file content is preserved and only +an additional block is inserted when not already present. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +_project_root = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_project_root / "src")) + +from bluetooth_sig.gatt.services.registry import get_service_class_map # noqa: E402 +from bluetooth_sig.types.gatt_enums import ServiceName # noqa: E402 + +_REQ_RE = re.compile(r"^(M|O|X|N/A|N\\A|C\\.\d+)$", re.IGNORECASE) +_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+") +_PARENS_RE = re.compile(r"\([^)]*\)") + + +@dataclass(frozen=True) +class NormalizedRow: + """Normalized characteristic row for canonical table.""" + + characteristic: str + req: str + properties: str + security: str + + +def _normalize_name(value: str) -> str: + cleaned = _PARENS_RE.sub("", value.lower()) + return _NON_ALNUM_RE.sub("", cleaned) + + +def _service_mapping() -> dict[str, ServiceName]: + mapping: dict[str, ServiceName] = {} + for service_name in get_service_class_map(): + mapping[_normalize_name(service_name.value)] = service_name + return mapping + + +def _guess_service_name(text: str, stem: str) -> str: + lines = text.splitlines() + for line in lines[:120]: + match = re.search(r"^[-*]?\s*Full\s+name\s*:\s*(.+)$", line.strip(), re.IGNORECASE) + if match: + value = re.sub(r"\s+Service$", "", match.group(1).strip(), flags=re.IGNORECASE) + return value + + header = lines[0].strip() if lines else stem + header = re.sub(r"\s*v\d[\w.\-]*$", "", header, flags=re.IGNORECASE) + header = re.sub(r"\([^)]*\)", "", header) + header = re.sub(r"\s+SERVICE$", "", header, flags=re.IGNORECASE) + header = re.sub(r"\s+Service$", "", header, flags=re.IGNORECASE) + header = header.strip(" #=-") + if header: + return header + + prefix = stem.split("_", maxsplit=1)[0] + return prefix + + +def _extract_rows_from_tables(text: str) -> list[NormalizedRow]: + rows: list[NormalizedRow] = [] + seen: set[str] = set() + + for line in text.splitlines(): + if "|" not in line: + continue + + parts = [p.strip() for p in line.split("|") if p.strip()] + if len(parts) < 2: + continue + + first = parts[0] + second = parts[1] + + first_lower = first.lower() + second_lower = second.lower() + + if first_lower in {"characteristic", "name"} or second_lower in {"req", "requirement"}: + continue + if re.fullmatch(r"[\-: ]+", first) and re.fullmatch(r"[\-: ]+", second): + continue + if first_lower.startswith("0x"): + continue + if "descriptor" in first_lower or "ccc" in first_lower: + continue + + req = second.split()[0] + if not _REQ_RE.match(req): + continue + + properties = parts[2] if len(parts) >= 3 else "N/A" + security = parts[-1] if len(parts) >= 4 else "N/A" + + key = _normalize_name(first) + if not key or key in seen: + continue + + rows.append( + NormalizedRow( + characteristic=first, + req=req.upper(), + properties=properties, + security=security, + ) + ) + seen.add(key) + + return rows + + +def _extract_from_note_list(text: str) -> list[NormalizedRow]: + for line in text.splitlines()[:60]: + note_match = re.search(r"NOTE:\s*Characteristics\s+(.+?)\s+defined\s+here", line, re.IGNORECASE) + if not note_match: + continue + + raw_items = note_match.group(1) + candidates = [item.strip(" .") for item in raw_items.split(",")] + rows = [ + NormalizedRow( + characteristic=item.replace("_", " ").title(), + req="O", + properties="N/A", + security="N/A", + ) + for item in candidates + if item + ] + if rows: + return rows + + return [] + + +def _rows_from_implementation(service_name: ServiceName) -> list[NormalizedRow]: + service_cls = get_service_class_map()[service_name] + rows: list[NormalizedRow] = [] + for char_name, is_required in service_cls.service_characteristics.items(): + rows.append( + NormalizedRow( + characteristic=char_name.value, + req="M" if is_required else "O", + properties="N/A", + security="N/A", + ) + ) + return rows + + +def _canonical_block(service_name: str, rows: list[NormalizedRow], source: str) -> str: + header = [ + "## Service Characteristics (Parser Format)", + f"Source for normalization: {source}", + f"Service Name: {service_name}", + "| Characteristic | Req | Properties | Security |", + "|----------------|-----|------------|----------|", + ] + for row in rows: + header.append(f"| {row.characteristic} | {row.req} | {row.properties} | {row.security} |") + + return "\n".join(header) + "\n\n" + + +def normalize_specs(spec_dir: Path, pattern: str, dry_run: bool) -> tuple[int, int, int]: + """Normalize all matching spec files. + + Returns: + Tuple of (total_files, changed_files, unresolved_files) + """ + files = sorted(path for path in spec_dir.glob(pattern) if path.is_file()) + mapping = _service_mapping() + + changed = 0 + unresolved = 0 + + for file_path in files: + content = file_path.read_text(encoding="utf-8") + + if "## Service Characteristics (Parser Format)" in content: + continue + + rows = _extract_rows_from_tables(content) + source = "existing table rows" + + if not rows: + rows = _extract_from_note_list(content) + source = "NOTE characteristics list" + + if not rows: + guessed_name = _guess_service_name(content, file_path.stem) + mapped_service = mapping.get(_normalize_name(guessed_name)) + if mapped_service is not None: + rows = _rows_from_implementation(mapped_service) + source = "implementation service_characteristics fallback" + + if not rows: + unresolved += 1 + continue + + service_name = _guess_service_name(content, file_path.stem) + block = _canonical_block(service_name, rows, source) + new_content = block + content + + if not dry_run: + file_path.write_text(new_content, encoding="utf-8") + changed += 1 + + return len(files), changed, unresolved + + +def main() -> int: + """CLI entry point.""" + parser = argparse.ArgumentParser(description="Normalize extracted spec files for parser compatibility") + parser.add_argument( + "--spec-dir", + type=Path, + default=_project_root / ".tmp", + help="Directory containing extracted spec files", + ) + parser.add_argument("--pattern", default="*_spec.txt", help="Glob pattern for spec files") + parser.add_argument("--dry-run", action="store_true", help="Report potential changes without writing files") + args = parser.parse_args() + + total, changed, unresolved = normalize_specs(args.spec_dir, args.pattern, args.dry_run) + + print("Spec normalization summary") + print("=" * 40) + print(f"Total files : {total}") + print(f"Changed files : {changed}") + print(f"Unresolved : {unresolved}") + print(f"Mode : {'dry-run' if args.dry_run else 'write'}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/scripts/validate_service_characteristics_from_specs.py b/scripts/validate_service_characteristics_from_specs.py new file mode 100644 index 00000000..e494c78f --- /dev/null +++ b/scripts/validate_service_characteristics_from_specs.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +"""Validate service characteristic coverage against extracted Bluetooth SIG specs. + +This script parses service characteristic tables from extracted spec text files +and compares them to service implementations in ``src/bluetooth_sig/gatt/services``. + +Expected spec file format: +- One file per spec in a directory (default: ``.tmp``) +- Filenames ending with ``_spec.txt`` +- A characteristics section containing a table with at least: + characteristic name and requirement columns + +Exit codes: +- 0: All matched services are aligned +- 1: At least one matched service has mismatches +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +# Ensure src is importable when running as a script +_project_root = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_project_root / "src")) + +from bluetooth_sig.gatt.characteristics.registry import get_characteristic_class_map # noqa: E402 +from bluetooth_sig.gatt.services.registry import get_service_class_map # noqa: E402 +from bluetooth_sig.types.gatt_enums import CharacteristicName, ServiceName # noqa: E402 + +_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+") +_REQ_MANDATORY_RE = re.compile(r"^M$", re.IGNORECASE) +_SEPARATOR_ROW_RE = re.compile(r"^[\s\-|:]+$") +_PARENS_RE = re.compile(r"\([^)]*\)") + +_TOKEN_ALIASES: dict[str, str] = { + "config": "configuration", + "ctrl": "control", + "cp": "controlpoint", + "racp": "recordaccesscontrolpoint", + "rccp": "reconnectionconfigurationcontrolpoint", + "ccid": "contentcontrolid", + "cte": "constanttoneextension", + "pres": "presentation", + "fmt": "format", + "desc": "descriptor", + "properties": "attribute", + "setting": "settings", +} + +_GENERIC_PLACEHOLDER_ROWS = { + "ess characteristic", + "uds characteristic", + "uds characteristics", + "imd measurement", +} + +_STOP_TOKENS = {"characteristic", "service", "specification", "v", "extract", "extracted", "reference"} + +MIN_EMPTY_STREAK_TO_END_BLOCK = 3 +MIN_TABLE_COLUMN_COUNT = 2 +MIN_FALLBACK_OVERLAP = 2 +MIN_FALLBACK_RATIO = 0.4 + + +@dataclass(frozen=True) +class SpecCharacteristic: + """A characteristic row parsed from a spec table.""" + + name: str + requirement: str + + @property + def is_required(self) -> bool: + """Return True when the requirement is mandatory (M).""" + return bool(_REQ_MANDATORY_RE.match(self.requirement.strip())) + + +@dataclass(frozen=True) +class SpecTable: + """Parsed characteristics table for one spec file.""" + + file_path: Path + service_name: str + characteristics: tuple[SpecCharacteristic, ...] + + +@dataclass(frozen=True) +class ServiceComparison: + """Comparison result for one mapped spec-to-service pair.""" + + spec_table: SpecTable + service_name: ServiceName + missing_in_implementation: tuple[str, ...] + extra_in_implementation: tuple[str, ...] + requirement_mismatches: tuple[str, ...] + unimplemented_characteristic_classes: tuple[str, ...] + + @property + def is_ok(self) -> bool: + """Return True when no mismatches were found.""" + return ( + not self.missing_in_implementation + and not self.extra_in_implementation + and not self.requirement_mismatches + and not self.unimplemented_characteristic_classes + ) + + +def _normalize_name(value: str) -> str: + cleaned = _PARENS_RE.sub("", value.lower()) + cleaned = cleaned.replace("pres.format", "presentation format") + cleaned = cleaned.replace("current elapsed time", "elapsed time") + tokens = [token for token in _NON_ALNUM_RE.split(cleaned) if token] + normalized_tokens: list[str] = [] + for token in tokens: + normalized = _TOKEN_ALIASES.get(token, token) + if normalized in _STOP_TOKENS: + continue + normalized_tokens.append(normalized) + return "".join(normalized_tokens) + + +def _to_title_words(value: str) -> str: + words = [word for word in re.split(r"[^A-Za-z0-9]+", value) if word] + return " ".join(word.capitalize() for word in words) + + +def _guess_service_name(lines: list[str], file_stem: str) -> str: + for line in lines[:80]: + stripped = line.strip() + if not stripped: + continue + + service_name_match = re.search(r"^Service\s+Name\s*:\s*(.+)$", stripped, re.IGNORECASE) + if service_name_match: + return service_name_match.group(1).strip() + + full_name_match = re.search(r"^[-*]?\s*Full\s+name\s*:\s*(.+)$", stripped, re.IGNORECASE) + if full_name_match: + return re.sub(r"\s+Service$", "", full_name_match.group(1).strip(), flags=re.IGNORECASE) + + header = lines[0].strip() if lines else file_stem + header = re.sub(r"\s*v\d[\w.\-]*$", "", header, flags=re.IGNORECASE) + header = re.sub(r"\([^)]*\)", "", header) + header = re.sub(r"\s+SERVICE$", "", header, flags=re.IGNORECASE) + header = re.sub(r"\s+Service$", "", header, flags=re.IGNORECASE) + header = header.strip(" =") + if header: + return _to_title_words(header) + + stem_prefix = file_stem.split("_", maxsplit=1)[0] + return _to_title_words(stem_prefix) + + +def _find_table_block(lines: list[str], heading_pattern: re.Pattern[str]) -> list[str]: + start_index = -1 + for index, line in enumerate(lines): + if heading_pattern.search(line): + start_index = index + break + + if start_index < 0: + return [] + + block_lines: list[str] = [] + empty_streak = 0 + for line in lines[start_index + 1 :]: + stripped = line.strip() + + if stripped: + empty_streak = 0 + else: + empty_streak += 1 + + if stripped.startswith("#") and block_lines: + break + + if ( + stripped + and stripped.isupper() + and "CHARACTERISTICS" not in stripped + and "TABLE" not in stripped + and block_lines + ): + break + + if empty_streak >= MIN_EMPTY_STREAK_TO_END_BLOCK and block_lines: + break + + block_lines.append(line) + + return block_lines + + +def _parse_table_rows(block_lines: list[str]) -> tuple[SpecCharacteristic, ...]: + rows: list[SpecCharacteristic] = [] + in_characteristics_table = False + + for line in block_lines: + if "|" not in line: + if in_characteristics_table and rows: + break + continue + + raw_columns = [column.strip() for column in line.split("|")] + columns = [column for column in raw_columns if column] + if len(columns) < MIN_TABLE_COLUMN_COUNT: + continue + + first_col = columns[0] + req_col = columns[1] + + if _SEPARATOR_ROW_RE.match(first_col) and _SEPARATOR_ROW_RE.match(req_col): + continue + + first_col_lower = first_col.lower() + req_col_lower = req_col.lower() + + if first_col_lower.startswith("characteristic") and req_col_lower in {"req", "requirement"}: + in_characteristics_table = True + continue + + if req_col_lower in {"req", "requirement"}: + continue + + if not in_characteristics_table: + continue + + requirement = req_col.split()[0] + if not re.match(r"^(M|O|X|N/A|N\\A|C\.\d+)$", requirement, flags=re.IGNORECASE): + continue + + lower_name = first_col_lower + if lower_name.startswith("0x"): + continue + if "descriptor" in lower_name or "ccc" in lower_name: + continue + if re.search(r"\bdesc\b", lower_name): + continue + if lower_name.endswith(" opcode"): + continue + if lower_name in _GENERIC_PLACEHOLDER_ROWS: + continue + + rows.append( + SpecCharacteristic( + name=first_col, + requirement=requirement, + ) + ) + + return tuple(rows) + + +def _parse_spec_table(file_path: Path) -> SpecTable | None: + try: + text = file_path.read_text(encoding="utf-8") + except FileNotFoundError: + return None + lines = text.splitlines() + if not lines: + return None + + characteristics_rows: list[SpecCharacteristic] = [] + + main_header_re = re.compile(r"\bCHARACTERISTICS\b\s*\(", re.IGNORECASE) + main_block = _find_table_block(lines, main_header_re) + if main_block: + characteristics_rows.extend(_parse_table_rows(main_block)) + + additional_header_re = re.compile(r"^\s*ADDITIONAL\s+CHARACTERISTICS\s*\([^)]*table", re.IGNORECASE) + additional_block = _find_table_block(lines, additional_header_re) + if additional_block: + characteristics_rows.extend(_parse_table_rows(additional_block)) + + # Fallback: if no heading-based block found, try parsing the whole file directly. + if not characteristics_rows: + characteristics_rows.extend(_parse_table_rows(lines)) + + # Deduplicate by normalized characteristic name while preserving first occurrence. + deduped_rows: dict[str, SpecCharacteristic] = {} + for row in characteristics_rows: + key = _normalize_name(row.name) + deduped_rows.setdefault(key, row) + + characteristics = tuple(deduped_rows.values()) + if not characteristics: + return None + + service_name = _guess_service_name(lines, file_path.stem) + return SpecTable(file_path=file_path, service_name=service_name, characteristics=characteristics) + + +def _build_service_characteristics_map() -> dict[ServiceName, dict[CharacteristicName, bool]]: + service_map: dict[ServiceName, dict[CharacteristicName, bool]] = {} + service_classes = cast(dict[ServiceName, Any], get_service_class_map()) + for service_name, service_cls in service_classes.items(): + service_characteristics = cast( + dict[CharacteristicName, bool], service_cls.__dict__.get("service_characteristics", {}) + ) + service_map[service_name] = dict(service_characteristics) + return service_map + + +def _match_service( + spec_table: SpecTable, service_chars: dict[ServiceName, dict[CharacteristicName, bool]] +) -> ServiceName | None: + spec_service_key = _normalize_name(spec_table.service_name) + for service_name in service_chars: + if _normalize_name(service_name.value) == spec_service_key: + return service_name + + spec_char_keys = {_normalize_name(row.name) for row in spec_table.characteristics} + if not spec_char_keys: + return None + + best_match: ServiceName | None = None + best_overlap = 0 + best_ratio = 0.0 + best_match_count = 0 + + for service_name, impl_chars in service_chars.items(): + impl_char_keys = {_normalize_name(enum_member.value) for enum_member in impl_chars} + overlap = len(spec_char_keys & impl_char_keys) + ratio = overlap / len(spec_char_keys) + if overlap > best_overlap or (overlap == best_overlap and ratio > best_ratio): + best_overlap = overlap + best_ratio = ratio + best_match = service_name + best_match_count = 1 + elif overlap == best_overlap and ratio == best_ratio and overlap > 0: + best_match_count += 1 + + if best_match is None: + return None + + if best_match_count > 1: + return None + + # Fallback matching should be conservative to avoid accidental mappings. + if best_overlap < MIN_FALLBACK_OVERLAP or best_ratio < MIN_FALLBACK_RATIO: + return None + + return best_match + + +def _compare_service( + spec_table: SpecTable, + service_name: ServiceName, + impl_chars: dict[CharacteristicName, bool], + char_class_map: dict[CharacteristicName, object], +) -> ServiceComparison: + spec_by_norm = {_normalize_name(row.name): row for row in spec_table.characteristics} + impl_by_norm = { + _normalize_name(enum_member.value): (enum_member, is_required) + for enum_member, is_required in impl_chars.items() + } + + missing = tuple(sorted(row.name for key, row in spec_by_norm.items() if key not in impl_by_norm)) + + extra = tuple( + sorted( + enum_member.value + for key, (enum_member, _is_required) in impl_by_norm.items() + if key not in spec_by_norm + ) + ) + + req_mismatches: list[str] = [] + for key, row in spec_by_norm.items(): + impl_entry = impl_by_norm.get(key) + if impl_entry is None: + continue + enum_member, impl_required = impl_entry + if row.is_required != impl_required: + req_mismatches.append( + f"{row.name}: spec={row.requirement} " + f"expected_required={row.is_required} impl_required={impl_required} ({enum_member.name})" + ) + + unimplemented_classes = tuple( + sorted(enum_member.value for enum_member in impl_chars if enum_member not in char_class_map) + ) + + return ServiceComparison( + spec_table=spec_table, + service_name=service_name, + missing_in_implementation=missing, + extra_in_implementation=extra, + requirement_mismatches=tuple(sorted(req_mismatches)), + unimplemented_characteristic_classes=unimplemented_classes, + ) + + +def _iter_spec_files(spec_dir: Path, pattern: str) -> list[Path]: + return sorted(path for path in spec_dir.glob(pattern) if path.is_file()) + + +def main(spec_dir: Path, pattern: str, verbose: bool) -> int: + """Run service characteristic validation. + + Args: + spec_dir: Directory containing extracted spec files + pattern: Glob pattern for spec files + verbose: Print per-file parse skips and matched details + + Returns: + Process exit code + """ + spec_files = _iter_spec_files(spec_dir, pattern) + if not spec_files: + print(f"No spec files found in {spec_dir} matching pattern '{pattern}'.") + return 0 + + service_chars = _build_service_characteristics_map() + char_class_map = get_characteristic_class_map() + + parsed_tables: list[SpecTable] = [] + skipped_files: list[Path] = [] + for file_path in spec_files: + parsed = _parse_spec_table(file_path) + if parsed is None: + skipped_files.append(file_path) + continue + parsed_tables.append(parsed) + + comparisons: list[ServiceComparison] = [] + unmatched_specs: list[SpecTable] = [] + + for spec_table in parsed_tables: + matched_service = _match_service(spec_table, service_chars) + if matched_service is None: + unmatched_specs.append(spec_table) + continue + comparisons.append( + _compare_service( + spec_table=spec_table, + service_name=matched_service, + impl_chars=service_chars[matched_service], + char_class_map=cast(dict[CharacteristicName, object], char_class_map), + ) + ) + + failures = [comparison for comparison in comparisons if not comparison.is_ok] + + print("\nService Characteristics Validation") + print("=" * 72) + print(f"Spec files scanned : {len(spec_files)}") + print(f"Spec tables parsed : {len(parsed_tables)}") + print(f"Specs matched to service: {len(comparisons)}") + print(f"Unmatched specs : {len(unmatched_specs)}") + print(f"Parse skips : {len(skipped_files)}") + print(f"Service mismatches : {len(failures)}") + + if verbose and skipped_files: + print("\nSkipped (no parseable characteristics table):") + for file_path in skipped_files: + print(f" - {file_path.name}") + + if unmatched_specs: + print("\nUnmatched spec tables:") + for spec_table in unmatched_specs: + print(f" - {spec_table.file_path.name}: service='{spec_table.service_name}'") + + if failures: + print("\nMismatches:") + for comparison in failures: + print( + f"\n[{comparison.spec_table.file_path.name}] " + f"spec='{comparison.spec_table.service_name}' -> impl='{comparison.service_name.value}'" + ) + if comparison.missing_in_implementation: + print(" Missing in implementation:") + for item in comparison.missing_in_implementation: + print(f" - {item}") + if comparison.extra_in_implementation: + print(" Extra in implementation:") + for item in comparison.extra_in_implementation: + print(f" - {item}") + if comparison.requirement_mismatches: + print(" Requirement flag mismatches:") + for item in comparison.requirement_mismatches: + print(f" - {item}") + if comparison.unimplemented_characteristic_classes: + print(" Service references characteristics without class implementation:") + for item in comparison.unimplemented_characteristic_classes: + print(f" - {item}") + elif comparisons: + print("\nAll matched services are aligned with parsed spec characteristic tables.") + + return 1 if failures else 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Validate service characteristic definitions against extracted spec tables" + ) + parser.add_argument( + "--spec-dir", + type=Path, + default=_project_root / ".tmp", + help="Directory containing extracted spec text files (default: ./.tmp)", + ) + parser.add_argument( + "--pattern", + default="*_spec.txt", + help="Glob pattern used to find spec files (default: *_spec.txt)", + ) + parser.add_argument("--verbose", "-v", action="store_true", help="Print skipped files and extra details") + cli_args = parser.parse_args() + raise SystemExit(main(spec_dir=cli_args.spec_dir, pattern=cli_args.pattern, verbose=cli_args.verbose)) diff --git a/src/bluetooth_sig/gatt/characteristics/base.py b/src/bluetooth_sig/gatt/characteristics/base.py index 037c3f2f..d2f60eaa 100644 --- a/src/bluetooth_sig/gatt/characteristics/base.py +++ b/src/bluetooth_sig/gatt/characteristics/base.py @@ -34,6 +34,7 @@ from .context_lookup import ContextLookupMixin from .descriptor_mixin import DescriptorMixin from .pipeline import CharacteristicValidator, EncodePipeline, ParsePipeline +from ..resolver import NameNormalizer from .role_classifier import classify_role from .templates import CodingTemplate @@ -307,10 +308,11 @@ def role(self) -> CharacteristicRole: def display_name(self) -> str: """Get the display name for this characteristic. - Uses explicit _characteristic_name if set, otherwise falls back - to class name. + Uses the canonical SIG/YAML name for lookup fidelity, then strips + supported display markup for human-readable output. """ - return self._characteristic_name or self.__class__.__name__ + raw_name = self._characteristic_name or self.__class__.__name__ + return NameNormalizer.sanitize_display_markup(raw_name) @cached_property def gss_special_values(self) -> dict[int, str]: diff --git a/src/bluetooth_sig/gatt/characteristics/co2_concentration.py b/src/bluetooth_sig/gatt/characteristics/co2_concentration.py index 44710140..6bbcd3eb 100644 --- a/src/bluetooth_sig/gatt/characteristics/co2_concentration.py +++ b/src/bluetooth_sig/gatt/characteristics/co2_concentration.py @@ -17,7 +17,7 @@ class CO2ConcentrationCharacteristic(BaseCharacteristic[float]): with a resolution of 1 ppm. """ - # Explicit YAML/registry name (contains LaTeX markup not derivable heuristically) + # Exact YAML/registry name (contains LaTeX markup not derivable heuristically) _characteristic_name = "CO\\textsubscript{2} Concentration" _template = ConcentrationTemplate() diff --git a/src/bluetooth_sig/gatt/resolver.py b/src/bluetooth_sig/gatt/resolver.py index e287c797..03e03c25 100644 --- a/src/bluetooth_sig/gatt/resolver.py +++ b/src/bluetooth_sig/gatt/resolver.py @@ -55,6 +55,19 @@ def camel_case_to_display_name(name: str) -> str: # Insert space before trailing numbers return re.sub(r"([a-z])(\d+)", r"\1 \2", result) + @classmethod + def sanitize_display_markup(cls, name: str) -> str: + """Convert SIG LaTeX-style display markup into plain-text output. + + Args: + name: Raw SIG/YAML name or description fragment. + + Returns: + Plain-text display string with supported LaTeX markup removed. + + """ + return re.sub(r"\\textsubscript\{([^}]+)\}", r"\1", name) + @staticmethod def remove_suffix(name: str, suffix: str) -> str: """Remove suffix from name if present. diff --git a/tests/docs/test_generated_characteristics_docs.py b/tests/docs/test_generated_characteristics_docs.py new file mode 100644 index 00000000..08b9f55e --- /dev/null +++ b/tests/docs/test_generated_characteristics_docs.py @@ -0,0 +1,62 @@ +"""Tests for generated characteristics documentation and docs HTML post-processing.""" + +from __future__ import annotations + +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from types import ModuleType +from types import SimpleNamespace + +from scripts import generate_char_service_list + + +def _load_docs_conf_module() -> ModuleType: + """Load the Sphinx conf module without requiring package installation.""" + conf_path = Path(__file__).parent.parent.parent / "docs" / "source" / "conf.py" + spec = spec_from_file_location("docs_source_conf", conf_path) + if spec is None or spec.loader is None: + raise AssertionError(f"Unable to load docs conf module from {conf_path}") + + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_generated_characteristics_docs_include_registry_coverage() -> None: + """Generated reference docs should describe implementation coverage accurately.""" + markdown = generate_char_service_list.generate_markdown() + ( + implemented_characteristics, + total_characteristics, + missing_characteristics, + implemented_services, + total_services, + missing_services, + ) = generate_char_service_list.get_coverage_summary() + sig_sha, sig_commit_url = generate_char_service_list.get_sig_submodule_context() + + assert f"**{implemented_characteristics} of {total_characteristics}**" in markdown + assert f"**{implemented_services} of {total_services}**" in markdown + assert f"[`{sig_sha[:7]}`]({sig_commit_url})" in markdown + assert "## Not Yet Implemented Characteristics" in markdown + assert "## Not Yet Implemented Services" in markdown + assert missing_characteristics[0] in markdown + assert missing_services[0] in markdown + + +def test_add_external_link_security_preserves_external_href(tmp_path: Path) -> None: + """External link hardening must not rewrite the link destination.""" + conf = _load_docs_conf_module() + html_file = tmp_path / "index.html" + html_file.write_text( + 'Bluetooth SIG', + encoding="utf-8", + ) + + app = SimpleNamespace(outdir=str(tmp_path)) + conf.add_external_link_security(app, None) + + content = html_file.read_text(encoding="utf-8") + assert 'href="https://www.bluetooth.com/specifications/assigned-numbers/"' in content + assert 'rel="noopener noreferrer"' in content + assert "[%22" not in content \ No newline at end of file diff --git a/tests/gatt/characteristics/test_co2_concentration.py b/tests/gatt/characteristics/test_co2_concentration.py index f520fba1..bd4cd591 100644 --- a/tests/gatt/characteristics/test_co2_concentration.py +++ b/tests/gatt/characteristics/test_co2_concentration.py @@ -63,3 +63,11 @@ def test_co2_concentration_validation_limits(self, characteristic: CO2Concentrat characteristic.parse_value(unknown_data) assert exc_info.value.special_value.raw_value == 65535 assert "not known" in exc_info.value.special_value.meaning.lower() + + def test_co2_concentration_display_name_uses_plain_text( + self, characteristic: CO2ConcentrationCharacteristic + ) -> None: + """Test plain-text display name while preserving YAML lookup name.""" + assert characteristic.display_name == "CO2 Concentration" + assert characteristic._characteristic_name == "CO\\textsubscript{2} Concentration" + assert characteristic.uuid == "2B8C" diff --git a/tests/gatt/test_resolver.py b/tests/gatt/test_resolver.py index 939cc4c8..a1b59d27 100644 --- a/tests/gatt/test_resolver.py +++ b/tests/gatt/test_resolver.py @@ -44,6 +44,15 @@ def test_camel_case_to_display_name_edge_cases(self) -> None: assert NameNormalizer.camel_case_to_display_name("ABC") == "ABC" assert NameNormalizer.camel_case_to_display_name("") == "" + def test_sanitize_display_markup_subscripts(self) -> None: + """Test LaTeX subscript markup is converted to plain text.""" + assert NameNormalizer.sanitize_display_markup(r"CO\textsubscript{2} Concentration") == "CO2 Concentration" + assert NameNormalizer.sanitize_display_markup(r"Gravity (g\textsubscript{n})") == "Gravity (gn)" + + def test_sanitize_display_markup_leaves_plain_text_unchanged(self) -> None: + """Test display markup sanitisation is a no-op for plain text.""" + assert NameNormalizer.sanitize_display_markup("Battery Level") == "Battery Level" + def test_remove_suffix_characteristic(self) -> None: """Test removing 'Characteristic' suffix.""" assert NameNormalizer.remove_suffix("BatteryLevelCharacteristic", "Characteristic") == "BatteryLevel"