diff --git a/.addonmergeignore b/.addonmergeignore new file mode 100644 index 0000000..e69de29 diff --git a/docs/managementFromGit/updatingExistingAddons.md b/docs/managementFromGit/updatingExistingAddons.md index 4895903..f6ef139 100644 --- a/docs/managementFromGit/updatingExistingAddons.md +++ b/docs/managementFromGit/updatingExistingAddons.md @@ -1,102 +1,307 @@ # Integrating the add-on template in your add-on -## Pre-requisites +## Pre-requisites for initial setup -1. Create a repository, for example on GitHub, providing readme and license files. -1. Clone the repository: +1. Create a repository, for example on GitHub, providing README and LICENSE files. + +2. Clone the repository to your local computer: + + ``` + git clone https://github.com/{repoName}.git + ``` + +3. Go to the folder where your repository was cloned: + + ``` + cd {repoFolder} + ``` + +4. In this folder, create an `addon` subfolder and store the code for your add-on. + +5. Commit your initial changes: + + ``` + git add . + git commit -m "Initial commit" + ``` + +## Updating an Existing Add-on + +AddonTemplate evolves over time and regularly receives improvements, bug fixes, new GitHub workflows, and build system updates. + +You can merge the latest template changes into your repository instead of manually copying updated files. +This document explains both the recommended automated update procedure and the manual Git-based workflow. + +> [!NOTE] +> Updating from AddonTemplate only affects your project's infrastructure (build scripts, GitHub workflows, configuration files, etc.). +> It does **not** modify your add-on's source code. + +--- + +## Before you begin + +Before initiating any update workflow (automated or manual), please complete these safety checks: + +* **Check repository status**: + Ensure your working tree is clean. + + ``` + git status + ``` + +* **Commit or stash**: + Save or stash any pending local modifications. + +* **Use a dedicated branch**: + It is highly recommended to perform the update on a separate, dedicated branch to isolate changes. + +--- + +## Recommended Method: Automated Update Using the Companion Tool + +To streamline the synchronization process and avoid dealing with syntax errors or manual merge conflicts in infrastructure files, a companion utility script is included in AddonTemplate: `syncAddonWithTemplate.py`. + +This script automatically extracts your legacy project settings (such as the add-on name, summary, authors, and repository URL from `buildVars.py`) and merges them cleanly into the newly generated `pyproject.toml` file, while safely preserving empty values if certain metadata is not set. + +This automation ensures a seamless transition to the new template infrastructure without losing your original configuration. + +The script automatically supports updating two types of legacy add-ons: + +* **Legacy Structure (Dictionary-based without pyproject.toml):** + For older add-ons where `addon_info` was defined as a standard dictionary, the tool automatically migrates the metadata to the modern `AddonInfo` object structure. + It generates a new, fully populated `pyproject.toml` file matching the latest template standards, and synchronizes all infrastructure files. + +* **Modern Structure (AddonInfo-based):** + For newer add-ons that already use the `AddonInfo` object but need upstream template updates, the tool checks for any missing metadata keys in `buildVars.py` to insert them. + It safely updates `pyproject.toml` dependencies and versions while preserving your custom configuration rules for tools like `pyright` and `ruff`. + +### Prerequisites + +Before running the tool, ensure your system meets the following requirements: + +* **Python**: + Version **3.13** or newer must be installed (matching the template's required Python version). + +* **Git**: + Git must be installed and available in your system `PATH`. + +* For add-ons without a pyproject.toml file, **Dependency Management (tomlkit)**: + Because the automated script relies on the third-party `tomlkit` library to safely parse and merge configurations, it must be installed in your global Python environment before execution. + This is necessary because legacy add-on repositories do not include this dependency yet, and it is not yet present by default in the template's stable `master` branch. + + To install or update `tomlkit` globally, run the following command in your terminal: - ```sh - git clone https://github.com/{repoName}.git ``` + python -m pip install -U tomlkit + ``` + + *(Note: Windows users using the standard Python launcher can replace `python` with `py` if needed: `py -m pip install -U tomlkit`)* + +### Running the automated tool + +The script is highly flexible and supports two execution modes: + +1. **Standard Mode (No arguments):** + Run the script directly from the root of your repository or from any of its subdirectories. + It will automatically locate the project root by searching for `buildVars.py`. + + ``` + uv run python syncAddonWithTemplate.py -ad . + ``` + +2. **Target Directory Mode (With argument):** + Run the script from any working directory by supplying the optional `addonDir` path (relative or absolute) pointing to the add-on repository you wish to update. + + ``` + uv run python syncAddonWithTemplate.py -ad ../MyAddon + ``` + +> [!NOTE] +> Before applying any modifications, the script creates an untracked backup directory located next to the add-on folder named `_bak_`. +> This directory contains a copy of the entire project before the update, allowing you to restore the previous state manually if necessary. + +Once the update has completed, verify that the add-on still builds correctly: + +``` +uv sync +uv run scons +``` + +If everything builds successfully, remove the `_bak_` directory, stage and commit the updated infrastructure: + +``` +git clean -f +git add . +git commit -m "chore: sync infrastructure with AddonTemplate" +``` + +### Using the Update Tool via Command Line + +The `syncAddonWithTemplate.py` script provides a non-destructive industrial update engine to align your local add-on repository layout with the latest structure of the official NVDA `AddonTemplate`. + +You can execute the script with various command-line arguments to customize the update workflow. + +#### Available Options + +| Short Flag | Long Argument | Description | Default Value | +| :--- | :--- | :--- | :--- | +| `-ad` | `--addon-dir` | Path to the root directory of the local add-on you want to update. If not specified, the script automatically walks up from your current directory to find `buildVars.py`. | Current working directory | +| `-td` | `--template-dir` | Path to a local clone/directory of the NVDA `AddonTemplate`. When provided, the tool skips fetching the template via Git and synchronizes directly using this local reference. | None (clones from GitHub) | +| `-dr` | `--dry-run` | Simulates the execution. It analyzes structure, logs planned changes, and builds reports without writing or modifying any file on disk. | Disabled | +| `-s` | `--skip-backup` | Disables the automatic creation of a timestamped backup directory (e.g., `addonName_bak_YYYYMMDD_HHMMSS`) before processing updates. | Disabled (Backup is created) | +| `-h` | `--help` | Displays the default automated help menu listing all available parameters. | N/A | + +#### Customizing Exclusions with `.addonmergeignore` + +Rather than modifying the `syncAddonWithTemplate.py` core source code or changing its internal `PROTECTED_ELEMENTS` array, the update tool includes a robust file-exclusion system driven by a local file named `.addonmergeignore`. + +This architectural design allows developers to cleanly decouple their project-specific freeze preferences from the update engine machinery. + +##### How to Use `.addonmergeignore` + +To declare custom exceptions, create a plain text file named `.addonmergeignore` and place it directly **at the root of your target add-on repository**. -1. In the folder where your add-on repository is cloned, create an `addon` submolder and store the code for your add-on. +* Inside this file, list the names of the files or folders you want the tool to skip during synchronization. -1. Go to the folder where your repository was cloned: +* You can write one pattern per line. + Empty lines and lines starting with `#` are automatically treated as comments and ignored. - ```sh - cd {repoFolder} +For instance, if you wish to prevent the synchronization process from overwriting your custom execution scripts or your exclusion mapping file itself, simply add them to the file: + +``` +# Freeze the synchronization script version +syncAddonWithTemplate.py +# Protect your local merge settings file from being replaced +.addonmergeignore +``` + +##### Crucial Requirements & Design Constraints + +1. **Case-Insensitivity:** + The update tool evaluates exclusions using a standardized, case-insensitive matching algorithm. + This ensures maximum cross-platform reliability (especially between Windows and Unix-like environments). + Since the script automatically normalizes all inputs to lowercase during execution, **you can write your rules using any casing you prefer** (e.g., `UpdateAddonFromTemplate.py` or `updateaddonfromtemplate.py` will both work perfectly). + +2. **File Location Requirement:** + The update engine always loads custom exclusions from the target add-on's root folder being updated. + Therefore, **the `.addonmergeignore` file must always reside inside the destination add-on directory**, even if you are executing the `syncAddonWithTemplate.py` script from a completely different directory or an external workspace. + +#### Usage Examples + +Depending on your workflow, the script can be executed either directly from within your add-on repository or from an external directory. + +##### 1. Standard Automatic Update + +Downloads the latest remote template, creates a safety backup of your repository, and non-destructively synchronizes the machinery files. + +* **Syntax A (Script inside the add-on repository):** + + ``` + uv run python syncAddonWithTemplate.py -ad . ``` -1. Commit your changes: +* **Syntax B (Script outside the add-on repository):** - ```sh - git add . - git commit -m "Initial commit" ``` + uv run python /path/to/syncAddonWithTemplate.py -ad /path/to/my-nvda-addon + ``` + +##### 2. Updating from a Local Template Cache (Offline/Development) -1. Add the template as a remote: +Useful when testing local modifications applied to the `AddonTemplate` or when working without an active internet connection. - ```sh - git remote add template https://github.com/nvaccess/addonTemplate.git +* **Syntax A (Script inside the add-on repository):** + + ``` + uv run python syncAddonWithTemplate.py -ad . -td /path/to/local/AddonTemplate ``` -1. Fetch the add-on template: +* **Syntax B (Script outside the add-on repository):** - ```sh - git fetch template + ``` + uv run python /path/to/syncAddonWithTemplate.py -ad /path/to/my-nvda-addon -td /path/to/local/AddonTemplate ``` -## Updating an Existing Add-on +##### 3. Simulating Changes Safely (Dry Run) -As AddonTemplate evolves, it receives improvements, bug fixes, new GitHub workflows, and build system updates. +Analyzes structural layouts, evaluates configurations, reads the `.addonmergeignore` directives, and builds reports without writing anything to disk. -You can merge the latest template changes into your repository instead of manually copying updated files. +* **Syntax A (Script inside the add-on repository):** -This document explains the recommended update procedure. + ``` + uv run python syncAddonWithTemplate.py -ad . --dry-run + ``` -> [!NOTE] -> Updating from AddonTemplate only affects your project's infrastructure (build scripts, GitHub workflows, configuration files, etc.). It does **not** modify your add-on's source code. +* **Syntax B (Script outside the add-on repository):** -## Before you begin + ``` + uv run python /path/to/syncAddonWithTemplate.py --dry-run -ad /path/to/my-nvda-addon + ``` -Before updating your repository: +##### 4. Speeding Up with Backup Omission -- Ensure your working tree is clean. +Target a project repository while skipping the automated safety backup creation phase to speed up execution. + +* **Syntax A (Script inside the add-on repository):** - ```sh - git status ``` + uv run python syncAddonWithTemplate.py -ad . --skip-backup + ``` + +* **Syntax B (Script outside the add-on repository):** -- Commit or stash any pending changes. + ``` + uv run python /path/to/syncAddonWithTemplate.py -ad /path/to/my-nvda-addon --skip-backup + ``` -- It is recommended to perform the update on a dedicated branch. +##### 5. Run without Installation (`--with` option) -If anything goes wrong before the merge commit is created, if you haven't passed the `--squash- flag, you can safely cancel the operation using: +If you wish to execute the synchronization script directly without installing its mandatory dependencies (like `tomlkit`) into your current environment beforehand, you can request `uv` to fetch and expose the packages temporarily during the command lifetime by using the `--with` flag: -```sh -git merge --abort +``` +uv run --with tomlkit python syncAddonWithTemplate.py -ad . ``` -## Adding the template repository +--- -If you have not already done so, add AddonTemplate as a remote: +## Alternative Method: Manual Update Using Git Merge + +If you prefer not to use the automated tool, you can manually merge the latest version of AddonTemplate into your repository. + +### Fetching the add-on template repository + +1. If you haven't done it yet, from your add-on repository, add the addonTemplate as a remote. -```sh -git remote add template https://github.com/nvaccess/AddonTemplate.git +``` +git remote add template https://github.com/nvaccess/addonTemplate.git ``` -Then fetch the latest changes: +2. Fetch the template: -```sh +``` git fetch template ``` -## Merging the latest template +### Merging the latest template Merge the latest version of AddonTemplate: -```sh +``` git merge template/master --allow-unrelated-histories --squash ``` -The `--allow-unrelated-histories` option is required because your add-on repository and AddonTemplate do not share a common Git history. +* **Why `--allow-unrelated-histories`?** + This option is required because your add-on repository and AddonTemplate do not share a common Git history. -The `--squash` flags will add changes from the template as a unique commit, instead of several ones, what may be useful to keep a cleaner history on your repository. +* **Why `--squash`?** + This option stages all changes from the template as a single uncommitted change, helping keep your repository history cleaner. + It compiles the template updates into a unique commit, which is useful to keep a cleaner history on your repository. At this stage, Git may report merge conflicts. - This is completely normal. +--- + ## Understanding merge conflicts During the merge, Git attempts to combine the contents of both repositories automatically. @@ -106,27 +311,26 @@ When Git cannot determine which version should be kept, it reports a merge confl A conflict does **not** mean that something went wrong. It simply means that some files require manual review. -## Resolving the merge +### Resolving the merge -### Using the restore command +#### Using the restore command The `restore` command can be used to update files on your working directory, i.e., the folder where your add-on repository was cloned. -The `--source` flag is used to determine where files to be restored can be found. +The `--source` option is used to determine where files to be restored can be found. -### Keep your add-on documentation +#### Keep your add-on documentation Your add-on documentation should not be replaced by the template. To keep your `.md` files from your add-on repository, ensuring they aren't replaced with files from the template, you can use the following command: -```sh +``` git restore *.md --source=HEAD ``` -### Remove the template documentation +#### Remove the template documentation The `docs/` directory belongs to AddonTemplate itself. - It is not intended to become part of your add-on repository. Remove it: @@ -137,73 +341,76 @@ git rm -r docs Or use the restore command: -```sh +``` git restore docs --source=HEAD ``` -### Resolve buildVars.py +#### Resolve `buildVars.py` `buildVars.py` usually contains merge conflicts because it includes both: -- information specific to your add-on; -- variables introduced by newer versions of AddonTemplate. +* information specific to your add-on; +* variables introduced by newer versions of AddonTemplate. Review the file carefully. In general: -- keep your add-on metadata; -- preserve your version number; -- keep your custom settings; -- add any new variables introduced by the template. +* keep your add-on metadata; +* preserve your version number; +* keep your custom settings; +* add any new variables introduced by the template. -### Resolve pyproject.toml +#### Resolve `pyproject.toml` `pyproject.toml` is another file that commonly requires manual review. Keep your project-specific configuration while incorporating any new settings required by the updated template. -### Other files +#### Other files -For most remaining files, the version provided by AddonTemplate is generally the correct one. +For most remaining infrastructure files, the version provided by AddonTemplate is generally the correct one. Typical examples include: -- `.github/` -- `.gitignore` -- `manifest.ini.tpl` -- `manifest-translated.ini.tpl` -- `site_scons/` -- `sconstruct` +* `.github/` +* `.gitignore` +* `manifest.ini.tpl` +* `manifest-translated.ini.tpl` +* `site_scons/` +* `sconstruct` Review any conflicts if necessary before completing the merge. -## Completing the merge +--- + +### Completing the merge Once all conflicts have been resolved, check if the add-on can be built properly: -```sh -uv sync # Update dependencies -uv run scons # Build the add-on +``` +uv sync +uv run scons ``` +If everything builds successfully, stage the modified files: -If all is right, stage the modified files: - -```sh +``` git add . ``` Then create the merge commit: -```sh -git commit ``` +git commit -m "chore: sync infrastructure with AddonTemplate" +``` + +--- -## Summary +## Summary of File Actions | File or directory | Recommended action | -|-------------------|--------------------| +| :--- | :--- | | `README.md` | Keep the add-on version | | `CHANGELOG.md` | Keep the add-on version | | `docs/` | Remove | @@ -211,6 +418,8 @@ git commit | `pyproject.toml` | Merge manually | | Other template files | Usually accept the template version | +--- + ## Troubleshooting ### I don't understand a merge conflict @@ -221,27 +430,45 @@ Most conflicts occur in `buildVars.py` and `pyproject.toml`. Review the conflicting sections carefully and combine the changes from both versions. +If you are unsure whether a change comes from your add-on or from AddonTemplate, compare the conflicting section with the latest version of AddonTemplate before resolving it. + ### I want to cancel the update -If you have not yet committed the merge, and you haven't passed the `--squash` flag to `git merge`, you can restore your repository to its previous state: +#### If using the Automated update: + +Since the automated script creates an untracked timestamped full copy backup directory named `_bak_` before modifying any infrastructure files, you can restore your previous state manually from that folder if you decide not to keep the update. + +If you have already staged some changes, you can also discard them using: -```sh -git merge --abort +``` +git restore . --staged ``` -If you passed the `--squash` flag, `git merge --abort` won't work. -In this case, you can use the restore command: +Then restore your working tree: -```sh -git restore . --staged # Discard changes added to the staging area (after using `git add .`) ``` +git restore . --source=HEAD +``` + +#### If using the Manual update: + +If you have not yet committed the merge and **did not** use the `--squash` option, you can cancel it with: -```sh -git restore . --source=HEAD # Restores the working directory to the last commit made in your add-on repository ``` +git merge --abort +``` + +If you performed a squash merge, `git merge --abort` is no longer available because no merge state is recorded in Git. + +In this case, restore your repository manually with: -If you committed changes, you can use: +``` +git restore . --staged +git restore . --source=HEAD +``` -```sh +If you have already committed the update and want to return to the previous state, you can reset your branch: + +``` git reset --hard {cleanBranch} ``` diff --git a/pyproject.toml b/pyproject.toml index 8766194..52cd0ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,10 +29,11 @@ Repository = "https://github.com/nvaccess/addonTemplate" # PEP 735 dependency groups. `uv sync` installs the `dev` group by default, which # pulls in every tool needed to build, translate, and lint the add-on. [dependency-groups] -# Build add-on +# Build add-on & repository synchronization machinery build = [ "scons==4.10.1", "Markdown==3.10", + "tomlkit==0.15.0", ] # Translations management l10n = [ @@ -48,7 +49,7 @@ lint = [ "uv==0.11.15", "ruff==0.14.5", "prek==0.4.8", - "pyright[nodejs]==1.1.407", + "pyright[nodejs]==1.1.411", ] dev = [ { include-group = "build" }, @@ -80,6 +81,7 @@ exclude = [ "__pycache__", ".venv", "buildVars.py", + "syncAddonWithTemplate.py", ] [tool.ruff.format] @@ -118,6 +120,7 @@ exclude = [ ".venv", "site_scons", ".github/scripts", + "syncAddonWithTemplate.py", # When excluding concrete paths relative to a directory, # not matching multiple folders by name e.g. `__pycache__`, # paths are relative to the configuration file. @@ -226,3 +229,6 @@ reportMissingTypeStubs = false # Bad rules # These are sorted alphabetically and should be enabled and moved to compliant rules section when resolved. + +[tool.setuptools] +py-modules = [] diff --git a/syncAddonWithTemplate.py b/syncAddonWithTemplate.py new file mode 100644 index 0000000..edc07e0 --- /dev/null +++ b/syncAddonWithTemplate.py @@ -0,0 +1,700 @@ +# Copyright (C) 2026 NV Access Limited, Abdel +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. + +from collections.abc import MutableMapping, MutableSequence +import argparse +import ast +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +from datetime import datetime +from pathlib import Path +from typing import Any, cast + +import tomlkit + +logger = logging.getLogger("syncAddon") + + +def buildArgParser() -> argparse.ArgumentParser: + """Build and configure the command-line argument parser. + + :return: Configured ArgumentParser object. + """ + parser = argparse.ArgumentParser( + description="Non-destructive industrial update tool for NVDA Add-ons.", + ) + parser.add_argument( + "-ad", "--addon-dir", + dest="addonDir", + default=None, + help="Path to the root directory of the add-on to update (defaults to current directory).", + ) + parser.add_argument( + "-td", "--template-dir", + dest="templateDir", + default=None, + help="Path to a local directory containing the NVDA AddonTemplate to use instead of fetching it via Git.", + ) + parser.add_argument( + "-dr", "--dry-run", + dest="dryRun", + action="store_true", + help="Simulate execution without modifying any files.", + ) + parser.add_argument( + "-s", "--skip-backup", + dest="skipBackup", + action="store_true", + help="Disable safety automatic project backup.", + ) + return parser + + +def parseAstDict(node: ast.Dict) -> dict[str, Any]: + """Extract key-value pairs from an AST Dict node. + + :param node: The ast.Dict node to parse. + :return: A dictionary containing the extracted keys and values. + """ + extracted: dict[str, Any] = {} + for keyNode, valNode in zip(node.keys, node.values): + if keyNode is None: + continue + key = getattr(keyNode, "value", None) + if isinstance(valNode, ast.Call) and getattr(valNode.func, "id", None) == "_": + valNode = valNode.args[0] + val = getattr(valNode, "value", None) + if key is not None: + extracted[key] = val + return extracted + + +def parseAstKeywords(keywords: list[ast.keyword]) -> dict[str, Any]: + """Extract key-value pairs from a list of AST keyword nodes. + + :param keywords: The list of ast.keyword nodes to parse. + :return: A dictionary containing the extracted keys and values. + """ + extracted: dict[str, Any] = {} + for keyword in keywords: + key = keyword.arg + valNode = keyword.value + if isinstance(valNode, ast.Call) and getattr(valNode.func, "id", None) == "_": + valNode = valNode.args[0] + val = getattr(valNode, "value", None) + if key is not None: + extracted[key] = val + return extracted + + +def usesOsModule(node: ast.AST) -> bool: + """Check recursively if an AST node contains an actual reference to the 'os' module. + + :param node: The AST node to inspect. + :return: True if the node references 'os', False otherwise. + """ + for child in ast.walk(node): + if isinstance(child, ast.Name) and child.id == "os": + return True + return False + + +def formatAuthorList(rawAuthors: str) -> tomlkit.items.Array: + """Convert a comma-separated string of authors into a tomlkit multiline array. + + :param rawAuthors: The raw authors string (e.g., "Author Name , Another"). + :return: A tomlkit.items.Array object containing inline tables. + """ + authorsList = tomlkit.array() + authorsList.multiline(True) + + parts = [p.strip() for p in rawAuthors.split(",") if p.strip()] + + for part in parts: + m = re.match(r"^(.*?)\s*<(.*?)>$", part) + if m: + t = tomlkit.inline_table() + t.update({"name": m.group(1).strip(), "email": m.group(2).strip()}) + authorsList.append(t) + elif part: + t = tomlkit.inline_table() + t.update({"name": part, "email": ""}) + authorsList.append(t) + + return authorsList + + +def fixTomlIndentation(tomlText: str) -> str: + """Replace leading 4-space indentations with a tab inside maintainers/authors array blocks. + + :param tomlText: The raw TOML string generated by tomlkit. + :return: The TOML string with strictly enforced tab indentation. + """ + lines = tomlText.splitlines(keepends=True) + fixedLines = [] + inTargetArray = False + + for line in lines: + stripped = line.strip() + if re.match(r"^(?:maintainers|authors)\s*=\s*\[", stripped): + inTargetArray = True + fixedLines.append(line) + continue + + if inTargetArray: + if stripped == "]": + inTargetArray = False + fixedLines.append(line) + else: + fixedLines.append(re.sub(r"^ {4}", "\t", line)) + else: + fixedLines.append(line) + + return "".join(fixedLines) + + +def createPyprojectFromTemplate(templatePath: Path, metadata: dict[str, Any]) -> tomlkit.TOMLDocument: + """Create a new pyproject.toml document based on the official template, preserving tab indentation. + + :param templatePath: Path to the reference template pyproject.toml file. + :param metadata: Extracted metadata dictionary from legacy buildVars/manifest. + :return: A tomlkit TOMLDocument adhering to template tab formatting and populated with metadata. + """ + with templatePath.open("r", encoding="utf-8") as f: + doc = tomlkit.parse(f.read()) + + if "project" not in doc: + doc["project"] = tomlkit.table() + + projectSection = doc["project"] + + if "addon_name" in metadata and metadata["addon_name"]: + projectSection["name"] = metadata["addon_name"] + + if "addon_summary" in metadata and metadata["addon_summary"]: + projectSection["description"] = metadata["addon_summary"] + + addonUrl = str(metadata.get("addon_url", "")).strip() + if addonUrl: + if "urls" not in projectSection: + projectSection["urls"] = tomlkit.table() + projectSection["urls"]["Repository"] = addonUrl + + if "addon_author" in metadata and metadata["addon_author"]: + projectSection["maintainers"] = formatAuthorList(metadata["addon_author"]) + + return doc + + +def cleanupPlaceholderAuthors(projectSection: dict[str, Any]) -> None: + """Remove NV Access placeholder entries from authors and maintainers fields in-place. + + :param projectSection: The project table/dictionary within the TOML structure. + :return: None + """ + for field in ["authors", "maintainers"]: + if field in projectSection and isinstance(projectSection[field], (list, MutableSequence)): + tomlList = projectSection[field] + for i in range(len(tomlList) - 1, -1, -1): + item = tomlList[i] + name = item.get("name", "") if hasattr(item, "get") else "" + if not name and isinstance(item, dict): + name = item.get("name", "") + + if str(name).strip().lower() in ["nv access", "nvaccess"]: + tomlList.pop(i) + + +def getBasePackageName(s: str) -> str: + """Extract base package name robustly (handles !=, <=, ~=, @ URLs, markers, etc.). + + :param s: The raw dependency string. + :return: The normalized base package name in lowercase. + """ + m = re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]*", s.strip()) + return m.group(0).lower() if m else s.strip().lower() + + +def replaceAstRange(tplLines: list[str], replacements: dict[tuple[int, int], str]) -> None: + """Apply AST line replacements on a line-by-line list in reverse order. + + :param tplLines: The list of lines representing the file content. + :param replacements: A dictionary mapping (start_line, end_line) tuples to the replacing string. + :return: None + """ + sortedRanges = sorted(replacements.keys(), key=lambda x: x[0], reverse=True) + for start, end in sortedRanges: + tplLines[start:end] = [replacements[(start, end)]] + + +def deepMergeDicts(dictProj: dict[str, Any], dictTpl: dict[str, Any]) -> dict[str, Any]: + """Recursively merges dictTpl into dictProj. + + :param dictProj: The original dictionary to be updated. + :param dictTpl: The template dictionary whose values will be merged into dictProj. + :return: The updated dictProj with merged values from dictTpl. + """ + for key, value in dictTpl.items(): + if key in dictProj: + projVal = dictProj[key] + if isinstance(projVal, MutableMapping) and isinstance(value, MutableMapping): + deepMergeDicts(projVal, value) + elif isinstance(projVal, MutableSequence) and isinstance(value, MutableSequence): + for item in value: + if item not in projVal: + projVal.append(item) + else: + pass + else: + dictProj[key] = value + return dictProj + + +def extractBuildvarsMetadata(filePath: str | Path) -> tuple[dict[str, Any], dict[str, tuple[ast.AST, str]]]: + """Extract metadata and raw assignment expressions along with AST nodes from buildVars.py safely. + + :param filePath: The path to the buildVars.py file. + :return: A tuple containing two dictionaries: metadata and globalVars mapping var_name -> (ast_node, unparsed_expr). + """ + p = Path(filePath) + if not p.exists(): + return {}, {} + + with p.open("r", encoding="utf-8") as f: + try: + tree = ast.parse(f.read()) + except SyntaxError as e: + logger.error("Syntax error while reading %s: %s", p, e) + return {}, {} + + metadata: dict[str, Any] = {} + globalVars: dict[str, tuple[ast.AST, str]] = {} + topLevelVars: set[str] = { + "pythonSources", + "excludedFiles", + "baseLanguage", + "markdownExtensions", + "brailleTables", + "symbolDictionaries", + "speechDictionaries", + } + + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if not isinstance(target, ast.Name): + continue + varName = target.id + + if varName == "addon_info": + if isinstance(node.value, ast.Dict): + metadata.update(parseAstDict(node.value)) + elif isinstance(node.value, ast.Call) and getattr(node.value.func, "id", None) == "AddonInfo": + metadata.update(parseAstKeywords(node.value.keywords)) + elif varName in topLevelVars: + globalVars[varName] = (node.value, ast.unparse(node.value)) + elif isinstance(node, ast.AnnAssign): + if isinstance(node.target, ast.Name) and node.target.id in topLevelVars: + if node.value is not None: + globalVars[node.target.id] = (node.value, ast.unparse(node.value)) + + return metadata, globalVars + + +def mergePyprojectToml(projPath: str | Path, tplPath: str | Path, metadata: dict[str, Any], dryRun: bool = False) -> str: + """Merge template pyproject.toml configuration into the developer's file. + + :param projPath: Path to the existing pyproject.toml file. + :param tplPath: Path to the template pyproject.toml file. + :param metadata: Dictionary containing legacy metadata values from buildVars.py. + :param dryRun: If True, simulate the merge without writing changes to disk. + :return: A string indicating the result of the merge operation. + """ + pTpl = Path(tplPath) + pProj = Path(projPath) + + if not pTpl.exists(): + return "skipped (no template found)" + + if not pProj.exists(): + try: + projData = createPyprojectFromTemplate(pTpl, metadata) + if not dryRun: + tomlOutput = fixTomlIndentation(tomlkit.dumps(projData)) + pProj.parent.mkdir(parents=True, exist_ok=True) + with pProj.open("w", encoding="utf-8") as f: + f.write(tomlOutput) + logger.info("Created missing pyproject.toml at %s", pProj) + return "created from template" + except Exception as e: + logger.error("Failed to create pyproject.toml: %s", e) + return f"failed to create from template ({str(e)})" + + try: + with pProj.open("r", encoding="utf-8") as f: + projData = tomlkit.parse(f.read()) + with pTpl.open("r", encoding="utf-8") as f: + tplData = tomlkit.parse(f.read()) + + wasOriginallyNvaccess = False + if "project" in projData: + for field in ["authors", "maintainers"]: + if field in projData["project"] and isinstance(projData["project"][field], (list, MutableSequence)): + for item in projData["project"][field]: + name = item.get("name", "") if hasattr(item, "get") else "" + if not name and isinstance(item, dict): + name = item.get("name", "") + if str(name).strip().lower() in ["nv access", "nvaccess"]: + wasOriginallyNvaccess = True + break + + projDeps = [] + if "project" in projData and "dependencies" in projData["project"]: + projDeps = list(projData["project"]["dependencies"]) + del projData["project"]["dependencies"] + + mergedData = deepMergeDicts(cast(dict[str, Any], projData), cast(dict[str, Any], tplData)) + + if "project" in mergedData: + projectSection = mergedData["project"] + + if not wasOriginallyNvaccess: + cleanupPlaceholderAuthors(projectSection) + + # Filter and merge custom dependencies not managed by PEP 735 dependency groups + if isinstance(projectSection.get("dependencies"), (list, MutableSequence)): + tplDeps = projectSection["dependencies"] + tplBases = {getBasePackageName(d) for d in tplDeps} + + # Collect all tooling package names managed by template dependency groups + groupBases: set[str] = set() + if "dependency-groups" in mergedData and isinstance(mergedData["dependency-groups"], MutableMapping): + for grp in mergedData["dependency-groups"].values(): + if isinstance(grp, (list, MutableSequence)): + for item in grp: + if isinstance(item, str): + groupBases.add(getBasePackageName(item)) + + # Known template tooling packages across historical versions + # used to filter out deprecated tools (e.g., pre-commit) no longer present in the template. + legacyToolingBases = { + "pre-commit", + "scons", + "markdown", + "nh3", + "crowdin-api-client", + "lxml", + "mdx_truly_sane_lists", + "markdown-link-attr-modifier", + "mdx-gh-links", + "uv", + "ruff", + "prek", + "pyright", + } + + for dep in projDeps: + base = getBasePackageName(dep) + # Check if dependency is present in template dependencies or dependency groups + isInTemplate = base in tplBases or base in groupBases + # Check if dependency is a legacy tooling package dropped by the template + isDroppedTooling = base in legacyToolingBases and not isInTemplate + + # Only append to project.dependencies if not already present in template + # dependencies nor managed by PEP 735 dependency groups + if not isInTemplate and not isDroppedTooling: + tplDeps.append(dep) + + if not dryRun: + tomlOutput = fixTomlIndentation(tomlkit.dumps(cast(tomlkit.TOMLDocument, mergedData))) + with pProj.open("w", encoding="utf-8") as f: + f.write(tomlOutput) + return "merged intelligently (tomlkit)" + except Exception as e: + logger.error("Failed to merge pyproject.toml: %s", e) + return f"failed to merge ({str(e)})" + + +def mergeBuildvarsFile( + projPath: str | Path, + tplPath: str | Path, + metadata: dict[str, Any], + globalVars: dict[str, tuple[ast.AST, str]], + dryRun: bool = False, +) -> str: + """Merge template buildVars.py using precise AST range tracking to prevent multiline leaks. + + :param projPath: Path to the existing buildVars.py file. + :param tplPath: Path to the template buildVars.py file. + :param metadata: Dictionary containing metadata values to update. + :param globalVars: Dictionary containing global variables mapping var_name -> (ast_node, unparsed_expr). + :param dryRun: If True, simulate the merge without writing changes to disk. + :return: A string indicating the result of the merge operation. + """ + pTpl = Path(tplPath) + pProj = Path(projPath) + + if not pTpl.exists(): + return "failed (no template found)" + + with pTpl.open("r", encoding="utf-8") as f: + tplContent = f.read() + + try: + tree = ast.parse(tplContent) + except SyntaxError as e: + return f"failed (template syntax error: {e})" + + tplLines = tplContent.splitlines(keepends=True) + replacements: dict[tuple[int, int], str] = {} + requiresOsImport = False + + for node in ast.walk(tree): + if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "AddonInfo": + for kw in node.keywords: + if kw.arg in metadata: + key = kw.arg + val = metadata[key] + if val is None: + formattedVal = "None" + elif isinstance(val, str): + isTranslatable = key in ["addon_summary", "addon_description", "addon_changelog"] + formattedVal = f"_({val!r})" if isTranslatable else repr(val) + else: + formattedVal = str(val) + + if kw.end_lineno is not None: + lineContent = tplLines[kw.lineno - 1] + indent = lineContent[: len(lineContent) - len(lineContent.lstrip())] + replacements[(kw.lineno - 1, kw.end_lineno)] = f"{indent}{key}={formattedVal},\n" + + elif isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name) and target.id in globalVars: + key = target.id + valNode, valExpression = globalVars[key] + if usesOsModule(valNode): + requiresOsImport = True + if node.end_lineno is not None: + lineContent = tplLines[node.lineno - 1] + indent = lineContent[: len(lineContent) - len(lineContent.lstrip())] + replacements[(node.lineno - 1, node.end_lineno)] = ( + f"{indent}{key} = {valExpression}\n" + ) + + elif isinstance(node, ast.AnnAssign): + if isinstance(node.target, ast.Name) and node.target.id in globalVars: + key = node.target.id + valNode, valExpression = globalVars[key] + if usesOsModule(valNode): + requiresOsImport = True + if node.end_lineno is not None: + lineContent = tplLines[node.lineno - 1] + indent = lineContent[: len(lineContent) - len(lineContent.lstrip())] + typeStr = ast.unparse(node.annotation) + replacements[(node.lineno - 1, node.end_lineno)] = ( + f"{indent}{key}: {typeStr} = {valExpression}\n" + ) + + replaceAstRange(tplLines, replacements) + + if requiresOsImport: + hasOsImport = any("import os" in line for line in tplLines[:15]) + if not hasOsImport: + tplLines.insert(0, "import os\n") + + if not dryRun: + with pProj.open("w", encoding="utf-8") as f: + f.writelines(tplLines) + return "merged & structured (AST verified)" + + +def runSynchronization(tempDir: str, addonDir: str, dryRun: bool) -> None: + """Synchronizes template machinery files from the temporary workspace into the target directory. + + :param tempDir: Path to the local temporary directory containing the template files. + :param addonDir: Path to the target add-on root directory. + :param dryRun: If True, simulate the sync without writing changes to disk. + :return: None + """ + logger.info("Synchronizing template machinery files...") + protectedElements = { + "readme.md", + "changelog.md", + "addontemplate.egg-info", + "addon", + ".git", + "__pycache__", + ".venv", + "docs", + ".ruff_cache", + "tests", + } + + ignoreFilePath = os.path.join(addonDir, ".addonmergeignore") + if os.path.exists(ignoreFilePath): + logger.info("Reading local custom exclusions from .addonmergeignore...") + try: + with open(ignoreFilePath, "r", encoding="utf-8") as f: + for line in f: + cleanLine = line.strip().lower() + if cleanLine and not cleanLine.startswith("#"): + protectedElements.add(cleanLine) + except Exception as e: + logger.warning("Failed to parse .addonmergeignore (%s)", e) + + syncReport: list[str] = [] + + for item in os.listdir(tempDir): + if item.lower() in protectedElements: + syncReport.append(f"- **{item}**: skipped (protected scope)") + continue + + if item in ["buildVars.py", "pyproject.toml"]: + continue + + srcItem = os.path.join(tempDir, item) + dstItem = os.path.join(addonDir, item) + + try: + if os.path.isdir(srcItem): + if not dryRun: + os.makedirs(dstItem, exist_ok=True) + shutil.copytree(srcItem, dstItem, dirs_exist_ok=True) + syncReport.append(f"- **{item}/**: merged safely") + else: + if not dryRun: + shutil.copy2(srcItem, dstItem) + syncReport.append(f"- **{item}**: synchronized") + except Exception as e: + syncReport.append(f"- **{item}**: failed ({str(e)})") + + logger.info("Processing structural configuration merges...") + templateBuildvars = os.path.join(tempDir, "buildVars.py") + templatePyproject = os.path.join(tempDir, "pyproject.toml") + + oldBuildvars = os.path.join(addonDir, "buildVars.py") + oldPyproject = os.path.join(addonDir, "pyproject.toml") + + bvMeta, bvGlobals = extractBuildvarsMetadata(oldBuildvars) + addonName = bvMeta.get("addon_name", os.path.basename(addonDir)) + + bvStatus = mergeBuildvarsFile(oldBuildvars, templateBuildvars, bvMeta, bvGlobals, dryRun) + ppStatus = mergePyprojectToml(oldPyproject, templatePyproject, bvMeta, dryRun) + + logger.info("=" * 50) + logger.info("UPDATE REPORT") + logger.info("=" * 50) + logger.info("Add-on: %s", addonName) + logger.info("\nTemplate synchronization:") + for entry in syncReport: + logger.info(" %s", entry) + logger.info( + "\nConfiguration files:\n - **buildVars.py**: %s\n - **pyproject.toml**: %s", + bvStatus, + ppStatus, + ) + + +def main() -> None: + """Execute main CLI entry point for the NVDA Add-on update tool.""" + logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s") + + parser = buildArgParser() + args = parser.parse_args() + + addonDirInput = args.addonDir + if addonDirInput: + addonDir = os.path.abspath(addonDirInput) + else: + cwd = Path(os.getcwd()).resolve() + addonRoot = next((p for p in (cwd, *cwd.parents) if (p / "buildVars.py").exists()), None) + addonDir = str(addonRoot) if addonRoot is not None else str(cwd) + + logger.info("=== NVDA ADD-ON UPDATE TOOL ===") + logger.info("Target Directory: %s", addonDir) + + oldBuildvars = os.path.join(addonDir, "buildVars.py") + + if not os.path.exists(oldBuildvars): + logger.error("'%s' does not appear to be a valid NVDA Add-on (missing buildVars.py).", addonDir) + if sys.stdin.isatty(): + input("\nPress Enter to exit...") + sys.exit(1) + + logger.info("Phase 1: Analyzing existing project structure and metadata...") + bvMeta, _ = extractBuildvarsMetadata(oldBuildvars) + addonName = bvMeta.get("addon_name", os.path.basename(addonDir)) + logger.info("Target Add-on Identified: %s", addonName) + + if args.dryRun: + logger.info("RUNNING IN SIMULATION MODE (--dry-run). No files will be modified.") + + logger.info("Phase 2: Safety backup verification...") + if args.dryRun: + logger.info("Safety backup skipped (simulation mode active).") + elif args.skipBackup: + logger.info("Safety backup skipped (--skip-backup requested by user).") + else: + backupDir = f"{addonDir}_bak_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + logger.info("Creating safety automatic backup in: %s...", os.path.basename(backupDir)) + try: + shutil.copytree( + addonDir, + backupDir, + ignore=shutil.ignore_patterns(".git", "__pycache__", ".venv", "*_bak_*"), + ) + logger.info("Backup created successfully.") + except Exception as e: + logger.error("Critical: Backup failed (%s). Aborting update.", e) + if sys.stdin.isatty(): + input("\nPress Enter to exit...") + sys.exit(1) + + if args.templateDir: + templatePath = os.path.abspath(args.templateDir) + logger.info("Phase 3: Using local template directory: %s", templatePath) + if not os.path.exists(os.path.join(templatePath, "buildVars.py")): + logger.error("Provided template directory does not appear to be a valid NVDA AddonTemplate (missing buildVars.py).") + if sys.stdin.isatty(): + input("\nPress Enter to exit...") + sys.exit(1) + runSynchronization(templatePath, addonDir, args.dryRun) + else: + logger.info("Phase 3: Provisioning latest official NVDA AddonTemplate via Git...") + with tempfile.TemporaryDirectory() as tempDir: + logger.info("Cloning template into temporary workspace...") + templateUrl = "https://github.com/nvaccess/AddonTemplate.git" + + try: + subprocess.run( + ["git", "clone", "--depth", "1", templateUrl, tempDir], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + logger.info("Template cloned successfully.") + except (subprocess.CalledProcessError, FileNotFoundError) as e: + logger.error("Failed to execute git clone. Make sure Git is available in your PATH.") + if isinstance(e, subprocess.CalledProcessError) and e.stderr: + logger.error("Details: %s", e.stderr.decode("utf-8", errors="ignore")) + if sys.stdin.isatty(): + input("\nPress Enter to exit...") + sys.exit(1) + + runSynchronization(tempDir, addonDir, args.dryRun) + + if not args.dryRun: + logger.info("Project successfully updated. Workspace cleared.") + else: + logger.info("Simulation finished. Workspace cleared.") + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock index 4484ce7..8aa2402 100644 --- a/uv.lock +++ b/uv.lock @@ -10,6 +10,7 @@ source = { editable = "." } build = [ { name = "markdown" }, { name = "scons" }, + { name = "tomlkit" }, ] dev = [ { name = "crowdin-api-client" }, @@ -23,6 +24,7 @@ dev = [ { name = "pyright", extra = ["nodejs"] }, { name = "ruff" }, { name = "scons" }, + { name = "tomlkit" }, { name = "uv" }, ] l10n = [ @@ -46,6 +48,7 @@ lint = [ build = [ { name = "markdown", specifier = "==3.10" }, { name = "scons", specifier = "==4.10.1" }, + { name = "tomlkit", specifier = "==0.15.0" }, ] dev = [ { name = "crowdin-api-client", specifier = "==1.24.1" }, @@ -56,9 +59,10 @@ dev = [ { name = "mdx-truly-sane-lists", specifier = "==1.3" }, { name = "nh3", specifier = "==0.3.2" }, { name = "prek", specifier = "==0.4.8" }, - { name = "pyright", extras = ["nodejs"], specifier = "==1.1.407" }, + { name = "pyright", extras = ["nodejs"], specifier = "==1.1.411" }, { name = "ruff", specifier = "==0.14.5" }, { name = "scons", specifier = "==4.10.1" }, + { name = "tomlkit", specifier = "==0.15.0" }, { name = "uv", specifier = "==0.11.15" }, ] l10n = [ @@ -71,7 +75,7 @@ l10n = [ ] lint = [ { name = "prek", specifier = "==0.4.8" }, - { name = "pyright", extras = ["nodejs"], specifier = "==1.1.407" }, + { name = "pyright", extras = ["nodejs"], specifier = "==1.1.411" }, { name = "ruff", specifier = "==0.14.5" }, { name = "uv", specifier = "==0.11.15" }, ] @@ -289,15 +293,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.407" +version = "1.1.411" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/1b/0aa08ee42948b61745ac5b5b5ccaec4669e8884b53d31c8ec20b2fcd6b6f/pyright-1.1.407.tar.gz", hash = "sha256:099674dba5c10489832d4a4b2d302636152a9a42d317986c38474c76fe562262", size = 4122872, upload-time = "2025-10-24T23:17:15.145Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/ab/265f7dc69d28113ebba19092e57b075f41543b2ed048429c5f56e2b88eac/pyright-1.1.411.tar.gz", hash = "sha256:d885a0551f2e763b089a02702174e7f4ba77548cddabc972ab86d1f7f1b0f998", size = 4112861, upload-time = "2026-06-25T02:14:06.37Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/93/b69052907d032b00c40cb656d21438ec00b3a471733de137a3f65a49a0a0/pyright-1.1.407-py3-none-any.whl", hash = "sha256:6dd419f54fcc13f03b52285796d65e639786373f433e243f8b94cf93a7444d21", size = 5997008, upload-time = "2025-10-24T23:17:13.159Z" }, + { url = "https://files.pythonhosted.org/packages/0a/49/385be530a6a5b78d1cbcd5c2e38debc8959a2fc6bdb716f4e581002979fc/pyright-1.1.411-py3-none-any.whl", hash = "sha256:dc7c72a8e2700c55baa127554040e067041ea53ccfd50bf96308cc4291c7d5d9", size = 6181526, upload-time = "2026-06-25T02:14:04.691Z" }, ] [package.optional-dependencies] @@ -355,6 +359,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/bf/931fb9fbb87234c32b8b1b1c15fba23472a10777c12043336675633809a7/scons-4.10.1-py3-none-any.whl", hash = "sha256:bd9d1c52f908d874eba92a8c0c0a8dcf2ed9f3b88ab956d0fce1da479c4e7126", size = 4136069, upload-time = "2025-11-16T22:43:35.933Z" }, ] +[[package]] +name = "tomlkit" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0"