Use components.json instead of github repo as modules version cache

.devcontainer/build-devcontainer/Dockerfile

@@ -27,7 +27,7 @@ RUN rm -f /etc/apt/sources.list.d/yarn.list && \
     rm -rf /var/lib/apt/lists/*
 
 # Final stage to copy only the required files after installation
-FROM mcr.microsoft.com/devcontainers/base:2.1.8-debian12@sha256:17e6cc517b483d1108b333d4c34352b0a21617f0117052e9b259d47113a9dc37 AS final
+FROM mcr.microsoft.com/devcontainers/base:2.1.11-debian12@sha256:bb7b81b6e5be17b5267f92f4ffda534fea37dab1df97b5e86c1f9b91da5c0b5d AS final
 
 # Copy only the conda environment and site-packages from build stage
 COPY --from=build /opt/conda /opt/conda

.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
           - types-setuptools
           - pydantic
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.11.30
+    rev: 0.11.31
     hooks:
       - id: uv-lock
         priority: 20

nf_core/components/components_command.py

@@ -6,7 +6,7 @@
 
 import nf_core.utils
 from nf_core.modules.modules_json import ModulesJson
-from nf_core.modules.modules_repo import ModulesRepo
+from nf_core.modules.modules_repo import ModulesRepoType, get_modules_repo
 from nf_core.modules.modules_utils import scan_modules_dir
 
 from .components_utils import get_repo_info
@@ -34,7 +34,7 @@ def __init__(
         """
         self.component_type: str = component_type
         self.directory: Path = Path(directory)
-        self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress)
+        self.modules_repo = get_modules_repo(remote_url, branch, no_pull, hide_progress)
         self.hide_progress: bool = hide_progress
         self.no_prompts: bool = no_prompts or not nf_core.utils.is_interactive()
         self.repo_type: str | None = None
@@ -156,7 +156,7 @@ def components_from_repo(self, install_dir: str) -> list[str]:
         return scan_modules_dir(repo_dir)
 
     def install_component_files(
-        self, component_name: str, component_version: str, modules_repo: ModulesRepo, install_dir: str | Path
+        self, component_name: str, component_version: str, modules_repo: ModulesRepoType, install_dir: str | Path
     ) -> bool:
         """
         Installs a module/subworkflow into the given directory
@@ -206,11 +206,12 @@ def check_modules_structure(self) -> None:
             # If there are modules installed in the wrong location
             if len(wrong_location_modules) > 0:
                 log.info("The modules folder structure is outdated. Reinstalling modules.")
-                # Remove the local copy of the modules repository
-                log.info(f"Updating '{self.modules_repo.local_repo_dir}'")
-                self.modules_repo.setup_local_repo(
-                    self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress
-                )
+                # Remove the local copy of the modules repository (git backend only)
+                if self.modules_repo.local_repo_dir is not None:
+                    log.info(f"Updating '{self.modules_repo.local_repo_dir}'")
+                    self.modules_repo.setup_local_repo(  # type: ignore[union-attr]
+                        self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress
+                    )
                 # Move wrong modules to the right directory
                 for module in wrong_location_modules:
                     modules_dir = Path("modules").resolve()

nf_core/components/components_utils.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import re
 from pathlib import Path
@@ -8,7 +9,8 @@
 import ruamel.yaml
 
 import nf_core.utils
-from nf_core.modules.modules_repo import ModulesRepo
+from nf_core.modules.modules_repo import ModulesRepoType, get_modules_repo
+from nf_core.modules.registry_client import RegistryClient
 
 log = logging.getLogger(__name__)
 
@@ -24,6 +26,30 @@
 yaml.width = 4096
 
 
+def load_schema_from_repo(modules_repo: "ModulesRepoType", relative_path: str) -> dict:
+    """
+    Load a JSON schema shipped in the modules repository.
+
+    Reads from the local git clone when available, otherwise fetches the file
+    over HTTP (registry mode).
+
+    Raises:
+        LookupError: If the schema cannot be read or fetched.
+    """
+    if isinstance(modules_repo, RegistryClient):
+        try:
+            return json.loads(modules_repo.get_file_content(relative_path))
+        except requests.RequestException as e:
+            raise LookupError(f"Could not fetch '{relative_path}' from '{modules_repo.remote_url}': {e}") from e
+    if modules_repo.local_repo_dir is None:
+        raise LookupError(f"Local clone of the modules repo not found, cannot load '{relative_path}'")
+    try:
+        with open(Path(modules_repo.local_repo_dir, relative_path)) as fh:
+            return json.load(fh)
+    except FileNotFoundError as e:
+        raise LookupError(f"Could not find '{relative_path}' in '{modules_repo.local_repo_dir}'") from e
+
+
 def get_repo_info(directory: Path, use_prompt: bool | None = True) -> tuple[Path, str | None, str]:
     """
     Determine whether this is a pipeline repository or a clone of
@@ -96,7 +122,7 @@ def get_repo_info(directory: Path, use_prompt: bool | None = True) -> tuple[Path
 def prompt_component_version_sha(
     component_name: str,
     component_type: str,
-    modules_repo: "ModulesRepo",
+    modules_repo: "ModulesRepoType",
     installed_sha: str | None = None,
 ) -> str:
     """
@@ -193,7 +219,7 @@ def get_components_to_install(
                         component_name = list(component.keys())[0].lower()
                         branch = component[component_name].get("branch")
                         git_remote = component[component_name]["git_remote"]
-                        modules_repo = ModulesRepo(git_remote, branch=branch)
+                        modules_repo = get_modules_repo(git_remote, branch=branch)
                         current_comp_dict = subworkflows if component_name in subworkflows else modules
 
                         component_dict = {

nf_core/components/install.py

@@ -19,7 +19,7 @@
     NF_CORE_MODULES_NAME,
 )
 from nf_core.modules.modules_json import ModulesJson
-from nf_core.modules.modules_repo import ModulesRepo
+from nf_core.modules.modules_repo import get_modules_repo
 from nf_core.pipelines.containers_utils import try_generate_container_configs
 
 log = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ def __init__(
         skip_deps: bool = False,
     ):
         super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
-        self.current_remote = ModulesRepo(remote_url, branch)
+        self.current_remote = get_modules_repo(remote_url, branch)
         self.branch = branch
         self.force = force
         self.prompt = prompt
@@ -57,7 +57,7 @@ def install(self, component: str | dict[str, str], silent: bool = False) -> bool
             # Override modules_repo when the component to install is a dependency from a subworkflow.
             remote_url = component.get("git_remote", self.current_remote.remote_url)
             branch = component.get("branch", self.branch)
-            self.modules_repo = ModulesRepo(remote_url, branch)
+            self.modules_repo = get_modules_repo(remote_url, branch)
             component = component["name"]
 
         if self.current_remote is None:
@@ -232,7 +232,9 @@ def install_included_components(self, subworkflow_dir):
         self.modules_repo = ini_modules_repo
 
     def collect_and_verify_name(
-        self, component: str | None, modules_repo: "nf_core.modules.modules_repo.ModulesRepo"
+        self,
+        component: str | None,
+        modules_repo: "nf_core.modules.modules_repo.ModulesRepo | nf_core.modules.registry_client.RegistryClient",
     ) -> str:
         """
         Collect component name.

nf_core/components/list.py

@@ -7,7 +7,7 @@
 
 from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson, ModulesJsonModuleEntry
-from nf_core.modules.modules_repo import ModulesRepo
+from nf_core.modules.modules_repo import get_modules_repo
 
 log = logging.getLogger(__name__)
 
@@ -135,7 +135,7 @@ def pattern_msg(keywords: list[str]) -> str:
                             version_sha = component_entry["git_sha"]
                             try:
                                 # pass repo_name to get info on modules even outside nf-core/modules
-                                module = ModulesRepo(
+                                module = get_modules_repo(
                                     remote_url=repo_url,
                                     branch=component_entry["branch"],
                                 )

nf_core/components/update.py

@@ -15,7 +15,7 @@
 from nf_core.components.install import ComponentInstall
 from nf_core.components.remove import ComponentRemove
 from nf_core.modules.modules_json import ModulesJson
-from nf_core.modules.modules_repo import ModulesRepo
+from nf_core.modules.modules_repo import get_modules_repo
 from nf_core.pipelines.containers_utils import try_generate_container_configs
 from nf_core.utils import plural_es, plural_s, plural_y
 
@@ -41,7 +41,7 @@ def __init__(
         skip_deps=False,
     ):
         super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
-        self.current_remote = ModulesRepo(remote_url, branch)
+        self.current_remote = get_modules_repo(remote_url, branch)
         self.branch = branch
         self.force = force
         self.prompt = prompt
@@ -105,7 +105,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
             # Override modules_repo when the component to install is a dependency from a subworkflow.
             remote_url = component.get("git_remote", self.current_remote.remote_url)
             branch = component.get("branch", self.branch)
-            self.modules_repo = ModulesRepo(remote_url, branch)
+            self.modules_repo = get_modules_repo(remote_url, branch)
             component = component["name"]
 
         self.component = component
@@ -708,7 +708,7 @@ def get_all_components_info(self, branch=None):
         repo_objs_comps = []
         for (repo_url, branch), comps_shas in repos_and_branches.items():
             try:
-                modules_repo = ModulesRepo(remote_url=repo_url, branch=branch)
+                modules_repo = get_modules_repo(remote_url=repo_url, branch=branch)
             except LookupError as e:
                 log.warning(e)
                 log.info(f"Skipping {self.component_type} in '{repo_url}'")

nf_core/modules/lint/__init__.py

@@ -20,7 +20,7 @@
 
 import nf_core.modules.modules_utils
 import nf_core.utils
-from nf_core.components.components_utils import get_biotools_id, get_biotools_response, yaml
+from nf_core.components.components_utils import get_biotools_id, get_biotools_response, load_schema_from_repo, yaml
 from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult
 from nf_core.components.nfcore_component import NFCoreComponent
 from nf_core.pipelines.lint_utils import console, run_prettier_on_file
@@ -285,24 +285,20 @@ def lint_module(
 
     def load_meta_schema(self) -> Mapping[str, Any]:
         """
-        Load the meta.yml JSON schema from the local modules repository cache.
+        Load the meta.yml JSON schema from the modules repository (local clone or HTTP registry).
         The schema is cached in self.meta_schema to avoid reloading.
 
         Returns:
             dict: The meta.yml JSON schema
 
         Raises:
-            LookupError: If the local module cache is not found
+            LookupError: If the schema cannot be read or fetched
         """
         # Return cached schema if already loaded
         if self.meta_schema is not None:
             return self.meta_schema
 
-        if self.modules_repo.local_repo_dir is None:
-            raise LookupError("Local module cache not found")
-
-        with open(Path(self.modules_repo.local_repo_dir, "modules/meta-schema.json")) as fh:
-            self.meta_schema = json.load(fh)
+        self.meta_schema = load_schema_from_repo(self.modules_repo, "modules/meta-schema.json")
         return self.meta_schema
 
     def sort_meta_yml(self, meta_yml: dict) -> dict:

nf_core/modules/lint/environment_yml.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from pathlib import Path
 
@@ -7,6 +6,7 @@
 from rich.progress import Progress
 
 import nf_core.utils
+from nf_core.components.components_utils import load_schema_from_repo
 from nf_core.components.lint import ComponentLint
 from nf_core.components.nfcore_component import NFCoreComponent
 from nf_core.modules.modules_utils import module_uses_dockerfile
@@ -199,8 +199,7 @@ def lint_environment_yml(
     if env_yml:
         valid_env_yml = False
         try:
-            with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/environment-schema.json")) as fh:
-                schema = json.load(fh)
+            schema = load_schema_from_repo(module_lint_object.modules_repo, "modules/environment-schema.json")
             validators.validate(instance=env_yml, schema=schema)
             module.passed.append(
                 (

nf_core/modules/lint/module_changes.py

@@ -57,7 +57,7 @@ def module_changes(module_lint_object, module):
     module.branch = module_lint_object.modules_json.get_component_branch(
         "modules", module.component_name, module.repo_url, module.org
     )
-    modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=module.repo_url, branch=module.branch)
+    modules_repo = nf_core.modules.modules_repo.get_modules_repo(remote_url=module.repo_url, branch=module.branch)
 
     for f, same in modules_repo.component_files_identical(
         module.component_name, tempdir, module.git_sha, "modules"

nf_core/modules/lint/module_version.py

@@ -50,7 +50,7 @@ def module_version(module_lint_object: "nf_core.modules.lint.ModuleLint", module
         module.branch = module_lint_object.modules_json.get_component_branch(
             "modules", module.component_name, module.repo_url, module.org
         )
-        modules_repo = nf_core.modules.modules_repo.ModulesRepo(remote_url=module.repo_url, branch=module.branch)
+        modules_repo = nf_core.modules.modules_repo.get_modules_repo(remote_url=module.repo_url, branch=module.branch)
 
         module_git_log = list(modules_repo.get_component_git_log(module.component_name, "modules"))
         if version == module_git_log[0]["git_sha"]:

nf_core/modules/modules_json.py

@@ -16,7 +16,8 @@
 import nf_core.utils
 from nf_core.components.components_utils import get_components_to_install
 from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
-from nf_core.modules.modules_repo import ModulesRepo
+from nf_core.modules.modules_repo import ModulesRepo, get_modules_repo
+from nf_core.modules.registry_client import RegistryClient
 from nf_core.pipelines.lint_utils import dump_json_with_prettier
 
 from ..components.components_differ import ComponentsDiffer
@@ -138,7 +139,7 @@ def get_component_names_from_repo(
         """
         names = []
         for repo_url in repos:
-            modules_repo = ModulesRepo(repo_url)
+            modules_repo = get_modules_repo(repo_url)
             if modules_repo is None:
                 raise UserWarning(f"Could not find module repository for '{repo_url}' in '{directory}'")
             if modules_repo.repo_path is None:
@@ -180,7 +181,7 @@ def get_pipeline_module_repositories(
         renamed_dirs = {}
         # Check if there are any untracked repositories
 
-        dirs_not_covered = self.dir_tree_uncovered(directory, [Path(ModulesRepo(url).repo_path) for url in repos])
+        dirs_not_covered = self.dir_tree_uncovered(directory, [Path(get_modules_repo(url).repo_path) for url in repos])
         if len(dirs_not_covered) > 0:
             log.info(f"Found custom {component_type[:-1]} repositories when creating 'modules.json'")
             # Loop until all directories in the base directory are covered by a remote
@@ -212,7 +213,7 @@ def get_pipeline_module_repositories(
                         ).unsafe_ask()
 
                 # Verify that there is a directory corresponding the remote
-                nrepo_name = ModulesRepo(nrepo_remote).repo_path
+                nrepo_name = get_modules_repo(nrepo_remote).repo_path
                 if nrepo_name is None:
                     raise UserWarning(f"Could not find the repository name for '{nrepo_remote}'")
                 if not (directory / nrepo_name).exists():
@@ -300,7 +301,7 @@ def determine_branches_and_shas(
             (dict[str, dict[str, str]]): The module.json entries for the modules/subworkflows
                                          from the repository
         """
-        default_modules_repo = ModulesRepo(remote_url=remote_url)
+        default_modules_repo = get_modules_repo(remote_url=remote_url)
         if component_type == "modules":
             repo_path = self.modules_dir / install_dir
         elif component_type == "subworkflows":
@@ -370,7 +371,9 @@ def determine_branches_and_shas(
                             dead_components.append(component)
                         break
                     # Create a new modules repo with the selected branch, and retry find the sha
-                    modules_repo = ModulesRepo(remote_url=remote_url, branch=branch, no_pull=True, hide_progress=True)
+                    modules_repo = get_modules_repo(
+                        remote_url=remote_url, branch=branch, no_pull=True, hide_progress=True
+                    )
                 else:
                     found_sha = True
                     break
@@ -397,7 +400,7 @@ def find_correct_commit_sha(
         component_type: str,
         component_name: str | Path,
         component_path: str | Path,
-        modules_repo: ModulesRepo,
+        modules_repo: "ModulesRepo | RegistryClient",
     ) -> str | None:
         """
         Returns the SHA for the latest commit where the local files are identical to the remote files
@@ -598,7 +601,7 @@ def reinstall_repo(self, install_dir, remote_url, module_entries):
 
         for branch, modules in branches_and_mods.items():
             try:
-                modules_repo = ModulesRepo(remote_url=remote_url, branch=branch)
+                modules_repo = get_modules_repo(remote_url=remote_url, branch=branch)
             except LookupError as e:
                 log.error(e)
                 failed_to_install.extend(modules)
@@ -722,7 +725,7 @@ def load(self) -> None:
     def update(
         self,
         component_type: str,
-        modules_repo: ModulesRepo,
+        modules_repo: "ModulesRepo | RegistryClient",
         component_name: str,
         component_version: str,
         installed_by: list[str] | None,
@@ -1079,7 +1082,7 @@ def get_dependent_components(
         # Find all components that have an entry of install by of  a given component, recursively call this function for subworkflows
         for comp_type in component_types:
             for repo_url in self.modules_json["repos"]:
-                modules_repo = ModulesRepo(repo_url)
+                modules_repo = get_modules_repo(repo_url)
                 install_dir = modules_repo.repo_path
                 try:
                     for comp in self.modules_json["repos"][repo_url][comp_type][install_dir]:
@@ -1216,7 +1219,7 @@ def resolve_missing_from_modules_json(self, missing_from_modules_json, component
         def components_with_repos():
             for directory in missing_from_modules_json:
                 for repo_url in repos:
-                    modules_repo = ModulesRepo(repo_url)
+                    modules_repo = get_modules_repo(repo_url)
                     paths_in_directory = []
                     repo_url_path = Path(
                         self.modules_dir,