From 9ff61f52d201b3813c964fcddb9e99e24bcf440b Mon Sep 17 00:00:00 2001 From: nextstrain-bot <78992647+nextstrain-bot@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:21:16 +0000 Subject: [PATCH 1/2] git subrepo pull (merge) shared/vendored subrepo: subdir: "shared/vendored" merged: "6e82a95" upstream: origin: "https://github.com/nextstrain/shared" branch: "main" commit: "6e82a95" git-subrepo: version: "0.4.9" origin: "https://github.com/ingydotnet/git-subrepo" commit: "5e0f401" --- shared/vendored/.gitrepo | 4 +- shared/vendored/snakemake/remote_files.smk | 93 +++++++++++++++++++++- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/shared/vendored/.gitrepo b/shared/vendored/.gitrepo index e28f4cc..3f1a4b4 100644 --- a/shared/vendored/.gitrepo +++ b/shared/vendored/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/nextstrain/shared branch = main - commit = ef9afdb9387c9ae2a9e2b5081025c4af241a9b47 - parent = 2262103f1368f266b8a7391d8bf62d3db0af9b5c + commit = 6e82a95b4e1e1a9b2ae42bb9a63c458a0a0ac7c0 + parent = d69e5f61ceddb2d39d57a5dcb7c74e3264b09aba method = merge cmdver = 0.4.9 diff --git a/shared/vendored/snakemake/remote_files.smk b/shared/vendored/snakemake/remote_files.smk index 844f80e..5c0414d 100644 --- a/shared/vendored/snakemake/remote_files.smk +++ b/shared/vendored/snakemake/remote_files.smk @@ -7,8 +7,43 @@ The errors raised by storage plugins are often confusing. For instance, a HTTP underlying issue. S3 credentials errors are similarly confusing and we attempt to check these ourselves to improve UX here. """ - +import os +import socket from urllib.parse import urlparse +from snakemake.io import get_flag_value, AnnotatedString +from snakemake.logging import logger + + +def is_online(host="8.8.8.8", port=53, timeout=3) -> bool: + """ + Check if workflow has network connection. + Based on + Host: 8.8.8.8 (google-public-dns-a.google.com) + OpenPort: 53/tcp + Service: domain (DNS/TCP) + """ + try: + socket.setdefaulttimeout(timeout) + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) + return True + except socket.error as err: + logger.debug(f"Assuming no internet connection with error: {err}") + return False + + +# Determine whether workflow should try to fetch or revalidate remote input +# files based on prioritization order: +# 1. Environment variable `NEXTSTRAIN_REMOTE_FILES` +# 2. Per workflow config parameter `use_remote_files` +# 3. Whether the workflow has network connection +USE_REMOTE_FILES = True +if "NEXTSTRAIN_USE_REMOTE_FILES" in os.environ: + USE_REMOTE_FILES = bool(os.environ["NEXTSTRAIN_USE_REMOTE_FILES"]) +elif "use_remote_files" in config: + USE_REMOTE_FILES = bool(config["use_remote_files"]) +else: + USE_REMOTE_FILES = is_online() + # Keep a list of known public buckets, which we'll allow uncredentialled (unsigned) access to # We could make this config-definable in the future @@ -99,7 +134,7 @@ def _storage_http(*, keep_local, retries) -> snakemake.storage.StorageProviderPr return _storage_registry['http'] -def path_or_url(uri, *, keep_local=True, retries=2) -> str: +def path_or_url(uri, *, keep_local=True, retries=2) -> str | AnnotatedString: """ Intended for use in Snakemake inputs / outputs to transparently use remote resources. Returns the URI wrapped by an applicable storage plugin. Local @@ -142,12 +177,12 @@ def path_or_url(uri, *, keep_local=True, retries=2) -> str: if info.scheme=='s3': try: - return _storage_s3(bucket=info.netloc, keep_local=keep_local, retries=retries)(uri) + return _local_or_storage(_storage_s3(bucket=info.netloc, keep_local=keep_local, retries=retries)(uri)) except RemoteFilesMissingCredentials as e: raise Exception(f"AWS credentials are required to access {uri!r}") from e if info.scheme=='https': - return _storage_http(keep_local=keep_local, retries=retries)(uri) + return _local_or_storage(_storage_http(keep_local=keep_local, retries=retries)(uri)) elif info.scheme=='http': raise Exception(f"HTTP remote file support is not implemented in nextstrain workflows (attempting to access {uri!r}).\n" "Please use an HTTPS address instead.") @@ -157,3 +192,53 @@ def path_or_url(uri, *, keep_local=True, retries=2) -> str: "Please get in touch if you require this functionality and we can add it to our workflows") raise Exception(f"Input address {uri!r} (scheme={info.scheme!r}) is from a non-supported remote") + + +def _local_or_storage(wrapped: AnnotatedString) -> str | AnnotatedString: + """ + Given a storage-wrapped input (an AnnotatedString carrying a `storage_object` + flag), transparently fall back to the local cache copy as determined by + USE_REMOTE_FILES: + * If USE_REMOTE_FILES == True, then return object unchanged and let the + allow normal Snakemake retrieval, revalidation, or + MissingInputException behaviour. + * If USE_REMOTE_FILES == False and cache does not exist, raise an error. + * If USE_REMOTE_FILES == False and cache does exist, then return the + plain local path. Snakemake then treats it as an ordinary local input + (existence via `os.path.exists`, no network), allowing the workflow to + run offline from cache. + """ + so = get_flag_value(wrapped, "storage_object") + if so is None: + raise Exception("_local_or_storage must be called with a storage object AnnotatedString. Provided argument: ", wrapped) + + # Return StorageObject as-is since remote files are expected and allow it + # to fetch or revalidate cached files as needed. + # Any connection errors will be raised downstream + if USE_REMOTE_FILES: + return wrapped + + # The keep_local flag is managed by the workflow author and this cannot be + # changed by the user. So output warning for user and return the StorageObject as-is. + if not so.keep_local: + logger.warning( + f"Unable to use cached copy of {so.query!r} because the workflow " + "author has opted out of keeping a local copy for this input. " + "Please contact the workflow author to enable use of local cached files." + ) + return wrapped + + local = so.local_path() + if not local.exists(): + raise Exception( + f"Unable to find local cached file for {so.query!r}. " + "Please make sure you have run the workflow at least once with " + "internet connection and config.use_remote_files set to True to " + "to fetch and save the remote file locally.") + + logger.warning( + f"Using the cached copy of {so.query} at {local} without revalidation. " + "Set config.use_remote_files to True if you want to force revalidation." + ) + + return str(local) From ad0cc8c5570c77dc488b0ca99f4ab4705effab16 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Wed, 14 Oct 2026 12:00:46 -0700 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee02de..46e8961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ the outputs of the workflows. ## 2026 +* 14 October 2026: phylogenetic - support use of cached remote inputs + * See https://github.com/nextstrain/shared/pull/87 for more details. * 20 July 2026: `filter` is replaced with `subsample`. See updated default config for the expected schema. **This is a breaking change**. * 20 July 2026: Command-line parameters for `filter`, `refine`, `ancestral`, `traits`, and `tip_frequencies` are now defined as a single argument string under `.` in config. This reverts the change from 14 July 2026 and provides a consistent and flexible pattern for passing **non-file** arguments to Augur **This is a breaking change**. * 14 July 2026: The refine option for clock filtering has moved from `refine.: "--clock-filter-iqd "` to `refine..clock_filter_iqd: `. **This is a breaking change**.