Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
- accept `process_low_memory` as a standard module label ([#4264](https://github.com/nf-core/tools/pull/4264))
- improve linting for `modules.json` to add support for only installing subworkflows from a repository and provide more explicit error messages ([#4287](https://github.com/nf-core/tools/pull/4287))
- improve singularity_tag linting check (add skip, handle exceptions correctly, check against oras) ([#4358](https://github.com/nf-core/tools/pull/4358))
- accept axis-decomposed `process_cpus_*` and `process_mem_*` labels alongside the legacy combined labels ([#XXXX](https://github.com/nf-core/tools/pull/XXXX))
- accept axis-decomposed `process_cpus_*` and `process_mem_*` labels alongside the legacy combined labels ([#4265](https://github.com/nf-core/tools/pull/4265))
- accept axis-decomposed `process_cpus_*`, `process_mem_*` and `process_time_*` labels alongside the legacy combined labels ([#4265](https://github.com/nf-core/tools/pull/4265))

### Modules

Expand All @@ -33,6 +36,9 @@
- fix dead link ([#4307](https://github.com/nf-core/tools/pull/4307))
- remove date placeholder from changelog title ([#4333](https://github.com/nf-core/tools/pull/4333))
- Fix publishDir configuration in modules.config for multiqc ([#4347](https://github.com/nf-core/tools/pull/4347))
- add axis-decomposed `process_cpus_{single,low,medium,high}` and `process_mem_{low,medium,high}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#XXXX](https://github.com/nf-core/tools/pull/XXXX))
- add axis-decomposed `process_cpus_{single,low,medium,high}` and `process_mem_{low,medium,high}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#4265](https://github.com/nf-core/tools/pull/4265))
- add axis-decomposed `process_cpus_{single,low,medium,high}`, `process_mem_{low,medium,high}` and `process_time_{short,medium,long}` labels to `base.config`; legacy combined labels remain in place pending deprecation ([#4265](https://github.com/nf-core/tools/pull/4265))

#### Version updates

Expand Down
12 changes: 12 additions & 0 deletions nf_core/components/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,18 @@ def _get_bioconda_tool(self):

def _get_module_structure_components(self):
process_label_defaults = [
# Axis-decomposed labels (preferred)
"process_cpus_single",
"process_cpus_low",
"process_cpus_medium",
"process_cpus_high",
"process_mem_low",
"process_mem_medium",
"process_mem_high",
"process_time_short",
"process_time_medium",
"process_time_long",
# Legacy combined labels (kept for backwards compatibility, to be deprecated)
"process_single",
"process_low",
"process_medium",
Expand Down
76 changes: 67 additions & 9 deletions nf_core/modules/lint/main_nf.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,26 @@ def main_nf(
equal the number of ``emit:`` outputs whose name starts with ``versions``.
A warning is issued if a legacy YAML-based ``versions`` emit is used instead
of a topic output.

* ``process_standard_label``: Process labels should follow the standard format.
A warning is issued if a legacy label is used. Allowed standard labels are:
"process_cpus_single",
"process_cpus_low",
"process_cpus_medium",
"process_cpus_high",
"process_mem_low",
"process_mem_medium",
"process_mem_high",
"process_time_short",
"process_time_medium",
"process_time_long"
Legacy labels are:
"process_single",
"process_low",
"process_medium",
"process_high",
"process_long",
"process_low_memory",
"process_high_memory"
"""

inputs: list[str] = []
Expand Down Expand Up @@ -558,6 +577,19 @@ def check_process_name_format(self, process_name, component_name):

def check_process_labels(self, lines):
correct_process_labels = [
# Axis-decomposed labels (preferred)
"process_cpus_single",
"process_cpus_low",
"process_cpus_medium",
"process_cpus_high",
"process_mem_low",
"process_mem_medium",
"process_mem_high",
"process_time_short",
"process_time_medium",
"process_time_long",
]
legacy_process_labels = [
"process_single",
"process_low",
"process_medium",
Expand All @@ -569,11 +601,11 @@ def check_process_labels(self, lines):
all_labels = [line.strip() for line in lines if line.lstrip().startswith("label ")]
bad_labels = []
good_labels = []
legacy_labels = []
if len(all_labels) > 0:
for label in all_labels:
try:
label = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label).group(1)
except AttributeError:
match = re.match(r"^label\s+'?\"?([a-zA-Z0-9_-]+)'?\"?$", label)
if match is None:
self.warned.append(
(
"main_nf",
Expand All @@ -583,21 +615,46 @@ def check_process_labels(self, lines):
)
)
continue
if label not in correct_process_labels:
label = match.group(1)
if label not in correct_process_labels and label not in legacy_process_labels:
bad_labels.append(label)
if label in legacy_process_labels:
legacy_labels.append(label)
else:
good_labels.append(label)
if len(good_labels) > 1:
if legacy_labels:
self.warned.append(
(
"main_nf",
"process_standard_label",
f"Deprecated process label found: `{legacy_labels[0]}`. Use the new standard labels instead: https://nf-co.re/docs/developing/migration-guides/resource-labels",
self.main_nf,
)
)
axes = [label.split("_")[1] for label in good_labels if len(label.split("_")) > 1]
if len(axes) != len(set(axes)):
conflicting = [
label
for label in good_labels
if axes.count(label.split("_")[1] if len(label.split("_")) > 1 else "") > 1
]
self.warned.append(
(
"main_nf",
"process_standard_label",
f"Conflicting process labels found: `{'`,`'.join(good_labels)}`",
f"Conflicting process labels found: `{'`,`'.join(conflicting)}`",
self.main_nf,
)
)
elif good_labels:
self.passed.append(
(
"main_nf",
"process_standard_label",
f"Correct process labels: `{'`,`'.join(good_labels)}`",
self.main_nf,
)
)
elif len(good_labels) == 1:
self.passed.append(("main_nf", "process_standard_label", "Correct process label", self.main_nf))
else:
self.warned.append(("main_nf", "process_standard_label", "Standard process label not found", self.main_nf))
if len(bad_labels) > 0:
Expand All @@ -618,6 +675,7 @@ def check_process_labels(self, lines):
self.main_nf,
)
)

else:
self.warned.append(("main_nf", "process_standard_label", "Process label not specified", self.main_nf))

Expand Down
45 changes: 45 additions & 0 deletions nf_core/pipeline-template/conf/base.config
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,51 @@ process {
// adding in your local modules too.
// TODO nf-core: Customise requirements for specific processes.
// See https://www.nextflow.io/docs/latest/config.html#config-process-selectors

//
// Resource labels for independently controlling the default amount
// of CPUs, memory, and time requests a process makes to a scheduler.
//
// TODO nf-core: adjust all CPU, memory, and time resource requests to work with 'average' data for the pipeline.
// Default resource usage limits are intentionally generous and are may not be suitable.
// It is highly recommended to modify these settings to suit your pipeline.
//
withLabel:process_cpus_single {
cpus = { 1 }
}
withLabel:process_cpus_low {
cpus = { 2 * task.attempt }
}
withLabel:process_cpus_medium {
cpus = { 6 * task.attempt }
}
withLabel:process_cpus_high {
cpus = { 12 * task.attempt }
}
withLabel:process_mem_low {
memory = { 1.GB * task.attempt }
}
withLabel:process_mem_medium {
memory = { 12.GB * task.attempt }
}
withLabel:process_mem_high {
memory = { 72.GB * task.attempt }
}
withLabel:process_time_short {
time = { 1.h * task.attempt }
}
withLabel:process_time_medium {
time = { 8.h * task.attempt }
}
withLabel:process_time_long {
time = { 20.h * task.attempt }
}

//
// Legacy combined labels. Kept for backwards compatibility while the
// ecosystem migrates to the axis-decomposed labels above; these will be
// deprecated in a future release.
//
withLabel:process_single {
cpus = { 1 }
memory = { 6.GB * task.attempt }
Expand Down
36 changes: 23 additions & 13 deletions tests/modules/lint/test_main_nf.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,19 +67,25 @@ def test_process_name_format(process_name, component_name, passed, warned, faile
@pytest.mark.parametrize(
"content,passed,warned,failed",
[
# Valid process label
("label 'process_high'\ncpus 12", 1, 0, 0),
# Non-alphanumeric characters in label
# Valid new-style axis-decomposed label
("label 'process_cpus_high'\ncpus 12", 1, 0, 0),
# Multiple axis-decomposed labels on different axes — allowed
("label 'process_cpus_high'\nlabel 'process_mem_low'\ncpus 12", 1, 0, 0),
# Same axis twice — conflict
("label 'process_cpus_high'\nlabel 'process_cpus_low'\ncpus 12", 0, 1, 0),
# Legacy label warns (no new-style label present, so also warns "not found")
("label 'process_high'\ncpus 12", 0, 2, 0),
# Two legacy labels warn legacy + not-found (distinct, so no dup warn)
("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 2, 0),
# Duplicate legacy labels: not-found + legacy + duplicate
("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 3, 0),
# Non-alphanumeric characters in label: non-alphanumeric warn + not-found warn
("label 'a:label:with:colons'\ncpus 12", 0, 2, 0),
# Conflicting labels
("label 'process_high'\nlabel 'process_low'\ncpus 12", 0, 1, 0),
# Duplicate labels
("label 'process_high'\nlabel 'process_high'\ncpus 12", 0, 2, 0),
# Valid and non-standard labels
("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 1, 0),
# Non-standard label only
("label 'process_extra_label'\ncpus 12", 0, 2, 0),
# Non-standard duplicates without quotes
# Non-standard (bad) label only: passes axis check but warns non-standard
("label 'process_extra_label'\ncpus 12", 1, 1, 0),
# Legacy + non-standard: passes axis check, warns legacy and non-standard
("label 'process_high'\nlabel 'process_extra_label'\ncpus 12", 1, 2, 0),
# Non-standard duplicates without quotes: conflict + non-standard + duplicate
("label process_extra_label\nlabel process_extra_label\ncpus 12", 0, 3, 0),
# No label found
("cpus 12", 0, 1, 0),
Expand Down Expand Up @@ -386,7 +392,11 @@ def test_topics_and_emits_version_check(self):
module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir)
module_lint.lint(print_results=False, module="bamstats/generalstats")
assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"
# assert len(module_lint.warned) == 0, f"Expected 0 warnings, got {[x.__dict__ for x in module_lint.warned]}"
# TODO: once modules are migrated to axis-decomposed labels, remove the filter and assert len(module_lint.warned) == 0
non_label_warned = [w for w in module_lint.warned if w.lint_test != "process_standard_label"]
assert len(non_label_warned) == 0, (
f"Expected 0 non-label warnings, got {[x.__dict__ for x in non_label_warned]}"
)

assert len(module_lint.passed) > 0

Expand Down
Loading