From c6cf9ec7bbff8d25cc6e1d2e0033b91b648725a1 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 7 Aug 2026 20:41:54 +0200 Subject: [PATCH 01/35] add CRAB workflow support via law.contrib.cms --- AnaProd/tasks.py | 13 +- Analysis/tasks.py | 25 +- docs/workflow/arguments.md | 18 +- docs/workflow/crab.md | 115 ++++++++ docs/workflow/htcondor.md | 4 + mkdocs.yml | 1 + run_tools/law_customizations.py | 474 ++++++++++++++++++++++++++------ run_tools/stageout_logs.sh | 35 ++- test/hello_world_task.py | 9 +- 9 files changed, 580 insertions(+), 114 deletions(-) create mode 100644 docs/workflow/crab.md diff --git a/AnaProd/tasks.py b/AnaProd/tasks.py index c70982a2..61de46aa 100644 --- a/AnaProd/tasks.py +++ b/AnaProd/tasks.py @@ -15,7 +15,12 @@ check_root_file_integrity, get_tree_entries, ) -from FLAF.run_tools.law_customizations import Task, HTCondorWorkflow, copy_param +from FLAF.run_tools.law_customizations import ( + Task, + HTCondorWorkflow, + CrabWorkflow, + copy_param, +) from FLAF.Common.Utilities import getCustomisationSplit, ServiceThread from .AnaTupleFileList import CreateMergePlan from .MergeAnaTuples import mergeAnaTuples @@ -111,7 +116,7 @@ def WF_complete(ref_task): return InputFileTask.WF_complete_ -class AnaTupleFileTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class AnaTupleFileTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 40.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 2) @@ -325,7 +330,7 @@ def run(self): shutil.rmtree(job_home) -class AnaTupleFileListBuilderTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class AnaTupleFileListBuilderTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 24.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) bundle_flavours = ["core", "inputFileList"] @@ -536,7 +541,7 @@ def run(self): shutil.copy(input_local.abspath, self.output().abspath) -class AnaTupleMergeTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class AnaTupleMergeTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 48.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 2) delete_inputs_after_merge = luigi.BoolParameter(default=False) diff --git a/Analysis/tasks.py b/Analysis/tasks.py index cfd54e18..1df044f2 100644 --- a/Analysis/tasks.py +++ b/Analysis/tasks.py @@ -10,6 +10,7 @@ from FLAF.run_tools.law_customizations import ( Task, HTCondorWorkflow, + CrabWorkflow, copy_param, ) from FLAF.AnaProd.tasks import ( @@ -61,7 +62,7 @@ def _anaTuple_outputs(task): return cache -class HistTupleProducerTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class HistTupleProducerTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 5.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 4) # many short per-file branches: group several per HTCondor job to bound nJobs. @@ -515,7 +516,7 @@ def _split_merged_marker(split_target): return split_target.sibling(split_target.basename + ".merged", type="f") -class HistFromNtupleProducerTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class HistFromNtupleProducerTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 10.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 2) variables = luigi.Parameter(default="") @@ -688,7 +689,11 @@ def run(self): HistFromNtupleProducer = os.path.join( self._flaf_root(), "Analysis", "HistProducerFromNTuple.py" ) - nMT = self.n_cpus * 2 if self.effective_workflow == "htcondor" else 8 + nMT = ( + self.n_cpus * 2 + if self.effective_workflow in ("htcondor", "crab") + else 8 + ) # Determine which variables still need to be produced for this (dataset, chunk). outputs = self.output() @@ -766,7 +771,7 @@ def _localize(inp): shutil.rmtree(job_home) -class HistMergerTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class HistMergerTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 5.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 2) variables = luigi.Parameter(default="") @@ -986,7 +991,7 @@ def _localize(var_file): ) -class AnalysisCacheTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class AnalysisCacheTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 2.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) producer_to_run = luigi.Parameter() @@ -1003,10 +1008,14 @@ def bundle_flavours(self): flavours.append("cmssw") return flavours - # Need to override this from HTCondorWorkflow to have separate data pathways for different cache tasks + # Need to override this from HTCondorWorkflow/CrabWorkflow to have separate data + # pathways for different cache tasks def htcondor_output_directory(self): return law.LocalDirectoryTarget(self.local_path(self.producer_to_run)) + def crab_output_directory(self): + return law.LocalDirectoryTarget(self.local_path(self.producer_to_run)) + def __init__(self, *args, **kwargs): ana_v = kwargs.get("ana_version") or kwargs.get("anaCache_version") if ana_v: @@ -1269,7 +1278,7 @@ def run(self): shutil.rmtree(job_home) -class HistPlotTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class HistPlotTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 2.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) variables = luigi.Parameter(default="") @@ -1518,7 +1527,7 @@ def bool_flag(key, default): ps_call(cmd, verbose=1) -class AnalysisCacheAggregationTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class AnalysisCacheAggregationTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 2.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) producer_to_aggregate = luigi.Parameter() diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index d0972c46..836fa4f8 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -14,7 +14,7 @@ also provides built-in options for status and cleanup. |---|---|---| | `--version` | *(required)* | Label that namespaces this run's outputs. Different versions never collide. | | `--period` | *(required)* | The [era](../concepts/eras.md), e.g. `Run3_2022`. | -| `--workflow` | `local` | `local` (this machine) or `htcondor` (batch). See [HTCondor](htcondor.md). | +| `--workflow` | `local` | `local` (this machine), `htcondor` (CERN batch), or `crab` (WLCG). See [HTCondor](htcondor.md) and [CRAB](crab.md). | | `--branches` | *(all)* | Which branches to run, e.g. `0`, `0,2`, `5-7`. Restricts only the launched task, not its dependencies. | | `--test` | `-1` | Process only N events per input file (`-1` = all). Great for smoke tests. | | `--process` | `""` | Restrict to one process (e.g. `custom_CI_Signal`). | @@ -31,10 +31,22 @@ also provides built-in options for status and cleanup. | `--parallel-jobs` | *(unbounded)* | Cap concurrent branches, e.g. `--parallel-jobs 100`. | | `--max-runtime` | *(task default)* | Per-job wall-clock limit. | | `--n-cpus` | `1` | CPUs requested per job. | -| `--priority` | `0` | Job priority. | -| `--bundle` | off | Ship a code/environment tarball to the worker. See [HTCondor → bundles](htcondor.md#bundles-shipping-the-code-to-workers). | +| `--priority` | `0` | Job priority (HTCondor). | +| `--bundle` | off | Ship a code/environment tarball to the worker. See [HTCondor → bundles](htcondor.md#bundles-shipping-the-code-to-workers). Always on for `--workflow crab`. | | `--htcondor-spool` | off | Spool job files to the schedd. | +## CRAB options (on every workflow task) + +| Option | Default | Meaning | +|---|---|---| +| `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | +| `--crab-memory` | `-1` | Max memory per job in MB (`-1` → `n_cpus * 2000` or `crab.max_memory_mb`). | +| `--crab-whitelist` | empty | Comma-separated site whitelist. | +| `--crab-blacklist` | empty | Comma-separated site blacklist (ignored if whitelist is set). | + +CRAB also needs `crab.storage_site` and `crab.out_lfn_base` in config (or the +`FLAF_CRAB_STORAGE_SITE` / `FLAF_CRAB_OUT_LFN_BASE` environment variables). + ## Status & cleanup (LAW built-ins) | Option | Meaning | diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md new file mode 100644 index 00000000..cdecf820 --- /dev/null +++ b/docs/workflow/crab.md @@ -0,0 +1,115 @@ +# Running on CRAB (WLCG) + +HTCondor covers the CERN local batch farm. For jobs that should run anywhere on the +**WLCG** (CMS CRAB), FLAF tasks can be submitted with `--workflow crab`. The implementation +uses [law's CMS CRAB workflow](https://github.com/riga/law) (`law.contrib.cms.CrabWorkflow`). + +Analysis **outputs** still go through the normal FLAF remote targets (`fs_default` via +gfal/`davs://` etc.). CRAB's own stageout path is only used for CRAB bookkeeping (automatic +output collection is disabled). + +## Prerequisites + +1. A valid **VOMS proxy** for the CMS VO (`voms-proxy-init --voms cms -valid 192:00`). +2. A **MyProxy** credential valid for **at least 5 days**. The CRAB *server* pulls the proxy + from `myproxy.cern.ch`; without it the client can accept the task and the server then + returns `SUBMITFAILED`. Set up once: + + ```sh + myproxy-init -d -n -s myproxy.cern.ch + # or, for non-interactive delegation, put the grid cert passphrase in a file and set + # job.crab_password_file in law.cfg + ``` + + FLAF fails early with a clear error if MyProxy is missing (instead of waiting for + `SUBMITFAILED`). +3. CRAB client available (via CMSSW / the law CMSSW sandbox; default sandbox name is set in + law's config as `job.crab_sandbox_name`). +4. **Bundles**: CRAB workers do not mount AFS. FLAF always ships code via `BundleTask` + when `--workflow crab` is used (same tarballs as HTCondor `--bundle`). Tasks already + declare `bundle_flavours`. +5. Remote `fs_default` (e.g. `davs://eoshome-...`) so bundles and analysis outputs are on a + grid-accessible filesystem. + +## Config + +Add a `crab:` block to `user_custom.yaml` (or pass via `--user-custom`): + +```yaml +crab: + storage_site: T2_CH_CERN + out_lfn_base: /store/user//FLAF + # optional: + # whitelist: [T2_CH_CERN] + # blacklist: [T2_US_MIT] + # max_memory_mb: 4000 +``` + +Alternatively set environment variables: + +```sh +export FLAF_CRAB_STORAGE_SITE=T2_CH_CERN +export FLAF_CRAB_OUT_LFN_BASE=/store/user/$USER/FLAF +``` + +| Key | Meaning | +|---|---| +| `storage_site` | CRAB `Site.storageSite` (required for submission). | +| `out_lfn_base` | CRAB `Data.outLFNDirBase` (required; not where analysis outputs go). | +| `whitelist` / `blacklist` | Optional site lists. Whitelist implies `ignoreLocality`. | +| `max_memory_mb` | Default memory when `--crab-memory` is not set (`n_cpus * 2000` otherwise). | + +## Submit + +```sh +law run FLAF.Analysis.tasks.HistTupleProducerTask \ + --period Run3_2022EE --version my_crab \ + --workflow crab \ + --branches 0 \ + --test 1000 \ + --user-custom /path/to/user_custom_with_crab.yaml +``` + +| Option | Why | +|---|---| +| `--workflow crab` | Submit via CRAB instead of local/HTCondor. | +| `--crab-memory 4000` | Override max memory (MB) per job. | +| `--crab-whitelist T2_CH_CERN` | Restrict to listed sites. | +| `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory. | +| `--transfer-logs` | On by default; enables remote log stageout when `fs_default` is WLCG. | + +You do **not** need `--bundle` for CRAB — bundles are forced whenever the workflow is `crab`. + +## How it fits with HTCondor + bundles + +| Mode | Code on worker | Typical use | +|---|---|---| +| `--workflow local` | Submit machine | Development, small tests | +| `--workflow htcondor` | AFS (or `--bundle` tarball) | CERN farm production | +| `--workflow htcondor --bundle` | Tarball from `fs_default` | HTCondor without AFS dependency | +| `--workflow crab` | Tarball from `fs_default` (always) | Full WLCG via CRAB | + +## Monitor + +```sh +law run FLAF.Analysis.tasks.HistTupleProducerTask \ + --period Run3_2022EE --version my_crab --print-status 1,1 +``` + +CRAB project directories live under `data/jobs/` (see `job.job_file_dir` in `law.cfg`). You can +also use `crab status -d ` from a CMSSW environment. + +## Caveats + +!!! warning "MyProxy must stay valid" + CRAB polls through MyProxy. Delegate a long-lived proxy before large campaigns + (`myproxy-init -d -n` or law's password-file path). + +!!! warning "First-time CRAB / grid mapfile" + New users may need a CRAB username mapping and write access to the chosen storage site + LFN. Prefer a site you already use for CMS jobs (`T2_CH_CERN` is the usual CERN EOS + choice for `/store/user/...`). + +!!! note "Test small first" + Validate with `--workflow local --branches 0 --test 1000`, then a single CRAB branch, + before large submissions. diff --git a/docs/workflow/htcondor.md b/docs/workflow/htcondor.md index 50be0576..c575b13d 100644 --- a/docs/workflow/htcondor.md +++ b/docs/workflow/htcondor.md @@ -53,6 +53,10 @@ A batch worker needs your code and environment. FLAF supports two modes: For most work the defaults are correct; you only think about bundles when a stage explicitly needs one (e.g. it declares a CMSSW bundle flavour) or when AFS is not available on the target pool. +For jobs that should run on the full CMS WLCG (not only CERN HTCondor), use +[`--workflow crab`](crab.md) — that path always uses bundles. + + !!! tip "Your edits to FLAF *do* reach the workers" Thanks to the dev overlay, non-bundle jobs run your edited `FLAF`/`Corrections`, and bundle jobs include them in the tarball — so testing framework changes on HTCondor works without diff --git a/mkdocs.yml b/mkdocs.yml index f3cd8003..bcc840f9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -88,6 +88,7 @@ nav: - Full workflow: - Walkthrough: workflow/walkthrough.md - Running on HTCondor: workflow/htcondor.md + - Running on CRAB: workflow/crab.md - Command arguments: workflow/arguments.md - Configuration guide: - user_custom.yaml: configuration/user-custom.md diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index bc8affd8..327be137 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -17,6 +17,7 @@ from FLAF.Common.Setup import Setup law.contrib.load("htcondor") +law.contrib.load("cms") def copy_param(ref_param, new_default): @@ -589,7 +590,7 @@ class HTCondorWorkflow(law.htcondor.HTCondorWorkflow): default=False, significant=False, description="download pre-built bundle archives on workers instead of accessing AFS; " - "tasks declare which flavours they need via bundle_flavours", + "tasks declare which flavours they need via bundle_flavours. Always on for --workflow crab.", ) htcondor_spool = luigi.BoolParameter( default=True, @@ -606,35 +607,6 @@ class HTCondorWorkflow(law.htcondor.HTCondorWorkflow): ] bundle_flavours = [] - def workflow_requires(self): - if self.bundle and self.bundle_flavours: - bundles = [] - for item in self.bundle_flavours: - if isinstance(item, (list, tuple)) and len(item) == 2: - flavour, bversion = item - bundles.append( - BundleTask.req(self, flavour=flavour, version=bversion) - ) - else: - flavour = item - bundles.append(BundleTask.req(self, flavour=flavour)) - return {"bundles": bundles} - return {} - - def htcondor_check_job_completeness(self): - return False - - def htcondor_poll_callback(self, poll_data): - update_kinit(verbose=0) - return True - - def htcondor_output_directory(self): - # the directory where submission meta data should be stored - return law.LocalDirectoryTarget(self.local_path()) - - def htcondor_log_directory(self): - return None - def _flaf_root(self): # FLAF source root, respecting the dev overlay: flaf_dev.sh sets FLAF_PATH to # the top-level FLAF_all/FLAF, while the analysis env.sh sets it to the pinned @@ -646,32 +618,62 @@ def _flaf_root(self): os.getenv("ANALYSIS_PATH"), "FLAF" ) - def htcondor_stageout_file(self): - return os.path.join(self._flaf_root(), "run_tools", "stageout_logs.sh") + def _uses_bundles(self): + """Whether this submission should ship and unpack code bundles on the worker. - def htcondor_bootstrap_file(self): - # each job can define a bootstrap file that is executed prior to the actual job - # in order to setup software and environment variables - return os.path.join(self._flaf_root(), "bootstrap.sh") - - def htcondor_job_file_factory_cls(self): - return CERNHTCondorJobFileFactory - - def htcondor_job_config(self, config, job_num, branches): + Bundles are optional for HTCondor (shared AFS is available) but required for CRAB + (WLCG workers have no AFS mount). + """ + if not self.bundle_flavours: + return False + if getattr(self, "effective_workflow", None) == "crab": + return True + return bool(self.bundle) + + def _bundle_requirements(self): + """Return BundleTask requirements for configured flavours (empty if unused).""" + if not self._uses_bundles(): + return {} + bundles = [] + for item in self.bundle_flavours: + if isinstance(item, (list, tuple)) and len(item) == 2: + flavour, bversion = item + bundles.append(BundleTask.req(self, flavour=flavour, version=bversion)) + else: + bundles.append(BundleTask.req(self, flavour=item)) + return {"bundles": bundles} + + def _apply_bundle_render_variables(self, config): + """Set bootstrap render variables for bundle download (or clear them).""" + if not self._uses_bundles(): + config.render_variables["bundle_list"] = "" + return + if not isinstance(self.fs_default, WLCGFileSystem): + raise RuntimeError( + "bundle / crab workflows require fs_default to be a remote filesystem " + "(davs://, root://, ...)" + ) + bundle_parts = [] + for item in self.bundle_flavours: + if isinstance(item, (list, tuple)) and len(item) == 2: + flavour, bversion = item + else: + flavour = item + bversion = self.version + bundle_url = self.remote_target( + bversion, "bundles", self.period, f"{flavour}.tar.bz2" + ).uri() + bundle_parts.append(f"{flavour}:{bundle_url}") + config.render_variables["bundle_list"] = " ".join(bundle_parts) + + def _apply_bootstrap_path_render_variables(self, config): + """Set analysis_path / FLAF_PATH / CORRECTIONS_PATH / token-server for bootstrap.sh.""" ana_path = os.getenv("ANALYSIS_PATH") - # NON-bundle jobs run on the shared AFS workspace and source the analysis env.sh - # there. Forward FLAF_PATH / CORRECTIONS_PATH so the worker uses the same FLAF / - # Corrections as the submit side (the submodule copies in production, or the edited - # top-level copies when flaf_dev.sh is active) — bootstrap.sh exports them before - # sourcing env.sh. In production these equal $ANALYSIS_PATH/FLAF(/Corrections), so - # forwarding them is transparent. - # - # BUNDLE jobs instead set analysis_path=NONE and ship FLAF / Corrections inside the - # tarball; they must NOT receive FLAF_PATH / CORRECTIONS_PATH, otherwise the in-bundle - # env.sh would point them back at the AFS workspace and the worker would access AFS. + # Bundle (and always-on CRAB) jobs unpack code on the worker and must not point back + # at AFS. Non-bundle HTCondor jobs source the shared workspace and forward overlay paths. flaf_path = "" corrections_path = "" - if self.bundle and self.bundle_flavours: + if self._uses_bundles(): config.render_variables["analysis_path"] = "NONE" else: config.render_variables["analysis_path"] = ana_path @@ -680,22 +682,57 @@ def htcondor_job_config(self, config, job_num, branches): config.render_variables["flaf_path"] = flaf_path config.render_variables["corrections_path"] = corrections_path - # token server for rate-limiting job starts to avoid AFS overload. - # Not needed in bundle mode: workers never touch AFS, so there is no load concern. runTokenServer = self.global_params.get("runTokenServer", None) - if runTokenServer and not (self.bundle and self.bundle_flavours): + if runTokenServer and not self._uses_bundles(): config.render_variables["run_token_server_host"] = runTokenServer["host"] config.render_variables["run_token_server_port"] = str( runTokenServer["port"] ) - # ship get_token.py with the job so it is available before AFS is accessed config.input_files["get_token_script"] = os.path.join( - ana_path, "FLAF", "run_tools", "get_run_token.py" + self._flaf_root(), "run_tools", "get_run_token.py" ) else: config.render_variables["run_token_server_host"] = "" config.render_variables["run_token_server_port"] = "" + def _log_remote_base_url(self): + if isinstance(self.fs_default, WLCGFileSystem): + return self.remote_dir_target( + self.version, "logs", self.__class__.__name__, self.period + ).uri() + return "" + + def workflow_requires(self): + return self._bundle_requirements() + + def htcondor_check_job_completeness(self): + return False + + def htcondor_poll_callback(self, poll_data): + update_kinit(verbose=0) + return True + + def htcondor_output_directory(self): + # the directory where submission meta data should be stored + return law.LocalDirectoryTarget(self.local_path()) + + def htcondor_log_directory(self): + return None + + def htcondor_stageout_file(self): + return os.path.join(self._flaf_root(), "run_tools", "stageout_logs.sh") + + def htcondor_bootstrap_file(self): + # each job can define a bootstrap file that is executed prior to the actual job + # in order to setup software and environment variables + return os.path.join(self._flaf_root(), "bootstrap.sh") + + def htcondor_job_file_factory_cls(self): + return CERNHTCondorJobFileFactory + + def htcondor_job_config(self, config, job_num, branches): + self._apply_bootstrap_path_render_variables(config) + # force to run on AlmaLinux9, https://batchdocs.web.cern.ch/local/submit.html config.custom_content.append( ("requirements", 'TARGET.OpSysAndVer =?= "AlmaLinux9"') @@ -718,10 +755,7 @@ def htcondor_job_config(self, config, job_num, branches): ("environment", '"LAW_HTCONDOR_JOB_POSTFIX=$(law_job_postfix)"') ) - # Compute the remote destination directory for the stageout script. - log_remote_base_url = "" - if isinstance(self.fs_default, WLCGFileSystem): - log_remote_base_url = self.remote_log_dir_target().uri() + log_remote_base_url = self._log_remote_base_url() config.render_variables["log_remote_base_url"] = log_remote_base_url # Redirect the sandbox log copy to /dev/null only when stageout will @@ -730,31 +764,11 @@ def htcondor_job_config(self, config, job_num, branches): if log_remote_base_url: config.output_files["stdall.txt"] = "/dev/null" - # bundle: build a space-separated list of "flavour:url" pairs for bootstrap.sh. - if self.bundle and self.bundle_flavours: - if not isinstance(self.fs_default, WLCGFileSystem): - raise RuntimeError( - "--bundle requires fs_default to be a remote filesystem (davs://, root://, ...)" - ) - bundle_parts = [] - for item in self.bundle_flavours: - if isinstance(item, (list, tuple)) and len(item) == 2: - flavour, bversion = item - else: - flavour = item - bversion = self.version - bundle_url = self.remote_target( - bversion, "bundles", self.period, f"{flavour}.tar.bz2" - ).uri() - bundle_parts.append(f"{flavour}:{bundle_url}") - config.render_variables["bundle_list"] = " ".join(bundle_parts) - - if not self.htcondor_spool: - config._worker_files_remote_dir = self.remote_dir_target( - self.version, "worker_files", self.period - ) - else: - config.render_variables["bundle_list"] = "" + self._apply_bundle_render_variables(config) + if self._uses_bundles() and not self.htcondor_spool: + config._worker_files_remote_dir = self.remote_dir_target( + self.version, "worker_files", self.period + ) return config @@ -839,3 +853,291 @@ def _submit_group(self, *args, **kwargs): # our `_submit_group` override (remote log path rewrite) would never run. Flip the # flag so this class is recognised as the "htcondor" workflow provider. HTCondorWorkflow._defined_workflow_proxy = True + + +class FLAFCrabJobFileFactory(law.cms.CrabJobFileFactory): + """CrabJobFileFactory compatible with recent CRAB clients. + + law 0.1.20 still emits ``JobType.sendPythonFolder``, which modern CRAB rejects as + deprecated (the Python folder is always sandboxed). Strip that line after writing + the config file. + """ + + def create(self, **kwargs): + job_file, c = super().create(**kwargs) + try: + with open(job_file) as f: + lines = f.readlines() + new_lines = [ln for ln in lines if "sendPythonFolder" not in ln] + if len(new_lines) != len(lines): + with open(job_file, "w") as f: + f.writelines(new_lines) + if hasattr(c, "crab") and getattr(c.crab, "JobType", None) is not None: + c.crab.JobType.sendPythonFolder = None + except Exception as exc: + print(f"WARNING: could not strip deprecated sendPythonFolder from {job_file}: {exc}") + return job_file, c + + +# Soften MyProxy requirements: modern CRAB accepts a local VOMS proxy for submit/status. +# law's default CrabWorkflowProxy.setup_job_manager always tries interactive myproxy +# delegation; that blocks non-interactive CI-like runs when myproxy is not pre-loaded. +_FLAFCrabWorkflowProxyBase = law.cms.CrabWorkflow.workflow_proxy_cls + + +class _FLAFCrabWorkflowProxy(_FLAFCrabWorkflowProxyBase): + def setup_job_manager(self): + """Ensure VOMS + MyProxy before crab submit. + + The CRAB *server* retrieves the user proxy from myproxy.cern.ch and requires + at least ~5 days remaining. A local VOMS proxy alone is not enough: the client + may accept the task, then the server returns SUBMITFAILED. Fail early here. + """ + proxy = os.environ.get("X509_USER_PROXY", "") + if not proxy or not os.path.isfile(proxy): + raise RuntimeError( + "CRAB submission requires a valid VOMS proxy (X509_USER_PROXY). " + "Run: voms-proxy-init --voms cms -valid 192:00" + ) + if not law.wlcg.check_vomsproxy_validity(proxy_file=proxy): + raise RuntimeError( + f"VOMS proxy at {proxy} is missing or expired; run " + "`voms-proxy-init --voms cms -valid 192:00`" + ) + kwargs = {"proxy": proxy} + + # Prefer an existing MyProxy credential if still long-lived enough for CRAB. + try: + info = law.wlcg.get_myproxy_info(silent=True) or {} + except Exception: + info = {} + # CRAB server asks for >= 5 days; keep a small margin. + min_myproxy_seconds = 5 * 24 * 3600 + if info.get("username") and info.get("timeleft", 0) >= min_myproxy_seconds: + kwargs["myproxy_username"] = info["username"] + return kwargs + + # Non-interactive delegation when a password file is configured (law.cfg). + cfg = law.config.Config.instance() + password_file = cfg.get_expanded("job", "crab_password_file") + if password_file and os.path.isfile(password_file): + from law.contrib.cms.util import renew_vomsproxy, delegate_myproxy + + if not law.wlcg.check_vomsproxy_validity(): + renew_vomsproxy(password_file=password_file) + kwargs["myproxy_username"] = delegate_myproxy(password_file=password_file) + return kwargs + + raise RuntimeError( + "CRAB requires a MyProxy credential valid for at least 5 days " + "(CRAB server retrieves it from myproxy.cern.ch). " + "Run once interactively:\n" + " myproxy-init -d -n -s myproxy.cern.ch\n" + "or set job.crab_password_file in law.cfg to a file containing the " + "grid certificate passphrase for non-interactive delegation." + ) + + +class CrabWorkflow(law.cms.CrabWorkflow): + """CRAB (WLCG) remote workflow, built on law.contrib.cms.CrabWorkflow. + + Analysis outputs are still written via FLAF's normal remote targets (fs_default + through gfal/davs). CRAB's own stageout location is only required for submission + bookkeeping (automatic output collection is disabled by law). + + CRAB workers have no AFS, so code is always shipped via the existing BundleTask + mechanism (same as ``--bundle`` on HTCondor). Tasks must declare ``bundle_flavours``. + + Config (``global_params`` / user_custom YAML):: + + crab: + storage_site: T2_CH_CERN # Site.storageSite + out_lfn_base: /store/user//FLAF # Data.outLFNDirBase + # optional: + # whitelist: [T2_CH_CERN, T2_IT_Pisa] + # blacklist: [T2_US_MIT] + # max_memory_mb: 4000 + """ + + # Re-declare in the class body so law's metaclass sets _defined_workflow_proxy=True + # and find_workflow_cls('crab') resolves to *this* class (not law.cms.CrabWorkflow). + workflow_proxy_cls = _FLAFCrabWorkflowProxy + + poll_interval = copy_param(law.cms.CrabWorkflow.poll_interval, 5) + transfer_logs = luigi.BoolParameter( + default=True, + significant=False, + description="transfer job logs (stdall.txt) for stageout; default True", + ) + crab_memory = luigi.IntParameter( + default=-1, + significant=False, + description="max memory per CRAB job in MB; -1 = n_cpus * 2000", + ) + crab_whitelist = law.CSVParameter( + default=(), + significant=False, + description="comma-separated CRAB Site.whitelist; empty = no whitelist", + ) + crab_blacklist = law.CSVParameter( + default=(), + significant=False, + description="comma-separated CRAB Site.blacklist; ignored when whitelist is set", + ) + + # Parameters that are only meaningful on the workflow (not branch tasks). + exclude_params_branch = getattr( + law.cms.CrabWorkflow, "exclude_params_branch", set() + ) | {"crab_memory", "crab_whitelist", "crab_blacklist"} + + def _crab_cfg(self): + return self.global_params.get("crab") or {} + + def crab_stageout_location(self): + """Return (storageSite, outLFNDirBase) required by CRAB submission. + + Analysis outputs do not go here (disableAutomaticOutputCollection is True); + still required by the CRAB client for a valid config. + """ + cfg = self._crab_cfg() + site = cfg.get("storage_site") or os.environ.get("FLAF_CRAB_STORAGE_SITE") + lfn = cfg.get("out_lfn_base") or os.environ.get("FLAF_CRAB_OUT_LFN_BASE") + if not site or not lfn: + raise RuntimeError( + "CRAB requires crab.storage_site and crab.out_lfn_base in config " + "(user_custom / global_params), or FLAF_CRAB_STORAGE_SITE and " + "FLAF_CRAB_OUT_LFN_BASE environment variables. " + "Example: crab: {storage_site: T2_CH_CERN, " + "out_lfn_base: /store/user/$USER/FLAF}" + ) + return str(site), str(lfn) + + def crab_output_directory(self): + return law.LocalDirectoryTarget(self.local_path()) + + def crab_request_name(self, submit_jobs): + # CRAB: no dots, max 100 characters. + import uuid + + parts = [ + self.task_family.replace(".", "_"), + str(self.version).replace(".", "_"), + str(self.period).replace(".", "_"), + uuid.uuid4().hex[:8], + ] + name = "_".join(parts) + return re.sub(r"[^A-Za-z0-9_\-]", "_", name)[:100] + + def crab_bootstrap_file(self): + from law.job.base import JobInputFile + + return JobInputFile( + path=os.path.join(self._flaf_root(), "bootstrap.sh"), + copy=True, + share=True, + render_job=True, + ) + + def crab_stageout_file(self): + from law.job.base import JobInputFile + + return JobInputFile( + path=os.path.join(self._flaf_root(), "run_tools", "stageout_logs.sh"), + copy=True, + share=True, + render_job=True, + ) + + def crab_workflow_requires(self): + # Always require bundles for CRAB (no AFS on WLCG workers). + if not self.bundle_flavours: + raise RuntimeError( + f"{self.__class__.__name__}: --workflow crab requires bundle_flavours " + "on the task (code/environment shipped via BundleTask)" + ) + return self._bundle_requirements() + + def crab_check_job_completeness(self): + return False + + def crab_poll_callback(self, poll_data): + update_kinit(verbose=0) + return True + + def crab_job_file_factory_cls(self): + return FLAFCrabJobFileFactory + + def crab_job_file(self): + # Same deps_depth=0 patch as HTCondor: avoid huge print_deps on the worker. + from law.job.base import JobInputFile + + original = law.util.law_src_path("job", "law_job.sh") + custom = os.path.join( + os.getenv("ANALYSIS_DATA_PATH"), "law_job_no_print_deps.sh" + ) + if not os.path.exists(custom) or os.path.getmtime(original) > os.path.getmtime( + custom + ): + with open(original) as f: + content = f.read() + content = re.sub(r'\bdeps_depth="[0-9]+"', 'deps_depth="0"', content) + with open(custom, "w") as f: + f.write(content) + os.chmod(custom, 0o755) + return JobInputFile(path=custom, copy=True, share=True, render_job=True) + + def crab_job_config(self, config, job_nums, branches=None): + # law 0.1.20 calls crab_job_config(config, list(keys), list(values)); the base + # signature documents a single submit_jobs arg, but the call site passes two lists. + if not self.bundle_flavours: + raise RuntimeError( + f"{self.__class__.__name__}: --workflow crab requires bundle_flavours" + ) + + self._apply_bootstrap_path_render_variables(config) + self._apply_bundle_render_variables(config) + + log_remote_base_url = self._log_remote_base_url() + config.render_variables["log_remote_base_url"] = log_remote_base_url + + # Memory / cores + n_cpus = int(getattr(self, "n_cpus", 1) or 1) + mem = int(self.crab_memory) + if mem <= 0: + mem = int(self._crab_cfg().get("max_memory_mb", n_cpus * 2000)) + config.crab.JobType.maxMemoryMB = mem + if n_cpus > 1: + config.crab.JobType.numCores = n_cpus + + # Optional runtime limit (hours → minutes) + max_runtime = getattr(self, "max_runtime", None) + if max_runtime is not None and float(max_runtime) > 0: + try: + config.crab.JobType.maxJobRuntimeMin = int( + math.floor(float(max_runtime) * 60) + ) + except Exception: + # Older CRAB clients may not support maxJobRuntimeMin; ignore if rejected later. + pass + + # Site white/black lists: CLI params override config. + # CRAB requires a whitelist when jobs use synthetic userInputFiles (no inputDataset), + # which is always the case for law CRAB workflows. Default the whitelist to the + # storage site when nothing else is configured. + whitelist = list(self.crab_whitelist) or list( + self._crab_cfg().get("whitelist") or [] + ) + blacklist = list(self.crab_blacklist) or list( + self._crab_cfg().get("blacklist") or [] + ) + if not whitelist and not blacklist: + site, _ = self.crab_stageout_location() + whitelist = [site] + if whitelist: + config.crab.Site.whitelist = [str(s) for s in whitelist] + config.crab.Site.ignoreGlobalBlacklist = True + config.crab.Data.ignoreLocality = True + elif blacklist: + config.crab.Site.blacklist = [str(s) for s in blacklist] + + return config diff --git a/run_tools/stageout_logs.sh b/run_tools/stageout_logs.sh index a892d1c5..8c3fe8aa 100644 --- a/run_tools/stageout_logs.sh +++ b/run_tools/stageout_logs.sh @@ -7,23 +7,36 @@ if [ -z "${log_remote_base_url}" ]; then exit 0 fi -postfix="${LAW_HTCONDOR_JOB_POSTFIX}" -if [ -n "${postfix}" ]; then - log_file="stdall${postfix}.txt" +# Resolve local log path and remote basename. +# HTCondor: law may use a postfix / cluster_process name. +# CRAB: the sandbox log is always stdall.txt in the job dir; include the CRAB job +# number in the remote name so concurrent jobs do not overwrite each other. +local_log_file="" +remote_log_file="" + +if [ -n "${LAW_CRAB_JOB_NUMBER:-}" ]; then + local_log_file="stdall.txt" + remote_log_file="stdall_crab${LAW_CRAB_JOB_NUMBER}.txt" else - cluster="${LAW_HTCONDOR_JOB_CLUSTER}" - process="${LAW_HTCONDOR_JOB_PROCESS}" - if [ -n "${cluster}" ] && [ -n "${process}" ]; then - log_file="stdall_${cluster}_${process}.txt" + postfix="${LAW_HTCONDOR_JOB_POSTFIX}" + if [ -n "${postfix}" ]; then + local_log_file="stdall${postfix}.txt" else - log_file="stdall.txt" + cluster="${LAW_HTCONDOR_JOB_CLUSTER}" + process="${LAW_HTCONDOR_JOB_PROCESS}" + if [ -n "${cluster}" ] && [ -n "${process}" ]; then + local_log_file="stdall_${cluster}_${process}.txt" + else + local_log_file="stdall.txt" + fi fi + remote_log_file="${local_log_file}" fi if [ -n "${LAW_JOB_INIT_DIR}" ]; then - log_path="${LAW_JOB_INIT_DIR}/${log_file}" + log_path="${LAW_JOB_INIT_DIR}/${local_log_file}" else - log_path="${log_file}" + log_path="${local_log_file}" fi if [ ! -f "${log_path}" ]; then @@ -31,7 +44,7 @@ if [ ! -f "${log_path}" ]; then exit 0 fi -log_remote_url="${log_remote_base_url%/}/${log_file}" +log_remote_url="${log_remote_base_url%/}/${remote_log_file}" GFAL_COPY=$(which gfal-copy 2>/dev/null) if [ -z "${GFAL_COPY}" ]; then diff --git a/test/hello_world_task.py b/test/hello_world_task.py index e1f6045c..d3f19691 100644 --- a/test/hello_world_task.py +++ b/test/hello_world_task.py @@ -1,10 +1,15 @@ import law import luigi -from FLAF.run_tools.law_customizations import Task, HTCondorWorkflow, copy_param +from FLAF.run_tools.law_customizations import ( + Task, + HTCondorWorkflow, + CrabWorkflow, + copy_param, +) -class HelloWorldTask(Task, HTCondorWorkflow, law.LocalWorkflow): +class HelloWorldTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 0.1) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) poll_interval = copy_param(HTCondorWorkflow.poll_interval, 1) From 1f3e0f18a987a85c66c36cfd1c80e6bec674b29b Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 7 Aug 2026 22:53:27 +0200 Subject: [PATCH 02/35] accept DN-named MyProxy credentials for CRAB --- run_tools/law_customizations.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 327be137..34d4233f 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -875,7 +875,9 @@ def create(self, **kwargs): if hasattr(c, "crab") and getattr(c.crab, "JobType", None) is not None: c.crab.JobType.sendPythonFolder = None except Exception as exc: - print(f"WARNING: could not strip deprecated sendPythonFolder from {job_file}: {exc}") + print( + f"WARNING: could not strip deprecated sendPythonFolder from {job_file}: {exc}" + ) return job_file, c @@ -906,18 +908,24 @@ def setup_job_manager(self): ) kwargs = {"proxy": proxy} - # Prefer an existing MyProxy credential if still long-lived enough for CRAB. - try: - info = law.wlcg.get_myproxy_info(silent=True) or {} - except Exception: - info = {} - # CRAB server asks for >= 5 days; keep a small margin. + # CRAB server asks for >= 5 days remaining; keep a small margin. min_myproxy_seconds = 5 * 24 * 3600 - if info.get("username") and info.get("timeleft", 0) >= min_myproxy_seconds: - kwargs["myproxy_username"] = info["username"] - return kwargs + + # MyProxy usernames may be either the DN (`myproxy-init -d`) or a SHA1 of the DN + # (law's default encode_username=True / some crab helpers). Accept either form. + for encode in (False, True): + try: + info = ( + law.wlcg.get_myproxy_info(encode_username=encode, silent=True) or {} + ) + except Exception: + info = {} + if info.get("username") and info.get("timeleft", 0) >= min_myproxy_seconds: + kwargs["myproxy_username"] = info["username"] + return kwargs # Non-interactive delegation when a password file is configured (law.cfg). + # law.delegate_myproxy registers under the SHA1 username CRAB workers expect. cfg = law.config.Config.instance() password_file = cfg.get_expanded("job", "crab_password_file") if password_file and os.path.isfile(password_file): @@ -933,6 +941,7 @@ def setup_job_manager(self): "(CRAB server retrieves it from myproxy.cern.ch). " "Run once interactively:\n" " myproxy-init -d -n -s myproxy.cern.ch\n" + " # verify: myproxy-info -d -s myproxy.cern.ch (timeleft >= 5 days)\n" "or set job.crab_password_file in law.cfg to a file containing the " "grid certificate passphrase for non-interactive delegation." ) From 44ea1cbbb5cb77bc1bdd04cc040ebdafb3fdc034 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 7 Aug 2026 23:36:58 +0200 Subject: [PATCH 03/35] disable CRAB output transfer; use FLAF remote I/O only --- docs/workflow/crab.md | 54 ++++++++++---- run_tools/law_customizations.py | 127 +++++++++++++++++++++++++------- 2 files changed, 141 insertions(+), 40 deletions(-) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index cdecf820..900fbab9 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -4,25 +4,42 @@ HTCondor covers the CERN local batch farm. For jobs that should run anywhere on **WLCG** (CMS CRAB), FLAF tasks can be submitted with `--workflow crab`. The implementation uses [law's CMS CRAB workflow](https://github.com/riga/law) (`law.contrib.cms.CrabWorkflow`). -Analysis **outputs** still go through the normal FLAF remote targets (`fs_default` via -gfal/`davs://` etc.). CRAB's own stageout path is only used for CRAB bookkeeping (automatic -output collection is disabled). +Analysis **outputs and job logs** use FLAF remote I/O only (`fs_default` via gfal/`davs://`, +plus `stageout_logs.sh`). CRAB `transferOutputs` / `transferLogs` are forced **off** so +nothing is duplicated onto CRAB's stageout area. `crab.storage_site` / +`crab.out_lfn_base` are still required by the CRAB client for a valid config and a +submit-time write check — they are not where FLAF stores analysis products. ## Prerequisites 1. A valid **VOMS proxy** for the CMS VO (`voms-proxy-init --voms cms -valid 192:00`). -2. A **MyProxy** credential valid for **at least 5 days**. The CRAB *server* pulls the proxy - from `myproxy.cern.ch`; without it the client can accept the task and the server then - returns `SUBMITFAILED`. Set up once: +2. A **MyProxy** credential valid for **at least 5 days**, registered so **CRAB task + workers can retrieve it**. A plain `myproxy-init -d -n` is not enough: the credential + must use the SHA1 username law/CRAB expect and include CRAB retriever DNs. From an + existing VOMS proxy (no grid-cert passphrase): ```sh - myproxy-init -d -n -s myproxy.cern.ch - # or, for non-interactive delegation, put the grid cert passphrase in a file and set - # job.crab_password_file in law.cfg + export X509_USER_PROXY=/tmp/x509up_u$(id -u) # or your proxy path + # identity DN → SHA1 username (same as law) + SHA1=$(python3 - <<'PY' + import hashlib, subprocess + out = subprocess.check_output(["voms-proxy-info", "-identity"], text=True).strip() + print(hashlib.sha1(out.encode()).hexdigest()) + PY + ) + RETR='/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod|dev)-tw(01|02|03).cern.ch|/DC=ch/DC=cern/OU=computers/CN=stefanov(m|m2).cern.ch|/DC=ch/DC=cern/OU=computers/CN=dciangot-tw.cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod)-tw(01|02).cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-dev-tw(01|02|03|04).cern.ch|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cmscrab/CN=(817881|373708)/CN=Robot: cms crab|/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=crabint1/CN=373708/CN=Robot: CMS CRAB Integration 1' + GT_PROXY_MODE=rfc myproxy-init -n -s myproxy.cern.ch \ + -C "$X509_USER_PROXY" -y "$X509_USER_PROXY" \ + -l "$SHA1" -t 168 -c 168 \ + -x -R "$RETR" -x -Z "$RETR" -m cms + myproxy-info -s myproxy.cern.ch -l "$SHA1" # expect timeleft >= 5 days + retrieval policy ``` - FLAF fails early with a clear error if MyProxy is missing (instead of waiting for - `SUBMITFAILED`). + Alternatively set `job.crab_password_file` in `law.cfg` to a file with the grid + certificate passphrase; law will call `delegate_myproxy` with the same CRAB retrievers. + + FLAF fails early if no suitable MyProxy credential is found (instead of waiting for + server-side `SUBMITFAILED`). 3. CRAB client available (via CMSSW / the law CMSSW sandbox; default sandbox name is set in law's config as `job.crab_sandbox_name`). 4. **Bundles**: CRAB workers do not mount AFS. FLAF always ships code via `BundleTask` @@ -37,18 +54,27 @@ Add a `crab:` block to `user_custom.yaml` (or pass via `--user-custom`): ```yaml crab: - storage_site: T2_CH_CERN + # Stageout site for CRAB bookkeeping (analysis outputs still use fs_default). + # At CERN, T3_CH_CERNBOX maps /store/user/ to personal EOS and usually + # passes `crab checkwrite`; T2_CH_CERN /store/user often does not exist. + storage_site: T3_CH_CERNBOX out_lfn_base: /store/user//FLAF # optional: - # whitelist: [T2_CH_CERN] + # whitelist: [T2_CH_CERN] # where jobs run (can differ from storage_site) # blacklist: [T2_US_MIT] # max_memory_mb: 4000 ``` +Verify write access before the first campaign: + +```sh +crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER +``` + Alternatively set environment variables: ```sh -export FLAF_CRAB_STORAGE_SITE=T2_CH_CERN +export FLAF_CRAB_STORAGE_SITE=T3_CH_CERNBOX export FLAF_CRAB_OUT_LFN_BASE=/store/user/$USER/FLAF ``` diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 34d4233f..42b56027 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -856,30 +856,95 @@ def _submit_group(self, *args, **kwargs): class FLAFCrabJobFileFactory(law.cms.CrabJobFileFactory): - """CrabJobFileFactory compatible with recent CRAB clients. + """CrabJobFileFactory for FLAF: no CRAB-side product/log stageout. - law 0.1.20 still emits ``JobType.sendPythonFolder``, which modern CRAB rejects as - deprecated (the Python folder is always sandboxed). Strip that line after writing - the config file. + Analysis products and job logs are written by FLAF itself (remote targets via + gfal + ``stageout_logs.sh``). CRAB is used only as a batch backend, so we force: + + - ``General.transferOutputs = False`` + - ``General.transferLogs = False`` + - no ``JobType.outputFiles`` + - ``JobType.disableAutomaticOutputCollection = True`` (law default) + + ``Site.storageSite`` / ``Data.outLFNDirBase`` remain required by the CRAB client + for a valid config and the submit-time write check, but FLAF never places analysis + outputs there. + + Also strips deprecated ``JobType.sendPythonFolder`` (rejected by modern CRAB). """ def create(self, **kwargs): + # Prevent law from promoting custom_log_file into CRAB JobType.outputFiles + # (which would set transferOutputs=True and duplicate FLAF log stageout). + kwargs = dict(kwargs) + kwargs["output_files"] = [] + # Keep a local log file name for the law job script if transfer_logs requested, + # but do not register it as a CRAB output. + custom_log = kwargs.get("custom_log_file") + job_file, c = super().create(**kwargs) + + if hasattr(c, "crab"): + c.crab.General.transferOutputs = False + c.crab.General.transferLogs = False + if getattr(c.crab, "JobType", None) is not None: + c.crab.JobType.sendPythonFolder = None + c.crab.JobType.outputFiles = None + c.crab.JobType.disableAutomaticOutputCollection = True + c.output_files = [] + if custom_log: + c.custom_log_file = custom_log + try: - with open(job_file) as f: - lines = f.readlines() - new_lines = [ln for ln in lines if "sendPythonFolder" not in ln] - if len(new_lines) != len(lines): - with open(job_file, "w") as f: - f.writelines(new_lines) - if hasattr(c, "crab") and getattr(c.crab, "JobType", None) is not None: - c.crab.JobType.sendPythonFolder = None + self._rewrite_crab_job_file(job_file) except Exception as exc: - print( - f"WARNING: could not strip deprecated sendPythonFolder from {job_file}: {exc}" - ) + print(f"WARNING: could not post-process crab job file {job_file}: {exc}") return job_file, c + @staticmethod + def _rewrite_crab_job_file(job_file): + """Rewrite the generated CRAB cfg to drop output transfer and deprecated keys.""" + with open(job_file) as f: + lines = f.readlines() + + new_lines = [] + skip_list = False + for ln in lines: + stripped = ln.strip() + + # Skip deprecated option entirely. + if "sendPythonFolder" in ln: + continue + + # Force no CRAB-side transfers (FLAF owns remote I/O). + if "General.transferOutputs" in ln: + new_lines.append("cfg.General.transferOutputs = False\n") + continue + if "General.transferLogs" in ln: + new_lines.append("cfg.General.transferLogs = False\n") + continue + + # Drop JobType.outputFiles (single line or multi-line list). + if "JobType.outputFiles" in ln: + if stripped.endswith("[") or ("[" in stripped and "]" not in stripped): + skip_list = True + continue + if skip_list: + if "]" in stripped: + skip_list = False + continue + + if "JobType.disableAutomaticOutputCollection" in ln: + new_lines.append( + "cfg.JobType.disableAutomaticOutputCollection = True\n" + ) + continue + + new_lines.append(ln) + + with open(job_file, "w") as f: + f.writelines(new_lines) + # Soften MyProxy requirements: modern CRAB accepts a local VOMS proxy for submit/status. # law's default CrabWorkflowProxy.setup_job_manager always tries interactive myproxy @@ -950,9 +1015,13 @@ def setup_job_manager(self): class CrabWorkflow(law.cms.CrabWorkflow): """CRAB (WLCG) remote workflow, built on law.contrib.cms.CrabWorkflow. - Analysis outputs are still written via FLAF's normal remote targets (fs_default - through gfal/davs). CRAB's own stageout location is only required for submission - bookkeeping (automatic output collection is disabled by law). + CRAB is only the batch backend. **All analysis products and logs use FLAF remote + I/O** (``fs_default`` / gfal via task targets and ``stageout_logs.sh``). CRAB + ``transferOutputs`` / ``transferLogs`` / ``JobType.outputFiles`` are forced off so + nothing is duplicated onto CRAB's stageout area. + + ``crab.storage_site`` / ``crab.out_lfn_base`` remain required by the CRAB client + (submit-time write check); they are not used for analysis outputs. CRAB workers have no AFS, so code is always shipped via the existing BundleTask mechanism (same as ``--bundle`` on HTCondor). Tasks must declare ``bundle_flavours``. @@ -960,8 +1029,8 @@ class CrabWorkflow(law.cms.CrabWorkflow): Config (``global_params`` / user_custom YAML):: crab: - storage_site: T2_CH_CERN # Site.storageSite - out_lfn_base: /store/user//FLAF # Data.outLFNDirBase + storage_site: T3_CH_CERNBOX # Site.storageSite (write-check only) + out_lfn_base: /store/user//FLAF # Data.outLFNDirBase (write-check only) # optional: # whitelist: [T2_CH_CERN, T2_IT_Pisa] # blacklist: [T2_US_MIT] @@ -973,10 +1042,13 @@ class CrabWorkflow(law.cms.CrabWorkflow): workflow_proxy_cls = _FLAFCrabWorkflowProxy poll_interval = copy_param(law.cms.CrabWorkflow.poll_interval, 5) + # When True, law names the worker log ``stdall.txt`` and FLAF stageout_logs.sh + # uploads it to fs_default. CRAB itself never transfers this file. transfer_logs = luigi.BoolParameter( default=True, significant=False, - description="transfer job logs (stdall.txt) for stageout; default True", + description="enable FLAF remote log stageout (stdall.txt via stageout_logs.sh); " + "CRAB transferLogs stays off", ) crab_memory = luigi.IntParameter( default=-1, @@ -1003,10 +1075,12 @@ def _crab_cfg(self): return self.global_params.get("crab") or {} def crab_stageout_location(self): - """Return (storageSite, outLFNDirBase) required by CRAB submission. + """Return (storageSite, outLFNDirBase) required by the CRAB client. - Analysis outputs do not go here (disableAutomaticOutputCollection is True); - still required by the CRAB client for a valid config. + FLAF does **not** store analysis outputs here (CRAB transferOutputs is forced + off; products go to ``fs_default``). CRAB still requires these fields and runs + a submit-time write check against them — pick a site you can write to + (e.g. ``T3_CH_CERNBOX`` + ``/store/user//...``). """ cfg = self._crab_cfg() site = cfg.get("storage_site") or os.environ.get("FLAF_CRAB_STORAGE_SITE") @@ -1015,8 +1089,9 @@ def crab_stageout_location(self): raise RuntimeError( "CRAB requires crab.storage_site and crab.out_lfn_base in config " "(user_custom / global_params), or FLAF_CRAB_STORAGE_SITE and " - "FLAF_CRAB_OUT_LFN_BASE environment variables. " - "Example: crab: {storage_site: T2_CH_CERN, " + "FLAF_CRAB_OUT_LFN_BASE environment variables. These are only for the " + "CRAB client write-check, not analysis outputs. " + "Example: crab: {storage_site: T3_CH_CERNBOX, " "out_lfn_base: /store/user/$USER/FLAF}" ) return str(site), str(lfn) From 79c77275044c96017db743e475830088f8f332e7 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 00:15:52 +0200 Subject: [PATCH 04/35] enforce minimum CRAB job runtime for bundle download --- run_tools/law_customizations.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 42b56027..695e2c00 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1193,13 +1193,15 @@ def crab_job_config(self, config, job_nums, branches=None): if n_cpus > 1: config.crab.JobType.numCores = n_cpus - # Optional runtime limit (hours → minutes) + # Runtime limit (hours → minutes). CRAB jobs must download/unpack bundles before + # the payload starts, so enforce a floor (default 60 min) even when the task's + # max_runtime is tiny (e.g. HelloWorld 0.1 h would otherwise be 6 min). max_runtime = getattr(self, "max_runtime", None) if max_runtime is not None and float(max_runtime) > 0: try: - config.crab.JobType.maxJobRuntimeMin = int( - math.floor(float(max_runtime) * 60) - ) + cfg_floor = int(self._crab_cfg().get("min_runtime_min", 60)) + minutes = max(int(math.floor(float(max_runtime) * 60)), cfg_floor) + config.crab.JobType.maxJobRuntimeMin = minutes except Exception: # Older CRAB clients may not support maxJobRuntimeMin; ignore if rejected later. pass From 67bdf718afc43b3d8a07aea8932602fb64dd2d22 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 00:38:08 +0200 Subject: [PATCH 05/35] ship user_custom to remote workers for CRAB/bundle jobs --- run_tools/law_customizations.py | 46 +++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 695e2c00..0b4806b5 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -150,11 +150,7 @@ def __init__(self, *args, **kwargs): super(Task, self).__init__(*args, **kwargs) user_custom_file = None if self.user_custom: - user_custom_file = self.user_custom - if not os.path.isabs(user_custom_file): - user_custom_file = os.path.join( - os.getenv("ANALYSIS_PATH"), user_custom_file - ) + user_custom_file = self._resolve_user_custom_path(self.user_custom) self.setup = Setup.getGlobal( os.getenv("ANALYSIS_PATH"), self.period, @@ -165,6 +161,44 @@ def __init__(self, *args, **kwargs): customisations=self.customisations, user_custom_file=user_custom_file, ) + + @staticmethod + def _resolve_user_custom_path(user_custom): + """Resolve --user-custom on submit host or remote worker. + + Absolute paths from the submit host are not available on CRAB/HTCondor + bundle workers. Remote jobs ship the file as a job input (basename only); + fall back to LAW_JOB_INIT_DIR / job home / ANALYSIS_PATH when the original + path is missing. + """ + path = user_custom + if not os.path.isabs(path): + ana = os.getenv("ANALYSIS_PATH") or "" + path = os.path.join(ana, path) if ana else path + if path and os.path.isfile(path): + return path + base = os.path.basename(user_custom) + for candidate in ( + base, + os.path.join(os.environ.get("LAW_JOB_INIT_DIR", ""), base), + os.path.join(os.environ.get("LAW_JOB_HOME", ""), base), + os.path.join(os.getenv("ANALYSIS_PATH") or "", base), + ): + if candidate and os.path.isfile(candidate): + return candidate + return path + + def _stage_user_custom_input(self, config): + """Ship user_custom yaml as a job input for remote workers (bundle/CRAB).""" + if not self.user_custom: + return + path = self.user_custom + if not os.path.isabs(path): + path = os.path.join(os.getenv("ANALYSIS_PATH") or "", path) + if not path or not os.path.isfile(path): + return + # Avoid absolute remote paths: worker will open by basename via _resolve_user_custom_path. + config.input_files["user_custom"] = path self._dataset_id_name_list = None self._dataset_id_name_dict = None self._dataset_name_id_dict = None @@ -732,6 +766,7 @@ def htcondor_job_file_factory_cls(self): def htcondor_job_config(self, config, job_num, branches): self._apply_bootstrap_path_render_variables(config) + self._stage_user_custom_input(config) # force to run on AlmaLinux9, https://batchdocs.web.cern.ch/local/submit.html config.custom_content.append( @@ -1180,6 +1215,7 @@ def crab_job_config(self, config, job_nums, branches=None): self._apply_bootstrap_path_render_variables(config) self._apply_bundle_render_variables(config) + self._stage_user_custom_input(config) log_remote_base_url = self._log_remote_base_url() config.render_variables["log_remote_base_url"] = log_remote_base_url From c8f736ed7918e06031abf92d3dfd89eb1fb9db16 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 01:13:11 +0200 Subject: [PATCH 06/35] fix: resolve hashed user_custom job inputs on CRAB workers --- AnaProd/tasks.py | 4 ++- Analysis/tasks.py | 14 ++++----- run_tools/law_customizations.py | 52 ++++++++++++++++++++++++--------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/AnaProd/tasks.py b/AnaProd/tasks.py index 61de46aa..7e86d9f2 100644 --- a/AnaProd/tasks.py +++ b/AnaProd/tasks.py @@ -330,7 +330,9 @@ def run(self): shutil.rmtree(job_home) -class AnaTupleFileListBuilderTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): +class AnaTupleFileListBuilderTask( + Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow +): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 24.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) bundle_flavours = ["core", "inputFileList"] diff --git a/Analysis/tasks.py b/Analysis/tasks.py index 1df044f2..72f82bc9 100644 --- a/Analysis/tasks.py +++ b/Analysis/tasks.py @@ -516,7 +516,9 @@ def _split_merged_marker(split_target): return split_target.sibling(split_target.basename + ".merged", type="f") -class HistFromNtupleProducerTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): +class HistFromNtupleProducerTask( + Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow +): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 10.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 2) variables = luigi.Parameter(default="") @@ -689,11 +691,7 @@ def run(self): HistFromNtupleProducer = os.path.join( self._flaf_root(), "Analysis", "HistProducerFromNTuple.py" ) - nMT = ( - self.n_cpus * 2 - if self.effective_workflow in ("htcondor", "crab") - else 8 - ) + nMT = self.n_cpus * 2 if self.effective_workflow in ("htcondor", "crab") else 8 # Determine which variables still need to be produced for this (dataset, chunk). outputs = self.output() @@ -1527,7 +1525,9 @@ def bool_flag(key, default): ps_call(cmd, verbose=1) -class AnalysisCacheAggregationTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): +class AnalysisCacheAggregationTask( + Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow +): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 2.0) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) producer_to_aggregate = luigi.Parameter() diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 0b4806b5..260e3fa4 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -167,9 +167,9 @@ def _resolve_user_custom_path(user_custom): """Resolve --user-custom on submit host or remote worker. Absolute paths from the submit host are not available on CRAB/HTCondor - bundle workers. Remote jobs ship the file as a job input (basename only); - fall back to LAW_JOB_INIT_DIR / job home / ANALYSIS_PATH when the original - path is missing. + bundle workers. Remote jobs ship the file as a job input; law may rename + it with a hash suffix (``name_.yaml``). Search common job dirs for + the exact basename or a hashed variant. """ path = user_custom if not os.path.isabs(path): @@ -178,14 +178,35 @@ def _resolve_user_custom_path(user_custom): if path and os.path.isfile(path): return path base = os.path.basename(user_custom) - for candidate in ( - base, - os.path.join(os.environ.get("LAW_JOB_INIT_DIR", ""), base), - os.path.join(os.environ.get("LAW_JOB_HOME", ""), base), - os.path.join(os.getenv("ANALYSIS_PATH") or "", base), - ): - if candidate and os.path.isfile(candidate): - return candidate + stem, ext = os.path.splitext(base) + search_dirs = [ + "", + os.environ.get("LAW_JOB_INIT_DIR", ""), + os.environ.get("LAW_JOB_HOME", ""), + os.getenv("ANALYSIS_PATH") or "", + "/srv", + ] + for d in search_dirs: + candidates = [] + if d: + candidates.append(os.path.join(d, base)) + else: + candidates.append(base) + for candidate in candidates: + if candidate and os.path.isfile(candidate): + return candidate + # Hashed JobInputFile variants: stem_.ext + dir_path = d if d else "." + if not os.path.isdir(dir_path): + continue + try: + for name in os.listdir(dir_path): + if name.startswith(stem + "_") and name.endswith(ext): + full = os.path.join(dir_path, name) + if os.path.isfile(full): + return full + except OSError: + pass return path def _stage_user_custom_input(self, config): @@ -197,8 +218,13 @@ def _stage_user_custom_input(self, config): path = os.path.join(os.getenv("ANALYSIS_PATH") or "", path) if not path or not os.path.isfile(path): return - # Avoid absolute remote paths: worker will open by basename via _resolve_user_custom_path. - config.input_files["user_custom"] = path + from law.job.base import JobInputFile + + # share=True, increment=False keeps a stable basename when possible; resolve + # still accepts law's hashed names if increment is forced elsewhere. + config.input_files["user_custom"] = JobInputFile( + path=path, copy=True, share=True, render=False, increment=False + ) self._dataset_id_name_list = None self._dataset_id_name_dict = None self._dataset_name_id_dict = None From d03593c384a0a9da8969d25767752cf4f5894dae Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 01:33:52 +0200 Subject: [PATCH 07/35] fix: return absolute paths when resolving staged user_custom --- run_tools/law_customizations.py | 51 +++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 260e3fa4..c1c08971 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -167,44 +167,53 @@ def _resolve_user_custom_path(user_custom): """Resolve --user-custom on submit host or remote worker. Absolute paths from the submit host are not available on CRAB/HTCondor - bundle workers. Remote jobs ship the file as a job input; law may rename - it with a hash suffix (``name_.yaml``). Search common job dirs for - the exact basename or a hashed variant. + bundle workers. Remote jobs ship the file as a job input; law renames + it with a content-hash suffix (``name_.yaml``). Search common job + dirs for the exact basename or a hashed variant. + + Always returns an absolute path when a file is found: a relative hit + (e.g. ``./name_hash.yaml`` from CWD) must not be re-joined onto + ANALYSIS_PATH by Setup/Config. """ + + def _abs_if_file(p): + if p and os.path.isfile(p): + return os.path.abspath(p) + return None + path = user_custom if not os.path.isabs(path): ana = os.getenv("ANALYSIS_PATH") or "" path = os.path.join(ana, path) if ana else path - if path and os.path.isfile(path): - return path + found = _abs_if_file(path) + if found: + return found base = os.path.basename(user_custom) stem, ext = os.path.splitext(base) + # Prefer job input locations over CWD/ANALYSIS_PATH (bundle cwd would + # incorrectly win a relative match that Setup then re-roots under bundle). search_dirs = [ - "", os.environ.get("LAW_JOB_INIT_DIR", ""), os.environ.get("LAW_JOB_HOME", ""), - os.getenv("ANALYSIS_PATH") or "", "/srv", + os.getcwd(), + os.getenv("ANALYSIS_PATH") or "", ] for d in search_dirs: - candidates = [] - if d: - candidates.append(os.path.join(d, base)) - else: - candidates.append(base) - for candidate in candidates: - if candidate and os.path.isfile(candidate): - return candidate + if not d: + continue + found = _abs_if_file(os.path.join(d, base)) + if found: + return found # Hashed JobInputFile variants: stem_.ext - dir_path = d if d else "." - if not os.path.isdir(dir_path): + if not os.path.isdir(d): continue try: - for name in os.listdir(dir_path): + for name in os.listdir(d): if name.startswith(stem + "_") and name.endswith(ext): - full = os.path.join(dir_path, name) - if os.path.isfile(full): - return full + found = _abs_if_file(os.path.join(d, name)) + if found: + return found except OSError: pass return path From 0917f551d949009847a942e97ff3f6c7b1f59fcf Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 01:48:34 +0200 Subject: [PATCH 08/35] fix: restore dataset id/name map init after user_custom resolve --- run_tools/law_customizations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index c1c08971..628b4269 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -161,6 +161,9 @@ def __init__(self, *args, **kwargs): customisations=self.customisations, user_custom_file=user_custom_file, ) + self._dataset_id_name_list = None + self._dataset_id_name_dict = None + self._dataset_name_id_dict = None @staticmethod def _resolve_user_custom_path(user_custom): From ecef1d36ab7f9548666feace8b076e9c7a52b744 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 03:42:47 +0200 Subject: [PATCH 09/35] fix: do not set CRAB numCores (PSet threads always 1) --- run_tools/law_customizations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 628b4269..edbc7515 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1258,14 +1258,15 @@ def crab_job_config(self, config, job_nums, branches=None): log_remote_base_url = self._log_remote_base_url() config.render_variables["log_remote_base_url"] = log_remote_base_url - # Memory / cores + # Memory. Do not set JobType.numCores: law's dummy CMSSW PSet leaves + # process.options.numberOfThreads at 1, and modern CRAB rejects a mismatch + # (exit 192). FLAF payloads are not multi-threaded CMSSW jobs; scale memory + # with the task's n_cpus instead. n_cpus = int(getattr(self, "n_cpus", 1) or 1) mem = int(self.crab_memory) if mem <= 0: mem = int(self._crab_cfg().get("max_memory_mb", n_cpus * 2000)) config.crab.JobType.maxMemoryMB = mem - if n_cpus > 1: - config.crab.JobType.numCores = n_cpus # Runtime limit (hours → minutes). CRAB jobs must download/unpack bundles before # the payload starts, so enforce a floor (default 60 min) even when the task's From ba21a730780a6f6a37c06e855fbe562aa5a90587 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 03:47:52 +0200 Subject: [PATCH 10/35] fix: cap CRAB single-core job memory to 2500 MB --- run_tools/law_customizations.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index edbc7515..9197f212 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1260,12 +1260,15 @@ def crab_job_config(self, config, job_nums, branches=None): # Memory. Do not set JobType.numCores: law's dummy CMSSW PSet leaves # process.options.numberOfThreads at 1, and modern CRAB rejects a mismatch - # (exit 192). FLAF payloads are not multi-threaded CMSSW jobs; scale memory - # with the task's n_cpus instead. + # (exit 192). Without multi-core, CRAB also caps memory (~3 GB for 1-core + # jobs), so clamp to that ceiling. FLAF payloads are not multi-threaded CMSSW. n_cpus = int(getattr(self, "n_cpus", 1) or 1) mem = int(self.crab_memory) if mem <= 0: mem = int(self._crab_cfg().get("max_memory_mb", n_cpus * 2000)) + single_core_cap = int(self._crab_cfg().get("max_memory_mb_single_core", 2500)) + if mem > single_core_cap: + mem = single_core_cap config.crab.JobType.maxMemoryMB = mem # Runtime limit (hours → minutes). CRAB jobs must download/unpack bundles before From 909bfd741b752ca45c359c911180af3eb74cd583 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 04:00:49 +0200 Subject: [PATCH 11/35] fix: resolve staged user_custom in Setup for remote subprocesses --- Common/Setup.py | 63 +++++++++++++++++++++++++++++---- run_tools/law_customizations.py | 54 ++-------------------------- 2 files changed, 59 insertions(+), 58 deletions(-) diff --git a/Common/Setup.py b/Common/Setup.py index 640b93ee..181693cb 100644 --- a/Common/Setup.py +++ b/Common/Setup.py @@ -10,6 +10,57 @@ from FLAF.Common.Utilities import create_processor_instances +def resolve_user_custom_path(user_custom): + """Resolve --user-custom on the submit host or a remote worker. + + Absolute submit-host paths are missing on CRAB/HTCondor bundle workers. + Jobs stage the file as a job input; law renames it with a content-hash + suffix (``name_.yaml``). Search common job dirs for the basename or + a hashed variant. Returns an absolute path when a file is found. + """ + + def _abs_if_file(p): + if p and os.path.isfile(p): + return os.path.abspath(p) + return None + + if not user_custom: + return user_custom + path = user_custom + if not os.path.isabs(path): + ana = os.getenv("ANALYSIS_PATH") or "" + path = os.path.join(ana, path) if ana else path + found = _abs_if_file(path) + if found: + return found + base = os.path.basename(user_custom) + stem, ext = os.path.splitext(base) + search_dirs = [ + os.environ.get("LAW_JOB_INIT_DIR", ""), + os.environ.get("LAW_JOB_HOME", ""), + "/srv", + os.getcwd(), + os.getenv("ANALYSIS_PATH") or "", + ] + for d in search_dirs: + if not d: + continue + found = _abs_if_file(os.path.join(d, base)) + if found: + return found + if not os.path.isdir(d): + continue + try: + for name in os.listdir(d): + if name.startswith(stem + "_") and name.endswith(ext): + found = _abs_if_file(os.path.join(d, name)) + if found: + return found + except OSError: + pass + return path + + def select_items(all_items, filters): def name_match(name, pattern): if pattern[0] == "^": @@ -265,12 +316,12 @@ def __init__( self.period = period self.law_run_version = law_run_version - # Resolve a relative user_custom file against the analysis path. The path is passed - # through to subprocesses (e.g. HistTupleProducer.py) verbatim, where the working - # directory is not the analysis directory (on HTCondor it is the job scratch dir), so - # it must be made absolute here for every caller, not only in Task.__init__. - if user_custom_file is not None and not os.path.isabs(user_custom_file): - user_custom_file = os.path.join(ana_path, user_custom_file) + # Resolve user_custom for every caller (Task and standalone scripts). Relative + # paths are rooted under ANALYSIS_PATH; absolute submit-host paths that are + # missing on remote workers are remapped to staged job inputs (see + # resolve_user_custom_path). + if user_custom_file: + user_custom_file = resolve_user_custom_path(user_custom_file) self.config_path_order = [ os.path.join(ana_path, "FLAF", "config"), diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 9197f212..cebfd8cb 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -167,59 +167,9 @@ def __init__(self, *args, **kwargs): @staticmethod def _resolve_user_custom_path(user_custom): - """Resolve --user-custom on submit host or remote worker. + from FLAF.Common.Setup import resolve_user_custom_path - Absolute paths from the submit host are not available on CRAB/HTCondor - bundle workers. Remote jobs ship the file as a job input; law renames - it with a content-hash suffix (``name_.yaml``). Search common job - dirs for the exact basename or a hashed variant. - - Always returns an absolute path when a file is found: a relative hit - (e.g. ``./name_hash.yaml`` from CWD) must not be re-joined onto - ANALYSIS_PATH by Setup/Config. - """ - - def _abs_if_file(p): - if p and os.path.isfile(p): - return os.path.abspath(p) - return None - - path = user_custom - if not os.path.isabs(path): - ana = os.getenv("ANALYSIS_PATH") or "" - path = os.path.join(ana, path) if ana else path - found = _abs_if_file(path) - if found: - return found - base = os.path.basename(user_custom) - stem, ext = os.path.splitext(base) - # Prefer job input locations over CWD/ANALYSIS_PATH (bundle cwd would - # incorrectly win a relative match that Setup then re-roots under bundle). - search_dirs = [ - os.environ.get("LAW_JOB_INIT_DIR", ""), - os.environ.get("LAW_JOB_HOME", ""), - "/srv", - os.getcwd(), - os.getenv("ANALYSIS_PATH") or "", - ] - for d in search_dirs: - if not d: - continue - found = _abs_if_file(os.path.join(d, base)) - if found: - return found - # Hashed JobInputFile variants: stem_.ext - if not os.path.isdir(d): - continue - try: - for name in os.listdir(d): - if name.startswith(stem + "_") and name.endswith(ext): - found = _abs_if_file(os.path.join(d, name)) - if found: - return found - except OSError: - pass - return path + return resolve_user_custom_path(user_custom) def _stage_user_custom_input(self, config): """Ship user_custom yaml as a job input for remote workers (bundle/CRAB).""" From 5dc69474202b68b9a9ecd463a2b92376e978b948 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 8 Aug 2026 19:07:23 +0200 Subject: [PATCH 12/35] fix: materialize RDF hists without ImplicitMT in HistFromNtuple --- Analysis/HistProducerFromNTuple.py | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/Analysis/HistProducerFromNTuple.py b/Analysis/HistProducerFromNTuple.py index 1df76138..dee00529 100644 --- a/Analysis/HistProducerFromNTuple.py +++ b/Analysis/HistProducerFromNTuple.py @@ -33,6 +33,20 @@ def find_keys(inFiles_list): return sorted(unique_keys) +def _materialize_hist(unit_hist): + """Turn an RDF RResultPtr[TH*] into a concrete histogram. + + Do not use ``hasattr`` on an unevaluated RResultPtr: under current ROOT/cppyy + that can trigger ``operator*`` via a broken attribute probe and fail with + misleading Filter/column errors (e.g. Legacy_region) even when the graph is + valid. ``GetValue()`` is the supported materialization path. + """ + get_value = getattr(unit_hist, "GetValue", None) + if callable(get_value): + return get_value() + return unit_hist + + def SaveHist(key_tuple, outFile, hist_list, hist_name, unc, scale, verbose=0): model, unit_hist, rdf = hist_list[0] if verbose > 0: @@ -42,16 +56,15 @@ def SaveHist(key_tuple, outFile, hist_list, hist_name, unc, scale, verbose=0): dir_name = "/".join(key_tuple) dir_ptr = Utilities.mkdir(outFile, dir_name) + unit_hist = _materialize_hist(unit_hist) merged_hist = model.GetHistogram().Clone() # Detach from the current ROOT directory: the histogram is persisted explicitly via # WriteTObject below, so it must not also be auto-flushed into the output file's root # (which would leave one stray, unnamed histogram per call when writing directly). merged_hist.SetDirectory(0) - N_bins = ( - unit_hist.GetNbins() - if hasattr(unit_hist, "GetNbins") - else unit_hist.GetNcells() - ) + # GetNcells covers TH1/TH2/TH3 (incl. under/overflow); do not probe GetNbins via + # hasattr on RResultPtr (see _materialize_hist). + N_bins = unit_hist.GetNcells() for i in range(0, N_bins): bin_content = unit_hist.GetBinContent(i) bin_error = unit_hist.GetBinError(i) @@ -61,6 +74,7 @@ def SaveHist(key_tuple, outFile, hist_list, hist_name, unc, scale, verbose=0): nentries = unit_hist.GetEntries() if len(hist_list) > 1: for model, unit_hist in hist_list[1:]: + unit_hist = _materialize_hist(unit_hist) hist = model.GetHistogram() for i in range(0, N_bins): bin_content = unit_hist.GetBinContent(i) @@ -201,7 +215,16 @@ def BuildAllHistActions( ) args = parser.parse_args() - ROOT.EnableImplicitMT(args.nMT) + # Do not call EnableImplicitMT here. Multi-tree RDataFrame graphs that Filter + # on bool region columns (e.g. Legacy_region) fail at evaluation under + # ImplicitMT — including EnableImplicitMT(1) — with TTreeReader errors. This + # producer books Central + weight + shape (shifted-tree) actions together, so + # it must stay single-threaded until ROOT fixes that combination. + if args.nMT and args.nMT > 1: + print( + f"Note: --nMT={args.nMT} ignored; HistProducerFromNTuple runs without " + "ROOT ImplicitMT (multi-tree region filters)." + ) start = time.time() From 59da4693936e22cec378b94a4e39457da528ce37 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sun, 9 Aug 2026 00:14:40 +0200 Subject: [PATCH 13/35] test: HelloWorld grid X509 download/rucio probe options --- test/hello_world_task.py | 152 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 2 deletions(-) diff --git a/test/hello_world_task.py b/test/hello_world_task.py index d3f19691..c49e7ac5 100644 --- a/test/hello_world_task.py +++ b/test/hello_world_task.py @@ -1,3 +1,8 @@ +import json +import os +import tempfile +import traceback + import law import luigi @@ -10,7 +15,7 @@ class HelloWorldTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): - max_runtime = copy_param(HTCondorWorkflow.max_runtime, 0.1) + max_runtime = copy_param(HTCondorWorkflow.max_runtime, 0.5) n_cpus = copy_param(HTCondorWorkflow.n_cpus, 1) poll_interval = copy_param(HTCondorWorkflow.poll_interval, 1) bundle_flavours = ["core"] @@ -19,6 +24,22 @@ class HelloWorldTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): significant=False, description="raise an exception in run() to test log transfer on crash", ) + # Remote X509 / grid access probe (significant so different probes do not collide). + download_url = luigi.Parameter( + default="", + significant=True, + description="if set, probe VOMS/X509 access: gfal_stat/sum and download when small", + ) + max_download_bytes = luigi.IntParameter( + default=20 * 1024 * 1024, + significant=False, + description="full download only if remote size is at most this many bytes", + ) + test_rucio = luigi.BoolParameter( + default=False, + significant=True, + description="if true, attempt Rucio Client() authentication on the worker", + ) def create_branch_map(self): return {0: "hello"} @@ -28,12 +49,139 @@ def output(self): self.version, self.__class__.__name__, self.period, "hello_world_done.txt" ) + def _probe_grid_access(self): + """Return a JSON-serializable report of X509/gfal/Rucio status on this host.""" + from FLAF.RunKit.grid_tools import ( + get_voms_proxy_info, + gfal_stat, + gfal_sum, + gfal_copy, + get_rucio_client, + copy_remote_file, + ) + + report = { + "host": os.uname().nodename if hasattr(os, "uname") else "unknown", + "cwd": os.getcwd(), + "X509_USER_PROXY": os.environ.get("X509_USER_PROXY"), + "proxy_file_exists": False, + "voms_proxy_info": None, + "voms_proxy_error": None, + "download_url": self.download_url, + "gfal_stat": None, + "gfal_stat_error": None, + "gfal_sum": None, + "gfal_sum_error": None, + "download": None, + "download_error": None, + "rucio": None, + "rucio_error": None, + } + + proxy = os.environ.get("X509_USER_PROXY", "") + report["proxy_file_exists"] = bool(proxy) and os.path.isfile(proxy) + + try: + info = get_voms_proxy_info() + # keep only plain types + report["voms_proxy_info"] = { + k: (float(v) if k == "timeleft" else str(v)) + for k, v in info.items() + if k + in ( + "path", + "timeleft", + "identity", + "issuer", + "type", + "strength", + "VO", + ) + } + except Exception as e: + report["voms_proxy_error"] = f"{type(e).__name__}: {e}" + + if self.download_url: + try: + st = gfal_stat(self.download_url) + report["gfal_stat"] = { + k: (int(v) if k == "size" else str(v)) for k, v in st.items() + } + except Exception as e: + report["gfal_stat_error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + + try: + asum = gfal_sum(self.download_url, sum_type="adler32") + report["gfal_sum"] = {"adler32": str(asum)} + except Exception as e: + report["gfal_sum_error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + + size = None + if report.get("gfal_stat") and "size" in report["gfal_stat"]: + size = int(report["gfal_stat"]["size"]) + do_full = size is not None and size <= int(self.max_download_bytes) + report["download"] = { + "attempted": do_full, + "reason": ( + "size_ok" + if do_full + else ( + "size_unknown" + if size is None + else f"size {size} > max_download_bytes {self.max_download_bytes}" + ) + ), + "local_size": None, + } + if do_full: + try: + tmpdir = tempfile.mkdtemp(prefix="hello_dl_") + local = os.path.join(tmpdir, "download.bin") + # Protocol URLs (davs/root/srm): gfal_copy. Pure /store/ LFNs: Rucio path. + if self.download_url.startswith("/store/"): + copy_remote_file(self.download_url, local, verbose=1) + else: + gfal_copy(self.download_url, local, verbose=1) + report["download"]["local_size"] = os.path.getsize(local) + report["download"]["ok"] = True + except Exception as e: + report["download"]["ok"] = False + report["download_error"] = ( + f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + ) + + if self.test_rucio: + try: + client = get_rucio_client() + # Lightweight authenticated call: whoami if available, else ping list. + who = None + if hasattr(client, "whoami"): + who = client.whoami() + report["rucio"] = { + "ok": True, + "whoami": str(who) if who is not None else None, + "account": os.environ.get("RUCIO_ACCOUNT"), + } + except Exception as e: + report["rucio"] = {"ok": False} + report["rucio_error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + + return report + def run(self): if self.force_fail: raise RuntimeError( f"Forced failure for testing log transfer on crash. version = {self.version}" ) print(f"hello world from {self.version}") + + body = "done\n" + if self.download_url or self.test_rucio: + report = self._probe_grid_access() + print("=== grid access probe report ===") + print(json.dumps(report, indent=2, default=str)) + body = json.dumps(report, indent=2, default=str) + "\n" + with self.output().localize("w") as tmp: with open(tmp.path, "w") as f: - f.write("done\n") + f.write(body) From b3f79bfa52c3dc1bee3713665f37bbfb52dbdcc9 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sun, 9 Aug 2026 01:04:15 +0200 Subject: [PATCH 14/35] fix: set RUCIO_ACCOUNT in remote bootstrap for CRAB pilots --- bootstrap.sh | 23 +++++++++++++++++++++++ run_tools/law_customizations.py | 5 +++++ 2 files changed, 28 insertions(+) diff --git a/bootstrap.sh b/bootstrap.sh index 9e7b0519..83e114c9 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -29,6 +29,24 @@ action() { export X509_USER_PROXY="${LAW_JOB_INIT_DIR}/voms.proxy" fi + # Rucio: pilot accounts (e.g. CRAB cmsplt01) are not valid RUCIO_ACCOUNT values. + # Prefer submit-time account (rendered below), then VOMS DN username, never pilot USER. + local rucio_account_submit="{{rucio_account}}" + if [ -z "${RUCIO_ACCOUNT:-}" ]; then + if [ -n "${rucio_account_submit}" ]; then + export RUCIO_ACCOUNT="${rucio_account_submit}" + elif [ -n "${X509_USER_PROXY:-}" ] && command -v voms-proxy-info >/dev/null 2>&1; then + # Standard CERN user cert: .../OU=Users/CN=/CN=/... + export RUCIO_ACCOUNT="$(voms-proxy-info -identity 2>/dev/null \ + | sed -n 's|.*/OU=Users/CN=\([^/]*\).*|\1|p' | head -1)" + fi + if [ -n "${RUCIO_ACCOUNT:-}" ]; then + echo "bootstrap: RUCIO_ACCOUNT=${RUCIO_ACCOUNT}" + else + echo "bootstrap: WARNING: RUCIO_ACCOUNT unset (Rucio client may fail on this worker)" + fi + fi + if [ -n "${bundle_list}" ]; then local lcg_setup="/cvmfs/sft.cern.ch/lcg/views/LCG_108a/x86_64-el9-gcc15-opt/setup.sh" if [ -f "${lcg_setup}" ]; then @@ -117,5 +135,10 @@ action() { [ -n "${corrections_path}" ] && export CORRECTIONS_PATH="${corrections_path}" source "${analysis_path}/env.sh" fi + + # Re-assert after env.sh (Rucio cvmfs init may re-read USER on some pilots). + if [ -n "${rucio_account_submit}" ]; then + export RUCIO_ACCOUNT="${rucio_account_submit}" + fi } action diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index cebfd8cb..1de37a9e 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -703,6 +703,11 @@ def _apply_bootstrap_path_render_variables(self, config): corrections_path = os.getenv("CORRECTIONS_PATH", "") or "" config.render_variables["flaf_path"] = flaf_path config.render_variables["corrections_path"] = corrections_path + # Rucio account for workers: CRAB pilots have USER=cmsplt01, which is not a Rucio + # account. Bake the submitter account so bootstrap can export RUCIO_ACCOUNT. + config.render_variables["rucio_account"] = ( + os.environ.get("RUCIO_ACCOUNT") or os.environ.get("USER") or "" + ) runTokenServer = self.global_params.get("runTokenServer", None) if runTokenServer and not self._uses_bundles(): From 6309dcee4d1f220994c78a3fedf959565efe3585 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sun, 9 Aug 2026 02:09:58 +0200 Subject: [PATCH 15/35] fix: multi-core CRAB PSet+memory for AnaTuple OOM --- run_tools/law_customizations.py | 61 ++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 1de37a9e..03dfad66 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1102,6 +1102,34 @@ class CrabWorkflow(law.cms.CrabWorkflow): def _crab_cfg(self): return self.global_params.get("crab") or {} + def _ensure_crab_pset(self, n_threads): + """Write a minimal CRAB PSet with numberOfThreads matching JobType.numCores.""" + n_threads = max(1, int(n_threads)) + out_dir = self.local_path() + os.makedirs(out_dir, exist_ok=True) + path = os.path.join(out_dir, f"crab_PSet_threads{n_threads}.py") + content = f"""# Auto-generated by FLAF for CRAB (threads must match JobType.numCores). +import FWCore.ParameterSet.Config as cms + +process = cms.Process("LAW") +process.source = cms.Source("PoolSource", fileNames=cms.untracked.vstring([""])) +process.output = cms.OutputModule( + "PoolOutputModule", fileName=cms.untracked.string("out.root") +) +process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(1)) +process.options = cms.untracked.PSet( + allowUnscheduled=cms.untracked.bool(True), + wantSummary=cms.untracked.bool(False), + numberOfThreads=cms.untracked.uint32({n_threads}), + numberOfStreams=cms.untracked.uint32(0), +) +process.out = cms.EndPath(process.output) +""" + if (not os.path.exists(path)) or open(path).read() != content: + with open(path, "w") as f: + f.write(content) + return path + def crab_stageout_location(self): """Return (storageSite, outLFNDirBase) required by the CRAB client. @@ -1213,17 +1241,32 @@ def crab_job_config(self, config, job_nums, branches=None): log_remote_base_url = self._log_remote_base_url() config.render_variables["log_remote_base_url"] = log_remote_base_url - # Memory. Do not set JobType.numCores: law's dummy CMSSW PSet leaves - # process.options.numberOfThreads at 1, and modern CRAB rejects a mismatch - # (exit 192). Without multi-core, CRAB also caps memory (~3 GB for 1-core - # jobs), so clamp to that ceiling. FLAF payloads are not multi-threaded CMSSW. - n_cpus = int(getattr(self, "n_cpus", 1) or 1) + # Cores + memory. CRAB enforces ~2500 MB per core and requires + # JobType.numCores == process.options.numberOfThreads in the PSet. + # AnaTuple/CMSSW jobs need several GB → request enough cores and bake a + # matching PSet (law's default PSet always has threads=1). + n_cpus = max(1, int(getattr(self, "n_cpus", 1) or 1)) mem = int(self.crab_memory) if mem <= 0: - mem = int(self._crab_cfg().get("max_memory_mb", n_cpus * 2000)) - single_core_cap = int(self._crab_cfg().get("max_memory_mb_single_core", 2500)) - if mem > single_core_cap: - mem = single_core_cap + mem = int(self._crab_cfg().get("max_memory_mb", n_cpus * 2500)) + # Production tasks that routinely OOM at 2.5 GB on CRAB. + heavy = self.__class__.__name__ in ( + "AnaTupleFileTask", + "AnaTupleMergeTask", + "HistTupleProducerTask", + "HistFromNtupleProducerTask", + ) + if heavy and mem < 8000: + mem = 8000 + mb_per_core = int(self._crab_cfg().get("max_memory_mb_per_core", 2500)) + max_cores = int(self._crab_cfg().get("max_cores", 8)) + n_cores = max(n_cpus, (mem + mb_per_core - 1) // mb_per_core) + n_cores = max(1, min(n_cores, max_cores)) + mem = max(mem, n_cores * mb_per_core) + # Write a per-thread-count PSet next to the analysis job dir. + pset_path = self._ensure_crab_pset(n_cores) + config.crab.JobType.psetName = pset_path + config.crab.JobType.numCores = n_cores config.crab.JobType.maxMemoryMB = mem # Runtime limit (hours → minutes). CRAB jobs must download/unpack bundles before From 6155e73d695bb85ab9ccb916bcec6e08893827cf Mon Sep 17 00:00:00 2001 From: Konstantin Date: Wed, 12 Aug 2026 15:42:06 +0200 Subject: [PATCH 16/35] =?UTF-8?q?fix:=20CRAB=20CI=20=E2=80=94=20CMSSW=20re?= =?UTF-8?q?locate,=20user=5Fcustom=20for=20AnalysisCache,=20variables?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Analysis/AnalysisCacheProducer.py | 7 +++- Analysis/tasks.py | 8 +++++ Common/Setup.py | 37 +++++++++++++++++---- RunKit/includeCMSSWlibs.py | 11 ++++++- bootstrap.sh | 55 +++++++++++++++++++++++++++++-- 5 files changed, 107 insertions(+), 11 deletions(-) diff --git a/Analysis/AnalysisCacheProducer.py b/Analysis/AnalysisCacheProducer.py index f035038f..ac14c4f0 100644 --- a/Analysis/AnalysisCacheProducer.py +++ b/Analysis/AnalysisCacheProducer.py @@ -269,7 +269,12 @@ def createAnalysisCache( parser.add_argument("--workingDir", required=True, type=str) parser.add_argument("--histTupleDef", type=str) parser.add_argument("--LAWrunVersion", required=True, type=str) - parser.add_argument("--user-custom", type=str, default=None) + parser.add_argument( + "--user-custom", + type=str, + default=None, + help="Optional user_custom yaml (staged on remote workers; resolved by Setup).", + ) args = parser.parse_args() startTime = time.time() diff --git a/Analysis/tasks.py b/Analysis/tasks.py index 72f82bc9..f33582fe 100644 --- a/Analysis/tasks.py +++ b/Analysis/tasks.py @@ -1247,6 +1247,14 @@ def run(self): analysisCacheProducer_cmd.extend( ["--cacheFiles", local_anacaches_str] ) + if self.user_custom: + # Must pass user_custom into the producer subprocess: Setup inside + # AnalysisCacheProducer is independent of the law Task Setup. Without + # this, keys only present in user_custom (e.g. compute_unc_histograms) + # are missing and BtagShape/histTupleDef raise KeyError on workers. + analysisCacheProducer_cmd.extend( + ["--user-custom", self.user_custom] + ) # Check if cmssw env is required prod_env = ( self.cmssw_env diff --git a/Common/Setup.py b/Common/Setup.py index 181693cb..bc2c8011 100644 --- a/Common/Setup.py +++ b/Common/Setup.py @@ -426,12 +426,37 @@ def __init__( self.histTuple_flavor = self.global_params["histTuple_flavor"] print(f"Using histTuple flavor {self.histTuple_flavor}") - self.histTuple_plot_vars = self.global_params["histTuple_flavors"][ - self.histTuple_flavor - ]["variables"] - self.histTuple_fullres_vars = self.global_params["histTuple_flavors"][ - self.histTuple_flavor - ]["fullResolution_variables"] + self.histTuple_plot_vars = list( + self.global_params["histTuple_flavors"][self.histTuple_flavor]["variables"] + ) + self.histTuple_fullres_vars = list( + self.global_params["histTuple_flavors"][self.histTuple_flavor][ + "fullResolution_variables" + ] + ) + + # Optional top-level `variables:` from user_custom / global.yaml. + # - When the active flavor already lists variables: treat as a restriction + # (keep only names in the list) — used by CI user_custom flavors. + # - When the flavor list is empty (e.g. H_mumu default): use the list as the + # active plot/full-res set so user_custom alone can drive a short CI chain + # without requiring histTuple_flavor: CI. + user_vars = self.global_params.get("variables") + if user_vars: + + def _var_name(v): + return v["name"] if isinstance(v, dict) else v + + selected = {_var_name(v) for v in user_vars} + if self.histTuple_plot_vars: + self.histTuple_plot_vars = [ + v for v in self.histTuple_plot_vars if _var_name(v) in selected + ] + self.histTuple_fullres_vars = [ + v for v in self.histTuple_fullres_vars if _var_name(v) in selected + ] + else: + self.histTuple_plot_vars = list(user_vars) # Whether up/down-variation histograms are produced. The histTuple flavor usually # dictates this (uncertainties are only needed for the limit-setting shape variable), diff --git a/RunKit/includeCMSSWlibs.py b/RunKit/includeCMSSWlibs.py index 96eb754d..4b97d68e 100644 --- a/RunKit/includeCMSSWlibs.py +++ b/RunKit/includeCMSSWlibs.py @@ -8,7 +8,16 @@ def includeLibTool(tool="", wantLib=False): command = ["scram", "tool", "info", tool] - directory = os.environ["CMSSW_BASE"] + # Prefer FLAF_CMSSW_BASE (always the analysis soft/CMSSW release, correctly + # relocated on CRAB/HTCondor bundle workers). CMSSW_BASE may still point at the + # submit-host AFS path if scram ProjectRename did not fully re-export the env. + directory = os.environ.get("FLAF_CMSSW_BASE") or os.environ["CMSSW_BASE"] + if not os.path.isdir(directory): + raise FileNotFoundError( + f"CMSSW release directory not found: {directory} " + f"(FLAF_CMSSW_BASE={os.environ.get('FLAF_CMSSW_BASE')!r}, " + f"CMSSW_BASE={os.environ.get('CMSSW_BASE')!r})" + ) returncode, output, err = ps_call( command, catch_stdout=True, cwd=directory, verbose=0 ) diff --git a/bootstrap.sh b/bootstrap.sh index 83e114c9..1f966f27 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -104,20 +104,69 @@ action() { if [ -n "${flaf_cmssw_version}" ]; then local cmssw_dir="${bundle_dir}/soft/${flaf_cmssw_version}" if [ -d "${cmssw_dir}/src" ]; then - echo "bootstrap: relocating CMSSW ${flaf_cmssw_version}" + # Relocate a tarball-shipped CMSSW release to this worker path. + # Do NOT `eval scram runtime` before ProjectRename: that bakes the + # submit-host (AFS) LOCALTOP into the shell as CMSSW_BASE, and + # anaTupleProducer (loadTF / scram tool info) then fails with + # FileNotFoundError on the AFS path (no AFS on CRAB workers). + echo "bootstrap: relocating CMSSW ${flaf_cmssw_version} -> ${cmssw_dir}" source /cvmfs/cms.cern.ch/cmsset_default.sh 2>/dev/null || true local prev_dir="${PWD}" + # Discover submit-host LOCALTOP from the shipped Self file (if any). + local old_localtop="" + if [ -f "${cmssw_dir}/config/Self" ]; then + old_localtop=$(sed -n 's/.*LOCALTOP="\([^"]*\)".*/\1/p' \ + "${cmssw_dir}/config/Self" 2>/dev/null | head -1) + # XML form: + if [ -z "${old_localtop}" ]; then + old_localtop=$(sed -n 's/.*path="\([^"]*\)".*/\1/p' \ + "${cmssw_dir}/config/Self" 2>/dev/null | head -1) + fi + fi cd "${cmssw_dir}/src" - eval "$(scramv1 runtime -sh 2>/dev/null)" || true - scramv1 b ProjectRename "${cmssw_dir}" 2>/dev/null || true + # ProjectRename updates scram's Self/LOCALTOP to cmssw_dir. + if ! scramv1 b ProjectRename "${cmssw_dir}"; then + echo "bootstrap: WARNING: scram ProjectRename failed; applying path sed fallback" + if [ -n "${old_localtop}" ] && [ "${old_localtop}" != "${cmssw_dir}" ]; then + # Escape sed separators in paths. + local old_esc new_esc + old_esc=$(printf '%s' "${old_localtop}" | sed 's/[.[\*^$\/&]/g') + new_esc=$(printf '%s' "${cmssw_dir}" | sed 's/[.[\*^$\/&]/g') + # Patch Self + .SCRAM metadata that embed the submit-host path. + if [ -f "${cmssw_dir}/config/Self" ]; then + sed -i "s|${old_esc}|${new_esc}|g" "${cmssw_dir}/config/Self" + fi + find "${cmssw_dir}/.SCRAM" -type f 2>/dev/null \ + | xargs -r sed -i "s|${old_esc}|${new_esc}|g" + fi + fi + # Drop any AFS-tainted CMSSW_* vars that a prior runtime might have set. + unset CMSSW_BASE CMSSW_RELEASE_BASE CMSSW_SEARCH_PATH RELEASETOP LOCALTOP 2>/dev/null || true cd "${prev_dir}" + # Verify scram now resolves to the bundle path (best-effort). + if [ -n "${old_localtop}" ]; then + local new_self + new_self=$(sed -n 's/.*path="\([^"]*\)".*/\1/p' \ + "${cmssw_dir}/config/Self" 2>/dev/null | head -1) + [ -z "${new_self}" ] && new_self=$(sed -n 's/.*LOCALTOP="\([^"]*\)".*/\1/p' \ + "${cmssw_dir}/config/Self" 2>/dev/null | head -1) + echo "bootstrap: CMSSW Self path now: ${new_self:-unknown}" + fi fi fi fi echo "bootstrap: sourcing env.sh from bundle" + # Ensure no leftover AFS CMSSW_BASE from the pilot / prior steps. + unset CMSSW_BASE CMSSW_RELEASE_BASE RELEASETOP LOCALTOP 2>/dev/null || true export FLAF_NO_INSTALL=1 source "${bundle_dir}/env.sh" + # After env.sh, force CMSSW_BASE to the bundled release when present so + # subprocesses (includeLibTool, HHbtag models) never see an AFS path. + if [ -n "${FLAF_CMSSW_BASE:-}" ] && [ -d "${FLAF_CMSSW_BASE}" ]; then + export CMSSW_BASE="${FLAF_CMSSW_BASE}" + echo "bootstrap: CMSSW_BASE=${CMSSW_BASE}" + fi else if [ "${analysis_path}" = "NONE" ]; then echo "ERROR: analysis_path is NONE but no bundle_list was provided" From e03d8d22306dfd0673d073af8041a48589c8f2c7 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Wed, 12 Aug 2026 15:44:45 +0200 Subject: [PATCH 17/35] docs: user_custom variables/histTuple_flavor behavior --- docs/configuration/user-custom.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/configuration/user-custom.md b/docs/configuration/user-custom.md index 8ec854ab..598e8aff 100644 --- a/docs/configuration/user-custom.md +++ b/docs/configuration/user-custom.md @@ -36,7 +36,8 @@ Replace ``/`` with yours (e.g. `k` / `kandroso`). With just this, | `compute_unc_histograms` | bool | Whether to also fill histograms for those variations. Prefer setting this **per histTuple flavor** in `global.yaml` (`histTuple_flavors..compute_unc_histograms`) — uncertainties are usually only needed for the limit-setting shape flavor, so the flavor should dictate it. The value here (or in `global.yaml`) is used as the fallback when the active flavor does not set it. | | `store_noncentral` | bool | Whether to keep the non-central (systematic-shift) outputs, not just the central one. | | `remove_merged_inputs` | bool | If `true`, `HistMergerTask` deletes each variable's per-chunk split histograms (`HistFromNtupleProducerTask` outputs) after merging, to save space, leaving a tiny per-chunk `.merged` marker in place of each. Safe: the producer stays "complete" for exactly the chunks that were merged (it finds the split *or* its marker), so the task graph stays consistent (no re-run); a chunk that was never produced has no marker and is still produced. Default `false` — intermediates are kept. | -| `variables` | list | Restrict which variables are produced/plotted. Omit for the full set. | +| `variables` | list | Restrict which variables are produced/plotted (applied to the active `histTuple_flavor` list). If that flavor's variable list is empty (e.g. H_mumu `default`), this list is used as the active set. Omit for the full flavor set. | +| `histTuple_flavor` | string | Optional. Selects which `histTuple_flavors` entry drives the variable lists (e.g. `CI` for the short H_mumu CI set). | | `hist_from_ntuple_max_hists` | int | Max histograms `HistFromNtupleProducerTask` books in one RDataFrame pass. The count is variables × selections × (Central + every Up/Down). Default `4000`; `0` disables batching. Lower this (do not raise CI memory) if a job OOMs. | !!! tip "`TestModel` is the fast path" From be0386797ad3ae61934bc02a319f945277e809b4 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Wed, 12 Aug 2026 16:04:49 +0200 Subject: [PATCH 18/35] fix: materialize AFS CMSSW src symlinks in cmssw bundle --- run_tools/law_customizations.py | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 03dfad66..94001958 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -495,6 +495,17 @@ def _get_bundle_source(pat: str) -> str: f"No files found for bundle flavour '{self.flavour}'" ) + # CMSSW analysis customisations (HHbtag, ClassicSVfit, …) are installed as + # absolute AFS symlinks under soft/CMSSW_*/src. On CRAB those targets do not + # exist. Materialize any absolute symlink that points outside the staging + # tree so the tarball is self-contained. Relative / internal links stay. + if self.flavour == "cmssw": + n_mat = self._materialize_external_symlinks(staging) + if n_mat: + print( + f"bundle[cmssw]: materialized {n_mat} external symlink(s)" + ) + subprocess.run( [ "tar", @@ -511,6 +522,51 @@ def _get_bundle_source(pat: str) -> str: ) print(f"bundle[{self.flavour}]: done") + @staticmethod + def _materialize_external_symlinks(root: str) -> int: + """Replace absolute external symlinks under *root* with real file/dir copies. + + Returns the number of symlinks replaced. Relative symlinks and absolute ones + that already resolve inside *root* are left unchanged. + """ + root_real = os.path.realpath(root) + n = 0 + # Collect first so we do not walk into trees we just replaced. + external = [] + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + for name in dirnames + filenames: + path = os.path.join(dirpath, name) + if not os.path.islink(path): + continue + target = os.readlink(path) + if not os.path.isabs(target): + continue + # Resolve once; skip broken links with a warning. + try: + resolved = os.path.realpath(path) + except OSError: + print(f"bundle[cmssw]: warning: broken symlink {path} -> {target}") + continue + if not os.path.exists(resolved): + print( + f"bundle[cmssw]: warning: dangling symlink {path} -> {target}" + ) + continue + # Already points inside the staging tree → fine to keep. + if resolved == root_real or resolved.startswith(root_real + os.sep): + continue + external.append((path, resolved)) + + for path, resolved in external: + os.unlink(path) + if os.path.isdir(resolved): + shutil.copytree(resolved, path, symlinks=True) + else: + shutil.copy2(resolved, path) + n += 1 + print(f"bundle[cmssw]: materialized {path} <- {resolved}") + return n + class CERNHTCondorJobFileFactory(law.htcondor.HTCondorJobFileFactory): """HTCondor job file factory that stages transfer_input_files to EOS and uses protocol URLs. From bc64be34c6c5da6df56de82827abad3552bc822f Mon Sep 17 00:00:00 2001 From: Konstantin Date: Wed, 12 Aug 2026 17:00:35 +0200 Subject: [PATCH 19/35] fix: force CMSSW_BASE=FLAF_CMSSW_BASE in cmssw_env for CRAB --- Common/Setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Common/Setup.py b/Common/Setup.py index bc2c8011..0ea8f22c 100644 --- a/Common/Setup.py +++ b/Common/Setup.py @@ -716,7 +716,8 @@ def fs_rucio(self): @property def cmssw_env(self): if self.cmssw_env_ is None: - self.cmssw_env_ = get_cmsenv(cmssw_path=os.getenv("FLAF_CMSSW_BASE")) + flaf_cmssw = os.getenv("FLAF_CMSSW_BASE") + self.cmssw_env_ = get_cmsenv(cmssw_path=flaf_cmssw) for var in [ "HOME", "FLAF_PATH", @@ -728,6 +729,13 @@ def cmssw_env(self): ]: if var in os.environ: self.cmssw_env_[var] = os.environ[var] + # scram runtime (inside get_cmsenv) may still emit the submit-host AFS + # CMSSW_BASE when ProjectRename failed on a CRAB worker. Force the + # relocated release path so HHbtag models / includeLibTool resolve + # inside the bundle, not /afs/... + if flaf_cmssw and os.path.isdir(flaf_cmssw): + self.cmssw_env_["CMSSW_BASE"] = flaf_cmssw + self.cmssw_env_["FLAF_CMSSW_BASE"] = flaf_cmssw if "PYTHONPATH" not in self.cmssw_env_: self.cmssw_env_["PYTHONPATH"] = self.ana_path else: From 6afa48176b74f33f2f47eb83b327fae9534afb70 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 13 Aug 2026 06:38:46 +0200 Subject: [PATCH 20/35] fix: address PR #299 review and black formatting --- bootstrap.sh | 7 ++++--- docs/workflow/arguments.md | 2 +- docs/workflow/crab.md | 6 +++--- run_tools/law_customizations.py | 11 ++++------- test/hello_world_task.py | 12 +++++++++--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 1f966f27..fe11673c 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -128,10 +128,11 @@ action() { if ! scramv1 b ProjectRename "${cmssw_dir}"; then echo "bootstrap: WARNING: scram ProjectRename failed; applying path sed fallback" if [ -n "${old_localtop}" ] && [ "${old_localtop}" != "${cmssw_dir}" ]; then - # Escape sed separators in paths. + # Escape for sed s|old|new|g: BRE metacharacters on the + # pattern side; \, &, | on both sides (delimiter specials). local old_esc new_esc - old_esc=$(printf '%s' "${old_localtop}" | sed 's/[.[\*^$\/&]/g') - new_esc=$(printf '%s' "${cmssw_dir}" | sed 's/[.[\*^$\/&]/g') + old_esc=$(printf '%s' "${old_localtop}" | sed 's/[.[\*^$\\|\&]/\\&/g') + new_esc=$(printf '%s' "${cmssw_dir}" | sed 's/[\\|\&]/\\&/g') # Patch Self + .SCRAM metadata that embed the submit-host path. if [ -f "${cmssw_dir}/config/Self" ]; then sed -i "s|${old_esc}|${new_esc}|g" "${cmssw_dir}/config/Self" diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index 836fa4f8..d92380b6 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -40,7 +40,7 @@ also provides built-in options for status and cleanup. | Option | Default | Meaning | |---|---|---| | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | -| `--crab-memory` | `-1` | Max memory per job in MB (`-1` → `n_cpus * 2000` or `crab.max_memory_mb`). | +| `--crab-memory` | `-1` | Max memory per job in MB (`-1` → `n_cpus * 2500` or `crab.max_memory_mb`). | | `--crab-whitelist` | empty | Comma-separated site whitelist. | | `--crab-blacklist` | empty | Comma-separated site blacklist (ignored if whitelist is set). | diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 900fbab9..06671ea5 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -83,7 +83,7 @@ export FLAF_CRAB_OUT_LFN_BASE=/store/user/$USER/FLAF | `storage_site` | CRAB `Site.storageSite` (required for submission). | | `out_lfn_base` | CRAB `Data.outLFNDirBase` (required; not where analysis outputs go). | | `whitelist` / `blacklist` | Optional site lists. Whitelist implies `ignoreLocality`. | -| `max_memory_mb` | Default memory when `--crab-memory` is not set (`n_cpus * 2000` otherwise). | +| `max_memory_mb` | Default memory when `--crab-memory` is not set (`n_cpus * 2500` otherwise). | ## Submit @@ -133,8 +133,8 @@ also use `crab status -d ` from a CMSSW environment. !!! warning "First-time CRAB / grid mapfile" New users may need a CRAB username mapping and write access to the chosen storage site - LFN. Prefer a site you already use for CMS jobs (`T2_CH_CERN` is the usual CERN EOS - choice for `/store/user/...`). + LFN. At CERN, prefer `T3_CH_CERNBOX` for `/store/user/...` (maps to personal EOS and + usually passes `crab checkwrite`); `T2_CH_CERN /store/user` often does not exist. !!! note "Test small first" Validate with `--workflow local --branches 0 --test 1000`, then a single CRAB branch, diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 94001958..acc1263b 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -187,9 +187,6 @@ def _stage_user_custom_input(self, config): config.input_files["user_custom"] = JobInputFile( path=path, copy=True, share=True, render=False, increment=False ) - self._dataset_id_name_list = None - self._dataset_id_name_dict = None - self._dataset_name_id_dict = None # Process-local memoization of create_branch_map results, shared across task # instances. The same branch map is otherwise rebuilt many times during task @@ -779,10 +776,10 @@ def _apply_bootstrap_path_render_variables(self, config): config.render_variables["run_token_server_port"] = "" def _log_remote_base_url(self): + # Must match remote_log_dir_target() (used by --print-status and the + # HTCondor submit proxy) so producer sub-paths stay consistent. if isinstance(self.fs_default, WLCGFileSystem): - return self.remote_dir_target( - self.version, "logs", self.__class__.__name__, self.period - ).uri() + return self.remote_log_dir_target().uri() return "" def workflow_requires(self): @@ -1137,7 +1134,7 @@ class CrabWorkflow(law.cms.CrabWorkflow): crab_memory = luigi.IntParameter( default=-1, significant=False, - description="max memory per CRAB job in MB; -1 = n_cpus * 2000", + description="max memory per CRAB job in MB; -1 = n_cpus * 2500", ) crab_whitelist = law.CSVParameter( default=(), diff --git a/test/hello_world_task.py b/test/hello_world_task.py index c49e7ac5..06bca493 100644 --- a/test/hello_world_task.py +++ b/test/hello_world_task.py @@ -108,13 +108,17 @@ def _probe_grid_access(self): k: (int(v) if k == "size" else str(v)) for k, v in st.items() } except Exception as e: - report["gfal_stat_error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + report["gfal_stat_error"] = ( + f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + ) try: asum = gfal_sum(self.download_url, sum_type="adler32") report["gfal_sum"] = {"adler32": str(asum)} except Exception as e: - report["gfal_sum_error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + report["gfal_sum_error"] = ( + f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + ) size = None if report.get("gfal_stat") and "size" in report["gfal_stat"]: @@ -164,7 +168,9 @@ def _probe_grid_access(self): } except Exception as e: report["rucio"] = {"ok": False} - report["rucio_error"] = f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + report["rucio_error"] = ( + f"{type(e).__name__}: {e}\n{traceback.format_exc()}" + ) return report From a3ca01b62c08b65f7dae44c237ade7ac3f414f2f Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 13 Aug 2026 17:26:14 +0200 Subject: [PATCH 21/35] fix: address remaining PR #299 review comments --- Analysis/HistProducerFromNTuple.py | 35 +----- Analysis/tasks.py | 8 -- Common/Setup.py | 2 +- docs/workflow/arguments.md | 8 +- docs/workflow/crab.md | 54 ++++----- run_tools/law_customizations.py | 178 ++++++++++++----------------- 6 files changed, 104 insertions(+), 181 deletions(-) diff --git a/Analysis/HistProducerFromNTuple.py b/Analysis/HistProducerFromNTuple.py index dee00529..1df76138 100644 --- a/Analysis/HistProducerFromNTuple.py +++ b/Analysis/HistProducerFromNTuple.py @@ -33,20 +33,6 @@ def find_keys(inFiles_list): return sorted(unique_keys) -def _materialize_hist(unit_hist): - """Turn an RDF RResultPtr[TH*] into a concrete histogram. - - Do not use ``hasattr`` on an unevaluated RResultPtr: under current ROOT/cppyy - that can trigger ``operator*`` via a broken attribute probe and fail with - misleading Filter/column errors (e.g. Legacy_region) even when the graph is - valid. ``GetValue()`` is the supported materialization path. - """ - get_value = getattr(unit_hist, "GetValue", None) - if callable(get_value): - return get_value() - return unit_hist - - def SaveHist(key_tuple, outFile, hist_list, hist_name, unc, scale, verbose=0): model, unit_hist, rdf = hist_list[0] if verbose > 0: @@ -56,15 +42,16 @@ def SaveHist(key_tuple, outFile, hist_list, hist_name, unc, scale, verbose=0): dir_name = "/".join(key_tuple) dir_ptr = Utilities.mkdir(outFile, dir_name) - unit_hist = _materialize_hist(unit_hist) merged_hist = model.GetHistogram().Clone() # Detach from the current ROOT directory: the histogram is persisted explicitly via # WriteTObject below, so it must not also be auto-flushed into the output file's root # (which would leave one stray, unnamed histogram per call when writing directly). merged_hist.SetDirectory(0) - # GetNcells covers TH1/TH2/TH3 (incl. under/overflow); do not probe GetNbins via - # hasattr on RResultPtr (see _materialize_hist). - N_bins = unit_hist.GetNcells() + N_bins = ( + unit_hist.GetNbins() + if hasattr(unit_hist, "GetNbins") + else unit_hist.GetNcells() + ) for i in range(0, N_bins): bin_content = unit_hist.GetBinContent(i) bin_error = unit_hist.GetBinError(i) @@ -74,7 +61,6 @@ def SaveHist(key_tuple, outFile, hist_list, hist_name, unc, scale, verbose=0): nentries = unit_hist.GetEntries() if len(hist_list) > 1: for model, unit_hist in hist_list[1:]: - unit_hist = _materialize_hist(unit_hist) hist = model.GetHistogram() for i in range(0, N_bins): bin_content = unit_hist.GetBinContent(i) @@ -215,16 +201,7 @@ def BuildAllHistActions( ) args = parser.parse_args() - # Do not call EnableImplicitMT here. Multi-tree RDataFrame graphs that Filter - # on bool region columns (e.g. Legacy_region) fail at evaluation under - # ImplicitMT — including EnableImplicitMT(1) — with TTreeReader errors. This - # producer books Central + weight + shape (shifted-tree) actions together, so - # it must stay single-threaded until ROOT fixes that combination. - if args.nMT and args.nMT > 1: - print( - f"Note: --nMT={args.nMT} ignored; HistProducerFromNTuple runs without " - "ROOT ImplicitMT (multi-tree region filters)." - ) + ROOT.EnableImplicitMT(args.nMT) start = time.time() diff --git a/Analysis/tasks.py b/Analysis/tasks.py index f33582fe..72f82bc9 100644 --- a/Analysis/tasks.py +++ b/Analysis/tasks.py @@ -1247,14 +1247,6 @@ def run(self): analysisCacheProducer_cmd.extend( ["--cacheFiles", local_anacaches_str] ) - if self.user_custom: - # Must pass user_custom into the producer subprocess: Setup inside - # AnalysisCacheProducer is independent of the law Task Setup. Without - # this, keys only present in user_custom (e.g. compute_unc_histograms) - # are missing and BtagShape/histTupleDef raise KeyError on workers. - analysisCacheProducer_cmd.extend( - ["--user-custom", self.user_custom] - ) # Check if cmssw env is required prod_env = ( self.cmssw_env diff --git a/Common/Setup.py b/Common/Setup.py index 0ea8f22c..0cbf6670 100644 --- a/Common/Setup.py +++ b/Common/Setup.py @@ -731,7 +731,7 @@ def cmssw_env(self): self.cmssw_env_[var] = os.environ[var] # scram runtime (inside get_cmsenv) may still emit the submit-host AFS # CMSSW_BASE when ProjectRename failed on a CRAB worker. Force the - # relocated release path so HHbtag models / includeLibTool resolve + # relocated release path so CMSSW-dependent models / includeLibTool resolve # inside the bundle, not /afs/... if flaf_cmssw and os.path.isdir(flaf_cmssw): self.cmssw_env_["CMSSW_BASE"] = flaf_cmssw diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index d92380b6..04e39d67 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -40,12 +40,10 @@ also provides built-in options for status and cleanup. | Option | Default | Meaning | |---|---|---| | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | -| `--crab-memory` | `-1` | Max memory per job in MB (`-1` → `n_cpus * 2500` or `crab.max_memory_mb`). | -| `--crab-whitelist` | empty | Comma-separated site whitelist. | -| `--crab-blacklist` | empty | Comma-separated site blacklist (ignored if whitelist is set). | -CRAB also needs `crab.storage_site` and `crab.out_lfn_base` in config (or the -`FLAF_CRAB_STORAGE_SITE` / `FLAF_CRAB_OUT_LFN_BASE` environment variables). +Site white/black lists go in `global.yaml` under `crab:` (not CLI flags). +`Site.storageSite` / `Data.outLFNDirBase` are derived from `fs_default`. +Memory is `2 GB * n_cpus`. ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 06671ea5..07f9512e 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -6,9 +6,9 @@ uses [law's CMS CRAB workflow](https://github.com/riga/law) (`law.contrib.cms.Cr Analysis **outputs and job logs** use FLAF remote I/O only (`fs_default` via gfal/`davs://`, plus `stageout_logs.sh`). CRAB `transferOutputs` / `transferLogs` are forced **off** so -nothing is duplicated onto CRAB's stageout area. `crab.storage_site` / -`crab.out_lfn_base` are still required by the CRAB client for a valid config and a -submit-time write check — they are not where FLAF stores analysis products. +nothing is duplicated onto CRAB's stageout area. The CRAB client still needs +`Site.storageSite` / `Data.outLFNDirBase` for a submit-time write check — those +fields are derived from `fs_default`, not configured separately. ## Prerequisites @@ -35,11 +35,9 @@ submit-time write check — they are not where FLAF stores analysis products. myproxy-info -s myproxy.cern.ch -l "$SHA1" # expect timeleft >= 5 days + retrieval policy ``` - Alternatively set `job.crab_password_file` in `law.cfg` to a file with the grid - certificate passphrase; law will call `delegate_myproxy` with the same CRAB retrievers. - - FLAF fails early if no suitable MyProxy credential is found (instead of waiting for - server-side `SUBMITFAILED`). + FLAF fails early if the VOMS proxy or a suitable MyProxy credential is missing + (instead of waiting for server-side `SUBMITFAILED`). There is no password-file + fallback. 3. CRAB client available (via CMSSW / the law CMSSW sandbox; default sandbox name is set in law's config as `job.crab_sandbox_name`). 4. **Bundles**: CRAB workers do not mount AFS. FLAF always ships code via `BundleTask` @@ -50,40 +48,34 @@ submit-time write check — they are not where FLAF stores analysis products. ## Config -Add a `crab:` block to `user_custom.yaml` (or pass via `--user-custom`): +CRAB's write-check site is taken from `fs_default`: + +| `fs_default` | CRAB `storageSite` + `outLFNDirBase` | +|---|---| +| `T3_CH_CERNBOX:/store/user//...` | as written | +| `davs://eoshome-.cern.ch:.../eos/user///...` | `T3_CH_CERNBOX` + `/store/user//...` | + +Site lists belong in `global.yaml` (or `user_custom.yaml`) under `crab:`: ```yaml crab: - # Stageout site for CRAB bookkeeping (analysis outputs still use fs_default). - # At CERN, T3_CH_CERNBOX maps /store/user/ to personal EOS and usually - # passes `crab checkwrite`; T2_CH_CERN /store/user often does not exist. - storage_site: T3_CH_CERNBOX - out_lfn_base: /store/user//FLAF - # optional: - # whitelist: [T2_CH_CERN] # where jobs run (can differ from storage_site) + whitelist: [T2_CH_CERN] # where jobs run (required) # blacklist: [T2_US_MIT] - # max_memory_mb: 4000 ``` +Memory is `2 GB * n_cpus` (the existing `--n-cpus` parameter). There is no separate +CRAB memory flag. + Verify write access before the first campaign: ```sh crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER ``` -Alternatively set environment variables: - -```sh -export FLAF_CRAB_STORAGE_SITE=T3_CH_CERNBOX -export FLAF_CRAB_OUT_LFN_BASE=/store/user/$USER/FLAF -``` - | Key | Meaning | |---|---| -| `storage_site` | CRAB `Site.storageSite` (required for submission). | -| `out_lfn_base` | CRAB `Data.outLFNDirBase` (required; not where analysis outputs go). | -| `whitelist` / `blacklist` | Optional site lists. Whitelist implies `ignoreLocality`. | -| `max_memory_mb` | Default memory when `--crab-memory` is not set (`n_cpus * 2500` otherwise). | +| `whitelist` | CRAB `Site.whitelist`. Required (or set `blacklist`) when submitting. | +| `blacklist` | CRAB `Site.blacklist`. Used only when `whitelist` is empty. | ## Submit @@ -99,9 +91,7 @@ law run FLAF.Analysis.tasks.HistTupleProducerTask \ | Option | Why | |---|---| | `--workflow crab` | Submit via CRAB instead of local/HTCondor. | -| `--crab-memory 4000` | Override max memory (MB) per job. | -| `--crab-whitelist T2_CH_CERN` | Restrict to listed sites. | -| `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory. | +| `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory (`2 GB * n_cpus`). | | `--transfer-logs` | On by default; enables remote log stageout when `fs_default` is WLCG. | You do **not** need `--bundle` for CRAB — bundles are forced whenever the workflow is `crab`. @@ -129,7 +119,7 @@ also use `crab status -d ` from a CMSSW environment. !!! warning "MyProxy must stay valid" CRAB polls through MyProxy. Delegate a long-lived proxy before large campaigns - (`myproxy-init -d -n` or law's password-file path). + (`myproxy-init` as in Prerequisites). !!! warning "First-time CRAB / grid mapfile" New users may need a CRAB username mapping and write access to the chosen storage site diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index acc1263b..06e74701 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1027,20 +1027,16 @@ def _rewrite_crab_job_file(job_file): f.writelines(new_lines) -# Soften MyProxy requirements: modern CRAB accepts a local VOMS proxy for submit/status. -# law's default CrabWorkflowProxy.setup_job_manager always tries interactive myproxy -# delegation; that blocks non-interactive CI-like runs when myproxy is not pre-loaded. +# Require VOMS + MyProxy before submit. The CRAB server retrieves the user proxy +# from myproxy.cern.ch (>= ~5 days remaining). A local VOMS proxy alone is not +# enough: the client may accept the task, then the server returns SUBMITFAILED. +# Do not fall back to interactive delegation or a law.cfg password file. _FLAFCrabWorkflowProxyBase = law.cms.CrabWorkflow.workflow_proxy_cls class _FLAFCrabWorkflowProxy(_FLAFCrabWorkflowProxyBase): def setup_job_manager(self): - """Ensure VOMS + MyProxy before crab submit. - - The CRAB *server* retrieves the user proxy from myproxy.cern.ch and requires - at least ~5 days remaining. A local VOMS proxy alone is not enough: the client - may accept the task, then the server returns SUBMITFAILED. Fail early here. - """ + """Require a valid VOMS proxy and a MyProxy credential (>= 5 days).""" proxy = os.environ.get("X509_USER_PROXY", "") if not proxy or not os.path.isfile(proxy): raise RuntimeError( @@ -1054,11 +1050,10 @@ def setup_job_manager(self): ) kwargs = {"proxy": proxy} - # CRAB server asks for >= 5 days remaining; keep a small margin. min_myproxy_seconds = 5 * 24 * 3600 - # MyProxy usernames may be either the DN (`myproxy-init -d`) or a SHA1 of the DN - # (law's default encode_username=True / some crab helpers). Accept either form. + # MyProxy usernames may be either the DN (`myproxy-init -d`) or a SHA1 of + # the DN (law encode_username=True / some crab helpers). Accept either form. for encode in (False, True): try: info = ( @@ -1070,29 +1065,57 @@ def setup_job_manager(self): kwargs["myproxy_username"] = info["username"] return kwargs - # Non-interactive delegation when a password file is configured (law.cfg). - # law.delegate_myproxy registers under the SHA1 username CRAB workers expect. - cfg = law.config.Config.instance() - password_file = cfg.get_expanded("job", "crab_password_file") - if password_file and os.path.isfile(password_file): - from law.contrib.cms.util import renew_vomsproxy, delegate_myproxy - - if not law.wlcg.check_vomsproxy_validity(): - renew_vomsproxy(password_file=password_file) - kwargs["myproxy_username"] = delegate_myproxy(password_file=password_file) - return kwargs - raise RuntimeError( "CRAB requires a MyProxy credential valid for at least 5 days " "(CRAB server retrieves it from myproxy.cern.ch). " "Run once interactively:\n" " myproxy-init -d -n -s myproxy.cern.ch\n" " # verify: myproxy-info -d -s myproxy.cern.ch (timeleft >= 5 days)\n" - "or set job.crab_password_file in law.cfg to a file containing the " - "grid certificate passphrase for non-interactive delegation." + "See docs/workflow/crab.md for the CRAB-retriever form." ) +_EOSHOME_FS_RE = re.compile( + r"^davs://eoshome-[a-z0-9]+\.cern\.ch(?::\d+)?/eos/user/[a-z0-9]/([^/]+)(/.*)?$", + re.IGNORECASE, +) + + +def _crab_stageout_from_fs_spec(fs_spec): + """Map ``fs_default`` to CRAB ``(storageSite, outLFNDirBase)``. + + Accepted forms (same as storage docs): + + - ``T3_CH_CERNBOX:/store/user//...`` + - ``davs://eoshome-.cern.ch:.../eos/user///...`` + → ``T3_CH_CERNBOX`` + ``/store/user//...`` + """ + if isinstance(fs_spec, (list, tuple)): + if not fs_spec: + raise RuntimeError("fs_default is empty; CRAB needs a remote filesystem") + fs_spec = fs_spec[0] + if not isinstance(fs_spec, str) or not fs_spec.strip(): + raise RuntimeError("fs_default must be a string (or list of strings)") + spec = fs_spec.strip().rstrip("/") + + if "://" not in spec and ":" in spec: + site, lfn = spec.split(":", 1) + site, lfn = site.strip(), lfn.strip() + if site and lfn.startswith("/"): + return site, lfn + + m = _EOSHOME_FS_RE.match(spec) + if m: + user, rest = m.group(1), m.group(2) or "" + return "T3_CH_CERNBOX", f"/store/user/{user}{rest}" + + raise RuntimeError( + "CRAB derives Site.storageSite and Data.outLFNDirBase from fs_default. " + "Use a WLCG site path (T3_CH_CERNBOX:/store/user//...) or a CERN " + f"EOS davs://eoshome-... URL. Got: {fs_spec}" + ) + + class CrabWorkflow(law.cms.CrabWorkflow): """CRAB (WLCG) remote workflow, built on law.contrib.cms.CrabWorkflow. @@ -1101,21 +1124,18 @@ class CrabWorkflow(law.cms.CrabWorkflow): ``transferOutputs`` / ``transferLogs`` / ``JobType.outputFiles`` are forced off so nothing is duplicated onto CRAB's stageout area. - ``crab.storage_site`` / ``crab.out_lfn_base`` remain required by the CRAB client - (submit-time write check); they are not used for analysis outputs. + ``Site.storageSite`` / ``Data.outLFNDirBase`` are derived from ``fs_default`` + (submit-time write check only). Site white/black lists come from the ``crab:`` + section of ``global.yaml``. Memory is ``2 GB * n_cpus``. CRAB workers have no AFS, so code is always shipped via the existing BundleTask mechanism (same as ``--bundle`` on HTCondor). Tasks must declare ``bundle_flavours``. - Config (``global_params`` / user_custom YAML):: + Config (``global.yaml`` / user_custom YAML):: crab: - storage_site: T3_CH_CERNBOX # Site.storageSite (write-check only) - out_lfn_base: /store/user//FLAF # Data.outLFNDirBase (write-check only) - # optional: - # whitelist: [T2_CH_CERN, T2_IT_Pisa] + whitelist: [T2_CH_CERN] # blacklist: [T2_US_MIT] - # max_memory_mb: 4000 """ # Re-declare in the class body so law's metaclass sets _defined_workflow_proxy=True @@ -1131,26 +1151,6 @@ class CrabWorkflow(law.cms.CrabWorkflow): description="enable FLAF remote log stageout (stdall.txt via stageout_logs.sh); " "CRAB transferLogs stays off", ) - crab_memory = luigi.IntParameter( - default=-1, - significant=False, - description="max memory per CRAB job in MB; -1 = n_cpus * 2500", - ) - crab_whitelist = law.CSVParameter( - default=(), - significant=False, - description="comma-separated CRAB Site.whitelist; empty = no whitelist", - ) - crab_blacklist = law.CSVParameter( - default=(), - significant=False, - description="comma-separated CRAB Site.blacklist; ignored when whitelist is set", - ) - - # Parameters that are only meaningful on the workflow (not branch tasks). - exclude_params_branch = getattr( - law.cms.CrabWorkflow, "exclude_params_branch", set() - ) | {"crab_memory", "crab_whitelist", "crab_blacklist"} def _crab_cfg(self): return self.global_params.get("crab") or {} @@ -1184,26 +1184,13 @@ def _ensure_crab_pset(self, n_threads): return path def crab_stageout_location(self): - """Return (storageSite, outLFNDirBase) required by the CRAB client. + """Return (storageSite, outLFNDirBase) derived from ``fs_default``. FLAF does **not** store analysis outputs here (CRAB transferOutputs is forced off; products go to ``fs_default``). CRAB still requires these fields and runs - a submit-time write check against them — pick a site you can write to - (e.g. ``T3_CH_CERNBOX`` + ``/store/user//...``). + a submit-time write check against them. """ - cfg = self._crab_cfg() - site = cfg.get("storage_site") or os.environ.get("FLAF_CRAB_STORAGE_SITE") - lfn = cfg.get("out_lfn_base") or os.environ.get("FLAF_CRAB_OUT_LFN_BASE") - if not site or not lfn: - raise RuntimeError( - "CRAB requires crab.storage_site and crab.out_lfn_base in config " - "(user_custom / global_params), or FLAF_CRAB_STORAGE_SITE and " - "FLAF_CRAB_OUT_LFN_BASE environment variables. These are only for the " - "CRAB client write-check, not analysis outputs. " - "Example: crab: {storage_site: T3_CH_CERNBOX, " - "out_lfn_base: /store/user/$USER/FLAF}" - ) - return str(site), str(lfn) + return _crab_stageout_from_fs_spec(self.global_params.get("fs_default")) def crab_output_directory(self): return law.LocalDirectoryTarget(self.local_path()) @@ -1294,32 +1281,13 @@ def crab_job_config(self, config, job_nums, branches=None): log_remote_base_url = self._log_remote_base_url() config.render_variables["log_remote_base_url"] = log_remote_base_url - # Cores + memory. CRAB enforces ~2500 MB per core and requires - # JobType.numCores == process.options.numberOfThreads in the PSet. - # AnaTuple/CMSSW jobs need several GB → request enough cores and bake a - # matching PSet (law's default PSet always has threads=1). + # Cores + memory. CRAB requires JobType.numCores == PSet numberOfThreads. + # Memory is 2 GB per CPU from the existing n_cpus parameter. n_cpus = max(1, int(getattr(self, "n_cpus", 1) or 1)) - mem = int(self.crab_memory) - if mem <= 0: - mem = int(self._crab_cfg().get("max_memory_mb", n_cpus * 2500)) - # Production tasks that routinely OOM at 2.5 GB on CRAB. - heavy = self.__class__.__name__ in ( - "AnaTupleFileTask", - "AnaTupleMergeTask", - "HistTupleProducerTask", - "HistFromNtupleProducerTask", - ) - if heavy and mem < 8000: - mem = 8000 - mb_per_core = int(self._crab_cfg().get("max_memory_mb_per_core", 2500)) - max_cores = int(self._crab_cfg().get("max_cores", 8)) - n_cores = max(n_cpus, (mem + mb_per_core - 1) // mb_per_core) - n_cores = max(1, min(n_cores, max_cores)) - mem = max(mem, n_cores * mb_per_core) - # Write a per-thread-count PSet next to the analysis job dir. - pset_path = self._ensure_crab_pset(n_cores) + mem = n_cpus * 2000 + pset_path = self._ensure_crab_pset(n_cpus) config.crab.JobType.psetName = pset_path - config.crab.JobType.numCores = n_cores + config.crab.JobType.numCores = n_cpus config.crab.JobType.maxMemoryMB = mem # Runtime limit (hours → minutes). CRAB jobs must download/unpack bundles before @@ -1335,19 +1303,17 @@ def crab_job_config(self, config, job_nums, branches=None): # Older CRAB clients may not support maxJobRuntimeMin; ignore if rejected later. pass - # Site white/black lists: CLI params override config. - # CRAB requires a whitelist when jobs use synthetic userInputFiles (no inputDataset), - # which is always the case for law CRAB workflows. Default the whitelist to the - # storage site when nothing else is configured. - whitelist = list(self.crab_whitelist) or list( - self._crab_cfg().get("whitelist") or [] - ) - blacklist = list(self.crab_blacklist) or list( - self._crab_cfg().get("blacklist") or [] - ) + # Site white/black lists come from global.yaml ``crab:`` (no CLI overrides). + # CRAB requires a whitelist when jobs use synthetic userInputFiles (no + # inputDataset), which is always the case for law CRAB workflows. + whitelist = list(self._crab_cfg().get("whitelist") or []) + blacklist = list(self._crab_cfg().get("blacklist") or []) if not whitelist and not blacklist: - site, _ = self.crab_stageout_location() - whitelist = [site] + raise RuntimeError( + "CRAB requires crab.whitelist (or crab.blacklist) in global.yaml " + "under the crab: section. Example:\n" + " crab:\n whitelist: [T2_CH_CERN]" + ) if whitelist: config.crab.Site.whitelist = [str(s) for s in whitelist] config.crab.Site.ignoreGlobalBlacklist = True From ce2de0667a6187cb60233ab1e39bab2140a57d24 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 13 Aug 2026 17:32:57 +0200 Subject: [PATCH 22/35] restore user_custom pass; drop only the verbose comment --- Analysis/tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Analysis/tasks.py b/Analysis/tasks.py index 72f82bc9..eeeabac2 100644 --- a/Analysis/tasks.py +++ b/Analysis/tasks.py @@ -1247,6 +1247,10 @@ def run(self): analysisCacheProducer_cmd.extend( ["--cacheFiles", local_anacaches_str] ) + if self.user_custom: + analysisCacheProducer_cmd.extend( + ["--user-custom", self.user_custom] + ) # Check if cmssw env is required prod_env = ( self.cmssw_env From 65d863aa7a20845387e246b9ccb3231309b8272c Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 13 Aug 2026 17:45:59 +0200 Subject: [PATCH 23/35] skip remotePathCacheHost on CRAB workers --- Common/Setup.py | 7 +++++++ docs/workflow/crab.md | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/Common/Setup.py b/Common/Setup.py index 0cbf6670..8d747774 100644 --- a/Common/Setup.py +++ b/Common/Setup.py @@ -657,6 +657,11 @@ def _create_fs_instance(self, path_or_paths): cache_validity = cfg.get("localPathCacheValidity", 600) host = cfg.get("remotePathCacheHost", None) port = cfg.get("remotePathCachePort", None) + # cms-flaf.cern.ch is behind the CERN firewall; CRAB workers at other + # sites cannot reach it. Fall back to the in-process PathCache. + if os.environ.get("LAW_CRAB_JOB_NUMBER") or os.environ.get("CRAB_Id"): + host = None + port = None verbose = cfg.get("verbose", 0) return WLCGFileSystem( path_or_paths, @@ -726,6 +731,8 @@ def cmssw_env(self): "X509_USER_PROXY", "FLAF_CMSSW_BASE", "FLAF_CMSSW_ARCH", + "LAW_CRAB_JOB_NUMBER", + "CRAB_Id", ]: if var in os.environ: self.cmssw_env_[var] = os.environ[var] diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 07f9512e..16ee4784 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -121,6 +121,11 @@ also use `crab status -d ` from a CMSSW environment. CRAB polls through MyProxy. Delegate a long-lived proxy before large campaigns (`myproxy-init` as in Prerequisites). +!!! note "Path-existence cache is local on CRAB workers" + `WLCGFileSystem.remotePathCacheHost` (`cms-flaf.cern.ch`) is behind the CERN + firewall, so CRAB workers do not use it. Existence checks use the in-process + cache and gfal only. + !!! warning "First-time CRAB / grid mapfile" New users may need a CRAB username mapping and write access to the chosen storage site LFN. At CERN, prefer `T3_CH_CERNBOX` for `/store/user/...` (maps to personal EOS and From 5a4345ab86163a5a59a37b7aa46f28c2c41c5112 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 13 Aug 2026 18:03:03 +0200 Subject: [PATCH 24/35] ship submit path cache with CRAB jobs --- Common/Setup.py | 25 +++++++- RunKit/law_gfal.py | 105 ++++++++++++++++++++++++++++++++ docs/workflow/crab.md | 8 ++- run_tools/law_customizations.py | 18 ++++++ 4 files changed, 150 insertions(+), 6 deletions(-) diff --git a/Common/Setup.py b/Common/Setup.py index 8d747774..c63363e4 100644 --- a/Common/Setup.py +++ b/Common/Setup.py @@ -658,18 +658,34 @@ def _create_fs_instance(self, path_or_paths): host = cfg.get("remotePathCacheHost", None) port = cfg.get("remotePathCachePort", None) # cms-flaf.cern.ch is behind the CERN firewall; CRAB workers at other - # sites cannot reach it. Fall back to the in-process PathCache. - if os.environ.get("LAW_CRAB_JOB_NUMBER") or os.environ.get("CRAB_Id"): + # sites cannot reach it. Use the in-process PathCache, seeded from the + # snapshot shipped at submit, with a longer TTL so many jobs do not + # re-stat the same remote paths. + on_crab_worker = bool( + os.environ.get("LAW_CRAB_JOB_NUMBER") or os.environ.get("CRAB_Id") + ) + if on_crab_worker: host = None port = None + cache_validity = int( + cfg.get( + "crabLocalPathCacheValidity", + max(cache_validity * 24, 86400), + ) + ) verbose = cfg.get("verbose", 0) - return WLCGFileSystem( + fs = WLCGFileSystem( path_or_paths, local_path_cache_validity_period=cache_validity, path_cache_host=host, path_cache_port=port, verbose=verbose, ) + if on_crab_worker: + from FLAF.RunKit.law_gfal import apply_shipped_path_cache + + apply_shipped_path_cache(fs) + return fs def get_fs(self, fs_name, custom_paths=None): fs_instance = None @@ -733,6 +749,9 @@ def cmssw_env(self): "FLAF_CMSSW_ARCH", "LAW_CRAB_JOB_NUMBER", "CRAB_Id", + "LAW_JOB_INIT_DIR", + "LAW_JOB_HOME", + "FLAF_SHIPPED_PATH_CACHE", ]: if var in os.environ: self.cmssw_env_[var] = os.environ[var] diff --git a/RunKit/law_gfal.py b/RunKit/law_gfal.py index 0343a6b0..06d9c3d6 100644 --- a/RunKit/law_gfal.py +++ b/RunKit/law_gfal.py @@ -1,3 +1,4 @@ +import json import time import os import sys @@ -94,6 +95,28 @@ def get(self, path): def get_many(self, paths): return {path: self.get(path)[0] for path in paths} + def iter_valid(self): + for path, entry in list(self.cache.items()): + if entry.is_valid(): + yield path, entry.exists + + def load_entries(self, entries): + """Refresh entries with this cache's validity period (snapshot has no timestamps).""" + negatives = [] + positives = [] + for item in entries: + path = item.get("path") + if not path: + continue + if item.get("exists"): + positives.append(path) + else: + negatives.append(path) + for path in negatives: + self.set(path, False) + for path in positives: + self.set(path, True) + def invalidate(self, path): to_remove = [] for p in self.cache: @@ -185,6 +208,88 @@ def invalidate(self, path): self.local_cache.invalidate(path) +SHIPPED_PATH_CACHE_BASENAME = "path_cache.json" +SHIPPED_PATH_CACHE_ENV = "FLAF_SHIPPED_PATH_CACHE" + +_shipped_path_cache_entries = None + + +def local_path_cache(fs): + """Return the in-process PathCache for a WLCG/GFAL filesystem, if any.""" + fi = getattr(fs, "file_interface", None) + pc = getattr(fi, "path_cache", None) + if pc is None: + return None + return getattr(pc, "local_cache", pc) + + +def collect_setup_path_cache_entries(setup): + """Union of valid path-cache entries from every FS the Setup has already created.""" + entries = {} + for fs in getattr(setup, "fs_dict", {}).values(): + pc = local_path_cache(fs) + if pc is None: + continue + for path, exists in pc.iter_valid(): + entries[path] = exists + return [{"path": path, "exists": exists} for path, exists in entries.items()] + + +def write_path_cache_file(path, entries): + with open(path, "w") as f: + json.dump({"entries": entries}, f) + + +def _resolve_shipped_path_cache_file(): + env_path = os.environ.get(SHIPPED_PATH_CACHE_ENV, "") + if env_path and os.path.isfile(env_path): + return env_path + stem, ext = os.path.splitext(SHIPPED_PATH_CACHE_BASENAME) + search_dirs = [ + os.environ.get("LAW_JOB_INIT_DIR", ""), + os.environ.get("LAW_JOB_HOME", ""), + "/srv", + os.getcwd(), + ] + for d in search_dirs: + if not d: + continue + direct = os.path.join(d, SHIPPED_PATH_CACHE_BASENAME) + if os.path.isfile(direct): + return direct + if not os.path.isdir(d): + continue + try: + for name in os.listdir(d): + if name.startswith(stem + "_") and name.endswith(ext): + cand = os.path.join(d, name) + if os.path.isfile(cand): + return cand + except OSError: + pass + return None + + +def apply_shipped_path_cache(fs): + """Load a submit-time path-cache snapshot into ``fs`` (once per process).""" + global _shipped_path_cache_entries + if _shipped_path_cache_entries is None: + _shipped_path_cache_entries = [] + path = _resolve_shipped_path_cache_file() + if path: + try: + with open(path) as f: + data = json.load(f) + _shipped_path_cache_entries = data.get("entries") or [] + os.environ[SHIPPED_PATH_CACHE_ENV] = path + except (OSError, ValueError, TypeError): + _shipped_path_cache_entries = [] + pc = local_path_cache(fs) + if pc is None or not _shipped_path_cache_entries: + return + pc.load_entries(_shipped_path_cache_entries) + + class GFALFileInterface(RemoteFileInterface): local_prefix = "file://" diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 16ee4784..c5005e02 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -121,10 +121,12 @@ also use `crab status -d ` from a CMSSW environment. CRAB polls through MyProxy. Delegate a long-lived proxy before large campaigns (`myproxy-init` as in Prerequisites). -!!! note "Path-existence cache is local on CRAB workers" +!!! note "Path-existence cache is shipped with the job" `WLCGFileSystem.remotePathCacheHost` (`cms-flaf.cern.ch`) is behind the CERN - firewall, so CRAB workers do not use it. Existence checks use the in-process - cache and gfal only. + firewall, so CRAB workers do not use it. At submit time FLAF dumps the + in-process path cache and ships it with the job; the worker loads that + snapshot and uses a longer local TTL (`24 × localPathCacheValidity`, at + least 24 h) so concurrent jobs do not re-stat the same remote paths. !!! warning "First-time CRAB / grid mapfile" New users may need a CRAB username mapping and write access to the chosen storage site diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 06e74701..58a30be9 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -188,6 +188,23 @@ def _stage_user_custom_input(self, config): path=path, copy=True, share=True, render=False, increment=False ) + def _stage_path_cache_input(self, config): + """Dump the submit-process path cache and ship it with the CRAB job.""" + from law.job.base import JobInputFile + from FLAF.RunKit.law_gfal import ( + SHIPPED_PATH_CACHE_BASENAME, + collect_setup_path_cache_entries, + write_path_cache_file, + ) + + out_dir = self.local_path() + os.makedirs(out_dir, exist_ok=True) + path = os.path.join(out_dir, SHIPPED_PATH_CACHE_BASENAME) + write_path_cache_file(path, collect_setup_path_cache_entries(self.setup)) + config.input_files["path_cache"] = JobInputFile( + path=path, copy=True, share=True, render=False, increment=False + ) + # Process-local memoization of create_branch_map results, shared across task # instances. The same branch map is otherwise rebuilt many times during task # initialization because every `X.req(...).create_branch_map()` constructs a fresh @@ -1277,6 +1294,7 @@ def crab_job_config(self, config, job_nums, branches=None): self._apply_bootstrap_path_render_variables(config) self._apply_bundle_render_variables(config) self._stage_user_custom_input(config) + self._stage_path_cache_input(config) log_remote_base_url = self._log_remote_base_url() config.render_variables["log_remote_base_url"] = log_remote_base_url From 057e167a578224d41ef1d9994064a99d77b3d8d8 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 14 Aug 2026 22:04:44 +0200 Subject: [PATCH 25/35] default CRAB site whitelist to all T1/T2/T3 sites --- docs/workflow/arguments.md | 6 +++--- docs/workflow/crab.md | 11 +++++++---- run_tools/law_customizations.py | 35 ++++++++++++++++----------------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index 04e39d67..26972645 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -41,9 +41,9 @@ also provides built-in options for status and cleanup. |---|---|---| | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | -Site white/black lists go in `global.yaml` under `crab:` (not CLI flags). -`Site.storageSite` / `Data.outLFNDirBase` are derived from `fs_default`. -Memory is `2 GB * n_cpus`. +Optional site white/black lists go in `global.yaml` under `crab:` (not CLI flags). +Unset whitelist ⇒ all T1/T2/T3 sites. `Site.storageSite` / `Data.outLFNDirBase` +are derived from `fs_default`. Memory is `2 GB * n_cpus`. ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index c5005e02..2a2cd95a 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -55,11 +55,14 @@ CRAB's write-check site is taken from `fs_default`: | `T3_CH_CERNBOX:/store/user//...` | as written | | `davs://eoshome-.cern.ch:.../eos/user///...` | `T3_CH_CERNBOX` + `/store/user//...` | -Site lists belong in `global.yaml` (or `user_custom.yaml`) under `crab:`: +The CRAB client requires `Site.whitelist` because law uses dummy `userInputFiles` +(no input dataset). FLAF defaults that list to `T1_*`, `T2_*`, `T3_*` so jobs +can run at every CMS processing site. Restrict or exclude sites only if you need +to, in `global.yaml` / `user_custom.yaml`: ```yaml crab: - whitelist: [T2_CH_CERN] # where jobs run (required) + # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites # blacklist: [T2_US_MIT] ``` @@ -74,8 +77,8 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER | Key | Meaning | |---|---| -| `whitelist` | CRAB `Site.whitelist`. Required (or set `blacklist`) when submitting. | -| `blacklist` | CRAB `Site.blacklist`. Used only when `whitelist` is empty. | +| `whitelist` | Optional. Restricts `Site.whitelist`. Default: `T1_*`, `T2_*`, `T3_*`. | +| `blacklist` | Optional. CRAB `Site.blacklist` (applied on top of the whitelist). | ## Submit diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 58a30be9..7ae22a5a 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1142,16 +1142,20 @@ class CrabWorkflow(law.cms.CrabWorkflow): nothing is duplicated onto CRAB's stageout area. ``Site.storageSite`` / ``Data.outLFNDirBase`` are derived from ``fs_default`` - (submit-time write check only). Site white/black lists come from the ``crab:`` - section of ``global.yaml``. Memory is ``2 GB * n_cpus``. + (submit-time write check only). Memory is ``2 GB * n_cpus``. + + CRAB's client requires ``Site.whitelist`` when law uses dummy ``userInputFiles``. + If ``crab.whitelist`` is unset, FLAF defaults to ``T1_*`` / ``T2_*`` / ``T3_*`` + so jobs can run at every CMS processing site. Optional ``crab.blacklist`` + still excludes sites. CRAB workers have no AFS, so code is always shipped via the existing BundleTask mechanism (same as ``--bundle`` on HTCondor). Tasks must declare ``bundle_flavours``. - Config (``global.yaml`` / user_custom YAML):: + Config (``global.yaml`` / user_custom YAML), all optional:: crab: - whitelist: [T2_CH_CERN] + # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites # blacklist: [T2_US_MIT] """ @@ -1321,22 +1325,17 @@ def crab_job_config(self, config, job_nums, branches=None): # Older CRAB clients may not support maxJobRuntimeMin; ignore if rejected later. pass - # Site white/black lists come from global.yaml ``crab:`` (no CLI overrides). - # CRAB requires a whitelist when jobs use synthetic userInputFiles (no - # inputDataset), which is always the case for law CRAB workflows. + # Law always sets dummy userInputFiles (no inputDataset). The CRAB client + # then requires Site.whitelist. Default to every CMS processing site so + # analyses need not pin T2_CH_CERN. An explicit crab.whitelist still + # restricts; crab.blacklist excludes sites on top of the list used. whitelist = list(self._crab_cfg().get("whitelist") or []) blacklist = list(self._crab_cfg().get("blacklist") or []) - if not whitelist and not blacklist: - raise RuntimeError( - "CRAB requires crab.whitelist (or crab.blacklist) in global.yaml " - "under the crab: section. Example:\n" - " crab:\n whitelist: [T2_CH_CERN]" - ) - if whitelist: - config.crab.Site.whitelist = [str(s) for s in whitelist] - config.crab.Site.ignoreGlobalBlacklist = True - config.crab.Data.ignoreLocality = True - elif blacklist: + if not whitelist: + whitelist = ["T1_*", "T2_*", "T3_*"] + config.crab.Site.whitelist = [str(s) for s in whitelist] + config.crab.Data.ignoreLocality = True + if blacklist: config.crab.Site.blacklist = [str(s) for s in blacklist] return config From 36eba5f72935b24f8532b77e8ce405baedab7138 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 14 Aug 2026 22:11:04 +0200 Subject: [PATCH 26/35] use real TT inputDataset so CRAB needs no site whitelist --- docs/workflow/arguments.md | 5 ++- docs/workflow/crab.md | 17 +++++---- run_tools/law_customizations.py | 65 ++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index 26972645..807ec725 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -42,8 +42,9 @@ also provides built-in options for status and cleanup. | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | Optional site white/black lists go in `global.yaml` under `crab:` (not CLI flags). -Unset whitelist ⇒ all T1/T2/T3 sites. `Site.storageSite` / `Data.outLFNDirBase` -are derived from `fs_default`. Memory is `2 GB * n_cpus`. +A real TT `inputDataset` plus `ignoreLocality` means no whitelist is required. +`Site.storageSite` / `Data.outLFNDirBase` are derived from `fs_default`. +Memory is `2 GB * n_cpus`. ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 2a2cd95a..b599a9b2 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -55,14 +55,16 @@ CRAB's write-check site is taken from `fs_default`: | `T3_CH_CERNBOX:/store/user//...` | as written | | `davs://eoshome-.cern.ch:.../eos/user///...` | `T3_CH_CERNBOX` + `/store/user//...` | -The CRAB client requires `Site.whitelist` because law uses dummy `userInputFiles` -(no input dataset). FLAF defaults that list to `T1_*`, `T2_*`, `T3_*` so jobs -can run at every CMS processing site. Restrict or exclude sites only if you need -to, in `global.yaml` / `user_custom.yaml`: +FLAF sets a real TT NanoAOD `Data.inputDataset` (from this era's `TTto2L2Nu` / +`TT`, or `crab.input_dataset`) and `ignoreLocality: True`. That stops law from +injecting dummy `userInputFiles`, so **no site whitelist is required** and jobs +are not limited to the TT sample's sites. Restrict or exclude sites only if you +need to: ```yaml crab: - # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites + # input_dataset: /TTto2L2Nu_.../NANOAODSIM + # whitelist: [T2_CH_CERN] # blacklist: [T2_US_MIT] ``` @@ -77,8 +79,9 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER | Key | Meaning | |---|---| -| `whitelist` | Optional. Restricts `Site.whitelist`. Default: `T1_*`, `T2_*`, `T3_*`. | -| `blacklist` | Optional. CRAB `Site.blacklist` (applied on top of the whitelist). | +| `input_dataset` | Optional. Official DAS dataset used only so CRAB has an `inputDataset`. Default: this era's `TTto2L2Nu` / `TT` NanoAOD. | +| `whitelist` | Optional. Restricts `Site.whitelist`. Omit to leave sites unrestricted. | +| `blacklist` | Optional. CRAB `Site.blacklist`. | ## Submit diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 7ae22a5a..523a51a8 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1144,10 +1144,12 @@ class CrabWorkflow(law.cms.CrabWorkflow): ``Site.storageSite`` / ``Data.outLFNDirBase`` are derived from ``fs_default`` (submit-time write check only). Memory is ``2 GB * n_cpus``. - CRAB's client requires ``Site.whitelist`` when law uses dummy ``userInputFiles``. - If ``crab.whitelist`` is unset, FLAF defaults to ``T1_*`` / ``T2_*`` / ``T3_*`` - so jobs can run at every CMS processing site. Optional ``crab.blacklist`` - still excludes sites. + Law would inject dummy ``userInputFiles`` when ``Data.inputDataset`` is empty, + and the CRAB client then requires ``Site.whitelist``. FLAF sets a real TT + NanoAOD ``inputDataset`` from this era (override with ``crab.input_dataset``) + and ``ignoreLocality = True`` so jobs are not tied to that sample's sites and + no whitelist is required. Optional ``crab.whitelist`` / ``crab.blacklist`` + still restrict or exclude sites. CRAB workers have no AFS, so code is always shipped via the existing BundleTask mechanism (same as ``--bundle`` on HTCondor). Tasks must declare ``bundle_flavours``. @@ -1155,7 +1157,8 @@ class CrabWorkflow(law.cms.CrabWorkflow): Config (``global.yaml`` / user_custom YAML), all optional:: crab: - # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites + # input_dataset: /TTto2L2Nu_.../NANOAODSIM + # whitelist: [T2_CH_CERN] # blacklist: [T2_US_MIT] """ @@ -1176,6 +1179,43 @@ class CrabWorkflow(law.cms.CrabWorkflow): def _crab_cfg(self): return self.global_params.get("crab") or {} + def _crab_input_dataset(self): + """Return a real official DAS dataset so law does not inject userInputFiles. + + FLAF never reads these files (remote I/O is via ``fs_*``). The name only + satisfies CRAB's Analysis plugin / DBS lookup. Prefer ``crab.input_dataset``, + else the first NanoAOD path of TTto2L2Nu / TT / TTtoLNu2Q / TTto4Q. + """ + override = self._crab_cfg().get("input_dataset") + if override: + return str(override) + datasets = getattr(self.setup, "datasets", None) + if datasets is None: + raise RuntimeError( + "CRAB needs a real Data.inputDataset; Setup.datasets is missing. " + "Set crab.input_dataset in global.yaml." + ) + for key in ("TTto2L2Nu", "TT", "TTtoLNu2Q", "TTto4Q"): + try: + entry = datasets[key] + except KeyError: + continue + nano = entry.get("nanoAOD") if isinstance(entry, dict) else None + if isinstance(nano, dict): + candidates = nano.values() + elif isinstance(nano, (list, tuple)): + candidates = nano + else: + candidates = [nano] + for path in candidates: + if isinstance(path, str) and path.startswith("/"): + return path + raise RuntimeError( + "CRAB needs a real Data.inputDataset so law does not inject dummy " + "userInputFiles. Set crab.input_dataset or add TTto2L2Nu/TT to " + "datasets.yaml for this era." + ) + def _ensure_crab_pset(self, n_threads): """Write a minimal CRAB PSet with numberOfThreads matching JobType.numCores.""" n_threads = max(1, int(n_threads)) @@ -1325,16 +1365,15 @@ def crab_job_config(self, config, job_nums, branches=None): # Older CRAB clients may not support maxJobRuntimeMin; ignore if rejected later. pass - # Law always sets dummy userInputFiles (no inputDataset). The CRAB client - # then requires Site.whitelist. Default to every CMS processing site so - # analyses need not pin T2_CH_CERN. An explicit crab.whitelist still - # restricts; crab.blacklist excludes sites on top of the list used. + # Real TT NanoAOD so law does not add dummy userInputFiles (that path + # forces Site.whitelist). ignoreLocality=True: do not pin jobs to the + # sample's Rucio sites; FLAF I/O does not use these files. + config.crab.Data.inputDataset = self._crab_input_dataset() + config.crab.Data.ignoreLocality = True whitelist = list(self._crab_cfg().get("whitelist") or []) blacklist = list(self._crab_cfg().get("blacklist") or []) - if not whitelist: - whitelist = ["T1_*", "T2_*", "T3_*"] - config.crab.Site.whitelist = [str(s) for s in whitelist] - config.crab.Data.ignoreLocality = True + if whitelist: + config.crab.Site.whitelist = [str(s) for s in whitelist] if blacklist: config.crab.Site.blacklist = [str(s) for s in blacklist] From 737ed84612de8bc74672eb8d22841d433f496856 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 14 Aug 2026 22:13:28 +0200 Subject: [PATCH 27/35] require crab.input_dataset in global.yaml --- docs/workflow/arguments.md | 8 ++-- docs/workflow/crab.md | 14 +++--- run_tools/law_customizations.py | 80 ++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index 807ec725..bd8b4090 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -41,10 +41,10 @@ also provides built-in options for status and cleanup. |---|---|---| | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | -Optional site white/black lists go in `global.yaml` under `crab:` (not CLI flags). -A real TT `inputDataset` plus `ignoreLocality` means no whitelist is required. -`Site.storageSite` / `Data.outLFNDirBase` are derived from `fs_default`. -Memory is `2 GB * n_cpus`. +Set `crab.input_dataset` in `global.yaml` (dataset key or DAS path). Combined +with `ignoreLocality`, no site whitelist is required. Optional white/black lists +also go under `crab:`. `Site.storageSite` / `Data.outLFNDirBase` are derived +from `fs_default`. Memory is `2 GB * n_cpus`. ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index b599a9b2..982da719 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -55,15 +55,15 @@ CRAB's write-check site is taken from `fs_default`: | `T3_CH_CERNBOX:/store/user//...` | as written | | `davs://eoshome-.cern.ch:.../eos/user///...` | `T3_CH_CERNBOX` + `/store/user//...` | -FLAF sets a real TT NanoAOD `Data.inputDataset` (from this era's `TTto2L2Nu` / -`TT`, or `crab.input_dataset`) and `ignoreLocality: True`. That stops law from -injecting dummy `userInputFiles`, so **no site whitelist is required** and jobs -are not limited to the TT sample's sites. Restrict or exclude sites only if you -need to: +Set `crab.input_dataset` in `global.yaml` to a real official sample (a +`datasets.yaml` key, or a DAS path). FLAF passes that as `Data.inputDataset` +with `ignoreLocality: True`, so law does not inject dummy `userInputFiles` and +**no site whitelist is required**. Jobs are not limited to that sample's sites. +Restrict or exclude sites only if you need to: ```yaml crab: - # input_dataset: /TTto2L2Nu_.../NANOAODSIM + input_dataset: TTto2L2Nu # whitelist: [T2_CH_CERN] # blacklist: [T2_US_MIT] ``` @@ -79,7 +79,7 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER | Key | Meaning | |---|---| -| `input_dataset` | Optional. Official DAS dataset used only so CRAB has an `inputDataset`. Default: this era's `TTto2L2Nu` / `TT` NanoAOD. | +| `input_dataset` | Required. `datasets.yaml` key (e.g. `TTto2L2Nu`) or a DAS path. Used only so CRAB has an `inputDataset`. | | `whitelist` | Optional. Restricts `Site.whitelist`. Omit to leave sites unrestricted. | | `blacklist` | Optional. CRAB `Site.blacklist`. | diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 523a51a8..e0a1a326 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1145,19 +1145,20 @@ class CrabWorkflow(law.cms.CrabWorkflow): (submit-time write check only). Memory is ``2 GB * n_cpus``. Law would inject dummy ``userInputFiles`` when ``Data.inputDataset`` is empty, - and the CRAB client then requires ``Site.whitelist``. FLAF sets a real TT - NanoAOD ``inputDataset`` from this era (override with ``crab.input_dataset``) - and ``ignoreLocality = True`` so jobs are not tied to that sample's sites and - no whitelist is required. Optional ``crab.whitelist`` / ``crab.blacklist`` + and the CRAB client then requires ``Site.whitelist``. Set + ``crab.input_dataset`` in ``global.yaml`` to a real official sample (dataset + key such as ``TTto2L2Nu``, or a DAS path). Combined with + ``ignoreLocality = True``, jobs are not tied to that sample's sites and no + whitelist is required. Optional ``crab.whitelist`` / ``crab.blacklist`` still restrict or exclude sites. CRAB workers have no AFS, so code is always shipped via the existing BundleTask mechanism (same as ``--bundle`` on HTCondor). Tasks must declare ``bundle_flavours``. - Config (``global.yaml`` / user_custom YAML), all optional:: + Config (``global.yaml``):: crab: - # input_dataset: /TTto2L2Nu_.../NANOAODSIM + input_dataset: TTto2L2Nu # whitelist: [T2_CH_CERN] # blacklist: [T2_US_MIT] """ @@ -1180,40 +1181,47 @@ def _crab_cfg(self): return self.global_params.get("crab") or {} def _crab_input_dataset(self): - """Return a real official DAS dataset so law does not inject userInputFiles. + """Resolve ``crab.input_dataset`` from global.yaml to a DAS path. - FLAF never reads these files (remote I/O is via ``fs_*``). The name only - satisfies CRAB's Analysis plugin / DBS lookup. Prefer ``crab.input_dataset``, - else the first NanoAOD path of TTto2L2Nu / TT / TTtoLNu2Q / TTto4Q. + The value is a datasets.yaml key (e.g. ``TTto2L2Nu``) or a full DAS path. + FLAF never reads these files; they only satisfy CRAB's Analysis plugin. """ - override = self._crab_cfg().get("input_dataset") - if override: - return str(override) + spec = self._crab_cfg().get("input_dataset") + if not spec: + raise RuntimeError( + "CRAB requires crab.input_dataset in global.yaml (a datasets.yaml " + "key such as TTto2L2Nu, or a DAS path). Example:\n" + " crab:\n input_dataset: TTto2L2Nu" + ) + spec = str(spec).strip() + if spec.startswith("/"): + return spec datasets = getattr(self.setup, "datasets", None) if datasets is None: raise RuntimeError( - "CRAB needs a real Data.inputDataset; Setup.datasets is missing. " - "Set crab.input_dataset in global.yaml." + "CRAB crab.input_dataset=%r needs Setup.datasets to resolve a " + "datasets.yaml key." % spec ) - for key in ("TTto2L2Nu", "TT", "TTtoLNu2Q", "TTto4Q"): - try: - entry = datasets[key] - except KeyError: - continue - nano = entry.get("nanoAOD") if isinstance(entry, dict) else None - if isinstance(nano, dict): - candidates = nano.values() - elif isinstance(nano, (list, tuple)): - candidates = nano - else: - candidates = [nano] - for path in candidates: - if isinstance(path, str) and path.startswith("/"): - return path + try: + entry = datasets[spec] + except KeyError: + raise RuntimeError( + "CRAB crab.input_dataset=%r is not a DAS path and was not found " + "in datasets.yaml for this era." % spec + ) + nano = entry.get("nanoAOD") if isinstance(entry, dict) else None + if isinstance(nano, dict): + candidates = nano.values() + elif isinstance(nano, (list, tuple)): + candidates = nano + else: + candidates = [nano] + for path in candidates: + if isinstance(path, str) and path.startswith("/"): + return path raise RuntimeError( - "CRAB needs a real Data.inputDataset so law does not inject dummy " - "userInputFiles. Set crab.input_dataset or add TTto2L2Nu/TT to " - "datasets.yaml for this era." + "CRAB crab.input_dataset=%r has no nanoAOD DAS path in datasets.yaml " + "for this era." % spec ) def _ensure_crab_pset(self, n_threads): @@ -1365,9 +1373,9 @@ def crab_job_config(self, config, job_nums, branches=None): # Older CRAB clients may not support maxJobRuntimeMin; ignore if rejected later. pass - # Real TT NanoAOD so law does not add dummy userInputFiles (that path - # forces Site.whitelist). ignoreLocality=True: do not pin jobs to the - # sample's Rucio sites; FLAF I/O does not use these files. + # crab.input_dataset (global.yaml) so law does not add dummy + # userInputFiles (that path forces Site.whitelist). ignoreLocality=True: + # do not pin jobs to that sample's Rucio sites; FLAF I/O ignores these files. config.crab.Data.inputDataset = self._crab_input_dataset() config.crab.Data.ignoreLocality = True whitelist = list(self._crab_cfg().get("whitelist") or []) From 49320c2490cc679f87ee4c23229904003d6497b5 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 14 Aug 2026 22:15:09 +0200 Subject: [PATCH 28/35] crab.input_dataset is a full DAS path, not a dataset key --- docs/workflow/arguments.md | 4 +-- docs/workflow/crab.md | 13 ++++----- run_tools/law_customizations.py | 50 +++++++++------------------------ 3 files changed, 22 insertions(+), 45 deletions(-) diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index bd8b4090..3e3c3ea0 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -41,8 +41,8 @@ also provides built-in options for status and cleanup. |---|---|---| | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | -Set `crab.input_dataset` in `global.yaml` (dataset key or DAS path). Combined -with `ignoreLocality`, no site whitelist is required. Optional white/black lists +Set `crab.input_dataset` in `global.yaml` to a full DAS path. Combined with +`ignoreLocality`, no site whitelist is required. Optional white/black lists also go under `crab:`. `Site.storageSite` / `Data.outLFNDirBase` are derived from `fs_default`. Memory is `2 GB * n_cpus`. diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 982da719..8a87fb35 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -55,15 +55,14 @@ CRAB's write-check site is taken from `fs_default`: | `T3_CH_CERNBOX:/store/user//...` | as written | | `davs://eoshome-.cern.ch:.../eos/user///...` | `T3_CH_CERNBOX` + `/store/user//...` | -Set `crab.input_dataset` in `global.yaml` to a real official sample (a -`datasets.yaml` key, or a DAS path). FLAF passes that as `Data.inputDataset` -with `ignoreLocality: True`, so law does not inject dummy `userInputFiles` and -**no site whitelist is required**. Jobs are not limited to that sample's sites. -Restrict or exclude sites only if you need to: +Set `crab.input_dataset` in `global.yaml` to a **full DAS path** of any official +sample (a CRAB placeholder only; FLAF does not read it). Combined with +`ignoreLocality: True`, law does not inject dummy `userInputFiles` and **no site +whitelist is required**. Restrict or exclude sites only if you need to: ```yaml crab: - input_dataset: TTto2L2Nu + input_dataset: /TTto2L2Nu_TuneCP5_13p6TeV_powheg-pythia8/Run3Summer22NanoAODv12-130X_mcRun3_2022_realistic_v5-v2/NANOAODSIM # whitelist: [T2_CH_CERN] # blacklist: [T2_US_MIT] ``` @@ -79,7 +78,7 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER | Key | Meaning | |---|---| -| `input_dataset` | Required. `datasets.yaml` key (e.g. `TTto2L2Nu`) or a DAS path. Used only so CRAB has an `inputDataset`. | +| `input_dataset` | Required. Full official DAS path. CRAB placeholder only; not used for FLAF I/O. | | `whitelist` | Optional. Restricts `Site.whitelist`. Omit to leave sites unrestricted. | | `blacklist` | Optional. CRAB `Site.blacklist`. | diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index e0a1a326..dc5ed9e1 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1146,8 +1146,8 @@ class CrabWorkflow(law.cms.CrabWorkflow): Law would inject dummy ``userInputFiles`` when ``Data.inputDataset`` is empty, and the CRAB client then requires ``Site.whitelist``. Set - ``crab.input_dataset`` in ``global.yaml`` to a real official sample (dataset - key such as ``TTto2L2Nu``, or a DAS path). Combined with + ``crab.input_dataset`` in ``global.yaml`` to a real official DAS path (a + placeholder only; FLAF does not read it). Combined with ``ignoreLocality = True``, jobs are not tied to that sample's sites and no whitelist is required. Optional ``crab.whitelist`` / ``crab.blacklist`` still restrict or exclude sites. @@ -1158,7 +1158,7 @@ class CrabWorkflow(law.cms.CrabWorkflow): Config (``global.yaml``):: crab: - input_dataset: TTto2L2Nu + input_dataset: /TTto2L2Nu_TuneCP5_13p6TeV_powheg-pythia8/.../NANOAODSIM # whitelist: [T2_CH_CERN] # blacklist: [T2_US_MIT] """ @@ -1181,48 +1181,26 @@ def _crab_cfg(self): return self.global_params.get("crab") or {} def _crab_input_dataset(self): - """Resolve ``crab.input_dataset`` from global.yaml to a DAS path. + """Return the official DAS path from ``crab.input_dataset`` in global.yaml. - The value is a datasets.yaml key (e.g. ``TTto2L2Nu``) or a full DAS path. - FLAF never reads these files; they only satisfy CRAB's Analysis plugin. + Placeholder for CRAB's Analysis plugin only; FLAF never reads these files. """ spec = self._crab_cfg().get("input_dataset") if not spec: raise RuntimeError( - "CRAB requires crab.input_dataset in global.yaml (a datasets.yaml " - "key such as TTto2L2Nu, or a DAS path). Example:\n" - " crab:\n input_dataset: TTto2L2Nu" + "CRAB requires crab.input_dataset in global.yaml as a full DAS " + "path. Example:\n" + " crab:\n" + " input_dataset: /TTto2L2Nu_TuneCP5_13p6TeV_powheg-pythia8/" + "Run3Summer22NanoAODv12-130X_mcRun3_2022_realistic_v5-v2/NANOAODSIM" ) spec = str(spec).strip() - if spec.startswith("/"): - return spec - datasets = getattr(self.setup, "datasets", None) - if datasets is None: + if not spec.startswith("/"): raise RuntimeError( - "CRAB crab.input_dataset=%r needs Setup.datasets to resolve a " - "datasets.yaml key." % spec + "CRAB crab.input_dataset must be a full DAS path starting with " + "'/', not a datasets.yaml key. Got: %r" % spec ) - try: - entry = datasets[spec] - except KeyError: - raise RuntimeError( - "CRAB crab.input_dataset=%r is not a DAS path and was not found " - "in datasets.yaml for this era." % spec - ) - nano = entry.get("nanoAOD") if isinstance(entry, dict) else None - if isinstance(nano, dict): - candidates = nano.values() - elif isinstance(nano, (list, tuple)): - candidates = nano - else: - candidates = [nano] - for path in candidates: - if isinstance(path, str) and path.startswith("/"): - return path - raise RuntimeError( - "CRAB crab.input_dataset=%r has no nanoAOD DAS path in datasets.yaml " - "for this era." % spec - ) + return spec def _ensure_crab_pset(self, n_threads): """Write a minimal CRAB PSet with numberOfThreads matching JobType.numCores.""" From cde13655dd6026a83c0bd921681b39924c655deb Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 14 Aug 2026 22:25:45 +0200 Subject: [PATCH 29/35] rollback CRAB to dummy userInputFiles and everywhere whitelist --- docs/workflow/arguments.md | 7 ++--- docs/workflow/crab.md | 16 +++++----- run_tools/law_customizations.py | 54 +++++++++------------------------ 3 files changed, 25 insertions(+), 52 deletions(-) diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index 3e3c3ea0..26972645 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -41,10 +41,9 @@ also provides built-in options for status and cleanup. |---|---|---| | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | -Set `crab.input_dataset` in `global.yaml` to a full DAS path. Combined with -`ignoreLocality`, no site whitelist is required. Optional white/black lists -also go under `crab:`. `Site.storageSite` / `Data.outLFNDirBase` are derived -from `fs_default`. Memory is `2 GB * n_cpus`. +Optional site white/black lists go in `global.yaml` under `crab:` (not CLI flags). +Unset whitelist ⇒ all T1/T2/T3 sites. `Site.storageSite` / `Data.outLFNDirBase` +are derived from `fs_default`. Memory is `2 GB * n_cpus`. ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 8a87fb35..2a2cd95a 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -55,15 +55,14 @@ CRAB's write-check site is taken from `fs_default`: | `T3_CH_CERNBOX:/store/user//...` | as written | | `davs://eoshome-.cern.ch:.../eos/user///...` | `T3_CH_CERNBOX` + `/store/user//...` | -Set `crab.input_dataset` in `global.yaml` to a **full DAS path** of any official -sample (a CRAB placeholder only; FLAF does not read it). Combined with -`ignoreLocality: True`, law does not inject dummy `userInputFiles` and **no site -whitelist is required**. Restrict or exclude sites only if you need to: +The CRAB client requires `Site.whitelist` because law uses dummy `userInputFiles` +(no input dataset). FLAF defaults that list to `T1_*`, `T2_*`, `T3_*` so jobs +can run at every CMS processing site. Restrict or exclude sites only if you need +to, in `global.yaml` / `user_custom.yaml`: ```yaml crab: - input_dataset: /TTto2L2Nu_TuneCP5_13p6TeV_powheg-pythia8/Run3Summer22NanoAODv12-130X_mcRun3_2022_realistic_v5-v2/NANOAODSIM - # whitelist: [T2_CH_CERN] + # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites # blacklist: [T2_US_MIT] ``` @@ -78,9 +77,8 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER | Key | Meaning | |---|---| -| `input_dataset` | Required. Full official DAS path. CRAB placeholder only; not used for FLAF I/O. | -| `whitelist` | Optional. Restricts `Site.whitelist`. Omit to leave sites unrestricted. | -| `blacklist` | Optional. CRAB `Site.blacklist`. | +| `whitelist` | Optional. Restricts `Site.whitelist`. Default: `T1_*`, `T2_*`, `T3_*`. | +| `blacklist` | Optional. CRAB `Site.blacklist` (applied on top of the whitelist). | ## Submit diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index dc5ed9e1..359e8367 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1144,22 +1144,19 @@ class CrabWorkflow(law.cms.CrabWorkflow): ``Site.storageSite`` / ``Data.outLFNDirBase`` are derived from ``fs_default`` (submit-time write check only). Memory is ``2 GB * n_cpus``. - Law would inject dummy ``userInputFiles`` when ``Data.inputDataset`` is empty, - and the CRAB client then requires ``Site.whitelist``. Set - ``crab.input_dataset`` in ``global.yaml`` to a real official DAS path (a - placeholder only; FLAF does not read it). Combined with - ``ignoreLocality = True``, jobs are not tied to that sample's sites and no - whitelist is required. Optional ``crab.whitelist`` / ``crab.blacklist`` - still restrict or exclude sites. + Law injects dummy ``userInputFiles`` when ``Data.inputDataset`` is empty, + and the CRAB client then requires ``Site.whitelist``. If ``crab.whitelist`` + is unset, FLAF defaults to ``T1_*`` / ``T2_*`` / ``T3_*`` so jobs can run + at every CMS processing site. Optional ``crab.blacklist`` still excludes + sites. CRAB workers have no AFS, so code is always shipped via the existing BundleTask mechanism (same as ``--bundle`` on HTCondor). Tasks must declare ``bundle_flavours``. - Config (``global.yaml``):: + Config (``global.yaml`` / user_custom YAML), all optional:: crab: - input_dataset: /TTto2L2Nu_TuneCP5_13p6TeV_powheg-pythia8/.../NANOAODSIM - # whitelist: [T2_CH_CERN] + # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites # blacklist: [T2_US_MIT] """ @@ -1180,28 +1177,6 @@ class CrabWorkflow(law.cms.CrabWorkflow): def _crab_cfg(self): return self.global_params.get("crab") or {} - def _crab_input_dataset(self): - """Return the official DAS path from ``crab.input_dataset`` in global.yaml. - - Placeholder for CRAB's Analysis plugin only; FLAF never reads these files. - """ - spec = self._crab_cfg().get("input_dataset") - if not spec: - raise RuntimeError( - "CRAB requires crab.input_dataset in global.yaml as a full DAS " - "path. Example:\n" - " crab:\n" - " input_dataset: /TTto2L2Nu_TuneCP5_13p6TeV_powheg-pythia8/" - "Run3Summer22NanoAODv12-130X_mcRun3_2022_realistic_v5-v2/NANOAODSIM" - ) - spec = str(spec).strip() - if not spec.startswith("/"): - raise RuntimeError( - "CRAB crab.input_dataset must be a full DAS path starting with " - "'/', not a datasets.yaml key. Got: %r" % spec - ) - return spec - def _ensure_crab_pset(self, n_threads): """Write a minimal CRAB PSet with numberOfThreads matching JobType.numCores.""" n_threads = max(1, int(n_threads)) @@ -1351,15 +1326,16 @@ def crab_job_config(self, config, job_nums, branches=None): # Older CRAB clients may not support maxJobRuntimeMin; ignore if rejected later. pass - # crab.input_dataset (global.yaml) so law does not add dummy - # userInputFiles (that path forces Site.whitelist). ignoreLocality=True: - # do not pin jobs to that sample's Rucio sites; FLAF I/O ignores these files. - config.crab.Data.inputDataset = self._crab_input_dataset() - config.crab.Data.ignoreLocality = True + # Law always sets dummy userInputFiles (no inputDataset). The CRAB client + # then requires Site.whitelist. Default to every CMS processing site so + # analyses need not pin T2_CH_CERN. An explicit crab.whitelist still + # restricts; crab.blacklist excludes sites on top of the list used. whitelist = list(self._crab_cfg().get("whitelist") or []) blacklist = list(self._crab_cfg().get("blacklist") or []) - if whitelist: - config.crab.Site.whitelist = [str(s) for s in whitelist] + if not whitelist: + whitelist = ["T1_*", "T2_*", "T3_*"] + config.crab.Site.whitelist = [str(s) for s in whitelist] + config.crab.Data.ignoreLocality = True if blacklist: config.crab.Site.blacklist = [str(s) for s in blacklist] From 7e5d718071460fe7a7ff5c3fd3679230a962dedb Mon Sep 17 00:00:00 2001 From: Konstantin Date: Fri, 14 Aug 2026 23:30:52 +0200 Subject: [PATCH 30/35] CRAB default 5000 parallel jobs and 20 percent refill threshold --- docs/workflow/arguments.md | 8 ++-- docs/workflow/crab.md | 5 ++ run_tools/law_customizations.py | 83 +++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 3 deletions(-) diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index 26972645..849ea8d7 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -28,7 +28,7 @@ also provides built-in options for status and cleanup. | Option | Default | Meaning | |---|---|---| | `--transfer-logs` | off | Bring job logs back to `data/`. Recommended. | -| `--parallel-jobs` | *(unbounded)* | Cap concurrent branches, e.g. `--parallel-jobs 100`. | +| `--parallel-jobs` | unbounded (HTCondor) / **5000** (CRAB) | Cap concurrent jobs. On CRAB this is also the max size of each CRAB task. | | `--max-runtime` | *(task default)* | Per-job wall-clock limit. | | `--n-cpus` | `1` | CPUs requested per job. | | `--priority` | `0` | Job priority (HTCondor). | @@ -42,8 +42,10 @@ also provides built-in options for status and cleanup. | `--workflow crab` | — | Submit branches via CMS CRAB (WLCG). See [CRAB](crab.md). | Optional site white/black lists go in `global.yaml` under `crab:` (not CLI flags). -Unset whitelist ⇒ all T1/T2/T3 sites. `Site.storageSite` / `Data.outLFNDirBase` -are derived from `fs_default`. Memory is `2 GB * n_cpus`. +Unset whitelist ⇒ all T1/T2/T3 sites. Default `--parallel-jobs` on CRAB is 5000 +(`crab.parallel_jobs`); a new CRAB task is submitted only when at least +`crab.refill_fraction` (default 0.2) of those slots are free. `Site.storageSite` +/ `Data.outLFNDirBase` are derived from `fs_default`. Memory is `2 GB * n_cpus`. ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 2a2cd95a..81fa5924 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -64,6 +64,8 @@ to, in `global.yaml` / `user_custom.yaml`: crab: # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites # blacklist: [T2_US_MIT] + # parallel_jobs: 5000 # default --parallel-jobs; CLI wins if set + # refill_fraction: 0.2 # new CRAB task only when this fraction of slots is free ``` Memory is `2 GB * n_cpus` (the existing `--n-cpus` parameter). There is no separate @@ -79,6 +81,8 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER |---|---| | `whitelist` | Optional. Restricts `Site.whitelist`. Default: `T1_*`, `T2_*`, `T3_*`. | | `blacklist` | Optional. CRAB `Site.blacklist` (applied on top of the whitelist). | +| `parallel_jobs` | Optional. Default for `--parallel-jobs` on CRAB (CLI wins). Default: `5000`. Caps how many CRAB jobs are in flight and thus the size of each CRAB task. CRAB itself refuses more than 10 000 jobs in one task. | +| `refill_fraction` | Optional. Submit a new CRAB task only when `parallel_jobs - n_active >= refill_fraction * parallel_jobs`. Default: `0.2`. Prevents a 1-job task every time a single job finishes. | ## Submit @@ -94,6 +98,7 @@ law run FLAF.Analysis.tasks.HistTupleProducerTask \ | Option | Why | |---|---| | `--workflow crab` | Submit via CRAB instead of local/HTCondor. | +| `--parallel-jobs` | Jobs in flight (default **5000** on CRAB, unlimited on HTCondor). Each refill is one CRAB task. Also `crab.parallel_jobs` in `global.yaml`. | | `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory (`2 GB * n_cpus`). | | `--transfer-logs` | On by default; enables remote log stageout when `fs_default` is WLCG. | diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 359e8367..059fa139 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -9,6 +9,8 @@ import subprocess import tempfile +from collections import OrderedDict + from law.parser import global_cmdline_values from FLAF.RunKit.run_tools import natural_sort @@ -1051,7 +1053,86 @@ def _rewrite_crab_job_file(job_file): _FLAFCrabWorkflowProxyBase = law.cms.CrabWorkflow.workflow_proxy_cls +_CRAB_DEFAULT_PARALLEL_JOBS = 5000 +_CRAB_DEFAULT_REFILL_FRACTION = 0.2 + + +def _cli_has_parallel_jobs(): + """True when the user passed ``--parallel-jobs`` (or a task-prefixed form).""" + parser = luigi.cmdline_parser.CmdlineParser.get_instance() + tokens = list(getattr(parser, "cmdline_args", None) or []) + for tok in tokens: + if tok in ("--parallel-jobs", "--parallel_jobs"): + return True + if tok.startswith("--parallel-jobs=") or tok.startswith("--parallel_jobs="): + return True + if tok.endswith("-parallel-jobs") or tok.endswith("-parallel_jobs"): + return True + if "-parallel-jobs=" in tok or "-parallel_jobs=" in tok: + return True + return False + + class _FLAFCrabWorkflowProxy(_FLAFCrabWorkflowProxyBase): + def __init__(self, *args, **kwargs): + super(_FLAFCrabWorkflowProxy, self).__init__(*args, **kwargs) + self._apply_crab_parallel_jobs() + + def _crab_refill_fraction(self): + raw = self.task._crab_cfg().get( + "refill_fraction", _CRAB_DEFAULT_REFILL_FRACTION + ) + try: + frac = float(raw) + except (TypeError, ValueError): + frac = _CRAB_DEFAULT_REFILL_FRACTION + return min(max(frac, 0.0), 1.0) + + def _apply_crab_parallel_jobs(self): + """CRAB default is 5000 jobs in flight; yaml then CLI override. + + Multi-workflow tasks inherit HTCondor's unlimited ``parallel_jobs``, so + the CrabWorkflow class default never wins. Apply the CRAB default here. + """ + if _cli_has_parallel_jobs(): + return + yaml_n = self.task._crab_cfg().get("parallel_jobs") + if yaml_n is not None: + self._set_parallel_jobs(int(yaml_n)) + return + if self.poll_data.n_parallel == self.n_parallel_max: + self._set_parallel_jobs(_CRAB_DEFAULT_PARALLEL_JOBS) + + def _should_submit_crab_group(self): + """Refill only when enough slots are free (default 20% of parallel_jobs). + + The first wave always submits. Unlimited ``parallel_jobs`` keeps law's + original behaviour (one group with every remaining job). + """ + n_parallel = self.poll_data.n_parallel + if n_parallel >= self.n_parallel_max: + return True + is_first_wave = (not self.job_data.jobs) and (not self._submitted) + if is_first_wave: + return True + free = n_parallel - self.poll_data.n_active + return free >= self._crab_refill_fraction() * n_parallel + + def submit(self, retry_jobs=None): + if self._should_submit_crab_group(): + return super(_FLAFCrabWorkflowProxy, self).submit(retry_jobs) + + # Park retries as unsubmitted so the next eligible refill picks them up + # as one larger CRAB task instead of a 1-job task now. + if retry_jobs: + for job_num, branches in retry_jobs.items(): + if self._can_skip_job(job_num, branches): + continue + self.job_data.jobs.pop(job_num, None) + self.job_data.unsubmitted_jobs[job_num] = branches + self.dump_job_data() + return OrderedDict() + def setup_job_manager(self): """Require a valid VOMS proxy and a MyProxy credential (>= 5 days).""" proxy = os.environ.get("X509_USER_PROXY", "") @@ -1158,6 +1239,8 @@ class CrabWorkflow(law.cms.CrabWorkflow): crab: # whitelist: [T2_CH_CERN] # omit to use all T1/T2/T3 sites # blacklist: [T2_US_MIT] + # parallel_jobs: 5000 # --parallel-jobs default; CLI wins + # refill_fraction: 0.2 # refill when free slots >= this * parallel_jobs """ # Re-declare in the class body so law's metaclass sets _defined_workflow_proxy=True From 9d07bae84b28f3071411188facd3880b4319b270 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 15 Aug 2026 05:05:30 +0200 Subject: [PATCH 31/35] raise CRAB AnaTuple memory to 4 cores within client cap --- AnaProd/tasks.py | 3 ++- docs/workflow/arguments.md | 4 +++- docs/workflow/crab.md | 10 +++++++--- run_tools/law_customizations.py | 15 ++++++++++++--- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/AnaProd/tasks.py b/AnaProd/tasks.py index 7e86d9f2..9bd5a2bc 100644 --- a/AnaProd/tasks.py +++ b/AnaProd/tasks.py @@ -118,7 +118,8 @@ def WF_complete(ref_task): class AnaTupleFileTask(Task, HTCondorWorkflow, CrabWorkflow, law.LocalWorkflow): max_runtime = copy_param(HTCondorWorkflow.max_runtime, 40.0) - n_cpus = copy_param(HTCondorWorkflow.n_cpus, 2) + # tautau CMSSW AnaTuple used ~7.5 GB RSS on CRAB; 2 cores cap at 5000 MB. + n_cpus = copy_param(HTCondorWorkflow.n_cpus, 4) @property def bundle_flavours(self): diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index 849ea8d7..daf6af35 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -45,7 +45,9 @@ Optional site white/black lists go in `global.yaml` under `crab:` (not CLI flags Unset whitelist ⇒ all T1/T2/T3 sites. Default `--parallel-jobs` on CRAB is 5000 (`crab.parallel_jobs`); a new CRAB task is submitted only when at least `crab.refill_fraction` (default 0.2) of those slots are free. `Site.storageSite` -/ `Data.outLFNDirBase` are derived from `fs_default`. Memory is `2 GB * n_cpus`. +/ `Data.outLFNDirBase` are derived from `fs_default`. Memory is +`3000 MB * n_cpus` (`crab.memory_mb_per_cpu`), capped at the CRAB client +limit (5000 MB for 1 core, `2500 MB * n_cpus` otherwise). ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 81fa5924..5c29ddfd 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -66,10 +66,13 @@ crab: # blacklist: [T2_US_MIT] # parallel_jobs: 5000 # default --parallel-jobs; CLI wins if set # refill_fraction: 0.2 # new CRAB task only when this fraction of slots is free + # memory_mb_per_cpu: 3000 # CRAB maxMemoryMB / n_cpus ``` -Memory is `2 GB * n_cpus` (the existing `--n-cpus` parameter). There is no separate -CRAB memory flag. +Memory is `3000 MB * n_cpus` (override with `crab.memory_mb_per_cpu`), then capped +at the CRAB client limit: 5000 MB for 1 core, `2500 MB * n_cpus` otherwise. +There is no separate CRAB memory CLI flag. AnaTuple production defaults to +4 cores (10 GB) so tautau CMSSW jobs fit; 2 cores only allow 5 GB. Verify write access before the first campaign: @@ -83,6 +86,7 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER | `blacklist` | Optional. CRAB `Site.blacklist` (applied on top of the whitelist). | | `parallel_jobs` | Optional. Default for `--parallel-jobs` on CRAB (CLI wins). Default: `5000`. Caps how many CRAB jobs are in flight and thus the size of each CRAB task. CRAB itself refuses more than 10 000 jobs in one task. | | `refill_fraction` | Optional. Submit a new CRAB task only when `parallel_jobs - n_active >= refill_fraction * parallel_jobs`. Default: `0.2`. Prevents a 1-job task every time a single job finishes. | +| `memory_mb_per_cpu` | Optional. CRAB `JobType.maxMemoryMB` is this times `--n-cpus`, capped at 5000 MB (1 core) or `2500 MB * n_cpus`. Default: `3000`. | ## Submit @@ -99,7 +103,7 @@ law run FLAF.Analysis.tasks.HistTupleProducerTask \ |---|---| | `--workflow crab` | Submit via CRAB instead of local/HTCondor. | | `--parallel-jobs` | Jobs in flight (default **5000** on CRAB, unlimited on HTCondor). Each refill is one CRAB task. Also `crab.parallel_jobs` in `global.yaml`. | -| `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory (`2 GB * n_cpus`). | +| `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory (`3000 MB * n_cpus`, CRAB-capped). | | `--transfer-logs` | On by default; enables remote log stageout when `fs_default` is WLCG. | You do **not** need `--bundle` for CRAB — bundles are forced whenever the workflow is `crab`. diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 059fa139..1d376c23 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1223,7 +1223,8 @@ class CrabWorkflow(law.cms.CrabWorkflow): nothing is duplicated onto CRAB's stageout area. ``Site.storageSite`` / ``Data.outLFNDirBase`` are derived from ``fs_default`` - (submit-time write check only). Memory is ``2 GB * n_cpus``. + (submit-time write check only). Memory is ``3000 MB * n_cpus`` (override + with ``crab.memory_mb_per_cpu``). Law injects dummy ``userInputFiles`` when ``Data.inputDataset`` is empty, and the CRAB client then requires ``Site.whitelist``. If ``crab.whitelist`` @@ -1241,6 +1242,7 @@ class CrabWorkflow(law.cms.CrabWorkflow): # blacklist: [T2_US_MIT] # parallel_jobs: 5000 # --parallel-jobs default; CLI wins # refill_fraction: 0.2 # refill when free slots >= this * parallel_jobs + # memory_mb_per_cpu: 3000 # CRAB JobType.maxMemoryMB / n_cpus """ # Re-declare in the class body so law's metaclass sets _defined_workflow_proxy=True @@ -1388,9 +1390,16 @@ def crab_job_config(self, config, job_nums, branches=None): config.render_variables["log_remote_base_url"] = log_remote_base_url # Cores + memory. CRAB requires JobType.numCores == PSet numberOfThreads. - # Memory is 2 GB per CPU from the existing n_cpus parameter. + # Default 3000 MB/CPU, then clamp to the CRAB client max + # (5000 MB for 1 core, 2500 MB * n_cpus otherwise). n_cpus = max(1, int(getattr(self, "n_cpus", 1) or 1)) - mem = n_cpus * 2000 + try: + mb_per_cpu = int(self._crab_cfg().get("memory_mb_per_cpu", 3000)) + except (TypeError, ValueError): + mb_per_cpu = 3000 + # CRAB client cap: 5000 MB (1 core) or 2500 MB * n_cpus (multi-core). + crab_max = 5000 if n_cpus == 1 else 2500 * n_cpus + mem = min(n_cpus * max(mb_per_cpu, 1), crab_max) pset_path = self._ensure_crab_pset(n_cpus) config.crab.JobType.psetName = pset_path config.crab.JobType.numCores = n_cpus From a89a7b48e922b4b770db36477a7a67c3459ff6b7 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 15 Aug 2026 05:26:45 +0200 Subject: [PATCH 32/35] put overlay FLAF on CMSSW PYTHONPATH for AnaTuple --- Common/Setup.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Common/Setup.py b/Common/Setup.py index c63363e4..4b231a3f 100644 --- a/Common/Setup.py +++ b/Common/Setup.py @@ -742,11 +742,13 @@ def cmssw_env(self): for var in [ "HOME", "FLAF_PATH", + "CORRECTIONS_PATH", "ANALYSIS_PATH", "ANALYSIS_DATA_PATH", "X509_USER_PROXY", "FLAF_CMSSW_BASE", "FLAF_CMSSW_ARCH", + "PYTHONSAFEPATH", "LAW_CRAB_JOB_NUMBER", "CRAB_Id", "LAW_JOB_INIT_DIR", @@ -762,11 +764,22 @@ def cmssw_env(self): if flaf_cmssw and os.path.isdir(flaf_cmssw): self.cmssw_env_["CMSSW_BASE"] = flaf_cmssw self.cmssw_env_["FLAF_CMSSW_BASE"] = flaf_cmssw + # Prepend overlay parents (if set) then ANALYSIS_PATH so `import FLAF` + # / `import Corrections` match env.sh. Without this, CMSSW python + # resolves the submodule copy and misses overlay-only modules. + py_prefix = [self.ana_path] + for env_key in ("FLAF_PATH", "CORRECTIONS_PATH"): + overlay = os.environ.get(env_key) + if overlay and os.path.isdir(overlay): + parent = os.path.dirname(os.path.abspath(overlay)) + if parent and parent not in py_prefix: + py_prefix.insert(0, parent) + py_prefix_str = ":".join(py_prefix) if "PYTHONPATH" not in self.cmssw_env_: - self.cmssw_env_["PYTHONPATH"] = self.ana_path + self.cmssw_env_["PYTHONPATH"] = py_prefix_str else: self.cmssw_env_["PYTHONPATH"] = ( - f'{self.ana_path}:{self.cmssw_env["PYTHONPATH"]}' + f"{py_prefix_str}:{self.cmssw_env_['PYTHONPATH']}" ) return self.cmssw_env_ From c83c14760c5f6a7f6d429ee4b977a92c12e5d994 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 15 Aug 2026 06:49:04 +0200 Subject: [PATCH 33/35] pin merge n_cpus in HistTuple requires so CRAB workers reuse it --- AnaProd/MergeAnaTuples.py | 6 ++++++ Analysis/tasks.py | 1 + 2 files changed, 7 insertions(+) diff --git a/AnaProd/MergeAnaTuples.py b/AnaProd/MergeAnaTuples.py index 793d23c4..68bb5bcf 100644 --- a/AnaProd/MergeAnaTuples.py +++ b/AnaProd/MergeAnaTuples.py @@ -209,6 +209,12 @@ def mergeAnaTuples( else: tree_list = [(central, central, "Events")] + if not tree_list: + raise RuntimeError( + f"No trees to merge for dataset {dataset_name} " + f"(empty or missing report tree lists)." + ) + if len(root_outputs) > 1 and len(tree_list) > 1: raise NotImplementedError( "Cannot write multiple output files when there are multiple uncertainties." diff --git a/Analysis/tasks.py b/Analysis/tasks.py index eeeabac2..ce074738 100644 --- a/Analysis/tasks.py +++ b/Analysis/tasks.py @@ -202,6 +202,7 @@ def requires(self): "anaTuple": AnaTupleMergeTask.req( self, max_runtime=AnaTupleMergeTask.max_runtime._default, + n_cpus=AnaTupleMergeTask.n_cpus._default, branch=prod_br, branches=(prod_br,), customisations=self.customisations, From 214cc746490fc5c4b02162903f1a6c2ce13c8cee Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 15 Aug 2026 10:39:09 +0200 Subject: [PATCH 34/35] document CRAB distant-site EOS I/O and live-bundle replace --- docs/workflow/crab.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index 5c29ddfd..c01262dc 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -145,6 +145,18 @@ also use `crab status -d ` from a CMSSW environment. LFN. At CERN, prefer `T3_CH_CERNBOX` for `/store/user/...` (maps to personal EOS and usually passes `crab checkwrite`); `T2_CH_CERN /store/user` often does not exist. +!!! note "Distant sites still read `fs_default`" + The default whitelist lets jobs run anywhere, but the bundle and outputs stay + on `fs_default`. Personal EOS (`davs://eoshome-*.cern.ch`) can fail or stall + from far-away sites (gfal 112, HTTP 404, hung DNN). Law retries usually + recover; set `crab.whitelist` closer to CERN if that I/O is a problem. + +!!! warning "Do not replace a live bundle mid-campaign" + `BundleTask` can stay DONE after `core.tar.bz2` is deleted because of the + path-existence cache. Workers then get HTTP 404. Rebuild into a sibling file + and `mv` it over the live path; do not `cp` onto a file jobs may be + downloading (a mid-copy can stage out 0 bytes). + !!! note "Test small first" Validate with `--workflow local --branches 0 --test 1000`, then a single CRAB branch, before large submissions. From ba368e20dff2127a67f45fe27756234eae5de140 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Sat, 15 Aug 2026 14:43:30 +0200 Subject: [PATCH 35/35] default CRAB memory to 2000 MB per core --- docs/workflow/arguments.md | 4 ++-- docs/workflow/crab.md | 15 ++++++++------- run_tools/law_customizations.py | 15 ++++++++------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/workflow/arguments.md b/docs/workflow/arguments.md index daf6af35..3b63f56d 100644 --- a/docs/workflow/arguments.md +++ b/docs/workflow/arguments.md @@ -46,8 +46,8 @@ Unset whitelist ⇒ all T1/T2/T3 sites. Default `--parallel-jobs` on CRAB is 500 (`crab.parallel_jobs`); a new CRAB task is submitted only when at least `crab.refill_fraction` (default 0.2) of those slots are free. `Site.storageSite` / `Data.outLFNDirBase` are derived from `fs_default`. Memory is -`3000 MB * n_cpus` (`crab.memory_mb_per_cpu`), capped at the CRAB client -limit (5000 MB for 1 core, `2500 MB * n_cpus` otherwise). +`2000 MB * n_cpus` (`crab.memory_mb_per_cpu`; CRAB / site-guaranteed default), +capped at the CRAB client limit (5000 MB for 1 core, `2500 MB * n_cpus` otherwise). ## Status & cleanup (LAW built-ins) diff --git a/docs/workflow/crab.md b/docs/workflow/crab.md index c01262dc..c416ecac 100644 --- a/docs/workflow/crab.md +++ b/docs/workflow/crab.md @@ -66,13 +66,14 @@ crab: # blacklist: [T2_US_MIT] # parallel_jobs: 5000 # default --parallel-jobs; CLI wins if set # refill_fraction: 0.2 # new CRAB task only when this fraction of slots is free - # memory_mb_per_cpu: 3000 # CRAB maxMemoryMB / n_cpus + # memory_mb_per_cpu: 2000 # CRAB maxMemoryMB / n_cpus ``` -Memory is `3000 MB * n_cpus` (override with `crab.memory_mb_per_cpu`), then capped -at the CRAB client limit: 5000 MB for 1 core, `2500 MB * n_cpus` otherwise. -There is no separate CRAB memory CLI flag. AnaTuple production defaults to -4 cores (10 GB) so tautau CMSSW jobs fit; 2 cores only allow 5 GB. +Memory is `2000 MB * n_cpus` (override with `crab.memory_mb_per_cpu`), matching +the CRAB default that all sites guarantee per core. Then capped at the CRAB +client limit: 5000 MB for 1 core, `2500 MB * n_cpus` otherwise. There is no +separate CRAB memory CLI flag. AnaTuple production defaults to 4 cores (8 GB) +so tautau CMSSW jobs fit; 2 cores only allow 5 GB. Verify write access before the first campaign: @@ -86,7 +87,7 @@ crab checkwrite --site=T3_CH_CERNBOX --lfn=/store/user/$USER | `blacklist` | Optional. CRAB `Site.blacklist` (applied on top of the whitelist). | | `parallel_jobs` | Optional. Default for `--parallel-jobs` on CRAB (CLI wins). Default: `5000`. Caps how many CRAB jobs are in flight and thus the size of each CRAB task. CRAB itself refuses more than 10 000 jobs in one task. | | `refill_fraction` | Optional. Submit a new CRAB task only when `parallel_jobs - n_active >= refill_fraction * parallel_jobs`. Default: `0.2`. Prevents a 1-job task every time a single job finishes. | -| `memory_mb_per_cpu` | Optional. CRAB `JobType.maxMemoryMB` is this times `--n-cpus`, capped at 5000 MB (1 core) or `2500 MB * n_cpus`. Default: `3000`. | +| `memory_mb_per_cpu` | Optional. CRAB `JobType.maxMemoryMB` is this times `--n-cpus`, capped at 5000 MB (1 core) or `2500 MB * n_cpus`. Default: `2000` (CRAB / site-guaranteed per-core default). | ## Submit @@ -103,7 +104,7 @@ law run FLAF.Analysis.tasks.HistTupleProducerTask \ |---|---| | `--workflow crab` | Submit via CRAB instead of local/HTCondor. | | `--parallel-jobs` | Jobs in flight (default **5000** on CRAB, unlimited on HTCondor). Each refill is one CRAB task. Also `crab.parallel_jobs` in `global.yaml`. | -| `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory (`3000 MB * n_cpus`, CRAB-capped). | +| `--max-runtime` / `--n-cpus` | Same as HTCondor; mapped to CRAB `maxJobRuntimeMin` / `numCores` / memory (`2000 MB * n_cpus`, CRAB-capped). | | `--transfer-logs` | On by default; enables remote log stageout when `fs_default` is WLCG. | You do **not** need `--bundle` for CRAB — bundles are forced whenever the workflow is `crab`. diff --git a/run_tools/law_customizations.py b/run_tools/law_customizations.py index 1d376c23..5cdcfcdd 100644 --- a/run_tools/law_customizations.py +++ b/run_tools/law_customizations.py @@ -1223,8 +1223,8 @@ class CrabWorkflow(law.cms.CrabWorkflow): nothing is duplicated onto CRAB's stageout area. ``Site.storageSite`` / ``Data.outLFNDirBase`` are derived from ``fs_default`` - (submit-time write check only). Memory is ``3000 MB * n_cpus`` (override - with ``crab.memory_mb_per_cpu``). + (submit-time write check only). Memory is ``2000 MB * n_cpus`` (override + with ``crab.memory_mb_per_cpu``), matching the CRAB / site-guaranteed default. Law injects dummy ``userInputFiles`` when ``Data.inputDataset`` is empty, and the CRAB client then requires ``Site.whitelist``. If ``crab.whitelist`` @@ -1242,7 +1242,7 @@ class CrabWorkflow(law.cms.CrabWorkflow): # blacklist: [T2_US_MIT] # parallel_jobs: 5000 # --parallel-jobs default; CLI wins # refill_fraction: 0.2 # refill when free slots >= this * parallel_jobs - # memory_mb_per_cpu: 3000 # CRAB JobType.maxMemoryMB / n_cpus + # memory_mb_per_cpu: 2000 # CRAB JobType.maxMemoryMB / n_cpus """ # Re-declare in the class body so law's metaclass sets _defined_workflow_proxy=True @@ -1390,13 +1390,14 @@ def crab_job_config(self, config, job_nums, branches=None): config.render_variables["log_remote_base_url"] = log_remote_base_url # Cores + memory. CRAB requires JobType.numCores == PSet numberOfThreads. - # Default 3000 MB/CPU, then clamp to the CRAB client max - # (5000 MB for 1 core, 2500 MB * n_cpus otherwise). + # Default 2000 MB/CPU (CRAB default; all sites guarantee this per core), + # then clamp to the CRAB client max (5000 MB for 1 core, 2500 MB * n_cpus + # otherwise). n_cpus = max(1, int(getattr(self, "n_cpus", 1) or 1)) try: - mb_per_cpu = int(self._crab_cfg().get("memory_mb_per_cpu", 3000)) + mb_per_cpu = int(self._crab_cfg().get("memory_mb_per_cpu", 2000)) except (TypeError, ValueError): - mb_per_cpu = 3000 + mb_per_cpu = 2000 # CRAB client cap: 5000 MB (1 core) or 2500 MB * n_cpus (multi-core). crab_max = 5000 if n_cpus == 1 else 2500 * n_cpus mem = min(n_cpus * max(mb_per_cpu, 1), crab_max)