CosmoStat · martinkilbinger · Jan 9, 2023 · Jan 12, 2023 · Jan 12, 2023 · Jan 17, 2023
diff --git a/Dockerfile.jupyter b/Dockerfile.jupyter
@@ -1,7 +1,7 @@
 # Dockerfile for ShapePipe Jupyter Lab
 # Uses the shared base image
 
-FROM ghcr.io/cosmostat/shapepipe:v2.0
+FROM ghcr.io/cosmostat/shapepipe:develop
 
 # Default command starts Jupyter Lab
 ENTRYPOINT ["jupyter","lab","--ip=0.0.0.0","--port=8888","--no-browser","--allow-root"]
diff --git a/bin/canfar_monitor.py b/bin/canfar_monitor.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.canfar_run import run_log
+run_log()
diff --git a/bin/canfar_monitor_log.py b/bin/canfar_monitor_log.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.canfar_run import run_monitor_log
+run_monitor_log()
diff --git a/bin/canfar_submit_job.py b/bin/canfar_submit_job.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.canfar_run import run_job
+run_job()
diff --git a/bin/shapepipe_run.py b/bin/shapepipe_run.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.shapepipe_run import main
+main()
diff --git a/bin/summary_run.py b/bin/summary_run.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from shapepipe.summary_run import main
+main()
diff --git a/example/cfis/config_exp_mccd.ini b/example/cfis/config_exp_mccd.ini
@@ -236,5 +236,3 @@ PSF_MODEL_PATTERN = fitted_model
 # PSF model separator
 PSF_MODEL_SEPARATOR = -
 
-# For multi-epoch purposes
-ME_LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite
diff --git a/example/cfis/config_onthefly.mask b/example/cfis/config_onthefly.mask
@@ -3,7 +3,7 @@
 ## Paths to executables
 [PROGRAM_PATH]
 
-WW_PATH = ww
+WW_PATH = weightwatcher
 WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww
 
 # Indicate cds client executable if no external star catalogue is available

diff --git a/example/cfis/config_save.mask b/example/cfis/config_save.mask
@@ -3,7 +3,7 @@
 ## Paths to executables
 [PROGRAM_PATH]
 
-WW_PATH = ww
+WW_PATH = weightwatcher
 WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww
 
 # Indicate cds client executable if no external star catalogue is available

diff --git a/example/cfis/config_tile_Git_cat_vos.ini b/example/cfis/config_tile_Git_cat_vos.ini
@@ -0,0 +1,90 @@
+# ShapePipe configuration file for: get external tile catalogue
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = False
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_tile_Gic
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = get_images_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Get external tile catalogue
+[GET_IMAGES_RUNNER]
+
+FILE_PATTERN = tile_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME =
+
+# Paths
+
+# Input path where catalogues are stored
+INPUT_PATH = vos:cfis/tiles_DR6
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = CFIS.000.000.r
+
+# Input file extensions
+INPUT_FILE_EXT = .cat
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{3}\.\d{3}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = CFIS_cat-
+
+# Copy/download method, one in 'vos', 'symlink'
+RETRIEVE = vos
+
+# If RETRIEVE=vos, number of attempts to download
+# Optional, default=3
+N_TRY = 3
+
+# Copy command options, optional
+RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem
diff --git a/example/cfis/config_exp_Mh.ini → example/cfis/config_tile_Mh_exp.ini b/example/cfis/config_exp_Mh.ini → example/cfis/config_tile_Mh_exp.ini
@@ -11,7 +11,7 @@
 VERBOSE = True
 
 # Name of run (optional) default: shapepipe_run
-RUN_NAME = run_sp_exp_Mh
+RUN_NAME = run_sp_tile_Mh_exp
 
 # Add date and time to RUN_NAME, optional, default: True
 RUN_DATETIME = False

diff --git a/example/cfis/config_tile_MiViSmVi.ini b/example/cfis/config_tile_MiViSmVi.ini
@@ -57,11 +57,11 @@ TIMEOUT = 96:00:00
 
 [MCCD_INTERP_RUNNER]
 
-INPUT_DIR = last:sextractor_runner_run_1
+INPUT_DIR = last:sextractor_runner_run_1, run_sp_exp_Mh:merge_headers_runner
 
-FILE_PATTERN = sexcat
+FILE_PATTERN = sexcat, log_exp_headers
 
-FILE_EXT = .fits
+FILE_EXT = .fits, .sqlite
 
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
@@ -84,9 +84,6 @@ PSF_MODEL_PATTERN = fitted_model
 
 PSF_MODEL_SEPARATOR = -
 
-# Multi-epoch mode: Path to file with single-exposure WCS header information
-ME_LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite
-
 
 [VIGNETMAKER_RUNNER_RUN_1]
 
@@ -146,11 +143,11 @@ OUTPUT_MODE = new
 # Create multi-epoch vignets for tiles corresponding to
 # positions on single-exposures
 
-INPUT_DIR = last:sextractor_runner_run_1
+INPUT_DIR = last:sextractor_runner_run_1, run_sp_exp_Mh:merge_headers_runner
 
-FILE_PATTERN = sexcat
+FILE_PATTERN = sexcat, log_exp_headers
 
-FILE_EXT = .fits
+FILE_EXT = .fits, .sqlite
 
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
@@ -178,4 +175,3 @@ PREFIX =
 # run outputs
 ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2
 ME_IMAGE_PATTERN = flag, image, weight, background
-ME_LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite
diff --git a/example/cfis/config_tile_Ng_batch_psfex_uc.ini b/example/cfis/config_tile_Ng_batch_psfex_uc.ini
@@ -0,0 +1,76 @@
+# ShapePipe configuration file for tiles: ngmix
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_tile_Ng
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = ngmix_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 24
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Model-fitting shapes with ngmix
+[NGMIX_RUNNER]
+
+INPUT_DIR = run_sp_tile_Uc:read_ext_sexcat_runner,last:psfex_interp_runner,last:vignetmaker_runner_run_2,run_sp_tile_Mh_exp:merge_headers_runner
+
+FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers
+
+FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Magnitude zero-point
+MAG_ZP = 30.0
+
+# Pixel scale in arcsec
+PIXEL_SCALE = 0.186
+
+SAVE_BATCH = 1000
+
+ID_OBJ_MIN = -1
+ID_OBJ_MAX = -1
diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini
@@ -55,18 +55,15 @@ TIMEOUT = 96:00:00
 # Model-fitting shapes with ngmix
 [NGMIX_RUNNER]
 
-INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2,run_sp_exp_Mh:merge_headers_runner
 
-FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet
+FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers
 
-FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite
+FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite
 
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
 
-# Multi-epoch mode: Path to file with single-exposure WCS header information
-LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite
-
 # Magnitude zero-point
 MAG_ZP = 30.0
 

diff --git a/example/cfis/config_tile_Ng_template_batch.ini b/example/cfis/config_tile_Ng_template_batch.ini
@@ -55,18 +55,15 @@ TIMEOUT = 96:00:00
 # Model-fitting shapes with ngmix
 [NGMIX_RUNNER]
 
-INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2,run_sp_exp_Mh:merge_headers_runner
 
-FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet
+FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers
 
-FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite
+FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite
 
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
 
-# Multi-epoch mode: Path to file with single-exposure WCS header information
-LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite
-
 # Directory of previous run, optional.
 CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u_prev/ngmix_runner/output
 

diff --git a/example/cfis/config_tile_PiViVi_canfar.ini → ...ple/cfis/config_tile_PiViVi_canfar_sx.ini b/example/cfis/config_tile_PiViVi_canfar.ini → ...ple/cfis/config_tile_PiViVi_canfar_sx.ini
@@ -9,7 +9,7 @@
 VERBOSE = True
 
 # Name of run (optional) default: shapepipe_run
-RUN_NAME = run_sp_tile_PsViSmVi
+RUN_NAME = run_sp_tile_PiViVi
 
 # Add date and time to RUN_NAME, optional, default: False
 ; RUN_DATETIME = False
@@ -57,11 +57,11 @@ TIMEOUT = 96:00:00
 
 [PSFEX_INTERP_RUNNER]
 
-INPUT_DIR = run_sp_tile_Sx:sextractor_runner
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner
 
-FILE_PATTERN = sexcat
+FILE_PATTERN = sexcat, log_exp_headers, exp_numbers
 
-FILE_EXT = .fits
+FILE_EXT = .fits, .sqlite, .txt
 
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
@@ -86,14 +86,14 @@ CHI2_THRESH = 2
 
 # Multi-epoch mode parameters
 
-ME_DOT_PSF_DIR = all:psfex_runner
+# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR
+# for v2.0 per-exposure pipeline.  psfex_runner/output/ dirs are discovered
+# by scanning $SP_EXP for the exposures listed in the exp_numbers input file.
+ME_DOT_PSF_EXP_DIR = $SP_EXP
 
 # Input psf file pattern
 ME_DOT_PSF_PATTERN = star_split_ratio_80
 
-# Multi-epoch mode: Path to file with single-exposure WCS header information
-ME_LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite
-
 
 # Create vignets for tiles weights
 [VIGNETMAKER_RUNNER_RUN_1]
@@ -132,11 +132,11 @@ PREFIX = weight
 # Create multi-epoch vignets for tiles corresponding to
 # positions on single-exposures
 
-INPUT_DIR = run_sp_tile_Sx:sextractor_runner
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner, run_sp_tile_Mh_exp:merge_headers_runner, last:find_exposures_runner
 
-FILE_PATTERN = sexcat
+FILE_PATTERN = sexcat, log_exp_headers, exp_numbers
 
-FILE_EXT = .fits
+FILE_EXT = .fits, .sqlite, .txt
 
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
@@ -161,7 +161,8 @@ STAMP_SIZE = 51
 PREFIX =
 
 # Additional parameters for path and file pattern corresponding to single-exposure
-# run outputs
-ME_IMAGE_DIR = all:split_exp_runner, all:split_exp_runner, all:split_exp_runner, all:sextractor_runner
+# run outputs.  ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for
+# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP.
+ME_IMAGE_EXP_DIR = $SP_EXP
+ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner
 ME_IMAGE_PATTERN = flag, image, weight, background
-ME_LOG_WCS = $SP_RUN/output/run_sp_exp_Mh/merge_headers_runner/output/log_exp_headers.sqlite