From 356036bcf40ee5c099a8ae65b788b6db8a4b597f Mon Sep 17 00:00:00 2001 From: Thomas Williams <32936518+thomaswilliamsastro@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:16:07 +0100 Subject: [PATCH 1/2] Support for gridding full singledish mosaics This PR adds support for combining multiple different TP mosaics together at once. It essentially loops over each MOUS, performing the calibration, before splitting out the field name (as defined in the ms_keys) and performing one big tsdimaging call at the end. This seems to be performing better than trying to linearly mosaic at the end. Because the imsize is set directly from the image, the phase centre is also no longer passed along to tsdimaging, to avoid potential mismatches leading to the edges being cut off. This is important for the way we have things set up, where individual tiles have their own coordinates separate from the full mosaic centre. It also takes a bit of time to try and improve on the "overwrite" switch in the pipeline. If False it should now just zoom through and not do any heavy processing. Default remains True, for backwards compatibility. - Add support for gridding multiple mosaics in the SD pipeline - Do a better job with the overwrite switch for the SD pipeline - Force phase centre to centre of fields in image --- CHANGELOG.md | 1 + phangsPipeline/casaSingleDishALMAWrapper.py | 303 ++++++++++++++------ phangsPipeline/handlerSingleDish.py | 51 ++-- 3 files changed, 255 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27037e9f..a1d9dc24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use SD pipeline weights in single-dish linear mosaicking (#397). - Expose ``min_pixperbeam`` for cube compression in ``loop_postprocess`` (#411). - Allow override parameters in chunked imaging handler (#417). +- Added support for gridding multiple singledish mosaics in the singledish pipeline (#414). ### Changed diff --git a/phangsPipeline/casaSingleDishALMAWrapper.py b/phangsPipeline/casaSingleDishALMAWrapper.py index 6cdf779d..4dafb691 100644 --- a/phangsPipeline/casaSingleDishALMAWrapper.py +++ b/phangsPipeline/casaSingleDishALMAWrapper.py @@ -1,3 +1,4 @@ +import gc import glob import logging import os @@ -41,22 +42,87 @@ def extractJyperK(base_path=PATH_CALIBRATION): # Copying the jyperk.csv file to the working directory shutil.copyfile(file_script, 'jyperk.csv') +def calculate_phase_center(vis): + """Calculate the phase center from the mean positions of the fields in the MS. + + Args: + vis (str): Path to the measurement set (MS) file. + + Returns: + str: Phase center + """ + + # Get the phase directions + tb = casaStuff.tbtool() + tb.open(f"{vis}/FIELD") + try: + # Shape: (2, n_poly_terms, n_fields), in radians + phase_dir: np.ndarray = tb.getcol("PHASE_DIR") + finally: + tb.close() + + # Get the reference frames + msmd = casaStuff.msmdtool() + msmd.open(vis) + try: + ref_frames = [msmd.phasecenter(fid)["refer"] for fid in range(msmd.nfields())] + finally: + msmd.close() + + # Get the unique reference frames. If there are multiple, we will use the first one and log a warning. + ref_frames = np.unique(ref_frames) + ref_frames = [str(x) for x in ref_frames] + if len(ref_frames) > 1: + logger.warning("Multiple reference frames found. Using the first one.") + ref_frame = ref_frames[0] + + ra: np.ndarray = phase_dir[0, 0, :] + dec: np.ndarray = phase_dir[1, 0, :] + + # Convert to Cartesian unit vectors, average, then convert back + x = np.mean(np.cos(dec) * np.cos(ra)) + y = np.mean(np.cos(dec) * np.sin(ra)) + z = np.mean(np.sin(dec)) + + mean_ra = float(np.mod(np.arctan2(y, x), 2 * np.pi)) + mean_dec = float(np.arctan2(z, np.hypot(x, y))) + + qa = casaStuff.qatool() + ra_str = qa.formxxx(f"{mean_ra}rad", format="hms", prec=4) + dec_str = qa.formxxx(f"{mean_dec}rad", format="dms", prec=4) + + phase_center = f"{ref_frame} {ra_str} {dec_str}" + + return phase_center + + +def SDImaging( + filename, + source, + name_line, + vel_cube_range, + chan_dv_kms, + restfreq, + phcenter="", + freq_Hz=None, + overwrite=True, + keep_only_trimmed=True, +): + + # Only include the source name in the output if it's not empty + if source != "": + outname = f"ALMA_TP.{source}.{name_line}" + else: + outname = f"ALMA_TP.{name_line}" -def SDImaging(filename, - source, - name_line, - vel_cube_range, - chan_dv_kms, - restfreq, - phcenter='', - freq_Hz=None, - overwrite=True, - keep_only_trimmed=True): - - outname = f'ALMA_TP.{source}.{name_line}' outimage = f'{outname}.image' outweight = f'{outname}.weight' + # If the image already exists and overwrite is False, skip imaging + if os.path.exists(outimage) and not overwrite: + logger.info(f"Image {outimage} already exists and overwrite is False. Skipping.") + return outimage + # Setup imaging: start_vel = min(vel_cube_range) vwidth_kms = abs(vel_cube_range[1]-vel_cube_range[0]) @@ -86,7 +152,7 @@ def SDImaging(filename, imsize = int(round(maxsize/cell)*1.5) - if phcenter == '': + if phcenter == "": coord_phase = read_source_coordinates(filename, source) else: coord_phase = phcenter @@ -111,8 +177,8 @@ def SDImaging(filename, imsize=imsize, cell=str(cell)+'arcsec', overwrite=overwrite, - outfile=outname) - + outfile=outname, + ) # Trim the image: myia = casaStuff.iatool() @@ -159,16 +225,18 @@ def SDImaging(filename, return outimage -def runALMAPipeline(path_galaxy, - ms_dirs: list | None = None, - in_source='all', - baseline_fit_func='poly', - baseline_fit_order=1, - baseline_linewindowmode='replace', - baseline_linewindow=None, - product_dict=None, - overwrite=True): - ''' +def runALMAPipeline( + path_galaxy, + ms_dirs: list | None = None, + in_source: dict | str = "all", + baseline_fit_func="poly", + baseline_fit_order=1, + baseline_linewindowmode="replace", + baseline_linewindow=None, + product_dict=None, + overwrite=True, +): + """ Wraps the ALMA SD pipeline recipes into a single function. This it to pass custom parameters to the pipeline, primarily the baseline correction. @@ -178,6 +246,8 @@ def runALMAPipeline(path_galaxy, Path to the galaxy folder. ms_dirs : list List of individual MSs + in_source : dict or str + Either a dictionary mapping a source name to an MS, or a string to apply to all baseline_fit_func : str Baseline fit function. baseline_fit_order : int @@ -191,9 +261,8 @@ def runALMAPipeline(path_galaxy, product_dict : dict Dictionary of line products information. overwrite : bool - Overwrite previous results. - - ''' + Overwrite previous results. Defaults to True. + """ if ms_dirs is None: ms_dirs = [] @@ -201,18 +270,20 @@ def runALMAPipeline(path_galaxy, logger.error("No input MSs folder provided.") return - from pipeline.cli import (h_init, - hsd_importdata, - hsd_flagdata, - h_tsyscal, - hsd_tsysflag, - hsd_skycal, - hsd_k2jycal, - hsd_applycal, - hsd_atmcor, - hsd_baseline, - hsd_blflag, - h_save) + from pipeline.cli import ( + h_init, + hsd_importdata, + hsd_flagdata, + h_tsyscal, + hsd_tsysflag, + hsd_skycal, + hsd_k2jycal, + hsd_applycal, + hsd_atmcor, + hsd_baseline, + hsd_blflag, + h_save, + ) all_eb_names = [] @@ -247,10 +318,17 @@ def runALMAPipeline(path_galaxy, ms_workdir = os.path.join(ms_dir, 'calibrated') workdir = os.path.join(path_galaxy, ms_workdir) - # If we already have this directory, then clear out to avoid rerunning calibration + # If we already have this directory, either clear out if overwrite is True or skip + do_calibration = True if os.path.exists(workdir): - logger.warning(f"Directory {workdir} already exists. Will delete") - shutil.rmtree(workdir) + + if overwrite: + logger.warning(f"Directory {workdir} already exists. Will delete") + shutil.rmtree(workdir) + + else: + logger.warning(f"Directory {workdir} already exists and overwrite is False. Will skip processing") + do_calibration = False if not os.path.exists(workdir): os.makedirs(workdir) @@ -265,8 +343,9 @@ def runALMAPipeline(path_galaxy, logger.info("No ASDM files found.") raise ValueError("No ASDM files found.") - # Retrieve the k2jy scaling from the auxiliary calibration products. - extractJyperK() + if do_calibration: + # Retrieve the k2jy scaling from the auxiliary calibration products. + extractJyperK() # Extract MS names from the JyperK table jyperk_tab = Table.read("jyperk.csv") @@ -305,31 +384,35 @@ def runALMAPipeline(path_galaxy, logger.info(f"Using these ASDM files: {eb_names}") # Run pipeline recipe. - context = h_init() - try: - hsd_importdata(vis=eb_names) - - hsd_flagdata(pipelinemode="automatic") - h_tsyscal(pipelinemode="automatic") - hsd_tsysflag(pipelinemode="automatic") - hsd_skycal(pipelinemode="automatic") - hsd_k2jycal(dbservice=False) - hsd_applycal(pipelinemode="automatic") - hsd_atmcor(pipelinemode="automatic") - - # NOTE: single baseline fit only. The pipeline repeats after 1 flagging step. - # May need to revisit if flagging is missed. - hsd_baseline(pipelinemode="automatic", - fitfunc=baseline_fit_func, - fitorder=baseline_fit_order, - linewindowmode=baseline_linewindowmode, - linewindow=baseline_linewindow, - ) - hsd_blflag(pipelinemode="automatic") - - - finally: - h_save() + if do_calibration: + context = h_init() + try: + hsd_importdata(vis=eb_names) + + hsd_flagdata(pipelinemode="automatic") + h_tsyscal(pipelinemode="automatic") + hsd_tsysflag(pipelinemode="automatic") + hsd_skycal(pipelinemode="automatic") + hsd_k2jycal(dbservice=False) + hsd_applycal(pipelinemode="automatic") + hsd_atmcor(pipelinemode="automatic") + + # NOTE: single baseline fit only. The pipeline repeats after 1 flagging step. + # May need to revisit if flagging is missed. + hsd_baseline(pipelinemode="automatic", + fitfunc=baseline_fit_func, + fitorder=baseline_fit_order, + linewindowmode=baseline_linewindowmode, + linewindow=baseline_linewindow, + ) + hsd_blflag(pipelinemode="automatic") + + finally: + h_save() + + # Remove the pipeline context, just in case this causes memory use to grow + del context + gc.collect() # Move back to the overall galaxy path os.chdir(path_galaxy) @@ -338,8 +421,55 @@ def runALMAPipeline(path_galaxy, this_vis = [] for filename in all_eb_names: + # Find baselined files that end with _bl filename_pattern = f"{filename}.ms.atmcor.atmtype*_bl" + bl_files = glob.glob(filename_pattern) + for bl_file in bl_files: + + # Split out the source from the EBs + output_file = bl_file.replace("_bl", "_bl.split") + + # If the file exists and we're overwriting, remove + if os.path.exists(output_file) and overwrite: + logger.warning(f"{output_file} already exists and overwrite is True. Will remove") + shutil.rmtree(output_file) + + if not os.path.exists(output_file): + + # Split out the files, taking only the specified field name + if isinstance(in_source, str): + source = get_sourcename(bl_file, source=in_source) + elif isinstance(in_source, dict): + bl_file_split = bl_file.split(os.path.sep) + + # Pull out the particular MS (if it exists) and get a source name + in_source_ms = in_source.get(bl_file_split[-3], "all") + source = get_sourcename( + bl_file, + source=in_source_ms, + ) + else: + raise TypeError("in_source must be either a string or a dictionary.") + + # Get the right data column out of the MS + mytb = casaStuff.tbtool() + mytb.open(bl_file) + colnames = mytb.colnames() + colnames = [x.lower() for x in colnames] + mytb.close() + + datacolumn = "corrected" if "corrected" in colnames else "data" + + casaStuff.split( + vis=bl_file, + outputvis=output_file, + field=source, + datacolumn=datacolumn, + ) + + # Recreate the list of baselined files, now they've been appended with .split + filename_pattern = f"{filename}.ms.atmcor.atmtype*_bl.split" bl_files = glob.glob(filename_pattern) # Add to list if we only have 1 entry @@ -361,20 +491,26 @@ def runALMAPipeline(path_galaxy, logger.error("No valid MSs found for imaging.") raise ValueError("No valid MSs found for imaging.") - # Avoid too many files open issues + # Avoid too many files open issues by concatenating ms_concat_filename = 'ALMA_TP_concat.ms' - if os.path.exists(ms_concat_filename): + + # Remove the concatenated MS if it already exists and overwrite is True + if os.path.exists(ms_concat_filename) and overwrite: + logger.warning(f"{ms_concat_filename} already exists, overwrite is True. Will remove") shutil.rmtree(ms_concat_filename) - casaStuff.concat( - vis=this_vis, - concatvis=ms_concat_filename, - respectname=True,) - logger.info('Msnames: %s, N=%d, concatenated: %s'%(this_vis, len(this_vis), ms_concat_filename)) + + if not os.path.exists(ms_concat_filename): + casaStuff.concat( + vis=this_vis, + concatvis=ms_concat_filename, + respectname=True, + ) + logger.info('Msnames: %s, N=%d, concatenated: %s'%(this_vis, len(this_vis), ms_concat_filename)) this_vis = ms_concat_filename - # read the source name directly from the ms - source = get_sourcename(this_vis, source=in_source) + # Calculate the phase center from the concatenated MS + phase_center = calculate_phase_center(vis=this_vis) for this_product in product_dict: @@ -383,21 +519,20 @@ def runALMAPipeline(path_galaxy, freq_rest_GHz = product_dict[this_product]['freq_rest_MHz'] / 1.e3 vel_cube = product_dict[this_product]['vel_cube'] - phase_center = product_dict[this_product]['phase_center'] - output_file = product_dict[this_product]['output_file'] - # Setup imaging call: + # Setup imaging call. Here, we use empty source, since we've already split + # out the source earlier outimage = SDImaging( filename=this_vis, - source=source, + source="", name_line=name_line, vel_cube_range=vel_cube, chan_dv_kms=chan_dv_kms, restfreq=freq_rest_GHz, phcenter=phase_center, freq_Hz=None, - overwrite=overwrite + overwrite=overwrite, ) # Export to fits diff --git a/phangsPipeline/handlerSingleDish.py b/phangsPipeline/handlerSingleDish.py index a4ce808b..0ac9c592 100644 --- a/phangsPipeline/handlerSingleDish.py +++ b/phangsPipeline/handlerSingleDish.py @@ -100,10 +100,11 @@ def _fname_dict( if sd_file is not None: fname_dict[tag] = sd_file - fname_dict['source'] = [] + # Link the source name to the raw data input + fname_dict['source'] = {} tag = 'sd_raw_data_list' - fname_dict[tag] = [] + fname_dict[tag] = {} if has_sd: for this_target, this_project, this_arraytag, this_obsnum in self._kh.loop_over_input_ms(target=target, config='tp'): sd_file = self._kh.get_file_for_input_ms(target=this_target, @@ -118,12 +119,12 @@ def _fname_dict( obsnum=this_obsnum, ) - fname_dict[tag].append(sd_file) - fname_dict['source'].append(source) + # Keep track of the observation number so everything is unique + fname_dict[tag][this_obsnum] = sd_file + obsnum_sd_file = f"{os.path.basename(sd_file)}_{this_obsnum}" + fname_dict['source'][obsnum_sd_file] = source - # Return - - return(fname_dict) + return fname_dict #endregion @@ -137,9 +138,14 @@ def task_execute_single_dish_pipeline( extra_ext_in='', extra_ext_out='', line_wing_kms=200.0, + overwrite: bool = True ): """ Execute single dish data reduction for one target. + + Args: + overwrite (bool, optional): If True, will overwrite existing output files. + Defaults to True. """ if product == 'all': @@ -169,12 +175,7 @@ def task_execute_single_dish_pipeline( logger.info("&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%") logger.info(" target: "+str(target)) logger.info(" product: "+str(product_list)) - logger.info(" raw data: "+str(input_raw_data)) - # logger.info(" output file: "+str(output_file)) - - # if os.path.isfile(output_file): - # logger.info('Output file already exists: '+str(output_file)+'. Will not re-process it.') - # return + logger.info(" raw data: "+str(list(input_raw_data.values()))) for product in product_list: if product not in self._kh.get_line_products(): @@ -184,6 +185,10 @@ def task_execute_single_dish_pipeline( parameters = self._kh.get_params_for_singledish(singledish_config='tp') product_dict = {} + + # Check if we've already produced all the output files + all_output_files_exist = True + for product in product_list: fname_dict = self._fname_dict( @@ -191,6 +196,9 @@ def task_execute_single_dish_pipeline( product=product, ) output_file = fname_dict['sd_file'] + + if not os.path.exists(output_file): + all_output_files_exist = False product_dict[product] = self._kh._config_dict['line_product'][product] @@ -233,14 +241,20 @@ def task_execute_single_dish_pipeline( if key not in product_dict[product]: product_dict[product][key] = parameters[key] + # If we already have all the output files, then skip if we're not overwriting + if all_output_files_exist and not overwrite: + logger.info(f"All output files already exist for {target}. Will not re-process it.") + return + # copy raw data over. Use a different folder for each MS path_galaxy_root = self._kh.get_singledish_dir_for_target(target=target, changeto=False) + os.sep + 'processing_singledish_'+target + os.sep ms_dirs = [] - for ird in input_raw_data: + for obs_num, ird in input_raw_data.items(): - ms_dir = str(os.path.basename(os.path.normpath(ird))) + # Make sure the output has the observation number in there + ms_dir = f"{str(os.path.basename(os.path.normpath(ird)))}_{obs_num}" ms_dirs.append(ms_dir) path_galaxy = os.path.join(str(os.path.abspath(path_galaxy_root)), ms_dir) + os.sep @@ -310,12 +324,13 @@ def task_execute_single_dish_pipeline( sdalma.runALMAPipeline(path_galaxy=path_galaxy_root, ms_dirs=ms_dirs, - in_source=fname_dict['source'][0], + in_source=fname_dict['source'], baseline_fit_func='poly', baseline_fit_order=parameters['bl_order'] if 'bl_order' in parameters else 1, baseline_linewindowmode='replace', baseline_linewindow=None, product_dict=product_dict, + overwrite=overwrite, ) @@ -367,6 +382,7 @@ def loop_singledish( self, do_all: bool = True, make_directories: bool = True, + overwrite: bool = True, ): """ Loops over the full set of targets, products, and @@ -377,6 +393,8 @@ def loop_singledish( Args: do_all (bool): If True, will actually run the singledish pipeline make_directories (bool): If True, will create missing directories + overwrite (bool): If True, will overwrite existing output files. + Defaults to True. """ if do_all: @@ -414,4 +432,5 @@ def loop_singledish( self.task_execute_single_dish_pipeline( target=this_target, product='all', + overwrite=overwrite, ) From 179e6287fa57043247bce00739108ba50f002caf Mon Sep 17 00:00:00 2001 From: Thomas Williams <32936518+thomaswilliamsastro@users.noreply.github.com> Date: Wed, 11 Nov 2026 08:52:16 +0000 Subject: [PATCH 2/2] Rename calculate_phase_center to calculate_image_center --- phangsPipeline/casaSingleDishALMAWrapper.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/phangsPipeline/casaSingleDishALMAWrapper.py b/phangsPipeline/casaSingleDishALMAWrapper.py index 4dafb691..b2d7b93f 100644 --- a/phangsPipeline/casaSingleDishALMAWrapper.py +++ b/phangsPipeline/casaSingleDishALMAWrapper.py @@ -42,14 +42,14 @@ def extractJyperK(base_path=PATH_CALIBRATION): # Copying the jyperk.csv file to the working directory shutil.copyfile(file_script, 'jyperk.csv') -def calculate_phase_center(vis): - """Calculate the phase center from the mean positions of the fields in the MS. +def calculate_image_center(vis): + """Calculate the image center from the mean positions of the fields in the MS. Args: vis (str): Path to the measurement set (MS) file. Returns: - str: Phase center + str: Image center """ # Get the phase directions @@ -91,9 +91,9 @@ def calculate_phase_center(vis): ra_str = qa.formxxx(f"{mean_ra}rad", format="hms", prec=4) dec_str = qa.formxxx(f"{mean_dec}rad", format="dms", prec=4) - phase_center = f"{ref_frame} {ra_str} {dec_str}" + image_center = f"{ref_frame} {ra_str} {dec_str}" - return phase_center + return image_center def SDImaging( @@ -509,8 +509,8 @@ def runALMAPipeline( this_vis = ms_concat_filename - # Calculate the phase center from the concatenated MS - phase_center = calculate_phase_center(vis=this_vis) + # Calculate the image center from the concatenated MS + image_center = calculate_image_center(vis=this_vis) for this_product in product_dict: @@ -530,7 +530,7 @@ def runALMAPipeline( vel_cube_range=vel_cube, chan_dv_kms=chan_dv_kms, restfreq=freq_rest_GHz, - phcenter=phase_center, + phcenter=image_center, freq_Hz=None, overwrite=overwrite, )