diff --git a/CHANGELOG.md b/CHANGELOG.md index 27037e9f..a1d9dc24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use SD pipeline weights in single-dish linear mosaicking (#397). - Expose ``min_pixperbeam`` for cube compression in ``loop_postprocess`` (#411). - Allow override parameters in chunked imaging handler (#417). +- Added support for gridding multiple singledish mosaics in the singledish pipeline (#414). ### Changed diff --git a/phangsPipeline/casaSingleDishALMAWrapper.py b/phangsPipeline/casaSingleDishALMAWrapper.py index 6cdf779d..b2d7b93f 100644 --- a/phangsPipeline/casaSingleDishALMAWrapper.py +++ b/phangsPipeline/casaSingleDishALMAWrapper.py @@ -1,3 +1,4 @@ +import gc import glob import logging import os @@ -41,22 +42,87 @@ def extractJyperK(base_path=PATH_CALIBRATION): # Copying the jyperk.csv file to the working directory shutil.copyfile(file_script, 'jyperk.csv') +def calculate_image_center(vis): + """Calculate the image center from the mean positions of the fields in the MS. + + Args: + vis (str): Path to the measurement set (MS) file. + + Returns: + str: Image center + """ + + # Get the phase directions + tb = casaStuff.tbtool() + tb.open(f"{vis}/FIELD") + try: + # Shape: (2, n_poly_terms, n_fields), in radians + phase_dir: np.ndarray = tb.getcol("PHASE_DIR") + finally: + tb.close() + + # Get the reference frames + msmd = casaStuff.msmdtool() + msmd.open(vis) + try: + ref_frames = [msmd.phasecenter(fid)["refer"] for fid in range(msmd.nfields())] + finally: + msmd.close() + + # Get the unique reference frames. If there are multiple, we will use the first one and log a warning. + ref_frames = np.unique(ref_frames) + ref_frames = [str(x) for x in ref_frames] + if len(ref_frames) > 1: + logger.warning("Multiple reference frames found. Using the first one.") + ref_frame = ref_frames[0] + + ra: np.ndarray = phase_dir[0, 0, :] + dec: np.ndarray = phase_dir[1, 0, :] + + # Convert to Cartesian unit vectors, average, then convert back + x = np.mean(np.cos(dec) * np.cos(ra)) + y = np.mean(np.cos(dec) * np.sin(ra)) + z = np.mean(np.sin(dec)) + + mean_ra = float(np.mod(np.arctan2(y, x), 2 * np.pi)) + mean_dec = float(np.arctan2(z, np.hypot(x, y))) + + qa = casaStuff.qatool() + ra_str = qa.formxxx(f"{mean_ra}rad", format="hms", prec=4) + dec_str = qa.formxxx(f"{mean_dec}rad", format="dms", prec=4) + + image_center = f"{ref_frame} {ra_str} {dec_str}" + + return image_center + + +def SDImaging( + filename, + source, + name_line, + vel_cube_range, + chan_dv_kms, + restfreq, + phcenter="", + freq_Hz=None, + overwrite=True, + keep_only_trimmed=True, +): + + # Only include the source name in the output if it's not empty + if source != "": + outname = f"ALMA_TP.{source}.{name_line}" + else: + outname = f"ALMA_TP.{name_line}" -def SDImaging(filename, - source, - name_line, - vel_cube_range, - chan_dv_kms, - restfreq, - phcenter='', - freq_Hz=None, - overwrite=True, - keep_only_trimmed=True): - - outname = f'ALMA_TP.{source}.{name_line}' outimage = f'{outname}.image' outweight = f'{outname}.weight' + # If the image already exists and overwrite is False, skip imaging + if os.path.exists(outimage) and not overwrite: + logger.info(f"Image {outimage} already exists and overwrite is False. Skipping.") + return outimage + # Setup imaging: start_vel = min(vel_cube_range) vwidth_kms = abs(vel_cube_range[1]-vel_cube_range[0]) @@ -86,7 +152,7 @@ def SDImaging(filename, imsize = int(round(maxsize/cell)*1.5) - if phcenter == '': + if phcenter == "": coord_phase = read_source_coordinates(filename, source) else: coord_phase = phcenter @@ -111,8 +177,8 @@ def SDImaging(filename, imsize=imsize, cell=str(cell)+'arcsec', overwrite=overwrite, - outfile=outname) - + outfile=outname, + ) # Trim the image: myia = casaStuff.iatool() @@ -159,16 +225,18 @@ def SDImaging(filename, return outimage -def runALMAPipeline(path_galaxy, - ms_dirs: list | None = None, - in_source='all', - baseline_fit_func='poly', - baseline_fit_order=1, - baseline_linewindowmode='replace', - baseline_linewindow=None, - product_dict=None, - overwrite=True): - ''' +def runALMAPipeline( + path_galaxy, + ms_dirs: list | None = None, + in_source: dict | str = "all", + baseline_fit_func="poly", + baseline_fit_order=1, + baseline_linewindowmode="replace", + baseline_linewindow=None, + product_dict=None, + overwrite=True, +): + """ Wraps the ALMA SD pipeline recipes into a single function. This it to pass custom parameters to the pipeline, primarily the baseline correction. @@ -178,6 +246,8 @@ def runALMAPipeline(path_galaxy, Path to the galaxy folder. ms_dirs : list List of individual MSs + in_source : dict or str + Either a dictionary mapping a source name to an MS, or a string to apply to all baseline_fit_func : str Baseline fit function. baseline_fit_order : int @@ -191,9 +261,8 @@ def runALMAPipeline(path_galaxy, product_dict : dict Dictionary of line products information. overwrite : bool - Overwrite previous results. - - ''' + Overwrite previous results. Defaults to True. + """ if ms_dirs is None: ms_dirs = [] @@ -201,18 +270,20 @@ def runALMAPipeline(path_galaxy, logger.error("No input MSs folder provided.") return - from pipeline.cli import (h_init, - hsd_importdata, - hsd_flagdata, - h_tsyscal, - hsd_tsysflag, - hsd_skycal, - hsd_k2jycal, - hsd_applycal, - hsd_atmcor, - hsd_baseline, - hsd_blflag, - h_save) + from pipeline.cli import ( + h_init, + hsd_importdata, + hsd_flagdata, + h_tsyscal, + hsd_tsysflag, + hsd_skycal, + hsd_k2jycal, + hsd_applycal, + hsd_atmcor, + hsd_baseline, + hsd_blflag, + h_save, + ) all_eb_names = [] @@ -247,10 +318,17 @@ def runALMAPipeline(path_galaxy, ms_workdir = os.path.join(ms_dir, 'calibrated') workdir = os.path.join(path_galaxy, ms_workdir) - # If we already have this directory, then clear out to avoid rerunning calibration + # If we already have this directory, either clear out if overwrite is True or skip + do_calibration = True if os.path.exists(workdir): - logger.warning(f"Directory {workdir} already exists. Will delete") - shutil.rmtree(workdir) + + if overwrite: + logger.warning(f"Directory {workdir} already exists. Will delete") + shutil.rmtree(workdir) + + else: + logger.warning(f"Directory {workdir} already exists and overwrite is False. Will skip processing") + do_calibration = False if not os.path.exists(workdir): os.makedirs(workdir) @@ -265,8 +343,9 @@ def runALMAPipeline(path_galaxy, logger.info("No ASDM files found.") raise ValueError("No ASDM files found.") - # Retrieve the k2jy scaling from the auxiliary calibration products. - extractJyperK() + if do_calibration: + # Retrieve the k2jy scaling from the auxiliary calibration products. + extractJyperK() # Extract MS names from the JyperK table jyperk_tab = Table.read("jyperk.csv") @@ -305,31 +384,35 @@ def runALMAPipeline(path_galaxy, logger.info(f"Using these ASDM files: {eb_names}") # Run pipeline recipe. - context = h_init() - try: - hsd_importdata(vis=eb_names) - - hsd_flagdata(pipelinemode="automatic") - h_tsyscal(pipelinemode="automatic") - hsd_tsysflag(pipelinemode="automatic") - hsd_skycal(pipelinemode="automatic") - hsd_k2jycal(dbservice=False) - hsd_applycal(pipelinemode="automatic") - hsd_atmcor(pipelinemode="automatic") - - # NOTE: single baseline fit only. The pipeline repeats after 1 flagging step. - # May need to revisit if flagging is missed. - hsd_baseline(pipelinemode="automatic", - fitfunc=baseline_fit_func, - fitorder=baseline_fit_order, - linewindowmode=baseline_linewindowmode, - linewindow=baseline_linewindow, - ) - hsd_blflag(pipelinemode="automatic") - - - finally: - h_save() + if do_calibration: + context = h_init() + try: + hsd_importdata(vis=eb_names) + + hsd_flagdata(pipelinemode="automatic") + h_tsyscal(pipelinemode="automatic") + hsd_tsysflag(pipelinemode="automatic") + hsd_skycal(pipelinemode="automatic") + hsd_k2jycal(dbservice=False) + hsd_applycal(pipelinemode="automatic") + hsd_atmcor(pipelinemode="automatic") + + # NOTE: single baseline fit only. The pipeline repeats after 1 flagging step. + # May need to revisit if flagging is missed. + hsd_baseline(pipelinemode="automatic", + fitfunc=baseline_fit_func, + fitorder=baseline_fit_order, + linewindowmode=baseline_linewindowmode, + linewindow=baseline_linewindow, + ) + hsd_blflag(pipelinemode="automatic") + + finally: + h_save() + + # Remove the pipeline context, just in case this causes memory use to grow + del context + gc.collect() # Move back to the overall galaxy path os.chdir(path_galaxy) @@ -338,8 +421,55 @@ def runALMAPipeline(path_galaxy, this_vis = [] for filename in all_eb_names: + # Find baselined files that end with _bl filename_pattern = f"{filename}.ms.atmcor.atmtype*_bl" + bl_files = glob.glob(filename_pattern) + for bl_file in bl_files: + + # Split out the source from the EBs + output_file = bl_file.replace("_bl", "_bl.split") + + # If the file exists and we're overwriting, remove + if os.path.exists(output_file) and overwrite: + logger.warning(f"{output_file} already exists and overwrite is True. Will remove") + shutil.rmtree(output_file) + + if not os.path.exists(output_file): + + # Split out the files, taking only the specified field name + if isinstance(in_source, str): + source = get_sourcename(bl_file, source=in_source) + elif isinstance(in_source, dict): + bl_file_split = bl_file.split(os.path.sep) + + # Pull out the particular MS (if it exists) and get a source name + in_source_ms = in_source.get(bl_file_split[-3], "all") + source = get_sourcename( + bl_file, + source=in_source_ms, + ) + else: + raise TypeError("in_source must be either a string or a dictionary.") + + # Get the right data column out of the MS + mytb = casaStuff.tbtool() + mytb.open(bl_file) + colnames = mytb.colnames() + colnames = [x.lower() for x in colnames] + mytb.close() + + datacolumn = "corrected" if "corrected" in colnames else "data" + + casaStuff.split( + vis=bl_file, + outputvis=output_file, + field=source, + datacolumn=datacolumn, + ) + + # Recreate the list of baselined files, now they've been appended with .split + filename_pattern = f"{filename}.ms.atmcor.atmtype*_bl.split" bl_files = glob.glob(filename_pattern) # Add to list if we only have 1 entry @@ -361,20 +491,26 @@ def runALMAPipeline(path_galaxy, logger.error("No valid MSs found for imaging.") raise ValueError("No valid MSs found for imaging.") - # Avoid too many files open issues + # Avoid too many files open issues by concatenating ms_concat_filename = 'ALMA_TP_concat.ms' - if os.path.exists(ms_concat_filename): + + # Remove the concatenated MS if it already exists and overwrite is True + if os.path.exists(ms_concat_filename) and overwrite: + logger.warning(f"{ms_concat_filename} already exists, overwrite is True. Will remove") shutil.rmtree(ms_concat_filename) - casaStuff.concat( - vis=this_vis, - concatvis=ms_concat_filename, - respectname=True,) - logger.info('Msnames: %s, N=%d, concatenated: %s'%(this_vis, len(this_vis), ms_concat_filename)) + + if not os.path.exists(ms_concat_filename): + casaStuff.concat( + vis=this_vis, + concatvis=ms_concat_filename, + respectname=True, + ) + logger.info('Msnames: %s, N=%d, concatenated: %s'%(this_vis, len(this_vis), ms_concat_filename)) this_vis = ms_concat_filename - # read the source name directly from the ms - source = get_sourcename(this_vis, source=in_source) + # Calculate the image center from the concatenated MS + image_center = calculate_image_center(vis=this_vis) for this_product in product_dict: @@ -383,21 +519,20 @@ def runALMAPipeline(path_galaxy, freq_rest_GHz = product_dict[this_product]['freq_rest_MHz'] / 1.e3 vel_cube = product_dict[this_product]['vel_cube'] - phase_center = product_dict[this_product]['phase_center'] - output_file = product_dict[this_product]['output_file'] - # Setup imaging call: + # Setup imaging call. Here, we use empty source, since we've already split + # out the source earlier outimage = SDImaging( filename=this_vis, - source=source, + source="", name_line=name_line, vel_cube_range=vel_cube, chan_dv_kms=chan_dv_kms, restfreq=freq_rest_GHz, - phcenter=phase_center, + phcenter=image_center, freq_Hz=None, - overwrite=overwrite + overwrite=overwrite, ) # Export to fits diff --git a/phangsPipeline/handlerSingleDish.py b/phangsPipeline/handlerSingleDish.py index a4ce808b..0ac9c592 100644 --- a/phangsPipeline/handlerSingleDish.py +++ b/phangsPipeline/handlerSingleDish.py @@ -100,10 +100,11 @@ def _fname_dict( if sd_file is not None: fname_dict[tag] = sd_file - fname_dict['source'] = [] + # Link the source name to the raw data input + fname_dict['source'] = {} tag = 'sd_raw_data_list' - fname_dict[tag] = [] + fname_dict[tag] = {} if has_sd: for this_target, this_project, this_arraytag, this_obsnum in self._kh.loop_over_input_ms(target=target, config='tp'): sd_file = self._kh.get_file_for_input_ms(target=this_target, @@ -118,12 +119,12 @@ def _fname_dict( obsnum=this_obsnum, ) - fname_dict[tag].append(sd_file) - fname_dict['source'].append(source) + # Keep track of the observation number so everything is unique + fname_dict[tag][this_obsnum] = sd_file + obsnum_sd_file = f"{os.path.basename(sd_file)}_{this_obsnum}" + fname_dict['source'][obsnum_sd_file] = source - # Return - - return(fname_dict) + return fname_dict #endregion @@ -137,9 +138,14 @@ def task_execute_single_dish_pipeline( extra_ext_in='', extra_ext_out='', line_wing_kms=200.0, + overwrite: bool = True ): """ Execute single dish data reduction for one target. + + Args: + overwrite (bool, optional): If True, will overwrite existing output files. + Defaults to True. """ if product == 'all': @@ -169,12 +175,7 @@ def task_execute_single_dish_pipeline( logger.info("&%&%&%&%&%&%&%&%&%&%&%&%&%&%&%") logger.info(" target: "+str(target)) logger.info(" product: "+str(product_list)) - logger.info(" raw data: "+str(input_raw_data)) - # logger.info(" output file: "+str(output_file)) - - # if os.path.isfile(output_file): - # logger.info('Output file already exists: '+str(output_file)+'. Will not re-process it.') - # return + logger.info(" raw data: "+str(list(input_raw_data.values()))) for product in product_list: if product not in self._kh.get_line_products(): @@ -184,6 +185,10 @@ def task_execute_single_dish_pipeline( parameters = self._kh.get_params_for_singledish(singledish_config='tp') product_dict = {} + + # Check if we've already produced all the output files + all_output_files_exist = True + for product in product_list: fname_dict = self._fname_dict( @@ -191,6 +196,9 @@ def task_execute_single_dish_pipeline( product=product, ) output_file = fname_dict['sd_file'] + + if not os.path.exists(output_file): + all_output_files_exist = False product_dict[product] = self._kh._config_dict['line_product'][product] @@ -233,14 +241,20 @@ def task_execute_single_dish_pipeline( if key not in product_dict[product]: product_dict[product][key] = parameters[key] + # If we already have all the output files, then skip if we're not overwriting + if all_output_files_exist and not overwrite: + logger.info(f"All output files already exist for {target}. Will not re-process it.") + return + # copy raw data over. Use a different folder for each MS path_galaxy_root = self._kh.get_singledish_dir_for_target(target=target, changeto=False) + os.sep + 'processing_singledish_'+target + os.sep ms_dirs = [] - for ird in input_raw_data: + for obs_num, ird in input_raw_data.items(): - ms_dir = str(os.path.basename(os.path.normpath(ird))) + # Make sure the output has the observation number in there + ms_dir = f"{str(os.path.basename(os.path.normpath(ird)))}_{obs_num}" ms_dirs.append(ms_dir) path_galaxy = os.path.join(str(os.path.abspath(path_galaxy_root)), ms_dir) + os.sep @@ -310,12 +324,13 @@ def task_execute_single_dish_pipeline( sdalma.runALMAPipeline(path_galaxy=path_galaxy_root, ms_dirs=ms_dirs, - in_source=fname_dict['source'][0], + in_source=fname_dict['source'], baseline_fit_func='poly', baseline_fit_order=parameters['bl_order'] if 'bl_order' in parameters else 1, baseline_linewindowmode='replace', baseline_linewindow=None, product_dict=product_dict, + overwrite=overwrite, ) @@ -367,6 +382,7 @@ def loop_singledish( self, do_all: bool = True, make_directories: bool = True, + overwrite: bool = True, ): """ Loops over the full set of targets, products, and @@ -377,6 +393,8 @@ def loop_singledish( Args: do_all (bool): If True, will actually run the singledish pipeline make_directories (bool): If True, will create missing directories + overwrite (bool): If True, will overwrite existing output files. + Defaults to True. """ if do_all: @@ -414,4 +432,5 @@ def loop_singledish( self.task_execute_single_dish_pipeline( target=this_target, product='all', + overwrite=overwrite, )