From 6f506a78b36d6e5140f65ce42d2b7567c00ff053 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Wed, 27 Nov 2024 14:06:42 -0500 Subject: [PATCH 1/4] Drop support for old deblender outputs in measureCoaddSources. --- python/lsst/pipe/tasks/multiBand.py | 50 +++-------------------------- 1 file changed, 4 insertions(+), 46 deletions(-) diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index feabea023..cd3b4b2cb 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -34,7 +34,7 @@ PipelineTaskConnections ) import lsst.pipe.base.connectionTypes as cT -from lsst.pex.config import Field, ConfigurableField, ChoiceField +from lsst.pex.config import Field, ConfigurableField from lsst.meas.algorithms import ( DynamicDetectionTask, ExceedsMaxVarianceScaleError, @@ -434,11 +434,6 @@ class MeasureMergedCoaddSourcesConnections( defaultTemplates={ "inputCoaddName": "deep", "outputCoaddName": "deep", - "deblendedCatalog": "deblendedFlux", - }, - deprecatedTemplates={ - # TODO[DM-47797]: remove this deprecated connection template. - "deblendedCatalog": "Support for old deblender outputs will be removed after v29." }, ): inputSchema = cT.InitInput( @@ -521,19 +516,6 @@ class MeasureMergedCoaddSourcesConnections( multiple=True, deferLoad=True, ) - # TODO[DM-47797]: remove this deprecated connection. - inputCatalog = cT.Input( - doc=("Name of the input catalog to use." - "If the single band deblender was used this should be 'deblendedFlux." - "If the multi-band deblender was used this should be 'deblendedModel, " - "or deblendedFlux if the multiband deblender was configured to output " - "deblended flux catalogs. If no deblending was performed this should " - "be 'mergeDet'"), - name="{inputCoaddName}Coadd_{deblendedCatalog}", - storageClass="SourceCatalog", - deprecated="Support for old deblender outputs will be removed after v29.", - dimensions=("tract", "patch", "band", "skymap"), - ) scarletCatalog = cT.Input( doc="Catalogs produced by multiband deblending", name="{inputCoaddName}Coadd_deblendedCatalog", @@ -583,13 +565,7 @@ def __init__(self, *, config=None): del self.sourceTableHandles if not config.propagateFlags.finalized_source_flags: del self.finalizedSourceTableHandles - # TODO[DM-47797]: only the 'if' block contents here should survive. - if config.inputCatalog == "deblendedCatalog": - del self.inputCatalog - if not config.doAddFootprints: - del self.scarletModels - else: - del self.deblendedCatalog + if not config.doAddFootprints: del self.scarletModels # TODO[DM-47797]: delete the conditionals below. @@ -611,18 +587,6 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig, pipelineConnections=MeasureMergedCoaddSourcesConnections): """Configuration parameters for the MeasureMergedCoaddSourcesTask """ - inputCatalog = ChoiceField( - dtype=str, - default="deblendedCatalog", - allowed={ - "deblendedCatalog": "Output catalog from ScarletDeblendTask", - "deblendedFlux": "Output catalog from SourceDeblendTask", - "mergeDet": "The merged detections before deblending." - }, - doc="The name of the input catalog.", - # TODO[DM-47797]: remove this config option and anything using it. - deprecated="Support for old deblender outputs will be removed after v29.", - ) doAddFootprints = Field(dtype=bool, default=True, doc="Whether or not to add footprints to the input catalog from scarlet models. " @@ -768,8 +732,6 @@ class MeasureMergedCoaddSourcesTask(PipelineTask): def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=None, **kwargs): super().__init__(**kwargs) - self.deblended = self.config.inputCatalog.startswith("deblended") - self.inputCatalog = "Coadd_" + self.config.inputCatalog if initInputs is not None: schema = initInputs['inputSchema'].schema if schema is None: @@ -833,12 +795,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): table = afwTable.SourceTable.make(self.schema, idGenerator.make_table_id_factory()) sources = afwTable.SourceCatalog(table) # Load the correct input catalog - if "scarletCatalog" in inputs: - inputCatalog = inputs.pop("scarletCatalog") - catalogRef = inputRefs.scarletCatalog - else: - inputCatalog = inputs.pop("inputCatalog") - catalogRef = inputRefs.inputCatalog + inputCatalog = inputs.pop("scarletCatalog") + catalogRef = inputRefs.scarletCatalog sources.extend(inputCatalog, self.schemaMapper) del inputCatalog # Add the HeavyFootprints to the deblended sources From f3dd922ed6ecffe604959accf9cc4a9458265447 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Wed, 27 Nov 2024 14:09:43 -0500 Subject: [PATCH 2/4] Drop refcat-object matching in measureCoaddSources. --- python/lsst/pipe/tasks/multiBand.py | 110 +--------------------------- 1 file changed, 2 insertions(+), 108 deletions(-) diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index cd3b4b2cb..efaa39d8e 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -40,7 +40,6 @@ ExceedsMaxVarianceScaleError, InsufficientSourcesError, PsfGenerationError, - ReferenceObjectLoader, ScaleVarianceTask, SetPrimaryFlagsTask, TooManyMaskedPixelsError, @@ -53,7 +52,6 @@ SkyMapIdGeneratorConfig, ) from lsst.meas.extensions.scarlet.io import updateCatalogFootprints -from lsst.meas.astrom import DirectMatchTask, denormalizeMatches from lsst.pipe.tasks.propagateSourceFlags import PropagateSourceFlagsTask import lsst.afw.image as afwImage import lsst.afw.math as afwMath @@ -446,16 +444,6 @@ class MeasureMergedCoaddSourcesConnections( name="{inputCoaddName}Coadd_meas_schema", storageClass="SourceCatalog" ) - # TODO[DM-47797]: remove this deprecated connection. - refCat = cT.PrerequisiteInput( - doc="Reference catalog used to match measured sources against known sources", - name="ref_cat", - storageClass="SimpleCatalog", - dimensions=("skypix",), - deferLoad=True, - multiple=True, - deprecated="Reference matching in measureCoaddSources will be removed after v29.", - ) exposure = cT.Input( doc="Input non-cell-based coadd image", name="{inputCoaddName}Coadd_calexp", @@ -534,23 +522,6 @@ class MeasureMergedCoaddSourcesConnections( dimensions=("tract", "patch", "band", "skymap"), storageClass="SourceCatalog", ) - # TODO[DM-47797]: remove this deprecated connection. - matchResult = cT.Output( - doc="Match catalog produced by configured matcher, optional on doMatchSources", - name="{outputCoaddName}Coadd_measMatch", - dimensions=("tract", "patch", "band", "skymap"), - storageClass="Catalog", - deprecated="Reference matching in measureCoaddSources will be removed after v29.", - ) - # TODO[DM-47797]: remove this deprecated connection. - denormMatches = cT.Output( - doc="Denormalized Match catalog produced by configured matcher, optional on " - "doWriteMatchesDenormalized", - name="{outputCoaddName}Coadd_measMatchFull", - dimensions=("tract", "patch", "band", "skymap"), - storageClass="Catalog", - deprecated="Reference matching in measureCoaddSources will be removed after v29.", - ) def __init__(self, *, config=None): super().__init__(config=config) @@ -568,14 +539,6 @@ def __init__(self, *, config=None): if not config.doAddFootprints: del self.scarletModels - # TODO[DM-47797]: delete the conditionals below. - if not config.doMatchSources: - del self.refCat - del self.matchResult - - if not config.doWriteMatchesDenormalized: - del self.denormMatches - if config.useCellCoadds: del self.exposure else: @@ -608,24 +571,6 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig, doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" ) propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc="Propagate source flags to coadd") - doMatchSources = Field( - dtype=bool, - default=False, - doc="Match sources to reference catalog?", - deprecated="Reference matching in measureCoaddSources will be removed after v29.", - ) - match = ConfigurableField( - target=DirectMatchTask, - doc="Matching to reference catalog", - deprecated="Reference matching in measureCoaddSources will be removed after v29.", - ) - doWriteMatchesDenormalized = Field( - dtype=bool, - default=False, - doc=("Write reference matches in denormalized format? " - "This format uses more disk space, but is more convenient to read."), - deprecated="Reference matching in measureCoaddSources will be removed after v29.", - ) coaddName = Field(dtype=str, default="deep", doc="Name of coadd") psfCache = Field(dtype=int, default=100, doc="Size of psfCache") checkUnitsParseStrict = Field( @@ -659,10 +604,6 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig, ) idGenerator = SkyMapIdGeneratorConfig.make_field() - @property - def refObjLoader(self): - return self.match.refObjLoader - def setDefaults(self): super().setDefaults() self.measurement.plugins.names |= ['base_InputCount', @@ -678,12 +619,6 @@ def setDefaults(self): self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE', 'INEXACT_PSF'] - def validate(self): - super().validate() - - if not self.doMatchSources and self.doWriteMatchesDenormalized: - raise ValueError("Cannot set doWriteMatchesDenormalized if doMatchSources is False.") - class MeasureMergedCoaddSourcesTask(PipelineTask): """Deblend sources from main catalog in each coadd seperately and measure. @@ -701,9 +636,6 @@ class MeasureMergedCoaddSourcesTask(PipelineTask): flag (indicating sources with no children) is set. Visit flags are propagated to the coadd sources. - Optionally, we can match the coadd sources to an external reference - catalog. - After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. The next stage in the multi-band processing procedure will merge these measurements into a suitable catalog @@ -715,10 +647,6 @@ class MeasureMergedCoaddSourcesTask(PipelineTask): The schema of the merged detection catalog used as input to this one. peakSchema : ``lsst.afw.table.Schema`, optional The schema of the PeakRecords in the Footprints in the merged detection catalog. - refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional - An instance of ReferenceObjectLoader that supplies an external reference - catalog. May be None if the loader can be constructed from the butler argument or all steps - requiring a reference catalog are disabled. initInputs : `dict`, optional Dictionary that can contain a key ``inputSchema`` containing the input schema. If present will override the value of ``schema``. @@ -729,8 +657,7 @@ class MeasureMergedCoaddSourcesTask(PipelineTask): _DefaultName = "measureCoaddSources" ConfigClass = MeasureMergedCoaddSourcesConfig - def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=None, - **kwargs): + def __init__(self, schema=None, peakSchema=None, initInputs=None, **kwargs): super().__init__(**kwargs) if initInputs is not None: schema = initInputs['inputSchema'].schema @@ -742,9 +669,6 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N self.algMetadata = PropertyList() self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata) self.makeSubtask("setPrimaryFlags", schema=self.schema) - # TODO[DM-47797]: remove match subtask - if self.config.doMatchSources: - self.makeSubtask("match", refObjLoader=refObjLoader) if self.config.doPropagateFlags: self.makeSubtask("propagateFlags", schema=self.schema) self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict) @@ -758,15 +682,6 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs = butlerQC.get(inputRefs) - # TODO[DM-47797]: remove this block - if self.config.doMatchSources: - refObjLoader = ReferenceObjectLoader([ref.datasetRef.dataId for ref in inputRefs.refCat], - inputs.pop('refCat'), - name=self.config.connections.refCat, - config=self.config.refObjLoader, - log=self.log) - self.match.setRefObjLoader(refObjLoader) - if self.config.useCellCoadds: multiple_cell_coadd = inputs.pop("exposure_cells") stitched_coadd = multiple_cell_coadd.stitch() @@ -904,9 +819,7 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, ------- results : `lsst.pipe.base.Struct` Results of running measurement task. Will contain the catalog in the - sources attribute. Optionally will have results of matching to a - reference catalog in the matchResults attribute, and denormalized - matches in the denormMatches attribute. + sources attribute. """ if self.config.doPropagateFlags: # These mask planes may not be defined on the coadds always. @@ -948,24 +861,5 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, ) results = Struct() - - # TODO[DM-47797]: remove this block - if self.config.doMatchSources: - matchResult = self.match.run(sources, exposure.getInfo().getFilter().bandLabel) - matches = afwTable.packMatches(matchResult.matches) - matches.table.setMetadata(matchResult.matchMeta) - results.matchResult = matches - if self.config.doWriteMatchesDenormalized: - if matchResult.matches: - denormMatches = denormalizeMatches(matchResult.matches, matchResult.matchMeta) - else: - self.log.warning("No matches, so generating dummy denormalized matches file") - denormMatches = afwTable.BaseCatalog(afwTable.Schema()) - denormMatches.setMetadata(PropertyList()) - denormMatches.getMetadata().add("COMMENT", - "This catalog is empty because no matches were found.") - results.denormMatches = denormMatches - results.denormMatches = denormMatches - results.outputSources = sources return results From f35207453616917a425a3a41cd90ccaef927a575 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Fri, 17 Jul 2026 14:30:19 -0400 Subject: [PATCH 3/4] Drop deprecated and unused visitCatalogs connection. --- python/lsst/pipe/tasks/multiBand.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index efaa39d8e..87055ea42 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -468,15 +468,6 @@ class MeasureMergedCoaddSourcesConnections( storageClass="SkyMap", dimensions=("skymap",), ) - # TODO[DM-47424]: remove this deprecated connection. - visitCatalogs = cT.Input( - doc="Deprecated and unused.", - name="src", - dimensions=("instrument", "visit", "detector"), - storageClass="SourceCatalog", - multiple=True, - deprecated="Deprecated and unused. Will be removed after v29.", - ) sourceTableHandles = cT.Input( doc=("Source tables that are derived from the ``CalibrateTask`` sources. " "These tables contain astrometry and photometry flags, and optionally " @@ -525,7 +516,6 @@ class MeasureMergedCoaddSourcesConnections( def __init__(self, *, config=None): super().__init__(config=config) - del self.visitCatalogs if not config.doPropagateFlags: del self.sourceTableHandles del self.finalizedSourceTableHandles From ddbb5bafb2547faa6d7df5a432b361c33d07ba71 Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Sun, 19 Jul 2026 11:24:25 -0400 Subject: [PATCH 4/4] Removed already-deprecated coadd-deblending connections. --- .../pipe/tasks/deblendCoaddSourcesPipeline.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/python/lsst/pipe/tasks/deblendCoaddSourcesPipeline.py b/python/lsst/pipe/tasks/deblendCoaddSourcesPipeline.py index 6c8646fd8..afde0f140 100644 --- a/python/lsst/pipe/tasks/deblendCoaddSourcesPipeline.py +++ b/python/lsst/pipe/tasks/deblendCoaddSourcesPipeline.py @@ -91,24 +91,6 @@ class DeblendCoaddSourcesMultiConnections(PipelineTaskConnections, name="{outputCoaddName}Coadd_deblendedFlux_schema", storageClass="SourceCatalog" ) - # TODO[DM-47405]: remove this deprecated connection. - fluxCatalogs = cT.Output( - doc="Flux weighted catalogs produced by multiband deblending", - name="{outputCoaddName}Coadd_deblendedFlux", - storageClass="SourceCatalog", - dimensions=("tract", "patch", "band", "skymap"), - multiple=True, - deprecated="Deprecated and unused; will be removed after v29." - ) - # TODO[DM-47405]: remove this deprecated connection. - templateCatalogs = cT.Output( - doc="Template catalogs produced by multiband deblending", - name="{outputCoaddName}Coadd_deblendedModel", - storageClass="SourceCatalog", - dimensions=("tract", "patch", "band", "skymap"), - multiple=True, - deprecated="Deprecated and unused; will be removed after v29." - ) deblendedCatalog = cT.Output( doc="Catalogs produced by multiband deblending", name="{outputCoaddName}Coadd_deblendedCatalog", @@ -130,9 +112,6 @@ class DeblendCoaddSourcesMultiConnections(PipelineTaskConnections, def __init__(self, *, config=None): super().__init__(config=config) - del self.fluxCatalogs - del self.templateCatalogs - if config: if config.useCellCoadds: del self.coadds