Skip to content

DM-47797: remove already-deprecated connections and config options #1333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TallJimbo wants to merge 4 commits into
base: main
Choose a base branch
from
Open

DM-47797: remove already-deprecated connections and config options #1333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
20e517f
Drop support for old deblender outputs in measureCoaddSources.
TallJimbo Nov 27, 2024
bf46ba8
Drop refcat-object matching in measureCoaddSources.
TallJimbo Nov 27, 2024
674e0b3
Drop deprecated and unused visitCatalogs connection.
TallJimbo Jul 17, 2026
22b35b0
Removed already-deprecated coadd-deblending connections.
TallJimbo Jul 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 0 additions & 21 deletions python/lsst/pipe/tasks/deblendCoaddSourcesPipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,24 +91,6 @@ class DeblendCoaddSourcesMultiConnections(PipelineTaskConnections,
name="{outputCoaddName}Coadd_deblendedFlux_schema",
storageClass="SourceCatalog"
)
# TODO[DM-47405]: remove this deprecated connection.
fluxCatalogs = cT.Output(
doc="Flux weighted catalogs produced by multiband deblending",
name="{outputCoaddName}Coadd_deblendedFlux",
storageClass="SourceCatalog",
dimensions=("tract", "patch", "band", "skymap"),
multiple=True,
deprecated="Deprecated and unused; will be removed after v29."
)
# TODO[DM-47405]: remove this deprecated connection.
templateCatalogs = cT.Output(
doc="Template catalogs produced by multiband deblending",
name="{outputCoaddName}Coadd_deblendedModel",
storageClass="SourceCatalog",
dimensions=("tract", "patch", "band", "skymap"),
multiple=True,
deprecated="Deprecated and unused; will be removed after v29."
)
deblendedCatalog = cT.Output(
doc="Catalogs produced by multiband deblending",
name="{outputCoaddName}Coadd_deblendedCatalog",
Expand All @@ -130,9 +112,6 @@ class DeblendCoaddSourcesMultiConnections(PipelineTaskConnections,

def __init__(self, *, config=None):
super().__init__(config=config)
del self.fluxCatalogs
del self.templateCatalogs

if config:
if config.useCellCoadds:
del self.coadds
Expand Down
170 changes: 6 additions & 164 deletions python/lsst/pipe/tasks/multiBand.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,12 @@
PipelineTaskConnections
)
import lsst.pipe.base.connectionTypes as cT
from lsst.pex.config import Field, ConfigurableField, ChoiceField
from lsst.pex.config import Field, ConfigurableField
from lsst.meas.algorithms import (
DynamicDetectionTask,
ExceedsMaxVarianceScaleError,
InsufficientSourcesError,
PsfGenerationError,
ReferenceObjectLoader,
ScaleVarianceTask,
SetPrimaryFlagsTask,
TooManyMaskedPixelsError,
Expand All @@ -53,7 +52,6 @@
SkyMapIdGeneratorConfig,
)
from lsst.meas.extensions.scarlet.io import updateCatalogFootprints
from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
from lsst.pipe.tasks.propagateSourceFlags import PropagateSourceFlagsTask
import lsst.afw.image as afwImage
import lsst.afw.math as afwMath
Expand Down Expand Up @@ -434,11 +432,6 @@ class MeasureMergedCoaddSourcesConnections(
defaultTemplates={
"inputCoaddName": "deep",
"outputCoaddName": "deep",
"deblendedCatalog": "deblendedFlux",
},
deprecatedTemplates={
# TODO[DM-47797]: remove this deprecated connection template.
"deblendedCatalog": "Support for old deblender outputs will be removed after v29."
},
):
inputSchema = cT.InitInput(
Expand All @@ -451,16 +444,6 @@ class MeasureMergedCoaddSourcesConnections(
name="{inputCoaddName}Coadd_meas_schema",
storageClass="SourceCatalog"
)
# TODO[DM-47797]: remove this deprecated connection.
refCat = cT.PrerequisiteInput(
doc="Reference catalog used to match measured sources against known sources",
name="ref_cat",
storageClass="SimpleCatalog",
dimensions=("skypix",),
deferLoad=True,
multiple=True,
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
exposure = cT.Input(
doc="Input non-cell-based coadd image",
name="{inputCoaddName}Coadd_calexp",
Expand All @@ -485,15 +468,6 @@ class MeasureMergedCoaddSourcesConnections(
storageClass="SkyMap",
dimensions=("skymap",),
)
# TODO[DM-47424]: remove this deprecated connection.
visitCatalogs = cT.Input(
doc="Deprecated and unused.",
name="src",
dimensions=("instrument", "visit", "detector"),
storageClass="SourceCatalog",
multiple=True,
deprecated="Deprecated and unused. Will be removed after v29.",
)
sourceTableHandles = cT.Input(
doc=("Source tables that are derived from the ``CalibrateTask`` sources. "
"These tables contain astrometry and photometry flags, and optionally "
Expand Down Expand Up @@ -521,19 +495,6 @@ class MeasureMergedCoaddSourcesConnections(
multiple=True,
deferLoad=True,
)
# TODO[DM-47797]: remove this deprecated connection.
inputCatalog = cT.Input(
doc=("Name of the input catalog to use."
"If the single band deblender was used this should be 'deblendedFlux."
"If the multi-band deblender was used this should be 'deblendedModel, "
"or deblendedFlux if the multiband deblender was configured to output "
"deblended flux catalogs. If no deblending was performed this should "
"be 'mergeDet'"),
name="{inputCoaddName}Coadd_{deblendedCatalog}",
storageClass="SourceCatalog",
deprecated="Support for old deblender outputs will be removed after v29.",
dimensions=("tract", "patch", "band", "skymap"),
)
scarletCatalog = cT.Input(
doc="Catalogs produced by multiband deblending",
name="{inputCoaddName}Coadd_deblendedCatalog",
Expand All @@ -552,27 +513,9 @@ class MeasureMergedCoaddSourcesConnections(
dimensions=("tract", "patch", "band", "skymap"),
storageClass="SourceCatalog",
)
# TODO[DM-47797]: remove this deprecated connection.
matchResult = cT.Output(
doc="Match catalog produced by configured matcher, optional on doMatchSources",
name="{outputCoaddName}Coadd_measMatch",
dimensions=("tract", "patch", "band", "skymap"),
storageClass="Catalog",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
# TODO[DM-47797]: remove this deprecated connection.
denormMatches = cT.Output(
doc="Denormalized Match catalog produced by configured matcher, optional on "
"doWriteMatchesDenormalized",
name="{outputCoaddName}Coadd_measMatchFull",
dimensions=("tract", "patch", "band", "skymap"),
storageClass="Catalog",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)

def __init__(self, *, config=None):
super().__init__(config=config)
del self.visitCatalogs
if not config.doPropagateFlags:
del self.sourceTableHandles
del self.finalizedSourceTableHandles
Expand All @@ -583,23 +526,9 @@ def __init__(self, *, config=None):
del self.sourceTableHandles
if not config.propagateFlags.finalized_source_flags:
del self.finalizedSourceTableHandles
# TODO[DM-47797]: only the 'if' block contents here should survive.
if config.inputCatalog == "deblendedCatalog":
del self.inputCatalog
if not config.doAddFootprints:
del self.scarletModels
else:
del self.deblendedCatalog
if not config.doAddFootprints:
del self.scarletModels

# TODO[DM-47797]: delete the conditionals below.
if not config.doMatchSources:
del self.refCat
del self.matchResult

if not config.doWriteMatchesDenormalized:
del self.denormMatches

if config.useCellCoadds:
del self.exposure
else:
Expand All @@ -611,18 +540,6 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
pipelineConnections=MeasureMergedCoaddSourcesConnections):
"""Configuration parameters for the MeasureMergedCoaddSourcesTask
"""
inputCatalog = ChoiceField(
dtype=str,
default="deblendedCatalog",
allowed={
"deblendedCatalog": "Output catalog from ScarletDeblendTask",
"deblendedFlux": "Output catalog from SourceDeblendTask",
"mergeDet": "The merged detections before deblending."
},
doc="The name of the input catalog.",
# TODO[DM-47797]: remove this config option and anything using it.
deprecated="Support for old deblender outputs will be removed after v29.",
)
doAddFootprints = Field(dtype=bool,
default=True,
doc="Whether or not to add footprints to the input catalog from scarlet models. "
Expand All @@ -644,24 +561,6 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
)
propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc="Propagate source flags to coadd")
doMatchSources = Field(
dtype=bool,
default=False,
doc="Match sources to reference catalog?",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
match = ConfigurableField(
target=DirectMatchTask,
doc="Matching to reference catalog",
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
doWriteMatchesDenormalized = Field(
dtype=bool,
default=False,
doc=("Write reference matches in denormalized format? "
"This format uses more disk space, but is more convenient to read."),
deprecated="Reference matching in measureCoaddSources will be removed after v29.",
)
coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
psfCache = Field(dtype=int, default=100, doc="Size of psfCache")
checkUnitsParseStrict = Field(
Expand Down Expand Up @@ -695,10 +594,6 @@ class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
)
idGenerator = SkyMapIdGeneratorConfig.make_field()

@property
def refObjLoader(self):
return self.match.refObjLoader

def setDefaults(self):
super().setDefaults()
self.measurement.plugins.names |= ['base_InputCount',
Expand All @@ -714,12 +609,6 @@ def setDefaults(self):
self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
'INEXACT_PSF']

def validate(self):
super().validate()

if not self.doMatchSources and self.doWriteMatchesDenormalized:
raise ValueError("Cannot set doWriteMatchesDenormalized if doMatchSources is False.")


class MeasureMergedCoaddSourcesTask(PipelineTask):
"""Deblend sources from main catalog in each coadd seperately and measure.
Expand All @@ -737,9 +626,6 @@ class MeasureMergedCoaddSourcesTask(PipelineTask):
flag (indicating sources with no children) is set. Visit flags are
propagated to the coadd sources.

Optionally, we can match the coadd sources to an external reference
catalog.

After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we
have a set of per-band catalogs. The next stage in the multi-band
processing procedure will merge these measurements into a suitable catalog
Expand All @@ -751,10 +637,6 @@ class MeasureMergedCoaddSourcesTask(PipelineTask):
The schema of the merged detection catalog used as input to this one.
peakSchema : ``lsst.afw.table.Schema`, optional
The schema of the PeakRecords in the Footprints in the merged detection catalog.
refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
An instance of ReferenceObjectLoader that supplies an external reference
catalog. May be None if the loader can be constructed from the butler argument or all steps
requiring a reference catalog are disabled.
initInputs : `dict`, optional
Dictionary that can contain a key ``inputSchema`` containing the
input schema. If present will override the value of ``schema``.
Expand All @@ -765,11 +647,8 @@ class MeasureMergedCoaddSourcesTask(PipelineTask):
_DefaultName = "measureCoaddSources"
ConfigClass = MeasureMergedCoaddSourcesConfig

def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
**kwargs):
def __init__(self, schema=None, peakSchema=None, initInputs=None, **kwargs):
super().__init__(**kwargs)
self.deblended = self.config.inputCatalog.startswith("deblended")
self.inputCatalog = "Coadd_" + self.config.inputCatalog
if initInputs is not None:
schema = initInputs['inputSchema'].schema
if schema is None:
Expand All @@ -780,9 +659,6 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N
self.algMetadata = PropertyList()
self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
self.makeSubtask("setPrimaryFlags", schema=self.schema)
# TODO[DM-47797]: remove match subtask
if self.config.doMatchSources:
self.makeSubtask("match", refObjLoader=refObjLoader)
if self.config.doPropagateFlags:
self.makeSubtask("propagateFlags", schema=self.schema)
self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
Expand All @@ -796,15 +672,6 @@ def __init__(self, schema=None, peakSchema=None, refObjLoader=None, initInputs=N
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)

# TODO[DM-47797]: remove this block
if self.config.doMatchSources:
refObjLoader = ReferenceObjectLoader([ref.datasetRef.dataId for ref in inputRefs.refCat],
inputs.pop('refCat'),
name=self.config.connections.refCat,
config=self.config.refObjLoader,
log=self.log)
self.match.setRefObjLoader(refObjLoader)

if self.config.useCellCoadds:
multiple_cell_coadd = inputs.pop("exposure_cells")
stitched_coadd = multiple_cell_coadd.stitch()
Expand Down Expand Up @@ -833,12 +700,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
table = afwTable.SourceTable.make(self.schema, idGenerator.make_table_id_factory())
sources = afwTable.SourceCatalog(table)
# Load the correct input catalog
if "scarletCatalog" in inputs:
inputCatalog = inputs.pop("scarletCatalog")
catalogRef = inputRefs.scarletCatalog
else:
inputCatalog = inputs.pop("inputCatalog")
catalogRef = inputRefs.inputCatalog
inputCatalog = inputs.pop("scarletCatalog")
catalogRef = inputRefs.scarletCatalog
sources.extend(inputCatalog, self.schemaMapper)
del inputCatalog
# Add the HeavyFootprints to the deblended sources
Expand Down Expand Up @@ -946,9 +809,7 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None,
-------
results : `lsst.pipe.base.Struct`
Results of running measurement task. Will contain the catalog in the
sources attribute. Optionally will have results of matching to a
reference catalog in the matchResults attribute, and denormalized
matches in the denormMatches attribute.
sources attribute.
"""
if self.config.doPropagateFlags:
# These mask planes may not be defined on the coadds always.
Expand Down Expand Up @@ -990,24 +851,5 @@ def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None,
)

results = Struct()

# TODO[DM-47797]: remove this block
if self.config.doMatchSources:
matchResult = self.match.run(sources, exposure.getInfo().getFilter().bandLabel)
matches = afwTable.packMatches(matchResult.matches)
matches.table.setMetadata(matchResult.matchMeta)
results.matchResult = matches
if self.config.doWriteMatchesDenormalized:
if matchResult.matches:
denormMatches = denormalizeMatches(matchResult.matches, matchResult.matchMeta)
else:
self.log.warning("No matches, so generating dummy denormalized matches file")
denormMatches = afwTable.BaseCatalog(afwTable.Schema())
denormMatches.setMetadata(PropertyList())
denormMatches.getMetadata().add("COMMENT",
"This catalog is empty because no matches were found.")
results.denormMatches = denormMatches
results.denormMatches = denormMatches

results.outputSources = sources
return results
Loading