diff --git a/doc/lsst.ip.diffim/tasks/lsst.ip.diffim.SimplifiedSubtractTask.rst b/doc/lsst.ip.diffim/tasks/lsst.ip.diffim.SimplifiedSubtractTask.rst index 8ad68e8d3..dd0812546 100644 --- a/doc/lsst.ip.diffim/tasks/lsst.ip.diffim.SimplifiedSubtractTask.rst +++ b/doc/lsst.ip.diffim/tasks/lsst.ip.diffim.SimplifiedSubtractTask.rst @@ -36,7 +36,7 @@ Use a local version of the AP pipeline configuration to run the pipeline by reus rewarpTemplate: class: lsst.ip.diffim.getTemplate.GetTemplateTask config: - connections.bbox: preliminary_visit_image.bbox + connections.detector: preliminary_visit_image.detector connections.wcs: preliminary_visit_image.wcs connections.coaddExposures: template_coadd connections.template: template_detector @@ -77,7 +77,7 @@ Alternatively, the pipeline can compute the PSF matching kernel internally by pe rewarpTemplate: class: lsst.ip.diffim.getTemplate.GetTemplateTask config: - connections.bbox: preliminary_visit_image.bbox + connections.detector: preliminary_visit_image.detector connections.wcs: preliminary_visit_image.wcs connections.coaddExposures: template_coadd connections.template: template_detector diff --git a/python/lsst/ip/diffim/computeSpatiallySampledMetrics.py b/python/lsst/ip/diffim/computeSpatiallySampledMetrics.py index dd8949f6e..3006991e6 100644 --- a/python/lsst/ip/diffim/computeSpatiallySampledMetrics.py +++ b/python/lsst/ip/diffim/computeSpatiallySampledMetrics.py @@ -19,6 +19,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import dataclasses + import numpy as np import scipy.signal @@ -29,7 +31,7 @@ import lsst.pipe.base as pipeBase import lsst.pex.config as pexConfig -from lsst.ip.diffim.utils import getPsfFwhm, angleMean, evaluateMaskFraction, getKernelCenterDisplacement +from lsst.ip.diffim.utils import getPsfFwhm, angleMean, getKernelCenterDisplacement from lsst.meas.algorithms import SkyObjectsTask from lsst.pex.exceptions import InvalidParameterError, RangeError from lsst.utils.timer import timeMethod @@ -81,6 +83,12 @@ class SpatiallySampledMetricsConnections(pipeBase.PipelineTaskConnections, name="{fakesType}{coaddName}Diff_spatiallySampledMetrics", ) + def __init__(self, *, config=None): + super().__init__(config=config) + if config.image_type == "future": + self.psfMatchingKernel = dataclasses.replace(self.psfMatchingKernel, + storageClass="ConvolutionKernel") + class SpatiallySampledMetricsConfig(pipeBase.PipelineTaskConfig, pipelineConnections=SpatiallySampledMetricsConnections): @@ -88,7 +96,8 @@ class SpatiallySampledMetricsConfig(pipeBase.PipelineTaskConfig, """ metricsMaskPlanes = lsst.pex.config.ListField( dtype=str, - doc="List of mask planes to include in metrics", + doc="List of mask planes to include in metrics. " + "A mask plane that is not registered reports 0 pixels set.", default=('BAD', 'CLIPPED', 'CR', 'DETECTED', 'DETECTED_NEGATIVE', 'EDGE', 'INEXACT_PSF', 'INJECTED', 'INJECTED_TEMPLATE', 'INTRP', 'NOT_DEBLENDED', 'NO_DATA', 'REJECTED', 'SAT', 'SAT_TEMPLATE', 'SENSOR_EDGE', 'STREAK', 'SUSPECT', @@ -99,6 +108,16 @@ class SpatiallySampledMetricsConfig(pipeBase.PipelineTaskConfig, target=SkyObjectsTask, doc="Generate QA metric sources", ) + image_type = pexConfig.ChoiceField[str]( + doc="Type of the input PSF matching kernel. The input images are read" + " as lsst.afw.image.ExposureF either way.", + allowed={ + "legacy": "Read lsst.afw.math.Kernel.", + "future": "Read lsst.images.convolution_kernels.ConvolutionKernel.", + }, + optional=False, + default="legacy", + ) def setDefaults(self): self.metricSources.avoidMask = ["NO_DATA", "EDGE"] @@ -215,6 +234,13 @@ def __init__(self, **kwargs): " relative to the science PSF L2 norm. Larger values indicate worse" " PSF matching. Assumes the kernel was solved to convolve the template.") + def runQuantum(self, butlerQC, inputRefs, outputRefs): + inputs = butlerQC.get(inputRefs) + if self.config.image_type == "future": + inputs["psfMatchingKernel"] = inputs["psfMatchingKernel"].to_legacy() + outputs = self.run(**inputs) + butlerQC.put(outputs, outputRefs) + @timeMethod def run(self, science, template, difference, diaSources, psfMatchingKernel): """Calculate difference image metrics on specific locations across the images @@ -247,12 +273,16 @@ def run(self, science, template, difference, diaSources, psfMatchingKernel): self.metricSources.run(mask=science.mask, seed=difference.info.id, catalog=spatiallySampledMetrics) - metricsMaskPlanes = [] + # Mask plane bits are looked up in the global afw dict, not the + # image's own. A plane that is not registered in this process has no + # pixels set in any image. + metricsMaskPlanes = {} for maskPlane in self.config.metricsMaskPlanes: try: - metricsMaskPlanes.append(maskPlane) + metricsMaskPlanes[maskPlane] = afwImage.Mask.getPlaneBitMask(maskPlane) except InvalidParameterError: - self.log.info("Unable to calculate metrics for mask plane %s: not in image"%maskPlane) + self.log.info("Mask plane %s is not registered; its metrics will be zero.", maskPlane) + metricsMaskPlanes[maskPlane] = 0 for src in spatiallySampledMetrics: self._evaluateLocalMetric(src, science, template, difference, diaSources, @@ -275,8 +305,9 @@ def _evaluateLocalMetric(self, src, science, template, difference, diaSources, The science image. difference : `lsst.afw.image.Exposure` Result of subtracting template from the science image. - metricsMaskPlanes : `list` of `str` - Mask planes to calculate metrics from. + metricsMaskPlanes : `dict` [`str`, `int`] + Mask planes to calculate metrics from, mapped to their bitmask. + The bitmask is 0 for a plane that is not registered. psfMatchingKernel : `~lsst.afw.math.LinearCombinationKernel` The PSF matching kernel of the subtraction to evaluate. """ @@ -324,9 +355,10 @@ def _evaluateLocalMetric(self, src, science, template, difference, diaSources, src.set('diffim_value', diffimVal) src.set('diffim_variance', diffimVar) src.set('diffim_chi2PerPix', self._diffimChi2PerPix(difference[bbox])) - for maskPlane in metricsMaskPlanes: + differenceMask = difference.mask[bbox].array + for maskPlane, bitMask in metricsMaskPlanes.items(): src.set("%s_mask_fraction"%maskPlane.lower(), - evaluateMaskFraction(difference.mask[bbox], maskPlane) + np.count_nonzero(differenceMask & bitMask)/differenceMask.size ) krnlSum, dx, dy, direction, length = getKernelCenterDisplacement( diff --git a/python/lsst/ip/diffim/detectAndMeasure.py b/python/lsst/ip/diffim/detectAndMeasure.py index 78ea68652..719b80a92 100644 --- a/python/lsst/ip/diffim/detectAndMeasure.py +++ b/python/lsst/ip/diffim/detectAndMeasure.py @@ -19,18 +19,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import dataclasses import numpy as np import requests import os + import lsst.afw.detection as afwDetection import lsst.afw.image as afwImage import lsst.afw.math as afwMath import lsst.afw.table as afwTable import lsst.daf.base as dafBase import lsst.geom +from lsst.images import Mask, get_legacy_difference_image_mask_planes +from lsst.images.fields import field_from_legacy_background from lsst.ip.diffim.utils import (evaluateMaskFraction, computeDifferenceImageMetrics, - populate_sattle_visit_cache) + populate_sattle_visit_cache, + get_difference_image_provenance) from lsst.meas.algorithms import SkyObjectsTask, SourceDetectionTask, SetPrimaryFlagsTask, MaskStreaksTask from lsst.meas.algorithms import FindGlintTrailsTask, FindCosmicRaysConfig, findCosmicRays from lsst.meas.base import ForcedMeasurementTask, ApplyApCorrTask, DetectorVisitIdGeneratorConfig @@ -179,10 +184,16 @@ def __init__(self, *, config): self.inputs.remove("kernelSources") if not (self.config.writeStreakInfo and self.config.doMaskStreaks): self.outputs.remove("maskedStreaks") - if not (self.config.doSubtractBackground and self.config.doWriteBackground): + # In future mode the background is recorded on the output images. + if (self.config.image_type == "future" + or not (self.config.doSubtractBackground and self.config.doWriteBackground)): self.outputs.remove("differenceBackground") if not (self.config.writeGlintInfo): self.outputs.remove("glintTrailInfo") + if self.config.image_type == "future": + self.difference = dataclasses.replace(self.difference, storageClass="DifferenceImage") + self.subtractedMeasuredExposure = dataclasses.replace(self.subtractedMeasuredExposure, + storageClass="DifferenceImage") class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, @@ -212,8 +223,11 @@ class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, ) doWriteBackground = pexConfig.Field( dtype=bool, - doc="Persist the fitted background model?", + doc="Persist the fitted background model? Ignored when" + " image_type='future', which records it on the output images.", default=False, + deprecated="Use image_type='future', which records the background on the output images." + " Will be removed after v31.", ) doCalculateResidualMetics = pexConfig.Field( dtype=bool, @@ -399,6 +413,16 @@ class DetectAndMeasureConfig(pipeBase.PipelineTaskConfig, "closest in time to the exposure." ) idGenerator = DetectorVisitIdGeneratorConfig.make_field() + image_type = pexConfig.ChoiceField[str]( + "Image type of the input and measured difference images (and, for" + " DetectAndMeasureScoreTask, the input and measured score images).", + allowed={ + "legacy": "Read and write lsst.afw.image.ExposureF.", + "future": "Read and write lsst.images.DifferenceImage.", + }, + optional=False, + default="legacy", + ) def setDefaults(self): # Background subtraction @@ -592,10 +616,28 @@ def runQuantum(self, butlerQC: pipeBase.QuantumContext, inputs = butlerQC.get(inputRefs) idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) idFactory = idGenerator.make_table_id_factory() - # Specify the fields that `annotate` needs below, to ensure they - # exist, even as None. + + # Input image that each output is measured from, keyed by output name. + inputImages = {} + if self.config.image_type == "future": + dataId = butlerQC.quantum.dataId + for output, connection in (("subtractedMeasuredExposure", "difference"), + ("scoreMeasuredExposure", "scoreExposure")): + if connection not in inputs: + continue + image = inputs[connection] + # The output is written back into this image, so it must + # already carry the kernel and the template records. + get_difference_image_provenance(image, "kernel", connection, dataId) + get_difference_image_provenance(image, "templates", connection, dataId) + inputImages[output] = image + inputs[connection] = image.to_legacy() + + # Specify the fields that `annotate` and `convert_outputs_to_future` + # need below, to ensure they exist, even as None. measurementResults = pipeBase.Struct( subtractedMeasuredExposure=None, + scoreMeasuredExposure=None, diaSources=None, maskedStreaks=None, differenceBackground=None, @@ -603,18 +645,75 @@ def runQuantum(self, butlerQC: pipeBase.QuantumContext, try: self.run(**inputs, idFactory=idFactory, measurementResults=measurementResults) except pipeBase.AlgorithmError as e: + # Annotate before converting error = pipeBase.AnnotatedPartialOutputsError.annotate( e, self, measurementResults.subtractedMeasuredExposure, + measurementResults.scoreMeasuredExposure, measurementResults.diaSources, measurementResults.maskedStreaks, log=self.log ) + if self.config.image_type == "future": + self.convert_outputs_to_future(measurementResults, inputImages) butlerQC.put(measurementResults, outputRefs) raise error from e + if self.config.image_type == "future": + self.convert_outputs_to_future(measurementResults, inputImages) butlerQC.put(measurementResults, outputRefs) + def convert_outputs_to_future(self, results, inputImages): + """Write the output images in a result struct back into the + `lsst.images` images they were measured from. + + The image, variance, and mask planes of each input image are replaced + by those of the corresponding legacy output, and the background + subtracted by this task is attached. The kernel, template records, + and observation metadata of the input are kept. Outputs that are + absent or `None` are skipped. + + Parameters + ---------- + results : `lsst.pipe.base.Struct` + Output struct to read and modify in place. + inputImages : `dict` [`str`, `lsst.images.DifferenceImage`] + Input image each output was measured from, keyed by the name of + the output. Each is modified in place. + + Notes + ----- + This uses the default legacy mask plane map of + `lsst.images.get_legacy_difference_image_mask_planes` and adds any + planes from `~lsst.images.get_legacy_optional_mask_planes` that are set + for any pixels in the image. + """ + background = getattr(results, "differenceBackground", None) + for name, image in inputImages.items(): + exposure = getattr(results, name, None) + if exposure is None: + continue + # These are no-ops when the legacy output is a view of the input, + # as returned by `~lsst.images.DifferenceImage.to_legacy`. + image.image.array[...] = exposure.image.array + image.variance.array[...] = exposure.variance.array + image.mask = Mask.from_legacy(exposure.mask, get_legacy_difference_image_mask_planes(), + sky_projection=image.sky_projection) + if background is not None and len(background) > 0: + # The same background is subtracted from the difference image + # and the score image, so it applies to both. + image.backgrounds.add( + "subtracted", + field_from_legacy_background(background, bounds=image.bbox, unit=image.unit), + description="Background subtracted from this image before detection.", + is_subtracted=True, + ) + setattr(results, name, image) + if background is not None and len(background) == 0: + # An empty BackgroundList cannot be converted to a field, and means + # that no background was subtracted. + self.log.debug("No background model to attach to the output images.") + @timeMethod def run(self, science, matchedTemplate, difference, kernelSources=None, idFactory=None, measurementResults=None): @@ -782,16 +881,19 @@ def _fitAndSubtractBackground(self, differenceExposure, scoreExposure=None): background=background, clearMask=True, ) - # Use the temporary detection mask for the final background subtraction. - # The detection mask planes will be cleared before the final detection - # step, so it is OK if they get set for differenceExposure. - detectedBit = differenceExposure.mask.getPlaneBitMask(["DETECTED"]) + # Use the temporary detection mask for the final background subtraction, + # then restore the original mask. The score image variant never + # clears the detection planes of the difference image. + mask = differenceExposure.mask + originalMask = mask.array.copy() + detectedBit = mask.getPlaneBitMask(["DETECTED"]) detectedPix = detectionExposure.mask.array & detectedBit > 0 - detectedNegativeBit = differenceExposure.mask.getPlaneBitMask(["DETECTED_NEGATIVE"]) + detectedNegativeBit = mask.getPlaneBitMask(["DETECTED_NEGATIVE"]) detectedNegativePix = detectionExposure.mask.array & detectedNegativeBit > 0 - differenceExposure.mask.array[detectedPix] |= detectedBit - differenceExposure.mask.array[detectedNegativePix] |= detectedNegativeBit + mask.array[detectedPix] |= detectedBit + mask.array[detectedNegativePix] |= detectedNegativeBit background = self.subtractFinalBackground.run(differenceExposure).background + mask.array[...] = originalMask if scoreExposure is not None: # The preconvolution kernel is normalized to 1, so the same # background level applies to the difference and score images. @@ -1483,6 +1585,13 @@ class DetectAndMeasureScoreConnections(DetectAndMeasureConnections): name="{fakesType}{coaddName}Diff_scoreExp", ) + def __init__(self, *, config): + super().__init__(config=config) + if self.config.image_type == "future": + self.scoreExposure = dataclasses.replace(self.scoreExposure, storageClass="DifferenceImage") + self.scoreMeasuredExposure = dataclasses.replace(self.scoreMeasuredExposure, + storageClass="DifferenceImage") + class DetectAndMeasureScoreConfig(DetectAndMeasureConfig, pipelineConnections=DetectAndMeasureScoreConnections): diff --git a/python/lsst/ip/diffim/getTemplate.py b/python/lsst/ip/diffim/getTemplate.py index 74e7a7588..92d18dd77 100644 --- a/python/lsst/ip/diffim/getTemplate.py +++ b/python/lsst/ip/diffim/getTemplate.py @@ -21,6 +21,7 @@ import collections import dataclasses +import astropy.units from astropy.stats import gaussian_sigma_to_fwhm import numpy as np @@ -36,9 +37,15 @@ import lsst.pex.exceptions import lsst.pipe.base as pipeBase +from lsst.images import ( + DifferenceImage, + DifferenceImageTemplateInfo, + get_legacy_optional_mask_planes, + get_legacy_template_mask_planes, +) from lsst.skymap import BaseSkyMap from lsst.ip.diffim.dcrModel import DcrModel -from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm +from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm, record_from_obs_info from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig, SubtractBackgroundTask, ScaleVarianceTask from lsst.utils.timer import timeMethod @@ -63,6 +70,22 @@ class GetTemplateConnections( name="{fakesType}calexp.bbox", storageClass="Box2I", dimensions=("instrument", "visit", "detector"), + deprecated="Replaced by the `detector` connection. Will be removed after v31.", + ) + detector = pipeBase.connectionTypes.Input( + doc="Detector of the exposure that we will construct the template for." + " Its bounding box sets the geometry of the output template.", + name="{fakesType}calexp.detector", + storageClass="Detector", + dimensions=("instrument", "visit", "detector"), + ) + obs_info = pipeBase.connectionTypes.Input( + doc="Observation metadata of the exposure the template is built for." + " Only read with output_image_type='future', which needs this to set" + " the obs_info component of a lsst.images.VisitImage.", + name="{fakesType}calexp.obs_info", + storageClass="ObservationInfo", + dimensions=("instrument", "visit", "detector"), ) wcs = pipeBase.connectionTypes.Input( doc="WCS of the exposure that we will construct the template for.", @@ -96,11 +119,17 @@ class GetTemplateConnections( def __init__(self, *, config=None): super().__init__(config=config) + # Kept only so that older config files still load. + del self.bbox if config.requireCoaddAtGraphBuild: self.coaddExposures = dataclasses.replace( self.coaddExposures, deferGraphConstraint=False, ) + if config.output_image_type == "future": + self.template = dataclasses.replace(self.template, storageClass="DifferenceImage") + else: + del self.obs_info class GetTemplateConfig( @@ -172,6 +201,23 @@ class GetTemplateConfig( doc="If True, include the coadd dataset existence in the" " initial butler query during QuantumGraph generation.", ) + output_image_type = pexConfig.ChoiceField[str]( + "Which image type to use for the output template.", + allowed={ + "legacy": "Write as a lsst.afw.image.ExposureF.", + "future": "Write as a lsst.images.DifferenceImage.", + }, + optional=False, + default="legacy", + ) + raiseOnUndefinedMaskMap = pexConfig.Field( + dtype=bool, + default=False, + doc="Raise if the template has pixels set in a mask plane that" + " `lsst.images` does not map? If False, log a warning and drop that" + " plane from the output instead." + " Only used with output_image_type='future'.", + ) def setDefaults(self): # Use a smaller cache: per SeparableKernel.computeCache, this should @@ -225,8 +271,10 @@ def __init__(self, *args, **kwargs): def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs = butlerQC.get(inputRefs) - bbox = inputs.pop("bbox") + detector = inputs.pop("detector") + bbox = detector.getBBox() wcs = inputs.pop("wcs") + obsInfo = inputs.pop("obs_info", None) coaddExposures = inputs.pop("coaddExposures") skymap = inputs.pop("skyMap") @@ -243,8 +291,105 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): physical_filter=physical_filter, visit=outputRefs.template.dataId["visit"], ) + if self.config.output_image_type == "future": + dataId = butlerQC.quantum.dataId + exposureRecord = record_from_obs_info(obsInfo, dataId["instrument"], dataId["visit"], + dataId.universe) + outputs.template = self.convert_outputs_to_future(outputs.template, + inputRefs.coaddExposures, + detector=detector, + exposureRecord=exposureRecord) butlerQC.put(outputs, outputRefs) + def convert_outputs_to_future(self, template, coadd_refs, detector, exposureRecord): + """Convert a template image to DifferenceImage format. + + This replaces ``template`` with an `lsst.images.DifferenceImage` + instance and records the coadds contributed to it. The Gaussian PSF + set by `run` is converted to an + `lsst.images.psfs.GaussianPointSpreadFunction`. + + Parameters + ---------- + template : `lsst.afw.image.Exposure` + Template to convert. Its detector is replaced with ``detector`` + and its mask is modified in place. + coadd_refs : `list` [`lsst.daf.butler.DatasetRef`] + References to the coadds that may have gone into the template. + May be a superset of the coadds that were used; supplies the + dataset id and RUN collection recorded for each coadd that was. + detector : `lsst.afw.cameraGeom.Detector` + Detector of the science image that the template was built for. + exposureRecord : `lsst.daf.butler.DimensionRecord` + The ``exposure`` record of the observation the template was built + for, which supplies the observation metadata recorded on the + output. + + Returns + ------- + convertedTemplate : `lsst.images.DifferenceImage` + The converted image. + + Raises + ------ + RuntimeError + Raised if the template has pixels set in a mask plane that + `lsst.images.get_legacy_template_mask_planes` does not map and + ``config.raiseOnUndefinedMaskMap`` is True, or if ``coadd_refs`` + spans more than one skymap. + """ + # The template PSF is a Gaussian, so rebuild the CoaddPsf from the + # template inputs to record which coadds contributed. + coaddPsf = self._makePsf(template, template.getInfo().getCoaddInputs().ccds, template.getWcs()) + template.setDetector(detector) + skymaps = {ref.dataId["skymap"] for ref in coadd_refs} + if len(skymaps) != 1: + raise RuntimeError("Expected the template inputs to come from one skymap;" + f" got {sorted(skymaps)}.") + butler_info = {(ref.dataId["tract"], ref.dataId["patch"]): (ref.id, ref.run) + for ref in coadd_refs} + planeMap = get_legacy_template_mask_planes() + if not self.config.raiseOnUndefinedMaskMap: + self._clearUnmappedMaskPlanes(template.mask, planeMap) + convertedTemplate = DifferenceImage.from_legacy( + template, + exposure_record=exposureRecord, + unit=astropy.units.nJy, + plane_map=planeMap, + ) + del template + convertedTemplate.templates = DifferenceImageTemplateInfo.from_legacy_psf( + convertedTemplate.sky_projection.pixel_frame, + coaddPsf, + skymaps.pop(), + butler_info, + log=self.log, + ) + return convertedTemplate + + def _clearUnmappedMaskPlanes(self, mask, planeMap): + """Clear the pixels of every mask plane that conversion would reject. + + Parameters + ---------- + mask : `lsst.afw.image.Mask` + Template mask to modify in place. + planeMap : `dict` [`str`, `lsst.images.MaskPlane`] + Plane map the template will be converted with. The optional + planes of `lsst.images.get_legacy_optional_mask_planes` are kept + as well, since conversion adds them when they have pixels set. + """ + keep = planeMap.keys() | get_legacy_optional_mask_planes().keys() + for name in mask.getMaskPlaneDict(): + if name in keep: + continue + bitmask = mask.getPlaneBitMask(name) + nSet = np.count_nonzero(mask.array & bitmask) + if nSet: + self.log.warning("Dropping mask plane %s from the template: %d pixels are set, but" + " the template plane map does not include it.", name, nSet) + mask.array &= ~bitmask + def getExposures(self, coaddExposureHandles, bbox, skymap, wcs): """Return a data structure containing the coadds that overlap the specified bbox projected onto the sky, and a corresponding data @@ -350,9 +495,9 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi `lsst.afw.image.Exposure`]] Coadds to be mosaicked, indexed on tract id. bbox : `lsst.geom.Box2I` - Template Bounding box of the detector geometry onto which to - resample the ``coaddExposureHandles``. Modified in-place to include the - template border. + Input Bounding box of the detector geometry onto which to + resample the ``coaddExposureHandles``. The output template bbox + will be grown by ``templateBorderSize``. wcs : `lsst.afw.geom.SkyWcs` Template WCS onto which to resample the ``coaddExposureHandles``. dataIds : `dict` [`int`, `list` [`lsst.daf.butler.DataCoordinate`]] @@ -382,6 +527,7 @@ def run(self, *, coaddExposureHandles, bbox, wcs, dataIds, physical_filter, visi """ band, photoCalib = self._checkInputs(dataIds, coaddExposureHandles) + bbox = geom.Box2I(bbox) bbox.grow(self.config.templateBorderSize) warped = {} @@ -864,6 +1010,12 @@ def __init__(self, *, config=None): class GetDcrTemplateConfig( GetTemplateConfig, pipelineConnections=GetDcrTemplateConnections ): + """Config for GetDcrTemplateTask. + + Only ``output_image_type="legacy"`` is supported: the task always writes + an `lsst.afw.image.ExposureF`. + """ + numSubfilters = pexConfig.Field( doc="Number of subfilters in the DcrCoadd.", dtype=int, @@ -895,11 +1047,15 @@ class GetDcrTemplateTask(GetTemplateTask): def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs = butlerQC.get(inputRefs) - bbox = inputs.pop("bbox") + detector = inputs.pop("detector") + bbox = detector.getBBox() wcs = inputs.pop("wcs") dcrCoaddExposureHandles = inputs.pop("dcrCoadds") skymap = inputs.pop("skyMap") visitInfo = inputs.pop("visitInfo") + # This task never converts its output, so the observation metadata + # that the conversion needs is dropped here. + inputs.pop("obs_info", None) # This should not happen with a properly configured execution context. assert not inputs, "runQuantum got more inputs than expected" diff --git a/python/lsst/ip/diffim/subtractImages.py b/python/lsst/ip/diffim/subtractImages.py index 4c39a924b..3ca166dea 100644 --- a/python/lsst/ip/diffim/subtractImages.py +++ b/python/lsst/ip/diffim/subtractImages.py @@ -19,6 +19,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import dataclasses + from astropy import units as u from astropy.stats import gaussian_fwhm_to_sigma import numpy as np @@ -27,9 +29,12 @@ import lsst.afw.image import lsst.afw.math import lsst.geom +from lsst.images import DifferenceImage +from lsst.images.convolution_kernels import ImageBasisConvolutionKernel from lsst.ip.diffim.utils import (evaluateMeanPsfFwhm, getPsfFwhm, computeDifferenceImageMetrics, computePSFNoiseEquivalentArea, - checkMask, setSourceFootprints) + checkMask, record_from_obs_info, setSourceFootprints, + get_difference_image_provenance) from lsst.meas.algorithms import ScaleVarianceTask, ScienceSourceSelectorTask import lsst.pex.config import lsst.pipe.base @@ -69,13 +74,17 @@ class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections, dimensions=_dimensions, defaultTemplates=_defaultTemplates): template = connectionTypes.Input( - doc="Input warped template to subtract.", + doc="Input warped template to subtract." + " With image_type='future' this is read as a lsst.images.DifferenceImage, so that the" + " record of the coadds that went into it can be passed on to the outputs.", dimensions=("instrument", "visit", "detector"), storageClass="ExposureF", name="{fakesType}{coaddName}Diff_templateExp" ) science = connectionTypes.Input( - doc="Input science exposure to subtract from.", + doc="Input science exposure to subtract from." + " With image_type='future' this is read as a lsst.images.VisitImage, so that its" + " photometric scaling can be passed on to the outputs.", dimensions=("instrument", "visit", "detector"), storageClass="ExposureF", name="{fakesType}calexp" @@ -100,6 +109,9 @@ def __init__(self, *, config=None): super().__init__(config=config) if not config.doApplyExternalCalibrations: del self.visitSummary + if config.image_type == "future": + self.template = dataclasses.replace(self.template, storageClass="DifferenceImage") + self.science = dataclasses.replace(self.science, storageClass="VisitImage") class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections, @@ -130,6 +142,15 @@ class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections, name="{fakesType}{coaddName}Diff_psfMatchSources" ) + def __init__(self, *, config=None): + super().__init__(config=config) + if config.image_type == "future": + self.difference = dataclasses.replace(self.difference, storageClass="DifferenceImage") + self.matchedTemplate = dataclasses.replace(self.matchedTemplate, + storageClass="DifferenceImage") + self.psfMatchingKernel = dataclasses.replace(self.psfMatchingKernel, + storageClass="ConvolutionKernel") + class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections, dimensions=_dimensions, @@ -153,6 +174,13 @@ class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections, name="{fakesType}{coaddName}Diff_psfScoreMatchSources" ) + def __init__(self, *, config=None): + super().__init__(config=config) + if config.image_type == "future": + self.scoreExposure = dataclasses.replace(self.scoreExposure, storageClass="DifferenceImage") + self.psfMatchingKernel = dataclasses.replace(self.psfMatchingKernel, + storageClass="ConvolutionKernel") + class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections): pass @@ -172,6 +200,9 @@ def __init__(self, *, config=None): if config.useExistingKernel: del self.psfMatchingKernel del self.kernelSources + if config.image_type == "future": + self.inputPsfMatchingKernel = dataclasses.replace(self.inputPsfMatchingKernel, + storageClass="ConvolutionKernel") else: del self.inputPsfMatchingKernel @@ -219,6 +250,7 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): "It is generally better to instead subtract the background in detectAndMeasure.", dtype=bool, default=False, + deprecated="Subtract the background in detectAndMeasure instead. Will be removed after v31.", ) doApplyExternalCalibrations = lsst.pex.config.Field( doc=( @@ -302,6 +334,19 @@ class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config): doc="Re-run source detection for kernel candidates if an error is" " encountered while calculating the matching kernel." ) + image_type = lsst.pex.config.ChoiceField[str]( + doc="Image type of the input template, the image outputs of this" + " task (difference and matchedTemplate, or scoreExposure), and the PSF" + " matching kernel.", + allowed={ + "legacy": "Read and write lsst.afw.image.ExposureF and" + " lsst.afw.math.Kernel.", + "future": "Read and write lsst.images.DifferenceImage and" + " lsst.images.convolution_kernels.ConvolutionKernel.", + }, + optional=False, + default="legacy", + ) def setDefaults(self): self.makeKernel.kernel.name = "AL" @@ -334,6 +379,15 @@ def setDefaults(self): self.fallbackSourceSelector.signalToNoise.minimum = signalToNoiseMinimum self.fallbackSourceSelector.signalToNoise.maximum = signalToNoiseMaximum + def validate(self): + super().validate() + if self.doSubtractBackground and self.image_type == "future": + raise lsst.pex.config.FieldValidationError( + AlardLuptonSubtractBaseConfig.doSubtractBackground, self, + "The background subtracted when solving the kernel is not recorded on" + " image_type='future' outputs; subtract the background in detectAndMeasure instead." + ) + class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig, pipelineConnections=AlardLuptonSubtractConnections): @@ -358,6 +412,11 @@ class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask): before kernel-matching. Subclasses that preconvolve override this to `True`.""" + futureImageOutputs = ("difference", "matchedTemplate") + """Names of the images in the results struct that are converted in future + mode (`tuple` [`str`]). Subclasses with different image outputs override + this.""" + def __init__(self, **kwargs): super().__init__(**kwargs) self.makeSubtask("decorrelate") @@ -416,16 +475,107 @@ def _applyExternalCalibrations(self, exposure, visitSummary): def runQuantum(self, butlerQC, inputRefs, outputRefs): inputs = butlerQC.get(inputRefs) + templateInfo = None + exposureRecord = None + photometricScaling = None + if self.config.image_type == "future": + template = inputs["template"] + dataId = butlerQC.quantum.dataId + # Read the `TemplateInfo` and the observation metadata from the + # template while it is still in the `DifferenceImage` format. + templateInfo = get_difference_image_provenance(template, "templates", "template", dataId) + exposureRecord = record_from_obs_info(template.obs_info, dataId["instrument"], + dataId["visit"], dataId.universe) + inputs["template"] = template.to_legacy() + photometricScaling = inputs["science"].photometric_scaling + inputs["science"] = inputs["science"].to_legacy() + if "inputPsfMatchingKernel" in inputs: + inputs["inputPsfMatchingKernel"] = inputs["inputPsfMatchingKernel"].to_legacy() try: results = self.run(**inputs) except lsst.pipe.base.AlgorithmError as e: error = lsst.pipe.base.AnnotatedPartialOutputsError.annotate(e, self, log=self.log) - # No partial outputs for butler to put + # No partial outputs for butler to put, so nothing to convert. raise error from e + if self.config.image_type == "future": + self.convert_outputs_to_future(results, exposureRecord, templateInfo=templateInfo, + photometricScaling=photometricScaling) + butlerQC.put(results, outputRefs) + def convert_outputs_to_future(self, results, exposureRecord, templateInfo=None, + photometricScaling=None): + """Convert the image outputs to `lsst.images` types. + + Each image named in `futureImageOutputs` that is present on + ``results`` is replaced by an `lsst.images.DifferenceImage`, with the + PSF matching kernel attached to it. ``psfMatchingKernel`` is replaced + by the same kernel as an + `lsst.images.convolution_kernels.ImageBasisConvolutionKernel`. + + Parameters + ---------- + results : `lsst.pipe.base.Struct` + Output struct to read and modify in place. + exposureRecord : `lsst.daf.butler.DimensionRecord` + The ``exposure`` record of the observation, which supplies the + observation metadata recorded on each converted output. + templateInfo : `list` [`lsst.images.DifferenceImageTemplateInfo`], \ + optional + Record of the coadds that went into the template, taken from the + template this task subtracted. + photometricScaling : `lsst.images.fields.BaseField`, optional + Photometric scaling of the science image. + + Raises + ------ + TypeError + Raised if the PSF matching kernel cannot be stored on an + `lsst.images.DifferenceImage`. + RuntimeError + Raised if an output has pixels set in a mask plane that + `lsst.images.get_legacy_difference_image_mask_planes` does not + map. + + Notes + ----- + This uses the default legacy mask plane map of + `lsst.images.get_legacy_difference_image_mask_planes` and adds any + planes from `~lsst.images.get_legacy_optional_mask_planes` that are set + for any pixels in the image. + """ + # The science image and every output built from it share the science + # pixel grid, so its detector applies to all of them. + science = results.matchedScience + try: + kernel = ImageBasisConvolutionKernel.from_legacy(results.psfMatchingKernel) + except (TypeError, ValueError, AttributeError) as e: + raise TypeError("The supplied PSF matching kernel cannot be attached to an" + f" lsst.images.DifferenceImage: {e}") from e + for name in self.futureImageOutputs: + exposure = getattr(results, name, None) + if exposure is None: + continue + if exposure.getDetector() is None: + # Fall back on the science detector for images without a + # detector of their own (e.g. matchedTemplate) + exposure.setDetector(science.getDetector()) + image = DifferenceImage.from_legacy( + exposure, + unit=u.nJy, + exposure_record=exposureRecord, + ) + if kernel is not None: + image.kernel = kernel + if templateInfo: + image.templates = templateInfo + if photometricScaling is not None: + image.photometric_scaling = photometricScaling + setattr(results, name, image) + results.psfMatchingKernel = kernel + @timeMethod def run(self, template, science, sources, visitSummary=None): """PSF match, subtract, and decorrelate two images. @@ -1137,10 +1287,15 @@ def updateMasks(self, template, science): # the `renameTemplateMask` config will be copied to new planes with # "_TEMPLATE" appended to their names, and the original mask plane will # be cleared. - clearMaskPlanes = [mp for mp in template.mask.getMaskPlaneDict().keys() - if mp not in self.config.preserveTemplateMask] renameMaskPlanes = [mp for mp in self.config.renameTemplateMask if mp in template.mask.getMaskPlaneDict().keys()] + # afw's mask planes are global, so if the "*_TEMPLATE" mask planes were + # defined anywhere previously they would now also be cleared unless we + # exclude them. + renamedMaskPlanes = {maskPlane + "_TEMPLATE" for maskPlane in renameMaskPlanes} + clearMaskPlanes = [mp for mp in template.mask.getMaskPlaneDict().keys() + if mp not in self.config.preserveTemplateMask + and mp not in renamedMaskPlanes] # propagate the mask plane related to Fake source injection # NOTE: the fake source injection sets FAKE plane, but it should be INJECTED @@ -1215,6 +1370,9 @@ class AlardLuptonPreconvolveSubtractTask(AlardLuptonSubtractTask): ConfigClass = AlardLuptonPreconvolveSubtractConfig _DefaultName = "alardLuptonPreconvolveSubtract" usePreconvolution = True + # This task just writes the score image instead of the difference image and + # matchedTemplate + futureImageOutputs = ("scoreExposure",) def run(self, template, science, sources, visitSummary=None): """Preconvolve the science image with its own PSF, diff --git a/python/lsst/ip/diffim/utils.py b/python/lsst/ip/diffim/utils.py index 6e3823d82..9b0940749 100644 --- a/python/lsst/ip/diffim/utils.py +++ b/python/lsst/ip/diffim/utils.py @@ -24,6 +24,7 @@ __all__ = ["evaluateMeanPsfFwhm", "getPsfFwhm", "getKernelCenterDisplacement", "computeDifferenceImageMetrics", "checkMask", "setSourceFootprints", + "record_from_obs_info", "get_difference_image_provenance", ] import itertools @@ -31,11 +32,13 @@ import requests from astropy.stats import gaussian_sigma_to_fwhm +import astropy.units as u import numpy as np import lsst.afw.detection as afwDetection import lsst.afw.image as afwImage import lsst.afw.math as afwMath +from lsst.daf.butler import Timespan import lsst.geom as geom from lsst.pex.exceptions import InvalidParameterError, RangeError import lsst.pipe.base @@ -45,6 +48,106 @@ _LOG = getLogger(__name__) +def record_from_obs_info(obs_info, instrument, visit, universe): + """Build an ``exposure`` dimension record from an observation info. + + Parameters + ---------- + obs_info : `astro_metadata_translator.ObservationInfo` + Observation info of an image that the image to be converted is + derived from. + instrument : `str` + Name of the instrument, usually from the quantum data ID. + visit : `int` + Id of the visit, usually from the quantum data ID. It is also used + as the id of the exposure. + universe : `lsst.daf.butler.DimensionUniverse` + Dimension universe that defines the record's schema, usually + ``butlerQC.quantum.dataId.universe``. + + Returns + ------- + record : `lsst.daf.butler.DimensionRecord` + An ``exposure`` record holding the fields that `lsst.images` reads + when it converts a legacy image. Fields it does not read, such as + ``dark_time`` and ``target_name``, are `None`. + + Raises + ------ + ValueError + Raised if ``obs_info`` is `None`, which means the image it came from + was not in an `lsst.images` format. + + Notes + ----- + The instrument and the visit are passed separately because an + observation info does not carry them in the form the conversion needs: + its ``instrument`` is read from the legacy FITS header rather than from + the record, and its ``exposure_id`` is the id that the visit info held. + """ + if obs_info is None: + raise ValueError("Cannot rebuild an exposure record without an observation info.") + azimuth = zenithAngle = None + if obs_info.altaz_begin is not None: + azimuth = obs_info.altaz_begin.az.deg + zenithAngle = 90.0 - obs_info.altaz_begin.alt.deg + exposureTime = obs_info.exposure_time_requested + return universe["exposure"].RecordClass( + instrument=instrument, + id=visit, + obs_id=obs_info.observation_id, + group=obs_info.exposure_group, + day_obs=obs_info.observing_day, + physical_filter=obs_info.physical_filter, + exposure_time=None if exposureTime is None else exposureTime.to_value(u.s), + seq_num=obs_info.observation_counter, + seq_start=obs_info.group_counter_start, + seq_end=obs_info.group_counter_end, + can_see_sky=obs_info.can_see_sky, + azimuth=azimuth, + zenith_angle=zenithAngle, + timespan=Timespan(begin=obs_info.datetime_begin, end=obs_info.datetime_end), + ) + + +def get_difference_image_provenance(image, attribute, connection, dataId): + """Get the PSF matching kernel or the template records of an input + difference image. + + Parameters + ---------- + image : `lsst.images.DifferenceImage` + Input image read by a task in future mode. + attribute : {"kernel", "templates"} + Name of the attribute to get. + connection : `str` + Name of the input connection the image was read from. + dataId : `lsst.daf.butler.DataCoordinate` + Data ID of the quantum, for the error message. + + Returns + ------- + value : `lsst.images.convolution_kernels.ConvolutionKernel` or \ + `list` [`lsst.images.DifferenceImageTemplateInfo`] + Value of the attribute. + + Raises + ------ + lsst.pipe.base.InvalidQuantumError + Raised if the image does not have the attribute, which happens when + the input was written as a legacy Exposure and converted on read. + """ + try: + return getattr(image, attribute) + except AttributeError as e: + what = "PSF matching kernel" if attribute == "kernel" else "template coadd records" + raise lsst.pipe.base.InvalidQuantumError( + f"Input {connection!r} for {dataId} has no {what}, which is required for image_type='future'." + " This is likely due to reading a legacy Exposure, which lacks this component. To fix," + " reprocess this image starting from `getTemplate`." + ) from e + + def getKernelCenterDisplacement(kernel, x, y, image=None): """Calculate the PSF matching kernel peak offset from the nominal position. diff --git a/tests/test_computeSpatiallySampledMetrics.py b/tests/test_computeSpatiallySampledMetrics.py new file mode 100644 index 000000000..8bb99c1e3 --- /dev/null +++ b/tests/test_computeSpatiallySampledMetrics.py @@ -0,0 +1,153 @@ +# This file is part of ip_diffim. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import types +import unittest + +import numpy as np + +import lsst.afw.image +import lsst.afw.table +import lsst.ip.diffim +import lsst.utils.tests +from lsst.images.convolution_kernels import ImageBasisConvolutionKernel + +from utils import makeTestImage + + +class ComputeSpatiallySampledMetricsTest(lsst.utils.tests.TestCase): + """Test the metrics that are sampled across the difference image.""" + + def setUp(self): + noiseLevel = 1. + science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6) + template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, + templateBorderSize=20, doApplyCalibration=True) + config = lsst.ip.diffim.AlardLuptonSubtractTask.ConfigClass() + config.sourceSelector.signalToNoise.fluxField = "truth_instFlux" + config.sourceSelector.signalToNoise.errField = "truth_instFluxErr" + subtraction = lsst.ip.diffim.AlardLuptonSubtractTask(config=config).run(template, science, sources) + self.science = subtraction.matchedScience + self.template = template + self.difference = subtraction.difference + self.psfMatchingKernel = subtraction.psfMatchingKernel + self.diaSources = self._makeDiaSources() + + @staticmethod + def _makeDiaSources(): + """Return an empty catalog with the fields the metrics read from the + detected sources. + """ + schema = lsst.afw.table.SourceTable.makeMinimalSchema() + schema.addField("x", "D", "Centroid x.") + schema.addField("y", "D", "Centroid y.") + schema.addField("isDipole", "Flag", "Is this source a dipole?") + schema.addField("dipoleAngle", "D", "Dipole orientation.") + schema.addField("dipoleLength", "D", "Dipole separation.") + return lsst.afw.table.SourceCatalog(schema) + + def _run(self, **kwargs): + """Run the metrics task on the images made in `setUp`.""" + config = lsst.ip.diffim.SpatiallySampledMetricsTask.ConfigClass() + config.update(**kwargs) + task = lsst.ip.diffim.SpatiallySampledMetricsTask(config=config) + return task.run(self.science, self.template, self.difference, self.diaSources, + self.psfMatchingKernel).spatiallySampledMetrics + + def testMaskPlaneNotInImage(self): + """A configured mask plane that is not registered has a fraction of + zero, rather than raising. + + An image converted from `lsst.images.DifferenceImage` only defines the + planes that have pixels set, so INJECTED is missing from an ordinary + AP run. + """ + present = "DETECTED" + missing = "NOT_A_MASK_PLANE" + self.assertIn(present, self.difference.mask.getMaskPlaneDict()) + self.assertNotIn(missing, self.difference.mask.getMaskPlaneDict()) + + metrics = self._run(metricsMaskPlanes=[present, missing]) + + self.assertGreater(len(metrics), 0) + self.assertTrue(np.all(metrics[f"{missing.lower()}_mask_fraction"] == 0)) + self.assertTrue(np.all(np.isfinite(metrics[f"{present.lower()}_mask_fraction"]))) + + def testConnections(self): + """``image_type`` sets the storage class of the kernel input only.""" + for imageType, kernelStorageClass in (("legacy", "MatchingKernel"), + ("future", "ConvolutionKernel")): + with self.subTest(imageType=imageType): + config = lsst.ip.diffim.SpatiallySampledMetricsTask.ConfigClass() + config.image_type = imageType + connections = config.ConnectionsClass(config=config) + self.assertEqual(connections.psfMatchingKernel.storageClass, kernelStorageClass) + for name in ("science", "template", "difference"): + self.assertEqual(getattr(connections, name).storageClass, "ExposureF") + + def testRunQuantumFuture(self): + """``runQuantum`` converts a future kernel back to the legacy type, + giving the same metrics as a legacy run. + """ + inputs = {"science": self.science, "template": self.template, "difference": self.difference, + "diaSources": self.diaSources} + metrics = {} + for imageType, kernel in ( + ("legacy", self.psfMatchingKernel), + ("future", ImageBasisConvolutionKernel.from_legacy(self.psfMatchingKernel)), + ): + config = lsst.ip.diffim.SpatiallySampledMetricsTask.ConfigClass() + config.image_type = imageType + task = lsst.ip.diffim.SpatiallySampledMetricsTask(config=config) + butlerQC = _RecordingQuantumContext() + task.runQuantum(butlerQC, types.SimpleNamespace(**inputs, psfMatchingKernel=kernel), None) + metrics[imageType] = butlerQC.put_values.spatiallySampledMetrics + + self.assertGreater(len(metrics["legacy"]), 0) + for name in ("psfMatchingKernel_sum", "psfMatchingKernel_dx", "psfMatchingKernel_dy", + "psfMatchingKernel_residualNorm"): + np.testing.assert_allclose(metrics["future"][name], metrics["legacy"][name], + rtol=1e-6, err_msg=name) + + +class _RecordingQuantumContext: + """Minimal `~lsst.pipe.base.QuantumContext` that records what was put.""" + def __init__(self): + self.put_values = None + + def get(self, refs): + return dict(refs.__dict__) + + def put(self, values, refs): + self.put_values = values + + +def setup_module(module): + lsst.utils.tests.init() + + +class MemoryTestCase(lsst.utils.tests.MemoryTestCase): + pass + + +if __name__ == "__main__": + lsst.utils.tests.init() + unittest.main() diff --git a/tests/test_detectAndMeasure.py b/tests/test_detectAndMeasure.py index 58315d35d..a2f26858f 100644 --- a/tests/test_detectAndMeasure.py +++ b/tests/test_detectAndMeasure.py @@ -22,19 +22,31 @@ import numpy as np import os import unittest +import uuid from unittest import mock import requests +import astropy.units + import lsst.afw.detection as afwDetection import lsst.afw.geom as afwGeom import lsst.afw.image as afwImage import lsst.afw.table as afwTable import lsst.afw.math as afwMath import lsst.geom +from lsst.images import (Box, DifferenceImage, DifferenceImageTemplateInfo, Polygon, + get_legacy_difference_image_mask_planes, get_legacy_optional_mask_planes) +from lsst.images.convolution_kernels import ImageBasisConvolutionKernel +from lsst.images.fields import ChebyshevField +import lsst.images.psfs +import lsst.images.serialization +from lsst.images.tests import compare_masked_image_to_legacy from lsst.ip.diffim import detectAndMeasure, subtractImages from lsst.afw.table import IdFactory -from lsst.afw.cameraGeom.testUtils import DetectorWrapper +from lsst.afw.cameraGeom.testUtils import CameraWrapper, DetectorWrapper import lsst.meas.algorithms as measAlg +import lsst.pipe.base +import lsst.pipe.base.testUtils from lsst.pipe.base import InvalidQuantumError, UpstreamFailureNoWorkFound, AlgorithmError import lsst.utils.tests import lsst.meas.base.tests @@ -43,7 +55,7 @@ import lsst.geom as geom import lsst.pex.config as pexConfig -from utils import makeTestImage, checkMask +from utils import makeTestExposureRecord, makeTestImage, makeTestVisitInfo, checkMask class DetectAndMeasureTestBase: @@ -1307,6 +1319,36 @@ def _run_and_check_detections(positive=True): _run_and_check_detections(positive=True) _run_and_check_detections(positive=False) + def test_background_detection_mask_not_kept(self): + """The detection masks of the difference image should match those of + the score image, without the detections used to fit the background. + """ + kwargs = {"seed": 1, "psfSize": 2.4, "fluxLevel": 500, "kernelSize": 31, + "templateBorderSize": 15} + science, sources = makeTestImage(noiseLevel=1., noiseSeed=6, **kwargs) + matchedTemplate, _ = makeTestImage(noiseLevel=0.25, noiseSeed=7, **kwargs) + difference = science.clone() + # Small-scale structure that the first-pass background fit removes + # but the final fit does not, so the two passes detect different + # pixels. + difference.image.array += 0.3*np.sin(np.arange(difference.getWidth())/20.) + subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() + score = subtractTask._convolveExposure(difference, science.psf.getKernel(), + subtractTask.convolutionControl) + detectionTask = self._setup_detection(doSubtractBackground=True, doSkySources=False, + badSubtractionRatioThreshold=1.) + + output = detectionTask.run(science, matchedTemplate, difference, score, sources) + + differenceMask = output.subtractedMeasuredExposure.mask + scoreMask = output.scoreMeasuredExposure.mask + for plane in ("DETECTED", "DETECTED_NEGATIVE"): + with self.subTest(plane=plane): + differencePix = differenceMask.array & differenceMask.getPlaneBitMask(plane) > 0 + scorePix = scoreMask.array & scoreMask.getPlaneBitMask(plane) > 0 + self.assertGreater(scorePix.sum(), 0) + np.testing.assert_array_equal(differencePix, scorePix) + def test_mask_cosmic_rays(self): """Cosmic rays detected on the difference image should propagate to the mask of the returned (measured) exposure. This version creates @@ -1557,6 +1599,478 @@ def makeVisitInfo(id=1): ) +class DetectAndMeasureOutputFormatTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase): + """Tests of the conversion of the output images to `lsst.images` types. + """ + detectionTask = detectAndMeasure.DetectAndMeasureTask + + @classmethod + def setUpClass(cls): + super().setUpClass() + # `DifferenceImage.from_legacy` needs a detector with per-amplifier + # raw geometry and a FIELD_ANGLE transform; the simpler + # `DetectorWrapper` detector used elsewhere in this file does not + # have either, and fails to convert. + cls.detector = list(CameraWrapper().camera)[0] + + def setUp(self): + self.dataId = lsst.daf.butler.DataCoordinate.standardize( + instrument="I", + visit=42, + detector=12, + universe=lsst.daf.butler.DimensionUniverse(), + ) + # The exposure record that `runQuantum` rebuilds from the + # observation metadata of the input difference image. + self.exposureRecord = makeTestExposureRecord(self.dataId.universe, + instrument=self.dataId["instrument"], + visit=self.dataId["visit"]) + + @staticmethod + def _makeTemplateInfo(patch=42): + """Return one record of a coadd that went into a template. + + Parameters + ---------- + patch : `int`, optional + Identifier of the patch the record describes. + + Returns + ------- + info : `lsst.images.DifferenceImageTemplateInfo` + The record. + """ + return DifferenceImageTemplateInfo( + skymap="skymap", tract=9813, patch=patch, dataset_id=uuid.uuid4(), + dataset_run="a/template/run", bounds=Polygon.from_box(Box.factory[0:50, 0:50]), + psf_shape_xx=4.0, psf_shape_yy=4.0, psf_shape_xy=0.0, psf_shape_flag=False) + + @staticmethod + def _makeKernel(scale=1.0): + """Return a PSF matching kernel of the one form that can be attached + to a difference image. + + Parameters + ---------- + scale : `float`, optional + Coefficient of the first basis kernel, to tell two kernels apart. + + Returns + ------- + kernel : `lsst.images.convolution_kernels.ImageBasisConvolutionKernel` + The kernel. + """ + basis = [] + for sigma in (1.0, 2.0): + image = afwImage.ImageD(5, 5) + y, x = np.mgrid[-2:3, -2:3] + image.array[:] = np.exp(-(x**2 + y**2)/(2*sigma**2)) + basis.append(afwMath.FixedKernel(image)) + bbox = lsst.geom.Box2D(lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Extent2I(100, 100))) + legacy = afwMath.LinearCombinationKernel(basis, afwMath.Chebyshev1Function2D(1, bbox)) + legacy.setSpatialParameters([[scale, 0.0, 0.0], [0.5, 0.0, 0.0]]) + return ImageBasisConvolutionKernel.from_legacy(legacy) + + @staticmethod + def _makePhotometricScaling(exposure): + """Return a constant photometric scaling for an input image. + + Parameters + ---------- + exposure : `lsst.afw.image.ExposureF` + Image from `_make_images`. + + Returns + ------- + photometricScaling : `lsst.images.fields.ChebyshevField` + The scaling. + """ + return ChebyshevField(Box.from_legacy(exposure.getBBox()), np.array([[0.25]]), + unit=astropy.units.nJy/astropy.units.electron) + + def _makeFutureInput(self, exposure, templateInfo=None, kernel=None, photometricScaling=None): + """Convert a legacy image to a `lsst.images.DifferenceImage`. + + Parameters + ---------- + exposure : `lsst.afw.image.ExposureF` + Image from `_make_images`. + templateInfo : `list` [`lsst.images.DifferenceImageTemplateInfo`], + optional + Record of the coadds that went into the template. + kernel : `lsst.images.convolution_kernels.ImageBasisConvolutionKernel`, + optional + PSF matching kernel. + photometricScaling : `lsst.images.fields.BaseField`, optional + Photometric scaling of the science image. + + Returns + ------- + image : `lsst.images.DifferenceImage` + The converted image. + """ + image = DifferenceImage.from_legacy( + exposure, unit=astropy.units.nJy, exposure_record=self.exposureRecord) + if templateInfo is not None: + image.templates = templateInfo + if kernel is not None: + image.kernel = kernel + if photometricScaling is not None: + image.photometric_scaling = photometricScaling + return image + + def _make_images(self, withScore=False): + """Make a science/template/difference image set for detection. + + The template is not actually subtracted from the science image, so + that there are plenty of detections; configure the detection task + with ``badSubtractionRatioThreshold=1.`` to match. + + Parameters + ---------- + withScore : `bool`, optional + Also make and return a score (maximum likelihood) image? + + Returns + ------- + images : `tuple` + The science exposure, the source catalog, the matched template, + the difference image, and (if ``withScore``) the score image. + """ + kernelSize = 31 + kwargs = {"seed": 1, "psfSize": 2.4, "fluxLevel": 500, "x0": 12345, "y0": 67890, + "kernelSize": kernelSize, "templateBorderSize": kernelSize//2} + science, sources = makeTestImage(noiseLevel=1., noiseSeed=6, **kwargs) + matchedTemplate, _ = makeTestImage(noiseLevel=0.25, noiseSeed=7, **kwargs) + # Add components to the basic test image + science.setDetector(self.detector) + matchedTemplate.setDetector(self.detector) + for exposure in (science, matchedTemplate): + exposure.info.setVisitInfo(makeTestVisitInfo(self.dataId["visit"])) + difference = science.clone() + if not withScore: + return science, sources, matchedTemplate, difference + subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() + score = subtractTask._convolveExposure(difference, science.psf.getKernel(), + subtractTask.convolutionControl) + score.setDetector(self.detector) + return science, sources, matchedTemplate, difference, score + + def _setup_task(self, **kwargs): + """Configure a detection task to go with `_make_images`. + + Parameters + ---------- + **kwargs + Any additional config parameters to set. + + Returns + ------- + task : `lsst.pipe.base.PipelineTask` + The configured task to use for detection and measurement. + """ + # The template is not actually subtracted from the science image in + # `_make_images`, so the subtraction residual metric threshold has to + # be set high. + kwargs.setdefault("badSubtractionRatioThreshold", 1.) + return self._setup_detection(doSkySources=False, **kwargs) + + def _check_converted(self, image, legacy): + """Check the properties common to every converted output image. + + Parameters + ---------- + image : `lsst.images.DifferenceImage` + The converted image. + legacy : `lsst.afw.image.ExposureF` + The legacy output exposure it was converted from. + """ + self.assertIsInstance(image, DifferenceImage) + self.assertEqual(image.unit, astropy.units.nJy) + self.assertIsNotNone(image.detector) + # `_makeFutureInput` was not given a photometric scaling. + self.assertIsNone(image.photometric_scaling) + compare_masked_image_to_legacy(image, legacy.maskedImage, + plane_map=get_legacy_difference_image_mask_planes()) + self.assertTrue(np.any(image.mask.get("DETECTED"))) + + def test_connections(self): + """Check that the output storage classes follow ``image_type``, + and that the background is a separate output only in legacy mode. + """ + for configClass, connectionsClass in ( + (detectAndMeasure.DetectAndMeasureConfig, detectAndMeasure.DetectAndMeasureConnections), + (detectAndMeasure.DetectAndMeasureScoreConfig, + detectAndMeasure.DetectAndMeasureScoreConnections), + ): + for imageType, storageClass in (("legacy", "ExposureF"), ("future", "DifferenceImage")): + with self.subTest(configClass=configClass.__name__, imageType=imageType): + config = configClass() + config.image_type = imageType + config.doSubtractBackground = True + with self.assertWarns(FutureWarning): + config.doWriteBackground = True + connections = connectionsClass(config=config) + self.assertEqual(connections.subtractedMeasuredExposure.storageClass, storageClass) + if connectionsClass is detectAndMeasure.DetectAndMeasureScoreConnections: + self.assertEqual(connections.scoreMeasuredExposure.storageClass, storageClass) + if imageType == "legacy": + self.assertIn("differenceBackground", connections.outputs) + self.assertEqual(connections.differenceBackground.storageClass, "Background") + else: + # The background is recorded on the output images. + self.assertNotIn("differenceBackground", connections.outputs) + + def test_run_returns_legacy_types(self): + """``run`` always returns legacy types, even in future mode.""" + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(image_type="future") + output = task.run(science, matchedTemplate, difference, sources) + self.assertIsInstance(output.subtractedMeasuredExposure, afwImage.ExposureF) + + def test_legacy_mode_does_not_convert(self): + """Legacy mode leaves the outputs unconverted.""" + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(doSubtractBackground=True, image_type="legacy") + output = task.run(science, matchedTemplate, difference, sources) + self.assertIsInstance(output.subtractedMeasuredExposure, afwImage.ExposureF) + self.assertIsInstance(output.differenceBackground, afwMath.BackgroundList) + lsst.pipe.base.testUtils.assertValidOutput(task, output) + + def test_convert_outputs_to_future(self): + """Convert the outputs of ``DetectAndMeasureTask``.""" + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(doSubtractBackground=True, image_type="future") + inputImage = self._makeFutureInput(difference) + output = task.run(science, matchedTemplate, inputImage.to_legacy(), sources) + legacy = output.subtractedMeasuredExposure + background = output.differenceBackground + task.convert_outputs_to_future(output, {"subtractedMeasuredExposure": inputImage}) + + self.assertIs(output.subtractedMeasuredExposure, inputImage) + self._check_converted(inputImage, legacy) + self.assertEqual(list(inputImage.backgrounds), ["subtracted"]) + # The separate background dataset is still written in future mode. + self.assertIs(output.differenceBackground, background) + self.assertIsInstance(output.differenceBackground, afwMath.BackgroundList) + + def test_convert_outputs_to_future_copied_pixels(self): + """The output pixels are copied into the input image when the legacy + image that was measured does not share them. + """ + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(doSubtractBackground=True, image_type="future") + inputImage = self._makeFutureInput(difference) + original = inputImage.image.array.copy() + output = task.run(science, matchedTemplate, inputImage.to_legacy(copy=True), sources) + legacy = output.subtractedMeasuredExposure + # Background subtraction changed the pixels of the copy only. + self.assertFalse(np.array_equal(legacy.image.array, original)) + np.testing.assert_array_equal(inputImage.image.array, original) + + task.convert_outputs_to_future(output, {"subtractedMeasuredExposure": inputImage}) + + self._check_converted(inputImage, legacy) + + def test_convert_outputs_to_future_source_injection(self): + """Convert a difference image from a pipeline that injects sources. + + The injection planes are optional, so the conversion adds them for + an image that has pixels set in them instead of raising. + """ + planes = ("INJECTED", "INJECTED_CORE", "INJECTED_TEMPLATE", "INJECTED_CORE_TEMPLATE") + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(image_type="future") + inputImage = self._makeFutureInput(difference) + output = task.run(science, matchedTemplate, inputImage.to_legacy(), sources) + legacy = output.subtractedMeasuredExposure + for n, plane in enumerate(planes): + legacy.mask.addMaskPlane(plane) + legacy.mask.array[0, n] |= legacy.mask.getPlaneBitMask(plane) + + task.convert_outputs_to_future(output, {"subtractedMeasuredExposure": inputImage}) + + image = output.subtractedMeasuredExposure + self.assertIsInstance(image, DifferenceImage) + # Name the optional planes in the map, so that dropping one fails + # the comparison instead of skipping it. + planeMap = get_legacy_difference_image_mask_planes() | get_legacy_optional_mask_planes() + compare_masked_image_to_legacy(image, legacy.maskedImage, plane_map=planeMap) + + def test_convert_outputs_to_future_score(self): + """Convert the outputs of ``DetectAndMeasureScoreTask``.""" + self.detectionTask = detectAndMeasure.DetectAndMeasureScoreTask + science, sources, matchedTemplate, difference, score = self._make_images(withScore=True) + task = self._setup_task(doSubtractBackground=True, image_type="future") + inputImages = {"subtractedMeasuredExposure": self._makeFutureInput(difference), + "scoreMeasuredExposure": self._makeFutureInput(score)} + output = task.run(science, matchedTemplate, + inputImages["subtractedMeasuredExposure"].to_legacy(), + inputImages["scoreMeasuredExposure"].to_legacy(), sources) + legacyOutputs = {name: getattr(output, name) for name in inputImages} + task.convert_outputs_to_future(output, inputImages) + + for name, legacy in legacyOutputs.items(): + with self.subTest(name=name): + self.assertIs(getattr(output, name), inputImages[name]) + self._check_converted(getattr(output, name), legacy) + # The same background is subtracted from both images. + self.assertIn("subtracted", getattr(output, name).backgrounds) + + def test_convert_outputs_to_future_empty_background(self): + """An empty background list cannot be converted to a field, and must + not break the conversion of the images. + """ + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(doSubtractBackground=False, image_type="future") + inputImage = self._makeFutureInput(difference) + output = task.run(science, matchedTemplate, inputImage.to_legacy(), sources) + self.assertEqual(len(output.differenceBackground), 0) + legacy = output.subtractedMeasuredExposure + task.convert_outputs_to_future(output, {"subtractedMeasuredExposure": inputImage}) + self._check_converted(output.subtractedMeasuredExposure, legacy) + self.assertEqual(len(output.subtractedMeasuredExposure.backgrounds), 0) + + def test_stray_struct_field_is_ignored(self): + """``DetectAndMeasureTask`` seeds ``scoreMeasuredExposure`` in its + result struct even though it has no such output connection; check + that such a stray field is ignored. + """ + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(image_type="legacy") + output = task.run(science, matchedTemplate, difference, sources) + output.scoreMeasuredExposure = difference + connections = task.config.ConnectionsClass(config=task.config) + self.assertNotIn("scoreMeasuredExposure", connections.outputs) + lsst.pipe.base.testUtils.assertValidOutput(task, output) + + def test_run_quantum(self): + """``runQuantum`` converts before putting in future mode, and leaves + the legacy types alone otherwise. + """ + templateInfo = [self._makeTemplateInfo()] + kernel = self._makeKernel() + for imageType, expectedType in (("legacy", afwImage.ExposureF), + ("future", DifferenceImage)): + with self.subTest(imageType=imageType): + science, sources, matchedTemplate, difference = self._make_images() + photometricScaling = self._makePhotometricScaling(difference) + if imageType == "future": + difference = self._makeFutureInput(difference, templateInfo, kernel, + photometricScaling) + task = self._setup_task(doSubtractBackground=True, image_type=imageType) + butlerQC = _RecordingQuantumContext(self.dataId) + task.runQuantum(butlerQC, + _FakeRefs(science=science, matchedTemplate=matchedTemplate, + difference=difference, kernelSources=sources), + _FakeRefs()) + out = butlerQC.put_values.subtractedMeasuredExposure + self.assertIsInstance(out, expectedType) + if imageType == "legacy": + self.assertIsInstance(butlerQC.put_values.differenceBackground, + afwMath.BackgroundList) + else: + self.assertIsNotNone(out.backgrounds.subtracted) + if imageType == "future": + # The provenance of the image this was measured from + # survives, rather than being worked out again. + self.assertEqual(out.templates, templateInfo) + self.assertEqual(out.kernel, kernel) + self.assertIs(out.photometric_scaling, photometricScaling) + + def test_run_quantum_without_provenance(self): + """An input difference image with no kernel or template records + raises an error that names what is missing, and writes nothing. + """ + templateInfo = [self._makeTemplateInfo()] + kernel = self._makeKernel() + for missing, templates, inputKernel in (("PSF matching kernel", templateInfo, None), + ("template coadd records", None, kernel)): + with self.subTest(missing=missing): + science, sources, matchedTemplate, difference = self._make_images() + task = self._setup_task(image_type="future") + butlerQC = _RecordingQuantumContext(self.dataId) + refs = _FakeRefs(science=science, matchedTemplate=matchedTemplate, + difference=self._makeFutureInput(difference, templates, inputKernel), + kernelSources=sources) + + with self.assertRaisesRegex(lsst.pipe.base.InvalidQuantumError, f"'difference'.*{missing}"): + task.runQuantum(butlerQC, refs, _FakeRefs()) + self.assertIsNone(butlerQC.put_values) + + def test_run_quantum_score_task_provenance(self): + """Each output takes the kernel of the image it is measured from, + which for the score task are two different kernels. + """ + self.detectionTask = detectAndMeasure.DetectAndMeasureScoreTask + science, sources, matchedTemplate, difference, score = self._make_images(withScore=True) + templateInfo = [self._makeTemplateInfo()] + differenceKernel = self._makeKernel(scale=1.0) + scoreKernel = self._makeKernel(scale=3.0) + self.assertNotEqual(differenceKernel, scoreKernel) + photometricScaling = self._makePhotometricScaling(difference) + task = self._setup_task(image_type="future") + butlerQC = _RecordingQuantumContext(self.dataId) + + task.runQuantum(butlerQC, + _FakeRefs(science=science, matchedTemplate=matchedTemplate, + difference=self._makeFutureInput(difference, templateInfo, + differenceKernel, photometricScaling), + scoreExposure=self._makeFutureInput(score, templateInfo, scoreKernel, + photometricScaling), + kernelSources=sources), + _FakeRefs()) + + out = butlerQC.put_values.subtractedMeasuredExposure + scoreOut = butlerQC.put_values.scoreMeasuredExposure + self.assertEqual(out.templates, templateInfo) + self.assertEqual(scoreOut.templates, templateInfo) + self.assertEqual(out.kernel, differenceKernel) + self.assertEqual(scoreOut.kernel, scoreKernel) + self.assertIs(out.photometric_scaling, photometricScaling) + self.assertIs(scoreOut.photometric_scaling, photometricScaling) + + def test_connections_future_reads_difference_image(self): + """The inputs whose provenance is copied ask for the type that + carries it, and only in future mode. + """ + for imageType, expected in (("legacy", "ExposureF"), ("future", "DifferenceImage")): + with self.subTest(imageType=imageType): + config = detectAndMeasure.DetectAndMeasureScoreTask.ConfigClass() + config.image_type = imageType + connections = config.connections.ConnectionsClass(config=config) + self.assertEqual(connections.difference.storageClass, expected) + self.assertEqual(connections.scoreExposure.storageClass, expected) + # Nothing is copied off these, so the butler converts them. + self.assertEqual(connections.science.storageClass, "ExposureF") + self.assertEqual(connections.matchedTemplate.storageClass, "ExposureF") + + +class _FakeRefs: + """Stand-in for an input or output ``QuantizedConnection``.""" + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +class _RecordingQuantumContext: + """Minimal `~lsst.pipe.base.QuantumContext` that records what was put. + + Parameters + ---------- + dataId : `lsst.daf.butler.DataCoordinate` + Data ID to report as the quantum data ID. + """ + def __init__(self, dataId): + self.quantum = _FakeRefs(dataId=dataId) + self.put_values = None + + def get(self, refs): + return dict(refs.__dict__) + + def put(self, values, refs): + self.put_values = values + + class MockResponse: """Provide a mock for requests.put calls""" def __init__(self, json_data, status_code, text): diff --git a/tests/test_getTemplate.py b/tests/test_getTemplate.py index 3b407eec5..1bb0a81be 100644 --- a/tests/test_getTemplate.py +++ b/tests/test_getTemplate.py @@ -24,25 +24,31 @@ import unittest import unittest.mock +import astropy.units as u from astropy.stats import gaussian_sigma_to_fwhm import numpy as np +import lsst.afw.cameraGeom.testUtils import lsst.afw.detection import lsst.afw.geom import lsst.afw.image import lsst.afw.math import lsst.afw.table -from lsst.daf.butler import DataCoordinate, DimensionUniverse +from lsst.daf.butler import DataCoordinate, DatasetRef, DatasetType, DimensionUniverse import lsst.geom +import lsst.images +import lsst.images.psfs +from lsst.images.tests import compare_masked_image_to_legacy import lsst.ip.diffim import lsst.meas.algorithms import lsst.meas.base.tests import lsst.pex.exceptions import lsst.pipe.base as pipeBase +import lsst.pipe.base.testUtils import lsst.skymap import lsst.utils.tests -from utils import generate_data_id +from utils import generate_data_id, makeTestExposureRecord # Change this to True, `setup display_ds9`, and open ds9 (or use another afw # display backend) to show the tract/patch layouts on the image. @@ -77,6 +83,7 @@ def setUp(self): self.skymap = self._makeSkymap() self.patches = collections.defaultdict(list) self.dataIds = collections.defaultdict(list) + self.coaddRefs = {} self.exposure = self._makeExposure() self.varianceBox = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180)) @@ -260,9 +267,8 @@ def testRunOneTractInput(self): box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180)) task = lsst.ip.diffim.GetTemplateTask() # Restrict to tract 0, since the box fits in just that tract. - # Task modifies the input bbox, so pass a copy. result = task.run(coaddExposureHandles={0: self.patches[0]}, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds={0: self.dataIds[0]}, physical_filter="a_test") @@ -278,9 +284,8 @@ def testRunOneTractMultipleInputs(self): """ box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180)) task = lsst.ip.diffim.GetTemplateTask() - # Task modifies the input bbox, so pass a copy. result = task.run(coaddExposureHandles=self.patches, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds=self.dataIds, physical_filter="a_test") @@ -294,9 +299,8 @@ def testRunTwoTracts(self): """ box = lsst.geom.Box2I(lsst.geom.Point2I(200, 200), lsst.geom.Point2I(600, 600)) task = lsst.ip.diffim.GetTemplateTask() - # Task modifies the input bbox, so pass a copy. result = task.run(coaddExposureHandles=self.patches, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds=self.dataIds, physical_filter="a_test") @@ -316,9 +320,8 @@ def testRunCalibratesToNanojansky(self): templates = {} for photoCalib in (self.exposure.photoCalib, lsst.afw.image.PhotoCalib(1.0)): self._remakePatches(photoCalib) - # Task modifies the input bbox, so pass a copy. result = task.run(coaddExposureHandles={0: self.patches[0]}, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds={0: self.dataIds[0]}, physical_filter="a_test") @@ -347,7 +350,7 @@ def testGaussianPsfGridFallback(self): box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180)) task = lsst.ip.diffim.GetTemplateTask() result = task.run(coaddExposureHandles={0: self.patches[0]}, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds={0: self.dataIds[0]}, physical_filter="a_test") @@ -376,7 +379,7 @@ def testRunNoTemplate(self): task = lsst.ip.diffim.GetTemplateTask() with self.assertRaisesRegex(lsst.pipe.base.NoWorkFound, "No patches found"): task.run(coaddExposureHandles=self.patches, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds=self.dataIds, physical_filter="a_test") @@ -391,9 +394,8 @@ def testMissingPatches(self): self.patches[0].pop(1) box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180)) task = lsst.ip.diffim.GetTemplateTask() - # Task modifies the input bbox, so pass a copy. result = task.run(coaddExposureHandles=self.patches, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds=self.dataIds, physical_filter="a_test") @@ -424,7 +426,7 @@ def testNanInputs(self, box=None, nInput=None): box = lsst.geom.Box2I(lsst.geom.Point2I(200, 200), lsst.geom.Point2I(600, 600)) task = lsst.ip.diffim.GetTemplateTask() result = task.run(coaddExposureHandles=self.patches, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds=self.dataIds, physical_filter="a_test") @@ -593,9 +595,8 @@ def _configureAndRunTask(doScaleVariance, varianceScale=1.): config = lsst.ip.diffim.GetTemplateTask.ConfigClass() config.doScaleVariance = doScaleVariance task = lsst.ip.diffim.GetTemplateTask(config=config) - # Task modifies the input bbox, so pass a copy. result = task.run(coaddExposureHandles={0: self._scaleInputVariance(0, varianceScale)}, - bbox=lsst.geom.Box2I(box), + bbox=box, wcs=self.exposure.wcs, dataIds={0: self.dataIds[0]}, physical_filter="a_test") @@ -638,6 +639,287 @@ def _configureAndRunTask(doScaleVariance, varianceScale=1.): factor*scaleFactor, rtol=1e-5) self.assertImagesAlmostEqual(templateLow.variance, templateOn.variance, rtol=1e-5) + def _runLegacyForFuture(self, raiseOnUndefinedMaskMap=True): + """Build a template from tract 0 with a task configured for the + future output type, without converting it. + + The task is kept as ``self.futureTask``. + + Parameters + ---------- + raiseOnUndefinedMaskMap : `bool`, optional + Value of the task config field of the same name. + + Returns + ------- + result : `lsst.pipe.base.Struct` + The legacy output struct. + box : `lsst.geom.Box2I` + The bounding box the template was requested on, before the task + grew it by the template border. + """ + config = lsst.ip.diffim.GetTemplateTask.ConfigClass() + config.output_image_type = "future" + config.raiseOnUndefinedMaskMap = raiseOnUndefinedMaskMap + task = lsst.ip.diffim.GetTemplateTask(config=config) + self.visit = 9876 + box = lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Point2I(180, 180)) + result = task.run(coaddExposureHandles={0: self.patches[0]}, + bbox=box, + wcs=self.exposure.wcs, + dataIds={0: self.dataIds[0]}, + physical_filter="a_test", + visit=self.visit) + # run() always produces legacy types, whatever output_image_type is. + self.assertIsInstance(result.template, lsst.afw.image.ExposureF) + self.futureTask = task + return result, box + + def _runFuture(self): + """Build a template and convert it, the way ``runQuantum`` does. + + Returns + ------- + result : `lsst.pipe.base.Struct` + The converted output struct. + box : `lsst.geom.Box2I` + The bounding box the template was requested on, before the task + grew it by the template border. + """ + # lsst.images needs per-amplifier raw geometry and a field angle + # transform, which the trivial test detectors do not have. + detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + result, box = self._runLegacyForFuture() + result.template = self.futureTask.convert_outputs_to_future( + result.template, self._coaddRefs(0), detector=detector, + exposureRecord=self._exposureRecord()) + return result, box, detector + + def _exposureRecord(self): + """Return the exposure record that ``runQuantum`` rebuilds from the + observation metadata of the science image. + """ + return makeTestExposureRecord(DimensionUniverse(), visit=self.visit) + + def _coaddRefs(self, tract): + """Return butler references for the coadds of one tract, like the + ones ``runQuantum`` passes on from its ``coaddExposures`` input. + + Parameters + ---------- + tract : `int` + Id of the tract whose coadds to make references for. + + Returns + ------- + refs : `list` [`lsst.daf.butler.DatasetRef`] + One reference per patch of that tract. The same references are + returned on every call, because each new one would get a new + dataset id. + """ + if tract not in self.coaddRefs: + datasetType = DatasetType("template_coadd", ("tract", "patch", "band", "skymap"), + "ExposureF", universe=DimensionUniverse()) + self.coaddRefs[tract] = [DatasetRef(datasetType, dataId, run="test_run") + for dataId in self.dataIds[tract]] + return self.coaddRefs[tract] + + def testConvertOutputsToFuture(self): + """Test that the output template is converted to a DifferenceImage + that keeps the pixels and the detector it was built on. + """ + detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + result, box = self._runLegacyForFuture() + legacy = result.template.clone() + result.template = self.futureTask.convert_outputs_to_future( + result.template, self._coaddRefs(0), detector=detector, + exposureRecord=self._exposureRecord()) + + self.assertIsInstance(result.template, lsst.images.DifferenceImage) + compare_masked_image_to_legacy(result.template, legacy.maskedImage, + plane_map=lsst.images.get_legacy_template_mask_planes()) + self.assertEqual(result.template.unit, u.nJy) + self.assertIsNotNone(result.template.detector) + self.assertEqual(result.template.detector.name, detector.getName()) + # The template is grown by the border, so it is larger than both the + # requested box and the detector. + expectedBox = lsst.geom.Box2I(box) + expectedBox.grow(lsst.ip.diffim.GetTemplateTask.ConfigClass().templateBorderSize) + self.assertEqual(result.template.bbox.to_legacy(), expectedBox) + self.assertEqual(result.template.obs_info.visit_id, self.visit) + + def testConvertOutputsToFutureCoaddMaskPlanes(self): + """Test that the coadd mask planes a warped template carries are + converted, and come back when the image is converted to legacy. + + The template is converted with the template plane map, so the coadd + planes are named there; the source injection planes are optional, and + are added because this template has pixels set in them. + """ + detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + planes = ("CLIPPED", "REJECTED", "INEXACT_PSF", "SENSOR_EDGE", "HIGH_VARIANCE", + "INJECTED", "INJECTED_CORE") + result, _ = self._runLegacyForFuture() + mask = result.template.mask + for n, plane in enumerate(planes): + mask.addMaskPlane(plane) + mask.array[0, n] |= mask.getPlaneBitMask(plane) + legacyTemplate = result.template.clone() + + result.template = self.futureTask.convert_outputs_to_future( + result.template, self._coaddRefs(0), detector=detector, + exposureRecord=self._exposureRecord()) + + template = result.template + self.assertIsInstance(template, lsst.images.DifferenceImage) + # Name the optional planes that have pixels set in the map, so that + # dropping one fails the comparison instead of skipping it. The others + # are left out because they may be registered by unrelated tests. + optional = lsst.images.get_legacy_optional_mask_planes() + planeMap = lsst.images.get_legacy_template_mask_planes() + planeMap.update({plane: optional[plane] for plane in planes if plane in optional}) + compare_masked_image_to_legacy(template, legacyTemplate.maskedImage, plane_map=planeMap) + # The butler converts a DifferenceImage to an ExposureF with the + # difference image plane map, which does not name the coadd planes; + # they keep their own names instead of being dropped. + legacy = template.to_legacy() + for n, plane in enumerate(planes): + with self.subTest(plane=plane): + self.assertIn(plane, legacy.mask.getMaskPlaneDict()) + self.assertEqual( + np.count_nonzero(legacy.mask.array & legacy.mask.getPlaneBitMask(plane)), 1) + + def _addUnmappedMaskPlane(self, template): + """Set one pixel in the template's BRIGHT_OBJECT plane, which the + template plane map does not include, and one in its CLIPPED plane, + which it does. + """ + mask = template.mask + for n, plane in enumerate(("BRIGHT_OBJECT", "CLIPPED")): + mask.addMaskPlane(plane) + mask.array[0, n] |= mask.getPlaneBitMask(plane) + + def testConvertOutputsToFutureUnmappedMaskPlaneRaises(self): + """With raiseOnUndefinedMaskMap=True, a template with pixels set in + an unmapped mask plane cannot be converted. + """ + detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + result, _ = self._runLegacyForFuture() + self._addUnmappedMaskPlane(result.template) + with self.assertRaisesRegex(RuntimeError, "BRIGHT_OBJECT"): + self.futureTask.convert_outputs_to_future( + result.template, self._coaddRefs(0), detector=detector, + exposureRecord=self._exposureRecord()) + + def testConvertOutputsToFutureUnmappedMaskPlaneWarns(self): + """With raiseOnUndefinedMaskMap=False, an unmapped mask plane is + dropped with a warning and the mapped planes are kept. + """ + detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + result, _ = self._runLegacyForFuture(raiseOnUndefinedMaskMap=False) + self._addUnmappedMaskPlane(result.template) + with self.assertLogs(self.futureTask.log.name, level="WARNING") as cm: + template = self.futureTask.convert_outputs_to_future( + result.template, self._coaddRefs(0), detector=detector, + exposureRecord=self._exposureRecord()) + self.assertIn("BRIGHT_OBJECT", "\n".join(cm.output)) + self.assertIsInstance(template, lsst.images.DifferenceImage) + self.assertNotIn("BRIGHT_OBJECT", template.mask.schema.names) + self.assertEqual(np.count_nonzero(template.mask.get("CLIPPED")), 1) + + def testConvertOutputsToFutureLosesProvenance(self): + """The coadd inputs that `run` attaches are not carried by + `lsst.images.DifferenceImage`. + """ + legacy, _ = self._runLegacyForFuture() + self.assertTrue(legacy.template.getInfo().hasCoaddInputs()) + result, _, _ = self._runFuture() + self.assertFalse(result.template.to_legacy().getInfo().hasCoaddInputs()) + + def testConvertOutputsToFutureTemplates(self): + """The coadds that went into the template are recorded on the + converted image. + """ + result, _, _ = self._runFuture() + refs = {(ref.dataId["tract"], ref.dataId["patch"]): ref for ref in self._coaddRefs(0)} + + templates = result.template.templates + self.assertGreater(len(templates), 0) + for template in templates: + with self.subTest(tract=template.tract, patch=template.patch): + ref = refs[template.tract, template.patch] + self.assertEqual(template.skymap, "skymap") + self.assertEqual(template.dataset_id, ref.id) + self.assertEqual(template.dataset_run, ref.run) + self.assertFalse(template.psf_shape_flag) + self.assertGreater(template.psf_shape_xx, 0) + self.assertGreater(template.bounds.area, 0) + + def testConvertOutputsToFuturePsf(self): + """The Gaussian PSF set by `run` is kept, defined over the detector. + """ + result, _ = self._runLegacyForFuture() + detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + legacyPsf = result.template.getPsf() + + result.template = self.futureTask.convert_outputs_to_future( + result.template, self._coaddRefs(0), detector=detector, + exposureRecord=self._exposureRecord()) + + psf = result.template.psf + self.assertIsInstance(psf, lsst.images.psfs.GaussianPointSpreadFunction) + self.assertFloatsAlmostEqual(psf.sigma, legacyPsf.getSigma()) + self.assertEqual(psf.bounds.to_legacy(), detector.getBBox()) + self.assertEqual(psf.kernel_bbox.to_legacy().getDimensions(), legacyPsf.getDimensions()) + + def testConvertOutputsToFuturePsfToLegacy(self): + """The Gaussian survives conversion back to an Exposure, which is how + a task that has not been converted reads the template. + """ + result, _, _ = self._runFuture() + + psf = result.template.to_legacy().getPsf() + self.assertIsInstance(psf, lsst.afw.detection.GaussianPsf) + self.assertFloatsAlmostEqual(psf.getSigma(), result.template.psf.sigma) + + +class GetTemplateConnectionsTestCase(lsst.utils.tests.TestCase): + """Test the connections that ``output_image_type`` switches on. + + These only need the config classes, not a built template. + """ + + def testOutputImageType(self): + """Test that the future output type changes the template storage + class, and leaves the input connections alone. + """ + Connections = lsst.ip.diffim.GetTemplateTask.ConfigClass.ConnectionsClass + config = lsst.ip.diffim.GetTemplateTask.ConfigClass() + + connections = Connections(config=config) + self.assertEqual(connections.template.storageClass, "ExposureF") + self.assertNotIn("obs_info", connections.inputs) + + config.output_image_type = "future" + connections = Connections(config=config) + self.assertEqual(connections.template.storageClass, "DifferenceImage") + # The dataset name is unchanged; only its storage class differs. + self.assertEqual(connections.template.name, "goodSeeingDiff_templateExp") + # The detector the conversion needs comes from the science image. + self.assertEqual(connections.detector.name, "calexp.detector") + self.assertEqual(connections.detector.storageClass, "Detector") + # The observation metadata is only read in future mode, where it is + # the one input the conversion adds. + self.assertEqual(connections.obs_info.name, "calexp.obs_info") + self.assertEqual(connections.obs_info.storageClass, "ObservationInfo") + + def testLintConnections(self): + """Check that the connections are self-consistent in both modes. + """ + for task in (lsst.ip.diffim.GetTemplateTask, lsst.ip.diffim.GetDcrTemplateTask): + with self.subTest(task=task.__name__): + lsst.pipe.base.testUtils.lintConnections(task.ConfigClass.ConnectionsClass) + def setup_module(module): lsst.utils.tests.init() diff --git a/tests/test_subtractTask.py b/tests/test_subtractTask.py index cf46b6794..ab318d9b1 100644 --- a/tests/test_subtractTask.py +++ b/tests/test_subtractTask.py @@ -20,25 +20,34 @@ # along with this program. If not, see . import unittest +import uuid from astropy import units as u import lsst.afw.detection as afwDetection import lsst.afw.image as afwImage +import lsst.afw.cameraGeom.testUtils import lsst.afw.math as afwMath import lsst.afw.table as afwTable import lsst.geom import lsst.meas.algorithms as measAlg +from lsst.daf.butler import DataCoordinate, DimensionUniverse +from lsst.images import (Box, DifferenceImage, DifferenceImageTemplateInfo, Polygon, VisitImage, + get_legacy_difference_image_mask_planes) +from lsst.images.convolution_kernels import ImageBasisConvolutionKernel +from lsst.images.fields import ChebyshevField +from lsst.images.tests import compare_masked_image_to_legacy from lsst.ip.diffim import subtractImages, InsufficientKernelSourcesError from lsst.pex.config import FieldValidationError -from lsst.pipe.base import NoWorkFound +from lsst.pipe.base import InvalidQuantumError, NoWorkFound import lsst.utils.tests import numpy as np from lsst.ip.diffim.utils import (computeRobustStatistics, computePSFNoiseEquivalentArea, evaluateMeanPsfFwhm, getPsfFwhm) from lsst.pex.exceptions import InvalidParameterError -from utils import makeStats, makeTestImage, CustomCoaddPsf +from utils import (makeStats, makeTestExposureRecord, makeTestImage, makeTestVisitInfo, + CustomCoaddPsf) class AlardLuptonSubtractTestBase: @@ -64,7 +73,6 @@ def _setup_subtraction(self, fluxField="truth_instFlux", errField="truth_instFlu The configured Task to use for detection and measurement. """ config = self.subtractTask.ConfigClass() - config.doSubtractBackground = False config.restrictKernelEdgeSources = False config.sourceSelector.signalToNoise.fluxField = fluxField config.sourceSelector.signalToNoise.errField = errField @@ -198,6 +206,36 @@ def test_clear_template_mask(self): else: self.assertTrue(diffimSum >= scienceSum) + def test_renamed_template_mask_survives_registered_plane(self): + """A renamed template plane keeps its pixels whether or not the plane + it is renamed to is already registered. + + afw's mask plane dictionary is shared by every mask in the process, so + reading an image that defines SAT_TEMPLATE puts that plane in the + template's dictionary as well, with no pixels set. + """ + for preRegistered in (False, True): + with self.subTest(preRegistered=preRegistered): + science, _ = makeTestImage(psfSize=self.midPsfSize, noiseLevel=1., noiseSeed=6) + template, _ = makeTestImage(psfSize=self.goodPsfSize, noiseLevel=1., noiseSeed=7, + templateBorderSize=20, doApplyCalibration=True) + template.mask.addMaskPlane("SAT") + satBitMask = template.mask.getPlaneBitMask("SAT") + template.mask.array[10:20, 10:20] |= satBitMask + expected = np.count_nonzero(template.mask.array & satBitMask) + self.assertGreater(expected, 0) + + if preRegistered: + afwImage.Mask(1, 1).addMaskPlane("SAT_TEMPLATE") + + task = self._setup_subtraction(mode="convolveTemplate") + task.updateMasks(template, science) + + satTemplateBitMask = template.mask.getPlaneBitMask("SAT_TEMPLATE") + self.assertEqual(np.count_nonzero(template.mask.array & satTemplateBitMask), expected) + # The plane the pixels were renamed from is still cleared. + self.assertEqual(np.count_nonzero(template.mask.array & satBitMask), 0) + def test_equal_images(self): """Test that running with enough sources produces reasonable output, with the same size psf in the template and science. @@ -1431,6 +1469,494 @@ def test_runSimplifiedTaskWithSourceDetection(self): self.assertFloatsAlmostEqual(differenceStd, np.sqrt(2)*noiseLevel, rtol=0.1) +class OutputImageTypeTest(lsst.utils.tests.TestCase): + """Tests of the optional conversion of the image outputs to + `lsst.images` types. + """ + def setUp(self): + # `makeTestImage` does not attach a detector or a VisitInfo, so + # supply a detector with real amplifier geometry, a data id, and the + # exposure record the conversion takes its metadata from. + self.detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + self.dataId = DataCoordinate.standardize( + instrument="testCam", + visit=12345, + detector=self.detector.getId(), + universe=DimensionUniverse(), + ) + self.exposureRecord = makeTestExposureRecord(self.dataId.universe, + instrument=self.dataId["instrument"], + visit=self.dataId["visit"]) + + def _setup_subtraction(self, taskClass, **kwargs): + """Configure one of the subtraction tasks with the shared test + settings. + + Parameters + ---------- + taskClass : `lsst.pipe.base.PipelineTask` + The subtraction task class to configure. + **kwargs + Any additional config parameters to set. + + Returns + ------- + task : `lsst.pipe.base.PipelineTask` + The configured task. + """ + helper = AlardLuptonSubtractTestBase() + helper.subtractTask = taskClass + return helper._setup_subtraction(**kwargs) + + def _make_images(self): + """Make a science and template image pair to subtract. + + Returns + ------- + science : `lsst.afw.image.ExposureF` + The science image, with a detector attached. + template : `lsst.afw.image.ExposureF` + The template image, with no detector attached. + sources : `lsst.afw.table.SourceCatalog` + Sources detected on the science image. + """ + noiseLevel = 1. + science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6) + template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, + templateBorderSize=20, doApplyCalibration=True) + science.setDetector(self.detector) + for exposure in (science, template): + exposure.info.setVisitInfo(makeTestVisitInfo(self.dataId["visit"])) + return science, template, sources + + def _check_converted_kernel(self, kernel, legacyKernel, bbox): + """Check that a converted kernel evaluates the same as the legacy + kernel it came from. + + Parameters + ---------- + kernel : `lsst.images.convolution_kernels.ImageBasisConvolutionKernel` + The converted kernel to check. + legacyKernel : `lsst.afw.math.LinearCombinationKernel` + The kernel before conversion. + bbox : `lsst.geom.Box2I` + Region of the image the kernel was fit on. + """ + self.assertIsInstance(kernel, ImageBasisConvolutionKernel) + roundTrip = kernel.to_legacy() + for point in (bbox.getMin(), bbox.getCenter(), bbox.getMax()): + expected = afwImage.ImageD(legacyKernel.getDimensions()) + legacyKernel.computeImage(expected, False, point.x, point.y) + actual = afwImage.ImageD(roundTrip.getDimensions()) + roundTrip.computeImage(actual, False, point.x, point.y) + np.testing.assert_allclose(actual.array, expected.array, rtol=0, atol=1e-12) + + def _check_converted(self, image, kernelExpected=True, photometricScaling=None): + """Check that one converted image is a fully-formed DifferenceImage. + + Parameters + ---------- + image : `lsst.images.DifferenceImage` + The converted output to check. + kernelExpected : `bool`, optional + Should the PSF matching kernel be attached? + photometricScaling : `lsst.images.fields.BaseField`, optional + Photometric scaling the image should carry, or `None` if it should + have none. + """ + self.assertIsInstance(image, DifferenceImage) + self.assertEqual(image.unit, u.nJy) + # The instrument and visit come from the exposure record passed to + # the conversion. + frame = image.sky_projection.pixel_frame + self.assertEqual(frame.instrument, self.dataId["instrument"]) + self.assertEqual(frame.visit, self.dataId["visit"]) + self.assertEqual(image.detector.id, self.detector.getId()) + if kernelExpected: + self.assertIsNotNone(image.kernel) + else: + # An unset kernel raises, rather than returning None. + with self.assertRaises(AttributeError): + image.kernel + if photometricScaling is None: + self.assertIsNone(image.photometric_scaling) + else: + self.assertIs(image.photometric_scaling, photometricScaling) + + def test_connections_legacy(self): + """Test that the default configuration leaves the output connections + with the legacy storage classes. + """ + for configClass, connectionsClass in ( + (subtractImages.AlardLuptonSubtractConfig, + subtractImages.AlardLuptonSubtractConnections), + (subtractImages.AlardLuptonPreconvolveSubtractConfig, + subtractImages.AlardLuptonPreconvolveSubtractConnections), + (subtractImages.SimplifiedSubtractConfig, + subtractImages.SimplifiedSubtractConnections), + ): + with self.subTest(configClass=configClass.__name__): + config = configClass() + self.assertEqual(config.image_type, "legacy") + connections = connectionsClass(config=config) + for name in connections.outputs: + if name in ("difference", "matchedTemplate", "scoreExposure"): + self.assertEqual(getattr(connections, name).storageClass, "ExposureF") + self.assertEqual(connections.science.storageClass, "ExposureF") + for name in ("psfMatchingKernel", "inputPsfMatchingKernel"): + if name in connections.outputs or name in connections.inputs: + self.assertEqual(getattr(connections, name).storageClass, "MatchingKernel") + + def test_connections_future(self): + """Test that ``image_type="future"`` swaps the storage class of + every image and kernel output. + """ + for configClass, connectionsClass, expected in ( + (subtractImages.AlardLuptonSubtractConfig, + subtractImages.AlardLuptonSubtractConnections, + ("difference", "matchedTemplate")), + (subtractImages.AlardLuptonPreconvolveSubtractConfig, + subtractImages.AlardLuptonPreconvolveSubtractConnections, + ("scoreExposure",)), + (subtractImages.SimplifiedSubtractConfig, + subtractImages.SimplifiedSubtractConnections, + ("difference", "matchedTemplate")), + ): + with self.subTest(configClass=configClass.__name__): + config = configClass() + config.image_type = "future" + connections = connectionsClass(config=config) + for name in expected: + self.assertEqual(getattr(connections, name).storageClass, "DifferenceImage") + # The science image is read in the new type to get its + # photometric scaling. + self.assertEqual(connections.science.storageClass, "VisitImage") + if "psfMatchingKernel" in connections.outputs: + self.assertEqual(connections.psfMatchingKernel.storageClass, "ConvolutionKernel") + + def test_connections_future_simplified_existing_kernel(self): + """Test the storage class swap for the connection variant that + consumes an existing kernel. + """ + config = subtractImages.SimplifiedSubtractConfig() + config.image_type = "future" + config.useExistingKernel = True + connections = subtractImages.SimplifiedSubtractConnections(config=config) + self.assertEqual(connections.difference.storageClass, "DifferenceImage") + self.assertEqual(connections.matchedTemplate.storageClass, "DifferenceImage") + self.assertEqual(connections.inputPsfMatchingKernel.storageClass, "ConvolutionKernel") + self.assertNotIn("psfMatchingKernel", connections.outputs) + + def test_convert_outputs_to_future(self): + """Test converting the outputs of `AlardLuptonSubtractTask`. + """ + science, template, sources = self._make_images() + task = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask, + image_type="future") + results = task.run(template, science, sources) + # The template has no detector, so neither does the matched template. + self.assertIsNone(results.matchedTemplate.getDetector()) + legacy = {name: getattr(results, name).clone() for name in ("difference", "matchedTemplate")} + legacyKernel = results.psfMatchingKernel + + task.convert_outputs_to_future(results, self.exposureRecord) + + for name, exposure in legacy.items(): + with self.subTest(name=name): + self._check_converted(getattr(results, name)) + compare_masked_image_to_legacy(getattr(results, name), exposure.maskedImage, + plane_map=get_legacy_difference_image_mask_planes()) + self.assertEqual(results.difference.bbox.to_legacy(), science.getBBox()) + self._check_converted_kernel(results.psfMatchingKernel, legacyKernel, science.getBBox()) + self.assertIsInstance(results.kernelSources, afwTable.SourceCatalog) + + def test_convert_outputs_to_future_score(self): + """Test converting the outputs of + `AlardLuptonPreconvolveSubtractTask`, which writes a score image. + """ + science, template, sources = self._make_images() + task = self._setup_subtraction(subtractImages.AlardLuptonPreconvolveSubtractTask, + image_type="future") + results = task.run(template, science, sources) + legacyKernel = results.psfMatchingKernel + + task.convert_outputs_to_future(results, self.exposureRecord) + + self._check_converted(results.scoreExposure) + self._check_converted_kernel(results.psfMatchingKernel, legacyKernel, science.getBBox()) + + def test_convert_outputs_to_future_simplified(self): + """Test converting the outputs of `SimplifiedSubtractTask` running + with a pre-existing kernel. + """ + science, template, sources = self._make_images() + alTask = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask) + alResults = alTask.run(template.clone(), science.clone(), sources) + task = self._setup_subtraction(subtractImages.SimplifiedSubtractTask, + useExistingKernel=True, + image_type="future") + results = task.run(template.clone(), science.clone(), + inputPsfMatchingKernel=alResults.psfMatchingKernel) + + task.convert_outputs_to_future(results, self.exposureRecord) + + self._check_converted(results.difference) + self._check_converted(results.matchedTemplate) + + def test_convert_outputs_to_future_photometric_scaling(self): + """Test that every converted output carries the photometric scaling + of the science image. + """ + science, template, sources = self._make_images() + photometricScaling = self._makePhotometricScaling(science) + for taskClass, names in ((subtractImages.AlardLuptonSubtractTask, ("difference", "matchedTemplate")), + (subtractImages.AlardLuptonPreconvolveSubtractTask, ("scoreExposure",))): + with self.subTest(taskClass=taskClass.__name__): + task = self._setup_subtraction(taskClass, image_type="future") + results = task.run(template.clone(), science.clone(), sources) + + task.convert_outputs_to_future(results, self.exposureRecord, + photometricScaling=photometricScaling) + + for name in names: + self._check_converted(getattr(results, name), photometricScaling=photometricScaling) + + def test_convert_outputs_to_future_missing_output(self): + """Test that conversion skips outputs that are absent from the result + struct, instead of failing. + """ + science, template, sources = self._make_images() + task = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask, + image_type="future") + results = task.run(template, science, sources) + del results.matchedTemplate + + task.convert_outputs_to_future(results, self.exposureRecord) + + self._check_converted(results.difference) + self.assertFalse(hasattr(results, "matchedTemplate")) + + @staticmethod + def _makeTemplateInfo(tract=9813, patch=42): + """Return one record of a coadd that went into a template. + + Parameters + ---------- + tract, patch : `int`, optional + Identifiers of the coadd the record describes. + + Returns + ------- + info : `lsst.images.DifferenceImageTemplateInfo` + The record. + """ + return DifferenceImageTemplateInfo( + skymap="skymap", tract=tract, patch=patch, dataset_id=uuid.uuid4(), + dataset_run="a/template/run", bounds=Polygon.from_box(Box.factory[0:50, 0:50]), + psf_shape_xx=4.0, psf_shape_yy=4.0, psf_shape_xy=0.0, psf_shape_flag=False) + + @staticmethod + def _makePhotometricScaling(science): + """Return a photometric scaling that is easy to tell apart from the + one converted from the science image's own calibration. + + Parameters + ---------- + science : `lsst.afw.image.ExposureF` + Science image from `_make_images`. + + Returns + ------- + photometricScaling : `lsst.images.fields.ChebyshevField` + A constant scaling over the science image. + """ + return ChebyshevField(Box.from_legacy(science.getBBox()), np.array([[0.25]]), + unit=u.nJy/u.electron) + + def _make_future_science(self, science, photometricScaling=None): + """Convert a legacy science image to the type the ``future`` mode + reads. + + Parameters + ---------- + science : `lsst.afw.image.ExposureF` + Science image from `_make_images`. + photometricScaling : `lsst.images.fields.BaseField`, optional + Photometric scaling to attach in place of the converted one. + + Returns + ------- + science : `lsst.images.VisitImage` + The converted science image. + """ + image = VisitImage.from_legacy(science.clone(), unit=u.nJy, exposure_record=self.exposureRecord) + if photometricScaling is not None: + image.photometric_scaling = photometricScaling + return image + + def _make_future_template(self, template, templateInfo=None): + """Convert a legacy template to the type the ``future`` mode reads. + + Parameters + ---------- + template : `lsst.afw.image.ExposureF` + Template from `_make_images`, which has no detector. + templateInfo : `list` [`lsst.images.DifferenceImageTemplateInfo`], + optional + Record of the coadds that went into it, left off if `None`. + + Returns + ------- + template : `lsst.images.DifferenceImage` + The converted template. + """ + template = template.clone() + template.setDetector(self.detector) + image = DifferenceImage.from_legacy(template, unit=u.nJy, exposure_record=self.exposureRecord) + if templateInfo is not None: + image.templates = templateInfo + return image + + def test_convert_outputs_to_future_templates(self): + """The record of the coadds that went into the template is copied to + every converted output. + """ + science, template, sources = self._make_images() + task = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask, + image_type="future") + results = task.run(template, science, sources) + templateInfo = [self._makeTemplateInfo(patch=41), self._makeTemplateInfo(patch=42)] + + task.convert_outputs_to_future(results, self.exposureRecord, templateInfo=templateInfo) + + for image in (results.difference, results.matchedTemplate): + self.assertEqual([(t.tract, t.patch) for t in image.templates], + [(9813, 41), (9813, 42)]) + + def test_convert_outputs_to_future_no_templates(self): + """A template with no record of its coadds leaves the outputs without + one, rather than with an empty list. + """ + science, template, sources = self._make_images() + task = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask, + image_type="future") + results = task.run(template, science, sources) + + task.convert_outputs_to_future(results, self.exposureRecord) + + with self.assertRaises(AttributeError): + results.difference.templates + + def test_run_quantum_puts_future_types(self): + """``runQuantum`` converts before putting in future mode, and does not + convert in legacy mode. + """ + science, template, sources = self._make_images() + templateInfo = [self._makeTemplateInfo()] + photometricScaling = self._makePhotometricScaling(science) + for imageType, expectedType in (("legacy", afwImage.ExposureF), + ("future", DifferenceImage)): + with self.subTest(imageType=imageType): + task = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask, + image_type=imageType) + if imageType == "future": + inputTemplate = self._make_future_template(template, templateInfo) + inputScience = self._make_future_science(science, photometricScaling) + else: + inputTemplate = template.clone() + inputScience = science.clone() + butlerQC = _RecordingQuantumContext(self.dataId) + task.runQuantum(butlerQC, + _FakeRefs(template=inputTemplate, science=inputScience, + sources=sources), + _FakeRefs()) + self.assertIsInstance(butlerQC.put_values.difference, expectedType) + self.assertIsInstance(butlerQC.put_values.matchedTemplate, expectedType) + # The kernel is a separate output in both modes. + self.assertIsInstance(butlerQC.put_values.psfMatchingKernel, + ImageBasisConvolutionKernel if imageType == "future" + else afwMath.LinearCombinationKernel) + if imageType == "future": + for name in ("difference", "matchedTemplate"): + image = getattr(butlerQC.put_values, name) + self.assertEqual(image.templates, templateInfo) + self.assertIs(image.photometric_scaling, photometricScaling) + + def test_run_quantum_template_without_provenance(self): + """A template with no record of its coadds raises an error that + names what is missing, and writes nothing. + """ + science, template, sources = self._make_images() + task = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask, image_type="future") + butlerQC = _RecordingQuantumContext(self.dataId) + refs = _FakeRefs(template=self._make_future_template(template), + science=self._make_future_science(science), sources=sources) + + with self.assertRaisesRegex(InvalidQuantumError, "'template'.*template coadd records"): + task.runQuantum(butlerQC, refs, _FakeRefs()) + self.assertIsNone(butlerQC.put_values) + + def test_run_quantum_simplified_puts_future_types(self): + """``runQuantum`` converts the outputs of `SimplifiedSubtractTask`, + which loads an existing kernel rather than fitting one. + """ + science, template, sources = self._make_images() + templateInfo = [self._makeTemplateInfo()] + photometricScaling = self._makePhotometricScaling(science) + alTask = self._setup_subtraction(subtractImages.AlardLuptonSubtractTask) + alResults = alTask.run(template.clone(), science.clone(), sources) + for useExistingKernel in (True, False): + with self.subTest(useExistingKernel=useExistingKernel): + task = self._setup_subtraction(subtractImages.SimplifiedSubtractTask, + useExistingKernel=useExistingKernel, + image_type="future") + refs = {"template": self._make_future_template(template, templateInfo), + "science": self._make_future_science(science, photometricScaling)} + if useExistingKernel: + refs["inputPsfMatchingKernel"] = ImageBasisConvolutionKernel.from_legacy( + alResults.psfMatchingKernel + ) + butlerQC = _RecordingQuantumContext(self.dataId) + + task.runQuantum(butlerQC, _FakeRefs(**refs), _FakeRefs()) + + self._check_converted(butlerQC.put_values.difference, + photometricScaling=photometricScaling) + self._check_converted(butlerQC.put_values.matchedTemplate, + photometricScaling=photometricScaling) + self.assertEqual(butlerQC.put_values.difference.templates, templateInfo) + self.assertIsInstance(butlerQC.put_values.psfMatchingKernel, ImageBasisConvolutionKernel) + if useExistingKernel: + # The input kernel is the one the subtraction used. + self._check_converted_kernel(butlerQC.put_values.difference.kernel, + alResults.psfMatchingKernel, science.getBBox()) + + +class _FakeRefs: + """Stand-in for an input or output ``QuantizedConnection``.""" + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +class _RecordingQuantumContext: + """Minimal `~lsst.pipe.base.QuantumContext` that records what was put. + + Parameters + ---------- + dataId : `dict` + Data ID to report as the quantum data ID. + """ + def __init__(self, dataId): + self.quantum = _FakeRefs(dataId=dataId) + self.put_values = None + + def get(self, refs): + return dict(refs.__dict__) + + def put(self, values, refs): + self.put_values = values + + def setup_module(module): lsst.utils.tests.init() diff --git a/tests/test_utils.py b/tests/test_utils.py index 34a54b7d9..2d04fc5a2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -23,13 +23,65 @@ import requests from unittest import mock -from lsst.ip.diffim.utils import populate_sattle_visit_cache +import lsst.afw.cameraGeom.testUtils +import lsst.daf.butler +from lsst.images import obs_info_from_legacy +from lsst.images.tests import get_dp2_exposure_record +from lsst.ip.diffim.utils import populate_sattle_visit_cache, record_from_obs_info import lsst.utils.tests from test_detectAndMeasure import makeVisitInfo, MockResponse from utils import makeTestImage +class ExposureRecordTest(lsst.utils.tests.TestCase): + """Tests of rebuilding an exposure record from an observation info.""" + + def setUp(self): + self.universe = lsst.daf.butler.DimensionUniverse() + self.record = get_dp2_exposure_record(self.universe) + # lsst.images needs per-amplifier raw geometry and a field angle + # transform, which the trivial test detectors do not have. + self.detector = list(lsst.afw.cameraGeom.testUtils.CameraWrapper().camera)[0] + self.obsInfo = self._makeObsInfo(self.record) + + def _makeObsInfo(self, record): + """Return the observation info that a converted image built from + ``record`` would carry. + """ + # The conversion requires the visit info and the record to agree + # on the exposure id. + return obs_info_from_legacy(makeVisitInfo(id=record.id), record, self.detector, + detector_exposure_id=12345) + + def _rebuild(self): + """Return the record rebuilt from the observation info, with additional + identifiers added. + """ + return record_from_obs_info(self.obsInfo, self.record.instrument, self.record.id, self.universe) + + def test_fields(self): + """Every field the conversion reads comes back with the value the + original record had. + """ + record = self._rebuild() + for field in ("instrument", "id", "obs_id", "group", "day_obs", "physical_filter", + "exposure_time", "seq_num", "seq_start", "seq_end", "can_see_sky", "timespan"): + self.assertEqual(getattr(record, field), getattr(self.record, field), msg=field) + self.assertFloatsAlmostEqual(record.azimuth, self.record.azimuth, rtol=1e-14) + self.assertFloatsAlmostEqual(record.zenith_angle, self.record.zenith_angle, rtol=1e-14) + + def test_conversion_unchanged(self): + """Converting with the rebuilt record gives the same observation + info as converting with the original one. + """ + self.assertEqual(self._makeObsInfo(self._rebuild()), self.obsInfo) + + def test_missing_obs_info(self): + with self.assertRaises(ValueError): + record_from_obs_info(None, self.record.instrument, self.record.id, self.universe) + + class UtilsTest(lsst.utils.tests.TestCase): def test_populate_sattle(self): diff --git a/tests/utils.py b/tests/utils.py index 47f5fce8b..bfc28ff99 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -24,6 +24,7 @@ __all__ = ["DipoleTestImage"] +import astropy.time import numpy as np import lsst.geom as geom import lsst.afw.detection as afwDet @@ -31,9 +32,11 @@ import lsst.afw.detection as afwDetection import lsst.afw.geom as afwGeom import lsst.afw.image as afwImage +from lsst.afw.coord import Observatory +import lsst.daf.base as dafBase import lsst.afw.math as afwMath import lsst.afw.table as afwTable -from lsst.daf.butler import DataCoordinate, DimensionUniverse +from lsst.daf.butler import DataCoordinate, DimensionUniverse, Timespan import lsst.meas.algorithms as measAlg import lsst.meas.base as measBase from lsst.meas.algorithms.testUtils import plantSources @@ -1152,6 +1155,77 @@ def getAveragePosition(self): return geom.Point2D(-10000, -10000) +def makeTestVisitInfo(visit=12345): + """Return a visit info for the test images. + + Converting an exposure to an `lsst.images` type reads the observation + metadata it still carries from its visit info, so an exposure that will + be converted needs one. The date matches the timespan of + `makeTestExposureRecord`. + + Parameters + ---------- + visit : `int`, optional + Id of the visit to record. + + Returns + ------- + visitInfo : `lsst.afw.image.VisitInfo` + The visit info. + """ + return afwImage.VisitInfo( + id=visit, + exposureTime=30.0, + date=dafBase.DateTime(60815.000174, dafBase.DateTime.MJD, dafBase.DateTime.TAI), + boresightRaDec=geom.SpherePoint(23.1, 73.2, geom.degrees), + boresightAzAlt=geom.SpherePoint(120.0, 60.0, geom.degrees), + boresightRotAngle=73.2*geom.degrees, + rotType=afwImage.RotType.SKY, + observatory=Observatory(11.1*geom.degrees, 22.2*geom.degrees, 0.333), + ) + + +def makeTestExposureRecord(universe, instrument="testCam", visit=12345): + """Return an ``exposure`` dimension record for the test images. + + The record holds the observation metadata that converting an image to an + `lsst.images` type needs. + + Parameters + ---------- + universe : `lsst.daf.butler.DimensionUniverse` + Dimension universe that defines the record's schema. + instrument : `str`, optional + Name of the instrument to record. + visit : `int`, optional + Id of the visit to record. The conversion takes the visit id of an + image from the id of its exposure record. + + Returns + ------- + record : `lsst.daf.butler.DimensionRecord` + The ``exposure`` record. + """ + begin = astropy.time.Time(60815.0, format="mjd", scale="tai") + exposureTime = 30.0 + return universe["exposure"].RecordClass( + instrument=instrument, + id=visit, + obs_id=f"test_{visit}", + group=str(visit), + day_obs=20250520, + physical_filter="test filter", + exposure_time=exposureTime, + seq_num=1, + seq_start=1, + seq_end=1, + can_see_sky=True, + azimuth=120.0, + zenith_angle=30.0, + timespan=Timespan(begin=begin, end=begin + astropy.time.TimeDelta(exposureTime, format="sec")), + ) + + def generate_data_id(*, tract: int = 9813, patch: int = 42, diff --git a/ups/ip_diffim.table b/ups/ip_diffim.table index 12a20e86a..3802bd9d0 100644 --- a/ups/ip_diffim.table +++ b/ups/ip_diffim.table @@ -1,5 +1,6 @@ setupRequired(afw) setupRequired(daf_base) +setupRequired(images) setupRequired(log) setupRequired(meas_astrom) setupRequired(meas_algorithms)