Initial commit

.dockerignore

@@ -0,0 +1,7 @@
+README.md
+Dockerfile
+camera/subaru_hsc_camera_0_00.fits
+camera/buildDetector_hack.py
+camera/buildDetector.py
+camera/1_Build_detectors.ipynb
+test_data

Dockerfile

@@ -0,0 +1,8 @@
+FROM lsstsqre/centos:7-stack-lsst_distrib-v21_0_0
+
+RUN echo "source /opt/lsst/software/stack/loadLSST.bash" >> /home/lsst/.bashrc
+RUN mkdir -p /opt/lsst/software/stack/stack/miniconda3-py37_4.8.2-cb4e2dc/Linux64/obs_ctmo/v1
+COPY . /opt/lsst/software/stack/stack/miniconda3-py37_4.8.2-cb4e2dc/Linux64/obs_ctmo/v1
+RUN eups declare obs_ctmo v1 -r ${EUPS_PATH}/Linux64/obs_ctmo/v1
+RUN eups declare -t current obs_ctmo v1
+RUN setup obs_ctmo v1

README.md

@@ -0,0 +1,73 @@
+# obs_ctmo
+
+This repo contains code to adapt the Cristina Torres Memorial Observatory in
+Brownsville, Texas to work with the LSST software stack.
+
+## Generate Test Images
+
+To test the pipeline, you may want to first execute the script to generate the test fake images:
+
+    % python test_data/generate_images.py
+
+This will create a `CTMO_TEST` folder inside `test_data` and generate 3 dark, 3 bias, 3 flat and 3 light frames with random 16-bit integer data.
+
+## Docker Image
+
+The `obs_ctmo` package is under development with version 21 of the docker image of the LSST Software Stack.
+We provide a Dockerfile that creates an image with `obs_ctmo` package installed by EUPS.
+You can create it, by running:
+
+    % docker build --tag ctmo/lsstpipe:latest .
+
+### Create a Detachable Container
+
+Prepare to run it in [detached mode](https://pipelines.lsst.io/install/docker.html).
+Make sure you are in this package folder (`obs_ctmo`) so that `pwd` resolves correctly.
+
+    % docker run -itd --name ctmo \
+    -v `pwd`/test_data:/home/lsst/pipe \
+    -v `pwd`:/opt/lsst/software/stack/stack/miniconda3-py37_4.8.2-cb4e2dc/Linux64/obs_ctmo/v1 \
+    ctmo/lsstpipe:latest
+
+This will create a container from the ctmo image in detached mode.
+
+The two `-v` arguments are optional. The first one mounts the directory with the test data inside the container. The second will mirror the `obs_ctmo` package inside the EUPS manager so that it can immediately pick up the edits when it's changed (useful when debugging).
+
+_Note: Check the `init.sh` that summarizes most of these initial steps._
+
+### Hop In and Out the Detachable Container
+
+From a shell on your host system, open a shell in the container with the docker exec command:
+
+    % docker exec -it ctmo /bin/bash
+
+Your prompt is now a prompt in the container with `obs_ctmo` installed.
+
+You can repeat this process, attaching to the container multiple times, to open multiple container shells.
+To close a container shell, terminate the session with `exit`.
+
+To stop the container entirely, run this command from your host’s shell:
+
+    % docker stop ctmo
+
+And delete the container after you no longer need it.
+
+    % docker rm ctmo
+
+### Run LSST Stack Commands
+
+From inside the `ctmo` container:
+
+    $ cd ~/pipe
+    $ setup lsst_distrib
+    $ setup obs_ctmo
+    $ setup -j -r CTMO_TEST
+    $ ingestImages.py DATA $CTMO_TEST_DIR/*.fits --mode=link
+
+You should see the ingested images in `~/pipe/DATA` inside the container (`test_data/DATA` in your local machine.)
+
+_Note: These commands are summarized in `test_data/prepare.sh`._
+
+---
+
+(c) CTMO Dev Team

camera/buildDetector.py

@@ -0,0 +1,83 @@
+import lsst.afw.table as afwTable
+import lsst.afw.geom as afwGeom
+import numpy as np
+
+# This is copying from afw/tests/testAmpInfoTable.py:
+readout = [[20.0]]
+gain_all = [[0.5]]
+
+
+def addAmp(ampCatalog, i, rN, gain_s):
+    record = ampCatalog.addNew()
+
+    width = 4096
+    height = 4096
+
+    os = 0  # pixels of overscan
+
+    bbox = afwGeom.Box2I(
+        afwGeom.Point2I(0, 0), afwGeom.Extent2I(width, height)
+    )
+    bbox.shift(afwGeom.Extent2I(width * i, 0))
+
+    gain = gain_s
+    saturation = 65535
+    readNoise = rN
+    readoutCorner = afwTable.LL if i == 0 else afwTable.LR
+    linearityCoeffs = (1.0, np.nan, np.nan, np.nan)
+    linearityType = "None"
+    rawBBox = afwGeom.Box2I(
+        afwGeom.Point2I(0, 0), afwGeom.Extent2I(width, height)
+    )
+    rawXYOffset = afwGeom.Extent2I(0, 0)
+    rawDataBBox = afwGeom.Box2I(
+        afwGeom.Point2I(0 if i == 0 else 0, 0), afwGeom.Extent2I(width, height)
+    )
+    rawHorizontalOverscanBBox = afwGeom.Box2I(
+        afwGeom.Point2I(0 if i == 0 else width - os - 1, 0),
+        afwGeom.Extent2I(os, 6220),
+    )
+    # rawVerticalOverscanBBox = afwGeom.Box2I(afwGeom.Point2I(50, 6132), afwGeom.Extent2I(0, 0))
+    # rawPrescanBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(0, 0))
+    emptyBox = afwGeom.BoxI()
+
+    shiftp = afwGeom.Extent2I((width) * i, 0)
+    rawBBox.shift(shiftp)
+    rawDataBBox.shift(shiftp)
+    rawHorizontalOverscanBBox.shift(shiftp)
+
+    record.setHasRawInfo(True)  # Sets the first Flag=True
+    record.setRawFlipX(False)  # Sets the second Flag=False
+    record.setRawFlipY(False)  # Sets the third Flag=False
+    record.setBBox(bbox)
+    record.setName("left" if i == 0 else "right")
+    record.setGain(gain)
+    record.setSaturation(saturation)
+    record.setReadNoise(readNoise)
+    record.setReadoutCorner(readoutCorner)
+    record.setLinearityCoeffs(linearityCoeffs)
+    record.setLinearityType(linearityType)
+    record.setRawBBox(rawBBox)
+    record.setRawXYOffset(rawXYOffset)
+    record.setRawDataBBox(rawDataBBox)
+    record.setRawHorizontalOverscanBBox(rawHorizontalOverscanBBox)
+    record.setRawVerticalOverscanBBox(emptyBox)
+    record.setRawPrescanBBox(emptyBox)
+
+
+def makeCcd(ccdId):
+    schema = afwTable.AmpInfoTable.makeMinimalSchema()
+    ampCatalog = afwTable.AmpInfoCatalog(schema)
+    ccdName = ccdId + 1
+    for i in range(1):
+        addAmp(ampCatalog, i, readout[ccdId - 1][i], gain_all[ccdId - 1][i])
+    return ampCatalog.writeFits("n%s_necam.fits" % ccdName)
+
+
+def main():
+    for i in range(1):
+        camera = makeCcd(i)
+
+
+if __name__ == "__main__":
+    main()

camera/buildDetector_hack.py

@@ -0,0 +1,60 @@
+import lsst.afw.table as afwTable
+import numpy as np
+
+# Gain: (0.722 pm 0.006) ADU/electron
+gain = 0.722
+# Readout noise: (15.8 pm 0.2) ADU
+readout_noise = 15.8
+# Saturation level: 65535 ADU
+saturation = 65535
+
+sample_fits_path = "subaru_hsc_camera_0_00.fits"
+hscAfw = afwTable.BaseCatalog.readFits(sample_fits_path)
+ctmoAfw = hscAfw[:1]
+
+# Change the pixel coord stuff for every ccd
+h, w = 4096, 4096
+ctmoAfw["bbox_extent_x"] = w
+ctmoAfw["bbox_extent_y"] = h
+ctmoAfw["raw_bbox_extent_x"] = w
+ctmoAfw["raw_bbox_extent_y"] = h
+ctmoAfw["raw_databbox_min_x"] = 0
+ctmoAfw["raw_databbox_min_y"] = 0
+ctmoAfw["raw_databbox_extent_x"] = w
+ctmoAfw["raw_databbox_extent_y"] = h
+ctmoAfw["raw_horizontaloverscanbbox_min_x"] = 0
+ctmoAfw["raw_horizontaloverscanbbox_min_y"] = 0
+ctmoAfw["raw_horizontaloverscanbbox_extent_x"] = 0
+ctmoAfw["raw_horizontaloverscanbbox_extent_y"] = 0
+ctmoAfw["raw_verticaloverscanbbox_min_x"] = 0
+ctmoAfw["raw_verticaloverscanbbox_min_y"] = 0
+ctmoAfw["raw_verticaloverscanbbox_extent_x"] = 0
+ctmoAfw["raw_verticaloverscanbbox_extent_y"] = 0
+ctmoAfw["raw_prescanbbox_min_x"] = 0
+ctmoAfw["raw_prescanbbox_min_y"] = 0
+ctmoAfw["raw_prescanbbox_extent_x"] = 0
+ctmoAfw["raw_prescanbbox_extent_y"] = 0
+
+ctmoAfw["gain"] = gain
+ctmoAfw["linearity_coeffs"] = [1.0, np.nan, np.nan, np.nan]
+ctmoAfw["readnoise"] = readout_noise
+ctmoAfw["saturation"] = saturation
+ctmoAfw.writeFits("PL16803.fits")
+
+# record.setHasRawInfo(True) #Sets the first Flag=True
+# record.setRawFlipX(False)  #Sets the second Flag=False
+# record.setRawFlipY(False)  #Sets the third Flag=False
+# record.setBBox(bbox)
+# record.setName('left' if i == 0 else 'right')
+# record.setGain(gain)
+# record.setSaturation(saturation)
+# record.setReadNoise(readNoise)
+# record.setReadoutCorner(readoutCorner)
+# record.setLinearityCoeffs(linearityCoeffs)
+# record.setLinearityType(linearityType)
+# record.setRawBBox(rawBBox)
+# record.setRawXYOffset(rawXYOffset)
+# record.setRawDataBBox(rawDataBBox)
+# record.setRawHorizontalOverscanBBox(rawHorizontalOverscanBBox)
+# record.setRawVerticalOverscanBBox(emptyBox)
+# record.setRawPrescanBBox(emptyBox)

camera/camera.py

@@ -0,0 +1,90 @@
+import lsst.afw.cameraGeom.cameraConfig
+
+# This simply asserts whether the config class is of the right format.
+assert type(config) == lsst.afw.cameraGeom.cameraConfig.CameraConfig, (
+    "config is of type %s.%s instead of lsst.afw.cameraGeom.cameraConfig.CameraConfig"
+    % (type(config).__module__, type(config).__name__)
+)
+
+# Sets the plate scale in arcsec/mm:
+# Pixel scale: 0.63 arcsec/pixel, and 9 microns each pixel
+config.plateScale = 70
+
+# This defines the native coordinate system:
+# FocalPlane is (x,y) in mm (rather than radians or pixels, for example).
+config.transformDict.nativeSys = "FocalPlane"
+
+# For some reason, it must have "Pupil" defined:
+config.transformDict.transforms = {}
+config.transformDict.transforms[
+    "FieldAngle"
+] = lsst.afw.geom.transformConfig.TransformConfig()
+
+# coeffs = [0,1] is the default. This is only necessary if you want to convert
+# between positions on the focal plane.
+config.transformDict.transforms["FieldAngle"].transform[
+    "inverted"
+].transform.retarget(target=lsst.afw.geom.transformRegistry["radial"])
+config.transformDict.transforms["FieldAngle"].transform[
+    "inverted"
+].transform.coeffs = [0.0, 1.0]
+config.transformDict.transforms["FieldAngle"].transform.name = "inverted"
+
+h, w = 4096, 4096
+
+# Define a list of detectors:
+config.detectorList = {}
+config.detectorList[0] = lsst.afw.cameraGeom.cameraConfig.DetectorConfig()
+
+# All non-commented lines ARE REQUIRED for CameraMapper:
+# y0 of pixel bounding box
+config.detectorList[0].bbox_y0 = 0
+
+# y1 of pixel bounding box
+config.detectorList[0].bbox_y1 = h
+
+# x0 of pixel bounding box
+config.detectorList[0].bbox_x0 = 0
+
+# x1 of pixel bounding box
+config.detectorList[0].bbox_x1 = w
+
+# Name of detector slot
+config.detectorList[0].name = "PL16803"
+
+# Pixel size in mm
+config.detectorList[0].pixelSize_x = 0.009
+config.detectorList[0].pixelSize_y = 0.009
+
+# Name of native coordinate system
+config.detectorList[0].transformDict.nativeSys = "Pixels"
+
+# x position of the reference point in the detector in pixels in transposed coordinates.
+config.detectorList[0].refpos_x = w // 2
+
+# y position of the reference point in the detector in pixels in transposed coordinates.
+config.detectorList[0].refpos_y = h // 2
+
+# Detector type: SCIENCE=0, FOCUS=1, GUIDER=2, WAVEFRONT=3
+config.detectorList[0].detectorType = 0
+
+# x offset from the origin of the camera in mm in the transposed system.
+config.detectorList[0].offset_x = 0.0
+
+# y offset from the origin of the camera in mm in the transposed system.
+config.detectorList[0].offset_y = 0.0
+
+config.detectorList[0].yawDeg = 0.0
+config.detectorList[0].rollDeg = 0.0
+config.detectorList[0].pitchDeg = 0.0
+
+# Serial string associated with this specific detector
+config.detectorList[0].serial = "1"
+
+# ID of detector slot
+config.detectorList[0].id = 1
+
+# Name of this config
+# This isn't strictly required for CameraMapper
+# but I'm keeping it there as it seems like a good idea:
+config.name = "CTMO"

config/ingest.py

@@ -0,0 +1,37 @@
+from lsst.obs.ctmo.ingest import CtmoParseTask
+
+config.parse.retarget(CtmoParseTask)
+
+config.parse.translation = {
+    "filter": "FILTER",
+    "object": "OBJECT",
+    "run": "RUN-ID",
+}
+config.parse.translators = {
+    "imageType": "translate_imageType",
+    "expTime": "translate_expTime",
+    "date": "translate_date",
+    "dateObs": "translate_dateObs",
+    "expId": "translate_expId",
+    "detector": "translate_detector",
+    "visit": "translate_visit",
+}
+config.parse.defaults = {
+}
+config.parse.hdu = 0
+
+config.register.columns = {
+    'run': 'text',
+    'visit': 'int',
+    'filter': 'text',
+    'date': 'text',
+    'dateObs': 'text',
+    'expTime': 'double',
+    'detector': 'int',
+    'object': 'text',
+    'imageType': 'text',
+    'expId': 'int',
+}
+
+config.register.unique = ["expId", "detector", "visit"]
+config.register.visit = ['visit', 'filter', 'dateObs', 'expTime']

init.sh

@@ -0,0 +1,6 @@
+python test_data/generate_images.py
+docker build --tag ctmo/lsstpipe:latest .
+docker run -itd --name ctmo \
+-v `pwd`/test_data:/home/lsst/pipe \
+-v `pwd`:/opt/lsst/software/stack/stack/miniconda3-py37_4.8.2-cb4e2dc/Linux64/obs_ctmo/v1 \
+ctmo/lsstpipe:latest