From 1435d4d78bd0511b1b4ff0a1200f85e16d04a045 Mon Sep 17 00:00:00 2001 From: watsonjj Date: Sun, 14 Apr 2019 14:48:54 +0200 Subject: [PATCH] Clean up notebook diff --- .../calibrated_data_exploration.ipynb | 825 ++++++++---------- 1 file changed, 382 insertions(+), 443 deletions(-) diff --git a/docs/tutorials/calibrated_data_exploration.ipynb b/docs/tutorials/calibrated_data_exploration.ipynb index cb3984b4035..27f8af30317 100644 --- a/docs/tutorials/calibrated_data_exploration.ipynb +++ b/docs/tutorials/calibrated_data_exploration.ipynb @@ -1,444 +1,383 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "# Explore Calibrated Data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "import ctapipe\n", - "from ctapipe.utils.datasets import get_dataset_path\n", - "from ctapipe.io import event_source, EventSeeker\n", - "from ctapipe.visualization import CameraDisplay\n", - "from ctapipe.instrument import CameraGeometry\n", - "from matplotlib import pyplot as plt\n", - "from astropy import units as u\n", - "import numpy as np\n", - "%matplotlib inline\n", - "plt.style.use(\"ggplot\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "print(ctapipe.__version__)\n", - "print(ctapipe.__file__)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Let\u0027s first open a raw event file and get an event out of it:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "filename \u003d get_dataset_path(\"gamma_test_large.simtel.gz\")\n", - "source \u003d event_source(filename, max_events\u003d2)\n", - "\n", - "for event in source:\n", - " print(event.r0.event_id)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "filename" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "source" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "event" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "print(event.r1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "## Perform basic calibration:\n", - "\n", - "Here we will use a `CameraCalibrator` which is just a simple wrapper that runs the three calibraraton and trace-integration phases of the pipeline, taking the data from levels:\n", - "\n", - " **R0** \u0026rightarrow; **R1** \u0026rightarrow; **DL0** \u0026rightarrow; **DL1**\n", - "\n", - "You could of course do these each separately, by using the classes `R1Calibrator`, `DL0Reducer`, and `DL1Calibrator`.\n", - "Note that we have not specified any configuration to the `CameraCalibrator`, so it will be using the default algorithms and thresholds, other than specifying that the product is a \"HESSIOR1Calibrator\" (hopefully in the near future that will be automatic)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": "from ctapipe.calib import CameraCalibrator\n\ncalib \u003d CameraCalibrator()\ncalib.calibrate(event)" - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Now the *r1*, *dl0* and *dl1* containers are filled in the event\n", - "\n", - "* **r1.tel[x]**: contains the \"r1-calibrated\" waveforms, after gain-selection, pedestal subtraciton, and gain-correction\n", - "* **dl0.tel[x]**: is the same but with optional data volume reduction (some pixels not filled), in this case this is not performed by default, so it is the same as r1\n", - "* **dl1.tel[x]**: contains the (possibly re-calibrated) waveforms as dl0, but also the time-integrated *image* that has been calculated using a `ImageExtractor` (a `NeighborPeakWindowSum` by default)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "for tel_id in event.dl1.tel:\n", - " print(\"TEL{:03}: {}\".format(tel_id, event.inst.subarray.tel[tel_id]))\n", - " print(\" - r0 wave shape : {}\".format(event.r0.tel[tel_id].waveform.shape))\n", - " print(\" - r1 wave shape : {}\".format(event.r1.tel[tel_id].waveform.shape))\n", - " print(\" - dl1 image shape : {}\".format(event.dl1.tel[tel_id].image.shape))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "## Some image processing:\n", - "\n", - "Let\u0027s look at the image" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "from ctapipe.visualization import CameraDisplay\n", - "tel_id \u003d sorted(event.r1.tels_with_data)[1]\n", - "sub \u003d event.inst.subarray\n", - "camera \u003d sub.tel[tel_id].camera\n", - "image \u003d event.dl1.tel[tel_id].image[0] " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "disp \u003d CameraDisplay(camera, image\u003dimage)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "from ctapipe.image import tailcuts_clean, hillas_parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "mask \u003d tailcuts_clean(camera, image, picture_thresh\u003d10, boundary_thresh\u003d5, min_number_picture_neighbors\u003d2)\n", - "cleaned \u003d image.copy()\n", - "cleaned[~mask] \u003d 0\n", - "disp \u003d CameraDisplay(camera, image\u003dcleaned)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "params \u003d hillas_parameters(camera, cleaned)\n", - "print(params)\n", - "params" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "params \u003d hillas_parameters(camera, cleaned)\n", - "\n", - "plt.figure(figsize\u003d(10,10))\n", - "disp \u003d CameraDisplay(camera, image\u003dimage)\n", - "disp.add_colorbar()\n", - "disp.overlay_moments(params, color\u003d\u0027red\u0027, lw\u003d3)\n", - "disp.highlight_pixels(mask, color\u003d\u0027white\u0027, alpha\u003d0.3, linewidth\u003d2)\n", - "\n", - "plt.xlim(params.x.to_value(u.m) - 0.5, params.x.to_value(u.m) + 0.5)\n", - "plt.ylim(params.y.to_value(u.m) - 0.5, params.y.to_value(u.m) + 0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "source.metadata" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "## More complex image processing:\n", - "\n", - "Let\u0027s now explore how stereo reconstruction works. \n", - "\n", - "### first, look at a summed image from multiple telescopes\n", - "\n", - "For this, we want to use a `CameraDisplay` again, but since we can\u0027t sum and display images with different cameras, we\u0027ll just sub-select images from a particular camera type\n", - "\n", - "These are the telescopes that are in this event:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "tels_in_event \u003d set(event.dl1.tel.keys()) # use a set here, so we can intersect it later\n", - "tels_in_event" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "cam_ids \u003d set(sub.get_tel_ids_for_type(\"MST:FlashCam\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "cams_in_event \u003d tels_in_event.intersection(cam_ids)\n", - "first_tel_id \u003d list(cams_in_event)[0]\n", - "tel \u003d sub.tel[first_tel_id]\n", - "print(\"{}s in event: {}\".format(tel, cams_in_event))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "Now, let\u0027s sum those images:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "image_sum \u003d np.zeros_like(tel.camera.pix_x.value) # just make an array of 0\u0027s in the same shape as the camera \n", - "\n", - "for tel_id in cams_in_event:\n", - " image_sum +\u003d event.dl1.tel[tel_id].image[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "And finally display the sum of those images" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "plt.figure(figsize\u003d(8,8))\n", - "\n", - "disp \u003d CameraDisplay(tel.camera, image\u003dimage_sum)\n", - "disp.overlay_moments(params, with_label\u003dFalse)\n", - "plt.title(\"Sum of {}x {}\".format(len(cams_in_event), tel))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": {} - }, - "source": [ - "let\u0027s also show which telescopes those were. Note that currently ArrayDisplay\u0027s value field is a vector by `tel_index`, not `tel_id`, so we have to convert to a tel_index. (this may change in a future version to be more user-friendly)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "from ctapipe.visualization import ArrayDisplay" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [ - "nectarcam_subarray \u003d sub.select_subarray(\"FlashCam\", cam_ids)\n", - "\n", - "hit_pattern \u003d np.zeros(shape\u003dnectarcam_subarray.num_tels)\n", - "hit_pattern[[nectarcam_subarray.tel_indices[x] for x in cams_in_event ]] \u003d 100\n", - "\n", - "plt.set_cmap(plt.cm.Accent)\n", - "plt.figure(figsize\u003d(8,8))\n", - "\n", - "ad \u003d ArrayDisplay(nectarcam_subarray)\n", - "ad.values \u003d hit_pattern\n", - "ad.add_labels()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": {} - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Explore Calibrated Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ctapipe\n", + "from ctapipe.utils.datasets import get_dataset_path\n", + "from ctapipe.io import event_source, EventSeeker\n", + "from ctapipe.visualization import CameraDisplay\n", + "from ctapipe.instrument import CameraGeometry\n", + "from matplotlib import pyplot as plt\n", + "from astropy import units as u\n", + "import numpy as np\n", + "%matplotlib inline\n", + "plt.style.use(\"ggplot\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(ctapipe.__version__)\n", + "print(ctapipe.__file__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's first open a raw event file and get an event out of it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filename = get_dataset_path(\"gamma_test_large.simtel.gz\")\n", + "source = event_source(filename, max_events=2)\n", + "\n", + "for event in source:\n", + " print(event.r0.event_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filename" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "source" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "event" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(event.r1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Perform basic calibration:\n", + "\n", + "Here we will use a `CameraCalibrator` which is just a simple wrapper that runs the three calibraraton and trace-integration phases of the pipeline, taking the data from levels:\n", + "\n", + " **R0** → **R1** → **DL0** → **DL1**\n", + "\n", + "You could of course do these each separately, by using the classes `R1Calibrator`, `DL0Reducer`, and `DL1Calibrator`.\n", + "Note that we have not specified any configuration to the `CameraCalibrator`, so it will be using the default algorithms and thresholds, other than specifying that the product is a \"HESSIOR1Calibrator\" (hopefully in the near future that will be automatic)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ctapipe.calib import CameraCalibrator\n", + "\n", + "calib = CameraCalibrator()\n", + "calib.calibrate(event)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now the *r1*, *dl0* and *dl1* containers are filled in the event\n", + "\n", + "* **r1.tel[x]**: contains the \"r1-calibrated\" waveforms, after gain-selection, pedestal subtraciton, and gain-correction\n", + "* **dl0.tel[x]**: is the same but with optional data volume reduction (some pixels not filled), in this case this is not performed by default, so it is the same as r1\n", + "* **dl1.tel[x]**: contains the (possibly re-calibrated) waveforms as dl0, but also the time-integrated *image* that has been calculated using a `ImageExtractor` (a `NeighborPeakWindowSum` by default)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for tel_id in event.dl1.tel:\n", + " print(\"TEL{:03}: {}\".format(tel_id, event.inst.subarray.tel[tel_id]))\n", + " print(\" - r0 wave shape : {}\".format(event.r0.tel[tel_id].waveform.shape))\n", + " print(\" - r1 wave shape : {}\".format(event.r1.tel[tel_id].waveform.shape))\n", + " print(\" - dl1 image shape : {}\".format(event.dl1.tel[tel_id].image.shape))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Some image processing:\n", + "\n", + "Let's look at the image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ctapipe.visualization import CameraDisplay\n", + "tel_id = sorted(event.r1.tels_with_data)[1]\n", + "sub = event.inst.subarray\n", + "camera = sub.tel[tel_id].camera\n", + "image = event.dl1.tel[tel_id].image[0] " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "disp = CameraDisplay(camera, image=image)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ctapipe.image import tailcuts_clean, hillas_parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mask = tailcuts_clean(camera, image, picture_thresh=10, boundary_thresh=5, min_number_picture_neighbors=2)\n", + "cleaned = image.copy()\n", + "cleaned[~mask] = 0\n", + "disp = CameraDisplay(camera, image=cleaned)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "params = hillas_parameters(camera, cleaned)\n", + "print(params)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "params = hillas_parameters(camera, cleaned)\n", + "\n", + "plt.figure(figsize=(10,10))\n", + "disp = CameraDisplay(camera, image=image)\n", + "disp.add_colorbar()\n", + "disp.overlay_moments(params, color='red', lw=3)\n", + "disp.highlight_pixels(mask, color='white', alpha=0.3, linewidth=2)\n", + "\n", + "plt.xlim(params.x.to_value(u.m) - 0.5, params.x.to_value(u.m) + 0.5)\n", + "plt.ylim(params.y.to_value(u.m) - 0.5, params.y.to_value(u.m) + 0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "source.metadata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## More complex image processing:\n", + "\n", + "Let's now explore how stereo reconstruction works. \n", + "\n", + "### first, look at a summed image from multiple telescopes\n", + "\n", + "For this, we want to use a `CameraDisplay` again, but since we can't sum and display images with different cameras, we'll just sub-select images from a particular camera type\n", + "\n", + "These are the telescopes that are in this event:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tels_in_event = set(event.dl1.tel.keys()) # use a set here, so we can intersect it later\n", + "tels_in_event" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cam_ids = set(sub.get_tel_ids_for_type(\"MST:FlashCam\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cams_in_event = tels_in_event.intersection(cam_ids)\n", + "first_tel_id = list(cams_in_event)[0]\n", + "tel = sub.tel[first_tel_id]\n", + "print(\"{}s in event: {}\".format(tel, cams_in_event))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, let's sum those images:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "image_sum = np.zeros_like(tel.camera.pix_x.value) # just make an array of 0's in the same shape as the camera \n", + "\n", + "for tel_id in cams_in_event:\n", + " image_sum += event.dl1.tel[tel_id].image[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And finally display the sum of those images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(8,8))\n", + "\n", + "disp = CameraDisplay(tel.camera, image=image_sum)\n", + "disp.overlay_moments(params, with_label=False)\n", + "plt.title(\"Sum of {}x {}\".format(len(cams_in_event), tel))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "let's also show which telescopes those were. Note that currently ArrayDisplay's value field is a vector by `tel_index`, not `tel_id`, so we have to convert to a tel_index. (this may change in a future version to be more user-friendly)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ctapipe.visualization import ArrayDisplay" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nectarcam_subarray = sub.select_subarray(\"FlashCam\", cam_ids)\n", + "\n", + "hit_pattern = np.zeros(shape=nectarcam_subarray.num_tels)\n", + "hit_pattern[[nectarcam_subarray.tel_indices[x] for x in cams_in_event ]] = 100\n", + "\n", + "plt.set_cmap(plt.cm.Accent)\n", + "plt.figure(figsize=(8,8))\n", + "\n", + "ad = ArrayDisplay(nectarcam_subarray)\n", + "ad.values = hit_pattern\n", + "ad.add_labels()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}