IVOA-Fornax-demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49057ac1-1d20-411b-878b-68cce77b191a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for inspiration: https://colab.research.google.com/drive/1jGja4e-x97wxfdLyAA_XIGLwAd35JUNb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cddca2bf-cd81-4467-87ab-c26b1280823f",
   "metadata": {},
   "source": [
    "Install lsdb (and transitively hipscat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67d7dca3-9ef8-43fb-a66c-a8caa40d8e26",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install -r requirements.txt --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2839f9b-73ed-4653-ac58-457ec72e1a74",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import lsdb\n",
    "from lsdb.core.search import BoxSearch, ConeSearch, PolygonSearch\n",
    "from hipscat.inspection import plot_pixels\n",
    "import hipscat as hc\n",
    "from hipscat.inspection.almanac import Almanac\n",
    "from hipscat.inspection.almanac_info import AlmanacInfo\n",
    "\n",
    "\"Version of lsdb is %s\" % lsdb.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4e755bf-bc30-4070-871a-3478a3220a6f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import dask.distributed\n",
    "\n",
    "# initialize dask client\n",
    "Client = dask.distributed.Client(n_workers=2, threads_per_worker=1, memory_limit=\"8Gb\")\n",
    "\n",
    "Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06790ed5-de0d-4aa1-8bef-6ebc462d38f1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# These are locations where the catalogs are stored if you want to access via internet\n",
    "\n",
    "# GAIA\n",
    "# gaia_path = \"https://epyc.astro.washington.edu/~lincc-frameworks/hipscat_surveys/gaia_dr3/gaia\"\n",
    "\n",
    "# ZTF\n",
    "# ztf_object_path = \"https://epyc.astro.washington.edu/~lincc-frameworks/hipscat_surveys/ztf/ztf_dr14/\"\n",
    "# ztf_source_path = \"https://epyc.astro.washington.edu/~lincc-frameworks/hipscat_surveys/ztf/ztf_source/\"\n",
    "\n",
    "# but we have these catalogs available via AWS buckets\n",
    "BUCKET = \"irsa-mast-tike-spitzer-data\"\n",
    "\n",
    "GAIA_BASE = f\"s3://{BUCKET}/data/GAIA/dr3/gaiasource/hipscat\"\n",
    "GAIA_OBJECT_PATH = f\"{GAIA_BASE}/gaia-dr3-gaiasource-hipscat\"\n",
    "GAIA_OBJECT_MARGIN_PATH = f\"{GAIA_BASE}/gaia-dr3-gaiasource-hipscat_margin_10arcsec\"\n",
    "\n",
    "ZTF_BASE = f\"s3://{BUCKET}/data/ZTF/dr20\"\n",
    "ZTF_OBJECT_PATH = f\"{ZTF_BASE}/objects/hipscat/ztf-dr20-objects-hipscat\"\n",
    "ZTF_OBJECT_MARGIN_PATH = f\"{ZTF_BASE}/objects/hipscat/ztf-dr20-objects-hipscat_margin_10arcsec\"\n",
    "ZTF_SOURCE_PATH = f\"{ZTF_BASE}/lc/hipscat/ztf-dr20-lc-hipscat\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6545ef59-f273-4bd7-bc70-c24488e41804",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Load Gaia DR3 margin cache\n",
    "gaia_margin = lsdb.read_hipscat(GAIA_OBJECT_MARGIN_PATH)\n",
    "# Load lite version of Gaia DR3 for quick cone searches\n",
    "# we load only ra, dec columns, and small sections of the sky\n",
    "gaia_lite = lsdb.read_hipscat(\n",
    "    GAIA_OBJECT_PATH,\n",
    "    columns=[\"ra\", \"dec\"],\n",
    "    margin_cache=gaia_margin,\n",
    "    search_filter=ConeSearch(ra=-60, dec=20, radius_arcsec=1 * 3600),\n",
    ")\n",
    "gaia_cone = lsdb.read_hipscat(\n",
    "    GAIA_OBJECT_PATH,\n",
    "    margin_cache=gaia_margin,\n",
    "    search_filter=ConeSearch(ra=-60, dec=20, radius_arcsec=1 * 3600),\n",
    ")\n",
    "# This is loading the full GAIA catalog\n",
    "gaia = lsdb.read_hipscat(GAIA_OBJECT_PATH, margin_cache=gaia_margin)\n",
    "\n",
    "# This is loading the full ZTF catalog\n",
    "ztf_margin = lsdb.read_hipscat(ZTF_OBJECT_MARGIN_PATH)\n",
    "ztf = lsdb.read_hipscat(ZTF_OBJECT_PATH, margin_cache=ztf_margin)\n",
    "ztf_source = lsdb.read_hipscat(ZTF_SOURCE_PATH, columns=[\"objra\", \"objdec\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3b62fda-0aa5-4d1e-9a20-09775741fa03",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Plot the Pixel Density Maps for Gaia and ZTF\n",
    "\n",
    "plot_pixels(gaia.hc_structure)\n",
    "plot_pixels(ztf.hc_structure)\n",
    "plot_pixels(ztf_source.hc_structure)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8df8847f-5325-447e-afaa-095786b63799",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5a0d17d-5452-4f38-ae21-d3d893854965",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Cross-match Gaia and ZTF\n",
    "# Observe this is only lazy loaded\n",
    "xmatch_object = gaia_lite.crossmatch(ztf)\n",
    "xmatch_object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1db905e9-3cf6-4841-8af3-ad5ffab5ba1d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Load into memory, the first partition\n",
    "xmatch_object.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3521ed83-c182-414f-b947-9cb75b6c277e",
   "metadata": {},
   "source": [
    "We can maintain an \"almanac\" of known catalogs. In this way, we can find new catalogs, or refer to them with simpler names."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c07e1482-d890-44a0-b68c-88bda2384621",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "almanac = Almanac(dirs=\"/home/jovyan/efs/lincc/almanac\")\n",
    "almanac.catalogs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "788d51a3-531f-4dbc-9df1-d956b9ec6ce1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# gaia_hc = almanac.get_catalog(\"gaia\")\n",
    "# gaia = lsdb.read_hipscat(gaia_hc.catalog_base_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9eb2226-b70d-4e67-99c6-45cc1b5487c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "ztf.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bb6fc80-e676-4617-a372-121292fbf7a1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ztf.head(1).columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ec506e1-2fe3-4fd8-ae9f-3b5de80599b6",
   "metadata": {},
   "source": [
    "You'll need to set the suffixes of the resulting join table, so you can query using those column names.\n",
    "\n",
    "```\n",
    "... , suffixes=(\"_gaia\", \"_ztf\") ...\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07c0dbb8-95e4-4e56-b591-6d57a5c8676b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# crossmatch ZTF + Gaia\n",
    "_all_sky_object = gaia_cone.crossmatch(ztf, suffixes=(\"_gaia\", \"_ztf\")).query(\n",
    "    \"parallax_gaia > 0 and parallax_over_error_gaia > 5 and \\\n",
    "    teff_gspphot_gaia > 5380 and teff_gspphot_gaia < 7220 and logg_gspphot_gaia > 4.5 \\\n",
    "    and logg_gspphot_gaia < 4.72 and classprob_dsc_combmod_star_gaia > 0.5\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "017c0397-d493-42ff-b2e5-e1dc3ab9bd73",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "all_sky_object = _all_sky_object.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db997b17-1273-4ee2-b3bd-c46f411cf811",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_sky_object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa02b8e1-1a64-4919-ba55-3b32b49ae93e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot the distribution\n",
    "# p"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "root *",
   "language": "python",
   "name": "conda-root-py"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}