notebooks/erl_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook allows you to compare the performance of segmentations generated from new affinity networks against the old one trained on JWR dataset. It assumes that the error analysis modules in the ecerebellum package have been run and results have been saved in the respective folders."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_chunks = range(12,16) # IDs of validation chunks\n",
    "train_chunks = range(0,12) # IDs of training chunks\n",
    "block_size = 60 # number of slices per chunk\n",
    "resdir = '/mnt/coxfs01/donglai/srujan/err-analysis/' # base directory with error analysis results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_erl_stats(resdir, seg_names, block_size, aff_offset):\n",
    "    \"\"\"\n",
    "    Reads err-analysis results and evaluates mean and std ERL across many chunks\n",
    "    \"\"\"\n",
    "    vi = []\n",
    "    erl_gt = []\n",
    "    erl_pred = []\n",
    "    for i in range(len(seg_names)):\n",
    "        print \"On evaluated chunk: %d\"%i\n",
    "        base_seg = seg_names[i]\n",
    "        vi_dict = json.load(open(resdir+base_seg+\"/vi.json\"))\n",
    "        vi.append((vi_dict[\"VI split\"], vi_dict[\"VI merge\"]))\n",
    "        print \"VI: \", vi[i]\n",
    "        erl_dict = json.load(open(resdir+base_seg+\"/skeleton-analysis-summary.json\"))\n",
    "        erl_gt.append(erl_dict[\"results\"][\"erl-gt\"])\n",
    "        print \"GT ERL in nm: %d\"%(erl_gt[i])\n",
    "        erl_pred.append(erl_dict[\"results\"][\"erl-pred\"])\n",
    "        print \"Predicted ERL in nm: %d\"%erl_pred[i]\n",
    "    erl_gt_mean = np.mean(erl_gt)\n",
    "    std_gt_mean = np.std(erl_gt)\n",
    "    erl_pred_mean = np.mean(erl_pred)\n",
    "    std_pred_mean = np.std(erl_pred)\n",
    "    print \"Mean GT ERL in nm: %d +/- %d\"%(erl_gt_mean, std_gt_mean)\n",
    "    print \"Mean pred ERL in nm: %d +/- %d\"%(erl_pred_mean, std_pred_mean)\n",
    "    return erl_pred_mean, std_pred_mean#, np.sum(erl_gt), np.sum(erl_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## JWR model on current test chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On evaluated chunk: 0\n",
      "VI:  (0.6249742436145521, 0.2612395624648516)\n",
      "GT ERL in nm: 3303\n",
      "Predicted ERL in nm: 1543\n",
      "On evaluated chunk: 1\n",
      "VI:  (0.48167293075122775, 0.2046038754984993)\n",
      "GT ERL in nm: 2768\n",
      "Predicted ERL in nm: 1078\n",
      "On evaluated chunk: 2\n",
      "VI:  (0.6155921005704638, 0.3002865526185817)\n",
      "GT ERL in nm: 2789\n",
      "Predicted ERL in nm: 871\n",
      "On evaluated chunk: 3\n",
      "VI:  (0.2886886095581719, 0.062055236428578985)\n",
      "GT ERL in nm: 2376\n",
      "Predicted ERL in nm: 1479\n",
      "Mean GT ERL in nm: 2809 +/- 329\n",
      "Mean pred ERL in nm: 1243 +/- 279\n"
     ]
    }
   ],
   "source": [
    "wz_thresh = 0.5\n",
    "aff_offset = 14\n",
    "zz = [chunk_id*block_size + aff_offset for chunk_id in eval_chunks]\n",
    "seg_names = [\"waterz%.2f-48nm-crop2gt-%04d\"%(wz_thresh, zz[i]) for i in range(len(eval_chunks))]\n",
    "erl_jwr_test, std_jwr_test = get_erl_stats(resdir,seg_names, block_size, aff_offset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## JWR model on current training chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On evaluated chunk: 0\n",
      "VI:  (0.08441740089257588, 0.007626745858172034)\n",
      "GT ERL in nm: 2005\n",
      "Predicted ERL in nm: 1661\n",
      "On evaluated chunk: 1\n",
      "VI:  (0.11385432584697436, 0.009933146890894319)\n",
      "GT ERL in nm: 2299\n",
      "Predicted ERL in nm: 1693\n",
      "On evaluated chunk: 2\n",
      "VI:  (0.3095177768012949, 0.11856814009009421)\n",
      "GT ERL in nm: 2877\n",
      "Predicted ERL in nm: 1450\n",
      "On evaluated chunk: 3\n",
      "VI:  (0.35223887775989937, 0.13939526327023172)\n",
      "GT ERL in nm: 3227\n",
      "Predicted ERL in nm: 1406\n",
      "On evaluated chunk: 4\n",
      "VI:  (0.6920218043220697, 0.10305426689668407)\n",
      "GT ERL in nm: 2976\n",
      "Predicted ERL in nm: 778\n",
      "On evaluated chunk: 5\n",
      "VI:  (0.6140835097411244, 0.21371108306675823)\n",
      "GT ERL in nm: 4032\n",
      "Predicted ERL in nm: 1460\n",
      "On evaluated chunk: 6\n",
      "VI:  (0.3425577640998605, 0.06824590574256541)\n",
      "GT ERL in nm: 2417\n",
      "Predicted ERL in nm: 1164\n",
      "On evaluated chunk: 7\n",
      "VI:  (0.2627277007388039, 0.05366486375807491)\n",
      "GT ERL in nm: 2078\n",
      "Predicted ERL in nm: 1645\n",
      "On evaluated chunk: 8\n",
      "VI:  (0.3230931044722458, 0.03600218501039468)\n",
      "GT ERL in nm: 1964\n",
      "Predicted ERL in nm: 1454\n",
      "On evaluated chunk: 9\n",
      "VI:  (0.13433709906287752, 0.02703319402139094)\n",
      "GT ERL in nm: 1902\n",
      "Predicted ERL in nm: 1588\n",
      "On evaluated chunk: 10\n",
      "VI:  (0.22497567413230213, 0.03223503796167928)\n",
      "GT ERL in nm: 2516\n",
      "Predicted ERL in nm: 1983\n",
      "On evaluated chunk: 11\n",
      "VI:  (0.35698573968985, 0.2196112145279527)\n",
      "GT ERL in nm: 2963\n",
      "Predicted ERL in nm: 1788\n",
      "Mean GT ERL in nm: 2605 +/- 608\n",
      "Mean pred ERL in nm: 1506 +/- 296\n"
     ]
    }
   ],
   "source": [
    "wz_thresh = 0.5\n",
    "aff_offset = 14\n",
    "zz = [chunk_id*block_size + aff_offset for chunk_id in train_chunks]\n",
    "seg_names = [\"waterz%.2f-48nm-crop2gt-%04d\"%(wz_thresh, zz[i]) for i in range(len(train_chunks))]\n",
    "erl_jwr_train, std_jwr_train = get_erl_stats(resdir,seg_names, block_size, aff_offset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## UnetPNI3DM2 - 11K epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On evaluated chunk: 0\n",
      "VI:  (0.6334221632069712, 0.4837357038319403)\n",
      "GT ERL in nm: 3303\n",
      "Predicted ERL in nm: 1474\n",
      "On evaluated chunk: 1\n",
      "VI:  (1.1411665942507934, 0.9876153705385816)\n",
      "GT ERL in nm: 2768\n",
      "Predicted ERL in nm: 781\n",
      "On evaluated chunk: 2\n",
      "VI:  (0.6832663498113299, 0.35468840385482203)\n",
      "GT ERL in nm: 2789\n",
      "Predicted ERL in nm: 1004\n",
      "On evaluated chunk: 3\n",
      "VI:  (0.24291393107932652, 0.06120300584460292)\n",
      "GT ERL in nm: 2376\n",
      "Predicted ERL in nm: 1635\n",
      "Mean GT ERL in nm: 2809 +/- 329\n",
      "Mean pred ERL in nm: 1223 +/- 345\n"
     ]
    }
   ],
   "source": [
    "seg_type = 'wtdil2'\n",
    "wz_thresh = 0.5\n",
    "zz = [chunk_id*block_size for chunk_id in eval_chunks]\n",
    "seg_names = [\"retrained-\"+seg_type+\"-%04d-wz%.2f\"%(zz[i], wz_thresh) for i in range(len(eval_chunks))]\n",
    "erl_jwr, std_jwr = get_erl_stats(resdir,seg_names, block_size, aff_offset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unet3DPniM2 - 49K epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On evaluated chunk: 0\n",
      "VI:  (0.4562201154182672, 0.9971963062090315)\n",
      "GT ERL in nm: 3303\n",
      "Predicted ERL in nm: 1210\n",
      "On evaluated chunk: 1\n",
      "VI:  (0.4221393722892554, 0.8698675382109224)\n",
      "GT ERL in nm: 2768\n",
      "Predicted ERL in nm: 806\n",
      "On evaluated chunk: 2\n",
      "VI:  (0.5432551474648435, 0.5770045089681198)\n",
      "GT ERL in nm: 2789\n",
      "Predicted ERL in nm: 873\n",
      "On evaluated chunk: 3\n",
      "VI:  (0.1993912599505692, 0.0928270150688557)\n",
      "GT ERL in nm: 2376\n",
      "Predicted ERL in nm: 1605\n",
      "Mean GT ERL in nm: 2809 +/- 329\n",
      "Mean pred ERL in nm: 1124 +/- 317\n"
     ]
    }
   ],
   "source": [
    "seg_type = '49Ktrain'\n",
    "wz_thresh = 0.5\n",
    "zz = [chunk_id*block_size for chunk_id in eval_chunks]\n",
    "seg_names = [\"retrained-\"+seg_type+\"-%04d-wz%.2f\"%(zz[i], wz_thresh) for i in range(len(eval_chunks))]\n",
    "erl_jwr, std_jwr = get_erl_stats(resdir,seg_names, block_size, aff_offset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## UnetFiber - 12K epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "On evaluated chunk: 0\n",
      "VI:  (0.4896863462445564, 0.8585366370428827)\n",
      "GT ERL in nm: 3303\n",
      "Predicted ERL in nm: 1247\n",
      "On evaluated chunk: 1\n",
      "VI:  (0.7305311816485477, 0.8552270099191793)\n",
      "GT ERL in nm: 2768\n",
      "Predicted ERL in nm: 763\n",
      "On evaluated chunk: 2\n",
      "VI:  (0.6323407030539118, 0.8016363419502773)\n",
      "GT ERL in nm: 2789\n",
      "Predicted ERL in nm: 753\n",
      "On evaluated chunk: 3\n",
      "VI:  (0.2374998483233842, 0.3080794344300453)\n",
      "GT ERL in nm: 2376\n",
      "Predicted ERL in nm: 1310\n",
      "Mean GT ERL in nm: 2809 +/- 329\n",
      "Mean pred ERL in nm: 1018 +/- 261\n"
     ]
    }
   ],
   "source": [
    "seg_type = '12Ktrain'\n",
    "wz_thresh = 0.5\n",
    "zz = [chunk_id*block_size for chunk_id in eval_chunks]\n",
    "seg_names = [\"retrained-\"+seg_type+\"-%04d-wz%.2f\"%(zz[i], wz_thresh) for i in range(len(eval_chunks))]\n",
    "erl_jwr, std_jwr = get_erl_stats(resdir,seg_names, block_size, aff_offset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}