RemoveSaturation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7361580-ae83-448e-b092-8bb032023539",
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "from glob import glob\n",
    "import numpy as np\n",
    "\n",
    "from os.path import join\n",
    "\n",
    "import os\n",
    "\n",
    "def safe_mkdir(path):\n",
    "    \"\"\" Create a directory if there isn't one already. \"\"\"\n",
    "    try:\n",
    "        os.makedirs(path)\n",
    "    except OSError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a661486c-f40b-47fb-8018-9aef516cb5e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_crop = glob(\"hdr-dsp-real-dataset/crop/*\")\n",
    "dir_crop.sort()\n",
    "\n",
    "dir_ratio = glob(\"hdr-dsp-real-dataset/ratios/*\")\n",
    "dir_ratio.sort()\n",
    "\n",
    "dir_sat = glob(\"hdr-dsp-real-dataset/satmask/*\")\n",
    "dir_sat.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dcd7031-aed3-4dc2-9457-a3c1d2f30875",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dir_crop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "659e59e7-793d-43ff-b599-001d7642c059",
   "metadata": {},
   "outputs": [],
   "source": [
    "ssc13index = []\n",
    "for i,crop in enumerate(dir_crop):\n",
    "    if \"ssc13\" in crop:\n",
    "        ssc13index.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bedf34e-32aa-458d-aa24-045eae394414",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(ssc13index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e30cf77-eae6-4368-9e2e-8943052eb135",
   "metadata": {},
   "outputs": [],
   "source": [
    "TrainData = {}\n",
    "TrainRatios = {}\n",
    "\n",
    "ValData = {}\n",
    "ValRatios = {}\n",
    "\n",
    "TestData = {}\n",
    "TestRatios = {}\n",
    "\n",
    "for i in range(3, 16):\n",
    "    TrainData[str(i)] = []\n",
    "    TrainRatios[str(i)] = []\n",
    "    \n",
    "    ValData[str(i)] = []\n",
    "    ValRatios[str(i)] = []\n",
    "    \n",
    "    TestData[str(i)] = []\n",
    "    TestRatios[str(i)] = []\n",
    "\n",
    "    \n",
    "    \n",
    "for i in range(2500):\n",
    "    data = np.load(dir_crop[i])\n",
    "    ratio = np.load(dir_ratio[i])\n",
    "    sat = np.load(dir_sat[i])\n",
    "    \n",
    "    nbFrames = len(ratio)\n",
    "    \n",
    "    nonSatIndex = list(range(nbFrames))\n",
    "    \n",
    "    if i not in ssc13index:\n",
    "        for j in range(nbFrames):\n",
    "            if False in sat[j]:\n",
    "                nonSatIndex.remove(j)\n",
    "    else:\n",
    "        for j in range(nbFrames):\n",
    "            if data[j].max() > 2600:\n",
    "                nonSatIndex.remove(j)\n",
    "                \n",
    "    if len(nonSatIndex) >= 3:\n",
    "        if i < 2000:\n",
    "            TrainData[str(len(nonSatIndex))].append(data[nonSatIndex])\n",
    "            TrainRatios[str(len(nonSatIndex))].append(ratio[nonSatIndex])\n",
    "        elif i <2200:\n",
    "            ValData[str(len(nonSatIndex))].append(data[nonSatIndex])\n",
    "            ValRatios[str(len(nonSatIndex))].append(ratio[nonSatIndex])\n",
    "        else:\n",
    "            TestData[str(len(nonSatIndex))].append(data[nonSatIndex])\n",
    "            TestRatios[str(len(nonSatIndex))].append(ratio[nonSatIndex])\n",
    "\n",
    "\n",
    "for j in range(3, 16):\n",
    "    if len(TrainRatios[str(j)]) > 0:\n",
    "        folder = \"SkySat_ME_noSaturation/train/{}\".format(j)\n",
    "        safe_mkdir(folder)\n",
    "        np.save(join(folder, \"trainLR.npy\"), np.array(TrainData[str(j)]))\n",
    "        np.save(join(folder, \"trainRatio.npy\"), np.array(TrainRatios[str(j)]))\n",
    "\n",
    "    if len(ValRatios[str(j)]) > 0:\n",
    "        folder = \"SkySat_ME_noSaturation/val/{}\".format(j)\n",
    "        safe_mkdir(folder)\n",
    "        np.save(join(folder, \"valLR.npy\"), np.array(ValData[str(j)]))\n",
    "        np.save(join(folder, \"valRatio.npy\"), np.array(ValRatios[str(j)]))\n",
    "        \n",
    "    if len(TestRatios[str(j)]) > 0:\n",
    "        folder = \"SkySat_ME_noSaturation/test/{}\".format(j)\n",
    "        safe_mkdir(folder)\n",
    "        np.save(join(folder, \"testLR.npy\"), np.array(TestData[str(j)]))\n",
    "        np.save(join(folder, \"testRatio.npy\"), np.array(TestRatios[str(j)]))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2210f4f8-bda4-46b3-8d46-0b0d891796a3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "HDR-DSP",
   "language": "python",
   "name": "hdr-dsp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}