TimeLens.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TimeLens.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMnd5T7fQS8WahELgHmI7KI",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/uzh-rpg/rpg_timelens/blob/main/TimeLens.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s4PPH-NTx3dF"
      },
      "source": [
        "# TimeLens: Event-based Video Frame Interpolation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9PnK9tknyDza"
      },
      "source": [
        "## Installation\n",
        "\n",
        "First clone the repo and install an environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kg_xBex5yJZx",
        "outputId": "94062ba2-efdf-403d-de5e-d979c835012e"
      },
      "source": [
        "# Check your current GPU\n",
        "# If you are lucky, you get 16GB VRAM. If you are not lucky, you get less. VRAM is important. The more VRAM, the higher the maximum resolution will go\n",
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Wed Jun 16 13:00:22 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.27       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   59C    P8    10W /  70W |      0MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XUtaNkg0GEt4"
      },
      "source": [
        "# Connect Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CTGU8anrzVh-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10d6e7e3-52d0-4622-dda6-81f9756268a1"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive', force_remount=True)\n",
        "print('Google Drive connected.')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n",
            "Google Drive connected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lQ_4dtx9Fb1v",
        "outputId": "e0456561-1db3-4bf4-99ad-e4b54e16dfb6"
      },
      "source": [
        "!ls /content/gdrive"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MyDrive  Shareddrives\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bNvjLMgM1siz"
      },
      "source": [
        "# Install Dependencies and Repo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qSmCBWoy1l_E",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "da95724a-3ccf-40ca-9b1b-253de0fdedcd"
      },
      "source": [
        "%cd /content\n",
        "!git clone https://github.com/uzh-rpg/rpg_timelens\n",
        "!wget http://rpg.ifi.uzh.ch/timelens/data2/checkpoint.bin\n",
        "!CUDA_VISIBLE_DEVICES=0"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "Cloning into 'rpg_timelens'...\n",
            "remote: Enumerating objects: 88, done.\u001b[K\n",
            "remote: Counting objects: 100% (88/88), done.\u001b[K\n",
            "remote: Compressing objects: 100% (66/66), done.\u001b[K\n",
            "remote: Total 88 (delta 19), reused 79 (delta 17), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (88/88), done.\n",
            "--2021-06-16 14:18:02--  http://rpg.ifi.uzh.ch/timelens/data2/checkpoint.bin\n",
            "Resolving rpg.ifi.uzh.ch (rpg.ifi.uzh.ch)... 130.60.155.125\n",
            "Connecting to rpg.ifi.uzh.ch (rpg.ifi.uzh.ch)|130.60.155.125|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 475379363 (453M) [application/octet-stream]\n",
            "Saving to: ‘checkpoint.bin’\n",
            "\n",
            "checkpoint.bin      100%[===================>] 453.36M   101MB/s    in 4.6s    \n",
            "\n",
            "2021-06-16 14:18:07 (98.5 MB/s) - ‘checkpoint.bin’ saved [475379363/475379363]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QuRNCO7Z2bAn"
      },
      "source": [
        "################# Required Configurations ############################\n",
        "\n",
        "#@markdown # Required Configuration\n",
        "#@markdown Use the values in here to configure what you'd like TimeLens to do. If you do not have your own data, just leave the values below and it will download an example.\n",
        "\n",
        "#@markdown ## Output file\n",
        "#@markdown Output file path: path (relative to the root of your Google Drive) for the output file. It will also determine the filetype in the destination. `.mp4` is recommended for video input, `.gif` for gif inputs.\n",
        "OUTPUT_FILE_PATH = \"/content/rpg_timelens/out.mp4\" #@param{type:\"string\"}\n",
        "\n",
        "################# Optional configurations ############################\n",
        "\n",
        "#@markdown # Optional Configuration\n",
        "#@markdown Parameters below can be left with their defaults, but feel free to adapt them to your needs.\n",
        "\n",
        "#@markdown ## Target FPS\n",
        "#@markdown  how many frames per second should the result have. This will determine how many intermediate images are interpolated.\n",
        "TARGET_FPS =  960#@param{type:\"number\"}\n",
        "\n",
        "#@markdown ## Frame input directory\n",
        "#@markdown A path, relative to your GDrive root, where you already have the list of frames in the format 00001.png, 00002.png, etc.\n",
        "FRAME_INPUT_DIR = '/content/rpg_timelens/input_frames' #@param{type:\"string\"}\n",
        "EVENTS_INPUT_DIR = '/content/rpg_timelens/input_events' #@param{type:\"string\"}\n",
        "\n",
        "#@markdown ## Frame output directory\n",
        "#@markdown A path, relative to your GDrive root, where you want the generated frame.\n",
        "FRAME_OUTPUT_DIR = '/content/rpg_timelens/output_frames' #@param{type:\"string\"}\n"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fZJHZ2T4Hdgu"
      },
      "source": [
        "# Set Up Example\n",
        "If you do not have data to process, just leave the default values in the form above and the cell below will download an example."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "418W2cPS2fla"
      },
      "source": [
        "# Example \n",
        "%%bash\n",
        "wget http://rpg.ifi.uzh.ch/timelens/data2/example_github.zip -O /content/example_github.zip\n",
        "unzip /content/example_github.zip -d /content/\n",
        "rm -rf /content/example_github.zip\n",
        "mv /content/example/events /content/rpg_timelens/input_events\n",
        "mv /content/example/images /content/rpg_timelens/input_frames\n",
        "mkdir /content/rpg_timelens/output_frames"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R_e_NEoO9Erq",
        "outputId": "a996f19d-88d9-465a-b247-ff194d8f7c34"
      },
      "source": [
        "# find the fps of the input images \n",
        "import numpy as np\n",
        "from os.path import join\n",
        "\n",
        "timestamps = np.genfromtxt(join(FRAME_INPUT_DIR, \"timestamp.txt\"))\n",
        "fps = int(1000000.0 / np.diff(timestamps).mean())\n",
        "factor = int(TARGET_FPS / fps)\n",
        "inserts = factor-1\n",
        "print(\"Upsampling from \", fps, \" to \", TARGET_FPS, \" with \", inserts , \" inserts.\")\n",
        "\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Upsampling from  159  to  960  with  5  inserts.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZBWMSweHSKUv"
      },
      "source": [
        "# Run TimeLens"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7NisgZgq-jLN",
        "outputId": "c1864805-1ea0-4296-d181-93e428001017"
      },
      "source": [
        "%cd /content/rpg_timelens/\n",
        "!mkdir $FRAME_OUTPUT_DIR\n",
        "!python -m timelens.run_timelens ../checkpoint.bin $EVENTS_INPUT_DIR $FRAME_INPUT_DIR $FRAME_OUTPUT_DIR 0 $inserts"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/rpg_timelens\n",
            "mkdir: cannot create directory ‘/content/rpg_timelens/output_frames’: File exists\n",
            "Processing .\n",
            "100% 50/50 [00:01<00:00, 43.86it/s]\n",
            "100% 50/50 [00:00<00:00, 150.52it/s]\n",
            "Processing /content/rpg_timelens/output_frames/.\n",
            "Counter: 0000\n",
            "Events left:  6153 Events right:  31367\n",
            "/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py:3458: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
            "  \"See the documentation of nn.Upsample for details.\".format(mode)\n",
            "Events left:  12482 Events right:  25038\n",
            "Events left:  18783 Events right:  18737\n",
            "Events left:  25037 Events right:  12483\n",
            "Events left:  31292 Events right:  6228\n",
            "Counter: 0006\n",
            "Events left:  6314 Events right:  30593\n",
            "Events left:  12529 Events right:  24378\n",
            "Events left:  18630 Events right:  18277\n",
            "Events left:  24734 Events right:  12173\n",
            "Events left:  30844 Events right:  6063\n",
            "Counter: 0012\n",
            "Events left:  6036 Events right:  29509\n",
            "Events left:  11936 Events right:  23609\n",
            "Events left:  17774 Events right:  17771\n",
            "Events left:  23759 Events right:  11786\n",
            "Events left:  29675 Events right:  5870\n",
            "Counter: 0018\n",
            "Events left:  6037 Events right:  29217\n",
            "Events left:  11927 Events right:  23327\n",
            "Events left:  17704 Events right:  17550\n",
            "Events left:  23748 Events right:  11506\n",
            "Events left:  29619 Events right:  5635\n",
            "Counter: 0024\n",
            "Events left:  5840 Events right:  28583\n",
            "Events left:  11817 Events right:  22606\n",
            "Events left:  17390 Events right:  17033\n",
            "Events left:  23055 Events right:  11368\n",
            "Events left:  28603 Events right:  5820\n",
            "Counter: 0030\n",
            "Events left:  5614 Events right:  28011\n",
            "Events left:  11336 Events right:  22289\n",
            "Events left:  16927 Events right:  16698\n",
            "Events left:  22649 Events right:  10976\n",
            "Events left:  28163 Events right:  5462\n",
            "Counter: 0036\n",
            "Events left:  5790 Events right:  28489\n",
            "Events left:  11533 Events right:  22746\n",
            "Events left:  17404 Events right:  16875\n",
            "Events left:  23228 Events right:  11051\n",
            "Events left:  28803 Events right:  5476\n",
            "Counter: 0042\n",
            "Events left:  5607 Events right:  26338\n",
            "Events left:  10939 Events right:  21006\n",
            "Events left:  16309 Events right:  15636\n",
            "Events left:  21586 Events right:  10359\n",
            "Events left:  26760 Events right:  5185\n",
            "Counter: 0048\n",
            "Events left:  5066 Events right:  26813\n",
            "Events left:  10347 Events right:  21532\n",
            "Events left:  15462 Events right:  16417\n",
            "Events left:  20430 Events right:  11449\n",
            "Events left:  26128 Events right:  5751\n",
            "Counter: 0054\n",
            "Events left:  5782 Events right:  31598\n",
            "Events left:  12054 Events right:  25326\n",
            "Events left:  18686 Events right:  18694\n",
            "Events left:  25014 Events right:  12366\n",
            "Events left:  31148 Events right:  6232\n",
            "Counter: 0060\n",
            "Events left:  6369 Events right:  31824\n",
            "Events left:  12589 Events right:  25604\n",
            "Events left:  18742 Events right:  19451\n",
            "Events left:  25214 Events right:  12979\n",
            "Events left:  31616 Events right:  6577\n",
            "Counter: 0066\n",
            "Events left:  6573 Events right:  34470\n",
            "Events left:  13158 Events right:  27885\n",
            "Events left:  20103 Events right:  20940\n",
            "Events left:  26950 Events right:  14093\n",
            "Events left:  33803 Events right:  7240\n",
            "Counter: 0072\n",
            "Events left:  7302 Events right:  38905\n",
            "Events left:  14838 Events right:  31369\n",
            "Events left:  22427 Events right:  23780\n",
            "Events left:  30292 Events right:  15915\n",
            "Events left:  38159 Events right:  8048\n",
            "Counter: 0078\n",
            "Events left:  8348 Events right:  42862\n",
            "Events left:  16750 Events right:  34460\n",
            "Events left:  25144 Events right:  26066\n",
            "Events left:  33699 Events right:  17511\n",
            "Events left:  42488 Events right:  8722\n",
            "Counter: 0084\n",
            "Events left:  9155 Events right:  47252\n",
            "Events left:  18287 Events right:  38120\n",
            "Events left:  27597 Events right:  28810\n",
            "Events left:  37125 Events right:  19282\n",
            "Events left:  46737 Events right:  9670\n",
            "Counter: 0090\n",
            "Events left:  9771 Events right:  49642\n",
            "Events left:  19494 Events right:  39919\n",
            "Events left:  29258 Events right:  30155\n",
            "Events left:  39030 Events right:  20383\n",
            "Events left:  49129 Events right:  10284\n",
            "Counter: 0096\n",
            "Events left:  10422 Events right:  54613\n",
            "Events left:  21001 Events right:  44034\n",
            "Events left:  31897 Events right:  33138\n",
            "Events left:  42836 Events right:  22199\n",
            "Events left:  53989 Events right:  11046\n",
            "Counter: 0102\n",
            "Events left:  11163 Events right:  58752\n",
            "Events left:  22718 Events right:  47197\n",
            "Events left:  34274 Events right:  35641\n",
            "Events left:  46036 Events right:  23879\n",
            "Events left:  58094 Events right:  11821\n",
            "Counter: 0108\n",
            "Events left:  11956 Events right:  61746\n",
            "Events left:  24085 Events right:  49617\n",
            "Events left:  36264 Events right:  37438\n",
            "Events left:  48605 Events right:  25097\n",
            "Events left:  61019 Events right:  12683\n",
            "Counter: 0114\n",
            "Events left:  12725 Events right:  65593\n",
            "Events left:  25702 Events right:  52616\n",
            "Events left:  38557 Events right:  39761\n",
            "Events left:  51580 Events right:  26738\n",
            "Events left:  64780 Events right:  13538\n",
            "Counter: 0120\n",
            "Events left:  13453 Events right:  71853\n",
            "Events left:  27358 Events right:  57948\n",
            "Events left:  41460 Events right:  43846\n",
            "Events left:  55898 Events right:  29408\n",
            "\n",
            "Aborted!\n",
            "^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MEZEm8rq-uvV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        },
        "outputId": "30fce0d9-1abb-4c5a-abb6-23118b247c56"
      },
      "source": [
        "!ffmpeg -i \"$FRAME_OUTPUT_DIR\"/%06d.png timelens.mp4\n",
        "from google.colab import files\n",
        "files.download(\"timelens.mp4\")"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ffmpeg version 3.4.8-0ubuntu0.2 Copyright (c) 2000-2020 the FFmpeg developers\n",
            "  built with gcc 7 (Ubuntu 7.5.0-3ubuntu1~18.04)\n",
            "  configuration: --prefix=/usr --extra-version=0ubuntu0.2 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --enable-gpl --disable-stripping --enable-avresample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librubberband --enable-librsvg --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-omx --enable-openal --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libopencv --enable-libx264 --enable-shared\n",
            "  libavutil      55. 78.100 / 55. 78.100\n",
            "  libavcodec     57.107.100 / 57.107.100\n",
            "  libavformat    57. 83.100 / 57. 83.100\n",
            "  libavdevice    57. 10.100 / 57. 10.100\n",
            "  libavfilter     6.107.100 /  6.107.100\n",
            "  libavresample   3.  7.  0 /  3.  7.  0\n",
            "  libswscale      4.  8.100 /  4.  8.100\n",
            "  libswresample   2.  9.100 /  2.  9.100\n",
            "  libpostproc    54.  7.100 / 54.  7.100\n",
            "Input #0, image2, from '/content/rpg_timelens/output_frames/%06d.png':\n",
            "  Duration: 00:00:04.96, start: 0.000000, bitrate: N/A\n",
            "    Stream #0:0: Video: png, rgb24(pc), 957x856, 25 fps, 25 tbr, 25 tbn, 25 tbc\n",
            "File 'timelens.mp4' already exists. Overwrite ? [y/N] N\n",
            "\u001b[4;31mNot overwriting - exiting\n",
            "\u001b[0m"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_f7568802-e5dc-4b94-9ac9-e02e03a08112\", \"timelens.mp4\", 587110)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P2pHPnBgNTuh"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}