catafest_0059.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/catafest/colab_google/blob/master/catafest_0059.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "56sK2HevvVlb"
      },
      "source": [
        "If you want to use next cell then you need to set runtime to T4 GPU.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3OSB-zNeuTjN",
        "outputId": "b26dbf98-9cd4-4c4f-b0be-878d2b3cbd91"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content\n",
            "Current GPU: Tesla T4\n"
          ]
        }
      ],
      "source": [
        "%cd /content\n",
        "\n",
        "gpu = !nvidia-smi --query-gpu=gpu_name --format=csv,noheader\n",
        "print(\"Current GPU: \" + gpu[0])\n",
        "\n",
        "if gpu[0] == \"Tesla K80\" and optix_enabled:\n",
        "  print(\"OptiX disabled because of unsupported GPU\")\n",
        "  optix_enabled = False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "id": "9nAPj8A3Xe0i",
        "outputId": "ddbceb54-61bd-425b-d34d-9b958100a400"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<table>\n",
              "    <tr>\n",
              "        <th colspan=\"2\">Memory Information</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "        <th>Total</th>\n",
              "        <td>12.67GB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "        <th>Available</th>\n",
              "        <td>10.86GB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "        <th>Used</th>\n",
              "        <td>1.50GB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "        <th>Percentage</th>\n",
              "        <td>14.3%</td>\n",
              "    </tr>\n",
              "</table>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# @title Memory Information\n",
        "\n",
        "import psutil\n",
        "from IPython.display import HTML, display\n",
        "\n",
        "def get_size(bytes, suffix=\"B\"):\n",
        "    factor = 1024\n",
        "    for unit in [\"\", \"K\", \"M\", \"G\", \"T\", \"P\"]:\n",
        "        if bytes < factor:\n",
        "            return f\"{bytes:.2f}{unit}{suffix}\"\n",
        "        bytes /= factor\n",
        "\n",
        "# Retrieve memory information\n",
        "svmem = psutil.virtual_memory()\n",
        "total = get_size(svmem.total)\n",
        "available = get_size(svmem.available)\n",
        "used = get_size(svmem.used)\n",
        "percentage = svmem.percent\n",
        "\n",
        "# Create an HTML table with memory information (Rounded Corners variant with dark theme)\n",
        "table_content = f\"\"\"\n",
        "<table>\n",
        "    <tr>\n",
        "        <th colspan=\"2\">Memory Information</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <th>Total</th>\n",
        "        <td>{total}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <th>Available</th>\n",
        "        <td>{available}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <th>Used</th>\n",
        "        <td>{used}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <th>Percentage</th>\n",
        "        <td>{percentage}%</td>\n",
        "    </tr>\n",
        "</table>\n",
        "\"\"\"\n",
        "\n",
        "# Display the table\n",
        "display(HTML(table_content))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHN3lDrS88FT"
      },
      "source": [
        "Let's start"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jw8y4OQt8-0G"
      },
      "outputs": [],
      "source": [
        "#@title\n",
        "import os\n",
        "from os.path import exists, join, basename, splitext\n",
        "\n",
        "git_repo_url = 'https://github.com/NVIDIA/tacotron2.git'\n",
        "project_name = splitext(basename(git_repo_url))[0]\n",
        "if not exists(project_name):\n",
        "  # clone and install\n",
        "  !git clone -q --recursive {git_repo_url}\n",
        "  !cd {project_name}/waveglow && git checkout 9168aea\n",
        "  !pip install -q librosa unidecode\n",
        "  !pip install -q --upgrade gdown\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_yIDnl9-BL3V"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "sys.path.append(join(project_name, 'waveglow/'))\n",
        "sys.path.append(project_name)\n",
        "import time\n",
        "import gdown\n",
        "import matplotlib\n",
        "import matplotlib.pylab as plt\n",
        "plt.rcParams[\"axes.grid\"] = False\n",
        "#@title\n",
        "tacotron2_pretrained_model = 'tacotron2_statedict.pt'\n",
        "if not exists(tacotron2_pretrained_model):\n",
        "  # download the Tacotron2 pretrained model\n",
        "  gdown.download('https://drive.google.com/uc?id=1c5ZTuT7J08wLUoVZ2KkUs_VdZuJ86ZqA', tacotron2_pretrained_model, quiet=False)\n",
        "waveglow_pretrained_model = 'waveglow_old.pt'\n",
        "if not exists(waveglow_pretrained_model):\n",
        "  # download the Waveglow pretrained model\n",
        "  gdown.download('https://drive.google.com/uc?id=1WsibBTsuRg_SF2Z6L6NFRTT-NjEy1oTx', waveglow_pretrained_model, quiet=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 76,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PuzHpeGMAPbQ",
        "outputId": "ba593189-fd6d-4d68-b10e-d85e50cb0dd6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-03-30 20:36:11.334420: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-03-30 20:36:11.334480: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-03-30 20:36:11.335826: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-03-30 20:36:12.772044: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Output directory '/content/tacotron2_v2' must not already exist.\n"
          ]
        }
      ],
      "source": [
        "\n",
        "!tf_upgrade_v2 --intree \"/content/tacotron2\" --outtree \"/content/tacotron2_v2\" --reportfile \"/content/report.txt\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 78,
      "metadata": {
        "id": "WDTFwDzc_3LX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "500732ea-9e9e-45f4-cec2-f428e271563f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch.autograd.grad_mode.set_grad_enabled at 0x7b3ebe417d00>"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ],
      "source": [
        "#@title\n",
        "import IPython.display as ipd\n",
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "from model import Tacotron2\n",
        "from layers import TacotronSTFT\n",
        "from audio_processing import griffin_lim\n",
        "from text import text_to_sequence\n",
        "from denoiser import Denoiser\n",
        "\n",
        "def plot_data(data, figsize=(16, 4)):\n",
        "    fig, axes = plt.subplots(1, len(data), figsize=figsize)\n",
        "    for i in range(len(data)):\n",
        "        axes[i].imshow(data[i], aspect='auto', origin='bottom',\n",
        "                       interpolation='none', cmap='viridis')\n",
        "\n",
        "torch.set_grad_enabled(False)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 79,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 332
        },
        "id": "27X4nCPJBxGP",
        "outputId": "d8814b4f-bad9-4541-8443-607f8282d3ea"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "module 'tensorflow' has no attribute 'contrib'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-79-474e2af39ea2>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mhparams\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcreate_hparams\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mhparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_hparams\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mhparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msampling_rate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m22050\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTacotron2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/tacotron2/hparams.py\u001b[0m in \u001b[0;36mcreate_hparams\u001b[0;34m(hparams_string, verbose)\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;34m\"\"\"Create model hyperparameters. Parse nondefault from given string.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     hparams = tf.training.HParams(\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0;31m################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0;31m# Experiment Parameters        #\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: module 'tensorflow' has no attribute 'contrib'"
          ]
        }
      ],
      "source": [
        "# initialize Tacotron2 with the pretrained model\n",
        "from hparams import create_hparams\n",
        "\n",
        "hparams = hparams.create_hparams()\n",
        "hparams.sampling_rate = 22050\n",
        "model = Tacotron2(hparams)\n",
        "model.load_state_dict(torch.load(tacotron2_pretrained_model)['state_dict'])\n",
        "_ = model.cuda().eval()#.half()\n",
        "\n",
        "# initialize Waveglow with the pretrained model\n",
        "# waveglow = torch.load(waveglow_pretrained_model)['model']\n",
        "# WORKAROUND for: https://github.com/NVIDIA/tacotron2/issues/182\n",
        "import json\n",
        "from glow import WaveGlow\n",
        "waveglow_config = json.load(open('%s/waveglow/config.json' % project_name))['waveglow_config']\n",
        "waveglow = WaveGlow(**waveglow_config)\n",
        "waveglow.load_state_dict(torch.load(waveglow_pretrained_model)['model'].state_dict())\n",
        "_ = waveglow.cuda().eval()#.half()\n",
        "for k in waveglow.convinv:\n",
        "    k.float()\n",
        "denoiser = Denoiser(waveglow)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T413pYks9RpB"
      },
      "outputs": [],
      "source": [
        "INPUT_TEXT = \"I speak about how Waveglow is really awesome!\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "yHUfqTJi9XpS",
        "outputId": "b1b2a581-e2b0-45c8-f4b5-74afcdfd00bb"
      },
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'waveglow' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-75-3b1c4919859e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0maudio\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwaveglow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmel_outputs_postnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msigma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.666\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mipd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAudio\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maudio\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhparams\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msampling_rate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'waveglow' is not defined"
          ]
        }
      ],
      "source": [
        "audio = waveglow.infer(mel_outputs_postnet, sigma=0.666)\n",
        "ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HadVBXFf9aWU"
      },
      "outputs": [],
      "source": [
        "# remove waveglow bias\n",
        "audio_denoised = denoiser(audio, strength=0.01)[:, 0]\n",
        "ipd.Audio(audio_denoised.cpu().numpy(), rate=hparams.sampling_rate)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E4rDyhlW9T60"
      },
      "outputs": [],
      "source": [
        "#@title\n",
        "sequence = np.array(text_to_sequence(INPUT_TEXT, ['english_cleaners']))[None, :]\n",
        "sequence = torch.autograd.Variable(torch.from_numpy(sequence)).long()\n",
        "sequence = sequence.cuda()\n",
        "\n",
        "mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)\n",
        "plot_data((mel_outputs.data.cpu().numpy()[0],\n",
        "           mel_outputs_postnet.data.cpu().numpy()[0],\n",
        "           alignments.data.cpu().numpy()[0].T))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": [],
      "authorship_tag": "ABX9TyNvL8whUG7lwwA6nYoyJey2",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}