SilenceRemovalMethods.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://github.com/kylebgorman/textgrid\n",
    "#Autosegmenteerija poolt loodud TextGridide lihtsustamiseks\n",
    "#!pip install textgrid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.signal import butter, lfilter\n",
    "import librosa\n",
    "import librosa.display\n",
    "import IPython.display as ipd\n",
    "import soundfile as sf\n",
    "import textgrid\n",
    "import os\n",
    "import errno\n",
    "import glob\n",
    "import os.path\n",
    "import shutil\n",
    "import subprocess, shlex\n",
    "import multiprocessing\n",
    "import time\n",
    "import re\n",
    "import math\n",
    "\n",
    "#https://github.com/TartuNLP/tts_preprocess_et\n",
    "from tts_preprocess_et.convert import convert_sentence\n",
    "from multiprocessing import Process\n",
    "from copy import copy\n",
    "\n",
    "%matplotlib inline\n",
    "matplotlib.rcParams['figure.figsize'] = [20, 15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MILLISEID EELTÖÖTLUS MEETODEID SOOVITE KASUTADA\n",
    "preprocessUsingRMSZCR = False #Kas rakendada kõnekorpusele RMS + ZCR põhist vaikuse eemaldamist\n",
    "                             #Bakalaureuse töös on kasutatud terminit RMSE, kuid librosa meetodi nimi on RMS (valem on sellele vaatamata sama)\n",
    "preprocessUsingAutosegment = False #Kas rakendada kõnekorpusele Autosegmenteerija põhist eemaldamist\n",
    "\n",
    "#KÕNEKORPUSTE JUURKAUST\n",
    "originalAudioRootFolder = '/gpfs/hpc/projects/nlpgroup/speech-synthesis/data/original/'\n",
    "\n",
    "#RAAMI JA DISKREETIMISSAGEDUSE SÄTTED\n",
    "SR = 22050 #Diskreetimissagedus (sampling rate)\n",
    "N_FFT = 512 #Raami suurus nii STFT kui ka RMS + ZCR puhul\n",
    "HOP_LENGTH = 128 #Raami nihutus nii STFT kui ka RMS + ZCR puhul, seega kattuvus (overlap) on järgmise / eelmise raamiga on 75%\n",
    "\n",
    "#AUTOSEGMENTEERIJA MUUTUJAD\n",
    "autosegmentMaxSilence = 0.5 #Defineerib keskmiste vaikuste maksimaalse pikkuse sekundites\n",
    "autosegmentAudioOutput = \"Preprocessed-Autosegment/\" #Väljundi kaust\n",
    "textGridsFolder =  \"TextGridsWithPreprocessedSents/\" #Kaust, kus asuvad TextGridid\n",
    "\n",
    "#LIHTSAKOELISE PUHTALT RMS MUUTUJAD\n",
    "simpleTrimMaxSilence = 0.35 #Eeldades, et keskmised vaikused on leebelt jagatud, on mingi vaikus niikuinii sees, seega max on natuke suurem\n",
    "simpleTrimMaxSilenceInFrames = int(simpleTrimMaxSilence*SR / HOP_LENGTH)\n",
    "simpleTrimOutputFolderName = \"Preprocessed-Simple-RMS/\" #Väljundi kaust\n",
    "\n",
    "#RMS + ZCR MEETODI MUUTUJAD\n",
    "newTrimMaxSilence = 0.4 #Eeldades, et keskmised vaikused on leebelt jagatud, on mingi vaikus niikuinii sees\n",
    "newTrimMaxSilenceInFrames = int(newTrimMaxSilence*SR / HOP_LENGTH)\n",
    "newTrimOutputFolderName = \"Preprocessed-RMS-ZCR/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hääle lugejate dataframe kirjutamine\n",
    "\n",
    "#Kuna iga kõnekorpus on erinev, siis on raske luua universaalset lahendust\n",
    "#Seega kui Teie kõnekorpuse ülesehitus erineb drastiliselt  nt. UT uudiskorpusest, siis peate meetodit muutma\n",
    "\n",
    "#Tähtis on koodiploki lõpuks luua Dataframe data, kus on vaikuse eemaldamise eeltöötlust soovivate failide nimekiri\n",
    "#Vajalikud veerud on \"speaker\", kus on rääkija nimi, \"filename\", kus on helifaili asukoht \"sentence\", kus on faili loetud lause (töödeldud)\n",
    "\n",
    "#NB! Lause all mõeldakse kogu helifailis etteloetud teksti. Kui helifailis loetakse ette mitu lauset, siis see loeb kui 1 lause.\n",
    "\n",
    "#Kõnelejate nimed\n",
    "UTspeakers = ['Mari', 'Kalev', 'Albert', 'Vesta']\n",
    "EKIspeakers = [\"Kylli\",\"Meelis\"]\n",
    "\n",
    "preprocessSentences = True #Kas lauseid sisse lugedes rakendada lausete eeltöötlust (lühendid lahti kirjutada) (võib võtta aega (antud korpustel 15 minutit))\n",
    "\n",
    "\n",
    "data = pd.DataFrame()\n",
    "baseSentencesFiles = set() #Salvetame juurkausta, kus on korpuseid kirjeldavad csv failid. Kaustade nimekirjaga saab hiljem vajadusel csv faile ringi kopeerida\n",
    "\n",
    "for speaker in UTspeakers:\n",
    "    sentencesFile = originalAudioRootFolder + 'UT-uudised-' + speaker +  '/sentences.csv'\n",
    "    sentences = pd.read_csv(sentencesFile, sep='|', header=None)\n",
    "    sentences.columns = ['filename', 'uneditedSentence']\n",
    "    sentences['speaker'] = speaker\n",
    "    \n",
    "    sentences[\"filename\"] = sentences[\"filename\"].map(lambda filename: \"UT-uudised-\" + speaker + '/' + filename)\n",
    "    \n",
    "    #https://stackoverflow.com/questions/55902042/python-keep-only-alphanumeric-and-space-and-ignore-non-ascii/55902074\n",
    "    if preprocessSentences:\n",
    "        sentences[\"sentence\"] = sentences[\"uneditedSentence\"].map(lambda sent: re.sub(r'[^A-Za-zšŠžŽöäüõÖÄÜÕ0-9 ]+', '', convert_sentence(sent)))\n",
    "    else:\n",
    "        sentences[\"sentence\"] = sentences[\"uneditedSentence\"].map(lambda sent: re.sub(r'[^A-Za-zšŠžŽöäüõÖÄÜÕ0-9 ]+', '', sent))\n",
    "    \n",
    "\n",
    "    \n",
    "    data = data.append(sentences)\n",
    "    \n",
    "    sentencesFileRoot = originalAudioRootFolder + 'UT-uudised-' + speaker\n",
    "    baseSentencesFiles.add(sentencesFileRoot)\n",
    "    \n",
    "    \n",
    "for speaker in EKIspeakers:\n",
    "    sentencesFile = originalAudioRootFolder + 'EKI-ilukirjandus-' + speaker +  '/sentences.csv'\n",
    "    sentences = pd.read_csv(sentencesFile, sep='|', header=None)\n",
    "    sentences.columns = ['filename', 'uneditedSentence']\n",
    "    sentences['speaker'] = speaker\n",
    "    \n",
    "    sentences[\"filename\"] = sentences[\"filename\"].map(lambda filename: \"EKI-ilukirjandus-\" + speaker + '/' + filename)\n",
    "    if preprocessSentences:\n",
    "        sentences[\"sentence\"] = sentences[\"uneditedSentence\"].map(lambda sent: re.sub(r'[^A-Za-zšŠžŽöäüõÖÄÜÕ0-9 ]+', '', convert_sentence(sent)))\n",
    "    else:\n",
    "        sentences[\"sentence\"] = sentences[\"uneditedSentence\"].map(lambda sent: re.sub(r'[^A-Za-zšŠžŽöäüõÖÄÜÕ0-9 ]+', '', sent))\n",
    "    \n",
    "    data = data.append(sentences)\n",
    "    \n",
    "    sentencesFileRoot = originalAudioRootFolder + 'EKI-ilukirjandus-' + speaker\n",
    "    baseSentencesFiles.add(sentencesFileRoot)\n",
    "\n",
    "#Edasipidi tegeleme combinedData dataframega\n",
    "#Notebooki lõpus olevate koodiblokkidega saab teha combinedDatat testimise eesmärgil väiksemaks, seega teeme igaks juhuks koopia\n",
    "combinedData = data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Kohustuslik abistav meetodit failide ja kaustade loomiseks\n",
    "\n",
    "#Loob vajalikud kaustad nende puudumisel, et saaks luua faili outputPath\n",
    "#https://stackoverflow.com/questions/273192/how-can-i-safely-create-a-nested-directory\n",
    "def ifDirNotExistCreate(outputPath):\n",
    "    if not os.path.exists(os.path.dirname(outputPath)):\n",
    "        try:\n",
    "            os.makedirs(os.path.dirname(outputPath))\n",
    "        except OSError as exc: # Rallikonditsiooni vastane kaitse\n",
    "            if exc.errno != errno.EEXIST:\n",
    "                raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Kasutab autosegmenteerija tagastatud textgrid faili, et eemaldada failist vaikust\n",
    "# Row on combinedData dataframe rida\n",
    "def removeSilenceUsingAutoSegment(row):\n",
    "    name = (\".\").join(row.filename.split(\".\")[0:-1]) #Kogu faili path v.a. tüüp (.wav)\n",
    "    try:\n",
    "        tg = textgrid.TextGrid.fromFile(textGridsFolder +name+'.TextGrid')\n",
    "    except: #Kui textgrid pole loodud, nt. mingi karakter oli illegaalne\n",
    "        print(\"Textgrid loading went wrong, tried to load \"+ textGridsFolder +name+'.TextGrid')\n",
    "        trimAndSave(row, False, autosegmentAudioOutput+name+\".wav\") #Kasutame oma loodud trimmerit\n",
    "        return\n",
    "    y, sr = librosa.load(name+\".wav\")\n",
    "    y = librosa.util.normalize(y)\n",
    "    filteredAudio = []\n",
    "    for index, interval in enumerate(tg[0]):\n",
    "        if interval.mark != \"<sil>\":\n",
    "            filteredAudio.extend(y[librosa.core.time_to_samples(interval.minTime,sr):librosa.core.time_to_samples(interval.maxTime,sr)])\n",
    "        else: #vaikus\n",
    "            if (index == 0 or index == len(tg[0]) -1): #Kui tegemist on vaikusega ja see vaikus on esimene või viimane sektor, mida me kuuleme\n",
    "                continue #Ei lisa leitud vaikust\n",
    "            if(interval.maxTime - interval.minTime >= autosegmentMaxSilence): #Kui keskmine vaikus on liiga pikk, jätame vaikuse keskmise osa välja\n",
    "                filteredAudio.extend(y[librosa.core.time_to_samples(interval.minTime,sr):\n",
    "                                       librosa.core.time_to_samples(interval.minTime + (autosegmentMaxSilence / 2),sr)])\n",
    "                filteredAudio.extend(y[librosa.core.time_to_samples(interval.maxTime - (autosegmentMaxSilence / 2),sr):\n",
    "                                       librosa.core.time_to_samples(interval.maxTime,sr)])\n",
    "            else: #piisavalt lühike, jätame kogu vaikuse sisse\n",
    "                filteredAudio.extend(y[librosa.core.time_to_samples(interval.minTime,sr):librosa.core.time_to_samples(interval.maxTime,sr)])\n",
    "    outputFileName =  autosegmentAudioOutput+name+\".wav\"\n",
    "    ifDirNotExistCreate(outputFileName)\n",
    "    sf.write(outputFileName, filteredAudio, sr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Eeldades, et igal helifailil on olemas sama nimega TextGrid fail, saame rakendada vaikuse eemaldajat\n",
    "\n",
    "#Lõimede ärakasutamise eesmärgil meetod, mis võtab järjekorrast failinime ja töötleb vastavat faili (loob uue töödeldud faili)\n",
    "def oneFileFromQueueToAutosegmentSilRemoval(taskQueue):\n",
    "    while not taskQueue.empty():\n",
    "        if(taskQueue.qsize() % 50 == 0):\n",
    "            print(\"Files left to remove silence from:\",taskQueue.qsize())\n",
    "        rowWithIndex = taskQueue.get()\n",
    "        removeSilenceUsingAutoSegment(rowWithIndex[0])\n",
    "    return True\n",
    "\n",
    "if preprocessUsingAutosegment: #Kui soovitakse kasutada Autosegmenteerijat\n",
    "    start = time.time() #Kulunud aja mõõtmiseks\n",
    "\n",
    "    taskQueue = multiprocessing.Queue()\n",
    "    processes = 10 #Maksimaalne lõimede arv, 10 tundus optimaalne, liiga kõrge arv aeglustas\n",
    "    proc = [] #Lõimelised protessid lähevad siia\n",
    "\n",
    "\n",
    "    for index, row in combinedData.iterrows():\n",
    "        name = (\".\").join(row.filename.split(\".\")[0:-1])\n",
    "        taskQueue.put((row,index))\n",
    "            \n",
    "            \n",
    "    for n in range(processes):\n",
    "        p = Process(target=oneFileFromQueueToAutosegmentSilRemoval, args=(taskQueue,))\n",
    "        proc.append(p)\n",
    "        p.start()\n",
    "\n",
    "    for p in proc:\n",
    "        p.join()\n",
    "\n",
    "    print(\"Time taken = \",time.time() - start, \"s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RMS ja ZCR meetodi puhul keskmisi vaikusi trimmiv meetod\n",
    "#states on list nullidest ja ühtedest, mis näitab, mis raamis on vaikus. 0 tähendab vaikust.\n",
    "def applyMaxSilenceDuration(states, maxSilenceFrames):\n",
    "    \n",
    "    newStates = copy(states) #Teeme koopia, et põhimeetodis mitte mõjutada originaalset states olekut\n",
    "                             #Sellesse koopiasse teeme muudatusi, et nt. lausete sisemist vaikust alles hoida\n",
    "    startOfSilenceFrame = -1 #Jätame meelde, mis raamis algas viimati nähtus vaikus\n",
    "    durationOfSilenceFrame = 0 #Praeguse vaikuse kestus raamides\n",
    "    \n",
    "    for index, val in enumerate(states):\n",
    "        if val == 1: #Heli\n",
    "            if durationOfSilenceFrame > maxSilenceFrames: #Enne heli oli vaikus, mis oli pikem kui lubatud\n",
    "                newStates[startOfSilenceFrame : startOfSilenceFrame + int(maxSilenceFrames / 2) + 1] = 1\n",
    "                newStates[index - int(maxSilenceFrames / 2) - 1 : index] = 1\n",
    "            elif durationOfSilenceFrame > 0: #Enne heli oli vaikus, mis oli lubatu piires\n",
    "                newStates[startOfSilenceFrame : index] = 1\n",
    "            durationOfSilenceFrame = 0\n",
    "            startOfSilenceFrame = -1\n",
    "        else: #Vaikus\n",
    "            durationOfSilenceFrame += 1\n",
    "            if startOfSilenceFrame == -1:\n",
    "                startOfSilenceFrame = index\n",
    "    return newStates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uus loodav trimmija (kasutab ka ZCR ja arvestab heli alguses oleva vaikusega)\n",
    "#Kui logida tulemusi, siis tehakse mitu joonist, nii et logimine on kasulik üksikute helifailide testimisel\n",
    "def trimAndSave(row, log,outputName = None):\n",
    "    \n",
    "    if log:\n",
    "        print(row.speaker + '/' + row.filename + '\\n' + \"Sent: \"+ row.sentence)\n",
    "    \n",
    "    #Eeltöötlus\n",
    "    audioFile = originalAudioRootFolder + row.filename\n",
    "    transcript  = row.sentence #Ehk helifailis loetud lause\n",
    "    \n",
    "    y, _ = librosa.load(audioFile, sr=SR) #Laadime helifaili sisse diskreetimissagedusega SR\n",
    "    yOriginal = y.copy() #Hiljem eemaldame vaikuse filtreerimata helisignaalist, seega vaja koopiat\n",
    "    \n",
    "    #Rakendame kõrg- ja madalpääs filtrit, et jätta alles vaid inimhääle sagedused\n",
    "    #https://stackoverflow.com/questions/48101564/create-a-band-pass-filter-via-scipy-in-python\n",
    "    lo,hi=250,6000\n",
    "    b,a=butter(N=6, Wn=[2*lo/SR, 2*hi/SR], btype='band')\n",
    "    yFiltered = lfilter(b,a,y)\n",
    "    \n",
    "    y = librosa.util.normalize(yFiltered)\n",
    "    yOriginal = librosa.util.normalize(yOriginal)\n",
    "    \n",
    "    #RMS leidmine\n",
    "    \n",
    "    #Et lihtsalt rakendada aknafunktsiooni RMS leidmisel, teeme enne STFT (kus on automaatselt sees akna funktsioon (Hanni aken))\n",
    "    #Samuti ei ole meil vaja STFT poolt antud faasi iga sageduskomponendi juures, ainult magnituude\n",
    "    S, phase = librosa.magphase(librosa.stft(y, n_fft=N_FFT, hop_length=HOP_LENGTH))\n",
    "    rms = librosa.feature.rms(S=S, frame_length=N_FFT, hop_length=HOP_LENGTH) [0]\n",
    "    \n",
    "    #Lihtsustatud versiooni kasutamise otsustamine\n",
    "    rmsNoiseThresholdBeginning = np.average(rms[17:34]) #Esimesed 0.1 - 0.2 sekundid helifailist\n",
    "    \n",
    "    if(rmsNoiseThresholdBeginning / max(rms) > 0.1): #Kui juba helifaili alguses algab mitte vaikus, ei saa tulevasi meetodeid rakendada\n",
    "        print(\"Soundfile starts with speech \"+ row.speaker + '/' + row.filename)\n",
    "        trimAndSaveSimple(row, log, newTrimOutputFolderName+row.filename) #Kasutame oma varem loodud trimmerit\n",
    "        return\n",
    "    \n",
    "    #Läve leidmine\n",
    "    rmsNoiseThreshold = 0.06 * (max(rms) - np.average(rms[17:34]))\n",
    "    \n",
    "    #Viterbi kasutamine leebe ja range märgenduse loomiseks\n",
    "    #1 - Range vaikuse märgendaja äärmiste vaikuste jaoks\n",
    "    #2 - Leebem vaikuse märgendaja keskmiste vaikuste jaoks\n",
    "    \n",
    "    \n",
    "    # https://librosa.org/doc/main/auto_examples/plot_viterbi.html#sphx-glr-auto-examples-plot-viterbi-py\n",
    "    times = librosa.times_like(rms, sr=SR, hop_length=HOP_LENGTH)\n",
    "    \n",
    "    #Leiame RMS erinevuse läve ja vaadeldava raami vahel\n",
    "    #Siis laiendame/normaliseerime väärtuste vahemikku, kuna rms väärtused on tavaliselt väiksed ([0,0.3])\n",
    "    rNormalized1 = (rms - 1.2*rmsNoiseThreshold) / np.std(rms) #1.2*rmsNoiseThreshold on RMS väärtus, mida saab kõneks lugeda\n",
    "    rNormalized2 = (rms - 1.1*rmsNoiseThreshold) / np.std(rms)\n",
    "    \n",
    "    \n",
    "    p1 = np.exp(rNormalized1) / (1 + np.exp(rNormalized1))\n",
    "    p2= np.exp(rNormalized2) / (1 + np.exp(rNormalized2))\n",
    "\n",
    "    \n",
    "    transition1 = librosa.sequence.transition_loop(2, [0.5, 0.58])\n",
    "    transition2 = librosa.sequence.transition_loop(2, [0.5, 0.6]) # Kui enne oli heli, eeldame, et pigem tuleb uuesti heli\n",
    "\n",
    "    \n",
    "    fullP1 = np.vstack([1 - p1, p1])\n",
    "    fullP2 = np.vstack([1 - p2, p2])\n",
    "\n",
    "    states1 = librosa.sequence.viterbi_discriminative(fullP1, transition1)\n",
    "    states2 = librosa.sequence.viterbi_discriminative(fullP2, transition2)\n",
    "    \n",
    "    \n",
    "    if log:\n",
    "        fig, ax = plt.subplots()\n",
    "        ax.set_title(row.filename)\n",
    "        ax.plot(times, p1, label='P[V=1|x]')\n",
    "        ax.axhline(0.5, color='r', alpha=0.5, label='Descision threshold')\n",
    "        ax.set(xlabel='Time')\n",
    "        ax.legend();\n",
    "\n",
    "    #Range märgenduse töötlemine ja kõne otspunktide leidmine\n",
    "    firstStrictSilenceFound = False # Kas oleme leidnud esimese vaikuse\n",
    "    lastFittingIndex = len(states1) - 1 #Salvestame siia viimase mittevaikuse raami\n",
    "    lastFittingIndexFrameCounter=0 #Mitu raami on viimati leitud mittevaikus kestnud? (Liiga väiksed helid ei loe)\n",
    "    firstFittingIndex = 0 #Salvestame siia esimese mittevaikuse raami\n",
    "    for index, state in enumerate(states1): # Range heli tunnetaja\n",
    "        if  state == 0: #Kui vaikus\n",
    "            lastFittingIndexFrameCounter = 0\n",
    "            if not firstStrictSilenceFound: #Kuna helifail võib alata mehhaanilise müraga, enne kõne märkimist võiks oodata esimese vaikuseni\n",
    "                firstStrictSilenceFound = True\n",
    "                \n",
    "        #Kui esimene vaikus on leitud ja tegemist on esimese mitte vaikusega\n",
    "        #Samuti on tegu kõnega ja kõne on kestnud vähemalt 12 raami (0,07 s)\n",
    "        if  (firstStrictSilenceFound and firstFittingIndex == 0 and \n",
    "            state == 1 and lastFittingIndexFrameCounter > 12):\n",
    "            firstFittingIndex = index - lastFittingIndexFrameCounter\n",
    "        if  state == 1: #Kui kõne\n",
    "            lastFittingIndexFrameCounter += 1\n",
    "            if lastFittingIndexFrameCounter > 1: #Ja kõne on kestnud vähemalt 1 raami ehk 0.023 s\n",
    "                lastFittingIndex = index\n",
    "        \n",
    "    #Leebe märgenduse töötlemine\n",
    "    states2MaxSilenced = applyMaxSilenceDuration(states2, newTrimMaxSilenceInFrames)\n",
    "    \n",
    "\n",
    "    #zero crossing rate leidmine ja läve arvutamine\n",
    "    zeroCR = librosa.feature.zero_crossing_rate(y, frame_length=N_FFT, hop_length=HOP_LENGTH)[0]\n",
    "    ZCRThreshold = np.average(zeroCR[17:34]) + 2.1* np.std(zeroCR[17:34])\n",
    "    \n",
    "    #ZCR ajavahemiku arvutamine\n",
    "    newTrimBaseZCRBonusWindowDuration = 0.12 #sekundites\n",
    "    \n",
    "    if len({\"g\",\"b\",\"d\",\"k\",\"p\",\"t\",\"f\",\"h\"} - set(transcript[-2:])) != 8 and transcript[-1] not in [\"a\",\"e\",\"i\",\"o\",\"u\",\"õ\",\"ä\",\"ö\",\"ü\"]:\n",
    "        newTrimBaseZCRBonusWindowDuration += 0.04\n",
    "    if(transcript[-2:] in [\"kk\",\"pp\",\"tt\",\"ll\"]):\n",
    "        newTrimBaseZCRBonusWindowDuration += 0.04\n",
    "    \n",
    "    newTrimZCRBonusWindowInFrames = int(newTrimBaseZCRBonusWindowDuration*SR / HOP_LENGTH)\n",
    "    \n",
    "    #Kõne otspunktide täpsustamine\n",
    "    firstFittingIndex = max(firstFittingIndex - 12,0)\n",
    "    \n",
    "    updatedLastFittingIndex = lastFittingIndex\n",
    "    for ind, zcr in enumerate(zeroCR[lastFittingIndex:lastFittingIndex + newTrimZCRBonusWindowInFrames +1], start=lastFittingIndex):\n",
    "        if zcr > ZCRThreshold:\n",
    "            updatedLastFittingIndex = ind\n",
    "        \n",
    "\n",
    "    updatedLastFittingIndex = min(updatedLastFittingIndex + 9,len(states2))\n",
    "\n",
    "    #Lõpliku märgenduse loomine\n",
    "    states2MaxSilenced[0:firstFittingIndex] = 0\n",
    "    states2MaxSilenced[lastFittingIndex:updatedLastFittingIndex] = 1\n",
    "    states2MaxSilenced[updatedLastFittingIndex:] = 0\n",
    "    \n",
    "    \n",
    "    if log:\n",
    "        fig, ax = plt.subplots(nrows=1, sharex=True)\n",
    "        #ax.set_title(row.filename + \"\\n\" + row.sentence)\n",
    "        ax.set_title(row.filename)\n",
    "        ax.label_outer()\n",
    "        \n",
    "        #ax.step(times, zeroCR, linestyle='--', color='pink', label='ZCR')\n",
    "        librosa.display.waveplot(y,alpha=0.5)\n",
    "        #ax.step(times, states2, linestyle='--', color='green', label='ViterbiForgiving')\n",
    "        ax.step(times, states1, linestyle='--', color='red', label='ViterbiStrict')\n",
    "        reversedScale = [-x for x in states2MaxSilenced]\n",
    "        ax.step(times, reversedScale, linestyle='--', color='black', label='ViterbiFinal')\n",
    "        ax.set(ylim=[-1.1, 1.1])\n",
    "        ax.legend(loc=\"upper right\")\n",
    "\n",
    "\n",
    "        \n",
    "    if log:\n",
    "        plt.figure(figsize=(25, 20))\n",
    "        librosa.display.waveplot(y,alpha=0.5)\n",
    "        plt.title(\"Final Viterby with ZCR and threshold\")\n",
    "        plt.hlines(ZCRThreshold,xmin=0,xmax=librosa.get_duration(yFiltered), colors=[\"black\"])\n",
    "        plt.plot(times, zeroCR, color=\"r\", label = \"ZCR\")\n",
    "        plt.plot(times, states2MaxSilenced, linestyle='--', color='black', label='ViterbiFinal')\n",
    "        plt.legend()\n",
    "        plt.show()\n",
    "\n",
    "    \n",
    "    #Siia loome uue helifaili\n",
    "    newSound = []\n",
    "    \n",
    "    #Käime uuendatud üldise helimärgenduste listi läbi ja lisame ainult need heliosad, kus oleme heli märgendanud\n",
    "    for index,state in  enumerate(states2MaxSilenced):\n",
    "        if(state == 1):\n",
    "            newSound.extend(yOriginal[index*HOP_LENGTH:((index+1)*HOP_LENGTH)])\n",
    "\n",
    "    #Loome kausta kuhu panna failid ja kirjutame need sinna\n",
    "    outputPath = newTrimOutputFolderName+row.filename\n",
    "    if outputName != None:\n",
    "        outputPath = outputName\n",
    "    ifDirNotExistCreate(outputPath)\n",
    "    sf.write(outputPath, newSound, SR)\n",
    "    return\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Uus loodav trimmija (ainult RMS põhine ja lävi ei arvesta helifaili alguses oleva vaikusega)\n",
    "def trimAndSaveSimple(row, log,outputName = None):\n",
    "    if log:\n",
    "        print(row.speaker + '/' + row.filename + '\\n' + \"Sent: \"+ row.sentence)\n",
    "    \n",
    "    #Eeltöötlus\n",
    "    audioFile = originalAudioRootFolder + row.filename\n",
    "    transcript  = row.sentence\n",
    "    \n",
    "    \n",
    "    y, _ = librosa.load(audioFile, sr=SR)\n",
    "    y = librosa.util.normalize(y)\n",
    "\n",
    "    #RMS leidmine\n",
    "    S, phase = librosa.magphase(librosa.stft(y, n_fft=N_FFT, hop_length=HOP_LENGTH))\n",
    "    rms = librosa.feature.rms(S=S, frame_length=N_FFT, hop_length=HOP_LENGTH)  [0]\n",
    "    \n",
    "    \n",
    "    #Arvestame läve arvutamisel kõneleja keskkonna ja kõneleja valjusega\n",
    "    shouldBeAvg = 0.085 # => Kui helifaili keskmine RMS on 0.085, ei muudeta läve kordjat\n",
    "    actualAvg = rms[rms > 0.01].mean()\n",
    "    multiplier = 1 + (actualAvg - shouldBeAvg) * 8\n",
    "    \n",
    "    #Viterbi kasutamine range ja leebe märgendamise jaoks\n",
    "    # https://librosa.org/doc/main/auto_examples/plot_viterbi.html#sphx-glr-auto-examples-plot-viterbi-py\n",
    "    times = librosa.times_like(rms, sr=SR, hop_length=HOP_LENGTH)\n",
    "    \n",
    "    \n",
    "    rNormalized1 = (rms - 0.012*multiplier) / np.std(rms)\n",
    "    rNormalized2 = (rms - 0.011*multiplier) / np.std(rms)\n",
    "    p1 = np.exp(rNormalized1) / (1 + np.exp(rNormalized1))\n",
    "    p2 = np.exp(rNormalized2) / (1 + np.exp(rNormalized2))\n",
    "\n",
    "    \n",
    "    transition1 = librosa.sequence.transition_loop(2, [0.5, 0.50]) # Ignoreerime minevikku faase\n",
    "    transition2 = librosa.sequence.transition_loop(2, [0.5, 0.58]) # Kui enne oli heli, eeldame, et pigem tuleb uuesti heli\n",
    "\n",
    "    \n",
    "    fullP1 = np.vstack([1 - p1, p1])\n",
    "    fullP2 = np.vstack([1 - p2, p2])\n",
    "\n",
    "    states1 = librosa.sequence.viterbi_discriminative(fullP1, transition1)\n",
    "    states2 = librosa.sequence.viterbi_discriminative(fullP2, transition2)\n",
    "    \n",
    "    \n",
    "    if log:\n",
    "        fig, ax = plt.subplots()\n",
    "        ax.set_title(row.filename)\n",
    "        ax.plot(times, p1, label='P[V=1|x]')\n",
    "        ax.axhline(0.5, color='r', alpha=0.5, label='Descision threshold')\n",
    "        ax.set(xlabel='Time')\n",
    "        ax.legend();\n",
    "\n",
    "    #Range märgendamise töötlemine ja kõne otspunktide leidmine\n",
    "    firstStrictSilenceFound = False # Kas oleme leidnud esimese vaikuse\n",
    "    lastFittingIndex = len(states1) - 1 #Salvestame siia viimase mittevaikuse raami\n",
    "    lastFittingIndexFrameCounter=0 #Mitu raami on viimati leitud vaikus kestnud? (Liiga väiksed helid ei loe)\n",
    "    firstFittingIndex = 0 #Salvestame siia esimese mittevaikuse raami\n",
    "    for index, state in enumerate(states1): # Range heli tunnetaja\n",
    "        if  state == 0: #Kui vaikus\n",
    "            lastFittingIndexFrameCounter = 0\n",
    "            if not firstStrictSilenceFound: #If we have not yet found first silence for the trimmer (to  ignore instant  noise)\n",
    "                firstStrictSilenceFound = True\n",
    "        if  firstStrictSilenceFound and firstFittingIndex == 0 and state == 1 and lastFittingIndexFrameCounter > 12: #Kui esimene vaikus on leitud ja tegemist on esimese mitte vaikusega\n",
    "            firstFittingIndex = index - lastFittingIndexFrameCounter\n",
    "        if  state == 1: #Kui kõne\n",
    "            lastFittingIndexFrameCounter += 1\n",
    "            if lastFittingIndexFrameCounter > 12:\n",
    "                lastFittingIndex = index\n",
    "           \n",
    "        \n",
    "    #Leebe märgendamise töötlemine\n",
    "    states2MaxSilenced = applyMaxSilenceDuration(states2, simpleTrimMaxSilenceInFrames)\n",
    "    \n",
    "    #Transkriptsiooniga arvestamine (helitud häälikud)\n",
    "    bonusFrames = 36 #Lause lõpus jätame nii palju diskreete alles ka pärast vaikuse tundmist (et kompenseerida agressiivsust) [0.21 sekundit]\n",
    "    if len({\"g\",\"b\",\"d\",\"k\",\"p\",\"t\",\"f\",\"h\"} - set(transcript[-2:])) != 8 and transcript[-1] not in [\"a\",\"e\",\"i\",\"o\",\"u\",\"õ\",\"ä\",\"ö\",\"ü\"]:\n",
    "        bonusFrames += 24 #0.14 sekundit\n",
    "    if(transcript[-2:] in [\"kk\",\"pp\",\"tt\",\"ll\"]):\n",
    "        bonusFrames += 20 #0.12 sekundit\n",
    "    \n",
    "    \n",
    "    #Kõne otspunktide parandamine\n",
    "    firstFittingIndex = max(firstFittingIndex - 15,0)\n",
    "    lastFittingIndex = min(lastFittingIndex + bonusFrames,len(states2))\n",
    "    \n",
    "    #Lisame leebele vaikuse leidajale vajalikud märkmed kõne otspunktidest\n",
    "    states2MaxSilenced[0:firstFittingIndex] = 0\n",
    "    states2MaxSilenced[lastFittingIndex:] = 0\n",
    "    \n",
    "    \n",
    "    if log:\n",
    "        fig, ax = plt.subplots(nrows=1, sharex=True)\n",
    "        ax.set_title(row.filename)\n",
    "        librosa.display.waveplot(y,alpha=0.5)\n",
    "        ax.label_outer()\n",
    "        \n",
    "        #ax.step(times, p2>=0.5, label='Frame-wise')\n",
    "        #ax.step(times, states2, linestyle='--', color='green', label='ViterbiForgiving')\n",
    "        ax.step(times, states1, linestyle='--', color='red', label='ViterbiStrict')\n",
    "        reversedScale = [-x for x in states2MaxSilenced]\n",
    "        ax.step(times, reversedScale, linestyle='--', color='black', label='ViterbiFinal')\n",
    "        ax.set(ylim=[-1.1, 1.1])\n",
    "        ax.legend()\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    #Siia loome uue helifaili\n",
    "    newSound = []\n",
    "    \n",
    "    #Käime lõpliku helimärgenduste listi läbi ja lisame ainult need heliosad, kus oleme heli märgendanud\n",
    "    for index,state in  enumerate(states2MaxSilenced):\n",
    "        if(state == 1):\n",
    "            #Tegelikult peaks lisama natuke väärtust diskreetide indeksitele, et need oleksid raami keskel, mitte alguses\n",
    "            newSound.extend(y[index*HOP_LENGTH:((index+1)*HOP_LENGTH)])\n",
    "\n",
    "    #Loome kausta kuhu panna failid ja kirjutame need sinna\n",
    "    outputPath = simpleTrimOutputFolderName+row.filename\n",
    "    if outputName != None:\n",
    "        outputPath = outputName\n",
    "        \n",
    "    ifDirNotExistCreate(outputPath)\n",
    "    sf.write(outputPath, newSound, SR)\n",
    "    return\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def oneFileFromQueueToRMS_ZCRSilRemoval(taskQueue):\n",
    "    while not taskQueue.empty():\n",
    "        if(taskQueue.qsize() % 10 == 0):\n",
    "            print(\"Files left to remove silence from:\",taskQueue.qsize())\n",
    "        rowWithIndex = taskQueue.get()\n",
    "        trimAndSave(rowWithIndex[0], False)\n",
    "    return True\n",
    "\n",
    "\n",
    "if preprocessUsingRMSZCR: #Kui soovitakse kasutada RMS + ZCR meetodit eeltöötluseks\n",
    "    start = time.time()\n",
    "\n",
    "\n",
    "    taskQueue = multiprocessing.Queue()\n",
    "    processes = 10\n",
    "    proc = []\n",
    "\n",
    "\n",
    "    for index, row in combinedData.iterrows():\n",
    "        taskQueue.put((row,index))\n",
    "\n",
    "    for n in range(processes):\n",
    "        p = Process(target=oneFileFromQueueToRMS_ZCRSilRemoval, args=(taskQueue,))\n",
    "        proc.append(p)\n",
    "        p.start()\n",
    "\n",
    "    for p in proc:\n",
    "        p.join()\n",
    "\n",
    "    print(\"Time taken = \",time.time() - start, \"s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Lisa tööriistad, nt. failide kopeerimiseks, combinedData muutmiseks või informatsiooni saamiseks helifailidest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Mitte kohustuslikud abistavad meetodid, kui on vaja faile kopeerida kaustast kausta, näiteks kodukausta kopeerimiseks / teise serverisse kopeerimiseks\n",
    "\n",
    "\n",
    "def writeSentenceCSVFilesFromFolderToFolder(originFolder,destinationFolder):\n",
    "    listOfCSVFiles =  glob.glob(originFolder +\"/*.csv\")\n",
    "    for csvFile in listOfCSVFiles:\n",
    "        ifDirNotExistCreate(destinationFolder)\n",
    "        shutil.copy(csvFile,destinationFolder)\n",
    "        \n",
    "def writeSentenceOfAudioFileToFolder(folderName,filename,sentence):\n",
    "    ifDirNotExistCreate(folderName+\"/\"+filename)\n",
    "    with open(folderName+\"/\"+ ((\".\").join(filename.split(\".\")[0:-1])) + \".txt\", 'w') as writer:\n",
    "        writer.write(sentence)\n",
    "        \n",
    "#loob kodukaustas vajalikud kaustad, kopeerib helifailid kodukausta, kirjutab samasse kausta eraldi tekstifaili igale lausele\n",
    "#Tagastab tegeletud kaustade hulga\n",
    "def copyAudioFromOriginalFolderAndWriteSentenceToSeperateFile(data):\n",
    "    for index, row in data.iterrows():\n",
    "        original_audio_file_path = originalAudioRootFolder + row.filename\n",
    "\n",
    "        #Teeme kindalks, et kodukaustas vajalikud kaustad eksisteerivad ja kui ei ekisteeri, siis loome'\n",
    "        #https://stackoverflow.com/questions/12517451/automatically-creating-directories-with-file-output\n",
    "        filename = row.filename\n",
    "        foldername = (\"/\").join(filename.split(\"/\")[0:-1]) #Ignoreerib faili nime osa (nt. UT-uudised-Albert)\n",
    "        ifDirNotExistCreate(filename)\n",
    "        shutil.copy(original_audio_file_path,foldername)\n",
    "        with open((\".\").join(filename.split(\".\")[0:-1]) + \".txt\", 'w') as writer:\n",
    "            writer.write(row.sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Kui on soov sättida valikuks kõik data kirjed\n",
    "\n",
    "# combinedData = data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Kui on soov sättida valikuks osa data kirjetest, ning see valik teha suvaliselt\n",
    "#from sklearn.utils import shuffle\n",
    "\n",
    "# limit = 100\n",
    "# VestaData= shuffle(data[data[\"speaker\"]==\"Vesta\"])\n",
    "# VestaData.reset_index(inplace=True, drop=True)\n",
    "# VestaData= VestaData[0:limit]\n",
    "\n",
    "# AlbertData= shuffle(data[data[\"speaker\"]==\"Albert\"])\n",
    "# AlbertData.reset_index(inplace=True, drop=True)\n",
    "# AlbertData= AlbertData[0:limit]\n",
    "\n",
    "# KalevData= shuffle(data[data[\"speaker\"]==\"Kalev\"])\n",
    "# KalevData.reset_index(inplace=True, drop=True)\n",
    "# KalevData= KalevData[0:limit]\n",
    "\n",
    "# MariData= shuffle(data[data[\"speaker\"]==\"Mari\"])\n",
    "# MariData.reset_index(inplace=True, drop=True)\n",
    "# MariData= MariData[0:limit]\n",
    "\n",
    "# KylliData= shuffle(data[data[\"speaker\"]==\"Kylli\"])\n",
    "# KylliData.reset_index(inplace=True, drop=True)\n",
    "# KylliData= KylliData[0:limit]\n",
    "\n",
    "# MeelisData= shuffle(data[data[\"speaker\"]==\"Meelis\"])\n",
    "# MeelisData.reset_index(inplace=True, drop=True)\n",
    "# MeelisData= MeelisData[0:limit]\n",
    "\n",
    "# combinedData = VestaData.append([AlbertData,KalevData,MariData,MeelisData,KylliData])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Kui on soov sättida valikuks osa data kirjetest, ning see valik oleks esimesed limit kirjet\n",
    "\n",
    "# limit = 15\n",
    "# VestaData= (data[data[\"speaker\"]==\"Vesta\"])\n",
    "# VestaData= VestaData[0:limit]\n",
    "\n",
    "# AlbertData= (data[data[\"speaker\"]==\"Albert\"])\n",
    "# AlbertData= AlbertData[0:limit]\n",
    "\n",
    "# KalevData= (data[data[\"speaker\"]==\"Kalev\"])\n",
    "# KalevData= KalevData[0:limit]\n",
    "\n",
    "# MariData= (data[data[\"speaker\"]==\"Mari\"])\n",
    "# MariData= MariData[0:limit]\n",
    "\n",
    "# KylliData= (data[data[\"speaker\"]==\"Kylli\"])\n",
    "# KylliData= KylliData[0:limit]\n",
    "\n",
    "# MeelisData= (data[data[\"speaker\"]==\"Meelis\"])\n",
    "# MeelisData= MeelisData[0:limit]\n",
    "\n",
    "# combinedData = VestaData.append([AlbertData,KalevData,MariData,KylliData,MeelisData])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Kirjutab ühe kausta csv failid teise kausta\n",
    "\n",
    "# destinationBase = './SentsForERR/'\n",
    "\n",
    "# for origin in baseSentencesFiles:\n",
    "#     writeSentenceCSVFilesFromFolderToFolder(origin,destinationBase + origin.split(\"/\")[-1] + \"/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Kirjutab iga rea kohta datas lause faili\n",
    "\n",
    "# for index, row in combinedData.iterrows():\n",
    "#     writeSentenceOfAudioFileToFolder(\"ERRSentencesWithoutPreprocessingTextFilesUpdated\",row.filename,row.sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Esiteks kopeeritakse vajalikud helifailid kodukausta ja luuakse igale helifailile eraldi tekstifail lausega (kirjutatavaks lauseks on datas olev sentence väärtus)\n",
    "\n",
    "# copyAudioFromOriginalFolderAndWriteSentenceToSeperateFile(combinedData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Oli kasutusel et leida, kas pärast helifailide normaliseerimist on RMS väärtus sarane, sõltumata lugejast\n",
    "#Selgus, et ikkagi mingi vahe on, seega on mõtekas tõsta RMS läve, kui üldiselt on RMS väärtus helifailis kõrge\n",
    "def findAvgRMS(data):\n",
    "    counter = 0\n",
    "    for index, row in data.iterrows():\n",
    "        audio_file = originalAudioRootFolder + row.filename\n",
    "        y, samplerate = librosa.load(audio_file)\n",
    "        y = librosa.util.normalize(y)\n",
    "        S, phase = librosa.magphase(librosa.stft(y, n_fft=N_FFT, hop_length=HOP_LENGTH))\n",
    "        rms = librosa.feature.rms(S=S, frame_length=N_FFT, hop_length=HOP_LENGTH)  [0]\n",
    "        rms = rms[rms > 0.01]\n",
    "        counter = counter + rms.mean()\n",
    "        print(index,\" -> \",rms.mean())\n",
    "    return counter / data.shape[0]\n",
    "  \n",
    "# AvgRMSVesta = findAvgRMS(VestaData[0:300])  \n",
    "# AvgRMSAlbert = findAvgRMS(AlbertData[0:300])  \n",
    "# AvgRMSKalev = findAvgRMS(KalevData[0:300])  \n",
    "# AvgRMSMari = findAvgRMS(MariData[0:300])  \n",
    "\n",
    "#print(\"Vesta -> \",AvgRMSVesta) #0.08319149660553583\n",
    "#print(\"Albert -> \",AvgRMSAlbert)#0.07957464025042947\n",
    "#print(\"Kalev -> \",AvgRMSKalev)#0.08875116641977554\n",
    "#print(\"Mari -> \",AvgRMSMari)#0.0805271124581126"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}