Colab Notebooks/https_github.com_s-ankur_fairseq-gec.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"https://github.com/s-ankur/fairseq-gec","provenance":[{"file_id":"1Nm7VyfAOmChyIo69Pr5jKGF-6XR8eb56","timestamp":1594373254464}],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"j2wKM_JmPcBS"},"source":["### Setup commands\n","!git clone https://github.com/s-ankur/fairseq-gec\n","%cd fairseq-gec\n","!pip install -e .\n","!sudo apt-get install libgoogle-perftools-dev libsparsehash-dev\n","%mkdir -p software/\n","%cd software\n","\n","%cd mosesdecoder\n","!bash bjam -j4\n","%cd ..\n","\n","!git clone https://github.com/clab/fast_align\n","%cd fast_align/\n","!mkdir -p build\n","%cd build\n","!cmake ..\n","!make\n","%cd ../..\n","\n","!chmod +x fast_align/build/fast_align mosesdecoder/scripts/ems/support/symmetrize-fast-align.perl\n","!chmod +x mosesdecoder/bin/*\n","%cd .."],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TdAT92v0mmjV"},"source":["!bash align.sh"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Vw6K-AI0Vqvn"},"source":["!bash preprocess.sh"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_R0r1so8R4kA"},"source":["!nohup bash train.sh>tmp.log &"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7j6qKcElGpu5"},"source":["!tail -f tmp.log"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a2S7N3ydH6nW","executionInfo":{"status":"ok","timestamp":1604675687822,"user_tz":-330,"elapsed":895,"user":{"displayName":"Ankur Sonawane IIT BHU","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiomZZ49qkpD-2EzVuzljSKs6l31EBMkNaQw4YeAQ=s64","userId":"12554047614810864421"}},"outputId":"ffc6f28f-ce53-4cb8-e483-4c79ee1dfba5","colab":{"base_uri":"https://localhost:8080/"}},"source":["cd fairseq-gec"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/content/drive/My Drive/fairseq-gec\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"k0QZ-ZOlYQDN"},"source":["import os\n","!rm -rf out/data_raw out/result\n","!mkdir -p out/data_raw/ out/result/\n","for err_type in os.listdir('../data/out'):\n","  print(err_type)\n","  !cp ../data/out/{err_type}/{err_type}.trg ../data/out/{err_type}/{err_type}.tgt\n","  !bash preprocess_hiwiki.sh ../data/out/{err_type}/{err_type}\n","  !bash generate.sh\n","  !mv out/result/outputema_last.txt ../data/out/{err_type}/fairseq.out\n","  !rm -rf out/data_raw out/result  \n","  !mkdir -p out/data_raw/ out/result/"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"o7Wf_NfmegYq"},"source":["!bash interactive.sh"],"execution_count":null,"outputs":[]}]}