reutersdata_keras_processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "reutersdata-keras processing.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNQOg+4TUDQUt19NIP027fJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sadidhasan/text-classification/blob/main/reutersdata_keras_processing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IBljSOmIu7CE"
      },
      "source": [
        "!mkdir reuters21578"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Lbym226fiAD",
        "outputId": "8945b2e4-68a1-4f25-cd3f-fab13e13eb42"
      },
      "source": [
        "%cd reuters21578/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/reuters21578\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QsFeBZ6Gfo2m",
        "outputId": "88b270ef-0347-478c-ecab-c7c5207b7a82"
      },
      "source": [
        "!pwd"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/reuters21578\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zfphS12rfz-C",
        "outputId": "b7ba6623-3a00-4568-e0a1-aedcbb607e69"
      },
      "source": [
        "!wget http://kdd.ics.uci.edu/databases/reuters21578/reuters21578.tar.gz"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-09-11 18:16:23--  http://kdd.ics.uci.edu/databases/reuters21578/reuters21578.tar.gz\n",
            "Resolving kdd.ics.uci.edu (kdd.ics.uci.edu)... 128.195.1.86\n",
            "Connecting to kdd.ics.uci.edu (kdd.ics.uci.edu)|128.195.1.86|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 8150596 (7.8M) [application/x-gzip]\n",
            "Saving to: ‘reuters21578.tar.gz’\n",
            "\n",
            "reuters21578.tar.gz 100%[===================>]   7.77M  4.00MB/s    in 1.9s    \n",
            "\n",
            "2021-09-11 18:16:25 (4.00 MB/s) - ‘reuters21578.tar.gz’ saved [8150596/8150596]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DOiAqGRqgFnT",
        "outputId": "608a9c28-d6a7-4279-c319-296cded5f8c4"
      },
      "source": [
        "!ls -l"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 7960\n",
            "-rw-r--r-- 1 root root 8150596 Mar 12  1999 reuters21578.tar.gz\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g2U7JTEUgI7t",
        "outputId": "89d2278c-4640-48f4-9221-5e38e58fd2e9"
      },
      "source": [
        "!tar -xvzf reuters21578.tar.gz"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "README.txt\n",
            "all-exchanges-strings.lc.txt\n",
            "all-orgs-strings.lc.txt\n",
            "all-people-strings.lc.txt\n",
            "all-places-strings.lc.txt\n",
            "all-topics-strings.lc.txt\n",
            "cat-descriptions_120396.txt\n",
            "feldman-cia-worldfactbook-data.txt\n",
            "lewis.dtd\n",
            "reut2-000.sgm\n",
            "reut2-001.sgm\n",
            "reut2-002.sgm\n",
            "reut2-003.sgm\n",
            "reut2-004.sgm\n",
            "reut2-005.sgm\n",
            "reut2-006.sgm\n",
            "reut2-007.sgm\n",
            "reut2-008.sgm\n",
            "reut2-009.sgm\n",
            "reut2-010.sgm\n",
            "reut2-011.sgm\n",
            "reut2-012.sgm\n",
            "reut2-013.sgm\n",
            "reut2-014.sgm\n",
            "reut2-015.sgm\n",
            "reut2-016.sgm\n",
            "reut2-017.sgm\n",
            "reut2-018.sgm\n",
            "reut2-019.sgm\n",
            "reut2-020.sgm\n",
            "reut2-021.sgm\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nMu-chuhgdqz",
        "outputId": "49c7c74a-5dbc-47f7-fc03-d518d36c6e1d"
      },
      "source": [
        "!ls"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "all-exchanges-strings.lc.txt\t    reut2-002.sgm  reut2-013.sgm\n",
            "all-orgs-strings.lc.txt\t\t    reut2-003.sgm  reut2-014.sgm\n",
            "all-people-strings.lc.txt\t    reut2-004.sgm  reut2-015.sgm\n",
            "all-places-strings.lc.txt\t    reut2-005.sgm  reut2-016.sgm\n",
            "all-topics-strings.lc.txt\t    reut2-006.sgm  reut2-017.sgm\n",
            "cat-descriptions_120396.txt\t    reut2-007.sgm  reut2-018.sgm\n",
            "feldman-cia-worldfactbook-data.txt  reut2-008.sgm  reut2-019.sgm\n",
            "lewis.dtd\t\t\t    reut2-009.sgm  reut2-020.sgm\n",
            "README.txt\t\t\t    reut2-010.sgm  reut2-021.sgm\n",
            "reut2-000.sgm\t\t\t    reut2-011.sgm  reuters21578.tar.gz\n",
            "reut2-001.sgm\t\t\t    reut2-012.sgm\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ikj4dPhBggoK",
        "outputId": "72912e9a-d79f-4d85-9e44-3d554914d0d4"
      },
      "source": [
        "import os\n",
        "import re\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "import collections\n",
        "\n",
        "def make_reuters_dataset(path=os.path.join('/content/reuters21578'), min_samples_per_topic=15):\n",
        "\n",
        "    wire_topics = []\n",
        "    topic_counts = {}\n",
        "    wire_bodies = []\n",
        "\n",
        "    for fname in sorted(os.listdir(path)):\n",
        "        if 'sgm' in fname:\n",
        "            s = open(os.path.join(path, fname),'rb').read().decode('latin-1')\n",
        "            #print(s)\n",
        "            tag = '<TOPICS>'\n",
        "            while tag in s:\n",
        "                s = s[s.find(tag)+len(tag):]\n",
        "                topics = s[:s.find('</')]\n",
        "                if topics and '</D><D>' not in topics:\n",
        "                    topic = topics.replace('<D>', '').replace('</D>', '')\n",
        "                    wire_topics.append(topic)\n",
        "                    topic_counts[topic] = topic_counts.get(topic, 0) + 1\n",
        "                else:\n",
        "                    continue\n",
        "\n",
        "                bodytag = '<BODY>'\n",
        "                body = s[s.find(bodytag)+len(bodytag):]\n",
        "                body = body[:body.find('</')]\n",
        "                wire_bodies.append(body)\n",
        "\n",
        "    # only keep most common topics\n",
        "    items = list(topic_counts.items())\n",
        "    items.sort(key=lambda x: x[1], reverse=True)\n",
        "    kept_topics = set()\n",
        "    for x in items:\n",
        "        print(x[0] + ': ' + str(x[1])),\n",
        "        if x[1] >= min_samples_per_topic:\n",
        "            kept_topics.add(x[0])\n",
        "    print\n",
        "    print('-')\n",
        "    print('Kept topics:', len(kept_topics))\n",
        "    \n",
        "    # filter wires with rare topics\n",
        "    kept_wires = []\n",
        "    labels = []\n",
        "    topic_indexes = {}\n",
        "    for t, b in zip(wire_topics, wire_bodies):\n",
        "        if t in kept_topics:\n",
        "            if t not in topic_indexes:\n",
        "                topic_index = len(topic_indexes)\n",
        "                topic_indexes[t] = topic_index\n",
        "            else:\n",
        "                topic_index = topic_indexes[t]\n",
        "\n",
        "            labels.append(topic_index)\n",
        "            kept_wires.append(b)\n",
        "\n",
        "    print('Kept wires:', len(kept_wires))\n",
        "    print('-')\n",
        "    print('Topic mapping:', sorted(topic_indexes.items(), key=lambda x:x[1]))\n",
        "    print('-')\n",
        "\n",
        "    # vectorize wires\n",
        "    tokenizer = Tokenizer()\n",
        "    tokenizer.fit_on_texts(kept_wires)\n",
        "    X = tokenizer.texts_to_sequences(kept_wires)\n",
        "\n",
        "    print('Sanity check:')\n",
        "    for w in [\"banana\", \"oil\", \"chocolate\", \"the\", \"dsft\"]:\n",
        "        print('...index of', w, ':', tokenizer.word_index.get(w))\n",
        "    print('text reconstruction:')\n",
        "    reverse_word_index = dict([(v, k) for k, v in tokenizer.word_index.items()])\n",
        "    print(' '.join(reverse_word_index[i] for i in X[10]))\n",
        "\n",
        "    print(collections.Counter(labels))\n",
        "    print(\"number of labels:\", len(labels), \"\\n\")\n",
        "    print(\"number of data points:\", len(X), \"\\n\")\n",
        "    for i in range(len(X)):\n",
        "      print(\"data point \", i, \":\", X[i], labels[i])\n",
        "      if i==3:\n",
        "        break\n",
        "    dataset = (X, labels)\n",
        "    print('-')\n",
        "    \n",
        "    for a in dataset:\n",
        "      print(a[0], a[1])\n",
        "      \n",
        "if __name__ == \"__main__\":\n",
        "    make_reuters_dataset()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "earn: 3972\n",
            "acq: 2423\n",
            "money-fx: 682\n",
            "crude: 543\n",
            "grain: 537\n",
            "trade: 473\n",
            "interest: 339\n",
            "ship: 209\n",
            "money-supply: 177\n",
            "sugar: 154\n",
            "gnp: 127\n",
            "coffee: 126\n",
            "gold: 123\n",
            "veg-oil: 94\n",
            "cpi: 86\n",
            "oilseed: 81\n",
            "cocoa: 67\n",
            "copper: 62\n",
            "reserves: 62\n",
            "bop: 60\n",
            "livestock: 58\n",
            "ipi: 57\n",
            "jobs: 57\n",
            "alum: 53\n",
            "iron-steel: 52\n",
            "nat-gas: 51\n",
            "dlr: 46\n",
            "rubber: 42\n",
            "gas: 38\n",
            "tin: 32\n",
            "carcass: 29\n",
            "pet-chem: 29\n",
            "cotton: 28\n",
            "wpi: 27\n",
            "retail: 23\n",
            "wheat: 22\n",
            "meal-feed: 22\n",
            "orange: 22\n",
            "zinc: 21\n",
            "housing: 19\n",
            "strategic-metal: 19\n",
            "lead: 19\n",
            "hog: 17\n",
            "heat: 16\n",
            "lei: 16\n",
            "silver: 16\n",
            "lumber: 13\n",
            "fuel: 13\n",
            "income: 13\n",
            "tea: 9\n",
            "corn: 9\n",
            "instal-debt: 7\n",
            "yen: 6\n",
            "soybean: 5\n",
            "potato: 5\n",
            "nickel: 5\n",
            "stg: 5\n",
            "cpu: 4\n",
            "jet: 4\n",
            "platinum: 4\n",
            "propane: 3\n",
            "rice: 3\n",
            "inventories: 3\n",
            "palm-oil: 3\n",
            "groundnut: 3\n",
            "l-cattle: 2\n",
            "coconut: 2\n",
            "rapeseed: 2\n",
            "plywood: 2\n",
            "f-cattle: 2\n",
            "fishmeal: 1\n",
            "tapioca: 1\n",
            "rand: 1\n",
            "saudriyal: 1\n",
            "nzdlr: 1\n",
            "wool: 1\n",
            "austdlr: 1\n",
            "soy-meal: 1\n",
            "barley: 1\n",
            "cruzado: 1\n",
            "hk: 1\n",
            "naphtha: 1\n",
            "-\n",
            "Kept topics: 46\n",
            "Kept wires: 11228\n",
            "-\n",
            "Topic mapping: [('cocoa', 0), ('grain', 1), ('veg-oil', 2), ('earn', 3), ('acq', 4), ('wheat', 5), ('copper', 6), ('housing', 7), ('money-supply', 8), ('coffee', 9), ('sugar', 10), ('trade', 11), ('reserves', 12), ('ship', 13), ('cotton', 14), ('carcass', 15), ('crude', 16), ('nat-gas', 17), ('cpi', 18), ('money-fx', 19), ('interest', 20), ('gnp', 21), ('meal-feed', 22), ('alum', 23), ('oilseed', 24), ('gold', 25), ('tin', 26), ('strategic-metal', 27), ('livestock', 28), ('retail', 29), ('ipi', 30), ('iron-steel', 31), ('rubber', 32), ('heat', 33), ('jobs', 34), ('lei', 35), ('bop', 36), ('zinc', 37), ('orange', 38), ('pet-chem', 39), ('dlr', 40), ('gas', 41), ('silver', 42), ('wpi', 43), ('hog', 44), ('lead', 45)]\n",
            "-\n",
            "Sanity check:\n",
            "...index of banana : 19764\n",
            "...index of oil : 62\n",
            "...index of chocolate : 7202\n",
            "...index of the : 1\n",
            "...index of dsft : None\n",
            "text reconstruction:\n",
            "the commodity credit corporation ccc has accepted an export bonus offer to cover the sale of 37 000 long tons of wheat flour to north yemen the u s agriculture department said the wheat four is for shipment march may and the bonus awarded was 119 05 dlrs per tonnes and will be paid in the form of commodities from the ccc inventory the bonus was awarded to the pillsbury company the wheat flour purchases complete the export enhancement program initiative announced in april 1986 it said reuter 3\n",
            "Counter({3: 3972, 4: 2423, 19: 682, 16: 543, 1: 537, 11: 473, 20: 339, 13: 209, 8: 177, 10: 154, 21: 127, 9: 126, 25: 123, 2: 94, 18: 86, 24: 81, 0: 67, 6: 62, 12: 62, 36: 60, 28: 58, 30: 57, 34: 57, 23: 53, 31: 52, 17: 51, 40: 46, 32: 42, 41: 38, 26: 32, 15: 29, 39: 29, 14: 28, 43: 27, 29: 23, 5: 22, 22: 22, 38: 22, 37: 21, 7: 19, 27: 19, 45: 19, 44: 17, 33: 16, 35: 16, 42: 16})\n",
            "number of labels: 11228 \n",
            "\n",
            "number of data points: 11228 \n",
            "\n",
            "data point  0 : [6164, 666, 1903, 1, 118, 4, 1, 6422, 662, 2330, 20672, 1, 1715, 221, 375, 107, 6, 2331, 1162, 10, 1, 1178, 5113, 567, 1109, 10066, 407, 54, 42, 82, 4419, 10996, 1800, 5, 4, 22, 1207, 903, 1, 1702, 242, 1074, 1, 5113, 30, 27, 578, 49, 26, 6693, 10, 1, 118, 270, 101, 269, 59, 2544, 3801, 1227, 2, 373, 2274, 529, 7, 1357, 132, 10, 1, 756, 2, 41, 1031, 8, 108, 41, 1058, 25, 1, 311, 1878, 51, 26, 928, 13, 2275, 21, 662, 2766, 154, 18, 13796, 31, 773, 4, 1, 6693, 282, 10996, 1800, 5, 141, 20, 322, 123, 2891, 40, 3, 695, 386, 1351, 460, 662, 20, 322, 551, 40, 3291, 37, 10997, 712, 3, 39, 150, 28, 132, 6422, 460, 824, 225, 56, 44, 8, 1227, 6, 85, 3802, 25, 964, 56, 29, 8, 141, 52, 7, 800, 6423, 4677, 1227, 322, 4, 1, 1904, 2, 680, 16260, 968, 6, 2806, 141, 52, 4550, 40, 3, 695, 386, 2, 49, 662, 38, 27, 4205, 10, 195, 40, 6959, 52, 186, 7247, 20673, 4, 4420, 6422, 5293, 1164, 4, 1102, 2, 1, 243, 1341, 114, 326, 584, 680, 54, 382, 7, 543, 353, 2, 122, 662, 378, 18, 13796, 10996, 1800, 5, 1088, 5689, 84, 135, 3, 2332, 3, 1267, 8352, 87, 20674, 2, 99, 2274, 5689, 6959, 59, 3478, 3, 113, 2307, 947, 6, 207, 919, 85, 59, 4206, 10, 91, 947, 25, 16, 1435, 3, 16, 4207, 12, 87, 659, 3, 1828, 3, 27, 1202, 70, 460, 85, 59, 72, 1068, 6, 134, 3, 496, 1828, 28, 248, 424, 811, 25, 16, 2219, 6, 16, 4950, 12, 6, 25, 349, 6, 422, 12, 133, 70, 415, 424, 2979, 1801, 25, 16, 4421, 16, 2850, 6, 16, 4950, 12, 87, 659, 2096, 5690, 85, 2, 6165, 59, 167, 91, 90, 382, 25, 44, 2332, 44, 3632, 6, 44, 1267, 12, 90, 88, 6165, 1593, 25, 29, 315, 1144, 70, 415, 88, 248, 424, 25, 44, 623, 6, 44, 5294, 12, 2979, 1801, 25, 44, 4678, 3, 44, 1656, 12, 6, 25, 29, 315, 6, 29, 281, 1144, 70, 415, 1801, 6, 2660, 1548, 25, 44, 1916, 12, 6, 29, 315, 1144, 70, 415, 1548, 10996, 1800, 5, 3028, 59, 1, 34, 35, 20675, 224, 711, 2730, 6, 496, 1828, 4679, 85, 59, 1569, 25, 3189, 3, 5940, 12, 10, 91, 90, 3189, 12, 10, 88, 5295, 12, 10, 2979, 6, 67, 744, 1144, 70, 415, 1548, 10, 2660, 1548, 1317, 59, 1, 34, 35, 2545, 2730, 6, 1047, 224, 711, 7593, 85, 59, 919, 28, 91, 90, 657, 25, 29, 2807, 6, 29, 2689, 12, 248, 424, 25, 29, 2276, 12, 6, 25, 16, 144, 1144, 70, 415, 424, 2979, 1801, 25, 29, 623, 12, 6, 25, 16, 144, 1144, 70, 415, 1801, 6, 2660, 1548, 25, 16, 144, 1144, 70, 415, 1548, 10996, 1800, 5, 132, 6422, 85, 52, 589, 345, 25, 56, 162, 8, 1227, 108, 1, 45, 190, 460, 6, 16, 1416, 8, 1227, 108, 1, 86, 540, 460, 579, 282, 10, 1, 242, 3, 101, 281, 52, 130, 3, 27, 2003, 24, 1, 1053, 662, 63, 256, 89, 10067, 47, 2077, 3551, 18, 101, 315, 14, 9] 0\n",
            "data point  1 : [1, 34, 35, 306, 210, 254, 1, 4819, 321, 406, 229, 139, 286, 302, 93, 247, 101, 144, 40, 1231, 12, 4551, 1777, 6960, 12089, 374, 1477, 1059, 10068, 98, 1709, 317, 93, 93, 226, 29, 575, 29, 276, 2428, 44, 563, 2277, 44, 563, 7248, 44, 422, 396, 16, 349, 16, 663, 2428, 9, 99, 9, 99, 2277, 9, 144, 1709, 45, 137, 12089, 374, 1477, 1059, 10068, 98, 1709, 317, 93, 93, 3479, 16, 293, 67, 812, 2277, 16, 563, 1422, 647, 7, 16, 755, 2428, 29, 575, 29, 575, 2277, 29, 563, 1777, 29, 489, 9, 144, 2461, 2428, 41, 594, 41, 594, 2277, 41, 748, 279, 265, 3891, 6, 4552, 54, 10069, 317, 2428, 1963, 357, 1291, 89, 2660, 56, 1964, 10, 7594, 6, 89, 424, 318, 1964, 10, 226, 317, 2277, 226, 1422, 89, 41, 183, 989, 396, 1777, 89, 58, 16, 989, 317, 7248, 4007, 226, 1291, 89, 107, 255, 1043, 1709, 45, 137, 2461, 12, 87, 6960, 249, 1594, 647, 7, 42, 551, 14, 9] 1\n",
            "data point  2 : [3127, 357, 163, 282, 548, 460, 2394, 2, 1640, 2429, 6, 122, 244, 3, 101, 156, 4, 3633, 2, 79, 2462, 464, 10, 477, 895, 136, 45, 190, 132, 6, 102, 457, 132, 3, 101, 131, 45, 4, 4101, 5503, 226, 2430, 16, 6424, 60, 1208, 4820, 67, 91, 2661, 56, 132, 29, 3803, 44, 44, 2630, 67, 1060, 5296, 717, 67, 132, 717, 67, 316, 1777, 316, 316, 3400, 195, 2394, 59, 4680, 132, 99, 67, 58, 69, 702, 88, 116, 67, 132, 116, 67, 316, 1, 163, 72, 2690, 195, 2394, 10, 12090, 40, 1231, 12090, 226, 2430, 744, 69, 1208, 717, 58, 91, 162, 29, 5297, 80, 67, 132, 1984, 60, 989, 58, 7595, 2430, 489, 60, 1208, 469, 69, 5296, 56, 60, 5297, 56, 9, 132, 398, 60, 190, 44, 702, 2430, 249, 69, 1208, 422, 16, 5296, 316, 5297, 316, 88, 116, 67, 132, 2980, 16, 3480, 41, 4680, 2430, 717, 56, 1208, 855, 41, 5296, 144, 16, 5297, 183, 41, 132, 1829, 60, 1436, 9, 1437, 62, 2394, 59, 20676, 2430, 536, 44, 1208, 2563, 9, 5296, 293, 41, 5297, 9, 29, 88, 316, 10998, 80, 67, 132, 2731, 44, 2078, 56, 20677, 2430, 99, 69, 1208, 318, 56, 5296, 116, 44, 5297, 29, 67, 132, 855, 60, 950, 16, 702, 62, 2430, 9, 58, 1208, 358, 16, 5296, 316, 5297, 29, 67, 88, 69, 67, 10998, 162, 67, 16261, 58, 67, 132, 575, 60, 499, 58, 14, 9] 2\n",
            "data point  3 : [5504, 244, 68, 5, 22, 163, 2, 859, 520, 7, 75, 10, 65, 77, 465, 2, 22, 172, 76, 10, 258, 2, 143, 40, 2, 90, 16, 86, 1, 46, 72, 5, 22, 163, 1749, 3, 2936, 3, 258, 25, 1, 300, 160, 90, 318, 39, 145, 4, 1, 1527, 241, 77, 19, 139, 8, 3, 144, 8, 76, 14, 9] 3\n",
            "-\n",
            "[6164, 666, 1903, 1, 118, 4, 1, 6422, 662, 2330, 20672, 1, 1715, 221, 375, 107, 6, 2331, 1162, 10, 1, 1178, 5113, 567, 1109, 10066, 407, 54, 42, 82, 4419, 10996, 1800, 5, 4, 22, 1207, 903, 1, 1702, 242, 1074, 1, 5113, 30, 27, 578, 49, 26, 6693, 10, 1, 118, 270, 101, 269, 59, 2544, 3801, 1227, 2, 373, 2274, 529, 7, 1357, 132, 10, 1, 756, 2, 41, 1031, 8, 108, 41, 1058, 25, 1, 311, 1878, 51, 26, 928, 13, 2275, 21, 662, 2766, 154, 18, 13796, 31, 773, 4, 1, 6693, 282, 10996, 1800, 5, 141, 20, 322, 123, 2891, 40, 3, 695, 386, 1351, 460, 662, 20, 322, 551, 40, 3291, 37, 10997, 712, 3, 39, 150, 28, 132, 6422, 460, 824, 225, 56, 44, 8, 1227, 6, 85, 3802, 25, 964, 56, 29, 8, 141, 52, 7, 800, 6423, 4677, 1227, 322, 4, 1, 1904, 2, 680, 16260, 968, 6, 2806, 141, 52, 4550, 40, 3, 695, 386, 2, 49, 662, 38, 27, 4205, 10, 195, 40, 6959, 52, 186, 7247, 20673, 4, 4420, 6422, 5293, 1164, 4, 1102, 2, 1, 243, 1341, 114, 326, 584, 680, 54, 382, 7, 543, 353, 2, 122, 662, 378, 18, 13796, 10996, 1800, 5, 1088, 5689, 84, 135, 3, 2332, 3, 1267, 8352, 87, 20674, 2, 99, 2274, 5689, 6959, 59, 3478, 3, 113, 2307, 947, 6, 207, 919, 85, 59, 4206, 10, 91, 947, 25, 16, 1435, 3, 16, 4207, 12, 87, 659, 3, 1828, 3, 27, 1202, 70, 460, 85, 59, 72, 1068, 6, 134, 3, 496, 1828, 28, 248, 424, 811, 25, 16, 2219, 6, 16, 4950, 12, 6, 25, 349, 6, 422, 12, 133, 70, 415, 424, 2979, 1801, 25, 16, 4421, 16, 2850, 6, 16, 4950, 12, 87, 659, 2096, 5690, 85, 2, 6165, 59, 167, 91, 90, 382, 25, 44, 2332, 44, 3632, 6, 44, 1267, 12, 90, 88, 6165, 1593, 25, 29, 315, 1144, 70, 415, 88, 248, 424, 25, 44, 623, 6, 44, 5294, 12, 2979, 1801, 25, 44, 4678, 3, 44, 1656, 12, 6, 25, 29, 315, 6, 29, 281, 1144, 70, 415, 1801, 6, 2660, 1548, 25, 44, 1916, 12, 6, 29, 315, 1144, 70, 415, 1548, 10996, 1800, 5, 3028, 59, 1, 34, 35, 20675, 224, 711, 2730, 6, 496, 1828, 4679, 85, 59, 1569, 25, 3189, 3, 5940, 12, 10, 91, 90, 3189, 12, 10, 88, 5295, 12, 10, 2979, 6, 67, 744, 1144, 70, 415, 1548, 10, 2660, 1548, 1317, 59, 1, 34, 35, 2545, 2730, 6, 1047, 224, 711, 7593, 85, 59, 919, 28, 91, 90, 657, 25, 29, 2807, 6, 29, 2689, 12, 248, 424, 25, 29, 2276, 12, 6, 25, 16, 144, 1144, 70, 415, 424, 2979, 1801, 25, 29, 623, 12, 6, 25, 16, 144, 1144, 70, 415, 1801, 6, 2660, 1548, 25, 16, 144, 1144, 70, 415, 1548, 10996, 1800, 5, 132, 6422, 85, 52, 589, 345, 25, 56, 162, 8, 1227, 108, 1, 45, 190, 460, 6, 16, 1416, 8, 1227, 108, 1, 86, 540, 460, 579, 282, 10, 1, 242, 3, 101, 281, 52, 130, 3, 27, 2003, 24, 1, 1053, 662, 63, 256, 89, 10067, 47, 2077, 3551, 18, 101, 315, 14, 9] [1, 34, 35, 306, 210, 254, 1, 4819, 321, 406, 229, 139, 286, 302, 93, 247, 101, 144, 40, 1231, 12, 4551, 1777, 6960, 12089, 374, 1477, 1059, 10068, 98, 1709, 317, 93, 93, 226, 29, 575, 29, 276, 2428, 44, 563, 2277, 44, 563, 7248, 44, 422, 396, 16, 349, 16, 663, 2428, 9, 99, 9, 99, 2277, 9, 144, 1709, 45, 137, 12089, 374, 1477, 1059, 10068, 98, 1709, 317, 93, 93, 3479, 16, 293, 67, 812, 2277, 16, 563, 1422, 647, 7, 16, 755, 2428, 29, 575, 29, 575, 2277, 29, 563, 1777, 29, 489, 9, 144, 2461, 2428, 41, 594, 41, 594, 2277, 41, 748, 279, 265, 3891, 6, 4552, 54, 10069, 317, 2428, 1963, 357, 1291, 89, 2660, 56, 1964, 10, 7594, 6, 89, 424, 318, 1964, 10, 226, 317, 2277, 226, 1422, 89, 41, 183, 989, 396, 1777, 89, 58, 16, 989, 317, 7248, 4007, 226, 1291, 89, 107, 255, 1043, 1709, 45, 137, 2461, 12, 87, 6960, 249, 1594, 647, 7, 42, 551, 14, 9]\n",
            "0 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mQpBR3peeawo"
      },
      "source": [
        "import os, sys\n",
        "import keras\n",
        "import numpy as np\n",
        "import collections\n",
        "import statistics\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bzdracYEevtM",
        "outputId": "476a4c34-b22f-4f50-a5ae-9794ac5ca99e"
      },
      "source": [
        "(X_train, y_train), (X_test, y_test) = keras.datasets.reuters.load_data()\n",
        "\n",
        "print(X_train.shape)\n",
        "print(y_test.shape)\n",
        "print(max(len(e) for e in X_train))\n",
        "print(max(len(e) for e in X_test))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/reuters.npz\n",
            "2113536/2110848 [==============================] - 0s 0us/step\n",
            "2121728/2110848 [==============================] - 0s 0us/step\n",
            "(8982,)\n",
            "(2246,)\n",
            "2376\n",
            "1032\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "od5w0CfCfvRO",
        "outputId": "1dd7ac19-b249-44c4-9b09-7dc58d62ed43"
      },
      "source": [
        "print(\"Raw training entry No 0: {}\".format(X_train[0]))\n",
        "print\n",
        "print(\"Raw training label No 0: '{}'\".format(y_train[0]))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Raw training entry No 0: [1, 27595, 28842, 8, 43, 10, 447, 5, 25, 207, 270, 5, 3095, 111, 16, 369, 186, 90, 67, 7, 89, 5, 19, 102, 6, 19, 124, 15, 90, 67, 84, 22, 482, 26, 7, 48, 4, 49, 8, 864, 39, 209, 154, 6, 151, 6, 83, 11, 15, 22, 155, 11, 15, 7, 48, 9, 4579, 1005, 504, 6, 258, 6, 272, 11, 15, 22, 134, 44, 11, 15, 16, 8, 197, 1245, 90, 67, 52, 29, 209, 30, 32, 132, 6, 109, 15, 17, 12]\n",
            "Raw training label No 0: '3'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5OTDVK2egHs_",
        "outputId": "c7255d4d-4756-4e03-dacb-dbfee1fb8b8b"
      },
      "source": [
        "raw_word_index = keras.datasets.reuters.get_word_index()\n",
        "word_index = {v+3:k for k,v in raw_word_index.items()}\n",
        "word_index[0] = '-PAD-'\n",
        "word_index[1] = '-START-'\n",
        "word_index[2] = '-UNK-'\n",
        "\n",
        "# Reconstruct train data entry as string\n",
        "entry = 202\n",
        "print(\"Newswire category: {}\".format(y_train[entry]))\n",
        "print(\" \".join(word_index.get(w, 2) for w in X_train[entry]))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/reuters_word_index.json\n",
            "557056/550378 [==============================] - 0s 0us/step\n",
            "565248/550378 [==============================] - 0s 0us/step\n",
            "Newswire category: 21\n",
            "-START- high labour costs and slower corporate investment could hinder sweden's economic growth after 1987 the organisation for economic cooperation and development oecd said the swedish economy grew at a slower rate in 1986 than in previous years gdp rose about 1 7 pct in 1986 compared with 2 2 pct in 1985 but this growth depended largely on external factors particularly lower oil prices the oecd secretariat said in its latest annual report on sweden it warned that labour costs had risen more rapidly in sweden than in other oecd countries because of high labour costs swedish industry which largely relies upon export markets was losing market share wages in the manufacturing sector grew by seven pct in 1986 in line with 1985 increases while public sector wages rose an estimated 9 2 pct in 1986 up from six pct in 1985 this was significantly higher than average wage increases of 3 75 pct for the seven largest members of the oecd in 1986 the report said wage moderation was central to maintaining economic growth in sweden it suggested that wage negotiations should be at least partly centralised to control the total wages bill and hold down inflation and unemployment levels helped by low oil prices and the government's tight fiscal policy inflation fell to just over three pct in 1986 from almost six pct in 1985 but remained higher than in most other oecd countries the report said unemployment a principal policy target was at 2 7 pct in 1986 in line with 1985's 2 8 pct and well below the oecd average of 8 6 pct the report said sweden's employment policies accounted for the high levels of wage inflation it also said that economic growth in 1986 relied increasingly on private consumption because corporate investment in machinery and equipment had shrunk the total volume of industrial investments dropped by two pct in 1986 with sharp declines in spending by the wood pulp and paper industries this compared to a 19 pct rise in 1985 when there was heavy investment in these industries the oecd said sweden should now make an effort to boost corporate investments and reduce its dependence on domestic consumption for economic growth it suggested there was room for reform in the tax system sweden should continue to cut public sector spending especially in local government to keep in line with its tighter fiscal policy the report said reuter 3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eO6DF_Sggnzz",
        "outputId": "c9257d6c-874d-4c5d-89bd-c4d5d1381621"
      },
      "source": [
        "category = 5\n",
        "\n",
        "train_elabels = [(c, i) for i,c in enumerate(y_train)]\n",
        "cat = [e[1] for e in filter(lambda x: x[0]==category, train_elabels)]\n",
        "print(\"Number of entries for category {}: {}\".format(category, len(cat)))\n",
        "print()\n",
        "for c in cat:\n",
        "    print(\" \".join(word_index.get(w, 2) for w in X_train[c]))\n",
        "    print()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of entries for category 5: 17\n",
            "\n",
            "-START- the european commission's decision to release an additional 300 000 tonnes of british intervention feed wheat for the home market will provide only moderate relief in an increasingly tight market traders said some operators had been anticipating a larger tonnage pointing out that at this week's u k intervention tender the market sought to buy 340 000 tonnes but only 126 000 tonnes were granted the new tranche of intervention grain is unlikely to satisfy demand they said and keen buying competition for supplies in stores is expected to keep prices firm the release of the feed wheat followed recent strong representations by the u k grain trade to the commission there has been growing concern that rising internal prices triggered by heavy exports were creating areas of shortage in interior markets the latest ec authorisation will add 70 000 tonnes at the april 14 tender and a further 30 000 tonnes later in the month the remaining 200 000 tonnes will be made available in may and june news of the release produced an early downward reaction in local physical markets but by midday some sections had halved early two stg losses while others were unchanged ministry of agriculture figures for march indicate 1 85 mln tonnes of wheat and 1 74 mln tonnes of barley remain in the free market however some traders believe these figures are overstated and while some may still be held on the farm the bulk of wheat is already sold some of the grain is also off the market in futures stores a total of 2 10 mln tonnes of intervention wheat has been sold for export or to the home market since the season started july 1 leaving an unsold balance in intervention of about 1 59 mln tonnes intervention barley sales have reached just over 1 0 mln tonnes leaving about 753 000 tonnes traders said this season's u k export performance has surpassed all early expectations and has created the present nervous situation in domestic markets where the fear now is free market supplies may not last out until new crop becomes available in august the market is sticking to its recent prediction of total barley and wheat exports of around 10 5 mln tonnes a new record and nearly double the previous record of 5 9 mln tonnes achieved in the 1984 85 season traders expect u k wheat exports to reach 6 0 mln and barley around 4 50 mln tonnes the soviet union has booked a record total of 2 5 mln tonnes of british wheat and barley this season but only 1 28 mln had surfaced in customs export figures by march 25 traders said other ec countries have bought large amounts of british grain and for the july 1 march 25 period had taken 2 59 mln tonnes of wheat and 2 06 mln tonnes of barley this compares with 1 28 mln and 868 700 tonnes last season the market is expecting prices particularly wheat to stay buoyant for the remaining few months of the season if supplies become more difficult and prices strengthen further feed compounders may increase cereal substitute usage traders said reuter 3\n",
            "\n",
            "-START- a total of 126 031 tonnes of u k intervention feed wheat was sold to the home market at this week's tender provisional results show the home grown cereals authority hgca said actual prices were not reported but the wheat was sold at or above the march intervention price of 119 17 stg per tonne grain traders sought to buy about 340 000 tonnes reuter 3\n",
            "\n",
            "-START- the u s agriculture department said it has accepted a bid for an export bonus to cover the sale of 50 000 tonnes of hard red winter wheat to sri lanka a bonus of 37 44 dlrs per tonne was awarded to continental grain co on the shipment scheduled for april 8 16 melvin sims usda general sales manager said an additional 10 000 tonnes of wheat are still available to sri lanka under the export enhancement program sims said reuter 3\n",
            "\n",
            "-START- speculation the united states will offer subsidized wheat to the soviet union appears to have reached a new level of intensity in the run up to secretary of state george shultz' visit later this month to moscow rumors of an impending deal have coursed through wheat markets since officials from the two countries held their customary semi annual grain talks in february moscow's decision at that time to reenter the u s corn market strengthened the perception of warming farm trade prospects shultz is set to arrive in moscow april 13 shultz' statement two weeks ago that he would not stand in the way of a wheat subsidy offer under the export enhancement program eep coupled with the announcement of his visit to moscow was interpreted by many grain trade representatives here as a clear signal that the reagan administration was preparing an offer administration officials in and out of the u s agriculture department have been extremely tight lipped about the prospects of a subsidy offer but usda officials for the most part have abandoned the contention the proposal is dormant suggesting that an offer while not a done deal is a live possibility prominent u s grain trade representatives many of whom asked not to be identified continue to maintain that an offer to subsidize four mln tonnes of wheat is imminent others who one month ago claimed a deal was not possible are saying they would not rule one out rep pat roberts r kan yesterday went so far as to predict a subsidy offer would be made within the next ten days to two weeks aides to roberts said he had spoken to republican leaders who had been in contact with administration officials richard fritz director of international marketing at u s wheat associates said he was confident an export enhancement offer would be made by the middle of this month fritz also said he thought the value of the bonus would end up being close to the offer washington made peking earlier this year when usda approved subsidies to china of around 36 dlrs per tonne on one mln tonnes of wheat some grain trade representatives say a four mln tonne wheat subsidy offer might help stimulate more soviet purchases of u s corn and open the door to u s sales of soybeans as ever one of the crucial sticking points in a wheat deal would appear to be price last summer the administration took the controversial step of offering the soviets subsidized wheat but were embarrassed when moscow spurned the proposal on the grounds that the 15 dlr per tonne subsidy still left u s wheat prices far above world market prices the administration's decision to set the subsidy level up front instead of accepting bids from exporters appeared to be a means of controlling the price while attempting to dampen criticism grain trade sources said nonetheless the pricing procedure did not prevent shultz from saying the soviets were chortling because washington was offering soviet housewives cheaper grain than that available to u s housewives the conventional wisdom among grain trade representatives here is that a general warming of relations between the two countries since last summer combined with continued hard times in the u s grain belt would favor a subsidy offer in addition the ussr has made it clear it would consider buying u s wheat if it were priced more competitively however observers have not forgotten the circumstances surrounding the administration's announcement of the wheat subsidy offer last summer up until the time of the announcment congressional and industry leaders were led to believe the white house had decided to expand the export enhancement program to include not only the soviets but also a much broader list of countries instead the administration scaled back the offer to include only the soviets that last minute change of heart adds a measure of uncertainty even to the predictions of those most convinced that the administration will not now pass up the opportunity to sell four mln tonnes of wheat to the soviet union reuter 3\n",
            "\n",
            "-START- egypt has been authorized to purchase about 125 000 tonnes of u s wheat flour under an existing pl 480 agreement the u s agriculture department said it may buy the wheat flour valued at 23 0 mln dlrs between april 8 and august 31 1987 and ship it by september 30 the department said reuter 3\n",
            "\n",
            "-START- speculation the united states will offer subsidized wheat to the soviet union appears to have reached a new level of intensity in the run up to secretary of state george shultz' visit later this month to moscow rumors of an impending deal have coursed through wheat markets since officials from the two countries held their customary semi annual grain talks in february moscow's decision at that time to reenter the u s corn market strengthened the perception of warming farm trade prospects shultz is set to arrive in moscow april 13 reuter 3\n",
            "\n",
            "-START- the u n food and agriculture organisation fao said global wheat and coarse grain output was likely to fall in 1987 but supplies would remain adequate to meet demand fao said in its monthly food outlook bulletin total world grain output was expected to fall 38 mln tonnes to 1 353 mln in 1987 due mainly to unusually high winter losses in the soviet union drought in china and reduced plantings in north america world cereal stocks at the end of 1986 87 were forecast to rise 47 mln tonnes to a record 452 mln tonnes softening the impact of reduced production but stocks are unevenly distributed with about 50 pct held by the u s thus the food security prospects in 1987 88 for many developing countries particularly in africa depend crucially on the outcome of this year's harvests fao said fao said world cereal supplies in 1986 87 were estimated at a record 2 113 mln tonnes about five pct higher than last season and due mainly to large stocks and a record 1986 harvest estimated at 1 865 mln tonnes fao's forecast of 1986 87 world cereals trade was revised upwards by eight mln tonnes to 179 mln due to the likelihood of substantial buying by china and the soviet union reuter 3\n",
            "\n",
            "-START- u s commerce secretary malcolm baldrige said the dispute between the u s and japan over computer chips will not develop into a full scale trade war i think both sides will try and make sure that this doesn't start a trade war as a matter of fact i'm positive that there won't be a trade war with japan baldrige said in a television interview president reagan last week imposed penalties over japan's alleged failure to adhere to a july 1986 agreement to stop selling semiconductors in third countries at below cost and to open its own market to u s made chips despite tokyo's displeasure with the penalties baldrige said japan would refrain from taking retaliatory steps that would lead to a trade war largely because japan itself would have the most to lose in such a conflict japan i think will because they have the means to do it face up to the fact that as the second largest industrial trading power in the world they have to live up to their responsibilities of free trade he said i think we'll work our way out of this he added it's not going to be easy but i guarantee you we're not going to have a trade war reuter 3\n",
            "\n",
            "-START- a cold air mass working its way south from canada may pose a threat to developing hard red winter wheat in oklahoma according to eugene krenzler wheat specialist for the oklahoma cooperative extension there is some threat some of the crop is far enough along so that it's probably vulnerable krenzler said accu weather meteorologist dale mohler said cold air moving from the north could put temperatures in the middle 20's fahrenheit as early as tonight depending on development of a low pressure area over southeast colorado which could stall the system if impeded by the low pressure area the cold air could hit oklahoma and texas as late as monday morning temperatures could stay in the mid 20's for up to eight hours mohler said krenzler said less than 10 pct of the oklahoma wheat crop has advanced to boot stage the closer to that stage the more vulnerable the head of the wheat is to cold weather he said we can handle probably an hour or so down to 25 or 26 degrees fahrenheit krenzler said as long as we don't have a lot of wind if we do get six hours down below 25 degrees i'd say we have a good chance of significant damage to the heads he said krenzler said early planted stands in the north central and southwestern parts of the state are probably most vulnerable crops in the panhandle of oklahoma and texas are less developed and have some snow cover protection from the cold he noted reuter 3\n",
            "\n",
            "-START- yugoslavia will tender april 14 for 100 000 tonnes of wheat the u s agriculture department's counselor in belgrade said in a field report the report dated april 7 said the wheat must be from 1986 and 1987 harvest and imports of soft wheat from europe and from other suppliers will not be considered it said the imports will be used to rebuild the federal reserves and as a result will not be subject to import surcharges reuter 3\n",
            "\n",
            "-START- u k domestic wheat markets dropped about one stg per tonne early this morning following overnight news that the ec is releasing a further 300 000 tonnes of wheat from british intervention stores for the home market over a three month period april deliveries of denaturable wheat were offered in east anglia at 124 stg and may at 125 stg per tonne both one stg down on yesterday's traded rates the market is expecting u k wheat futures to show a similar loss at today's opening reuter 3\n",
            "\n",
            "-START- the u s agriculture department said it has accepted a bid for an export bonus to cover the sale of 18 000 tonnes of u s durum wheat to algeria a bonus of 43 25 dlrs per tonne was awarded to cam usa inc on the shipment scheduled for june 20 30 melvin sims usda general sales manager said an additional 228 000 tonnes of durum wheat are available to algeria under the department's export enhancement program sims said reuter 3\n",
            "\n",
            "-START- pakistan is not emerging as a major wheat exporter as world market prospects are not good enough sartaj aziz special assistant on food and agriculture to the pakistani prime minister said in an interview no exports are planned for the next 12 months or so and plans last year to sell one mln tonnes to iran came to nothing because they could not agree a price he said aziz forecast that pakistan may have exportable surpluses of one mln tonnes or a half mln tonnes over the next few harvests in years when the weather is favourable the government does not wish to increase output much above this because of low world prices and the land would be better used for other crops aziz said the pakistani government does not want area sown to wheat to increase from the current seven mln hectares some 10 pct of that area which gives low yields could be switched to more profitable crops such as oilseeds the aim is to concentrate on raising yields from the current 1 8 to 1 9 tonnes per ha to at least 2 5 tonnes per ha over the next five to seven years he said aziz said the current 1986 87 crop harvesting of which is just beginning is expected to yield around a record 14 5 mln tonnes this compares with a target of 14 7 mln and last year's yield of 14 0 mln he said rains some six weeks ago helped the crop but more recent rains reduced prospects slightly the long term wheat production target is for some 17 mln tonnes by mid 1993 taking into account pakistan's annual population growth rate of more than three pct current consumption is some 12 5 mln tonnes the current wheat reserve is 2 5 mln tonnes aziz said this compares with a minimum reserve commitment of one mln tonnes which pakistan will maintain at all costs and a strategic reserve target of two mln tonnes despite the fact that stocks are a half mln tonnes over target the surplus will not be exported at present he said the government wants to keep an extra safety margin until it sees what effect the abolition of a 44 year old wheat rationing system will have on domestic consumption new exports will be considered only in about a year's time when the 1987 88 crop can be gauged as well he said the new domestic policy introduced on march 15 is for the government to supply unlimited quantities of wheat at two rupees per kilo with other costs this means a price in pakistani markets of between 2 30 and 2 50 rupees per kilo under the old system introduced during world war two and due to be phased out by april 15 some 50 mln ration cards were issued enabling poor people to buy wheat cheaply aziz said following the introduction of a government support price in the 1970s the system become so corrupted that only 20 to 25 pct of subsidised wheat was actually reaching the consumer the rest being diverted illicitly to the mills the ration system had also not had the stabilising effect on the internal wheat market that was intended aziz said prices have already begun to fall with the introduction of the new system the wheat price in karachi the most expensive pakistani city had dropped from 3 11 rupees per kilo on march 1 to 2 85 rupees on march 30 aziz said he does not expect the change in system to have a major effect on total consumption but it may encourage better use of side products such as bran reuter 3\n",
            "\n",
            "-START- the commodity credit corporation ccc has accepted an export bonus offer to cover the sale of 37 000 long tons of wheat flour to north yemen the u s agriculture department said the wheat four is for shipment march may and the bonus awarded was 119 05 dlrs per tonnes and will be paid in the form of commodities from the ccc inventory the bonus was awarded to the pillsbury company the wheat flour purchases complete the export enhancement program initiative announced in april 1986 it said reuter 3\n",
            "\n",
            "-START- the united nations food and agriculture organization fao said it approved emergency food aid worth more than 14 3 mln dlrs for drought victims in ethiopia the aid will include 30 000 tonnes of wheat and 1 200 tonnes of vegetable oil for farmers in the wollo and illubabor regions fao said it has also approved more than 1 4 mln dlrs of food aid for 8 000 families in sri lanka in addition 583 225 dlrs of aid will be made available to malawi to feed 96 700 people displaced from mozambique and a further 340 200 dlrs for cyclone victims in vanuatu in the south pacific reuter 3\n",
            "\n",
            "-START- french cereals exports through rouen port rose to 751 563 tonnes between march 1 and march 25 from 603 413 tonnes in the same 1986 period freight sources said the soviet union took 263 051 tonnes of wheat and barley saudi arabia 90 944 tonnes barley china 87 259 wheat algeria 64 896 wheat bangladesh 30 000 wheat morocco 27 500 maize greece 26 700 wheat and barley spain 25 124 wheat poland 24 683 wheat brazil 24 082 wheat italy 21 659 wheat cyprus 20 700 wheat and maize israel 16 500 maize and the u k 8 797 tonnes wheat six ships are loading 120 000 tonnes of wheat the sources said they include 30 000 tonnes for china 31 000 for the soviet union 25 000 for turkey and 35 000 for italy another ship is loading 17 000 tonnes of colza for the soviet union another 12 ships should arrive to load 344 000 tonnes of cereals by the end of the week six are to load 186 000 tonnes of wheat for the soviet union two will load 60 000 tonnes of barley for saudi arabia one 28 000 tonnes of wheat for china two 25 000 tonnes of wheat each for algeria and turkey and one 20 000 tonnes of wheat for italy another is expected to load 20 000 tonnes of colza for the soviet union flour exports through rouen rose to 23 457 tonnes in the 25 day period from 5 500 in the equivalent 1986 period the sources said sudan took 19 327 tonnes and west coast africa 4 130 four ships are currently loading 32 000 tonnes including 24 000 for egypt 6 000 for tanzania and 2 000 for mauritania a ship is expected later this week to load 12 000 tonnes for china reuter 3\n",
            "\n",
            "-START- sesostris the spanish subsidiary of the international grain trader dreyfus sold 18 000 tonnes of barley to greece for delivery from mediterranean ports from april 14 to 30 a company spokesman said he gave no details on price the barley has a 14 pct humidity reuter 3\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ea0w0AchEGg",
        "outputId": "a55f7673-26fa-4e18-d9b8-19648159aaa7"
      },
      "source": [
        "mapping = ['cocoa','grain','veg-oil','earn','acq','wheat','copper','housing','money-supply',\n",
        "           'coffee','sugar','trade','reserves','ship','cotton','carcass','crude','nat-gas',\n",
        "           'cpi','money-fx','interest','gnp','meal-feed','alum','oilseed','gold','tin',\n",
        "           'strategic-metal','livestock','retail','ipi','iron-steel','rubber','heat','jobs',\n",
        "           'lei','bop','zinc','orange','pet-chem','dlr','gas','silver','wpi','hog','lead']\n",
        "\n",
        "train_count = collections.Counter(y_train)\n",
        "test_count = collections.Counter(y_test)\n",
        "total_words = [statistics.mean([len(e) for e in X_train[y_train.flatten() == i]]) for i in range(46)]\n",
        "\n",
        "print(\"{:5s} {:20s} {:5s} {:5s}  {:7s}\".format(\" \"    ,\" \"         , \"Nr of\", \"docs\", \"Mean nr of words\"))\n",
        "print(\"{:5s} {:20s} {:5s}  {:5s} {:7s}\".format(\"Index\",\"Class name\", \"train\", \"test\", \"in train set\"))\n",
        "for i in range(46):\n",
        "    print(\"{:5d} {:20s} {:5d} {:5d}   {:6.2f}\".format(i,mapping[i], train_count[i], test_count[i], total_words[i]))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                           Nr of docs   Mean nr of words\n",
            "Index Class name           train  test  in train set\n",
            "    0 cocoa                   55    12   225.78\n",
            "    1 grain                  432   105   188.67\n",
            "    2 veg-oil                 74    20   184.86\n",
            "    3 earn                  3159   813    87.67\n",
            "    4 acq                   1949   474   135.83\n",
            "    5 wheat                   17     5   213.35\n",
            "    6 copper                  48    14   154.46\n",
            "    7 housing                 16     3   180.38\n",
            "    8 money-supply           139    38   191.48\n",
            "    9 coffee                 101    25   225.87\n",
            "   10 sugar                  124    30   184.73\n",
            "   11 trade                  390    83   253.80\n",
            "   12 reserves                49    13   186.92\n",
            "   13 ship                   172    37   164.66\n",
            "   14 cotton                  26     2   142.69\n",
            "   15 carcass                 20     9   170.45\n",
            "   16 crude                  444    99   219.79\n",
            "   17 nat-gas                 39    12   149.82\n",
            "   18 cpi                     66    20   146.85\n",
            "   19 money-fx               549   133   185.34\n",
            "   20 interest               269    70   201.00\n",
            "   21 gnp                    100    27   281.83\n",
            "   22 meal-feed               15     7   183.73\n",
            "   23 alum                    41    12   157.34\n",
            "   24 oilseed                 62    19   151.24\n",
            "   25 gold                    92    31   152.38\n",
            "   26 tin                     24     8   259.88\n",
            "   27 strategic-metal         15     4   145.27\n",
            "   28 livestock               48    10   177.48\n",
            "   29 retail                  19     4   258.32\n",
            "   30 ipi                     45    12   175.78\n",
            "   31 iron-steel              39    13   157.51\n",
            "   32 rubber                  32    10   207.44\n",
            "   33 heat                    11     5   115.55\n",
            "   34 jobs                    50     7   152.94\n",
            "   35 lei                     10     6   142.30\n",
            "   36 bop                     49    11   228.45\n",
            "   37 zinc                    19     2   164.74\n",
            "   38 orange                  19     3   130.21\n",
            "   39 pet-chem                24     5   153.96\n",
            "   40 dlr                     36    10   278.39\n",
            "   41 gas                     30     8   175.07\n",
            "   42 silver                  13     3   197.92\n",
            "   43 wpi                     21     6   152.71\n",
            "   44 hog                     12     5    90.75\n",
            "   45 lead                    18     1   159.89\n"
          ]
        }
      ]
    }
  ]
}