pokedex.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pokedex\n",
    "This notebook is used to collect the full pokedex dataset from Gen I to Gen VIII"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use bulbapedia.bulbagarden.net to collect the name of all the pokemon."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://bulbapedia.bulbagarden.net/wiki/List_of_Pokémon_by_National_Pokédex_number\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get(url)\n",
    "soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "rows = soup.find_all(\"tr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "for row in rows:\n",
    "    if len(row.find_all(\"th\")) > 1:\n",
    "        headers = [header.text.replace('\\n','').replace(' ','') for header in row.find_all(\"th\")]\n",
    "    elif len(row.find_all(\"td\")) > 1: \n",
    "        record = {}\n",
    "        for i in range(len(row.find_all(\"td\"))):\n",
    "            record[headers[i]] = row.find_all(\"td\")[i].text.replace('\\n','').replace(' ','')\n",
    "        df = df.append(pd.Series(record), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['Ndex', 'MS']\n",
    "df = df[cols]\n",
    "df.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "pokemon_names = df['MS'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "pokemon_names = [item.lower() for item in pokemon_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bulbasaur', 'ivysaur', 'venusaur', 'charmander', 'charmeleon']\n"
     ]
    }
   ],
   "source": [
    "print(pokemon_names[0:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will use the primary data source of Serebii.net using their Gen VIII pokedex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.serebii.net/pokedex-swsh/{}/#stats\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_number(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    num = table.find_all(\"tr\")[1].find_all(\"td\", class_=\"fooinfo\")[2].text.strip().split('\\n')[0].split(\"#\")[-1]\n",
    "    \n",
    "    return num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_name(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    name = table.find_all(\"tr\")[1].find_all(\"td\")[0].text\n",
    "    \n",
    "    return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_altnames(soup):\n",
    "    altnames = {}\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    rows = table.find_all(\"tr\")[1].find_all(\"td\")[1].find_all(\"tr\")\n",
    "    for row in rows:\n",
    "        key = row.find_all(\"td\")[0].text.strip().replace(\":\",\"\")\n",
    "        contents = row.find_all(\"td\")[1].contents\n",
    "        value = [x for x in contents if isinstance(x, type(contents[0]))]\n",
    "        altnames[key] = value\n",
    "    \n",
    "    return altnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_type(soup):\n",
    "    types = []\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    imgs = table.find_all(\"img\")\n",
    "    for img in imgs:\n",
    "        types.append(img[\"src\"].split(\"/\")[-1].split(\".\")[0])\n",
    "    \n",
    "    return types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gender(soup):\n",
    "    gender_ratios = {}\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    try:  #genderless pokemon will cause exception\n",
    "        rows = table.find_all(\"tr\")[1].find_all(\"td\", class_=\"fooinfo\")[3].contents[0].find_all(\"tr\")\n",
    "        for row in rows:\n",
    "            key = row.find_all(\"td\")[0].text.split(\" \")[0]\n",
    "            value = row.find_all(\"td\")[1].text\n",
    "            gender_ratios[key] = value\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "    return gender_ratios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_classification(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    classification = table.find_all(\"td\", class_=\"fooinfo\")[4].text\n",
    "    \n",
    "    return classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_height(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    height = table.find_all(\"td\", class_=\"fooinfo\")[5].text.split(\"\\t\")[-1]\n",
    "    \n",
    "    return height"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    weight = table.find_all(\"td\", class_=\"fooinfo\")[6].text.split(\"\\t\")[-1]\n",
    "    \n",
    "    return weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_capture_rate(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    rate = table.find_all(\"td\", class_=\"fooinfo\")[7].text.split(\"\\t\")[-1]\n",
    "    \n",
    "    return rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_base_egg_steps(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    steps = table.find_all(\"td\", class_=\"fooinfo\")[8].text.split(\"\\t\")[-1].replace(\",\",\"\")\n",
    "    \n",
    "    return steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_base_stats(soup):\n",
    "    base_stats = {}\n",
    "    stat_names = [\"HP\",\"Attack\",\"Defense\",\"Sp. Attack\",\"Sp. Defense\",\"Speed\"]\n",
    "\n",
    "    stats_indices = []\n",
    "    for row in soup.find_all(\"tr\"):\n",
    "        if 'Stats' == row.text.replace('\\n',''):\n",
    "            stats_indices.append(soup.find_all(\"tr\").index(row))\n",
    "\n",
    "    try:\n",
    "        columns = soup.find_all(\"tr\")[stats_indices[0]+2].find_all(\"td\")\n",
    "\n",
    "    except:\n",
    "        columns = soup.find_all(\"tr\")[stats_indices[1]+2].find_all(\"td\")\n",
    "\n",
    "    for i in range(len(stat_names)):\n",
    "        base_stats[stat_names[i]] = columns[i+1].text\n",
    "\n",
    "    return base_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_legendary_status(name):\n",
    "                \n",
    "    legendary_status = {\"sublegendary\": 0, \"legendary\": 0, \"mythical\": 0}\n",
    "    if name in status_dict[\"sublegendary\"]:\n",
    "        legendary_status[\"sublegendary\"] = 1\n",
    "    elif name in status_dict[\"legendary\"]:\n",
    "        legendary_status[\"legendary\"] = 1\n",
    "    elif name in status_dict[\"mythical\"]:\n",
    "        legendary_status[\"mythical\"] = 1\n",
    "    \n",
    "    return legendary_status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_experience_growth(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[2].find_all(\"tr\", recursive=False)[3]\n",
    "    exp = table.find_all(\"tr\", recursive=False)[3].td.contents[0].split(\" \")[0].replace(\",\",\"\")\n",
    "    \n",
    "    return exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_base_happiness(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[2].find_all(\"tr\", recursive=False)[3]\n",
    "    happiness = table.find_all(\"tr\", recursive=False)[3].find_all(\"td\",recursive=False)[1].text\n",
    "    \n",
    "    return happiness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_against(soup):\n",
    "    headers = []\n",
    "    against_dict = {}\n",
    "    links = soup.find_all(\"table\", class_=\"dextable\")[3].find_all(\"tr\",recursive=False)[1].find_all(\"a\")\n",
    "    for link in links:\n",
    "        header = link['href'].split(\"/\")[-1].split(\".\")[0]\n",
    "        headers.append(header)\n",
    "    \n",
    "    columns = soup.find_all(\"table\", class_=\"dextable\")[3].find_all(\"tr\",recursive=False)[2].find_all('td')\n",
    "    for i in range(len(headers)):\n",
    "        against_dict[headers[i]] = columns[i].text.split('*')[-1]\n",
    "    \n",
    "    return against_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_abilities(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"dextable\")[2]\n",
    "    abilities = table.find_all('tr')[0].text.split(\": \")[1].strip().split(\" - \")\n",
    "    \n",
    "    return abilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gen(number):\n",
    "    if int(number) <= 151:\n",
    "        gen = 'I'\n",
    "    elif int(number) <= 251:\n",
    "        gen = 'II'\n",
    "    elif int(number) <= 386:\n",
    "        gen = 'III'\n",
    "    elif int(number) <= 493:\n",
    "        gen = 'IV'\n",
    "    elif int(number) <= 649:\n",
    "        gen = 'V'\n",
    "    elif int(number) <= 721:\n",
    "        gen = 'VI'\n",
    "    elif int(number) <= 809:\n",
    "        gen = 'VII'\n",
    "    else:\n",
    "        gen = 'VIII'\n",
    "    \n",
    "    return gen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_description(name):\n",
    "    name = name.replace(\"'\",'').replace(' ','-').replace('.',' ').strip().replace(' ','-').replace(':','-')\n",
    "    if name == 'nidoran♀':\n",
    "        name = 'nidoran-female'\n",
    "    elif name == 'nidoran♂':\n",
    "        name = 'nidoran-male'\n",
    "    elif name == 'mimejr':\n",
    "        name = 'mime-jr'\n",
    "    elif name[:4] == 'tapu':\n",
    "        name = name[:4] + '-' + name[4:]\n",
    "    url = 'https://www.pokemon.com/us/pokedex/{}'\n",
    "    r = requests.get(url.format(name))\n",
    "    soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "    desc = soup.find('div', class_='version-descriptions active').find('p', class_='active').contents[0].strip()\n",
    "\n",
    "    return desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_evochain(soup):\n",
    "    table = soup.find_all(\"table\", class_=\"evochain\")[0]\n",
    "    chain = []\n",
    "    if len(table.find_all('tr')) == 1:\n",
    "        for img in table.find_all('img'):\n",
    "            try:\n",
    "                a = img['title']\n",
    "                chain.append(a)   \n",
    "            except:\n",
    "                try:\n",
    "                    a = img['alt']\n",
    "                    chain.append(a) \n",
    "                except:\n",
    "                    pass\n",
    "    \n",
    "    elif table.find('a')['href'].split('/')[1] == 'pokedex-sm':\n",
    "        row = table.find('tr')\n",
    "        for col in row.find_all('td')[:7]:\n",
    "            try:\n",
    "                a = col.find('img')['title'].strip()\n",
    "            except:\n",
    "                try:\n",
    "                    a = col.find('img')['alt'].strip()\n",
    "                except:\n",
    "                    number = col.find('a')['href'].split('/')[-1].split('.')[0]\n",
    "                    a = pokemon_names[int(number)-1].capitalize()\n",
    "            chain.append(a)\n",
    "    \n",
    "    else:\n",
    "        for col in table.find_all('td')[:7]:\n",
    "            try:\n",
    "                a = col.find('img')['title'].strip()\n",
    "            except:\n",
    "                try:\n",
    "                    a = col.find('img')['alt'].strip()\n",
    "                except:\n",
    "                    try:\n",
    "                        number = col.find('img')['src'].split('/')[-1].split('.')[0]\n",
    "                        a = pokemon_names[int(number)-1].capitalize()\n",
    "                    except:\n",
    "                        pass\n",
    "                    #a = col.find('a')['href'].split('/')[-1].capitalize()\n",
    "            chain.append(a)\n",
    "    \n",
    "    if len(chain)>1:\n",
    "        if chain[0] == chain[1]:\n",
    "            chain.pop(0)\n",
    "    \n",
    "        if len(chain)>3 and chain[2] == chain[3]:\n",
    "            try:\n",
    "                chain.pop(5)\n",
    "            except:\n",
    "                pass\n",
    "            chain.pop(4)\n",
    "            chain.pop(3)\n",
    "\n",
    "        try:\n",
    "            if chain[4] == chain[5]:\n",
    "                chain.pop(6)\n",
    "                chain.pop(5)\n",
    "        except:\n",
    "            pass \n",
    "    \n",
    "    return chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pokedex = pd.DataFrame()\n",
    "url = \"https://www.serebii.net/pokemon/legendary.shtml\"\n",
    "r = requests.get(url)\n",
    "soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "tables = soup.find_all(\"table\", class_=\"trainer\")\n",
    "status_list = [\"sublegendary\",\"legendary\",\"mythical\"]\n",
    "status_dict = {}\n",
    "for i in range(len(tables)):\n",
    "    status_dict[status_list[i]] = []\n",
    "    for row in tables[i].find_all(\"tr\", recursive=False)[1:-1]:\n",
    "        for column in row.find_all(\"td\", recursive=False):\n",
    "            item = column.find_all('td')[1].text\n",
    "            status_dict[status_list[i]].append(item)\n",
    "            \n",
    "for pokemon in pokemon_names:\n",
    "    stats = {}\n",
    "    \n",
    "    try:\n",
    "        url = \"https://www.serebii.net/pokedex-swsh/{}/#stats\"\n",
    "        r = requests.get(url.format(pokemon))\n",
    "        soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "        check = soup.find_all(\"table\", class_=\"dextable\")[1]\n",
    "    except:\n",
    "        url = \"https://www.serebii.net/pokedex-sm/{}.shtml\"\n",
    "        number = \"{0:0=3d}\".format(pokemon_names.index(pokemon)+1)\n",
    "        r = requests.get(url.format(number))\n",
    "        soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "    \n",
    "    stats['national_number'] = get_number(soup)\n",
    "    stats['gen'] = get_gen(stats['national_number'])\n",
    "    stats['english_name'] = get_name(soup)\n",
    "    stats['japanese_name'] = get_altnames(soup)['Japan'][0]\n",
    "    stats['primary_type'] = get_type(soup)[0]\n",
    "    stats['secondary_type'] = get_type(soup)[1] if len(get_type(soup))>1 else None \n",
    "    stats['percent_male'] = get_gender(soup)['Male'].replace('%','') if len(get_gender(soup))>1 else None\n",
    "    stats['percent_female'] = get_gender(soup)['Female'].replace('%','') if len(get_gender(soup))>1 else None\n",
    "    stats['classification'] = get_classification(soup)\n",
    "    stats['height_m'] = get_height(soup).split('m')[0]\n",
    "    stats['weight_kg'] = get_weight(soup).split('kg')[0]\n",
    "    stats['capture_rate'] = get_capture_rate(soup)\n",
    "    stats['base_egg_steps'] = get_base_egg_steps(soup)\n",
    "    stats['hp'] = get_base_stats(soup)[\"HP\"]\n",
    "    stats['attack'] = get_base_stats(soup)[\"Attack\"]\n",
    "    stats['defense'] = get_base_stats(soup)[\"Defense\"]\n",
    "    stats['sp_attack'] = get_base_stats(soup)[\"Sp. Attack\"]\n",
    "    stats['sp_defense'] = get_base_stats(soup)[\"Sp. Defense\"]\n",
    "    stats['speed'] = get_base_stats(soup)[\"Speed\"]\n",
    "    stats['description'] = get_description(pokemon)\n",
    "    \n",
    "    abilities = get_abilities(soup)\n",
    "    for i in range(3):\n",
    "        try:\n",
    "            if 'Hidden' not in abilities[i]:\n",
    "                stats['abilities_{}'.format(i)] = abilities[i]\n",
    "            elif 'Hidden' in abilities[i]:\n",
    "                stats['abilities_hidden'] = abilities[i].split('(')[0].strip()\n",
    "        except:\n",
    "             stats['abilities_{}'.format(i)] = None\n",
    "                \n",
    "    for i in range(7):\n",
    "        try:\n",
    "            stats['evochain_{}'.format(i)] = get_evochain(soup)[i]\n",
    "        except:\n",
    "            stats['evochain_{}'.format(i)] = None\n",
    "    \n",
    "    legend = get_legendary_status(stats['english_name'])\n",
    "    for i in legend.keys():\n",
    "        stats['is_{}'.format(i)] = legend[i]\n",
    "    \n",
    "    against = get_against(soup)\n",
    "    for i in against.keys():\n",
    "        stats['against_{}'.format(i)] = against[i]\n",
    "    \n",
    "    pokedex = pokedex.append(pd.Series(stats), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lele'"
      ]
     },
     "execution_count": 385,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pokemon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\n",
    "    'national_number',\n",
    "    'gen',\n",
    "    'english_name',\n",
    "    'japanese_name',\n",
    "    'primary_type',\n",
    "    'secondary_type',\n",
    "    'classification',\n",
    "    'percent_male',\n",
    "    'percent_female',\n",
    "    'height_m',\n",
    "    'weight_kg',\n",
    "    'capture_rate',\n",
    "    'base_egg_steps',\n",
    "    'hp',\n",
    "    'attack',\n",
    "    'defense',\n",
    "    'sp_attack',\n",
    "    'sp_defense',\n",
    "    'speed'\n",
    "    ]\n",
    "\n",
    "for i in range(3):\n",
    "    cols.append('abilities_{}'.format(i))\n",
    "\n",
    "cols.append('abilities_hidden')\n",
    "\n",
    "for i in against.keys():\n",
    "    cols.append('against_{}'.format(i))\n",
    "\n",
    "for i in legend.keys():\n",
    "    cols.append('is_{}'.format(i))\n",
    "\n",
    "for i in range(7):\n",
    "    cols.append('evochain_{}'.format(i))\n",
    "\n",
    "cols.append('description')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [],
   "source": [
    "pokedex[cols].to_csv('pokemon.csv', index=False, encoding='utf-16')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alter data set for radar plot "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('pokemon.csv', encoding='utf-16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.melt(df, \n",
    "        id_vars=['national_number','english_name'], \n",
    "        value_vars=['hp', 'attack', 'defense', 'sp_attack', 'sp_defense','speed'], \n",
    "        var_name='stat_name', \n",
    "        value_name='stat_value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('pokemon_stats.csv', index=False, encoding='utf-16')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Obtain small images of all 898 pokemon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "\n",
    "for i in range(898):\n",
    "    url = \"https://assets.pokemon.com/assets/cms2/img/pokedex/detail/{0:0=3d}.png\".format(i+1)\n",
    "    output = \"{0:0=3d}.png\".format(i+1)\n",
    "    urllib.request.urlretrieve(url, output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Obtain large images of all 898 pokemon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "cwd = os.getcwd()\n",
    "\n",
    "for i in range(len(pokemon_names)):\n",
    "    try:\n",
    "        url = \"https://bulbapedia.bulbagarden.net/wiki/File:{}.png\".format(\"{0:0=3d}\".format(i+1)+pokemon_names[i].replace(\"'\",\"%27\").replace(\".\",\"._\").capitalize())\n",
    "        r = requests.get(url)\n",
    "        soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "        div = soup.find(\"div\", class_=\"fullImageLink\")\n",
    "        target = div.a['href']\n",
    "        output = \"/images/{0:0=3d}.png\".format(i+1)\n",
    "    except:\n",
    "        try:\n",
    "            url = \"https://bulbapedia.bulbagarden.net/wiki/File:{}.png\".format(\"{0:0=3d}\".format(i+1)+pokemon_names[i].replace(\"'\",\"%27\").replace(\".\",\"._\").title())\n",
    "            r = requests.get(url)\n",
    "            soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "            div = soup.find(\"div\", class_=\"fullImageLink\")\n",
    "            target = div.a['href']\n",
    "            output = \"/images/{0:0=3d}.png\".format(i+1)\n",
    "        except:\n",
    "            if pokemon_names[i]=='mimejr.':\n",
    "                name = 'Mime_Jr'\n",
    "                url = \"https://bulbapedia.bulbagarden.net/wiki/File:{}.png\".format(\"{0:0=3d}\".format(i+1)+name)\n",
    "                r = requests.get(url)\n",
    "                soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "                div = soup.find(\"div\", class_=\"fullImageLink\")\n",
    "                target = div.a['href']\n",
    "                output = \"/images/{0:0=3d}.png\".format(i+1)\n",
    "            else:\n",
    "                name = \"Giratina\"\n",
    "                url = \"https://bulbapedia.bulbagarden.net/wiki/File:{}.png\".format(\"{0:0=3d}\".format(i+1)+name+\"-Origin\")\n",
    "                r = requests.get(url)\n",
    "                soup = BeautifulSoup(r.text, \"html.parser\")\n",
    "                div = soup.find(\"div\", class_=\"fullImageLink\")\n",
    "                target = div.a['href']\n",
    "                output = \"/images/{0:0=3d}.png\".format(i+1+485)\n",
    "                \n",
    "    try:\n",
    "        urllib.request.urlretrieve(target, output)\n",
    "    except:\n",
    "        if target[0:4] != 'http':\n",
    "            target = \"https:\" + target\n",
    "        r = requests.get(target)\n",
    "        with open(cwd+output, 'wb') as outfile:\n",
    "            outfile.write(r.content)\n",
    "            outfile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Collect alternative images for pokemon with alt forms (e.g. regional variants, mega evolution, gigantamax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://bulbapedia.bulbagarden.net/wiki/Mega_Evolution'\n",
    "base = 'https://bulbapedia.bulbagarden.net'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get(url)\n",
    "soup = BeautifulSoup(r.text, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "cwd = os.getcwd()\n",
    "\n",
    "for i in soup.find_all('a', class_='image'):\n",
    "    if '-Mega' in i['href']:\n",
    "        url = base+i['href']\n",
    "        output = i['href'].split(':')[-1]\n",
    "        r = requests.get(url)\n",
    "        soup2 = BeautifulSoup(r.text, \"html.parser\")\n",
    "        div = soup2.find(\"div\", class_=\"fullImageLink\")\n",
    "        target = div.a['href']\n",
    "        if target[0:4] != 'http':\n",
    "            target = \"https:\" + target\n",
    "        try:\n",
    "            urllib.request.urlretrieve(target, output)\n",
    "        except:\n",
    "            r = requests.get(target)\n",
    "        with open(cwd+output, 'wb') as outfile:\n",
    "            outfile.write(r.content)\n",
    "            outfile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://bulbapedia.bulbagarden.net/wiki/Primal_Reversion'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "cwd = os.getcwd()\n",
    "\n",
    "for i in soup.find_all('a', class_='image'):\n",
    "    if '-Primal' in i['href']:\n",
    "        url = base+i['href']\n",
    "        output = i['href'].split(':')[-1]\n",
    "        r = requests.get(url)\n",
    "        soup2 = BeautifulSoup(r.text, \"html.parser\")\n",
    "        div = soup2.find(\"div\", class_=\"fullImageLink\")\n",
    "        target = div.a['href']\n",
    "        if target[0:4] != 'http':\n",
    "            target = \"https:\" + target\n",
    "        try:\n",
    "            urllib.request.urlretrieve(target, output)\n",
    "        except:\n",
    "            r = requests.get(target)\n",
    "        with open(cwd+output, 'wb') as outfile:\n",
    "            outfile.write(r.content)\n",
    "            outfile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "url = 'https://bulbapedia.bulbagarden.net/wiki/List_of_Pok%C3%A9mon_with_form_differences'\n",
    "r = requests.get(url)\n",
    "soup = BeautifulSoup(r.text, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "cwd = os.getcwd()\n",
    "\n",
    "for i in soup.find_all('a', class_='image'):\n",
    "    if '-' in i['href'] and '.png' in i['href']:\n",
    "        url = base+i['href']\n",
    "        output = i['href'].split(':')[-1]\n",
    "        r = requests.get(url)\n",
    "        soup2 = BeautifulSoup(r.text, \"html.parser\")\n",
    "        div = soup2.find(\"div\", class_=\"fullImageLink\")\n",
    "        target = div.a['href']\n",
    "        if target[0:4] != 'http':\n",
    "            target = \"https:\" + target\n",
    "        try:\n",
    "            urllib.request.urlretrieve(target, output)\n",
    "        except:\n",
    "            r = requests.get(target)\n",
    "        with open(cwd+output, 'wb') as outfile:\n",
    "            outfile.write(r.content)\n",
    "            outfile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://bulbapedia.bulbagarden.net/wiki/Regional_form'\n",
    "r = requests.get(url)\n",
    "soup = BeautifulSoup(r.text, \"html.parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'There is a plant seed on its back right from the day this Pokémon is born. The seed slowly grows larger.'"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find('div', class_='version-descriptions active').find('p', class_='active').contents[0].strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "\n",
    "for i in range(898):\n",
    "    url = \"https://assets.pokemon.com/assets/cms2/img/pokedex/detail/{0:0=3d}.png\".format(i+1)\n",
    "    output = \"{0:0=3d}.png\".format(i+1)\n",
    "    urllib.request.urlretrieve(url, output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}