GHSL_downloader.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "import zipfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "unzip = True\n",
    "dst = 'data'\n",
    "\n",
    "years = [str(year) for year in range(1975, 2030 + 1, 5)]\n",
    "\n",
    "rows = (4, )\n",
    "cols = range(19, 20+1)\n",
    "\n",
    "root = 'https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL'\n",
    "\n",
    "products = {\n",
    "    'GHS_BUILT_S_GLOBE_R2023A': [f'GHS_BUILT_S_E{year}_GLOBE_R2023A_54009_100' for year in years] + \n",
    "                                [f'GHS_BUILT_S_NRES_E{year}_GLOBE_R2023A_54009_100' for year in years] +\n",
    "                                [f'GHS_BUILT_S_E2018_GLOBE_R2023A_54009_10'] +\n",
    "                                [f'GHS_BUILT_S_NRES_E2018_GLOBE_R2023A_54009_10'],\n",
    "    'GHS_BUILT_V_GLOBE_R2023A': [f'GHS_BUILT_V_E{year}_GLOBE_R2023A_54009_100' for year in years] + \n",
    "                                [f'GHS_BUILT_V_NRES_E{year}_GLOBE_R2023A_54009_100' for year in years] +\n",
    "                                [f'GHS_BUILT_V_NRES_E2018_GLOBE_R2023A_54009_10'] +\n",
    "                                [f'GHS_BUILT_V_E2018_GLOBE_R2023A_54009_10'],\n",
    "    'GHS_POP_GLOBE_R2023A': [f'GHS_POP_E{year}_GLOBE_R2023A_54009_100' for year in years],\n",
    "    'GHS_SMOD_GLOBE_R2023A': [f'GHS_SMOD_E{year}_GLOBE_R2023A_54009_1000' for year in years],\n",
    "    'GHS_BUILT_H_GLOBE_R2023A': ('GHS_BUILT_H_AGBH_E2018_GLOBE_R2023A_54009_100', 'GHS_BUILT_H_ANBH_E2018_GLOBE_R2023A_54009_100', ),\n",
    "    'GHS_BUILT_C_GLOBE_R2023A': ('GHS_BUILT_C_FUN_E2018_GLOBE_R2023A_54009_10', 'GHS_BUILT_C_MSZ_E2018_GLOBE_R2023A_54009_10', ),\n",
    "    'GHS_SDATA_GLOBE_R2023A': ('GHS_SDATA_LDS_QUANT_MT_GLOBE_R2023A_54009_100', ),\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for product in products:\n",
    "    datasets = products.get(product)\n",
    "    for dataset in datasets:\n",
    "        for row in rows:\n",
    "            for col in cols:\n",
    "                if product == 'GHS_SDATA_GLOBE_R2023A':\n",
    "                    item = f'{dataset}_R{row}_C{col}.zip'\n",
    "                else:\n",
    "                    item = f'{dataset}_V1_0_R{row}_C{col}.zip'\n",
    "                url = f'{root}/{product}/{dataset}/V1-0/tiles/{item}'\n",
    "                target = os.path.join(dst, product, dataset, item)\n",
    "                \n",
    "                if os.path.exists(target.removesuffix('.zip')):\n",
    "                    continue\n",
    "                else:\n",
    "                    os.makedirs(os.path.dirname(target), exist_ok=True)\n",
    "                    try:\n",
    "                        r = requests.get(url)\n",
    "                        if r:\n",
    "                            with open(target, 'wb') as f:\n",
    "                                f.write(r.content)\n",
    "                        else:\n",
    "                            continue\n",
    "\n",
    "                    except requests.exceptions.RequestException as e:\n",
    "                        print(target, e)\n",
    "                        continue\n",
    "\n",
    "                    if unzip:\n",
    "                        with zipfile.ZipFile(target, 'r') as zip_f:\n",
    "                            zip_f.extractall(target.removesuffix('.zip'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/GHS_BUILT_S_GLOBE_R2023A/GHS_BUILT_S_NRES_E2020_GLOBE_R2023A_54009_100/GHS_BUILT_S_NRES_E2020_GLOBE_R2023A_54009_100_V1_0_R3_C20.zip'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_urls = [\n",
    "    'https://geodata.ucdavis.edu/gadm/gadm4.1/gadm_410-gpkg.zip',\n",
    "    'https://ghsl.jrc.ec.europa.eu/download/GHSL_data_54009_shapefile.zip',\n",
    "    'https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL//GHS_DUC_GLOBE_R2023A/V1-0/GHS_DUC_MT_GLOBE_R2023A_V1_0.zip',\n",
    "    'https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_SDATA_GLOBE_R2023A/GHS_SDATA_WUP2018_BOUNDARIES_MT_GLOBE_R2023A/V1-0/GHS_SDATA_WUP2018_BOUNDARIES_MT_GLOBE_R2023A_V1_0.zip',\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for url in other_urls:\n",
    "    target = os.path.join(dst, 'other', os.path.basename(url))\n",
    "    if os.path.exists(target):\n",
    "        continue\n",
    "    else:\n",
    "        os.makedirs(os.path.dirname(target), exist_ok=True)\n",
    "        try:\n",
    "            r = requests.get(url)\n",
    "            with open(target, 'wb') as f:\n",
    "                f.write(r.content)\n",
    "\n",
    "            if unzip:\n",
    "                with zipfile.ZipFile(target, 'r') as zip_f:\n",
    "                    zip_f.extractall(target.removesuffix('.zip'))\n",
    "\n",
    "        except requests.exceptions.RequestException as e:\n",
    "            print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "vscode": {
     "languageId": "bat"
    }
   },
   "source": [
    "TEMPLATES/EXAMPLES:\n",
    "\n",
    "GHS_BUILT_S:\n",
    "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_S_GLOBE_R2023A/GHS_BUILT_S_E2030_GLOBE_R2023A_54009_100/V1-0/tiles/GHS_BUILT_S_E2030_GLOBE_R2023A_54009_100_V1_0_R4_C19.zip\n",
    "\n",
    "GHS_BUILT_H:\n",
    "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_H_GLOBE_R2023A/GHS_BUILT_H_AGBH_E2018_GLOBE_R2023A_54009_100/V1-0/tiles/GHS_BUILT_H_AGBH_E2018_GLOBE_R2023A_54009_100_V1_0_R4_C19.zip\n",
    "\n",
    "GHS_BUILT_V:\n",
    "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_BUILT_V_GLOBE_R2023A/GHS_BUILT_V_E2030_GLOBE_R2023A_54009_100/V1-0/tiles/GHS_BUILT_V_E2030_GLOBE_R2023A_54009_100_V1_0_R4_C19.zip\n",
    "\n",
    "\n",
    "GHS_SDATA:\n",
    "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_SDATA_GLOBE_R2023A/GHS_SDATA_LDS_QUANT_MT_GLOBE_R2023A_54009_100/V1-0/tiles/GHS_SDATA_LDS_QUANT_MT_GLOBE_R2023A_54009_100_R4_C19.zip"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}