solar_dataset_preparation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4R5ncJpCtmmn"
      },
      "source": [
        "# Solar Dataset Preparation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PIvkOvchuMNx"
      },
      "source": [
        "Run only one time.\n",
        "\n",
        "Downloads the WorldView-3 15 cm HD and 30 cm native imagery dataset of solar panels for Germany.\n",
        "\n",
        "Prepares the data to be used by YOLOv8."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RNPJ0GL1vB_Z"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ELcSbM2tvLoO"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Some libraries were ignored.\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import shutil\n",
        "from urllib.request import urlretrieve\n",
        "import glob\n",
        "from sklearn.model_selection import train_test_split\n",
        "try:\n",
        "    from google.colab import files \n",
        "    from google.colab import drive \n",
        "except:\n",
        "    print(\"Some libraries were ignored.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9jT6cPoMvCD8"
      },
      "source": [
        "## User Inputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "9sbySaXktm7M"
      },
      "outputs": [],
      "source": [
        "# Datasets folder\n",
        "# Folder where all the datasets will be saved\n",
        "datasets_folder = \"datasets\"\n",
        "\n",
        "# Dataset spatial resolution\n",
        "# \"native\" - Native 30 cm\n",
        "# \"hd\" - HD 15 cm\n",
        "spatial_res = \"native\"\n",
        "\n",
        "# Split\n",
        "# Separate the dataset into Train, Validation and Test (total is 100%)\n",
        "split = (50, 25, 25)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dcm4DnO2vP8H"
      },
      "source": [
        "## Code"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "wk0ZcDqitm-r"
      },
      "outputs": [],
      "source": [
        "# Create new folder (if does not exist) to store the downloaded dataset\n",
        "if not os.path.exists(datasets_folder):\n",
        "  os.mkdir(datasets_folder)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K_IkZ9gIxgXr",
        "outputId": "f156290c-c921-4d5e-f95d-9f43986b9598"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "('datasets/Maxar_HD_and_Native_Solar_Panel_Image_Chips.zip',\n",
              " <http.client.HTTPMessage at 0x7fd698467b20>)"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Download dataset (images_chips)\n",
        "# Be patient... Takes around 4 min with 900Mbps to download 3.85GB \n",
        "dataset_images_url = \"https://maxar-marketing.s3.amazonaws.com/product-samples/Maxar_HD_and_Native_Solar_Panel_Image_Chips.zip\"\n",
        "dataset_images_path = os.path.join(datasets_folder, \"Maxar_HD_and_Native_Solar_Panel_Image_Chips.zip\")\n",
        "urlretrieve(dataset_images_url, dataset_images_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "02ms8_4aypt0",
        "outputId": "6a364ea2-2c40-44e8-c87c-b75530c135ad"
      },
      "outputs": [],
      "source": [
        "# Unzip dataset (images_chips)\n",
        "# Quiet and overwrite\n",
        "# Be patient... Takes around 2 min to unzip\n",
        "!unzip -q -o {dataset_images_path} -d {datasets_folder}\n",
        "os.remove(dataset_images_path)\n",
        "os.remove(os.path.join(datasets_folder, \"README_ImageChips.txt\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XXYcR_a1KgiX",
        "outputId": "49440b7c-9058-4cce-cb81-d219665dd048"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "('datasets/labels.zip', <http.client.HTTPMessage at 0x7fd6984b49d0>)"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Download dataset (labels)\n",
        "# Be patient... Fast\n",
        "dataset_labels_url = \"https://figshare.com/ndownloader/files/39255599\"\n",
        "dataset_labels_path = os.path.join(datasets_folder, \"labels.zip\")\n",
        "urlretrieve(dataset_labels_url, dataset_labels_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "om2czyLp0bqX",
        "outputId": "a786c0c2-5e3c-48e3-b7b7-b21bbb09ac21"
      },
      "outputs": [],
      "source": [
        "# Unzip dataset (labels)\n",
        "# Quiet and overwrite\n",
        "!unzip -q -o {dataset_labels_path} -d {datasets_folder}\n",
        "os.remove(dataset_labels_path)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "rBrO_Gcd3DlO"
      },
      "outputs": [],
      "source": [
        "# Select folders based on spatial resolution\n",
        "images_folder = os.path.join(datasets_folder, \"image_chips\")\n",
        "labels_folder = os.path.join(datasets_folder, \"labels\")\n",
        "if spatial_res == \"hd\":\n",
        "  images_folder = os.path.join(images_folder, \"image_chips_hd\")\n",
        "  labels_folder = os.path.join(labels_folder, \"labels_hd\")\n",
        "else:\n",
        "  images_folder = os.path.join(images_folder, \"image_chips_native\")\n",
        "  labels_folder = os.path.join(labels_folder, \"labels_native\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qd5BJ0qfHSjx",
        "outputId": "d2e5db44-87e9-4b3b-90b1-8bb990f76ca4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of images files: 2542\n",
            "Number of labels files: 2542\n"
          ]
        }
      ],
      "source": [
        "# Check if number of images files match labels files\n",
        "images_list = glob.glob(os.path.join(images_folder, '*'))\n",
        "images_number = len(images_list)\n",
        "print(\"Number of images files: \" + str(images_number))\n",
        "\n",
        "labels_list = glob.glob(os.path.join(labels_folder, '*'))\n",
        "labels_number = len(labels_list)\n",
        "print(\"Number of labels files: \" + str(labels_number))\n",
        "assert images_number == labels_number, \"The number of files do not match.\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "TwpxnkiaNoBw"
      },
      "outputs": [],
      "source": [
        "# Save to a list only the names of all images\n",
        "images_names_list = []\n",
        "for image_path in images_list:\n",
        "    images_names_list.append(os.path.basename(image_path).split('.')[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "gk0EAyv1Npl2"
      },
      "outputs": [],
      "source": [
        "# Split into Train and Other\n",
        "x_train, x_other = train_test_split(images_names_list, random_state=42, test_size=(100-split[0])/100)\n",
        "\n",
        "# Split Other into Validation and Test\n",
        "x_valid, x_test = train_test_split(x_other, random_state=42, test_size=split[2]/(100-split[0]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "Vx8dBTwVNp3A"
      },
      "outputs": [],
      "source": [
        "# Create the Train-Validation-Test folders structure\n",
        "for folder_name in [\"train\", \"validation\", \"test\"]:\n",
        "    folder_path = os.path.join(datasets_folder, folder_name)\n",
        "    if not os.path.exists(folder_path):\n",
        "        os.mkdir(folder_path)\n",
        "    else:\n",
        "        shutil.rmtree(folder_path)\n",
        "        os.mkdir(folder_path)\n",
        "    os.mkdir(os.path.join(folder_path, \"images\"))\n",
        "    os.mkdir(os.path.join(folder_path, \"labels\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "skuk2FuqPI91"
      },
      "outputs": [],
      "source": [
        "# Function to copy files\n",
        "def copy_images_labels_files(images_folder, labels_folder, dst_folder, file_name):\n",
        "  # Images\n",
        "  image_src = os.path.join(images_folder, file_name+\".tif\")\n",
        "  image_dst = os.path.join(dst_folder, \"images\", file_name+\".tif\")\n",
        "  shutil.copy(image_src, image_dst)\n",
        "\n",
        "  # Labels\n",
        "  labels_src = os.path.join(labels_folder, file_name+\".txt\")\n",
        "  labels_dst = os.path.join(dst_folder, \"labels\", file_name+\".txt\")\n",
        "  shutil.copy(labels_src, labels_dst)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "TH7nfb9QO4ci"
      },
      "outputs": [],
      "source": [
        "# Copy the images and labels files\n",
        "# Train\n",
        "train_folder = os.path.join(datasets_folder, \"train\")\n",
        "for file_name in x_train:\n",
        "  copy_images_labels_files(images_folder, labels_folder, train_folder, file_name)\n",
        "\n",
        "# Validation\n",
        "valid_folder = os.path.join(datasets_folder, \"validation\")\n",
        "for file_name in x_valid:\n",
        "  copy_images_labels_files(images_folder, labels_folder, valid_folder, file_name)\n",
        "\n",
        "# Test\n",
        "test_folder = os.path.join(datasets_folder, \"test\")\n",
        "for file_name in x_test:\n",
        "  copy_images_labels_files(images_folder, labels_folder, test_folder, file_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y47okIbKQHei",
        "outputId": "47538029-2ab4-4c35-e2ed-43e2413e63fe"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of images files: 2542\n",
            "Number of labels files: 2542\n"
          ]
        }
      ],
      "source": [
        "# Check if number of images files match labels files\n",
        "# Images\n",
        "train_images_number = len(glob.glob(os.path.join(datasets_folder, \"train\", \"images\", \"*\")))\n",
        "valid_images_number = len(glob.glob(os.path.join(datasets_folder, \"validation\", \"images\", \"*\")))\n",
        "test_images_number = len(glob.glob(os.path.join(datasets_folder, \"test\", \"images\", \"*\")))\n",
        "images_number = train_images_number + valid_images_number + test_images_number\n",
        "print(\"Number of images files: \" + str(images_number))\n",
        "\n",
        "# Labels\n",
        "train_labels_number = len(glob.glob(os.path.join(datasets_folder, \"train\", \"labels\", \"*\")))\n",
        "valid_labels_number = len(glob.glob(os.path.join(datasets_folder, \"validation\", \"labels\", \"*\")))\n",
        "test_labels_number = len(glob.glob(os.path.join(datasets_folder, \"test\", \"labels\", \"*\")))\n",
        "labels_number = train_labels_number + valid_labels_number + test_labels_number\n",
        "print(\"Number of labels files: \" + str(labels_number))\n",
        "\n",
        "assert images_number == labels_number, \"The number of files do not match.\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "Gko02OueQHtp"
      },
      "outputs": [],
      "source": [
        "# The category for each solar panel object is 0 for objects identified with high confidence,\n",
        "# 1 for objects identified with moderate confidence, and 2 for objects identified with low confidence.\n",
        "# Consider only high confidence (0)\n",
        "for folder in [\"train/labels\", \"validation/labels\", \"test/labels\"]:\n",
        "    labels_list = glob.glob(os.path.join(datasets_folder, folder, \"*\"))\n",
        "    for labels_file in labels_list:\n",
        "        # Open the text file for reading\n",
        "        with open(labels_file, 'r') as file:\n",
        "            # Read all lines from the file\n",
        "            lines = file.readlines()\n",
        "\n",
        "        # Filter out lines where the first element is 0\n",
        "        filtered_lines = [line for line in lines if line.strip().split(' ')[0] == '0']\n",
        "\n",
        "        # Open the text file for writing\n",
        "        with open(labels_file, 'w') as file:\n",
        "            # Write the filtered lines to the output file\n",
        "            file.writelines(filtered_lines)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Zip final datasets\n",
        "# Be patient... Takes around 2 min to zip\n",
        "# Train\n",
        "!cd {train_folder} && zip -q -r {\"../train.zip\"} *\n",
        "\n",
        "# Validation\n",
        "!cd {valid_folder} && zip -q -r {\"../validation.zip\"} *\n",
        "\n",
        "# Test\n",
        "!cd {test_folder} && zip -q -r {\"../test.zip\"} *"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving to drive ignored.\n"
          ]
        }
      ],
      "source": [
        "# Save the zip datasets on your drive\n",
        "try:\n",
        "    drive.mount(\"/content/drive\")\n",
        "    save_driver_folder = \"/content/drive/MyDrive/datasets\"\n",
        "    if not os.path.exists(save_driver_folder):\n",
        "        os.mkdir(save_driver_folder)\n",
        "    # Train\n",
        "    !cp {train_folder+\".zip\"} {save_driver_folder}\n",
        "\n",
        "    # Validation\n",
        "    !cp {valid_folder+\".zip\"} {save_driver_folder}\n",
        "\n",
        "    # Test\n",
        "    !cp {test_folder+\".zip\"} {save_driver_folder}\n",
        "except:\n",
        "    print(\"Saving to drive ignored.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "riNOaHhgQIvI",
        "outputId": "8af49d75-773c-4d35-91f1-08c2e9f11231"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Download to local machine ignored.\n"
          ]
        }
      ],
      "source": [
        "# Download the final datasets to your local machine\n",
        "try:\n",
        "    # Train\n",
        "    files.download(train_folder+\".zip\")\n",
        "\n",
        "    # Validation\n",
        "    files.download(valid_folder+\".zip\")\n",
        "\n",
        "    # Test\n",
        "    files.download(test_folder+\".zip\")\n",
        "except:\n",
        "    print(\"Download to local machine ignored.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Delete remaining intermediate files and folders\n",
        "if os.path.exists(\"datasets/image_chips\"):\n",
        "        shutil.rmtree(\"datasets/image_chips\")\n",
        "if os.path.exists(\"datasets/labels\"):\n",
        "        shutil.rmtree(\"datasets/labels\")\n",
        "if os.path.exists(\"datasets/Maxar_HD_and_Native_Solar_Panel_Image_Chips.zip\"):\n",
        "        os.remove(\"datasets/Maxar_HD_and_Native_Solar_Panel_Image_Chips.zip\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}