increasing_sample_size.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/boothmanrylan/accuracyAssessmentTools/blob/main/increasing_sample_size.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9E5EFNO6Dzfx"
      },
      "source": [
        "# Installs and Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ogwaLH3BC4xH",
        "tags": []
      },
      "outputs": [],
      "source": [
        "%cd /content\n",
        "!git clone https://github.com/boothmanrylan/accuracyAssessmentTools.git\n",
        "%cd accuracyAssessmentTools\n",
        "!pip install ."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from acc_assessment.olofsson import Olofsson\n",
        "from acc_assessment.utils import expand_error_matrix"
      ],
      "metadata": {
        "id": "IrLA0qviG2v2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tASR5FX-GITX"
      },
      "source": [
        "# Create Assessments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wtKKZboXGHcM",
        "tags": []
      },
      "outputs": [],
      "source": [
        "MIN = 10 # with less than this very likely to encounter a divide by zero error\n",
        "\n",
        "data = {\n",
        "    \"Deforestation\":     [66, 0, 1, 2],\n",
        "    \"Forest gain\":       [0, 55, 0, 1],\n",
        "    \"Stable forest\":     [5, 8, 153, 9],\n",
        "    \"Stable non-forest\": [4, 12, 11, 313],\n",
        "}\n",
        "mapped_area = {\n",
        "    \"Deforestation\":     200000,\n",
        "    \"Forest gain\":       150000,\n",
        "    \"Stable forest\":     3200000,\n",
        "    \"Stable non-forest\": 6450000,\n",
        "}\n",
        "\n",
        "df = pd.DataFrame(data)\n",
        "df.index = df.columns\n",
        "\n",
        "# sample(frac=1) shuffles the data to ensure they are in a random order\n",
        "longform_data = expand_error_matrix(df, \"map\", \"ref\").sample(frac=1)\n",
        "\n",
        "# create n assessments where the ith assessment is based on the first i samples\n",
        "all_assessments = [\n",
        "    Olofsson(longform_data.iloc[:i, :], mapped_area, \"map\", \"ref\")\n",
        "    for i in range(MIN, longform_data.shape[0] + 1)\n",
        "]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Plot results"
      ],
      "metadata": {
        "id": "XPBbZDOJe9uI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def make_plot(x, value, std, name, force_ylim=True):\n",
        "    fig, ax = plt.subplots()\n",
        "\n",
        "    twin_axis = ax.twinx()\n",
        "\n",
        "    p1, = ax.plot(x, value, label=name)\n",
        "    ax.fill_between(x, value - std, value + std, alpha=0.3)\n",
        "    p2, = twin_axis.plot(x, std, label=\"Standard Deviation\", color=\"red\")\n",
        "\n",
        "    ax.set(xlim=(x[0], x[-1]), xlabel=\"Sample Size\")\n",
        "    ax.set(ylabel=name)\n",
        "    if force_ylim:\n",
        "        ax.set(ylim=(0.5, 1.0))\n",
        "    ax.yaxis.label.set_color(p1.get_color())\n",
        "    ax.tick_params(axis=\"y\", colors=p1.get_color())\n",
        "\n",
        "    twin_axis.set(ylabel=\"Standard Deviation\")\n",
        "    twin_axis.yaxis.label.set_color(p2.get_color())\n",
        "    twin_axis.tick_params(axis=\"y\", colors=p2.get_color())\n",
        "\n",
        "    ax.legend(handles=[p1, p2])\n",
        "\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "MQVNBhvrWeNv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# get the users accuracy and standard error for the deforestation classes\n",
        "deforestation_users_accuracy = [x.users_accuracy(\"Deforestation\") for x in all_assessments]\n",
        "ua = np.array([x[0] for x in deforestation_users_accuracy])\n",
        "se = np.array([x[1] for x in deforestation_users_accuracy])\n",
        "x = np.arange(0, ua.shape[0]) + MIN\n",
        "\n",
        "make_plot(x, ua, se, \"Deforestation User's Accuracy\")"
      ],
      "metadata": {
        "id": "pH2lr9Zke_oE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# plot the overall accuracy\n",
        "overall_accuracy = [x.overall_accuracy() for x in all_assessments]\n",
        "oa = np.array([x[0] for x in overall_accuracy])\n",
        "oa_se = np.array([x[1] for x in overall_accuracy])\n",
        "x = np.arange(0, ua.shape[0]) + MIN\n",
        "\n",
        "make_plot(x, oa, oa_se, \"Overall Accuracy\")"
      ],
      "metadata": {
        "id": "e8EulEgskiL4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deforested_area = [x.area(\"Deforestation\") for x in all_assessments]\n",
        "area = np.array([x[0] for x in deforested_area])\n",
        "error = np.array([x[1] for x in deforested_area])\n",
        "x = np.arange(0, area.shape[0]) + MIN\n",
        "\n",
        "make_plot(x, area, error, \"Deforested Area (Pixels)\", force_ylim=False)"
      ],
      "metadata": {
        "id": "wf54CUTvcCjO"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "AccuracyAssessmentTools.ipynb",
      "private_outputs": true,
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.16"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}