calc_linear_trends.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:39:07.220357Z",
     "iopub.status.busy": "2024-10-08T03:39:07.220191Z",
     "iopub.status.idle": "2024-10-08T03:39:09.521485Z",
     "shell.execute_reply": "2024-10-08T03:39:09.520978Z"
    }
   },
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import os\n",
    "from glob import glob\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from tqdm.auto import tqdm\n",
    "from tqdm.contrib.concurrent import process_map\n",
    "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error, root_mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:39:09.523487Z",
     "iopub.status.busy": "2024-10-08T03:39:09.523256Z",
     "iopub.status.idle": "2024-10-08T03:39:09.641021Z",
     "shell.execute_reply": "2024-10-08T03:39:09.640592Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>sediment_area_ha</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>85da9213fffffff</td>\n",
       "      <td>84da921ffffffff</td>\n",
       "      <td>210.88</td>\n",
       "      <td>POLYGON ((1528885.242 5425964.833, 1524656.417...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>85bb58c7fffffff</td>\n",
       "      <td>84bb58dffffffff</td>\n",
       "      <td>253.24</td>\n",
       "      <td>POLYGON ((1595414.183 6163032.018, 1591203.235...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               index        parent_id  sediment_area_ha  \\\n",
       "95   85da9213fffffff  84da921ffffffff            210.88   \n",
       "128  85bb58c7fffffff  84bb58dffffffff            253.24   \n",
       "\n",
       "                                              geometry  \n",
       "95   POLYGON ((1528885.242 5425964.833, 1524656.417...  \n",
       "128  POLYGON ((1595414.183 6163032.018, 1591203.235...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sites = pd.read_csv('global-inputs/sites.csv')\n",
    "hr5_cells = gpd.read_file('global-inputs/HR5-change-cells-aoi.gpkg')\n",
    "\n",
    "cells = hr5_cells[hr5_cells['index'].isin(sites['cell_id'])]\n",
    "\n",
    "cells\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:39:09.642547Z",
     "iopub.status.busy": "2024-10-08T03:39:09.642337Z",
     "iopub.status.idle": "2024-10-08T03:39:09.646223Z",
     "shell.execute_reply": "2024-10-08T03:39:09.645845Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    data/HR5/85bb58c7fffffff/shoreline_timeseries_...\n",
       "1    data/HR5/85da9213fffffff/shoreline_timeseries_...\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shoreline_files = pd.Series(sorted(glob(\"data/HR5/*/shoreline_timeseries_tidal_correction.csv\")))\n",
    "\n",
    "shoreline_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:39:09.647609Z",
     "iopub.status.busy": "2024-10-08T03:39:09.647447Z",
     "iopub.status.idle": "2024-10-08T03:39:09.652755Z",
     "shell.execute_reply": "2024-10-08T03:39:09.652363Z"
    }
   },
   "outputs": [],
   "source": [
    "def calc_trends(file):\n",
    "    df  = pd.read_csv(file)\n",
    "    try:\n",
    "        df.date = pd.to_datetime(df.date)\n",
    "    except:\n",
    "        print(file)\n",
    "    df.index = (df.date - df.date.min()).dt.days / 365.25\n",
    "    df.drop(columns=['date', 'Unnamed: 0', 'corrected_IW_shoreline_position', 'EOV_shoreline_chg'], inplace=True, errors='ignore')\n",
    "    trends = {}\n",
    "    for col in df.columns:\n",
    "        sub_df = df[col].dropna()\n",
    "        \n",
    "        if not len(sub_df):\n",
    "            continue\n",
    "        x = sub_df.index.to_numpy().reshape(-1,1)\n",
    "        y = sub_df\n",
    "        linear_trend = LinearRegression().fit(x,y)\n",
    "        pred = linear_trend.predict(x)\n",
    "\n",
    "        trends['index'] = file.split('/')[2]\n",
    "        if col == 'normalised_IW_shoreline_position':\n",
    "            trends[\"IW shoreline trend\"] = linear_trend.coef_[0]\n",
    "            trends[\"IW shoreline intercept\"] = linear_trend.intercept_\n",
    "            trends[\"IW shoreline n_points\"] = len(df[col])\n",
    "            trends[\"IW shoreline n_points_nonan\"] = len(sub_df)\n",
    "            trends[\"IW shoreline r2_score\"] = r2_score(y, pred)\n",
    "            trends[\"IW shoreline mae\"] = mean_absolute_error(y, pred)\n",
    "            trends[\"IW shoreline mse\"] = mean_squared_error(y, pred)\n",
    "            trends[\"IW shoreline rmse\"] = root_mean_squared_error(y, pred)\n",
    "        else:\n",
    "            trends[\"EOV shoreline trend\"] = linear_trend.coef_[0]\n",
    "            trends[\"EOV shoreline intercept\"] = linear_trend.intercept_\n",
    "            trends[\"EOV shoreline n_points\"] = len(df[col])\n",
    "            trends[\"EOV shoreline n_points_nonan\"] = len(sub_df)\n",
    "            trends[\"EOV shoreline r2_score\"] = r2_score(y, pred)\n",
    "            trends[\"EOV shoreline mae\"] = mean_absolute_error(y, pred)\n",
    "            trends[\"EOV shoreline mse\"] = mean_squared_error(y, pred)\n",
    "            trends[\"EOV shoreline rmse\"] = root_mean_squared_error(y, pred)\n",
    "\n",
    "    return pd.DataFrame([trends])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:39:09.654131Z",
     "iopub.status.busy": "2024-10-08T03:39:09.653979Z",
     "iopub.status.idle": "2024-10-08T03:39:09.834030Z",
     "shell.execute_reply": "2024-10-08T03:39:09.833668Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>IW shoreline trend</th>\n",
       "      <th>IW shoreline intercept</th>\n",
       "      <th>IW shoreline n_points</th>\n",
       "      <th>IW shoreline n_points_nonan</th>\n",
       "      <th>IW shoreline r2_score</th>\n",
       "      <th>IW shoreline mae</th>\n",
       "      <th>IW shoreline mse</th>\n",
       "      <th>IW shoreline rmse</th>\n",
       "      <th>EOV shoreline trend</th>\n",
       "      <th>EOV shoreline intercept</th>\n",
       "      <th>EOV shoreline n_points</th>\n",
       "      <th>EOV shoreline n_points_nonan</th>\n",
       "      <th>EOV shoreline r2_score</th>\n",
       "      <th>EOV shoreline mae</th>\n",
       "      <th>EOV shoreline mse</th>\n",
       "      <th>EOV shoreline rmse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>85bb58c7fffffff</td>\n",
       "      <td>-0.518494</td>\n",
       "      <td>7.886078</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>0.004687</td>\n",
       "      <td>16.093085</td>\n",
       "      <td>948.484685</td>\n",
       "      <td>30.797479</td>\n",
       "      <td>0.108178</td>\n",
       "      <td>5.619808</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>0.011170</td>\n",
       "      <td>3.302646</td>\n",
       "      <td>17.213889</td>\n",
       "      <td>4.148962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>85da9213fffffff</td>\n",
       "      <td>0.262913</td>\n",
       "      <td>23.784081</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>0.017620</td>\n",
       "      <td>11.031605</td>\n",
       "      <td>169.460165</td>\n",
       "      <td>13.017687</td>\n",
       "      <td>0.079789</td>\n",
       "      <td>-1.491009</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>0.026733</td>\n",
       "      <td>2.288275</td>\n",
       "      <td>10.191473</td>\n",
       "      <td>3.192409</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             index  IW shoreline trend  IW shoreline intercept  \\\n",
       "0  85bb58c7fffffff           -0.518494                7.886078   \n",
       "0  85da9213fffffff            0.262913               23.784081   \n",
       "\n",
       "   IW shoreline n_points  IW shoreline n_points_nonan  IW shoreline r2_score  \\\n",
       "0                     37                           37               0.004687   \n",
       "0                     81                           81               0.017620   \n",
       "\n",
       "   IW shoreline mae  IW shoreline mse  IW shoreline rmse  EOV shoreline trend  \\\n",
       "0         16.093085        948.484685          30.797479             0.108178   \n",
       "0         11.031605        169.460165          13.017687             0.079789   \n",
       "\n",
       "   EOV shoreline intercept  EOV shoreline n_points  \\\n",
       "0                 5.619808                      37   \n",
       "0                -1.491009                      81   \n",
       "\n",
       "   EOV shoreline n_points_nonan  EOV shoreline r2_score  EOV shoreline mae  \\\n",
       "0                            37                0.011170           3.302646   \n",
       "0                            81                0.026733           2.288275   \n",
       "\n",
       "   EOV shoreline mse  EOV shoreline rmse  \n",
       "0          17.213889            4.148962  \n",
       "0          10.191473            3.192409  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trends = pd.concat([calc_trends(f) for f in shoreline_files])\n",
    "\n",
    "trends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-08T03:39:09.835592Z",
     "iopub.status.busy": "2024-10-08T03:39:09.835308Z",
     "iopub.status.idle": "2024-10-08T03:39:09.875498Z",
     "shell.execute_reply": "2024-10-08T03:39:09.875126Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>sediment_area_ha</th>\n",
       "      <th>geometry</th>\n",
       "      <th>IW shoreline trend</th>\n",
       "      <th>IW shoreline intercept</th>\n",
       "      <th>IW shoreline n_points</th>\n",
       "      <th>IW shoreline n_points_nonan</th>\n",
       "      <th>IW shoreline r2_score</th>\n",
       "      <th>IW shoreline mae</th>\n",
       "      <th>IW shoreline mse</th>\n",
       "      <th>IW shoreline rmse</th>\n",
       "      <th>EOV shoreline trend</th>\n",
       "      <th>EOV shoreline intercept</th>\n",
       "      <th>EOV shoreline n_points</th>\n",
       "      <th>EOV shoreline n_points_nonan</th>\n",
       "      <th>EOV shoreline r2_score</th>\n",
       "      <th>EOV shoreline mae</th>\n",
       "      <th>EOV shoreline mse</th>\n",
       "      <th>EOV shoreline rmse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>85da9213fffffff</td>\n",
       "      <td>84da921ffffffff</td>\n",
       "      <td>210.88</td>\n",
       "      <td>POLYGON ((172.15037 -41.31465, 172.101 -41.230...</td>\n",
       "      <td>0.262913</td>\n",
       "      <td>23.784081</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>0.017620</td>\n",
       "      <td>11.031605</td>\n",
       "      <td>169.460165</td>\n",
       "      <td>13.017687</td>\n",
       "      <td>0.079789</td>\n",
       "      <td>-1.491009</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>0.026733</td>\n",
       "      <td>2.288275</td>\n",
       "      <td>10.191473</td>\n",
       "      <td>3.192409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>85bb58c7fffffff</td>\n",
       "      <td>84bb58dffffffff</td>\n",
       "      <td>253.24</td>\n",
       "      <td>POLYGON ((172.94994 -34.67468, 172.90408 -34.5...</td>\n",
       "      <td>-0.518494</td>\n",
       "      <td>7.886078</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>0.004687</td>\n",
       "      <td>16.093085</td>\n",
       "      <td>948.484685</td>\n",
       "      <td>30.797479</td>\n",
       "      <td>0.108178</td>\n",
       "      <td>5.619808</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "      <td>0.011170</td>\n",
       "      <td>3.302646</td>\n",
       "      <td>17.213889</td>\n",
       "      <td>4.148962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             index        parent_id  sediment_area_ha  \\\n",
       "0  85da9213fffffff  84da921ffffffff            210.88   \n",
       "1  85bb58c7fffffff  84bb58dffffffff            253.24   \n",
       "\n",
       "                                            geometry  IW shoreline trend  \\\n",
       "0  POLYGON ((172.15037 -41.31465, 172.101 -41.230...            0.262913   \n",
       "1  POLYGON ((172.94994 -34.67468, 172.90408 -34.5...           -0.518494   \n",
       "\n",
       "   IW shoreline intercept  IW shoreline n_points  IW shoreline n_points_nonan  \\\n",
       "0               23.784081                     81                           81   \n",
       "1                7.886078                     37                           37   \n",
       "\n",
       "   IW shoreline r2_score  IW shoreline mae  IW shoreline mse  \\\n",
       "0               0.017620         11.031605        169.460165   \n",
       "1               0.004687         16.093085        948.484685   \n",
       "\n",
       "   IW shoreline rmse  EOV shoreline trend  EOV shoreline intercept  \\\n",
       "0          13.017687             0.079789                -1.491009   \n",
       "1          30.797479             0.108178                 5.619808   \n",
       "\n",
       "   EOV shoreline n_points  EOV shoreline n_points_nonan  \\\n",
       "0                      81                            81   \n",
       "1                      37                            37   \n",
       "\n",
       "   EOV shoreline r2_score  EOV shoreline mae  EOV shoreline mse  \\\n",
       "0                0.026733           2.288275          10.191473   \n",
       "1                0.011170           3.302646          17.213889   \n",
       "\n",
       "   EOV shoreline rmse  \n",
       "0            3.192409  \n",
       "1            4.148962  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trends_file_path = 'shoreline_trends_by_cell.geojson'\n",
    "# if os.path.exists(trends_file_path):\n",
    "#     cell_trends = gpd.read_file('shoreline_trends_by_cell.geojson')\n",
    "#     cell_trends.to_crs(\"EPSG:4326\", inplace=True)\n",
    "#     cell_trends.update(trends)\n",
    "#     cell_trends.to_file(trends_file_path)\n",
    "# else:\n",
    "cell_trends = cells.merge(trends, on='index')\n",
    "cell_trends.to_crs(\"EPSG:4326\", inplace=True)\n",
    "cell_trends.to_file(trends_file_path)\n",
    "    \n",
    "\n",
    "cell_trends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}