Boosted Random Forest Simplified.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load libraries\n",
    "\n",
    "### Need arcpy license and geostatistical analyst license to compare against kriging\n",
    "\n",
    "Current citation\n",
    "\n",
    "Lamb, D. S. 2018. Random Forest Trees for Spatial Interpolation. Presented at the Meeting of the Association of American Geographers, New Orleans, LA.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from scipy import spatial\n",
    "import arcpy\n",
    "import seaborn as sns\n",
    "import geopandas as gpd\n",
    "import matplotlib.pyplot as plt\n",
    "from shapely.geometry import LineString, Point, shape, mapping\n",
    "from scipy.interpolate import Rbf\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions for formatting map output if needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def meters_formatter(x, p):\n",
    "    strRes = '{:,}m'.format(int(x)) \n",
    "    return strRes\n",
    "\n",
    "def scaleBar(x,y,mapdistance,ax,trans,subdivision=1,height=.02):\n",
    "    \"\"\"x - lower left corner of arrow in trans coordinates\n",
    "       y - lower left corner of arrow in trans coordiantes\n",
    "       mapdistance - maximum distance to show on the scalebar\n",
    "       ax - axes to add patch and text\n",
    "       trans - transformation the coordinates are in\n",
    "       subdivision - number of subdivisions to show in the scalebar\n",
    "       height - height of the bar part of the scalebar\"\"\"\n",
    "    xmin, xmax = ax.get_xlim() #returns left,right\n",
    "    abs_width = abs(xmax-xmin)\n",
    "    length = 1.0/abs_width * mapdistance\n",
    "    if subdivision > 1.0:\n",
    "        sublength = float(length)/subdivision\n",
    "        fColor = 'black'\n",
    "        subx = x\n",
    "        for i in range(0,subdivision):\n",
    "            ax.add_patch(mpatches.Rectangle((subx,y), sublength, height, transform=trans,facecolor=fColor,edgecolor='black',lw=.5,clip_on=False))\n",
    "            subx += sublength\n",
    "            if fColor == 'black':\n",
    "                fColor = 'white'\n",
    "            else:\n",
    "                fColor = 'black'\n",
    "            \n",
    "    else:\n",
    "        ax.add_patch(mpatches.Rectangle((x,y), length, height, transform=trans,facecolor='black',edgecolor='black',clip_on=False))\n",
    "        \n",
    "    ax.text(x,y+height*1.5,'0',transform=trans,ha='center')\n",
    "    ax.text(x+length,y+height*1.5,meters_formatter(mapdistance,None),transform=trans,ha='center')\n",
    "    \n",
    "def northArrowPatch(x,y,width,height,ax,trans):\n",
    "    \"\"\"x - lower left corner of arrow in trans coordinates\n",
    "       y - lower left corner of arrow in trans coordiantes\n",
    "       width - estimated width of arrow in trans coordiantes\n",
    "       height - estimated height of arrow in trans coordiantes\n",
    "       ax - axes to add patch and text\n",
    "       trans - transformation the coordinates are in\"\"\"\n",
    "    verts = []\n",
    "    codes = []\n",
    "    for part in range(0,4):\n",
    "        if part == 0:\n",
    "            verts.append((x,y))\n",
    "            codes.append(Path.MOVETO)\n",
    "        if part == 1:\n",
    "            verts.append((x+(width/2.0),y-(height)))\n",
    "            codes.append(Path.LINETO)\n",
    "        if part == 2:\n",
    "            verts.append((x,y-(height-(1.0/5.0)*height)))\n",
    "            codes.append(Path.LINETO)\n",
    "        if part == 3:\n",
    "            verts.append((x-(width/2.0),y-(height)))\n",
    "            codes.append(Path.LINETO)\n",
    "            verts.append((x,y))\n",
    "            codes.append(Path.CLOSEPOLY)\n",
    "\n",
    "    path = Path(verts, codes)\n",
    "    northPatch = mpatches.PathPatch(path, facecolor='k', lw=0,transform=trans,clip_on=False)\n",
    "    ax.add_patch(northPatch)\n",
    "    lbl = ax.text(x,y+((1.0/5.0)*height),'N',transform=trans,ha='center')\n",
    "    return northPatch,lbl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simulate surface gaussian random noise, including an explanatory variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n,m = 100,100\n",
    "F = 2\n",
    "x_c = np.linspace(1,n,n)\n",
    "\n",
    "y_c = np.linspace(1,m,m)\n",
    "X_c, Y_c = np.meshgrid(x_c,y_c)\n",
    "\n",
    "i = np.minimum(X_c-1,n-X_c+1)\n",
    "j = np.minimum(Y_c-1,m-Y_c+1)\n",
    "\n",
    "isq = np.square(i)\n",
    "jsq = np.square(j)\n",
    "\n",
    "H = np.exp(-.5 * (isq+jsq) / (F**2))\n",
    "Z = np.real(np.fft.ifft2(H*np.fft.fft2(np.random.randn(n,m))))\n",
    "ya = np.random.rand(m)*100\n",
    "xa = np.random.rand(n)*100\n",
    "adjust =np.sqrt(xa**2+ya**2)\n",
    "rbf = Rbf(xa, ya, adjust, epsilon=5)\n",
    "ZI = rbf(X_c, Y_c)\n",
    "rows={\"xcoord\":[],\"ycoord\":[],\"zval\":[],\"dv\":[],\"dv2\":[],\"ROW\":[],\"COL\":[]}\n",
    "for i in range(n):\n",
    "    row = X_c[i]\n",
    "    #print row\n",
    "    for j in range(n):\n",
    "        rows['xcoord'].append(row[j])\n",
    "        rows['ycoord'].append(Y_c[i][j])\n",
    "        rows['dv'].append(Z[i][j])\n",
    "        rows['dv2'].append(ZI[i][j])\n",
    "        rows['zval'].append(Z[i][j]*ZI[i][j])\n",
    "        rows['ROW'].append(i)\n",
    "        rows['COL'].append(j)\n",
    "df = pd.DataFrame(rows)\n",
    "gridz= df.pivot('ROW','COL','zval')\n",
    "#print grid\n",
    "\n",
    "ax = sns.heatmap(gridz,cmap='Greys',xticklabels=False, yticklabels=False)\n",
    "\n",
    "ax.set_aspect('equal')\n",
    "ax.set_xlabel(\"\")\n",
    "ax.set_ylabel(\"\")\n",
    "ax.set_title(\"Simulated Gaussian Noise\")\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot out the trend surface / explanatory variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gridz= df.pivot('ROW','COL','dv2')\n",
    "#print grid\n",
    "\n",
    "ax = sns.heatmap(gridz,cmap='Greys',xticklabels=False, yticklabels=False)\n",
    "\n",
    "ax.set_aspect('equal')\n",
    "ax.set_xlabel(\"\")\n",
    "ax.set_ylabel(\"\")\n",
    "ax.set_title(\"Trend Variable\")\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot out figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, ((ax1, ax2,ax3)) = plt.subplots(1,3)\n",
    "fig.set_size_inches(8, 4)\n",
    "gridz= df.pivot('ROW','COL','zval')\n",
    "sns.heatmap(gridz,cmap='Greys',xticklabels=False, yticklabels=False,ax=ax1,cbar=False)\n",
    "ax1.set_aspect('equal')\n",
    "ax1.set_xlabel(\"\")\n",
    "ax1.set_ylabel(\"\")\n",
    "ax1.set_title(\"Dependent Variable\")\n",
    "gridv= df.pivot('ROW','COL','dv')\n",
    "sns.heatmap(gridv,cmap='Greys',xticklabels=False, yticklabels=False,ax=ax2,cbar=False)\n",
    "ax2.set_aspect('equal')\n",
    "ax2.set_xlabel(\"\")\n",
    "ax2.set_ylabel(\"\")\n",
    "ax2.set_title(\"Independent Variable\")\n",
    "gridt= df.pivot('ROW','COL','dv2')\n",
    "sns.heatmap(gridt,cmap='Greys',xticklabels=False, yticklabels=False,ax=ax3,cbar=False)\n",
    "ax3.set_aspect('equal')\n",
    "ax3.set_xlabel(\"\")\n",
    "ax3.set_ylabel(\"\")\n",
    "ax3.set_title(\" Trend Variable\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"GaussianFFTWithIV.png\",dpi=300)\n",
    "plt.savefig(\"GaussianFFTWithIV.pdf\",dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Randomly create missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Create missing values list\n",
    "bins = []\n",
    "for k in range(len(rows['xcoord'])):\n",
    "    if np.random.random() > .02:\n",
    "        bins.append(1)\n",
    "    else:\n",
    "        bins.append(0)\n",
    "df['missing'] = bins\n",
    "print len(bins)\n",
    "print df['missing'].sum()\n",
    "print float(df['missing'].sum())/len(bins)\n",
    "\n",
    "imput = []\n",
    "for index,row in df.iterrows():\n",
    "    if row['missing'] ==1:\n",
    "        imput.append(df['dv'].median())\n",
    "    else:\n",
    "        imput.append(row['dv'])\n",
    "df['dv_m'] = imput\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions for sampling data, creating a grid of points to account for spatial dependence, calculting inverse distance weighted surface, and developing Random Forest model and comparison metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def performance_metric_r2(y_true, y_predict):\n",
    "    \"\"\" Calculates and returns the performance score between \n",
    "        true and predicted values based on the metric chosen. \"\"\"\n",
    "    \n",
    "    # TODO: Calculate the performance score between 'y_true' and 'y_predict'\n",
    "    score = r2_score(y_true, y_predict)\n",
    "    \n",
    "    # Return the score\n",
    "    return score\n",
    "\n",
    "def performance_metric_mae(y_true, y_predict):\n",
    "    \"\"\" Calculates and returns the performance score between \n",
    "        true and predicted values based on the metric chosen. \"\"\"\n",
    "    \n",
    "    # TODO: Calculate the performance score between 'y_true' and 'y_predict'\n",
    "    score = -1*mean_absolute_error(y_true, y_predict)\n",
    "    \n",
    "    # Return the score\n",
    "    return score\n",
    "\n",
    "def performance_metric_mse(y_true, y_predict):\n",
    "    \"\"\" Calculates and returns the performance score between \n",
    "        true and predicted values based on the metric chosen. \"\"\"\n",
    "    \n",
    "    # TODO: Calculate the performance score between 'y_true' and 'y_predict'\n",
    "    score = -1*mean_squared_error(y_true, y_predict)\n",
    "    \n",
    "    # Return the score\n",
    "    return score\n",
    "\n",
    "#score = performance_metric([3, -0.5, 2, 7, 4.2], [2.5, 0.0, 2.1, 7.8, 5.3])\n",
    "#print(\"Model has a coefficient of determination, R^2, of {:.3f}.\".format(score))\n",
    "\n",
    "def fit_model(X, y,params=None,r2=True,mae=False,mse=False):\n",
    "    \"\"\" Performs grid search over the 'max_depth' parameter for a \n",
    "        decision tree regressor trained on the input data [X, y]. \"\"\"\n",
    "    \n",
    "    # Create cross-validation sets from the training data\n",
    "    cv_sets = ShuffleSplit(X.shape[0], test_size = 0.20, random_state = 0)\n",
    "\n",
    "    # TODO: Create a decision tree regressor object\n",
    "    regressor = GradientBoostingRegressor()#DecisionTreeRegressor()\n",
    "\n",
    "    # TODO: Create a dictionary for the parameter 'max_depth' with a range from 1 to 10\n",
    "    if params == None:\n",
    "        params = {'n_estimators': [300,400],\n",
    "                  'max_depth': list(range(4,35)),\n",
    "                  'min_samples_leaf':list(range(1,3)),\n",
    "                  'max_features':[\"auto\"],\n",
    "                  'loss':['lad'],\n",
    "                 \"learning_rate\":[.01,.1],\n",
    "                 \"subsample\":[0.5,.75,1.0]}\n",
    "        #\n",
    "    # TODO: Transform 'performance_metric' into a scoring function using 'make_scorer' \n",
    "    if r2:\n",
    "        scoring_fnc = make_scorer(performance_metric_r2)\n",
    "    if mae:\n",
    "        scoring_fnc = make_scorer(performance_metric_mae)\n",
    "    if mse:\n",
    "        scoring_fnc = make_scorer(performance_metric_mse)\n",
    "    else:\n",
    "        scoring_fnc = make_scorer(performance_metric_r2)\n",
    "    # TODO: Create the grid search object\n",
    "    grid = GridSearchCV(regressor, params, scoring = scoring_fnc,cv=5)\n",
    "\n",
    "    # Fit the grid search object to the data to compute the optimal model\n",
    "    grid = grid.fit(X, y)\n",
    "\n",
    "    # Return the optimal model after fitting the data\n",
    "    return grid.best_estimator_\n",
    "\n",
    "def createXYGrid(llx,lly,urx,ury,columnCount,rowCount):\n",
    "    x = np.linspace(llx,urx, columnCount)\n",
    "    y = np.linspace(lly, ury, rowCount)\n",
    "    xv, yv = np.meshgrid(x, y, sparse=False, indexing='ij')\n",
    "    outCoords = []\n",
    "    for i in range(len(x)):\n",
    "        for j in range(len(y)):\n",
    "            outCoords.append([xv[i,j], yv[i,j]])\n",
    "    outCoords = np.array(outCoords)\n",
    "    #plt.scatter(outCoords[:,0],outCoords[:,1])\n",
    "    #plt.grid(True)\n",
    "    #plt.axes().set_aspect('equal', 'datalim')\n",
    "    \n",
    "    return outCoords\n",
    "\n",
    "def getDatasetCoords(df,yindex,xindices,coordindices,gcoords,distance= 'minkowski',p=-.5,inv=False,invp=1.0):\n",
    "    if len(xindices) >0:\n",
    "        X = df.iloc[:,xindices].values\n",
    "        print X.shape\n",
    "    coords =  df.iloc[:,coordindices].values\n",
    "    Y = df.iloc[:,yindex].values\n",
    "    if distance == 'minkowski':\n",
    "        sp_dist = spatial.distance.cdist(coords,gcoords, distance,p=p)\n",
    "    else:\n",
    "        sp_dist = spatial.distance.cdist(coords,gcoords, distance)#,p=)\n",
    "    sp_dist = np.square(sp_dist)\n",
    "    if inv == True:\n",
    "        sp_dist = np.divide(1.0,np.power(sp_dist,invp))\n",
    "    #intern_dist = spatial.distance.cdist(coords,coords, 'euclidean')\n",
    "    if len(xindices) >0:\n",
    "        X=np.concatenate((X, sp_dist), axis=1)\n",
    "    else:\n",
    "        X = sp_dist\n",
    "    return X, Y\n",
    "\n",
    "def getDatasetList(df,coordindices,gcoords):\n",
    "    coords =  df.iloc[:,coordindices].values\n",
    "    np.append(coords,gcoords)\n",
    "    return coords\n",
    "\n",
    "\n",
    "def getDatasetSample(df,nsample,yindex,xindices,coordindices,gcoords):\n",
    "    dataset_sub = df.sample(n=nsample, replace=False).copy()\n",
    "    X,Y = getDatasetCoords(dataset_sub,yindex,xindices,coordindices,gcoords)\n",
    "    return dataset_sub, X, Y\n",
    "\n",
    "def getDatasetSampleSub(df,nsample,yindex,xindices,coordindices,subsamp):\n",
    "    dataset_sub = df.sample(n=nsample, replace=False).copy()\n",
    "    dataset_sub_sub = dataset_sub.sample(n=subsamp,replace=False).copy()\n",
    "    gcoords =  dataset_sub_sub.iloc[:,coordindices].values\n",
    "    X,Y = getDatasetCoords(dataset_sub,yindex,xindices,coordindices,gcoords)\n",
    "    return dataset_sub,gcoords, X, Y\n",
    "\n",
    "\n",
    "def getDatasetPred(df,yindex,xindices):\n",
    "    X = df.iloc[:,xindices].values\n",
    "    #print X.shape\n",
    "    Y = df.iloc[:,yindex].values\n",
    "    return X, Y\n",
    "\n",
    "\n",
    "def getDatasetSampleNoGrid(df,nsample,yindex,xindices):\n",
    "    dataset_sub = df.sample(n=nsample, replace=False).copy()\n",
    "    X,Y = getDatasetPred(dataset_sub,yindex,xindices)\n",
    "    return dataset_sub,X,Y\n",
    "\n",
    "def inverseDistanceValues(points,datalist,gridpoints,power=2,nn=12):\n",
    "    tree = spatial.KDTree(points)\n",
    "    q_res = tree.query(gridpoints,nn)\n",
    "    outVals = []\n",
    "    for i in range(0,len(q_res[0])):\n",
    "        if len(q_res[0][i][np.where(q_res[0][i]==0.0)]) >0:\n",
    "            res = datalist[q_res[1][i]][0]\n",
    "            outVals.append(res)\n",
    "        else:\n",
    "            invdist = np.divide(1.0,np.power(q_res[0][i],2))\n",
    "            value = datalist[q_res[1][i]]\n",
    "            res = np.divide(np.sum(np.multiply(value,invdist)),np.sum(invdist))\n",
    "            outVals.append(res)\n",
    "        if np.isnan(res):\n",
    "            print invdist\n",
    "            print value\n",
    "            print np.sum(invdist)\n",
    "            print np.multiply(value,invdist)\n",
    "            print np.sum(np.multiply(value,invdist))\n",
    "            print i\n",
    "    return np.array(outVals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print df.head()\n",
    "yind = [6]\n",
    "xyind = [4,5]\n",
    "dv = [8,9]\n",
    "dvxy = [8,9,4,5]\n",
    "#dv = [2]\n",
    "#dvxy = [2,4,5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "llx = min(df['xcoord'])\n",
    "\n",
    "lly = min(df['ycoord'])\n",
    "\n",
    "urx = max(df['xcoord'])\n",
    "\n",
    "ury = max(df['ycoord'])\n",
    "\n",
    "gridCoords = createXYGrid(llx,lly,urx,ury,4,4)\n",
    "#df,nsample,yindex,xindices,coordindices,gcoords\n",
    "#dataset_sub,X,Y= getDatasetSample(df,100,yind,dvxy,xyind,gridCoords)\n",
    "#df,nsample,yindex,xindices):\n",
    "dataset_sub,X,Y= getDatasetSampleNoGrid(df,100,yind,dvxy)\n",
    "plt.scatter(dataset_sub['xcoord'],dataset_sub['ycoord'])\n",
    "#plt.scatter(gridCoords[:,0],gridCoords[:,1],c='r')\n",
    "plt.grid(True)\n",
    "plt.axes().set_aspect('equal')\n",
    "\n",
    "#df,yindex,xindices,coordindices,gcoords,distance= 'minkowski',p=-.5,inv=False,invp=1.0\n",
    "#d='euclidean'\n",
    "#X,Y = getDatasetCoords(dataset_sub,yind,dvxy,xyind,gridCoords,distance=d,p=3,inv=False,invp=2)\n",
    "print X.shape\n",
    "sns.set()\n",
    "#for k in range(len(X)):\n",
    "    #if np.random.random() > .5:\n",
    "        #X[k][0]=-100000\n",
    "\n",
    "#x_pred,y_pred = getDatasetCoords(df,yind,dvxy,xyind,gridCoords,distance=d,p=3,inv=False,invp=2)\n",
    "x_pred,y_pred = getDatasetPred(df,yind,dvxy)\n",
    "#for k in range(len(x_pred)):\n",
    "    #if np.random.random() > .5:\n",
    "        #x_pred[k][0]=-100000\n",
    "        \n",
    "\n",
    "mod = GradientBoostingRegressor(n_estimators=400, learning_rate=0.01, max_features='auto',max_depth=10,min_samples_leaf=2,\n",
    "                                loss='lad',subsample=.7)\n",
    "mod.fit(X,Y)\n",
    "\n",
    "df['predicted']=mod.predict(x_pred)\n",
    "#alldata['msenb']=np.square(alldata['predicted']-alldata['rain'])\n",
    "tempPred = mod.predict(x_pred)\n",
    "print performance_metric_mse(y_pred,tempPred)*-1\n",
    "print mean_squared_error(y_pred, tempPred)\n",
    "print performance_metric_mae(y_pred,tempPred)*-1\n",
    "print performance_metric_r2(y_pred,tempPred)\n",
    "\n",
    "gridRain= df.pivot('ROW','COL','predicted')\n",
    "#print grid\n",
    "\n",
    "ax = sns.heatmap(gridRain,cmap='Blues',xticklabels=False, yticklabels=False)\n",
    "ax.set_aspect('equal')\n",
    "ax.set_xlabel(\"\")\n",
    "ax.set_ylabel(\"\")\n",
    "ax.set_title(\"Predicted DV\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fit_model(X, y,params=None,r2=True,mae=False,mse=False):\n",
    "    \"\"\" Performs grid search over the 'max_depth' parameter for a \n",
    "        decision tree regressor trained on the input data [X, y]. \"\"\"\n",
    "    \n",
    "    # Create cross-validation sets from the training data\n",
    "    cv_sets = ShuffleSplit(X.shape[0], test_size = 0.20, random_state = 0)\n",
    "\n",
    "    # TODO: Create a decision tree regressor object\n",
    "    regressor = GradientBoostingRegressor()#DecisionTreeRegressor()\n",
    "\n",
    "    # TODO: Create a dictionary for the parameter 'max_depth' with a range from 1 to 10\n",
    "    if params == None:\n",
    "        params = {'n_estimators': [400],\n",
    "                  'max_depth': list(range(8,25)),\n",
    "                  'min_samples_leaf':list(range(1,3)),\n",
    "                  'max_features':[\"auto\"],\n",
    "                  'loss':['lad'],\n",
    "                 \"learning_rate\":[.01,.1],\n",
    "                 \"subsample\":[0.5,.75,1.0]}\n",
    "        #\n",
    "    # TODO: Transform 'performance_metric' into a scoring function using 'make_scorer' \n",
    "    if r2:\n",
    "        scoring_fnc = make_scorer(performance_metric_r2)\n",
    "    if mae:\n",
    "        scoring_fnc = make_scorer(performance_metric_mae)\n",
    "    if mse:\n",
    "        scoring_fnc = make_scorer(performance_metric_mse)\n",
    "    else:\n",
    "        scoring_fnc = make_scorer(performance_metric_r2)\n",
    "    # TODO: Create the grid search object\n",
    "    grid = GridSearchCV(regressor, params, scoring = scoring_fnc,cv=5)\n",
    "\n",
    "    # Fit the grid search object to the data to compute the optimal model\n",
    "    grid = grid.fit(X, y)\n",
    "\n",
    "    # Return the optimal model after fitting the data\n",
    "    return grid.best_estimator_\n",
    "\n",
    "\n",
    "results3 = {\"rfmseg\":[],\"rfmaeg\":[],\"rfrsqg\":[],\"rfmsedv\":[],\"rfmaedv\":[],\"rfrsqdv\":[],\"idwmse\":[],\"idwmae\":[],\"idwrsq\":[],\n",
    "           \"samplesize\":[],\"gridsize\":[]}\n",
    "samplesizes = range(20,310,10)\n",
    "ss=100\n",
    "\n",
    "for i in range(0,100):\n",
    "    results3[\"samplesize\"].append(ss)\n",
    "    gs = int(ss*.05)\n",
    "    results3[\"gridsize\"].append(gs)\n",
    "    gridCoords = createXYGrid(llx,lly,urx,ury,4,4)\n",
    "    dataset_sub,X,Y= getDatasetSampleNoGrid(df,100,yindiv,xyind)\n",
    "    d='euclidean'\n",
    "    X,Y = getDatasetCoords(dataset_sub,yindiv,xyind,xyind,gridCoords,distance=d,p=3,inv=False,invp=2)\n",
    "    x_pred,y_pred = getDatasetCoords(df,yindiv,xyind,xyind,gridCoords,distance=d,p=3,inv=False,invp=2)\n",
    "    \n",
    "    mod = fit_model(X, Y,r2=False,mae=False,mse=True)\n",
    "    tempPred = mod.predict(x_pred)\n",
    "    results3[\"rfmseg\"].append(performance_metric_mse(y_pred,tempPred)*-1)\n",
    "    results3[\"rfmaeg\"].append(performance_metric_mae(y_pred,tempPred)*-1)\n",
    "    results3[\"rfrsqg\"].append(performance_metric_r2(y_pred,tempPred))\n",
    "    \n",
    "\n",
    "    d='euclidean'\n",
    "    X,Y = getDatasetCoords(dataset_sub,yind,dvxym,xyind,gridCoords,distance=d,p=3,inv=False,invp=2)\n",
    "\n",
    "    x_pred,y_pred = getDatasetCoords(df,yind,dvxym,xyind,gridCoords,distance=d,p=3,inv=False,invp=2)\n",
    "    mod = fit_model(X, Y,r2=False,mae=False,mse=True)\n",
    "    tempPred = mod.predict(x_pred)\n",
    "    results3[\"rfmsedv\"].append(performance_metric_mse(y_pred,tempPred)*-1)\n",
    "    results3[\"rfmaedv\"].append(performance_metric_mae(y_pred,tempPred)*-1)\n",
    "    results3[\"rfrsqdv\"].append(performance_metric_r2(y_pred,tempPred))\n",
    "    \n",
    "    subset_coords = dataset_sub.iloc[:,xyind].values\n",
    "    rain_values = np.hstack(dataset_sub.iloc[:,yindiv].values)\n",
    "    full_grid = df.iloc[:,xyind].values\n",
    "    tempPred = inverseDistanceValues(subset_coords,rain_values,full_grid)\n",
    "    results3[\"idwmse\"].append(performance_metric_mse(y_pred,tempPred)*-1)\n",
    "    results3[\"idwmae\"].append(performance_metric_mae(y_pred,tempPred)*-1)\n",
    "    results3[\"idwrsq\"].append(performance_metric_r2(y_pred,tempPred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}