tools.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from modules import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matplotlib.rc('font', **{'family' : 'sans-serif',\n",
    "        'serif'  : 'Helvetica Neue',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 16})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def similar(a, b):\n",
    "    '''\n",
    "    Computes simlarity of two strings a and b using Python SequenceMatcher'''\n",
    "    return SequenceMatcher(None, a, b).ratio()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bqutil_set_project(project_id):\n",
    "    if project_id not in ['mitx-research', 'mitx-data', 'harvardx-data']:\n",
    "        return\n",
    "    f = open(\"/Users/cgn/Dropbox (MIT)/cgn/bq/edx2bigquery/edx2bigquery/edx2bigquery_config.py\", 'w')\n",
    "    text = '''PROJECT_ID = \"{project_id}\"\n",
    "auth_key_file = \"USE_GCLOUD_AUTH\"\n",
    "auth_service_acct = None'''.format(project_id=project_id)\n",
    "    f.write(text)\n",
    "    f.close"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bqutil_get_table(dataset_id, list_of_table_ids, path, gs_folder=None, project_id='mitx-research'):\n",
    "    '''\n",
    "    Downloads large tables using Google Storage.\n",
    "    dataset_id: string of dataset_id, also supports a list of dataset_id strings \n",
    "    gs_folder: Specify a folder if persistent storage in Google Storage is needed. \n",
    "    \n",
    "    Need to add support for extracting and combining sharded tables\n",
    "    '''\n",
    "    \n",
    "    persistent = True\n",
    "    \n",
    "    if gs_folder is None:\n",
    "        gs_folder = 'temp'\n",
    "        persistent = False\n",
    "    \n",
    "    if type(dataset_id) == str:\n",
    "        list_of_dataset_ids = [dataset_id for i in range(len(list_of_table_ids))]\n",
    "    else:\n",
    "        list_of_dataset_ids = dataset_id\n",
    "        \n",
    "    for i in range(len(list_of_table_ids)):\n",
    "        table_id = list_of_table_ids[i]\n",
    "        dataset_id = list_of_dataset_ids[i]\n",
    "        gsfn = 'gs://mitx-research/' + gs_folder + '/' + dataset_id + \"_\" + table_id + '*.csv.gz'\n",
    "        bqutil.extract_table_to_gs(dataset_id=dataset_id, table_id=table_id, gsfn=gsfn, format='csv', do_gzip=True, project_id=project_id)\n",
    "   \n",
    "    # Download all tables to path\n",
    "    gsfn = 'gs://mitx-research/' + gs_folder\n",
    "    cmd = \"gsutil -m cp -r \" + gsfn + \" \" + path\n",
    "    os.system(cmd)\n",
    "    \n",
    "    if not persistent:\n",
    "        #If user did not ask for persitent storage in gs, remove files   \n",
    "        cmd = 'gsutil -m rm -r ' + gsfn\n",
    "        os.system(cmd)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bqutil_df2bq(df, schema, project_id, dataset_id, table_id):\n",
    "    '''Uploads a dataframe to a bigquery table\n",
    "    Example schema:\n",
    "        schema = [{'type': 'STRING', 'name': 'cameo_candidate'},\n",
    "              {'type': 'STRING', 'name': 'shadow_candidate'},\n",
    "              {'type': 'FLOAT', 'name': 'similarity'},]\n",
    "    Example usage:\n",
    "        df2bq(df, schema, 'mitx-research', '0_cgn_sample', table)\n",
    "    '''\n",
    "    \n",
    "    csv_filename = table_id + '.csv' \n",
    "    df.to_csv(csv_filename, header = True, index = False)\n",
    "    schema_filename = table_id + \"__schema.json\"\n",
    "    open(schema_filename, 'w').write(json.dumps(schema, indent=4))\n",
    "    \n",
    "    address = project_id + ':' + dataset_id + '.' + table_id\n",
    "    #Remove table first, otherwise it will append\n",
    "    try:\n",
    "        bqutil.delete_bq_table(dataset_id, table_id, project_id)\n",
    "        print 'Overwriting table:', address\n",
    "    except:\n",
    "        print 'Creating table:', address\n",
    "    \n",
    "    #Upload table\n",
    "    cmd = \"bq --project_id=mitx-research load --skip_leading_rows=1 %s %s %s\" % (address, csv_filename, schema_filename)\n",
    "    print cmd\n",
    "    sys.stdout.flush()\n",
    "    os.system(cmd)\n",
    "    os.remove(csv_filename)\n",
    "    os.remove(schema_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bqutil_bq2df(project_id, dataset_id, table_id):\n",
    "    '''Downloads a BigQuery Table and stores it in a DataFrame.\n",
    "    This method guarantees preservation of column and row orders.\n",
    "    Return: A Pandas dataframe.\n",
    "    '''\n",
    "    info = bqutil.get_bq_table_info(dataset_id, table_id, project_id)\n",
    "    megabytes = float(info['numBytes']) / 1.0e6\n",
    "    print project_id + ':' + dataset_id + '.' + table_id +' is', str(megabytes) + ' Mb and', info['numRows'],'rows'\n",
    "    estimated_seconds = megabytes / 0.35792536\n",
    "    \n",
    "    print \"Downloading...\"\n",
    "    print \"Estimated time to download table from BQ:\", str(dt.timedelta(seconds = estimated_seconds))\n",
    "    sys.stdout.flush()\n",
    "    \n",
    "    t = datetime.now()\n",
    " \n",
    "    data = bqutil.get_table_data(dataset_id=dataset_id, table_id = table_id, project_id=project_id, maxResults=5000000)\n",
    "    print \"Download completed in\", str(datetime.now() - t)\n",
    "    sys.stdout.flush()\n",
    "    \n",
    "    if data is None:\n",
    "        return pd.DataFrame()\n",
    "    return pd.DataFrame(data['data'], columns=data['field_names'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bqutil_SQL2df(project_id, SQL, temp_project_id = 'mitx-research', temp_dataset = '0_cgn_sybils', temp_table = 'temp', overwrite = True):\n",
    "    '''Executes Google BigQuery SQL and stores the results in a DataFrame.\n",
    "    This method guarantees preservation of column and row orders.\n",
    "    Return: A Pandas dataframe.\n",
    "    '''\n",
    "    bqutil.create_bq_table(temp_dataset, temp_table, SQL, overwrite = overwrite, project_id = project_id, output_project_id=temp_project_id, allowLargeResults=True, sql_for_description='Created by Curtis G. Northcutt\\n' + SQL)\n",
    "    return bqutil_bq2df(project_id = temp_project_id, dataset_id = temp_dataset, table_id = temp_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def haversine(lon1, lat1, lon2, lat2):\n",
    "    \"\"\"\n",
    "    Calculate the great circle distance between two points \n",
    "    on the earth (specified in decimal degrees)\n",
    "    \"\"\"\n",
    "    # convert decimal degrees to radians \n",
    "    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])\n",
    "    # haversine formula \n",
    "    dlon = lon2 - lon1 \n",
    "    dlat = lat2 - lat1 \n",
    "    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2\n",
    "    c = 2 * asin(sqrt(a)) \n",
    "    km = 6367 * c\n",
    "    miles = km / 1.6\n",
    "    return km"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def kmeans(X, n = 5, initialization = 'kmeans++',verbose = False, num_iterations = -1):\n",
    "    ''' kmeans algorithm. \n",
    "        X - a nparray with rows as data points and columns as dimensions/features\n",
    "        n - the number of clusters\n",
    "        initialization - 'kmeans++' or 'random'\n",
    "        verbose prints running times\n",
    "        number_iterations - stops after num_iterations. Set to -1 for local optima without stopping.\n",
    "    '''\n",
    "    if initialization not in ['random','kmeans++']:\n",
    "        print('Initialization must be either \"random\" or \"kmeans++.\"')\n",
    "              \n",
    "    #initialization\n",
    "    K = n #number of clusters is K\n",
    "    (N, D) = np.shape(X)\n",
    "    mu = np.zeros((K, D))\n",
    "    r = []\n",
    "    \n",
    "    if initialization == 'random':\n",
    "        #Randomly assign labels\n",
    "        for i in range(N):\n",
    "            label = np.zeros((1,K))\n",
    "            label[0, random.randint(K)] = 1.0\n",
    "            r.append(label)\n",
    "        r = np.vstack(r)\n",
    "    else:\n",
    "        r = np.zeros((N, K))\n",
    "        mu = kmeanspp(X, n = K, verbose = verbose)\n",
    "        if verbose:\n",
    "            print('Cluster means:', mu)\n",
    "        #Compute closest cluster for each point\n",
    "        for n in range(N):\n",
    "            min_dist = np.inf\n",
    "            min_k = -1\n",
    "            for k in range(K):\n",
    "                dist = np.linalg.norm(X[n] - mu[k])\n",
    "                if dist < min_dist:\n",
    "                    min_dist = dist\n",
    "                    min_k = k  \n",
    "            r[n, min_k] = 1.0\n",
    "    \n",
    "    #Iterative kmeans\n",
    "    distort = []\n",
    "    r_changed = True\n",
    "    t = dt.datetime.now()\n",
    "    iteration_count = 0\n",
    "    \n",
    "    while r_changed:\n",
    "        \n",
    "        iteration_count += 1\n",
    "        r_changed = False\n",
    "        \n",
    "        if num_iterations != -1 and iteration_count > num_iterations:\n",
    "            break\n",
    "        \n",
    "        #Calculate distortion\n",
    "        d = distortion(r, X, mu)\n",
    "        distort.append(d)\n",
    "        if verbose:\n",
    "            print('\\nIteration', iteration_count, ', Time Elapsed:', dt.datetime.now() - t)\n",
    "            print('Distortion:', d)\n",
    "        \n",
    "        #Compute means of clusters\n",
    "        for k in range(K):\n",
    "            count = np.sum(r[:, k])\n",
    "            if (count != 0):\n",
    "                mu[k] = np.sum(X[r[:,k] == 1.0], axis = 0) / count\n",
    "        \n",
    "        #Compute closest cluster for each point\n",
    "        for n in range(N):\n",
    "            min_dist = np.inf\n",
    "            min_k = -1\n",
    "            for k in range(K):\n",
    "                dist = np.linalg.norm(X[n] - mu[k])\n",
    "                if dist < min_dist:\n",
    "                    min_dist = dist\n",
    "                    min_k = k\n",
    "            cur_k = np.where(r[n] == 1.0)[0][0] #Finds current cluster (i.e. index of 1 in one-hot encoded label vector)\n",
    "            if (cur_k != min_k):     \n",
    "                r[n] = np.zeros((1,K))\n",
    "                r[n, min_k] = 1.0\n",
    "                r_changed = True\n",
    "            \n",
    "    #Return a list of cluster labels\n",
    "    return (np.array([np.where(i == 1)[0][0] for i in r]), distort)          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def distortion(r, X, mu):\n",
    "    '''Computes the total distortion in the clusters.\n",
    "    r - responsibility one-hot encoded vectors\n",
    "    X - np.array with rows as data points and columns as dimensions/features\n",
    "    mu - means of clusters\n",
    "    '''\n",
    "    result = 0\n",
    "    for n in range(len(X)):\n",
    "        for k in range(len(mu)):\n",
    "            result += r[n][k] * np.linalg.norm(X[n] - mu[k])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def kmeanspp(X, n = 10, z = 2, verbose = False, figures = False):\n",
    "    '''Runs kmeans plus plus for initialization of kmeans\n",
    "    X = (N, D) matrix, input data matrix. rows are datum. colums are features/dimensions\n",
    "    n = int, number of clusers.\n",
    "    z = power on the distance. If z is big, more likely to select points far away from a mean.\n",
    "    verbose = boolean, print out while running'''\n",
    "    \n",
    "    #initialization\n",
    "    K = n #number of clusters is K\n",
    "    (N, D) = np.shape(X)\n",
    "    mu = np.zeros((K, D))\n",
    "    \n",
    "    #Make random datum first cluster center\n",
    "    mu[0] = X[random.randint(N)] \n",
    "    d = np.zeros(N) #distance of each point to closest center\n",
    "    \n",
    "    if verbose:\n",
    "        print('K-means++ Initialization Started.')\n",
    "        k = 0\n",
    "        if (figures and D == 2):\n",
    "            ax = pd.DataFrame(X).plot(x = 0, y = 1, c = 'crimson', kind='scatter', figsize = (14,14), s= 500,\n",
    "            title = \"Kmeans++ by Curtis G. Northcutt - Iteration:\" + str(k))\n",
    "            ax.set_xlabel('Dimension 1')\n",
    "            ax.set_ylabel('Dimension 2')\n",
    "            savefig('kmeanspp' + str(k) + '.png')\n",
    "    \n",
    "    #kmeanspp algorithm\n",
    "    for k in range(1, K):\n",
    "        for n in range(N):\n",
    "            min_dist = np.inf\n",
    "            for k_ in range(k):\n",
    "                dist = np.linalg.norm(X[n] - mu[k_])\n",
    "                if dist < min_dist:\n",
    "                    min_dist = dist\n",
    "            d[n] = min_dist\n",
    "            \n",
    "        #Make sure all d at least 1 by dividing by the min\n",
    "        d = d / min(d[np.nonzero(d)])\n",
    "        \n",
    "        #Compute distribution for each point\n",
    "        p = (d)**z/np.sum((d)**z) \n",
    "        \n",
    "        mu[k] = X[np.random.choice(range(N), p=p)] #sample from distribution for next mean\n",
    "        \n",
    "        if verbose:\n",
    "            print('Cluster mean', k+1, 'added.', 100.0*k/K, '% complete.')\n",
    "            if (figures and D == 2):\n",
    "                ax = pd.DataFrame(X).plot(x = 0, y = 1, c = 'crimson', kind='scatter', figsize = (14,14), s= 80000*p,\n",
    "                title = \"Kmeans++ by Curtis G. Northcutt - Iteration:\" + str(k))\n",
    "                ax.set_xlabel('Dimension 1')\n",
    "                ax.set_ylabel('Dimension 2')\n",
    "                scatter(mu[:k,0], mu[:k,1], s = 120, marker='x')\n",
    "                savefig('kmeanspp' + str(k) + '.png')\n",
    "                \n",
    "    print('K-means++ Initialization Complete. K-means commencing.')           \n",
    "    return mu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get20colors():\n",
    "    '''# These are the \"Tableau 20\" colors as RGB.\n",
    "    tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),\n",
    "             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),\n",
    "             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),\n",
    "             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),\n",
    "             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]\n",
    " \n",
    "    # Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts.\n",
    "    for i in range(len(tableau20)):\n",
    "        r, g, b = tableau20[i]\n",
    "        tableau20[i] = (r / 255., g / 255., b / 255.)\n",
    "        '''\n",
    "    \n",
    "    # These are the \"Tableau 20\" colors as RGB.\n",
    "    tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),\n",
    "             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),\n",
    "             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),\n",
    "             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),\n",
    "             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]\n",
    " \n",
    "    # Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts.\n",
    "    for i in range(len(tableau20)):\n",
    "        r, g, b = tableau20[i]\n",
    "        tableau20[i] = (r / 255., g / 255., b / 255.)\n",
    "        \n",
    "    return tableau20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def setfont(size = 20, weight = 'normal', serif = 'Helvetica Neue', family = 'sans-serif'):\n",
    "    matplotlib.rc('font', **{'family' : family,\n",
    "        'serif'  : serif,\n",
    "        'weight' : weight,\n",
    "        'size'   : size})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getfont(family = 'sans-serif', serif = 'Helvetica Neue', weight = 'normal', size = 10):\n",
    "    '''{'family' : 'sans-serif',\n",
    "        'serif'  : 'Helvetica Neue',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 22}\n",
    "        \n",
    "        Call by: matplotlib.rc('font', **font)'''\n",
    "    \n",
    "    return {'family' : family,\n",
    "        'serif'  : serif,\n",
    "        'weight' : weight,\n",
    "        'size'   : size}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def multinomial(x):\n",
    "    '''Method which does not require cumulative sum\n",
    "    Implements numpy.random.choice()'''\n",
    "    i = 0\n",
    "    val = np.random.uniform()\n",
    "    while(val >=0):\n",
    "        val -= x[i]\n",
    "        i += 1\n",
    "    return i-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def multinomial2(x):\n",
    "    '''Uses cumulative sum and finds smallest value thats great\n",
    "    Implements numpy.random.choice()'''\n",
    "    val = min(x[(np.cumsum(x)/max(np.cumsum(x))) > rand()])\n",
    "    return np.where(x == val)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def prettyplotdf(df, title, xlabel, ylabel, kind = 'scatter', figsize = (16,10), legend = False, linewidth = 1, c = 'crimson', s = 100, cmap = cm.get_cmap('Spectral')):\n",
    "    '''df - pandas DataFrame\n",
    "        x will be first column, y will be next column\n",
    "    '''\n",
    "    ax = df.plot(x = df.columns[0], y = df.columns[1], kind = kind, figsize = figsize, legend = legend, linewidth = linewidth, c = c, title = title, s = s, cmap=cmap)\n",
    "    ax.set_xlabel(xlabel)\n",
    "    ax.set_ylabel(ylabel)\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def prettyplotseries(x, title, xlabel, ylabel, kind = 'line', figsize = (16,10), legend = False, linewidth = 5, c = 'crimson', cmap = cm.get_cmap('Spectral')):\n",
    "    '''x - pandas series\n",
    "    '''\n",
    "    if kind == 'bar':\n",
    "        ax = x.plot(figsize = figsize, legend = legend, title = title, kind = kind, color = c)\n",
    "    else:\n",
    "        ax = x.plot(figsize = figsize, legend = legend, linewidth = linewidth, c = c,\n",
    "      title = title, kind = kind, cmap = cmap)\n",
    "        \n",
    "    ax.set_xlabel(xlabel)\n",
    "    ax.set_ylabel(ylabel)\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def figexamplehist(niter = 1000, dof = [1,3,5,7,9,50]):\n",
    "    '''niter - number of smaples in each degree of freedom\n",
    "    dof - degrees of freedom\n",
    "    \n",
    "    #Compute niter sqrt of sums for each of d gaussians.\n",
    "    X = np.zeros((niter,6))\n",
    "    for i in range(niter):\n",
    "        for ix, dimension in enumerate(dof):\n",
    "            X[i, ix] = np.sqrt(sum(np.random.normal(size = dimension)**2))\n",
    "\n",
    "    #Plot the histograms\n",
    "    plt.figure(figsize = (16, 10))\n",
    "    for i, dimension in enumerate(dof):\n",
    "        plt.hist(X[:,i], alpha = .4, bins = 50, linewidth = .1, label = 'dimensions (df) =' +str(dimension))\n",
    "\n",
    "    plt.title('Sqrt of Squared Sum of \"d\" Independent Normal Distributions', fontsize=20)\n",
    "    plt.xlabel('Euclidean Distance from Origin', fontsize=18)\n",
    "    plt.ylabel('Histogram of', niter, 'samples for each dimension', fontsize=16)\n",
    "    legendcontent = ['d = ' + str(d) for d in dof]\n",
    "    plt.legend([legendcontent, title = 'Number of Dimensions (Independent Guassians in Sum)')\n",
    "    '''\n",
    "    #Compute niter sqrt of sums for each of d gaussians.\n",
    "    X = np.zeros((niter,len(dof)))\n",
    "    for i in range(niter):\n",
    "        for ix, dimension in enumerate(dof):\n",
    "            X[i, ix] = np.sqrt(sum(np.random.normal(size = dimension)**2))\n",
    "\n",
    "    #Plot the histograms\n",
    "    plt.figure(figsize = (16, 10))\n",
    "    for i, dimension in enumerate(dof):\n",
    "        plt.hist(X[:,i], alpha = .4, bins = 50, linewidth = .1, label = 'dimensions (df) =' +str(dimension))\n",
    "\n",
    "    plt.title('Sqrt of Squared Sum of \"d\" Independent Normal Distributions', fontsize=20)\n",
    "    plt.xlabel('Euclidean Distance from Origin', fontsize=18)\n",
    "    plt.ylabel('Histogram of '+ str(niter) + ' samples for each dimension', fontsize=16)\n",
    "    legendcontent = ['d = ' + str(d) for d in dof]\n",
    "    plt.legend(legendcontent, title = 'Number of Dimensions (Independent Guassians in Sum)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def figexamplemultiscatter(niter = 1000, dof = [1,3,5,7,9,50]):\n",
    "    '''\n",
    "    plt.figure(figsize = (14, 8))\n",
    "    dimension = 1\n",
    "    x = np.linspace(0, 10, 1000)\n",
    "    for dimension in dof:\n",
    "        y = [scipy.stats.chi.pdf(i, df = dimension) for i in x]\n",
    "        plt.plot(x,y)\n",
    "    plt.title('Chi Distribution', fontsize=20)\n",
    "    plt.xlabel('Random Variable d, Distance from Origin', fontsize=18)\n",
    "    plt.ylabel('Probability Density Function', fontsize=16)\n",
    "    legendcontent = ['df = ' + str(d) for d in dof]\n",
    "    plt.legend(legendcontent, title = 'Number of Dimensions (degrees of freedom for Chi)')\n",
    "   '''\n",
    "    plt.figure(figsize = (14, 8))\n",
    "    dimension = 1\n",
    "    x = np.linspace(0, 20, 1000)\n",
    "    for dimension in dof:\n",
    "        y = [scipy.stats.chi.pdf(i, df = dimension) for i in x]\n",
    "        plt.plot(x,y)\n",
    "    plt.title('Chi Distribution', fontsize=20)\n",
    "    plt.xlabel('Random Variable d, Distance from Origin', fontsize=18)\n",
    "    plt.ylabel('Probability Density Function', fontsize=16)\n",
    "    legendcontent = ['df = ' + str(d) for d in dof]\n",
    "    plt.legend(legendcontent, title = 'Number of Dimensions (degrees of freedom for Chi)') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generateRandomClusters(n = 100, seperation = 6, uniformity = 1, gridSize = 5, randomness = 1.5):\n",
    "    '''\n",
    "    seperation - how distinct the grid clusters are (higher value is more sperated)\n",
    "    uniformity - lower uniformity tends to cluster at (0,0), higher uniformity makes all grid spots equally likely\n",
    "    gridSize - (gridSize x gridSize) lattice where clusters can go.\n",
    "    randomness - Variation within each cluster (exhibits uniform distribution as randomness grows larger)\n",
    "    \n",
    "    Explanation of how this works: randint(randint(3) + 1)*5 + np.random.normal()\n",
    "    The innermost term,randint(3) + 1, generates a random max for randint from .\n",
    "    randint then generates some random number between 0 and the random number we just found.\n",
    "    This is multiplied by 5, this spaces out the results of the random. For example,\n",
    "    if you generate random numbers in range [0,1,2] --> x5 --> [0,5, 10] (better seperate clusters)\n",
    "    Finally, we add gaussian noise around this random location. \n",
    "    In summary, we have a bunch of points we could generate centers, and how many times\n",
    "    we generate points in each point is randomized. And then we also add random noise to each\n",
    "    point. This results in light clustering.'''\n",
    "    \n",
    "    gridSize -= uniformity\n",
    "    X = [[np.random.randint(np.random.randint(gridSize) + uniformity)*seperation + randomness*np.random.normal(), \n",
    "          np.random.randint(np.random.randint(gridSize) + uniformity)*seperation + randomness*np.random.normal()] \n",
    "         for i in range(n)]\n",
    "    return np.array(X)\n",
    "\n",
    "#pd.DataFrame(X).plot(x = 0, y = 1, kind = 'scatter', figsize=(14,8), s = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def countrymap(country = None):\n",
    "    cmap = buildcountrymap()\n",
    "    if country == None:\n",
    "        return cmap\n",
    "    else:\n",
    "        return cmap[country]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def buildcountrymap():\n",
    "    locmap = {unicodedata.normalize('NFKD', i.name).encode('ascii','ignore').decode('utf-8') : \n",
    "        int(i.numeric) for i in list(pycountry.countries)}\n",
    "    locmap['Venezuela'] = locmap['Venezuela, Bolivarian Republic of']\n",
    "    locmap['Bolivia'] = locmap['Bolivia, Plurinational State of']\n",
    "    locmap['Macedonia'] = locmap['Macedonia, Republic of']\n",
    "    locmap['Libyan Arab Jamahiriya'] = locmap['Libya']\n",
    "    locmap['Taiwan'] = locmap['Taiwan, Province of China']\n",
    "    locmap['Moldova'] = locmap['Moldova, Republic of']\n",
    "    locmap['Antigua'] = locmap['Antigua and Barbuda']\n",
    "    locmap['Barbuda'] = locmap['Antigua and Barbuda']\n",
    "    locmap['Bonaire'] = locmap['Bonaire, Sint Eustatius and Saba']\n",
    "    locmap['Bosnia'] = locmap['Bosnia and Herzegovina']\n",
    "    locmap['Herzegovina'] = locmap['Bosnia and Herzegovina']\n",
    "    locmap['Cocos Islands'] = locmap['Cocos (Keeling) Islands']\n",
    "    locmap['Congo'] = locmap['Congo, The Democratic Republic of the']\n",
    "    locmap['Falkland Islands'] = locmap['Falkland Islands (Malvinas)']\n",
    "    locmap['Iran'] = locmap['Iran, Islamic Republic of']\n",
    "    locmap['Holy See'] = locmap['Holy See (Vatican City State)']\n",
    "    locmap['Vatican City State'] = locmap['Holy See (Vatican City State)']\n",
    "    locmap['Heard Island'] = locmap['Heard Island and McDonald Islands']\n",
    "    locmap['McDonald Islands'] = locmap['Heard Island and McDonald Islands']\n",
    "    locmap['Korea'] = locmap['Korea, Democratic People\\'s Republic of']\n",
    "    locmap['Lao'] = locmap['Lao People\\'s Democratic Republic']\n",
    "    locmap['Micronesia'] = locmap['Micronesia, Federated States of']\n",
    "    locmap['Palestine'] = locmap['Palestine, State of']\n",
    "    locmap['Saint Helena'] = locmap['Saint Helena, Ascension and Tristan da Cunha']\n",
    "    locmap['Saint Martin'] = locmap['Saint Martin (French part)']\n",
    "    locmap['Saint Kitts'] = locmap['Saint Kitts and Nevis']\n",
    "    locmap['Nevis'] = locmap['Saint Kitts and Nevis']\n",
    "    locmap['Saint Pierre'] = locmap['Saint Pierre and Miquelon']\n",
    "    locmap['Miquelon'] = locmap['Saint Pierre and Miquelon']\n",
    "    locmap['Saint Vincent'] = locmap['Saint Vincent and the Grenadines']\n",
    "    locmap['Grenadines'] = locmap['Saint Vincent and the Grenadines']\n",
    "    locmap['Sao Tome'] = locmap['Sao Tome and Principe']\n",
    "    locmap['Principe'] = locmap['Sao Tome and Principe']\n",
    "    locmap['Sint Maarten'] = locmap['Sint Maarten (Dutch part)']\n",
    "    locmap['South Georgia'] = locmap['South Georgia and the South Sandwich Islands']\n",
    "    locmap['South Sandwich Islands'] = locmap['South Georgia and the South Sandwich Islands']\n",
    "    locmap['Svalbard'] = locmap['Svalbard and Jan Mayen']\n",
    "    locmap['Jan Mayen'] = locmap['Svalbard and Jan Mayen']\n",
    "    locmap['Tanzania'] = locmap['Tanzania, United Republic of']\n",
    "    locmap['Trinidad'] = locmap['Trinidad and Tobago']\n",
    "    locmap['Tobago'] = locmap['Trinidad and Tobago']\n",
    "    locmap['Turks'] = locmap['Turks and Caicos Islands']\n",
    "    locmap['Caicos Islands'] = locmap['Turks and Caicos Islands']\n",
    "    locmap['Wallis'] = locmap['Wallis and Futuna']\n",
    "    locmap['Futuna'] = locmap['Wallis and Futuna']\n",
    "    locmap['British Virgin Islands'] = locmap['Virgin Islands, British']\n",
    "    locmap['U.S. Virgin Islands'] = locmap['Virgin Islands, U.S.']\n",
    "    locmap['Virgin Islands'] = locmap['Virgin Islands, U.S.']\n",
    "    locmap['Heard Island and Mcdonald Islands'] = locmap['Heard Island and McDonald Islands']\n",
    "    locmap['Netherlands Antilles'] = locmap['Netherlands']\n",
    "    locmap['Palestinian Territory, Occupied'] = locmap['Palestine']\n",
    "    locmap['Congo, the Democratic Republic of the'] = locmap['Congo']\n",
    "    locmap['Cote D\\'Ivoire'] = locmap['Cote d\\'Ivoire']\n",
    "    locmap['Virgin Islands, U.s.'] = locmap['Virgin Islands, U.S.']\n",
    "    return locmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prettyhistpercent(\n",
    "    series,\n",
    "    title = 'Histogram of percent of total questions answered', \n",
    "    xlabel = 'X = Percent of total questions answered',\n",
    "    ylabel= 'Number of students who answered X percent of questions',\n",
    "    bins = 40,\n",
    "    titlefontsize = 20,\n",
    "    axisfontsize = 16,\n",
    "    figsize = (16,10),\n",
    "    legend = False,\n",
    "    linewidth = .1,\n",
    "    c = 'crimson',\n",
    "    s = 100,\n",
    "    alpha = .5,\n",
    "    legendloc = 2,\n",
    "    legendcontent = 'Content of legend',\n",
    "    legendtitle = 'About the data'):\n",
    "    \n",
    "    fig = plt.figure(figsize = figsize)\n",
    "    ax = fig.add_subplot(1,1,1)\n",
    "    plt.hist(series, alpha = alpha, bins = bins, linewidth = linewidth, color = c)\n",
    "    plt.title(title, fontsize=titlefontsize)\n",
    "    plt.xlabel(xlabel, fontsize=axisfontsize)\n",
    "    plt.ylabel(ylabel, fontsize=axisfontsize)\n",
    "    plt.legend(legendcontent, title = legendtitle, loc = legendloc)\n",
    "    fmt = '%.f%%' # Format you want the ticks, e.g. '40%'\n",
    "    xticks = mtick.FormatStrFormatter(fmt)\n",
    "    ax.xaxis.set_major_formatter(xticks)\n",
    "    #plt.savefig('chinax_percent_of_questions_answered_by_possible_cheaters.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}