Merge pull request #69 from drkane/django

Django

.env.example

@@ -0,0 +1,7 @@
+SECRET_KEY=blahblah
+DJANGO_SETTINGS_MODULE=findthatcharity.settings
+DATABASE_URL=postgres://postgres:postgres@localhost/ftc_dj
+CACHE_URL=dbcache://findthatcharity_cache
+ES_URL=localhost:9200
+ALLOWED_HOSTS='.ftc.dkane.net;.findthatcharity.uk'
+DEBUG=False

.gitignore

@@ -1,11 +1,16 @@
 # data
 
 data/
+!ftc/tests/data/
+!ftc/tests/data/*.zip
 
+# djangofiles
+/static/
 
 # python files
 __pycache__/
 env/
+envd/
 scratchpad/
 .cache
 

DOKKU_SCALE

@@ -0,0 +1 @@
+web=4

Procfile

@@ -1 +1,2 @@
-web: gunicorn --pythonpath server server:app
+web: gunicorn findthatcharity.wsgi:application
+release: python manage.py migrate --noinput

addtocsv/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class AddtocsvConfig(AppConfig):
+    name = "addtocsv"

addtocsv/jinja2/addtocsv.html.j2

@@ -0,0 +1,132 @@
+{# {% from '_utils.html' import info_block %} #}
+{% set title = 'Add fields to CSV' %}
+{% extends "base.html.j2" %}
+
+{% block headscripts %}
+<script src="https://cdnjs.cloudflare.com/ajax/libs/PapaParse/5.1.0/papaparse.min.js" integrity="sha256-Fh801SO9gqegfUdkDxyzXzIUPWzO/Vatqj8uN+5xcL4=" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.9-1/core.min.js" integrity="sha256-hV6Ff1ZbnLObO8BWHPZs1oA3aPZkX4bnnEKO4nX1sm0=" crossorigin="anonymous"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/crypto-js/3.1.9-1/md5.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
+{% endblock %}
+
+{% block content %}
+<form action="" method="post" enctype="multipart/form-data" class="w-100 w-70-l fl-l pr3-l">
+  <div class="w-100 mb4 entry-content cf">
+    <p class="measure">Add data to a CSV file by looking it up from a column with a charity number or other organisation identifier.</p>
+    <p class="measure">The file should be separated by commas (<code>,</code>) - not semicolons or tabs)
+    and the first row should contain the column names.</p>
+    <a class="dn-l link underline blue f5" href="#privacy">See important note on privacy below</a>
+  </div>
+  <div class="w-100 b--light-gray ba bw1 br3 mb4" id="addtocsv">
+
+    <div class="w-100 pa2 b--light-gray bb bw1 stage" id="stage-select-file">
+      <h3 class="pa0 ma0 dib">
+        Step 1: 
+        <span class="normal">Select CSV file</span>
+      </h3>
+      <span class="fr contents-top" v-if="stage > 0">
+        <a href="#" id="reset-select-file" class="stage-reset link f6 blue underline dim" v-on:click.prevent="csv = null">Change file</a>
+        <span class="file-name dib code pa1 bg-light-gray lh-solid" id="csvfilename">[[filename]]</span>
+      </span>
+      <div class="contents mv3" v-else-if="stage == 0">
+        <label class="file-label">
+          <input class="file-input w-100" type="file" name="csvfile" id="csvfile" accept="text/csv,.csv" v-on:change="selectFile">
+          <span class="file-cta">
+            <span class="file-icon">
+              <i class="fas fa-upload"></i>
+            </span>
+            <span class="file-label">
+              Choose a file…
+            </span>
+          </span>
+        </label>
+        <div class="file has-name is-fullwidth">
+        </div>
+        <p class="gray f6 pa0 ma0">
+          Files with a large number of rows will cause
+          your browser to slow and may not complete successfully.
+        </p>
+
+      </div>
+    </div>
+
+    <div class="w-100 pa2 b--light-gray bb bw1 stage" id="stage-select-identifier-field">
+      <h3 class="pa0 ma0 dib">
+        Step 2: 
+        <span class="normal">Select organisation identifier field</span>
+      </h3>
+      <span class="fr contents-top" v-if="stage > 1">
+        <a href="#" id="reset-select-identifier-field" class="stage-reset link f6 blue underline dim" v-on:click.prevent="column_to_use = null">Change field</a>
+        <span class="dib code pa1 bg-light-gray lh-solid" id="column-name-desc">[[column_to_use]]</span>
+        <input class="dn" type="text" name="column_name" id="column_name" :value="column_to_use">
+      </span>
+      <div class="contents mv4" v-else-if="stage == 1">
+        <div class="field mb3" id="csvpreview">
+          <ul class="list pa0 ma0">
+            <li v-for="f in field_select" class="mb2">
+                <label class="pointer">
+                    <input type="radio" name="identifier_field" :value=[[f.name]] v-on:click.prevent="column_to_use = f.name" />
+                    <span class="code pa1 bg-light-gray underline-hover">[[f.name]]</span>
+                    <ul class="list pa0 ma0">
+                        <li class="dib mr2 gray mt1 f6 tl truncate mw5" v-for="v in f.example_values">[[v]]</li>
+                    </ul>
+                </label>
+            </li>
+          </ul>
+        </div>
+      </div>
+    </div>
+
+    <div class="w-100 pa2 b--light-gray bb bw1 stage" id="stage-select-fields">
+      <h3 class="pa0 ma0 dib">
+        Step 3: 
+        <span class="normal">Select data to add</span>
+      </h3>
+      <span class="fr contents-top" v-if="stage > 2">
+        <span class="dib code pa1 bg-light-gray" id="fields_to_add"></span>
+      </span>
+      <div class="contents mv4" v-else-if="stage == 2">
+        <ul class="list ma0 pa0">
+            <li v-for="field in fields" :key="field.id">
+                <label>
+                    <input type="checkbox" name="fields" :value="field.id" v-model="fields_to_add">
+                    [[field.name]]
+                </label>
+            </li>
+        </ul>
+
+        <div class="contents mv4">
+            <input class="button-reset bn pv3 ph4 b tc bg-animate bg-yellow dim near-black pointer br2-ns ml4-l" type="submit" 
+                   value="Add data to CSV" id='fetch_identifiers' v-on:click.prevent="getResults" />
+            <div id="result" v-if="progress !== null">
+                <p class="pa0 mv3 mh2" id="result-text">Creating file…</p>
+                <div id="progress-bar" class="bg-light-blue h2 mt4 mh2">
+                    <div class="progress-bar-inner bg-blue h2 ph3 pv1 f6 tr white" v-bind:style="{ width: progress + '%' }" id="progress-bar-inner">[[progress]]%</div>
+                </div>
+            </div>
+        </div>
+      </div>
+
+    </div>
+
+  </div>
+</form>
+
+<div class="content w-100 w-30-l fl-l pa3 bg-light-gray f5" id="privacy">
+  <h3 class="pa0 ma0 header-font">Privacy</h3>
+  <p>Your file will not leave your computer, but the organsiation identifiers from
+    the column you select will be sent to the findthatcharity server in order to 
+    retrieve the information. No other data is sent to the server.</p>
+  <p>It could therefore be possible to reconstruct the organisations contained
+    in your file. <strong>You should think carefully before using this tool
+    for any personal or sensitive information.</strong></p>
+</div>
+{% endblock %}
+
+{% block bodyscripts %}
+<script type="text/javascript">
+const PROPERTIES_URL = {{ request.build_absolute_uri(url('propose_properties'))|tojson }};
+const EXTEND_URL = {{ request.build_absolute_uri(url('reconcile'))|tojson }};
+</script>
+<script src='{{ static("js/csv.js") }}' type="text/javascript"></script>
+{% endblock %}

addtocsv/static/js/csv.js

@@ -0,0 +1,157 @@
+const DEFAULT_HASH_LENGTH = 4;
+const ARRAY_CHUNK_SIZE = 100;
+const FIELD_MATCH_REGEX = /(organi[sz]ation|org_?id)/g;
+
+function chunk_array(arr, len) {
+    var chunks = [],
+        i = 0,
+        n = arr.length;
+    while (i < n) {
+        chunks.push(arr.slice(i, i += len));
+    }
+    return chunks;
+}
+
+
+function openSaveFileDialog(data, filename, mimetype) {
+    // from: https://github.com/mholt/PapaParse/issues/175#issuecomment-395978144
+
+    if (!data) return;
+
+    var blob = data.constructor !== Blob
+        ? new Blob([data], { type: mimetype || 'application/octet-stream' })
+        : data;
+
+    if (navigator.msSaveBlob) {
+        navigator.msSaveBlob(blob, filename);
+        return;
+    }
+
+    var lnk = document.createElement('a'),
+        url = window.URL,
+        objectURL;
+
+    if (mimetype) {
+        lnk.type = mimetype;
+    }
+
+    lnk.download = filename || 'untitled';
+    lnk.href = objectURL = url.createObjectURL(blob);
+    lnk.dispatchEvent(new MouseEvent('click'));
+    setTimeout(url.revokeObjectURL.bind(url, objectURL));
+
+}
+
+var app = new Vue({
+    el: '#addtocsv',
+    data: {
+        fields: [],
+        csv: null,
+        csv_results: null,
+        field_select: [],
+        fields_to_add: [],
+        column_to_use: null,
+        column_to_use_auto: false,
+        progress: null,
+    },
+    delimiters: ['[[', ']]'],
+    mounted() {
+        fetch(PROPERTIES_URL)
+            .then(r => r.json())
+            .then(r => (this.fields = r.properties))
+    },
+    methods: {
+        resetFile: function(){
+            this.csv = null;
+            this.csv_results = null;
+            this.field_select = [];
+            this.fields_to_add = [];
+            this.column_to_use = null;
+            this.column_to_use_auto = false;
+            this.progress = null;
+        },
+        selectFile: function(ev) {
+            this.csv = ev.target.files[0];
+            var component = this;
+
+            Papa.parse(this.csv, {
+                header: true,
+                worker: true,
+                complete: function (results) {
+                    component.csv_results = results;
+                    component.field_select = results.meta.fields.map((f) => ({
+                        name: f,
+                        slug: f.toLowerCase().replace(/"\s+"/gi, "-").replace(/[^a-z0-9]/gi, ''),
+                        example_values: results.data
+                            .map((d) => d[f])
+                            .filter((v, i, a) => a.indexOf(v) === i)
+                            .slice(0, 5)
+                    }));
+                    could_use = component.field_select.filter((f) => FIELD_MATCH_REGEX.test(f.name));
+                    if(could_use.length > 0){
+                        component.column_to_use = could_use[0].name;
+                        component.column_to_use_auto = true;
+                    }
+                }
+            });
+        },
+        getResults: function(){
+            var component = this;
+            var ids = this.csv_results.data.map((d) => d[component.column_to_use]);
+            var id_chunks = chunk_array(ids, ARRAY_CHUNK_SIZE);
+            var rows_done = 0;
+            component.progress = 0;
+            return Promise.all(id_chunks.map(id_chunk => {
+                var body = new FormData();
+                body.append("extend", JSON.stringify({
+                    "ids": id_chunk,
+                    "properties": component.fields_to_add.map((p) => ({id: p})),
+                }));
+                return fetch(EXTEND_URL, {
+                    method: 'POST',
+                    body: body,
+                })
+                    .then((res) => {
+                        rows_done += id_chunk.length;
+                        component.progress = ((rows_done / ids.length) * 100).toFixed(1);
+                        return res.json()
+                    })
+                    .then((data) => data.rows)
+                    .catch((error) => {
+                        console.log('Error: ', error)
+                    })
+                }))
+                .then(data => Object.assign({}, ...data))
+                .then(new_data => openSaveFileDialog(
+                    Papa.unparse({
+                        fields: component.csv_results.meta.fields.concat(component.fields_to_add),
+                        data: component.csv_results.data.map(row => Object.assign(
+                            row,
+                            new_data[row[component.column_to_use]]
+                        ))
+                    }),
+                    component.csv.name.replace(".csv", "-org.csv")
+                ));
+        },
+    },
+    computed: {
+        stage: function () {
+            if(!this.csv){
+                return 0;
+            }
+            if(this.column_to_use){
+                return 2;
+            }
+            return 1;
+        },
+        filename: function () {
+            if (!this.csv) {
+                return null;
+            }
+            if (this.csv_results) {
+                return this.csv.name + " (" + this.csv_results.data.length + " rows)";
+            }
+            return this.csv.name;
+        }
+    }
+})

addtocsv/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

addtocsv/views.py

@@ -0,0 +1,6 @@
+from django.shortcuts import render
+
+
+def index(request):
+    context = {}
+    return render(request, "addtocsv.html.j2", context)