From 749a4afbd88527077ef51c006242809623dd9d82 Mon Sep 17 00:00:00 2001
From: Greg Heinrich <gheinrich@nvidia.com>
Date: Mon, 27 Jul 2015 17:26:01 +0200
Subject: [PATCH] Enable use of server files in dataset creation - closes #106

---
 digits/dataset/images/classification/forms.py | 44 ++++++++-
 digits/dataset/images/classification/views.py | 49 +++++----
 .../datasets/images/classification/new.html   | 99 ++++++++++++++-----
 3 files changed, 144 insertions(+), 48 deletions(-)

diff --git a/digits/dataset/images/classification/forms.py b/digits/dataset/images/classification/forms.py
index d08840119..d3864b3eb 100644
--- a/digits/dataset/images/classification/forms.py
+++ b/digits/dataset/images/classification/forms.py
@@ -106,9 +106,19 @@ def validate_folder_path(form, field):
 
     ### Method - textfile
 
+    textfile_use_local_files = wtforms.BooleanField(u'Use local files',
+        default=False)
+
     textfile_train_images = wtforms.FileField(u'Training images',
             validators=[
-                validate_required_iff(method='textfile')
+                validate_required_iff(method='textfile',
+                                      textfile_use_local_files=False)
+                ]
+            )
+    textfile_local_train_images = wtforms.StringField(u'Training images',
+            validators=[
+                validate_required_iff(method='textfile',
+                                      textfile_use_local_files=True)
                 ]
             )
     textfile_train_folder = wtforms.StringField(u'Training images folder')
@@ -134,7 +144,16 @@ def validate_textfile_train_folder(form, field):
             validators=[
                 validate_required_iff(
                     method='textfile',
-                    textfile_use_val=True)
+                    textfile_use_val=True,
+                    textfile_use_local_files=False)
+                ]
+            )
+    textfile_local_val_images = wtforms.StringField(u'Validation images',
+            validators=[
+                validate_required_iff(
+                    method='textfile',
+                    textfile_use_val=True,
+                    textfile_use_local_files=True)
                 ]
             )
     textfile_val_folder = wtforms.StringField(u'Validation images folder')
@@ -160,7 +179,16 @@ def validate_textfile_val_folder(form, field):
             validators=[
                 validate_required_iff(
                     method='textfile',
-                    textfile_use_test=True)
+                    textfile_use_test=True,
+                    textfile_use_local_files=False)
+                ]
+            )
+    textfile_local_test_images = wtforms.StringField(u'Test images',
+            validators=[
+                validate_required_iff(
+                    method='textfile',
+                    textfile_use_test=True,
+                    textfile_use_local_files=True)
                 ]
             )
     textfile_test_folder = wtforms.StringField(u'Test images folder')
@@ -191,7 +219,15 @@ def validate_textfile_test_folder(form, field):
 
     textfile_labels_file = wtforms.FileField(u'Labels',
             validators=[
-                validate_required_iff(method='textfile')
+                validate_required_iff(method='textfile',
+                                      textfile_use_local_files=False)
+                ]
+            )
+
+    textfile_local_labels_file = wtforms.StringField(u'Labels',
+            validators=[
+                validate_required_iff(method='textfile',
+                                      textfile_use_local_files=True)
                 ]
             )
 
diff --git a/digits/dataset/images/classification/views.py b/digits/dataset/images/classification/views.py
index ac36e4b8c..ef5c9a77e 100644
--- a/digits/dataset/images/classification/views.py
+++ b/digits/dataset/images/classification/views.py
@@ -118,20 +118,25 @@ def from_files(job, form):
     Add tasks for creating a dataset by reading textfiles
     """
     ### labels
-
-    flask.request.files[form.textfile_labels_file.name].save(
-            os.path.join(job.dir(), utils.constants.LABELS_FILE)
-            )
-    job.labels_file = utils.constants.LABELS_FILE
+    if form.textfile_use_local_files.data:
+        job.labels_file = form.textfile_local_labels_file.data.strip()
+    else:
+        flask.request.files[form.textfile_labels_file.name].save(
+                os.path.join(job.dir(), utils.constants.LABELS_FILE)
+                )
+        job.labels_file = utils.constants.LABELS_FILE
 
     encoding = form.encoding.data
     shuffle = bool(form.textfile_shuffle.data)
 
     ### train
-
-    flask.request.files[form.textfile_train_images.name].save(
-            os.path.join(job.dir(), utils.constants.TRAIN_FILE)
-            )
+    if form.textfile_use_local_files.data:
+        train_file = form.textfile_local_train_images.data.strip()
+    else:
+        flask.request.files[form.textfile_train_images.name].save(
+                os.path.join(job.dir(), utils.constants.TRAIN_FILE)
+                )
+        train_file = utils.constants.TRAIN_FILE
 
     image_folder = form.textfile_train_folder.data.strip()
     if not image_folder:
@@ -140,7 +145,7 @@ def from_files(job, form):
     job.tasks.append(
             tasks.CreateDbTask(
                 job_dir     = job.dir(),
-                input_file  = utils.constants.TRAIN_FILE,
+                input_file  = train_file,
                 db_name     = utils.constants.TRAIN_DB,
                 image_dims  = job.image_dims,
                 image_folder= image_folder,
@@ -155,9 +160,13 @@ def from_files(job, form):
     ### val
 
     if form.textfile_use_val.data:
-        flask.request.files[form.textfile_val_images.name].save(
-                os.path.join(job.dir(), utils.constants.VAL_FILE)
-                )
+        if form.textfile_use_local_files.data:
+            val_file = form.textfile_local_val_images.data.strip()
+        else:
+            flask.request.files[form.textfile_val_images.name].save(
+                    os.path.join(job.dir(), utils.constants.VAL_FILE)
+                    )
+            val_file = utils.constants.VAL_FILE
 
         image_folder = form.textfile_val_folder.data.strip()
         if not image_folder:
@@ -166,7 +175,7 @@ def from_files(job, form):
         job.tasks.append(
                 tasks.CreateDbTask(
                     job_dir     = job.dir(),
-                    input_file  = utils.constants.VAL_FILE,
+                    input_file  = val_file,
                     db_name     = utils.constants.VAL_DB,
                     image_dims  = job.image_dims,
                     image_folder= image_folder,
@@ -180,9 +189,13 @@ def from_files(job, form):
     ### test
 
     if form.textfile_use_test.data:
-        flask.request.files[form.textfile_test_images.name].save(
-                os.path.join(job.dir(), utils.constants.TEST_FILE)
-                )
+        if form.textfile_use_local_files.data:
+            test_file = form.textfile_local_test_images.data.strip()
+        else:
+            flask.request.files[form.textfile_test_images.name].save(
+                    os.path.join(job.dir(), utils.constants.TEST_FILE)
+                    )
+            test_file = utils.constants.TEST_FILE
 
         image_folder = form.textfile_test_folder.data.strip()
         if not image_folder:
@@ -191,7 +204,7 @@ def from_files(job, form):
         job.tasks.append(
                 tasks.CreateDbTask(
                     job_dir     = job.dir(),
-                    input_file  = utils.constants.TEST_FILE,
+                    input_file  = test_file,
                     db_name     = utils.constants.TEST_DB,
                     image_dims  = job.image_dims,
                     image_folder= image_folder,
diff --git a/digits/templates/datasets/images/classification/new.html b/digits/templates/datasets/images/classification/new.html
index 3c6e61a5d..156f627d0 100644
--- a/digits/templates/datasets/images/classification/new.html
+++ b/digits/templates/datasets/images/classification/new.html
@@ -127,7 +127,7 @@ <h1>New Image Classification Dataset</h1>
             <!-- Nav tabs -->
             <ul class="nav nav-tabs" id="db-tabs" role="tablist">
                 <li><a href="#db-folder" role="tab" data-toggle="tab">Use Image Folder</a></li>
-                <li><a href="#db-textfile" role="tab" data-toggle="tab">Upload Text Files</a></li>
+                <li><a href="#db-textfile" role="tab" data-toggle="tab">Use Text Files</a></li>
             </ul>
 
             <!-- Tab panes -->
@@ -234,7 +234,7 @@ <h1>New Image Classification Dataset</h1>
                                 <th>Set</th>
 
                                 <th>Text file <span name="text_file_explanation"
-                                    class="explanation-tooltip glyphicon glyphicon-question-sign"
+                          class="explanation-tooltip glyphicon glyphicon-question-sign"
                                     data-container="body"
                                     title="Each line in the file should be formatted as '<PATH> <LABEL>' where <PATH> specifies image path either on the local filesystem or a URL, and the <LABEL> is a numeric label that matches the uploaded labels file"
                                 ></span></th>
@@ -245,11 +245,29 @@ <h1>New Image Classification Dataset</h1>
                                     title="Paths in the text files will be appended with this value before reading"
                                 ></span></th>
                             </tr>
+                            <tr>
+                                <td></td>
+                                <td>
+                                    <label for="{{form.textfile_use_local_files.name}}">
+                                        {{form.textfile_use_local_files}}
+                                        Use local paths on server
+                                    </label>
+                                    <span name="text_file_explanation"
+                                          class="explanation-tooltip glyphicon glyphicon-question-sign"
+                                          data-container="body"
+                                          title="Check to use local paths on server, uncheck to upload files from client machine"
+                                    ></span>
+                                </td>
+                                <td></td>
+                            </tr>
                             <tr>
                                 <td><b>Training</b></td>
-                                <td class="form-group{{' has-error' if form.textfile_train_images.errors}}">
+                                <td class="form-group{{' has-error' if form.textfile_train_images.errors}} cl-upload-files">
                                     {{ form.textfile_train_images(class='form-control') }}
                                 </td>
+                                <td class="form-group{{' has-error' if form.textfile_local_train_images.errors}} cl-local-files">
+                                    {{ form.textfile_local_train_images(class='form-control',placeholder='enter path') }}
+                                </td>
                                 <td class="form-group{{' has-error' if form.textfile_train_folder.errors}}">
                                     {{ form.textfile_train_folder(class='form-control') }}
                                 </td>
@@ -261,9 +279,12 @@ <h1>New Image Classification Dataset</h1>
                                         Validation
                                     </label>
                                 </td>
-                                <td class="form-group{{' has-error' if form.textfile_val_images.errors}}">
+                                <td class="form-group{{' has-error' if form.textfile_val_images.errors}} cl-upload-files">
                                     {{ form.textfile_val_images(class='form-control') }}
                                 </td>
+                                <td class="form-group{{' has-error' if form.textfile_local_val_images.errors}} cl-local-files">
+                                    {{ form.textfile_local_val_images(class='form-control',placeholder='enter path') }}
+                                </td>
                                 <td class="form-group{{' has-error' if form.textfile_val_folder.errors}}">
                                     {{ form.textfile_val_folder(class='form-control') }}
                                 </td>
@@ -275,35 +296,18 @@ <h1>New Image Classification Dataset</h1>
                                         Test
                                     </label>
                                 </td>
-                                <td class="form-group{{' has-error' if form.textfile_test_images.errors}}">
+                                <td class="form-group{{' has-error' if form.textfile_test_images.errors}} cl-upload-files">
                                     {{ form.textfile_test_images(class='form-control') }}
                                 </td>
+                                <td class="form-group{{' has-error' if form.textfile_local_test_images.errors}} cl-local-files">
+                                    {{ form.textfile_local_test_images(class='form-control',placeholder='enter path') }}
+                                </td>
                                 <td class="form-group{{' has-error' if form.textfile_test_folder.errors}}">
                                     {{ form.textfile_test_folder(class='form-control') }}
                                 </td>
                             </tr>
                         </table>
                     </div>
-                    <script>
-// Disable form fields according to the checkboxes
-function textfile_image_sets_disabled() {
-    var value = $("#textfile_use_val").prop("checked");
-    $("#textfile_val_images").prop("disabled", !value);
-    $("#textfile_val_folder").prop("disabled", !value);
-    value = $("#textfile_use_test").prop("checked");
-    $("#textfile_test_images").prop("disabled", !value);
-    $("#textfile_test_folder").prop("disabled", !value);
-}
-
-$("#textfile_use_val").click(function() {
-        textfile_image_sets_disabled();
-        });
-$("#textfile_use_test").click(function() {
-        textfile_image_sets_disabled();
-        });
-textfile_image_sets_disabled();
-                    </script>
-
                     <div class="row">
                         <div class="col-sm-6 col-sm-offset-1">
                             <div class="form-group{{ ' has-error' if form.textfile_shuffle.errors else '' }}">
@@ -317,7 +321,7 @@ <h1>New Image Classification Dataset</h1>
                                         ></span>
                                 </div>
                             </div>
-                            <div class="form-group{{ ' has-error' if form.textfile_labels_file.errors else '' }}">
+                            <div class="form-group{{ ' has-error' if form.textfile_labels_file.errors else '' }} cl-upload-files">
                                 {{ form.textfile_labels_file.label }}
                                 <div class="input-group">
                                     {{ form.textfile_labels_file(class='form-control') }}
@@ -328,6 +332,17 @@ <h1>New Image Classification Dataset</h1>
                                         ></span>
                                 </div>
                             </div>
+                            <div class="form-group{{ ' has-error' if form.textfile_local_labels_file.errors else '' }} cl-local-files">
+                                {{ form.textfile_local_labels_file.label }}
+                                <div class="input-group">
+                                    {{ form.textfile_local_labels_file(class='form-control',placeholder='enter path') }}
+                                    <span name="textfile_labels_file_explanation"
+                                        class="input-group-addon explanation-tooltip glyphicon glyphicon-question-sign"
+                                        data-container="body"
+                                        title="The 'i'th line of the file should give the string label associated with the '(i-1)'th numberic label. (E.g. the string label for the numeric label 0 is supposed to be on line 1.)"
+                                        ></span>
+                                </div>
+                            </div>
                         </div>
                     </div>
                 </div>
@@ -337,6 +352,38 @@ <h1>New Image Classification Dataset</h1>
     </div>
     {{ form.method(style="display:none;") }}
     <script>
+// Disable form fields according to the checkboxes
+function textfile_image_sets_disabled() {
+    var value = $("#textfile_use_val").prop("checked");
+    var useLocalFiles = $("#textfile_use_local_files").prop("checked");
+    if (useLocalFiles) {
+        $(".cl-upload-files").hide();
+        $(".cl-local-files").show();
+    }
+    else{
+        $(".cl-upload-files").show();
+        $(".cl-local-files").hide();
+    }
+    $("#textfile_val_images").prop("disabled", !value);
+    $("#textfile_local_val_images").prop("disabled", !value);
+    $("#textfile_val_folder").prop("disabled", !value);
+    value = $("#textfile_use_test").prop("checked");
+    $("#textfile_test_images").prop("disabled", !value);
+    $("#textfile_local_test_images").prop("disabled", !value);
+    $("#textfile_test_folder").prop("disabled", !value);
+}
+
+$("#textfile_use_val").click(function() {
+        textfile_image_sets_disabled();
+        });
+$("#textfile_use_test").click(function() {
+        textfile_image_sets_disabled();
+        });
+$("#textfile_use_local_files").click(function() {
+        textfile_image_sets_disabled();
+        });
+textfile_image_sets_disabled();
+
 function update_db_tab(tabName) {
     $('select[name=method]').val( tabName );
 }