support python3 in ailab, also fixes some minor issues

Summary: Pull Request resolved: #279

Reviewed By: hl475

Differential Revision: D15196928

Pulled By: sf-wind

fbshipit-source-id: 633e27c8cee4531aa782fc3c7a3db0d66456fcae

.gitignore

@@ -71,3 +71,6 @@ ailab/*/migrations/*.py
 
 # Django secret key
 ailab/ailab/secret_key.py
+
+ailab/static/
+ailab/media/

ailab/README.md

@@ -29,9 +29,15 @@ apt-get install python-dev libmysqlclient-dev
 
 
 - Install components for ailab:
+  - Python2
 ```
 pip install MySQL-python
 pip install django-tables2 "django<2" django-nvd3 django-widget-tweaks django-bootstrap3
+```
+  - Python3 (On Mac, follow instructions in https://pypi.org/project/mysqlclient/ to fix `mysqlclient` package)
+```
+pip install mysqlclient
+pip install django-tables2 django-nvd3 django-widget-tweaks django-bootstrap3
 ```
 - In the current directory (ailab), run the following commands:
 ```
@@ -88,19 +94,21 @@ nginx
 ```
 python manage.py collectstatic --clear
 ```
-2. Start uWSGI from this directory in production setting
+2. Start uWSGI from this directory in production setting.
+Need to increase the buffer size since the URL may be quite long.
 ```
-uwsgi --socket :8001 --module ailab.wsgi
+uwsgi --socket :8001 --module ailab.wsgi -b 32768
 ```
 
 Now, the server should be up and running, ready to receive requests.
 
 
 ## Start Lab
-On server, inside `benchmarking` directory, run:
+On server, inside `benchmarking` directory, run the following command to start an android lab:
 ```
-python run_bench.py --lab --claimer_id <claimer_id> --server_addr <server_name> --remote_reporter "<server_name>/benchmark/store-result|oss"
+python run_bench.py --lab --claimer_id <claimer_id> --server_addr <server_name> --remote_reporter "<server_name>/benchmark/store-result|oss" --platform android
 ```
+The lab needs to be able to build android binary.
 
 ## Run Benchmark Remotely
 On another machine, inside `benchmarking` directory, invoke the lab by running:

ailab/benchmark/benchmark_result_controller.py

@@ -11,7 +11,7 @@ def store_result(data):
     for log in logs:
         store_single_entry(log)
 
-    return {"status": "success"}
+    return {"status": "success", "count": len(logs)}
 
 
 def store_single_entry(log):

ailab/benchmark/templates/result_visualization.html

@@ -15,6 +15,9 @@
 
 
 <div class="container-fluid">
+  <nav class="navbar navbar-default">
+    <a class="navbar-brand" href="https://github.com/facebook/FAI-PEP">FAI-PEP</a>
+  </nav>
   <div class="row">
     <div class="col-md-2">
       <ul class="nav nav-tabs" id="configTab" role="tablist">
@@ -106,7 +109,7 @@ <h4 style="margin-top: 0">Graph Configuration</h4>
       <div id="builder"></div>
     </div>
 
-    <div class="col-md-8">
+    <div class="col-md-10">
       <div id='graph-view' class="panel panel-default">
         {% include 'graph_view.html' %}
       </div>

ailab/benchmark/views.py

@@ -9,12 +9,12 @@
 from django.views.decorators.csrf import csrf_exempt
 from django_tables2 import RequestConfig
 
-import db_controller
-import benchmark_result_controller
-from result_table import ResultTable
+from .db_controller import get_payload
+from .benchmark_result_controller import store_result
+from .result_table import ResultTable
 from .models import BenchmarkResult
 
-import visualize_utils as utils
+from .visualize_utils import construct_q
 
 
 PLOTABLE_COL_SET = {
@@ -37,15 +37,15 @@
 @csrf_exempt
 def handle_request(request):
     data = request.POST.copy()
-    results = db_controller.get_payload(data)
+    results = get_payload(data)
 
     return JsonResponse(results)
 
 
 @csrf_exempt
 def store_benchmark_result(request):
     data = json.loads(request.body)
-    results = benchmark_result_controller.store_result(data)
+    results = store_result(data)
 
     return JsonResponse(results)
 
@@ -69,7 +69,7 @@ def visualize(request):
     filters = {} if request.GET.get('filters') is None \
         else json.loads(request.GET.get('filters'))
     if len(filters) != 0 and filters['valid']:
-        result_q = utils.construct_q(filters)
+        result_q = construct_q(filters)
         qs = BenchmarkResult.objects.filter(result_q)
     else:
         qs = BenchmarkResult.objects.all()

benchmarking/bridge/upload_download_files_django.py

@@ -17,7 +17,7 @@ def __init__(self, args):
     def upload(self, **kwargs):
         path = kwargs['file']
         if self.server_addr:
-            storage_addr = self.server_addr + "upload/"
+            storage_addr = self.server_addr + "/upload/"
             getLogger().info("Uploading {} to {}".format(path, storage_addr))
             filename = os.path.basename(path)
 
@@ -27,7 +27,7 @@ def upload(self, **kwargs):
 
         url = ''
         if result_json['status'] == 'success':
-            url = self.server_addr + result_json['path']
+            url = self.server_addr + "/" + result_json['path']
         getLogger().info("File has been uploaded to {}".format(url))
         return url
 

benchmarking/remote/django_url_printer.py

@@ -26,7 +26,7 @@
 class DjangoURLPrinter(URLPrinterBase):
     def __init__(self, args):
         self.args = args
-        self.db_url = self.args.server_addr + DJANGO_SUB_URL
+        self.db_url = self.args.server_addr + "/" + DJANGO_SUB_URL
 
     def getColumnSelParams(self):
         col_sel_params = []
@@ -87,7 +87,12 @@ def getDjangoParams(self, user_identifier):
     def printURL(self, dataset, user_identifier, benchmarks):
         params = self.getDjangoParams(user_identifier)
 
-        param_string = urllib.urlencode(params)
+        try:
+            # pytyon 2
+            param_string = urllib.urlencode(params)
+        except Exception:
+            # python 3
+            param_string = urllib.parse.urlencode(params)
 
         url = (
             self.db_url + "?{}"

benchmarking/run_bench.py

@@ -117,6 +117,8 @@ def _saveDefaultArgs(self, new_args):
 
     def _askArgsFromUser(self, args, new_args):
         args.update(new_args)
+        if "--server_addr" in args and "--remote_reporter" not in args:
+            args["--remote_reporter"] = args["--server_addr"] + "/benchmark/store-result|oss"
         self._inputOneRequiredArg(
             "Please enter the directory the framework repo resides",
             "--repo_dir", args)
@@ -149,6 +151,7 @@ def _loadDefaultArgs(self):
             '--timeout': 300,
             '--logger_level': 'warning',
             '--server_addr': "http://127.0.0.1:8000",
+            '--result_db': "django",
         }
         return args
 

benchmarking/run_lab.py

@@ -22,6 +22,7 @@
 import multiprocessing
 import os
 import stat
+import sys
 import tempfile
 import threading
 import time
@@ -502,6 +503,7 @@ def _downloadFiles(self, jobs_queue):
                         control_info = job["benchmarks"]["info"]["control"]
                         self._downloadBinaries(control_info)
             except Exception:
+                getLogger().error("Unknown exception {}".format(sys.exc_info()[0]))
                 getLogger().error("File download failure")
         return benchmark_files
 

benchmarking/run_remote.py

@@ -165,7 +165,6 @@ def _buildProgram(self, tempdir):
                 allfiles = [program]
             else:
                 allfiles = [os.path.join(filedir, f) for f in os.listdir(filedir)]
-
             for fn in allfiles:
                 filename, _ = self.file_handler.uploadFile(fn, None, None, False)
                 getLogger().info("program: {}".format(filename))
@@ -183,7 +182,7 @@ def __init__(self, raw_args=None):
         if not self.args.benchmark_db_entry:
             assert self.args.server_addr is not None, \
                 "Either server_addr or benchmark_db_entry must be specified"
-            self.args.benchmark_db_entry = self.args.server_addr + "benchmark/"
+            self.args.benchmark_db_entry = self.args.server_addr + "/benchmark/"
         self.db = DBDriver(self.args.benchmark_db,
                            self.args.app_id,
                            self.args.token,
@@ -302,6 +301,7 @@ def run(self):
             self.db.submitBenchmarks(data, new_devices, user_identifier, user)
         if self.args.async_submit:
             return
+
         self.url_printer.printURL(self.scuba_dataset,
                                   user_identifier,
                                   benchmarks)