From f94ec5699d03b9cbeb1d56662d5e9cf44573a89f Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sun, 28 Jan 2018 11:33:43 +0800
Subject: [PATCH 01/19] Refactor expire_data_check_()

* Delete duplicated error checking
* Add docstring to describe what this function will do
---
 arrange_schedule.py | 37 +++++++++++--------------------------
 1 file changed, 11 insertions(+), 26 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 0b93060..43614c4 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -378,8 +378,11 @@ def find_activity(json_obj):
     return_msg["result"] = "success"
     return return_msg
 
-#The API connect mysql and clean expire data
 def expire_data_check_():
+    """
+    Check text and image data has expired, and if it is in schedule,
+    mark it expired, too.
+    """
     try:
         return_msg = {}
         return_msg["result"] = "fail"
@@ -391,14 +394,8 @@ def expire_data_check_():
         #update expire data
         for expired_image_id in pure_result:
             deal_result.append(expired_image_id[0])
-            try:
-                with ImageDao() as imageDao:
-                    imageDao.markExpired(expired_image_id[0])
-            except DB_Exception as e:
-                return_msg["error"] = gen_error_msg(e.args[1])
-                
-        if "error" in return_msg:
-            return return_msg
+            with ImageDao() as imageDao:
+                imageDao.markExpired(expired_image_id[0])
 
         #find expire text data
         with TextDao() as textDao:
@@ -407,24 +404,12 @@ def expire_data_check_():
         #update expire data
         for expired_text_id in pure_result:
             deal_result.append(expired_text_id[0])
-            try:
-                with TextDao() as textDao:
-                    textDao.markExpired(expired_text_id[0])
-            except DB_Exception as e:
-                return_msg["error"] = gen_error_msg(e.args[1])
-                
-        if "error" in return_msg:
-            return return_msg
-        
+            with TextDao() as textDao:
+                textDao.markExpired(expired_text_id[0])
+
         for target_id in deal_result:
-            try:
-                with ScheduleDao() as scheduleDao:
-                    scheduleDao.markExpiredSchedule(targetId=target_id)
-            except DB_Exception as e:
-                return_msg["error"] = gen_error_msg(e.args[1])
-                
-        if "error" in return_msg:
-            return return_msg
+            with ScheduleDao() as scheduleDao:
+                scheduleDao.markExpiredSchedule(targetId=target_id)
 
         return_msg["result"] = "success"
         return return_msg

From 8fd85be161a1b3885d90b99f1dfc4a736d503964 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sun, 28 Jan 2018 11:40:04 +0800
Subject: [PATCH 02/19] Add docstring for find_activity()

---
 arrange_schedule.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 43614c4..bdebab0 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -326,8 +326,11 @@ def mix_image_and_text(arrange_mode,deal_obj):
         deal_obj = new_list
     return deal_obj
 
-#The API connect mysql and find image data that can be scheduled
 def find_activity(json_obj):
+    """
+    According to the input arrange_mode setting, check the arrange mode consistence,
+    and find out the text and image can be add into schedule, return the candidates
+    """
     return_msg = {}
     return_msg["result"] = "fail"
     receive_obj = {}

From 41fab8bb8aef399cdfb1943d78d5ecc7d1fbae6e Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sun, 28 Jan 2018 12:31:46 +0800
Subject: [PATCH 03/19] Refactor set_system_log()

---
 arrange_schedule.py | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index bdebab0..68989fc 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -1255,29 +1255,30 @@ def CHLD_handler(para1, para2):
         send_obj["error"] = ("kill : ( " + str(para1) + ", " + str(para2)+" ) ")
         set_system_log(send_obj)
 
-#future can write to log.txt. now just print it
 def set_system_log(json_obj):
+    """
+    Print error message and write to log file
+    """
     return_msg = {}
     return_msg["result"] = "fail"
     file_name = "static/log/impossible_error.txt"
-    debug = 1
-    file_pointer = ""
 
-    if debug == 1:
-        try:
-            if json_obj["result"]=="fail":
-                print("#error : " + json_obj["error"])
-                if not os.path.isfile(file_name) :
-                    file_pointer = open(file_name, "w")
-                else :
-                    file_pointer = open(file_name, "a")
-                str_write = str(time.time()) + " fail : " + str(json_obj["error"]) + "\n"
-                file_pointer.write(str_write)
-                file_pointer.close()
-        except:
-            return_msg["result"] = "fail to print error"
+    if "result" not in json_obj:
+        return_msg["error"] = "set_system_log failed no `result` in input parameter"
+        return return_msg
+
+    if json_obj["result"] == "fail":
+        if "error" not in json_obj:
+            return_msg["result"] = "success"
             return return_msg
-    
+
+        print("#error : " + json_obj["error"])
+        mode = "a" if os.path.isfile(file_name) else "w"
+        with open(file_name,mode) as fp:
+            fp.write("{timestamp} fail: {err_msg}\n".format(
+                timestamp=time.time(),err_msg=json_obj["error"])
+            )
+
     return_msg["result"] = "success"
     return return_msg
         

From 8630ae64138243ba4f522cd25d0120b2abbc334a Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sun, 28 Jan 2018 23:05:53 +0800
Subject: [PATCH 04/19] Remove arrange_mode 6 and 7

---
 arrange_schedule.py  | 31 ++++---------------------------
 dataAccessObjects.py | 18 ------------------
 modeUtil.py          |  2 +-
 3 files changed, 5 insertions(+), 46 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 68989fc..86dd75c 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -204,7 +204,6 @@ def find_text_acticity(json_obj):
         return_msg = {}
         return_msg["result"] = "fail"
         deal_result = []
-        arrange_mode = 1
         arrange_condition = []
         try:
             arrange_mode = json_obj["arrange_mode"]
@@ -295,35 +294,13 @@ def find_image_acticity(json_obj):
         return return_msg
 
 def mix_image_and_text(arrange_mode,deal_obj):
-    if arrange_mode == 0 or arrange_mode == 3:
+    if arrange_mode in [0,3]:
         "DO NOTHING"
-    elif arrange_mode == 1 or arrange_mode == 4:
+    elif arrange_mode in [1,4]:
         deal_obj = sample(deal_obj, len(deal_obj))
-    elif arrange_mode == 2 or arrange_mode == 5:
+    elif arrange_mode in [2,5]:
         if len(deal_obj)>20:
             deal_obj = sample(deal_obj, 20)
-    elif arrange_mode == 6 or arrange_mode == 7:
-        img_start_num = 0
-        for num1 in range(len(deal_obj)-1):
-            if deal_obj[num1][2] < deal_obj[num1+1][2]:
-                img_start_num = num1 + 1
-        num1 = 0
-        num2 = img_start_num
-        new_list = []
-        for num3 in range(len(deal_obj)):
-            if num1 == img_start_num:
-                new_list.append(deal_obj[num2])
-                num2 += 1
-            elif num2 == len(deal_obj):
-                new_list.append(deal_obj[num1])
-                num1 += 1
-            elif deal_obj[num1][2] >= deal_obj[num2][2]:
-                new_list.append(deal_obj[num1])
-                num1 += 1
-            else :
-                new_list.append(deal_obj[num2])
-                num2 += 1
-        deal_obj = new_list
     return deal_obj
 
 def find_activity(json_obj):
@@ -346,7 +323,7 @@ def find_activity(json_obj):
         return_msg["error"] = "input parameter missing"
         return return_msg
 
-    if arrange_mode in [3,4,5,7] and len(arrange_condition) == 0:
+    if arrange_mode in [3,4,5] and len(arrange_condition) == 0:
         return_msg["error"] = 'Then arrange mode {mode} need to assgin condition'.format(mode=arrange_mode)
         return return_msg
     
diff --git a/dataAccessObjects.py b/dataAccessObjects.py
index 417a6b2..c6409e2 100644
--- a/dataAccessObjects.py
+++ b/dataAccessObjects.py
@@ -59,24 +59,6 @@ def findActivities(self,conditionAssigned,orderById,arrangeMode,arrangeCondition
                 sql += " and ({type_condition}) ".format(type_condition=type_condition)
             if orderById:
                 sql += " ORDER BY {dataName}_id ASC".format(dataName=self.dataName)
-        elif arrangeMode == 6:
-            sql = "SELECT a0.{dataName}_id, a0.{dataName}_display_time, a1.type_weight FROM " \
-                +" (SELECT {dataName}_id, type_id, {dataName}_display_time FROM {tableName} WHERE " \
-                +" {dataName}_is_delete=0 and {dataName}_is_expire=0 "\
-                +" and (TO_DAYS(NOW()) between TO_DAYS({dataName}_start_date) and TO_DAYS({dataName}_end_date)) " \
-                +" and (TIME_TO_SEC(DATE_FORMAT(NOW(), '%H:%i:%s')) between TIME_TO_SEC({dataName}_start_time) and TIME_TO_SEC({dataName}_end_time))) AS a0 "\
-                +" LEFT JOIN (SELECT type_id, type_weight FROM data_type ) AS a1 "\
-                +" ON a0.type_id=a1.type_id ORDER BY a1.type_weight ASC"
-        elif arrangeMode == 7:
-            sql = "SELECT a0.{dataName}_id, a0.{dataName}_display_time, a1.type_weight FROM " \
-                +" (SELECT {dataName}_id, type_id, {dataName}_display_time FROM {tableName} WHERE "\
-                +" ({type_condition})".format(type_condition=type_condition)\
-                +" and {dataName}_is_delete=0 and {dataName}_is_expire=0 "\
-                +" and (TO_DAYS(NOW()) between TO_DAYS({dataName}_start_date) and TO_DAYS({dataName}_end_date)) " \
-                +" and (TIME_TO_SEC(DATE_FORMAT(NOW(), '%H:%i:%s')) between TIME_TO_SEC({dataName}_start_time) and TIME_TO_SEC({dataName}_end_time))) AS a0 "\
-                +" LEFT JOIN (SELECT type_id, type_weight FROM data_type WHERE "\
-                +" ({type_condition})".format(type_condition=type_condition)\
-                +" ) AS a1 ON a0.type_id=a1.type_id ORDER BY a1.type_weight ASC"
 
         sql = sql.format(dataName=self.dataName,tableName=self.tableName)
         ret = self.db.query(sql)
diff --git a/modeUtil.py b/modeUtil.py
index 238f5e3..43fa566 100644
--- a/modeUtil.py
+++ b/modeUtil.py
@@ -10,7 +10,7 @@ def checkOrderById(arrangeMode):
 
     @staticmethod
     def checkConditionAssigned(arrangeMode):
-        if arrangeMode in [3,4,5,7]:
+        if arrangeMode in [3,4,5]:
             return True
         else:
             return False

From 8934c1fe0cf2e40f763bec00cc186fe755f2328a Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sun, 28 Jan 2018 23:13:17 +0800
Subject: [PATCH 05/19] Remove useless variable

---
 arrange_schedule.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 86dd75c..12b35fa 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -972,16 +972,12 @@ def crawler_google_drive_img(json_obj):
     try:
         return_msg = {}
         return_msg["result"] = "fail"
-        server_dir = ""
-        user_id = 1
         try:
             server_dir = json_obj["server_dir"]
             user_id = json_obj["user_id"]
         except:
             return_msg["error"] = "input parameter missing"
             return return_msg
-        data_type = 4
-        receive_obj = {}
 
         #find google_drive_image type id 
         receive_msg = find_drive_data_type()

From d48d8b69f862d0860b88d86e13c9b09512287829 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sun, 28 Jan 2018 23:21:28 +0800
Subject: [PATCH 06/19] Refactor google_calendar_text()

---
 arrange_schedule.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 12b35fa..cac4a09 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -710,6 +710,10 @@ def crawler_cwb_img(json_obj):
         return return_msg
 
 def google_calendar_text():
+    """
+    Get google api credential, and grab calendar's upcoming events,
+    then check existed or insert into DB
+    """
     try:
         return_msg = {}
         return_msg["result"] = "fail"
@@ -718,17 +722,13 @@ def google_calendar_text():
             return_msg["error"] = "No credential file"
             return return_msg
         else:
-            try:
-                events = get_upcoming_events(credentials)
-                for cal, events_value in events.items():
-                    for e in events_value:
-                        check_event_exist_or_insert(e, cal)
-                        sleep(1.5)
-                return_msg["result"] = "success"
-                return return_msg
-            except DB_Exception as e:
-                return_msg["error"] = e.arg[1]
-                return return_msg
+            events = get_upcoming_events(credentials)
+            for calendar, events_value in events.items():
+                for event in events_value:
+                    check_event_exist_or_insert(event, calendar)
+                    sleep(1.5)
+            return_msg["result"] = "success"
+            return return_msg
     except Exception as e:
         return_msg["error"] = str(e)
         return return_msg

From ca35a67296065ce34b1c51e44ae4d5c56ebe92f5 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 16:22:17 +0800
Subject: [PATCH 07/19] Add parameter typeName for getTypeDir()

---
 dataAccessObjects.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/dataAccessObjects.py b/dataAccessObjects.py
index c6409e2..4e1b14f 100644
--- a/dataAccessObjects.py
+++ b/dataAccessObjects.py
@@ -440,8 +440,13 @@ def insertUserPrefer(self,prefId,userId,prefStr):
         self.db.cmd(sql)
 
 class DataTypeDao(DefaultDao):
-    def getTypeDir(self,typeId):
-        sql = 'SELECT type_dir FROM data_type WHERE type_id={typeId}'.format(typeId=typeId)
+    def getTypeDir(self,typeId=None,typeName=None):
+        sql = 'SELECT type_dir FROM data_type WHERE {condition}'
+        if typeId:
+            condition = 'type_id={typeId}'.format(typeId=typeId)
+        elif typeName:
+            condition = 'type_name={typeName}'.format(typeName=typeName)
+        sql = sql.format(condition=condition)
         return self.queryOneValue(sql)
 
     def getTypeName(self,typeId):

From f657fcc7eaf3db8bfadab55947088bd0d6df5ac8 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 16:54:46 +0800
Subject: [PATCH 08/19] Add getDataType() to DataTypeDao

---
 dataAccessObjects.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/dataAccessObjects.py b/dataAccessObjects.py
index 4e1b14f..b442d67 100644
--- a/dataAccessObjects.py
+++ b/dataAccessObjects.py
@@ -493,6 +493,17 @@ def getDisplayDataType(self,info):
             #TODO check need to raise exception or not
             return None
 
+    def getDataType(self, typeId=None, typeName=None):
+        sql = 'SELECT type_id, type_name, type_dir FROM data_type WHERE {condition}'
+        if typeId:
+            condition = 'type_id = {typeId}'.format(typeId=typeId)
+        elif typeName:
+            condition = 'type_name = "{typeName}"'.format(typeName=typeName)
+        sql = sql.format(condition=condition)
+        ret = self.db.query(sql)
+        dataType = dict(typeId=ret[0][0],typeName=ret[0][1],typeDir=ret[0][2])
+        return dataType
+
 class ArrangeModeDao(DefaultDao):
     def getArrangeMode(self):
         sql = 'SELECT armd_sn, armd_mode, armd_condition FROM arrange_mode WHERE armd_is_delete=0 and armd_is_expire=0 and '\

From 08cbc75b4cdd4c489f7844d7920b2bbdc178f710 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 16:55:47 +0800
Subject: [PATCH 09/19] Use getDataType() to replace find_cwb_type()

Use more general function getDataType() to replace find_cwb_type()
---
 arrange_schedule.py              | 20 +++++++-------------
 test/test_02_arrange_schedule.py |  3 ---
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index cac4a09..8dc389f 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -610,15 +610,6 @@ def read_arrange_mode():
         return_msg["error"] = gen_error_msg(e.args[1])
         return return_msg
 
-def find_cwb_type():
-    return_msg = {}
-    with DataTypeDao() as dataTypeDao:
-        return_msg["typeId"] = dataTypeDao.getTypeId('氣像雲圖')
-        if return_msg["typeId"] == None:
-            return None
-        return_msg["typeDir"] = dataTypeDao.getTypeDir(return_msg["typeId"])
-    return return_msg
-
 def delete_old_cwb_img(server_dir,user_id):
     send_obj = {}
     error_list_id = []
@@ -657,8 +648,11 @@ def crawler_cwb_img(json_obj):
         send_obj = {}
         receive_obj = {}
 
-        data_type = find_cwb_type()
-        if data_type is None:
+        with DataTypeDao() as dataTypeDao:
+            weather_data_type = dataTypeDao.getDataType(typeName='氣像雲圖')
+
+        if weather_data_type is None:
+            #TODO create cwb data_type
             return_msg["error"] = "no cwb img data type"
             return return_msg
 
@@ -666,7 +660,7 @@ def crawler_cwb_img(json_obj):
             target_img = 'CV1_TW_3600_{timeStamp}.png'.format(timeStamp=time.strftime("%Y%m%d%H%M", time.localtime(now_time)))
             url = 'http://www.cwb.gov.tw/V7/observe/radar/Data/HD_Radar/' + target_img
             try:
-                target_img = os.path.join('static',data_type["typeDir"],target_img)
+                target_img = os.path.join('static',weather_data_type["typeDir"],target_img)
                 request.urlretrieve(url, target_img)
             except:
                 now_time -= 60
@@ -678,7 +672,7 @@ def crawler_cwb_img(json_obj):
 
             #upload new file
             send_obj["server_dir"] = server_dir
-            send_obj["file_type"] = data_type["typeId"]
+            send_obj["file_type"] = weather_data_type["typeId"]
             send_obj["filepath"] = target_img
             send_obj["start_date"] = time.strftime("%Y-%m-%d", time.localtime(time.time()))
             send_obj["end_date"] = time.strftime("%Y-%m-%d", time.localtime(time.time()+86400))
diff --git a/test/test_02_arrange_schedule.py b/test/test_02_arrange_schedule.py
index 2269785..18538de 100644
--- a/test/test_02_arrange_schedule.py
+++ b/test/test_02_arrange_schedule.py
@@ -72,9 +72,6 @@ def test_expire_data_check(self):
         receive_msg = expire_data_check_()
         self.assertEqual(receive_msg['result'],'success')
 
-    def test_find_cwb_type_id(self):
-        self.assertNotEqual(find_cwb_type(),-1)
-
     def test_mark_now_activity(self):
         receive_msg = mark_now_activity()
         self.assertEqual(receive_msg['result'], 'success')

From 67dc8ed0ac96ada51e4bb1b32871087b205c9233 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 16:57:22 +0800
Subject: [PATCH 10/19] Remove duplicate unittest test_crawler_schedule()

The test in test_crawler_schedule() is almost the same as
test_crawler_news() and test_crawler_ptt_news(), so remove it.
---
 test/test_02_arrange_schedule.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/test/test_02_arrange_schedule.py b/test/test_02_arrange_schedule.py
index 18538de..47af169 100644
--- a/test/test_02_arrange_schedule.py
+++ b/test/test_02_arrange_schedule.py
@@ -49,10 +49,6 @@ def test_crawler_ptt_news(self):
             receive_msg = crawler_ptt_news(boards)
             self.assertEqual(receive_msg['result'],'success')
 
-    def test_crawler_schedule(self):
-        receive_msg = crawler_schedule()
-        self.assertEqual(receive_msg['result'],'success')
-
     def test_check_fortune_table(self):
         receive_msg = check_table(fortune=True)
         self.assertEqual(receive_msg['result'],'success')

From 57c6696c0ba3956f806c122888d6cb36bba94bb1 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 17:12:00 +0800
Subject: [PATCH 11/19] Use getTypeId() to replace find_drive_data_type()

---
 arrange_schedule.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 8dc389f..487f423 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -852,19 +852,6 @@ def add_event_by_qrcode(eventInfo):
         .format(summary=summary,startDate=startDate,startTime=startTime,endDate=endDate,endTime=endTime,location=location,detail=description)
     qrcode.make_qrcode_image(link,target_dir,event=eventInfo['id'])
 
-def find_drive_data_type():
-    return_msg = {}
-    with DataTypeDao() as dataTypeDao:
-        typeId = dataTypeDao.getTypeId('google_drive_image')
-    if typeId != None:
-        return_msg['data_type'] = int(typeId)
-        return_msg['result'] = 'success'
-        return return_msg
-    else:
-        return_msg['error'] = "no google_drive_image data type"
-        return_msg['result'] = 'fail'
-        return return_msg
-
 def search_google_drive_folder(service):
     g_sql = "(name='1day' or name='3day' or name='7day' or name='14day')"
     results = service.files().list(
@@ -973,12 +960,14 @@ def crawler_google_drive_img(json_obj):
             return_msg["error"] = "input parameter missing"
             return return_msg
 
-        #find google_drive_image type id 
-        receive_msg = find_drive_data_type()
-        if receive_msg['result']=='fail':
-            return receive_msg
+        with DataTypeDao() as dataTypeDao:
+            googel_drive_type_id = dataTypeDao.getTypeId(typeName='google_drive_image')
+
+        if googel_drive_type_id:
+            json_obj['data_type'] = googel_drive_type_id
         else:
-            json_obj['data_type'] = receive_msg['data_type']
+            return_msg["error"] = "No such data_type: {}".format('google_drive_image')
+            return return_msg
         
         #get google credentials
         credentials = get_credentials()

From 7b907d138ced0489cec436eb7089cf951673b785 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 17:31:21 +0800
Subject: [PATCH 12/19] Refactor search_google_drive_file()

---
 arrange_schedule.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 487f423..a156c81 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -861,14 +861,16 @@ def search_google_drive_folder(service):
     return items
 
 def search_google_drive_file(service):
-    #set time
+    """
+    Search images which is modified or uploaded in last 12 hours on google drive
+    """
     now_time = time.time()
     start_time = time.strftime("%Y-%m-%dT%H:%M:%S+08:00", time.localtime(now_time-43200))
     results = service.files().list(
-        q="modifiedTime > '" + start_time + "' and mimeType contains 'image/'", 
+        q="modifiedTime > '{start_time}' and mimeType contains 'image/'".format(start_time=start_time),
         fields="nextPageToken, files(id, name, parents)").execute()
-    items = results.get('files', [])
-    return items
+    files = results.get('files', [])
+    return files
     
 def merge_files_and_days(days_limit, drive_file):
     for num1 in range(len(drive_file)):

From bb1bdc2c5c7b3782d61890da1f127f17939d6b72 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 17:39:59 +0800
Subject: [PATCH 13/19] Use dict type get() method

---
 arrange_schedule.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index a156c81..e76ca07 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -797,14 +797,9 @@ def check_event_exist_or_insert(event, calendar_name=None):
         receive_msg = upload_text_insert_db(send_msg)
         addition_msg = rule_base_agent(event)
         event_file_path = os.path.join('static','calendar_event','{name}.png'.format(name=event_id))
-        if 'description' in event:
-            description = event['description']
-        else:
-            description = addition_msg['description']
-        if 'location' in event:
-            location = event['location']
-        else:
-            location = ""
+        description = event.get('description',addition_msg['description'])
+        location = event.get('location','')
+
         if 'dateTime' in event['start'].keys() and event_start_date == event_end_date:
             detailtime = "{start} - {end}".format(start=event_start_time, end=event_end_time)
         else:

From 00526252e0e50888adc12370d216fd722e0691e7 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 17:54:49 +0800
Subject: [PATCH 14/19] Remove outdated file

---
 DB_consist.py | 60 ---------------------------------------------------
 1 file changed, 60 deletions(-)
 delete mode 100644 DB_consist.py

diff --git a/DB_consist.py b/DB_consist.py
deleted file mode 100644
index d88178d..0000000
--- a/DB_consist.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from mysql import mysql
-from env_init import create_data_type
-import os
-
-def create_user_data_file():
-    print("check user_data dir and file...")
-    try:
-        if not os.path.exists("setting"):
-            print('create dir setting"')
-            os.makedirs('setting')
-        if not os.path.isfile("setting/server_setting.txt"):
-            print('create file "setting/server_setting.txt"')
-            with open("setting/server_setting.txt", "w") as fp:
-                fp.write("bluetooth_enable 1")
-        print("check finish")
-    except Exception as e:
-        print("create user_data file failed",e)
-
-def check_column_exist_or_add(db,table,column_name,data_type):
-    if len(db.query('show columns from %s like "%s"' % (table,column_name))) == 0:
-        print('%s doesn\'t in table %s' %(column_name,table))
-        print('add %s %s into table %s' %(column_name,data_type,table))
-        db.cmd('alter table %s add column %s %s' % (table,column_name,data_type))
-
-def check_table_exist_or_create(db,table_name,sql):
-    if len(db.query('show tables like "%s"' % (table_name))) == 0:
-        print('Table %s doesn\'t exist' % table_name)
-        print('Create table %s' % table_name)
-        db.cmd(sql)
-
-def check_data_type_exist_or_create(db,data_type):
-    if len(db.query('select * from data_type where type_name = "%s"' % data_type)) == 0:
-        print("%s data_type doesn't exist" % data_type)
-        create_data_type(data_type)
-
-def check_bluetooth_DB(db):
-    create_user_data_file()
-    check_table_exist_or_create(db,'user_prefer','create table user_prefer ( \
-                                pref_id varchar(14) unique key, \
-                                user_id int default 0, \
-                                pref_data_type_01 varchar(100), \
-                                pref_data_type_02 varchar(100), \
-                                pref_data_type_03 varchar(100), \
-                                pref_data_type_04 varchar(100), \
-                                pref_data_type_05 varchar(100), \
-                                pref_set_time datetime default now(), \
-                                pref_is_delete bit(1) default 0)')
-    check_column_exist_or_add(db,'user','user_bluetooth_id','varchar(50)')
-    check_column_exist_or_add(db,'user','user_profession','int default 0 not null')
-    check_column_exist_or_add(db,'image_data','img_like_count','int default 0')
-    check_column_exist_or_add(db,'user','user_birthday','datetime')
-    check_column_exist_or_add(db,'text_data','text_like_count','int default 0')
-    check_data_type_exist_or_create(db,"customized_text")
-
-def main():
-    db = mysql()
-    db.connect()
-    check_bluetooth_DB(db)
-
-main()

From 87da6a1fd041a0c16d76d53610dfa62810857c23 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 18:06:09 +0800
Subject: [PATCH 15/19] Unify worker job function format

---
 arrange_schedule.py              | 8 ++++----
 test/test_02_arrange_schedule.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index e76ca07..2d80a7b 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -358,7 +358,7 @@ def find_activity(json_obj):
     return_msg["result"] = "success"
     return return_msg
 
-def expire_data_check_():
+def expire_data_check():
     """
     Check text and image data has expired, and if it is in schedule,
     mark it expired, too.
@@ -1235,8 +1235,8 @@ def set_system_log(json_obj):
     return_msg["result"] = "success"
     return return_msg
         
-def expire_data_check():
-    receive_obj = expire_data_check_()
+def do_expire_data_check():
+    receive_obj = expire_data_check()
     if receive_obj["result"] == "success":
         "DO NOTHING"
     else :
@@ -1356,7 +1356,7 @@ def main():
     alarm_crawler_functions = raw_time + 15.0
     alarm_auto_text_generator = raw_time + 97.0
 
-    check_expire_data_worker = Worker(job=expire_data_check,name='Check expired data')
+    check_expire_data_worker = Worker(job=do_expire_data_check,name='Check expired data')
     set_schedule_log_worker = Worker(job=do_set_schedule_log,name='Set schedule log')
     cwb_crawler_worker = Worker(job=do_cwb_crawler,name='Crawler for cwb image')
     google_calendar_worker = Worker(job=do_google_calendar,name='Grab Google calendar event')
diff --git a/test/test_02_arrange_schedule.py b/test/test_02_arrange_schedule.py
index 47af169..b252be8 100644
--- a/test/test_02_arrange_schedule.py
+++ b/test/test_02_arrange_schedule.py
@@ -65,7 +65,7 @@ def test_set_schedule_log(self):
         self.assertEqual(receive_msg['result'],'success')
 
     def test_expire_data_check(self):
-        receive_msg = expire_data_check_()
+        receive_msg = expire_data_check()
         self.assertEqual(receive_msg['result'],'success')
 
     def test_mark_now_activity(self):

From 26e9c7f32ee5135eb1b75df5a65ac330eb1480bc Mon Sep 17 00:00:00 2001
From: Billy SU <g4691821@gmail.com>
Date: Mon, 29 Jan 2018 20:49:56 +0800
Subject: [PATCH 16/19] Update README.md

---
 README.md | 67 ++++++++++++++++++++-----------------------------------
 1 file changed, 24 insertions(+), 43 deletions(-)

diff --git a/README.md b/README.md
index 48a86a8..1a7be07 100644
--- a/README.md
+++ b/README.md
@@ -9,41 +9,18 @@ The functions Electronic-Blackboard included are as follows
 * Announcement template
 * etc
 
-## Setup
+## Quick start
+### Prerequisities
+* docker
 
-### Prerequisites
-
-Since the project is totally written in python 3, pyhton 2 is not recommended.
-* Python3
-* MySQL
-
-### Installing
-
-1. For the libraries needed
-```
-# pip3 install -r requirements.txt
-```
-2. create nessasary files in this project
-  * create `mysql_auth.txt`
-3. The content inside `mysql_auth.txt` should be  
-  * Host
-  * User
-  * Password
-  * Database  -create the database first where to store your data
-```
-localhost
-root
-your_password
-yout_database_name
+### Starting up docker
 ```
-4. Execute `env_init.py` to automatically initialize environment settings
+./docker/run.sh
+[sudo] password for <username>: 
+root@XXXXXXXXXX:/home/EB# 
 ```
-$ python3 env_init.py
-```
-
-## Run test
-`$ pytest test/`
-
+### Testing
+`root@XXXXXXXXXX:/home/EB# ./docker/test.sh`
 if success, it should print
 ```
 ============================= test session starts ==============================
@@ -52,20 +29,24 @@ rootdir: /home/travis/build/SWLBot/electronic-blackboard, inifile:
 plugins: ordering-0.5
  ...
 
-=================== XX passed, XX warnings in 186.97 seconds ====================
+=================== XX passed, XX warnings in XXXX seconds ====================
 ```
-
-## Start
+### Running up 
 ```
-$ python3 server.py
-$ python3 arrange_schedule.py
-$ python3 board.py
+root@XXXXXXXXXX:/home/EB# python3 env_init.py
+...
+root@XXXXXXXXXX:/home/EB# ./autorun.sh
+root@XXXXXXXXXX:/home/EB# tmux ls
+blackboard: 4 windows (created XXXXXX ) [135x34]
 ```
-
-Open the web browser to check the implement result with the url `localhost:3000` and `localhost:4000`
-* port:3000 -the platform of uploading files
-* port:4000 -the broadcast display
-
+Now you can use your browser connect to following addresses:
+#### backend dashboard
+*Default* 
+Account: admin
+Password: admin
+http://localhost:3000/
+#### frontend display
+http://localhost:4000/
 ## Functional test
 ### Firefox
 1. Install geckodriver

From 107a9805e4d359082ab4390cd837e2f8d7d01e3b Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sat, 3 Feb 2018 11:31:11 +0800
Subject: [PATCH 17/19] Refactor broadcast_api.py

---
 broadcast_api.py | 39 +++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/broadcast_api.py b/broadcast_api.py
index fa98aa5..6b98696 100644
--- a/broadcast_api.py
+++ b/broadcast_api.py
@@ -8,7 +8,13 @@
 import json
 
 def getDisplayContent(sche_target_id,return_msg):
-    #get type from target id prefix
+    """
+    Get the display content of the display target.
+
+    Args:
+        sche_target_id: The id of display target
+        return_msg: Output paramter
+    """
     targetIdPrefix = sche_target_id[:4]
     if targetIdPrefix == "imge":
         with ImageDao() as imageDao:
@@ -19,41 +25,40 @@ def getDisplayContent(sche_target_id,return_msg):
             file_info = textDao.getIdSysName(Id=sche_target_id)
         return_msg["file_type"] = "text"
     else :
-        return_msg["error"] = "target id type error {}".format(targetIdPrefix)
-        return return_msg
+        raise Exception("target id type error {}".format(targetIdPrefix))
 
     try:
         type_id = file_info['typeId']
         system_file_name = file_info['systemName']
         return_msg["like_count"] = file_info['likeCount']
     except:
-        return_msg["error"] = "no file record"
-        return return_msg
+        raise Exception("no file record")
 
     with DataTypeDao() as dataTypeDao:
         type_dir = dataTypeDao.getTypeDir(typeId=type_id)
         type_name = dataTypeDao.getTypeName(typeId=type_id)
     if type_dir == None or type_name == None:
-        return_msg["error"] = "No such type id {}".format(type_id)
-        return return_msg
+        raise Exception("No such type id {}".format(type_id))
 
     targetFile = os.path.join("static", type_dir, system_file_name)
     return_msg["file"] = os.path.join(type_dir, system_file_name)
     return_msg["type_name"] = type_name
 
-    #if text read file
     if return_msg["file_type"] == "text":
         if not os.path.isfile(targetFile) :
-            return_msg["error"] = "no file"
-            return return_msg
+            raise Exception("Text file doesn't exists")
         else :
             with open(targetFile,"r") as fp:
                 file_content = json.load(fp)
             return_msg["file_text"] = file_content
-    return return_msg
 
-#The API load schedule.txt and find out the first image which has not print and the time limit still allow
 def load_schedule():
+    """
+    Get the next display target
+
+    Returns:
+        return_msg: Display target filled with its attributes
+    """
     try:
         return_msg = {}
         return_msg["result"] = "fail"
@@ -65,12 +70,11 @@ def load_schedule():
             return_msg["error"] = "no schedule"
             return return_msg
         return_msg["schedule_id"] = next_schedule['schedule_id']
-        sche_target_id = next_schedule['sche_target_id']
         return_msg["display_time"] = int(next_schedule['display_time'])
 
-        return_msg = getDisplayContent(sche_target_id,return_msg)
-        if "error" in return_msg:
-            return return_msg
+        sche_target_id = next_schedule['sche_target_id']
+
+        getDisplayContent(sche_target_id,return_msg)
 
         #update display count
         if return_msg["file_type"] == "image":
@@ -85,4 +89,7 @@ def load_schedule():
     except DB_Exception as e:
         return_msg["error"] = e.args[1]
         return return_msg
+    except Exception as e:
+        return_msg["error"] = str(e)
+        return return_msg
 

From 53a5e69213cec1d5bfe3a5c2b21ecb3286f83274 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sat, 3 Feb 2018 11:39:57 +0800
Subject: [PATCH 18/19] Remove duplicate function

---
 arrange_schedule.py  | 10 ++++------
 dataAccessObjects.py |  6 ------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 2d80a7b..3696f46 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -145,22 +145,20 @@ def load_next_schedule(json_obj):
                     if expired:
                         raise loadScheduleError("Schedule target expired {}".format(sche_target_id))
 
-                #find type dir
-                check_target_dir = ""
                 with DataTypeDao() as dataTypeDao:
                     type_dir = dataTypeDao.getTypeDir(type_id)
 
                 if type_dir:
-                    check_target_dir = os.path.join(target_dir,'static',
+                    target_filepath = os.path.join(target_dir,'static',
                                             type_dir,system_file_name)
                 else:
                     raise loadScheduleError("No such type dir for type id {}".format(type_id))
 
                 #if text read file
-                if not os.path.isfile(check_target_dir) :
-                    raise loadScheduleError("File {} doesn't exist".format(check_target_dir))
+                if not os.path.isfile(target_filepath) :
+                    raise loadScheduleError("File {} doesn't exist".format(target_filepath))
                 else :
-                    return_msg["file"] = check_target_dir
+                    return_msg["file"] = target_filepath
                     break
 
             except loadScheduleError as e:
diff --git a/dataAccessObjects.py b/dataAccessObjects.py
index b442d67..03fd174 100644
--- a/dataAccessObjects.py
+++ b/dataAccessObjects.py
@@ -284,9 +284,6 @@ class ImageDao(DataManipulateDao):
     prefix = 'imge'
     def markExpired(self,imgId):
         super().markExpired(targetId=imgId)
-        
-    def markDeleted(self,imgId,userId):
-        super().markDeleted(targetId=imgId,userId=userId)
 
     def checkExisted(self,typeId,fileName):
         sql = 'SELECT COUNT(*) FROM image_data WHERE img_is_expire=0 and img_is_delete=0 '\
@@ -384,9 +381,6 @@ def checkExisted(self,eventId):
     def markExpired(self,textId):
         super().markExpired(targetId=textId)
 
-    def markDeleted(self,textId,userId):
-        super().markDeleted(targetId=textId,userId=userId)
-
     def addLikeCount(self,targetId):
         sql = 'UPDATE text_data SET text_like_count=text_like_count+1 WHERE text_id="{targetId}"'.format(targetId=str(targetId))
         self.db.cmd(sql)

From 461adc2698c85356cd747aca5e886b5aea9bb084 Mon Sep 17 00:00:00 2001
From: Billy Su <g4691821@gmail.com>
Date: Sat, 3 Feb 2018 12:36:44 +0800
Subject: [PATCH 19/19] Refactor main() in arrange_schedule.py

Delete the useless part of checking next schdeule's attribute when
checking the number of remain undisplay schedules is less than
min_db_activity.
---
 arrange_schedule.py | 144 +++++---------------------------------------
 1 file changed, 14 insertions(+), 130 deletions(-)

diff --git a/arrange_schedule.py b/arrange_schedule.py
index 3696f46..dd29a9f 100644
--- a/arrange_schedule.py
+++ b/arrange_schedule.py
@@ -83,105 +83,6 @@ def find_next_schedule():
         return_msg["error"] = gen_error_msg(e.args[1])
         return return_msg
 
-def load_next_schedule(json_obj):
-    """
-    This function will call find_next_schedule() to decide next display
-    item, and check it has expired or not, and count the undisplayed
-    schedule
-    """
-    try:
-        return_msg = {}
-        return_msg["result"] = "fail"
-        try:
-            target_dir = json_obj["board_py_dir"]
-        except:
-            return_msg["error"] = "input parameter missing"
-            return return_msg
-        
-        while True:
-            #find schedule
-            receive_msg = find_next_schedule()
-            if receive_msg["result"]=="fail":
-                return_msg["error"] = receive_msg["error"]
-                return return_msg
-            
-            sche_target_id = receive_msg["sche_target_id"]
-            no_need_check_time = receive_msg["no_need_check_time"]
-            target_sn = receive_msg["target_sn"]
-            #fill return msg
-            return_msg["schedule_id"] = receive_msg["schedule_id"]
-            return_msg["display_time"] = receive_msg["display_time"]
-    
-            #get type id and filename accord to type
-            try:
-                if sche_target_id[0:4]=="imge":
-                    with ImageDao() as imageDao:
-                        file_info= imageDao.getFileInfo(sche_target_id)
-                    return_msg["file_type"] = "image"
-                elif sche_target_id[0:4]=="text":
-                    with TextDao() as textDao:
-                        file_info = textDao.getFileInfo(sche_target_id)
-                    return_msg["file_type"] = "text"
-                else :
-                    raise loadScheduleError("No such type data {}".format(sche_target_id[0:4]))
-
-                if file_info:
-                    type_id = file_info['typeId']
-                    system_file_name = file_info['systemFileName']
-                else:
-                    raise loadScheduleError("No file info of {}".format(sche_target_id))
-
-                # check display target expired
-                if no_need_check_time == b'\x00':
-                    if return_msg["file_type"]=="image":
-                        with ImageDao() as imageDao:
-                            expired = imageDao.checkExpired(sche_target_id)
-                    elif return_msg["file_type"]=="text":
-                        with TextDao() as textDao:
-                            expired = textDao.checkExpired(sche_target_id)
-                    else:
-                        "impossible"
-
-                    if expired:
-                        raise loadScheduleError("Schedule target expired {}".format(sche_target_id))
-
-                with DataTypeDao() as dataTypeDao:
-                    type_dir = dataTypeDao.getTypeDir(type_id)
-
-                if type_dir:
-                    target_filepath = os.path.join(target_dir,'static',
-                                            type_dir,system_file_name)
-                else:
-                    raise loadScheduleError("No such type dir for type id {}".format(type_id))
-
-                #if text read file
-                if not os.path.isfile(target_filepath) :
-                    raise loadScheduleError("File {} doesn't exist".format(target_filepath))
-                else :
-                    return_msg["file"] = target_filepath
-                    break
-
-            except loadScheduleError as e:
-                if target_sn != 0:
-                    with ScheduleDao() as scheduleDao:
-                        scheduleDao.markExpiredSchedule(target_sn)
-                    target_sn = 0
-                print(e)
-                continue
-        
-        #check less activity number
-        with ScheduleDao() as scheduleDao:
-            return_msg['last_activity'] = scheduleDao.countUndisplaySchedule()
-        if not return_msg['last_activity']:
-            return_msg["error"] = "sql error"
-            return return_msg
-
-        return_msg["result"] = "success"
-        return return_msg
-    except DB_Exception as e:
-        return_msg["error"] = gen_error_msg(e.args[1])
-        return return_msg
-
 def find_text_acticity(json_obj):
     """
     According current `arrange_mode` and `condition`, to find out
@@ -1311,11 +1212,9 @@ def fork_time_management(raw_time,now_time,alarm,worker,fail_time,success_time):
 min_db_activity = 10
 
 def main():
-    just_startup = 1
     arrange_mode_change = 0
     arrange_sn = 0
     arrange_mode = 0
-    check_file_dir = "NO_FILE"
     condition = []
     send_obj = {}
     receive_obj = {}
@@ -1346,7 +1245,7 @@ def main():
     alarm_read_system_setting = raw_time + 300.0
     alarm_expire_data_check = raw_time + 3.0
     alarm_set_schedule_log = raw_time + 10.0
-    alarm_load_next_schedule = raw_time
+    alarm_check_remain_schedule = raw_time
     alarm_add_schedule = 1960380833.0
     alarm_crawler_cwb_img = raw_time + 7.0
     alarm_google_calendar_text = raw_time + 5.0
@@ -1388,34 +1287,19 @@ def main():
         alarm_set_schedule_log = fork_time_management(raw_time,now_time,
             alarm_set_schedule_log,set_schedule_log_worker,3.0,1800.0)
 
-        #load next schedule
-        if not os.path.isfile(check_file_dir) or raw_time >= alarm_load_next_schedule:
-            print("#4 "+ time.strftime('%Y-%m-%dT%H:%M:%SZ',now_time))
-            #mark now activity
-            if just_startup == 0:
-                receive_obj = mark_now_activity()
-                if receive_obj["result"] == "success":
-                    "DO NOTHING"
-                else :
-                    receive_obj["error"] = "mark_now_activity : " + receive_obj["error"]
-                    set_system_log(receive_obj)
-            
-            #load next schedule
-            send_obj["board_py_dir"] = board_py_dir
-            receive_obj = load_next_schedule(send_obj)
-            if receive_obj["result"] == "success":
-                just_startup = 0
-                alarm_load_next_schedule = raw_time + int(receive_obj["display_time"])
-                check_file_dir = receive_obj["file"]
-                if int(receive_obj["last_activity"]) < min_db_activity:
-                    alarm_add_schedule = raw_time
-            else :
-                if receive_obj["error"] == "no schedule":
-                    alarm_add_schedule = raw_time
-                    alarm_load_next_schedule = raw_time + 1.0
-                    just_startup = 1
-                receive_obj["error"] = "load_next_schedule : " + receive_obj["error"]
-                set_system_log(receive_obj)
+        if raw_time >= alarm_check_remain_schedule:
+            print("[{timestamp}] Check remain schedule count".format(timestamp=time.strftime('%Y-%m-%dT%H:%M:%SZ',now_time)))
+
+            ret = mark_now_activity()
+            if "error" in ret:
+                set_system_log(ret)
+
+            with ScheduleDao() as scheduleDao:
+                remain_schedules = scheduleDao.countUndisplaySchedule()
+            if remain_schedules < min_db_activity:
+                alarm_add_schedule = raw_time
+
+            alarm_check_remain_schedule += 3
 
         #add_schedule
         if raw_time >= alarm_add_schedule: