RTC and Adafruit IO related improvements

enviro/__init__.py

@@ -172,6 +172,8 @@ def connect_to_wifi():
 
   logging.info("  - ip address: ", ip)
   """
+  import rp2
+  rp2.country("GB") 
   wlan = network.WLAN(network.STA_IF)
   wlan.active(True)
   wlan.connect(wifi_ssid, wifi_password)
@@ -219,9 +221,38 @@ def low_disk_space():
     return (os.statvfs(".")[3] / os.statvfs(".")[2]) < 0.1   
   return False
 
-# returns True if the rtc clock has been set
+# returns True if the rtc clock has been set recently 
 def is_clock_set():
-  return rtc.datetime()[0] > 2020 # year greater than 2020? we're golden!
+  # is the year on or before 2020?
+  if rtc.datetime()[0] <= 2020:
+    return False
+
+  if helpers.file_exists("sync_time.txt"):
+    now_str = helpers.datetime_string()
+    now = helpers.timestamp(now_str)
+
+    time_entries = []
+    with open("sync_time.txt", "r") as timefile:
+      time_entries = timefile.read().split("\n")
+
+    # read the first line from the time file
+    sync = now
+    for entry in time_entries:
+      if entry:
+        sync = helpers.timestamp(entry)
+        break
+
+    seconds_since_sync = now - sync
+    if seconds_since_sync >= 0:  # there's the rare chance of having a newer sync time than what the RTC reports
+      try:
+        if seconds_since_sync < (config.resync_frequency * 60 * 60):
+          return True
+
+        logging.info(f"  - rtc has not been synched for {config.resync_frequency} hour(s)")
+      except AttributeError:
+        return True
+
+  return False
 
 # connect to wifi and attempt to fetch the current time from an ntp server
 def sync_clock_from_ntp():
@@ -233,8 +264,27 @@ def sync_clock_from_ntp():
   if not timestamp:
     logging.error("  - failed to fetch time from ntp server")
     return False  
+
+  # fixes an issue where sometimes the RTC would not pick up the new time
+  i2c.writeto_mem(0x51, 0x00, b'\x10') # reset the rtc so we can change the time
   rtc.datetime(timestamp) # set the time on the rtc chip
+  i2c.writeto_mem(0x51, 0x00, b'\x00') # ensure rtc is running
+  rtc.enable_timer_interrupt(False)
+
+  # read back the RTC time to confirm it was updated successfully
+  dt = rtc.datetime()
+  if dt != timestamp[0:7]:
+    logging.error("  - failed to update rtc")
+    if helpers.file_exists("sync_time.txt"):
+      os.remove("sync_time.txt")
+    return False
+
   logging.info("  - rtc synched")
+
+  # write out the sync time log
+  with open("sync_time.txt", "w") as syncfile:
+    syncfile.write("{0:04d}-{1:02d}-{2:02d}T{3:02d}:{4:02d}:{5:02d}Z".format(*timestamp))  
+
   return True
 
 # set the state of the warning led (off, on, blinking)
@@ -302,7 +352,7 @@ def get_sensor_readings():
 
 
   readings = get_board().get_sensor_readings(seconds_since_last)
-  readings["voltage"] = 0.0 # battery_voltage #Temporarily removed until issue is fixed
+  # readings["voltage"] = 0.0 # battery_voltage #Temporarily removed until issue is fixed
 
   # write out the last time log
   with open("last_time.txt", "w") as timefile:
@@ -346,7 +396,10 @@ def cache_upload(readings):
 
 # return the number of cached results waiting to be uploaded
 def cached_upload_count():
-  return len(os.listdir("uploads"))
+  try:
+    return len(os.listdir("uploads"))
+  except OSError:
+    return 0
 
 # returns True if we have more cached uploads than our config allows
 def is_upload_needed():
@@ -376,8 +429,23 @@ def upload_readings():
             with open("reattempt_upload.txt", "w") as attemptfile:
               attemptfile.write("")
 
-            logging.info(f"  - rate limited, going to sleep for 1 minute")
+            logging.info(f"  - cannot upload '{cache_file[0]}' - rate limited")
             sleep(1)
+          elif status == UPLOAD_LOST_SYNC:
+            # remove the sync time file to trigger a resync on next boot
+            if helpers.file_exists("sync_time.txt"):
+              os.remove("sync_time.txt")
+
+            # write out that we want to attempt a reupload
+            with open("reattempt_upload.txt", "w") as attemptfile:
+              attemptfile.write("")
+
+            logging.info(f"  - cannot upload '{cache_file[0]}' - rtc has become out of sync")
+            sleep(1)
+          elif status == UPLOAD_SKIP_FILE:
+            logging.error(f"  ! cannot upload '{cache_file[0]}' to {destination}. Skipping file")
+            warn_led(WARN_LED_BLINK)
+            continue
           else:
             logging.error(f"  ! failed to upload '{cache_file[0]}' to {destination}")
             return False
@@ -425,20 +493,28 @@ def startup():
 
   # see if we were woken to attempt a reupload
   if helpers.file_exists("reattempt_upload.txt"):
-    os.remove("reattempt_upload.txt")
+    upload_count = cached_upload_count()
+    if upload_count == 0:
+      os.remove("reattempt_upload.txt")
+      return
 
-    logging.info(f"> {cached_upload_count()} cache file(s) still to upload")
+    logging.info(f"> {upload_count} cache file(s) still to upload")
     if not upload_readings():
       halt("! reading upload failed")
 
+    os.remove("reattempt_upload.txt")
+
     # if it was the RTC that woke us, go to sleep until our next scheduled reading
     # otherwise continue with taking new readings etc
     # Note, this *may* result in a missed reading
     if reason == WAKE_REASON_RTC_ALARM:
       sleep()
 
 def sleep(time_override=None):
-  logging.info("> going to sleep")
+  if time_override is not None:
+    logging.info(f"> going to sleep for {time_override} minute(s)")
+  else:
+    logging.info("> going to sleep")
 
   # make sure the rtc flags are cleared before going back to sleep
   logging.debug("  - clearing and disabling previous alarm")
@@ -447,12 +523,16 @@ def sleep(time_override=None):
 
   # set alarm to wake us up for next reading
   dt = rtc.datetime()
-  hour, minute = dt[3:5]
+  hour, minute, second = dt[3:6]
 
   # calculate how many minutes into the day we are
   if time_override is not None:
     minute += time_override
   else:
+    # if the time is very close to the end of the minute, advance to the next minute
+    # this aims to fix the edge case where the board goes to sleep right as the RTC triggers, thus never waking up
+    if second > 55:
+      minute += 1
     minute = math.floor(minute / config.reading_frequency) * config.reading_frequency
     minute += config.reading_frequency
 

enviro/config_template.py

@@ -15,6 +15,9 @@
 # how often to wake up and take a reading (in minutes)
 reading_frequency = 15
 
+# how often to trigger a resync of the onboard RTC (in hours)
+resync_frequency = 168
+
 # where to upload to ("http", "mqtt", "adafruit_io", "influxdb")
 destination = None
 

enviro/constants.py

@@ -40,3 +40,5 @@
 UPLOAD_SUCCESS = 0
 UPLOAD_FAILED = 1
 UPLOAD_RATE_LIMITED = 2
+UPLOAD_LOST_SYNC = 3
+UPLOAD_SKIP_FILE = 4

enviro/destinations/adafruit_io.py

@@ -1,5 +1,5 @@
 from enviro import logging
-from enviro.constants import UPLOAD_SUCCESS, UPLOAD_FAILED, UPLOAD_RATE_LIMITED
+from enviro.constants import *
 import urequests
 import config
 
@@ -29,15 +29,33 @@ def upload_reading(reading):
 
   try:
     result = urequests.post(url, json=payload, headers=headers)
+
+    error_message = ""    
+    try:
+      error_message = result.json()['error']
+    except (TypeError, KeyError):
+      pass
+
     result.close()
     if result.status_code == 429:
       return UPLOAD_RATE_LIMITED
 
     if result.status_code == 200:
       return UPLOAD_SUCCESS
 
-    logging.debug(f"  - upload issue ({result.status_code} {result.reason})")
-  except:
-    logging.debug(f"  - an exception occurred when uploading")
+    if result.status_code == 422:
+      if error_message.find("data created_at may not be in the future") == 0:
+        return UPLOAD_LOST_SYNC
+
+      logging.debug(f"  - upload issue '{error_message}' - You may have run out of feeds to upload data to")
+      return UPLOAD_SKIP_FILE
+
+    logging.debug(f"  - upload issue '{error_message}' ({result.status_code} - {result.reason.decode('utf-8')})")      
+
+  except Exception as exc:
+    import sys, io
+    buf = io.StringIO()
+    sys.print_exception(exc, buf)
+    logging.debug(f"  - an exception occurred when uploading.", buf.getvalue())
 
   return UPLOAD_FAILED