Fix scanner force-scanning

RELEASE.md

@@ -1,5 +1,9 @@
 # RELEASE NOTES
 
+## v1.14.1 - Scanner Fixes
+
+* Fix force-scanning bug in scanner introduced in last release by @uzlonewolf in https://github.com/jasonacox/tinytuya/pull/511.
+
 ## v1.14.0 - Command Line Updates
 
 * PyPI 1.14.0 rewrite of main to use argparse and add additional options by @uzlonewolf in https://github.com/jasonacox/tinytuya/pull/503

server/README.md

@@ -104,12 +104,21 @@ docker start tinytuya
 
 The UI at http://localhost:8888 allows you to view and control the devices.
 
-![image](https://user-images.githubusercontent.com/836718/227736045-adb6e359-c0c1-44b9-b9ad-7e978f6b7b84.png)
+![image](https://github.com/jasonacox/tinytuya/assets/836718/e00a1f9a-48e2-400c-afa1-7a81799efa89)
 
 ![image](https://user-images.githubusercontent.com/836718/227736057-e5392c13-554f-457e-9082-43c4d41a98ed.png)
 
 ## Release Notes
 
+### p12 - Force Scan
+
+* Added "Force Scan" button to cause server to run a network scan for devices not broadcasting.
+* Minor updates to UI for a cleaner title and footer to accommodate button.
+* Added logic to allow settings via environmental variables.
+* Add broadcast request to local network for 3.5 devices. 
+* Fix bug with cloud sync refresh losing device mappings.
+* Added "Cloud Sync" button to poll cloud for updated device data.
+
 ### t11 - Minimize Container
 
 * Reduce size of Docker container by removing rust build and using python:3.12-bookworm.

server/server.py

@@ -48,7 +48,7 @@
 import sys
 import os
 import urllib.parse
-from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer
+from http.server import BaseHTTPRequestHandler, HTTPServer
 from socketserver import ThreadingMixIn 
 
 # Terminal color capability for all platforms
@@ -59,30 +59,28 @@
     pass
 
 import tinytuya
+from tinytuya import scanner
+import os
 
-BUILD = "t11"
-
-# Defaults
-APIPORT = 8888
-DEBUGMODE = False
-DEVICEFILE = tinytuya.DEVICEFILE
-SNAPSHOTFILE = tinytuya.SNAPSHOTFILE
-CONFIGFILE = tinytuya.CONFIGFILE
-TCPTIMEOUT = tinytuya.TCPTIMEOUT    # Seconds to wait for socket open for scanning
-TCPPORT = tinytuya.TCPPORT          # Tuya TCP Local Port
-MAXCOUNT = tinytuya.MAXCOUNT        # How many tries before stopping
-UDPPORT = tinytuya.UDPPORT          # Tuya 3.1 UDP Port
-UDPPORTS = tinytuya.UDPPORTS        # Tuya 3.3 encrypted UDP Port
-UDPPORTAPP = tinytuya.UDPPORTAPP    # Tuya App
-TIMEOUT = tinytuya.TIMEOUT          # Socket Timeout
-RETRYTIME = 30
-RETRYCOUNT = 5
-SAVEDEVICEFILE = True
-
-# Check for Environmental Overrides
-debugmode = os.getenv("DEBUG", "no")
-if debugmode.lower() == "yes":
-    DEBUGMODE = True
+BUILD = "p12"
+
+# Defaults from Environment
+APIPORT = int(os.getenv("APIPORT", "8888"))
+DEBUGMODE = os.getenv("DEBUGMODE", "False").lower() == "true"
+DEVICEFILE = os.getenv("DEVICEFILE", tinytuya.DEVICEFILE)
+SNAPSHOTFILE = os.getenv("SNAPSHOTFILE", tinytuya.SNAPSHOTFILE)
+CONFIGFILE = os.getenv("CONFIGFILE", tinytuya.CONFIGFILE)
+TCPTIMEOUT = float(os.getenv("TCPTIMEOUT", str(tinytuya.TCPTIMEOUT)))
+TCPPORT = int(os.getenv("TCPPORT", str(tinytuya.TCPPORT)))
+MAXCOUNT = int(os.getenv("MAXCOUNT", str(tinytuya.MAXCOUNT)))
+UDPPORT = int(os.getenv("UDPPORT", str(tinytuya.UDPPORT)))
+UDPPORTS = int(os.getenv("UDPPORTS", str(tinytuya.UDPPORTS)))
+UDPPORTAPP = int(os.getenv("UDPPORTAPP", str(tinytuya.UDPPORTAPP)))
+TIMEOUT = float(os.getenv("TIMEOUT", str(tinytuya.TIMEOUT)))
+RETRYTIME = int(os.getenv("RETRYTIME", "30"))
+RETRYCOUNT = int(os.getenv("RETRYCOUNT", "5"))
+SAVEDEVICEFILE = os.getenv("SAVEDEVICEFILE", "True").lower() == "true"
+DEBUGMODE = os.getenv("DEBUGMODE", "no").lower() == "yes"
 
 # Logging
 log = logging.getLogger(__name__)
@@ -124,7 +122,11 @@ def sig_term_handle(signum, frame):
 retrydevices = {}
 retrytimer = 0
 cloudconfig = {'apiKey':'', 'apiSecret':'', 'apiRegion':'', 'apiDeviceID':''}
-
+forcescan = False
+forcescandone = True
+cloudsync = False
+cloudsyncdone = True
+cloudcreds = True
 
 # Terminal formatting
 (bold, subbold, normal, dim, alert, alertdim, cyan, red, yellow) = tinytuya.termcolor(True)
@@ -238,19 +240,25 @@ def tuyaLoadConfig():
 tuyadevices = tuyaLoadJson()
 cloudconfig = tuyaLoadConfig()
 
+# Start with Cloud API credentials
+if cloudconfig['apiKey'] == '' or cloudconfig['apiSecret'] == '' or cloudconfig['apiRegion'] == '' or cloudconfig['apiDeviceID'] == '':
+    cloudcreds = False
+
 def tuyaCloudRefresh():
+    global tuyadevices
+    print(" + Cloud Refresh Requested")
     log.debug("Calling Cloud Refresh")
     if cloudconfig['apiKey'] == '' or cloudconfig['apiSecret'] == '' or cloudconfig['apiRegion'] == '' or cloudconfig['apiDeviceID'] == '':
         log.debug("Cloud API config missing, not loading")
         return {'Error': 'Cloud API config missing'}
 
-    global tuyadevices
     cloud = tinytuya.Cloud( **cloudconfig )
     # on auth error, getdevices() will implode
     if cloud.error:
         return cloud.error
-    tuyadevices = cloud.getdevices(False)
+    tuyadevices = cloud.getdevices(verbose=False, oldlist=tuyadevices, include_map=True)
     tuyaSaveJson()
+    print(f" - Cloud Refresh Complete: {len(tuyadevices)} devices")
     return {'devices': tuyadevices}
 
 def getDeviceIdByName(name):
@@ -269,6 +277,7 @@ def tuyalisten(port):
     """
     log.debug("Started tuyalisten thread on %d", port)
     print(" - tuyalisten %d Running" % port)
+    last_broadcast = 0
 
     # Enable UDP listening broadcasting mode on UDP port 
     client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
@@ -282,6 +291,10 @@ def tuyalisten(port):
     client.settimeout(5)
 
     while(running):
+        if port == UDPPORTAPP and time.time() - last_broadcast > scanner.BROADCASTTIME:
+                log.debug("Sending discovery request to all 3.5 devices on the network")
+                scanner.send_discovery_request()
+                last_broadcast = time.time()
         try:
             data, addr = client.recvfrom(4048)
         except (KeyboardInterrupt, SystemExit) as err:
@@ -305,11 +318,14 @@ def tuyalisten(port):
             (dname, dkey, mac) = tuyaLookup(gwId)
         except:
             pass
+        if not gwId:
+            continue
         # set values
         result["name"] = dname
         result["mac"] = mac
         result["key"] = dkey
         result["id"] = gwId
+        result["forced"] = False
 
         # add device if new
         if not appenddevice(result, deviceslist):
@@ -323,7 +339,6 @@ def tuyalisten(port):
 
 class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
     daemon_threads = True
-    pass
 
 def delayoff(d, sw):
     d.turn_off(switch=sw, nowait=True)
@@ -355,8 +370,13 @@ def do_POST(self):
         self.wfile.write(bytes(message, "utf8"))
 
     def do_GET(self):
+        # pylint: disable=global-variable-not-assigned
         global retrytimer, retrydevices
         global cloudconfig, deviceslist
+        global forcescan, forcescandone
+        global serverstats, running
+        global cloudcreds, cloudsync, cloudsyncdone
+        global tuyadevices, newdevices
 
         self.send_response(200)
         message = "Error"
@@ -381,6 +401,11 @@ def do_GET(self):
             # Give Internal Stats
             serverstats['ts'] = int(time.time())
             serverstats['mem'] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+            serverstats['cloudcreds'] = cloudcreds
+            serverstats['cloudsync'] = cloudsync
+            serverstats['cloudsyncdone'] = cloudsyncdone
+            serverstats['forcescan'] = forcescan
+            serverstats['forcescandone'] = forcescandone
             message = json.dumps(serverstats)
         elif self.path.startswith('/set/'):
             try:
@@ -516,6 +541,11 @@ def do_GET(self):
             jout = {}
             jout["found"] = len(deviceslist)
             jout["registered"] = len(tuyadevices)
+            jout["forcescan"] = forcescan
+            jout["forcescandone"] = forcescandone
+            jout["cloudsync"] = cloudsync
+            jout["cloudsyncdone"] = cloudsyncdone
+            jout["cloudcreds"] = cloudcreds
             message = json.dumps(jout)
         elif self.path.startswith('/status/'):
             id = self.path.split('/status/')[1]
@@ -542,8 +572,14 @@ def do_GET(self):
                 message = json.dumps({"Error": "Device ID not found.", "id": id})
                 log.debug("Device ID not found: %s" % id)  
         elif self.path == '/sync':
-            message = json.dumps(tuyaCloudRefresh())
-            retrytimer = time.time() + RETRYTIME
+            if cloudconfig['apiKey'] == '' or cloudconfig['apiSecret'] == '' or cloudconfig['apiRegion'] == '' or cloudconfig['apiDeviceID'] == '':
+                message = json.dumps({"Error": "Cloud API config missing."})
+                log.debug("Cloud API config missing")
+            else:
+                message = json.dumps({"OK": "Cloud Sync Started."})
+            cloudsync = True
+            cloudsyncdone = False
+            retrytimer = 0
             retrydevices['*'] = 1
         elif self.path.startswith('/cloudconfig/'):
             cfgstr = self.path.split('/cloudconfig/')[1]
@@ -559,8 +595,14 @@ def do_GET(self):
                 message = json.dumps(tuyaCloudRefresh())
                 retrytimer = time.time() + RETRYTIME
                 retrydevices['*'] = 1
+                cloudcreds = all(cfg)
         elif self.path == '/offline':
             message = json.dumps(offlineDevices())
+        elif self.path == '/scan':
+            # Force Scan for new devices
+            forcescan = True
+            forcescandone = False
+            message = json.dumps({"OK": "Forcing a scan for new devices."})
         else:
             # Serve static assets from web root first, if found.
             fcontent, ftype = get_static(web_root, self.path)
@@ -571,7 +613,7 @@ def do_GET(self):
                 self.wfile.write(fcontent)
                 return
 
-        # Counts 
+        # Counts
         if "Error" in message:
             serverstats['errors'] = serverstats['errors'] + 1
         serverstats['gets'] = serverstats['gets'] + 1
@@ -606,7 +648,7 @@ def api(port):
     tuyaUDPs = threading.Thread(target=tuyalisten, args=(UDPPORTS,))
     tuyaUDP7 = threading.Thread(target=tuyalisten, args=(UDPPORTAPP,))
     apiServer = threading.Thread(target=api, args=(APIPORT,))
-    
+
     print(
         "\n%sTinyTuya %s(Server)%s [%s%s]\n"
         % (bold, normal, dim, tinytuya.__version__, BUILD)
@@ -626,17 +668,68 @@ def api(port):
 
     print(" * API and UI Endpoint on http://localhost:%d" % APIPORT)
     log.debug("Server URL http://localhost:%d" % APIPORT)
-    
+
     try:
         while(True):
             log.debug("Discovered Devices: %d   " % len(deviceslist))
+            if forcescan:
+                print(" + ForceScan: Scan for new devices started...")
+                forcescan = False
+                retrytimer = time.time() + RETRYTIME
+                # def devices(verbose=False, scantime=None, color=True, poll=True, forcescan=False, byID=False, show_timer=None, 
+                #   discover=True, wantips=None, wantids=None, snapshot=None, assume_yes=False, tuyadevices=[], 
+                #   maxdevices=0)
+                try:
+                    found = scanner.devices(forcescan=True, verbose=False, discover=False, assume_yes=True, tuyadevices=tuyadevices)
+                except:
+                    log.error("Error during scanner.devices()")
+                    found = []
+                print(f" - ForceScan: Found {len(found)} devices")
+                for f in found:
+                    log.debug(f"   - {found[f]}")
+                    gwId = found[f]["id"]
+                    result = {}
+                    dname = dkey = mac = ""
+                    try:
+                        # Try to pull name and key data
+                        (dname, dkey, mac) = tuyaLookup(gwId)
+                    except:
+                        pass
+                    # set values
+                    result["name"] = dname
+                    result["mac"] = mac
+                    result["key"] = dkey
+                    result["id"] = gwId
+                    result["ip"] = found[f]["ip"]
+                    result["version"] = found[f]["version"]
+                    result["forced"] = True
+
+                    # add device if new
+                    if not appenddevice(result, deviceslist):
+                        # Added device to list
+                        if dname == "" and dkey == "" and result["id"] not in newdevices:
+                            # If fetching the key failed, save it to retry later
+                            retrydevices[result["id"]] = RETRYCOUNT
+                            newdevices.append(result["id"])
+                forcescandone = True
+
+            if cloudsync:
+                cloudsync = False
+                cloudsyncdone = False
+                tuyaCloudRefresh()
+                cloudsyncdone = True
+                print(" - Cloud Sync Complete")
+                retrytimer = time.time() + RETRYTIME
+                retrydevices['*'] = 1
 
             if retrytimer <= time.time() or '*' in retrydevices:
                 if len(retrydevices) > 0:
                     # only refresh the cloud if we are not here because /sync was called
                     if '*' not in retrydevices:
+                        cloudsyncdone = False
                         tuyaCloudRefresh()
                         retrytimer = time.time() + RETRYTIME
+                        cloudsyncdone = True
                     found = []
                     # Try all unknown devices, even if the retry count expired
                     for devid in newdevices:
@@ -671,4 +764,4 @@ def api(port):
         # Close down API thread
         print("Stopping threads...")
         log.debug("Stoppping threads")
-        requests.get('http://localhost:%d/stop' % APIPORT)
+        requests.get('http://localhost:%d/stop' % APIPORT, timeout=5)