Tag the running image to latest if it's stable

src/sonic-ctrmgrd/.gitignore

@@ -10,3 +10,4 @@ tests/__pycache__/
 ctrmgr/__pycache__/
 venv
 tests/.coverage*
+.pytest_cache/

src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py

@@ -60,7 +60,7 @@
         CFG_SER_IP: "",
         CFG_SER_PORT: "6443",
         CFG_SER_DISABLE: "false",
-        CFG_SER_INSECURE: "false"
+        CFG_SER_INSECURE: "true"
         }
 
 dflt_st_ser = {
@@ -88,17 +88,20 @@
 JOIN_LATENCY = "join_latency_on_boot_seconds"
 JOIN_RETRY = "retry_join_interval_seconds"
 LABEL_RETRY = "retry_labels_update_seconds"
+TAG_IMAGE_LATEST = "tag_latest_image_on_wait_seconds"
 USE_K8S_PROXY = "use_k8s_as_http_proxy"
 
 remote_ctr_config = {
     JOIN_LATENCY: 10,
     JOIN_RETRY: 10,
     LABEL_RETRY: 2,
+    TAG_IMAGE_LATEST: 30,
     USE_K8S_PROXY: ""
     }
 
 def log_debug(m):
     msg = "{}: {}".format(inspect.stack()[1][3], m)
+    #print(msg)
     syslog.syslog(syslog.LOG_DEBUG, msg)
 
 
@@ -147,6 +150,8 @@ def init():
         with open(SONIC_CTR_CONFIG, "r") as s:
             d = json.load(s)
             remote_ctr_config.update(d)
+    if UNIT_TESTING:
+        remote_ctr_config[TAG_IMAGE_LATEST] = 0
 
 
 class MainServer:
@@ -171,11 +176,11 @@ def register_db(self, db_name):
             self.db_connectors[db_name] = swsscommon.DBConnector(db_name, 0)
 
 
-    def register_timer(self, ts, handler):
+    def register_timer(self, ts, handler, args=()):
         """ Register timer based handler. 
             The handler will be called on/after give timestamp, ts
         """
-        self.timer_handlers[ts].append(handler)
+        self.timer_handlers[ts].append((handler, args))
 
 
     def register_handler(self, db_name, table_name, handler):
@@ -234,7 +239,7 @@ def run(self):
                     lst = self.timer_handlers[k]
                     del self.timer_handlers[k]
                     for fn in lst:
-                        fn()
+                        fn[0](*fn[1])
                 else:
                     timeout = (k - ct_ts).seconds
                     break
@@ -425,6 +430,54 @@ def do_join(self, ip, port, insecure):
                     format(remote_ctr_config[JOIN_RETRY], self.start_time))
 
 
+def tag_latest_image(server, feat, docker_id, image_ver):
+    res = 1
+    if not UNIT_TESTING:
+        status = os.system("docker ps |grep {} >/dev/null".format(docker_id))
+        if status:
+            syslog.syslog(syslog.LOG_ERR,
+                    "Feature {}:{} is not stable".format(feat, image_ver))
+        else:
+            image_item = os.popen("docker inspect {} |jq -r .[].Image".format(docker_id)).read().strip()
+            if image_item:
+                image_id = image_item.split(":")[1][:12]
+                image_info = os.popen("docker images |grep {}".format(image_id)).read().split()
+                if image_info:
+                    image_rep = image_info[0]
+                    res = os.system("docker tag {} {}:latest".format(image_id, image_rep))
+                    if res != 0:
+                        syslog.syslog(syslog.LOG_ERR,
+                                "Failed to tag {}:{} to latest".format(image_rep, image_ver))
+                    else:
+                        syslog.syslog(syslog.LOG_INFO,
+                                "Successfully tag {}:{} to latest".format(image_rep, image_ver))
+                        feat_status = os.popen("docker inspect {} |jq -r .[].State.Running".format(feat)).read().strip()
+                        if feat_status:
+                            if feat_status == 'true':
+                                os.system("docker stop {}".format(feat))
+                                syslog.syslog(syslog.LOG_ERR,
+                                        "{} should not run, stop it".format(feat))
+                            os.system("docker rm {}".format(feat))
+                            syslog.syslog(syslog.LOG_INFO,
+                                        "Delete previous {} container".format(feat))
+                else:
+                    syslog.syslog(syslog.LOG_ERR,
+                            "Failed to docker images |grep {} to get image repo".format(image_id))
+            else:
+                syslog.syslog(syslog.LOG_ERR,
+                        "Failed to inspect container:{} to get image id".format(docker_id))
+    else:
+        server.mod_db_entry(STATE_DB_NAME,
+                FEATURE_TABLE, feat, {"tag_latest": "true"})
+        res = 0
+    if res:
+        log_debug("failed to tag {}:{} to latest".format(feat, image_ver))
+    else:
+        log_debug("successfully tag {}:{} to latest".format(feat, image_ver))
+
+    return res
+
+
 #
 # Feature changes
 #
@@ -522,6 +575,19 @@ def on_state_update(self, key, op, data):
         self.st_data[key] = _update_entry(dflt_st_feat, data)
         remote_state = self.st_data[key][ST_FEAT_REMOTE_STATE]
 
+        if (old_remote_state != remote_state) and (remote_state == "running"):
+            # Tag latest
+            start_time = datetime.datetime.now() + datetime.timedelta(
+                    seconds=remote_ctr_config[TAG_IMAGE_LATEST])
+            self.server.register_timer(start_time, tag_latest_image, (
+                    self.server,
+                    key, 
+                    self.st_data[key][ST_FEAT_CTR_ID],
+                    self.st_data[key][ST_FEAT_CTR_VER]))
+
+            log_debug("try to tag latest label after {} seconds @{}".format(
+                    remote_ctr_config[TAG_IMAGE_LATEST], start_time))
+
         if (not init) and (
                 (old_remote_state == remote_state) or (remote_state != "pending")):
             # no change or nothing to do.

src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json

@@ -3,6 +3,7 @@
     "retry_join_interval_seconds": 30,
     "retry_labels_update_seconds": 5,
     "revert_to_local_on_wait_seconds": 60,
+    "tag_latest_image_on_wait_seconds": 600,
     "use_k8s_as_http_proxy": "n"
 }
 

src/sonic-ctrmgrd/tests/ctrmgrd_test.py

@@ -106,7 +106,7 @@
             common_test.KUBE_JOIN: {
                 "ip": "10.10.10.10",
                 "port": "6443",
-                "insecure": "false"
+                "insecure": "true"
             }
         }
     },
@@ -151,7 +151,7 @@
             common_test.KUBE_JOIN: {
                 "ip": "10.10.10.10",
                 "port": "6443",
-                "insecure": "false"
+                "insecure": "true"
             },
             common_test.KUBE_RESET: {
                 "flag": "true"
@@ -276,6 +276,51 @@
                 }
             }
         }
+    },
+    3: {
+        common_test.DESCR: "Tag image latest when remote_state changes to running",
+        common_test.ARGS: "ctrmgrd",
+        common_test.PRE: {
+            common_test.CONFIG_DB_NO: {
+                common_test.FEATURE_TABLE: {
+                    "snmp": {
+                        "set_owner": "kube"
+                    }
+                }
+            },
+            common_test.STATE_DB_NO: {
+                common_test.FEATURE_TABLE: {
+                    "snmp": {
+                        "remote_state": "pending"
+                    }
+                }
+            }
+        },
+        common_test.UPD: {
+            common_test.CONFIG_DB_NO: {
+                common_test.FEATURE_TABLE: {
+                    "snmp": {
+                        "set_owner": "kube"
+                    }
+                }
+            },
+            common_test.STATE_DB_NO: {
+                common_test.FEATURE_TABLE: {
+                    "snmp": {
+                        "remote_state": "running"
+                    }
+                }
+            }
+        },
+        common_test.POST: {
+            common_test.STATE_DB_NO: {
+                common_test.FEATURE_TABLE: {
+                    "snmp": {
+                        "tag_latest": "true"
+                    }
+                }
+            }
+        }
     }
 }