diff --git a/src/sonic-ctrmgrd/.gitignore b/src/sonic-ctrmgrd/.gitignore index bdebd5e838cd..2f42f4d2c5ea 100644 --- a/src/sonic-ctrmgrd/.gitignore +++ b/src/sonic-ctrmgrd/.gitignore @@ -10,3 +10,4 @@ tests/__pycache__/ ctrmgr/__pycache__/ venv tests/.coverage* +.pytest_cache/ \ No newline at end of file diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py index 0de3297845e4..d53ca0b42e3a 100755 --- a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py @@ -60,7 +60,7 @@ CFG_SER_IP: "", CFG_SER_PORT: "6443", CFG_SER_DISABLE: "false", - CFG_SER_INSECURE: "false" + CFG_SER_INSECURE: "true" } dflt_st_ser = { @@ -88,17 +88,20 @@ JOIN_LATENCY = "join_latency_on_boot_seconds" JOIN_RETRY = "retry_join_interval_seconds" LABEL_RETRY = "retry_labels_update_seconds" +TAG_IMAGE_LATEST = "tag_latest_image_on_wait_seconds" USE_K8S_PROXY = "use_k8s_as_http_proxy" remote_ctr_config = { JOIN_LATENCY: 10, JOIN_RETRY: 10, LABEL_RETRY: 2, + TAG_IMAGE_LATEST: 30, USE_K8S_PROXY: "" } def log_debug(m): msg = "{}: {}".format(inspect.stack()[1][3], m) + #print(msg) syslog.syslog(syslog.LOG_DEBUG, msg) @@ -147,6 +150,8 @@ def init(): with open(SONIC_CTR_CONFIG, "r") as s: d = json.load(s) remote_ctr_config.update(d) + if UNIT_TESTING: + remote_ctr_config[TAG_IMAGE_LATEST] = 0 class MainServer: @@ -171,11 +176,11 @@ def register_db(self, db_name): self.db_connectors[db_name] = swsscommon.DBConnector(db_name, 0) - def register_timer(self, ts, handler): + def register_timer(self, ts, handler, args=()): """ Register timer based handler. The handler will be called on/after give timestamp, ts """ - self.timer_handlers[ts].append(handler) + self.timer_handlers[ts].append((handler, args)) def register_handler(self, db_name, table_name, handler): @@ -234,7 +239,7 @@ def run(self): lst = self.timer_handlers[k] del self.timer_handlers[k] for fn in lst: - fn() + fn[0](*fn[1]) else: timeout = (k - ct_ts).seconds break @@ -425,6 +430,54 @@ def do_join(self, ip, port, insecure): format(remote_ctr_config[JOIN_RETRY], self.start_time)) +def tag_latest_image(server, feat, docker_id, image_ver): + res = 1 + if not UNIT_TESTING: + status = os.system("docker ps |grep {} >/dev/null".format(docker_id)) + if status: + syslog.syslog(syslog.LOG_ERR, + "Feature {}:{} is not stable".format(feat, image_ver)) + else: + image_item = os.popen("docker inspect {} |jq -r .[].Image".format(docker_id)).read().strip() + if image_item: + image_id = image_item.split(":")[1][:12] + image_info = os.popen("docker images |grep {}".format(image_id)).read().split() + if image_info: + image_rep = image_info[0] + res = os.system("docker tag {} {}:latest".format(image_id, image_rep)) + if res != 0: + syslog.syslog(syslog.LOG_ERR, + "Failed to tag {}:{} to latest".format(image_rep, image_ver)) + else: + syslog.syslog(syslog.LOG_INFO, + "Successfully tag {}:{} to latest".format(image_rep, image_ver)) + feat_status = os.popen("docker inspect {} |jq -r .[].State.Running".format(feat)).read().strip() + if feat_status: + if feat_status == 'true': + os.system("docker stop {}".format(feat)) + syslog.syslog(syslog.LOG_ERR, + "{} should not run, stop it".format(feat)) + os.system("docker rm {}".format(feat)) + syslog.syslog(syslog.LOG_INFO, + "Delete previous {} container".format(feat)) + else: + syslog.syslog(syslog.LOG_ERR, + "Failed to docker images |grep {} to get image repo".format(image_id)) + else: + syslog.syslog(syslog.LOG_ERR, + "Failed to inspect container:{} to get image id".format(docker_id)) + else: + server.mod_db_entry(STATE_DB_NAME, + FEATURE_TABLE, feat, {"tag_latest": "true"}) + res = 0 + if res: + log_debug("failed to tag {}:{} to latest".format(feat, image_ver)) + else: + log_debug("successfully tag {}:{} to latest".format(feat, image_ver)) + + return res + + # # Feature changes # @@ -522,6 +575,19 @@ def on_state_update(self, key, op, data): self.st_data[key] = _update_entry(dflt_st_feat, data) remote_state = self.st_data[key][ST_FEAT_REMOTE_STATE] + if (old_remote_state != remote_state) and (remote_state == "running"): + # Tag latest + start_time = datetime.datetime.now() + datetime.timedelta( + seconds=remote_ctr_config[TAG_IMAGE_LATEST]) + self.server.register_timer(start_time, tag_latest_image, ( + self.server, + key, + self.st_data[key][ST_FEAT_CTR_ID], + self.st_data[key][ST_FEAT_CTR_VER])) + + log_debug("try to tag latest label after {} seconds @{}".format( + remote_ctr_config[TAG_IMAGE_LATEST], start_time)) + if (not init) and ( (old_remote_state == remote_state) or (remote_state != "pending")): # no change or nothing to do. diff --git a/src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json b/src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json index 3fb0c20fddcf..0b91fde36473 100644 --- a/src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json +++ b/src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json @@ -3,6 +3,7 @@ "retry_join_interval_seconds": 30, "retry_labels_update_seconds": 5, "revert_to_local_on_wait_seconds": 60, + "tag_latest_image_on_wait_seconds": 600, "use_k8s_as_http_proxy": "n" } diff --git a/src/sonic-ctrmgrd/tests/ctrmgrd_test.py b/src/sonic-ctrmgrd/tests/ctrmgrd_test.py index 171534b5a8d1..842b935396d1 100755 --- a/src/sonic-ctrmgrd/tests/ctrmgrd_test.py +++ b/src/sonic-ctrmgrd/tests/ctrmgrd_test.py @@ -106,7 +106,7 @@ common_test.KUBE_JOIN: { "ip": "10.10.10.10", "port": "6443", - "insecure": "false" + "insecure": "true" } } }, @@ -151,7 +151,7 @@ common_test.KUBE_JOIN: { "ip": "10.10.10.10", "port": "6443", - "insecure": "false" + "insecure": "true" }, common_test.KUBE_RESET: { "flag": "true" @@ -276,6 +276,51 @@ } } } + }, + 3: { + common_test.DESCR: "Tag image latest when remote_state changes to running", + common_test.ARGS: "ctrmgrd", + common_test.PRE: { + common_test.CONFIG_DB_NO: { + common_test.FEATURE_TABLE: { + "snmp": { + "set_owner": "kube" + } + } + }, + common_test.STATE_DB_NO: { + common_test.FEATURE_TABLE: { + "snmp": { + "remote_state": "pending" + } + } + } + }, + common_test.UPD: { + common_test.CONFIG_DB_NO: { + common_test.FEATURE_TABLE: { + "snmp": { + "set_owner": "kube" + } + } + }, + common_test.STATE_DB_NO: { + common_test.FEATURE_TABLE: { + "snmp": { + "remote_state": "running" + } + } + } + }, + common_test.POST: { + common_test.STATE_DB_NO: { + common_test.FEATURE_TABLE: { + "snmp": { + "tag_latest": "true" + } + } + } + } } }