From e6e34dbc47267edeba2a265d4824fb0d2248c6c2 Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Fri, 30 Jun 2023 07:26:32 +0000 Subject: [PATCH 1/7] Fix CONFIG_DB_INITIALIZED flag check logic and set/reset flag for warm-reboot --- files/build_templates/docker_image_ctl.j2 | 3 +++ files/image_config/config-setup/config-setup | 1 + files/image_config/warmboot-finalizer/finalize-warmboot.sh | 2 +- files/scripts/swss.sh | 2 +- files/scripts/syncd_common.sh | 2 +- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index c14bedfd6f32..26f61972f3ed 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -238,6 +238,9 @@ function postStartAction() if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast" || "$BOOT_TYPE" == "fast") && -f $WARM_DIR/dump.rdb ]]; then # retain the dump file from last boot for debugging purposes mv $WARM_DIR/dump.rdb $WARM_DIR/dump.rdb.old + # CONFIG_DB init flag will be set to 1 at the time when device warm-reboots and old config is loaded on DB + # Reset CONFIG_DB init value to 0 to indicate start of new system. This will be set to "1" after DB migration is completed + $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "0" else # If there is a config_db.json dump file, load it. if [ -r /etc/sonic/config_db$DEV.json ]; then diff --git a/files/image_config/config-setup/config-setup b/files/image_config/config-setup/config-setup index 0f5a248b05f8..0735d39b632e 100755 --- a/files/image_config/config-setup/config-setup +++ b/files/image_config/config-setup/config-setup @@ -327,6 +327,7 @@ do_config_migration() echo "Warm reboot detected..." disable_updategraph rm -f /tmp/pending_config_migration + sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" exit 0 elif check_all_config_db_present; then echo "Use config_db.json from old system..." diff --git a/files/image_config/warmboot-finalizer/finalize-warmboot.sh b/files/image_config/warmboot-finalizer/finalize-warmboot.sh index 783f3881816b..1446144aec34 100755 --- a/files/image_config/warmboot-finalizer/finalize-warmboot.sh +++ b/files/image_config/warmboot-finalizer/finalize-warmboot.sh @@ -74,7 +74,7 @@ function wait_for_database_service() done # Wait for configDB initialization - until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; + until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do sleep 1; done diff --git a/files/scripts/swss.sh b/files/scripts/swss.sh index 0e37fe1376cd..86c2bae2ddb8 100755 --- a/files/scripts/swss.sh +++ b/files/scripts/swss.sh @@ -87,7 +87,7 @@ function wait_for_database_service() done # Wait for configDB initialization - until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; + until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do sleep 1; done } diff --git a/files/scripts/syncd_common.sh b/files/scripts/syncd_common.sh index a850e31b207d..826cdd731ba6 100755 --- a/files/scripts/syncd_common.sh +++ b/files/scripts/syncd_common.sh @@ -66,7 +66,7 @@ function wait_for_database_service() done # Wait for configDB initialization - until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]]; + until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do sleep 1; done } From c05af46d02db315b3354a6f88eef45d49ff85b08 Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Fri, 30 Jun 2023 07:29:38 +0000 Subject: [PATCH 2/7] Fix db-cli usage --- files/image_config/warmboot-finalizer/finalize-warmboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/image_config/warmboot-finalizer/finalize-warmboot.sh b/files/image_config/warmboot-finalizer/finalize-warmboot.sh index 1446144aec34..b01832a6c7b8 100755 --- a/files/image_config/warmboot-finalizer/finalize-warmboot.sh +++ b/files/image_config/warmboot-finalizer/finalize-warmboot.sh @@ -74,7 +74,7 @@ function wait_for_database_service() done # Wait for configDB initialization - until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; + until [[ $(sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") -eq 1 ]]; do sleep 1; done From 65c65e13406270994f745b1f388a13262eee00d2 Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Fri, 7 Jul 2023 02:36:59 +0000 Subject: [PATCH 3/7] Handle same image warm-reboot and generalize handling of INIT flag --- files/build_templates/docker_image_ctl.j2 | 9 ++++++--- files/image_config/config-setup/config-setup | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index 688c1619ea41..dbf5e721cdd5 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -256,15 +256,16 @@ function postStartAction() # set the key to expire in 3 minutes $SONIC_DB_CLI STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180" fi - - $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" fi if [ -e /tmp/pending_config_migration ]; then # this is first boot to a new image, config-setup execution is pending. + # for warmboot case, DB is loaded but migration is still pending # For fast/cold reboot case, DB contains nothing at this point - # Call db_migrator after config-setup loads the config (from old config or minigraph) + # Call db_migrator and set CONFIG_DB_INITIALIZED after config-setup loads the config (from old config or minigraph) echo "Delaying db_migrator until config migration is over" + # unset CONFIG_DB_INITIALIZED to indicate pending config load and migration + $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "0" else # this is not a first time boot to a new image. Datbase container starts w/ old pre-existing config if [[ -x /usr/local/bin/db_migrator.py ]]; then @@ -273,6 +274,8 @@ function postStartAction() /usr/local/bin/db_migrator.py -o migrate fi fi + # set CONFIG_DB_INITIALIZED to indicate end of config load and migration + $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" fi # Add redis UDS to the redis group and give read/write access to the group REDIS_SOCK="/var/run/redis${DEV}/redis.sock" diff --git a/files/image_config/config-setup/config-setup b/files/image_config/config-setup/config-setup index 875beba45bb8..23b5d0ff3446 100755 --- a/files/image_config/config-setup/config-setup +++ b/files/image_config/config-setup/config-setup @@ -305,13 +305,14 @@ check_all_config_db_present() } # DB schema is subject to change between two images -# Perform DB schema migration after loading backup config from previous image +# Perform DB schema migration after loading backup config/minigraph from previous image do_db_migration() { if [[ -x /usr/local/bin/db_migrator.py ]]; then # Migrate the DB to the latest schema version if needed /usr/local/bin/db_migrator.py -o migrate fi + sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" } # Perform configuration migration from backup copy. @@ -338,7 +339,6 @@ do_config_migration() disable_updategraph do_db_migration rm -f /tmp/pending_config_migration - sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" exit 0 elif check_all_config_db_present; then echo "Use config_db.json from old system..." From 678e33ef830bdb314b618b51a13e8e28ef487ea6 Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Fri, 7 Jul 2023 18:25:51 +0000 Subject: [PATCH 4/7] Cover boot from ONIE case: set config init flag when minigraph, config_db are missing --- files/image_config/config-setup/config-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/files/image_config/config-setup/config-setup b/files/image_config/config-setup/config-setup index 23b5d0ff3446..7952d3aa64dc 100755 --- a/files/image_config/config-setup/config-setup +++ b/files/image_config/config-setup/config-setup @@ -354,6 +354,7 @@ do_config_migration() disable_updategraph else echo "Didn't found neither config_db.json nor minigraph.xml ..." + sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" fi rm -f /tmp/pending_config_migration From 27baf1514eb8e9b9b7473fc66ab203eff3f847f1 Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Fri, 7 Jul 2023 21:47:56 +0000 Subject: [PATCH 5/7] Handle case: first boot of SONiC --- files/image_config/config-setup/config-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/image_config/config-setup/config-setup b/files/image_config/config-setup/config-setup index 7952d3aa64dc..d4826d9b6279 100755 --- a/files/image_config/config-setup/config-setup +++ b/files/image_config/config-setup/config-setup @@ -256,6 +256,7 @@ do_config_initialization() fi rm -f /tmp/pending_config_initialization + sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" } # Restore config-setup post migration hooks from a backup copy @@ -354,7 +355,6 @@ do_config_migration() disable_updategraph else echo "Didn't found neither config_db.json nor minigraph.xml ..." - sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" fi rm -f /tmp/pending_config_migration From 5779bfce33a2b55d224b539cfabc94abf484ce12 Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Tue, 18 Jul 2023 09:09:07 +0000 Subject: [PATCH 6/7] Check for config init flag --- files/build_templates/docker_image_ctl.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index dbf5e721cdd5..bdcd4fed0e55 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -258,7 +258,7 @@ function postStartAction() fi fi - if [ -e /tmp/pending_config_migration ]; then + if [ -e /tmp/pending_config_migration ] || [ -e /tmp/pending_config_initialization ]; then # this is first boot to a new image, config-setup execution is pending. # for warmboot case, DB is loaded but migration is still pending # For fast/cold reboot case, DB contains nothing at this point From fdecad130b8ba8696be7daea677f96e43d4cedef Mon Sep 17 00:00:00 2001 From: Vaibhav Hemant Dixit Date: Wed, 19 Jul 2023 07:38:23 +0000 Subject: [PATCH 7/7] Simplify logic, and do not call db_migrator for same image reboot --- files/build_templates/docker_image_ctl.j2 | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index bdcd4fed0e55..23d111051f62 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -235,12 +235,10 @@ function postStartAction() ($(docker exec -i database$DEV sonic-db-cli PING | grep -c PONG) -gt 0) ]]; do sleep 1; done + if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast" || "$BOOT_TYPE" == "fast") && -f $WARM_DIR/dump.rdb ]]; then # retain the dump file from last boot for debugging purposes mv $WARM_DIR/dump.rdb $WARM_DIR/dump.rdb.old - # CONFIG_DB init flag will be set to 1 at the time when device warm-reboots and old config is loaded on DB - # Reset CONFIG_DB init value to 0 to indicate start of new system. This will be set to "1" after DB migration is completed - $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "0" else # If there is a config_db.json dump file, load it. if [ -r /etc/sonic/config_db$DEV.json ]; then @@ -251,29 +249,16 @@ function postStartAction() $SONIC_CFGGEN -j /etc/sonic/config_db$DEV.json --write-to-db fi fi - - if [[ "$BOOT_TYPE" == "fast" ]]; then - # set the key to expire in 3 minutes - $SONIC_DB_CLI STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180" - fi fi if [ -e /tmp/pending_config_migration ] || [ -e /tmp/pending_config_initialization ]; then # this is first boot to a new image, config-setup execution is pending. # for warmboot case, DB is loaded but migration is still pending - # For fast/cold reboot case, DB contains nothing at this point - # Call db_migrator and set CONFIG_DB_INITIALIZED after config-setup loads the config (from old config or minigraph) - echo "Delaying db_migrator until config migration is over" + # For firstbboot/fast/cold reboot case, DB contains nothing at this point # unset CONFIG_DB_INITIALIZED to indicate pending config load and migration + # This flag will be set to "1" after DB migration/initialization is completed as part of config-setup $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "0" else - # this is not a first time boot to a new image. Datbase container starts w/ old pre-existing config - if [[ -x /usr/local/bin/db_migrator.py ]]; then - # Migrate the DB to the latest schema version if needed - if [ -z "$DEV" ]; then - /usr/local/bin/db_migrator.py -o migrate - fi - fi # set CONFIG_DB_INITIALIZED to indicate end of config load and migration $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1" fi