From 3dc963c70e819d8a9145a2b2f57998cba1a7d5e8 Mon Sep 17 00:00:00 2001 From: dbelusky Date: Wed, 12 Oct 2022 14:17:53 +0200 Subject: [PATCH 1/3] implement scheduler to main.py --- composefiles/swarm-fm-workflows.yml | 1 + demo-workflows/workers/constants/scheduler.py | 14 +++++++++++ demo-workflows/workers/main.py | 23 +++++++++++++++++++ startup.sh | 8 ++++++- 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 demo-workflows/workers/constants/scheduler.py diff --git a/composefiles/swarm-fm-workflows.yml b/composefiles/swarm-fm-workflows.yml index 463b485..537542b 100644 --- a/composefiles/swarm-fm-workflows.yml +++ b/composefiles/swarm-fm-workflows.yml @@ -21,6 +21,7 @@ services: - UNICONFIG_URL_BASE=https://uniconfig:8181/rests - INSTANCES_TO_SIMULATE=${INSTANCES_TO_SIMULATE} - RUN_TESTTOOLS=${RUN_TESTTOOLS} + - TOPOLOGY_DISCOVERY_BACKUP=${TOPOLOGY_DISCOVERY_BACKUP} entrypoint: ["/set_env_secrets.sh", "python3 main.py"] secrets: - frinx_auth diff --git a/demo-workflows/workers/constants/scheduler.py b/demo-workflows/workers/constants/scheduler.py new file mode 100644 index 0000000..7516870 --- /dev/null +++ b/demo-workflows/workers/constants/scheduler.py @@ -0,0 +1,14 @@ +UNIFLOW_SCHEDULER_URL = "http://workflow-proxy:8088/schedule" + +UNIFLOW_SCHEDULER_LIST = [ + { + "Arangodb_create_backup_and_delete_old_backups": { + "workflowName": "Arangodb_create_backup_and_delete_old_backups", + "workflowVersion": "1", + "workflowContext": {"delete_age": "24"}, + "name": "Arangodb_create_backup_and_delete_old_backups:1", + "cronString": "0 * * * *", + "enabled": True, + } + } +] diff --git a/demo-workflows/workers/main.py b/demo-workflows/workers/main.py index fae41e9..0b181b0 100644 --- a/demo-workflows/workers/main.py +++ b/demo-workflows/workers/main.py @@ -12,6 +12,7 @@ from importDevices import import_devices from importDevices import import_blueprints +from constants.scheduler import UNIFLOW_SCHEDULER_LIST, UNIFLOW_SCHEDULER_URL # Suppress InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -72,6 +73,7 @@ def _register_workers(conductor) -> None: import device_worker import lldp_identification_worker import influx_db + import topology_discovery_worker # import vll_worker # import vll_service_worker # import vpls_worker @@ -92,6 +94,7 @@ def _register_workers(conductor) -> None: device_worker.start(conductor) lldp_identification_worker.start(conductor) influx_db.start(conductor) + topology_discovery_worker.start(conductor) # vll_worker.start(cc) # vll_service_worker.start(cc) # vpls_worker.start(cc) @@ -122,6 +125,22 @@ def _configure_healthcheck(file: Path = HEALTCHCHEK_FILE) -> None: pass +def _import_schedulers() -> dict: + """ + return: result_dict ex. {'Arangodb_create_backup_and_delete_old_backups:1': {'status_code': 200}} + """ + result_dict = {} + for scheduler_data in UNIFLOW_SCHEDULER_LIST: + # Convert dict values to dict + data = list(scheduler_data.values())[0] + # URL ex. http://workflow-proxy:8088/schedule/Arangodb_create_backup_and_delete_old_backups:1 + response = requests.put(f"{UNIFLOW_SCHEDULER_URL}/{data['name']}", + data=json.dumps(data), headers=conductor_headers) + result_dict[data["name"]] = {"status_code": response.status_code} + + return result_dict + + def main(): configure_logging() logger = logging.getLogger(__name__) @@ -145,6 +164,10 @@ def main(): _import_blueprints() logger.info("All blueprints are imported") + if int(os.getenv("TOPOLOGY_DISCOVERY_BACKUP")): + result_dict = _import_schedulers() + logger.info(f"Scheduler import status codes: {result_dict}") + _configure_healthcheck() logger.debug("Health check is configured") diff --git a/startup.sh b/startup.sh index d8ff849..481c004 100755 --- a/startup.sh +++ b/startup.sh @@ -13,7 +13,9 @@ DESCRIPTION: -h|--help Print this help and exit -d|--debug Enable verbose - -ci|--clean-inventory Clean device inventory + -ci|--clean-inventory Clean device inventory + -tdb|--topology-discovery-backup Create Schedule for topology discovery backup WF (every 1 hour) + SAMPLE-TOPOLOGY Parameters: -ad|--all-devices Run CLI and netconf devices @@ -28,6 +30,7 @@ EOF } export RUN_TESTTOOLS="./scripts/run_cli_devices/run_devices_docker.sh" +export TOPOLOGY_DISCOVERY_BACKUP="0" INSTANCES_TO_SIMULATE="" function argumentsCheck { @@ -55,6 +58,9 @@ function argumentsCheck { -ci|--clean-inventory) docker exec -it "$(docker ps -qf "name=fm_inventory-postgres")" psql -U postgres -a inventory -c 'delete from device_inventory;' echo "Device inventory has been cleaned.";; + -tdb|--topology-discovery-backup) + export TOPOLOGY_DISCOVERY_BACKUP="1" + echo "Run scheduler for topology discovery arangodb backup";; *) echo "Unknow option: ${1}" show_help From 183c52daa1811430be96105d240a5fbb963b116e Mon Sep 17 00:00:00 2001 From: dbelusky Date: Wed, 12 Oct 2022 14:40:47 +0200 Subject: [PATCH 2/3] rename TOPOLOGY_DISCOVERY_BACKUP to TOPOLOGY_DISCOVERY_BACKUP_ENABLED --- composefiles/swarm-fm-workflows.yml | 2 +- demo-workflows/workers/main.py | 2 +- startup.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/composefiles/swarm-fm-workflows.yml b/composefiles/swarm-fm-workflows.yml index 537542b..82dfad2 100644 --- a/composefiles/swarm-fm-workflows.yml +++ b/composefiles/swarm-fm-workflows.yml @@ -21,7 +21,7 @@ services: - UNICONFIG_URL_BASE=https://uniconfig:8181/rests - INSTANCES_TO_SIMULATE=${INSTANCES_TO_SIMULATE} - RUN_TESTTOOLS=${RUN_TESTTOOLS} - - TOPOLOGY_DISCOVERY_BACKUP=${TOPOLOGY_DISCOVERY_BACKUP} + - TOPOLOGY_DISCOVERY_BACKUP_ENABLED=${TOPOLOGY_DISCOVERY_BACKUP_ENABLED} entrypoint: ["/set_env_secrets.sh", "python3 main.py"] secrets: - frinx_auth diff --git a/demo-workflows/workers/main.py b/demo-workflows/workers/main.py index 0b181b0..989ad38 100644 --- a/demo-workflows/workers/main.py +++ b/demo-workflows/workers/main.py @@ -164,7 +164,7 @@ def main(): _import_blueprints() logger.info("All blueprints are imported") - if int(os.getenv("TOPOLOGY_DISCOVERY_BACKUP")): + if int(os.getenv("TOPOLOGY_DISCOVERY_BACKUP_ENABLED")): result_dict = _import_schedulers() logger.info(f"Scheduler import status codes: {result_dict}") diff --git a/startup.sh b/startup.sh index 481c004..1781091 100755 --- a/startup.sh +++ b/startup.sh @@ -30,7 +30,7 @@ EOF } export RUN_TESTTOOLS="./scripts/run_cli_devices/run_devices_docker.sh" -export TOPOLOGY_DISCOVERY_BACKUP="0" +export TOPOLOGY_DISCOVERY_BACKUP_ENABLED="0" INSTANCES_TO_SIMULATE="" function argumentsCheck { @@ -59,7 +59,7 @@ function argumentsCheck { docker exec -it "$(docker ps -qf "name=fm_inventory-postgres")" psql -U postgres -a inventory -c 'delete from device_inventory;' echo "Device inventory has been cleaned.";; -tdb|--topology-discovery-backup) - export TOPOLOGY_DISCOVERY_BACKUP="1" + export TOPOLOGY_DISCOVERY_BACKUP_ENABLED="1" echo "Run scheduler for topology discovery arangodb backup";; *) echo "Unknow option: ${1}" From db7ec38a1b76f5c411214c8a6b66badde90ad3e3 Mon Sep 17 00:00:00 2001 From: dbelusky Date: Wed, 12 Oct 2022 15:27:02 +0200 Subject: [PATCH 3/3] delete fm_prefix from TOPOLOGY_DISCOVERY_BASE_URL --- demo-workflows/workers/topology_discovery_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo-workflows/workers/topology_discovery_worker.py b/demo-workflows/workers/topology_discovery_worker.py index 28d3803..012316b 100644 --- a/demo-workflows/workers/topology_discovery_worker.py +++ b/demo-workflows/workers/topology_discovery_worker.py @@ -2,7 +2,7 @@ import requests import json -TOPOLOGY_DISCOVERY_BASE_URL = "http://fm_topology-discovery:5000/api" +TOPOLOGY_DISCOVERY_BASE_URL = "http://topology-discovery:5000/api" TOPOLOGY_DISCOVERY_HEADERS = {"Content-Type": "application/json"}