From 3a8c82edbda79fa43bdd2dce4c6fec58e63bb711 Mon Sep 17 00:00:00 2001 From: Pranav-b-7 Date: Thu, 29 Jul 2021 14:02:08 +0530 Subject: [PATCH] migration script from v3.8.x to v3.9 --- .../migration_v3.8.x_to_v3.9.py | 324 ++++++++++++++++++ .../steps_to_migrate_from_v3.8.x_to_v3.9.txt | 10 + 2 files changed, 334 insertions(+) create mode 100644 scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py create mode 100644 scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt diff --git a/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py new file mode 100644 index 00000000..97423df6 --- /dev/null +++ b/scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py @@ -0,0 +1,324 @@ +import json +import psycopg2 +import sys +import datetime +import string +import random +import requests + +def perform_migration(input_string): + try: + print ('Migrating from v3.8.x to v3.9') + + # Modify existing user_group table + modifyUserGroupTable() + + # add admin groups configuration to table + print ('Printing all admin group names given:: ', input_string) + admin_groups = input_string.split(',') + for group_name in admin_groups: + map_user_group_admin(group_name) + print("Successfully configured admin groups to table to v3.9") + + print("Migrating approval gate parameter entity") + alter_approval_gate_parameter() + approval_gate_instance_ids = fetch_distinct_gate_instances() + for approval_gate_instance_id in approval_gate_instance_ids: + connectors = fetch_distinct_connectors(approval_gate_instance_id[0]) + param_group_id = 1 + for connector in connectors: + update_param_group_id(param_group_id, connector[0], approval_gate_instance_id[0]) + param_group_id = param_group_id + 1 + + print("Migrating pipeline json") + pipeline_jsons = fetch_pipeline_json() + for pipeline_json in pipeline_jsons: + migrate_pipeline_json(pipeline_json[0], pipeline_json[1]) + print("Successfully migrated to v3.9") + + platform_conn.commit() + visibility_conn.commit() + except Exception as e: + platform_conn.rollback() + visibility_conn.rollback() + print ('Exception occured during migration : ', e) + finally: + platform_conn.close() + visibility_conn.close() + + +def alter_approval_gate_parameter(): + try: + cur = visibility_conn.cursor() + cur.execute("ALTER TABLE approval_gate_parameter ADD COLUMN IF NOT EXISTS param_group_id INT NOT NULL DEFAULT 0") + except Exception as e: + print("Exception occured while altering the approval_gate_parameter table : ", e) + raise e + + +def fetch_distinct_gate_instances(): + try: + cur = visibility_conn.cursor() + cur.execute("select distinct(approval_gate_instance_id) from approval_gate_parameter ORDER BY approval_gate_instance_id") + return cur.fetchall() + except Exception as e: + print("Exception occured while fetching distinct gate instances : ", e) + raise e + + +def fetch_distinct_connectors(gate_instance_id): + try: + cur = visibility_conn.cursor() + cur.execute("select distinct(connector_type) from approval_gate_parameter where approval_gate_instance_id = "+str(gate_instance_id)) + return cur.fetchall() + except Exception as e: + print("Exception occured while fetching distinct connectors : ", e) + raise e + + +def update_param_group_id(param_group_id, connector_type, approval_gate_instance_id): + try: + cur = visibility_conn.cursor() + cur.execute("update approval_gate_parameter set param_group_id = " + str(param_group_id) + " where connector_type = '" + str(connector_type) + "' AND approval_gate_instance_id = "+str(approval_gate_instance_id)) + except Exception as e: + print("Exception occured while updating param group id : ", e) + raise e + + +def fetch_pipeline_json(): + try: + cur = platform_conn.cursor() + cur.execute("select id, pipeline_json from pipeline") + return cur.fetchall() + except Exception as e: + print("Exception occured while fetching pipeline jsons : ", e) + raise e + + +def migrate_pipeline_json(id, pipeline_json): + try: + cur = platform_conn.cursor() + pipe_json = json.loads(pipeline_json) + stages = pipe_json['stages'] + if stages is not None and len(stages)>0: + for stage in stages: + gate_type = stage['type'] + if gate_type is not None and len(gate_type)>0: + if str(gate_type).strip() == 'Visibility Approval': + migrate_approval_gate(stage) + elif str(gate_type).strip() == 'Verification': + migrate_verification_gate(stage) + elif str(gate_type).strip() == 'Test Verification': + migrate_test_verification_gate(stage) + elif str(gate_type).strip() == 'Policy Stage': + migrate_policy_gate(stage) + pipe_json['stages'] = stages + cur.execute("update pipeline set pipeline_json = '"+str(json.dumps(pipe_json))+"' where id="+str(id)) + + except Exception as e: + print("Exception occured while migrating pipeline json : ", e) + raise e + + +def migrate_policy_gate(stage): + del stage['alias'] + stage['type'] = 'policy' + + +def migrate_test_verification_gate(stage): + del stage['alias'] + del stage['comments'] + stage['type'] = 'testVerification' + + +def migrate_verification_gate(stage): + del stage['alias'] + del stage['comments'] + stage['type'] = 'verification' + + +def migrate_approval_gate(stage): + try: + del stage['alias'] + stage['type'] = 'approval' + parameters = stage['parameters'] + if parameters is not None and len(parameters) > 0: + canaryid = parameters['canaryid'] + gitrepo = parameters['gitrepo'] + jenkinsartifact = parameters['jenkinsartifact'] + jenkinsbuild = parameters['jenkinsbuild'] + appscanid = parameters['appscanid'] + aquawave = parameters['aquawave'] + jenkinsjob = parameters['jenkinsjob'] + projectkey = parameters['projectkey'] + connector = parameters['connector'] + jiraid = parameters['jiraid'] + header = parameters['header'] + gitcommitid = parameters['gitcommitid'] + connectordata = parameters['connectordata'] + imageids = parameters['imageids'] + gateurl = parameters['gateurl'] + + connectors = [] + if gateurl is not None and len(gateurl)>0: + url_components = str(gateurl).split("/") + approval_gate_id = url_components[len(url_components) - 2] + configured_tool_connectors = fetch_configured_tool_connectors(approval_gate_id) + for configured_tool_connector in configured_tool_connectors: + connector_type = configured_tool_connector['connectorType'] + if connector_type is not None: + if str(connector_type).strip().upper() == 'JIRA': + configure_jira(configured_tool_connector, jiraid) + elif str(connector_type).strip().upper() == 'GIT': + configure_git(configured_tool_connector, gitcommitid, gitrepo) + elif str(connector_type).strip().upper() == 'JENKINS': + configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob) + elif str(connector_type).strip().upper() == 'AUTOPILOT': + configure_autopilot(canaryid, configured_tool_connector) + elif str(connector_type).strip().upper() == 'SONARQUBE': + configure_sonarqube(configured_tool_connector, projectkey) + elif str(connector_type).strip().upper() == 'APPSCAN': + configure_appscan(appscanid, configured_tool_connector) + elif str(connector_type).strip().upper() == 'AQUAWAVE': + configure_aquawave(aquawave, configured_tool_connector) + connectors.append(configured_tool_connector) + gateurl = str(gateurl).strip().replace("v4", "v5") + gateurl = str(gateurl).strip().replace("v3", "v5") + gateurl = str(gateurl).strip().replace("v2", "v5") + gateurl = str(gateurl).strip().replace("v1", "v5") + new_parameters = {"imageIds": imageids, + "connectors": connectors, + "gateUrl": gateurl} + stage['parameters'] = new_parameters + + except Exception as e: + print("Exception occured while migrating approval gate : ", e) + raise e + + +def configure_aquawave(aquawave, configured_tool_connector): + aqua = [] + param = {"imageId": aquawave} + aqua.append(param) + configured_tool_connector['values'] = aqua + + +def configure_appscan(appscanid, configured_tool_connector): + appscan = [] + param = {"id": appscanid} + appscan.append(param) + configured_tool_connector['values'] = appscan + + +def configure_sonarqube(configured_tool_connector, projectkey): + sonarqube = [] + param = {"projectKey": projectkey} + sonarqube.append(param) + configured_tool_connector['values'] = sonarqube + + +def configure_autopilot(canaryid, configured_tool_connector): + autopilot = [] + param = {"canaryId": canaryid} + autopilot.append(param) + configured_tool_connector['values'] = autopilot + + +def configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob): + jenkins = [] + param = {"job": jenkinsjob, + "buildId": jenkinsbuild, + "artifact": jenkinsartifact} + jenkins.append(param) + configured_tool_connector['values'] = jenkins + + +def configure_git(configured_tool_connector, gitcommitid, gitrepo): + git = [] + param = {"repo": gitrepo, + "commitId": gitcommitid} + git.append(param) + configured_tool_connector['values'] = git + + +def configure_jira(configured_tool_connector, jiraid): + jira_tickets = [] + if jiraid is not None and len(jiraid) > 0: + jiraids = str(jiraid).split(",") + for id in jiraids: + param = {"jira_ticket_no": id} + jira_tickets.append(param) + configured_tool_connector['values'] = jira_tickets + + +def fetch_configured_tool_connectors(approval_gate_id): + try: + url = oes_visibility_url + "/visibilityservice/v1/approvalGates/"+str(approval_gate_id)+"/configuredtoolConnectors" + headers = {'x-spinnaker-user': oes_admin_user} + response = requests.get(url=url, headers=headers) + return json.loads(response.content) + except Exception as e: + print("Exception occured while fetching configured tool connectors : ", e) + raise e + +def modifyUserGroupTable(): + try: + cur = platform_conn.cursor() + cur.execute("""ALTER TABLE user_group ADD COLUMN IF NOT EXISTS "is_admin" BOOLEAN NOT NULL DEFAULT FALSE""" + ) + except Exception as e: + print ('Exception occured while modifying user_group table: ', + e) + raise e + + +def map_user_group_admin(group_name): + try: + cur = platform_conn.cursor() + cur.execute("SELECT id FROM user_group Where name='" + + group_name + "'") + user_group_id = cur.fetchone() + update_user_group(user_group_id) + except Exception as e: + print ('Exception occured while fetching data from user_group table: ' + , e) + raise e + + +def update_user_group(user_group_id): + try: + cur = platform_conn.cursor() + cur.execute('UPDATE user_group SET is_admin = true WHERE id = '+str(user_group_id[0])) + except Exception as e: + print ('Exception occured while updating user_group table: ', e) + raise e + +if __name__ == '__main__': + n = len(sys.argv) + if n != 8: + print ('Please pass valid 7 arguments ') + + platform_db = sys.argv[1] + platform_host = sys.argv[2] + visibility_db = sys.argv[3] + visibility_host = sys.argv[4] + port = sys.argv[5] + oes_visibility_url = sys.argv[6] + oes_admin_user = sys.argv[7] + + # Establishing the platform db connection + + platform_conn = psycopg2.connect(database=platform_db, + user='postgres', password='networks123', + host=platform_host, port=port) + print ('Opened platform database connection successfully') + + visibility_conn = psycopg2.connect(database=visibility_db, user='postgres', password='networks123', host=visibility_host, port=port) + print("Visibility database connection established successfully") + + # Getting input of admin groups + + input_string = \ + input('Enter admin groups to be configure in separated by comma:: ' + ) + perform_migration(input_string) diff --git a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt new file mode 100644 index 00000000..333c802e --- /dev/null +++ b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt @@ -0,0 +1,10 @@ +Steps to migrate from v3.8.x to v3.9 + +1. Copy the file : migration_v3.8.x_to_v3.9.py to some path in platform pod. +2. Run the below command to execute the script. + + python3 + + Replace with valid values + + Ex: python3 migration_v3.8.x_to_v3.9.py platformdb oes-db visibilitydb oes-db 5432 http://oes-visibility:8096 admin