-
Notifications
You must be signed in to change notification settings - Fork 57
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
migration script from v3.8.x to v3.9
- Loading branch information
1 parent
330e123
commit 3a8c82e
Showing
2 changed files
with
334 additions
and
0 deletions.
There are no files selected for viewing
324 changes: 324 additions & 0 deletions
324
scripts/oes-data-migration-scripts/migration_v3.8.x_to_v3.9.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,324 @@ | ||
import json | ||
import psycopg2 | ||
import sys | ||
import datetime | ||
import string | ||
import random | ||
import requests | ||
|
||
def perform_migration(input_string): | ||
try: | ||
print ('Migrating from v3.8.x to v3.9') | ||
|
||
# Modify existing user_group table | ||
modifyUserGroupTable() | ||
|
||
# add admin groups configuration to table | ||
print ('Printing all admin group names given:: ', input_string) | ||
admin_groups = input_string.split(',') | ||
for group_name in admin_groups: | ||
map_user_group_admin(group_name) | ||
print("Successfully configured admin groups to table to v3.9") | ||
|
||
print("Migrating approval gate parameter entity") | ||
alter_approval_gate_parameter() | ||
approval_gate_instance_ids = fetch_distinct_gate_instances() | ||
for approval_gate_instance_id in approval_gate_instance_ids: | ||
connectors = fetch_distinct_connectors(approval_gate_instance_id[0]) | ||
param_group_id = 1 | ||
for connector in connectors: | ||
update_param_group_id(param_group_id, connector[0], approval_gate_instance_id[0]) | ||
param_group_id = param_group_id + 1 | ||
|
||
print("Migrating pipeline json") | ||
pipeline_jsons = fetch_pipeline_json() | ||
for pipeline_json in pipeline_jsons: | ||
migrate_pipeline_json(pipeline_json[0], pipeline_json[1]) | ||
print("Successfully migrated to v3.9") | ||
|
||
platform_conn.commit() | ||
visibility_conn.commit() | ||
except Exception as e: | ||
platform_conn.rollback() | ||
visibility_conn.rollback() | ||
print ('Exception occured during migration : ', e) | ||
finally: | ||
platform_conn.close() | ||
visibility_conn.close() | ||
|
||
|
||
def alter_approval_gate_parameter(): | ||
try: | ||
cur = visibility_conn.cursor() | ||
cur.execute("ALTER TABLE approval_gate_parameter ADD COLUMN IF NOT EXISTS param_group_id INT NOT NULL DEFAULT 0") | ||
except Exception as e: | ||
print("Exception occured while altering the approval_gate_parameter table : ", e) | ||
raise e | ||
|
||
|
||
def fetch_distinct_gate_instances(): | ||
try: | ||
cur = visibility_conn.cursor() | ||
cur.execute("select distinct(approval_gate_instance_id) from approval_gate_parameter ORDER BY approval_gate_instance_id") | ||
return cur.fetchall() | ||
except Exception as e: | ||
print("Exception occured while fetching distinct gate instances : ", e) | ||
raise e | ||
|
||
|
||
def fetch_distinct_connectors(gate_instance_id): | ||
try: | ||
cur = visibility_conn.cursor() | ||
cur.execute("select distinct(connector_type) from approval_gate_parameter where approval_gate_instance_id = "+str(gate_instance_id)) | ||
return cur.fetchall() | ||
except Exception as e: | ||
print("Exception occured while fetching distinct connectors : ", e) | ||
raise e | ||
|
||
|
||
def update_param_group_id(param_group_id, connector_type, approval_gate_instance_id): | ||
try: | ||
cur = visibility_conn.cursor() | ||
cur.execute("update approval_gate_parameter set param_group_id = " + str(param_group_id) + " where connector_type = '" + str(connector_type) + "' AND approval_gate_instance_id = "+str(approval_gate_instance_id)) | ||
except Exception as e: | ||
print("Exception occured while updating param group id : ", e) | ||
raise e | ||
|
||
|
||
def fetch_pipeline_json(): | ||
try: | ||
cur = platform_conn.cursor() | ||
cur.execute("select id, pipeline_json from pipeline") | ||
return cur.fetchall() | ||
except Exception as e: | ||
print("Exception occured while fetching pipeline jsons : ", e) | ||
raise e | ||
|
||
|
||
def migrate_pipeline_json(id, pipeline_json): | ||
try: | ||
cur = platform_conn.cursor() | ||
pipe_json = json.loads(pipeline_json) | ||
stages = pipe_json['stages'] | ||
if stages is not None and len(stages)>0: | ||
for stage in stages: | ||
gate_type = stage['type'] | ||
if gate_type is not None and len(gate_type)>0: | ||
if str(gate_type).strip() == 'Visibility Approval': | ||
migrate_approval_gate(stage) | ||
elif str(gate_type).strip() == 'Verification': | ||
migrate_verification_gate(stage) | ||
elif str(gate_type).strip() == 'Test Verification': | ||
migrate_test_verification_gate(stage) | ||
elif str(gate_type).strip() == 'Policy Stage': | ||
migrate_policy_gate(stage) | ||
pipe_json['stages'] = stages | ||
cur.execute("update pipeline set pipeline_json = '"+str(json.dumps(pipe_json))+"' where id="+str(id)) | ||
|
||
except Exception as e: | ||
print("Exception occured while migrating pipeline json : ", e) | ||
raise e | ||
|
||
|
||
def migrate_policy_gate(stage): | ||
del stage['alias'] | ||
stage['type'] = 'policy' | ||
|
||
|
||
def migrate_test_verification_gate(stage): | ||
del stage['alias'] | ||
del stage['comments'] | ||
stage['type'] = 'testVerification' | ||
|
||
|
||
def migrate_verification_gate(stage): | ||
del stage['alias'] | ||
del stage['comments'] | ||
stage['type'] = 'verification' | ||
|
||
|
||
def migrate_approval_gate(stage): | ||
try: | ||
del stage['alias'] | ||
stage['type'] = 'approval' | ||
parameters = stage['parameters'] | ||
if parameters is not None and len(parameters) > 0: | ||
canaryid = parameters['canaryid'] | ||
gitrepo = parameters['gitrepo'] | ||
jenkinsartifact = parameters['jenkinsartifact'] | ||
jenkinsbuild = parameters['jenkinsbuild'] | ||
appscanid = parameters['appscanid'] | ||
aquawave = parameters['aquawave'] | ||
jenkinsjob = parameters['jenkinsjob'] | ||
projectkey = parameters['projectkey'] | ||
connector = parameters['connector'] | ||
jiraid = parameters['jiraid'] | ||
header = parameters['header'] | ||
gitcommitid = parameters['gitcommitid'] | ||
connectordata = parameters['connectordata'] | ||
imageids = parameters['imageids'] | ||
gateurl = parameters['gateurl'] | ||
|
||
connectors = [] | ||
if gateurl is not None and len(gateurl)>0: | ||
url_components = str(gateurl).split("/") | ||
approval_gate_id = url_components[len(url_components) - 2] | ||
configured_tool_connectors = fetch_configured_tool_connectors(approval_gate_id) | ||
for configured_tool_connector in configured_tool_connectors: | ||
connector_type = configured_tool_connector['connectorType'] | ||
if connector_type is not None: | ||
if str(connector_type).strip().upper() == 'JIRA': | ||
configure_jira(configured_tool_connector, jiraid) | ||
elif str(connector_type).strip().upper() == 'GIT': | ||
configure_git(configured_tool_connector, gitcommitid, gitrepo) | ||
elif str(connector_type).strip().upper() == 'JENKINS': | ||
configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob) | ||
elif str(connector_type).strip().upper() == 'AUTOPILOT': | ||
configure_autopilot(canaryid, configured_tool_connector) | ||
elif str(connector_type).strip().upper() == 'SONARQUBE': | ||
configure_sonarqube(configured_tool_connector, projectkey) | ||
elif str(connector_type).strip().upper() == 'APPSCAN': | ||
configure_appscan(appscanid, configured_tool_connector) | ||
elif str(connector_type).strip().upper() == 'AQUAWAVE': | ||
configure_aquawave(aquawave, configured_tool_connector) | ||
connectors.append(configured_tool_connector) | ||
gateurl = str(gateurl).strip().replace("v4", "v5") | ||
gateurl = str(gateurl).strip().replace("v3", "v5") | ||
gateurl = str(gateurl).strip().replace("v2", "v5") | ||
gateurl = str(gateurl).strip().replace("v1", "v5") | ||
new_parameters = {"imageIds": imageids, | ||
"connectors": connectors, | ||
"gateUrl": gateurl} | ||
stage['parameters'] = new_parameters | ||
|
||
except Exception as e: | ||
print("Exception occured while migrating approval gate : ", e) | ||
raise e | ||
|
||
|
||
def configure_aquawave(aquawave, configured_tool_connector): | ||
aqua = [] | ||
param = {"imageId": aquawave} | ||
aqua.append(param) | ||
configured_tool_connector['values'] = aqua | ||
|
||
|
||
def configure_appscan(appscanid, configured_tool_connector): | ||
appscan = [] | ||
param = {"id": appscanid} | ||
appscan.append(param) | ||
configured_tool_connector['values'] = appscan | ||
|
||
|
||
def configure_sonarqube(configured_tool_connector, projectkey): | ||
sonarqube = [] | ||
param = {"projectKey": projectkey} | ||
sonarqube.append(param) | ||
configured_tool_connector['values'] = sonarqube | ||
|
||
|
||
def configure_autopilot(canaryid, configured_tool_connector): | ||
autopilot = [] | ||
param = {"canaryId": canaryid} | ||
autopilot.append(param) | ||
configured_tool_connector['values'] = autopilot | ||
|
||
|
||
def configure_jenkins(configured_tool_connector, jenkinsartifact, jenkinsbuild, jenkinsjob): | ||
jenkins = [] | ||
param = {"job": jenkinsjob, | ||
"buildId": jenkinsbuild, | ||
"artifact": jenkinsartifact} | ||
jenkins.append(param) | ||
configured_tool_connector['values'] = jenkins | ||
|
||
|
||
def configure_git(configured_tool_connector, gitcommitid, gitrepo): | ||
git = [] | ||
param = {"repo": gitrepo, | ||
"commitId": gitcommitid} | ||
git.append(param) | ||
configured_tool_connector['values'] = git | ||
|
||
|
||
def configure_jira(configured_tool_connector, jiraid): | ||
jira_tickets = [] | ||
if jiraid is not None and len(jiraid) > 0: | ||
jiraids = str(jiraid).split(",") | ||
for id in jiraids: | ||
param = {"jira_ticket_no": id} | ||
jira_tickets.append(param) | ||
configured_tool_connector['values'] = jira_tickets | ||
|
||
|
||
def fetch_configured_tool_connectors(approval_gate_id): | ||
try: | ||
url = oes_visibility_url + "/visibilityservice/v1/approvalGates/"+str(approval_gate_id)+"/configuredtoolConnectors" | ||
headers = {'x-spinnaker-user': oes_admin_user} | ||
response = requests.get(url=url, headers=headers) | ||
return json.loads(response.content) | ||
except Exception as e: | ||
print("Exception occured while fetching configured tool connectors : ", e) | ||
raise e | ||
|
||
def modifyUserGroupTable(): | ||
try: | ||
cur = platform_conn.cursor() | ||
cur.execute("""ALTER TABLE user_group ADD COLUMN IF NOT EXISTS "is_admin" BOOLEAN NOT NULL DEFAULT FALSE""" | ||
) | ||
except Exception as e: | ||
print ('Exception occured while modifying user_group table: ', | ||
e) | ||
raise e | ||
|
||
|
||
def map_user_group_admin(group_name): | ||
try: | ||
cur = platform_conn.cursor() | ||
cur.execute("SELECT id FROM user_group Where name='" | ||
+ group_name + "'") | ||
user_group_id = cur.fetchone() | ||
update_user_group(user_group_id) | ||
except Exception as e: | ||
print ('Exception occured while fetching data from user_group table: ' | ||
, e) | ||
raise e | ||
|
||
|
||
def update_user_group(user_group_id): | ||
try: | ||
cur = platform_conn.cursor() | ||
cur.execute('UPDATE user_group SET is_admin = true WHERE id = '+str(user_group_id[0])) | ||
except Exception as e: | ||
print ('Exception occured while updating user_group table: ', e) | ||
raise e | ||
|
||
if __name__ == '__main__': | ||
n = len(sys.argv) | ||
if n != 8: | ||
print ('Please pass valid 7 arguments <platform-db-name> <platform-db-host> <visibility-db-name> <visibility-db-host> <db-port> <oes-visibility-url> <oes-admin-user>') | ||
|
||
platform_db = sys.argv[1] | ||
platform_host = sys.argv[2] | ||
visibility_db = sys.argv[3] | ||
visibility_host = sys.argv[4] | ||
port = sys.argv[5] | ||
oes_visibility_url = sys.argv[6] | ||
oes_admin_user = sys.argv[7] | ||
|
||
# Establishing the platform db connection | ||
|
||
platform_conn = psycopg2.connect(database=platform_db, | ||
user='postgres', password='networks123', | ||
host=platform_host, port=port) | ||
print ('Opened platform database connection successfully') | ||
|
||
visibility_conn = psycopg2.connect(database=visibility_db, user='postgres', password='networks123', host=visibility_host, port=port) | ||
print("Visibility database connection established successfully") | ||
|
||
# Getting input of admin groups | ||
|
||
input_string = \ | ||
input('Enter admin groups to be configure in separated by comma:: ' | ||
) | ||
perform_migration(input_string) |
10 changes: 10 additions & 0 deletions
10
scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.8.x_to_v3.9.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
Steps to migrate from v3.8.x to v3.9 | ||
|
||
1. Copy the file : migration_v3.8.x_to_v3.9.py to some path in platform pod. | ||
2. Run the below command to execute the script. | ||
|
||
python3 <scriptfilename> <platform-db-name> <platform-db-host> <visibility-db-name> <visibility-db-host> <db-port> <oes-visibility-url> <oes-admin-user> | ||
|
||
Replace <scriptfilename> <platform-db-name> <platform-db-host> <visibility-db-name> <visibility-db-host> <db-port> <oes-visibility-url> <oes-admin-user> with valid values | ||
|
||
Ex: python3 migration_v3.8.x_to_v3.9.py platformdb oes-db visibilitydb oes-db 5432 http://oes-visibility:8096 admin |