From 3f8a75b6271dfe4316e576070b6a2a65714afeb6 Mon Sep 17 00:00:00 2001 From: ramya Date: Fri, 19 Aug 2022 18:34:22 +0530 Subject: [PATCH 1/3] worked on script enchancement --- .../migration_v3.11.x_to_v3.12.py | 55 ++++++++++++------- ...steps_to_migrate_from_v3.11.x_to_v3.12.txt | 4 +- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py index 84bdecaa..2f12b7bb 100644 --- a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py +++ b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py @@ -4,9 +4,15 @@ import sys import shlex import subprocess +from itertools import repeat + import psycopg2 import requests from psycopg2.extras import execute_values +from concurrent.futures import ThreadPoolExecutor +from time import sleep + + class bcolors: @@ -23,9 +29,7 @@ class bcolors: def perform_migration(): global is_error_occurred - gate_pipeline_maps = None - try: try: drop_table_user_group_permission_3_11() @@ -270,7 +274,7 @@ def update_spinnaker_gate_url(): def get_gate_pipeline_map(): try: - platform_cursor.execute("SELECT pipeline_id, service_gate_id FROM gate_pipeline_map") + platform_cursor.execute("SELECT pipeline_id, service_gate_id FROM gate_pipeline_map ") return platform_cursor.fetchall() except Exception as e: platform_conn.rollback() @@ -314,7 +318,7 @@ def update_pipeline_json_with_unified_url(pipelines): domain_name = host_url_comps[2] else: domain_name = host_url_comps[2] + "/" + host_url_comps[3] - + url_comps = str(gate_url).split("/") url_comps[2] = domain_name gate_url = "/" @@ -341,19 +345,28 @@ def update_pipeline_json_with_unified_url(pipelines): def update_pipeline_with_unified_url(gate_pipeline_maps, cookie): try: - for gate_pipeline_map in gate_pipeline_maps: - pipeline_id = gate_pipeline_map[0] - gate_id = gate_pipeline_map[1] - url = host_url + "/dashboardservice/v4/pipelines/" + str(pipeline_id) + "/gates/" + str(gate_id) - headers = {'Cookie': cookie} - response = requests.get(url=url, headers=headers) - gate_response = json.loads(response.content) - logging.info("Gate data from dashboard api : \n" + str(gate_response) + '\n') - update_gate(gate_response, pipeline_id, gate_id, cookie) + with ThreadPoolExecutor(max_workers = thread_pool_size) as executor: + executor.map(update_gate_information,repeat(cookie), gate_pipeline_maps) except Exception as e: raise e +def update_gate_information(cookie, gate_pipeline_map): + try: + sleep(2) + pipeline_id = gate_pipeline_map[0] + gate_id = gate_pipeline_map[1] + url = host_url + "/dashboardservice/v4/pipelines/" + str(pipeline_id) + "/gates/" + str(gate_id) + headers = {'Cookie': cookie} + response = requests.get(url=url, headers=headers) + gate_response = json.loads(response.content) + logging.info("Gate data from dashboard api : \n" + str(gate_response) + '\n') + update_gate(gate_response, pipeline_id, gate_id, cookie) + except Exception as e: + logging.error("Exception occurred while updating gate information : ", exc_info=True) + raise e + + def update_gate(gate_response, pipeline_id, gate_id, cookie): try: platform_cursor.execute("SELECT service_id FROM service_pipeline_map where pipeline_id = " + str(pipeline_id)) @@ -716,8 +729,10 @@ def updateApprovalAuditJson(audit_events_table_id, updateJson): def drop_constraints_feature_permission(): try: - platform_cursor.execute(" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fkdap3k7dwyp8yq5sn0kjf6eo42 ") - platform_cursor.execute(" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fklantt6pc0wjwueula2lt5vmt8 ") + platform_cursor.execute( + " ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fkdap3k7dwyp8yq5sn0kjf6eo42 ") + platform_cursor.execute( + " ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fklantt6pc0wjwueula2lt5vmt8 ") except Exception as e: platform_conn.rollback() raise e @@ -741,15 +756,14 @@ def login_to_isd(): raise e - if __name__ == '__main__': n = len(sys.argv) - if n != 19: + if n != 20: print( - "Please pass valid 18 arguments " + "Please pass valid 19 arguments " " " - " ") + " ") exit(1) global is_error_occurred @@ -777,6 +791,7 @@ def login_to_isd(): isd_admin_password = sys.argv[16] db_username = sys.argv[17] db_password = sys.argv[18] + thread_pool_size = sys.argv[19] # Establishing the platform db connection platform_conn = psycopg2.connect(database=platform_db, user=db_username, password=db_password, host=platform_host, @@ -820,4 +835,4 @@ def login_to_isd(): visibility_cursor = visibility_conn.cursor() - perform_migration() \ No newline at end of file + perform_migration() diff --git a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt index c39d646a..42d7ce9b 100644 --- a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt +++ b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt @@ -3,7 +3,7 @@ Steps to migrate from v3.11.x to v3.12 1. Copy the file : migration_v3.11.x_to_v3.12.py file to platform pod 2. Run the below command to execute the script. - python3 + python3 - ex: python3 migration_v3.11_to_v3.12.py platformdb oes-db oesdb oes-db opsmx oes-db auditdb oes-db visibilitydb oes-db 5432 https://isd-gate.dev.opsmx.net http://sapor-gate:8084 https://upgrade-ui.isd-dev.opsmx.net/gate isd-admin-username isd-admin-password dbusername dbpassword + ex: python3 migration_v3.11_to_v3.12.py platformdb oes-db oesdb oes-db opsmx oes-db auditdb oes-db visibilitydb oes-db 5432 https://isd-gate.dev.opsmx.net http://sapor-gate:8084 https://upgrade-ui.isd-dev.opsmx.net/gate isd-admin-username isd-admin-password dbusername dbpassword 20 From ef2fbd71f5df11cc5d75c395a76030bc0b0c4d43 Mon Sep 17 00:00:00 2001 From: ramya Date: Fri, 19 Aug 2022 18:44:39 +0530 Subject: [PATCH 2/3] worked on script enchancement --- scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py index 2f12b7bb..0d744c90 100644 --- a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py +++ b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py @@ -353,7 +353,6 @@ def update_pipeline_with_unified_url(gate_pipeline_maps, cookie): def update_gate_information(cookie, gate_pipeline_map): try: - sleep(2) pipeline_id = gate_pipeline_map[0] gate_id = gate_pipeline_map[1] url = host_url + "/dashboardservice/v4/pipelines/" + str(pipeline_id) + "/gates/" + str(gate_id) From 10664f95ec7be9adf720837512744dd9afa1a5bc Mon Sep 17 00:00:00 2001 From: ramya Date: Fri, 19 Aug 2022 21:26:39 +0530 Subject: [PATCH 3/3] worked on script enchancement phase1 --- .../migration_v3.11.x_to_v3.12.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py index 0d744c90..ff86f5c9 100644 --- a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py +++ b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py @@ -280,6 +280,14 @@ def get_gate_pipeline_map(): platform_conn.rollback() raise e +def get_service_pipeline_data(): + try: + platform_cursor.execute("SELECT service_id,pipeline_id FROM service_pipeline_map ") + return platform_cursor.fetchall() + except Exception as e: + platform_conn.rollback() + raise e + def get_pipelines(gate_pipeline_maps): try: @@ -345,13 +353,15 @@ def update_pipeline_json_with_unified_url(pipelines): def update_pipeline_with_unified_url(gate_pipeline_maps, cookie): try: + serviceIds_pipeline_data = get_service_pipeline_data(); + serviceIds_pipeline_map = dict((y, x) for x, y in serviceIds_pipeline_data) with ThreadPoolExecutor(max_workers = thread_pool_size) as executor: - executor.map(update_gate_information,repeat(cookie), gate_pipeline_maps) + executor.map(update_gate_information,repeat(cookie),repeat(serviceIds_pipeline_map),gate_pipeline_maps) except Exception as e: raise e -def update_gate_information(cookie, gate_pipeline_map): +def update_gate_information(cookie,serviceIds_pipeline_map,gate_pipeline_map): try: pipeline_id = gate_pipeline_map[0] gate_id = gate_pipeline_map[1] @@ -360,16 +370,16 @@ def update_gate_information(cookie, gate_pipeline_map): response = requests.get(url=url, headers=headers) gate_response = json.loads(response.content) logging.info("Gate data from dashboard api : \n" + str(gate_response) + '\n') - update_gate(gate_response, pipeline_id, gate_id, cookie) + update_gate(gate_response, pipeline_id, gate_id, cookie,serviceIds_pipeline_map) except Exception as e: logging.error("Exception occurred while updating gate information : ", exc_info=True) raise e -def update_gate(gate_response, pipeline_id, gate_id, cookie): +def update_gate(gate_response, pipeline_id, gate_id, cookie,serviceIds_pipeline_map): try: - platform_cursor.execute("SELECT service_id FROM service_pipeline_map where pipeline_id = " + str(pipeline_id)) - service_id = platform_cursor.fetchone()[0] + if list(serviceIds_pipeline_map.keys())[list(serviceIds_pipeline_map.values()).index(pipeline_id)] is not None: + service_id = list(serviceIds_pipeline_map.keys())[list(serviceIds_pipeline_map.values()).index(pipeline_id)] try: if gate_response["dependsOn"] is None: gate_response["dependsOn"] = [] @@ -435,7 +445,7 @@ def update_gate(gate_response, pipeline_id, gate_id, cookie): requests.put(url=url, data=req_payload, headers=headers) logging.info("Updated the gate data : \n" + str(req_payload) + '\n') except Exception as e: - platform_conn.rollback() + logging.error("Exception occurred while updating gate information with dashboard api : ", exc_info=True) raise e @@ -790,7 +800,7 @@ def login_to_isd(): isd_admin_password = sys.argv[16] db_username = sys.argv[17] db_password = sys.argv[18] - thread_pool_size = sys.argv[19] + thread_pool_size = int(sys.argv[19]) # Establishing the platform db connection platform_conn = psycopg2.connect(database=platform_db, user=db_username, password=db_password, host=platform_host,