From 5cba3f6a91290909e557ef196c7706ba77851ffd Mon Sep 17 00:00:00 2001 From: Ashley Date: Fri, 6 Dec 2024 03:08:13 +0000 Subject: [PATCH 1/8] Update: management scripts for clean and export now can use env vars --- .../management/commands/import_lotus_notes.py | 5 +++-- .../management/commands/ml_migration_script.py | 3 +++ mooringlicensing/utils/export_clean.py | 9 +++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mooringlicensing/management/commands/import_lotus_notes.py b/mooringlicensing/management/commands/import_lotus_notes.py index d7242e3e..4df08ea8 100755 --- a/mooringlicensing/management/commands/import_lotus_notes.py +++ b/mooringlicensing/management/commands/import_lotus_notes.py @@ -1,6 +1,7 @@ from django.core.management.base import BaseCommand import logging from mooringlicensing.utils.export_clean import clean +from confy import env logger = logging.getLogger(__name__) @@ -10,6 +11,6 @@ class Command(BaseCommand): help = 'Run Mooring Licensing Export to Mooring Booking Cron tasks' def handle(self, *args, **options): - src_path = '/data/LotusNotes/2024_02_06' - out_path = src_path + '/clean' + src_path = env('LOTUS_NOTES_PATH', '/data/data/projects/mooringlicensing/tmp/ml_export') + out_path = env('MIGRATION_DATA_PATH', '/data/data/projects/mooringlicensing/tmp/clean') clean(srcpath=src_path, outpath=out_path) \ No newline at end of file diff --git a/mooringlicensing/management/commands/ml_migration_script.py b/mooringlicensing/management/commands/ml_migration_script.py index 0662783c..cce02901 100644 --- a/mooringlicensing/management/commands/ml_migration_script.py +++ b/mooringlicensing/management/commands/ml_migration_script.py @@ -2,6 +2,7 @@ from django.conf import settings from mooringlicensing.utils.mooring_licence_migrate_pd import MooringLicenceReader import time +from confy import env import logging logger = logging.getLogger(__name__) @@ -16,6 +17,8 @@ def add_arguments(self, parser): def handle(self, *args, **options): path = options['path'] + if not path: + path = env('MIGRATION_DATA_PATH', '/data/data/projects/mooringlicensing/tmp/clean') t_start = time.time() mlr=MooringLicenceReader('PersonDets.txt', 'MooringDets.txt', 'VesselDets.txt', 'UserDets.txt', 'ApplicationDets.txt','annual_admissions_booking_report.csv', path=path) diff --git a/mooringlicensing/utils/export_clean.py b/mooringlicensing/utils/export_clean.py index 90910603..948668ce 100644 --- a/mooringlicensing/utils/export_clean.py +++ b/mooringlicensing/utils/export_clean.py @@ -2,6 +2,7 @@ import os import logging import re +from confy import env logger = logging.getLogger(__name__) @@ -51,10 +52,10 @@ def clean(srcpath='/data/data/projects/mooringlicensing/tmp/ml_export', outpath= line_col_count = temp.count('-!-') if line_col_count > col_count: - print("\n___Line",line_number,"has the wrong number of -!- -", line_col_count, "when it should be",col_count,"___") - print("\n",header_line,"\n") - print(temp, "\n") - print("__________________________________________________\n") + logger.error("\n___Line",line_number,"has the wrong number of -!- -", line_col_count, "when it should be",col_count,"___") + logger.error("\n",header_line,"\n") + logger.error(temp, "\n") + logger.error("__________________________________________________\n") wr.writerow([temp]) except Exception as e: From 41ef3bf56fa28bd424bd843d1a07b6f8b3647e53 Mon Sep 17 00:00:00 2001 From: Ashley Date: Fri, 6 Dec 2024 05:10:55 +0000 Subject: [PATCH 2/8] Added: summary text file output for migrations --- .../commands/ml_migration_script.py | 3 + .../utils/mooring_licence_migrate_pd.py | 191 +++++++++++++++++- 2 files changed, 193 insertions(+), 1 deletion(-) diff --git a/mooringlicensing/management/commands/ml_migration_script.py b/mooringlicensing/management/commands/ml_migration_script.py index cce02901..b57e9d24 100644 --- a/mooringlicensing/management/commands/ml_migration_script.py +++ b/mooringlicensing/management/commands/ml_migration_script.py @@ -40,3 +40,6 @@ def handle(self, *args, **options): t_end = time.time() logger.info('TIME TAKEN: {}'.format(t_end - t_start)) + f = open(mlr.summary_file, "a") + f.write("\n\nTotal time taken for migration: {}".format(t_end - t_start)) + f.close() \ No newline at end of file diff --git a/mooringlicensing/utils/mooring_licence_migrate_pd.py b/mooringlicensing/utils/mooring_licence_migrate_pd.py index d1c6884a..b1ed27c3 100644 --- a/mooringlicensing/utils/mooring_licence_migrate_pd.py +++ b/mooringlicensing/utils/mooring_licence_migrate_pd.py @@ -1,5 +1,6 @@ from ledger_api_client.ledger_models import EmailUserRO as EmailUser from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist +import time import datetime import pandas as pd import numpy as np @@ -286,6 +287,9 @@ class MooringLicenceReader(): """ def __init__(self, fname_user, fname_ml, fname_ves, fname_authuser, fname_wl, fname_aa, path='/data/data/projects/mooringlicensing/tmp/clean/'): + start_time = time.time() + start_datetime = datetime.datetime.now() + self.df_user=pd.read_csv(path+fname_user, delimiter='-!-', dtype=str) self.df_ml=pd.read_csv(path+fname_ml, delimiter='-!-', dtype=str) self.df_ves=pd.read_csv(path+fname_ves, delimiter='-!-', dtype=str) @@ -322,6 +326,17 @@ def __init__(self, fname_user, fname_ml, fname_ves, fname_authuser, fname_wl, fn self.ml_user_not_found = [] self.ml_no_ves_rows = [] + #summary_file + self.summary_file = "migration_summary_" + str(start_datetime).replace(" ", "_").replace(".", ":") + ".txt" + end_time = time.time() + + #create summary file, add init time + f = open(self.summary_file, "w") + f.write("Mooring Licensing Migration Summary\n") + f.write("Initialisation started at:", start_datetime) + f.write("Time taken for initialistation: {}s".format(end_time-start_time)) + f.close() + def __get_phone_number(self, row): if 'work_number' in row and row.work_number: row.work_number.replace(' ', '') @@ -581,6 +596,9 @@ def create_system_user_address_dict(self, applicant): return residential_address_dict, postal_address_dict, use_for_postal def create_users(self): + + start_time = time.time() + logger.info('Creating DCV users ...') df = self.df_dcv.groupby('pers_no').first() self._create_users_df(df) @@ -598,6 +616,28 @@ def create_users(self): logger.info('Creating AA users ...') self._create_users_df_aa(self.df_aa) + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nMigrate Users Summary\n") + f.write("Migrate Users started at:", start_time,"\n") + + f.write(f'users created: {len(self.user_created)}') + f.write(f'users existing: {len(self.user_existing)}') + f.write(f'users errors: {len(self.user_errors)}') + f.write(f'system users created: {len(self.system_user_created)}') + f.write(f'system users existing: {len(self.system_user_existing)}') + f.write(f'no_email errors: {len(self.no_email)}') + f.write(f'users errors: {self.user_errors}') + f.write(f'no_email errors: {self.no_email}') + + if self.user_error_details: + f.write('\nusers error details\n') + for i in self.user_error_details: + f.write(i) + + f.write("\nTime taken for migrating users: {}s".format(end_time-start_time)) + f.close() + def _create_users_df(self, df): # Iterate through the dataframe and create non-existent users for index, row in tqdm(df.iterrows(), total=df.shape[0]): @@ -768,9 +808,22 @@ def _create_users_df_aa(self, df): print(i) def create_vessels(self): + start_time = time.time() self._create_vessels() self._create_vessels_wl() + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nMigrate Vessels Summary\n") + f.write("Migrate Vessels started at:", start_time,"\n") + + f.write(f'vessels created: {len(self.vessels_created)}') + f.write(f'vessels errors: {self.vessels_errors}') + f.write(f'vessels errors: {len(self.vessels_errors)}') + + f.write("\nTime taken for migrating vessels: {}s".format(end_time-start_time)) + f.close() + def _create_vessels(self): def _ves_fields(prefix, ves_row, pers_no): @@ -1056,6 +1109,7 @@ def ves_fields(ves_row): print(f'vessels_dict dict size: {len(self.vessels_dict)}') def create_mooring_licences(self): + start_time = time.time() expiry_date = EXPIRY_DATE start_date = START_DATE date_applied = DATE_APPLIED @@ -1282,7 +1336,29 @@ def create_mooring_licences(self): print(f'Vessels not Found: {vessel_not_found}') print(f'Vessels not Found: {len(vessel_not_found)}') + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nMigrate Mooring Licenses Summary\n") + f.write("Migrate Mooring Licenses started at:", start_time,"\n") + + f.write(f'ml_user_not_found: {self.ml_user_not_found}') + f.write(f'Duplicate Pers_No in ves_row: {self.ml_no_ves_rows}') + f.write(f'ml_user_not_found: {len(self.ml_user_not_found)}') + f.write(f'Duplicate Pers_No in ves_row: {len(self.ml_no_ves_rows)}') + f.write(f'Moorings not Found: {mooring_not_found}') + f.write(f'Moorings not Found: {len(mooring_not_found)}') + f.write(f'Owners not Found: {owner_not_found}') + f.write(f'VesselOwnership not Found: {len(vessel_ownership_not_found)}') + f.write(f'VesselOwnership not Found: {vessel_ownership_not_found}') + f.write(f'Owners not Found: {len(owner_not_found)}') + f.write(f'Vessels not Found: {vessel_not_found}') + f.write(f'Vessels not Found: {len(vessel_not_found)}') + + f.write("\nTime taken for migrating mooring licenses: {}s".format(end_time-start_time)) + f.close() + def create_authuser_permits(self): + start_time = time.time() expiry_date = EXPIRY_DATE start_date = START_DATE date_applied = DATE_APPLIED @@ -1515,8 +1591,27 @@ def create_authuser_permits(self): print(f'no_au_stickers: no_au_stickers, {len(no_au_stickers)}') for i in errors: print(i) + + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nMigrate Authorised User Permits Summary\n") + f.write("Migrate Authorised User Permits started at:", start_time,"\n") + + f.write(f'vessel_not_found: {vessel_not_found}') + f.write(f'vessel_not_found: {len(vessel_not_found)}') + f.write(f'aup_created: {len(aup_created)}') + f.write(f'au_stickers: au_stickers, {len(au_stickers)}') + f.write(f'no_au_stickers: no_au_stickers, {len(no_au_stickers)}') + if errors: + f.write('\naup error details\n') + for i in errors: + f.write(i) + + f.write("\nTime taken for migrating authorised user permits: {}s".format(end_time-start_time)) + f.close() def create_waiting_list(self): + start_time = time.time() expiry_date = EXPIRY_DATE start_date = START_DATE date_applied = DATE_APPLIED @@ -1662,8 +1757,27 @@ def create_waiting_list(self): print(f'wl_created: {len(wl_created)}') for i in errors: print(i) + + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nMigrate Waiting List Allocations Summary\n") + f.write("Migrate Waiting List Allocations started at:", start_time,"\n") + + f.write(f'vessel_not_found: {vessel_not_found}') + f.write(f'vessel_not_found: {len(vessel_not_found)}') + f.write(f'user_not_found: {user_not_found}') + f.write(f'user_not_found: {len(user_not_found)}') + f.write(f'wl_created: {len(wl_created)}') + if errors: + f.write('\nwla error details\n') + for i in errors: + f.write(i) + + f.write("\nTime taken for migrating waiting list allocations: {}s".format(end_time-start_time)) + f.close() def create_dcv(self): + start_time = time.time() expiry_date = EXPIRY_DATE start_date = START_DATE fee_season = FeeSeason.objects.filter(application_type__code='dcvp', name=FEE_SEASON)[0] @@ -1773,6 +1887,24 @@ def create_dcv(self): print(f'dcv_created: {len(dcv_created)}') for i in errors: print(i) + + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nMigrate DCV Permits Summary\n") + f.write("Migrate DCV Permits started at:", start_time,"\n") + + f.write(f'vessel_not_found: {vessel_not_found}') + f.write(f'vessel_not_found: {len(vessel_not_found)}') + f.write(f'user_not_found: {user_not_found}') + f.write(f'user_not_found: {len(user_not_found)}') + f.write(f'dcv_created: {len(dcv_created)}') + if errors: + f.write('\ndcv error details\n') + for i in errors: + f.write(i) + + f.write("\nTime taken for migrating dcv permits: {}s".format(end_time-start_time)) + f.close() def _create_single_vessel(self, user, rego_no, ves_name, ves_type, length, draft, beam, weight, pct_interest, berth_mooring=''): @@ -1820,6 +1952,7 @@ def try_except(value): return vessel, vessel_details, vessel_ownership def create_annual_admissions(self): + start_time = time.time() """ mlr=MooringLicenceReader('PersonDets20221222-125823.txt', 'MooringDets20221222-125546.txt', 'VesselDets20221222-125823.txt', 'UserDets20221222-130353.txt', 'ApplicationDets20221222-125157.txt','annual_admissions_booking_report_20230125084027.csv') mlr.create_users() @@ -2016,7 +2149,23 @@ def create_annual_admissions(self): print(f'Vessel Details Not Found: {len(vessel_details_not_found)}') for i in errors: print(i) - + + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nMigrate Annual Admissions Summary\n") + f.write("Migrate Annual Admissions started at:", start_time,"\n") + + f.write(f'rego_aa_created: {len(rego_aa_created)}') + f.write(f'total_aa_created: {len(total_aa_created)}') + f.write(f'Vessel Details Not Found: {vessel_details_not_found}') + f.write(f'Vessel Details Not Found: {len(vessel_details_not_found)}') + if errors: + f.write('\naa error details\n') + for i in errors: + f.write(i) + + f.write("\nTime taken for migrating annual admissions: {}s".format(end_time-start_time)) + f.close() def create_licence_pdfs(self): self.create_pdf_ml() @@ -2029,32 +2178,72 @@ def create_licence_pdfs(self): def create_pdf_ml(self): """ MooringLicenceReader.create_pdf_ml() """ + start_time = time.time() self._create_pdf_licence(MooringLicence.objects.filter(migrated=True)) + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nCreate Mooring License PDFs Summary\n") + f.write("Create Mooring License PDFs started at:", start_time,"\n") + f.write("\nTime taken for creating mooring license pdfs: {}s".format(end_time-start_time)) + f.close() + @classmethod def create_pdf_aup(self): """ MooringLicenceReader.create_pdf_aup() """ + start_time = time.time() self._create_pdf_licence(AuthorisedUserPermit.objects.filter(migrated=True)) + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nCreate Authorised User Permit PDFs Summary\n") + f.write("Create Authorised User Permit PDFs started at:", start_time,"\n") + f.write("\nTime taken for creating authorised user permit pdfs: {}s".format(end_time-start_time)) + f.close() + @classmethod def create_pdf_wl(self): """ MooringLicenceReader.create_pdf_wl() """ + start_time = time.time() self._create_pdf_licence(WaitingListAllocation.objects.filter(migrated=True)) + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nCreate Waiting List Allocation PDFs Summary\n") + f.write("Create Waiting List Allocation PDFs started at:", start_time,"\n") + f.write("\nTime taken for creating waiting list allocation pdfs: {}s".format(end_time-start_time)) + f.close() + @classmethod def create_pdf_aa(self): """ MooringLicenceReader.create_pdf_aa() """ + start_time = time.time() self._create_pdf_licence(AnnualAdmissionPermit.objects.filter(migrated=True)) + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nCreate Annual Admission PDFs Summary\n") + f.write("Create Annual Admission PDFs started at:", start_time,"\n") + f.write("\nTime taken for creating annual admission pdfs: {}s".format(end_time-start_time)) + f.close() + @classmethod def create_pdf_dcv(self): """ MooringLicenceReader.create_pdf_dcv() """ + start_time = time.time() self._create_pdf_licence(DcvPermit.objects.filter(migrated=True)) + end_time = time.time() + f = open(self.summary_file, "a") + f.write("\nCreate DCV Permit PDFs Summary\n") + f.write("Create DCV Permit PDFs started at:", start_time,"\n") + f.write("\nTime taken for creating dcv permit pdfs: {}s".format(end_time-start_time)) + f.close() + @staticmethod def _create_pdf_licence(approvals_migrated): """ MooringLicenceReader._create_pdf_licence(MooringLicence.objects.filter(migrated=True), current_proposal__processing_status=Proposal.PROCESSING_STATUS_APPROVED) From 1dced96d2f0c645924277c9f2d7c5ab7fe69c94b Mon Sep 17 00:00:00 2001 From: Ashley Date: Fri, 6 Dec 2024 07:17:26 +0000 Subject: [PATCH 3/8] Added: migration (and re-added lotus notes import) script to management commands, adjusted summary txt output --- .../mooringlicensing/mgt-commands.html | 9 + .../commands/ml_migration_script.py | 2 +- .../utils/mooring_licence_migrate_pd.py | 174 +++++++++--------- 3 files changed, 97 insertions(+), 88 deletions(-) diff --git a/mooringlicensing/components/proposals/templates/mooringlicensing/mgt-commands.html b/mooringlicensing/components/proposals/templates/mooringlicensing/mgt-commands.html index 25bcaa08..8089c69d 100755 --- a/mooringlicensing/components/proposals/templates/mooringlicensing/mgt-commands.html +++ b/mooringlicensing/components/proposals/templates/mooringlicensing/mgt-commands.html @@ -19,6 +19,15 @@

Hover over button to see help text.

{% endif %} +
+ +
+ +
+ +
+ +
diff --git a/mooringlicensing/management/commands/ml_migration_script.py b/mooringlicensing/management/commands/ml_migration_script.py index b57e9d24..ae09a3a4 100644 --- a/mooringlicensing/management/commands/ml_migration_script.py +++ b/mooringlicensing/management/commands/ml_migration_script.py @@ -18,7 +18,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): path = options['path'] if not path: - path = env('MIGRATION_DATA_PATH', '/data/data/projects/mooringlicensing/tmp/clean') + path = env('MIGRATION_DATA_PATH', '/data/data/projects/mooringlicensing/tmp/clean/') t_start = time.time() mlr=MooringLicenceReader('PersonDets.txt', 'MooringDets.txt', 'VesselDets.txt', 'UserDets.txt', 'ApplicationDets.txt','annual_admissions_booking_report.csv', path=path) diff --git a/mooringlicensing/utils/mooring_licence_migrate_pd.py b/mooringlicensing/utils/mooring_licence_migrate_pd.py index b1ed27c3..82181fef 100644 --- a/mooringlicensing/utils/mooring_licence_migrate_pd.py +++ b/mooringlicensing/utils/mooring_licence_migrate_pd.py @@ -327,14 +327,14 @@ def __init__(self, fname_user, fname_ml, fname_ves, fname_authuser, fname_wl, fn self.ml_no_ves_rows = [] #summary_file - self.summary_file = "migration_summary_" + str(start_datetime).replace(" ", "_").replace(".", ":") + ".txt" + self.summary_file = path + "migration_summary_" + str(start_datetime).replace(" ", "_").replace(".", ":") + ".txt" end_time = time.time() #create summary file, add init time f = open(self.summary_file, "w") f.write("Mooring Licensing Migration Summary\n") - f.write("Initialisation started at:", start_datetime) - f.write("Time taken for initialistation: {}s".format(end_time-start_time)) + f.write("\nInitialisation started at: {}".format(start_datetime)) + f.write("\nTime taken for initialisation: {}s".format(end_time-start_time)) f.close() def __get_phone_number(self, row): @@ -618,24 +618,24 @@ def create_users(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nMigrate Users Summary\n") - f.write("Migrate Users started at:", start_time,"\n") - - f.write(f'users created: {len(self.user_created)}') - f.write(f'users existing: {len(self.user_existing)}') - f.write(f'users errors: {len(self.user_errors)}') - f.write(f'system users created: {len(self.system_user_created)}') - f.write(f'system users existing: {len(self.system_user_existing)}') - f.write(f'no_email errors: {len(self.no_email)}') - f.write(f'users errors: {self.user_errors}') - f.write(f'no_email errors: {self.no_email}') + f.write("\n\nMigrate Users Summary\n") + f.write("Migrate Users started at: {}\n".format(start_time)) + + f.write(f'\nusers created: {len(self.user_created)}') + f.write(f'\nusers existing: {len(self.user_existing)}') + f.write(f'\nusers errors: {len(self.user_errors)}') + f.write(f'\nsystem users created: {len(self.system_user_created)}') + f.write(f'\nsystem users existing: {len(self.system_user_existing)}') + f.write(f'\nno_email errors: {len(self.no_email)}') + f.write(f'\nusers errors: {self.user_errors}') + f.write(f'\nno_email errors: {self.no_email}') if self.user_error_details: f.write('\nusers error details\n') for i in self.user_error_details: - f.write(i) + f.write("\n"+i) - f.write("\nTime taken for migrating users: {}s".format(end_time-start_time)) + f.write("\n\nTime taken for migrating users: {}s".format(end_time-start_time)) f.close() def _create_users_df(self, df): @@ -814,14 +814,14 @@ def create_vessels(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nMigrate Vessels Summary\n") - f.write("Migrate Vessels started at:", start_time,"\n") + f.write("\n\nMigrate Vessels Summary\n") + f.write("Migrate Vessels started at: {}\n".format(start_time)) - f.write(f'vessels created: {len(self.vessels_created)}') - f.write(f'vessels errors: {self.vessels_errors}') - f.write(f'vessels errors: {len(self.vessels_errors)}') + f.write(f'\nvessels created: {len(self.vessels_created)}') + f.write(f'\nvessels errors: {self.vessels_errors}') + f.write(f'\nvessels errors: {len(self.vessels_errors)}') - f.write("\nTime taken for migrating vessels: {}s".format(end_time-start_time)) + f.write("\n\nTime taken for migrating vessels: {}s".format(end_time-start_time)) f.close() def _create_vessels(self): @@ -1338,23 +1338,23 @@ def create_mooring_licences(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nMigrate Mooring Licenses Summary\n") - f.write("Migrate Mooring Licenses started at:", start_time,"\n") - - f.write(f'ml_user_not_found: {self.ml_user_not_found}') - f.write(f'Duplicate Pers_No in ves_row: {self.ml_no_ves_rows}') - f.write(f'ml_user_not_found: {len(self.ml_user_not_found)}') - f.write(f'Duplicate Pers_No in ves_row: {len(self.ml_no_ves_rows)}') - f.write(f'Moorings not Found: {mooring_not_found}') - f.write(f'Moorings not Found: {len(mooring_not_found)}') - f.write(f'Owners not Found: {owner_not_found}') - f.write(f'VesselOwnership not Found: {len(vessel_ownership_not_found)}') - f.write(f'VesselOwnership not Found: {vessel_ownership_not_found}') - f.write(f'Owners not Found: {len(owner_not_found)}') - f.write(f'Vessels not Found: {vessel_not_found}') - f.write(f'Vessels not Found: {len(vessel_not_found)}') - - f.write("\nTime taken for migrating mooring licenses: {}s".format(end_time-start_time)) + f.write("\n\nMigrate Mooring Licenses Summary\n") + f.write("Migrate Mooring Licenses started at: {}\n".format(start_time)) + + f.write(f'\nml_user_not_found: {self.ml_user_not_found}') + f.write(f'\nDuplicate Pers_No in ves_row: {self.ml_no_ves_rows}') + f.write(f'\nml_user_not_found: {len(self.ml_user_not_found)}') + f.write(f'\nDuplicate Pers_No in ves_row: {len(self.ml_no_ves_rows)}') + f.write(f'\nMoorings not Found: {mooring_not_found}') + f.write(f'\nMoorings not Found: {len(mooring_not_found)}') + f.write(f'\nOwners not Found: {owner_not_found}') + f.write(f'\nVesselOwnership not Found: {len(vessel_ownership_not_found)}') + f.write(f'\nVesselOwnership not Found: {vessel_ownership_not_found}') + f.write(f'\nOwners not Found: {len(owner_not_found)}') + f.write(f'\nVessels not Found: {vessel_not_found}') + f.write(f'\nVessels not Found: {len(vessel_not_found)}') + + f.write("\n\nTime taken for migrating mooring licenses: {}s".format(end_time-start_time)) f.close() def create_authuser_permits(self): @@ -1594,20 +1594,20 @@ def create_authuser_permits(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nMigrate Authorised User Permits Summary\n") - f.write("Migrate Authorised User Permits started at:", start_time,"\n") - - f.write(f'vessel_not_found: {vessel_not_found}') - f.write(f'vessel_not_found: {len(vessel_not_found)}') - f.write(f'aup_created: {len(aup_created)}') - f.write(f'au_stickers: au_stickers, {len(au_stickers)}') - f.write(f'no_au_stickers: no_au_stickers, {len(no_au_stickers)}') + f.write("\n\nMigrate Authorised User Permits Summary\n") + f.write("Migrate Authorised User Permits started at: {}\n".format(start_time)) + + f.write(f'\nvessel_not_found: {vessel_not_found}') + f.write(f'\nvessel_not_found: {len(vessel_not_found)}') + f.write(f'\naup_created: {len(aup_created)}') + f.write(f'\nau_stickers: au_stickers, {len(au_stickers)}') + f.write(f'\nno_au_stickers: no_au_stickers, {len(no_au_stickers)}') if errors: f.write('\naup error details\n') for i in errors: - f.write(i) + f.write("\n"+i) - f.write("\nTime taken for migrating authorised user permits: {}s".format(end_time-start_time)) + f.write("\n\nTime taken for migrating authorised user permits: {}s".format(end_time-start_time)) f.close() def create_waiting_list(self): @@ -1760,20 +1760,20 @@ def create_waiting_list(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nMigrate Waiting List Allocations Summary\n") - f.write("Migrate Waiting List Allocations started at:", start_time,"\n") - - f.write(f'vessel_not_found: {vessel_not_found}') - f.write(f'vessel_not_found: {len(vessel_not_found)}') - f.write(f'user_not_found: {user_not_found}') - f.write(f'user_not_found: {len(user_not_found)}') - f.write(f'wl_created: {len(wl_created)}') + f.write("\n\nMigrate Waiting List Allocations Summary\n") + f.write("Migrate Waiting List Allocations started at: {}\n".format(start_time)) + + f.write(f'\nvessel_not_found: {vessel_not_found}') + f.write(f'\nvessel_not_found: {len(vessel_not_found)}') + f.write(f'\nuser_not_found: {user_not_found}') + f.write(f'\nuser_not_found: {len(user_not_found)}') + f.write(f'\nwl_created: {len(wl_created)}') if errors: f.write('\nwla error details\n') for i in errors: - f.write(i) + f.write("\n"+i) - f.write("\nTime taken for migrating waiting list allocations: {}s".format(end_time-start_time)) + f.write("\n\nTime taken for migrating waiting list allocations: {}s".format(end_time-start_time)) f.close() def create_dcv(self): @@ -1890,20 +1890,20 @@ def create_dcv(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nMigrate DCV Permits Summary\n") - f.write("Migrate DCV Permits started at:", start_time,"\n") - - f.write(f'vessel_not_found: {vessel_not_found}') - f.write(f'vessel_not_found: {len(vessel_not_found)}') - f.write(f'user_not_found: {user_not_found}') - f.write(f'user_not_found: {len(user_not_found)}') - f.write(f'dcv_created: {len(dcv_created)}') + f.write("\n\nMigrate DCV Permits Summary\n") + f.write("Migrate DCV Permits started at: {}\n".format(start_time)) + + f.write(f'\nvessel_not_found: {vessel_not_found}') + f.write(f'\nvessel_not_found: {len(vessel_not_found)}') + f.write(f'\nuser_not_found: {user_not_found}') + f.write(f'\nuser_not_found: {len(user_not_found)}') + f.write(f'\ndcv_created: {len(dcv_created)}') if errors: f.write('\ndcv error details\n') for i in errors: - f.write(i) + f.write("\n"+i) - f.write("\nTime taken for migrating dcv permits: {}s".format(end_time-start_time)) + f.write("\n\nTime taken for migrating dcv permits: {}s".format(end_time-start_time)) f.close() def _create_single_vessel(self, user, rego_no, ves_name, ves_type, length, draft, beam, weight, pct_interest, berth_mooring=''): @@ -2152,19 +2152,19 @@ def create_annual_admissions(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nMigrate Annual Admissions Summary\n") - f.write("Migrate Annual Admissions started at:", start_time,"\n") + f.write("\n\nMigrate Annual Admissions Summary\n") + f.write("Migrate Annual Admissions started at: {}\n".format(start_time)) - f.write(f'rego_aa_created: {len(rego_aa_created)}') - f.write(f'total_aa_created: {len(total_aa_created)}') - f.write(f'Vessel Details Not Found: {vessel_details_not_found}') - f.write(f'Vessel Details Not Found: {len(vessel_details_not_found)}') + f.write(f'\nrego_aa_created: {len(rego_aa_created)}') + f.write(f'\ntotal_aa_created: {len(total_aa_created)}') + f.write(f'\nVessel Details Not Found: {vessel_details_not_found}') + f.write(f'\nVessel Details Not Found: {len(vessel_details_not_found)}') if errors: f.write('\naa error details\n') for i in errors: - f.write(i) + f.write("\n"+i) - f.write("\nTime taken for migrating annual admissions: {}s".format(end_time-start_time)) + f.write("\n\nTime taken for migrating annual admissions: {}s".format(end_time-start_time)) f.close() def create_licence_pdfs(self): @@ -2183,8 +2183,8 @@ def create_pdf_ml(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nCreate Mooring License PDFs Summary\n") - f.write("Create Mooring License PDFs started at:", start_time,"\n") + f.write("\n\nCreate Mooring License PDFs Summary\n") + f.write("Create Mooring License PDFs started at: {}\n".format(start_time)) f.write("\nTime taken for creating mooring license pdfs: {}s".format(end_time-start_time)) f.close() @@ -2197,8 +2197,8 @@ def create_pdf_aup(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nCreate Authorised User Permit PDFs Summary\n") - f.write("Create Authorised User Permit PDFs started at:", start_time,"\n") + f.write("\n\nCreate Authorised User Permit PDFs Summary\n") + f.write("Create Authorised User Permit PDFs started at: {}\n".format(start_time)) f.write("\nTime taken for creating authorised user permit pdfs: {}s".format(end_time-start_time)) f.close() @@ -2211,8 +2211,8 @@ def create_pdf_wl(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nCreate Waiting List Allocation PDFs Summary\n") - f.write("Create Waiting List Allocation PDFs started at:", start_time,"\n") + f.write("\n\nCreate Waiting List Allocation PDFs Summary\n") + f.write("Create Waiting List Allocation PDFs started at: {}\n".format(start_time)) f.write("\nTime taken for creating waiting list allocation pdfs: {}s".format(end_time-start_time)) f.close() @@ -2225,8 +2225,8 @@ def create_pdf_aa(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nCreate Annual Admission PDFs Summary\n") - f.write("Create Annual Admission PDFs started at:", start_time,"\n") + f.write("\n\nCreate Annual Admission PDFs Summary\n") + f.write("Create Annual Admission PDFs started at: {}\n".format(start_time)) f.write("\nTime taken for creating annual admission pdfs: {}s".format(end_time-start_time)) f.close() @@ -2239,8 +2239,8 @@ def create_pdf_dcv(self): end_time = time.time() f = open(self.summary_file, "a") - f.write("\nCreate DCV Permit PDFs Summary\n") - f.write("Create DCV Permit PDFs started at:", start_time,"\n") + f.write("\n\nCreate DCV Permit PDFs Summary\n") + f.write("Create DCV Permit PDFs started at: {}\n".format(start_time)) f.write("\nTime taken for creating dcv permit pdfs: {}s".format(end_time-start_time)) f.close() From b290a166d3958d885f951d2196d12d4958d50eaa Mon Sep 17 00:00:00 2001 From: Ashley Date: Fri, 6 Dec 2024 07:55:42 +0000 Subject: [PATCH 4/8] Update: create pdf functions no longer class functions - they use the start time of the mlr for tracking --- mooringlicensing/components/proposals/models.py | 2 +- .../management/commands/ml_migration_script.py | 10 +++++----- .../utils/mooring_licence_migrate_pd.py | 15 +++++---------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/mooringlicensing/components/proposals/models.py b/mooringlicensing/components/proposals/models.py index da53a984..f25ce0a3 100644 --- a/mooringlicensing/components/proposals/models.py +++ b/mooringlicensing/components/proposals/models.py @@ -1747,7 +1747,7 @@ def final_approval_for_AUA_MLA(self, request=None): 'system': settings.PAYMENT_SYSTEM_ID, 'custom_basket': True, 'booking_reference': reference, - 'no_payment': True, + 'no_payment': True, #TODO remove this or determine why it is here 'tax_override': True, } logger.info(f'basket_params: {basket_params}') diff --git a/mooringlicensing/management/commands/ml_migration_script.py b/mooringlicensing/management/commands/ml_migration_script.py index ae09a3a4..857dc7d5 100644 --- a/mooringlicensing/management/commands/ml_migration_script.py +++ b/mooringlicensing/management/commands/ml_migration_script.py @@ -31,11 +31,11 @@ def handle(self, *args, **options): mlr.create_dcv() mlr.create_annual_admissions() - MooringLicenceReader.create_pdf_ml() - MooringLicenceReader.create_pdf_aup() - MooringLicenceReader.create_pdf_wl() - MooringLicenceReader.create_pdf_dcv() - MooringLicenceReader.create_pdf_aa() + mlr.create_pdf_ml() + mlr.create_pdf_aup() + mlr.create_pdf_wl() + mlr.create_pdf_dcv() + mlr.create_pdf_aa() t_end = time.time() logger.info('TIME TAKEN: {}'.format(t_end - t_start)) diff --git a/mooringlicensing/utils/mooring_licence_migrate_pd.py b/mooringlicensing/utils/mooring_licence_migrate_pd.py index 82181fef..9b5929c2 100644 --- a/mooringlicensing/utils/mooring_licence_migrate_pd.py +++ b/mooringlicensing/utils/mooring_licence_migrate_pd.py @@ -631,7 +631,7 @@ def create_users(self): f.write(f'\nno_email errors: {self.no_email}') if self.user_error_details: - f.write('\nusers error details\n') + f.write('\n\nusers error details\n') for i in self.user_error_details: f.write("\n"+i) @@ -1603,7 +1603,7 @@ def create_authuser_permits(self): f.write(f'\nau_stickers: au_stickers, {len(au_stickers)}') f.write(f'\nno_au_stickers: no_au_stickers, {len(no_au_stickers)}') if errors: - f.write('\naup error details\n') + f.write('\n\naup error details\n') for i in errors: f.write("\n"+i) @@ -1769,7 +1769,7 @@ def create_waiting_list(self): f.write(f'\nuser_not_found: {len(user_not_found)}') f.write(f'\nwl_created: {len(wl_created)}') if errors: - f.write('\nwla error details\n') + f.write('\n\nwla error details\n') for i in errors: f.write("\n"+i) @@ -1899,7 +1899,7 @@ def create_dcv(self): f.write(f'\nuser_not_found: {len(user_not_found)}') f.write(f'\ndcv_created: {len(dcv_created)}') if errors: - f.write('\ndcv error details\n') + f.write('\n\ndcv error details\n') for i in errors: f.write("\n"+i) @@ -2160,7 +2160,7 @@ def create_annual_admissions(self): f.write(f'\nVessel Details Not Found: {vessel_details_not_found}') f.write(f'\nVessel Details Not Found: {len(vessel_details_not_found)}') if errors: - f.write('\naa error details\n') + f.write('\n\naa error details\n') for i in errors: f.write("\n"+i) @@ -2174,7 +2174,6 @@ def create_licence_pdfs(self): self.create_pdf_aa() self.create_pdf_dcv() - @classmethod def create_pdf_ml(self): """ MooringLicenceReader.create_pdf_ml() """ @@ -2188,7 +2187,6 @@ def create_pdf_ml(self): f.write("\nTime taken for creating mooring license pdfs: {}s".format(end_time-start_time)) f.close() - @classmethod def create_pdf_aup(self): """ MooringLicenceReader.create_pdf_aup() """ @@ -2202,7 +2200,6 @@ def create_pdf_aup(self): f.write("\nTime taken for creating authorised user permit pdfs: {}s".format(end_time-start_time)) f.close() - @classmethod def create_pdf_wl(self): """ MooringLicenceReader.create_pdf_wl() """ @@ -2216,7 +2213,6 @@ def create_pdf_wl(self): f.write("\nTime taken for creating waiting list allocation pdfs: {}s".format(end_time-start_time)) f.close() - @classmethod def create_pdf_aa(self): """ MooringLicenceReader.create_pdf_aa() """ @@ -2230,7 +2226,6 @@ def create_pdf_aa(self): f.write("\nTime taken for creating annual admission pdfs: {}s".format(end_time-start_time)) f.close() - @classmethod def create_pdf_dcv(self): """ MooringLicenceReader.create_pdf_dcv() """ From 0d458915c0d2a7425a82c9860630891d74e0a372 Mon Sep 17 00:00:00 2001 From: Ashley Date: Mon, 9 Dec 2024 04:28:03 +0000 Subject: [PATCH 5/8] Added: no email notification checkbox and model field for proposal (can only be changed internally) --- mooringlicensing/components/approvals/api.py | 2 ++ mooringlicensing/components/proposals/api.py | 24 ++++++++++++------- .../components/proposals/models.py | 2 ++ .../components/proposals/serializers.py | 1 + .../components/proposals/utils.py | 7 ++++++ .../src/components/external/proposal.vue | 2 +- .../components/external/proposal_apply.vue | 20 ++++++++++------ .../approvals/offer_mooring_licence.vue | 9 ++++++- .../internal/proposals/proposal.vue | 5 ++++ .../0378_proposal_no_email_notifications.py | 18 ++++++++++++++ 10 files changed, 72 insertions(+), 18 deletions(-) create mode 100644 mooringlicensing/migrations/0378_proposal_no_email_notifications.py diff --git a/mooringlicensing/components/approvals/api.py b/mooringlicensing/components/approvals/api.py index a2ecee70..3f38201d 100755 --- a/mooringlicensing/components/approvals/api.py +++ b/mooringlicensing/components/approvals/api.py @@ -1815,6 +1815,7 @@ def create_mooring_licence_application(self, request, *args, **kwargs): waiting_list_allocation = self.get_object() proposal_type = ProposalType.objects.get(code=PROPOSAL_TYPE_NEW) selected_mooring_id = request.data.get("selected_mooring_id") + no_email_notifications = request.data.get("no_emails") allocated_mooring = Mooring.objects.get(id=selected_mooring_id) current_date = datetime.now(pytz.timezone(TIME_ZONE)).date() @@ -1839,6 +1840,7 @@ def create_mooring_licence_application(self, request, *args, **kwargs): allocated_mooring=allocated_mooring, waiting_list_allocation=waiting_list_allocation, date_invited=current_date, + no_email_notifications=no_email_notifications, ) waiting_list_allocation.proposal_applicant.copy_self_to_proposal(new_proposal) logger.info(f'Offering new Mooring Site Licence application: [{new_proposal}], which has been created from the waiting list allocation: [{waiting_list_allocation}].') diff --git a/mooringlicensing/components/proposals/api.py b/mooringlicensing/components/proposals/api.py index d4856b77..9f46628e 100755 --- a/mooringlicensing/components/proposals/api.py +++ b/mooringlicensing/components/proposals/api.py @@ -735,6 +735,7 @@ def create(self, request, *args, **kwargs): if is_internal(request): applicant_system_id = request.data.get("applicant_system_id") + no_email_notifications = request.data.get("no_emails", False) if not applicant_system_id: raise serializers.ValidationError("no system user id provided") @@ -749,9 +750,10 @@ def create(self, request, *args, **kwargs): raise serializers.ValidationError("proposal type does not exist") obj = AnnualAdmissionApplication.objects.create( - submitter=request.user.id, - proposal_type=proposal_type - ) + submitter=request.user.id, + no_email_notifications=no_email_notifications, + proposal_type=proposal_type + ) logger.info(f'Annual Admission Application: [{obj}] has been created by the user: [{request.user}].') @@ -821,6 +823,7 @@ def create(self, request, *args, **kwargs): if is_internal(request): applicant_system_id = request.data.get("applicant_system_id") + no_email_notifications = request.data.get("no_emails", False) if not applicant_system_id: raise serializers.ValidationError("no system user id provided") @@ -835,9 +838,10 @@ def create(self, request, *args, **kwargs): raise serializers.ValidationError("proposal type does not exist") obj = AuthorisedUserApplication.objects.create( - submitter=request.user.id, - proposal_type=proposal_type - ) + submitter=request.user.id, + no_email_notifications=no_email_notifications, + proposal_type=proposal_type + ) logger.info(f'Authorised User Permit Application: [{obj}] has been created by the user: [{request.user}].') obj.log_user_action(f'Authorised User Permit Application: {obj.lodgement_number} has been created.', request) @@ -926,6 +930,7 @@ def create(self, request, *args, **kwargs): if is_internal(request): applicant_system_id = request.data.get("applicant_system_id") + no_email_notifications = request.data.get("no_emails", False) if not applicant_system_id: raise serializers.ValidationError("no system user id provided") @@ -947,9 +952,10 @@ def create(self, request, *args, **kwargs): raise serializers.ValidationError("user not permitted to create WLA at this time") obj = WaitingListApplication.objects.create( - submitter=request.user.id, - proposal_type=proposal_type - ) + submitter=request.user.id, + no_email_notifications=no_email_notifications, + proposal_type=proposal_type + ) logger.info(f'Waiting List Application: [{obj}] has been created by the user: [{request.user}].') obj.log_user_action(f'Waiting List Application: {obj.lodgement_number} has been created.', request) diff --git a/mooringlicensing/components/proposals/models.py b/mooringlicensing/components/proposals/models.py index f25ce0a3..fbc90e61 100644 --- a/mooringlicensing/components/proposals/models.py +++ b/mooringlicensing/components/proposals/models.py @@ -359,6 +359,8 @@ class Proposal(DirtyFieldsMixin, RevisionedMixin): payment_reminder_sent = models.BooleanField(default=False) payment_due_date = models.DateField(blank=True, null=True) #date when payment is due for future invoices + no_email_notifications = models.BooleanField(default=False) + class Meta: app_label = 'mooringlicensing' verbose_name = "Application" diff --git a/mooringlicensing/components/proposals/serializers.py b/mooringlicensing/components/proposals/serializers.py index 09afc467..7ada468e 100644 --- a/mooringlicensing/components/proposals/serializers.py +++ b/mooringlicensing/components/proposals/serializers.py @@ -967,6 +967,7 @@ class Meta: 'uuid', 'allocated_mooring', 'has_unactioned_endorsements', + 'no_email_notifications', ) read_only_fields = ( 'documents', diff --git a/mooringlicensing/components/proposals/utils.py b/mooringlicensing/components/proposals/utils.py index 01a86174..82d93341 100644 --- a/mooringlicensing/components/proposals/utils.py +++ b/mooringlicensing/components/proposals/utils.py @@ -70,6 +70,13 @@ def save_proponent_data(instance, request, action, being_auto_approved=False): save_proponent_data_aua(instance, request, action) elif type(instance.child_obj) == MooringLicenceApplication: save_proponent_data_mla(instance, request, action) + + if instance.has_assessor_mode(request.user): + instance.refresh_from_db() + proposal_data = request.data.get('proposal') if request.data.get('proposal') else {} + if proposal_data and "no_email_notifications" in proposal_data: + instance.no_email_notifications = proposal_data["no_email_notifications"] + instance.save() else: raise serializers.ValidationError("user not authorised to update applicant details") diff --git a/mooringlicensing/frontend/mooringlicensing/src/components/external/proposal.vue b/mooringlicensing/frontend/mooringlicensing/src/components/external/proposal.vue index db3cc730..5610e416 100755 --- a/mooringlicensing/frontend/mooringlicensing/src/components/external/proposal.vue +++ b/mooringlicensing/frontend/mooringlicensing/src/components/external/proposal.vue @@ -100,7 +100,7 @@

+

@@ -229,9 +232,9 @@