From 267fd94ebdd164060fd2e4e7981158bcb00d517b Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 05:12:43 +0000 Subject: [PATCH 01/10] Removed: cron.py and mooring_booking_utils.py --- mooringlicensing/cron.py | 24 - mooringlicensing/mooring_booking_utils.py | 1656 --------------------- 2 files changed, 1680 deletions(-) delete mode 100644 mooringlicensing/cron.py delete mode 100644 mooringlicensing/mooring_booking_utils.py diff --git a/mooringlicensing/cron.py b/mooringlicensing/cron.py deleted file mode 100644 index 489b0e5d1..000000000 --- a/mooringlicensing/cron.py +++ /dev/null @@ -1,24 +0,0 @@ -from datetime import date, timedelta - -from django.db.models import Q -from django.core import management -from django_cron import CronJobBase, Schedule -from django.conf import settings - -#from mooringlicensing.components.main.api import oracle_integration - -#TODO does not appear to be in use- remove -class OracleIntegrationCronJob(CronJobBase): - """ - To Test (shortly after RUN_AT_TIMES): - ./manage_ml.py runcrons - """ - RUN_AT_TIMES = [settings.CRON_RUN_AT_TIMES] - - schedule = Schedule(run_at_times=RUN_AT_TIMES) - code = 'ml.oracle_integration' - - def do(self): - pass - #oracle_integration(str(date.today()-timedelta(days=1)),False) - diff --git a/mooringlicensing/mooring_booking_utils.py b/mooringlicensing/mooring_booking_utils.py deleted file mode 100644 index 219ae1f8b..000000000 --- a/mooringlicensing/mooring_booking_utils.py +++ /dev/null @@ -1,1656 +0,0 @@ -#TODO - review, determine if any of this is needed - -from datetime import datetime, timedelta, date -import logging -import traceback -from decimal import * -import json -import calendar -import geojson -import requests -import io -from django.conf import settings -# from django.core.urlresolvers import reverse, reverse_lazy -from django.urls import reverse -from django.core.exceptions import ValidationError -from django.db import transaction -from django.db.models import Q -from django.http import HttpResponseRedirect, HttpResponse -from django.shortcuts import redirect -from django.utils import timezone -from dateutil.tz.tz import tzoffset -from pytz import timezone as pytimezone -# from ledger.payments.models import Invoice,OracleInterface,CashTransaction -# from ledger.payments.utils import oracle_parser_on_invoice, update_payments -# from ledger.checkout.utils import create_basket_session, create_checkout_session, place_order_submission, get_cookie_basket -from ledger_api_client.utils import create_basket_session, create_checkout_session, place_order_submission, update_payments -#from mooring.models import (MooringArea, Mooringsite, MooringsiteRate, MooringsiteBooking, Booking, BookingInvoice, MooringsiteBookingRange, Rate, MooringAreaBookingRange,MooringAreaStayHistory, MooringsiteRate, MarinaEntryRate, BookingVehicleRego, AdmissionsBooking, AdmissionsOracleCode, AdmissionsRate, AdmissionsLine, ChangePricePeriod, CancelPricePeriod, GlobalSettings, MooringAreaGroup, AdmissionsLocation, ChangeGroup, CancelGroup, BookingPeriod, BookingPeriodOption, AdmissionsBookingInvoice, BookingAnnualAdmission) -#from mooring import models -#from mooring.serialisers import BookingRegoSerializer, MooringsiteRateSerializer, MarinaEntryRateSerializer, RateSerializer, MooringsiteRateReadonlySerializer, AdmissionsRateSerializer -#from mooring.emails import send_booking_invoice,send_booking_confirmation -#from mooring import emails -# <<<<<<< HEAD -# from oscar.apps.order.models import Order -from ledger_api_client.order import Order -# from ledger.payments.invoice import utils -# ||||||| 741adce2 -# from oscar.apps.order.models import Order -# from ledger.payments.invoice import utils -# ======= -#from oscar.apps.order.models import Order -# from ledger.order.models import Order -# from ledger.payments.invoice import utils -# >>>>>>> main -#from mooring import models -from mooringlicensing.components.proposals.models import Proposal -from mooringlicensing.components.payments_ml.models import ApplicationFee -from mooringlicensing.settings import MOORING_LICENSING_EXTERNAL_URL - -logger = logging.getLogger('booking_checkout') - -def override_lineitems(override_price, override_reason, total_price, oracle_code, override_reason_info=""): - invoice_line = [] - if oracle_code: - #if override_reason: - discount = Decimal(override_price) - Decimal(override_price) - Decimal(override_price) - invoice_line.append({"ledger_description": '{} - {}'.format(override_reason.text, override_reason_info), "quantity": 1, 'price_incl_tax': discount, 'oracle_code': oracle_code, 'line_status': 1}) - return invoice_line - -def nononline_booking_lineitems(oracle_code, request): - invoice_line = [] - if oracle_code: - group = MooringAreaGroup.objects.filter(members__in=[request.user]) - value = GlobalSettings.objects.get(mooring_group=group, key=0).value - if Decimal(value) > 0: - invoice_line.append({'ledger_description': 'Phone Booking Fee', 'quantity': 1, 'price_incl_tax': Decimal(value), 'oracle_code': oracle_code, 'line_status': 1}) -# invoice_line.append({'ledger_description': 'Phone Booking Fee', 'quantity': 1, 'price_incl_tax': Decimal(value), 'oracle_code': oracle_code}) - return invoice_line - -def admission_lineitems(lines): - invoice_lines = [] - if lines: - for line in lines: - if line['guests'] > 0: - invoice_lines.append({'ledger_description': 'Admissions {} - {} ({} guests)'.format(line['from'], line['to'], line['guests']), "quantity": 1, 'price_incl_tax': line['admissionFee'], "oracle_code": line['oracle_code'], 'line_status': 1}) - -# invoice_lines.append({'ledger_description': 'Admissions {} - {} ({} guests)'.format(line['from'], line['to'], line['guests']), "quantity": 1, 'price_incl_tax': line['admissionFee'], "oracle_code": line['oracle_code']}) - - return invoice_lines - - -def calculate_price_booking_cancellation(booking, overide_cancel_fees=False): - current_date_time = datetime.strptime(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S') - nowtime = datetime.today() - nowtimec = datetime.strptime(nowtime.strftime('%Y-%m-%d'),'%Y-%m-%d') - mg = MooringAreaGroup.objects.all() - - booking = MooringsiteBooking.objects.filter(booking=booking) - cancellation_fees = [] - adjustment_fee = Decimal('0.00') - #{'additional_fees': 'true', 'description': 'Booking Change Fee','amount': Decimal('0.00')} - - for ob in booking: - changed = True - #for bc in booking_changes: - # if bc.campsite == ob.campsite and ob.from_dt == bc.from_dt and ob.to_dt == bc.to_dt and ob.booking_period_option == bc.booking_period_option: - # changed = False - from_dt = datetime.strptime(ob.from_dt.strftime('%Y-%m-%d'),'%Y-%m-%d') - daystillbooking = (from_dt-nowtimec).days - - cancel_policy = None - cancel_fee_amount = '0.00' - #change_price_period = CancelPricePeriod.objects.filter(id=ob.booking_period_option.cancel_group_id).order_by('days') - cancel_group = CancelGroup.objects.get(id=ob.booking_period_option.cancel_group_id) - cancel_price_period = cancel_group.cancel_period.all().order_by('days') - mooring_group =None - for i in mg: - if i.moorings.count() > 0: - mooring_group = i.moorings.all()[0].id - - - for cpp in cancel_price_period: - if daystillbooking < 0: - daystillbooking = 0 - if daystillbooking >= cpp.days: - cancel_policy =cpp - - if cancel_policy: - if cancel_policy.calulation_type == 0: - # Percentage - cancel_fee_amount = float(ob.amount) * (cancel_policy.percentage / 100) - elif cancel_policy.calulation_type == 1: - cancel_fee_amount = cancel_policy.amount - # Fixed Pricing - description = 'Mooring {} ({} - {})'.format(ob.campsite.mooringarea.name,ob.from_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p'),ob.to_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p')) - - if overide_cancel_fees is True: - cancellation_fees.append({'additional_fees': 'true', 'description': 'Refund - '+description,'amount': str(ob.amount - ob.amount - ob.amount), 'mooring_group': mooring_group, 'oracle_code': str(ob.campsite.mooringarea.oracle_code)}) - else: - - if datetime.strptime(ob.from_dt.astimezone(pytimezone('Australia/Perth')).strftime('%Y-%m-%d %H:%M:%S'),'%Y-%m-%d %H:%M:%S') < current_date_time: - #cancellation_fees.append({'additional_fees': 'true', 'description': 'Past Booking - '+description,'amount': Decimal('0.00'), 'mooring_group': mooring_group}) - cancellation_fees.append({'additional_fees': 'true', 'description': 'Past Booking - '+description,'amount': Decimal('0.00'), 'mooring_group': mooring_group, 'oracle_code': str(ob.campsite.mooringarea.oracle_code)}) - else: - #change_fees['amount'] = str(refund_amount) - cancellation_fees.append({'additional_fees': 'true', 'description': 'Cancel Fee - '+description,'amount': cancel_fee_amount, 'mooring_group': mooring_group, 'oracle_code': str(ob.campsite.mooringarea.oracle_code)}) - cancellation_fees.append({'additional_fees': 'true', 'description': 'Refund - '+description,'amount': str(ob.amount - ob.amount - ob.amount), 'mooring_group': mooring_group, 'oracle_code': str(ob.campsite.mooringarea.oracle_code)}) - #cancellation_fees.append({'additional_fees': 'true', 'description': 'Cancel Fee - '+description,'amount': cancel_fee_amount, 'mooring_group': mooring_group}) - #cancellation_fees.append({'additional_fees': 'true', 'description': 'Refund - '+description,'amount': str(ob.amount - ob.amount - ob.amount), 'mooring_group': mooring_group}) - else: - - print ("NO CANCELATION POLICY") - - #else: - # adjustment_fee = ob.amount + adjustment_fee - #change_fees.append({'additional_fees': 'true', 'description': 'Mooring Adjustment Credit' ,'amount': str(adjustment_fee - adjustment_fee - adjustment_fee)}) - - return cancellation_fees - - - -def calculate_price_booking_change(old_booking, new_booking,overide_change_fees=False): - nowtime = datetime.today() - nowtimec = datetime.strptime(nowtime.strftime('%Y-%m-%d'),'%Y-%m-%d') - - old_booking_mooring = MooringsiteBooking.objects.filter(booking=old_booking) - booking_changes = MooringsiteBooking.objects.filter(booking=new_booking) - change_fees = [] - adjustment_fee = Decimal('0.00') - mg = MooringAreaGroup.objects.all() - #{'additional_fees': 'true', 'description': 'Booking Change Fee','amount': Decimal('0.00')} - - for ob in old_booking_mooring: - changed = True - for bc in booking_changes: - if bc.campsite == ob.campsite and ob.from_dt == bc.from_dt and ob.to_dt == bc.to_dt and ob.booking_period_option == bc.booking_period_option: - changed = False - from_dt = datetime.strptime(ob.from_dt.strftime('%Y-%m-%d'),'%Y-%m-%d') - daystillbooking = (from_dt-nowtimec).days - refund_policy = None - - for i in mg: - if i.moorings.count() > 0: - mooring_group = i.moorings.all()[0].id - - - if changed is True: - change_fee_amount = '0.00' - # change_price_period = ChangePricePeriod.objects.filter(id=ob.booking_period_option.change_group_id).order_by('-days') - change_group = ChangeGroup.objects.get(id=ob.booking_period_option.change_group_id) - change_price_period = change_group.change_period.all().order_by('days') - for cpp in change_price_period: - if daystillbooking < 0: - daystillbooking = 0 -# if cpp.days >= daystillbooking: - if daystillbooking >= cpp.days: - refund_policy =cpp - if refund_policy: - if refund_policy.calulation_type == 0: - # Percentage - change_fee_amount = float(ob.amount) * (refund_policy.percentage / 100) - elif refund_policy.calulation_type == 1: - change_fee_amount = refund_policy.amount - # Fixed Pricing - - description = 'Mooring {} ({} - {})'.format(ob.campsite.mooringarea.name,ob.from_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p'),ob.to_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p')) - - if overide_change_fees is True: - change_fees.append({'additional_fees': 'true', 'description': 'Refund - '+description,'amount': str(format(ob.amount - ob.amount - ob.amount, '.2f')), 'oracle_code': str(ob.campsite.mooringarea.oracle_code), 'mooring_group': mooring_group, 'line_status': 3}) - else: - #change_fees['amount'] = str(refund_amount) - #change_fees.append({'additional_fees': 'true', 'description': 'Change Fee - '+description,'amount': float(change_fee_amount), 'oracle_code': str(ob.campsite.mooringarea.oracle_code), 'mooring_group': mooring_group}) - #change_fees.append({'additional_fees': 'true', 'description': 'Refund - '+description,'amount': str(ob.amount - ob.amount - ob.amount), 'oracle_code': str(ob.campsite.mooringarea.oracle_code), 'mooring_group': mooring_group}) - change_fees.append({'additional_fees': 'true', 'description': 'Change Fee - '+description,'amount': str(format(change_fee_amount, '.2f')), 'oracle_code': str(ob.campsite.mooringarea.oracle_code), 'mooring_group': mooring_group, 'line_status': 2}) - change_fees.append({'additional_fees': 'true', 'description': 'Refund - '+description,'amount': str(format(ob.amount - ob.amount - ob.amount, '.2f')), 'oracle_code': str(ob.campsite.mooringarea.oracle_code), 'mooring_group': mooring_group, 'line_status': 3}) - else: - print ("NO REFUND POLICY") - - else: - #description = 'Mooring {} ({} - {})'.format(ob.campsite.mooringarea.name,ob.from_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p'),ob.to_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p')) - adjustment_fee = float('0.00') - adjustment_fee = float(ob.amount) + adjustment_fee - description = 'Mooring {} ({} - {})'.format(ob.campsite.mooringarea.name,ob.from_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p'),ob.to_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p')) -# change_fees.append({'additional_fees': 'true', 'description': 'Adjustment - '+description ,'amount': str(adjustment_fee - adjustment_fee - adjustment_fee), 'oracle_code': str(ob.campsite.mooringarea.oracle_code), 'mooring_group': mooring_group}) - change_fees.append({'additional_fees': 'true', 'description': 'Adjustment - '+description ,'amount': str(format(adjustment_fee - adjustment_fee - adjustment_fee, '.2f')), 'oracle_code': str(ob.campsite.mooringarea.oracle_code), 'mooring_group': mooring_group, 'line_status': 3}) - - return change_fees - -def calculate_price_admissions_cancel(adBooking, change_fees, overide_cancel_fees=False): - ad_lines = AdmissionsLine.objects.filter(admissionsBooking=adBooking) - for line in ad_lines: - if line.arrivalDate > date.today() or overide_cancel_fees is True: - - - description = "Admission ({}) for {} guest(s)".format(datetime.strftime(line.arrivalDate, '%d/%m/%Y'), adBooking.total_admissions) - oracle_code = AdmissionsOracleCode.objects.filter(mooring_group=line.location.mooring_group)[0] - - change_fees.append({'additional_fees': 'true', 'description': 'Refund - ' + description,'amount': str(line.cost - line.cost - line.cost), 'oracle_code': str(oracle_code.oracle_code), 'mooring_group': line.location.mooring_group.id, 'line_status': 3}) - return change_fees - - -def calculate_price_admissions_change(adBooking, change_fees): - ad_lines = AdmissionsLine.objects.filter(admissionsBooking=adBooking) - for line in ad_lines: - - description = "Admission ({}) for {} guest(s)".format(datetime.strftime(line.arrivalDate, '%d/%m/%Y'), adBooking.total_admissions) - oracle_code = AdmissionsOracleCode.objects.filter(mooring_group=line.location.mooring_group)[0] - - # Fees - change_fees.append({'additional_fees': 'true', 'description': 'Adjustment - ' + description,'amount': str(line.cost - line.cost - line.cost), 'oracle_code': str(oracle_code.oracle_code), 'mooring_group': line.location.mooring_group.id, 'line_status': 3 }) - - - return change_fees - -def price_or_lineitems(request,booking,campsite_list,lines=True,old_booking=None): - total_price = Decimal(0) - booking_mooring = MooringsiteBooking.objects.filter(booking=booking) - booking_mooring_old = [] - if booking.old_booking: - booking_mooring_old = MooringsiteBooking.objects.filter(booking=booking.old_booking) - - invoice_lines = [] - if lines: - for bm in booking_mooring: - line_status = 1 - amount = bm.amount - if str(bm.id) in booking.override_lines: - amount = Decimal(booking.override_lines[str(bm.id)]) - for ob in booking_mooring_old: - if bm.campsite == ob.campsite and ob.from_dt == bm.from_dt and ob.to_dt == bm.to_dt and ob.booking_period_option == bm.booking_period_option: - line_status = 2 - invoice_lines.append({'ledger_description':'Mooring {} ({} - {})'.format(bm.campsite.mooringarea.name,bm.from_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p'),bm.to_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p')),"quantity":1,"price_incl_tax":amount,"oracle_code":bm.campsite.mooringarea.oracle_code, 'line_status': line_status}) - - - -# invoice_lines.append({'ledger_description':'Mooring {} ({} - {})'.format(bm.campsite.mooringarea.name,bm.from_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p'),bm.to_dt.astimezone(pytimezone('Australia/Perth')).strftime('%d/%m/%Y %H:%M %p')),"quantity":1,"price_incl_tax":bm.amount,"oracle_code":bm.campsite.mooringarea.oracle_code}) - return invoice_lines - else: - return total_price - -def price_or_lineitems_extras(request,booking,change_fees,invoice_lines=[]): - total_price = Decimal(0) - booking_mooring = MooringsiteBooking.objects.filter(booking=booking) - for cf in change_fees: -# invoice_lines.append({'ledger_description':cf['description'],"quantity":1,"price_incl_tax":cf['amount'],"oracle_code":cf['oracle_code']}) - invoice_lines.append({'ledger_description':cf['description'],"quantity":1,"price_incl_tax":cf['amount'],"oracle_code":cf['oracle_code'], 'line_status': cf['line_status']}) - return invoice_lines - -def old_price_or_lineitems(request,booking,campsite_list,lines=True,old_booking=None): - total_price = Decimal(0) - rate_list = {} - invoice_lines = [] - if not lines and not old_booking: - raise Exception('An old booking is required if lines is set to false') - # Create line items for customers - daily_rates = [get_campsite_current_rate(request,c,booking.arrival.strftime('%Y-%m-%d'),booking.departure.strftime('%Y-%m-%d')) for c in campsite_list] - if not daily_rates: - raise Exception('There was an error while trying to get the daily rates.') - for rates in daily_rates: - for c in rates: - if c['rate']['campsite'] not in rate_list.keys(): - rate_list[c['rate']['campsite']] = {c['rate']['id']:{'start':c['date'],'end':c['date'],'mooring': c['rate']['mooring'] ,'adult':c['rate']['adult'],'concession':c['rate']['concession'],'child':c['rate']['child'],'infant':c['rate']['infant']}} - else: - if c['rate']['id'] not in rate_list[c['rate']['campsite']].keys(): - rate_list[c['rate']['campsite']] = {c['rate']['id']:{'start':c['date'],'end':c['date'],'mooring': c['rate']['mooring'], 'adult':c['rate']['adult'],'concession':c['rate']['concession'],'child':c['rate']['child'],'infant':c['rate']['infant']}} - else: - rate_list[c['rate']['campsite']][c['rate']['id']]['end'] = c['date'] - # Get Guest Details - #guests = {} - #for k,v in booking.details.items(): - # if 'num_' in k: - # guests[k.split('num_')[1]] = v - ##### Above is for poeple quantity (mooring are not based on people.. based on vessels) - - # guess is used as the quantity items for the check out basket. - guests = {} - guests['mooring'] = 1 - for k,v in guests.items(): - if int(v) > 0: - for c,p in rate_list.items(): - for i,r in p.items(): - price = Decimal(0) - end = datetime.strptime(r['end'],"%Y-%m-%d").date() - start = datetime.strptime(r['start'],"%Y-%m-%d").date() - num_days = int ((end - start).days) + 1 - campsite = Mooringsite.objects.get(id=c) - if lines: - price = str((num_days * Decimal(r[k]))) - #if not booking.mooringarea.oracle_code: - # raise Exception('The mooringarea selected does not have an Oracle code attached to it.') - end_date = end + timedelta(days=1) -# invoice_lines.append({'ledger_description':'Mooring fee {} ({} - {})'.format(k,start.strftime('%d-%m-%Y'),end_date.strftime('%d-%m-%Y')),"quantity":v,"price_incl_tax":price,"oracle_code":booking.mooringarea.oracle_code}) - invoice_lines.append({'ledger_description':'Admission fee on {} ({}) {}'.format(adLine.arrivalDate, group, overnightStay),"quantity":amount,"price_incl_tax":price, "oracle_code":oracle_code, 'line_status': 1}) - else: - price = (num_days * Decimal(r[k])) * v - total_price += price - # Create line items for vehicles - if lines: - vehicles = booking.regos.all() - else: - vehicles = old_booking.regos.all() - if vehicles: - if booking.mooringarea.park.entry_fee_required: - # Update the booking vehicle regos with the park entry requirement - vehicles.update(park_entry_fee=True) - if not booking.mooringarea.park.oracle_code: - raise Exception('A marine park entry Oracle code has not been set for the park that the mooringarea belongs to.') - park_entry_rate = get_park_entry_rate(request,booking.arrival.strftime('%Y-%m-%d')) - vehicle_dict = { - 'vessel' : vehicles.filter(entry_fee=True, type='vessel'), - #'vehicle': vehicles.filter(entry_fee=True, type='vehicle'), - 'motorbike': vehicles.filter(entry_fee=True, type='motorbike'), - 'concession': vehicles.filter(entry_fee=True, type='concession') - } - - for k,v in vehicle_dict.items(): - if v.count() > 0: - if lines: - price = park_entry_rate[k] - regos = ', '.join([x[0] for x in v.values_list('rego')]) - invoice_lines.append({ - 'ledger_description': 'Mooring fee - {}'.format(k), - 'quantity': v.count(), - 'price_incl_tax': price, - 'oracle_code': booking.mooringarea.park.oracle_code - }) - else: - price = Decimal(park_entry_rate[k]) * v.count() - total_price += price - if lines: - return invoice_lines - else: - return total_price - -def get_admissions_entry_rate(request,start_date, location): - res = [] - if start_date: - start_date = datetime.strptime(start_date,"%Y-%m-%d").date() - group = location.mooring_group - price_history = AdmissionsRate.objects.filter(mooring_group__in=[group,], period_start__lte = start_date).order_by('-period_start') - if price_history: - serializer = AdmissionsRateSerializer(price_history,many=True,context={'request':request}) - res = serializer.data[0] - return res - -def admissions_price_or_lineitems(request, admissionsBooking,lines=True): - total_price = Decimal(0) - rate_list = {} - invoice_lines = [] - line = lines - daily_rates = [] - # Create line items for customers - admissionsLines = AdmissionsLine.objects.filter(admissionsBooking=admissionsBooking) - for adLine in admissionsLines: - rate = get_admissions_entry_rate(request,adLine.arrivalDate.strftime('%Y-%m-%d'), adLine.location) - daily_rate = {'date' : adLine.arrivalDate.strftime('%d/%m/%Y'), 'rate' : rate} - daily_rates.append(daily_rate) - oracle_codes = AdmissionsOracleCode.objects.filter(mooring_group__in=[adLine.location.mooring_group,]) - if not oracle_codes.count() > 0: - if request.user.is_staff: - raise Exception('Admissions Oracle Code missing, please set up in administration tool.') - else: - raise Exception('Please alert {} of the following error message:\nAdmissions Oracle Code missing.'.format(adLine['group'])) - if not daily_rates or daily_rates == []: - raise Exception('There was an error while trying to get the daily rates.') - family = 0 - adults = admissionsBooking.noOfAdults - children = admissionsBooking.noOfChildren - if adults > 1 and children > 1: - if adults == children: - if adults % 2 == 0: - family = adults//2 - adults = 0 - children = 0 - else: - adults -= 1 - family = adults//2 - adults = 1 - children = 1 - elif adults > children: #Adults greater - tickets based on children - if children % 2 == 0: - family = children//2 - adults -= children - children = 0 - else: - children -= 1 - family = children//2 - adults -= children - children = 1 - else: #Children greater - tickets based on adults - if adults % 2 == 0: - family = adults//2 - children -= adults - adults = 0 - else: - adults -= 1 - family = adults//2 - children -= adults - adults = 1 - - people = {'Adults': adults,'Concessions': admissionsBooking.noOfConcessions,'Children': children,'Infants': admissionsBooking.noOfInfants, 'Family': family} - for adLine in admissionsLines: - for group, amount in people.items(): - if line: - if (amount > 0): - if group == 'Adults': - gr = 'adult' - elif group == 'Children': - gr = group - elif group == 'Infants': - gr = 'infant' - elif group == 'Family': - gr = 'family' - if adLine.overnightStay: - costfield = gr.lower() + "_overnight_cost" - overnightStay = "Overnight Included" - else: - costfield = gr.lower() + "_cost" - overnightStay = "Day Visit Only" - daily_rate = next(item for item in daily_rates if item['date'] == adLine.arrivalDate.strftime('%d/%m/%Y'))['rate'] - price = daily_rate.get(costfield) - oracle_codes = AdmissionsOracleCode.objects.filter(mooring_group=adLine.location.mooring_group) - if oracle_codes.count() > 0: - oracle_code = oracle_codes[0].oracle_code - invoice_lines.append({'ledger_description':'Admission fee on {} ({}) {}'.format(adLine.arrivalDate, group, overnightStay),"quantity":amount,"price_incl_tax":price, "oracle_code":oracle_code, 'line_status': 1}) - - else: - daily_rate = daily_rates[adLine.arrivalDate.strftime('%d/%m/%Y')] - price = Decimal(daily_rate) - total_cost += price - if line: - return invoice_lines - else: - return total_price - -def check_date_diff(old_booking,new_booking): - if old_booking.arrival == new_booking.arrival and old_booking.departure == new_booking.departure: - return 4 # same days - elif old_booking.arrival == new_booking.arrival: - old_booking_days = int((old_booking.departure - old_booking.arrival).days) - new_days = int((new_booking.departure - new_booking.arrival).days) - if new_days > old_booking_days: - return 1 #additional days - else: - return 2 #reduced days - elif old_booking.departure == new_booking.departure: - old_booking_days = int((old_booking.departure - old_booking.arrival).days) - new_days = int((new_booking.departure - new_booking.arrival).days) - if new_days > old_booking_days: - return 1 #additional days - else: - return 2 #reduced days - else: - return 3 # different days - -def get_diff_days(old_booking,new_booking,additional=True): - if additional: - return int((new_booking.departure - old_booking.departure).days) - return int((old_booking.departure - new_booking.departure).days) - -def create_temp_bookingupdate(request,arrival,departure,booking_details,old_booking,total_price): - # delete all the campsites in the old moving so as to transfer them to the new booking - old_booking.campsites.all().delete() - booking = create_booking_by_site(booking_details['campsites'], - start_date = arrival, - end_date = departure, - num_adult = booking_details['num_adult'], - num_concession= booking_details['num_concession'], - num_child= booking_details['num_child'], - num_infant= booking_details['num_infant'], - num_mooring = booking_details['num_mooring'], - cost_total = total_price, - customer = old_booking.customer, - override_price=old_booking.override_price, - updating_booking = True, - override_checks=True - ) - # Move all the vehicles to the new booking - for r in old_booking.regos.all(): - r.booking = booking - r.save() - - lines = price_or_lineitems(request,booking,booking.campsite_id_list) - booking_arrival = booking.arrival.strftime('%d-%m-%Y') - booking_departure = booking.departure.strftime('%d-%m-%Y') - reservation = u'Reservation for {} confirmation PS{}'.format( - u'{} {}'.format(booking.customer.first_name, booking.customer.last_name), booking.id) - # Proceed to generate invoice - - checkout_response = checkout(request,booking,lines,invoice_text=reservation,internal=True) - - # FIXME: replace with session check - invoice = None - if 'invoice=' in checkout_response.url: - invoice = checkout_response.url.split('invoice=', 1)[1] - else: - for h in reversed(checkout_response.history): - if 'invoice=' in h.url: - invoice = h.url.split('invoice=', 1)[1] - break - - # create the new invoice - new_invoice = internal_create_booking_invoice(booking, invoice) - - # Check if the booking is a legacy booking and doesn't have an invoice - if old_booking.legacy_id and old_booking.invoices.count() < 1: - # Create a cash transaction in order to fix the outstnding invoice payment - CashTransaction.objects.create( - invoice = Invoice.objects.get(reference=new_invoice.invoice_reference), - amount = old_booking.cost_total, - type = 'move_in', - source = 'cash', - details = 'Transfer of funds from migrated booking', - movement_reference='Migrated Booking Funds' - ) - # Update payment details for the new invoice - update_payments(new_invoice.invoice_reference) - - # Attach new invoices to old booking - for i in old_booking.invoices.all(): - inv = Invoice.objects.get(reference=i.invoice_reference) - inv.voided = True - #transfer to the new invoice - inv.move_funds(inv.transferable_amount,Invoice.objects.get(reference=new_invoice.invoice_reference),'Transfer of funds from {}'.format(inv.reference)) - inv.save() - # Change the booking for the selected invoice - new_invoice.booking = old_booking - new_invoice.save() - - return booking - - -def get_annual_admissions_pricing_info(annual_booking_period_id,vessel_size): - nowdt = datetime.now() - price = '0.00' - annual_admissions = {'response': 'error', 'abpg': {}, 'abpo': {}, 'abpovc': {}} - if models.AnnualBookingPeriodGroup.objects.filter(id=int(annual_booking_period_id)).count() > 0: - abpg = models.AnnualBookingPeriodGroup.objects.get(id=int(annual_booking_period_id)) - vsc = models.VesselSizeCategory.objects.filter(start_size__lte=Decimal(vessel_size),end_size__gte=Decimal(vessel_size)) - abpo= models.AnnualBookingPeriodOption.objects.filter(start_time__lte=nowdt,finish_time__gte=nowdt,annual_booking_period_group=abpg) - if abpo.count() > 0 and vsc.count() > 0: - abpovc = models.AnnualBookingPeriodOptionVesselCategoryPrice.objects.filter(annual_booking_period_option=abpo[0],vessel_category=vsc[0]) - price = abpovc[0].price - annual_admissions['abpg'] = abpg - if abpo.count() > 0: - annual_admissions['abpo'] = abpo[0] - if abpovc.count() > 0: - annual_admissions['abpovc'] = abpovc[0] - annual_admissions['response'] = 'success' - return annual_admissions - - -def iiiicreate_temp_bookingupdate(request,arrival,departure,booking_details,old_booking,total_price): - # delete all the campsites in the old moving so as to transfer them to the new booking - old_booking.campsites.all().delete() - booking = create_booking_by_site(booking_details['campsites'][0], - start_date = arrival, - end_date = departure, - num_adult = booking_details['num_adult'], - num_concession= booking_details['num_concession'], - num_child= booking_details['num_child'], - num_infant= booking_details['num_infant'], - num_mooring = booking_details['num_mooring'], - cost_total = total_price, - customer = old_booking.customer, - updating_booking = True - ) - - # Move all the vehicles to the new booking - for r in old_booking.regos.all(): - r.booking = booking - r.save() - - lines = price_or_lineitems(request,booking,booking.campsite_id_list) - booking_arrival = booking.arrival.strftime('%d-%m-%Y') - booking_departure = booking.departure.strftime('%d-%m-%Y') - reservation = "Reservation for {} from {} to {} at {}".format('{} {}'.format(booking.customer.first_name,booking.customer.last_name),booking_arrival,booking_departure,booking.mooringarea.name) - - # Proceed to generate invoice - checkout_response = checkout(request,booking,lines,invoice_text=reservation,internal=True) - internal_create_booking_invoice(booking, checkout_response) - - # Get the new invoice - new_invoice = booking.invoices.first() - - # Check if the booking is a legacy booking and doesn't have an invoice - if old_booking.legacy_id and old_booking.invoices.count() < 1: - # Create a cash transaction in order to fix the outstnding invoice payment - CashTransaction.objects.create( - invoice = Invoice.objects.get(reference=new_invoice.invoice_reference), - amount = old_booking.cost_total, - type = 'move_in', - source = 'cash', - details = 'Transfer of funds from migrated booking', - movement_reference='Migrated Booking Funds' - ) - # Update payment details for the new invoice - update_payments(new_invoice.invoice_reference) - - # Attach new invoices to old booking - for i in old_booking.invoices.all(): - inv = Invoice.objects.get(reference=i.invoice_reference) - inv.voided = True - #transfer to the new invoice - inv.move_funds(inv.transferable_amount,Invoice.objects.get(reference=new_invoice.invoice_reference),'Transfer of funds from {}'.format(inv.reference)) - inv.save() - # Change the booking for the selected invoice - new_invoice.booking = old_booking - new_invoice.save() - return booking - -def update_booking(request,old_booking,booking_details): - same_dates = False - same_campsites = False - same_campground = False - same_details = False - same_vehicles = True - - with transaction.atomic(): - try: - set_session_booking(request.session, old_booking) - new_details = {} - new_details.update(old_booking.details) - # Update the guests - new_details['num_adult'] = booking_details['num_adult'] - new_details['num_concession'] = booking_details['num_concession'] - new_details['num_child'] = booking_details['num_child'] - new_details['num_infant'] = booking_details['num_infant'] - booking = Booking( - arrival = booking_details['start_date'], - departure =booking_details['end_date'], - details = new_details, - customer=old_booking.customer, - mooringarea = MooringArea.objects.get(id=booking_details['mooringarea'])) - # Check that the departure is not less than the arrival - if booking.departure < booking.arrival: - raise Exception('The departure date cannot be before the arrival date') - today = datetime.now().date() - if today > old_booking.departure: - raise ValidationError('You cannot change a booking past the departure date.') - # Check if it is the same campground - if old_booking.mooringarea.id == booking.mooringarea.id: - same_campground = True - # Check if dates are the same - if (old_booking.arrival == booking.arrival) and (old_booking.departure == booking.departure): - same_dates = True - # Check if the campsite is the same - if sorted(old_booking.campsite_id_list) == sorted(booking_details['campsites']): - same_campsites = True - # Check if the details have changed - if new_details == old_booking.details: - same_details = True - # Check if the vehicles have changed - current_regos = old_booking.regos.all() - current_vehicle_regos= sorted([r.rego for r in current_regos]) - - # Add history - new_history = old_booking._generate_history(user=request.user) - - if request.data.get('entryFees').get('regos'): - new_regos = request.data['entryFees'].pop('regos') - sent_regos = [r['rego'] for r in new_regos] - regos_serializers = [] - update_regos_serializers = [] - for n in new_regos: - if n['rego'] not in current_vehicle_regos: - n['booking'] = old_booking.id - regos_serializers.append(BookingRegoSerializer(data=n)) - same_vehicles = False - else: - booking_rego = BookingVehicleRego.objects.get(booking=old_booking,rego=n['rego']) - n['booking'] = old_booking.id - if booking_rego.type != n['type'] or booking_rego.entry_fee != n['entry_fee']: - update_regos_serializers.append(BookingRegoSerializer(booking_rego,data=n)) - # Create the new regos if they are there - if regos_serializers: - for r in regos_serializers: - r.is_valid(raise_exception=True) - r.save() - # Update the new regos if they are there - if update_regos_serializers: - for r in update_regos_serializers: - r.is_valid(raise_exception=True) - r.save() - same_vehicles = False - - # Check if there are regos in place that need to be removed - stale_regos = [] - for r in current_regos: - if r.rego not in sent_regos: - stale_regos.append(r.id) - # delete stale regos - if stale_regos: - same_vehicles = False - BookingVehicleRego.objects.filter(id__in=stale_regos).delete() - else: - same_vehicles = False - if current_regos: - current_regos.delete() - - if same_campsites and same_dates and same_vehicles and same_details: - if new_history is not None: - new_history.delete() - return old_booking - - # Check difference of dates in booking - old_booking_days = int((old_booking.departure - old_booking.arrival).days) - new_days = int((booking_details['end_date'] - booking_details['start_date']).days) - date_diff = check_date_diff(old_booking,booking) - - total_price = price_or_lineitems(request,booking,booking_details['campsites'],lines=False,old_booking=old_booking) - price_diff = True - if old_booking.cost_total != total_price: - price_diff = True - if price_diff: - - booking = create_temp_bookingupdate(request,booking.arrival,booking.departure,booking_details,old_booking,total_price) - # Attach campsite booking objects to old booking - for c in booking.campsites.all(): - c.booking = old_booking - c.save() - # Move all the vehicles to the in new booking to the old booking - for r in booking.regos.all(): - r.booking = old_booking - r.save() - old_booking.cost_total = booking.cost_total - old_booking.departure = booking.departure - old_booking.arrival = booking.arrival - old_booking.details.update(booking.details) - if not same_campground: - old_booking.campground = booking.campground - old_booking.save() - booking.delete() - delete_session_booking(request.session) - send_booking_invoice(old_booking) - # send out the confirmation email if the booking is paid or over paid - if old_booking.status == 'Paid' or old_booking.status == 'Over Paid': - send_booking_confirmation(old_booking,request) - return old_booking - except: - delete_session_booking(request.session) - print(traceback.print_exc()) - raise - -def create_or_update_booking(request,booking_details,updating=False,override_checks=False): - booking = None - if not updating: - booking = create_booking_by_site(booking_details['campsites'], - start_date = booking_details['start_date'], - end_date=booking_details['end_date'], - num_adult=booking_details['num_adult'], - num_concession=booking_details['num_concession'], - num_child=booking_details['num_child'], - num_infant=booking_details['num_infant'], - num_mooring=booking_details['num_mooring'], - vessel_size=booking_details['vessel_size'], - cost_total=booking_details['cost_total'], - override_price=booking_details['override_price'], - override_reason=booking_details['override_reason'], - override_reason_info=booking_details['override_reason_info'], - overridden_by=booking_details['overridden_by'], - customer=booking_details['customer'], - override_checks=override_checks - ) - - booking.details['first_name'] = booking_details['first_name'] - booking.details['last_name'] = booking_details['last_name'] - booking.details['phone'] = booking_details['phone'] - booking.details['country'] = booking_details['country'] - booking.details['postcode'] = booking_details['postcode'] - - # Add booking regos - if 'regos' in booking_details: - regos = booking_details['regos'] - for r in regos: - r['booking'] = booking.id - regos_serializers = [BookingRegoSerializer(data=r) for r in regos] - for r in regos_serializers: - r.is_valid(raise_exception=True) - r.save() - booking.save() - return booking - -def old_create_or_update_booking(request,booking_details,updating=False): - booking = None - if not updating: - booking = create_booking_by_site(campsite_id= booking_details['campsite_id'], - start_date = booking_details['start_date'], - end_date=booking_details['end_date'], - num_adult=booking_details['num_adult'], - num_concession=booking_details['num_concession'], - num_child=booking_details['num_child'], - num_infant=booking_details['num_infant'], - num_mooring=booking_details['num_mooring'], - vessel_size=booking_details['vessel_size'], - cost_total=booking_details['cost_total'], - customer=booking_details['customer']) - - booking.details['first_name'] = booking_details['first_name'] - booking.details['last_name'] = booking_details['last_name'] - booking.details['phone'] = booking_details['phone'] - booking.details['country'] = booking_details['country'] - booking.details['postcode'] = booking_details['postcode'] - - # Add booking regos - if request.data.get('parkEntry').get('regos'): - regos = request.data['parkEntry'].pop('regos') - for r in regos: - r[u'booking'] = booking.id - regos_serializers = [BookingRegoSerializer(data=r) for r in regos] - for r in regos_serializers: - r.is_valid(raise_exception=True) - r.save() - booking.save() - return booking - -def admissionsCheckout(request, admissionsBooking, lines, invoice_text=None, vouchers=[], internal=False): - basket_params = { - 'products': lines, - 'vouchers': vouchers, - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'custom_basket': True, - 'booking_reference': 'AD-'+str(admissionsBooking.id), - 'tax_override': True, - } - - basket, basket_hash = create_basket_session(request, basket_params) - return_preload_url= MOORING_LICENSING_EXTERNAL_URL + reverse("public_admissions_success") - checkout_params = { - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'fallback_url': request.build_absolute_uri('/'), - 'return_url': request.build_absolute_uri(reverse('public_admissions_success')), - 'return_preload_url': return_preload_url, - 'force_redirect': True, - 'proxy': True if internal else False, - 'invoice_text': invoice_text, - } - - if internal or request.user.is_anonymous(): - checkout_params['basket_owner'] = admissionsBooking.customer.id - create_checkout_session(request, checkout_params) - - if internal: - responseJson = place_order_submission(request) - else: - print(reverse('checkout:index')) - responseJson = HttpResponse(geojson.dumps({'status': 'success','redirect': reverse('checkout:index'),}), content_type='application/json') - # response = HttpResponseRedirect(reverse('checkout:index')) - - # inject the current basket into the redirect response cookies - # or else, anonymous users will be directionless - responseJson.set_cookie( - settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - ) - return responseJson - -def get_basket(request): - return get_cookie_basket(settings.OSCAR_BASKET_COOKIE_OPEN,request) - -def annual_admission_checkout(request, booking, lines, invoice_text=None, vouchers=[], internal=False): - - basket_params = { - 'products': lines, - 'vouchers': vouchers, - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'custom_basket': True, - 'booking_reference': 'AA-'+str(booking.id), - 'tax_override': True, - } - basket, basket_hash = create_basket_session(request, basket_params) - return_preload_url = MOORING_LICENSING_EXTERNAL_URL + reverse("public_booking_annual_admission_success") - checkout_params = { - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'fallback_url': request.build_absolute_uri('/'), - 'return_url': request.build_absolute_uri(reverse('public_booking_annual_admission_success')), - 'return_preload_url': return_preload_url, - 'force_redirect': True, - 'proxy': True if internal else False, - 'invoice_text': invoice_text, - } - if internal or request.user.is_anonymous(): - checkout_params['basket_owner'] = booking.customer.id - - create_checkout_session(request, checkout_params) - - - -# if internal: -# response = place_order_submission(request) -# else: - response = HttpResponseRedirect(reverse('checkout:index')) - # inject the current basket into the redirect response cookies - # or else, anonymous users will be directionless - response.set_cookie( - settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - ) - - #if booking.cost_total < 0: - # response = HttpResponseRedirect('/refund-payment') - # response.set_cookie( - # settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - # max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - # secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - # ) - - ## Zero booking costs - #if booking.cost_total < 1 and booking.cost_total > -1: - # response = HttpResponseRedirect('/no-payment') - # response.set_cookie( - # settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - # max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - # secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - # ) - return response - - -def checkout(request, booking, lines, invoice_text=None, vouchers=[], internal=False): - basket_params = { - 'products': lines, - 'vouchers': vouchers, - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'custom_basket': True, - 'booking_reference': 'PS-'+str(booking.id), - 'tax_override': True, - } - - basket, basket_hash = create_basket_session(request, basket_params) - return_preload_url = MOORING_LICENSING_EXTERNAL_URL + reverse("public_booking_success") - checkout_params = { - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'fallback_url': request.build_absolute_uri('/'), - 'return_url': request.build_absolute_uri(reverse('public_booking_success')), - 'return_preload_url': return_preload_url, - 'force_redirect': True, - 'proxy': True if internal else False, - 'invoice_text': invoice_text, - } -# if not internal: -# checkout_params['check_url'] = request.build_absolute_uri('/api/booking/{}/booking_checkout_status.json'.format(booking.id)) - if internal or request.user.is_anonymous(): - checkout_params['basket_owner'] = booking.customer.id - - print ("BOOKING ID 3") - print (request.session['ps_booking']) - - create_checkout_session(request, checkout_params) - print ("BOOKING ID 4") - print (request.session['ps_booking']) - - - -# if internal: -# response = place_order_submission(request) -# else: - #response = HttpResponseRedirect(reverse('checkout:index')) - response = HttpResponse(" Redirecting please wait: "+reverse('checkout:index')+"") - - # inject the current basket into the redirect response cookies - # or else, anonymous users will be directionless - response.set_cookie( - settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - ) - - if booking.cost_total < 0: - response = HttpResponseRedirect('/refund-payment') - response.set_cookie( - settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - ) - - # Zero booking costs - if booking.cost_total < 1 and booking.cost_total > -1: - response = HttpResponseRedirect('/no-payment') - response.set_cookie( - settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - ) - return response - - -def allocate_failedrefund_to_unallocated(request, booking, lines, invoice_text=None, internal=False, order_total='0.00',user=None): - booking_reference = None - if booking.__class__.__name__ == 'AdmissionsBooking': - booking_reference = 'AD-'+str(booking.id) - elif booking.__class__.__name__ == 'BookingAnnualAdmission': - booking_reference = 'AA-'+str(booking.id) - else: - booking_reference = 'PS-'+str(booking.id) - - basket_params = { - 'products': lines, - 'vouchers': [], - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'custom_basket': True, - 'booking_reference': booking_reference, - 'tax_override': True, - } - - basket, basket_hash = create_basket_session(request, basket_params) - ci = utils.CreateInvoiceBasket() - order = ci.create_invoice_and_order(basket, total=None, shipping_method='No shipping required',shipping_charge=False, user=user, status='Submitted', invoice_text='Refund Allocation Pool', ) - #basket.status = 'Submitted' - #basket.save() - #new_order = Order.objects.get(basket=basket) - new_invoice = Invoice.objects.get(order_number=order.number) - update_payments(new_invoice.reference) - if booking.__class__.__name__ == 'AdmissionsBooking': - print ("AdmissionsBooking") - book_inv, created = AdmissionsBookingInvoice.objects.get_or_create(admissions_booking=booking, invoice_reference=new_invoice.reference, system_invoice=True) - elif booking.__class__.__name__ == 'BookingAnnualAdmission': - print ("BookingAnnualAdmission") - book_inv, created = models.BookingAnnualInvoice.objects.get_or_create(booking_annual_admission=booking, invoice_reference=new_invoice.reference, system_invoice=True) - else: - book_inv, created = BookingInvoice.objects.get_or_create(booking=booking, invoice_reference=new_invoice.reference, system_invoice=True) - - return order - -# Booking changed to Proposal -def allocate_refund_to_invoice(request, booking, lines, invoice_text=None, internal=False, order_total='0.00',user=None, system_invoice=False): - booking_reference = None - #if booking.__class__.__name__ == 'AdmissionsBooking': - # booking_reference = 'AD-'+str(booking.id) - #elif booking.__class__.__name__ == 'BookingAnnualAdmission': - # booking_reference = 'AA-'+str(booking.id) - #else: - #booking_reference = 'PS-'+str(booking.id) - booking_reference = booking.lodgement_number - - basket_params = { - 'products': lines, - 'vouchers': [], - 'system': settings.PS_PAYMENT_SYSTEM_ID, - 'custom_basket': True, - 'booking_reference': booking_reference, - 'tax_override': True, - } - - basket, basket_hash = create_basket_session(request, basket_params) - ci = utils.CreateInvoiceBasket() - order = ci.create_invoice_and_order(basket, total=None, shipping_method='No shipping required',shipping_charge=False, user=user, status='Submitted', invoice_text='Oracle Allocation Pools', ) - #basket.status = 'Submitted' - #basket.save() - #new_order = Order.objects.get(basket=basket) - new_invoice = Invoice.objects.get(order_number=order.number) - update_payments(new_invoice.reference) - #if booking.__class__.__name__ == 'AdmissionsBooking': - # print ("AdmissionsBooking") - # book_inv, created = AdmissionsBookingInvoice.objects.get_or_create(admissions_booking=booking, invoice_reference=new_invoice.reference, system_invoice=True) - #elif booking.__class__.__name__ == 'BookingAnnualAdmission': - # print ("BookingAnnualAdmission") - # book_inv, created = models.BookingAnnualInvoice.objects.get_or_create(booking_annual_admission=booking, invoice_reference=new_invoice.reference, system_invoice=True) - #else: - - book_inv, created = ApplicationFee.objects.get_or_create(proposal=booking, invoice_reference=new_invoice.reference, system_invoice=system_invoice) - - return order - -def old_internal_create_booking_invoice(booking, checkout_response): - if not checkout_response.history: - raise Exception('There was a problem retrieving the invoice for this booking') - last_redirect = checkout_response.history[-2] - reference = last_redirect.url.split('=')[1] - try: - Invoice.objects.get(reference=reference) - except Invoice.DoesNotExist: - raise Exception("There was a problem attaching an invoice for this booking") - book_inv = BookingInvoice.objects.get_or_create(booking=booking,invoice_reference=reference) - return book_inv - -def internal_create_booking_invoice(booking, reference): - try: - Invoice.objects.get(reference=reference) - except Invoice.DoesNotExist: - raise Exception("There was a problem attaching an invoice for this booking") - book_inv = BookingInvoice.objects.get_or_create(booking=booking,invoice_reference=reference) - return book_inv - - - -def internal_booking(request,booking_details,internal=True,updating=False): - json_booking = request.data - booking = None - try: - booking = create_or_update_booking(request, booking_details, updating, override_checks=internal) - with transaction.atomic(): - set_session_booking(request.session,booking) - # Get line items - booking_arrival = booking.arrival.strftime('%d-%m-%Y') - booking_departure = booking.departure.strftime('%d-%m-%Y') - reservation = u"Reservation for {} confirmation PS{}".format(u'{} {}'.format(booking.customer.first_name,booking.customer.last_name), booking.id) - lines = price_or_lineitems(request,booking,booking.campsite_id_list) - - # Proceed to generate invoice - checkout_response = checkout(request,booking,lines,invoice_text=reservation,internal=True) - # Change the type of booking - booking.booking_type = 0 - booking.save() - - # FIXME: replace with session check - invoice = None - if 'invoice=' in checkout_response.url: - invoice = checkout_response.url.split('invoice=', 1)[1] - else: - for h in reversed(checkout_response.history): - if 'invoice=' in h.url: - invoice = h.url.split('invoice=', 1)[1] - break - print ("-== internal_booking ==-") - internal_create_booking_invoice(booking, invoice) - delete_session_booking(request.session) - send_booking_invoice(booking) - return booking - - except: - if booking: - booking.delete() - raise - - -def old_internal_booking(request,booking_details,internal=True,updating=False): - json_booking = request.data - booking = None - try: - booking = create_or_update_booking(request,booking_details,updating) - with transaction.atomic(): - set_session_booking(request.session,booking) - # Get line items - booking_arrival = booking.arrival.strftime('%d-%m-%Y') - booking_departure = booking.departure.strftime('%d-%m-%Y') - reservation = u"Reservation for {} from {} to {} at {}".format(u'{} {}'.format(booking.customer.first_name,booking.customer.last_name),booking_arrival,booking_departure,booking.mooringarea.name) - lines = price_or_lineitems(request,booking,booking.campsite_id_list) - - # Proceed to generate invoice - checkout_response = checkout(request,booking,lines,invoice_text=reservation,internal=True) - # Change the type of booking - booking.booking_type = 0 - booking.save() - internal_create_booking_invoice(booking, checkout_response) - delete_session_booking(request.session) - send_booking_invoice(booking) - return booking - - except: - if booking: - booking.delete() - raise - -def set_session_booking(session, booking): - session['ps_booking'] = booking.id - session.modified = True - -def get_session_admissions_booking(session): - if 'ad_booking' in session: - booking_id = session['ad_booking'] - else: - raise Exception('Admissions booking not in Session') - - try: - return AdmissionsBooking.objects.get(id=booking_id) - except AdmissionsBooking.DoesNotExist: - raise Exception('Admissions booking not found for booking_id {}'.format(booking_id)) - -def get_annual_admission_session_booking(session): - if 'annual_admission_booking' in session: - booking_id = session['annual_admission_booking'] - else: - raise Exception('Annual Admission Booking not in Session') - - try: - return BookingAnnualAdmission.objects.get(id=booking_id) - except BookingAnnualAdmission.DoesNotExist: - raise Exception('Annual Admission Booking not found for booking_id {}'.format(booking_id)) - -def delete_annual_admission_session_booking(session): - if 'annual_admission_booking' in session: - del session['annual_admission_booking'] - session.modified = True - -def delete_session_admissions_booking(session): - if 'ad_booking' in session: - del session['ad_booking'] - session.modified = True - -def get_session_booking(session): - if 'ps_booking' in session: - booking_id = session['ps_booking'] - else: - raise Exception('Booking not in Session') - - try: - return Booking.objects.get(id=booking_id) - except Booking.DoesNotExist: - raise Exception('Booking not found for booking_id {}'.format(booking_id)) - -def delete_session_booking(session): - if 'ps_booking' in session: - del session['ps_booking'] - session.modified = True - -def daterange(start_date, end_date): - for n in range(int ((end_date - start_date).days)): - yield start_date + timedelta(n) - - -def admissions_lines(booking_mooring): - lines = [] - for bm in booking_mooring: - # Convert the from and to dates of this booking to just plain dates in local time. - # Append them to a list. - if bm.campsite.mooringarea.park.entry_fee_required: - from_dt = bm.from_dt - timestamp = calendar.timegm(from_dt.timetuple()) - local_dt = datetime.fromtimestamp(timestamp) - from_dt = local_dt.replace(microsecond=from_dt.microsecond) - to_dt = bm.to_dt - timestamp = calendar.timegm(to_dt.timetuple()) - local_dt = datetime.fromtimestamp(timestamp) - to_dt = local_dt.replace(microsecond=to_dt.microsecond) - group = MooringAreaGroup.objects.filter(moorings__in=[bm.campsite.mooringarea,])[0].id - lines.append({'from': from_dt, 'to': to_dt, 'group':group}) - # Sort the list by date from. - new_lines = sorted(lines, key=lambda line: line['from']) - i = 0 - lines = [] - latest_from = None - latest_to = None - # Loop through the list, if first instance, then this line's from date is the first admission fee. - # Then compare this TO value to the next FROM value. If they are not the same or overlapping dates - # add this date to the list, using the latest from and this TO value. - while i < len(new_lines): - if i == 0: - latest_from = new_lines[i]['from'].date() - if i < len(new_lines)-1: - if new_lines[i]['to'].date() < new_lines[i+1]['from'].date(): - latest_to = new_lines[i]['to'].date() - else: - # if new_lines[i]['from'].date() > new_lines[i-1]['to'].date(): - latest_to = new_lines[i]['to'].date() - - if latest_to: - lines.append({"rowid":'admission_fee_id'+str(i), 'id': i,'from':datetime.strftime(latest_from, '%d %b %Y'), 'to': datetime.strftime(latest_to, '%d %b %Y'), 'admissionFee': 0, 'group': new_lines[i]['group']}) - if i < len(new_lines)-1: - latest_from = new_lines[i+1]['from'].date() - latest_to = None - i+= 1 - return lines - -# Access Level check for Group -def mooring_group_access_level_change(pk,request): - mooring_groups = MooringAreaGroup.objects.filter(members__in=[request.user,]) - if request.user.is_superuser is True: - return True - else: - if ChangeGroup.objects.filter(pk=pk,mooring_group__in=mooring_groups).count() > 0: - return True - - return False - -def mooring_group_access_level_cancel(pk,request): - mooring_groups = MooringAreaGroup.objects.filter(members__in=[request.user,]) - if request.user.is_superuser is True: - return True - else: - if CancelGroup.objects.filter(pk=pk,mooring_group__in=mooring_groups).count() > 0: - return True - - return False - -def mooring_group_access_level_change_options(cg,pk,request): - mooring_groups = MooringAreaGroup.objects.filter(members__in=[request.user,]) - if request.user.is_superuser is True: - return True - else: - cpp = ChangePricePeriod.objects.get(id=pk) - if ChangeGroup.objects.filter(id=cg,change_period__in=[cpp],mooring_group__in=mooring_groups).count() > 0: - return True - - return False - -def mooring_group_access_level_cancel_options(cg,pk,request): - mooring_groups = MooringAreaGroup.objects.filter(members__in=[request.user,]) - if request.user.is_superuser is True: - return True - else: - cpp = CancelPricePeriod.objects.get(id=pk) - if CancelGroup.objects.filter(id=cg,cancel_period__in=[cpp],mooring_group__in=mooring_groups).count() > 0: - return True - - return False - -def mooring_group_access_level_booking_period(pk,request): - mooring_groups = MooringAreaGroup.objects.filter(members__in=[request.user,]) - if request.user.is_superuser is True: - return True - else: - if BookingPeriod.objects.filter(pk=pk,mooring_group__in=mooring_groups).count() > 0: - return True - - return False - -def mooring_group_access_level_annual_booking_period(pk,request): - mooring_groups = MooringAreaGroup.objects.filter(members__in=[request.user,]) - if request.user.is_superuser is True: - return True - else: - if models.AnnualBookingPeriodGroup.objects.filter(pk=pk,mooring_group__in=mooring_groups).count() > 0: - return True - - return False - - -def mooring_group_access_level_booking_period_option(pk,bp_group_id,request): - mooring_groups = MooringAreaGroup.objects.filter(members__in=[request.user,]) - if request.user.is_superuser is True: - return True - else: - bpo = BookingPeriodOption.objects.get(id=pk) - if BookingPeriod.objects.filter(pk=bp_group_id,booking_period__in=[bpo],mooring_group__in=mooring_groups).count() > 0: - return True - return False - - -def check_mooring_admin_access(request): - if request.user.is_superuser is True: - return True - else: - if request.user.groups.filter(name__in=['Mooring Admin']).exists(): - return True - return False - - -def get_provinces(country_code): - provinces = [] - read_data = "" - json_response = [] - with io.open(settings.BASE_DIR+'/mooring/data/provinces.json', "r", encoding="utf-8") as my_file: - read_data = my_file.read() - provinces = json.loads(read_data) - - for p in provinces: - if p['country'] == country_code: - if 'short' in p: - json_response.append(p) - - return json_response - - - - -def booking_success(basket, booking, context_processor): - - print("MLINE 1.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - order = Order.objects.get(basket=basket[0]) - invoice = Invoice.objects.get(order_number=order.number) - print("MLINE 1.02", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - invoice_ref = invoice.reference - book_inv, created = BookingInvoice.objects.get_or_create(booking=booking, invoice_reference=invoice_ref) - print("MLINE 1.03", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - #invoice_ref = request.GET.get('invoice') - if booking.booking_type == 3: - print("MLINE 2.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - try: - inv = Invoice.objects.get(reference=invoice_ref) - order = Order.objects.get(number=inv.order_number) - order.user = booking.customer - order.save() - except Invoice.DoesNotExist: - print ("INVOICE ERROR") - logger.error('{} tried making a booking with an incorrect invoice'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user')) - return redirect('public_make_booking') - if inv.system not in ['0516']: - print ("SYSTEM ERROR") - logger.error('{} tried making a booking with an invoice from another system with reference number {}'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user',inv.reference)) - return redirect('public_make_booking') - print("MLINE 3.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - if book_inv: - print("MLINE 4.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - if booking.old_booking: - old_booking = Booking.objects.get(id=booking.old_booking.id) - old_booking.booking_type = 4 - old_booking.cancelation_time = datetime.now() - old_booking.canceled_by = booking.created_by #request.user - old_booking.save() - booking_items = MooringsiteBooking.objects.filter(booking=old_booking) - # Find admissions booking for old booking - if old_booking.admission_payment: - old_booking.admission_payment.booking_type = 4 - old_booking.admission_payment.cancelation_time = datetime.now() - old_booking.admission_payment.canceled_by = booking.created_by #request.user - old_booking.admission_payment.save() - for bi in booking_items: - bi.booking_type = 4 - bi.save() - print("MLINE 5.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - booking_items_current = MooringsiteBooking.objects.filter(booking=booking) - for bi in booking_items_current: - if str(bi.id) in booking.override_lines: - bi.amount = Decimal(booking.override_lines[str(bi.id)]) - bi.save() - print("MLINE 6.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - msb = MooringsiteBooking.objects.filter(booking=booking).order_by('from_dt') - from_date = msb[0].from_dt - to_date = msb[msb.count()-1].to_dt - timestamp = calendar.timegm(from_date.timetuple()) - local_dt = datetime.fromtimestamp(timestamp) - from_dt = local_dt.replace(microsecond=from_date.microsecond) - from_date_converted = from_dt.date() - timestamp = calendar.timegm(to_date.timetuple()) - local_dt = datetime.fromtimestamp(timestamp) - to_dt = local_dt.replace(microsecond=to_date.microsecond) - to_date_converted = to_dt.date() - booking.arrival = from_date_converted - booking.departure = to_date_converted - # set booking to be permanent fixture - booking.booking_type = 1 # internet booking - booking.expiry_time = None - print("MLINE 7.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - update_payments(invoice_ref) - print("MLINE 8.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - #Calculate Admissions and create object - if booking.admission_payment: - ad_booking = AdmissionsBooking.objects.get(pk=booking.admission_payment.pk) - ad_booking.created_by = booking.created_by - ad_booking.booking_type=1 - print("MLINE 8.02", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - ad_booking.save() - print("MLINE 8.03", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - ad_invoice = AdmissionsBookingInvoice.objects.get_or_create(admissions_booking=ad_booking, invoice_reference=invoice_ref) - print("MLINE 8.04", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - - for al in ad_booking.override_lines.keys(): - ad_line = AdmissionsLine.objects.get(id=int(al)) - ad_line.cost = ad_booking.override_lines[str(al)] - ad_line.save() - print("MLINE 8.05", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - # booking.admission_payment = ad_booking - booking.save() - print("MLINE 9.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - #if not request.user.is_staff: - # print "USER IS NOT STAFF." - #request.session['ps_last_booking'] = booking.id - #utils.delete_session_booking(request.session) - # send out the invoice before the confirmation is sent if total is greater than zero - #if booking.cost_total > 0: - print("MLINE 10.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - try: - emails.send_booking_invoice(booking,context_processor) - except Exception as e: - print ("Error Sending Invoice ("+str(booking.id)+") :"+str(e)) - # for fully paid bookings, fire off confirmation emaili - #if booking.invoice_status == 'paid': - print("MLINE 11.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - try: - emails.send_booking_confirmation(booking,context_processor) - except Exception as e: - print ("Error Sending Booking Confirmation ("+str(booking.id)+") :"+str(e)) - print("MLINE 12.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - refund_failed = None - if models.RefundFailed.objects.filter(booking=booking).count() > 0: - refund_failed = models.RefundFailed.objects.filter(booking=booking) - # Create/Update Vessel in VesselDetails Table - print("MLINE 13.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - try: - - if models.VesselDetail.objects.filter(rego_no=booking.details['vessel_rego']).count() > 0: - vd = models.VesselDetail.objects.filter(rego_no=booking.details['vessel_rego']) - p = vd[0] - p.vessel_size=booking.details['vessel_size'] - p.vessel_draft=booking.details['vessel_draft'] - p.vessel_beam=booking.details['vessel_beam'] - p.vessel_weight=booking.details['vessel_weight'] - p.save() - else: - models.VesselDetail.objects.create(rego_no=booking.details['vessel_rego'], - vessel_size=booking.details['vessel_size'], - vessel_draft=booking.details['vessel_draft'], - vessel_beam=booking.details['vessel_beam'], - vessel_weight=booking.details['vessel_weight'] - ) - print("MLINE 14.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - except: - print ("ERROR: create vesseldetails on booking success") - - context = { - 'booking': booking, - 'book_inv': [book_inv], - 'refund_failed' : refund_failed - } - print("MLINE 15.01", datetime.now().strftime("%d/%m/%Y %H:%M:%S")) - return context - - - -def booking_annual_admission_success(basket, booking, context_processor): - - order = Order.objects.get(basket=basket[0]) - invoice = Invoice.objects.get(order_number=order.number) - invoice_ref = invoice.reference - book_inv, created = models.BookingAnnualInvoice.objects.get_or_create(booking_annual_admission=booking, invoice_reference=invoice_ref) - - #invoice_ref = request.GET.get('invoice') - if booking.booking_type == 3: - try: - inv = Invoice.objects.get(reference=invoice_ref) - order = Order.objects.get(number=inv.order_number) - order.user = booking.customer - order.save() - except Invoice.DoesNotExist: - print ("INVOICE ERROR") - logger.error('{} tried making a booking with an incorrect invoice'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user')) - return redirect('public_make_booking') - if inv.system not in ['0516']: - print ("SYSTEM ERROR") - logger.error('{} tried making a booking with an invoice from another system with reference number {}'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user',inv.reference)) - return redirect('public_make_booking') - - if book_inv: - # set booking to be permanent fixture - booking.booking_type = 1 # internet booking - booking.expiry_time = None - update_payments(invoice_ref) - #Calculate Admissions and create object - booking.save() - #if not request.user.is_staff: - # print "USER IS NOT STAFF." - print ("SEND EMAIL") - - try: - emails.send_annual_admission_booking_invoice(booking,context_processor) - except Exception as e: - print ("Error Sending Invoice ("+str(booking.id)+") :"+str(e)) - - try: - emails.send_new_annual_admission_booking_internal(booking,context_processor) - except Exception as e: - print ("Error Sending Booking Confirmation ("+str(booking.id)+") :"+str(e)) - - # for fully paid bookings, fire off confirmation emaili - #if booking.invoice_status == 'paid': - context = { - 'booking': booking, - 'book_inv': [book_inv], - } - - try: - - if models.VesselDetail.objects.filter(rego_no=booking.details['vessel_rego']).count() > 0: - vd = models.VesselDetail.objects.filter(rego_no=booking.details['vessel_rego']) - p = vd[0] - p.vessel_name=booking.details['vessel_name'] - p.save() - except: - print ("ERROR: create vesseldetails on booking success") - - - print ("COMPLETED SUCCESS") - return context - - -def booking_admission_success(basket, booking, context_processor): - - arrival = models.AdmissionsLine.objects.filter(admissionsBooking=booking)[0].arrivalDate - overnight = models.AdmissionsLine.objects.filter(admissionsBooking=booking)[0].overnightStay - - order = Order.objects.get(basket=basket[0]) - invoice = Invoice.objects.get(order_number=order.number) - invoice_ref = invoice.reference - - #invoice_ref = request.GET.get('invoice') - - if booking.booking_type == 3: - try: - inv = Invoice.objects.get(reference=invoice_ref) - order = Order.objects.get(number=inv.order_number) - order.user = booking.customer - order.save() - except Invoice.DoesNotExist: - logger.error('{} tried making a booking with an incorrect invoice'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user')) - return redirect('admissions', args=(booking.location.key,)) - - if inv.system not in ['0516']: - logger.error('{} tried making a booking with an invoice from another system with reference number {}'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user',inv.reference)) - return redirect('admissions', args=(booking.location.key,)) - - try: - b = AdmissionsBookingInvoice.objects.get(invoice_reference=invoice_ref) - logger.error('{} tried making an admission booking with an already used invoice with reference number {}'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user',inv.reference)) - return redirect('admissions', args=(booking.location.key,)) - except AdmissionsBookingInvoice.DoesNotExist: - logger.info('{} finished temporary booking {}, creating new AdmissionBookingInvoice with reference {}'.format('User {} with id {}'.format(booking.customer.get_full_name(),booking.customer.id) if booking.customer else 'An anonymous user',booking.id, invoice_ref)) - # FIXME: replace with server side notify_url callback - admissionsInvoice = AdmissionsBookingInvoice.objects.get_or_create(admissions_booking=booking, invoice_reference=invoice_ref) - - # set booking to be permanent fixture - booking.booking_type = 1 # internet booking - booking.save() - #request.session['ad_last_booking'] = booking.id - #utils.delete_session_admissions_booking(request.session) - - try: - # send out the invoice before the confirmation is sent - emails.send_admissions_booking_invoice(booking,context_processor) - except Exception as e: - print ("Error Sending Invoice ("+str(booking.id)+") :"+str(e)) - - try: - # for fully paid bookings, fire off confirmation email - emails.send_admissions_booking_confirmation(booking,context_processor) - except Exception as e: - print ("Error Sending Booking Confirmation ("+str(booking.id)+") :"+str(e)) - - - context = { - 'admissionsBooking': booking, - 'arrival' : arrival, - 'overnight': overnight, - 'admissionsInvoice': [invoice_ref] - } - From 44df2a65d155bc227f58e55f8ce7061dcbb26acd Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 05:20:45 +0000 Subject: [PATCH 02/10] Removed: TemporaryDocuments --- .../components/approvals/models.py | 1 - mooringlicensing/components/main/admin.py | 7 +------ mooringlicensing/components/main/models.py | 20 ------------------- .../components/main/serializers.py | 5 +---- .../components/proposals/utils.py | 3 --- .../0372_delete_temporarydocument_and_more.py | 19 ++++++++++++++++++ 6 files changed, 21 insertions(+), 34 deletions(-) create mode 100644 mooringlicensing/migrations/0372_delete_temporarydocument_and_more.py diff --git a/mooringlicensing/components/approvals/models.py b/mooringlicensing/components/approvals/models.py index bddf1407b..bbd9fc9bd 100755 --- a/mooringlicensing/components/approvals/models.py +++ b/mooringlicensing/components/approvals/models.py @@ -71,7 +71,6 @@ def update_approval_doc_filename(instance, filename): def update_approval_comms_log_filename(instance, filename): return '{}/proposals/{}/approvals/communications/{}'.format(settings.MEDIA_APP_DIR, instance.log_entry.approval.current_proposal.id,filename) -#TODO this is not used or accessed anywhere except when a mooring is offered (upload) - consider removing or using class WaitingListOfferDocument(Document): @staticmethod def relative_path_to_file(approval_id, filename): diff --git a/mooringlicensing/components/main/admin.py b/mooringlicensing/components/main/admin.py index 62e871763..81e01b457 100644 --- a/mooringlicensing/components/main/admin.py +++ b/mooringlicensing/components/main/admin.py @@ -2,7 +2,7 @@ from django.contrib import admin from mooringlicensing.components.main.models import VesselSizeCategory, VesselSizeCategoryGroup, ApplicationType, \ - NumberOfDaysSetting, NumberOfDaysType, TemporaryDocument + NumberOfDaysSetting, NumberOfDaysType from mooringlicensing.components.payments_ml.models import OracleCodeItem from django.utils.html import mark_safe @@ -111,11 +111,6 @@ def get_vessel_size_categories(self, obj): get_vessel_size_categories.short_description = 'vessel size category' -@admin.register(TemporaryDocument) -class TemporaryDocumentAdmin(admin.ModelAdmin): - pass - - class OracleCodeItemInline(admin.TabularInline): model = OracleCodeItem extra = 0 diff --git a/mooringlicensing/components/main/models.py b/mooringlicensing/components/main/models.py index f032fd9aa..df62a84c3 100755 --- a/mooringlicensing/components/main/models.py +++ b/mooringlicensing/components/main/models.py @@ -207,24 +207,6 @@ class Meta: def __str__(self): return 'System Maintenance: {} ({}) - starting {}, ending {}'.format(self.name, self.description, self.start_date, self.end_date) -#TODO appears to be unused - check if needed -class TemporaryDocumentCollection(models.Model): - - class Meta: - app_label = 'mooringlicensing' - -#TODO appears to be unused - check if needed -class TemporaryDocument(Document): - temp_document_collection = models.ForeignKey( - TemporaryDocumentCollection, - related_name='documents', - on_delete=models.CASCADE, - ) - _file = models.FileField(storage=private_storage,max_length=255) - - class Meta: - app_label = 'mooringlicensing' - def update_electoral_roll_doc_filename(instance, filename): return '{}/emailusers/{}/documents/{}'.format(settings.MEDIA_APP_DIR, instance.emailuser.id,filename) @@ -435,8 +417,6 @@ def get_setting_by_date(number_of_days_type, target_date=datetime.now(pytz.timez reversion.register(ApplicationType, follow=['proposalstandardrequirement_set', 'feeseason_set', 'feeconstructor_set', 'oracle_code_items']) reversion.register(GlobalSettings, follow=[]) reversion.register(SystemMaintenance, follow=[]) -reversion.register(TemporaryDocumentCollection, follow=['documents']) -reversion.register(TemporaryDocument, follow=[]) reversion.register(VesselSizeCategoryGroup, follow=['vessel_size_categories', 'fee_constructors']) reversion.register(VesselSizeCategory, follow=['feeitem_set']) reversion.register(NumberOfDaysType, follow=['settings']) diff --git a/mooringlicensing/components/main/serializers.py b/mooringlicensing/components/main/serializers.py index 294b5c887..570cf8f65 100755 --- a/mooringlicensing/components/main/serializers.py +++ b/mooringlicensing/components/main/serializers.py @@ -1,9 +1,6 @@ from rest_framework import serializers from mooringlicensing import settings -from mooringlicensing.components.main.models import ( - CommunicationsLogEntry, - TemporaryDocumentCollection, - ) +from mooringlicensing.components.main.models import CommunicationsLogEntry from ledger_api_client.ledger_models import EmailUserRO, Invoice from mooringlicensing.ledger_api_utils import get_invoice_payment_status diff --git a/mooringlicensing/components/proposals/utils.py b/mooringlicensing/components/proposals/utils.py index ec6d5bce0..44a0145db 100644 --- a/mooringlicensing/components/proposals/utils.py +++ b/mooringlicensing/components/proposals/utils.py @@ -8,7 +8,6 @@ from mooringlicensing import settings import json from mooringlicensing.components.main.utils import get_dot_vessel_information -from mooringlicensing.components.main.models import TemporaryDocumentCollection from mooringlicensing.components.main.process_document import save_vessel_registration_document_obj from mooringlicensing.components.proposals.models import ( VesselOwnershipCompanyOwnership, @@ -630,8 +629,6 @@ def handle_document(instance, vessel_ownership, request_data, *args, **kwargs): temporary_document_collection_id = request_data.get('proposal', {}).get('temporary_document_collection_id') if temporary_document_collection_id: temp_doc_collection = None - if TemporaryDocumentCollection.objects.filter(id=temporary_document_collection_id): - temp_doc_collection = TemporaryDocumentCollection.objects.filter(id=temporary_document_collection_id)[0] if temp_doc_collection: for doc in temp_doc_collection.documents.all(): save_vessel_registration_document_obj(vessel_ownership, doc) diff --git a/mooringlicensing/migrations/0372_delete_temporarydocument_and_more.py b/mooringlicensing/migrations/0372_delete_temporarydocument_and_more.py new file mode 100644 index 000000000..de4f34cde --- /dev/null +++ b/mooringlicensing/migrations/0372_delete_temporarydocument_and_more.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.9 on 2024-10-29 05:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mooringlicensing', '0371_complianceamendmentrequest'), + ] + + operations = [ + migrations.DeleteModel( + name='TemporaryDocument', + ), + migrations.DeleteModel( + name='TemporaryDocumentCollection', + ), + ] From 0ffe8751d466a46f04022bb510efa7f037d5ad3f Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 05:23:01 +0000 Subject: [PATCH 03/10] Removed: On-hold status functionality --- .../components/proposals/models.py | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/mooringlicensing/components/proposals/models.py b/mooringlicensing/components/proposals/models.py index d90fe2acb..20f30c382 100644 --- a/mooringlicensing/components/proposals/models.py +++ b/mooringlicensing/components/proposals/models.py @@ -288,9 +288,6 @@ class Proposal(DirtyFieldsMixin, RevisionedMixin): processing_status = models.CharField('Processing Status', max_length=40, choices=PROCESSING_STATUS_CHOICES, default=PROCESSING_STATUS_CHOICES[0][0]) - - #TODO used by on_hold - which is not implemented (remove if on_hold removed) - prev_processing_status = models.CharField(max_length=40, blank=True, null=True) approval = models.ForeignKey('mooringlicensing.Approval',null=True,blank=True, on_delete=models.SET_NULL) previous_application = models.ForeignKey('self', on_delete=models.SET_NULL, blank=True, null=True, related_name="succeeding_proposals") @@ -298,9 +295,6 @@ class Proposal(DirtyFieldsMixin, RevisionedMixin): proposed_decline_status = models.BooleanField(default=False) title = models.CharField(max_length=255,null=True,blank=True) - #TODO currently not in use, but may be required (remove if not) - approval_comment = models.TextField(blank=True) - #If the proposal is created as part of migration of approvals migrated = models.BooleanField(default=False) @@ -1386,39 +1380,6 @@ def final_decline(self, request, details): except: raise - #TODO not in use and has outdated code - remove or check if required - def on_hold(self,request): - with transaction.atomic(): - try: - if not self.can_assess(request.user): - raise exceptions.ProposalNotAuthorized() - if not (self.processing_status == 'with_assessor' or self.processing_status == 'with_referral'): - raise ValidationError('You cannot put on hold if it is not with assessor or with referral') - - self.prev_processing_status = self.processing_status - self.processing_status = self.PROCESSING_STATUS_ONHOLD - self.save() - # Log proposal action - self.log_user_action(ProposalUserAction.ACTION_PUT_ONHOLD.format(self.id),request) - except: - raise - - #TODO not in use and has outdated code - remove or check if required - def on_hold_remove(self,request): - with transaction.atomic(): - try: - if not self.can_assess(request.user): - raise exceptions.ProposalNotAuthorized() - if self.processing_status != 'on_hold': - raise ValidationError('You cannot remove on hold if it is not currently on hold') - - self.processing_status = self.prev_processing_status - self.prev_processing_status = self.PROCESSING_STATUS_ONHOLD - self.save() - # Log proposal action - self.log_user_action(ProposalUserAction.ACTION_REMOVE_ONHOLD.format(self.id),request) - except: - raise def proposed_approval(self, request, details): from mooringlicensing.components.approvals.models import MooringOnApproval From 7647a436235d877c840d0f316fff569b31d66365 Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 05:41:49 +0000 Subject: [PATCH 04/10] Removed: unused exported field for several models --- .../components/approvals/models.py | 4 --- .../components/proposals/models.py | 8 ----- .../0373_remove_approval_exported_and_more.py | 33 +++++++++++++++++++ 3 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 mooringlicensing/migrations/0373_remove_approval_exported_and_more.py diff --git a/mooringlicensing/components/approvals/models.py b/mooringlicensing/components/approvals/models.py index bbd9fc9bd..56495ff19 100755 --- a/mooringlicensing/components/approvals/models.py +++ b/mooringlicensing/components/approvals/models.py @@ -318,10 +318,6 @@ class Approval(RevisionedMixin): renewal_count = models.PositiveSmallIntegerField('Number of times an Approval has been renewed', default=0) migrated = models.BooleanField(default=False) - #TODO review this - look in to how exports are supposed to work, remove if not required - # for cron job - exported = models.BooleanField(default=False) # must be False after every add/edit - moorings = models.ManyToManyField(Mooring, through=MooringOnApproval) vessel_ownerships = models.ManyToManyField(VesselOwnership, through=VesselOwnershipOnApproval) wla_order = models.PositiveIntegerField(help_text='wla order per mooring bay', null=True) diff --git a/mooringlicensing/components/proposals/models.py b/mooringlicensing/components/proposals/models.py index 20f30c382..257c6f1e1 100644 --- a/mooringlicensing/components/proposals/models.py +++ b/mooringlicensing/components/proposals/models.py @@ -4419,10 +4419,6 @@ class VesselDetails(RevisionedMixin): # ManyToManyField link in Proposal created = models.DateTimeField(default=timezone.now) updated = models.DateTimeField(auto_now=True) - # TODO review how this works and determine if needed - # for cron job - exported = models.BooleanField(default=False) # must be False after every add/edit - objects = models.Manager() filtered_objects = VesselDetailsManager() @@ -4512,10 +4508,6 @@ class VesselOwnership(RevisionedMixin): created = models.DateTimeField(default=timezone.now) updated = models.DateTimeField(auto_now=True) - # TODO review how this works and determine if needed - # for cron job - exported = models.BooleanField(default=False) # must be False after every add/edit - objects = models.Manager() filtered_objects = VesselOwnershipManager() ## Name as shown on DoT registration papers diff --git a/mooringlicensing/migrations/0373_remove_approval_exported_and_more.py b/mooringlicensing/migrations/0373_remove_approval_exported_and_more.py new file mode 100644 index 000000000..e2c94c16e --- /dev/null +++ b/mooringlicensing/migrations/0373_remove_approval_exported_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.0.9 on 2024-10-29 05:36 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mooringlicensing', '0372_delete_temporarydocument_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='approval', + name='exported', + ), + migrations.RemoveField( + model_name='proposal', + name='approval_comment', + ), + migrations.RemoveField( + model_name='proposal', + name='prev_processing_status', + ), + migrations.RemoveField( + model_name='vesseldetails', + name='exported', + ), + migrations.RemoveField( + model_name='vesselownership', + name='exported', + ), + ] From 93f0d4227f0f34323ece01a74128141422313ce7 Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 06:08:14 +0000 Subject: [PATCH 05/10] Removed: unused cronjob file (referencing das) --- mooringlicensing/utils/mooringlicensing-cronjobs | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 mooringlicensing/utils/mooringlicensing-cronjobs diff --git a/mooringlicensing/utils/mooringlicensing-cronjobs b/mooringlicensing/utils/mooringlicensing-cronjobs deleted file mode 100755 index 782981280..000000000 --- a/mooringlicensing/utils/mooringlicensing-cronjobs +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -# Execute every minute. Polls the CommercialOperator Admin table SystemMaintenance, and checks if the application can be taken down at the time indicated in the Admin table -* * * * * root cd /var/www/commercialoperator-prod.8088 && source venv/bin/activate && python manage_ds.py system_maintenance_check >/dev/null 2>&1 -# -# Every day (at 02:00 hrs), run the daily commercialoperator management commands batch -0 2 * * * www-data cd /var/www/commercialoperator-prod.8088 && source venv/bin/activate && python manage_ds.py cron_tasks >/dev/null 2>&1 From 63de7ba8f386326f0b8fc7c5e9380bf9da45d738 Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 06:37:16 +0000 Subject: [PATCH 06/10] Update: cleared out commented out code in management commands --- .../commands/approval_renewal_notices.py | 2 -- ...l_approvals_due_to_no_vessels_nominated.py | 9 --------- ...licence_application_due_to_no_documents.py | 14 ------------- ...ng_licence_application_due_to_no_submit.py | 16 --------------- .../commands/export_and_email_sticker_data.py | 9 +-------- .../commands/export_to_mooring_booking.py | 7 ------- .../export_to_mooring_booking_cron_task.py | 12 ++--------- .../management/commands/import_lotus_notes.py | 4 +--- .../commands/import_mooring_bookings_data.py | 2 -- .../commands/import_sticker_data.py | 13 ------------ .../commands/ml_migration_export_script.py | 4 ---- .../commands/remove_unpaid_dcv_admissions.py | 3 --- .../commands/send_compliance_reminder.py | 2 -- .../commands/send_endorser_reminder.py | 12 ----------- ...licence_application_submit_due_reminder.py | 3 --- .../send_reminder_submission_of_mla.py | 10 ---------- .../commands/send_vessel_nominate_reminder.py | 20 ------------------- .../commands/system_maintenance_check.py | 3 --- .../commands/update_approval_status.py | 5 ----- .../commands/update_compliance_status.py | 19 ------------------ 20 files changed, 4 insertions(+), 165 deletions(-) diff --git a/mooringlicensing/management/commands/approval_renewal_notices.py b/mooringlicensing/management/commands/approval_renewal_notices.py index af075430a..edebf550b 100755 --- a/mooringlicensing/management/commands/approval_renewal_notices.py +++ b/mooringlicensing/management/commands/approval_renewal_notices.py @@ -96,8 +96,6 @@ def handle(self, *args, **options): self.perform_per_type(CODE_DAYS_FOR_RENEWAL_ML, MooringLicence, updates, errors) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors)>0 else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/cancel_approvals_due_to_no_vessels_nominated.py b/mooringlicensing/management/commands/cancel_approvals_due_to_no_vessels_nominated.py index 236de51d2..8187ec235 100755 --- a/mooringlicensing/management/commands/cancel_approvals_due_to_no_vessels_nominated.py +++ b/mooringlicensing/management/commands/cancel_approvals_due_to_no_vessels_nominated.py @@ -29,19 +29,13 @@ def perform(self, approval_type, today, **options): # Retrieve the number of days before expiry date of the approvals to email if approval_type == WaitingListAllocation.code: - # days_type = NumberOfDaysType.objects.get(code=CODE_DAYS_BEFORE_END_OF_SIX_MONTH_PERIOD_WLA) approval_class = WaitingListAllocation elif approval_type == MooringLicence.code: - # days_type = NumberOfDaysType.objects.get(code=CODE_DAYS_BEFORE_END_OF_SIX_MONTH_PERIOD_ML) approval_class = MooringLicence else: # Do nothing return - # days_setting = NumberOfDaysSetting.get_setting_by_date(days_type, today) - # if not days_setting: - # # No number of days found - # raise ImproperlyConfigured("NumberOfDays: {} is not defined for the date: {}".format(days_type.name, today)) # NOTE: When sending the reminder: # sold_date + 6months < today # sold_date < today - 6months @@ -86,9 +80,6 @@ def perform(self, approval_type, today, **options): errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len( - # errors) > 0 else 'Errors: 0' - # msg = '

{} ({}) completed. {}. IDs updated: {}.

'.format(cmd_name, approval_type, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_documents.py b/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_documents.py index 866a1a553..661918a86 100755 --- a/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_documents.py +++ b/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_documents.py @@ -2,7 +2,6 @@ from django.utils import timezone from django.core.management.base import BaseCommand -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db.models import Q @@ -46,21 +45,10 @@ def handle(self, *args, **options): queries &= Q(processing_status=Proposal.PROCESSING_STATUS_AWAITING_DOCUMENTS) queries &= Q(lodgement_date__lt=boundary_date) - # For debug - # params = options.get('params') - # debug = True if params.get('debug', 'f').lower() in ['true', 't', 'yes', 'y'] else False - # approval_lodgement_number = params.get('expire_mooring_licence_application_due_to_no_documents_lodgement_number', 'no-lodgement-number') - # if debug: - # queries = queries | Q(lodgement_number__iexact=approval_lodgement_number) - for a in MooringLicenceApplication.objects.filter(queries): try: a.processing_status = Proposal.PROCESSING_STATUS_EXPIRED a.save() - # update WLA internal_status and queue date - # a.waiting_list_allocation.internal_status = 'waiting' - # a.waiting_list_allocation.wla_queue_date = today - # a.waiting_list_allocation.save() # reset Waiting List order a.waiting_list_allocation.set_wla_order() due_date = a.lodgement_date + timedelta(days=days_setting.number_of_days) @@ -74,8 +62,6 @@ def handle(self, *args, **options): errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors) > 0 else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_submit.py b/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_submit.py index 4abb2ea4c..5a523b7e4 100755 --- a/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_submit.py +++ b/mooringlicensing/management/commands/expire_mooring_licence_application_due_to_no_submit.py @@ -2,7 +2,6 @@ from django.utils import timezone from django.core.management.base import BaseCommand -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db.models import Q @@ -43,23 +42,10 @@ def handle(self, *args, **options): queries &= Q(processing_status__in=(Proposal.PROCESSING_STATUS_DRAFT, Proposal.PROCESSING_STATUS_AWAITING_DOCUMENTS)) queries &= Q(date_invited__lt=boundary_date) - # For debug - # params = options.get('params') - # debug = True if params.get('debug', 'f').lower() in ['true', 't', 'yes', 'y'] else False - # approval_lodgement_number = params.get('expire_mooring_licence_application_due_to_no_submit_lodgement_number', 'no-lodgement-number') - # if debug: - # queries = queries | Q(lodgement_number__iexact=approval_lodgement_number) - for a in MooringLicenceApplication.objects.filter(queries): try: a.processing_status = Proposal.PROCESSING_STATUS_EXPIRED a.save() - # update WLA internal_status and queue date - # a.waiting_list_allocation.internal_status = 'waiting' - # a.waiting_list_allocation.wla_queue_date = today - # a.waiting_list_allocation.save() - # reset Waiting List order - # a.waiting_list_allocation.set_wla_order() due_date = a.date_invited + timedelta(days=days_setting.number_of_days) send_expire_mooring_licence_application_email(a, MooringLicenceApplication.REASON_FOR_EXPIRY_NOT_SUBMITTED, due_date) @@ -72,8 +58,6 @@ def handle(self, *args, **options): errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors) > 0 else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/export_and_email_sticker_data.py b/mooringlicensing/management/commands/export_and_email_sticker_data.py index f0c790d16..95fd75786 100644 --- a/mooringlicensing/management/commands/export_and_email_sticker_data.py +++ b/mooringlicensing/management/commands/export_and_email_sticker_data.py @@ -20,13 +20,6 @@ def handle(self, *args, **options): success_filenames, error_filenames = email_stickers_document() cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - error_count = len(errors) + len(error_filenames) - # err_str = 'Errors: {}'.format(error_count) if error_count else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) - cron_email.info(msg) - - # ml1 = Approval.objects.get(lodgement_number='MOL000417') - # ml2 = Approval.objects.get(lodgement_number='MOL000445') - # ml1.child_obj.swap_moorings(ml2.child_obj) + cron_email.info(msg) \ No newline at end of file diff --git a/mooringlicensing/management/commands/export_to_mooring_booking.py b/mooringlicensing/management/commands/export_to_mooring_booking.py index ba15757b3..c8f26ca75 100644 --- a/mooringlicensing/management/commands/export_to_mooring_booking.py +++ b/mooringlicensing/management/commands/export_to_mooring_booking.py @@ -1,11 +1,6 @@ from django.core.management.base import BaseCommand -from django.utils import timezone -from django.conf import settings from mooringlicensing.components.main.utils import export_to_mooring_booking from mooringlicensing.components.approvals.models import Approval -from datetime import date, timedelta - -import itertools import logging logger = logging.getLogger(__name__) @@ -15,7 +10,6 @@ class Command(BaseCommand): help = 'Export to Mooring Bookings VesselLicence' def handle(self, *args, **options): - #errors, updates = import_mooring_bookings_data() approvals_to_export = Approval.objects.filter(export_to_mooring_booking=True) if approvals_to_export: errors = [] @@ -31,5 +25,4 @@ def handle(self, *args, **options): cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() err_str = 'Errors: {}'.format(len(errors)) if len(errors)>0 else 'Errors: 0' msg = '

{} completed. Total Errors: {}:{}. IDs updated: {}.

'.format(cmd_name, err_str, errors, updates) - #logger.info(msg) print(msg) # will redirect to run_cron_tasks.log file, by the parent script diff --git a/mooringlicensing/management/commands/export_to_mooring_booking_cron_task.py b/mooringlicensing/management/commands/export_to_mooring_booking_cron_task.py index 91c5ad4a8..43f89bdea 100755 --- a/mooringlicensing/management/commands/export_to_mooring_booking_cron_task.py +++ b/mooringlicensing/management/commands/export_to_mooring_booking_cron_task.py @@ -1,22 +1,18 @@ from django.core.management.base import BaseCommand from django.conf import settings -#from django.core.mail import send_mail -from django.core.mail import EmailMultiAlternatives, EmailMessage +from django.core.mail import EmailMultiAlternatives from pathlib import Path import subprocess import logging logger = logging.getLogger(__name__) -#LOGFILE = 'logs/run_cron_tasks.log' LOGFILE = 'logs/run_export_to_mooring_booking_cron_task.log' - class Command(BaseCommand): help = 'Run Mooring Licensing Export to Mooring Booking Cron tasks' def handle(self, *args, **options): - #print("cron tasks") stdout_redirect = ' | tee -a {}'.format(LOGFILE) subprocess.call('cat /dev/null > {}'.format(LOGFILE), shell=True) # empty the log file @@ -33,11 +29,7 @@ def send_email(self): body = '' to = settings.CRON_NOTIFICATION_EMAIL if isinstance(settings.NOTIFICATION_EMAIL, list) else [settings.CRON_NOTIFICATION_EMAIL] msg = EmailMultiAlternatives(subject, log_txt, settings.EMAIL_FROM, to, - #attachments=_attachments, cc=cc, bcc=bcc, - #reply_to=reply_to, headers={'System-Environment': email_instance} ) msg.attach_alternative(log_txt, "text/html") - msg.send() - #send_mail(subject, body, settings.EMAIL_FROM, to, fail_silently=False, html_message=log_txt,) - + msg.send() \ No newline at end of file diff --git a/mooringlicensing/management/commands/import_lotus_notes.py b/mooringlicensing/management/commands/import_lotus_notes.py index b53cdbaf3..690599ed7 100755 --- a/mooringlicensing/management/commands/import_lotus_notes.py +++ b/mooringlicensing/management/commands/import_lotus_notes.py @@ -1,14 +1,12 @@ +#TODO is this required? from django.core.management.base import BaseCommand import logging from mooringlicensing.utils.export_clean import clean - logger = logging.getLogger(__name__) -#LOGFILE = 'logs/run_cron_tasks.log' LOGFILE = 'logs/run_export_to_mooring_booking_cron_task.log' - class Command(BaseCommand): help = 'Run Mooring Licensing Export to Mooring Booking Cron tasks' diff --git a/mooringlicensing/management/commands/import_mooring_bookings_data.py b/mooringlicensing/management/commands/import_mooring_bookings_data.py index dfe24a375..e47bdb86f 100644 --- a/mooringlicensing/management/commands/import_mooring_bookings_data.py +++ b/mooringlicensing/management/commands/import_mooring_bookings_data.py @@ -15,8 +15,6 @@ def handle(self, *args, **options): errors, updates = import_mooring_bookings_data() cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors)>0 else 'Errors: 0' - # msg = '

{} completed. Errors: {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/import_sticker_data.py b/mooringlicensing/management/commands/import_sticker_data.py index 732c0483e..124d26818 100644 --- a/mooringlicensing/management/commands/import_sticker_data.py +++ b/mooringlicensing/management/commands/import_sticker_data.py @@ -65,7 +65,6 @@ def handle(self, *args, **options): # 2. Save the attached files into the database ########## typ, data = imapclient.fetch(num, '(RFC822)') - # typ, data = imapclient.fetch(num, '(BODY[HEADER.FIELDS (MESSAGE-ID)])') raw_email = data[0][1] # converts byte literal to string removing b'' @@ -121,7 +120,6 @@ def handle(self, *args, **options): logger.exception('Exception has been raised when importing .xlsx file') continue - # imapclient.store(num, "+FLAGS", "\\Deleted") imapclient.copy(num, "Archive") imapclient.store(num, "+FLAGS", "\\Deleted") except: @@ -145,10 +143,6 @@ def handle(self, *args, **options): send_sticker_import_batch_email(process_summary) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # error_count = len(errors) + len(error_filenames) - # error_count = len(errors) - # err_str = 'Errors: {}'.format(error_count) if error_count else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) @@ -255,10 +249,6 @@ def process_sticker_printing_response(process_summary): if sticker.status in (Sticker.STICKER_STATUS_AWAITING_PRINTING, Sticker.STICKER_STATUS_READY): # sticker should not be in READY status though. sticker.status = Sticker.STICKER_STATUS_CURRENT - #if sticker.sticker_to_replace: # new sticker has the old sticker here if it's created for renewal - # # When this sticker is created for renewal, set 'expiry' status to the old sticker. - # sticker.sticker_to_replace.status = Sticker.STICKER_STATUS_EXPIRED - # sticker.sticker_to_replace.save() sticker.save() process_summary['stickers'].append(sticker) @@ -284,9 +274,6 @@ def process_sticker_printing_response(process_summary): response.processed = True # Update response obj not to process again response.save() - else: - # No fild is saved in the _file field - pass return updates, errors diff --git a/mooringlicensing/management/commands/ml_migration_export_script.py b/mooringlicensing/management/commands/ml_migration_export_script.py index 83829ee9f..7bb0110ce 100644 --- a/mooringlicensing/management/commands/ml_migration_export_script.py +++ b/mooringlicensing/management/commands/ml_migration_export_script.py @@ -1,8 +1,4 @@ from django.core.management.base import BaseCommand -from django.utils import timezone -from django.conf import settings -from django.core.mail import send_mail -from pathlib import Path from mooringlicensing.utils.excel_export import mla_to_excel from mooringlicensing.utils.excel_export import authusers_to_excel from mooringlicensing.utils.excel_export import dcvpermits_to_excel diff --git a/mooringlicensing/management/commands/remove_unpaid_dcv_admissions.py b/mooringlicensing/management/commands/remove_unpaid_dcv_admissions.py index b605463b7..3ac916d2c 100644 --- a/mooringlicensing/management/commands/remove_unpaid_dcv_admissions.py +++ b/mooringlicensing/management/commands/remove_unpaid_dcv_admissions.py @@ -1,7 +1,5 @@ import datetime from django.utils import timezone - -from confy import env from django.core.management.base import BaseCommand from django.db.models import Q @@ -12,7 +10,6 @@ logger = logging.getLogger('cron_tasks') cron_email = logging.getLogger('cron_email') - class Command(BaseCommand): help = 'Remove Unpaid DCV admission applications which has been created before 24 hours' diff --git a/mooringlicensing/management/commands/send_compliance_reminder.py b/mooringlicensing/management/commands/send_compliance_reminder.py index 75a211dfb..7d8b4c5f5 100755 --- a/mooringlicensing/management/commands/send_compliance_reminder.py +++ b/mooringlicensing/management/commands/send_compliance_reminder.py @@ -93,8 +93,6 @@ def handle(self, *args, **options): logger.error('{}\n{}'.format(err_msg, str(e))) errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors)>0 else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/send_endorser_reminder.py b/mooringlicensing/management/commands/send_endorser_reminder.py index e28a5ade4..00813b642 100755 --- a/mooringlicensing/management/commands/send_endorser_reminder.py +++ b/mooringlicensing/management/commands/send_endorser_reminder.py @@ -2,7 +2,6 @@ from django.core.management.base import BaseCommand from django.utils import timezone -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.db.models import Q @@ -37,23 +36,14 @@ def handle(self, *args, **options): logger.info('Running command {}'.format(__name__)) - # For debug - # params = options.get('params') - # debug = True if params.get('debug', 'f').lower() in ['true', 't', 'yes', 'y'] else False - # proposal_id = int(params.get('send_endorser_reminder_id', 0)) - # Construct queries queries = Q() queries &= Q(processing_status=Proposal.PROCESSING_STATUS_AWAITING_ENDORSEMENT) queries &= Q(lodgement_date__lt=boundary_date) - #queries &= Q(endorser_reminder_sent=False) - # if debug: - # queries = queries | Q(id=proposal_id) for a in AuthorisedUserApplication.objects.filter(queries): try: send_endorser_reminder_email(a) - #a.endorser_reminder_sent = True a.save() logger.info('Reminder to endorser sent for Proposal {}'.format(a.lodgement_number)) updates.append(a.lodgement_number) @@ -63,8 +53,6 @@ def handle(self, *args, **options): errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors) > 0 else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/send_mooring_licence_application_submit_due_reminder.py b/mooringlicensing/management/commands/send_mooring_licence_application_submit_due_reminder.py index 970dbff12..cb267ff9d 100755 --- a/mooringlicensing/management/commands/send_mooring_licence_application_submit_due_reminder.py +++ b/mooringlicensing/management/commands/send_mooring_licence_application_submit_due_reminder.py @@ -92,9 +92,6 @@ def perform(self, approval_type, today, **options): cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len( - # errors) > 0 else 'Errors: 0' - # msg = '

{} ({}) completed. {}. IDs updated: {}.

'.format(cmd_name, approval_type, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/send_reminder_submission_of_mla.py b/mooringlicensing/management/commands/send_reminder_submission_of_mla.py index 641085e9f..5b9d780ac 100644 --- a/mooringlicensing/management/commands/send_reminder_submission_of_mla.py +++ b/mooringlicensing/management/commands/send_reminder_submission_of_mla.py @@ -6,7 +6,6 @@ import logging -from mooringlicensing.components.approvals.email import send_reminder_submission_of_mla_mail from mooringlicensing.components.proposals.email import send_invitee_reminder_email from mooringlicensing.components.approvals.models import Approval, WaitingListAllocation, MooringLicence from mooringlicensing.components.main.models import NumberOfDaysType, NumberOfDaysSetting @@ -25,7 +24,6 @@ def handle(self, *args, **options): self.perform(WaitingListAllocation.code, today, **options) self.perform(MooringLicence.code, today, **options) - # self.perform(AuthorisedUserPermit.code, today, **options) def perform(self, approval_type, today, **options): errors = [] @@ -65,11 +63,6 @@ def perform(self, approval_type, today, **options): notification_date_final = today + timedelta(days=days_final_reminder-total_expire_period) logger.info('Running command {}'.format(__name__)) - # For debug - # params = options.get('params') - # debug = True if params.get('debug', 'f').lower() in ['true', 't', 'yes', 'y'] else False - # approval_lodgement_number = params.get('send_vessel_nominate_reminder_lodgement_number', 'no-number') - # Get approvals if approval_type == WaitingListAllocation.code: queries = Q() @@ -94,9 +87,6 @@ def perform(self, approval_type, today, **options): cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len( - # errors) > 0 else 'Errors: 0' - # msg = '

{} ({}) completed. {}. IDs updated: {}.

'.format(cmd_name, approval_type, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/send_vessel_nominate_reminder.py b/mooringlicensing/management/commands/send_vessel_nominate_reminder.py index 1e58ea056..4e2570f97 100755 --- a/mooringlicensing/management/commands/send_vessel_nominate_reminder.py +++ b/mooringlicensing/management/commands/send_vessel_nominate_reminder.py @@ -28,7 +28,6 @@ def handle(self, *args, **options): self.perform(WaitingListAllocation.code, today, **options) self.perform(MooringLicence.code, today, **options) - # self.perform(AuthorisedUserPermit.code, today, **options) def perform(self, approval_type, today, **options): errors = [] @@ -41,9 +40,6 @@ def perform(self, approval_type, today, **options): elif approval_type == MooringLicence.code: days_type = NumberOfDaysType.objects.get(code=CODE_DAYS_BEFORE_END_OF_SIX_MONTH_PERIOD_ML) approval_class = MooringLicence - # elif approval_type == AuthorisedUserPermit.code: - # days_type = NumberOfDaysType.objects.get(code=CODE_DAYS_BEFORE_END_OF_SIX_MONTH_PERIOD_AUP) - # approval_class = AuthorisedUserPermit else: # Do nothing return @@ -59,11 +55,6 @@ def perform(self, approval_type, today, **options): logger.info('Running command {}'.format(__name__)) - # For debug - # params = options.get('params') - # debug = True if params.get('debug', 'f').lower() in ['true', 't', 'yes', 'y'] else False - # approval_lodgement_number = params.get('send_vessel_nominate_reminder_lodgement_number', 'no-number') - # Get approvals if approval_type == WaitingListAllocation.code: queries = Q() @@ -71,8 +62,6 @@ def perform(self, approval_type, today, **options): queries &= Q(current_proposal__vessel_ownership__end_date__isnull=False) queries &= Q(current_proposal__vessel_ownership__end_date__lt=boundary_date) queries &= Q(vessel_nomination_reminder_sent=False) - # if debug: - # queries = queries | Q(lodgement_number__iexact=approval_lodgement_number) approvals = approval_class.objects.filter(queries) elif approval_type == MooringLicence.code: queries = Q() @@ -85,17 +74,11 @@ def perform(self, approval_type, today, **options): # Check if there is at least one vessel which meets the ML vessel requirement if not ml_meet_vessel_requirement(approval, boundary_date): approvals.append(approval) - # if debug: - # apps = MooringLicence.objects.filter(lodgement_number__iexact=approval_lodgement_number) - # if apps: - # approvals.append(apps[0]) elif approval_type == AuthorisedUserPermit.code: queries = Q() queries &= Q(status__in=(Approval.APPROVAL_STATUS_CURRENT, Approval.APPROVAL_STATUS_SUSPENDED)) queries &= Q(current_proposal__vessel_ownership__end_date__lt=boundary_date) queries &= Q(vessel_nomination_reminder_sent=False) - # if debug: - # queries = queries | Q(lodgement_number__iexact=approval_lodgement_number) approvals = approval_class.objects.filter(queries) for a in approvals: @@ -111,9 +94,6 @@ def perform(self, approval_type, today, **options): errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len( - # errors) > 0 else 'Errors: 0' - # msg = '

{} ({}) completed. {}. IDs updated: {}.

'.format(cmd_name, approval_type, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/system_maintenance_check.py b/mooringlicensing/management/commands/system_maintenance_check.py index b5556f047..d67b38472 100755 --- a/mooringlicensing/management/commands/system_maintenance_check.py +++ b/mooringlicensing/management/commands/system_maintenance_check.py @@ -1,8 +1,6 @@ from django.core.management.base import BaseCommand -from django.conf import settings import subprocess import os -from mooringlicensing.components.main.models import SystemMaintenance from mooringlicensing.templatetags.users import system_maintenance_can_start import itertools @@ -29,6 +27,5 @@ def handle(self, *args, **options): if system_maintenance_can_start(): logger.info('Running command {}'.format(__name__)) subprocess.Popen('date 2>&1 | tee -a {}'.format(self.log_file), shell=True) - #subprocess.Popen(settings.SUPERVISOR_STOP_CMD + ' 2>&1 | tee -a {}'.format(self.log_file), shell=True) subprocess.Popen(cmd, shell=True) diff --git a/mooringlicensing/management/commands/update_approval_status.py b/mooringlicensing/management/commands/update_approval_status.py index 5e78b8c10..bbe02f687 100755 --- a/mooringlicensing/management/commands/update_approval_status.py +++ b/mooringlicensing/management/commands/update_approval_status.py @@ -2,13 +2,11 @@ from django.utils import timezone from django.db.models import Q -from mooringlicensing import settings from mooringlicensing.components.approvals.models import ( Approval, ApprovalUserAction, WaitingListAllocation, ) from mooringlicensing.components.proposals.models import ProposalUserAction -from ledger_api_client.models import EmailUser import datetime from mooringlicensing.components.approvals.email import ( send_approval_cancel_email_notification, @@ -42,7 +40,6 @@ def handle(self, *args, **options): for approval in approvals: try: approval.expire_approval() - # a.save() # Saved in the above function...? logger.info('Updated Approval {} status to {}'.format(approval.id, approval.status)) updates.append(approval.lodgement_number) except Exception as e: @@ -178,8 +175,6 @@ def handle(self, *args, **options): errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors)>0 else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) diff --git a/mooringlicensing/management/commands/update_compliance_status.py b/mooringlicensing/management/commands/update_compliance_status.py index 10d1c4f77..18655c451 100755 --- a/mooringlicensing/management/commands/update_compliance_status.py +++ b/mooringlicensing/management/commands/update_compliance_status.py @@ -1,10 +1,6 @@ from django.core.management.base import BaseCommand from django.db.models import Q from django.utils import timezone -from django.conf import settings -from ledger_api_client.models import EmailUser - -from mooringlicensing.components.approvals.models import Approval from mooringlicensing.components.compliances.models import Compliance, ComplianceUserAction from mooringlicensing.components.main.models import NumberOfDaysSetting, NumberOfDaysType import datetime @@ -30,19 +26,6 @@ def handle(self, *args, **options): errors = [] updates = [] logger.info('Running command {}'.format(__name__)) - #for c in Compliance.objects.filter(processing_status=Compliance.PROCESSING_STATUS_FUTURE): - # if c.due_date <= compare_date and c.due_date <= c.approval.expiry_date and c.approval.status == Approval.APPROVAL_STATUS_CURRENT: - # try: - # c.processing_status = Compliance.PROCESSING_STATUS_DUE - # c.customer_status = Compliance.CUSTOMER_STATUS_DUE - # c.save() - # ComplianceUserAction.log_action(c, ComplianceUserAction.ACTION_STATUS_CHANGE.format(c.id), user) - # logger.info('updated Compliance {} status to {}'.format(c.id, c.processing_status)) - # updates.append(c.lodgement_number) - # except Exception as e: - # err_msg = 'Error updating Compliance {} status'.format(c.lodgement_number) - # logger.error('{}\n{}'.format(err_msg, str(e))) - # errors.append(err_msg) # Future --> Due queries = Q() @@ -84,8 +67,6 @@ def handle(self, *args, **options): errors.append(err_msg) cmd_name = __name__.split('.')[-1].replace('_', ' ').upper() - # err_str = 'Errors: {}'.format(len(errors)) if len(errors) > 0 else 'Errors: 0' - # msg = '

{} completed. {}. IDs updated: {}.

'.format(cmd_name, err_str, updates) msg = construct_email_message(cmd_name, errors, updates) logger.info(msg) cron_email.info(msg) From 6c741eedaeabda623a120d1e2718f9781ca23c4c Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 07:40:43 +0000 Subject: [PATCH 07/10] Update: minor adjustments and test session set removal --- mooringlicensing/components/payments_ml/views.py | 4 ++-- mooringlicensing/components/proposals/serializers.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mooringlicensing/components/payments_ml/views.py b/mooringlicensing/components/payments_ml/views.py index 4413bcf10..0578e52b8 100644 --- a/mooringlicensing/components/payments_ml/views.py +++ b/mooringlicensing/components/payments_ml/views.py @@ -69,7 +69,7 @@ def post(self, request, *args, **kwargs): try: with transaction.atomic(): #TODO what does this do? - set_session_dcv_admission_invoice(request.session, dcv_admission_fee) + #set_session_dcv_admission_invoice(request.session, dcv_admission_fee) lines, db_processes = dcv_admission.create_fee_lines() @@ -394,7 +394,7 @@ def post(self, request, *args, **kwargs): try: with transaction.atomic(): #TODO what does this do? - set_session_application_invoice(request.session, application_fee) + #set_session_application_invoice(request.session, application_fee) try: lines, db_processes_after_success = proposal.child_obj.create_fee_lines() # Accessed by WL and AA diff --git a/mooringlicensing/components/proposals/serializers.py b/mooringlicensing/components/proposals/serializers.py index a5f250faa..e8e0140cc 100644 --- a/mooringlicensing/components/proposals/serializers.py +++ b/mooringlicensing/components/proposals/serializers.py @@ -1451,7 +1451,6 @@ class Meta: 'berth_mooring', 'created', 'updated', - 'exported', 'read_only', 'vessel_type_display', ) From a2c893f28dfbb455ce54a5d538c51e2c183504bb Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 08:01:15 +0000 Subject: [PATCH 08/10] Removed: unused invoice session vars --- .../components/payments_ml/utils.py | 112 ------------------ .../components/payments_ml/views.py | 7 +- 2 files changed, 1 insertion(+), 118 deletions(-) diff --git a/mooringlicensing/components/payments_ml/utils.py b/mooringlicensing/components/payments_ml/utils.py index 43848ed23..baf3fc1ec 100644 --- a/mooringlicensing/components/payments_ml/utils.py +++ b/mooringlicensing/components/payments_ml/utils.py @@ -106,118 +106,6 @@ def generate_line_item(application_type, fee_amount_adjusted, fee_constructor, i } -NAME_SESSION_APPLICATION_INVOICE = 'mooringlicensing_app_invoice' -NAME_SESSION_DCV_PERMIT_INVOICE = 'mooringlicensing_dcv_permit_invoice' -NAME_SESSION_DCV_ADMISSION_INVOICE = 'mooringlicensing_dcv_admission_invoice' -NAME_SESSION_STICKER_ACTION_INVOICE = 'mooringlicensing_sticker_action_invoice' - -#TODO review - does not appear to be required -def set_session_sticker_action_invoice(session, application_fee): - """ Application Fee session ID """ - session[NAME_SESSION_STICKER_ACTION_INVOICE] = application_fee.id - session.modified = True - -#TODO review - does not appear to be required -def get_session_sticker_action_invoice(session): - """ Application Fee session ID """ - if NAME_SESSION_STICKER_ACTION_INVOICE in session: - application_fee_id = session[NAME_SESSION_STICKER_ACTION_INVOICE] - else: - raise Exception('Application not in Session') - - try: - return StickerActionFee.objects.get(id=application_fee_id) - except StickerActionFee.DoesNotExist: - raise Exception('StickerActionFee not found for id: {}'.format(application_fee_id)) - -#TODO review - does not appear to be required -def delete_session_sticker_action_invoice(session): - """ Application Fee session ID """ - if NAME_SESSION_STICKER_ACTION_INVOICE in session: - del session[NAME_SESSION_STICKER_ACTION_INVOICE] - session.modified = True - - -def set_session_application_invoice(session, application_fee): - print('in set_session_application_invoice') - - """ Application Fee session ID """ - session[NAME_SESSION_APPLICATION_INVOICE] = application_fee.id - session.modified = True - -#TODO review - does not appear to be required -def get_session_application_invoice(session): - print('in get_session_application_invoice') - - """ Application Fee session ID """ - if NAME_SESSION_APPLICATION_INVOICE in session: - application_fee_id = session[NAME_SESSION_APPLICATION_INVOICE] - else: - # Reach here when the ApplicationFeeSuccessView is accessed 2nd time. Which is correct. - raise Exception('Application not in Session') - - try: - return ApplicationFee.objects.get(id=application_fee_id) - except ApplicationFee.DoesNotExist: - raise - - -#TODO review - does not appear to be required -def delete_session_application_invoice(session): - print('in delete_session_application_invoice') - - """ Application Fee session ID """ - if NAME_SESSION_APPLICATION_INVOICE in session: - del session[NAME_SESSION_APPLICATION_INVOICE] - session.modified = True - -#TODO review - does not appear to be required -def set_session_dcv_permit_invoice(session, dcv_permit_fee): - session[NAME_SESSION_DCV_PERMIT_INVOICE] = dcv_permit_fee.id - session.modified = True - -#TODO review - does not appear to be required -def get_session_dcv_permit_invoice(session): - if NAME_SESSION_DCV_PERMIT_INVOICE in session: - dcv_permit_fee_id = session[NAME_SESSION_DCV_PERMIT_INVOICE] - else: - raise Exception('DcvPermit not in Session') - - try: - return DcvPermitFee.objects.get(id=dcv_permit_fee_id) - except DcvPermitFee.DoesNotExist: - raise Exception('DcvPermit not found for application {}'.format(dcv_permit_fee_id)) - -#TODO review - does not appear to be required -def delete_session_dcv_permit_invoice(session): - if NAME_SESSION_DCV_PERMIT_INVOICE in session: - del session[NAME_SESSION_DCV_PERMIT_INVOICE] - session.modified = True - - -def set_session_dcv_admission_invoice(session, dcv_admission_fee): - session[NAME_SESSION_DCV_ADMISSION_INVOICE] = dcv_admission_fee.id - session.modified = True - -#TODO review - does not appear to be required -def get_session_dcv_admission_invoice(session): - if NAME_SESSION_DCV_ADMISSION_INVOICE in session: - dcv_admission_fee_id = session[NAME_SESSION_DCV_ADMISSION_INVOICE] - else: - raise Exception('DcvAdmission not in Session') - - try: - return DcvAdmissionFee.objects.get(id=dcv_admission_fee_id) - except DcvAdmissionFee.DoesNotExist: - raise Exception('DcvAdmission not found for application {}'.format(dcv_admission_fee_id)) - -#TODO review - does not appear to be required -def delete_session_dcv_admission_invoice(session): - if NAME_SESSION_DCV_ADMISSION_INVOICE in session: - del session[NAME_SESSION_DCV_ADMISSION_INVOICE] - session.modified = True - - def make_serializable(line_items): for line in line_items: for key in line: diff --git a/mooringlicensing/components/payments_ml/views.py b/mooringlicensing/components/payments_ml/views.py index 0578e52b8..c9927eaf1 100644 --- a/mooringlicensing/components/payments_ml/views.py +++ b/mooringlicensing/components/payments_ml/views.py @@ -38,7 +38,7 @@ FeeItemStickerReplacement, FeeItemApplicationFee, FeeCalculation ) from mooringlicensing.components.payments_ml.utils import ( - checkout, set_session_application_invoice, set_session_dcv_admission_invoice, + checkout ) from mooringlicensing.components.proposals.models import ( Proposal, ProposalUserAction, @@ -68,8 +68,6 @@ def post(self, request, *args, **kwargs): try: with transaction.atomic(): - #TODO what does this do? - #set_session_dcv_admission_invoice(request.session, dcv_admission_fee) lines, db_processes = dcv_admission.create_fee_lines() @@ -393,9 +391,6 @@ def post(self, request, *args, **kwargs): try: with transaction.atomic(): - #TODO what does this do? - #set_session_application_invoice(request.session, application_fee) - try: lines, db_processes_after_success = proposal.child_obj.create_fee_lines() # Accessed by WL and AA except Exception as e: From 1156f8bf1d0f370599f99ac6460ef15a6e6c966b Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 29 Oct 2024 08:30:08 +0000 Subject: [PATCH 09/10] Removed: unused pay existing invoice func --- .../components/payments_ml/utils.py | 42 +------------------ 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/mooringlicensing/components/payments_ml/utils.py b/mooringlicensing/components/payments_ml/utils.py index baf3fc1ec..837201d83 100644 --- a/mooringlicensing/components/payments_ml/utils.py +++ b/mooringlicensing/components/payments_ml/utils.py @@ -8,8 +8,7 @@ from ledger_api_client.settings_base import * from mooringlicensing import settings from mooringlicensing.components.payments_ml.models import ( - ApplicationFee, DcvPermitFee, - DcvAdmissionFee, StickerActionFee + ApplicationFee ) logger = logging.getLogger(__name__) @@ -112,41 +111,4 @@ def make_serializable(line_items): if isinstance(line[key], decimal.Decimal): # Convert Decimal to str line[key] = float(line[key]) - return line_items - -#TODO review - does not appear to be required -def checkout_existing_invoice(request, invoice, return_url_ns='public_booking_success'): - - basket, basket_hash = use_existing_basket_from_invoice(invoice.reference) - return_preload_url = settings.MOORING_LICENSING_EXTERNAL_URL + reverse(return_url_ns) - checkout_params = { - 'system': settings.PAYMENT_SYSTEM_ID, - 'fallback_url': request.build_absolute_uri('/'), - 'return_url': request.build_absolute_uri(reverse(return_url_ns)), - 'return_preload_url': return_preload_url, - 'force_redirect': True, - 'invoice_text': invoice.text, - } - - if request.user.is_anonymous(): - # We need to determine the basket owner and set it to the checkout_params to proceed the payment - application_fee = ApplicationFee.objects.filter(invoice_reference=invoice.reference) - if application_fee: - application_fee = application_fee[0] - checkout_params['basket_owner'] = application_fee.proposal.applicant_id - - create_checkout_session(request, checkout_params) - - # use HttpResponse instead of HttpResponseRedirect - HttpResonseRedirect does not pass cookies which is important for ledger to get the correct basket - response = HttpResponse( - " Redirecting please wait: " + reverse('checkout:index') + "") - - # inject the current basket into the redirect response cookies - # or else, anonymous users will be directionless - response.set_cookie( - settings.OSCAR_BASKET_COOKIE_OPEN, basket_hash, - max_age=settings.OSCAR_BASKET_COOKIE_LIFETIME, - secure=settings.OSCAR_BASKET_COOKIE_SECURE, httponly=True - ) - return response \ No newline at end of file + return line_items \ No newline at end of file From 2cbc73a39537ec9ecc6faa2f2a9e418841a6fe05 Mon Sep 17 00:00:00 2001 From: Ashley Date: Wed, 30 Oct 2024 01:21:10 +0000 Subject: [PATCH 10/10] Removed: old util scripts that are no longer needed (most functionality integrated in to one script, export function not required and not maintained) refer to this commit if restoration required --- .../components/payments_ml/utils.py | 6 +- .../includes/staff_menu_extras.html | 3 +- .../utils/annual_admission_migrate.py | 318 -- .../utils/archive/annual_admission_migrate.py | 243 -- .../authorised_user_migrate.15Oct2021.py | 489 --- .../utils/archive/authorised_user_migrate.py | 675 ----- mooringlicensing/utils/archive/dcv_migrate.py | 246 -- mooringlicensing/utils/archive/json_to_csv.py | 72 - mooringlicensing/utils/archive/make_unique.py | 45 - .../utils/archive/ml_csvwriter.py | 100 - .../utils/archive/mooring_licence_migrate.py | 482 --- .../archive/mooring_licence_migrate_pd.py | 2659 ----------------- .../mooring_licence_migrate_write_csv.py | 348 --- .../utils/archive/mooringlicensing-cronjobs | 6 - .../utils/archive/vessel_migrate.py | 564 ---- .../utils/archive/waiting_list_migrate.py | 363 --- .../authorised_user_migrate.15Oct2021.py | 489 --- .../utils/authorised_user_migrate.py | 675 ----- mooringlicensing/utils/dcv_migrate.py | 246 -- .../utils/excel_export/__init__.py | 0 .../excel_export/annualadmission_to_excel.py | 13 - .../utils/excel_export/authusers_to_excel.py | 195 -- .../utils/excel_export/dcvpermits_to_excel.py | 104 - .../utils/excel_export/mla_to_excel.py | 189 -- .../utils/excel_export/wla_to_excel.py | 167 -- mooringlicensing/utils/json_to_csv.py | 72 - mooringlicensing/utils/make_unique.py | 45 - mooringlicensing/utils/ml_csvwriter.py | 100 - .../utils/mooring_licence_migrate.py | 482 --- .../mooring_licence_migrate_write_csv.py | 301 -- mooringlicensing/utils/vessel_migrate.py | 564 ---- .../utils/waiting_list_migrate.py | 363 --- 32 files changed, 2 insertions(+), 10622 deletions(-) delete mode 100644 mooringlicensing/utils/annual_admission_migrate.py delete mode 100644 mooringlicensing/utils/archive/annual_admission_migrate.py delete mode 100644 mooringlicensing/utils/archive/authorised_user_migrate.15Oct2021.py delete mode 100644 mooringlicensing/utils/archive/authorised_user_migrate.py delete mode 100644 mooringlicensing/utils/archive/dcv_migrate.py delete mode 100644 mooringlicensing/utils/archive/json_to_csv.py delete mode 100644 mooringlicensing/utils/archive/make_unique.py delete mode 100644 mooringlicensing/utils/archive/ml_csvwriter.py delete mode 100644 mooringlicensing/utils/archive/mooring_licence_migrate.py delete mode 100644 mooringlicensing/utils/archive/mooring_licence_migrate_pd.py delete mode 100644 mooringlicensing/utils/archive/mooring_licence_migrate_write_csv.py delete mode 100755 mooringlicensing/utils/archive/mooringlicensing-cronjobs delete mode 100644 mooringlicensing/utils/archive/vessel_migrate.py delete mode 100644 mooringlicensing/utils/archive/waiting_list_migrate.py delete mode 100644 mooringlicensing/utils/authorised_user_migrate.15Oct2021.py delete mode 100644 mooringlicensing/utils/authorised_user_migrate.py delete mode 100644 mooringlicensing/utils/dcv_migrate.py delete mode 100644 mooringlicensing/utils/excel_export/__init__.py delete mode 100644 mooringlicensing/utils/excel_export/annualadmission_to_excel.py delete mode 100644 mooringlicensing/utils/excel_export/authusers_to_excel.py delete mode 100644 mooringlicensing/utils/excel_export/dcvpermits_to_excel.py delete mode 100644 mooringlicensing/utils/excel_export/mla_to_excel.py delete mode 100644 mooringlicensing/utils/excel_export/wla_to_excel.py delete mode 100644 mooringlicensing/utils/json_to_csv.py delete mode 100644 mooringlicensing/utils/make_unique.py delete mode 100644 mooringlicensing/utils/ml_csvwriter.py delete mode 100644 mooringlicensing/utils/mooring_licence_migrate.py delete mode 100644 mooringlicensing/utils/mooring_licence_migrate_write_csv.py delete mode 100644 mooringlicensing/utils/vessel_migrate.py delete mode 100644 mooringlicensing/utils/waiting_list_migrate.py diff --git a/mooringlicensing/components/payments_ml/utils.py b/mooringlicensing/components/payments_ml/utils.py index 837201d83..2f8b85c73 100644 --- a/mooringlicensing/components/payments_ml/utils.py +++ b/mooringlicensing/components/payments_ml/utils.py @@ -4,16 +4,12 @@ import pytz from django.http import HttpResponse from django.urls import reverse -from ledger_api_client.utils import create_basket_session, create_checkout_session, calculate_excl_gst, use_existing_basket_from_invoice +from ledger_api_client.utils import create_basket_session, create_checkout_session, calculate_excl_gst from ledger_api_client.settings_base import * from mooringlicensing import settings -from mooringlicensing.components.payments_ml.models import ( - ApplicationFee -) logger = logging.getLogger(__name__) - def checkout(request, email_user, lines, return_url, return_preload_url, booking_reference, invoice_text=None, vouchers=[], proxy=False,): basket_params = { 'products': make_serializable(lines), diff --git a/mooringlicensing/templates/webtemplate_dbca/includes/staff_menu_extras.html b/mooringlicensing/templates/webtemplate_dbca/includes/staff_menu_extras.html index 4ab2a28f8..edf599dd8 100644 --- a/mooringlicensing/templates/webtemplate_dbca/includes/staff_menu_extras.html +++ b/mooringlicensing/templates/webtemplate_dbca/includes/staff_menu_extras.html @@ -1,5 +1,4 @@ - {% if is_mooringlicensing_admin_user or request.user.is_superuser or is_payment_officer %}
  • Financial Reports
  • Failed Refunds {% if ledger_totals.total_failed > 0 %}{{ ledger_totals.total_failed }}{% endif %}
    • -{% endif %} +{% endif %} \ No newline at end of file diff --git a/mooringlicensing/utils/annual_admission_migrate.py b/mooringlicensing/utils/annual_admission_migrate.py deleted file mode 100644 index 0a47b5680..000000000 --- a/mooringlicensing/utils/annual_admission_migrate.py +++ /dev/null @@ -1,318 +0,0 @@ -import os -import subprocess -import json -import csv -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - AnnualAdmissionApplication, - ProposalUserAction, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, AnnualAdmissionPermit - - -class AnnualAdmissionMigration(object): - ''' - from mooringlicensing.utils.annual_admission_migrate import AnnualAdmissionMigration - AnnualAdmissionMigration(test=False) - ''' - - def __init__(self, filename='mooringlicensing/utils/csv/annual_admissions_booking_report_20230125084027.csv', test=False): - """ - NOTE: - filename='mooringlicensing/utils/csv/annual_admissions_booking_report_20230125084027.csv' comes from Moorings RIA system (??) - (https://mooring-ria-internal.dbca.wa.gov.au/dashboard/bookings/annual-admissions/) - """ - self.filename = filename - self.test = test - - self.migrate() - - def migrate(self): - - expiry_date = datetime.date(2023,8,31) - start_date = datetime.date(2021,8,31) - - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - - added = [] - errors = [] - count_no_mooring = 0 - idx = -1 - with transaction.atomic(): - with open(self.filename) as csvfile: - reader = csv.reader(csvfile, delimiter=str(',')) - header = next(reader) # skip header - for row in reader: - #import ipdb; ipdb.set_trace() - if not row[0].startswith('#'): - idx += 1 - - ID = row[0] - created = row[1].strip() - firstname = row[2].strip() - lastname = row[3].strip() - address1 = row[4].strip() - suburb = row[5].strip() - state = row[6].strip() - postcode = row[7].strip() - sticker_no = row[8].strip() - rego_no = row[9].strip() - email = row[10].lower().strip() - mobile_no = row[11].strip() - phone_no = row[12].strip() - vessel_name = row[13].strip() - length = row[14].strip() - year = row[15].strip() - status = row[16].strip() - booking_period = row[17].strip() - country = row[18].strip() - -# if status != 'expired': -# # ignore everything, except 'expired' records -# continue -# elif idx % 50 == 0: -# print(f"{idx} {status}") - -# if status == 'current' and idx % 50 == 0: -# print(f"{idx} {status}") -# else: -# # ignore everything, except 'current' records -# continue - - if idx % 50 == 0: - print(f"{idx} {status}") - - if self.test: - continue - - try: - #import ipdb; ipdb.set_trace() - - vessel_type = 'other' - vessel_weight = Decimal( 0.0 ) - vessel_draft = Decimal( 0.0 ) - percentage = None # force user to set at renewal time - - address = None - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) - - country = Country.objects.get(printable_name='Australia') - address, address_created = Address.objects.get_or_create(line1=address1, line3=suburb, postcode=postcode, state=state, country=country, user=user) - - user.residential_address = address - user.postal_address = address - user.save() - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - except ObjectDoesNotExist: - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage) - - try: - vessel_details = VesselDetails.objects.get(vessel=vessel) - except MultipleObjectsReturned as e: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - except ObjectDoesNotExist as e: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - berth_mooring='home' - ) - - proposal=AnnualAdmissionApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.datetime.now(datetime.timezone.utc), - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=length, - vessel_draft=vessel_draft, - vessel_weight=vessel_weight, - berth_mooring='home', - percentage=percentage, - individual_owner=True, - processing_status='approved', - customer_status='approved', - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - }, - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Annual Admission - Migrated Application', - ) - - #approval = WaitingListAllocation.objects.create( - approval = AnnualAdmissionPermit.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - #proposal.allocated_mooring.mooring_licence = approval - #proposal.allocated_mooring.save() - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - if address: - address_list.append(address.id) - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - except Exception as e: - errors.append(e) - #import ipdb; ipdb.set_trace() - #raise Exception(str(e)) - print(f'{e}') - continue - - print(f'Address.objects.get(id__in={address_list}).delete()') - print(f'EmailUser.objects.get(id__in={user_list}).delete()') - print(f'Vessel.objects.get(id__in={vessel_list}).delete()') - print(f'Owner.objects.get(id__in={owner_list}).delete()') - print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') - print(f'VesselDetails.objects.get(id__in={details_list}).delete()') - print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') - print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') - print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') - print(f'errors: {errors}') - -def clear_record(): - Address.objects.last().delete() - EmailUser.objects.last().delete() - Vessel.objects.last().delete() - Owner.objects.last().delete() - VesselOwnership.objects.last().delete() - VesselDetails.objects.last().delete() - WaitingListApplication.objects.last().delete() - ProposalUserAction.objects.last().delete() - WaitingListAllocation.objects.last().delete() - ApprovalHistory.objects.last().delete() - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - #_files = self.get_files(self.search_str1) - files=[ - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - #print(f'fname: {fname}, key2: {key2}') - return record - - diff --git a/mooringlicensing/utils/archive/annual_admission_migrate.py b/mooringlicensing/utils/archive/annual_admission_migrate.py deleted file mode 100644 index 92a55e8de..000000000 --- a/mooringlicensing/utils/archive/annual_admission_migrate.py +++ /dev/null @@ -1,243 +0,0 @@ -import os -import subprocess -import json -import csv -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -#from oscar.apps.address.models import Country -#from ledger.accounts.models import EmailUser, Address -from ledger_api_client.ledger_models import EmailUserRO -from mooringlicensing.components.proposals.models import ( - Proposal, - ProposalType, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - AnnualAdmissionApplication, - ProposalUserAction, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, AnnualAdmissionPermit - - -class AnnualAdmissionMigration(object): - ''' - from mooringlicensing.utils.annual_admission_migrate import AnnualAdmissionMigration - AnnualAdmissionMigration(test=False) - ''' - - def __init__(self, filename='mooringlicensing/utils/csv/clean/annual_admissions_booking_report.csv', test=False): - """ - NOTE: - filename='mooringlicensing/utils/csv/clean/annual_admissions_booking_report.csv' comes from Moorings RIA system (??) - (https://mooring-ria-internal.dbca.wa.gov.au/dashboard/bookings/annual-admissions/) - """ - self.filename = filename - self.test = test - - self.migrate() - - def migrate(self): - - expiry_date = datetime.date(2023,8,31) - start_date = datetime.date(2022,9,1) - - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - - added = [] - errors = [] - count_no_mooring = 0 - idx = -1 - with transaction.atomic(): - with open(self.filename) as csvfile: - reader = csv.reader(csvfile, delimiter=str('|')) - header = next(reader) # skip header - for row in reader: - #import ipdb; ipdb.set_trace() - if not row[0].startswith('#'): - idx += 1 - - ID = row[0] - created = row[1].strip() - firstname = row[2].strip() - lastname = row[3].strip() - address1 = row[4].strip() - suburb = row[5].strip() - state = row[6].strip() - postcode = row[7].strip() - sticker_no = row[8].strip() - rego_no = row[9].strip() - email = row[10].lower().strip() - mobile_no = row[11].strip() - phone_no = row[12].strip() - vessel_name = row[13].strip() - length = row[14].strip() - year = row[15].strip() - status = row[16].strip() - booking_period = row[17].strip() - country = row[18].strip() - -# if status != 'expired': -# # ignore everything, except 'expired' records -# continue -# elif idx % 50 == 0: -# print(f"{idx} {status}") - -# if status == 'current' and idx % 50 == 0: -# print(f"{idx} {status}") -# else: -# # ignore everything, except 'current' records -# continue - - if idx % 50 == 0: - print(f"{idx} {status}") - - if self.test: - continue - - try: - #import ipdb; ipdb.set_trace() - - vessel_type = 'other' - vessel_weight = Decimal( 0.0 ) - vessel_draft = Decimal( 0.0 ) - percentage = 100 # force user to set at renewal time - - address = None - try: - user = EmailUserRO.objects.get(email=email) - except: - import ipdb; ipdb.set_trace() - logger.warn(f'User not found: {user}') - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - except ObjectDoesNotExist: - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage) - - try: - vessel_details = VesselDetails.objects.get(vessel=vessel) - except MultipleObjectsReturned as e: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - except ObjectDoesNotExist as e: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - berth_mooring=None - ) - - proposal=AnnualAdmissionApplication.objects.create( - proposal_type=ProposalType.objects.get(code='new'), # new application - submitter=user.id, - lodgement_date=datetime.datetime.now(datetime.timezone.utc), - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=length, - vessel_draft=vessel_draft, - vessel_weight=vessel_weight, - berth_mooring=None, #'home', - percentage=percentage, - individual_owner=True, - processing_status='approved', - customer_status='approved', - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - }, - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Annual Admission - Migrated Application', - ) - - #approval = WaitingListAllocation.objects.create( - approval = AnnualAdmissionPermit.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user.id, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - #proposal.allocated_mooring.mooring_licence = approval - #proposal.allocated_mooring.save() - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - except Exception as e: - errors.append(e) - #import ipdb; ipdb.set_trace() - #raise Exception(str(e)) - print(f'{e}') - continue - - print(f'EmailUser.objects.get(id__in={user_list}).delete()') - print(f'Vessel.objects.get(id__in={vessel_list}).delete()') - print(f'Owner.objects.get(id__in={owner_list}).delete()') - print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') - print(f'VesselDetails.objects.get(id__in={details_list}).delete()') - print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') - print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') - print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') - print(f'errors: {errors}') - diff --git a/mooringlicensing/utils/archive/authorised_user_migrate.15Oct2021.py b/mooringlicensing/utils/archive/authorised_user_migrate.15Oct2021.py deleted file mode 100644 index c2b43a7bd..000000000 --- a/mooringlicensing/utils/archive/authorised_user_migrate.15Oct2021.py +++ /dev/null @@ -1,489 +0,0 @@ -import os -import subprocess -import json -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - AuthorisedUserApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringOnApproval, AuthorisedUserPermit - - -class AuthUserPermitMigration(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - WaitingListMigration(test=False) - ''' - - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021', test=False): - self.path = path - self.test = test - - #self.waitlist = self.read_dict('Waitlist___Bay.json') - self.auth_users_with_L = self.read_dict('Auth_Users___Surname___with_L.json') - self.auth_users_No_L = self.read_dict('Auth_Users___Surname___No_L.json') - self.moorings = self.read_dict('Auth_Users___Surname___Moorings.json') - self.vessel_rego = self.read_dict('Auth_Users___Vessel_Rego.json') - - self.canc_lic = self.read_dict('Auth_Users___Cancelled_Lic.json') - self.canc = self.read_dict('Auth_Users___Cancelled.json') - - - self._auth_users = [ - #... can be found in directory tests/test.json - ] - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def search_multiple(self, key, value, records): - return [record for record in records if value == record.get(key)] - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2021,11,30) - date_applied = '2020-08-31' - - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - wl_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - wla_list = [] - - added = [] - errors = [] - no_records = [] - fnames = [] - count_no_mooring = 0 - with transaction.atomic(): - for idx, record in enumerate(self.auth_users_No_L[33:200], 33): - #for idx, record in enumerate(self.auth_users_No_L[46:], 46): - #for idx, record in enumerate(self.auth_users_with_L, 1): - #for idx, record in enumerate(self.auth_users_with_L[2:3], 1): - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - no_auth_permits = int(record['NoAuth']) - if no_auth_permits == 0: - continue - - address = record.get('_1') - - email_record, fname1 = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - phonemobile_record, fname2 = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneMobile') - phonehome_record, fname3 = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneHome') -# if fname1 not in fnames: -# fnames.append(fname1) -# if fname2 not in fnames: -# fnames.append(fname2) -# if fname3 not in fnames: -# fnames.append(fname3) - - - email = email_record.get('EMail').lower() - mobile_no = phonemobile_record.get('PhoneMobile') - username = record.get('UserName') #.lower() - firstname = username.split(' ')[-1] - lastname = ' '.join(username.split(' ')[:-1]) - - try: - phone_no = phonehome_record.get('PhoneHome') if phonehome_record else '' - #address = address_record.get('_1') - except: - import ipdb; ipdb.set_trace() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {username}, {no_auth_permits}') - - #for no_auth in no_auth_permits: - vessel_rego_records = self.search_multiple('_1', username, self.vessel_rego) - moorings_records = self.search_multiple('_1', username, self.moorings) - if len(vessel_rego_records) == no_auth_permits: - #print(f'{len(vessel_rego_records)}, {no_auth_permits}') - #print(f'Vessel Rego: {vessel_rego_records}') - aup_records = vessel_rego_records - elif len(moorings_records) == no_auth_permits: - #print(f'{len(moorings_records)}, {no_auth_permits}') - #print(f'Moorings: {moorings_records}') - aup_records = moorings_records - else: - canc = self.search_multiple('_1', username, self.canc) - canc_lic = self.search_multiple('_12', username, self.canc_lic) - - num_canc = len(canc) + len(canc_lic) - rec = dict(username=username, noauth=no_auth_permits, num_cancelled=num_canc) - no_records.append(rec) - print(f'ERROR: aup_records not matched: {rec}') - #import ipdb; ipdb.set_trace() - continue - - for i in aup_records: - permit_type = aup_records[0]['_6'] # RIA or Lic - mooring_no = aup_records[0]['MooringNo'] - vessel_name = aup_records[0]['VesName'] - #vessel_len = aup_records[0]['VesLen'] - vessel_rego = aup_records[0]['VesRego'] - - #if i['_1'] == username: - # print(i) - - if self.test: - #import ipdb; ipdb.set_trace() - continue - - -# mooring_record = self.search('_1', username, self.moorings) -# try: -# mooring = mooring_record['MooringNo'] -# vessel_name = mooring_record['VesName'] -# rego_no = mooring_record['VesRego'] -# except: -# count_no_mooring += 1 -# print(f'*************** {idx}, {pers_no}, {username}, {count_no_mooring}') -# continue -# #vessel_name = record.get('_11').split('-')[1].strip() -# #rego_no = record.get('_11').split('-')[0].strip() -# -# vessel_type = 'other' -# vessel_overall_length = Decimal( record.get('RegLength1') ) -# #vessel_draft = Decimal( record.get('Draft') ) -# vessel_draft = Decimal( 0.0 ) -# vessel_weight = Decimal( 0.0 ) -# #berth_mooring = record.get('_5') -# #mooring = ?? # record.get('_5') -# -# # see mooringlicensing/utils/tests/mooring_names.txt -# if Mooring.objects.filter(name=mooring).count()>0: -# mooring = Mooring.objects.filter(name=mooring)[0] -# else: -# import ipdb; ipdb.set_trace() -# print(f'Mooring not found: {mooring}') -# #mooring_bay = MooringBay.objects.get(name='Rottnest Island') -# #mooring_bay = MooringBay.objects.get(name='Thomson Bay') -# -# # needed ?? -# #date_applied = record.get('DateApplied') -# #position_no = int(record.get('BayPosNo')) -# #trim_no = record.get('TrimNo') -# percentage = None # force user to set at renewal time -# -# -# items = address.split(',') -# line1 = items[0].strip() -# line2 = items[1].strip() if len(items) > 3 else '' -# line3 = items[2].strip() if len(items) > 4 else '' -# state = items[-2].strip() -# postcode = items[-1].strip() -# -# #import ipdb; ipdb.set_trace() -# print(f'{idx}, {pers_no}, {username}, {state}, {postcode}') -# if self.test: -# continue -# -# try: -# user = EmailUser.objects.get(email=email) -# except: -# user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) -# -# country = Country.objects.get(printable_name='Australia') -# address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) -# user.residential_address = address -# user.postal_address = address -# user.save() -# -# try: -# vessel = Vessel.objects.get(rego_no=rego_no) -# except ObjectDoesNotExist: -# vessel = Vessel.objects.create(rego_no=rego_no) -# -# -# try: -# owner = Owner.objects.get(emailuser=user) -# except ObjectDoesNotExist: -# owner = Owner.objects.create(emailuser=user) -# -# -# try: -# vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) -# except ObjectDoesNotExist: -# vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage) -# -# try: -# vessel_details = VesselDetails.objects.get(vessel=vessel) -# except ObjectDoesNotExist: -# vessel_details = VesselDetails.objects.create( -# vessel_type=vessel_type, -# vessel=vessel, -# vessel_name=vessel_name, -# vessel_length=vessel_overall_length, -# vessel_draft=vessel_draft, -# vessel_weight= vessel_weight, -# #berth_mooring=berth_mooring -# ) -# -# proposal=AuthorisedUserApplication.objects.create( -# proposal_type_id=1, # new application -# submitter=user, -# migrated=True, -# vessel_details=vessel_details, -# vessel_ownership=vessel_ownership, -# rego_no=rego_no, -# vessel_type=vessel_type, -# vessel_name=vessel_name, -# vessel_length=vessel_overall_length, -# vessel_draft=vessel_draft, -# #vessel_beam='', -# vessel_weight=vessel_weight, -# berth_mooring='home', -# preferred_bay= mooring.mooring_bay, -# percentage=percentage, -# individual_owner=True, -# #proposed_issuance_approval={}, -# processing_status='approved', -# customer_status='approved', -# proposed_issuance_approval={ -# "details": None, -# "cc_email": None, -# "mooring_id": mooring.id, -# "ria_mooring_name": mooring.name, -# "mooring_bay_id": mooring.mooring_bay.id, -# "vessel_ownership": [], -# "mooring_on_approval": [] -# }, -# ) -##> "proposed_issuance_approval": { -##> "details": "dd", -##> "cc_email": null, -##> "mooring_id": 127, -##> "mooring_bay_id": 7, -##> "ria_mooring_name": "CB005", -##> "vessel_ownership": [], -##> "mooring_on_approval": [] -##> }, -## -##> "berth_mooring": "home", -##> "dot_name": "abc", -##> "insurance_choice": "over_ten", -##> "preferred_bay": null, -##> "silent_elector": null, -##> "mooring_authorisation_preference": "ria", -##> "bay_preferences_numbered": "[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\"]", -# -# ua=ProposalUserAction.objects.create( -# proposal=proposal, -# who=user, -# what='Authorised User Permit - Migrated Application', -# ) -# -# try: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() -# except: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() -# -# #approval = WaitingListAllocation.objects.create( -# approval = AuthorisedUserPermit.objects.create( -# status='current', -# #internal_status=None, -# current_proposal=proposal, -# issue_date = datetime.datetime.now(datetime.timezone.utc), -# #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), -# start_date = start_date, -# expiry_date = expiry_date, -# submitter=user, -# migrated=True, -# export_to_mooring_booking=True, -# ) -# #print(f'wla_order: {position_no}') -# -# proposal.approval = approval -# proposal.save() -# -# moa = MooringOnApproval.objects.create( -# approval=approval, -# mooring=mooring, -# sticker=None, -# site_licensee=True, # ??? -# end_date=expiry_date -# ) -# -# try: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) -# except: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) -# -# approval_history = ApprovalHistory.objects.create( -# reason='new', -# approval=approval, -# vessel_ownership = vessel_ownership, -# proposal = proposal, -# #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) -# start_date = start_date, -# ) -# -# added.append(proposal.id) -# -# address_list.append(address.id) -# user_list.append(user.id) -# vessel_list.append(vessel.id) -# owner_list.append(owner.id) -# ownership_list.append(vessel_ownership.id) -# details_list.append(vessel_details.id) -# proposal_list.append(proposal.proposal.id) -# wl_list.append(proposal.id) -# user_action_list.append(ua.id) -# approval_list.append(approval.id) -# approval_history_list.append(approval_history.id) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - -# print(f'Address.objects.get(id__in={address_list}).delete()') -# print(f'EmailUser.objects.get(id__in={user_list}).delete()') -# print(f'Vessel.objects.get(id__in={vessel_list}).delete()') -# print(f'Owner.objects.get(id__in={owner_list}).delete()') -# print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') -# print(f'VesselDetails.objects.get(id__in={details_list}).delete()') -# print(f'WaitingListApplication.objects.get(id__in={wl_list}).delete()') -# print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') -# print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') -# print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') -# print(f'count_no_mooring: {count_no_mooring}') - print(f'no_records: {no_records}') - print(f'fnames_records: {fnames}') - -def clear_record(): - Address.objects.last().delete() - EmailUser.objects.last().delete() - Vessel.objects.last().delete() - Owner.objects.last().delete() - VesselOwnership.objects.last().delete() - VesselDetails.objects.last().delete() - WaitingListApplication.objects.last().delete() - ProposalUserAction.objects.last().delete() - WaitingListAllocation.objects.last().delete() - ApprovalHistory.objects.last().delete() - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - _files = self.get_files(self.search_str1) - files=[ - - #self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleName.json', - self.path + os.sep + 'PeopleLicType.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'HINCurrent.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'Vessel___Vessel_Name___Current.json', - -# self.path + os.sep + 'Vessel___Rego___Current.json', -# self.path + os.sep + 'PeopleNo.json', -# self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', -# -# self.path + os.sep + 'Auth_User_Details.json', -# self.path + os.sep + 'Auth_User___Inv.json', -# self.path + os.sep + 'Auth_User_Payment_Summary.json', -# self.path + os.sep + 'Auth_Users___AutoCanc.json', -# self.path + os.sep + 'Auth_Users___Cancelled.json', -# self.path + os.sep + 'Auth_Users___Cancelled_Lic.json', -# self.path + os.sep + 'Auth_Users__Cancelled_Lic_Name.json', -# self.path + os.sep + 'Auth_Users___Cancelled_PersNo.json', -# self.path + os.sep + 'Auth_Users___Lic_Pers_No.json', -# self.path + os.sep + 'Auth_Users___Mooring.json', -# self.path + os.sep + 'Auth_Users___Sticker_No.json', -# self.path + os.sep + 'Auth_Users___Surname___Moorings.json', -# self.path + os.sep + 'Auth_Users___Surname___No.json', -# self.path + os.sep + 'Auth_Users___Surname___No_L.json', -# self.path + os.sep + 'Auth_Users___Surname___with_L.json', -# self.path + os.sep + 'Auth_Users___User_Pers_No.json', -# self.path + os.sep + 'Auth_Users___Vessel_Rego.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - #print('****************' + fname) - return record, fname - - diff --git a/mooringlicensing/utils/archive/authorised_user_migrate.py b/mooringlicensing/utils/archive/authorised_user_migrate.py deleted file mode 100644 index fa365ae81..000000000 --- a/mooringlicensing/utils/archive/authorised_user_migrate.py +++ /dev/null @@ -1,675 +0,0 @@ -import os -import subprocess -import json -import ast -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - AuthorisedUserApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringOnApproval, AuthorisedUserPermit - - -class AuthUserPermitMigration(object): - ''' - PRELIM: - from mooringlicensing.utils.json_to_csv import JsonToCsv - JsonToCsv().convert() - - from mooringlicensing.utils.authorised_user_migrate import AuthUserPermitMigration, GrepSearch - AuthUserPermitMigration(test=False) - ''' - - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021', path_csv='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv', test=False): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.path_csv = path + '_all_csv' if not path.endswith('/') else path.strip('/') + '_all_csv' - self.test = test - - #self.waitlist = self.read_dict('Waitlist___Bay.json') - self.auth_users_with_L = self.read_dict('Auth_Users___Surname___with_L.json') - self.auth_users_No_L = self.read_dict('Auth_Users___Surname___No_L.json') - self.moorings = self.read_dict('Auth_Users___Surname___Moorings.json') - self.vessel_rego = self.read_dict('Auth_Users___Vessel_Rego.json') - - self.canc_lic = self.read_dict('Auth_Users___Cancelled_Lic.json') - self.canc = self.read_dict('Auth_Users___Cancelled.json') - - - self._auth_users = [ - #... can be found in directory tests/test.json - ] - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def search_multiple(self, key, value, records): - return [record for record in records if value == record.get(key)] - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - date_applied = '2021-08-31' - - address_not_found = [] - address_l_not_found = [] - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - aup_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - - aup_data = [] - aup_data.append(['PersNo', 'Email', 'DoB', 'Rego No', 'MooringNo', 'Mooring Authorisation (RIA/LIC)']) - - added = [] - errors = [] - no_records = [] - fnames = [] - no_persno = [] - no_email = [] - no_email_l = [] - no_licencee = [] - count_no_mooring = 0 - with transaction.atomic(): - #for idx, record in enumerate(self.moorings[827:], 827): - for idx, record in enumerate(self.moorings, 1): - try: - #import ipdb; ipdb.set_trace() - username = record.get('_1') #.lower() - permit_type = record['_6'] # RIA or Lic - mooring_no = record['MooringNo'] - vessel_name = record['VesName'] - #vessel_len = aup_records[0]['VesLen'] - rego_no = record['VesRego'] - - gs2 = GrepSearch2(username=username, mooring_no=mooring_no, path=self.path_csv) - try: - pers_no = gs2.get_persno() - if not pers_no and (username, pers_no) not in no_persno: - no_persno.append((username, pers_no)) - print(f'** NO PersNo FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - continue - except: - if (username, pers_no) not in no_persno: - no_persno.append((username, pers_no)) - print(f'NO PersNo FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - import ipdb; ipdb.set_trace() - continue - -# if (username, pers_no) not in no_email: -# no_email.append((username,pers_no)) -# print(f'NO EMAIL FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - - address, phone_home, phone_mobile, phone_work = gs2.get_address(pers_no) - if address is None: - address_not_found.append(pers_no) - print(f'ADDRESS NOT FOUND: {idx}, {pers_no}, {username} {permit_type}, {mooring_no}') - continue - email, username = gs2.get_email() - if not email: - continue - email = email.split(';')[0] - email = email.strip().replace('{','').replace('}','') - firstname = username.split(' ')[-1] - lastname = ' '.join(username.split(' ')[:-1]) - -# if (username, pers_no) not in no_email: -# no_email.append((username,pers_no)) -# print(f'NO EMAIL FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - - email_l, username_l = gs2.get_email(licencee=True) - if username == username_l: - print(f'{idx}, Licencee and AUP Same User: {username}, {mooring_no}') - continue - - #if not address_l and (username, pers_no, mooring_no) not in no_licencee and permit_type=='Lic': - if not email_l and permit_type=='Lic': - if (username, pers_no, mooring_no) not in no_licencee: - no_licencee.append((username, pers_no, mooring_no)) - permit_type = 'RIA' # override and save as RIA nominated mooring - print(f'{idx}, {pers_no}, {address}, {username}, {permit_type}, {mooring_no}') - elif permit_type=='Lic': - email_l = email_l.split(';')[0] - firstname_l = username.split(' ')[-1] - lastname_l = ' '.join(username.split(' ')[:-1]) - pers_no_l = gs2.get_persno(username_l) - #import ipdb; ipdb.set_trace() - address_l, phone_home_l, phone_mobile_l, phone_work_l = gs2.get_address(pers_no_l) - print(f'{idx}, {pers_no}, {address}, {username}, {permit_type}, {mooring_no}: Licencee - ({email_l} {username_l} {address_l}, {phone_mobile})') - if address_l is None: - address_l_not_found.append(pers_no_l) - print(f'ADDRESS_L NOT FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}: Licencee - ({pers_no_l})') - continue - else: - print(f'{idx}, {pers_no}, {address}, {username}, {permit_type}, {mooring_no}') - - - - vessel_overall_length, vessel_draft = gs2.get_vessel_size(rego_no) - vessel_type = 'other' - vessel_weight = Decimal( 0.0 ) - berth_mooring = '' - percentage = None # force user to set at renewal time - dot_name = gs2.get_dot_rego(rego_no) - - if self.test: - #import ipdb; ipdb.set_trace() - continue - - if Mooring.objects.filter(name=mooring_no).count()>0: - mooring = Mooring.objects.filter(name=mooring_no)[0] - elif mooring_no=='TB999': - print(f'{idx}, Mooring ignored: {mooring_no}') - continue - else: - import ipdb; ipdb.set_trace() - print(f'Mooring not found: {mooring_no}') - #mooring_bay = MooringBay.objects.get(name='Rottnest Island') - #mooring_bay = MooringBay.objects.get(name='Thomson Bay') - - # User email user - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=phone_mobile, phone_number=phone_home) - - country = Country.objects.get(printable_name='Australia') - try: - _address = Address.objects.get(user=user) - except: - _address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - user.residential_address = _address - user.postal_address = _address - user.save() - - #if permit_type=='Lic' and email_l: - bay_preferences_numbered = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"] - mooring_authorisation_preference = 'ria' - site_licensee_email = None - keep_existing_mooring = True - site_licensee = False - if permit_type=='Lic': - # Licensee email usee - items = address_l.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - try: - licensee = EmailUser.objects.get(email=email_l) - except: - licensee = EmailUser.objects.create(email=email_l, first_name=firstname_l, last_name=lastname_l, mobile_number=phone_mobile_l, phone_number=phone_home_l) - - country = Country.objects.get(printable_name='Australia') - try: - _address_l = Address.objects.get(user=licensee) - except: - _address_l, address_created_l = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=licensee) - licensee.residential_address = _address_l - licensee.postal_address = _address_l - licensee.save() - - mooring_authorisation_preference='site_licensee' - site_licensee_email=email_l - bay_preferences_numbered = None - site_licensee = True - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - vessel_ownership.dot_name = dot_name - vessel_ownership.save() - except ObjectDoesNotExist: - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage, dot_name=dot_name) - - if VesselDetails.objects.filter(vessel=vessel).count()>0: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - else: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - #berth_mooring=berth_mooring - ) - - proposal=AuthorisedUserApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.now(), - mooring_authorisation_preference=mooring_authorisation_preference, - site_licensee_email=site_licensee_email, - keep_existing_mooring=True, - mooring=mooring, - bay_preferences_numbered=bay_preferences_numbered, - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - #vessel_beam='', - vessel_weight=vessel_weight, - berth_mooring='home', - preferred_bay= mooring.mooring_bay, - percentage=percentage, - individual_owner=True, - dot_name=dot_name, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - proposed_issuance_approval={ - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - "vessel_ownership": [], - "mooring_on_approval": [] - }, - ) -#> "proposed_issuance_approval": { -#> "details": "dd", -#> "cc_email": null, -#> "mooring_id": 127, -#> "mooring_bay_id": 7, -#> "ria_mooring_name": "CB005", -#> "vesSEL_OWNERSHIP": [], -#> "mooring_on_approval": [] -#> }, -# -#> "berth_mooring": "home", -#> "dot_name": "abc", -#> "insurance_choice": "over_ten", -#> "preferred_bay": null, -#> "silent_elector": null, -#> "mooring_authorisation_preference": "ria", -#> "bay_preferences_numbered": "[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\"]", - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Authorised User Permit - Migrated Application', - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() - - #approval = WaitingListAllocation.objects.create( - approval = AuthorisedUserPermit.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - - moa = MooringOnApproval.objects.create( - approval=approval, - mooring=mooring, - sticker=None, - site_licensee=site_licensee, - #end_date=expiry_date - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - address_list.append(_address.id) - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - aup_list.append(proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - #date_invited = '' - aup_data.append([pers_no, email, user.dob, rego_no, mooring.name, 'LIC']) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - -# print(f'Address.objects.get(id__in={address_list}).delete()') -# print(f'EmailUser.objects.get(id__in={user_list}).delete()') -# print(f'Vessel.objects.get(id__in={vessel_list}).delete()') -# print(f'Owner.objects.get(id__in={owner_list}).delete()') -# print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') -# print(f'VesselDetails.objects.get(id__in={details_list}).delete()') - print(f'AuthorisedUserApplication.objects.get(id__in={aup_list}).delete()') -# print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') - print(f'AuthorisedUserPermit.objects.get(id__in={approval_list}).delete()') - print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') -# print(f'count_no_mooring: {count_no_mooring}') - print(f'no_records: {no_records}') - print(f'fnames_records: {fnames}') - print(f'no_persno: {no_persno}') - print(f'no_email: {no_email}') - print(f'no_licencee: {no_licencee}') - print(f'address_not_found: {address_not_found}') - print(f'address_l_not_found: {address_l_not_found}') - - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - #def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - def __init__(self, search_str1, path): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value in record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - _files = self.get_files(self.search_str1) - files=[ - - self.path + os.sep + 'Auth_Users___Surname___with_L.json', - self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'Vessel___People___All.json', - self.path + os.sep + 'PeopleName.json', - - self.path + os.sep + 'Vessel___People___All.json', - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleLicType.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'HINCurrent.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'Vessel___Vessel_Name___Current.json', - - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - #import ipdb; ipdb.set_trace() - record = find(key1, self.search_str1, f_json) - if not record or (key2=='_1' and len(record['_1'].split(',')) <= 1): - continue - #print('****************' + fname) - return record, fname - return None, None - -class GrepSearch2(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, username, mooring_no, path): - self.path = path - self.username = username.replace(' ', ' ') - self.mooring_no = mooring_no - - def _get_result1(self): - ''' Read all files in directory ''' - cmd = f"grep -r '{self.username}' {self.path} | grep \"'MooringNo': '{self.mooring_no}'\" | grep FirstNameL" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - #import ipdb; ipdb.set_trace() - res = output.decode('utf-8') - firstname_l = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['FirstNameL'] - lastname_l = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['LastNameL'] - username_l = '{} {}'.format(lastname_l, firstname_l) - return username_l - - except: - return None - - def _get_result2(self): - ''' Read all files in directory ''' - cmd = "grep -r '{self.username}' {self.path} | grep \"'MooringNo': '{self.mooring_no}'\" | grep '_12'" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - #import ipdb; ipdb.set_trace() - res = output.decode('utf-8') - username_l = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['_12'] - return username_l - - except: - return None - - - def get_email(self, licencee=False): - ''' Read all files in directory - Get Licensee email''' - if licencee: - username = self._get_result1() - if not username: - username = self._get_result2() - #import ipdb; ipdb.set_trace() - if not username: - #import ipdb; ipdb.set_trace() - return None, None - else: - username = self.username - - cmd = f"grep -r '{username}' {self.path} | grep '@'" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - #import ipdb; ipdb.set_trace() - try: - res = output.decode('utf-8') - email = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['EMail'] - email = email.lower().strip('>').strip('<').strip() - except: - email = None - - #import ipdb; ipdb.set_trace() - return email, username - - def get_persno(self, username=None): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - #cmd = "grep -r '{}' {} | grep -v PersNoL | grep 'PersNo'".format(self.username, self.path) - #cmd = "grep -ir '{}' {}/Auth_Users___*.* | grep -v PersNoL | grep 'PersNo'".format(self.username, self.path) - if not username: - username = self.username - - cmd = f"grep -ir '{username}' {self.path}/Auth_Users___*.* {self.path}/People___PersNo.csv {self.path}/People___Surname.csv | grep -v PersNoU | grep -v PersNoL | grep PersNo" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - res = output.decode('utf-8') - persno = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['PersNo'] - #persno = ast.literal_eval(res.splitlines()[0])['PersNo'] - except: - persno = None - - return persno - - def get_vessel_size(self, rego_no): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - cmd = f"grep -ir '{rego_no}' {self.path} | grep RegLength1 | grep Draft1" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - res = output.decode('utf-8') - ves_length = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['RegLength1'] - ves_draft = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['Draft1'] - if ves_length == 'None': - ves_length = 0.0 - if ves_draft == 'None': - ves_draft = 0.0 - except: - ves_length = 0.0 - ves_draft = 0.0 - - return Decimal(ves_length), Decimal(ves_draft) - - def get_dot_rego(self, rego_no): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - cmd = f"grep -ir '{rego_no}' {self.path} | grep DoTRego1" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - res = output.decode('utf-8') - dot_rego = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['DoTRego1'] - except: - dot_rego = None - - return dot_rego - - - def get_address(self, pers_no): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - cmd = f"grep -ir \"'PersNo': '{pers_no}'\" {self.path} | grep \"'_1'\" | grep UserName| grep Phone" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: -# res = output.decode('utf-8') -# lines = res.splitlines() -# for i in lines: -# line = ast.literal_eval(lines[i].split('csv:')[1]) -# address = line['_1'] -# if len(address.split(',')) > 2: -# phone_home = line['PhoneHome'] -# phone_mobile = line['PhoneMobile'] -# phone_work = line['PhoneWork'] -# break - - res = output.decode('utf-8') - lines = res.splitlines() - line = ast.literal_eval(lines[0].split('csv:')[1]) - address = line['_1'] - phone_home = line['PhoneHome'] - phone_mobile = line['PhoneMobile'] - phone_work = line['PhoneWork'] - - except: - address = None - phone_home = None - phone_mobile = None - phone_work = None - - return address, phone_home, phone_mobile, phone_work - diff --git a/mooringlicensing/utils/archive/dcv_migrate.py b/mooringlicensing/utils/archive/dcv_migrate.py deleted file mode 100644 index bd4a4ca78..000000000 --- a/mooringlicensing/utils/archive/dcv_migrate.py +++ /dev/null @@ -1,246 +0,0 @@ -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist - -import os -import subprocess -import json -import datetime -from decimal import Decimal - -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - WaitingListApplication, - ProposalUserAction, - MooringBay, -) -from mooringlicensing.components.approvals.models import DcvOrganisation, DcvVessel, DcvPermit -from mooringlicensing.components.payments_ml.models import FeeSeason - -class DcvMigration(object): - ''' - from mooringlicensing.utils.dcv_migrate import DcvMigration, GrepSearch - DcvMigration(test=False) - - # cp mooringlicensing/management/templates/Attachment\ Template\ -\ DCVP.docx /var/www/mooringlicensing/media/approval_permit_template/Attachment_Template_-_DCVP.docx - ''' - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.test = test - - self.company = self.read_dict('Charter_Operators___Company.json') - self.surname = self.read_dict('Charter_Operators___Surname.json') - - self._company = [ - #... can be found in directory tests/test.json - ] - - self._surname = [ - #... can be found in directory tests/test.json - ] - - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def migrate(self): - - #fee_season = FeeSeason.objects.get(name='2020-2021') - fee_season = FeeSeason.objects.filter(name='2022 - 2023')[0] - start_date = datetime.date(2021, 9, 1) - end_date = datetime.date(2023,8,31) - - address_list = [] - user_list = [] - dcv_vessel_list = [] - dcv_organisation_list = [] - dcv_permit_list = [] - - added = [] - errors = [] - with transaction.atomic(): - #for idx, record in enumerate(self.company[43:], 43): - for idx, record in enumerate(self.company, 1): - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - email = record.get('EMail').split(';')[0].strip().lower() - org_name = record.get('Company') if record.get('Company') != 'None' else None - num_vessels = int(record.get('NoVessel')) - address = record.get('_1') - phone_no = record.get('PhoneHome') - mobile_no = record.get('PhoneMobile') - worke_no = record.get('PhoneWork') - - surname_record = self.search('PersNo', pers_no, self.surname) - - username = surname_record.get('_2').lower() - firstname = username.split(' ')[-1].title() - lastname = ' '.join(username.split(' ')[:-1]).title() - - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {state}, {postcode}') - if self.test: - continue - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) - - country = Country.objects.get(printable_name='Australia') - address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - user.residential_address = address - user.postal_address = address - user.save() - - dcv_organisation = None - if org_name: - try: - dcv_organisation = DcvOrganisation.objects.get(name=org_name) - except ObjectDoesNotExist: - dcv_organisation = DcvOrganisation.objects.create(name=org_name) - #import ipdb; ipdb.set_trace() - - - for idx,i in enumerate(['_3', '_11', '_12', '_13', '_14'], 1): - items = surname_record[i].split('-') - rego_no = items[0].strip() - vessel_name = items[1].strip() - #print(surname[10][i]) - print(f'\t{idx}, {rego_no}, {vessel_name}') - - - try: - dcv_vessel = DcvVessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - dcv_vessel = DcvVessel.objects.create( - rego_no = rego_no, - vessel_name = vessel_name, - dcv_organisation = dcv_organisation, - ) - - dcv_permit = DcvPermit.objects.create( - submitter = user, - lodgement_datetime = datetime.datetime.now(datetime.timezone.utc), - fee_season = fee_season, - start_date = start_date, - end_date = end_date, - dcv_vessel = dcv_vessel, - dcv_organisation =dcv_organisation, - migrated = True - ) - #import ipdb; ipdb.set_trace() - dcv_permit.generate_dcv_permit_doc() - - dcv_vessel_list.append(dcv_vessel.id) - dcv_permit_list.append(dcv_permit.id) - - if idx == num_vessels: - break - - address_list.append(address.id) - user_list.append(user.id) - if dcv_organisation: - dcv_organisation_list.append(dcv_organisation.id) - - added.append(dcv_permit.id) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - - print(f'Address.objects.get(id__in={address_list}).delete()') - print(f'EmailUser.objects.get(id__in={user_list}).delete()') - print(f'DcvVessel.objects.get(id__in={dcv_vessel_list}).delete()') - print(f'DcvPermit.objects.get(id__in={dcv_permit_list}).delete()') - print(f'DcvOrganisation.objects.get(id__in={dcv_organisation_list}).delete()') - - -def clear_migrated_dcv(): - pass - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - #files = self.get_files(self.search_str1) - files=[ - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - return record - - diff --git a/mooringlicensing/utils/archive/json_to_csv.py b/mooringlicensing/utils/archive/json_to_csv.py deleted file mode 100644 index ce67ccaf0..000000000 --- a/mooringlicensing/utils/archive/json_to_csv.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import json -import csv -from os import listdir -from os.path import isfile, join - -class JsonToCsv(): - """ - from mooringlicensing.utils.json_to_csv import JsonToCsv - JsonToCsv().convert() - """ - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021_auth_users/'): - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021/'): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes/', csv_path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv/'): - self.path = path - self.csv_path = csv_path - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def convert(self): - #import ipdb; ipdb.set_trace() - files = [f for f in listdir(self.path) if isfile(join(self.path, f))] - - if os.path.exists(self.csv_path): - raise Exception('CVS Path already exists: {self.csv_path}') - else: - os.makedirs(self.csv_path) - - for filename in files: - try: - _list = self.read_dict(filename) - if _list: - -# #csv_filename = '/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_csv/' + filename.replace('json','csv') -# csv_filename = '/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv/' + filename.replace('json','csv') -# with open(csv_filename, 'w') as f: -# dict_writer = csv.DictWriter(f, fieldnames=_list[0].keys()) -# dict_writer.writeheader() -# for row in _list: -# dict_writer.writerow(row) - - #csv_filename = '/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv/' + filename.replace('json','csv') - #csv_path = self.path + 'all_csv/' if self.path.endswith(os.sep) else self.path + os.sep + 'all_csv/' - #if not os.path.exists(self.csv_path): - # os.makedirs(self.csv_path) - - csv_filename = self.csv_path + filename.replace('json','csv') - with open(csv_filename, 'w') as f: - #csvwriter = csv.writer(f) - #import ipdb; ipdb.set_trace() - for row in _list: - f.write(str(row) + '\n') - - except Exception as e: - print(e) - print(filename) - import ipdb; ipdb.set_trace() - - - diff --git a/mooringlicensing/utils/archive/make_unique.py b/mooringlicensing/utils/archive/make_unique.py deleted file mode 100644 index 538268802..000000000 --- a/mooringlicensing/utils/archive/make_unique.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import json - -class MakeUnique(object): - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/wait_list'): - self.path = path - self.make_unique() - - def get_files(self): - ''' Read all files in directory ''' - files = [] - with os.scandir(self.path) as it: - for entry in it: - if entry.name.endswith(".json") and entry.is_file(): - files.append(entry.name) - return files - - def make_unique(self): - ''' Remove duplicate entries in json file and write to unique folder ''' - tmp_list = [] - for fname in self.get_files(): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - out_path = self.path + os.sep + 'unique' if not self.path.endswith(os.sep) else self.path + 'unique' - if not os.path.exists(out_path): - os.makedirs(out_path) - - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - print(f'Original Length: {len(f_json)}\tNew Length: {len(_list)}\t({fname})') - - with open(out_path + os.sep + fname, 'w') as f: - print(json.dumps(_list), file=f) - - tmp_list.append( {fname : _list[:3]} ) - - with open(out_path + os.sep + 'summary.json', 'w') as f: - print(json.dumps(tmp_list), file=f) - - #return tmp diff --git a/mooringlicensing/utils/archive/ml_csvwriter.py b/mooringlicensing/utils/archive/ml_csvwriter.py deleted file mode 100644 index ccc94cbc4..000000000 --- a/mooringlicensing/utils/archive/ml_csvwriter.py +++ /dev/null @@ -1,100 +0,0 @@ -import csv -from datetime import datetime -import pytz -import os -# from ledger.settings_base import TIME_ZONE -from ledger_api_client.settings_base import TIME_ZONE -from mooringlicensing.settings import BASE_DIR -from mooringlicensing.components.approvals.models import ( - Approval, WaitingListAllocation, AnnualAdmissionPermit, - AuthorisedUserPermit, MooringLicence - ) - - -date_now = datetime.now(pytz.timezone(TIME_ZONE)).date() -date_now_str = date_now.strftime('%Y%m%d') - - -def write_wla(): - approval_type = WaitingListAllocation.code - approvals = WaitingListAllocation.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'WLA Queue Date', 'WLA Order', - 'First Name', 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessel Rego', 'Company', - ] - rows = [[wla.lodgement_number, wla.start_date, wla.expiry_date, wla.issue_date.strftime('%Y-%m-%d'), - wla.wla_queue_date.strftime('%Y-%m-%d') if wla.wla_queue_date else '', wla.wla_order, - wla.submitter_obj.first_name, wla.submitter_obj.last_name, wla.submitter_obj.residential_address.line1, - wla.submitter_obj.residential_address.locality, wla.submitter_obj.residential_address.state, - wla.submitter_obj.residential_address.country, wla.submitter_obj.residential_address.postcode, '', - wla.submitter_obj.phone_number, wla.submitter_obj.mobile_number, wla.submitter_obj.email, - wla.current_proposal.vessel_ownership.vessel.rego_no if wla.current_proposal.vessel_ownership else '', '', - ] for wla in WaitingListAllocation.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_aap(): - approval_type = AnnualAdmissionPermit.code - approvals = AnnualAdmissionPermit.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'First Name', - 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessel Rego', 'Company', - ] - rows = [[aap.lodgement_number, aap.start_date, aap.expiry_date, aap.issue_date.strftime('%Y-%m-%d'), - aap.submitter_obj.first_name, aap.submitter_obj.last_name, aap.submitter_obj.residential_address.line1, - aap.submitter_obj.residential_address.locality, aap.submitter_obj.residential_address.state, - aap.submitter_obj.residential_address.country, aap.submitter_obj.residential_address.postcode, '', - aap.submitter_obj.phone_number, aap.submitter_obj.mobile_number, aap.submitter_obj.email, - aap.current_proposal.vessel_ownership.vessel.rego_no if aap.current_proposal.vessel_ownership else '', '', - ] for aap in AnnualAdmissionPermit.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_aup(): - approval_type = AuthorisedUserPermit.code - approvals = AuthorisedUserPermit.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'Moorings', 'First Name', - 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessel Rego', 'Company', - ] - rows = [[aup.lodgement_number, aup.start_date, aup.expiry_date, aup.issue_date.strftime('%Y-%m-%d'), - ','.join(str(moa.mooring) for moa in aup.mooringonapproval_set.filter(mooring__mooring_licence__status='current')), - aup.submitter_obj.first_name, aup.submitter_obj.last_name, aup.submitter_obj.residential_address.line1, - aup.submitter_obj.residential_address.locality, aup.submitter_obj.residential_address.state, - aup.submitter_obj.residential_address.country, aup.submitter_obj.residential_address.postcode, '', - aup.submitter_obj.phone_number, aup.submitter_obj.mobile_number, aup.submitter_obj.email, - aup.current_proposal.vessel_ownership.vessel.rego_no if aup.current_proposal.vessel_ownership else '', '', - ] for aup in AuthorisedUserPermit.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_ml(): - approval_type = MooringLicence.code - approvals = MooringLicence.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'Mooring', 'First Name', - 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessels', - ] - rows = [[ml.lodgement_number, ml.start_date, ml.expiry_date, ml.issue_date.strftime('%Y-%m-%d'), - ml.mooring if hasattr(ml, 'mooring') else '', - ml.submitter_obj.first_name, ml.submitter_obj.last_name, ml.submitter_obj.residential_address.line1, - ml.submitter_obj.residential_address.locality, ml.submitter_obj.residential_address.state, - ml.submitter_obj.residential_address.country, ml.submitter_obj.residential_address.postcode, '', - ml.submitter_obj.phone_number, ml.submitter_obj.mobile_number, ml.submitter_obj.email, - ','.join(vessel.rego_no for vessel in ml.vessel_list), - ] for ml in MooringLicence.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_file(approval_type, approvals, header_row, rows): - if not os.path.isdir(os.path.join(BASE_DIR, 'mooringlicensing', 'utils', 'csv')): - os.mkdir(os.path.join(BASE_DIR, 'mooringlicensing', 'utils', 'csv')) - filename = os.path.join(BASE_DIR, 'mooringlicensing', 'utils', 'csv', '{}_{}.csv'.format(approval_type, date_now_str)) - - with open(filename, 'w', newline='') as csvfile: - csvwriter = csv.writer(csvfile, delimiter=':', quotechar='|', quoting=csv.QUOTE_MINIMAL) - csvwriter.writerow(header_row) - for row in rows: - csvwriter.writerow(row) - -write_wla() -write_aap() -write_aup() -write_ml() - diff --git a/mooringlicensing/utils/archive/mooring_licence_migrate.py b/mooringlicensing/utils/archive/mooring_licence_migrate.py deleted file mode 100644 index b40929483..000000000 --- a/mooringlicensing/utils/archive/mooring_licence_migrate.py +++ /dev/null @@ -1,482 +0,0 @@ -import os -import subprocess -import json -import csv -import datetime -from decimal import Decimal -from django.db import transaction, IntegrityError -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval - - -class MooringLicenceMigration(object): - ''' - from mooringlicensing.utils.mooring_licence_migrate import MooringLicenceMigration - MooringLicenceMigration(test=False) - ''' - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.test = test - - #self.waitlist = self.read_dict('Waitlist___Bay.json') - self.auth_users = self.read_dict('Licencees___Auth_User.json') - self.mooring_no = self.read_dict('Licencees___Mooring_No.json') - self.surname = self.read_dict('Licencees___Surname.json') - #self.vessel = self.read_dict('Vessel___HIN___Current.json') - self.vessel = self.read_dict('Vessel___Details_All.json') - self.owner = self.read_dict('Owner_Details.json') - self.allocated = self.read_dict('Admin___Application___Pers_No.json') - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - start_date = datetime.date(2021,8,31) - date_applied = '2021-08-31' - - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - - added = [] - errors = [] - count_no_mooring = 0 - - user_data = [] - vessel_data = [] - mla_data = [] - ml_all_data = [] - user_data.append(['PersNo', 'Email', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address']) - vessel_data.append(['Owner', 'Email', 'DoB', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring']) - mla.append(['PersNo', 'Email', 'DoB', 'Rego No', 'MooringNo', 'Date Invited']) - ml_all_data.append(['PersNo', 'Email', 'DoB', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring', 'MooringNo', 'Date Invited']) - with transaction.atomic(): - #for idx, record in enumerate(self.mooring_no, 1): - for idx, record in enumerate(self.mooring_no[257:260], 257): - #if idx==258: - # import ipdb; ipdb.set_trace() - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - address = record.get('_1') - mooring = record.get('MooringNo') - mooring_bay = record.get('_7') - username = record.get('UserName').lower() - rego_no = record.get('_4').split('-')[0].strip() - vessel_name = record.get('_4').split('-')[1].strip() - - owner_record = self.search('PersNoL', pers_no, self.owner) - if owner_record: - vessel_name = owner_record['VesName'] - rego_no = owner_record['VesRego'] - licence_pc = owner_record['LicPC'] - - surname_record = self.search('PersNo', pers_no, self.surname) - #email = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - #email_record.get('EMail').lower().strip() - #H1N = GrepSearch(pers_no, path=self.path).search('PersNo', 'H1N1') - email = surname_record['EMail'].split(',')[0].strip().lower() - if email=='{ronrouwenhorst@bigpond.com} ;arthur@technitemp.com': - errors.append(email) - continue - - HIN = surname_record['HIN1'] - - vessel_record = self.search('PersNo', pers_no, self.vessel) - if vessel_record: - vessel_dot = vessel_record.get('DoTRego1') - #email = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - vessel_overall_length = Decimal( vessel_record.get('RegLength1') ) - vessel_draft = Decimal( vessel_record.get('Draft1') ) - else: - vessel_dot = rego_no - #vessel_overall_length = Decimal( GrepSearch(pers_no, path=self.path).search('PersNo', 'RegLength1') ) - try: - vessel_overall_length = Decimal( surname_record.get('RegLength1') ) - except Exception as e: - vessel_overall_length = Decimal( 0.0 ) - vessel_draft = Decimal( 0.0 ) - - vessel_type = 'other' - vessel_weight = Decimal( 0.0 ) - - allocated_record = self.search('PersNo', pers_no, self.allocated) - try: - date_invited = datetime.datetime.strptime(allocated_record.get('DateAllocated'), '%Y-%m-%d').date() - except: - date_invited = None - - #email_record = GrepSearch(pers_no).search('PersNo', 'EMail') - phonemobile_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneMobile') - phonehome_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneHome') - - mobile_no = phonemobile_record.get('PhoneMobile') - firstname = username.split(' ')[-1].title() - lastname = ' '.join(username.split(' ')[:-1]).title() - - try: - phone_no = phonehome_record.get('PhoneHome') if phonehome_record else '' - #address = address_record.get('_1') - except: - import ipdb; ipdb.set_trace() - - #print(f'{idx}, {pers_no}, {username}') - #if self.test: - # import ipdb; ipdb.set_trace() - # continue - - -# mooring_record = self.search('_1', username, self.moorings) -# try: -# mooring = mooring_record['MooringNo'] -# vessel_name = mooring_record['VesName'] -# rego_no = mooring_record['VesRego'] -# except: -# count_no_mooring += 1 -# print(f'*************** {idx}, {pers_no}, {username}, {count_no_mooring}') -# continue -# #vessel_name = record.get('_11').split('-')[1].strip() -# #rego_no = record.get('_11').split('-')[0].strip() - - - # see mooringlicensing/utils/tests/mooring_names.txt - if Mooring.objects.filter(name=mooring).count()>0: - mooring = Mooring.objects.filter(name=mooring)[0] - else: - #import ipdb; ipdb.set_trace() - print(f'Mooring not found: {mooring}') - errors.append(dict(idx=idx, record=record)) - continue - - #mooring_bay = MooringBay.objects.get(name='Rottnest Island') - #mooring_bay = MooringBay.objects.get(name='Thomson Bay') - - # needed ?? - #date_applied = record.get('DateApplied') - #position_no = int(record.get('BayPosNo')) - #trim_no = record.get('TrimNo') - percentage = None # force user to set at renewal time - - - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {username}, {state}, {postcode}') - if self.test: - continue - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) - - country = Country.objects.get(printable_name='Australia') - address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - user.residential_address = address - user.postal_address = address - user.save() - - user_data.append([pers_no, email, user.dob, firstname, lastname, mobile_no, phone_no, address]) - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - except ObjectDoesNotExist: - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage) - - try: - vessel_details = VesselDetails.objects.get(vessel=vessel) - except MultipleObjectsReturned: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - except ObjectDoesNotExist: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - berth_mooring='home' - ) - - berth_mooring = '' - vessel_data.append([pers_no, email, user.dob, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring]) - - proposal=MooringLicenceApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.now(), - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - #vessel_beam='', - vessel_weight=vessel_weight, - berth_mooring='home', - preferred_bay= mooring.mooring_bay, - percentage=percentage, - individual_owner=True, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - allocated_mooring=mooring, - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - "vessel_ownership": [{ - "dot_name": vessel_dot - }], - "mooring_on_approval": [] - }, - date_invited=None #date_invited, - dot_name=vessel_dot, - ) - - date_invited = '' - mla_data.append([pers_no, email, user.dob, rego_no, mooring.mooring_bay, date_invited]) - -#> "proposed_issuance_approval": { -#> "details": "dd", -#> "cc_email": null, -#> "mooring_id": 127, -#> "mooring_bay_id": 7, -#> "ria_mooring_name": "CB005", -#> "vessel_ownership": [], -#> "mooring_on_approval": [] -#> }, -# -#> "berth_mooring": "home", -#> "dot_name": "abc", -#> "insurance_choice": "over_ten", -#> "preferred_bay": null, -#> "silent_elector": null, -#> "mooring_authorisation_preference": "ria", -#> "bay_preferences_numbered": "[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\"]", - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Mooring Site Licence - Migrated Application', - ) - - #approval = WaitingListAllocation.objects.create( - approval = MooringLicence.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - proposal.allocated_mooring.mooring_licence = approval - proposal.allocated_mooring.save() - - vooa = VesselOwnershipOnApproval.objects.create( - approval=approval, - vessel_ownership=vessel_ownership, - ) -# moa = MooringOnApproval.objects.create( -# approval=approval, -# mooring=mooring, -# sticker=None, -# site_licensee=True, # ??? -# end_date=expiry_date -# ) - - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - address_list.append(address.id) - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - ml_all_data.append([pers_no, email, user.dob, firstname, lastname, mobile_no, phone_no, address, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring, mooring.mooring_bay, date_invited]) - -# except IntegrityError as e: -# errors.append(repr(e)) -# continue - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - - with open('ml_all.csv', 'w') as f: - writer = csv.writer(f) - #writer.writerow(header) - writer.writerows(ml_all_data) - -# print(f'Address.objects.get(id__in={address_list}).delete()') -# print(f'EmailUser.objects.get(id__in={user_list}).delete()') -# print(f'Vessel.objects.get(id__in={vessel_list}).delete()') -# print(f'Owner.objects.get(id__in={owner_list}).delete()') -# print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') -# print(f'VesselDetails.objects.get(id__in={details_list}).delete()') -# print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') -# print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') -# print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') - print(f'errors: {errors}') - - -def clear_record(): - Address.objects.last().delete() - EmailUser.objects.last().delete() - Vessel.objects.last().delete() - Owner.objects.last().delete() - VesselOwnership.objects.last().delete() - VesselDetails.objects.last().delete() - WaitingListApplication.objects.last().delete() - ProposalUserAction.objects.last().delete() - WaitingListAllocation.objects.last().delete() - ApprovalHistory.objects.last().delete() - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - #_files = self.get_files(self.search_str1) - files=[ - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - #print(f'fname: {fname}, key2: {key2}') - return record - - diff --git a/mooringlicensing/utils/archive/mooring_licence_migrate_pd.py b/mooringlicensing/utils/archive/mooring_licence_migrate_pd.py deleted file mode 100644 index 3236704f0..000000000 --- a/mooringlicensing/utils/archive/mooring_licence_migrate_pd.py +++ /dev/null @@ -1,2659 +0,0 @@ -from ledger.accounts.models import Organisation as ledger_organisation -from ledger.accounts.models import OrganisationAddress -from ledger.accounts.models import EmailUser, Address -from ledger.payments.models import Invoice -from django.conf import settings -from django.db import connection -#from disturbance.components.organisations.models import Organisation, OrganisationContact, UserDelegation -#from disturbance.components.main.models import ApplicationType -#from disturbance.components.main.utils import get_category -#from disturbance.components.proposals.models import Proposal, ProposalType, ApiarySite, ApiarySiteOnProposal, ProposalApiary #, ProposalOtherDetails, ProposalPark -#from disturbance.components.approvals.models import Approval, MigratedApiaryLicence, ApiarySiteOnApproval -from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist -from django.db import IntegrityError, transaction -from django.contrib.gis.geos import GEOSGeometry -from django.utils import timezone -from django.db.models import Q -from ledger.address.models import Country -import csv -import os -import datetime -import time -import string -import pandas as pd -import numpy as np - -from dateutil.relativedelta import relativedelta -from decimal import Decimal - -from mooringlicensing.components.payments_ml.models import FeeSeason - -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - AuthorisedUserApplication, - WaitingListApplication, - AnnualAdmissionApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import ( - Approval, - ApprovalHistory, - MooringLicence, - AuthorisedUserPermit, - VesselOwnershipOnApproval, - MooringOnApproval, - WaitingListAllocation, - AnnualAdmissionPermit, - ApprovalUserAction, - Sticker, - DcvOrganisation, - DcvPermit, - DcvVessel, -) - -from tqdm import tqdm -import logging -logger = logging.getLogger(__name__) - -NL = '\n' - -TODAY = datetime.datetime.now(datetime.timezone.utc).date() -EXPIRY_DATE = datetime.date(2023,8,31) -START_DATE = datetime.date(2022,9,1) -DATE_APPLIED = '2022-09-01' -FEE_SEASON = '2022 - 2023' - -VESSEL_TYPE_MAPPING = { - 'A': 'Catamaran', - 'B': 'Bow Rider', - 'C': 'Cabin Cruiser', - 'E': 'Centre Console', - 'F': 'Ferry', - 'G': 'Rigid Inflatable', - 'H': 'Half Cabin', - 'I': 'Inflatable', - 'L': 'Launch', - 'M': 'Motor Sailer', - 'O': 'Open Boat', - 'P': 'Power Boat', - 'R': 'Runabout', - 'S': 'Fishing Boat', - 'T': 'Tender', - 'W': 'Walkaround', - 'Y': 'Yacht', -} - -USER_COLUMN_MAPPING = { - 'Person No': 'pers_no', - 'Type of Licences': 'licences_type', - 'Cancelled': 'cancelled', - 'Paid Up?': 'paid_up', - 'BP$': 'bp', - 'First Name': 'first_name', - 'Middle1 Initial': 'middle_name', - 'Last Name': 'last_name', - 'Age': 'age', - 'Date of Birth': 'dob', - 'Company': 'company', - 'Address': 'address', - 'Suburb': 'suburb', - 'State': 'state', - 'Post Code': 'postcode', - 'Postal Address': 'postal_address', - 'Postal Suburb': 'postal_suburb', - 'Postal State': 'postal_state', - 'Postal Post Code': 'postal_postcode', - 'Electoral Roll': 'electoral_role', - 'Phone Home': 'home_number', - 'Phone Work': 'work_number', - 'Phone Mobile': 'mobile_number', - 'Fax': 'fax', - 'e-Mail Address': 'email', - 'Email Promotions': 'email_prom', - 'Waitlist Bay': 'wl_bay', - 'Bays Auth for': 'bays_auth_for', - 'No Bays Auth for': 'no_bays_auth_for', - 'Vessel Sold': 'vessel_sold', - 'Current Mooring No': 'current_mooring_no', - 'Vsl Due Date': 'ves_due_date', - 'No Users (for Lics)': 'no_user_lics', -} - - -ML_COLUMN_MAPPING = { - 'Mooring Number': 'mooring_no', - 'Licencee First Name': 'first_name_l', - 'Licencee Last Name': 'last_name_l', - 'Pers No': 'pers_no', - 'Grandfather': 'grandfather', - 'Tonnage': 'tonnage', - 'Cert for addn Vessel': 'cert_addn_vessel', - 'Mooring Type': 'mooring_type', - 'Float Colour': 'float_colour', - 'Enviro Specs': 'enviro_specs', - 'Buoy Specs': 'buoy_specs', - 'Mooring Bay': 'mooring_bay', - 'Cancelled': 'cancelled', - 'No Auth Users 2012': 'no_auth_user_2012', - 'No Auth Users 2012 RIA': 'no_auth_users_2012_ria', - 'No Auth Users 2012 Lic': 'no_auth_user_2012_lic', - 'No Auth Users': 'no_auth_users', - 'No Auth Users Lic': 'no_auth_user_lic', - 'No Auth Users XL': 'no_auth_users_xl', - 'Colour Current': 'colour', - 'Current Max Vessel on Site': 'max_vessel_on_site', - 'Lic Ves Length': 'ves_length_lic', - 'Draft': 'draft', - 'Max Ves Length': 'max_ves_length', - 'Comments/Notes': 'comments_notes', - 'Comments': 'comments', -} - -AU_COLUMN_MAPPING = { - 'Mooring Number': 'mooring_no', - 'Cancelled': 'cancelled', - 'Licencee Approved Authorised User':'licencee_approved', - 'First Name (L)': 'first_name_l', - 'Last Name (L)': 'last_name_l', - 'Person No (L)': 'pers_no_l', - 'User Type': 'user_type', - 'Date Issued': 'date_issued', - 'Sticker No': 'sticker', - 'First Name (U)': 'first_name_u', - 'Last Name (U)': 'last_name_u', - 'Lic of Mooring No': 'lic_of_mooring_no', - 'PersNo (U)': 'pers_no_u', - 'Vessel Rego': 'vessel_rego', - 'Vessel Name': 'vessel_name', - 'Vessel Length': 'vessel_length', - 'Vessel Draft': 'vessel_draft', - 'Date Cancelled': 'date_cancelled', - 'Reason Cancelled': 'reason_cancelled', -} - -WL_COLUMN_MAPPING = { - 'FirstName': 'first_name', - 'LastName': 'last_name', - 'PersNo': 'pers_no', - 'TrimNo': 'trim_no', - 'Bay': 'bay', - 'AppStatus': 'app_status', - 'BayPosNo': 'bay_pos_no', - 'SeqNo': 'seq_no', - 'VesName': 'vessel_name', - 'RegLength': 'vessel_length', - 'VesRego': 'vessel_rego', - 'Draft': 'vessel_draft', - 'DateApplied': 'date_applied', - 'DateCancelled': 'date_cancelled', - 'DateAllocated': 'date_allocated', - 'DateAnnConfRecd': 'date_annconf_recd', - 'MooringNo': 'mooring_no', - 'Cancelled': 'cancelled', - 'Comment': 'comment', -} - - -AA_COLUMN_MAPPING = { - 'ID': 'id', - 'Date Created': 'date_created', - 'First Name': 'first_name', - 'Last Name': 'last_name', - 'Postal Address 1': 'address', - 'Suburb': 'suburb', - 'State': 'state', - 'Post Code': 'postcode', - 'Sticker No': 'sticker_no', - 'Vessel Rego': 'rego_no', - 'Email': 'email', - 'Mobile': 'mobile_number', - 'Phone': 'work_number', - 'Vessel Name': 'vessel_name', - 'Vessel Length': 'vessel_length', - 'Year': 'year', - 'Status': 'status', - 'Booking Period': 'booking_period', - 'Country': 'country', -} - -class MooringLicenceReader(): - """ - First need to run clean(). - This will write output files to /var/www/mooringlicensing/mooringlicensing/utils/csv/clean - from mooringlicensing.utils.export_clean import clean - clean() - - Notes: - Clean CSV - 1. open vessel_details in vi and replace the str chars --> ':%s/"//g' - 2. vessel_details - 'Person No' 210406 - has extra column delete cell B - - FROM shell_plus: - from mooringlicensing.utils.mooring_licence_migrate_pd import MooringLicenceReader - mlr=MooringLicenceReader('PersonDets.txt', 'MooringDets.txt', 'VesselDets.txt', 'UserDets.txt', 'ApplicationDets.txt', 'annual_admissions_booking_report.csv') - - mlr.create_users() - mlr.create_vessels() - mlr.create_mooring_licences() - mlr.create_authuser_permits() - mlr.create_waiting_list() - mlr.create_dcv() - mlr.create_annual_admissions() - - OR - from mooringlicensing.utils.mooring_licence_migrate_pd import MooringLicenceReader - mlr=MooringLicenceReader('PersonDets.txt', 'MooringDets.txt', 'VesselDets.txt', 'UserDets.txt', 'ApplicationDets.txt', 'annual_admissions_booking_report.csv') - mlr.run_migration - - FROM mgt-command: - python manage_ds.py mooring_migration_script --filename mooringlicensing/utils/csv/MooringDets20221201-083202.txt - - - Check for unique permit numbers - pm = df['permit_number'].value_counts().loc[lambda x: x>1][1:].astype('int') - pm.to_excel('/tmp/permit_numbers.xlsx') - - From RKS: - - from mooringlicensing.utils.mooring_licence_migrate_pd import MooringLicenceReader - mlr=MooringLicenceReader('PersonDets.txt','MooringDets.txt','VesselDets.txt','UserDets.txt','ApplicationDets.txt','annual_admissions_booking_report.csv',path='/app/shared/clean/clean_22Dec2022/') - - """ - - def __init__(self, fname_user, fname_ml, fname_ves, fname_authuser, fname_wl, fname_aa, path='mooringlicensing/utils/csv/clean/'): - #import ipdb; ipdb.set_trace() - self.df_user=pd.read_csv(path+fname_user, delimiter='|', dtype=str, low_memory=False) - self.df_ml=pd.read_csv(path+fname_ml, delimiter='|', dtype=str) - self.df_ves=pd.read_csv(path+fname_ves, delimiter='|', dtype=str) - self.df_authuser=pd.read_csv(path+fname_authuser, delimiter='|', dtype=str) - self.df_wl=pd.read_csv(path+fname_wl, delimiter='|', dtype=str) - self.df_aa=pd.read_csv(path+fname_aa, delimiter=',', dtype=str) - - #import ipdb; ipdb.set_trace() - self.df_user=self._read_users() - self.df_ml=self._read_ml() - self.df_ves=self._read_ves() - self.df_authuser=self._read_au() - self.df_wl=self._read_wl() - self.df_aa=self._read_aa() - self.df_dcv=self._read_dcv() - - # create_users - self.pers_ids = [] - self.user_created = [] - self.user_existing = [] - self.user_errors = [] - self.no_email = [] - - # create_vessels - self.vessels_created = [] - self.vessels_existing = [] - self.vessels_errors = [] - self.pct_interest_errors = [] - self.pct_interest_errors = [] - self.no_ves_rows = [] - - # create_vessels - self.ml_user_not_found = [] - self.ml_no_ves_rows = [] - - def _get_phone_number(self, row): - if 'work_number' in row and row.work_number: - row.work_number.replace(' ', '') - return row.work_number - - elif 'home_number' in row and row.home_number: - row.home_number.replace(' ', '') - return row.home_number - - def _get_work_number(self, row): - return row.work_number.replace(' ', '') - - - def _get_mobile_number(self, row): - return row.mobile_number.replace(' ', '') - -# def _read_excel(self, filename): -# def _get_country_code(x): -# try: -# country=Country.objects.get(iso_3166_1_a2=x.get('country')) -# except Exception as e: -# country=Country.objects.get(iso_3166_1_a2='AU') -# return country.code -# -# -# df = pd.read_excel(filename) -# -# # Rename the cols from Spreadsheet headers to Model fields names -# df = df.rename(columns=COLUMN_MAPPING) -# df[df.columns] = df.apply(lambda x: x.str.strip() if isinstance(x, str) else x) -# #df['start_date'] = pd.to_datetime(df['start_date'], errors='coerce') -# #df['expiry_date'] = pd.to_datetime(df['expiry_date'], errors='coerce') -# #df['issue_date'] = pd.to_datetime(df['issue_date'], errors='coerce') -# #df['approval_cpc_date'] = pd.to_datetime(df['approval_cpc_date'], errors='coerce') -# #df['approval_minister_date'] = pd.to_datetime(df['approval_minister_date'], errors='coerce') -# -# #df['issue_date'] = df.apply(lambda row: row.issue_date if isinstance(row.issue_date, datetime.datetime) else row.start_date, axis=1) -# #df['abn'] = df['abn'].str.replace(" ","").str.strip() -# #df['email'] = df['email'].str.replace(" ","").str.lower().str.strip() -# #df['first_name'] = df['first_name'].apply(lambda x: x.lower().capitalize().strip() if not pd.isnull(x) else 'No First Name') -# #df['last_name'] = df['last_name'].apply(lambda x: x.lower().capitalize().strip() if not pd.isnull(x) else 'No Last Name') -# #df['licencee'] = df['licencee'].apply(lambda x: x.strip() if not pd.isnull(x) else 'No Licencee Name') -# #df['postcode'] = df['postcode'].apply(lambda x: '0000' if pd.isnull(x) else x) -# #df['country'] = df['country'].apply(_get_country_code) -# -# # clean everything else -# df.fillna('', inplace=True) -# df.replace({np.NaN: ''}, inplace=True) -# -# # check excel column names are the same column_mappings -# #import ipdb; ipdb.set_trace() -# #if df.columns.values.tolist() != [*COLUMN_MAPPING.values()]: -# # raise Exception('Column Names have changed!') -# -# # add extra column -# #df['licencee_type'] = df['abn'].apply(lambda x: 'organisation' if x else 'individual') -# -# #import ipdb; ipdb.set_trace() -# #return df[:500] -# return df - - def _read_users(self): - def _get_country_code(x): - try: - country=Country.objects.get(iso_3166_1_a2=x.get('country')) - except Exception as e: - country=Country.objects.get(iso_3166_1_a2='AU') - return country.code - - - #import ipdb; ipdb.set_trace() - # Rename the cols from Spreadsheet headers to Model fields names - df_user = self.df_user.rename(columns=USER_COLUMN_MAPPING) - df_user[df_user.columns] = df_user.apply(lambda x: x.str.strip() if isinstance(x, str) else x) - - # clean everything else - df_user.fillna('', inplace=True) - df_user.replace({np.NaN: ''}, inplace=True) - - #return df[:500] - return df_user - - def _read_ml(self): - - # Rename the cols from Spreadsheet headers to Model fields names - df_ml = self.df_ml.rename(columns=ML_COLUMN_MAPPING) - df_ml[df_ml.columns] = df_ml.apply(lambda x: x.str.strip() if isinstance(x, str) else x) - - # clean everything else - df_ml.fillna('', inplace=True) - df_ml.replace({np.NaN: ''}, inplace=True) - - # filter cancelled and rows with no name - #import ipdb; ipdb.set_trace() - df_ml = df_ml[(df_ml['cancelled']=='N') & (df_ml['first_name_l'].isna()==False)] - - #return df[:500] - return df_ml - - def _read_ves(self): - - # clean everything else - self.df_ves.fillna('', inplace=True) - self.df_ves.replace({np.NaN: ''}, inplace=True) - - # filter cancelled and rows with no name - #df_ml = df_ml[(df_ml['cancelled']=='N') & (df_ml['first_name_l'].isna()==False)] - - #return df[:500] - return self.df_ves - - def _read_au(self): - """ Read Auth User file """ - - # Rename the cols from Spreadsheet headers to Model fields names - #import ipdb; ipdb.set_trace() - df_authuser = self.df_authuser.rename(columns=AU_COLUMN_MAPPING) - df_authuser[df_authuser.columns] = df_authuser.apply(lambda x: x.str.strip() if isinstance(x, str) else x) - - # clean everything else - df_authuser.fillna('', inplace=True) - df_authuser.replace({np.NaN: ''}, inplace=True) - - # filter cancelled and rows with no name - #import ipdb; ipdb.set_trace() - df_authuser = df_authuser[(df_authuser['cancelled']=='N') & (df_authuser['user_type']!='L')] - - #return df[:500] - return df_authuser - - def _read_wl(self): - """ Read Auth User file """ - - # Rename the cols from Spreadsheet headers to Model fields names - #import ipdb; ipdb.set_trace() - df_wl = self.df_wl.rename(columns=WL_COLUMN_MAPPING) - df_wl[df_wl.columns] = df_wl.apply(lambda x: x.str.strip() if isinstance(x, str) else x) - df_wl['bay_pos_no']= pd.to_numeric(df_wl['bay_pos_no']) - - # clean everything else - df_wl.fillna('', inplace=True) - df_wl.replace({np.NaN: ''}, inplace=True) - - # filter cancelled and rows with no name - #import ipdb; ipdb.set_trace() - df_wl = df_wl[(df_wl['app_status']=='W')] - df_wl = df_wl.sort_values(['bay_pos_no'],ascending=True).groupby('bay').head(1000) - - #return df[:500] - return df_wl - - def _read_dcv(self): - """ Read PersonDets file - for DCV Permits details """ - if len(self.df_user)==0: - _read_user() - - df_dcv = self.df_user[(self.df_user['company']!='') & (self.df_user['paid_up']=='Y') & (self.df_user['licences_type']=='C')] - return df_dcv - - def _read_aa(self): - """ Read Annual Admissions file created from - https://mooring-ria-internal.dbca.wa.gov.au/dashboard/bookings/annual-admissions/ - - Eg. mooringlicensing/utils/csv/clean/annual_admissions_booking_report_20230125084027.csv - """ - # Rename the cols from Spreadsheet headers to Model fields names - #import ipdb; ipdb.set_trace() - df_aa = self.df_aa.rename(columns=AA_COLUMN_MAPPING) - df_aa[df_aa.columns] = df_aa.apply(lambda x: x.str.strip() if isinstance(x, str) else x) - - # clean everything else - df_aa.fillna('', inplace=True) - df_aa.replace({np.NaN: ''}, inplace=True) - - # create dict of vessel details by rego_no --> self.vessels_dict['DO904'] - logger.info('Creating Vessels details Dictionary ...') - self.create_vessels_dict() - - #df_dcv = self.df_user[(self.df_user['company']!='') & (self.df_user['paid_up']=='Y') & (self.df_user['licences_type']=='C')] - return df_aa - - - def run_migration(self): - #mlr=MooringLicenceReader('PersonDets20221222-125823.txt', 'MooringDets20221222-125546.txt', 'VesselDets20221222-125823.txt', 'UserDets20221222-130353.txt') - self.create_users() - self.create_vessels() - self.create_mooring_licences() - self.create_authuser_permits() - self.create_waiting_list() - self.create_dcv() - self.create_annual_admissions() - - - def _run_migration(self): - - # create the users and organisations, if they don't already exist - t0_start = time.time() - #try: - self.create_users() - self.create_organisations() - #except Exception as e: - # print(e) - # import ipdb; ipdb.set_trace() - t0_end = time.time() - print('TIME TAKEN (Orgs and Users): {}'.format(t0_end - t0_start)) - - # create the Migratiom models - t1_start = time.time() - #try: - self.create_licences() - #except Exception as e: - # print(e) - # import ipdb; ipdb.set_trace() - t1_end = time.time() - print('TIME TAKEN (Create License Models): {}'.format(t1_end - t1_start)) - - # create the Licence/Permit PDFs - t2_start = time.time() - try: - self.create_licence_pdf() - except Exception as e: - print(e) - import ipdb; ipdb.set_trace() - t2_end = time.time() - print('TIME TAKEN (Create License PDFs): {}'.format(t2_end - t2_start)) - - print('TIME TAKEN (Total): {}'.format(t2_end - t0_start)) - - def create_users(self): - logger.info('Creating DCV users ...') - df = self.df_dcv.groupby('pers_no').first() - self._create_users_df(df) - self.pers_ids_dcv = self.pers_ids - - logger.info('Creating ML & AU users ...') - for pers_type in ['pers_no_u', 'pers_no_l']: - df = self.df_authuser.groupby(pers_type).first() - self._create_users_df(df) - - logger.info('Creating WL users ...') - df = self.df_wl.groupby('pers_no').first() - self._create_users_df(df) - - logger.info('Creating AA users ...') - #self._create_users_df_aa(self.df_aa) - self._create_users_df(self.df_aa) - - def _create_users_df(self, df): - #with transaction.atomic(): - for index, row in tqdm(df.iterrows(), total=df.shape[0]): - try: - if not row.name: - continue - - user_row = self.df_user[self.df_user['pers_no']==row.name] #.squeeze() # as Pandas Series - if user_row.empty: - continue - - if len(user_row)>1: - user_row = user_row[user_row['paid_up']=='Y'] - user_row = user_row.squeeze() # convert to Pandas Series - - email = user_row.email.lower().replace(' ','') - if not email: - self.no_email.append(user_row.pers_no) - continue - - first_name = user_row.first_name.lower().title().strip() - last_name = user_row.last_name.lower().title().strip() - -# if email == 'mch@iinet.net.au': -# import ipdb; ipdb.set_trace() - - if user_row.pers_no == '127286': - import ipdb; ipdb.set_trace() - - country = Country.objects.get(printable_name='Australia') - users = EmailUser.objects.filter(email=email) - if users.count() == 0: - user = EmailUser.objects.create( - email=email, - first_name=first_name, - last_name=last_name, - phone_number=self._get_phone_number(user_row), - mobile_number=self._get_mobile_number(user_row) - ) - - address = Address.objects.create(line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country, user=user) - - self.user_created.append(email) - #import ipdb; ipdb.set_trace() - #continue - else: - user = users[0] - #print(f'User exists: {user}') - #continue - # update user details - user.first_name = first_name - user.last_name = last_name - user.phone_number = self._get_phone_number(user_row) - user.mobile_number = self._get_mobile_number(user_row) - - items = [i.strip() for i in user_row.address.split(',')] - if len(items) == 1: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) - else: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) - - if addresses.count()>0: - address = addresses[0] - #address.line1 = user_row.postal_address - #address.save() - with connection.cursor() as cursor: - if user_row.address: - if len(items) == 1: - cursor.execute(f''' - UPDATE accounts_address SET - line1=\'{items[0].replace("'", "")}\', - line2=\'\', - line3=\'\', - locality=\'{user_row.suburb.replace("'", "")}\', - postcode=\'{user_row.postcode}\', - state=\'{user_row.state}\', - country=\'{country.code}\' - WHERE id={address.id} - ''') - elif len(items) == 2: - cursor.execute(f''' - UPDATE accounts_address SET - line1=\'{items[0].replace("'", "")}\', - line2=\'{items[1].replace("'", "")}\', - line3=\'\', - locality=\'{user_row.suburb.replace("'", "")}\' , - postcode=\'{user_row.postcode}\', - state=\'{user_row.state}\', - country=\'{country.code}\' - WHERE id={address.id} - ''') - elif len(items) > 2: - cursor.execute(f''' - UPDATE accounts_address SET - line1=\'{items[0].replace("'", "")}\', - line2=\'{items[1].replace("'", "")}\', - line3=\'{items[2].replace("'", "")}\', - locality=\'{user_row.suburb.replace("'", "")}\', - postcode=\'{user_row.postcode}\', - state=\'{user_row.state}\', - country=\'{country.code}\' - WHERE id={address.id} - ''') - - -# #addr = user_row.address.strip(''').strip('"') -# if len(items) == 1: -# cursor.execute(f'''UPDATE accounts_address SET line1=\'{items[0].replace("'", "")}\', line2=\'\', line3=\'\' WHERE id={address.id}''') -# elif len(items) == 2: -# cursor.execute(f'''UPDATE accounts_address SET line1=\'{items[0].replace("'", "")}\', line2=\'{items[1].replace("'", "")}\', line3=\'\' WHERE id={address.id}''') -# elif len(items) > 2: -# cursor.execute(f'''UPDATE accounts_address SET line1=\'{items[0].replace("'", "")}\', line2=\'{items[1].replace("'", "")}\', line3=\'{items[2].replace("'", "")}\' WHERE id={address.id}''') -# -# address.locality=user_row.suburb -# address.postcode=user_row.postcode -# address.state=user_row.state -# address.country=country -# address.save() - - else: - address = Address.objects.create(line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country, user=user) - - self.user_existing.append(email) - - postal_address = None - if user_row.postal_address: - items = [i.strip() for i in user_row.postal_address.split(',')] - if len(items) == 1: - postal_addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) - else: - postal_addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) - - if postal_addresses.count()>0: - postal_address = postal_addresses[0] - with connection.cursor() as cursor: - if user_row.address: - #addr = user_row.address.strip(''').strip('"') - if len(items) == 1: - cursor.execute(f''' - UPDATE accounts_address SET - line1=\'{items[0].replace("'", "")}\', - line2=\'\', - line3=\'\', - locality=\'{user_row.postal_suburb.replace("'", "")}\', - postcode=\'{user_row.postal_postcode}\', - state=\'{user_row.postal_state}\', - country=\'{country.code}\' - WHERE id={postal_address.id} - ''') - elif len(items) == 2: - cursor.execute(f''' - UPDATE accounts_address SET - line1=\'{items[0].replace("'", "")}\', - line2=\'{items[1].replace("'", "")}\', - line3=\'\', - locality=\'{user_row.postal_suburb.replace("'", "")}\' , - postcode=\'{user_row.postal_postcode}\', - state=\'{user_row.postal_state}\', - country=\'{country.code}\' - WHERE id={postal_address.id} - ''') - elif len(items) > 2: - cursor.execute(f''' - UPDATE accounts_address SET - line1=\'{items[0].replace("'", "")}\', - line2=\'{items[1].replace("'", "")}\', - line3=\'{items[2].replace("'", "")}\', - locality=\'{user_row.postal_suburb.replace("'", "")}\', - postcode=\'{user_row.postal_postcode}\', - state=\'{user_row.postal_state}\', - country=\'{country.code}\' - WHERE id={postal_address.id} - ''') - - -# postal_address.locality=user_row.postal_suburb -# postal_address.postcode=user_row.postal_postcode -# postal_address.state=user_row.postal_state -# postal_address.country=country -# postal_address.save() - else: - postal_address = Address.objects.create(line1=user_row.postal_address, locality=user_row.postal_suburb, postcode=user_row.postal_postcode, state=user_row.postal_state, country=country, user=user) - - user.residential_address = address - user.postal_address = postal_address - user.save() - - self.pers_ids.append((user.id, row.name)) - - except Exception as e: - import ipdb; ipdb.set_trace() - self.user_errors.append(user_row.email) - logger.error(f'user: {row.name} *********** 1 *********** FAILED. {e}') - - print(f'Users Created: {len(self.user_created)}') - print(f'Users Existing: {len(self.user_existing)}') - - def _create_users_df_aa(self, df): - """ Reads the annual_admissions file created from the Mooring Booking System """ - for index, row in tqdm(df.iterrows(), total=df.shape[0]): - try: - user_row = row.copy() - - email = user_row.email.lower().replace(' ','') - if not email: - self.no_email.append(user_row.pers_no) - continue - - if email == 'tim@paramountsafety.com.au': - import ipdb; ipdb.set_trace() - - first_name = user_row.first_name.lower().title().strip() - last_name = user_row.last_name.lower().title().strip() - - users = EmailUser.objects.filter(email=email) - if users.count() == 0: - continue - else: - user = users[0] - print(f'User exists: {user}') - continue - except Exception as e: - import ipdb; ipdb.set_trace() - self.user_errors.append(user_row.email) - logger.error(f'user: {row.name} *********** 1 *********** FAILED. {e}') - - def __create_users_df(self, df): - # Iterate through the dataframe and create non-existent users - #import ipdb; ipdb.set_trace() - for index, row in tqdm(df.iterrows(), total=df.shape[0]): - #if row.status != 'Vacant': - try: - #import ipdb; ipdb.set_trace() - #first_name = data['first_name'] if not pd.isnull(data['first_name']) else 'No First Name' - #last_name = data['last_name'] if not pd.isnull(data['last_name']) else 'No Last Name' - #email = df['email'] - #import ipdb; ipdb.set_trace() - - #if row.name == '206846': - # import ipdb; ipdb.set_trace() - - if not row.name: - continue - - user_row = self.df_user[self.df_user['pers_no']==row.name] #.squeeze() # as Pandas Series - if user_row.empty: - continue - - if len(user_row)>1: - user_row = user_row[user_row['paid_up']=='Y'] - user_row = user_row.squeeze() # convert to Pandas Series - - email = user_row.email.lower().replace(' ','') - if not email: - self.no_email.append(user_row.pers_no) - continue - - first_name = user_row.first_name.lower().title().strip() - last_name = user_row.last_name.lower().title().strip() - #first_name = ' '.join([i.lower().capitalize().replace(' ','') for i in user_row.first_name.split(' ')]) - #last_name = ' '.join([i.lower().capitalize().replace(' ','') for i in user_row.last_name.split(' ')]) - - #if email == 'tim@paramountsafety.com.au': - #if email == 'gill.herbert@nightingale.net.au': - if email == 'mch@iinet.net.au': - import ipdb; ipdb.set_trace() - - users = EmailUser.objects.filter(email=email) - if users.count() == 0: - #import ipdb; ipdb.set_trace() - user = EmailUser.objects.create( - email=email, - first_name=first_name, - last_name=last_name, - phone_number=self._get_phone_number(user_row), - mobile_number=self._get_mobile_number(user_row) - ) - - country = Country.objects.get(printable_name='Australia') - #addresses = Address.objects.filter(user=user) - #addresses = Address.objects.filter(locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, user=user) - items = [i.strip() for i in user_row.address.split(',')] - if len(items) == 1: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) - else: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) - - if addresses.count()>0: - address = addresses[0] - address.line1 = user_row.postal_address - address.save() - else: - address = Address.objects.create(line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country, user=user) - - - self.user_created.append(email) - else: - user = users[0] - # update user details - user.first_name = first_name - user.last_name = last_name - user.phone_number = self._get_phone_number(user_row) - user.mobile_number = self._get_mobile_number(user_row) - - country = Country.objects.get(printable_name='Australia') - #addresses = Address.objects.filter(user=user, line1=user_row.address) - #addresses = Address.objects.filter(locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, user=user) - items = [i.strip() for i in user_row.address.split(',')] - if len(items) == 1: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) - else: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) - - #addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=user_row.address.split(',')[0])) - if addresses.count()>0: - address = addresses[0] - address.line1 = user_row.postal_address - address.save() - else: - address = Address.objects.create(line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country, user=user) - -# try: -# address, address_created = Address.objects.get_or_create(user=user, line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country) -# except MultipleObjectsReturned as e: -# address = Address.objects.filter(user=user)[0] - - self.user_existing.append(email) - - # update existing address - address.locality=user_row.suburb - address.postcode=user_row.postcode - address.state=user_row.state - address.country=country - address.save() - - # update existing postal address - #postal_address = user.postal_address - postal_address = None - if user_row.postal_address: - #postal_addresses = Address.objects.filter(user=user, line1=user_row.address) - #postal_addresses = Address.objects.filter(locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, user=user) - #postal_addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=user_row.postal_address.split(',')[0])) - items = [i.strip() for i in user_row.postal_address.split(',')] - if len(items) == 1: - postal_addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) - else: - postal_addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) - - if postal_addresses.count()>0: - postal_address = postal_addresses[0] - postal_address.line1 = user_row.postal_address - postal_address.save() - else: - postal_address = Address.objects.create(line1=user_row.postal_address, locality=user_row.postal_suburb, postcode=user_row.postal_postcode, state=user_row.postal_state, country=country, user=user) - - #postal_address, created = Address.objects.update_or_create(user=user, line1=user_row.postal_address, locality=user_row.postal_suburb, postcode=user_row.postal_postcode, state=user_row.postal_state, country=country) - -# try: -# postal_address = Address.objects.create(user=user, line1=user_row.postal_address, locality=user_row.postal_suburb, postcode=user_row.postal_postcode, state=user_row.postal_state, country=country) -# except IntegrityError as e: -# postal_address.line1=user_row.postal_address -# postal_address.locality=user_row.postal_suburb -# postal_address.line1=user_row.postal_postcode -# postal_address.line1=user_row.postal_state -# postal_address.country=country -# postal_address.save() - - user.residential_address = address - user.postal_address = postal_address - user.save() - #print(f'{email}: {user.postal_address}') - - self.pers_ids.append((user.id, row.name)) - - - except Exception as e: - import ipdb; ipdb.set_trace() - self.user_errors.append(user_row.email) - logger.error(f'user: {row.name} *********** 1 *********** FAILED. {e}') - - print(f'users created: {len(self.user_created)}') - print(f'users existing: {len(self.user_existing)}') - print(f'users errors: {len(self.user_errors)}') - print(f'no_email errors: {len(self.no_email)}') - print(f'users errors: {self.user_errors}') - print(f'no_email errors: {self.no_email}') - - def __create_users_df_aa(self, df): - """ Reads the annual_admissions file created from the Mooring Booking System """ - # Iterate through the dataframe and create non-existent users - #import ipdb; ipdb.set_trace() - for index, row in tqdm(df.iterrows(), total=df.shape[0]): - #if row.status != 'Vacant': - try: - #import ipdb; ipdb.set_trace() - #first_name = data['first_name'] if not pd.isnull(data['first_name']) else 'No First Name' - #last_name = data['last_name'] if not pd.isnull(data['last_name']) else 'No Last Name' - #email = df['email'] - #import ipdb; ipdb.set_trace() - - #if row.name == '206846': - # import ipdb; ipdb.set_trace() - - #user_row = self.df_user[self.df_user['pers_no']==row.name] #.squeeze() # as Pandas Series - user_row = row.copy() - - email = user_row.email.lower().replace(' ','') - if not email: - self.no_email.append(user_row.pers_no) - continue - - if email == 'tim@paramountsafety.com.au': - import ipdb; ipdb.set_trace() - - first_name = user_row.first_name.lower().title().strip() - last_name = user_row.last_name.lower().title().strip() - #first_name = ' '.join([i.lower().capitialize().replace(' ','') for i in user_row.first_name.split(' ')]) - #last_name = ' '.join([i.lower().capitalize().replace(' ','') for i in user_row.last_name.split(' ')]) - - users = EmailUser.objects.filter(email=email) - if users.count() == 0: - #import ipdb; ipdb.set_trace() - continue - user = EmailUser.objects.create( - email=email, - first_name=first_name, - last_name=last_name, - #phone_number=self._get_work_number(user_row), - phone_number=self._get_phone_number(user_row), - mobile_number=self._get_mobile_number(user_row) - ) - - country = Country.objects.get(printable_name='Australia') - # address, address_created = Address.objects.get_or_create(line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country, user=user) - #addresses = Address.objects.filter(user=user, line1=user_row.address) - #addresses = Address.objects.filter(locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, user=user) - #addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=user_row.address.split(',')[0])) - items = [i.strip() for i in user_row.address.split(',')] - if len(items) == 1: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) - else: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) - - - if addresses.count()>0: - address = addresses[0] - address.line1 = user_row.postal_address - address.save() - else: - address = Address.objects.create(line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country, user=user) - - - self.user_created.append(email) - else: - user = users[0] - print(f'User exists: {user}') - continue - # update user details - user.first_name = first_name - user.last_name = last_name - #user.phone_number = self._get_work_number(user_row) - user.phone_number = self._get_phone_number(user_row) - user.mobile_number = self._get_mobile_number(user_row) - - country = Country.objects.get(printable_name='Australia') - #addresses = Address.objects.filter(user=user, line1=user_row.address) - #addresses = Address.objects.filter(locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, user=user) - #addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=user_row.address.split(',')[0])) - items = [i.strip() for i in user_row.address.split(',')] - if len(items) == 1: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) - else: - addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) - - - if addresses.count()>0: - address = addresses[0] - address.line1 = user_row.postal_address - address.save() - else: - address = Address.objects.create(line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country, user=user) - -# try: -# address, address_created = Address.objects.get_or_create(user=user, line1=user_row.address, locality=user_row.suburb, postcode=user_row.postcode, state=user_row.state, country=country) -# except MultipleObjectsReturned as e: -# address = Address.objects.filter(user=user)[0] - - self.user_existing.append(email) - - # update existing residential address - address.locality=user_row.suburb - address.postcode=user_row.postcode - address.state=user_row.state - address.country=country - address.save() - -# # update existing postal address -# postal_address = None -# if user_row.postal_address: -# items = [i.strip() for i in user_row.postal_address.split(',')] -# if len(items) == 1: -# postal_addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0])) -# else: -# postal_addresses = Address.objects.filter(Q(user=user), Q(search_text__icontains=items[0]) | Q(search_text__icontains=items[1])) -# -# -# if postal_addresses.count()>0: -# postal_address = postal_addresses[0] -# else: -# postal_address = Address.objects.create(line1=user_row.postal_address, locality=user_row.postal_suburb, postcode=user_row.postal_postcode, state=user_row.postal_state, country=country, user=user) - - user.residential_address = address - user.postal_address = None #postal_address - user.save() - #print(f'{email}: {user.postal_address}') - - self.pers_ids.append((user.id, row.name)) - - - except Exception as e: - import ipdb; ipdb.set_trace() - self.user_errors.append(user_row.email) - logger.error(f'user: {row.name} *********** 1 *********** FAILED. {e}') - - print(f'users created: {len(self.user_created)}') - print(f'users existing: {len(self.user_existing)}') - print(f'users errors: {len(self.user_errors)}') - print(f'no_email errors: {len(self.no_email)}') - print(f'users errors: {self.user_errors}') - print(f'no_email errors: {self.no_email}') - - def create_vessels(self): - - def _ves_fields(prefix, ves_row, pers_no): - ''' - Returns the 7 fields for a given field type (ves_dets file allows upto 7 vessels per pers_no/owner) - Eg. - ves_fields('DoT Rego', ves_row, '000477') - Out[201]: ['GC1', 'DW106', 'DK819', 'EZ315', 'EF82'] - ''' - field1 = prefix + ' Nominated Ves' - field2 = prefix + ' Ad Ves 2' - field3 = prefix + ' Ad Ves 3' - field4 = prefix + ' Ad Ves 4' - field5 = prefix + ' Ad Ves 5' - field6 = prefix + ' Ad Ves 6' - field7 = prefix + ' Ad Ves 7' - - - ves_list_raw = ves_row[[field1, field2, field3, field4, field5, field6, field7]].values.flatten().tolist() - #return [i.rstrip(pers_no) for i in ves_list_raw if i.rstrip(pers_no)] - return [i[:-len(pers_no)] for i in ves_list_raw if i!=pers_no] - - def ves_fields(ves_row): - #import ipdb; ipdb.set_trace() - ves_details = [] - for colname_postfix in ['Nominated Ves', 'Ad Ves 2', 'Ad Ves 3', 'Ad Ves 4', 'Ad Ves 5', 'Ad Ves 6', 'Ad Ves 7']: - #cols = [col for col in ves_row.columns if (colname_postfix in col)] - cols = [col for col in ves_row.index if (colname_postfix in col)] - - #ves_detail = ves_row[cols].to_dict(orient='records')[0] - #ves_detail = ves_row[cols].to_dict(orient='records') - ves_detail = ves_row[cols].to_dict() - - #if ves_row['H. I. N. ' + colname_postfix].iloc[0] != '' or ves_row['DoT Rego ' + colname_postfix].iloc[0] != '': - if ves_row['H. I. N. ' + colname_postfix] != '' or ves_row['DoT Rego ' + colname_postfix] != '': - ves_details.append(ves_detail) - #ves_details.append(ves_detail) - - return ves_details - - def try_except(value): - #if pers_no=='213162': - # import ipdb; ipdb.set_trace() - - try: - val = float(value) - except Exception: - val = 0.00 - return val - - print(f'Deleting existing Vessels ...') - Vessel.objects.all().delete() - - self.vessels_au = {} - self.vessels_an = {} - self.vessels_dcv = {} - postfix = ['Nominated Ves', 'Ad Ves 2', 'Ad Ves 3', 'Ad Ves 4', 'Ad Ves 5', 'Ad Ves 6', 'Ad Ves 7'] - for user_id, pers_no in tqdm(self.pers_ids): - try: - #if pers_no=='202600': - # import ipdb; ipdb.set_trace() - - #import ipdb; ipdb.set_trace() - ves_rows = self.df_ves[self.df_ves['Person No']==pers_no] - if len(ves_rows) > 1: - ves_rows = ves_rows[(ves_rows['Au Sticker No Nominated Ves']!='')] - self.no_ves_rows.append((pers_no, len(ves_rows))) - - for idx, row in ves_rows.iterrows(): - ves_row = row.to_frame() - - ves_list = ves_fields(row) - - try: - owner = Owner.objects.get(emailuser_id=user_id) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser_id=user_id) - - au_vessels = {} - c_vessels = [] - for i, ves in enumerate(ves_list): - ves_name=ves['Name ' + postfix[i]] - ves_type=ves['Type ' + postfix[i]] - rego_no=ves['DoT Rego ' + postfix[i]] - pct_interest=ves['%Interest ' + postfix[i]] - tot_length=ves['Reg Length ' + postfix[i]] - draft=ves['Draft ' + postfix[i]] - tonnage=ves['Tonnage ' + postfix[i]] - c_sticker=ves['C Sticker No ' + postfix[i]] - au_sticker=ves['Au Sticker No ' + postfix[i]] # for first 4 moorings - au_sticker_date=ves['Date Au Sticker Sent ' + postfix[i]] # for first 4 moorings - - an_sticker=ves['An Sticker No ' + postfix[i]] # additional stickers, where number of mooring is > 4 (4 mooring's per sticker) - an_sticker_date=ves['Date Au Sticker Sent ' + postfix[i]] # additional stickers, where number of mooring is > 4 (4 mooring's per sticker) - #import ipdb; ipdb.set_trace() - moorings=[i.strip() for i in ves['Au Sites ' + postfix[i]].split(',')] - - #if rego_no=='BZ314': - # import ipdb; ipdb.set_trace() - try: - ves_type = VESSEL_TYPE_MAPPING[ves_type] - except Exception as e: - ves_type = 'other' - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - except ObjectDoesNotExist: - pct_interest = int(round(float(try_except(pct_interest)),0)) - if pct_interest < 25: - self.pct_interest_errors.append((pers_no, rego_no, pct_interest)) - pct_interest = 100 - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=pct_interest) - - try: - vessel_details = VesselDetails.objects.get(vessel=vessel) - vessel_details.vessel_type = ves_type - vessel_details.save() - except MultipleObjectsReturned: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - vessel_details.vessel_type = ves_type - vessel_details.save() - except: - vessel_details = VesselDetails.objects.create( - vessel=vessel, - vessel_type=ves_type, - vessel_name=ves_name, - vessel_length=try_except(tot_length), - vessel_draft=try_except(draft), - vessel_weight=try_except(tonnage), - berth_mooring='' - ) - #au_vessels.update({rego_no:au_sticker, sticker_sent:}) - self.vessels_au.update({rego_no: dict(au_sticker=au_sticker, au_sticker_sent=au_sticker_date, moorings=moorings[:4])}) - self.vessels_an.update({rego_no: dict(an_sticker=an_sticker, an_sticker_sent=an_sticker_date, moorings=moorings[4:])}) - c_vessels.append((rego_no, c_sticker)) - #if pers_no=='200700': - # import ipdb; ipdb.set_trace() - - #self.vessels_au.update({pers_no:au_vessels}) - self.vessels_dcv.update({pers_no:c_vessels}) - self.vessels_created.append((pers_no, len(c_vessels), c_vessels)) - - except Exception as e: - self.vessels_errors.append((pers_no, str(e))) - continue - - print(f'vessels created: {len(self.vessels_created)}') - print(f'vessels errors: {self.vessels_errors}') - print(f'vessels errors: {len(self.vessels_errors)}') - print(f'vessels pct_int errors: {self.pct_interest_errors}') - print(f'vessels pct_int errors: {len(self.pct_interest_errors)}') - print(f'vessels no_ves_rows: {self.no_ves_rows}: Total len({self.no_ves_rows})') - - - def create_vessels_dict(self): - - def ves_fields(ves_row): - #import ipdb; ipdb.set_trace() - ves_details = [] - for colname_postfix in ['Nominated Ves', 'Ad Ves 2', 'Ad Ves 3', 'Ad Ves 4', 'Ad Ves 5', 'Ad Ves 6', 'Ad Ves 7']: - cols = [col for col in ves_row.index if (colname_postfix in col)] - - ves_detail = ves_row[cols].to_dict() - - if ves_row['H. I. N. ' + colname_postfix] != '' or ves_row['DoT Rego ' + colname_postfix] != '': - ves_details.append(ves_detail) - - return ves_details - - self.vessels_dict = {} - postfix = ['Nominated Ves', 'Ad Ves 2', 'Ad Ves 3', 'Ad Ves 4', 'Ad Ves 5', 'Ad Ves 6', 'Ad Ves 7'] - for idx, row in tqdm(self.df_ves.iterrows(), total=self.df_ves.shape[0]): - ves_row = row.to_frame() - - ves_list = ves_fields(row) - - au_vessels = {} - c_vessels = [] - for i, ves in enumerate(ves_list): - try: - ves_name=ves['Name ' + postfix[i]] - ves_type=ves['Type ' + postfix[i]] - rego_no=ves['DoT Rego ' + postfix[i]] - pct_interest=ves['%Interest ' + postfix[i]] - tot_length=ves['Reg Length ' + postfix[i]] - draft=ves['Draft ' + postfix[i]] - tonnage=ves['Tonnage ' + postfix[i]] - beam=ves['Beam ' + postfix[i]] - ml_sticker=ves['Lic Sticker Number ' + postfix[i]] - c_sticker=ves['C Sticker No ' + postfix[i]] - au_sticker=ves['Au Sticker No ' + postfix[i]] - au_sticker_date=ves['Date Au Sticker Sent ' + postfix[i]] - - an_sticker=ves['An Sticker No ' + postfix[i]] - an_sticker_date=ves['Date An Sticker Sent ' + postfix[i]] - - #if rego_no=='DO904': - # import ipdb; ipdb.set_trace() - - self.vessels_dict.update({rego_no: - { - 'length': tot_length, - 'draft': draft, - 'beam': beam, - 'weight': tonnage, - 'name': ves_name, - 'type': ves_type, - 'percentage': pct_interest, - 'ml_sticker': ml_sticker, - 'au_sticker': au_sticker, - 'c_sticker': c_sticker, - 'au_sticker_date': au_sticker_date, - 'an_sticker': an_sticker, - 'an_sticker_date': an_sticker_date, - } - }) - - except Exception as e: - print(f'ERROR: vessels_dict: {str(e)}') - if e.args[0]=='Name Ad Ves 2': - continue - else: - import ipdb; ipdb.set_trace() - pass - - - - print(f'vessels_dict dict size: {len(self.vessels_dict)}') - - def create_mooring_licences(self): - expiry_date = EXPIRY_DATE - start_date = START_DATE - date_applied = DATE_APPLIED - mooring_not_found = [] - vessel_not_found = [] - owner_not_found = [] - vessel_ownership_not_found = [] - errors = [] - - # Iterate through the ml dataframe and create MooringLicenceApplication's - df = self.df_ml.groupby('mooring_no').first() - for index, row in tqdm(df.iterrows(), total=df.shape[0]): - try: - - #if row.pers_no == '132322': - # import ipdb; ipdb.set_trace() - #else: - # continue - - #if row.name == 'CB033': - # import ipdb; ipdb.set_trace() - - if not row.name or len([_str for _str in ['KINGSTON REEF','NN64','PB 02','RIA'] if row.name in _str])>0: - continue - - user_row = self.df_user[self.df_user['pers_no']==row.pers_no] #.squeeze() # as Pandas Series - - if user_row.empty: - continue - - if len(user_row)>1: - #user_row = user_row[(user_row['paid_up']=='Y') | (user_row['current_mooring_no']!='')] - user_row = user_row[(user_row['paid_up']=='Y')] - user_row = user_row.squeeze() # convert to Pandas Series - - #if not user_row.empty and user_row.pers_no=='000036': - # import ipdb; ipdb.set_trace() - - email = user_row.email.lower().replace(' ','') - if not email: - self.no_email.append(user_row.pers_no) - continue - - users = EmailUser.objects.filter(email=email) - if users.count() == 0: - #import ipdb; ipdb.set_trace() - print(f'ml - email not found: {email}') - - self.ml_user_not_found.append(email) - continue - else: - user = users[0] - self.user_existing.append(email) - - #import ipdb; ipdb.set_trace() - ves_row = self.df_ves[self.df_ves['Person No']==row.pers_no] - if len(ves_row) > 1: - self.ml_no_ves_rows.append((user_row.pers_no, len(ves_row))) - ves_row = ves_row.iloc[0] - else: - ves_row = ves_row.squeeze() -# -# ves_list = ves_fields(ves_row) - - - - #import ipdb; ipdb.set_trace() - berth_mooring = '' - #vessel_data.append([pers_no, email, user.dob, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring]) - - #import ipdb; ipdb.set_trace() - postfix = 'Nominated Ves' - ves_name = ves_row['Name ' + postfix] - ves_type = ves_row['Type ' + postfix] - rego_no = ves_row['DoT Rego ' + postfix] - pct_interest = ves_row['%Interest ' + postfix] - tot_length = ves_row['Reg Length ' + postfix] - draft = ves_row['Draft ' + postfix] - tonnage = ves_row['Tonnage ' + postfix] - sticker_number = ves_row['Lic Sticker Number ' + postfix] #if ves_row['Lic Sticker Number ' + postfix] else None - sticker_sent = ves_row['Licencee Sticker Sent ' + postfix] #if ves_row['Licencee Sticker Sent ' + postfix] else None - - ves_type = VESSEL_TYPE_MAPPING.get(ves_type, 'other') -# sticker_numbers = ves_fields('Lic Sticker Number', ves_row, pers_no) -# sticker_sent = ves_fields('Licencee Sticker Sent', ves_row, pers_no) - - #if ves_name=='': - # continue - - #if not user_row.empty and user_row.pers_no=='000036': - # import ipdb; ipdb.set_trace() - - try: - owner = Owner.objects.get(emailuser=user) - except Exception as e: - owner_not_found.append((row.pers_no, user, rego_no)) - continue - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except Exception as e: - vessel_not_found.append(rego_no) - continue - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - except Exception as e: - vessel_ownership_not_found.append(rego_no) - continue - try: - vessel_details = VesselDetails.objects.get(vessel=vessel) - except MultipleObjectsReturned: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - vessel_details.vessel_type = ves_type - vessel_details.save() - - if Mooring.objects.filter(name=row.name).count()>0: - mooring = Mooring.objects.filter(name=row.name)[0] - else: - #import ipdb; ipdb.set_trace() - #print(f'Mooring not found: {row.name}') - #errors.append(dict(idx=index, record=ves_row)) - mooring_not_found.append(row.name) - continue - - -# mla=MooringLicenceApplication.objects.filter( -# vessel_details=vessel_details, -# vessel_ownership=vessel_ownership, -# rego_no=rego_no, -# ).exclude(current_proposal=Proposal.PROCESSING_STATUS_DECLINED) -# if mla.count()>0: -# # already exists -# continue - - proposal=MooringLicenceApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.datetime.now().astimezone(), - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=ves_type, - vessel_name=ves_name, - vessel_length=vessel_details.vessel_length, - vessel_draft=vessel_details.vessel_draft, - vessel_beam=vessel_details.vessel_beam, - vessel_weight=vessel_details.vessel_weight, - berth_mooring=vessel_details.berth_mooring, - preferred_bay= mooring.mooring_bay, - percentage=vessel_ownership.percentage, - individual_owner=True, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - allocated_mooring=mooring, - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - "vessel_ownership": [{ - "dot_name": rego_no - }], - "mooring_on_approval": [] - }, - date_invited=None, #date_invited, - dot_name=rego_no, - ) - - #if not user_row.empty and user_row.pers_no=='000036': - # import ipdb; ipdb.set_trace() - - date_invited = '' - - pua=ProposalUserAction.objects.create( - proposal=proposal, - who=user, - what='Mooring Licence - Migrated Application', - ) - - - approval = MooringLicence.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - - aua=ApprovalUserAction.objects.create( - - approval=approval, - who=user, - what='Mooring Licence - Migrated Application', - ) - - proposal.approval = approval - proposal.save() - alloc_mooring = proposal.allocated_mooring - alloc_mooring.mooring_licence = approval - alloc_mooring.save() - - vooa = VesselOwnershipOnApproval.objects.create( - approval=approval, - vessel_ownership=vessel_ownership, - ) - - if user_row.licences_type!='L': - moa = MooringOnApproval.objects.create( - approval=approval, - mooring=mooring, - sticker=None, - site_licensee=True, # ??? - end_date=expiry_date - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - -# if row.pers_no == '000036': -# import ipdb; ipdb.set_trace() - - if sticker_number: - sticker = Sticker.objects.create( - number=sticker_number, - status=Sticker.STICKER_STATUS_CURRENT, # 'current' - approval=approval, - proposal_initiated=proposal, - vessel_ownership=vessel_ownership, - printing_date=None, #TODAY, - #mailing_date=sticker_sent, #TODAY, - mailing_date=datetime.datetime.strptime(sticker_sent, '%d/%m/%Y').date() if sticker_sent else None, - sticker_printing_batch=None, - sticker_printing_response=None, - ) - -# if row.pers_no == '210758': -# if sticker_number: -# sticker2 = Sticker.objects.create( -# number='101010', -# status=Sticker.STICKER_STATUS_CURRENT, # 'current' -# approval=approval, -# proposal_initiated=proposal, -# vessel_ownership=vessel_ownership, -# printing_date=None, #TODAY, -# #mailing_date=sticker_sent, #TODAY, -# mailing_date=datetime.datetime.strptime(sticker_sent, '%d/%m/%Y').date() if sticker_sent else None, -# sticker_printing_batch=None, -# sticker_printing_response=None, -# ) - - approval_history.stickers.add(sticker.id) -# approval_history.stickers.add(sticker22.id) - #approval.generate_doc() - - except Exception as e: - logger.error(f'ERROR: {row.name}. {str(e)}') - import ipdb; ipdb.set_trace() - self.user_errors.append(user_row.email) - - print(f'ml_user_not_found: {self.ml_user_not_found}') - print(f'Duplicate Pers_No in ves_row: {self.ml_no_ves_rows}') - print(f'ml_user_not_found: {len(self.ml_user_not_found)}') - print(f'Duplicate Pers_No in ves_row: {len(self.ml_no_ves_rows)}') - print(f'Moorings not Found: {mooring_not_found}') - print(f'Moorings not Found: {len(mooring_not_found)}') - print(f'Owners not Found: {owner_not_found}') - print(f'VesselOwnership not Found: {len(vessel_ownership_not_found)}') - print(f'VesselOwnership not Found: {vessel_ownership_not_found}') - print(f'Owners not Found: {len(owner_not_found)}') - print(f'Vessels not Found: {vessel_not_found}') - print(f'Vessels not Found: {len(vessel_not_found)}') - - def create_authuser_permits(self): - expiry_date = EXPIRY_DATE - start_date = START_DATE - date_applied = DATE_APPLIED - - errors = [] - vessel_not_found = [] - aup_created = [] - au_stickers = [] - no_au_stickers = [] - - bay_preferences_numbered = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"] - vessel_type = 'other' - - #for index, row in tqdm(self.df_authuser.iterrows(), total=self.df_authuser.shape[0]): - df_authuser = self.df_authuser[(self.df_authuser['vessel_rego']!='0')].groupby('vessel_rego').first() - for index, row in tqdm(df_authuser.iterrows(), total=df_authuser.shape[0]): - #if row.status != 'Vacant': - try: - #if row.pers_no_l == '019626': - # print(f"rego_no: {row['vessel_rego']}") - # import ipdb; ipdb.set_trace() - rego_no = row.name - -# if row.mooring_no in ['CB033','NN126','TB255','TB271','TB280']: -# import ipdb; ipdb.set_trace() - #if row.mooring_no in ['CB033']: - # import ipdb; ipdb.set_trace() - - #if row.pers_no_u == '210758': - # import ipdb; ipdb.set_trace() - #else: - # continue - - if row.mooring_no == 'TB999': - # exclude mooring - continue - - #if rego_no == 'GM894': - # import ipdb; ipdb.set_trace() - - - mooring_authorisation_preference = 'site_licensee' if row['licencee_approved']=='Y' else 'ria' - mooring = Mooring.objects.filter(name=row['mooring_no'])[0] - #licensee = EmailUser.objects.get(first_name=row['first_name_l'].lower().capitalize(), last_name=row['last_name_l'].lower().capitalize()) - #email_l = self.df_user[(self.df_user['pers_no']==row['pers_no_l'])].iloc[0]['email'].strip() - email_l = self.df_user[(self.df_user['pers_no']==row['pers_no_l']) & (self.df_user['email']!='')].iloc[0]['email'].strip() - try: - licensee = EmailUser.objects.get(email=email_l.lower()) - except Exception as e: - licensee = EmailUser.objects.get(first_name=row['first_name_l'].lower().capitalize(), last_name=row['last_name_l'].lower().capitalize()) - #user = EmailUser.objects.filter(first_name=row['first_name_u'].lower().capitalize(), last_name=row['last_name_u'].lower().capitalize())[0] - if not row['first_name_u']: - # This record represents Mooring Licence Holder - No need for an Auth User Permit - continue - - email_u = self.df_user[(self.df_user['pers_no']==row.pers_no_u) & (self.df_user['email']!='')].iloc[0]['email'].strip() - try: - user = EmailUser.objects.get(email=email_u.lower()) - except Exception as e: - user = EmailUser.objects.get(first_name=row['first_name_u'].lower().capitalize(), last_name=row['last_name_u'].lower().capitalize()) - #email_u = self.df_user[(self.df_user['first_name']=='Alisa') & (self.df_user['last_name']=='Simich')].iloc[0]['email'] - #email_u = self.df_user[(self.df_user['first_name']==row['first_name_u']) & (self.df_user['last_name']==row['last_name_u'])].iloc[0]['email'] - #user = EmailUser.objects.get(email=email_u) - - #sticker_number = row['sticker'], - #rego_no = row['vessel_rego'] - rego_no = row.name - #if sticker_number == 31779: - # import ipdb; ipdb.set_trace() - sticker_info = self.vessels_au.get(rego_no) - if sticker_info: - sticker_number = sticker_info['au_sticker'] - sticker_sent = sticker_info['au_sticker_sent'] - - #import ipdb; ipdb.set_trace() - sticker_info2 = self.vessels_an.get(rego_no) - if sticker_info2: - sticker_number2 = sticker_info2['an_sticker'] - sticker_sent2 = sticker_info2['an_sticker_sent'] - - if sticker_number is None: - no_au_stickers.append(rego_no) - else: - au_stickers.append(rego_no) - -# if row.pers_no_u == '210758': -# import ipdb; ipdb.set_trace() - - date_issued = row['date_issued'].split(' ')[0] - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except Exception as e: - vessel_not_found.append(f'{row.pers_no_u} - {email_u}: {rego_no}') - continue - - vessel_ownership = vessel.vesselownership_set.all()[0] - vessel_details = vessel.vesseldetails_set.all()[0] - -# auth_user = AuthorisedUserApplication.objects.filter( -# site_licensee_email=licensee.email, -# mooring=mooring, -# vessel_details=vessel_details, -# vessel_ownership=vessel_ownership, -# rego_no=rego_no, -# ) -# if len(auth_user)>0: -# # already exists -# continue - - proposal=AuthorisedUserApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=TODAY, #datetime.datetime.now(), - mooring_authorisation_preference=mooring_authorisation_preference, - site_licensee_email=licensee.email, - keep_existing_mooring=True, - mooring=mooring, - bay_preferences_numbered=bay_preferences_numbered, - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=row['vessel_name'], - #vessel_length=row['vessel_length'], - #vessel_draft=row['vessel_draft'], - vessel_length=vessel_details.vessel_length, - vessel_draft=vessel_details.vessel_draft, - vessel_beam=vessel_details.vessel_beam, - vessel_weight=vessel_details.vessel_weight, - berth_mooring=vessel_details.berth_mooring, - preferred_bay= mooring.mooring_bay, - percentage=vessel_ownership.percentage, - individual_owner=True, - dot_name=vessel_ownership.dot_name, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - proposed_issuance_approval={ - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - "vessel_ownership": [], - "mooring_on_approval": [] - }, - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user, - what='Authorised User Permit - Migrated Application', - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() - - #approval = WaitingListAllocation.objects.create( - approval = AuthorisedUserPermit.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - aup_created.append(approval.id) - - aua=ApprovalUserAction.objects.create( - approval=approval, - who=user, - what='Authorised User Permit - Migrated Application', - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - -# approval_history = ApprovalHistory.objects.create( -# reason='new', -# approval=approval, -# vessel_ownership = vessel_ownership, -# proposal = proposal, -# #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) -# start_date = start_date, -# ) - - -# if sticker_number: -# sticker = Sticker.objects.create( -# number=sticker_number, -# status=Sticker.STICKER_STATUS_CURRENT, # 'current' -# approval=approval, -# proposal_initiated=proposal, -# vessel_ownership=vessel_ownership, -# printing_date=None, #TODAY, -# #mailing_date=sticker_sent, #TODAY, -# #mailing_date=datetime.datetime.strptime(date_issued, '%d/%m/%Y').date() if date_issued else None, -# mailing_date=datetime.datetime.strptime(sticker_sent, '%d/%m/%Y').date() if sticker_sent else None, -# sticker_printing_batch=None, -# sticker_printing_response=None, -# ) -# -# if sticker_number2: -# sticker = Sticker.objects.create( -# number=sticker_number2, -# status=Sticker.STICKER_STATUS_CURRENT, # 'current' -# approval=approval, -# proposal_initiated=proposal, -# vessel_ownership=vessel_ownership, -# printing_date=None, #TODAY, -# #mailing_date=sticker_sent, #TODAY, -# #mailing_date=datetime.datetime.strptime(date_issued, '%d/%m/%Y').date() if date_issued else None, -# mailing_date=datetime.datetime.strptime(sticker_sent2, '%d/%m/%Y').date() if sticker_sent2 else None, -# sticker_printing_batch=None, -# sticker_printing_response=None, -# ) - - - #if row.pers_no_u == '210758': - # import ipdb; ipdb.set_trace() - -# approval_history.stickers.add(sticker.id) - - #auth_user_moorings = self.df_authuser[(self.df_authuser['pers_no_u']==pers_no_u)] - #auth_user_moorings = self.df_authuser[(self.df_authuser['pers_no_u']==pers_no_u) & (self.df_authuser['vessel_rego']==rego_no)].drop_duplicates(subset=['mooring_no']) - #auth_user_moorings = self.df_authuser[(self.df_authuser['vessel_rego']==rego_no)] - auth_user_moorings = self.df_authuser[(self.df_authuser['vessel_rego']==rego_no)].drop_duplicates(subset=['mooring_no','vessel_rego']) - #for idx, auth_user in tqdm(self.df_authuser.iterrows(), total=self.df_authuser.shape[0]): - sticker = None - for idx, auth_user in auth_user_moorings.iterrows(): - mooring = Mooring.objects.filter(name=auth_user.mooring_no) - -# if row.pers_no_u == '210758': -# import ipdb; ipdb.set_trace() - if mooring[0].name in self.vessels_au.get(rego_no)['moorings']: - sticker_number = self.vessels_au.get(rego_no)['au_sticker'] - sticker_sent = self.vessels_au.get(rego_no)['au_sticker_sent'] - - elif mooring[0].name in self.vessels_an.get(rego_no)['moorings']: - sticker_number = self.vessels_an.get(rego_no)['an_sticker'] - sticker_sent = self.vessels_an.get(rego_no)['an_sticker_sent'] - - else: - #import ipdb; ipdb.set_trace() - sticker_number = '' - logger.error("Cannot get sticker number") - - if sticker_number: - sticker, created = Sticker.objects.get_or_create( - number=sticker_number, - approval=approval, - proposal_initiated=proposal, - vessel_ownership=vessel_ownership, - - defaults = dict( - status=Sticker.STICKER_STATUS_CURRENT, # 'current' - printing_date=None, #TODAY, - mailing_date=datetime.datetime.strptime(sticker_sent, '%d/%m/%Y').date() if sticker_sent else None, - sticker_printing_batch=None, - sticker_printing_response=None, - ) - ) - -# sticker = Sticker.objects.create( -# number=sticker_number, -# status=Sticker.STICKER_STATUS_CURRENT, # 'current' -# approval=approval, -# proposal_initiated=proposal, -# vessel_ownership=vessel_ownership, -# printing_date=None, #TODAY, -# #mailing_date=sticker_sent, #TODAY, -# #mailing_date=datetime.datetime.strptime(date_issued, '%d/%m/%Y').date() if date_issued else None, -# mailing_date=datetime.datetime.strptime(sticker_sent, '%d/%m/%Y').date() if sticker_sent else None, -# sticker_printing_batch=None, -# sticker_printing_response=None, -# ) - - moa = MooringOnApproval.objects.create( - approval=approval, - mooring=mooring[0], - sticker=sticker, - site_licensee=False, #site_licensee, - #end_date=expiry_date - ) - -# if row.pers_no_u == '210758': -# moa = MooringOnApproval.objects.create( -# approval=approval, -# mooring=mooring[0], -# sticker=sticker2, -# site_licensee=False, #site_licensee, -# #end_date=expiry_date -# ) - -# if row.pers_no_u == '210758': -# import ipdb; ipdb.set_trace() - - #approval.generate_doc() - - except Exception as e: - errors.append(str(e)) - import ipdb; ipdb.set_trace() - pass - #raise Exception(str(e)) - - print(f'vessel_not_found: {vessel_not_found}') - print(f'vessel_not_found: {len(vessel_not_found)}') - print(f'aup_created: {len(aup_created)}') - print(f'au_stickers: au_stickers, {len(au_stickers)}') - print(f'no_au_stickers: no_au_stickers, {len(no_au_stickers)}') - - def create_waiting_list(self): - expiry_date = EXPIRY_DATE - start_date = START_DATE - date_applied = DATE_APPLIED - - errors = [] - vessel_not_found = [] - user_not_found = [] - wl_created = [] - - vessel_type = 'other' - - for index, row in tqdm(self.df_wl.iterrows(), total=self.df_wl.shape[0]): - try: - #import ipdb; ipdb.set_trace() - #if row.mooring_no == 'TB999': - # continue - - pers_no = row['pers_no'] - mooring_bay = MooringBay.objects.get(code=row['bay']) - - email = self.df_user[(self.df_user['pers_no']==pers_no) & (self.df_user['email']!='')].iloc[0]['email'].strip() - first_name = row.first_name.lower().title().strip() - last_name = row.last_name.lower().title().strip() - - if email == 'tonya@thebarrfamily.net': - import ipdb; ipdb.set_trace() - - try: - user = EmailUser.objects.get(email=email.lower()) - user.first_name = first_name - user.last_name = last_name - user.save() - except Exception as e: - user = EmailUser.objects.get(first_name=first_name, last_name=last_name) - - rego_no = row['vessel_rego'] - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except Exception as e: - vessel_not_found.append(f'{pers_no} - {email}: {rego_no}') - continue - - vessel_ownership = vessel.vesselownership_set.all()[0] - vessel_details = vessel.vesseldetails_set.all()[0] - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() - - proposal=WaitingListApplication.objects.create( - proposal_type_id=1, - submitter=user, - lodgement_date=start_date, - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - - vessel_name=row['vessel_name'], - vessel_length=vessel_details.vessel_length, - vessel_draft=vessel_details.vessel_draft, - vessel_beam=vessel_details.vessel_beam, - vessel_weight=vessel_details.vessel_weight, - berth_mooring=vessel_details.berth_mooring, - - preferred_bay=mooring_bay, - percentage=vessel_ownership.percentage, - individual_owner=True, - proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user, - what='Waiting List - Migrated Application', - ) - - approval = WaitingListAllocation.objects.create( - status='current', - internal_status='waiting', - current_proposal=proposal, - issue_date = start_date, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - wla_order=row['bay_pos_no'], - wla_queue_date=TODAY + datetime.timedelta(seconds=int(row['bay_pos_no'])), - ) - #print(f'wla_order: {position_no}') - wl_created.append(approval.id) - - aua=ApprovalUserAction.objects.create( - approval=approval, - who=user, - what='Waiting List Allocation - Migrated Application', - ) - - proposal.approval = approval - proposal.save() - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - -# approval_history = ApprovalHistory.objects.create( -# reason='new', -# approval=approval, -# vessel_ownership = vessel_ownership, -# proposal = proposal, -# start_date = start_date, -# ) - - #approval.generate_doc() - - except Exception as e: - errors.append(str(e)) - import ipdb; ipdb.set_trace() - pass - #raise Exception(str(e)) - - print(f'vessel_not_found: {vessel_not_found}') - print(f'vessel_not_found: {len(vessel_not_found)}') - print(f'user_not_found: {user_not_found}') - print(f'user_not_found: {len(user_not_found)}') - print(f'wl_created: {len(wl_created)}') - - def create_dcv(self): - expiry_date = EXPIRY_DATE - start_date = START_DATE - date_applied = DATE_APPLIED - fee_season = FeeSeason.objects.filter(name=FEE_SEASON)[0] - - errors = [] - vessel_not_found = [] - user_not_found = [] - dcv_created = [] - - #vessel_type = 'other' - - for index, row in tqdm(self.df_dcv.iterrows(), total=self.df_dcv.shape[0]): - try: - #import ipdb; ipdb.set_trace() - #if row.pers_no == '200700': - # import ipdb; ipdb.set_trace() - - pers_no = row['pers_no'] - email = row['email'] - org_name = row['company'] - - #email = self.df_user[(self.df_user['pers_no']==pers_no) & (self.df_user['email']!='')].iloc[0]['email'].strip() - first_name = row.first_name.lower().title().strip() - last_name = row.last_name.lower().title().strip() - try: - user = EmailUser.objects.get(email=email.lower()) - user.first_name = first_name - user.last_name = last_name - user.save() - except Exception as e: - user = EmailUser.objects.get(first_name=first_name, last_name=last_name) - - try: - dcv_organisation = DcvOrganisation.objects.get(name=org_name) - except ObjectDoesNotExist: - dcv_organisation = DcvOrganisation.objects.create(name=org_name) - - - try: - vessels_dcv = self.vessels_dcv[pers_no] - except Exception as e: - vessel_not_found.append(pers_no) - continue - - for rego_no, sticker_no in vessels_dcv: - try: - dcv_vessel = DcvVessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel_details = VesselDetails.objects.get(vessel__rego_no=rego_no) - dcv_vessel = DcvVessel.objects.create( - rego_no = rego_no, - vessel_name = vessel_details.vessel_name, - dcv_organisation = dcv_organisation, - ) - - dcv_permit = DcvPermit.objects.create( - submitter = user, - lodgement_datetime = datetime.datetime.now(datetime.timezone.utc), - fee_season = fee_season, - start_date = start_date, - end_date = expiry_date, - dcv_vessel = dcv_vessel, - dcv_organisation =dcv_organisation, - migrated = True - ) - -# approval_history = ApprovalHistory.objects.create( -# reason='new', -# approval=approval, -# vessel_ownership = vessel_ownership, -# proposal = proposal, -# start_date = start_date, -# ) - - #import ipdb; ipdb.set_trace() - if sticker_no: - sticker = Sticker.objects.create( - number=sticker_no, - status=Sticker.STICKER_STATUS_CURRENT, # 'current' - dcv_permit=dcv_permit, - #vessel_ownership=vessel_ownership, - #printing_date=TODAY, - #mailing_date=datetime.datetime.strptime(date_issued, '%d/%m/%Y').date() if date_issued else None, - mailing_date=TODAY, - #sticker_printing_batch=None, - #sticker_printing_response=None, - ) - - #import ipdb; ipdb.set_trace() - #dcv_permit.generate_dcv_permit_doc() - dcv_created.append(dcv_permit.id) - - - - except Exception as e: - errors.append(str(e)) - import ipdb; ipdb.set_trace() - pass - #raise Exception(str(e)) - - print(f'vessel_not_found: {vessel_not_found}') - print(f'vessel_not_found: {len(vessel_not_found)}') - print(f'user_not_found: {user_not_found}') - print(f'user_not_found: {len(user_not_found)}') - print(f'dcv_created: {len(dcv_created)}') - - - def _create_single_vessel(self, user, rego_no, ves_name, ves_type, length, draft, beam, weight, pct_interest, berth_mooring=''): - - def try_except(value): - try: - val = float(value) - except Exception: - val = 0.00 - return val - - # check if vessel object exists - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - except ObjectDoesNotExist: - pct_interest = int(round(float(try_except(pct_interest)),0)) - if pct_interest < 25: - pct_interest = 100 - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=pct_interest) - - try: - vessel_details = VesselDetails.objects.get(vessel=vessel) - except MultipleObjectsReturned: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - except: - vessel_details = VesselDetails.objects.create( - vessel=vessel, - vessel_type=ves_type, - vessel_name=ves_name, - vessel_length=try_except(length), - vessel_draft=try_except(draft), - vessel_weight=try_except(weight), - vessel_beam=try_except(beam), - berth_mooring='' - ) - - return vessel, vessel_details, vessel_ownership - - - def create_annual_admissions(self): - """ - mlr=MooringLicenceReader('PersonDets20221222-125823.txt', 'MooringDets20221222-125546.txt', 'VesselDets20221222-125823.txt', 'UserDets20221222-130353.txt', 'ApplicationDets20221222-125157.txt','annual_admissions_booking_report_20230125084027.csv') - mlr.create_users() - mlr.create_vessels() - mlr.create_annual_admissions() - """ - expiry_date = EXPIRY_DATE - start_date = START_DATE - date_applied = DATE_APPLIED - fee_season = FeeSeason.objects.filter(name=FEE_SEASON)[0] - - errors = [] - vessel_not_found = [] - vessel_details_not_found = [] - user_not_found = [] - rego_aa_created = [] - total_aa_created = [] - - vessel_type = 'other' - - for index, row in tqdm(self.df_aa.iterrows(), total=self.df_aa.shape[0]): - try: - #if row.rego_no == 'GW113': - # import ipdb; ipdb.set_trace() - - rego_no = row['rego_no'] - email = row['email'] - sticker_no = row['sticker_no'] - date_created = row['date_created'] - vessel_name = row['vessel_name'] - vessel_length = row['vessel_length'] - - #email = self.df_user[(self.df_user['pers_no']==pers_no) & (self.df_user['email']!='')].iloc[0]['email'].strip() - first_name = row.first_name.lower().title().strip() - last_name = row.last_name.lower().title().strip() - try: - user = EmailUser.objects.get(email=email.lower().strip()) - user.first_name = first_name - user.last_name = last_name - user.save() - except Exception as e: - user = EmailUser.objects.get(first_name=first_name, last_name=last_name) - -#['id', 'date_created', 'first_name', 'last_name', 'address', 'suburb', -# 'state', 'postcode', 'sticker_no', 'rego_no', 'email', 'mobile_number', -# 'work_number', 'vessel_name', 'vessel_length', 'year', 'status', -# 'booking_period', 'country'] - - vessel_dict = self.vessels_dict.get(rego_no) - if vessel_dict is None: - vessel, vessel_details, vessel_ownership = self._create_single_vessel( - user, - rego_no, - ves_name=vessel_name, # use vessel_name from AA Moorings Spreasheet - ves_type=vessel_type, - length=vessel_length, # use vessel_length from AA Moorings Spreasheet - draft=Decimal(0.0), - beam=Decimal(0.0), - weight=Decimal(0.0), - pct_interest=100, - ) - rego_aa_created.append(rego_no) - else: - try: - vessel_details = VesselDetails.objects.get(vessel__rego_no=rego_no) - #vessel_ownership = VesselOwnership.objects.get(vessel__rego_no=rego_no) - owner = Owner.objects.get(emailuser=user) - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel__rego_no=rego_no) - except Exception as e: - vessel, vessel_details, vessel_ownership = self._create_single_vessel( - user, - rego_no, - ves_name=vessel_name, # use vessel_name from AA Moorings Spreasheet - ves_type=vessel_type, - length=vessel_length, # use vessel_length from AA Moorings Spreasheet - draft=Decimal(0.0), - beam=Decimal(0.0), - weight=Decimal(0.0), - pct_interest=100, - ) - - #vessel_details_not_found.append((user.email, rego_no)) - #continue - - #if user.email == 'kdeluca@iinet.net.au': - # import ipdb; ipdb.set_trace() - - # update length from AA Spreadsheet file - vessel_details.vessel_name=vessel_name - vessel_details.vessel_length=Decimal(vessel_length) if vessel_length else Decimal(0) - vessel_details.save() - - total_aa_created.append(rego_no) - - proposal=AnnualAdmissionApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=TODAY, - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_details.vessel_type, - vessel_name=vessel_details.vessel_name, - vessel_length=vessel_details.vessel_length, - vessel_draft=vessel_details.vessel_draft, - vessel_weight=vessel_details.vessel_weight, - percentage=vessel_ownership.percentage, - berth_mooring='', - individual_owner=True, - processing_status='approved', - customer_status='approved', - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - }, - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user, - what='Annual Admission - Migrated Application', - ) - - #approval = WaitingListAllocation.objects.create( - approval = AnnualAdmissionPermit.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = TODAY, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - aua=ApprovalUserAction.objects.create( - approval=approval, - who=user, - what='Annual Admission - Migrated Application', - ) - - proposal.approval = approval - proposal.save() - #proposal.allocated_mooring.mooring_licence = approval - #proposal.allocated_mooring.save() - -# approval_history = ApprovalHistory.objects.create( -# reason='new', -# approval=approval, -# vessel_ownership = vessel_ownership, -# proposal = proposal, -# #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) -# start_date = start_date, -# ) - - if sticker_no: - sticker = Sticker.objects.create( - number=sticker_no, - status=Sticker.STICKER_STATUS_CURRENT, # 'current' - approval=approval, - proposal_initiated=proposal, - vessel_ownership=vessel_ownership, - printing_date=datetime.datetime.strptime(date_created.split(' ')[0], '%d/%m/%Y').date() if date_created else None, - mailing_date=datetime.datetime.strptime(date_created.split(' ')[0], '%d/%m/%Y').date() if date_created else None, - sticker_printing_batch=None, - sticker_printing_response=None, - ) -# approval_history.stickers.add(sticker.id) - - #approval.generate_doc() - - except Exception as e: - errors.append(str(e)) - import ipdb; ipdb.set_trace() - pass - #raise Exception(str(e)) - - print(f'rego_aa_created: {len(rego_aa_created)}') - print(f'total_aa_created: {len(total_aa_created)}') - print(f'Vessel Details Not Found: {vessel_details_not_found}') - print(f'Vessel Details Not Found: {len(vessel_details_not_found)}') - - - def _migrate_approval(self, data, submitter, applicant=None, proxy_applicant=None): - from disturbance.components.approvals.models import Approval - #import ipdb; ipdb.set_trace() - try: - site_number = int(data.permit_number) if data.permit_number else int(data.licensed_site) - except Exception as e: - import ipdb; ipdb.set_trace() - print('ERROR: There is no site_number - Must provide a site number in migration spreadsheeet. {e}') - - try: - expiry_date = data['expiry_date'] if data['expiry_date'] else datetime.date.today() - start_date = data['start_date'] if data['start_date'] else datetime.date.today() - issue_date = data['issue_date'] if data['issue_date'] else start_date - site_status = 'not_to_be_reissued' if data['status'].lower().strip() == 'not to be reissued' else data['status'].lower().strip() - - except Exception as e: - import ipdb; ipdb.set_trace() - print(e) - - try: - - #import ipdb; ipdb.set_trace() - if applicant: - proposal, p_created = Proposal.objects.get_or_create( - application_type=self.application_type, - activity='Apiary', - submitter=submitter, - applicant=applicant, - schema=self.proposal_type.schema, - ) - approval, approval_created = Approval.objects.update_or_create( - applicant=applicant, - status=Approval.STATUS_CURRENT, - apiary_approval=True, - defaults = { - 'issue_date':issue_date, - 'expiry_date':expiry_date, - 'start_date':start_date, - #'submitter':submitter, - 'current_proposal':proposal, - } - ) - else: - import ipdb; ipdb.set_trace() - - #if 'PM' not in proposal.lodgement_number: - # proposal.lodgement_number = proposal.lodgement_number.replace('P', 'PM') # Application Migrated - proposal.approval= approval - proposal.processing_status='approved' - proposal.customer_status='approved' - proposal.migrated=True - proposal.proposed_issuance_approval = { - 'start_date': start_date.strftime('%d-%m-%Y'), - 'expiry_date': expiry_date.strftime('%d-%m-%Y'), - 'details': 'Migrated', - 'cc_email': 'Migrated', - } - - approval.migrated=True - - # create invoice for payment of zero dollars - order = create_invoice(proposal) - invoice = Invoice.objects.get(order_number=order.number) - proposal.fee_invoice_references = [invoice.reference] - - proposal.save() - approval.save() - - # create apiary sites and intermediate table entries - #geometry = GEOSGeometry('POINT(' + str(data['latitude']) + ' ' + str(data['longitude']) + ')', srid=4326) - geometry = GEOSGeometry('POINT(' + str(data['latitude']) + ' ' + str(data['longitude']) + ')', srid=4326) - #import ipdb; ipdb.set_trace() - apiary_site = ApiarySite.objects.create( - id=site_number, - is_vacant=True if site_status=='vacant' else False - ) - site_category = get_category(geometry) - intermediary_approval_site = ApiarySiteOnApproval.objects.create( - #id=site_number, - apiary_site=apiary_site, - approval=approval, - wkb_geometry=geometry, - site_category = site_category, - licensed_site=True if data['licensed_site'] else False, - batch_no=data['batch_no'], - approval_cpc_date=data['approval_cpc_date'] if data.approval_cpc_date else None, - approval_minister_date=data['approval_minister_date'] if data.approval_minister_date else None, - map_ref=data['map_ref'], - forest_block=data['forest_block'], - cog=data['cog'], - roadtrack=data['roadtrack'], - zone=data['zone'], - catchment=data['catchment'], - #dra_permit=data['dra_permit'], - site_status=site_status, - ) - #import ipdb; ipdb.set_trace() - pa, pa_created = ProposalApiary.objects.get_or_create(proposal=proposal) - - intermediary_proposal_site = ApiarySiteOnProposal.objects.create( - #id=site_number, - apiary_site=apiary_site, - #approval=approval, - proposal_apiary=pa, - wkb_geometry_draft=geometry, - site_category_draft = site_category, - wkb_geometry_processed=geometry, - site_category_processed = site_category, - licensed_site=True if data['licensed_site'] else False, - batch_no=data['batch_no'], - #approval_cpc_date=data['approval_cpc_date'], - #approval_minister_date=data['approval_minister_date'], - approval_cpc_date=data['approval_cpc_date'] if data.approval_cpc_date else None, - approval_minister_date=data['approval_minister_date'] if data.approval_minister_date else None, - map_ref=data['map_ref'], - forest_block=data['forest_block'], - cog=data['cog'], - roadtrack=data['roadtrack'], - zone=data['zone'], - catchment=data['catchment'], - site_status=site_status, - #dra_permit=data['dra_permit'], - ) - #import ipdb; ipdb.set_trace() - - apiary_site.latest_approval_link=intermediary_approval_site - apiary_site.latest_proposal_link=intermediary_proposal_site - if site_status == 'vacant': - apiary_site.approval_link_for_vacant=intermediary_approval_site - apiary_site.proposal_link_for_vacant=intermediary_proposal_site - apiary_site.save() - - - except Exception as e: - logger.error('{}'.format(e)) - import ipdb; ipdb.set_trace() - return None - - return approval - -# def create_licence_pdf(self): -# approvals_migrated = Approval.objects.filter(migrated=True) -# print('Total Approvals: {} - {}'.format(approvals_migrated.count(), approvals_migrated)) -# for idx, a in enumerate(approvals_migrated): -# a.generate_doc() -# print('{}, Created PDF for Approval {}'.format(idx, a)) -# -# def create_dcv_licence_pdf(self): -# approvals_migrated = DcvPermit.objects.filter(migrated=True) -# print('Total DCV Approvals: {} - {}'.format(approvals_migrated.count(), approvals_migrated)) -# for idx, a in enumerate(approvals_migrated): -# a.generate_dcv_permit_doc() -# print('{}, Created PDF for DCV Approval {}'.format(idx, a)) - - - def create_licence_pdfs(self): - MooringLicenceReader.create_pdf_ml() - MooringLicenceReader.create_pdf_aup() - MooringLicenceReader.create_pdf_wl() - MooringLicenceReader.create_pdf_aa() - MooringLicenceReader.create_pdf_dcv() - - @classmethod - def create_pdf_ml(self): - """ MooringLicenceReader.create_pdf_ml() - """ - self._create_pdf_licence(MooringLicence.objects.filter(migrated=True)) - - @classmethod - def create_pdf_aup(self): - """ MooringLicenceReader.create_pdf_aup() - """ - self._create_pdf_licence(AuthorisedUserPermit.objects.filter(migrated=True)) - - @classmethod - def create_pdf_wl(self): - """ MooringLicenceReader.create_pdf_wl() - """ - self._create_pdf_licence(WaitingListAllocation.objects.filter(migrated=True)) - - @classmethod - def create_pdf_aa(self): - """ MooringLicenceReader.create_pdf_aa() - """ - self._create_pdf_licence(AnnualAdmissionPermit.objects.filter(migrated=True)) - - @classmethod - def create_pdf_dcv(self): - """ MooringLicenceReader.create_pdf_dcv() - """ - self._create_pdf_licence(DcvPermit.objects.filter(migrated=True)) - - @staticmethod - def _create_pdf_licence(approvals_migrated): - """ MooringLicenceReader._create_pdf_licence(MooringLicence.objects.filter(migrated=True), current_proposal__processing_status=Proposal.PROCESSING_STATUS_APPROVED) - """ - #approvals_migrated = MooringLicence.objects.filter(migrated=True) - permit_name = approvals_migrated[0].__class__.__name__ - print(f'Total {permit_name}: {approvals_migrated.count()} - {approvals_migrated}') - if isinstance(approvals_migrated[0], DcvPermit): - approvals = approvals_migrated.filter(migrated=True) - else: - approvals = approvals_migrated.filter(migrated=True, current_proposal__processing_status=Proposal.PROCESSING_STATUS_APPROVED) - - for idx, a in enumerate(approvals): - if isinstance(a, DcvPermit) and len(a.dcv_permit_documents.all())==0: - a.generate_dcv_permit_doc() - elif not hasattr(a, 'licence_document') or a.licence_document is None: - a.generate_doc() - print(f'{idx}, Created PDF for {permit_name}: {a}') - - -def create_invoice(proposal, payment_method='other'): - """ - This will create and invoice and order from a basket bypassing the session - and payment bpoint code constraints. - """ - from ledger.checkout.utils import createCustomBasket - from ledger.payments.invoice.utils import CreateInvoiceBasket - from ledger.accounts.models import EmailUser - - now = timezone.now().date() - line_items = [ - {'ledger_description': 'Migration Licence Charge Waiver - {} - {}'.format(now, proposal.lodgement_number), - 'oracle_code': 'N/A', #proposal.application_type.oracle_code_application, - 'price_incl_tax': Decimal(0.0), - 'price_excl_tax': Decimal(0.0), - 'quantity': 1, - } - ] - - user = EmailUser.objects.get(email__icontains='das@dbca.wa.gov.au') - invoice_text = 'Migrated Permit/Licence Invoice' - - basket = createCustomBasket(line_items, user, settings.PAYMENT_SYSTEM_ID) - order = CreateInvoiceBasket(payment_method=payment_method, system=settings.PAYMENT_SYSTEM_PREFIX).create_invoice_and_order(basket, 0, None, None, user=user, invoice_text=invoice_text) - - return order - - - diff --git a/mooringlicensing/utils/archive/mooring_licence_migrate_write_csv.py b/mooringlicensing/utils/archive/mooring_licence_migrate_write_csv.py deleted file mode 100644 index 1905c6285..000000000 --- a/mooringlicensing/utils/archive/mooring_licence_migrate_write_csv.py +++ /dev/null @@ -1,348 +0,0 @@ -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval -from mooringlicensing.components.organisations.models import Organisation -# from ledger.accounts.models import Organisation as ledger_org - -import sys -import csv -import pandas as pd - -MODELS = [ - 'MooringLicenceApplication', - 'Mooring', - 'MooringBay', -] - -PROPOSAL_FIELDS = [ - 'id', - #'proposal_type__code', - #'assessor_data', - #'comment_data', - #'proposed_issuance_approval', - 'customer_status', - 'lodgement_number', - #'lodgement_sequence', - 'lodgement_date', - 'processing_status', - #'prev_processing_status', - #'proposed_decline_status', - 'title', - 'approval_level', - #'approval_comment', - 'migrated', - 'rego_no', - 'vessel_id', - 'vessel_type', - 'vessel_name', - 'vessel_length', - 'vessel_draft', - 'vessel_beam', - 'vessel_weight', - 'berth_mooring', - 'dot_name', - 'percentage', - 'individual_owner', - 'company_ownership_percentage', - 'company_ownership_name', - #'insurance_choice', - #'silent_elector', - 'mooring_authorisation_preference', - 'bay_preferences_numbered', - #'site_licensee_email', - #'endorser_reminder_sent', - #'date_invited', - #'invitee_reminder_sent', - #'temporary_document_collection_id', - #'keep_existing_mooring', - #'keep_existing_vessel', - #'auto_approve', - #'null_vessel_on_create', - 'proposal', - #'uuid', -] - -ORGANISATION_FIELDS = [ - #'org_applicant__' - 'org_applicant__organisation__id', - #'org_applicant__name', - 'org_applicant__organisation__abn', - 'org_applicant__organisation__identification', - 'org_applicant__organisation__email', - 'org_applicant__organisation__trading_name', -] - -ORGANISATION_POSTAL_ADDRESS_FIELDS = [ -# #'org_applicant__postal_address__' -# #'org_applicant__billing_address__' - 'org_applicant__organisation__postal_address__id', - 'org_applicant__organisation__postal_address__line1', - 'org_applicant__organisation__postal_address__line2', - 'org_applicant__organisation__postal_address__line3', - 'org_applicant__organisation__postal_address__locality', - 'org_applicant__organisation__postal_address__state', - 'org_applicant__organisation__postal_address__country', - 'org_applicant__organisation__postal_address__postcode', - #'org_applicant__postal_address__search_text', -] - -SUBMITTER_FIELDS = [ -# #'submitter__' - 'submitter__id', - 'submitter__email', - 'submitter__first_name', - 'submitter__last_name', - 'submitter__dob', - 'submitter__phone_number', - 'submitter__mobile_number', - 'submitter__fax_number', -] - -SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS = [ - 'submitter__residential_address__line1', - 'submitter__residential_address__line2', - 'submitter__residential_address__line3', - 'submitter__residential_address__locality', - 'submitter__residential_address__state', - 'submitter__residential_address__postcode', - 'submitter__residential_address__country', -] - -SUBMITTER_POSTAL_ADDRESS_FIELDS = [ - 'submitter__postal_address__line1', - 'submitter__postal_address__line2', - 'submitter__postal_address__line3', - 'submitter__postal_address__locality', - 'submitter__postal_address__state', - 'submitter__postal_address__postcode', - 'submitter__postal_address__country', -] - -APPROVAL_FIELDS = [ - 'approval__id', - 'approval__lodgement_number', - 'approval__status', - 'approval__internal_status', - 'approval__issue_date', - 'approval__start_date', - 'approval__expiry_date', - 'approval__wla_queue_date', - 'approval__migrated', - 'approval__submitter__email', - 'approval__org_applicant__organisation__name', - -# 'approval__proxy_applicant__', -# 'replaced_by', -# 'renewal_sent', -# 'original_issue_date', -# 'surrender_details', -# 'suspension_details', -# 'extracted_fields', -# 'cancellation_details', -# 'extend_details', -# 'cancellation_date', -# 'set_to_cancel', -# 'set_to_suspend', -# 'set_to_surrender', -# 'renewal_count', -# 'vessel_nomination_reminder_sent', -# 'reissued', -# 'export_to_mooring_booking', -# 'licence_document__', -# 'authorised_user_summary_document__', -# 'cover_letter_document__', -# 'current_proposal__', -# 'renewal_document__', - -] - -VESSEL_DETAILS_FIELDS = [ -# 'vessel_details__' - 'vessel_details__id', - 'vessel_details__vessel_type', - 'vessel_details__vessel_name', - 'vessel_details__vessel_length', - 'vessel_details__vessel_draft', - 'vessel_details__vessel_beam', - 'vessel_details__vessel_weight', - 'vessel_details__berth_mooring', - #'vessel_details__created', - #'vessel_details__updated', - 'vessel_details__exported', - 'vessel_details__vessel__rego_no', - #'vessel_details__vessel__blocking_owner__' -] - -VESSEL_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__id', - 'vessel_details__vessel__blocking_owner__percentage', - 'vessel_details__vessel__blocking_owner__dot_name', - 'vessel_details__vessel__blocking_owner__owner__emailuser__email', -] - -VESSEL_COMPANY_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__company_ownership__company__name', - #'vessel_details__vessel__blocking_owner__company_ownership__company__percentage', - -# 'company_ownership__' -# 'vessel_details__vessel__blocking_owner__start_date', -# 'vessel_details__vessel__blocking_owner__end_date', -# 'vessel_details__vessel__blocking_owner__created', -# 'vessel_details__vessel__blocking_owner__updated', -# 'vessel_details__vessel__blocking_owner__exported', -] - -VESSEL_PREFERRED_BAY_FIELDS = [ - 'preferred_bay__name', -] - -VESSEL_MOORING_FIELDS = [ - 'mooring__name', -] - -FEESEASON_FIELDS = [ - 'fee_season__name', -] - -# 'mooring__' -# 'allocated_mooring__' -# 'waiting_list_allocation__' -# 'fee_season__' - -# 'proxy_applicant__' -# 'assigned_officer__' -# 'assigned_approver__' -# 'previous_application__' -# 'approval_level_document__' - - -def write_csv3(): - """ - from mooringlicensing.utils.mooring_licence_migrate_write_csv import get_fields, write_csv3 - - """ - fields = PROPOSAL_FIELDS - fields += ORGANISATION_FIELDS - fields += ORGANISATION_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - fields += APPROVAL_FIELDS - fields += VESSEL_DETAILS_FIELDS - fields += VESSEL_OWNER_FIELDS - fields += VESSEL_COMPANY_OWNER_FIELDS - fields += VESSEL_PREFERRED_BAY_FIELDS - fields += VESSEL_MOORING_FIELDS - fields += FEESEASON_FIELDS - - mla_qs = MooringLicenceApplication.objects.filter()[:10].values_list(*fields) - #data = [fields].append(list(mla_qs)) - #import ipdb; ipdb.set_trace() - print(fields) - #return fields, mla_qs - -# with open("mla.csv", "w") as f: -# writer = csv.writer(f) -# #writer.writerows(fields) -# writer.writerows([fields]) -# writer.writerows(mla_qs) -# import ipdb; ipdb.set_trace() - - df = pd.DataFrame(mla_qs, columns=fields) - df['lodgement_date'] = df['lodgement_date'].dt.tz_localize(None) # remove timezone for excel output - df.to_excel('mla.xlsx', index=0) - return df - - -def write_csv(filename): - model_fields = [] - model_hdrs = [] - import ipdb; ipdb.set_trace() - for model in MODELS: - model_fields += [x.name for x in getattr(sys.modules[__name__], model)._meta.concrete_fields] - model_hdrs += [x.name + f' ({model})' for x in getattr(sys.modules[__name__], model)._meta.concrete_fields] - #model_fields += [''] - #model_hdrs += [''] - - model_fields_list = list(MooringLicenceApplication.objects.values_list(*model_fields)) - - with open(filename, 'w', newline='') as myfile: - for _list in model_fields_list: - wr = csv.writer(myfile, quoting=csv.QUOTE_ALL) - wr.writerow(_list) - - -from rest_framework.renderers import JSONRenderer -from mooringlicensing.components.proposals.serializers import VesselOwnershipSerializer, InternalProposalSerializer, MooringLicenceApplicationSerializer -import json - -def serialize(model, serializer): - """ - serialize('VesselOwnership', 'VesselOwnershipSerializer') - """ - #qs = VesselOwnership.objects.all() - #serializer = VesselOwnershipSerializer(qs, many=True) - qs = getattr(sys.modules[__name__], model).objects.all()[:2] - #serializer = getattr(sys.modules[__name__], serializer)(qs, many=True) - serializer = getattr(sys.modules[__name__], serializer)(qs, many=True) - #serializer.data - renderer = JSONRenderer() - rendered = renderer.render(serializer.data) - rendered = rendered.decode('utf-8') - #rendered - - return json.loads(rendered) - -def get_fields(model): - fk=[] - #for field in MooringLicenceApplication._meta.concrete_fields: - for field in model._meta.concrete_fields: - if field.get_internal_type() == 'ForeignKey': - #print(f'{field.name}__') - fk.append(f'\'{field.name}__\'',) - else: - print(f'\'{field.name}\',') - -# fk_fields = get_fields(ledger_org) -# for field in fk_fields: -# print(f'org_applicant__{field}') - - - - for field in fk: - print(field) - - -def write_csv2(): - fields = PROPOSAL_FIELDS - mla_qs = MooringLicenceApplication.objects.filter()[:1] -# for mla in mla_qs: -# if mla.org_applicant: -# fields += ORGANISATION_FIELDS -# if mla.org_applicant and mla.org_applicant.organisation.postal_address: -# fields += ORGANISATION_POSTAL_ADDRESS_FIELDS -# if mla.submitter: -# fields += SUBMITTER_FIELDS -# if mla.submitter and mla.submitter.postal_address: -# fields += SUBMITTER_POSTAL_ADDRESS_FIELDS -# if mla.submitter and mla.submitter.residential_address: -# fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - - - fields += ORGANISATION_FIELDS - fields += ORGANISATION_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - mla = MooringLicenceApplication.objects.filter()[:1].values_list(*fields) - #mla = .values_list(*fields) - print(mla) - - diff --git a/mooringlicensing/utils/archive/mooringlicensing-cronjobs b/mooringlicensing/utils/archive/mooringlicensing-cronjobs deleted file mode 100755 index 782981280..000000000 --- a/mooringlicensing/utils/archive/mooringlicensing-cronjobs +++ /dev/null @@ -1,6 +0,0 @@ -SHELL=/bin/bash -# Execute every minute. Polls the CommercialOperator Admin table SystemMaintenance, and checks if the application can be taken down at the time indicated in the Admin table -* * * * * root cd /var/www/commercialoperator-prod.8088 && source venv/bin/activate && python manage_ds.py system_maintenance_check >/dev/null 2>&1 -# -# Every day (at 02:00 hrs), run the daily commercialoperator management commands batch -0 2 * * * www-data cd /var/www/commercialoperator-prod.8088 && source venv/bin/activate && python manage_ds.py cron_tasks >/dev/null 2>&1 diff --git a/mooringlicensing/utils/archive/vessel_migrate.py b/mooringlicensing/utils/archive/vessel_migrate.py deleted file mode 100644 index 7835e7d51..000000000 --- a/mooringlicensing/utils/archive/vessel_migrate.py +++ /dev/null @@ -1,564 +0,0 @@ -import os -import subprocess -import json -import ast -import csv -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist -from oscar.apps.address.models import Country -from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - ProposalUserAction, - Mooring, - MooringBay, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval, Sticker - - - -class VesselMigration(object): - ''' - PRELIM: - from mooringlicensing.utils.json_to_csv import JsonToCsv - JsonToCsv().convert() - - from mooringlicensing.utils.authorised_user_migrate import AuthUserPermitMigration, GrepSearch - AuthUserPermitMigration(test=False) - ''' - - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021', path_csv='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv', test=False): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.path_csv = path + '_all_csv' if not path.endswith('/') else path.strip('/') + '_all_csv' - self.test = test - -# self.auth_users_with_L = self.read_dict('Auth_Users___Surname___with_L.json') -# self.auth_users_No_L = self.read_dict('Auth_Users___Surname___No_L.json') -# self.moorings = self.read_dict('Auth_Users___Surname___Moorings.json') -# self.vessel_rego = self.read_dict('Auth_Users___Vessel_Rego.json') -# -# self.canc_lic = self.read_dict('Auth_Users___Cancelled_Lic.json') -# self.canc = self.read_dict('Auth_Users___Cancelled.json') -# -# self.vessel_rego_all = self.read_dict('Vessel___Rego___All.json') -# self.vessel_rego_current = self.read_dict('Vessel___Rego___Current.json') -# self.vessel_details_all = self.read_dict('Vessel___Details_All.json') -# self.vessel_details_current = self.read_dict('Vessel___Details_Current.json') -# self.people_name = self.read_dict('PeopleName.json') - - #self.surname = self.read_dict('Licencees___Surname.json') - self.mooring_no = self.read_dict('Licencees___Mooring_No.json') - self.mooring_details = self.read_dict('Mooring_Details.json') - self.people_lictype = self.read_dict('PeopleLicType.json') - self.vessel_details = self.read_dict('Vessel___Details_All.json') - #self.vessel_multiple = self.read_dict('Vessel___Multiple_Vessels.json') # vessel rego and name - self.vessel_multiple = self.read_dict('Licencees___Sticker_No.json') # vessel rego and name - self.user_rego = self.read_dict('UserRego.json') # vessel length/draft - if not available in vessel_multiple - self.user_mooring = self.read_dict('UserMooring.json') # vessel length/draft - if not available in vessel_multiple - self.people_no = self.read_dict('PeopleNo.json') # address/phone details (if no vessel_multiple details) - self.people_cur_mooring = self.read_dict('People___Current_Mooring.json') # address/phone details (if no vessel_multiple details) - #self.people_email = self.read_dict('People___eMail.json') # address/phone details (if no vessel_multiple details) - self.people_email = self.read_dict('Licencees___Surname.json') # address/phone details (if no vessel_multiple details) - - - self._auth_users = [ - #... can be found in directory tests/test.json - ] - - #self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def search_multiple(self, key, value, records): - return [record for record in records if value == record.get(key)] - - - def get_vessels(self, no_vessel, record): - if not record: - # ML with no linked vessel - #return [('', '', 0.0, 0.0)] # rego/name/length/draft - return [] # rego/name/length/draft - - #keys = ['_6', '_7', '_8', '_9', '_10'] - ves_keys = ['_10', '_11', '_12', '_13', '_14'] - sticker_keys = ['StickerLNo1', 'StickerLNo2', 'StickerLNo3', 'StickerLNo4', 'StickerLNo5'] - _vessels = [] - for i in range(no_vessel): - items = record[ves_keys[i]].split('-') - rego = items[0].strip() - vessel_name = items[1].strip() - sticker_no = record[sticker_keys[i]].strip() - if sticker_no == 'None': - sticker_no = '' -# import ipdb; ipdb.set_trace() - _vessels.append((rego, vessel_name, 0.0, 0.0, sticker_no)) # rego/name/length/draft - return _vessels - -# def get_stickers(self, no_vessel, record): -# if not record: -# # ML with no linked vessel -# return [] -# -# keys = ['StickerLNo1', 'StickerLNo2', 'StickerLNo3', 'StickerLNo4', 'StickerLNo5'] -# _stickers = [] -# for i in range(no_vessel): -# sticker_no = record[keys[i]].strip() -# _stickers.append((rego, vessel_name, 0.0, 0.0)) # rego/name/length/draft -# return _vessels - - - def to_float(self, val): - try: - return float(val) - except Exception as e: - return 0.0 - - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - start_date = datetime.date(2022,8,31) - date_applied = '2022-08-31' - date_invited = None #date_invited, - migrated = False - - self.num_vessels = [] - self.vessels_found = [] - self.vessels_not_found = [] - self.vessel_details_not_found = [] - self.emailuser = [] - self.errors = [] - self.approvals = [] - - user_data = [] - vessel_data = [] - mla_data = [] - ml_all_data = [] - user_data.append(['PersNo', 'Email', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address', 'NoVessel']) - vessel_data.append(['Owner', 'Email', 'DoB', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring']) - mla_data.append(['PersNo', 'Email', 'DoB', 'Rego No', 'MooringNo', 'Date Invited']) - ml_all_data.append(['PersNo', 'Email', 'DoB', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring', 'MooringNo', 'Date Invited']) - - rego_no='' - dot_name='' - percentage='' - vessel_type='' - vessel_name='' - vessel_overall_length='' - vessel_draft='' - vessel_weight='' - berth_mooring='' - - with transaction.atomic(): - #for idx, record in enumerate(self.moorings[827:], 827): - import ipdb; ipdb.set_trace() - #for idx, record in enumerate(self.vessel_rego_current[:60], 0): - #for idx, record in enumerate(self.vessel_rego_current, 0): - #for idx, record in enumerate(self.mooring_no[:300], 0): - #for idx, record in enumerate(self.mooring_no[200:205], 0): - for idx, record in enumerate(self.mooring_no[0:], 0): - try: - mooring_no = record['MooringNo'] - username = record['UserName'] - pers_no = record['PersNo'] - address = record['_1'] - no_vessel = int(float(record['NoVessel'])) - - if no_vessel>5: - self.num_vessels.append(dict(username=username, pers_no=pers_no, no_vessel=no_vessel, mooring_no=mooring_no)) - no_vessel = 5 # ignore the remainder (JSON file only list max. 5) - - if pers_no not in ['000477', '206776', '207209', '030742']: - continue -# if pers_no == '073604': -# import ipdb; ipdb.set_trace() -# else: -# continue - - - - # assume first vessel is licenced to Mooring - record_ves_details = self.search('PersNo', pers_no, self.vessel_multiple) - vessels = self.get_vessels(no_vessel, record_ves_details) - print(f'{idx} Vessel details found: {username} {pers_no} {mooring_no} {no_vessel} {vessels}') - #vessel_mooring = vessels[0] - - #import ipdb; ipdb.set_trace() - record_people_no = self.search('PersNo', pers_no, self.people_cur_mooring) - address = record_people_no['_1'] - phone_home = record_people_no['PhoneHome'] - phone_mobile = record_people_no['PhoneMobile'] - phone_work = record_people_no['PhoneWork'] - - #{'LastName': 'Abbott', '_7': 'William', 'PersNo': '207907', '_8': 'W', 'NoVessel': '1', '_9': '1', 'EMail': 'wabbinator@gmail.com', 'LicContactEmail': 'None'} - record_people_email = self.search('PersNo', pers_no, self.people_email) - email = record_people_email['EMail'].split(';')[0].strip().lower() - - user = self.set_emailuser(username, email, address, phone_mobile, phone_home) - - # see mooringlicensing/utils/tests/mooring_names.txt - if Mooring.objects.filter(name=mooring_no).count()>0: - mooring = Mooring.objects.filter(name=mooring_no)[0] - else: - import ipdb; ipdb.set_trace() - print(f'Mooring not found: {mooring}') - self.errors.append(dict(idx=idx, record=record)) - continue - - approval = MooringLicence.objects.create( - status='current', - #internal_status=None, - #current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=migrated, - export_to_mooring_booking=True, - ) - self.approvals.append(approval.id) - - if not vessels: - proposal=MooringLicenceApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.datetime.now(datetime.timezone.utc), - migrated=migrated, - - preferred_bay= mooring.mooring_bay, - processing_status='approved', - customer_status='approved', - allocated_mooring=mooring, - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - - "vessel_ownership": [dict(dot_name=i[0]) for i in vessels], - "mooring_on_approval": [] - }, - date_invited=date_invited, - ) - - approval.current_proposal=proposal - approval.save() - - else: - #import ipdb; ipdb.set_trace() - vessel_objs = [] - #proposal = None - for ves_idx, ves in enumerate(vessels): - ves_rego_no = ves[0] - ves_name = ves[1] - ves_length = self.to_float(ves[2]) - ves_draft = self.to_float(ves[3]) - sticker_no = ves[4] - ves_type = '' - ves_weight = Decimal( 0.0 ) - berth_mooring = '' - percentage = None # force user to set at renewal time - vessel_type = '' - - try: - vessel = Vessel.objects.get(rego_no=ves_rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=ves_rego_no) - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - vessel_ownership.dot_name = ves_rego_no # rego_no - vessel_ownership.save() - except ObjectDoesNotExist: - #vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage, dot_name=dot_name) - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage, dot_name=ves_rego_no) - - if VesselDetails.objects.filter(vessel=vessel).count()>0: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - else: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=ves_name, - vessel_length=ves_length, - vessel_draft=ves_draft, - vessel_weight= ves_weight, - berth_mooring=berth_mooring - ) - - vessel_objs.append(dict( - vessel=vessel, - owner=owner, - vessel_ownership=vessel_ownership, - vessel_details=vessel_details, - )) - - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - - #import ipdb; ipdb.set_trace() - proposal=MooringLicenceApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.datetime.now(datetime.timezone.utc), - migrated=migrated, - - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=ves_rego_no, - vessel_type=ves_type, - vessel_name=ves_name, - vessel_length=ves_length, - vessel_draft=ves_draft, - #vessel_beam='', - vessel_weight=ves_weight, - berth_mooring=berth_mooring, - percentage=percentage, - individual_owner=True, - dot_name='', #vessel_mooring[0], #vessel_dot, - - preferred_bay= mooring.mooring_bay, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - allocated_mooring=mooring, - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - # "vessel_ownership": [{ - # "dot_name": vessel_mooring[0], # vessel_mooring[0] --> rego_no #vessel_dot - # }], - "vessel_ownership": [dict(dot_name=i[0]) for i in vessels], - "mooring_on_approval": [] - }, - date_invited=date_invited, - ) - -# proposal.vessel_details=vessel_details -# proposal.vessel_ownership=vessel_ownership -# proposal.rego_no=ves_rego_no -# proposal.vessel_type=ves_type -# proposal.vessel_name=ves_name -# proposal.vessel_length=ves_length -# proposal.vessel_draft=ves_draft -# #proposal.vessel_beam='' -# proposal.vessel_weight=ves_weight -# proposal.berth_mooring=berth_mooring -# proposal.percentage=percentage -# proposal.individual_owner=True -# proposal.dot_name='' #vessel_mooring[0], #vessel_dot, -# proposal.save() - - sticker=Sticker.objects.create( - number=sticker_no, - status='current', - approval=approval, - printing_date=start_date, - mailing_date=start_date, - vessel_ownership=vessel_ownership, - proposal_initiated=proposal, - #dcv_permit=null, - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user, - what='Mooring Site Licence - Migrated Application', - ) - - vooa = VesselOwnershipOnApproval.objects.create( - approval=approval, - vessel_ownership=vessel_ownership, - ) - - #import ipdb; ipdb.set_trace() - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership=vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - #stickers = [sticker.id], - ) - approval_history.stickers.add(sticker.id), - -# for approval_id in self.approvals: -# a = MooringLicence.objects.get(id=approval_id) -# import ipdb; ipdb.set_trace() -# a.generate_doc() - - - proposal.approval = approval - proposal.save() - #proposal.allocated_mooring.mooring_licence = approval - #proposal.allocated_mooring.save() - #approval.generate_doc() - -# moa = MooringOnApproval.objects.create( -# approval=approval, -# mooring=mooring, -# sticker=None, -# site_licensee=True, # ??? -# end_date=expiry_date -# ) - - ml_all_data.append([pers_no, email, user.dob, user.first_name, user.last_name, user.mobile_number, user.phone_number, user.residential_address, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring, mooring.name, date_invited]) - - user_data.append([pers_no, email, user.dob, user.first_name, user.last_name, user.mobile_number, user.phone_number, user.residential_address, no_vessel]) - - berth_mooring = '' - vessel_data.append([pers_no, email, user.dob, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring]) - - #date_invited = '' - mla_data.append([pers_no, email, user.dob, rego_no, mooring.name, date_invited]) - - except Exception as e: - self.errors.append(f'{idx} {record}\n{e}') - import ipdb; ipdb.set_trace() - print(e) - - #import ipdb; ipdb.set_trace() - with open('ml_all.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(ml_all_data) - - with open('user.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(user_data) - - with open('vessel.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(vessel_data) - - with open('mla.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(mla_data) - - - - - - print(f'vessels_found: {self.vessels_found}') - print(f'vessels_not_found: {self.vessels_not_found}') - print(f'vessel_details_not_found: {self.vessel_details_not_found}') - print(f'num_vessels: {self.num_vessels}') - print(f'errors: {self.errors}') - print(f'no. vessels_found: {len(self.vessels_found)}') - print(f'no. vessels_not_found: {len(self.vessels_not_found)}') - print(f'no. vessel_details_not_found: {len(self.vessel_details_not_found)}') - print(f'no. emailuser: {len(self.emailuser)}') - print(f'no. num_vessels: {len(self.num_vessels)}') - print(f'no. errors: {len(self.errors)}') - - def set_emailuser(self, username, email, address, phone_mobile, phone_home): - items = address.split(',') - line1 = ', '.join(items[:-2]).strip() - state = items[-2].strip() - postcode = items[-1].strip() - - firstname = username.split(' ')[-1].strip() - lastname = ' '.join(username.split(' ')[:-1]) - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=phone_mobile, phone_number=phone_home) - - country = Country.objects.get(printable_name='Australia') - try: - _address = Address.objects.get(user=user) - _address.line1 = line1 - _address.state = state - _address.postcode = postcode - _address.country = country - _address.save() - except: - _address, address_created = Address.objects.get_or_create(line1=line1, state=state, postcode=postcode, country=country, user=user) - user.residential_address = _address - user.postal_address = _address - user.save() - self.emailuser.append(f'{email}') - - return user - - - - def get_vessel_details(self, idx, pers_no, username, rego_no, record): - try: - #import ipdb; ipdb.set_trace() - if record: - if rego_no != '0': - dot_name = record['DoTRego1'] - ves_length = record['RegLength1'] - ves_draft = record['Draft1'] - - record_people_name = self.search('PersNo', pers_no, self.people_name) - ves_name = record_people_name['_3'] - email = record_people_name['EMail'] - - print(idx, email) - #import ipdb; ipdb.set_trace() - self.vessels_found.append(f'{pers_no} - {username} - {rego_no}') - else: - #print(idx, f'No Reg {rego_no} - {username}') - self.vessels_not_found.append(f'{pers_no} - {username} - {rego_no}') - #import ipdb; ipdb.set_trace() - else: - print(idx, f'No Vessel Details {pers_no} - {username}') - self.vessel_details_not_found.append(f'{pers_no} - {username} - {rego_no}') - - except Excception as e: - self.errors.append(f'{idx} {record}\n{e}') - import ipdb; ipdb.set_trace() - #print(e) - - - diff --git a/mooringlicensing/utils/archive/waiting_list_migrate.py b/mooringlicensing/utils/archive/waiting_list_migrate.py deleted file mode 100644 index 670f5f354..000000000 --- a/mooringlicensing/utils/archive/waiting_list_migrate.py +++ /dev/null @@ -1,363 +0,0 @@ -import os -import subprocess -import json -import datetime -from decimal import Decimal -from django.db import transaction -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - WaitingListApplication, - ProposalUserAction, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, WaitingListAllocation - -class WaitingListMigration(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - WaitingListMigration(test=False) - ''' - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.test = test - - self.waitlist = self.read_dict('Waitlist___Bay.json') - - self._waitlist = [ - #... can be found in directory tests/test.json - ] - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - - addresses_not_found = [] - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - wl_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - wla_list = [] - - added = [] - errors = [] - with transaction.atomic(): - #for idx, record in enumerate(self.waitlist[541:], 541): - for idx, record in enumerate(self.waitlist, 1): - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - email_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - address_record = GrepSearch(pers_no, path=self.path).search('PersNo', '_1') - if address_record is None: - print(f'Address Not Found: {idx}, {pers_no}') - addresses_not_found.append(pers_no) - continue - - - phonemobile_record = {} - if 'PhoneMobile' not in address_record: - phonemobile_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneMobile') - else: - phonemobile_record = address_record - - phonehome_record = {} - if 'PhoneHome' not in address_record: - phonehome_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneHome') - else: - phonehome_record = address_record - - email = email_record.get('EMail').lower().strip() - if email=='mjmatthews@me.com': - errors.append(email) - continue - - mobile_no = phonemobile_record.get('PhoneMobile') - username = record.get('UserName').lower() - firstname = username.split(' ')[-1].title() - lastname = ' '.join(username.split(' ')[:-1]).title() - - try: - phone_no = phonehome_record.get('PhoneHome') if phonehome_record else '' - address = address_record.get('_1') - except: - import ipdb; ipdb.set_trace() - - rego_no = record.get('VesRego') - vessel_type = 'other' - vessel_name = record.get('VesName') - vessel_overall_length = Decimal( record.get('RegLength') ) - vessel_draft = Decimal( record.get('Draft') ) - vessel_weight = Decimal( 0.0 ) - berth_mooring = record.get('_5') - - # see mooringlicensing/utils/tests/mooring_names.txt - if MooringBay.objects.filter(name=berth_mooring).count()>0: - mooring_bay = MooringBay.objects.filter(name=berth_mooring)[0] - else: - mooring_bay = MooringBay.objects.get(name='Rottnest Island') - #mooring_bay = MooringBay.objects.get(name='Thomson Bay') - - # needed ?? - date_applied = record.get('DateApplied') - position_no = int(record.get('BayPosNo')) - trim_no = record.get('TrimNo') - percentage = None # force user to set at renewal time - - - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {state}, {postcode}') - if self.test: - continue - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) - - country = Country.objects.get(printable_name='Australia') - #address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - address, address_created = Address.objects.get_or_create( - user=user, - defaults={ - 'line1':line1, - 'line2':line2, - 'line3':line3, - 'postcode':postcode, - 'state':state, - 'country':country - } - ) - user.residential_address = address - user.postal_address = address - user.save() - - vessel, rego_created = Vessel.objects.update_or_create(rego_no=rego_no) - owner, owner_created = Owner.objects.update_or_create(emailuser=user) - vessel_ownership, ownership_created = VesselOwnership.objects.update_or_create( - owner=owner, - vessel=vessel, - percentage=percentage - ) - - vessel_details, details_created = VesselDetails.objects.update_or_create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - berth_mooring=berth_mooring - ) - - proposal=WaitingListApplication.objects.create( - proposal_type_id=1, - submitter=user, - lodgement_date=datetime.now(), - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - #vessel_beam='', - vessel_weight=vessel_weight, - berth_mooring=berth_mooring, - preferred_bay=mooring_bay, - percentage=percentage, - individual_owner=True, - proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Waiting List - Migrated Application', - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() - - approval = WaitingListAllocation.objects.create( - status='current', - internal_status='waiting', - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - wla_order=position_no, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - address_list.append(address.id) - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - wl_list.append(proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - - print(f'Address.objects.get(id__in={address_list}).delete()') - print(f'EmailUser.objects.get(id__in={user_list}).delete()') - print(f'Vessel.objects.get(id__in={vessel_list}).delete()') - print(f'Owner.objects.get(id__in={owner_list}).delete()') - print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') - print(f'VesselDetails.objects.get(id__in={details_list}).delete()') - print(f'WaitingListApplication.objects.get(id__in={wl_list}).delete()') - print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') - print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') - print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') - print(f'Addresses Not Found: {addresses_not_found}') - -def clear_record(): - Address.objects.last().delete() - EmailUser.objects.last().delete() - Vessel.objects.last().delete() - Owner.objects.last().delete() - VesselOwnership.objects.last().delete() - VesselDetails.objects.last().delete() - WaitingListApplication.objects.last().delete() - ProposalUserAction.objects.last().delete() - WaitingListAllocation.objects.last().delete() - ApprovalHistory.objects.last().delete() - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' -# files = self.get_files(self.search_str1) - files=[ - self.path + os.sep + 'Admin___EContacts___6_Waitlist.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'Vessel___Vessel_Name___Current.json', - self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - self.path + os.sep + 'Admin___Interrogation___Lic___WL_DOB_check.json', - self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'Auth_Users___Surname___with_L.json', - self.path + os.sep + 'Vessel___HIN___Current.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - print(f'fname: {fname}, key2: {key2}') - return record - - diff --git a/mooringlicensing/utils/authorised_user_migrate.15Oct2021.py b/mooringlicensing/utils/authorised_user_migrate.15Oct2021.py deleted file mode 100644 index c2b43a7bd..000000000 --- a/mooringlicensing/utils/authorised_user_migrate.15Oct2021.py +++ /dev/null @@ -1,489 +0,0 @@ -import os -import subprocess -import json -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - AuthorisedUserApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringOnApproval, AuthorisedUserPermit - - -class AuthUserPermitMigration(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - WaitingListMigration(test=False) - ''' - - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021', test=False): - self.path = path - self.test = test - - #self.waitlist = self.read_dict('Waitlist___Bay.json') - self.auth_users_with_L = self.read_dict('Auth_Users___Surname___with_L.json') - self.auth_users_No_L = self.read_dict('Auth_Users___Surname___No_L.json') - self.moorings = self.read_dict('Auth_Users___Surname___Moorings.json') - self.vessel_rego = self.read_dict('Auth_Users___Vessel_Rego.json') - - self.canc_lic = self.read_dict('Auth_Users___Cancelled_Lic.json') - self.canc = self.read_dict('Auth_Users___Cancelled.json') - - - self._auth_users = [ - #... can be found in directory tests/test.json - ] - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def search_multiple(self, key, value, records): - return [record for record in records if value == record.get(key)] - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2021,11,30) - date_applied = '2020-08-31' - - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - wl_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - wla_list = [] - - added = [] - errors = [] - no_records = [] - fnames = [] - count_no_mooring = 0 - with transaction.atomic(): - for idx, record in enumerate(self.auth_users_No_L[33:200], 33): - #for idx, record in enumerate(self.auth_users_No_L[46:], 46): - #for idx, record in enumerate(self.auth_users_with_L, 1): - #for idx, record in enumerate(self.auth_users_with_L[2:3], 1): - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - no_auth_permits = int(record['NoAuth']) - if no_auth_permits == 0: - continue - - address = record.get('_1') - - email_record, fname1 = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - phonemobile_record, fname2 = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneMobile') - phonehome_record, fname3 = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneHome') -# if fname1 not in fnames: -# fnames.append(fname1) -# if fname2 not in fnames: -# fnames.append(fname2) -# if fname3 not in fnames: -# fnames.append(fname3) - - - email = email_record.get('EMail').lower() - mobile_no = phonemobile_record.get('PhoneMobile') - username = record.get('UserName') #.lower() - firstname = username.split(' ')[-1] - lastname = ' '.join(username.split(' ')[:-1]) - - try: - phone_no = phonehome_record.get('PhoneHome') if phonehome_record else '' - #address = address_record.get('_1') - except: - import ipdb; ipdb.set_trace() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {username}, {no_auth_permits}') - - #for no_auth in no_auth_permits: - vessel_rego_records = self.search_multiple('_1', username, self.vessel_rego) - moorings_records = self.search_multiple('_1', username, self.moorings) - if len(vessel_rego_records) == no_auth_permits: - #print(f'{len(vessel_rego_records)}, {no_auth_permits}') - #print(f'Vessel Rego: {vessel_rego_records}') - aup_records = vessel_rego_records - elif len(moorings_records) == no_auth_permits: - #print(f'{len(moorings_records)}, {no_auth_permits}') - #print(f'Moorings: {moorings_records}') - aup_records = moorings_records - else: - canc = self.search_multiple('_1', username, self.canc) - canc_lic = self.search_multiple('_12', username, self.canc_lic) - - num_canc = len(canc) + len(canc_lic) - rec = dict(username=username, noauth=no_auth_permits, num_cancelled=num_canc) - no_records.append(rec) - print(f'ERROR: aup_records not matched: {rec}') - #import ipdb; ipdb.set_trace() - continue - - for i in aup_records: - permit_type = aup_records[0]['_6'] # RIA or Lic - mooring_no = aup_records[0]['MooringNo'] - vessel_name = aup_records[0]['VesName'] - #vessel_len = aup_records[0]['VesLen'] - vessel_rego = aup_records[0]['VesRego'] - - #if i['_1'] == username: - # print(i) - - if self.test: - #import ipdb; ipdb.set_trace() - continue - - -# mooring_record = self.search('_1', username, self.moorings) -# try: -# mooring = mooring_record['MooringNo'] -# vessel_name = mooring_record['VesName'] -# rego_no = mooring_record['VesRego'] -# except: -# count_no_mooring += 1 -# print(f'*************** {idx}, {pers_no}, {username}, {count_no_mooring}') -# continue -# #vessel_name = record.get('_11').split('-')[1].strip() -# #rego_no = record.get('_11').split('-')[0].strip() -# -# vessel_type = 'other' -# vessel_overall_length = Decimal( record.get('RegLength1') ) -# #vessel_draft = Decimal( record.get('Draft') ) -# vessel_draft = Decimal( 0.0 ) -# vessel_weight = Decimal( 0.0 ) -# #berth_mooring = record.get('_5') -# #mooring = ?? # record.get('_5') -# -# # see mooringlicensing/utils/tests/mooring_names.txt -# if Mooring.objects.filter(name=mooring).count()>0: -# mooring = Mooring.objects.filter(name=mooring)[0] -# else: -# import ipdb; ipdb.set_trace() -# print(f'Mooring not found: {mooring}') -# #mooring_bay = MooringBay.objects.get(name='Rottnest Island') -# #mooring_bay = MooringBay.objects.get(name='Thomson Bay') -# -# # needed ?? -# #date_applied = record.get('DateApplied') -# #position_no = int(record.get('BayPosNo')) -# #trim_no = record.get('TrimNo') -# percentage = None # force user to set at renewal time -# -# -# items = address.split(',') -# line1 = items[0].strip() -# line2 = items[1].strip() if len(items) > 3 else '' -# line3 = items[2].strip() if len(items) > 4 else '' -# state = items[-2].strip() -# postcode = items[-1].strip() -# -# #import ipdb; ipdb.set_trace() -# print(f'{idx}, {pers_no}, {username}, {state}, {postcode}') -# if self.test: -# continue -# -# try: -# user = EmailUser.objects.get(email=email) -# except: -# user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) -# -# country = Country.objects.get(printable_name='Australia') -# address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) -# user.residential_address = address -# user.postal_address = address -# user.save() -# -# try: -# vessel = Vessel.objects.get(rego_no=rego_no) -# except ObjectDoesNotExist: -# vessel = Vessel.objects.create(rego_no=rego_no) -# -# -# try: -# owner = Owner.objects.get(emailuser=user) -# except ObjectDoesNotExist: -# owner = Owner.objects.create(emailuser=user) -# -# -# try: -# vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) -# except ObjectDoesNotExist: -# vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage) -# -# try: -# vessel_details = VesselDetails.objects.get(vessel=vessel) -# except ObjectDoesNotExist: -# vessel_details = VesselDetails.objects.create( -# vessel_type=vessel_type, -# vessel=vessel, -# vessel_name=vessel_name, -# vessel_length=vessel_overall_length, -# vessel_draft=vessel_draft, -# vessel_weight= vessel_weight, -# #berth_mooring=berth_mooring -# ) -# -# proposal=AuthorisedUserApplication.objects.create( -# proposal_type_id=1, # new application -# submitter=user, -# migrated=True, -# vessel_details=vessel_details, -# vessel_ownership=vessel_ownership, -# rego_no=rego_no, -# vessel_type=vessel_type, -# vessel_name=vessel_name, -# vessel_length=vessel_overall_length, -# vessel_draft=vessel_draft, -# #vessel_beam='', -# vessel_weight=vessel_weight, -# berth_mooring='home', -# preferred_bay= mooring.mooring_bay, -# percentage=percentage, -# individual_owner=True, -# #proposed_issuance_approval={}, -# processing_status='approved', -# customer_status='approved', -# proposed_issuance_approval={ -# "details": None, -# "cc_email": None, -# "mooring_id": mooring.id, -# "ria_mooring_name": mooring.name, -# "mooring_bay_id": mooring.mooring_bay.id, -# "vessel_ownership": [], -# "mooring_on_approval": [] -# }, -# ) -##> "proposed_issuance_approval": { -##> "details": "dd", -##> "cc_email": null, -##> "mooring_id": 127, -##> "mooring_bay_id": 7, -##> "ria_mooring_name": "CB005", -##> "vessel_ownership": [], -##> "mooring_on_approval": [] -##> }, -## -##> "berth_mooring": "home", -##> "dot_name": "abc", -##> "insurance_choice": "over_ten", -##> "preferred_bay": null, -##> "silent_elector": null, -##> "mooring_authorisation_preference": "ria", -##> "bay_preferences_numbered": "[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\"]", -# -# ua=ProposalUserAction.objects.create( -# proposal=proposal, -# who=user, -# what='Authorised User Permit - Migrated Application', -# ) -# -# try: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() -# except: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() -# -# #approval = WaitingListAllocation.objects.create( -# approval = AuthorisedUserPermit.objects.create( -# status='current', -# #internal_status=None, -# current_proposal=proposal, -# issue_date = datetime.datetime.now(datetime.timezone.utc), -# #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), -# start_date = start_date, -# expiry_date = expiry_date, -# submitter=user, -# migrated=True, -# export_to_mooring_booking=True, -# ) -# #print(f'wla_order: {position_no}') -# -# proposal.approval = approval -# proposal.save() -# -# moa = MooringOnApproval.objects.create( -# approval=approval, -# mooring=mooring, -# sticker=None, -# site_licensee=True, # ??? -# end_date=expiry_date -# ) -# -# try: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) -# except: -# start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) -# -# approval_history = ApprovalHistory.objects.create( -# reason='new', -# approval=approval, -# vessel_ownership = vessel_ownership, -# proposal = proposal, -# #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) -# start_date = start_date, -# ) -# -# added.append(proposal.id) -# -# address_list.append(address.id) -# user_list.append(user.id) -# vessel_list.append(vessel.id) -# owner_list.append(owner.id) -# ownership_list.append(vessel_ownership.id) -# details_list.append(vessel_details.id) -# proposal_list.append(proposal.proposal.id) -# wl_list.append(proposal.id) -# user_action_list.append(ua.id) -# approval_list.append(approval.id) -# approval_history_list.append(approval_history.id) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - -# print(f'Address.objects.get(id__in={address_list}).delete()') -# print(f'EmailUser.objects.get(id__in={user_list}).delete()') -# print(f'Vessel.objects.get(id__in={vessel_list}).delete()') -# print(f'Owner.objects.get(id__in={owner_list}).delete()') -# print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') -# print(f'VesselDetails.objects.get(id__in={details_list}).delete()') -# print(f'WaitingListApplication.objects.get(id__in={wl_list}).delete()') -# print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') -# print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') -# print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') -# print(f'count_no_mooring: {count_no_mooring}') - print(f'no_records: {no_records}') - print(f'fnames_records: {fnames}') - -def clear_record(): - Address.objects.last().delete() - EmailUser.objects.last().delete() - Vessel.objects.last().delete() - Owner.objects.last().delete() - VesselOwnership.objects.last().delete() - VesselDetails.objects.last().delete() - WaitingListApplication.objects.last().delete() - ProposalUserAction.objects.last().delete() - WaitingListAllocation.objects.last().delete() - ApprovalHistory.objects.last().delete() - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - _files = self.get_files(self.search_str1) - files=[ - - #self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleName.json', - self.path + os.sep + 'PeopleLicType.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'HINCurrent.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'Vessel___Vessel_Name___Current.json', - -# self.path + os.sep + 'Vessel___Rego___Current.json', -# self.path + os.sep + 'PeopleNo.json', -# self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', -# -# self.path + os.sep + 'Auth_User_Details.json', -# self.path + os.sep + 'Auth_User___Inv.json', -# self.path + os.sep + 'Auth_User_Payment_Summary.json', -# self.path + os.sep + 'Auth_Users___AutoCanc.json', -# self.path + os.sep + 'Auth_Users___Cancelled.json', -# self.path + os.sep + 'Auth_Users___Cancelled_Lic.json', -# self.path + os.sep + 'Auth_Users__Cancelled_Lic_Name.json', -# self.path + os.sep + 'Auth_Users___Cancelled_PersNo.json', -# self.path + os.sep + 'Auth_Users___Lic_Pers_No.json', -# self.path + os.sep + 'Auth_Users___Mooring.json', -# self.path + os.sep + 'Auth_Users___Sticker_No.json', -# self.path + os.sep + 'Auth_Users___Surname___Moorings.json', -# self.path + os.sep + 'Auth_Users___Surname___No.json', -# self.path + os.sep + 'Auth_Users___Surname___No_L.json', -# self.path + os.sep + 'Auth_Users___Surname___with_L.json', -# self.path + os.sep + 'Auth_Users___User_Pers_No.json', -# self.path + os.sep + 'Auth_Users___Vessel_Rego.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - #print('****************' + fname) - return record, fname - - diff --git a/mooringlicensing/utils/authorised_user_migrate.py b/mooringlicensing/utils/authorised_user_migrate.py deleted file mode 100644 index fa365ae81..000000000 --- a/mooringlicensing/utils/authorised_user_migrate.py +++ /dev/null @@ -1,675 +0,0 @@ -import os -import subprocess -import json -import ast -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - AuthorisedUserApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringOnApproval, AuthorisedUserPermit - - -class AuthUserPermitMigration(object): - ''' - PRELIM: - from mooringlicensing.utils.json_to_csv import JsonToCsv - JsonToCsv().convert() - - from mooringlicensing.utils.authorised_user_migrate import AuthUserPermitMigration, GrepSearch - AuthUserPermitMigration(test=False) - ''' - - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021', path_csv='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv', test=False): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.path_csv = path + '_all_csv' if not path.endswith('/') else path.strip('/') + '_all_csv' - self.test = test - - #self.waitlist = self.read_dict('Waitlist___Bay.json') - self.auth_users_with_L = self.read_dict('Auth_Users___Surname___with_L.json') - self.auth_users_No_L = self.read_dict('Auth_Users___Surname___No_L.json') - self.moorings = self.read_dict('Auth_Users___Surname___Moorings.json') - self.vessel_rego = self.read_dict('Auth_Users___Vessel_Rego.json') - - self.canc_lic = self.read_dict('Auth_Users___Cancelled_Lic.json') - self.canc = self.read_dict('Auth_Users___Cancelled.json') - - - self._auth_users = [ - #... can be found in directory tests/test.json - ] - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def search_multiple(self, key, value, records): - return [record for record in records if value == record.get(key)] - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - date_applied = '2021-08-31' - - address_not_found = [] - address_l_not_found = [] - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - aup_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - - aup_data = [] - aup_data.append(['PersNo', 'Email', 'DoB', 'Rego No', 'MooringNo', 'Mooring Authorisation (RIA/LIC)']) - - added = [] - errors = [] - no_records = [] - fnames = [] - no_persno = [] - no_email = [] - no_email_l = [] - no_licencee = [] - count_no_mooring = 0 - with transaction.atomic(): - #for idx, record in enumerate(self.moorings[827:], 827): - for idx, record in enumerate(self.moorings, 1): - try: - #import ipdb; ipdb.set_trace() - username = record.get('_1') #.lower() - permit_type = record['_6'] # RIA or Lic - mooring_no = record['MooringNo'] - vessel_name = record['VesName'] - #vessel_len = aup_records[0]['VesLen'] - rego_no = record['VesRego'] - - gs2 = GrepSearch2(username=username, mooring_no=mooring_no, path=self.path_csv) - try: - pers_no = gs2.get_persno() - if not pers_no and (username, pers_no) not in no_persno: - no_persno.append((username, pers_no)) - print(f'** NO PersNo FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - continue - except: - if (username, pers_no) not in no_persno: - no_persno.append((username, pers_no)) - print(f'NO PersNo FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - import ipdb; ipdb.set_trace() - continue - -# if (username, pers_no) not in no_email: -# no_email.append((username,pers_no)) -# print(f'NO EMAIL FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - - address, phone_home, phone_mobile, phone_work = gs2.get_address(pers_no) - if address is None: - address_not_found.append(pers_no) - print(f'ADDRESS NOT FOUND: {idx}, {pers_no}, {username} {permit_type}, {mooring_no}') - continue - email, username = gs2.get_email() - if not email: - continue - email = email.split(';')[0] - email = email.strip().replace('{','').replace('}','') - firstname = username.split(' ')[-1] - lastname = ' '.join(username.split(' ')[:-1]) - -# if (username, pers_no) not in no_email: -# no_email.append((username,pers_no)) -# print(f'NO EMAIL FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}') - - email_l, username_l = gs2.get_email(licencee=True) - if username == username_l: - print(f'{idx}, Licencee and AUP Same User: {username}, {mooring_no}') - continue - - #if not address_l and (username, pers_no, mooring_no) not in no_licencee and permit_type=='Lic': - if not email_l and permit_type=='Lic': - if (username, pers_no, mooring_no) not in no_licencee: - no_licencee.append((username, pers_no, mooring_no)) - permit_type = 'RIA' # override and save as RIA nominated mooring - print(f'{idx}, {pers_no}, {address}, {username}, {permit_type}, {mooring_no}') - elif permit_type=='Lic': - email_l = email_l.split(';')[0] - firstname_l = username.split(' ')[-1] - lastname_l = ' '.join(username.split(' ')[:-1]) - pers_no_l = gs2.get_persno(username_l) - #import ipdb; ipdb.set_trace() - address_l, phone_home_l, phone_mobile_l, phone_work_l = gs2.get_address(pers_no_l) - print(f'{idx}, {pers_no}, {address}, {username}, {permit_type}, {mooring_no}: Licencee - ({email_l} {username_l} {address_l}, {phone_mobile})') - if address_l is None: - address_l_not_found.append(pers_no_l) - print(f'ADDRESS_L NOT FOUND: {idx}, {pers_no}, {username}, {permit_type}, {mooring_no}: Licencee - ({pers_no_l})') - continue - else: - print(f'{idx}, {pers_no}, {address}, {username}, {permit_type}, {mooring_no}') - - - - vessel_overall_length, vessel_draft = gs2.get_vessel_size(rego_no) - vessel_type = 'other' - vessel_weight = Decimal( 0.0 ) - berth_mooring = '' - percentage = None # force user to set at renewal time - dot_name = gs2.get_dot_rego(rego_no) - - if self.test: - #import ipdb; ipdb.set_trace() - continue - - if Mooring.objects.filter(name=mooring_no).count()>0: - mooring = Mooring.objects.filter(name=mooring_no)[0] - elif mooring_no=='TB999': - print(f'{idx}, Mooring ignored: {mooring_no}') - continue - else: - import ipdb; ipdb.set_trace() - print(f'Mooring not found: {mooring_no}') - #mooring_bay = MooringBay.objects.get(name='Rottnest Island') - #mooring_bay = MooringBay.objects.get(name='Thomson Bay') - - # User email user - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=phone_mobile, phone_number=phone_home) - - country = Country.objects.get(printable_name='Australia') - try: - _address = Address.objects.get(user=user) - except: - _address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - user.residential_address = _address - user.postal_address = _address - user.save() - - #if permit_type=='Lic' and email_l: - bay_preferences_numbered = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"] - mooring_authorisation_preference = 'ria' - site_licensee_email = None - keep_existing_mooring = True - site_licensee = False - if permit_type=='Lic': - # Licensee email usee - items = address_l.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - try: - licensee = EmailUser.objects.get(email=email_l) - except: - licensee = EmailUser.objects.create(email=email_l, first_name=firstname_l, last_name=lastname_l, mobile_number=phone_mobile_l, phone_number=phone_home_l) - - country = Country.objects.get(printable_name='Australia') - try: - _address_l = Address.objects.get(user=licensee) - except: - _address_l, address_created_l = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=licensee) - licensee.residential_address = _address_l - licensee.postal_address = _address_l - licensee.save() - - mooring_authorisation_preference='site_licensee' - site_licensee_email=email_l - bay_preferences_numbered = None - site_licensee = True - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - vessel_ownership.dot_name = dot_name - vessel_ownership.save() - except ObjectDoesNotExist: - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage, dot_name=dot_name) - - if VesselDetails.objects.filter(vessel=vessel).count()>0: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - else: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - #berth_mooring=berth_mooring - ) - - proposal=AuthorisedUserApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.now(), - mooring_authorisation_preference=mooring_authorisation_preference, - site_licensee_email=site_licensee_email, - keep_existing_mooring=True, - mooring=mooring, - bay_preferences_numbered=bay_preferences_numbered, - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - #vessel_beam='', - vessel_weight=vessel_weight, - berth_mooring='home', - preferred_bay= mooring.mooring_bay, - percentage=percentage, - individual_owner=True, - dot_name=dot_name, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - proposed_issuance_approval={ - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - "vessel_ownership": [], - "mooring_on_approval": [] - }, - ) -#> "proposed_issuance_approval": { -#> "details": "dd", -#> "cc_email": null, -#> "mooring_id": 127, -#> "mooring_bay_id": 7, -#> "ria_mooring_name": "CB005", -#> "vesSEL_OWNERSHIP": [], -#> "mooring_on_approval": [] -#> }, -# -#> "berth_mooring": "home", -#> "dot_name": "abc", -#> "insurance_choice": "over_ten", -#> "preferred_bay": null, -#> "silent_elector": null, -#> "mooring_authorisation_preference": "ria", -#> "bay_preferences_numbered": "[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\"]", - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Authorised User Permit - Migrated Application', - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() - - #approval = WaitingListAllocation.objects.create( - approval = AuthorisedUserPermit.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - - moa = MooringOnApproval.objects.create( - approval=approval, - mooring=mooring, - sticker=None, - site_licensee=site_licensee, - #end_date=expiry_date - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - address_list.append(_address.id) - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - aup_list.append(proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - #date_invited = '' - aup_data.append([pers_no, email, user.dob, rego_no, mooring.name, 'LIC']) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - -# print(f'Address.objects.get(id__in={address_list}).delete()') -# print(f'EmailUser.objects.get(id__in={user_list}).delete()') -# print(f'Vessel.objects.get(id__in={vessel_list}).delete()') -# print(f'Owner.objects.get(id__in={owner_list}).delete()') -# print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') -# print(f'VesselDetails.objects.get(id__in={details_list}).delete()') - print(f'AuthorisedUserApplication.objects.get(id__in={aup_list}).delete()') -# print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') - print(f'AuthorisedUserPermit.objects.get(id__in={approval_list}).delete()') - print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') -# print(f'count_no_mooring: {count_no_mooring}') - print(f'no_records: {no_records}') - print(f'fnames_records: {fnames}') - print(f'no_persno: {no_persno}') - print(f'no_email: {no_email}') - print(f'no_licencee: {no_licencee}') - print(f'address_not_found: {address_not_found}') - print(f'address_l_not_found: {address_l_not_found}') - - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - #def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - def __init__(self, search_str1, path): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value in record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - _files = self.get_files(self.search_str1) - files=[ - - self.path + os.sep + 'Auth_Users___Surname___with_L.json', - self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'Vessel___People___All.json', - self.path + os.sep + 'PeopleName.json', - - self.path + os.sep + 'Vessel___People___All.json', - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleLicType.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'HINCurrent.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'Vessel___Vessel_Name___Current.json', - - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - #import ipdb; ipdb.set_trace() - record = find(key1, self.search_str1, f_json) - if not record or (key2=='_1' and len(record['_1'].split(',')) <= 1): - continue - #print('****************' + fname) - return record, fname - return None, None - -class GrepSearch2(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, username, mooring_no, path): - self.path = path - self.username = username.replace(' ', ' ') - self.mooring_no = mooring_no - - def _get_result1(self): - ''' Read all files in directory ''' - cmd = f"grep -r '{self.username}' {self.path} | grep \"'MooringNo': '{self.mooring_no}'\" | grep FirstNameL" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - #import ipdb; ipdb.set_trace() - res = output.decode('utf-8') - firstname_l = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['FirstNameL'] - lastname_l = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['LastNameL'] - username_l = '{} {}'.format(lastname_l, firstname_l) - return username_l - - except: - return None - - def _get_result2(self): - ''' Read all files in directory ''' - cmd = "grep -r '{self.username}' {self.path} | grep \"'MooringNo': '{self.mooring_no}'\" | grep '_12'" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - #import ipdb; ipdb.set_trace() - res = output.decode('utf-8') - username_l = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['_12'] - return username_l - - except: - return None - - - def get_email(self, licencee=False): - ''' Read all files in directory - Get Licensee email''' - if licencee: - username = self._get_result1() - if not username: - username = self._get_result2() - #import ipdb; ipdb.set_trace() - if not username: - #import ipdb; ipdb.set_trace() - return None, None - else: - username = self.username - - cmd = f"grep -r '{username}' {self.path} | grep '@'" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - #import ipdb; ipdb.set_trace() - try: - res = output.decode('utf-8') - email = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['EMail'] - email = email.lower().strip('>').strip('<').strip() - except: - email = None - - #import ipdb; ipdb.set_trace() - return email, username - - def get_persno(self, username=None): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - #cmd = "grep -r '{}' {} | grep -v PersNoL | grep 'PersNo'".format(self.username, self.path) - #cmd = "grep -ir '{}' {}/Auth_Users___*.* | grep -v PersNoL | grep 'PersNo'".format(self.username, self.path) - if not username: - username = self.username - - cmd = f"grep -ir '{username}' {self.path}/Auth_Users___*.* {self.path}/People___PersNo.csv {self.path}/People___Surname.csv | grep -v PersNoU | grep -v PersNoL | grep PersNo" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - res = output.decode('utf-8') - persno = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['PersNo'] - #persno = ast.literal_eval(res.splitlines()[0])['PersNo'] - except: - persno = None - - return persno - - def get_vessel_size(self, rego_no): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - cmd = f"grep -ir '{rego_no}' {self.path} | grep RegLength1 | grep Draft1" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - res = output.decode('utf-8') - ves_length = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['RegLength1'] - ves_draft = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['Draft1'] - if ves_length == 'None': - ves_length = 0.0 - if ves_draft == 'None': - ves_draft = 0.0 - except: - ves_length = 0.0 - ves_draft = 0.0 - - return Decimal(ves_length), Decimal(ves_draft) - - def get_dot_rego(self, rego_no): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - cmd = f"grep -ir '{rego_no}' {self.path} | grep DoTRego1" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: - res = output.decode('utf-8') - dot_rego = ast.literal_eval(res.splitlines()[0].split('csv:')[1])['DoTRego1'] - except: - dot_rego = None - - return dot_rego - - - def get_address(self, pers_no): - ''' Read all files in directory - Get PersNo''' - #import ipdb; ipdb.set_trace() - cmd = f"grep -ir \"'PersNo': '{pers_no}'\" {self.path} | grep \"'_1'\" | grep UserName| grep Phone" - ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) - output = ps.communicate()[0] - - try: -# res = output.decode('utf-8') -# lines = res.splitlines() -# for i in lines: -# line = ast.literal_eval(lines[i].split('csv:')[1]) -# address = line['_1'] -# if len(address.split(',')) > 2: -# phone_home = line['PhoneHome'] -# phone_mobile = line['PhoneMobile'] -# phone_work = line['PhoneWork'] -# break - - res = output.decode('utf-8') - lines = res.splitlines() - line = ast.literal_eval(lines[0].split('csv:')[1]) - address = line['_1'] - phone_home = line['PhoneHome'] - phone_mobile = line['PhoneMobile'] - phone_work = line['PhoneWork'] - - except: - address = None - phone_home = None - phone_mobile = None - phone_work = None - - return address, phone_home, phone_mobile, phone_work - diff --git a/mooringlicensing/utils/dcv_migrate.py b/mooringlicensing/utils/dcv_migrate.py deleted file mode 100644 index bd4a4ca78..000000000 --- a/mooringlicensing/utils/dcv_migrate.py +++ /dev/null @@ -1,246 +0,0 @@ -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist - -import os -import subprocess -import json -import datetime -from decimal import Decimal - -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - WaitingListApplication, - ProposalUserAction, - MooringBay, -) -from mooringlicensing.components.approvals.models import DcvOrganisation, DcvVessel, DcvPermit -from mooringlicensing.components.payments_ml.models import FeeSeason - -class DcvMigration(object): - ''' - from mooringlicensing.utils.dcv_migrate import DcvMigration, GrepSearch - DcvMigration(test=False) - - # cp mooringlicensing/management/templates/Attachment\ Template\ -\ DCVP.docx /var/www/mooringlicensing/media/approval_permit_template/Attachment_Template_-_DCVP.docx - ''' - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.test = test - - self.company = self.read_dict('Charter_Operators___Company.json') - self.surname = self.read_dict('Charter_Operators___Surname.json') - - self._company = [ - #... can be found in directory tests/test.json - ] - - self._surname = [ - #... can be found in directory tests/test.json - ] - - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def migrate(self): - - #fee_season = FeeSeason.objects.get(name='2020-2021') - fee_season = FeeSeason.objects.filter(name='2022 - 2023')[0] - start_date = datetime.date(2021, 9, 1) - end_date = datetime.date(2023,8,31) - - address_list = [] - user_list = [] - dcv_vessel_list = [] - dcv_organisation_list = [] - dcv_permit_list = [] - - added = [] - errors = [] - with transaction.atomic(): - #for idx, record in enumerate(self.company[43:], 43): - for idx, record in enumerate(self.company, 1): - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - email = record.get('EMail').split(';')[0].strip().lower() - org_name = record.get('Company') if record.get('Company') != 'None' else None - num_vessels = int(record.get('NoVessel')) - address = record.get('_1') - phone_no = record.get('PhoneHome') - mobile_no = record.get('PhoneMobile') - worke_no = record.get('PhoneWork') - - surname_record = self.search('PersNo', pers_no, self.surname) - - username = surname_record.get('_2').lower() - firstname = username.split(' ')[-1].title() - lastname = ' '.join(username.split(' ')[:-1]).title() - - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {state}, {postcode}') - if self.test: - continue - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) - - country = Country.objects.get(printable_name='Australia') - address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - user.residential_address = address - user.postal_address = address - user.save() - - dcv_organisation = None - if org_name: - try: - dcv_organisation = DcvOrganisation.objects.get(name=org_name) - except ObjectDoesNotExist: - dcv_organisation = DcvOrganisation.objects.create(name=org_name) - #import ipdb; ipdb.set_trace() - - - for idx,i in enumerate(['_3', '_11', '_12', '_13', '_14'], 1): - items = surname_record[i].split('-') - rego_no = items[0].strip() - vessel_name = items[1].strip() - #print(surname[10][i]) - print(f'\t{idx}, {rego_no}, {vessel_name}') - - - try: - dcv_vessel = DcvVessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - dcv_vessel = DcvVessel.objects.create( - rego_no = rego_no, - vessel_name = vessel_name, - dcv_organisation = dcv_organisation, - ) - - dcv_permit = DcvPermit.objects.create( - submitter = user, - lodgement_datetime = datetime.datetime.now(datetime.timezone.utc), - fee_season = fee_season, - start_date = start_date, - end_date = end_date, - dcv_vessel = dcv_vessel, - dcv_organisation =dcv_organisation, - migrated = True - ) - #import ipdb; ipdb.set_trace() - dcv_permit.generate_dcv_permit_doc() - - dcv_vessel_list.append(dcv_vessel.id) - dcv_permit_list.append(dcv_permit.id) - - if idx == num_vessels: - break - - address_list.append(address.id) - user_list.append(user.id) - if dcv_organisation: - dcv_organisation_list.append(dcv_organisation.id) - - added.append(dcv_permit.id) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - - print(f'Address.objects.get(id__in={address_list}).delete()') - print(f'EmailUser.objects.get(id__in={user_list}).delete()') - print(f'DcvVessel.objects.get(id__in={dcv_vessel_list}).delete()') - print(f'DcvPermit.objects.get(id__in={dcv_permit_list}).delete()') - print(f'DcvOrganisation.objects.get(id__in={dcv_organisation_list}).delete()') - - -def clear_migrated_dcv(): - pass - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - #files = self.get_files(self.search_str1) - files=[ - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - return record - - diff --git a/mooringlicensing/utils/excel_export/__init__.py b/mooringlicensing/utils/excel_export/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/mooringlicensing/utils/excel_export/annualadmission_to_excel.py b/mooringlicensing/utils/excel_export/annualadmission_to_excel.py deleted file mode 100644 index 10a68d3b3..000000000 --- a/mooringlicensing/utils/excel_export/annualadmission_to_excel.py +++ /dev/null @@ -1,13 +0,0 @@ -import pandas as pd - - -def write(filename): - """ - from mooringlicensing.utils.excel_export.annualadmission_to_excel import write - df = write() - """ - df = pd.read_csv(filename) - df.to_excel('annualadmission.xlsx', index=0) - return df - - diff --git a/mooringlicensing/utils/excel_export/authusers_to_excel.py b/mooringlicensing/utils/excel_export/authusers_to_excel.py deleted file mode 100644 index b175f3830..000000000 --- a/mooringlicensing/utils/excel_export/authusers_to_excel.py +++ /dev/null @@ -1,195 +0,0 @@ -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - AuthorisedUserApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval - -import pandas as pd - - -PROPOSAL_FIELDS = [ - #proposal_type_id - #submitter - 'lodgement_number', - 'lodgement_date', - 'migrated', - 'mooring_authorisation_preference', - 'site_licensee_email', - 'keep_existing_mooring', - 'bay_preferences_numbered', - #vessel_details - #vessel_ownership - 'rego_no', - 'vessel_type', - 'vessel_name', - 'vessel_length', - 'vessel_draft', - #vessel_beam='', - 'vessel_weight', - 'berth_mooring', - #preferred_bay - 'percentage', - 'individual_owner', - 'processing_status', - 'customer_status', - #allocated_mooring -# proposed_issuance_approval={ -# "start_date": start_date.strftime('%d/%m/%Y'), -# "expiry_date": expiry_date.strftime('%d/%m/%Y'), -# "details": None, -# "cc_email": None, -# "mooring_id": mooring.id, -# "ria_mooring_name": mooring.name, -# "mooring_bay_id": mooring.mooring_bay.id, -# "vessel_ownership": [{ -# "dot_name": vessel_dot -# }], -# "mooring_on_approval": [] -# }, - 'date_invited', - 'dot_name', -] - -SUBMITTER_FIELDS = [ -# #'submitter__' - 'submitter__id', - 'submitter__email', - 'submitter__first_name', - 'submitter__last_name', - 'submitter__dob', - 'submitter__phone_number', - 'submitter__mobile_number', - 'submitter__fax_number', -] - -SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS = [ - 'submitter__residential_address__line1', - 'submitter__residential_address__line2', - 'submitter__residential_address__line3', - 'submitter__residential_address__locality', - 'submitter__residential_address__state', - 'submitter__residential_address__postcode', - 'submitter__residential_address__country', -] - -SUBMITTER_POSTAL_ADDRESS_FIELDS = [ - 'submitter__postal_address__line1', - 'submitter__postal_address__line2', - 'submitter__postal_address__line3', - 'submitter__postal_address__locality', - 'submitter__postal_address__state', - 'submitter__postal_address__postcode', - 'submitter__postal_address__country', -] - - -APPROVAL_FIELDS = [ - 'approval__id', - 'approval__lodgement_number', - 'approval__status', - 'approval__issue_date', - 'approval__start_date', - 'approval__expiry_date', - 'approval__wla_queue_date', - #'approval__migrated', - 'approval__export_to_mooring_booking', - #'approval__mooringonapproval_set', -] - - -VESSEL_DETAILS_FIELDS = [ -# 'vessel_details__' - 'vessel_details__id', - 'vessel_details__vessel_type', - 'vessel_details__vessel_name', - 'vessel_details__vessel_length', - 'vessel_details__vessel_draft', - 'vessel_details__vessel_beam', - 'vessel_details__vessel_weight', - 'vessel_details__berth_mooring', - #'vessel_details__created', - #'vessel_details__updated', - 'vessel_details__exported', - 'vessel_details__vessel__rego_no', - #'vessel_details__vessel__blocking_owner__' -] - -VESSEL_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__id', - 'vessel_details__vessel__blocking_owner__percentage', - 'vessel_details__vessel__blocking_owner__dot_name', - 'vessel_details__vessel__blocking_owner__owner__emailuser__email', -] - -VESSEL_COMPANY_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__company_ownership__company__name', -] - -VESSEL_PREFERRED_BAY_FIELDS = [ - 'preferred_bay__name', -] - -VESSEL_MOORING_FIELDS = [ - 'mooring__name', - #'allocated_mooring__name', -] - -FEESEASON_FIELDS = [ - 'fee_season__name', -] - -def write(): - """ - from mooringlicensing.utils.excel_export.authusers_to_excel import write - df = write() - """ - fields = PROPOSAL_FIELDS - fields += SUBMITTER_FIELDS - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - fields += APPROVAL_FIELDS - fields += VESSEL_DETAILS_FIELDS - fields += VESSEL_OWNER_FIELDS - fields += VESSEL_COMPANY_OWNER_FIELDS - fields += VESSEL_PREFERRED_BAY_FIELDS - fields += VESSEL_MOORING_FIELDS - #fields += FEESEASON_FIELDS - - #import ipdb; ipdb.set_trace() - authusers_qs = AuthorisedUserApplication.objects.filter(migrated=True).values_list(*fields) - #print(fields) - - df = pd.DataFrame(authusers_qs, columns=fields) - if not df['lodgement_date'].empty: - df['lodgement_date'] = df['lodgement_date'].dt.tz_localize(None) # remove timezone for excel output - if not df['approval__issue_date'].empty: - df['approval__issue_date'] = df['approval__issue_date'].dt.tz_localize(None) # remove timezone for excel output - df.to_excel('aua.xlsx', index=0) - return df - - -def get_fields(model): - """ - from mooringlicensing.utils.excel_export.authusers_to_excel import get_fields - get_fields(MooringLicence) - """ - fk=[] - #for field in MooringLicenceApplication._meta.concrete_fields: - for field in model._meta.concrete_fields: - if field.get_internal_type() == 'ForeignKey': - #print(f'{field.name}__') - fk.append(f'\'{field.name}__\'',) - else: - print(f'\'{field.name}\',') - - for field in fk: - print(field) - diff --git a/mooringlicensing/utils/excel_export/dcvpermits_to_excel.py b/mooringlicensing/utils/excel_export/dcvpermits_to_excel.py deleted file mode 100644 index fe0cde222..000000000 --- a/mooringlicensing/utils/excel_export/dcvpermits_to_excel.py +++ /dev/null @@ -1,104 +0,0 @@ -from mooringlicensing.components.proposals.models import ( - Proposal -) -from mooringlicensing.components.approvals.models import DcvPermit, DcvVessel, DcvOrganisation - -import pandas as pd - - -DCV_PERMIT_FIELDS = [ - 'id', - 'lodgement_number', - 'lodgement_datetime', - 'start_date', - 'end_date', - 'migrated', -] - -SUBMITTER_FIELDS = [ -# #'submitter__' - 'submitter__id', - 'submitter__email', - 'submitter__first_name', - 'submitter__last_name', - 'submitter__dob', - 'submitter__phone_number', - 'submitter__mobile_number', - 'submitter__fax_number', -] - -SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS = [ - 'submitter__residential_address__line1', - 'submitter__residential_address__line2', - 'submitter__residential_address__line3', - 'submitter__residential_address__locality', - 'submitter__residential_address__state', - 'submitter__residential_address__postcode', - 'submitter__residential_address__country', -] - -SUBMITTER_POSTAL_ADDRESS_FIELDS = [ - 'submitter__postal_address__line1', - 'submitter__postal_address__line2', - 'submitter__postal_address__line3', - 'submitter__postal_address__locality', - 'submitter__postal_address__state', - 'submitter__postal_address__postcode', - 'submitter__postal_address__country', -] - -DCV_VESSEL_FIELDS = [ - 'dcv_vessel__vessel_name', - 'dcv_vessel__rego_no', -] - -DCV_ORGANISATION_FIELDS = [ - 'dcv_organisation__name', - 'dcv_organisation__abn', -] - -FEESEASON_FIELDS = [ - 'fee_season__name', -] - -def write(): - """ - from mooringlicensing.utils.excel_export.dvcpermits_to_excel import write - df = write() - """ - fields = DCV_PERMIT_FIELDS - fields += SUBMITTER_FIELDS - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - fields += DCV_VESSEL_FIELDS - fields += DCV_ORGANISATION_FIELDS - fields += FEESEASON_FIELDS - - #import ipdb; ipdb.set_trace() - dcvp_qs = DcvPermit.objects.filter(migrated=True).values_list(*fields) - #print(fields) - - df = pd.DataFrame(dcvp_qs, columns=fields) - if not df['lodgement_datetime'].empty: - df['lodgement_datetime'] = df['lodgement_datetime'].dt.tz_localize(None) # remove timezone for excel output - df.to_excel('dcv.xlsx', index=0) - return df - - -def get_fields(model): - """ - from mooringlicensing.utils.excel_export.dvcpermits_to_excel import get_fields - get_fields(MooringLicence) - """ - fk=[] - #for field in MooringLicenceApplication._meta.concrete_fields: - for field in model._meta.concrete_fields: - if field.get_internal_type() == 'ForeignKey': - #print(f'{field.name}__') - fk.append(f'\'{field.name}__\'',) - else: - print(f'\'{field.name}\',') - - for field in fk: - print(field) - diff --git a/mooringlicensing/utils/excel_export/mla_to_excel.py b/mooringlicensing/utils/excel_export/mla_to_excel.py deleted file mode 100644 index f1c12b3f3..000000000 --- a/mooringlicensing/utils/excel_export/mla_to_excel.py +++ /dev/null @@ -1,189 +0,0 @@ -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval - -import pandas as pd - - -PROPOSAL_FIELDS = [ - #proposal_type_id - #submitter - 'lodgement_number', - 'lodgement_date', - 'migrated', - #vessel_details - #vessel_ownership - 'rego_no', - 'vessel_type', - 'vessel_name', - 'vessel_length', - 'vessel_draft', - #vessel_beam='', - 'vessel_weight', - 'berth_mooring', - #preferred_bay - 'percentage', - 'individual_owner', - 'processing_status', - 'customer_status', - #allocated_mooring -# proposed_issuance_approval={ -# "start_date": start_date.strftime('%d/%m/%Y'), -# "expiry_date": expiry_date.strftime('%d/%m/%Y'), -# "details": None, -# "cc_email": None, -# "mooring_id": mooring.id, -# "ria_mooring_name": mooring.name, -# "mooring_bay_id": mooring.mooring_bay.id, -# "vessel_ownership": [{ -# "dot_name": vessel_dot -# }], -# "mooring_on_approval": [] -# }, - 'date_invited', - 'dot_name', -] - -SUBMITTER_FIELDS = [ -# #'submitter__' - 'submitter__id', - 'submitter__email', - 'submitter__first_name', - 'submitter__last_name', - 'submitter__dob', - 'submitter__phone_number', - 'submitter__mobile_number', - 'submitter__fax_number', -] - -SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS = [ - 'submitter__residential_address__line1', - 'submitter__residential_address__line2', - 'submitter__residential_address__line3', - 'submitter__residential_address__locality', - 'submitter__residential_address__state', - 'submitter__residential_address__postcode', - 'submitter__residential_address__country', -] - -SUBMITTER_POSTAL_ADDRESS_FIELDS = [ - 'submitter__postal_address__line1', - 'submitter__postal_address__line2', - 'submitter__postal_address__line3', - 'submitter__postal_address__locality', - 'submitter__postal_address__state', - 'submitter__postal_address__postcode', - 'submitter__postal_address__country', -] - - -APPROVAL_FIELDS = [ - 'approval__id', - 'approval__lodgement_number', - 'approval__status', - 'approval__issue_date', - 'approval__start_date', - 'approval__expiry_date', - 'approval__wla_queue_date', - #'approval__migrated', - 'approval__export_to_mooring_booking', -] - -VESSEL_DETAILS_FIELDS = [ -# 'vessel_details__' - 'vessel_details__id', - 'vessel_details__vessel_type', - 'vessel_details__vessel_name', - 'vessel_details__vessel_length', - 'vessel_details__vessel_draft', - 'vessel_details__vessel_beam', - 'vessel_details__vessel_weight', - 'vessel_details__berth_mooring', - #'vessel_details__created', - #'vessel_details__updated', - 'vessel_details__exported', - 'vessel_details__vessel__rego_no', - #'vessel_details__vessel__blocking_owner__' -] - -VESSEL_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__id', - 'vessel_details__vessel__blocking_owner__percentage', - 'vessel_details__vessel__blocking_owner__dot_name', - 'vessel_details__vessel__blocking_owner__owner__emailuser__email', -] - -VESSEL_COMPANY_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__company_ownership__company__name', -] - -VESSEL_PREFERRED_BAY_FIELDS = [ - 'preferred_bay__name', -] - -VESSEL_MOORING_FIELDS = [ - 'mooring__name', - 'allocated_mooring__name', -] - -FEESEASON_FIELDS = [ - 'fee_season__name', -] - -def write(): - """ - from mooringlicensing.utils.excel_export.mla_to_excel import write - df = write() - """ - fields = PROPOSAL_FIELDS - fields += SUBMITTER_FIELDS - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - fields += APPROVAL_FIELDS - fields += VESSEL_DETAILS_FIELDS - fields += VESSEL_OWNER_FIELDS - fields += VESSEL_COMPANY_OWNER_FIELDS - fields += VESSEL_PREFERRED_BAY_FIELDS - fields += VESSEL_MOORING_FIELDS - fields += FEESEASON_FIELDS - - #import ipdb; ipdb.set_trace() - #mla_qs = MooringLicenceApplication.objects.filter()[:10].values_list(*fields) - mla_qs = MooringLicenceApplication.objects.filter(migrated=True).values_list(*fields) - #print(fields) - - df = pd.DataFrame(mla_qs, columns=fields) - if not df['lodgement_date'].empty: - df['lodgement_date'] = df['lodgement_date'].dt.tz_localize(None) # remove timezone for excel output - if not df['approval__issue_date'].empty: - df['approval__issue_date'] = df['approval__issue_date'].dt.tz_localize(None) # remove timezone for excel output - df.to_excel('mla.xlsx', index=0) - return df - - -def get_fields(model): - """ - from mooringlicensing.utils.excel_export.mla_to_excel import get_fields - get_fields(MooringLicence) - """ - fk=[] - #for field in MooringLicenceApplication._meta.concrete_fields: - for field in model._meta.concrete_fields: - if field.get_internal_type() == 'ForeignKey': - #print(f'{field.name}__') - fk.append(f'\'{field.name}__\'',) - else: - print(f'\'{field.name}\',') - - for field in fk: - print(field) - diff --git a/mooringlicensing/utils/excel_export/wla_to_excel.py b/mooringlicensing/utils/excel_export/wla_to_excel.py deleted file mode 100644 index 2befe689c..000000000 --- a/mooringlicensing/utils/excel_export/wla_to_excel.py +++ /dev/null @@ -1,167 +0,0 @@ -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - WaitingListApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory - -import pandas as pd - - -PROPOSAL_FIELDS = [ - #proposal_type_id - #submitter - 'lodgement_number', - 'lodgement_date', - 'migrated', - #vessel_details - #vessel_ownership - 'rego_no', - 'vessel_type', - 'vessel_name', - 'vessel_length', - 'vessel_draft', - #vessel_beam='', - 'vessel_weight', - 'berth_mooring', - #preferred_bay - 'percentage', - 'individual_owner', - 'processing_status', - 'customer_status', - #allocated_mooring - #'date_invited', - #'dot_name', -] - - -SUBMITTER_FIELDS = [ -# #'submitter__' - 'submitter__id', - 'submitter__email', - 'submitter__first_name', - 'submitter__last_name', - 'submitter__dob', - 'submitter__phone_number', - 'submitter__mobile_number', - 'submitter__fax_number', -] - -SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS = [ - 'submitter__residential_address__line1', - 'submitter__residential_address__line2', - 'submitter__residential_address__line3', - 'submitter__residential_address__locality', - 'submitter__residential_address__state', - 'submitter__residential_address__postcode', - 'submitter__residential_address__country', -] - -SUBMITTER_POSTAL_ADDRESS_FIELDS = [ - 'submitter__postal_address__line1', - 'submitter__postal_address__line2', - 'submitter__postal_address__line3', - 'submitter__postal_address__locality', - 'submitter__postal_address__state', - 'submitter__postal_address__postcode', - 'submitter__postal_address__country', -] - - -APPROVAL_FIELDS = [ - 'approval__id', - 'approval__lodgement_number', - 'approval__status', - 'approval__internal_status', - 'approval__issue_date', - 'approval__start_date', - 'approval__expiry_date', - 'approval__wla_queue_date', - 'approval__wla_order', -] - -VESSEL_DETAILS_FIELDS = [ -# 'vessel_details__' - 'vessel_details__id', - 'vessel_details__vessel_type', - 'vessel_details__vessel_name', - 'vessel_details__vessel_length', - 'vessel_details__vessel_draft', - 'vessel_details__vessel_beam', - 'vessel_details__vessel_weight', - 'vessel_details__berth_mooring', - #'vessel_details__created', - #'vessel_details__updated', - 'vessel_details__exported', - 'vessel_details__vessel__rego_no', - #'vessel_details__vessel__blocking_owner__' -] - -VESSEL_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__id', - 'vessel_details__vessel__blocking_owner__percentage', - 'vessel_details__vessel__blocking_owner__dot_name', - 'vessel_details__vessel__blocking_owner__owner__emailuser__email', -] - -VESSEL_PREFERRED_BAY_FIELDS = [ - 'preferred_bay__name', -] - -VESSEL_MOORING_FIELDS = [ - 'mooring__name', - #'allocated_mooring__name', -] - -def write(): - """ - from mooringlicensing.utils.excel_export.wla_to_excel import write - df = write() - """ - fields = PROPOSAL_FIELDS - fields += SUBMITTER_FIELDS - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - fields += APPROVAL_FIELDS - fields += VESSEL_DETAILS_FIELDS - fields += VESSEL_OWNER_FIELDS - fields += VESSEL_PREFERRED_BAY_FIELDS - fields += VESSEL_MOORING_FIELDS - #fields += FEESEASON_FIELDS - - #import ipdb; ipdb.set_trace() - wla_qs = WaitingListApplication.objects.filter(migrated=True).values_list(*fields) - #print(fields) - - df = pd.DataFrame(wla_qs, columns=fields) - if not df['lodgement_date'].empty: - df['lodgement_date'] = df['lodgement_date'].dt.tz_localize(None) # remove timezone for excel output - if not df['approval__issue_date'].empty: - df['approval__issue_date'] = df['approval__issue_date'].dt.tz_localize(None) # remove timezone for excel output - df.to_excel('wla.xlsx', index=0) - return df - - -def get_fields(model): - """ - from mooringlicensing.utils.excel_export.wla_to_excel import get_fields - get_fields(MooringLicence) - """ - fk=[] - #for field in MooringLicenceApplication._meta.concrete_fields: - for field in model._meta.concrete_fields: - if field.get_internal_type() == 'ForeignKey': - #print(f'{field.name}__') - fk.append(f'\'{field.name}__\'',) - else: - print(f'\'{field.name}\',') - - for field in fk: - print(field) - diff --git a/mooringlicensing/utils/json_to_csv.py b/mooringlicensing/utils/json_to_csv.py deleted file mode 100644 index ce67ccaf0..000000000 --- a/mooringlicensing/utils/json_to_csv.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import json -import csv -from os import listdir -from os.path import isfile, join - -class JsonToCsv(): - """ - from mooringlicensing.utils.json_to_csv import JsonToCsv - JsonToCsv().convert() - """ - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021_auth_users/'): - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021/'): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes/', csv_path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv/'): - self.path = path - self.csv_path = csv_path - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def convert(self): - #import ipdb; ipdb.set_trace() - files = [f for f in listdir(self.path) if isfile(join(self.path, f))] - - if os.path.exists(self.csv_path): - raise Exception('CVS Path already exists: {self.csv_path}') - else: - os.makedirs(self.csv_path) - - for filename in files: - try: - _list = self.read_dict(filename) - if _list: - -# #csv_filename = '/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_csv/' + filename.replace('json','csv') -# csv_filename = '/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv/' + filename.replace('json','csv') -# with open(csv_filename, 'w') as f: -# dict_writer = csv.DictWriter(f, fieldnames=_list[0].keys()) -# dict_writer.writeheader() -# for row in _list: -# dict_writer.writerow(row) - - #csv_filename = '/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv/' + filename.replace('json','csv') - #csv_path = self.path + 'all_csv/' if self.path.endswith(os.sep) else self.path + os.sep + 'all_csv/' - #if not os.path.exists(self.csv_path): - # os.makedirs(self.csv_path) - - csv_filename = self.csv_path + filename.replace('json','csv') - with open(csv_filename, 'w') as f: - #csvwriter = csv.writer(f) - #import ipdb; ipdb.set_trace() - for row in _list: - f.write(str(row) + '\n') - - except Exception as e: - print(e) - print(filename) - import ipdb; ipdb.set_trace() - - - diff --git a/mooringlicensing/utils/make_unique.py b/mooringlicensing/utils/make_unique.py deleted file mode 100644 index 538268802..000000000 --- a/mooringlicensing/utils/make_unique.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -import json - -class MakeUnique(object): - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/wait_list'): - self.path = path - self.make_unique() - - def get_files(self): - ''' Read all files in directory ''' - files = [] - with os.scandir(self.path) as it: - for entry in it: - if entry.name.endswith(".json") and entry.is_file(): - files.append(entry.name) - return files - - def make_unique(self): - ''' Remove duplicate entries in json file and write to unique folder ''' - tmp_list = [] - for fname in self.get_files(): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - out_path = self.path + os.sep + 'unique' if not self.path.endswith(os.sep) else self.path + 'unique' - if not os.path.exists(out_path): - os.makedirs(out_path) - - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - print(f'Original Length: {len(f_json)}\tNew Length: {len(_list)}\t({fname})') - - with open(out_path + os.sep + fname, 'w') as f: - print(json.dumps(_list), file=f) - - tmp_list.append( {fname : _list[:3]} ) - - with open(out_path + os.sep + 'summary.json', 'w') as f: - print(json.dumps(tmp_list), file=f) - - #return tmp diff --git a/mooringlicensing/utils/ml_csvwriter.py b/mooringlicensing/utils/ml_csvwriter.py deleted file mode 100644 index ccc94cbc4..000000000 --- a/mooringlicensing/utils/ml_csvwriter.py +++ /dev/null @@ -1,100 +0,0 @@ -import csv -from datetime import datetime -import pytz -import os -# from ledger.settings_base import TIME_ZONE -from ledger_api_client.settings_base import TIME_ZONE -from mooringlicensing.settings import BASE_DIR -from mooringlicensing.components.approvals.models import ( - Approval, WaitingListAllocation, AnnualAdmissionPermit, - AuthorisedUserPermit, MooringLicence - ) - - -date_now = datetime.now(pytz.timezone(TIME_ZONE)).date() -date_now_str = date_now.strftime('%Y%m%d') - - -def write_wla(): - approval_type = WaitingListAllocation.code - approvals = WaitingListAllocation.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'WLA Queue Date', 'WLA Order', - 'First Name', 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessel Rego', 'Company', - ] - rows = [[wla.lodgement_number, wla.start_date, wla.expiry_date, wla.issue_date.strftime('%Y-%m-%d'), - wla.wla_queue_date.strftime('%Y-%m-%d') if wla.wla_queue_date else '', wla.wla_order, - wla.submitter_obj.first_name, wla.submitter_obj.last_name, wla.submitter_obj.residential_address.line1, - wla.submitter_obj.residential_address.locality, wla.submitter_obj.residential_address.state, - wla.submitter_obj.residential_address.country, wla.submitter_obj.residential_address.postcode, '', - wla.submitter_obj.phone_number, wla.submitter_obj.mobile_number, wla.submitter_obj.email, - wla.current_proposal.vessel_ownership.vessel.rego_no if wla.current_proposal.vessel_ownership else '', '', - ] for wla in WaitingListAllocation.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_aap(): - approval_type = AnnualAdmissionPermit.code - approvals = AnnualAdmissionPermit.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'First Name', - 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessel Rego', 'Company', - ] - rows = [[aap.lodgement_number, aap.start_date, aap.expiry_date, aap.issue_date.strftime('%Y-%m-%d'), - aap.submitter_obj.first_name, aap.submitter_obj.last_name, aap.submitter_obj.residential_address.line1, - aap.submitter_obj.residential_address.locality, aap.submitter_obj.residential_address.state, - aap.submitter_obj.residential_address.country, aap.submitter_obj.residential_address.postcode, '', - aap.submitter_obj.phone_number, aap.submitter_obj.mobile_number, aap.submitter_obj.email, - aap.current_proposal.vessel_ownership.vessel.rego_no if aap.current_proposal.vessel_ownership else '', '', - ] for aap in AnnualAdmissionPermit.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_aup(): - approval_type = AuthorisedUserPermit.code - approvals = AuthorisedUserPermit.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'Moorings', 'First Name', - 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessel Rego', 'Company', - ] - rows = [[aup.lodgement_number, aup.start_date, aup.expiry_date, aup.issue_date.strftime('%Y-%m-%d'), - ','.join(str(moa.mooring) for moa in aup.mooringonapproval_set.filter(mooring__mooring_licence__status='current')), - aup.submitter_obj.first_name, aup.submitter_obj.last_name, aup.submitter_obj.residential_address.line1, - aup.submitter_obj.residential_address.locality, aup.submitter_obj.residential_address.state, - aup.submitter_obj.residential_address.country, aup.submitter_obj.residential_address.postcode, '', - aup.submitter_obj.phone_number, aup.submitter_obj.mobile_number, aup.submitter_obj.email, - aup.current_proposal.vessel_ownership.vessel.rego_no if aup.current_proposal.vessel_ownership else '', '', - ] for aup in AuthorisedUserPermit.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_ml(): - approval_type = MooringLicence.code - approvals = MooringLicence.objects.filter(status='current') - header_row = ['Lodgement Number', 'Start Date', 'Expiry Date', 'Issue Date', 'Mooring', 'First Name', - 'Last Name', 'Address 1', 'Suburb', 'State', 'Country', 'Postcode', 'Postal Address', - 'Phone', 'Mobile', 'EMAIL', 'Vessels', - ] - rows = [[ml.lodgement_number, ml.start_date, ml.expiry_date, ml.issue_date.strftime('%Y-%m-%d'), - ml.mooring if hasattr(ml, 'mooring') else '', - ml.submitter_obj.first_name, ml.submitter_obj.last_name, ml.submitter_obj.residential_address.line1, - ml.submitter_obj.residential_address.locality, ml.submitter_obj.residential_address.state, - ml.submitter_obj.residential_address.country, ml.submitter_obj.residential_address.postcode, '', - ml.submitter_obj.phone_number, ml.submitter_obj.mobile_number, ml.submitter_obj.email, - ','.join(vessel.rego_no for vessel in ml.vessel_list), - ] for ml in MooringLicence.objects.filter(status='current')] - write_file(approval_type, approvals, header_row, rows) - -def write_file(approval_type, approvals, header_row, rows): - if not os.path.isdir(os.path.join(BASE_DIR, 'mooringlicensing', 'utils', 'csv')): - os.mkdir(os.path.join(BASE_DIR, 'mooringlicensing', 'utils', 'csv')) - filename = os.path.join(BASE_DIR, 'mooringlicensing', 'utils', 'csv', '{}_{}.csv'.format(approval_type, date_now_str)) - - with open(filename, 'w', newline='') as csvfile: - csvwriter = csv.writer(csvfile, delimiter=':', quotechar='|', quoting=csv.QUOTE_MINIMAL) - csvwriter.writerow(header_row) - for row in rows: - csvwriter.writerow(row) - -write_wla() -write_aap() -write_aup() -write_ml() - diff --git a/mooringlicensing/utils/mooring_licence_migrate.py b/mooringlicensing/utils/mooring_licence_migrate.py deleted file mode 100644 index b40929483..000000000 --- a/mooringlicensing/utils/mooring_licence_migrate.py +++ /dev/null @@ -1,482 +0,0 @@ -import os -import subprocess -import json -import csv -import datetime -from decimal import Decimal -from django.db import transaction, IntegrityError -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval - - -class MooringLicenceMigration(object): - ''' - from mooringlicensing.utils.mooring_licence_migrate import MooringLicenceMigration - MooringLicenceMigration(test=False) - ''' - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.test = test - - #self.waitlist = self.read_dict('Waitlist___Bay.json') - self.auth_users = self.read_dict('Licencees___Auth_User.json') - self.mooring_no = self.read_dict('Licencees___Mooring_No.json') - self.surname = self.read_dict('Licencees___Surname.json') - #self.vessel = self.read_dict('Vessel___HIN___Current.json') - self.vessel = self.read_dict('Vessel___Details_All.json') - self.owner = self.read_dict('Owner_Details.json') - self.allocated = self.read_dict('Admin___Application___Pers_No.json') - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - start_date = datetime.date(2021,8,31) - date_applied = '2021-08-31' - - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - - added = [] - errors = [] - count_no_mooring = 0 - - user_data = [] - vessel_data = [] - mla_data = [] - ml_all_data = [] - user_data.append(['PersNo', 'Email', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address']) - vessel_data.append(['Owner', 'Email', 'DoB', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring']) - mla.append(['PersNo', 'Email', 'DoB', 'Rego No', 'MooringNo', 'Date Invited']) - ml_all_data.append(['PersNo', 'Email', 'DoB', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring', 'MooringNo', 'Date Invited']) - with transaction.atomic(): - #for idx, record in enumerate(self.mooring_no, 1): - for idx, record in enumerate(self.mooring_no[257:260], 257): - #if idx==258: - # import ipdb; ipdb.set_trace() - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - address = record.get('_1') - mooring = record.get('MooringNo') - mooring_bay = record.get('_7') - username = record.get('UserName').lower() - rego_no = record.get('_4').split('-')[0].strip() - vessel_name = record.get('_4').split('-')[1].strip() - - owner_record = self.search('PersNoL', pers_no, self.owner) - if owner_record: - vessel_name = owner_record['VesName'] - rego_no = owner_record['VesRego'] - licence_pc = owner_record['LicPC'] - - surname_record = self.search('PersNo', pers_no, self.surname) - #email = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - #email_record.get('EMail').lower().strip() - #H1N = GrepSearch(pers_no, path=self.path).search('PersNo', 'H1N1') - email = surname_record['EMail'].split(',')[0].strip().lower() - if email=='{ronrouwenhorst@bigpond.com} ;arthur@technitemp.com': - errors.append(email) - continue - - HIN = surname_record['HIN1'] - - vessel_record = self.search('PersNo', pers_no, self.vessel) - if vessel_record: - vessel_dot = vessel_record.get('DoTRego1') - #email = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - vessel_overall_length = Decimal( vessel_record.get('RegLength1') ) - vessel_draft = Decimal( vessel_record.get('Draft1') ) - else: - vessel_dot = rego_no - #vessel_overall_length = Decimal( GrepSearch(pers_no, path=self.path).search('PersNo', 'RegLength1') ) - try: - vessel_overall_length = Decimal( surname_record.get('RegLength1') ) - except Exception as e: - vessel_overall_length = Decimal( 0.0 ) - vessel_draft = Decimal( 0.0 ) - - vessel_type = 'other' - vessel_weight = Decimal( 0.0 ) - - allocated_record = self.search('PersNo', pers_no, self.allocated) - try: - date_invited = datetime.datetime.strptime(allocated_record.get('DateAllocated'), '%Y-%m-%d').date() - except: - date_invited = None - - #email_record = GrepSearch(pers_no).search('PersNo', 'EMail') - phonemobile_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneMobile') - phonehome_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneHome') - - mobile_no = phonemobile_record.get('PhoneMobile') - firstname = username.split(' ')[-1].title() - lastname = ' '.join(username.split(' ')[:-1]).title() - - try: - phone_no = phonehome_record.get('PhoneHome') if phonehome_record else '' - #address = address_record.get('_1') - except: - import ipdb; ipdb.set_trace() - - #print(f'{idx}, {pers_no}, {username}') - #if self.test: - # import ipdb; ipdb.set_trace() - # continue - - -# mooring_record = self.search('_1', username, self.moorings) -# try: -# mooring = mooring_record['MooringNo'] -# vessel_name = mooring_record['VesName'] -# rego_no = mooring_record['VesRego'] -# except: -# count_no_mooring += 1 -# print(f'*************** {idx}, {pers_no}, {username}, {count_no_mooring}') -# continue -# #vessel_name = record.get('_11').split('-')[1].strip() -# #rego_no = record.get('_11').split('-')[0].strip() - - - # see mooringlicensing/utils/tests/mooring_names.txt - if Mooring.objects.filter(name=mooring).count()>0: - mooring = Mooring.objects.filter(name=mooring)[0] - else: - #import ipdb; ipdb.set_trace() - print(f'Mooring not found: {mooring}') - errors.append(dict(idx=idx, record=record)) - continue - - #mooring_bay = MooringBay.objects.get(name='Rottnest Island') - #mooring_bay = MooringBay.objects.get(name='Thomson Bay') - - # needed ?? - #date_applied = record.get('DateApplied') - #position_no = int(record.get('BayPosNo')) - #trim_no = record.get('TrimNo') - percentage = None # force user to set at renewal time - - - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {username}, {state}, {postcode}') - if self.test: - continue - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) - - country = Country.objects.get(printable_name='Australia') - address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - user.residential_address = address - user.postal_address = address - user.save() - - user_data.append([pers_no, email, user.dob, firstname, lastname, mobile_no, phone_no, address]) - - try: - vessel = Vessel.objects.get(rego_no=rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=rego_no) - - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - except ObjectDoesNotExist: - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage) - - try: - vessel_details = VesselDetails.objects.get(vessel=vessel) - except MultipleObjectsReturned: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - except ObjectDoesNotExist: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - berth_mooring='home' - ) - - berth_mooring = '' - vessel_data.append([pers_no, email, user.dob, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring]) - - proposal=MooringLicenceApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.now(), - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - #vessel_beam='', - vessel_weight=vessel_weight, - berth_mooring='home', - preferred_bay= mooring.mooring_bay, - percentage=percentage, - individual_owner=True, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - allocated_mooring=mooring, - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - "vessel_ownership": [{ - "dot_name": vessel_dot - }], - "mooring_on_approval": [] - }, - date_invited=None #date_invited, - dot_name=vessel_dot, - ) - - date_invited = '' - mla_data.append([pers_no, email, user.dob, rego_no, mooring.mooring_bay, date_invited]) - -#> "proposed_issuance_approval": { -#> "details": "dd", -#> "cc_email": null, -#> "mooring_id": 127, -#> "mooring_bay_id": 7, -#> "ria_mooring_name": "CB005", -#> "vessel_ownership": [], -#> "mooring_on_approval": [] -#> }, -# -#> "berth_mooring": "home", -#> "dot_name": "abc", -#> "insurance_choice": "over_ten", -#> "preferred_bay": null, -#> "silent_elector": null, -#> "mooring_authorisation_preference": "ria", -#> "bay_preferences_numbered": "[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\"]", - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Mooring Site Licence - Migrated Application', - ) - - #approval = WaitingListAllocation.objects.create( - approval = MooringLicence.objects.create( - status='current', - #internal_status=None, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - export_to_mooring_booking=True, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - proposal.allocated_mooring.mooring_licence = approval - proposal.allocated_mooring.save() - - vooa = VesselOwnershipOnApproval.objects.create( - approval=approval, - vessel_ownership=vessel_ownership, - ) -# moa = MooringOnApproval.objects.create( -# approval=approval, -# mooring=mooring, -# sticker=None, -# site_licensee=True, # ??? -# end_date=expiry_date -# ) - - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - address_list.append(address.id) - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - ml_all_data.append([pers_no, email, user.dob, firstname, lastname, mobile_no, phone_no, address, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring, mooring.mooring_bay, date_invited]) - -# except IntegrityError as e: -# errors.append(repr(e)) -# continue - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - - with open('ml_all.csv', 'w') as f: - writer = csv.writer(f) - #writer.writerow(header) - writer.writerows(ml_all_data) - -# print(f'Address.objects.get(id__in={address_list}).delete()') -# print(f'EmailUser.objects.get(id__in={user_list}).delete()') -# print(f'Vessel.objects.get(id__in={vessel_list}).delete()') -# print(f'Owner.objects.get(id__in={owner_list}).delete()') -# print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') -# print(f'VesselDetails.objects.get(id__in={details_list}).delete()') -# print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') -# print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') -# print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') - print(f'errors: {errors}') - - -def clear_record(): - Address.objects.last().delete() - EmailUser.objects.last().delete() - Vessel.objects.last().delete() - Owner.objects.last().delete() - VesselOwnership.objects.last().delete() - VesselDetails.objects.last().delete() - WaitingListApplication.objects.last().delete() - ProposalUserAction.objects.last().delete() - WaitingListAllocation.objects.last().delete() - ApprovalHistory.objects.last().delete() - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' - #_files = self.get_files(self.search_str1) - files=[ - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Admin___EContacts___5_Licencee.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - #print(f'fname: {fname}, key2: {key2}') - return record - - diff --git a/mooringlicensing/utils/mooring_licence_migrate_write_csv.py b/mooringlicensing/utils/mooring_licence_migrate_write_csv.py deleted file mode 100644 index d19ee0c5f..000000000 --- a/mooringlicensing/utils/mooring_licence_migrate_write_csv.py +++ /dev/null @@ -1,301 +0,0 @@ -#TODO: determine if this script is required or not - fix/remove as needed - -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - ProposalUserAction, - Mooring, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval - -import sys -import csv -import pandas as pd - -MODELS = [ - 'MooringLicenceApplication', - 'Mooring', - 'MooringBay', -] - -PROPOSAL_FIELDS = [ - 'id', - #'proposal_type__code', - #'assessor_data', - #'comment_data', - #'proposed_issuance_approval', - 'customer_status', - 'lodgement_number', - #'lodgement_sequence', - 'lodgement_date', - 'processing_status', - #'prev_processing_status', - #'proposed_decline_status', - 'title', - 'approval_level', - #'approval_comment', - 'migrated', - 'rego_no', - 'vessel_id', - 'vessel_type', - 'vessel_name', - 'vessel_length', - 'vessel_draft', - 'vessel_beam', - 'vessel_weight', - 'berth_mooring', - 'dot_name', - 'percentage', - 'individual_owner', - 'company_ownership_percentage', - 'company_ownership_name', - #'insurance_choice', - #'silent_elector', - 'mooring_authorisation_preference', - 'bay_preferences_numbered', - #'site_licensee_email', - #'endorser_reminder_sent', - #'date_invited', - #'invitee_reminder_sent', - #'temporary_document_collection_id', - #'keep_existing_mooring', - #'keep_existing_vessel', - #'auto_approve', - #'null_vessel_on_create', - 'proposal', - #'uuid', -] - -SUBMITTER_FIELDS = [ -# #'submitter__' - 'submitter__id', - 'submitter__email', - 'submitter__first_name', - 'submitter__last_name', - 'submitter__dob', - 'submitter__phone_number', - 'submitter__mobile_number', - 'submitter__fax_number', -] - -SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS = [ - 'submitter__residential_address__line1', - 'submitter__residential_address__line2', - 'submitter__residential_address__line3', - 'submitter__residential_address__locality', - 'submitter__residential_address__state', - 'submitter__residential_address__postcode', - 'submitter__residential_address__country', -] - -SUBMITTER_POSTAL_ADDRESS_FIELDS = [ - 'submitter__postal_address__line1', - 'submitter__postal_address__line2', - 'submitter__postal_address__line3', - 'submitter__postal_address__locality', - 'submitter__postal_address__state', - 'submitter__postal_address__postcode', - 'submitter__postal_address__country', -] - -APPROVAL_FIELDS = [ - 'approval__id', - 'approval__lodgement_number', - 'approval__status', - 'approval__internal_status', - 'approval__issue_date', - 'approval__start_date', - 'approval__expiry_date', - 'approval__wla_queue_date', - 'approval__migrated', - 'approval__submitter__email', - -# 'approval__proxy_applicant__', -# 'replaced_by', -# 'renewal_sent', -# 'original_issue_date', -# 'surrender_details', -# 'suspension_details', -# 'extracted_fields', -# 'cancellation_details', -# 'extend_details', -# 'cancellation_date', -# 'set_to_cancel', -# 'set_to_suspend', -# 'set_to_surrender', -# 'renewal_count', -# 'vessel_nomination_reminder_sent', -# 'reissued', -# 'export_to_mooring_booking', -# 'licence_document__', -# 'authorised_user_summary_document__', -# 'cover_letter_document__', -# 'current_proposal__', -# 'renewal_document__', - -] - -VESSEL_DETAILS_FIELDS = [ -# 'vessel_details__' - 'vessel_details__id', - 'vessel_details__vessel_type', - 'vessel_details__vessel_name', - 'vessel_details__vessel_length', - 'vessel_details__vessel_draft', - 'vessel_details__vessel_beam', - 'vessel_details__vessel_weight', - 'vessel_details__berth_mooring', - #'vessel_details__created', - #'vessel_details__updated', - 'vessel_details__exported', - 'vessel_details__vessel__rego_no', - #'vessel_details__vessel__blocking_owner__' -] - -VESSEL_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__id', - 'vessel_details__vessel__blocking_owner__percentage', - 'vessel_details__vessel__blocking_owner__dot_name', - 'vessel_details__vessel__blocking_owner__owner__emailuser__email', -] - -VESSEL_COMPANY_OWNER_FIELDS = [ - 'vessel_details__vessel__blocking_owner__company_ownership__company__name', - #'vessel_details__vessel__blocking_owner__company_ownership__company__percentage', - -# 'company_ownership__' -# 'vessel_details__vessel__blocking_owner__start_date', -# 'vessel_details__vessel__blocking_owner__end_date', -# 'vessel_details__vessel__blocking_owner__created', -# 'vessel_details__vessel__blocking_owner__updated', -# 'vessel_details__vessel__blocking_owner__exported', -] - -VESSEL_PREFERRED_BAY_FIELDS = [ - 'preferred_bay__name', -] - -VESSEL_MOORING_FIELDS = [ - 'mooring__name', -] - -FEESEASON_FIELDS = [ - 'fee_season__name', -] - -# 'mooring__' -# 'allocated_mooring__' -# 'waiting_list_allocation__' -# 'fee_season__' - -# 'proxy_applicant__' -# 'assigned_officer__' -# 'assigned_approver__' -# 'previous_application__' -# 'approval_level_document__' - - -def write_csv3(): - """ - from mooringlicensing.utils.mooring_licence_migrate_write_csv import get_fields, write_csv3 - - """ - fields = PROPOSAL_FIELDS - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - fields += APPROVAL_FIELDS - fields += VESSEL_DETAILS_FIELDS - fields += VESSEL_OWNER_FIELDS - fields += VESSEL_COMPANY_OWNER_FIELDS - fields += VESSEL_PREFERRED_BAY_FIELDS - fields += VESSEL_MOORING_FIELDS - fields += FEESEASON_FIELDS - - mla_qs = MooringLicenceApplication.objects.filter()[:10].values_list(*fields) - #data = [fields].append(list(mla_qs)) - #import ipdb; ipdb.set_trace() - print(fields) - #return fields, mla_qs - -# with open("mla.csv", "w") as f: -# writer = csv.writer(f) -# #writer.writerows(fields) -# writer.writerows([fields]) -# writer.writerows(mla_qs) -# import ipdb; ipdb.set_trace() - - df = pd.DataFrame(mla_qs, columns=fields) - df['lodgement_date'] = df['lodgement_date'].dt.tz_localize(None) # remove timezone for excel output - df.to_excel('mla.xlsx', index=0) - return df - - -def write_csv(filename): - model_fields = [] - model_hdrs = [] - import ipdb; ipdb.set_trace() - for model in MODELS: - model_fields += [x.name for x in getattr(sys.modules[__name__], model)._meta.concrete_fields] - model_hdrs += [x.name + f' ({model})' for x in getattr(sys.modules[__name__], model)._meta.concrete_fields] - #model_fields += [''] - #model_hdrs += [''] - - model_fields_list = list(MooringLicenceApplication.objects.values_list(*model_fields)) - - with open(filename, 'w', newline='') as myfile: - for _list in model_fields_list: - wr = csv.writer(myfile, quoting=csv.QUOTE_ALL) - wr.writerow(_list) - - -from rest_framework.renderers import JSONRenderer -from mooringlicensing.components.proposals.serializers import VesselOwnershipSerializer, InternalProposalSerializer, MooringLicenceApplicationSerializer -import json - -def serialize(model, serializer): - """ - serialize('VesselOwnership', 'VesselOwnershipSerializer') - """ - #qs = VesselOwnership.objects.all() - #serializer = VesselOwnershipSerializer(qs, many=True) - qs = getattr(sys.modules[__name__], model).objects.all()[:2] - #serializer = getattr(sys.modules[__name__], serializer)(qs, many=True) - serializer = getattr(sys.modules[__name__], serializer)(qs, many=True) - #serializer.data - renderer = JSONRenderer() - rendered = renderer.render(serializer.data) - rendered = rendered.decode('utf-8') - #rendered - - return json.loads(rendered) - -def get_fields(model): - fk=[] - #for field in MooringLicenceApplication._meta.concrete_fields: - for field in model._meta.concrete_fields: - if field.get_internal_type() == 'ForeignKey': - #print(f'{field.name}__') - fk.append(f'\'{field.name}__\'',) - else: - print(f'\'{field.name}\',') - - for field in fk: - print(field) - - -def write_csv2(): - fields = PROPOSAL_FIELDS - mla_qs = MooringLicenceApplication.objects.filter()[:1] - - fields += SUBMITTER_POSTAL_ADDRESS_FIELDS - fields += SUBMITTER_RESIDENTIAL_ADDRESS_FIELDS - mla = MooringLicenceApplication.objects.filter()[:1].values_list(*fields) - #mla = .values_list(*fields) - print(mla) - - diff --git a/mooringlicensing/utils/vessel_migrate.py b/mooringlicensing/utils/vessel_migrate.py deleted file mode 100644 index 7835e7d51..000000000 --- a/mooringlicensing/utils/vessel_migrate.py +++ /dev/null @@ -1,564 +0,0 @@ -import os -import subprocess -import json -import ast -import csv -import datetime -from decimal import Decimal -from django.db import transaction -from django.core.exceptions import ObjectDoesNotExist -from oscar.apps.address.models import Country -from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - MooringLicenceApplication, - ProposalUserAction, - Mooring, - MooringBay, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, MooringLicence, VesselOwnershipOnApproval, Sticker - - - -class VesselMigration(object): - ''' - PRELIM: - from mooringlicensing.utils.json_to_csv import JsonToCsv - JsonToCsv().convert() - - from mooringlicensing.utils.authorised_user_migrate import AuthUserPermitMigration, GrepSearch - AuthUserPermitMigration(test=False) - ''' - - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - #def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_07Oct2021', path_csv='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes_all_csv', test=False): - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.path_csv = path + '_all_csv' if not path.endswith('/') else path.strip('/') + '_all_csv' - self.test = test - -# self.auth_users_with_L = self.read_dict('Auth_Users___Surname___with_L.json') -# self.auth_users_No_L = self.read_dict('Auth_Users___Surname___No_L.json') -# self.moorings = self.read_dict('Auth_Users___Surname___Moorings.json') -# self.vessel_rego = self.read_dict('Auth_Users___Vessel_Rego.json') -# -# self.canc_lic = self.read_dict('Auth_Users___Cancelled_Lic.json') -# self.canc = self.read_dict('Auth_Users___Cancelled.json') -# -# self.vessel_rego_all = self.read_dict('Vessel___Rego___All.json') -# self.vessel_rego_current = self.read_dict('Vessel___Rego___Current.json') -# self.vessel_details_all = self.read_dict('Vessel___Details_All.json') -# self.vessel_details_current = self.read_dict('Vessel___Details_Current.json') -# self.people_name = self.read_dict('PeopleName.json') - - #self.surname = self.read_dict('Licencees___Surname.json') - self.mooring_no = self.read_dict('Licencees___Mooring_No.json') - self.mooring_details = self.read_dict('Mooring_Details.json') - self.people_lictype = self.read_dict('PeopleLicType.json') - self.vessel_details = self.read_dict('Vessel___Details_All.json') - #self.vessel_multiple = self.read_dict('Vessel___Multiple_Vessels.json') # vessel rego and name - self.vessel_multiple = self.read_dict('Licencees___Sticker_No.json') # vessel rego and name - self.user_rego = self.read_dict('UserRego.json') # vessel length/draft - if not available in vessel_multiple - self.user_mooring = self.read_dict('UserMooring.json') # vessel length/draft - if not available in vessel_multiple - self.people_no = self.read_dict('PeopleNo.json') # address/phone details (if no vessel_multiple details) - self.people_cur_mooring = self.read_dict('People___Current_Mooring.json') # address/phone details (if no vessel_multiple details) - #self.people_email = self.read_dict('People___eMail.json') # address/phone details (if no vessel_multiple details) - self.people_email = self.read_dict('Licencees___Surname.json') # address/phone details (if no vessel_multiple details) - - - self._auth_users = [ - #... can be found in directory tests/test.json - ] - - #self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def search_multiple(self, key, value, records): - return [record for record in records if value == record.get(key)] - - - def get_vessels(self, no_vessel, record): - if not record: - # ML with no linked vessel - #return [('', '', 0.0, 0.0)] # rego/name/length/draft - return [] # rego/name/length/draft - - #keys = ['_6', '_7', '_8', '_9', '_10'] - ves_keys = ['_10', '_11', '_12', '_13', '_14'] - sticker_keys = ['StickerLNo1', 'StickerLNo2', 'StickerLNo3', 'StickerLNo4', 'StickerLNo5'] - _vessels = [] - for i in range(no_vessel): - items = record[ves_keys[i]].split('-') - rego = items[0].strip() - vessel_name = items[1].strip() - sticker_no = record[sticker_keys[i]].strip() - if sticker_no == 'None': - sticker_no = '' -# import ipdb; ipdb.set_trace() - _vessels.append((rego, vessel_name, 0.0, 0.0, sticker_no)) # rego/name/length/draft - return _vessels - -# def get_stickers(self, no_vessel, record): -# if not record: -# # ML with no linked vessel -# return [] -# -# keys = ['StickerLNo1', 'StickerLNo2', 'StickerLNo3', 'StickerLNo4', 'StickerLNo5'] -# _stickers = [] -# for i in range(no_vessel): -# sticker_no = record[keys[i]].strip() -# _stickers.append((rego, vessel_name, 0.0, 0.0)) # rego/name/length/draft -# return _vessels - - - def to_float(self, val): - try: - return float(val) - except Exception as e: - return 0.0 - - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - start_date = datetime.date(2022,8,31) - date_applied = '2022-08-31' - date_invited = None #date_invited, - migrated = False - - self.num_vessels = [] - self.vessels_found = [] - self.vessels_not_found = [] - self.vessel_details_not_found = [] - self.emailuser = [] - self.errors = [] - self.approvals = [] - - user_data = [] - vessel_data = [] - mla_data = [] - ml_all_data = [] - user_data.append(['PersNo', 'Email', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address', 'NoVessel']) - vessel_data.append(['Owner', 'Email', 'DoB', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring']) - mla_data.append(['PersNo', 'Email', 'DoB', 'Rego No', 'MooringNo', 'Date Invited']) - ml_all_data.append(['PersNo', 'Email', 'DoB', 'DoB', 'FirstName', 'LastName', 'MobileNo', 'PhoneNo', 'Address', 'Rego No', 'DoT Name', 'Percentage Ownership', 'Vessel Type', 'Vessel Name', 'Length', 'Draft', 'Weight', 'berth_mooring', 'MooringNo', 'Date Invited']) - - rego_no='' - dot_name='' - percentage='' - vessel_type='' - vessel_name='' - vessel_overall_length='' - vessel_draft='' - vessel_weight='' - berth_mooring='' - - with transaction.atomic(): - #for idx, record in enumerate(self.moorings[827:], 827): - import ipdb; ipdb.set_trace() - #for idx, record in enumerate(self.vessel_rego_current[:60], 0): - #for idx, record in enumerate(self.vessel_rego_current, 0): - #for idx, record in enumerate(self.mooring_no[:300], 0): - #for idx, record in enumerate(self.mooring_no[200:205], 0): - for idx, record in enumerate(self.mooring_no[0:], 0): - try: - mooring_no = record['MooringNo'] - username = record['UserName'] - pers_no = record['PersNo'] - address = record['_1'] - no_vessel = int(float(record['NoVessel'])) - - if no_vessel>5: - self.num_vessels.append(dict(username=username, pers_no=pers_no, no_vessel=no_vessel, mooring_no=mooring_no)) - no_vessel = 5 # ignore the remainder (JSON file only list max. 5) - - if pers_no not in ['000477', '206776', '207209', '030742']: - continue -# if pers_no == '073604': -# import ipdb; ipdb.set_trace() -# else: -# continue - - - - # assume first vessel is licenced to Mooring - record_ves_details = self.search('PersNo', pers_no, self.vessel_multiple) - vessels = self.get_vessels(no_vessel, record_ves_details) - print(f'{idx} Vessel details found: {username} {pers_no} {mooring_no} {no_vessel} {vessels}') - #vessel_mooring = vessels[0] - - #import ipdb; ipdb.set_trace() - record_people_no = self.search('PersNo', pers_no, self.people_cur_mooring) - address = record_people_no['_1'] - phone_home = record_people_no['PhoneHome'] - phone_mobile = record_people_no['PhoneMobile'] - phone_work = record_people_no['PhoneWork'] - - #{'LastName': 'Abbott', '_7': 'William', 'PersNo': '207907', '_8': 'W', 'NoVessel': '1', '_9': '1', 'EMail': 'wabbinator@gmail.com', 'LicContactEmail': 'None'} - record_people_email = self.search('PersNo', pers_no, self.people_email) - email = record_people_email['EMail'].split(';')[0].strip().lower() - - user = self.set_emailuser(username, email, address, phone_mobile, phone_home) - - # see mooringlicensing/utils/tests/mooring_names.txt - if Mooring.objects.filter(name=mooring_no).count()>0: - mooring = Mooring.objects.filter(name=mooring_no)[0] - else: - import ipdb; ipdb.set_trace() - print(f'Mooring not found: {mooring}') - self.errors.append(dict(idx=idx, record=record)) - continue - - approval = MooringLicence.objects.create( - status='current', - #internal_status=None, - #current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=migrated, - export_to_mooring_booking=True, - ) - self.approvals.append(approval.id) - - if not vessels: - proposal=MooringLicenceApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.datetime.now(datetime.timezone.utc), - migrated=migrated, - - preferred_bay= mooring.mooring_bay, - processing_status='approved', - customer_status='approved', - allocated_mooring=mooring, - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - - "vessel_ownership": [dict(dot_name=i[0]) for i in vessels], - "mooring_on_approval": [] - }, - date_invited=date_invited, - ) - - approval.current_proposal=proposal - approval.save() - - else: - #import ipdb; ipdb.set_trace() - vessel_objs = [] - #proposal = None - for ves_idx, ves in enumerate(vessels): - ves_rego_no = ves[0] - ves_name = ves[1] - ves_length = self.to_float(ves[2]) - ves_draft = self.to_float(ves[3]) - sticker_no = ves[4] - ves_type = '' - ves_weight = Decimal( 0.0 ) - berth_mooring = '' - percentage = None # force user to set at renewal time - vessel_type = '' - - try: - vessel = Vessel.objects.get(rego_no=ves_rego_no) - except ObjectDoesNotExist: - vessel = Vessel.objects.create(rego_no=ves_rego_no) - - try: - owner = Owner.objects.get(emailuser=user) - except ObjectDoesNotExist: - owner = Owner.objects.create(emailuser=user) - - try: - vessel_ownership = VesselOwnership.objects.get(owner=owner, vessel=vessel) - vessel_ownership.dot_name = ves_rego_no # rego_no - vessel_ownership.save() - except ObjectDoesNotExist: - #vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage, dot_name=dot_name) - vessel_ownership = VesselOwnership.objects.create(owner=owner, vessel=vessel, percentage=percentage, dot_name=ves_rego_no) - - if VesselDetails.objects.filter(vessel=vessel).count()>0: - vessel_details = VesselDetails.objects.filter(vessel=vessel)[0] - else: - vessel_details = VesselDetails.objects.create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=ves_name, - vessel_length=ves_length, - vessel_draft=ves_draft, - vessel_weight= ves_weight, - berth_mooring=berth_mooring - ) - - vessel_objs.append(dict( - vessel=vessel, - owner=owner, - vessel_ownership=vessel_ownership, - vessel_details=vessel_details, - )) - - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - - #import ipdb; ipdb.set_trace() - proposal=MooringLicenceApplication.objects.create( - proposal_type_id=1, # new application - submitter=user, - lodgement_date=datetime.datetime.now(datetime.timezone.utc), - migrated=migrated, - - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=ves_rego_no, - vessel_type=ves_type, - vessel_name=ves_name, - vessel_length=ves_length, - vessel_draft=ves_draft, - #vessel_beam='', - vessel_weight=ves_weight, - berth_mooring=berth_mooring, - percentage=percentage, - individual_owner=True, - dot_name='', #vessel_mooring[0], #vessel_dot, - - preferred_bay= mooring.mooring_bay, - #proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - allocated_mooring=mooring, - proposed_issuance_approval={ - "start_date": start_date.strftime('%d/%m/%Y'), - "expiry_date": expiry_date.strftime('%d/%m/%Y'), - "details": None, - "cc_email": None, - "mooring_id": mooring.id, - "ria_mooring_name": mooring.name, - "mooring_bay_id": mooring.mooring_bay.id, - # "vessel_ownership": [{ - # "dot_name": vessel_mooring[0], # vessel_mooring[0] --> rego_no #vessel_dot - # }], - "vessel_ownership": [dict(dot_name=i[0]) for i in vessels], - "mooring_on_approval": [] - }, - date_invited=date_invited, - ) - -# proposal.vessel_details=vessel_details -# proposal.vessel_ownership=vessel_ownership -# proposal.rego_no=ves_rego_no -# proposal.vessel_type=ves_type -# proposal.vessel_name=ves_name -# proposal.vessel_length=ves_length -# proposal.vessel_draft=ves_draft -# #proposal.vessel_beam='' -# proposal.vessel_weight=ves_weight -# proposal.berth_mooring=berth_mooring -# proposal.percentage=percentage -# proposal.individual_owner=True -# proposal.dot_name='' #vessel_mooring[0], #vessel_dot, -# proposal.save() - - sticker=Sticker.objects.create( - number=sticker_no, - status='current', - approval=approval, - printing_date=start_date, - mailing_date=start_date, - vessel_ownership=vessel_ownership, - proposal_initiated=proposal, - #dcv_permit=null, - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user, - what='Mooring Site Licence - Migrated Application', - ) - - vooa = VesselOwnershipOnApproval.objects.create( - approval=approval, - vessel_ownership=vessel_ownership, - ) - - #import ipdb; ipdb.set_trace() - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership=vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - #stickers = [sticker.id], - ) - approval_history.stickers.add(sticker.id), - -# for approval_id in self.approvals: -# a = MooringLicence.objects.get(id=approval_id) -# import ipdb; ipdb.set_trace() -# a.generate_doc() - - - proposal.approval = approval - proposal.save() - #proposal.allocated_mooring.mooring_licence = approval - #proposal.allocated_mooring.save() - #approval.generate_doc() - -# moa = MooringOnApproval.objects.create( -# approval=approval, -# mooring=mooring, -# sticker=None, -# site_licensee=True, # ??? -# end_date=expiry_date -# ) - - ml_all_data.append([pers_no, email, user.dob, user.first_name, user.last_name, user.mobile_number, user.phone_number, user.residential_address, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring, mooring.name, date_invited]) - - user_data.append([pers_no, email, user.dob, user.first_name, user.last_name, user.mobile_number, user.phone_number, user.residential_address, no_vessel]) - - berth_mooring = '' - vessel_data.append([pers_no, email, user.dob, rego_no, dot_name, percentage, vessel_type, vessel_name, vessel_overall_length, vessel_draft, vessel_weight, berth_mooring]) - - #date_invited = '' - mla_data.append([pers_no, email, user.dob, rego_no, mooring.name, date_invited]) - - except Exception as e: - self.errors.append(f'{idx} {record}\n{e}') - import ipdb; ipdb.set_trace() - print(e) - - #import ipdb; ipdb.set_trace() - with open('ml_all.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(ml_all_data) - - with open('user.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(user_data) - - with open('vessel.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(vessel_data) - - with open('mla.csv', 'w') as f: - writer = csv.writer(f) - writer.writerows(mla_data) - - - - - - print(f'vessels_found: {self.vessels_found}') - print(f'vessels_not_found: {self.vessels_not_found}') - print(f'vessel_details_not_found: {self.vessel_details_not_found}') - print(f'num_vessels: {self.num_vessels}') - print(f'errors: {self.errors}') - print(f'no. vessels_found: {len(self.vessels_found)}') - print(f'no. vessels_not_found: {len(self.vessels_not_found)}') - print(f'no. vessel_details_not_found: {len(self.vessel_details_not_found)}') - print(f'no. emailuser: {len(self.emailuser)}') - print(f'no. num_vessels: {len(self.num_vessels)}') - print(f'no. errors: {len(self.errors)}') - - def set_emailuser(self, username, email, address, phone_mobile, phone_home): - items = address.split(',') - line1 = ', '.join(items[:-2]).strip() - state = items[-2].strip() - postcode = items[-1].strip() - - firstname = username.split(' ')[-1].strip() - lastname = ' '.join(username.split(' ')[:-1]) - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=phone_mobile, phone_number=phone_home) - - country = Country.objects.get(printable_name='Australia') - try: - _address = Address.objects.get(user=user) - _address.line1 = line1 - _address.state = state - _address.postcode = postcode - _address.country = country - _address.save() - except: - _address, address_created = Address.objects.get_or_create(line1=line1, state=state, postcode=postcode, country=country, user=user) - user.residential_address = _address - user.postal_address = _address - user.save() - self.emailuser.append(f'{email}') - - return user - - - - def get_vessel_details(self, idx, pers_no, username, rego_no, record): - try: - #import ipdb; ipdb.set_trace() - if record: - if rego_no != '0': - dot_name = record['DoTRego1'] - ves_length = record['RegLength1'] - ves_draft = record['Draft1'] - - record_people_name = self.search('PersNo', pers_no, self.people_name) - ves_name = record_people_name['_3'] - email = record_people_name['EMail'] - - print(idx, email) - #import ipdb; ipdb.set_trace() - self.vessels_found.append(f'{pers_no} - {username} - {rego_no}') - else: - #print(idx, f'No Reg {rego_no} - {username}') - self.vessels_not_found.append(f'{pers_no} - {username} - {rego_no}') - #import ipdb; ipdb.set_trace() - else: - print(idx, f'No Vessel Details {pers_no} - {username}') - self.vessel_details_not_found.append(f'{pers_no} - {username} - {rego_no}') - - except Excception as e: - self.errors.append(f'{idx} {record}\n{e}') - import ipdb; ipdb.set_trace() - #print(e) - - - diff --git a/mooringlicensing/utils/waiting_list_migrate.py b/mooringlicensing/utils/waiting_list_migrate.py deleted file mode 100644 index fe6e74aef..000000000 --- a/mooringlicensing/utils/waiting_list_migrate.py +++ /dev/null @@ -1,363 +0,0 @@ -import os -import subprocess -import json -import datetime -from decimal import Decimal -from django.db import transaction -# from oscar.apps.address.models import Country -# from ledger.accounts.models import EmailUser, Address -from mooringlicensing.components.proposals.models import ( - Proposal, - Vessel, - Owner, - VesselOwnership, - VesselDetails, - WaitingListApplication, - ProposalUserAction, - MooringBay, -) -from mooringlicensing.components.approvals.models import Approval, ApprovalHistory, WaitingListAllocation - -class WaitingListMigration(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - WaitingListMigration(test=False) - ''' - - def __init__(self, path='/var/www/mooringlicensing/mooringlicensing/utils/lotus_notes', test=False): - self.path = path - self.test = test - - self.waitlist = self.read_dict('Waitlist___Bay.json') - - self._waitlist = [ - #... can be found in directory tests/test.json - ] - - self.migrate() - - def read_dict(self, fname): - filename = self.path + os.sep + fname if not self.path.endswith(os.sep) else self.path + fname - with open(filename) as f: - f_json = json.load(f) - - # make list unique - _list = [] - for i in f_json: - if i not in _list: - _list.append(i) - - #import ipdb; ipdb.set_trace() - return _list - - def search(self, key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - def migrate(self): - - #submitter = EmailUser.objects.get(email='jawaid.mushtaq@dbca.wa.gov.au') - expiry_date = datetime.date(2023,8,31) - - addresses_not_found = [] - address_list = [] - user_list = [] - vessel_list = [] - owner_list = [] - ownership_list = [] - details_list = [] - proposal_list = [] - wl_list = [] - user_action_list = [] - approval_list = [] - approval_history_list = [] - wla_list = [] - - added = [] - errors = [] - with transaction.atomic(): - #for idx, record in enumerate(self.waitlist[541:], 541): - for idx, record in enumerate(self.waitlist, 1): - try: - #import ipdb; ipdb.set_trace() - pers_no = record.get('PersNo') - email_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'EMail') - address_record = GrepSearch(pers_no, path=self.path).search('PersNo', '_1') - if address_record is None: - print(f'Address Not Found: {idx}, {pers_no}') - addresses_not_found.append(pers_no) - continue - - - phonemobile_record = {} - if 'PhoneMobile' not in address_record: - phonemobile_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneMobile') - else: - phonemobile_record = address_record - - phonehome_record = {} - if 'PhoneHome' not in address_record: - phonehome_record = GrepSearch(pers_no, path=self.path).search('PersNo', 'PhoneHome') - else: - phonehome_record = address_record - - email = email_record.get('EMail').lower().strip() - if email=='mjmatthews@me.com': - errors.append(email) - continue - - mobile_no = phonemobile_record.get('PhoneMobile') - username = record.get('UserName').lower() - firstname = username.split(' ')[-1].title() - lastname = ' '.join(username.split(' ')[:-1]).title() - - try: - phone_no = phonehome_record.get('PhoneHome') if phonehome_record else '' - address = address_record.get('_1') - except: - import ipdb; ipdb.set_trace() - - rego_no = record.get('VesRego') - vessel_type = 'other' - vessel_name = record.get('VesName') - vessel_overall_length = Decimal( record.get('RegLength') ) - vessel_draft = Decimal( record.get('Draft') ) - vessel_weight = Decimal( 0.0 ) - berth_mooring = record.get('_5') - - # see mooringlicensing/utils/tests/mooring_names.txt - if MooringBay.objects.filter(name=berth_mooring).count()>0: - mooring_bay = MooringBay.objects.filter(name=berth_mooring)[0] - else: - mooring_bay = MooringBay.objects.get(name='Rottnest Island') - #mooring_bay = MooringBay.objects.get(name='Thomson Bay') - - # needed ?? - date_applied = record.get('DateApplied') - position_no = int(record.get('BayPosNo')) - trim_no = record.get('TrimNo') - percentage = None # force user to set at renewal time - - - items = address.split(',') - line1 = items[0].strip() - line2 = items[1].strip() if len(items) > 3 else '' - line3 = items[2].strip() if len(items) > 4 else '' - state = items[-2].strip() - postcode = items[-1].strip() - - #import ipdb; ipdb.set_trace() - print(f'{idx}, {pers_no}, {state}, {postcode}') - if self.test: - continue - - try: - user = EmailUser.objects.get(email=email) - except: - user = EmailUser.objects.create(email=email, first_name=firstname, last_name=lastname, mobile_number=mobile_no, phone_number=phone_no) - - country = Country.objects.get(printable_name='Australia') - #address, address_created = Address.objects.get_or_create(line1=line1, line2=line2, line3=line3, postcode=postcode, state=state, country=country, user=user) - address, address_created = Address.objects.get_or_create( - user=user, - defaults={ - 'line1':line1, - 'line2':line2, - 'line3':line3, - 'postcode':postcode, - 'state':state, - 'country':country - } - ) - user.residential_address = address - user.postal_address = address - user.save() - - vessel, rego_created = Vessel.objects.update_or_create(rego_no=rego_no) - owner, owner_created = Owner.objects.update_or_create(emailuser=user) - vessel_ownership, ownership_created = VesselOwnership.objects.update_or_create( - owner=owner, - vessel=vessel, - percentage=percentage - ) - - vessel_details, details_created = VesselDetails.objects.update_or_create( - vessel_type=vessel_type, - vessel=vessel, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - vessel_weight= vessel_weight, - berth_mooring=berth_mooring - ) - - proposal=WaitingListApplication.objects.create( - proposal_type_id=1, - submitter=user, - lodgement_date=datetime.now(), - migrated=True, - vessel_details=vessel_details, - vessel_ownership=vessel_ownership, - rego_no=rego_no, - vessel_type=vessel_type, - vessel_name=vessel_name, - vessel_length=vessel_overall_length, - vessel_draft=vessel_draft, - #vessel_beam='', - vessel_weight=vessel_weight, - berth_mooring=berth_mooring, - preferred_bay=mooring_bay, - percentage=percentage, - individual_owner=True, - proposed_issuance_approval={}, - processing_status='approved', - customer_status='approved', - ) - - ua=ProposalUserAction.objects.create( - proposal=proposal, - who=user.id, - what='Waiting List - Migrated Application', - ) - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date() - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').date() - - approval = WaitingListAllocation.objects.create( - status=Approval.APPROVAL_STATUS_CURRENT, - internal_status=Approval.INTERNAL_STATUS_WAITING, - current_proposal=proposal, - issue_date = datetime.datetime.now(datetime.timezone.utc), - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').date(), - start_date = start_date, - expiry_date = expiry_date, - submitter=user, - migrated=True, - wla_order=position_no, - ) - #print(f'wla_order: {position_no}') - - proposal.approval = approval - proposal.save() - - try: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - except: - start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d').astimezone(datetime.timezone.utc) - - approval_history = ApprovalHistory.objects.create( - reason='new', - approval=approval, - vessel_ownership = vessel_ownership, - proposal = proposal, - #start_date = datetime.datetime.strptime(date_applied, '%Y-%m-%d %H:%M:%S').astimezone(datetime.timezone.utc) - start_date = start_date, - ) - - added.append(proposal.id) - - address_list.append(address.id) - user_list.append(user.id) - vessel_list.append(vessel.id) - owner_list.append(owner.id) - ownership_list.append(vessel_ownership.id) - details_list.append(vessel_details.id) - proposal_list.append(proposal.proposal.id) - wl_list.append(proposal.id) - user_action_list.append(ua.id) - approval_list.append(approval.id) - approval_history_list.append(approval_history.id) - - except Exception as e: - #errors.append(str(e)) - import ipdb; ipdb.set_trace() - raise Exception(str(e)) - - print(f'Address.objects.get(id__in={address_list}).delete()') - print(f'EmailUser.objects.get(id__in={user_list}).delete()') - print(f'Vessel.objects.get(id__in={vessel_list}).delete()') - print(f'Owner.objects.get(id__in={owner_list}).delete()') - print(f'VesselOwnership.objects.get(id__in={ownership_list}).delete()') - print(f'VesselDetails.objects.get(id__in={details_list}).delete()') - print(f'WaitingListApplication.objects.get(id__in={wl_list}).delete()') - print(f'ProposalUserAction.objects.get(id__in={user_action_list}).delete()') - print(f'WaitingListAllocation.objects.get(id__in={approval_list}).delete()') - print(f'ApprovalHistory.objects.get(id__in={approval_history_list}).delete()') - print(f'Addresses Not Found: {addresses_not_found}') - -def clear_record(): - Address.objects.last().delete() - EmailUser.objects.last().delete() - Vessel.objects.last().delete() - Owner.objects.last().delete() - VesselOwnership.objects.last().delete() - VesselDetails.objects.last().delete() - WaitingListApplication.objects.last().delete() - ProposalUserAction.objects.last().delete() - WaitingListAllocation.objects.last().delete() - ApprovalHistory.objects.last().delete() - -class GrepSearch(object): - ''' - from mooringlicensing.utils.waiting_list_migrate import WaitingListMigration, GrepSearch - - GrepSearch('123').search('key', '_1') --> Search for string key='123' in all files. The result record/dict must also have key '_1' - ''' - - def __init__(self, search_str1, path='mooringlicensing/utils/lotus_notes'): - self.path = path - self.search_str1 = search_str1 - #self.search() - - def get_files(self, search_str): - ''' Read all files in directory ''' - r=subprocess.check_output(['grep', '-rl', search_str, self.path]) - files = [i for i in r.decode('UTF-8').split('\n') if i.endswith(".json") ] - return files - - def search(self, key1, key2): - ''' Iteratively search for key:value pair in grep'd files ''' - - def find(key, value, records): - for record in records: - if value == record.get(key): - return record - return None - - #import ipdb; ipdb.set_trace() - # Get all files that contains string 'self.search_str1' -# files = self.get_files(self.search_str1) - files=[ - self.path + os.sep + 'Admin___EContacts___6_Waitlist.json', - self.path + os.sep + 'People___Trim_File.json', - self.path + os.sep + 'Vessel___Rego___Current.json', - self.path + os.sep + 'Vessel___Vessel_Name___Current.json', - self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'PeopleNo.json', - self.path + os.sep + 'Admin___Labels___Postal.json', - self.path + os.sep + 'Admin___Interrogation___Lic___WL_DOB_check.json', - self.path + os.sep + 'Auth_Users___Surname___No_L.json', - self.path + os.sep + 'Auth_Users___Surname___with_L.json', - self.path + os.sep + 'Vessel___HIN___Current.json', - ] - - for fname in files: - with open(fname) as f: - f_json = json.load(f) - #import ipdb; ipdb.set_trace() - # check the files (json) also has the key2 - if key2 in f_json[0]: - # find record in json that contains key1:search_str1 - record = find(key1, self.search_str1, f_json) - if record: - #import ipdb; ipdb.set_trace() - if key2=='_1' and len(record['_1'].split(',')) <= 1: - continue - print(f'fname: {fname}, key2: {key2}') - return record - -