diff --git a/kustomize/overlays/prod/kustomization.yaml b/kustomize/overlays/prod/kustomization.yaml index 0ebff376..64195fda 100644 --- a/kustomize/overlays/prod/kustomization.yaml +++ b/kustomize/overlays/prod/kustomization.yaml @@ -28,4 +28,4 @@ patches: - path: geoserver_service_patch.yaml images: - name: ghcr.io/dbca-wa/prs - newTag: 2.5.38 + newTag: 2.5.39 diff --git a/poetry.lock b/poetry.lock index 066ecfba..427cee81 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1896,13 +1896,13 @@ msg-parse = ["extract-msg (>=0.27)"] [[package]] name = "sentry-sdk" -version = "1.40.3" +version = "1.40.4" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.40.3.tar.gz", hash = "sha256:3c2b027979bb400cd65a47970e64f8cef8acda86b288a27f42a98692505086cd"}, - {file = "sentry_sdk-1.40.3-py2.py3-none-any.whl", hash = "sha256:73383f28311ae55602bb6cc3b013830811135ba5521e41333a6e68f269413502"}, + {file = "sentry-sdk-1.40.4.tar.gz", hash = "sha256:657abae98b0050a0316f0873d7149f951574ae6212f71d2e3a1c4c88f62d6456"}, + {file = "sentry_sdk-1.40.4-py2.py3-none-any.whl", hash = "sha256:ac5cf56bb897ec47135d239ddeedf7c1c12d406fb031a4c0caa07399ed014d7e"}, ] [package.dependencies] @@ -1929,7 +1929,7 @@ huey = ["huey (>=2)"] loguru = ["loguru (>=0.5)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] -pure-eval = ["asttokens", "executing", "pure-eval"] +pure-eval = ["asttokens", "executing", "pure_eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -2164,16 +2164,16 @@ files = [ [[package]] name = "webtemplate-dbca" -version = "1.6.0" +version = "1.7.0" description = "Base HTML templates for DBCA Django projects" optional = false python-versions = "*" files = [ - {file = "webtemplate-dbca-1.6.0.tar.gz", hash = "sha256:6ab6f043ba62b178b5fbd85907d7e9a5ad6634aa231b5a11e3a5d35035ff8942"}, + {file = "webtemplate-dbca-1.7.0.tar.gz", hash = "sha256:36d0cf2110b948f4008d0761d770409ddd069c16fec4f539ab1bfaeb834ca23b"}, ] [package.dependencies] -Django = ">=3.2,<5" +Django = ">=3.2" [[package]] name = "whitenoise" @@ -2216,4 +2216,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "90b2cbfc7c45e10b39ea7b1f3e34c8d562fb0a47f1fc23f1ee5fb28a39a721a4" +content-hash = "f0bd90dbb2ab2d3efc8ccfdeff18827bbd0f27209b135d7b8cd723a4b6d6b010" diff --git a/prs2/harvester/management/commands/harvest_email_referrals.py b/prs2/harvester/management/commands/harvest_email_referrals.py index bfb99783..c3936867 100644 --- a/prs2/harvester/management/commands/harvest_email_referrals.py +++ b/prs2/harvester/management/commands/harvest_email_referrals.py @@ -28,12 +28,18 @@ def add_arguments(self, parser): def handle(self, *args, **options): logger = logging.getLogger("harvester") actions = [] + + # Download unread emails from each specifed source email address. for email in settings.PLANNING_EMAILS: if "purge_email" in options and options["purge_email"]: actions += harvest_unread_emails(from_email=email, purge_email=True) else: actions += harvest_unread_emails(from_email=email) + + # Import all harvested, unprocessed referrals. actions += import_harvested_refs() + + # Optionally email the power users group a report. if options["email"]: # Send an email to users in the "PRS power users" group. pu_group = Group.objects.get(name=settings.PRS_POWER_USER_GROUP) diff --git a/prs2/harvester/models.py b/prs2/harvester/models.py index ac79c436..2c870fe7 100644 --- a/prs2/harvester/models.py +++ b/prs2/harvester/models.py @@ -47,9 +47,11 @@ def save(self, force_insert=False, force_update=False, *args, **kwargs): def harvest(self, create_tasks=True, create_locations=True, create_records=True, assignee=False): """Undertake the harvest process for this emailed referral. """ + actions = [] if self.processed: - return + LOGGER.info(f'Emailed referral {self.pk} is already processed, aborting') + return actions User = get_user_model() dbca = Agency.objects.get(slug='dbca') @@ -59,18 +61,17 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, assignee_default = User.objects.get(username=settings.REFERRAL_ASSIGNEE_FALLBACK) else: assignee_default = assignee - actions = [] attachments = self.emailattachment_set.all() self.log = '' # Emails without attachments are usually reminder notices. if not attachments.exists(): - s = 'Skipping harvested referral {} (no attachments)'.format(self) + s = f'Skipping emailed referral {self.pk} (no attachments)' LOGGER.info(s) self.log = s self.processed = True self.save() - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + actions.append(f'{datetime.now().isoformat()} {s}') return actions # We don't want to harvest "overdue referral" reminders. @@ -78,23 +79,78 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, 'wapc eoverdue referral', 're: wapc eoverdue referral', ) - if self.subject.lower().startswith(overdue_subject_prefixes): - s = 'Skipping harvested referral {} (overdue notice)'.format(self) + if any([self.subject.lower().startswith(i) for i in overdue_subject_prefixes]): + s = f'Skipping harvested referral {self} (overdue notice)' LOGGER.info(s) self.log = s self.processed = True self.save() - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + actions.append(f'{datetime.now().isoformat()} {s}') + return actions + + # Some emailed referrals contain "additional documents" where the size + # limit of attachments have been exceeded for the first email. + additional_documents_subject = ( + 'additional documents', + 'additional referral documents', + ) + if any([i in self.subject.lower() for i in additional_documents_subject]) and attachments.exists(): + # Try to parse the referral reference from the email subject (it's normally the first + # element in the string). + reference = self.subject.split()[0] + if Referral.objects.current().filter(reference__iexact=reference): + referral = Referral.objects.current().filter(reference__iexact=reference).order_by('-pk').first() + s = f'Referral ref. {reference} is already in database; using existing referral {referral}' + LOGGER.info(s) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') + self.referral = referral + + # Save the EmailedReferral as a record on the referral. + if create_records: + new_record = Record.objects.create( + name=self.subject, referral=referral, order_date=datetime.today()) + file_name = f'emailed_referral_{reference}.html' + new_file = ContentFile(str.encode(self.body)) + new_record.uploaded_file.save(file_name, new_file) + with create_revision(): + new_record.save() + set_comment('Initial version.') + s = f'New PRS record generated: {new_record}' + LOGGER.info(s) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') + + # Add records to the referral (one per attachment). + for att in attachments: + new_record = Record.objects.create( + name=att.name, referral=referral, order_date=datetime.today()) + # Duplicate the uploaded file. + new_file = ContentFile(att.attachment.read()) + new_record.uploaded_file.save(att.name, new_file) + new_record.save() + s = f'New PRS record generated: {new_record}' + LOGGER.info(s) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') + # Link the attachment to the new, generated record. + att.record = new_record + att.save() + + LOGGER.info(f'Marking emailed referral {self.pk} as processed') + self.processed = True + self.save() + LOGGER.info('Done') return actions # Must be an attachment named 'Application.xml' present to import. if not attachments.filter(name__istartswith='application.xml'): - s = 'Skipping harvested referral {} (no XML attachment)'.format(self.pk) + s = f'Skipping harvested referral {self.pk} (no XML attachment)' LOGGER.info(s) self.log = s self.processed = True self.save() - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + actions.append(f'{datetime.now().isoformat()} {s}') return actions else: xml_file = attachments.get(name__istartswith='application.xml') @@ -103,31 +159,31 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, try: d = xmltodict.parse(xml_file.attachment.read()) except Exception as e: - s = 'Harvested referral {} parsing of application.xml failed'.format(self.pk) + s = f'Harvested referral {self.pk} parsing of application.xml failed' LOGGER.error(s) LOGGER.exception(e) - self.log = self.log + '{}\n{}\n'.format(s, e) + self.log = self.log + f'{s}\n{e}\n' self.processed = True self.save() - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + actions.append(f'{datetime.now().isoformat()} {s}') app = d['APPLICATION'] reference = app['WAPC_APPLICATION_NO'] # New/existing referral object. if Referral.objects.current().filter(reference__iexact=reference): # Note if the the reference no. exists in PRS already. - s = 'Referral ref. {} is already in database; using existing referral'.format(reference) + s = f'Referral ref. {reference} is already in database; using existing referral' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') new_ref = Referral.objects.current().filter(reference__iexact=reference).order_by('-pk').first() referral_preexists = True else: # No match with existing references. - s = 'Importing harvested referral ref. {} as new entity'.format(reference) + s = f'Importing harvested referral ref. {reference} as new entity' LOGGER.info(s) - self.log = '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') new_ref = Referral(reference=reference) referral_preexists = False @@ -135,12 +191,12 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, try: ref_type = ReferralType.objects.filter(name__istartswith=app['APP_TYPE'])[0] except Exception: - s = 'Referral type {} is not recognised type; skipping'.format(app['APP_TYPE']) + s = f'Referral type {app["APP_TYPE"]} is not recognised type; skipping' LOGGER.warning(s) - self.log = self.log + '{}\n'.format(s) + self.log = f'{s}\n' self.processed = True self.save() - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + actions.append(f'{datetime.now().isoformat()} {s}') return actions # Determine the intersecting region(s). @@ -164,10 +220,10 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, regions.append(r) intersected_region = True except Exception: - s = 'Address long/lat could not be parsed ({}, {})'.format(a['LONGITUDE'], a['LATITUDE']) + s = f'Address long/lat could not be parsed ({a["LONGITUDE"]}, {a["LATITUDE"]})' LOGGER.warning(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') intersected_region = False # Use the PIN field to try returning geometry from SLIP. if 'PIN' in a and a['PIN']: @@ -186,19 +242,19 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, for r in Region.objects.all(): if r.region_mpoly and r.region_mpoly.intersects(p) and r not in regions: regions.append(r) - s = 'Address PIN {} returned geometry from SLIP'.format(a['PIN']) - self.log = self.log + '{}\n'.format(s) + s = f'Address PIN {a["PIN"]} returned geometry from SLIP' + self.log = self.log + f'{s}\n' LOGGER.info(s) except Exception as e: - s = 'Error querying Landgate SLIP for spatial data (referral ref. {})'.format(reference) + s = f'Error querying Landgate SLIP for spatial data (referral ref. {reference})' LOGGER.error(s) LOGGER.error(resp.content) LOGGER.exception(e) - self.log = self.log + '{}\n{}\n{}\n'.format(s, resp.content, e) + self.log = self.log + f'{s}\n{resp.content}\n{e}\n' else: - s = 'Address PIN could not be parsed ({})'.format(a['PIN']) + s = f'Address PIN could not be parsed ({a["PIN"]})' LOGGER.warning(s) - self.log = self.log + '{}\n'.format(s) + self.log = self.log + f'{s}\n' regions = set(regions) # Business rules: # Didn't intersect a region? Might be bad geometry in the XML. @@ -207,24 +263,24 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, if len(regions) == 0: region = Region.objects.get(name='Swan') assigned = assignee_default - s = 'No regions were intersected, defaulting to {} ({})'.format(region, assigned) + s = f'No regions were intersected, defaulting to {region} ({assigned})' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) + self.log = self.log + f'{s}\n' elif len(regions) > 1: region = Region.objects.get(name='Swan') assigned = assignee_default - s = '>1 regions were intersected ({}), defaulting to {} ({})'.format(regions, region, assigned) + s = f'>1 regions were intersected ({regions}), defaulting to {region} ({assigned})' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) + self.log = self.log + f'{s}\n' else: region = regions.pop() try: assigned = RegionAssignee.objects.get(region=region).user except Exception: - s = 'No default assignee set for {}, defaulting to {}'.format(region, assignee_default) + s = f'No default assignee set for {region}, defaulting to {assignee_default}' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') assigned = assignee_default # Create/update the referral in PRS. @@ -240,15 +296,12 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, set_comment('Initial version.') if referral_preexists: - s = 'PRS referral updated: {}'.format(new_ref) - LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + s = f'PRS referral updated: {new_ref}' else: - s = 'New PRS referral generated: {}'.format(new_ref) - LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + s = f'New PRS referral generated: {new_ref}' + LOGGER.info(s) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Assign to a region. new_ref.regions.add(region) @@ -257,10 +310,10 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, new_ref.lga = LocalGovernment.objects.get(name=app['LOCAL_GOVERNMENT']) new_ref.save() except Exception: - s = 'LGA {} was not recognised'.format(app['LOCAL_GOVERNMENT']) + s = f'LGA {app["LOCAL_GOVERNMENT"]} was not recognised' LOGGER.warning(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Add triggers to the new referral. if 'MRSZONE_TEXT' in app: @@ -311,10 +364,10 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, new_loc.save() set_comment('Initial version.') new_locations.append(new_loc) - s = 'New PRS location generated: {}'.format(new_loc) + s = f'New PRS location generated: {new_loc}' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Check to see if new locations intersect with any existing locations. intersecting = [] @@ -326,10 +379,10 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, for l in intersecting: if l.referral.pk != new_ref.pk: new_ref.add_relationship(l.referral) - s = 'New referral {} related to existing referral {}'.format(new_ref.pk, l.referral.pk) + s = f'New referral {new_ref.pk} related to existing referral {l.referral.pk}' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Create an "Assess a referral" task and assign it to a user. if create_tasks: @@ -352,32 +405,32 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, with create_revision(): new_task.save() set_comment('Initial version.') - s = 'New PRS task generated: {} assigned to {}'.format(new_task, assigned.get_full_name()) + s = f'New PRS task generated: {new_task} assigned to {assigned.get_full_name()}' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Email the assigned user about the new task. new_task.email_user() - s = 'Task assignment email sent to {}'.format(assigned.email) + s = f'Task assignment email sent to {assigned.email}' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Save the EmailedReferral as a record on the referral. if create_records: new_record = Record.objects.create( name=self.subject, referral=new_ref, order_date=datetime.today()) - file_name = 'emailed_referral_{}.html'.format(reference) + file_name = f'emailed_referral_{reference}.html' new_file = ContentFile(str.encode(self.body)) new_record.uploaded_file.save(file_name, new_file) with create_revision(): new_record.save() set_comment('Initial version.') - s = 'New PRS record generated: {}'.format(new_record) + s = f'New PRS record generated: {new_record}' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Add records to the referral (one per attachment). for i in attachments: @@ -387,18 +440,20 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, new_file = ContentFile(i.attachment.read()) new_record.uploaded_file.save(i.name, new_file) new_record.save() - s = 'New PRS record generated: {}'.format(new_record) + s = f'New PRS record generated: {new_record}' LOGGER.info(s) - self.log = self.log + '{}\n'.format(s) - actions.append('{} {}'.format(datetime.now().isoformat(), s)) + self.log = self.log + f'{s}\n' + actions.append(f'{datetime.now().isoformat()} {s}') # Link the attachment to the new, generated record. i.record = new_record i.save() # Link the emailed referral to the new or existing referral. + LOGGER.info(f'Marking emailed referral {self.pk} as processed and linking it to {new_ref}') self.referral = new_ref self.processed = True self.save() + LOGGER.info('Done') return actions @@ -438,6 +493,6 @@ class RegionAssignee(models.Model): def __str__(self): if self.user: - return '{} -> {}'.format(self.region, self.user.get_full_name()) + return f'{self.region} -> {self.user.get_full_name()}' else: - return '{} -> none'.format(self.region) + return f'{self.region} -> none' diff --git a/prs2/harvester/utils.py b/prs2/harvester/utils.py index db6c14a1..a595be4e 100644 --- a/prs2/harvester/utils.py +++ b/prs2/harvester/utils.py @@ -9,9 +9,9 @@ from io import StringIO import logging from lxml.html import clean, fromstring -from pytz import timezone import requests import time +from zoneinfo import ZoneInfo LOGGER = logging.getLogger('harvester') @@ -29,7 +29,7 @@ def get_imap(mailbox='INBOX'): def unread_from_email(imap, from_email): """Returns (status, list of UIDs) of unread emails from a sender. """ - search = '(UNSEEN FROM "{}")'.format(from_email) + search = f'(UNSEEN FROM "{from_email}")' status, response = imap.search(None, search) if status != 'OK': return status, response @@ -66,7 +66,7 @@ def harvest_email(uid, message): if message.is_multipart(): # Should always be True. parts = [i for i in message.walk()] else: - LOGGER.error('Email UID {} is not of type multipart'.format(uid)) + LOGGER.error(f'Email UID {uid} is not of type multipart') return False message_body = None @@ -87,13 +87,13 @@ def harvest_email(uid, message): to_e = email.utils.parseaddr(message.get('To'))[1] # FIXME: skip the "To" whitelist check at present. # if not to_e.lower() in settings.ASSESSOR_EMAILS: - # LOGGER.info('Email UID {} to {} harvest was skipped'.format(uid, to_e)) + # LOGGER.info(f'Email UID {uid} to {to_e} harvest was skipped') # return None # Not in the whitelist; skip. from_e = email.utils.parseaddr(message.get('From'))[1] # Parse the 'sent' date & time (assume WST). - wa_tz = timezone('Australia/Perth') + wa_tz = ZoneInfo(settings.TIME_ZONE) ts = time.mktime(email.utils.parsedate(message.get('Date'))) - received = wa_tz.localize(datetime.fromtimestamp(ts)) + received = datetime.fromtimestamp(ts).astimezone(wa_tz) # Generate an EmailedReferral object. em_new = EmailedReferral( received=received, email_uid=str(uid), to_email=to_e, @@ -103,7 +103,7 @@ def harvest_email(uid, message): t = fromstring(clean.clean_html(message_body.get_payload())) em_new.body = t.text_content().replace('=\n', '').strip() em_new.save() - LOGGER.info('Email UID {} harvested: {}'.format(uid, em_new.subject)) + LOGGER.info(f'Email UID {uid} harvested: {em_new.subject}') for a in attachments: att_name = a.get_filename() att_new = EmailAttachment(emailed_referral=em_new, name=att_name) @@ -115,13 +115,13 @@ def harvest_email(uid, message): new_file = ContentFile(data) att_new.attachment.save(att_name, new_file) att_new.save() - LOGGER.info('Email attachment created: {}'.format(att_new.name)) + LOGGER.info(f'Email attachment created: {att_new.name}') except Exception as e: - LOGGER.error('Email UID {} generated exception during harvest'.format(uid)) + LOGGER.error(f'Email UID {uid} generated exception during harvest') LOGGER.exception(e) return None else: - LOGGER.error('Email UID {} had no message body'.format(uid)) + LOGGER.error(f'Email UID {uid} had no message body') return None return True @@ -155,17 +155,17 @@ def harvest_unread_emails(from_email, purge_email=False): actions = [] imap = get_imap() - LOGGER.info('Requesting unread emails from {}'.format(from_email)) - actions.append('{} Requesting unread emails from {}'.format(datetime.now().isoformat(), from_email)) + LOGGER.info(f'Requesting unread emails from {from_email}') + actions.append(f'{datetime.now().isoformat()} Requesting unread emails from {from_email}') status, uids = unread_from_email(imap, from_email) if status != 'OK': - LOGGER.error('Server response failure: {}'.status) - actions.append('{} Server response failure: {}'.format(datetime.now().isoformat(), status)) + LOGGER.error(f'Server response failure: {status}') + actions.append(f'{datetime.now().isoformat()} Server response failure: {status}') return actions - LOGGER.info('Server lists {} unread emails'.format(len(uids))) - actions.append('{} Server lists {} unread emails'.format(datetime.now().isoformat(), len(uids))) + LOGGER.info(f'Server lists {len(uids)} unread emails') + actions.append(f'{datetime.now().isoformat()} Server lists {len(uids)} unread emails') if uids: for uid in uids: @@ -174,13 +174,13 @@ def harvest_unread_emails(from_email, purge_email=False): uid = uid.decode('utf-8') # Fetch email message. - LOGGER.info('Fetching email UID {}'.format(uid)) + LOGGER.info(f'Fetching email UID {uid}') status, message = fetch_email(imap, uid) if status != 'OK': - LOGGER.error('Server response failure on fetching email UID {}: {}'.format(uid, status)) + LOGGER.error(f'Server response failure on fetching email UID {uid}: {status}') continue - LOGGER.info('Harvesting email UID {}'.format(uid)) - actions.append('{} Harvesting email UID {}'.format(datetime.now().isoformat(), uid)) + LOGGER.info(f'Harvesting email UID {uid}') + actions.append(f'{datetime.now().isoformat()} Harvesting email UID {uid}') harvest_email(uid, message) # Optionally mark email as read and flag it for deletion. @@ -188,14 +188,14 @@ def harvest_unread_emails(from_email, purge_email=False): # Mark email as read. status, response = email_mark_read(imap, uid) if status == 'OK': - LOGGER.info('Email UID {} was marked as "Read"'.format(uid)) + LOGGER.info(f'Email UID {uid} was marked as "Read"') # Mark email for deletion. status, response = email_delete(imap, uid) if status == 'OK': - LOGGER.info('Email UID {} was marked for deletion'.format(uid)) + LOGGER.info(f'Email UID {uid} was marked for deletion') - LOGGER.info('Harvest process completed ({})'.format(from_email)) - actions.append('{} Harvest process completed ({})'.format(datetime.now().isoformat(), from_email)) + LOGGER.info(f'Harvest process completed ({from_email})') + actions.append(f'{datetime.now().isoformat()} Harvest process completed ({from_email})') imap.expunge() imap.logout() @@ -209,16 +209,16 @@ def import_harvested_refs(): actions = [] LOGGER.info('Starting import of harvested referrals') - actions.append('{} Starting import of harvested referrals'.format(datetime.now().isoformat())) + actions.append(f'{datetime.now().isoformat()} Starting import of harvested referrals') # Process harvested refs that are unprocessed at present. for er in EmailedReferral.objects.filter(referral__isnull=True, processed=False): try: actions.append(er.harvest()) except Exception: - actions.append('Emailed referral {} failed to import; notify the custodian to investigate'.format(er)) + actions.append(f'Emailed referral {er} failed to import; notify the custodian to investigate') LOGGER.info('Import process completed') - actions.append('{} Import process completed'.format(datetime.now().isoformat())) + actions.append(f'{datetime.now().isoformat()} Import process completed') return actions @@ -226,7 +226,8 @@ def email_harvest_actions(to_emails, actions): """Function to email a log of harvest actions to users. Accepts a list of emails and list of actions to append. """ - subject = 'PRS emailed referral harvest log {}'.format(date.today().strftime('%x')) + ts = date.today().strftime('%x') + subject = f'PRS emailed referral harvest log {ts}' from_email = 'PRS-Alerts@dbca.wa.gov.au' text_content = '''This is an automated message to summarise harvest actions undertaken for emailed referrals.\n @@ -235,8 +236,8 @@ def email_harvest_actions(to_emails, actions): actions undertaken for emailed referrals.

Actions:

''' for l in actions: - text_content += '{}\n'.format(l) - html_content += '{}
'.format(l) + text_content += f'{l}\n' + html_content += f'{l}
' msg = EmailMultiAlternatives(subject, text_content, from_email, to_emails) msg.attach_alternative(html_content, 'text/html') # Email should fail gracefully (no Exception raised on failure). @@ -256,7 +257,7 @@ def query_slip(pin): 'typeName': type_name, 'request': 'getFeature', 'outputFormat': 'json', - 'cql_filter': 'polygon_number={}'.format(pin), + 'cql_filter': f'polygon_number={pin}', } resp = requests.get(url, auth=auth, params=params) return resp @@ -274,7 +275,7 @@ def query_slip_esri(pin): 'outSR': 4326, 'outFields': '*', 'returnGeometry': 'true', - 'where': 'polygon_number={}'.format(pin), + 'where': f'polygon_number={pin}', } resp = requests.get(url, auth=auth, params=params) resp.raise_for_status() diff --git a/pyproject.toml b/pyproject.toml index 8f034368..f542d322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "prs" -version = "2.5.38" +version = "2.5.39" description = "Planning Referral System corporate application" authors = ["Ashley Felton "] license = "Apache-2.0" @@ -25,7 +25,7 @@ openpyxl = "3.1.2" xmltodict = "0.13.0" extract-msg = "0.47.0" django-extensions = "3.2.3" -webtemplate-dbca = "1.6.0" +webtemplate-dbca = "1.7.0" mixer = "7.2.2" geojson = "3.1.0" typesense = "0.19.0" @@ -38,7 +38,7 @@ whitenoise = {version = "6.6.0", extras = ["brotli"]} django-crum = "0.7.9" django-storages = {version = "1.14.2", extras = ["azure"]} tomli = "2.0.1" -sentry-sdk = {version = "1.40.3", extras = ["django"]} +sentry-sdk = {version = "1.40.4", extras = ["django"]} crispy-bootstrap5 = "0.7" django-redis = "5.4.0"