From 102ddc5151f4b3a976728f4444aa98e688731d19 Mon Sep 17 00:00:00 2001 From: Ashley Felton Date: Mon, 19 Feb 2024 08:21:25 +0800 Subject: [PATCH 1/5] Add referral URL to RecordAdmin. --- prs2/referral/admin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/prs2/referral/admin.py b/prs2/referral/admin.py index bb21f550..6bce50ac 100644 --- a/prs2/referral/admin.py +++ b/prs2/referral/admin.py @@ -1,6 +1,8 @@ # Core Django imports from django.contrib.gis import admin from django.contrib.gis.admin import ModelAdmin +from django.urls import reverse +from django.utils.safestring import mark_safe # Third-party app imports from reversion.admin import VersionAdmin @@ -170,6 +172,7 @@ class RecordAdmin(ReferralBaseModelAdmin): list_display = ( "id", "name", + "referral_url", "infobase_id", "creator", "created", @@ -180,6 +183,13 @@ class RecordAdmin(ReferralBaseModelAdmin): date_hierarchy = "created" search_fields = ("id", "name", "infobase_id", "description") + def referral_url(self, instance): + if not instance.referral: + return "" + url = reverse("admin:referral_referral_change", args=[instance.referral.pk]) + return mark_safe(f"{instance.referral.pk}") + referral_url.short_description = "Referral" + class NoteAdmin(ReferralBaseModelAdmin): list_display = ( From e2989693978f512817d98c42570c94cdd30844e1 Mon Sep 17 00:00:00 2001 From: Ashley Felton Date: Mon, 19 Feb 2024 08:22:23 +0800 Subject: [PATCH 2/5] Add attachments_url to EmailedReferralAdmin. --- prs2/harvester/admin.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/prs2/harvester/admin.py b/prs2/harvester/admin.py index 3671a5c3..59540c46 100644 --- a/prs2/harvester/admin.py +++ b/prs2/harvester/admin.py @@ -29,21 +29,26 @@ def harvest_referral(modeladmin, request, queryset): actions = [harvest_referral] date_hierarchy = 'received' list_display = ( - 'received', 'from_email', 'subject', 'harvested', 'attachments', + 'received', 'from_email', 'subject', 'harvested', 'attachments_url', 'referral_url', 'processed') raw_id_fields = ('referral',) search_fields = ('subject',) readonly_fields = ['email_uid', 'to_email', 'from_email', 'subject', 'body', 'processed', 'log'] - def attachments(self, instance): - return instance.emailattachment_set.count() + def attachments_url(self, instance): + if not instance.emailattachment_set.exists(): + return "" + count = instance.emailattachment_set.count() + url = reverse("admin:harvester_emailattachment_changelist") + return mark_safe(f"{count}") + attachments_url.short_description = 'attachments' def referral_url(self, instance): if not instance.referral: - return '' - url = reverse('admin:referral_referral_change', args=[instance.referral.pk]) - return mark_safe('{}'.format(url, instance.referral.pk)) - referral_url.short_description = 'Referral' + return "" + url = reverse("admin:referral_referral_change", args=[instance.referral.pk]) + return mark_safe(f"{instance.referral.pk}") + referral_url.short_description = "Referral" @admin.register(EmailAttachment) @@ -55,14 +60,14 @@ class EmailAttachmentAdmin(admin.ModelAdmin): def emailed_referral_url(self, instance): url = reverse('admin:harvester_emailedreferral_change', args=[instance.emailed_referral.pk]) - return mark_safe('{}'.format(url, instance.emailed_referral)) + return mark_safe(f'{instance.emailed_referral}') emailed_referral_url.short_description = 'Emailed referral' def record_url(self, instance): if not instance.record: return '' url = reverse('admin:referral_record_change', args=[instance.record.pk]) - return mark_safe('{}'.format(url, instance.record.pk)) + return mark_safe(f'{instance.record.pk}') record_url.short_description = 'Record' From 3f46841e093ba45d5457fcf481f72ade561b3f73 Mon Sep 17 00:00:00 2001 From: Ashley Felton Date: Mon, 19 Feb 2024 10:42:02 +0800 Subject: [PATCH 3/5] harvest_email: also harvest plain text body emails. --- prs2/harvester/utils.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/prs2/harvester/utils.py b/prs2/harvester/utils.py index a595be4e..b6bbb374 100644 --- a/prs2/harvester/utils.py +++ b/prs2/harvester/utils.py @@ -72,13 +72,13 @@ def harvest_email(uid, message): message_body = None attachments = [] - for p in parts: - # 'text/html' content is the email body. - if p.get_content_type() == 'text/html': - message_body = p + for part in parts: + # 'text/html' or 'text/plain' content should be the email body. + if part.get_content_type() in ['text/html', 'text/plain']: + message_body = part # Other content types (not multipart/mixed) are attachments (normally application/octet-stream). - elif p.get_content_type() != 'multipart/mixed': - attachments.append(p) + elif part.get_content_type() != 'multipart/mixed': + attachments.append(part) # Create & return EmailedReferral from the email body (if found). if message_body: @@ -99,9 +99,12 @@ def harvest_email(uid, message): received=received, email_uid=str(uid), to_email=to_e, from_email=from_e, subject=message.get('Subject'), ) - # Strip the HTML from the message body and just save the text content. - t = fromstring(clean.clean_html(message_body.get_payload())) - em_new.body = t.text_content().replace('=\n', '').strip() + if message_body.get_content_type() == 'text/html': + # Strip the HTML from the message body and just save the text content. + t = fromstring(clean.clean_html(message_body.get_payload())) + em_new.body = t.text_content().replace('=\n', '').strip() + else: + em_new.body = message_body.get_payload() # Plain text body. em_new.save() LOGGER.info(f'Email UID {uid} harvested: {em_new.subject}') for a in attachments: @@ -164,7 +167,7 @@ def harvest_unread_emails(from_email, purge_email=False): actions.append(f'{datetime.now().isoformat()} Server response failure: {status}') return actions - LOGGER.info(f'Server lists {len(uids)} unread emails') + LOGGER.info(f'Server lists {len(uids)} unread emails from {from_email}') actions.append(f'{datetime.now().isoformat()} Server lists {len(uids)} unread emails') if uids: From ef6579ec58f81ceca6ca36d20e542a720c74fcea Mon Sep 17 00:00:00 2001 From: Ashley Felton Date: Mon, 19 Feb 2024 10:48:58 +0800 Subject: [PATCH 4/5] Add email harvest scenario: WAPC decision letter for existing referral. --- prs2/harvester/models.py | 347 +++++++++++++++++++++++++-------------- 1 file changed, 220 insertions(+), 127 deletions(-) diff --git a/prs2/harvester/models.py b/prs2/harvester/models.py index 2c870fe7..78f57fc5 100644 --- a/prs2/harvester/models.py +++ b/prs2/harvester/models.py @@ -64,30 +64,31 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, attachments = self.emailattachment_set.all() self.log = '' - # Emails without attachments are usually reminder notices. + # SCENARIO: no email attachments, skip harvest. if not attachments.exists(): - s = f'Skipping emailed referral {self.pk} (no attachments)' - LOGGER.info(s) - self.log = s + log = f'Skipping emailed referral {self.pk} (no attachments)' + LOGGER.info(log) + self.log = log self.processed = True self.save() - actions.append(f'{datetime.now().isoformat()} {s}') + actions.append(f'{datetime.now().isoformat()} {log}') return actions - # We don't want to harvest "overdue referral" reminders. + # SCENARIO: overdue referral reminders: skip harvest. overdue_subject_prefixes = ( 'wapc eoverdue referral', 're: wapc eoverdue referral', ) if any([self.subject.lower().startswith(i) for i in overdue_subject_prefixes]): - s = f'Skipping harvested referral {self} (overdue notice)' - LOGGER.info(s) - self.log = s + log = f'Skipping harvested referral {self} (overdue notice)' + LOGGER.info(log) + self.log = log self.processed = True self.save() - actions.append(f'{datetime.now().isoformat()} {s}') + actions.append(f'{datetime.now().isoformat()} {log}') return actions + # SCENARIO: email referral supplement. # Some emailed referrals contain "additional documents" where the size # limit of attachments have been exceeded for the first email. additional_documents_subject = ( @@ -95,15 +96,17 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, 'additional referral documents', ) if any([i in self.subject.lower() for i in additional_documents_subject]) and attachments.exists(): + log = f'Harvested referral {self} appears to be a supplement to an existing email' + LOGGER.info(log) # Try to parse the referral reference from the email subject (it's normally the first # element in the string). reference = self.subject.split()[0] if Referral.objects.current().filter(reference__iexact=reference): referral = Referral.objects.current().filter(reference__iexact=reference).order_by('-pk').first() - s = f'Referral ref. {reference} is already in database; using existing referral {referral}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'Referral ref. {reference} is already in database; using existing referral {referral.pk}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') self.referral = referral # Save the EmailedReferral as a record on the referral. @@ -116,10 +119,10 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, with create_revision(): new_record.save() set_comment('Initial version.') - s = f'New PRS record generated: {new_record}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'New PRS record generated: {new_record}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Add records to the referral (one per attachment). for att in attachments: @@ -129,28 +132,115 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, new_file = ContentFile(att.attachment.read()) new_record.uploaded_file.save(att.name, new_file) new_record.save() - s = f'New PRS record generated: {new_record}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'New PRS record generated: {new_record}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Link the attachment to the new, generated record. att.record = new_record att.save() + else: + LOGGER.info(f'Referral ref. {reference} not found, skipping') LOGGER.info(f'Marking emailed referral {self.pk} as processed') self.processed = True self.save() LOGGER.info('Done') + + return actions + + # SCENARIO: WAPC decision letter for a referral. + # Existing referral should exist, and the email should include application.xml + if self.subject.lower().startswith('wapc decision letter for application'): + if not attachments.filter(name__istartswith='application.xml'): + log = f'Skipping harvested decision letter {self.pk} (no XML attachment)' + LOGGER.info(log) + self.log = log + self.processed = True + self.save() + actions.append(f'{datetime.now().isoformat()} {log}') + return actions + else: + xml_file = attachments.get(name__istartswith='application.xml') + # Parse the attached XML file. + try: + d = xmltodict.parse(xml_file.attachment.read()) + except Exception as e: + log = f'Harvested decision letter {self.pk} parsing of application.xml failed' + LOGGER.error(log) + LOGGER.exception(e) + self.log = self.log + f'{log}\n{e}\n' + LOGGER.info(f'Marking emailed decision letter {self.pk} as processed') + self.processed = True + self.save() + LOGGER.info('Done') + actions.append(f'{datetime.now().isoformat()} {log}') + return actions + app = d['APPLICATION'] + reference = app['WAPC_APPLICATION_NO'] + # New/existing referral object. + if not Referral.objects.current().filter(reference__iexact=reference).exists(): + log = f'Skipping harvested decision letter {self.pk} (no existing record)' + LOGGER.info(log) + self.log = log + self.processed = True + self.save() + actions.append(f'{datetime.now().isoformat()} {log}') + return actions + else: + log = f'Referral ref. {reference} exists in database' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') + referral = Referral.objects.current().filter(reference__iexact=reference).order_by('-pk').first() + # Save the EmailedReferral as a record on the referral. + if create_records: + new_record = Record.objects.create( + name=self.subject, referral=referral, order_date=datetime.today()) + file_name = f'emailed_referral_{reference}.html' + new_file = ContentFile(str.encode(self.body)) + new_record.uploaded_file.save(file_name, new_file) + with create_revision(): + new_record.save() + set_comment('Initial version.') + log = f'New PRS record generated: {new_record}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') + + # Add records to the referral (one per attachment). + for i in attachments: + new_record = Record.objects.create( + name=i.name, referral=referral, order_date=datetime.today()) + # Duplicate the uploaded file. + new_file = ContentFile(i.attachment.read()) + new_record.uploaded_file.save(i.name, new_file) + new_record.save() + log = f'New PRS record generated: {new_record}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') + # Link the attachment to the new, generated record. + i.record = new_record + i.save() + + # Link the emailed referral to the new or existing referral. + LOGGER.info(f'Marking emailed referral {self.pk} as processed and linking it to {referral}') + self.referral = referral + self.processed = True + self.save() + LOGGER.info('Done') return actions + # SCENARIO: a standard emailed referral. # Must be an attachment named 'Application.xml' present to import. if not attachments.filter(name__istartswith='application.xml'): - s = f'Skipping harvested referral {self.pk} (no XML attachment)' - LOGGER.info(s) - self.log = s + log = f'Skipping harvested referral {self.pk} (no XML attachment)' + LOGGER.info(log) + self.log = log self.processed = True self.save() - actions.append(f'{datetime.now().isoformat()} {s}') + actions.append(f'{datetime.now().isoformat()} {log}') return actions else: xml_file = attachments.get(name__istartswith='application.xml') @@ -159,44 +249,48 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, try: d = xmltodict.parse(xml_file.attachment.read()) except Exception as e: - s = f'Harvested referral {self.pk} parsing of application.xml failed' - LOGGER.error(s) + log = f'Harvested referral {self.pk} parsing of application.xml failed' + LOGGER.error(log) LOGGER.exception(e) - self.log = self.log + f'{s}\n{e}\n' + self.log = self.log + f'{log}\n{e}\n' + LOGGER.info(f'Marking emailed referral {self.pk} as processed') self.processed = True self.save() - actions.append(f'{datetime.now().isoformat()} {s}') + LOGGER.info('Done') + actions.append(f'{datetime.now().isoformat()} {log}') + return actions + app = d['APPLICATION'] reference = app['WAPC_APPLICATION_NO'] # New/existing referral object. if Referral.objects.current().filter(reference__iexact=reference): # Note if the the reference no. exists in PRS already. - s = f'Referral ref. {reference} is already in database; using existing referral' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') - new_ref = Referral.objects.current().filter(reference__iexact=reference).order_by('-pk').first() + log = f'Referral ref. {reference} is already in database; using existing referral' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') + referral = Referral.objects.current().filter(reference__iexact=reference).order_by('-pk').first() referral_preexists = True else: # No match with existing references. - s = f'Importing harvested referral ref. {reference} as new entity' - LOGGER.info(s) - self.log = f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') - new_ref = Referral(reference=reference) + log = f'Importing harvested referral ref. {reference} as new entity' + LOGGER.info(log) + self.log = f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') + referral = Referral(reference=reference) referral_preexists = False # Referral type try: ref_type = ReferralType.objects.filter(name__istartswith=app['APP_TYPE'])[0] except Exception: - s = f'Referral type {app["APP_TYPE"]} is not recognised type; skipping' - LOGGER.warning(s) - self.log = f'{s}\n' + log = f'Referral type {app["APP_TYPE"]} is not recognised type; skipping' + LOGGER.warning(log) + self.log = f'{log}\n' self.processed = True self.save() - actions.append(f'{datetime.now().isoformat()} {s}') + actions.append(f'{datetime.now().isoformat()} {log}') return actions # Determine the intersecting region(s). @@ -220,10 +314,10 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, regions.append(r) intersected_region = True except Exception: - s = f'Address long/lat could not be parsed ({a["LONGITUDE"]}, {a["LATITUDE"]})' - LOGGER.warning(s) - self.log = f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'Address long/lat could not be parsed ({a["LONGITUDE"]}, {a["LATITUDE"]})' + LOGGER.warning(log) + self.log = f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') intersected_region = False # Use the PIN field to try returning geometry from SLIP. if 'PIN' in a and a['PIN']: @@ -242,19 +336,19 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, for r in Region.objects.all(): if r.region_mpoly and r.region_mpoly.intersects(p) and r not in regions: regions.append(r) - s = f'Address PIN {a["PIN"]} returned geometry from SLIP' - self.log = self.log + f'{s}\n' - LOGGER.info(s) + log = f'Address PIN {a["PIN"]} returned geometry from SLIP' + self.log = self.log + f'{log}\n' + LOGGER.info(log) except Exception as e: - s = f'Error querying Landgate SLIP for spatial data (referral ref. {reference})' - LOGGER.error(s) + log = f'Error querying Landgate SLIP for spatial data (referral ref. {reference})' + LOGGER.error(log) LOGGER.error(resp.content) LOGGER.exception(e) - self.log = self.log + f'{s}\n{resp.content}\n{e}\n' + self.log = self.log + f'{log}\n{resp.content}\n{e}\n' else: - s = f'Address PIN could not be parsed ({a["PIN"]})' - LOGGER.warning(s) - self.log = self.log + f'{s}\n' + log = f'Address PIN could not be parsed ({a["PIN"]})' + LOGGER.warning(log) + self.log = self.log + f'{log}\n' regions = set(regions) # Business rules: # Didn't intersect a region? Might be bad geometry in the XML. @@ -263,57 +357,57 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, if len(regions) == 0: region = Region.objects.get(name='Swan') assigned = assignee_default - s = f'No regions were intersected, defaulting to {region} ({assigned})' - LOGGER.info(s) - self.log = self.log + f'{s}\n' + log = f'No regions were intersected, defaulting to {region} ({assigned})' + LOGGER.info(log) + self.log = self.log + f'{log}\n' elif len(regions) > 1: region = Region.objects.get(name='Swan') assigned = assignee_default - s = f'>1 regions were intersected ({regions}), defaulting to {region} ({assigned})' - LOGGER.info(s) - self.log = self.log + f'{s}\n' + log = f'>1 regions were intersected ({regions}), defaulting to {region} ({assigned})' + LOGGER.info(log) + self.log = self.log + f'{log}\n' else: region = regions.pop() try: assigned = RegionAssignee.objects.get(region=region).user except Exception: - s = f'No default assignee set for {region}, defaulting to {assignee_default}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'No default assignee set for {region}, defaulting to {assignee_default}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') assigned = assignee_default # Create/update the referral in PRS. - new_ref.type = ref_type - new_ref.agency = dbca - new_ref.referring_org = wapc - new_ref.reference = reference - new_ref.description = app['DEVELOPMENT_DESCRIPTION'] if 'DEVELOPMENT_DESCRIPTION' in app else '' - new_ref.referral_date = self.received.date() - new_ref.address = app['LOCATION'] if 'LOCATION' in app else '' + referral.type = ref_type + referral.agency = dbca + referral.referring_org = wapc + referral.reference = reference + referral.description = app['DEVELOPMENT_DESCRIPTION'] if 'DEVELOPMENT_DESCRIPTION' in app else '' + referral.referral_date = self.received.date() + referral.address = app['LOCATION'] if 'LOCATION' in app else '' with create_revision(): - new_ref.save() + referral.save() set_comment('Initial version.') if referral_preexists: - s = f'PRS referral updated: {new_ref}' + log = f'PRS referral updated: {referral}' else: - s = f'New PRS referral generated: {new_ref}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'New PRS referral generated: {referral}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Assign to a region. - new_ref.regions.add(region) + referral.regions.add(region) # Assign an LGA. try: - new_ref.lga = LocalGovernment.objects.get(name=app['LOCAL_GOVERNMENT']) - new_ref.save() + referral.lga = LocalGovernment.objects.get(name=app['LOCAL_GOVERNMENT']) + referral.save() except Exception: - s = f'LGA {app["LOCAL_GOVERNMENT"]} was not recognised' - LOGGER.warning(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'LGA {app["LOCAL_GOVERNMENT"]} was not recognised' + LOGGER.warning(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Add triggers to the new referral. if 'MRSZONE_TEXT' in app: @@ -325,20 +419,20 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, # A couple of exceptions for DoP triggers follow (specific -> general trigger). if i.startswith('BUSH FOREVER SITE'): added_trigger = True - new_ref.dop_triggers.add(DopTrigger.objects.get(name='Bush Forever site')) + referral.dop_triggers.add(DopTrigger.objects.get(name='Bush Forever site')) elif i.startswith('DPW ESTATE'): added_trigger = True - new_ref.dop_triggers.add(DopTrigger.objects.get(name='Parks and Wildlife estate')) + referral.dop_triggers.add(DopTrigger.objects.get(name='Parks and Wildlife estate')) elif i.find('REGIONAL PARK') > -1: added_trigger = True - new_ref.dop_triggers.add(DopTrigger.objects.get(name='Regional Park')) + referral.dop_triggers.add(DopTrigger.objects.get(name='Regional Park')) # All other triggers (don't use exists() in case of duplicates). elif DopTrigger.objects.current().filter(name__istartswith=i).count() == 1: added_trigger = True - new_ref.dop_triggers.add(DopTrigger.objects.current().get(name__istartswith=i)) + referral.dop_triggers.add(DopTrigger.objects.current().get(name__istartswith=i)) # If we didn't link any DoP triggers, link the "No Parks and Wildlife trigger" tag. if not added_trigger: - new_ref.dop_triggers.add(DopTrigger.objects.get(name='No Parks and Wildlife trigger')) + referral.dop_triggers.add(DopTrigger.objects.get(name='No Parks and Wildlife trigger')) # Add locations to the new referral (one per polygon in each MP geometry). if create_locations: @@ -353,7 +447,7 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, road_suffix=l['STREET_SUFFIX'], locality=l['SUBURB'], postcode=l['POSTCODE'], - referral=new_ref, + referral=referral, poly=geom ) try: # NUMBER_FROM XML fields started to contain non-integer values :( @@ -364,10 +458,10 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, new_loc.save() set_comment('Initial version.') new_locations.append(new_loc) - s = f'New PRS location generated: {new_loc}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'New PRS location generated: {new_loc}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Check to see if new locations intersect with any existing locations. intersecting = [] @@ -377,20 +471,20 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, intersecting += list(other_l) # For any intersecting locations, relate the new and existing referrals. for l in intersecting: - if l.referral.pk != new_ref.pk: - new_ref.add_relationship(l.referral) - s = f'New referral {new_ref.pk} related to existing referral {l.referral.pk}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + if l.referral.pk != referral.pk: + referral.add_relationship(l.referral) + log = f'New referral {referral.pk} related to existing referral {l.referral.pk}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Create an "Assess a referral" task and assign it to a user. if create_tasks: new_task = Task( type=assess_task, - referral=new_ref, - start_date=new_ref.referral_date, - description=new_ref.description, + referral=referral, + start_date=referral.referral_date, + description=referral.description, assigned_user=assigned ) new_task.state = assess_task.initial_state @@ -405,56 +499,55 @@ def harvest(self, create_tasks=True, create_locations=True, create_records=True, with create_revision(): new_task.save() set_comment('Initial version.') - s = f'New PRS task generated: {new_task} assigned to {assigned.get_full_name()}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'New PRS task generated: {new_task} assigned to {assigned.get_full_name()}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Email the assigned user about the new task. new_task.email_user() - s = f'Task assignment email sent to {assigned.email}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'Task assignment email sent to {assigned.email}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Save the EmailedReferral as a record on the referral. if create_records: new_record = Record.objects.create( - name=self.subject, referral=new_ref, order_date=datetime.today()) + name=self.subject, referral=referral, order_date=datetime.today()) file_name = f'emailed_referral_{reference}.html' new_file = ContentFile(str.encode(self.body)) new_record.uploaded_file.save(file_name, new_file) with create_revision(): new_record.save() set_comment('Initial version.') - s = f'New PRS record generated: {new_record}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'New PRS record generated: {new_record}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Add records to the referral (one per attachment). for i in attachments: new_record = Record.objects.create( - name=i.name, referral=new_ref, order_date=datetime.today()) + name=i.name, referral=referral, order_date=datetime.today()) # Duplicate the uploaded file. new_file = ContentFile(i.attachment.read()) new_record.uploaded_file.save(i.name, new_file) new_record.save() - s = f'New PRS record generated: {new_record}' - LOGGER.info(s) - self.log = self.log + f'{s}\n' - actions.append(f'{datetime.now().isoformat()} {s}') + log = f'New PRS record generated: {new_record}' + LOGGER.info(log) + self.log = self.log + f'{log}\n' + actions.append(f'{datetime.now().isoformat()} {log}') # Link the attachment to the new, generated record. i.record = new_record i.save() # Link the emailed referral to the new or existing referral. - LOGGER.info(f'Marking emailed referral {self.pk} as processed and linking it to {new_ref}') - self.referral = new_ref + LOGGER.info(f'Marking emailed referral {self.pk} as processed and linking it to {referral}') + self.referral = referral self.processed = True self.save() LOGGER.info('Done') - return actions From 646b57595a13f86353dd82eadfce91b7eb552f64 Mon Sep 17 00:00:00 2001 From: Ashley Felton Date: Mon, 19 Feb 2024 11:05:46 +0800 Subject: [PATCH 5/5] Increment project minor version. --- kustomize/overlays/prod/kustomization.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize/overlays/prod/kustomization.yaml b/kustomize/overlays/prod/kustomization.yaml index 64195fda..000da491 100644 --- a/kustomize/overlays/prod/kustomization.yaml +++ b/kustomize/overlays/prod/kustomization.yaml @@ -28,4 +28,4 @@ patches: - path: geoserver_service_patch.yaml images: - name: ghcr.io/dbca-wa/prs - newTag: 2.5.39 + newTag: 2.5.40 diff --git a/pyproject.toml b/pyproject.toml index f542d322..eeb90c77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "prs" -version = "2.5.39" +version = "2.5.40" description = "Planning Referral System corporate application" authors = ["Ashley Felton "] license = "Apache-2.0"