From 4ab4b676af04038c8a3f0ecf38a809e427823f6c Mon Sep 17 00:00:00 2001 From: David Kane Date: Fri, 12 Feb 2021 13:51:26 +0000 Subject: [PATCH 01/19] reorganise models into separate files --- findthatcharity/settings.py | 35 ++ ftc/models.py | 850 ----------------------------- ftc/models/__init__.py | 21 + ftc/models/organisation.py | 361 ++++++++++++ ftc/models/organisation_link.py | 22 + ftc/models/organisation_type.py | 37 ++ ftc/models/orgid.py | 41 ++ ftc/models/orgid_scheme.py | 31 ++ ftc/models/related_organisation.py | 282 ++++++++++ ftc/models/scrape.py | 31 ++ ftc/models/source.py | 27 + 11 files changed, 888 insertions(+), 850 deletions(-) delete mode 100644 ftc/models.py create mode 100644 ftc/models/__init__.py create mode 100644 ftc/models/organisation.py create mode 100644 ftc/models/organisation_link.py create mode 100644 ftc/models/organisation_type.py create mode 100644 ftc/models/orgid.py create mode 100644 ftc/models/orgid_scheme.py create mode 100644 ftc/models/related_organisation.py create mode 100644 ftc/models/scrape.py create mode 100644 ftc/models/source.py diff --git a/findthatcharity/settings.py b/findthatcharity/settings.py index ab17243f..8852d41c 100644 --- a/findthatcharity/settings.py +++ b/findthatcharity/settings.py @@ -185,3 +185,38 @@ } CORS_ALLOW_ALL_ORIGINS = True + +IGNORE_DOMAINS = ( + "gmail.com", + "hotmail.com", + "btinternet.com", + "hotmail.co.uk", + "yahoo.co.uk", + "outlook.com", + "aol.com", + "btconnect.com", + "yahoo.com", + "googlemail.com", + "ntlworld.com", + "talktalk.net", + "sky.com", + "live.co.uk", + "ntlworld.com", + "tiscali.co.uk", + "icloud.com", + "btopenworld.com", + "blueyonder.co.uk", + "virginmedia.com", + "nhs.net", + "me.com", + "msn.com", + "talk21.com", + "aol.co.uk", + "mail.com", + "live.com", + "virgin.net", + "ymail.com", + "mac.com", + "waitrose.com", + "gmail.co.uk", +) diff --git a/ftc/models.py b/ftc/models.py deleted file mode 100644 index c910264f..00000000 --- a/ftc/models.py +++ /dev/null @@ -1,850 +0,0 @@ -import datetime -import re - -from django.contrib.postgres.indexes import GinIndex -from django.db import models -from django.urls import reverse -from django.utils.text import slugify -from django_better_admin_arrayfield.models.fields import ArrayField - -IGNORE_DOMAINS = ( - "gmail.com", - "hotmail.com", - "btinternet.com", - "hotmail.co.uk", - "yahoo.co.uk", - "outlook.com", - "aol.com", - "btconnect.com", - "yahoo.com", - "googlemail.com", - "ntlworld.com", - "talktalk.net", - "sky.com", - "live.co.uk", - "ntlworld.com", - "tiscali.co.uk", - "icloud.com", - "btopenworld.com", - "blueyonder.co.uk", - "virginmedia.com", - "nhs.net", - "me.com", - "msn.com", - "talk21.com", - "aol.co.uk", - "mail.com", - "live.com", - "virgin.net", - "ymail.com", - "mac.com", - "waitrose.com", - "gmail.co.uk", -) - - -EXTERNAL_LINKS = { - "GB-CHC": [ - [ - "https://register-of-charities.charitycommission.gov.uk/charity-details/?regId={}&subId=0", - "Charity Commission England and Wales", - ], - ["https://charitybase.uk/charities/{}", "CharityBase"], - ["http://opencharities.org/charities/{}", "OpenCharities"], - ["https://givingisgreat.org/charitydetail/?regNo={}", "Giving is Great"], - [ - "http://www.charitychoice.co.uk/charities/search?t=qsearch&q={}", - "Charities Direct", - ], - ], - "GB-COH": [ - ["https://beta.companieshouse.gov.uk/company/{}", "Companies House"], - ["https://opencorporates.com/companies/gb/{}", "Opencorporates"], - ], - "GB-NIC": [ - [ - "http://www.charitycommissionni.org.uk/charity-details/?regid={}&subid=0", - "Charity Commission Northern Ireland", - ], - ["https://givingisgreat.org/charitydetail/?regNo={}", "Giving is Great"], - ], - "GB-SC": [ - [ - "https://www.oscr.org.uk/about-charities/search-the-register/charity-details?number={}", - "Office of Scottish Charity Regulator", - ], - ["https://givingisgreat.org/charitydetail/?regNo={}", "Giving is Great"], - ], - "GB-EDU": [ - [ - "https://get-information-schools.service.gov.uk/Establishments/Establishment/Details/{}", - "Get information about schools", - ], - ], - "GB-UKPRN": [ - [ - "https://www.ukrlp.co.uk/ukrlp/ukrlp_provider.page_pls_provDetails?x=&pn_p_id={}&pv_status=VERIFIED&pv_vis_code=L", - "UK Register of Learning Providers", - ], - ], - "GB-NHS": [ - ["https://odsportal.hscic.gov.uk/Organisation/Details/{}", "NHS Digital"], - ], - "GB-LAE": [ - [ - "https://www.registers.service.gov.uk/registers/local-authority-eng/records/{}", - "Local authorities in England", - ], - ], - "GB-LAN": [ - [ - "https://www.registers.service.gov.uk/registers/local-authority-nir/records/{}", - "Local authorities in Northern Ireland", - ], - ], - "GB-LAS": [ - [ - "https://www.registers.service.gov.uk/registers/local-authority-sct/records/{}", - "Local authorities in Scotland", - ], - ], - "GB-PLA": [ - [ - "https://www.registers.service.gov.uk/registers/principal-local-authority/records/{}", - "Principal Local authorities in Wales", - ], - ], - "GB-GOR": [ - [ - "https://www.registers.service.gov.uk/registers/government-organisation/records/{}", - "Government organisations on GOV.UK", - ], - ], - "XI-GRID": [ - ["https://www.grid.ac/institutes/{}", "Global Research Identifier Database"], - ], -} - - -class Orgid(str): - def __new__(cls, content): - instance = super().__new__(cls, content) - instance._split_orgid(content) - return instance - - def _split_orgid(self, value): - self.scheme = None - self.id = value - if value is None: - return - split_orgid = value.split("-", maxsplit=3) - if len(split_orgid) > 2: - self.scheme = "-".join(split_orgid[:2]) - self.id = "-".join(split_orgid[2:]) - - -class OrgidField(models.CharField): - - description = "An orgid based on the format here: http://org-id.guide/about" - - def __init__(self, *args, **kwargs): - kwargs["max_length"] = 200 - super().__init__(*args, **kwargs) - - def from_db_value(self, value, expression, connection): - if value is None: - return value - return Orgid(value) - - def to_python(self, value): - if isinstance(value, Orgid): - return value - - if value is None: - return value - - return Orgid(value) - - -class Organisation(models.Model): - org_id = OrgidField(db_index=True, verbose_name="Organisation Identifier") - orgIDs = ArrayField( - OrgidField(blank=True), - verbose_name="Other organisation identifiers", - ) - linked_orgs = ArrayField( - models.CharField(max_length=100, blank=True), - blank=True, - null=True, - verbose_name="Linked organisations", - ) - name = models.CharField(max_length=255, verbose_name="Name") - alternateName = ArrayField( - models.CharField(max_length=255, blank=True), - blank=True, - null=True, - verbose_name="Other names", - ) - charityNumber = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Charity number" - ) - companyNumber = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Company number" - ) - streetAddress = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Address: street" - ) - addressLocality = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Address: locality" - ) - addressRegion = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Address: region" - ) - addressCountry = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Address: country" - ) - postalCode = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Postcode", db_index=True - ) - telephone = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Telephone" - ) - email = models.CharField( - max_length=255, null=True, blank=True, verbose_name="Email address" - ) - description = models.TextField(null=True, blank=True, verbose_name="Description") - url = models.URLField(null=True, blank=True, verbose_name="Website address") - domain = models.CharField( - max_length=255, - null=True, - blank=True, - db_index=True, - verbose_name="Website domain", - ) - latestIncome = models.BigIntegerField( - null=True, blank=True, verbose_name="Latest income" - ) - latestIncomeDate = models.DateField( - null=True, blank=True, verbose_name="Latest financial year end" - ) - dateRegistered = models.DateField( - null=True, blank=True, verbose_name="Date registered" - ) - dateRemoved = models.DateField(null=True, blank=True, verbose_name="Date removed") - active = models.BooleanField(null=True, blank=True, verbose_name="Active") - status = models.CharField( - max_length=200, null=True, blank=True, verbose_name="Status" - ) - parent = models.CharField( - max_length=200, null=True, blank=True, verbose_name="Parent organisation" - ) - dateModified = models.DateTimeField( - auto_now=True, verbose_name="Date record was last modified" - ) - source = models.ForeignKey( - "Source", - related_name="organisations", - on_delete=models.CASCADE, - ) - organisationType = ArrayField( - models.CharField(max_length=255, blank=True), - blank=True, - null=True, - verbose_name="Other organisation types", - ) - organisationTypePrimary = models.ForeignKey( - "OrganisationType", - on_delete=models.CASCADE, - related_name="organisations", - verbose_name="Primary organisation type", - ) - scrape = models.ForeignKey( - "Scrape", - related_name="organisations", - on_delete=models.CASCADE, - ) - spider = models.CharField(max_length=200, db_index=True) - location = models.JSONField(null=True, blank=True) - org_id_scheme = models.ForeignKey( - "OrgidScheme", - on_delete=models.CASCADE, - blank=True, - null=True, - ) - - # geography fields - geo_oa11 = models.CharField(max_length=9, null=True, blank=True) - geo_cty = models.CharField(max_length=9, null=True, blank=True) - geo_laua = models.CharField(max_length=9, null=True, blank=True, db_index=True) - geo_ward = models.CharField(max_length=9, null=True, blank=True) - geo_ctry = models.CharField(max_length=9, null=True, blank=True, db_index=True) - geo_rgn = models.CharField(max_length=9, null=True, blank=True, db_index=True) - geo_pcon = models.CharField(max_length=9, null=True, blank=True, db_index=True) - geo_ttwa = models.CharField(max_length=9, null=True, blank=True) - geo_lsoa11 = models.CharField(max_length=9, null=True, blank=True, db_index=True) - geo_msoa11 = models.CharField(max_length=9, null=True, blank=True, db_index=True) - geo_lep1 = models.CharField(max_length=9, null=True, blank=True) - geo_lep2 = models.CharField(max_length=9, null=True, blank=True) - geo_lat = models.FloatField(null=True, blank=True) - geo_long = models.FloatField(null=True, blank=True) - - class Meta: - unique_together = ( - "org_id", - "scrape", - ) - indexes = [ - GinIndex(fields=["orgIDs"]), - GinIndex(fields=["linked_orgs"]), - GinIndex(fields=["alternateName"]), - GinIndex(fields=["organisationType"]), - ] - - def __str__(self): - return "%s %s" % (self.organisationTypePrimary.title, self.org_id) - - @property - def org_links(self): - return OrganisationLink.objects.filter( - models.Q(org_id_a=self.org_id) | models.Q(org_id_b=self.org_id) - ) - - def get_priority(self): - if self.org_id.scheme in OrgidScheme.PRIORITIES: - prefix_order = OrgidScheme.PRIORITIES.index(self.org_id.scheme) - else: - prefix_order = len(OrgidScheme.PRIORITIES) + 1 - return ( - 0 if self.active else 1, - prefix_order, - self.dateRegistered if self.dateRegistered else datetime.date.min, - ) - - @property - def all_names(self): - if self.name is None: - return self.alternateName - if self.alternateName is None: - return [self.name] - return [self.name] + self.alternateName - - @classmethod - def get_fields_as_properties(cls): - internal_fields = ["scrape", "spider", "id"] - return [ - {"id": f.name, "name": f.verbose_name} - for f in cls._meta.get_fields() - if f.name not in internal_fields - ] - - def schema_dot_org(self, request=None): - """Return a schema.org Organisation object representing this organisation""" - - obj = { - "@context": "https://schema.org", - "@type": "Organization", - "name": self.name, - "identifier": self.org_id, - } - - if self.url: - obj["url"] = self.url - if self.description: - obj["description"] = self.description - if self.alternateName: - obj["alternateName"] = self.alternateName - if self.dateRegistered: - obj["foundingDate"] = self.dateRegistered.isoformat() - if not self.active and self.dateRemoved: - obj["dissolutionDate"] = self.dateRemoved.isoformat() - if self.orgIDs and len(self.orgIDs) > 1: - if request: - obj["sameAs"] = [request.build_absolute_uri(id) for id in self.sameAs] - else: - obj["sameAs"] = self.sameAs - return obj - - def get_links(self): - if self.url: - yield (self.cleanUrl, self.displayUrl, self.org_id) - if not self.orgIDs: - return - for o in self.orgIDs: - links = EXTERNAL_LINKS.get(o.scheme, []) - for link in links: - yield (link[0].format(o.id), link[1], o) - - @property - def sameAs(self): - return [ - reverse("orgid_html", kwargs=dict(org_id=o)) - for o in self.orgIDs - if o != self.org_id - ] - - @property - def cleanUrl(self): - if not self.url: - return None - if not self.url.startswith("http") and not self.url.startswith("//"): - return "http://" + self.url - return self.url - - @property - def displayUrl(self): - if not self.url: - return None - url = re.sub(r"(https?:)?//", "", self.url) - if url.startswith("www."): - url = url[4:] - if url.endswith("/"): - url = url[:-1] - return url - - @property - def sortedAlternateName(self): - if not self.alternateName: - return [] - return sorted( - self.alternateName, - key=lambda x: x[4:] if x.lower().startswith("the ") else x, - ) - - def geoCodes(self): - - special_cases = { - "K02000001": [ - "E92000001", - "N92000002", - "S92000003", - "W92000004", - ], # United Kingdom - # Great Britain - "K03000001": ["E92000001", "S92000003", "W92000004"], - "K04000001": ["E92000001", "W92000004"], # England and Wales - } - - for v in self.location: - if v.get("geoCode") and re.match("[ENWSK][0-9]{8}", v.get("geoCode")): - # special case for combinations of countries - if v["geoCode"] in special_cases: - for a in special_cases[v["geoCode"]]: - yield a - continue - yield v.get("geoCode") - - -class OrganisationType(models.Model): - slug = models.SlugField(max_length=255, editable=False, primary_key=True) - title = models.CharField(max_length=255) - - def save(self, *args, **kwargs): - value = self.title - self.slug = slugify(value, allow_unicode=True) - super().save(*args, **kwargs) - - KEY_TYPES = [ - "registered-charity", - "registered-charity-england-and-wales", - "registered-charity-scotland", - "registered-charity-northern-ireland", - "registered-company", - # "company limited by guarantee", - "charitable-incorporated-organisation", - "education", - "community-interest-company", - "health", - "registered-society", - "community-amateur-sports-club", - "registered-provider-of-social-housing", - "government-organisation", - "local-authority", - "university", - ] - - def is_keytype(self): - return self.slug in self.KEY_TYPES - - def __str__(self): - return self.title - - -class OrganisationLink(models.Model): - org_id_a = OrgidField(max_length=255, db_index=True) - org_id_b = OrgidField(max_length=255, db_index=True) - spider = models.CharField(max_length=200, db_index=True) - source = models.ForeignKey( - "Source", - related_name="organisation_links", - on_delete=models.CASCADE, - ) - scrape = models.ForeignKey( - "Scrape", - related_name="organisation_links", - on_delete=models.CASCADE, - ) - - def __str__(self): - return "From {} to {}".format(self.org_id_a, self.org_id_b) - - -class Source(models.Model): - id = models.CharField(max_length=200, unique=True, db_index=True, primary_key=True) - data = models.JSONField() - - @property - def title(self): - return self.data.get("title") - - @property - def publisher(self): - return self.data.get("publisher", {}).get("name") - - @property - def slug(self): - return self.id - - @property - def modified(self): - return datetime.datetime.fromisoformat(self.data.get("modified")) - - def __str__(self): - return self.publisher + " (" + self.title + ")" - - -class Scrape(models.Model): - class ScrapeStatus(models.TextChoices): - RUNNING = "running", "In progress" - SUCCESS = "success", "Finished successfully" - ERRORS = "errors", "Finished with errors" - FAILED = "failed", "Failed to complete" - - spider = models.CharField(max_length=200) - result = models.JSONField(null=True, blank=True, editable=False) - start_time = models.DateTimeField(auto_now_add=True, editable=False) - finish_time = models.DateTimeField( - auto_now=True, null=True, blank=True, editable=False - ) - log = models.TextField(null=True, blank=True, editable=False) - items = models.IntegerField(default=0, editable=False) - errors = models.IntegerField(default=0, editable=False) - status = models.CharField( - max_length=50, - null=True, - blank=True, - choices=ScrapeStatus.choices, - editable=False, - ) - - def __str__(self): - return "{} [{}] {:%Y-%m-%d %H:%M}".format( - self.spider, self.status, self.start_time - ) - - -class OrgidScheme(models.Model): - - PRIORITIES = [ - "GB-CHC", - "GB-SC", - "GB-NIC", - "GB-EDU", - "GB-LAE", - "GB-PLA", - "GB-LAS", - "GB-LANI", - "GB-GOR", - "GB-COH", - ] - - code = models.CharField(max_length=200, primary_key=True, db_index=True) - data = models.JSONField() - priority = models.IntegerField(null=True, blank=True) - - def save(self, *args, **kwargs): - if self.code in self.PRIORITIES: - self.priority = self.PRIORITIES.index(self.code) - else: - self.priority = len(self.PRIORITIES) + 1 - super().save(*args, **kwargs) - - def __str__(self): - return "{} - {}".format(self.code, self.data.get("name", {}).get("en")) - - -class RelatedOrganisation: - def __init__(self, orgs): - self.records = self.prioritise_orgs(orgs) - - @classmethod - def from_orgid(cls, org_id): - orgs = Organisation.objects.filter(linked_orgs__contains=[org_id]) - return cls(orgs) - - @property - def orgIDs(self): - return list(set(self.get_all("orgIDs"))) - - @property - def names(self): - names = {} - for r in self.records: - for n in r.all_names: - if n not in names or not n.isupper() or not n.islower(): - names[n.lower().strip()] = n - return names - - @property - def alternateName(self): - names = self.get_all("all_names") - return list( - set( - [ - self.names.get(n.lower().strip(), n) - for n in names - if n.lower().strip() != self.name.lower().strip() - ] - ) - ) - - @property - def name(self): - return self.names.get(self.records[0].name.lower(), self.records[0].name) - - @property - def sources(self): - sources = list(self.get_all("source")) - sources.extend([o.source for o in self.org_links]) - return list(set(sources)) - - @property - def org_links(self): - org_links = [] - for o in self.records: - org_links.extend(o.org_links) - return list(set(org_links)) - - def __getattr__(self, key, *args): - return getattr(self.records[0], key, *args) - - def first(self, field, justvalue=False): - for r in self.records: - if getattr(r, field, None): - if justvalue: - return getattr(r, field) - return { - "value": getattr(r, field), - "orgid": r.org_id, - "source": r.source, - } - if justvalue: - return None - return {} - - def get_all(self, field): - seen = set() - for r in self.records: - values = getattr(r, field, None) - if not isinstance(values, list): - values = [values] - for v in values: - if v not in seen: - yield v - seen.add(v) - - def prioritise_orgs(self, orgs): - # Decide what order a list of organisations should go in, - # based on their priority - return sorted(orgs, key=lambda o: o.get_priority()) - - def get_links(self): - links_seen = set() - for r in self.records: - for link in r.get_links(): - if link[1] not in links_seen: - yield link - links_seen.add(link[1]) - - @property - def sameAs(self): - return [ - reverse("orgid_html", kwargs=dict(org_id=o)) - for o in self.orgIDs - if o != self.org_id - ] - - @property - def activeRecords(self): - return [r for r in self.records if r.active] - - @property - def inactiveRecords(self): - return [r for r in self.records if not r.active] - - def schema_dot_org(self, request=None): - """Return a schema.org Organisation object representing this organisation""" - - obj = { - "@context": "https://schema.org", - "@type": "Organization", - "name": self.name, - "identifier": self.org_id, - } - - if self.first("url"): - obj["url"] = self.first("url").get("value") - if self.first("description"): - obj["description"] = self.first("description").get("value") - if self.alternateName: - obj["alternateName"] = self.alternateName - if self.dateRegistered: - obj["foundingDate"] = self.dateRegistered.isoformat() - if not self.active and self.first("dateRemoved"): - obj["dissolutionDate"] = self.first("dateRemoved").get("value").isoformat() - if len(self.orgIDs) > 1: - if request: - obj["sameAs"] = [request.build_absolute_uri(id) for id in self.sameAs] - else: - obj["sameAs"] = self.sameAs - return obj - - def to_json(self, charity=False, request=None): - address_fields = [ - "streetAddress", - "addressLocality", - "addressRegion", - "addressCountry", - "postalCode", - ] - orgtypes = [y for y in self.get_all("organisationType")] - orgtypes = [o.title for o in OrganisationType.objects.filter(slug__in=orgtypes)] - - def build_url(url): - if request: - return request.build_absolute_uri(url) - return url - - if charity: - ccew_number = None - ccew_link = None - oscr_number = None - oscr_link = None - ccni_number = None - ccni_link = None - company_numbers = [] - for o in self.linked_orgs: - if o.startswith("GB-CHC-"): - ccew_number = o.replace("GB-CHC-", "") - ccew_link = EXTERNAL_LINKS["GB-CHC"][1][0].format(ccew_number) - elif o.startswith("GB-NIC-"): - ccni_number = o.replace("GB-NIC-", "") - ccni_link = EXTERNAL_LINKS["GB-NIC"][0][0].format(ccni_number) - elif o.startswith("GB-SC-"): - oscr_number = o.replace("GB-SC-", "") - oscr_link = EXTERNAL_LINKS["GB-SC"][0][0].format(oscr_number) - elif o.startswith("GB-COH-"): - company_numbers.append( - { - "number": o.replace("GB-COH-", ""), - "url": EXTERNAL_LINKS["GB-COH"][0][0].format( - o.replace("GB-COH-", "") - ), - "source": self.source.id, - } - ) - names = [] - names_seen = set() - for r in self.records: - if r.name not in names_seen: - names.append( - { - "name": r.name, - "type": "registered name", - "source": r.source.id, - } - ) - names_seen.add(r.name) - for n in r.alternateName: - if n not in names_seen: - names.append( - {"name": n, "type": "other name", "source": r.source.id} - ) - names_seen.add(n) - - return { - "ccew_number": ccew_number, - "oscr_number": oscr_number, - "ccni_number": ccni_number, - "active": self.active, - "names": names, - "known_as": self.name, - "geo": { - "areas": [], - "postcode": self.postalCode, - "location": self.location, - "address": { - k: getattr(self, k) - for k in address_fields - if getattr(self, k, None) - }, - }, - "url": self.cleanUrl, - "domain": self.domain, - "latest_income": self.latestIncome, - "company_number": company_numbers, - "parent": self.parent, - "ccew_link": ccew_link, - "oscr_link": oscr_link, - "ccni_link": ccni_link, - "date_registered": self.dateRegistered, - "date_removed": self.dateRemoved, - "org-ids": self.linked_orgs, - "alt_names": self.alternateName, - "last_modified": self.dateModified, - } - - return { - "id": self.org_id, - "name": self.name, - "charityNumber": self.charityNumber, - "companyNumber": self.companyNumber, - "description": self.description, - "url": self.cleanUrl, - "latestIncome": self.latestIncome, - "dateRegistered": self.dateRegistered, - "dateRemoved": self.dateRemoved, - "active": self.active, - "parent": self.parent, - "organisationType": orgtypes, - "organisationTypePrimary": self.organisationTypePrimary.title, - "alternateName": self.alternateName, - "telephone": self.telephone, - "email": self.email, - "location": self.location, - "address": { - k: getattr(self, k) for k in address_fields if getattr(self, k, None) - }, - "sources": [s.id for s in self.sources], - "links": [ - { - "site": "Find that Charity", - "url": build_url( - reverse("orgid_json", kwargs={"org_id": self.org_id}) - ), - "orgid": self.org_id, - }, - ] - + [ - {"site": link[1], "url": link[0], "orgid": link[2]} - for link in self.get_links() - ], - "orgIDs": self.orgIDs, - "linked_records": [ - { - "orgid": orgid, - "url": build_url(reverse("orgid_json", kwargs={"org_id": orgid})), - } - for orgid in self.linked_orgs - ], - "dateModified": self.dateModified, - } diff --git a/ftc/models/__init__.py b/ftc/models/__init__.py new file mode 100644 index 00000000..87917059 --- /dev/null +++ b/ftc/models/__init__.py @@ -0,0 +1,21 @@ +from .organisation import Organisation, EXTERNAL_LINKS +from .organisation_link import OrganisationLink +from .organisation_type import OrganisationType +from .orgid import OrgidField, Orgid +from .orgid_scheme import OrgidScheme +from .related_organisation import RelatedOrganisation +from .scrape import Scrape +from .source import Source + +__all__ = [ + "Organisation", + "EXTERNAL_LINKS", + "OrganisationLink", + "OrganisationType", + "OrgidField", + "Orgid", + "OrgidScheme", + "RelatedOrganisation", + "Scrape", + "Source", +] diff --git a/ftc/models/organisation.py b/ftc/models/organisation.py new file mode 100644 index 00000000..47ef00d1 --- /dev/null +++ b/ftc/models/organisation.py @@ -0,0 +1,361 @@ +import datetime +import re + +from django.contrib.postgres.indexes import GinIndex +from django.db import models +from django.urls import reverse +from django_better_admin_arrayfield.models.fields import ArrayField + +from .orgid import OrgidField +from .orgid_scheme import OrgidScheme +from .organisation_link import OrganisationLink + +EXTERNAL_LINKS = { + "GB-CHC": [ + [ + "https://register-of-charities.charitycommission.gov.uk/charity-details/?regId={}&subId=0", + "Charity Commission England and Wales", + ], + ["https://charitybase.uk/charities/{}", "CharityBase"], + ["http://opencharities.org/charities/{}", "OpenCharities"], + ["https://givingisgreat.org/charitydetail/?regNo={}", "Giving is Great"], + [ + "http://www.charitychoice.co.uk/charities/search?t=qsearch&q={}", + "Charities Direct", + ], + ], + "GB-COH": [ + ["https://beta.companieshouse.gov.uk/company/{}", "Companies House"], + ["https://opencorporates.com/companies/gb/{}", "Opencorporates"], + ], + "GB-NIC": [ + [ + "http://www.charitycommissionni.org.uk/charity-details/?regid={}&subid=0", + "Charity Commission Northern Ireland", + ], + ["https://givingisgreat.org/charitydetail/?regNo={}", "Giving is Great"], + ], + "GB-SC": [ + [ + "https://www.oscr.org.uk/about-charities/search-the-register/charity-details?number={}", + "Office of Scottish Charity Regulator", + ], + ["https://givingisgreat.org/charitydetail/?regNo={}", "Giving is Great"], + ], + "GB-EDU": [ + [ + "https://get-information-schools.service.gov.uk/Establishments/Establishment/Details/{}", + "Get information about schools", + ], + ], + "GB-UKPRN": [ + [ + "https://www.ukrlp.co.uk/ukrlp/ukrlp_provider.page_pls_provDetails?x=&pn_p_id={}&pv_status=VERIFIED&pv_vis_code=L", + "UK Register of Learning Providers", + ], + ], + "GB-NHS": [ + ["https://odsportal.hscic.gov.uk/Organisation/Details/{}", "NHS Digital"], + ], + "GB-LAE": [ + [ + "https://www.registers.service.gov.uk/registers/local-authority-eng/records/{}", + "Local authorities in England", + ], + ], + "GB-LAN": [ + [ + "https://www.registers.service.gov.uk/registers/local-authority-nir/records/{}", + "Local authorities in Northern Ireland", + ], + ], + "GB-LAS": [ + [ + "https://www.registers.service.gov.uk/registers/local-authority-sct/records/{}", + "Local authorities in Scotland", + ], + ], + "GB-PLA": [ + [ + "https://www.registers.service.gov.uk/registers/principal-local-authority/records/{}", + "Principal Local authorities in Wales", + ], + ], + "GB-GOR": [ + [ + "https://www.registers.service.gov.uk/registers/government-organisation/records/{}", + "Government organisations on GOV.UK", + ], + ], + "XI-GRID": [ + ["https://www.grid.ac/institutes/{}", "Global Research Identifier Database"], + ], +} + + +class Organisation(models.Model): + org_id = OrgidField(db_index=True, verbose_name="Organisation Identifier") + orgIDs = ArrayField( + OrgidField(blank=True), + verbose_name="Other organisation identifiers", + ) + linked_orgs = ArrayField( + models.CharField(max_length=100, blank=True), + blank=True, + null=True, + verbose_name="Linked organisations", + ) + name = models.CharField(max_length=255, verbose_name="Name") + alternateName = ArrayField( + models.CharField(max_length=255, blank=True), + blank=True, + null=True, + verbose_name="Other names", + ) + charityNumber = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Charity number" + ) + companyNumber = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Company number" + ) + streetAddress = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Address: street" + ) + addressLocality = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Address: locality" + ) + addressRegion = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Address: region" + ) + addressCountry = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Address: country" + ) + postalCode = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Postcode", db_index=True + ) + telephone = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Telephone" + ) + email = models.CharField( + max_length=255, null=True, blank=True, verbose_name="Email address" + ) + description = models.TextField(null=True, blank=True, verbose_name="Description") + url = models.URLField(null=True, blank=True, verbose_name="Website address") + domain = models.CharField( + max_length=255, + null=True, + blank=True, + db_index=True, + verbose_name="Website domain", + ) + latestIncome = models.BigIntegerField( + null=True, blank=True, verbose_name="Latest income" + ) + latestIncomeDate = models.DateField( + null=True, blank=True, verbose_name="Latest financial year end" + ) + dateRegistered = models.DateField( + null=True, blank=True, verbose_name="Date registered" + ) + dateRemoved = models.DateField(null=True, blank=True, verbose_name="Date removed") + active = models.BooleanField(null=True, blank=True, verbose_name="Active") + status = models.CharField( + max_length=200, null=True, blank=True, verbose_name="Status" + ) + parent = models.CharField( + max_length=200, null=True, blank=True, verbose_name="Parent organisation" + ) + dateModified = models.DateTimeField( + auto_now=True, verbose_name="Date record was last modified" + ) + source = models.ForeignKey( + "Source", + related_name="organisations", + on_delete=models.CASCADE, + ) + organisationType = ArrayField( + models.CharField(max_length=255, blank=True), + blank=True, + null=True, + verbose_name="Other organisation types", + ) + organisationTypePrimary = models.ForeignKey( + "OrganisationType", + on_delete=models.CASCADE, + related_name="organisations", + verbose_name="Primary organisation type", + ) + scrape = models.ForeignKey( + "Scrape", + related_name="organisations", + on_delete=models.CASCADE, + ) + spider = models.CharField(max_length=200, db_index=True) + location = models.JSONField(null=True, blank=True) + org_id_scheme = models.ForeignKey( + "OrgidScheme", + on_delete=models.CASCADE, + blank=True, + null=True, + ) + + # geography fields + geo_oa11 = models.CharField(max_length=9, null=True, blank=True) + geo_cty = models.CharField(max_length=9, null=True, blank=True) + geo_laua = models.CharField(max_length=9, null=True, blank=True, db_index=True) + geo_ward = models.CharField(max_length=9, null=True, blank=True) + geo_ctry = models.CharField(max_length=9, null=True, blank=True, db_index=True) + geo_rgn = models.CharField(max_length=9, null=True, blank=True, db_index=True) + geo_pcon = models.CharField(max_length=9, null=True, blank=True, db_index=True) + geo_ttwa = models.CharField(max_length=9, null=True, blank=True) + geo_lsoa11 = models.CharField(max_length=9, null=True, blank=True, db_index=True) + geo_msoa11 = models.CharField(max_length=9, null=True, blank=True, db_index=True) + geo_lep1 = models.CharField(max_length=9, null=True, blank=True) + geo_lep2 = models.CharField(max_length=9, null=True, blank=True) + geo_lat = models.FloatField(null=True, blank=True) + geo_long = models.FloatField(null=True, blank=True) + + class Meta: + unique_together = ( + "org_id", + "scrape", + ) + indexes = [ + GinIndex(fields=["orgIDs"]), + GinIndex(fields=["linked_orgs"]), + GinIndex(fields=["alternateName"]), + GinIndex(fields=["organisationType"]), + ] + + def __str__(self): + return "%s %s" % (self.organisationTypePrimary.title, self.org_id) + + @property + def org_links(self): + return OrganisationLink.objects.filter( + models.Q(org_id_a=self.org_id) | models.Q(org_id_b=self.org_id) + ) + + def get_priority(self): + if self.org_id.scheme in OrgidScheme.PRIORITIES: + prefix_order = OrgidScheme.PRIORITIES.index(self.org_id.scheme) + else: + prefix_order = len(OrgidScheme.PRIORITIES) + 1 + return ( + 0 if self.active else 1, + prefix_order, + self.dateRegistered if self.dateRegistered else datetime.date.min, + ) + + @property + def all_names(self): + if self.name is None: + return self.alternateName + if self.alternateName is None: + return [self.name] + return [self.name] + self.alternateName + + @classmethod + def get_fields_as_properties(cls): + internal_fields = ["scrape", "spider", "id"] + return [ + {"id": f.name, "name": f.verbose_name} + for f in cls._meta.get_fields() + if f.name not in internal_fields + ] + + def schema_dot_org(self, request=None): + """Return a schema.org Organisation object representing this organisation""" + + obj = { + "@context": "https://schema.org", + "@type": "Organization", + "name": self.name, + "identifier": self.org_id, + } + + if self.url: + obj["url"] = self.url + if self.description: + obj["description"] = self.description + if self.alternateName: + obj["alternateName"] = self.alternateName + if self.dateRegistered: + obj["foundingDate"] = self.dateRegistered.isoformat() + if not self.active and self.dateRemoved: + obj["dissolutionDate"] = self.dateRemoved.isoformat() + if self.orgIDs and len(self.orgIDs) > 1: + if request: + obj["sameAs"] = [request.build_absolute_uri(id) for id in self.sameAs] + else: + obj["sameAs"] = self.sameAs + return obj + + def get_links(self): + if self.url: + yield (self.cleanUrl, self.displayUrl, self.org_id) + if not self.orgIDs: + return + for o in self.orgIDs: + links = EXTERNAL_LINKS.get(o.scheme, []) + for link in links: + yield (link[0].format(o.id), link[1], o) + + @property + def sameAs(self): + return [ + reverse("orgid_html", kwargs=dict(org_id=o)) + for o in self.orgIDs + if o != self.org_id + ] + + @property + def cleanUrl(self): + if not self.url: + return None + if not self.url.startswith("http") and not self.url.startswith("//"): + return "http://" + self.url + return self.url + + @property + def displayUrl(self): + if not self.url: + return None + url = re.sub(r"(https?:)?//", "", self.url) + if url.startswith("www."): + url = url[4:] + if url.endswith("/"): + url = url[:-1] + return url + + @property + def sortedAlternateName(self): + if not self.alternateName: + return [] + return sorted( + self.alternateName, + key=lambda x: x[4:] if x.lower().startswith("the ") else x, + ) + + def geoCodes(self): + + special_cases = { + "K02000001": [ + "E92000001", + "N92000002", + "S92000003", + "W92000004", + ], # United Kingdom + # Great Britain + "K03000001": ["E92000001", "S92000003", "W92000004"], + "K04000001": ["E92000001", "W92000004"], # England and Wales + } + + for v in self.location: + if v.get("geoCode") and re.match("[ENWSK][0-9]{8}", v.get("geoCode")): + # special case for combinations of countries + if v["geoCode"] in special_cases: + for a in special_cases[v["geoCode"]]: + yield a + continue + yield v.get("geoCode") diff --git a/ftc/models/organisation_link.py b/ftc/models/organisation_link.py new file mode 100644 index 00000000..519ffa32 --- /dev/null +++ b/ftc/models/organisation_link.py @@ -0,0 +1,22 @@ +from django.db import models + +from .orgid import OrgidField + + +class OrganisationLink(models.Model): + org_id_a = OrgidField(max_length=255, db_index=True) + org_id_b = OrgidField(max_length=255, db_index=True) + spider = models.CharField(max_length=200, db_index=True) + source = models.ForeignKey( + "Source", + related_name="organisation_links", + on_delete=models.CASCADE, + ) + scrape = models.ForeignKey( + "Scrape", + related_name="organisation_links", + on_delete=models.CASCADE, + ) + + def __str__(self): + return "From {} to {}".format(self.org_id_a, self.org_id_b) diff --git a/ftc/models/organisation_type.py b/ftc/models/organisation_type.py new file mode 100644 index 00000000..5211d6d6 --- /dev/null +++ b/ftc/models/organisation_type.py @@ -0,0 +1,37 @@ +from django.db import models +from django.utils.text import slugify + + +class OrganisationType(models.Model): + slug = models.SlugField(max_length=255, editable=False, primary_key=True) + title = models.CharField(max_length=255) + + def save(self, *args, **kwargs): + value = self.title + self.slug = slugify(value, allow_unicode=True) + super().save(*args, **kwargs) + + KEY_TYPES = [ + "registered-charity", + "registered-charity-england-and-wales", + "registered-charity-scotland", + "registered-charity-northern-ireland", + "registered-company", + # "company limited by guarantee", + "charitable-incorporated-organisation", + "education", + "community-interest-company", + "health", + "registered-society", + "community-amateur-sports-club", + "registered-provider-of-social-housing", + "government-organisation", + "local-authority", + "university", + ] + + def is_keytype(self): + return self.slug in self.KEY_TYPES + + def __str__(self): + return self.title diff --git a/ftc/models/orgid.py b/ftc/models/orgid.py new file mode 100644 index 00000000..ca7ed469 --- /dev/null +++ b/ftc/models/orgid.py @@ -0,0 +1,41 @@ +from django.db import models + + +class Orgid(str): + def __new__(cls, content): + instance = super().__new__(cls, content) + instance._split_orgid(content) + return instance + + def _split_orgid(self, value): + self.scheme = None + self.id = value + if value is None: + return + split_orgid = value.split("-", maxsplit=3) + if len(split_orgid) > 2: + self.scheme = "-".join(split_orgid[:2]) + self.id = "-".join(split_orgid[2:]) + + +class OrgidField(models.CharField): + + description = "An orgid based on the format here: http://org-id.guide/about" + + def __init__(self, *args, **kwargs): + kwargs["max_length"] = 200 + super().__init__(*args, **kwargs) + + def from_db_value(self, value, expression, connection): + if value is None: + return value + return Orgid(value) + + def to_python(self, value): + if isinstance(value, Orgid): + return value + + if value is None: + return value + + return Orgid(value) diff --git a/ftc/models/orgid_scheme.py b/ftc/models/orgid_scheme.py new file mode 100644 index 00000000..699a0fb4 --- /dev/null +++ b/ftc/models/orgid_scheme.py @@ -0,0 +1,31 @@ +from django.db import models + + +class OrgidScheme(models.Model): + + PRIORITIES = [ + "GB-CHC", + "GB-SC", + "GB-NIC", + "GB-EDU", + "GB-LAE", + "GB-PLA", + "GB-LAS", + "GB-LANI", + "GB-GOR", + "GB-COH", + ] + + code = models.CharField(max_length=200, primary_key=True, db_index=True) + data = models.JSONField() + priority = models.IntegerField(null=True, blank=True) + + def save(self, *args, **kwargs): + if self.code in self.PRIORITIES: + self.priority = self.PRIORITIES.index(self.code) + else: + self.priority = len(self.PRIORITIES) + 1 + super().save(*args, **kwargs) + + def __str__(self): + return "{} - {}".format(self.code, self.data.get("name", {}).get("en")) diff --git a/ftc/models/related_organisation.py b/ftc/models/related_organisation.py new file mode 100644 index 00000000..6512e890 --- /dev/null +++ b/ftc/models/related_organisation.py @@ -0,0 +1,282 @@ +from django.urls import reverse + +from .organisation import Organisation, EXTERNAL_LINKS +from .organisation_type import OrganisationType + + +class RelatedOrganisation: + def __init__(self, orgs): + self.records = self.prioritise_orgs(orgs) + + @classmethod + def from_orgid(cls, org_id): + orgs = Organisation.objects.filter(linked_orgs__contains=[org_id]) + return cls(orgs) + + @property + def orgIDs(self): + return list(set(self.get_all("orgIDs"))) + + @property + def names(self): + names = {} + for r in self.records: + for n in r.all_names: + if n not in names or not n.isupper() or not n.islower(): + names[n.lower().strip()] = n + return names + + @property + def alternateName(self): + names = self.get_all("all_names") + return list( + set( + [ + self.names.get(n.lower().strip(), n) + for n in names + if n.lower().strip() != self.name.lower().strip() + ] + ) + ) + + @property + def name(self): + return self.names.get(self.records[0].name.lower(), self.records[0].name) + + @property + def sources(self): + sources = list(self.get_all("source")) + sources.extend([o.source for o in self.org_links]) + return list(set(sources)) + + @property + def org_links(self): + org_links = [] + for o in self.records: + org_links.extend(o.org_links) + return list(set(org_links)) + + def __getattr__(self, key, *args): + return getattr(self.records[0], key, *args) + + def first(self, field, justvalue=False): + for r in self.records: + if getattr(r, field, None): + if justvalue: + return getattr(r, field) + return { + "value": getattr(r, field), + "orgid": r.org_id, + "source": r.source, + } + if justvalue: + return None + return {} + + def get_all(self, field): + seen = set() + for r in self.records: + values = getattr(r, field, None) + if not isinstance(values, list): + values = [values] + for v in values: + if v not in seen: + yield v + seen.add(v) + + def prioritise_orgs(self, orgs): + # Decide what order a list of organisations should go in, + # based on their priority + return sorted(orgs, key=lambda o: o.get_priority()) + + def get_links(self): + links_seen = set() + for r in self.records: + for link in r.get_links(): + if link[1] not in links_seen: + yield link + links_seen.add(link[1]) + + @property + def sameAs(self): + return [ + reverse("orgid_html", kwargs=dict(org_id=o)) + for o in self.orgIDs + if o != self.org_id + ] + + @property + def activeRecords(self): + return [r for r in self.records if r.active] + + @property + def inactiveRecords(self): + return [r for r in self.records if not r.active] + + def schema_dot_org(self, request=None): + """Return a schema.org Organisation object representing this organisation""" + + obj = { + "@context": "https://schema.org", + "@type": "Organization", + "name": self.name, + "identifier": self.org_id, + } + + if self.first("url"): + obj["url"] = self.first("url").get("value") + if self.first("description"): + obj["description"] = self.first("description").get("value") + if self.alternateName: + obj["alternateName"] = self.alternateName + if self.dateRegistered: + obj["foundingDate"] = self.dateRegistered.isoformat() + if not self.active and self.first("dateRemoved"): + obj["dissolutionDate"] = self.first("dateRemoved").get("value").isoformat() + if len(self.orgIDs) > 1: + if request: + obj["sameAs"] = [request.build_absolute_uri(id) for id in self.sameAs] + else: + obj["sameAs"] = self.sameAs + return obj + + def to_json(self, charity=False, request=None): + address_fields = [ + "streetAddress", + "addressLocality", + "addressRegion", + "addressCountry", + "postalCode", + ] + orgtypes = [y for y in self.get_all("organisationType")] + orgtypes = [o.title for o in OrganisationType.objects.filter(slug__in=orgtypes)] + + def build_url(url): + if request: + return request.build_absolute_uri(url) + return url + + if charity: + ccew_number = None + ccew_link = None + oscr_number = None + oscr_link = None + ccni_number = None + ccni_link = None + company_numbers = [] + for o in self.linked_orgs: + if o.startswith("GB-CHC-"): + ccew_number = o.replace("GB-CHC-", "") + ccew_link = EXTERNAL_LINKS["GB-CHC"][1][0].format(ccew_number) + elif o.startswith("GB-NIC-"): + ccni_number = o.replace("GB-NIC-", "") + ccni_link = EXTERNAL_LINKS["GB-NIC"][0][0].format(ccni_number) + elif o.startswith("GB-SC-"): + oscr_number = o.replace("GB-SC-", "") + oscr_link = EXTERNAL_LINKS["GB-SC"][0][0].format(oscr_number) + elif o.startswith("GB-COH-"): + company_numbers.append( + { + "number": o.replace("GB-COH-", ""), + "url": EXTERNAL_LINKS["GB-COH"][0][0].format( + o.replace("GB-COH-", "") + ), + "source": self.source.id, + } + ) + names = [] + names_seen = set() + for r in self.records: + if r.name not in names_seen: + names.append( + { + "name": r.name, + "type": "registered name", + "source": r.source.id, + } + ) + names_seen.add(r.name) + for n in r.alternateName: + if n not in names_seen: + names.append( + {"name": n, "type": "other name", "source": r.source.id} + ) + names_seen.add(n) + + return { + "ccew_number": ccew_number, + "oscr_number": oscr_number, + "ccni_number": ccni_number, + "active": self.active, + "names": names, + "known_as": self.name, + "geo": { + "areas": [], + "postcode": self.postalCode, + "location": self.location, + "address": { + k: getattr(self, k) + for k in address_fields + if getattr(self, k, None) + }, + }, + "url": self.cleanUrl, + "domain": self.domain, + "latest_income": self.latestIncome, + "company_number": company_numbers, + "parent": self.parent, + "ccew_link": ccew_link, + "oscr_link": oscr_link, + "ccni_link": ccni_link, + "date_registered": self.dateRegistered, + "date_removed": self.dateRemoved, + "org-ids": self.linked_orgs, + "alt_names": self.alternateName, + "last_modified": self.dateModified, + } + + return { + "id": self.org_id, + "name": self.name, + "charityNumber": self.charityNumber, + "companyNumber": self.companyNumber, + "description": self.description, + "url": self.cleanUrl, + "latestIncome": self.latestIncome, + "dateRegistered": self.dateRegistered, + "dateRemoved": self.dateRemoved, + "active": self.active, + "parent": self.parent, + "organisationType": orgtypes, + "organisationTypePrimary": self.organisationTypePrimary.title, + "alternateName": self.alternateName, + "telephone": self.telephone, + "email": self.email, + "location": self.location, + "address": { + k: getattr(self, k) for k in address_fields if getattr(self, k, None) + }, + "sources": [s.id for s in self.sources], + "links": [ + { + "site": "Find that Charity", + "url": build_url( + reverse("orgid_json", kwargs={"org_id": self.org_id}) + ), + "orgid": self.org_id, + }, + ] + + [ + {"site": link[1], "url": link[0], "orgid": link[2]} + for link in self.get_links() + ], + "orgIDs": self.orgIDs, + "linked_records": [ + { + "orgid": orgid, + "url": build_url(reverse("orgid_json", kwargs={"org_id": orgid})), + } + for orgid in self.linked_orgs + ], + "dateModified": self.dateModified, + } diff --git a/ftc/models/scrape.py b/ftc/models/scrape.py new file mode 100644 index 00000000..aba272b3 --- /dev/null +++ b/ftc/models/scrape.py @@ -0,0 +1,31 @@ +from django.db import models + + +class Scrape(models.Model): + class ScrapeStatus(models.TextChoices): + RUNNING = "running", "In progress" + SUCCESS = "success", "Finished successfully" + ERRORS = "errors", "Finished with errors" + FAILED = "failed", "Failed to complete" + + spider = models.CharField(max_length=200) + result = models.JSONField(null=True, blank=True, editable=False) + start_time = models.DateTimeField(auto_now_add=True, editable=False) + finish_time = models.DateTimeField( + auto_now=True, null=True, blank=True, editable=False + ) + log = models.TextField(null=True, blank=True, editable=False) + items = models.IntegerField(default=0, editable=False) + errors = models.IntegerField(default=0, editable=False) + status = models.CharField( + max_length=50, + null=True, + blank=True, + choices=ScrapeStatus.choices, + editable=False, + ) + + def __str__(self): + return "{} [{}] {:%Y-%m-%d %H:%M}".format( + self.spider, self.status, self.start_time + ) diff --git a/ftc/models/source.py b/ftc/models/source.py new file mode 100644 index 00000000..9ba708d0 --- /dev/null +++ b/ftc/models/source.py @@ -0,0 +1,27 @@ +import datetime + +from django.db import models + + +class Source(models.Model): + id = models.CharField(max_length=200, unique=True, db_index=True, primary_key=True) + data = models.JSONField() + + @property + def title(self): + return self.data.get("title") + + @property + def publisher(self): + return self.data.get("publisher", {}).get("name") + + @property + def slug(self): + return self.id + + @property + def modified(self): + return datetime.datetime.fromisoformat(self.data.get("modified")) + + def __str__(self): + return self.publisher + " (" + self.title + ")" From e6b7ed5246bb408662808b4553a874d4b7c58496 Mon Sep 17 00:00:00 2001 From: David Kane Date: Fri, 12 Feb 2021 14:42:02 +0000 Subject: [PATCH 02/19] run isort --- ftc/models/__init__.py | 4 ++-- ftc/models/organisation.py | 2 +- ftc/models/related_organisation.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ftc/models/__init__.py b/ftc/models/__init__.py index 87917059..a5f48783 100644 --- a/ftc/models/__init__.py +++ b/ftc/models/__init__.py @@ -1,7 +1,7 @@ -from .organisation import Organisation, EXTERNAL_LINKS +from .organisation import EXTERNAL_LINKS, Organisation from .organisation_link import OrganisationLink from .organisation_type import OrganisationType -from .orgid import OrgidField, Orgid +from .orgid import Orgid, OrgidField from .orgid_scheme import OrgidScheme from .related_organisation import RelatedOrganisation from .scrape import Scrape diff --git a/ftc/models/organisation.py b/ftc/models/organisation.py index 47ef00d1..03e09920 100644 --- a/ftc/models/organisation.py +++ b/ftc/models/organisation.py @@ -6,9 +6,9 @@ from django.urls import reverse from django_better_admin_arrayfield.models.fields import ArrayField +from .organisation_link import OrganisationLink from .orgid import OrgidField from .orgid_scheme import OrgidScheme -from .organisation_link import OrganisationLink EXTERNAL_LINKS = { "GB-CHC": [ diff --git a/ftc/models/related_organisation.py b/ftc/models/related_organisation.py index 6512e890..eee82f2c 100644 --- a/ftc/models/related_organisation.py +++ b/ftc/models/related_organisation.py @@ -1,6 +1,6 @@ from django.urls import reverse -from .organisation import Organisation, EXTERNAL_LINKS +from .organisation import EXTERNAL_LINKS, Organisation from .organisation_type import OrganisationType From 534c2d517a7a6259adf5c777e3dbc9329566baef Mon Sep 17 00:00:00 2001 From: David Kane Date: Fri, 12 Feb 2021 17:39:22 +0000 Subject: [PATCH 03/19] ensure all link records are found --- ftc/management/commands/_base_scraper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ftc/management/commands/_base_scraper.py b/ftc/management/commands/_base_scraper.py index f5dc0af8..0a99eba2 100644 --- a/ftc/management/commands/_base_scraper.py +++ b/ftc/management/commands/_base_scraper.py @@ -221,6 +221,7 @@ def add_org_record(self, record): self.object_count, ) ) + self.get_link_records() self.records = [] def save_sources(self): From 0f2439302c13c4ed58d597389475bdf2b1c9f291 Mon Sep 17 00:00:00 2001 From: David Kane Date: Fri, 12 Feb 2021 19:13:39 +0000 Subject: [PATCH 04/19] add organisation_location table --- charity/management/commands/_ccew_sql.py | 32 +++ .../management/commands/import_charities.py | 1 + ftc/management/commands/import_all.py | 1 + ftc/management/commands/update_orgids.py | 33 ---- ftc/migrations/0009_organisationlocation.py | 44 +++++ ftc/migrations/0010_auto_20210212_1524.py | 23 +++ .../0011_organisationlocation_scrape.py | 20 ++ ftc/models/__init__.py | 2 + ftc/models/organisation_location.py | 120 ++++++++++++ geo/management/commands/import_geolookups.py | 127 ++++++++++++ geo/management/commands/update_geodata.py | 184 ++++++++++++++++++ geo/management/commands/update_postcodes.py | 141 -------------- geo/migrations/0006_geolookup.py | 35 ++++ geo/migrations/0007_geolookup_name.py | 18 ++ geo/models.py | 80 ++++++++ 15 files changed, 687 insertions(+), 174 deletions(-) create mode 100644 ftc/migrations/0009_organisationlocation.py create mode 100644 ftc/migrations/0010_auto_20210212_1524.py create mode 100644 ftc/migrations/0011_organisationlocation_scrape.py create mode 100644 ftc/models/organisation_location.py create mode 100644 geo/management/commands/import_geolookups.py create mode 100644 geo/management/commands/update_geodata.py delete mode 100644 geo/management/commands/update_postcodes.py create mode 100644 geo/migrations/0006_geolookup.py create mode 100644 geo/migrations/0007_geolookup_name.py diff --git a/charity/management/commands/_ccew_sql.py b/charity/management/commands/_ccew_sql.py index 2ac371cd..a4b1b799 100644 --- a/charity/management/commands/_ccew_sql.py +++ b/charity/management/commands/_ccew_sql.py @@ -331,3 +331,35 @@ where company_number not in ('01234567', '12345678', '00000000') and cc.source = '{source}' """ + +UPDATE_CCEW[ + "Insert into organisation location table" +] = """ +insert into ftc_organisationlocation ( + organisation_id, + org_id, + name, + "geoCode", + "geoCodeType", + "locationType", + geo_iso, + "source_id", + "scrape_id" +) +select fo.id as organisation_id, + CONCAT('GB-CHC-', cc.regno) as org_id, + aooname as name, + coalesce(ca."GSS", ca."ISO3166_1") as "geoCode", + case when ca."GSS" is not null then 'ONS' + when ca."ISO3166_1" is not null then 'ISO' + else null end as "geoCodeType", + 'AOO' as "locationType", + ca."ISO3166_1" as geo_iso, + fo.source_id as source_id, + {scrape_id} as scrape_id +from charity_ccewcharityaoo cc + inner join charity_areaofoperation ca + on cc.aookey = ca.aookey and cc.aootype = ca.aootype + inner join ftc_organisation fo + on fo.org_id = CONCAT('GB-CHC-', cc.regno) +""" diff --git a/charity/management/commands/import_charities.py b/charity/management/commands/import_charities.py index 6e0b5c0c..aa44dc7c 100644 --- a/charity/management/commands/import_charities.py +++ b/charity/management/commands/import_charities.py @@ -16,3 +16,4 @@ def handle(self, *args, **options): except Exception: self.stdout.write(self.style.ERROR("Command {} failed".format(scraper))) management.call_command("update_orgids") + management.call_command("update_geodata") diff --git a/ftc/management/commands/import_all.py b/ftc/management/commands/import_all.py index 75893c42..651776f3 100644 --- a/ftc/management/commands/import_all.py +++ b/ftc/management/commands/import_all.py @@ -29,3 +29,4 @@ def handle(self, *args, **options): except Exception: self.stdout.write(self.style.ERROR("Command {} failed".format(scraper))) management.call_command("update_orgids") + management.call_command("update_geodata") diff --git a/ftc/management/commands/update_orgids.py b/ftc/management/commands/update_orgids.py index 16da58d1..471e572c 100644 --- a/ftc/management/commands/update_orgids.py +++ b/ftc/management/commands/update_orgids.py @@ -64,39 +64,6 @@ class Command(BaseCommand): set linked_orgs = string_to_array(org_id, '') where linked_orgs is null; """, - "Remove blank postcodes": """ - update ftc_organisation - set "postalCode" = null - where trim("postalCode") = ''; - """, - "Update misformatted postcodes": """ - update ftc_organisation - set "postalCode" = concat_ws( - ' ', - trim(left(replace("postalCode", ' ', ''), length(replace("postalCode", ' ', ''))-3)), - right(replace("postalCode", ' ', ''), 3) - ) - where "postalCode" is not null; - """, - "Add geodata to organisation records": """ - update ftc_organisation - set geo_oa11 = geo.oa11, - geo_cty = geo.cty, - geo_ctry = geo.ctry, - geo_laua = geo.laua, - geo_ward = geo.ward, - geo_rgn = geo.rgn, - geo_pcon = geo.pcon, - geo_ttwa = geo.ttwa, - geo_lsoa11 = geo.lsoa11, - geo_msoa11 = geo.msoa11, - geo_lep1 = geo.lep1, - geo_lep2 = geo.lep2, - geo_lat = geo.lat, - geo_long = geo.long - from geo_postcode geo - where ftc_organisation."postalCode" = geo.pcds; - """, } def __init__(self, *args, **kwargs): diff --git a/ftc/migrations/0009_organisationlocation.py b/ftc/migrations/0009_organisationlocation.py new file mode 100644 index 00000000..a0eb53c8 --- /dev/null +++ b/ftc/migrations/0009_organisationlocation.py @@ -0,0 +1,44 @@ +# Generated by Django 3.1.1 on 2021-02-12 14:41 + +from django.db import migrations, models +import django.db.models.deletion +import ftc.models.orgid + + +class Migration(migrations.Migration): + + dependencies = [ + ('ftc', '0008_auto_20201002_1408'), + ] + + operations = [ + migrations.CreateModel( + name='OrganisationLocation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('org_id', ftc.models.orgid.OrgidField(db_index=True, max_length=200, verbose_name='Organisation Identifier')), + ('name', models.CharField(max_length=255, verbose_name='Name')), + ('description', models.TextField(blank=True, null=True, verbose_name='Description')), + ('geoCode', models.CharField(blank=True, max_length=200, null=True, verbose_name='Geo Code')), + ('geoCodeType', models.CharField(choices=[('PC', 'Postcode'), ('ONS', 'ONS code'), ('ISO', 'ISO Country Code')], db_index=True, default='ONS', max_length=5, verbose_name='Geo Code Type')), + ('locationType', models.CharField(choices=[('HQ', 'Registered Office'), ('AOO', 'Area of operation'), ('SITE', 'Site')], db_index=True, default='HQ', max_length=5, verbose_name='Location Type')), + ('geo_iso', models.CharField(blank=True, db_index=True, max_length=3, null=True, verbose_name='ISO Country Code')), + ('geo_oa11', models.CharField(blank=True, max_length=9, null=True, verbose_name='Output Area')), + ('geo_cty', models.CharField(blank=True, max_length=9, null=True, verbose_name='County')), + ('geo_laua', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Local Authority')), + ('geo_ward', models.CharField(blank=True, max_length=9, null=True, verbose_name='Ward')), + ('geo_ctry', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Country')), + ('geo_rgn', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Region')), + ('geo_pcon', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Parliamentary Constituency')), + ('geo_ttwa', models.CharField(blank=True, max_length=9, null=True, verbose_name='Travel to Work Area')), + ('geo_lsoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Lower Super Output Area')), + ('geo_msoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Middle Super Output Area')), + ('geo_lep1', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 1')), + ('geo_lep2', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 2')), + ('geo_lat', models.FloatField(blank=True, null=True, verbose_name='Latitude')), + ('geo_long', models.FloatField(blank=True, null=True, verbose_name='Longitude')), + ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='locations', to='ftc.organisation')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='organisation_locations', to='ftc.source')), + ], + ), + ] diff --git a/ftc/migrations/0010_auto_20210212_1524.py b/ftc/migrations/0010_auto_20210212_1524.py new file mode 100644 index 00000000..1ef65a8e --- /dev/null +++ b/ftc/migrations/0010_auto_20210212_1524.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.1 on 2021-02-12 15:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ftc', '0009_organisationlocation'), + ] + + operations = [ + migrations.AlterField( + model_name='organisationlocation', + name='geoCodeType', + field=models.CharField(choices=[('PC', 'UK Postcode'), ('ONS', 'ONS code'), ('ISO', 'ISO3166-1 Country Code')], db_index=True, default='ONS', max_length=5, verbose_name='Geo Code Type'), + ), + migrations.AlterField( + model_name='organisationlocation', + name='geo_iso', + field=models.CharField(blank=True, db_index=True, max_length=3, null=True, verbose_name='ISO3166-1 Country Code'), + ), + ] diff --git a/ftc/migrations/0011_organisationlocation_scrape.py b/ftc/migrations/0011_organisationlocation_scrape.py new file mode 100644 index 00000000..6af6d495 --- /dev/null +++ b/ftc/migrations/0011_organisationlocation_scrape.py @@ -0,0 +1,20 @@ +# Generated by Django 3.1.1 on 2021-02-12 17:41 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('ftc', '0010_auto_20210212_1524'), + ] + + operations = [ + migrations.AddField( + model_name='organisationlocation', + name='scrape', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='organisation_locations', to='ftc.scrape'), + preserve_default=False, + ), + ] diff --git a/ftc/models/__init__.py b/ftc/models/__init__.py index a5f48783..5b425684 100644 --- a/ftc/models/__init__.py +++ b/ftc/models/__init__.py @@ -1,5 +1,6 @@ from .organisation import EXTERNAL_LINKS, Organisation from .organisation_link import OrganisationLink +from .organisation_location import OrganisationLocation from .organisation_type import OrganisationType from .orgid import Orgid, OrgidField from .orgid_scheme import OrgidScheme @@ -11,6 +12,7 @@ "Organisation", "EXTERNAL_LINKS", "OrganisationLink", + "OrganisationLocation", "OrganisationType", "OrgidField", "Orgid", diff --git a/ftc/models/organisation_location.py b/ftc/models/organisation_location.py new file mode 100644 index 00000000..e36f38c4 --- /dev/null +++ b/ftc/models/organisation_location.py @@ -0,0 +1,120 @@ +from django.db import models + +from .orgid import OrgidField + + +class OrganisationLocation(models.Model): + class LocationTypes(models.TextChoices): + REGISTERED_OFFICE = "HQ", "Registered Office" + AREA_OF_OPERATION = "AOO", "Area of operation" + SITE = "SITE", "Site" + + class GeoCodeTypes(models.TextChoices): + POSTCODE = "PC", "UK Postcode" + ONS_CODE = "ONS", "ONS code" + ISO_CODE = "ISO", "ISO3166-1 Country Code" + + organisation = models.ForeignKey( + "Organisation", + on_delete=models.CASCADE, + related_name="locations", + ) + org_id = OrgidField(db_index=True, verbose_name="Organisation Identifier") + name = models.CharField(max_length=255, verbose_name="Name") + description = models.TextField(null=True, blank=True, verbose_name="Description") + geoCode = models.CharField( + max_length=200, null=True, blank=True, verbose_name="Geo Code" + ) + geoCodeType = models.CharField( + max_length=5, + choices=GeoCodeTypes.choices, + default=GeoCodeTypes.ONS_CODE, + db_index=True, + verbose_name="Geo Code Type", + ) + locationType = models.CharField( + max_length=5, + choices=LocationTypes.choices, + default=LocationTypes.REGISTERED_OFFICE, + db_index=True, + verbose_name="Location Type", + ) + source = models.ForeignKey( + "Source", + related_name="organisation_locations", + on_delete=models.CASCADE, + ) + scrape = models.ForeignKey( + "Scrape", + related_name="organisation_locations", + on_delete=models.CASCADE, + ) + + # geography fields + geo_iso = models.CharField( + max_length=3, + null=True, + blank=True, + verbose_name="ISO3166-1 Country Code", + db_index=True, + ) + geo_oa11 = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Output Area" + ) + geo_cty = models.CharField( + max_length=9, null=True, blank=True, verbose_name="County" + ) + geo_laua = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Local Authority", + db_index=True, + ) + geo_ward = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Ward" + ) + geo_ctry = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Country", db_index=True + ) + geo_rgn = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Region", db_index=True + ) + geo_pcon = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Parliamentary Constituency", + db_index=True, + ) + geo_ttwa = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Travel to Work Area" + ) + geo_lsoa11 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Lower Super Output Area", + db_index=True, + ) + geo_msoa11 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Middle Super Output Area", + db_index=True, + ) + geo_lep1 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Local Enterprise Partnership 1", + ) + geo_lep2 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Local Enterprise Partnership 2", + ) + geo_lat = models.FloatField(null=True, blank=True, verbose_name="Latitude") + geo_long = models.FloatField(null=True, blank=True, verbose_name="Longitude") diff --git a/geo/management/commands/import_geolookups.py b/geo/management/commands/import_geolookups.py new file mode 100644 index 00000000..2c391c4f --- /dev/null +++ b/geo/management/commands/import_geolookups.py @@ -0,0 +1,127 @@ +import io +import csv +from collections import namedtuple + +from ftc.management.commands._base_scraper import BaseScraper +from geo.models import GeoLookup + +GeoSource = namedtuple('GeoSource', ['link', "type", 'codefield', 'namefield']) + + +class Command(BaseScraper): + + name = "geo_lookups" + bulk_limit = 50000 + GEO_SOURCES = { + "la": GeoSource("https://github.com/drkane/geo-lookups/raw/master/la_all_codes.csv", "la", "LADCD", "LADNM"), + "utla": GeoSource("https://github.com/drkane/geo-lookups/raw/master/utla_all_codes.csv", "utla", "UTLACD", "UTLANM"), + "lsoa": GeoSource("https://github.com/drkane/geo-lookups/raw/master/lsoa_la.csv", "lsoa", "LSOA11CD", "LSOA11NM"), + "msoa": GeoSource("https://github.com/drkane/geo-lookups/raw/master/msoa_la.csv", "msoa", "MSOA11CD", "MSOA11HCLNM"), + "ward": GeoSource("https://github.com/drkane/geo-lookups/raw/master/ward_all_codes.csv", "ward", "WDCD", "WDNM"), + } + + FIELD_MATCH = { + "WDCD": "geo_ward", # ward code (may be out of date) + "LSOA11CD": "geo_lsoa11", # Lower Super Output Area code + "MSOA11CD": "geo_msoa11", # Middle Super Output Area code + "LAD20CD": "geo_laua", # Local Authority (2020) code + "UTLACD": "geo_cty", # Upper tier local authority code + # "CAUTHCD": "geo_", # Combined authority code + "RGNCD": "geo_rgn", # Region code + "CTRYCD": "geo_ctry", # Country code + "TTWA11CD": "geo_ttwa", # Travel to work area code + } + + MANUAL_RECORDS = ( + {"geoCode": "E92000001", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "E92000001", "name": "England"}, + {"geoCode": "S92000003", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "S92000003", "name": "Scotland"}, + {"geoCode": "N92000002", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "N92000002", "name": "Northern Ireland"}, + {"geoCode": "W92000004", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "W92000004", "name": "Wales"}, + {"geoCode": "E12000001", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000001", "name": "North East"}, + {"geoCode": "E12000002", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000002", "name": "North West"}, + {"geoCode": "E12000003", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000003", "name": "Yorkshire and The Humber"}, + {"geoCode": "E12000004", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000004", "name": "East Midlands"}, + {"geoCode": "E12000005", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000005", "name": "West Midlands"}, + {"geoCode": "E12000006", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000006", "name": "East of England"}, + {"geoCode": "E12000007", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000007", "name": "London"}, + {"geoCode": "E12000008", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000008", "name": "South East"}, + {"geoCode": "E12000009", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000009", "name": "South West"}, + ) + + def run_scraper(self, *args, **options): + + # initialise records + self.files = {} + self.records = {} + self.object_count = 0 + + # delete existing records + GeoLookup.objects.all().delete() + self.logger.info("Deleting existing items") + + # create the manual records + for m in self.MANUAL_RECORDS: + self.add_record(**m) + + # download the files + for k, source in self.GEO_SOURCES.items(): + if source.link.startswith("http"): + self.set_session(False) + self.logger.info("Downloading from {}".format(source.link)) + response = self.session.get(source.link) + response.raise_for_status() + self.files[k] = io.BytesIO(response.content) + else: + self.logger.info("Opening {}".format(source.link)) + self.files[k] = open(source.link, "rb") + + # go through each geolookup file and create records + for k, f in self.files.items(): + reader = csv.DictReader( + io.TextIOWrapper(f, encoding="utf-8-sig") + ) + self.logger.info("Opening file {}".format(k)) + for row in reader: + self.parse_row(row, self.GEO_SOURCES[k]) + + self.close_spider() + + def parse_row(self, row, geocodetype): + + for k, v in row.items(): + if not v or v == "": + row[k] = None + + self.add_record( + geoCode=row[geocodetype.codefield], + geoCodeType=geocodetype.type, + geo_iso="GB", + name=row[geocodetype.namefield], + **{ + v: row[k] + for k, v in self.FIELD_MATCH.items() + if row.get(k) + } + ) + + def add_record(self, **kwargs): + self.records[kwargs["geoCode"]] = GeoLookup(**kwargs) + + def commit_records(self): + self.object_count += len(self.records) + self.logger.info("Saving {:,.0f} records".format(len(self.records))) + GeoLookup.objects.bulk_create(self.records.values()) + self.logger.info( + "Saved {:,.0f} records ({:,.0f} total)".format( + len(self.records), + self.object_count, + ) + ) + + def close_spider(self): + self.commit_records() + self.scrape.items = self.object_count + results = {"records": self.object_count} + self.scrape.errors = self.error_count + self.scrape.result = results + self.scrape_logger.teardown() diff --git a/geo/management/commands/update_geodata.py b/geo/management/commands/update_geodata.py new file mode 100644 index 00000000..4efea028 --- /dev/null +++ b/geo/management/commands/update_geodata.py @@ -0,0 +1,184 @@ +import logging + +from django.core.management.base import BaseCommand +from django.db import connection + + +class Command(BaseCommand): + + update_sql = { + "Remove blank postcodes": """ + update ftc_organisation + set "postalCode" = null + where trim("postalCode") = ''; + """, + "Update misformatted postcodes": """ + update ftc_organisation + set "postalCode" = concat_ws( + ' ', + trim(left(replace("postalCode", ' ', ''), length(replace("postalCode", ' ', ''))-3)), + right(replace("postalCode", ' ', ''), 3) + ) + where "postalCode" is not null; + """, + 'convert "ENGLAND AND WALES" to "ENGLAND" and "WALES"': """ + insert into ftc_organisationlocation ( + "organisation_id", + "org_id", + "name", + "description", + "geoCode", + "geoCodeType", + "locationType", + "geo_iso", + "source_id", + "scrape_id" + ) + select fo.organisation_id, + fo.org_id, + 'England' as name, + fo.description, + 'E92000001' as "geoCode", + fo."geoCodeType", + fo."locationType", + fo.geo_iso, + fo.source_id, + fo.scrape_id + from ftc_organisationlocation fo + where fo."geoCode" = 'K04000001' + and "geoCodeType" = 'ONS' + union + select fo.organisation_id, + fo.org_id, + 'Wales' as name, + fo.description, + 'W92000004' as "geoCode", + fo."geoCodeType", + fo."locationType", + fo.geo_iso, + fo.source_id, + fo.scrape_id + from ftc_organisationlocation fo + where fo."geoCode" = 'K04000001' + and "geoCodeType" = 'ONS'; + """, + "add postcode data to location from organisation table": """ + insert into ftc_organisationlocation ( + organisation_id, + org_id, + name, + "geoCode", + "geoCodeType", + "locationType", + geo_iso, + "geo_oa11", + "geo_cty", + "geo_ctry", + "geo_laua", + "geo_ward", + "geo_rgn", + "geo_pcon", + "geo_ttwa", + "geo_lsoa11", + "geo_msoa11", + "geo_lep1", + "geo_lep2", + "geo_lat", + "geo_long", + "source_id", + "scrape_id" + ) + select fo.id as organisation_id, + fo.org_id as org_id, + fo."postalCode" as name, + geo.pcds as "geoCode", + 'PC' as "geoCodeType", + 'HQ' as "locationType", + 'GB' as geo_iso, + geo.oa11 as "geo_oa11", + geo.cty as "geo_cty", + geo.ctry as "geo_ctry", + geo.laua as "geo_laua", + geo.ward as "geo_ward", + geo.rgn as "geo_rgn", + geo.pcon as "geo_pcon", + geo.ttwa as "geo_ttwa", + geo.lsoa11 as "geo_lsoa11", + geo.msoa11 as "geo_msoa11", + geo.lep1 as "geo_lep1", + geo.lep2 as "geo_lep2", + geo.lat as "geo_lat", + geo.long as "geo_long", + fo.source_id as source_id, + fo.scrape_id as scrape_id + from ftc_organisation fo + inner join geo_postcode geo + on fo."postalCode" = geo.pcds; + """, + "delete any records from location that aren't based on current scrapes": """ + delete from ftc_organisationlocation + where id in ( + select fol.id + from ftc_organisationlocation fol + left outer join ftc_organisation fo + on fol.organisation_id = fo.id + and fol.scrape_id = fo.scrape_id + where fo.scrape_id is null + ) + """, + "add missing area information for postcodes": """ + update ftc_organisationlocation + set geo_iso = 'GB', + geo_oa11 = geo.oa11, + geo_cty = geo.cty, + geo_ctry = geo.ctry, + geo_laua = geo.laua, + geo_ward = geo.ward, + geo_rgn = geo.rgn, + geo_pcon = geo.pcon, + geo_ttwa = geo.ttwa, + geo_lsoa11 = geo.lsoa11, + geo_msoa11 = geo.msoa11, + geo_lep1 = geo.lep1, + geo_lep2 = geo.lep2, + geo_lat = geo.lat, + geo_long = geo.long + from geo_postcode geo + where ftc_organisationlocation."geoCode" = geo.pcds + and ftc_organisationlocation."geoCodeType" = 'PC' + and ftc_organisationlocation.geo_oa11 is null; + """, + "add missing area information for lookups": """ + update ftc_organisationlocation + set geo_iso = 'GB', + geo_oa11 = geo.geo_oa11, + geo_cty = geo.geo_cty, + geo_ctry = geo.geo_ctry, + geo_laua = geo.geo_laua, + geo_ward = geo.geo_ward, + geo_rgn = geo.geo_rgn, + geo_pcon = geo.geo_pcon, + geo_ttwa = geo.geo_ttwa, + geo_lsoa11 = geo.geo_lsoa11, + geo_msoa11 = geo.geo_msoa11, + geo_lep1 = geo.geo_lep1, + geo_lep2 = geo.geo_lep2, + geo_lat = geo.geo_lat, + geo_long = geo.geo_long + from geo_geolookup geo + where ftc_organisationlocation."geoCode" = geo."geoCode" + and ftc_organisationlocation."geoCodeType" = 'ONS' + and ftc_organisationlocation.geo_ctry is null; + """ + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.logger = logging.getLogger(__name__) + + def handle(self, *args, **options): + with connection.cursor() as cursor: + for name, sql in self.update_sql.items(): + self.logger.info("Starting SQL {}".format(name)) + cursor.execute(sql) + self.logger.info("Finished SQL {}".format(name)) diff --git a/geo/management/commands/update_postcodes.py b/geo/management/commands/update_postcodes.py deleted file mode 100644 index 53d20acd..00000000 --- a/geo/management/commands/update_postcodes.py +++ /dev/null @@ -1,141 +0,0 @@ -import csv -import datetime -import io -import zipfile - -import tqdm - -from ftc.management.commands._base_scraper import BaseScraper -from geo.models import Postcode - - -class Command(BaseScraper): - - name = "postcodes" - bulk_limit = 50000 - - def add_arguments(self, parser): - parser.add_argument( - "source", - help="Source file for the NPSL data (should be a ZIP file)", - ) - - def run_scraper(self, *args, **options): - - # initialise records - self.records = [] - self.object_count = 0 - - # delete existing records - Postcode.objects.all().delete() - - # download the file - if options["source"].startswith("http"): - self.logger.info("Downloading from {}".format(options["source"])) - nspl = self.session.get(options["source"]) - nspl.raise_for_status() - nspl_f = io.BytesIO(nspl.content) - else: - self.logger.info("Opening {}".format(options["source"])) - nspl_f = open(options["source"], "rb") - - # go through each postcode file and create records - with zipfile.ZipFile(nspl_f) as z: - for f in z.infolist(): - if not f.filename.startswith( - "Data/multi_csv/" - ) or not f.filename.endswith(".csv"): - continue - with z.open(f) as csvfile: - reader = csv.DictReader( - io.TextIOWrapper(csvfile, encoding="latin1") - ) - self.logger.info("Opening file {}".format(f.filename)) - for row in tqdm.tqdm(reader): - self.parse_row(row) - - self.close_spider() - - def parse_row(self, row): - - for k, v in row.items(): - if not v or v == "": - row[k] = None - - def parse_date(value): - if not value or value == "": - return None - return datetime.date( - int(value[0:4]), - int(value[4:6]), - 1, - ) - - self.add_record( - pcd=row.get("pcd"), - pcd2=row.get("pcd2"), - pcds=row.get("pcds"), - dointr=parse_date(row.get("dointr")), - doterm=parse_date(row.get("doterm")), - usertype=row.get("usertype"), - oseast1m=row.get("oseast1m"), - osnrth1m=row.get("osnrth1m"), - osgrdind=row.get("osgrdind"), - oa11=row.get("oa11"), - cty=row.get("cty"), - ced=row.get("ced"), - laua=row.get("laua"), - ward=row.get("ward"), - hlthau=row.get("hlthau"), - nhser=row.get("nhser"), - ctry=row.get("ctry"), - rgn=row.get("rgn"), - pcon=row.get("pcon"), - eer=row.get("eer"), - teclec=row.get("teclec"), - ttwa=row.get("ttwa"), - pct=row.get("pct"), - nuts=row.get("nuts"), - npark=row.get("park"), - lsoa11=row.get("lsoa11"), - msoa11=row.get("msoa11"), - wz11=row.get("wz11"), - ccg=row.get("ccg"), - bua11=row.get("bua11"), - buasd11=row.get("buasd11"), - ru11ind=row.get("ru11ind"), - oac11=row.get("oac11"), - lat=row.get("lat"), - long=row.get("long"), - lep1=row.get("lep1"), - lep2=row.get("lep2"), - pfa=row.get("pfa"), - imd=row.get("imd"), - calncv=row.get("calncv"), - stp=row.get("stp"), - ) - - def add_record(self, **kwargs): - self.records.append(Postcode(**kwargs)) - if len(self.records) >= self.bulk_limit: - self.commit_records() - - def commit_records(self): - self.object_count += len(self.records) - self.logger.info("Saving {:,.0f} records".format(len(self.records))) - Postcode.objects.bulk_create(self.records) - self.logger.info( - "Saved {:,.0f} records ({:,.0f} total)".format( - len(self.records), - self.object_count, - ) - ) - self.records = [] - - def close_spider(self): - self.commit_records() - self.scrape.items = self.object_count - results = {"records": self.object_count} - self.scrape.errors = self.error_count - self.scrape.result = results - self.scrape_logger.teardown() diff --git a/geo/migrations/0006_geolookup.py b/geo/migrations/0006_geolookup.py new file mode 100644 index 00000000..c75b60b8 --- /dev/null +++ b/geo/migrations/0006_geolookup.py @@ -0,0 +1,35 @@ +# Generated by Django 3.1.1 on 2021-02-12 15:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geo', '0005_delete_organisationlocation'), + ] + + operations = [ + migrations.CreateModel( + name='geoLookup', + fields=[ + ('geoCode', models.CharField(max_length=200, primary_key=True, serialize=False, verbose_name='Geo Code')), + ('geoCodeType', models.CharField(db_index=True, max_length=25, verbose_name='Geo Code Type')), + ('geo_iso', models.CharField(db_index=True, default='GB', max_length=3, verbose_name='ISO3166-1 Country Code')), + ('geo_oa11', models.CharField(blank=True, max_length=9, null=True, verbose_name='Output Area')), + ('geo_cty', models.CharField(blank=True, max_length=9, null=True, verbose_name='County')), + ('geo_laua', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Local Authority')), + ('geo_ward', models.CharField(blank=True, max_length=9, null=True, verbose_name='Ward')), + ('geo_ctry', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Country')), + ('geo_rgn', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Region')), + ('geo_pcon', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Parliamentary Constituency')), + ('geo_ttwa', models.CharField(blank=True, max_length=9, null=True, verbose_name='Travel to Work Area')), + ('geo_lsoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Lower Super Output Area')), + ('geo_msoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Middle Super Output Area')), + ('geo_lep1', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 1')), + ('geo_lep2', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 2')), + ('geo_lat', models.FloatField(blank=True, null=True, verbose_name='Latitude')), + ('geo_long', models.FloatField(blank=True, null=True, verbose_name='Longitude')), + ], + ), + ] diff --git a/geo/migrations/0007_geolookup_name.py b/geo/migrations/0007_geolookup_name.py new file mode 100644 index 00000000..03f7b7cc --- /dev/null +++ b/geo/migrations/0007_geolookup_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2021-02-12 15:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('geo', '0006_geolookup'), + ] + + operations = [ + migrations.AddField( + model_name='geolookup', + name='name', + field=models.CharField(blank=True, db_index=True, max_length=255, null=True), + ), + ] diff --git a/geo/models.py b/geo/models.py index f8ccbb36..7fdf1d7a 100644 --- a/geo/models.py +++ b/geo/models.py @@ -74,3 +74,83 @@ class GridIndex(models.IntegerChoices): imd = models.IntegerField(null=True, blank=True) calncv = models.CharField(max_length=9, null=True, blank=True) stp = models.CharField(max_length=9, null=True, blank=True) + + +class GeoLookup(models.Model): + geoCode = models.CharField( + max_length=200, + verbose_name="Geo Code", + primary_key=True, + ) + geoCodeType = models.CharField( + max_length=25, + db_index=True, + verbose_name="Geo Code Type", + ) + name = models.CharField(max_length=255, db_index=True, null=True, blank=True) + geo_iso = models.CharField( + max_length=3, + verbose_name="ISO3166-1 Country Code", + db_index=True, + default='GB', + ) + geo_oa11 = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Output Area" + ) + geo_cty = models.CharField( + max_length=9, null=True, blank=True, verbose_name="County" + ) + geo_laua = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Local Authority", + db_index=True, + ) + geo_ward = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Ward" + ) + geo_ctry = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Country", db_index=True + ) + geo_rgn = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Region", db_index=True + ) + geo_pcon = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Parliamentary Constituency", + db_index=True, + ) + geo_ttwa = models.CharField( + max_length=9, null=True, blank=True, verbose_name="Travel to Work Area" + ) + geo_lsoa11 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Lower Super Output Area", + db_index=True, + ) + geo_msoa11 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Middle Super Output Area", + db_index=True, + ) + geo_lep1 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Local Enterprise Partnership 1", + ) + geo_lep2 = models.CharField( + max_length=9, + null=True, + blank=True, + verbose_name="Local Enterprise Partnership 2", + ) + geo_lat = models.FloatField(null=True, blank=True, verbose_name="Latitude") + geo_long = models.FloatField(null=True, blank=True, verbose_name="Longitude") From f6e4a486cd56ed8baafd3d1bbddc6f6a50b557bb Mon Sep 17 00:00:00 2001 From: David Kane Date: Fri, 12 Feb 2021 19:15:26 +0000 Subject: [PATCH 05/19] black and isort --- ftc/migrations/0009_organisationlocation.py | 226 +++++++++++++++--- ftc/migrations/0010_auto_20210212_1524.py | 30 ++- .../0011_organisationlocation_scrape.py | 15 +- geo/management/commands/import_geolookups.py | 162 ++++++++++--- geo/management/commands/update_geodata.py | 8 +- geo/migrations/0006_geolookup.py | 158 ++++++++++-- geo/migrations/0007_geolookup_name.py | 10 +- geo/models.py | 2 +- 8 files changed, 516 insertions(+), 95 deletions(-) diff --git a/ftc/migrations/0009_organisationlocation.py b/ftc/migrations/0009_organisationlocation.py index a0eb53c8..7c81a042 100644 --- a/ftc/migrations/0009_organisationlocation.py +++ b/ftc/migrations/0009_organisationlocation.py @@ -1,44 +1,216 @@ # Generated by Django 3.1.1 on 2021-02-12 14:41 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + import ftc.models.orgid class Migration(migrations.Migration): dependencies = [ - ('ftc', '0008_auto_20201002_1408'), + ("ftc", "0008_auto_20201002_1408"), ] operations = [ migrations.CreateModel( - name='OrganisationLocation', + name="OrganisationLocation", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('org_id', ftc.models.orgid.OrgidField(db_index=True, max_length=200, verbose_name='Organisation Identifier')), - ('name', models.CharField(max_length=255, verbose_name='Name')), - ('description', models.TextField(blank=True, null=True, verbose_name='Description')), - ('geoCode', models.CharField(blank=True, max_length=200, null=True, verbose_name='Geo Code')), - ('geoCodeType', models.CharField(choices=[('PC', 'Postcode'), ('ONS', 'ONS code'), ('ISO', 'ISO Country Code')], db_index=True, default='ONS', max_length=5, verbose_name='Geo Code Type')), - ('locationType', models.CharField(choices=[('HQ', 'Registered Office'), ('AOO', 'Area of operation'), ('SITE', 'Site')], db_index=True, default='HQ', max_length=5, verbose_name='Location Type')), - ('geo_iso', models.CharField(blank=True, db_index=True, max_length=3, null=True, verbose_name='ISO Country Code')), - ('geo_oa11', models.CharField(blank=True, max_length=9, null=True, verbose_name='Output Area')), - ('geo_cty', models.CharField(blank=True, max_length=9, null=True, verbose_name='County')), - ('geo_laua', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Local Authority')), - ('geo_ward', models.CharField(blank=True, max_length=9, null=True, verbose_name='Ward')), - ('geo_ctry', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Country')), - ('geo_rgn', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Region')), - ('geo_pcon', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Parliamentary Constituency')), - ('geo_ttwa', models.CharField(blank=True, max_length=9, null=True, verbose_name='Travel to Work Area')), - ('geo_lsoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Lower Super Output Area')), - ('geo_msoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Middle Super Output Area')), - ('geo_lep1', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 1')), - ('geo_lep2', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 2')), - ('geo_lat', models.FloatField(blank=True, null=True, verbose_name='Latitude')), - ('geo_long', models.FloatField(blank=True, null=True, verbose_name='Longitude')), - ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='locations', to='ftc.organisation')), - ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='organisation_locations', to='ftc.source')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "org_id", + ftc.models.orgid.OrgidField( + db_index=True, + max_length=200, + verbose_name="Organisation Identifier", + ), + ), + ("name", models.CharField(max_length=255, verbose_name="Name")), + ( + "description", + models.TextField(blank=True, null=True, verbose_name="Description"), + ), + ( + "geoCode", + models.CharField( + blank=True, max_length=200, null=True, verbose_name="Geo Code" + ), + ), + ( + "geoCodeType", + models.CharField( + choices=[ + ("PC", "Postcode"), + ("ONS", "ONS code"), + ("ISO", "ISO Country Code"), + ], + db_index=True, + default="ONS", + max_length=5, + verbose_name="Geo Code Type", + ), + ), + ( + "locationType", + models.CharField( + choices=[ + ("HQ", "Registered Office"), + ("AOO", "Area of operation"), + ("SITE", "Site"), + ], + db_index=True, + default="HQ", + max_length=5, + verbose_name="Location Type", + ), + ), + ( + "geo_iso", + models.CharField( + blank=True, + db_index=True, + max_length=3, + null=True, + verbose_name="ISO Country Code", + ), + ), + ( + "geo_oa11", + models.CharField( + blank=True, max_length=9, null=True, verbose_name="Output Area" + ), + ), + ( + "geo_cty", + models.CharField( + blank=True, max_length=9, null=True, verbose_name="County" + ), + ), + ( + "geo_laua", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Local Authority", + ), + ), + ( + "geo_ward", + models.CharField( + blank=True, max_length=9, null=True, verbose_name="Ward" + ), + ), + ( + "geo_ctry", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Country", + ), + ), + ( + "geo_rgn", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Region", + ), + ), + ( + "geo_pcon", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Parliamentary Constituency", + ), + ), + ( + "geo_ttwa", + models.CharField( + blank=True, + max_length=9, + null=True, + verbose_name="Travel to Work Area", + ), + ), + ( + "geo_lsoa11", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Lower Super Output Area", + ), + ), + ( + "geo_msoa11", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Middle Super Output Area", + ), + ), + ( + "geo_lep1", + models.CharField( + blank=True, + max_length=9, + null=True, + verbose_name="Local Enterprise Partnership 1", + ), + ), + ( + "geo_lep2", + models.CharField( + blank=True, + max_length=9, + null=True, + verbose_name="Local Enterprise Partnership 2", + ), + ), + ( + "geo_lat", + models.FloatField(blank=True, null=True, verbose_name="Latitude"), + ), + ( + "geo_long", + models.FloatField(blank=True, null=True, verbose_name="Longitude"), + ), + ( + "organisation", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="locations", + to="ftc.organisation", + ), + ), + ( + "source", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="organisation_locations", + to="ftc.source", + ), + ), ], ), ] diff --git a/ftc/migrations/0010_auto_20210212_1524.py b/ftc/migrations/0010_auto_20210212_1524.py index 1ef65a8e..7ffc56ef 100644 --- a/ftc/migrations/0010_auto_20210212_1524.py +++ b/ftc/migrations/0010_auto_20210212_1524.py @@ -6,18 +6,34 @@ class Migration(migrations.Migration): dependencies = [ - ('ftc', '0009_organisationlocation'), + ("ftc", "0009_organisationlocation"), ] operations = [ migrations.AlterField( - model_name='organisationlocation', - name='geoCodeType', - field=models.CharField(choices=[('PC', 'UK Postcode'), ('ONS', 'ONS code'), ('ISO', 'ISO3166-1 Country Code')], db_index=True, default='ONS', max_length=5, verbose_name='Geo Code Type'), + model_name="organisationlocation", + name="geoCodeType", + field=models.CharField( + choices=[ + ("PC", "UK Postcode"), + ("ONS", "ONS code"), + ("ISO", "ISO3166-1 Country Code"), + ], + db_index=True, + default="ONS", + max_length=5, + verbose_name="Geo Code Type", + ), ), migrations.AlterField( - model_name='organisationlocation', - name='geo_iso', - field=models.CharField(blank=True, db_index=True, max_length=3, null=True, verbose_name='ISO3166-1 Country Code'), + model_name="organisationlocation", + name="geo_iso", + field=models.CharField( + blank=True, + db_index=True, + max_length=3, + null=True, + verbose_name="ISO3166-1 Country Code", + ), ), ] diff --git a/ftc/migrations/0011_organisationlocation_scrape.py b/ftc/migrations/0011_organisationlocation_scrape.py index 6af6d495..b3b56e06 100644 --- a/ftc/migrations/0011_organisationlocation_scrape.py +++ b/ftc/migrations/0011_organisationlocation_scrape.py @@ -1,20 +1,25 @@ # Generated by Django 3.1.1 on 2021-02-12 17:41 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('ftc', '0010_auto_20210212_1524'), + ("ftc", "0010_auto_20210212_1524"), ] operations = [ migrations.AddField( - model_name='organisationlocation', - name='scrape', - field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='organisation_locations', to='ftc.scrape'), + model_name="organisationlocation", + name="scrape", + field=models.ForeignKey( + default=1, + on_delete=django.db.models.deletion.CASCADE, + related_name="organisation_locations", + to="ftc.scrape", + ), preserve_default=False, ), ] diff --git a/geo/management/commands/import_geolookups.py b/geo/management/commands/import_geolookups.py index 2c391c4f..cbab5373 100644 --- a/geo/management/commands/import_geolookups.py +++ b/geo/management/commands/import_geolookups.py @@ -1,11 +1,11 @@ -import io import csv +import io from collections import namedtuple from ftc.management.commands._base_scraper import BaseScraper from geo.models import GeoLookup -GeoSource = namedtuple('GeoSource', ['link', "type", 'codefield', 'namefield']) +GeoSource = namedtuple("GeoSource", ["link", "type", "codefield", "namefield"]) class Command(BaseScraper): @@ -13,11 +13,36 @@ class Command(BaseScraper): name = "geo_lookups" bulk_limit = 50000 GEO_SOURCES = { - "la": GeoSource("https://github.com/drkane/geo-lookups/raw/master/la_all_codes.csv", "la", "LADCD", "LADNM"), - "utla": GeoSource("https://github.com/drkane/geo-lookups/raw/master/utla_all_codes.csv", "utla", "UTLACD", "UTLANM"), - "lsoa": GeoSource("https://github.com/drkane/geo-lookups/raw/master/lsoa_la.csv", "lsoa", "LSOA11CD", "LSOA11NM"), - "msoa": GeoSource("https://github.com/drkane/geo-lookups/raw/master/msoa_la.csv", "msoa", "MSOA11CD", "MSOA11HCLNM"), - "ward": GeoSource("https://github.com/drkane/geo-lookups/raw/master/ward_all_codes.csv", "ward", "WDCD", "WDNM"), + "la": GeoSource( + "https://github.com/drkane/geo-lookups/raw/master/la_all_codes.csv", + "la", + "LADCD", + "LADNM", + ), + "utla": GeoSource( + "https://github.com/drkane/geo-lookups/raw/master/utla_all_codes.csv", + "utla", + "UTLACD", + "UTLANM", + ), + "lsoa": GeoSource( + "https://github.com/drkane/geo-lookups/raw/master/lsoa_la.csv", + "lsoa", + "LSOA11CD", + "LSOA11NM", + ), + "msoa": GeoSource( + "https://github.com/drkane/geo-lookups/raw/master/msoa_la.csv", + "msoa", + "MSOA11CD", + "MSOA11HCLNM", + ), + "ward": GeoSource( + "https://github.com/drkane/geo-lookups/raw/master/ward_all_codes.csv", + "ward", + "WDCD", + "WDNM", + ), } FIELD_MATCH = { @@ -33,19 +58,106 @@ class Command(BaseScraper): } MANUAL_RECORDS = ( - {"geoCode": "E92000001", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "E92000001", "name": "England"}, - {"geoCode": "S92000003", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "S92000003", "name": "Scotland"}, - {"geoCode": "N92000002", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "N92000002", "name": "Northern Ireland"}, - {"geoCode": "W92000004", "geoCodeType": "ctry", "geo_iso": "GB", "geo_ctry": "W92000004", "name": "Wales"}, - {"geoCode": "E12000001", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000001", "name": "North East"}, - {"geoCode": "E12000002", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000002", "name": "North West"}, - {"geoCode": "E12000003", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000003", "name": "Yorkshire and The Humber"}, - {"geoCode": "E12000004", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000004", "name": "East Midlands"}, - {"geoCode": "E12000005", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000005", "name": "West Midlands"}, - {"geoCode": "E12000006", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000006", "name": "East of England"}, - {"geoCode": "E12000007", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000007", "name": "London"}, - {"geoCode": "E12000008", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000008", "name": "South East"}, - {"geoCode": "E12000009", "geoCodeType": "rgn", "geo_iso": "GB", "geo_ctry": "E92000001", "geo_rgn": "E12000009", "name": "South West"}, + { + "geoCode": "E92000001", + "geoCodeType": "ctry", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "name": "England", + }, + { + "geoCode": "S92000003", + "geoCodeType": "ctry", + "geo_iso": "GB", + "geo_ctry": "S92000003", + "name": "Scotland", + }, + { + "geoCode": "N92000002", + "geoCodeType": "ctry", + "geo_iso": "GB", + "geo_ctry": "N92000002", + "name": "Northern Ireland", + }, + { + "geoCode": "W92000004", + "geoCodeType": "ctry", + "geo_iso": "GB", + "geo_ctry": "W92000004", + "name": "Wales", + }, + { + "geoCode": "E12000001", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000001", + "name": "North East", + }, + { + "geoCode": "E12000002", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000002", + "name": "North West", + }, + { + "geoCode": "E12000003", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000003", + "name": "Yorkshire and The Humber", + }, + { + "geoCode": "E12000004", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000004", + "name": "East Midlands", + }, + { + "geoCode": "E12000005", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000005", + "name": "West Midlands", + }, + { + "geoCode": "E12000006", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000006", + "name": "East of England", + }, + { + "geoCode": "E12000007", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000007", + "name": "London", + }, + { + "geoCode": "E12000008", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000008", + "name": "South East", + }, + { + "geoCode": "E12000009", + "geoCodeType": "rgn", + "geo_iso": "GB", + "geo_ctry": "E92000001", + "geo_rgn": "E12000009", + "name": "South West", + }, ) def run_scraper(self, *args, **options): @@ -77,9 +189,7 @@ def run_scraper(self, *args, **options): # go through each geolookup file and create records for k, f in self.files.items(): - reader = csv.DictReader( - io.TextIOWrapper(f, encoding="utf-8-sig") - ) + reader = csv.DictReader(io.TextIOWrapper(f, encoding="utf-8-sig")) self.logger.info("Opening file {}".format(k)) for row in reader: self.parse_row(row, self.GEO_SOURCES[k]) @@ -97,11 +207,7 @@ def parse_row(self, row, geocodetype): geoCodeType=geocodetype.type, geo_iso="GB", name=row[geocodetype.namefield], - **{ - v: row[k] - for k, v in self.FIELD_MATCH.items() - if row.get(k) - } + **{v: row[k] for k, v in self.FIELD_MATCH.items() if row.get(k)} ) def add_record(self, **kwargs): diff --git a/geo/management/commands/update_geodata.py b/geo/management/commands/update_geodata.py index 4efea028..ec59fcdb 100644 --- a/geo/management/commands/update_geodata.py +++ b/geo/management/commands/update_geodata.py @@ -118,11 +118,11 @@ class Command(BaseCommand): "delete any records from location that aren't based on current scrapes": """ delete from ftc_organisationlocation where id in ( - select fol.id + select fol.id from ftc_organisationlocation fol - left outer join ftc_organisation fo + left outer join ftc_organisation fo on fol.organisation_id = fo.id - and fol.scrape_id = fo.scrape_id + and fol.scrape_id = fo.scrape_id where fo.scrape_id is null ) """, @@ -169,7 +169,7 @@ class Command(BaseCommand): where ftc_organisationlocation."geoCode" = geo."geoCode" and ftc_organisationlocation."geoCodeType" = 'ONS' and ftc_organisationlocation.geo_ctry is null; - """ + """, } def __init__(self, *args, **kwargs): diff --git a/geo/migrations/0006_geolookup.py b/geo/migrations/0006_geolookup.py index c75b60b8..5e0c5271 100644 --- a/geo/migrations/0006_geolookup.py +++ b/geo/migrations/0006_geolookup.py @@ -6,30 +6,150 @@ class Migration(migrations.Migration): dependencies = [ - ('geo', '0005_delete_organisationlocation'), + ("geo", "0005_delete_organisationlocation"), ] operations = [ migrations.CreateModel( - name='geoLookup', + name="geoLookup", fields=[ - ('geoCode', models.CharField(max_length=200, primary_key=True, serialize=False, verbose_name='Geo Code')), - ('geoCodeType', models.CharField(db_index=True, max_length=25, verbose_name='Geo Code Type')), - ('geo_iso', models.CharField(db_index=True, default='GB', max_length=3, verbose_name='ISO3166-1 Country Code')), - ('geo_oa11', models.CharField(blank=True, max_length=9, null=True, verbose_name='Output Area')), - ('geo_cty', models.CharField(blank=True, max_length=9, null=True, verbose_name='County')), - ('geo_laua', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Local Authority')), - ('geo_ward', models.CharField(blank=True, max_length=9, null=True, verbose_name='Ward')), - ('geo_ctry', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Country')), - ('geo_rgn', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Region')), - ('geo_pcon', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Parliamentary Constituency')), - ('geo_ttwa', models.CharField(blank=True, max_length=9, null=True, verbose_name='Travel to Work Area')), - ('geo_lsoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Lower Super Output Area')), - ('geo_msoa11', models.CharField(blank=True, db_index=True, max_length=9, null=True, verbose_name='Middle Super Output Area')), - ('geo_lep1', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 1')), - ('geo_lep2', models.CharField(blank=True, max_length=9, null=True, verbose_name='Local Enterprise Partnership 2')), - ('geo_lat', models.FloatField(blank=True, null=True, verbose_name='Latitude')), - ('geo_long', models.FloatField(blank=True, null=True, verbose_name='Longitude')), + ( + "geoCode", + models.CharField( + max_length=200, + primary_key=True, + serialize=False, + verbose_name="Geo Code", + ), + ), + ( + "geoCodeType", + models.CharField( + db_index=True, max_length=25, verbose_name="Geo Code Type" + ), + ), + ( + "geo_iso", + models.CharField( + db_index=True, + default="GB", + max_length=3, + verbose_name="ISO3166-1 Country Code", + ), + ), + ( + "geo_oa11", + models.CharField( + blank=True, max_length=9, null=True, verbose_name="Output Area" + ), + ), + ( + "geo_cty", + models.CharField( + blank=True, max_length=9, null=True, verbose_name="County" + ), + ), + ( + "geo_laua", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Local Authority", + ), + ), + ( + "geo_ward", + models.CharField( + blank=True, max_length=9, null=True, verbose_name="Ward" + ), + ), + ( + "geo_ctry", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Country", + ), + ), + ( + "geo_rgn", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Region", + ), + ), + ( + "geo_pcon", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Parliamentary Constituency", + ), + ), + ( + "geo_ttwa", + models.CharField( + blank=True, + max_length=9, + null=True, + verbose_name="Travel to Work Area", + ), + ), + ( + "geo_lsoa11", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Lower Super Output Area", + ), + ), + ( + "geo_msoa11", + models.CharField( + blank=True, + db_index=True, + max_length=9, + null=True, + verbose_name="Middle Super Output Area", + ), + ), + ( + "geo_lep1", + models.CharField( + blank=True, + max_length=9, + null=True, + verbose_name="Local Enterprise Partnership 1", + ), + ), + ( + "geo_lep2", + models.CharField( + blank=True, + max_length=9, + null=True, + verbose_name="Local Enterprise Partnership 2", + ), + ), + ( + "geo_lat", + models.FloatField(blank=True, null=True, verbose_name="Latitude"), + ), + ( + "geo_long", + models.FloatField(blank=True, null=True, verbose_name="Longitude"), + ), ], ), ] diff --git a/geo/migrations/0007_geolookup_name.py b/geo/migrations/0007_geolookup_name.py index 03f7b7cc..664233ca 100644 --- a/geo/migrations/0007_geolookup_name.py +++ b/geo/migrations/0007_geolookup_name.py @@ -6,13 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('geo', '0006_geolookup'), + ("geo", "0006_geolookup"), ] operations = [ migrations.AddField( - model_name='geolookup', - name='name', - field=models.CharField(blank=True, db_index=True, max_length=255, null=True), + model_name="geolookup", + name="name", + field=models.CharField( + blank=True, db_index=True, max_length=255, null=True + ), ), ] diff --git a/geo/models.py b/geo/models.py index 7fdf1d7a..1a6c85cb 100644 --- a/geo/models.py +++ b/geo/models.py @@ -92,7 +92,7 @@ class GeoLookup(models.Model): max_length=3, verbose_name="ISO3166-1 Country Code", db_index=True, - default='GB', + default="GB", ) geo_oa11 = models.CharField( max_length=9, null=True, blank=True, verbose_name="Output Area" From f790ece139203646c0f3f39d49e7ca9055118a13 Mon Sep 17 00:00:00 2001 From: David Kane Date: Fri, 12 Feb 2021 22:09:22 +0000 Subject: [PATCH 06/19] display location in front end --- findthatcharity/jinja2.py | 6 +++ findthatcharity/utils.py | 2 +- ftc/jinja2/components/org_record.html.j2 | 28 -------------- ftc/jinja2/org.html.j2 | 4 +- ftc/jinja2/panels/locations.html.j2 | 40 +++++++++++--------- ftc/models/organisation.py | 24 +++++++++--- ftc/models/organisation_location.py | 5 +++ geo/management/commands/import_geolookups.py | 22 +++++------ 8 files changed, 66 insertions(+), 65 deletions(-) diff --git a/findthatcharity/jinja2.py b/findthatcharity/jinja2.py index 2865751e..b0531b76 100644 --- a/findthatcharity/jinja2.py +++ b/findthatcharity/jinja2.py @@ -18,6 +18,7 @@ url_replace, ) from ftc.models import Organisation, OrganisationType, OrgidScheme, Source +from geo.models import GeoLookup from jinja2 import Environment @@ -80,6 +81,10 @@ def get_orgidschemes(): return value +def get_geoname(code): + return GeoLookup.objects.get(geoCode=code).name + + def environment(**options): env = Environment(**options) @@ -104,6 +109,7 @@ def environment(**options): "slugify": slugify, "titlecase": to_titlecase, "pluralise": pluralise, + "get_geoname": get_geoname, } ) return env diff --git a/findthatcharity/utils.py b/findthatcharity/utils.py index 7df48935..88e39dbc 100644 --- a/findthatcharity/utils.py +++ b/findthatcharity/utils.py @@ -117,7 +117,7 @@ def regex_search(s, regex): def list_to_string(items, sep=", ", final_sep=" and "): - if not isinstance(items, list): + if isinstance(items, str): return items if len(items) == 1: diff --git a/ftc/jinja2/components/org_record.html.j2 b/ftc/jinja2/components/org_record.html.j2 index 5f527cf9..0fab8a9e 100644 --- a/ftc/jinja2/components/org_record.html.j2 +++ b/ftc/jinja2/components/org_record.html.j2 @@ -119,32 +119,4 @@ {% endif %}
- {# {% if charity["geo"]["location"] %} -
-
-

Location

-
-
-
- - - {% if charity.get("geo", {}).get("location") %} - - - - - {% endif %} - - - -
Latitude/longitude - - {{ charity.get("geo", {}).get("location",{}).get("lat") }}, - {{ charity.get("geo", {}).get("location",{}).get("lon") }} - -
Postcode{{ charity.get("geo", {}).get("postcode") }}
-
-
-
- {% endif %} #} {% endmacro %} \ No newline at end of file diff --git a/ftc/jinja2/org.html.j2 b/ftc/jinja2/org.html.j2 index b5ed9c3b..6eb9fc1f 100644 --- a/ftc/jinja2/org.html.j2 +++ b/ftc/jinja2/org.html.j2 @@ -30,7 +30,7 @@ {% block headscripts %} {{ super() }} -{% if org.location or org.postalCode %} +{% if org.locations.exists() or org.postalCode %} @@ -45,7 +45,7 @@ {% block bodyscripts %} {{ super() }} -{% if org.location or org.postalCode %} +{% if org.locations.exists() or org.postalCode %} {% endif %} diff --git a/ftc/jinja2/panels/locations.html.j2 b/ftc/jinja2/panels/locations.html.j2 index a1903e1f..f40300d1 100644 --- a/ftc/jinja2/panels/locations.html.j2 +++ b/ftc/jinja2/panels/locations.html.j2 @@ -1,43 +1,57 @@ {% if org.locations.exists() or org.postalCode %} {% call org_panel("Locations", org) %} -
+
-
+

Depending on the data source, location may describe the headquarters of the organisation rather than the area it operates in.

{% if org.postalCode %}
-
Postcode
+

Postcode

{{ org.postalCode }}

{% endif %} {% if org.locations.exists() %} -
-
Locations
- {% for country, areas in org.locations_group().items() %} -
    - {% if country != "GB" %} -
  • {{ country }}
    • - {% else %} - {% for l, types in areas.items() %} + {% for areatype, countries in org.locations_group()|dictsort %} + {% for country, areas in countries|dictsort|selectattr(0, "equalto", 'GB') %} + {% if loop.first %} +
    +

    {{ areatype }} in the UK

    +
      + {% endif %} + {% for l in areas|sort %} +
    • {% if l and l|regex_search("[ENWSK][0-9]{8}") %} -
    • - - {{ l|get_geoname }} - : {{ types|list_to_string }} -
      • + + {{ l|get_geoname }} + {% else %} -
    • - {{ l|get_geoname }} ({{ l }}) -
      • + {{ l|get_geoname }} ({{ l }}) {% endif %} + {% endfor %} - {% endif %} + {% if loop.last %}
    +
    + {% endif %} + {% endfor %} + {% for country, areas in countries|dictsort|rejectattr(0, "equalto", 'GB') %} + {% if loop.first %} +
    +

    {{ areatype }} outside the UK

    +
      + {% endif %} + {% for area in areas|sort %} +
    • {{ area|get_geoname }}
      • + {% endfor %} + {% if loop.last %} +
    +
    + {% endif %} + {% endfor %} {% endfor %} -
{% endif %}
{% endcall %} diff --git a/ftc/models/organisation.py b/ftc/models/organisation.py index 3ed2aa0c..837bc285 100644 --- a/ftc/models/organisation.py +++ b/ftc/models/organisation.py @@ -336,23 +336,47 @@ def geoCodes(self): "K04000001": ["E92000001", "W92000004"], # England and Wales } - for v in self.locations.all(): - if re.match("[ENWSK][0-9]{8}", v.geoCode): + for location in self.locations.all(): + if re.match("[ENWSK][0-9]{8}", location.geoCode): # special case for combinations of countries - if v.geoCode in special_cases: - for a in special_cases[v.geoCode]: + if location.geoCode in special_cases: + for a in special_cases[location.geoCode]: yield a continue - yield v.geoCode + yield location.geoCode def locations_group(self): locations = defaultdict(lambda: defaultdict(set)) - for v in self.locations.all(): - print(OrganisationLocation.LocationTypes) - location_type = OrganisationLocation.LocationTypes(v.locationType).label - if v.geo_laua: - locations[v.geo_iso][v.geo_laua].add(location_type) + for location in self.locations.all(): + location_type = OrganisationLocation.LocationTypes(location.locationType).label + if location.geoCodeType == OrganisationLocation.GeoCodeTypes.POSTCODE: + locations[location_type][location.geo_iso].add(location.geo_laua) else: - locations[v.geo_iso][v.geo_iso].add(location_type) + locations[location_type][location.geo_iso].add(location.geoCode) return locations + + @property + def location(self): + locations = [] + for location in self.locations.all(): + if location.geoCodeType == OrganisationLocation.GeoCodeTypes.POSTCODE: + geocode = location.geo_laua + else: + geocode = location.geoCode + locations.append({ + "id": location.geoCode, + "name": location.name, + "geoCode": geocode, + "type": location.locationType, + }) + return locations + + @property + def lat_lngs(self): + return_lat_lngs = [] + for location in self.locations.all(): + if location.geo_lat and location.geo_long: + location_type = OrganisationLocation.LocationTypes(location.locationType).label + return_lat_lngs.append((location.geo_lat, location.geo_long, location_type, location.name)) + return return_lat_lngs diff --git a/ftc/static/js/locationmap.js b/ftc/static/js/locationmap.js index e08c19c5..c985369f 100644 --- a/ftc/static/js/locationmap.js +++ b/ftc/static/js/locationmap.js @@ -5,7 +5,7 @@ var bounds = L.latLngBounds( L.latLng(49.8647440573549, -8.649995833304311), L.latLng(60.86078239016185, 1.763705609663519), ); -if (GEOCODES || (ORG_LAT && ORG_LONG)) { +if (GEOCODES || ORG_LAT_LONGS) { var map = L.map('locationmap').setView([51.505, -0.09], 13); map.scrollWheelZoom.disable(); L.tileLayer(TILES, { style: 'toner' }).addTo(map); @@ -30,16 +30,16 @@ if (GEOCODES || (ORG_LAT && ORG_LONG)) { }); } - if (ORG_LAT && ORG_LONG) { - var point = L.latLng([ - ORG_LAT, - ORG_LONG, - ]); - var marker = L.marker(point).addTo(map); - if(POSTCODE){ - marker.bindPopup(POSTCODE); - } - bounds.extend(point); - map.fitBounds(bounds); + if (ORG_LAT_LONGS) { + ORG_LAT_LONGS.forEach((latlng) => { + var point = L.latLng([ + latlng[0], + latlng[1], + ]); + var marker = L.marker(point).addTo(map); + marker.bindPopup(`${latlng[2]}: ${latlng[3]}`); + bounds.extend(point); + map.fitBounds(bounds); + }); } } \ No newline at end of file diff --git a/geo/management/commands/import_geolookups.py b/geo/management/commands/import_geolookups.py index 219582c7..c2342ded 100644 --- a/geo/management/commands/import_geolookups.py +++ b/geo/management/commands/import_geolookups.py @@ -2,6 +2,8 @@ import io from collections import namedtuple +import pycountry + from ftc.management.commands._base_scraper import BaseScraper from geo.models import GeoLookup @@ -171,6 +173,15 @@ def run_scraper(self, *args, **options): GeoLookup.objects.all().delete() self.logger.info("Deleting existing items") + # Import countries from pycountry + for country in pycountry.countries: + self.add_record(**{ + "geoCode": country.alpha_2, + "geoCodeType": "iso", + "geo_iso": country.alpha_2, + "name": country.name + }) + # create the manual records for m in self.MANUAL_RECORDS: self.add_record(**m) diff --git a/geo/management/commands/update_geodata.py b/geo/management/commands/update_geodata.py index ec59fcdb..0da8773c 100644 --- a/geo/management/commands/update_geodata.py +++ b/geo/management/commands/update_geodata.py @@ -62,6 +62,11 @@ class Command(BaseCommand): where fo."geoCode" = 'K04000001' and "geoCodeType" = 'ONS'; """, + 'remove any "ENGLAND AND WALES" references': """ + delete from ftc_organisationlocation fo + where fo."geoCode" = 'K04000001' + and "geoCodeType" = 'ONS'; + """, "add postcode data to location from organisation table": """ insert into ftc_organisationlocation ( organisation_id, diff --git a/requirements.in b/requirements.in index da67fb62..d5d6a30e 100644 --- a/requirements.in +++ b/requirements.in @@ -24,4 +24,5 @@ titlecase psycopg2 bcp-reader PyYAML -lxml==4.6.2 \ No newline at end of file +lxml==4.6.2 +pycountry \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7ff8ec94..613444b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,7 @@ markupsafe==1.1.1 # via jinja2 openpyxl==3.0.3 # via -r requirements.in parse==1.15.0 # via requests-html psycopg2==2.8.6 # via -r requirements.in +pycountry==20.7.3 # via -r requirements.in pydantic==1.7.3 # via django-ninja pyee==7.0.2 # via pyppeteer pyexcel-ezodf==0.3.4 # via pyexcel-ods3 From 3371683b0926831fe8db4934613f6f4851973a85 Mon Sep 17 00:00:00 2001 From: David Kane Date: Wed, 24 Mar 2021 17:17:28 +0000 Subject: [PATCH 09/19] add location filter and facet --- findthatcharity/jinja2.py | 18 ++++++++++++++ ftc/documents.py | 2 +- ftc/jinja2/components/search_facets.html.j2 | 17 ++++++++++++- ftc/jinja2/index.html.j2 | 1 + ftc/jinja2/orgtype.html.j2 | 13 ++++++++-- ftc/models/organisation.py | 27 +++++++++++++++++++++ ftc/query.py | 15 ++++++++++++ 7 files changed, 89 insertions(+), 4 deletions(-) diff --git a/findthatcharity/jinja2.py b/findthatcharity/jinja2.py index 7adb73e4..604ef7e9 100644 --- a/findthatcharity/jinja2.py +++ b/findthatcharity/jinja2.py @@ -81,6 +81,23 @@ def get_orgidschemes(): return value +def get_locations(): + cache_key = "locationnames" + value = cache.get(cache_key) + if value: + return value + if GeoLookup._meta.db_table in connection.introspection.table_names(): + value = {} + for s in GeoLookup.objects.all(): + if s.geoCodeType not in value: + value[s.geoCodeType] = {} + value[s.geoCodeType][s.geoCode] = s.name + cache.set(cache_key, value, 60 * 60) + else: + value = {} + return value + + def get_geoname(code): try: return GeoLookup.objects.get(geoCode=code).name @@ -99,6 +116,7 @@ def environment(**options): "get_orgtypes": get_orgtypes, "get_sources": get_sources, "get_orgidschemes": get_orgidschemes, + "get_locations": get_locations, "url_replace": url_replace, "url_remove": url_remove, "ga_tracking": settings.GOOGLE_ANALYTICS, diff --git a/ftc/documents.py b/ftc/documents.py index f9e4a138..0c163489 100644 --- a/ftc/documents.py +++ b/ftc/documents.py @@ -156,7 +156,7 @@ def prepare_domain(self, instance): return instance.domain def prepare_location(self, instance): - return list(set([location['geoCode'] for location in instance.location])) + return instance.allGeoCodes def prepare_latestIncome(self, instance): return instance.latestIncome diff --git a/ftc/jinja2/components/search_facets.html.j2 b/ftc/jinja2/components/search_facets.html.j2 index 1a7ae70c..0a58a5a2 100644 --- a/ftc/jinja2/components/search_facets.html.j2 +++ b/ftc/jinja2/components/search_facets.html.j2 @@ -72,4 +72,19 @@
{% endfor %} - \ No newline at end of file + +{% for locationtype, locationtypename in (('ctry', 'UK Country'), ('rgn', 'English Region'), ('la', 'UK Local Authority'), ('iso', 'Country')) %} +

{{ locationtypename }}

+
    +{% for areacode, areaname in locations[locationtype]|dictsort %} + {% if areacode in search.aggregation.by_location %} +
  • +
    + {{ areaname }} + {{ "{:,.0f}".format(search.aggregation.by_location[areacode]) }} +
    +
    • + {% endif %} +{% endfor %} +
+{% endfor %} \ No newline at end of file diff --git a/ftc/jinja2/index.html.j2 b/ftc/jinja2/index.html.j2 index 211ab7fc..a997e8c3 100644 --- a/ftc/jinja2/index.html.j2 +++ b/ftc/jinja2/index.html.j2 @@ -3,6 +3,7 @@ {% set sources = get_sources() %} {% set orgtypes = get_orgtypes() %} +{% set locations = get_locations() %} {% from 'components/org_id.html.j2' import orgid_link %} {% block content %} diff --git a/ftc/jinja2/orgtype.html.j2 b/ftc/jinja2/orgtype.html.j2 index b3827f58..2d1ef5ae 100644 --- a/ftc/jinja2/orgtype.html.j2 +++ b/ftc/jinja2/orgtype.html.j2 @@ -1,6 +1,7 @@ {% extends 'base.html.j2' %} {% set sources = get_sources() %} {% set orgtypes = get_orgtypes() %} +{% set locations = get_locations() %} {% set heading = '{}'.format( base_query.title, ) if base_query else 'Search Results' %} @@ -29,7 +30,7 @@
{% include 'components/search_form.html.j2' %} - {% if search.other_orgtypes or search.source %} + {% if search.other_orgtypes or search.source or search.location %}
Filters: {% if search.other_orgtypes %} @@ -48,7 +49,15 @@ {% endfor %} {% endif %} - Clear filters + {% if search.location %} + {% for l in search.location if base_query.slug != l %} + + Location: + {{ l|get_geoname }} + + {% endfor %} + {% endif %} + Clear filters
{% endif %}
diff --git a/ftc/models/organisation.py b/ftc/models/organisation.py index 837bc285..2245b022 100644 --- a/ftc/models/organisation.py +++ b/ftc/models/organisation.py @@ -345,6 +345,33 @@ def geoCodes(self): continue yield location.geoCode + @property + def allGeoCodes(self): + + location_fields = [ + "geo_iso", + "geo_oa11", + "geo_cty", + "geo_laua", + "geo_ward", + "geo_ctry", + "geo_rgn", + "geo_pcon", + "geo_ttwa", + "geo_lsoa11", + "geo_msoa11", + "geo_lep1", + "geo_lep2", + ] + + locations = set() + for location in self.locations.all(): + for f in location_fields: + value = getattr(location, f, None) + if value and not value.endswith("999999"): + locations.add(value) + return list(locations) + def locations_group(self): locations = defaultdict(lambda: defaultdict(set)) diff --git a/ftc/query.py b/ftc/query.py index 9af50620..a96ac2de 100644 --- a/ftc/query.py +++ b/ftc/query.py @@ -78,6 +78,7 @@ def __init__(self, results_per_page=25, **kwargs): self.active = None self.domain = None self.postcode = None + self.location = None self.query = None self.paginator = None @@ -95,6 +96,7 @@ def set_criteria( active=None, domain=None, postcode=None, + location=None, ): if term and isinstance(term, str): self.term = term @@ -103,6 +105,7 @@ def set_criteria( (base_orgtype, "base_orgtype"), (other_orgtypes, "other_orgtypes"), (source, "source"), + (location, "location"), ]: if t: if isinstance(t, str) and t != "all": @@ -127,6 +130,8 @@ def set_criteria_from_request(self, request): self.set_criteria(other_orgtypes=request.GET.getlist("orgtype")) if "source" in request.GET and request.GET.get("source") != "all": self.set_criteria(source=request.GET.getlist("source")) + if "location" in request.GET and request.GET.get("location") != "all": + self.set_criteria(location=request.GET.getlist("location")) if "q" in request.GET: self.set_criteria(term=request.GET["q"]) if request.GET.get("active", "").lower().startswith("t"): @@ -182,6 +187,10 @@ def run_es(self, with_pagination=False, with_aggregation=False): if self.source: filter_.append({"terms": {"source": self.source}}) + # check for source + if self.location: + filter_.append({"terms": {"location": self.location}}) + # check for active or inactive organisations if self.active is True: filter_.append({"match": {"active": True}}) @@ -206,9 +215,11 @@ def run_es(self, with_pagination=False, with_aggregation=False): by_source = A("terms", field="source", size=150) by_orgtype = A("terms", field="organisationType", size=150) by_active = A("terms", field="active", size=150) + by_location = A("terms", field="location", size=150) q.aggs.bucket("by_source", by_source) q.aggs.bucket("by_orgtype", by_orgtype) q.aggs.bucket("by_active", by_active) + q.aggs.bucket("by_location", by_location) self.query = q.execute(params=params) if with_pagination: @@ -223,6 +234,10 @@ def run_es(self, with_pagination=False, with_aggregation=False): {"orgtype": b["key"], "records": b["doc_count"]} for b in self.query.aggregations["by_orgtype"]["buckets"] ] + self.aggregation["by_location"] = { + b["key"]: b["doc_count"] + for b in self.query.aggregations["by_location"]["buckets"] + } self.aggregation["by_active"] = { "active": 0, "inactive": 0, From faf76dc5dae1dd5ee7ed4e69b89cc78bb14c38b8 Mon Sep 17 00:00:00 2001 From: David Kane Date: Wed, 24 Mar 2021 18:54:11 +0000 Subject: [PATCH 10/19] isort and black --- .../0012_remove_organisation_location.py | 6 +- ftc/migrations/0013_auto_20210212_2215.py | 58 +++++++++---------- ftc/models/organisation.py | 28 +++++---- ftc/models/organisation_location.py | 4 +- geo/management/commands/import_geolookups.py | 14 +++-- 5 files changed, 59 insertions(+), 51 deletions(-) diff --git a/ftc/migrations/0012_remove_organisation_location.py b/ftc/migrations/0012_remove_organisation_location.py index 0f1f6191..fcad1ff2 100644 --- a/ftc/migrations/0012_remove_organisation_location.py +++ b/ftc/migrations/0012_remove_organisation_location.py @@ -6,12 +6,12 @@ class Migration(migrations.Migration): dependencies = [ - ('ftc', '0011_organisationlocation_scrape'), + ("ftc", "0011_organisationlocation_scrape"), ] operations = [ migrations.RemoveField( - model_name='organisation', - name='location', + model_name="organisation", + name="location", ), ] diff --git a/ftc/migrations/0013_auto_20210212_2215.py b/ftc/migrations/0013_auto_20210212_2215.py index 6caca340..416f90db 100644 --- a/ftc/migrations/0013_auto_20210212_2215.py +++ b/ftc/migrations/0013_auto_20210212_2215.py @@ -6,64 +6,64 @@ class Migration(migrations.Migration): dependencies = [ - ('ftc', '0012_remove_organisation_location'), + ("ftc", "0012_remove_organisation_location"), ] operations = [ migrations.RemoveField( - model_name='organisation', - name='geo_ctry', + model_name="organisation", + name="geo_ctry", ), migrations.RemoveField( - model_name='organisation', - name='geo_cty', + model_name="organisation", + name="geo_cty", ), migrations.RemoveField( - model_name='organisation', - name='geo_lat', + model_name="organisation", + name="geo_lat", ), migrations.RemoveField( - model_name='organisation', - name='geo_laua', + model_name="organisation", + name="geo_laua", ), migrations.RemoveField( - model_name='organisation', - name='geo_lep1', + model_name="organisation", + name="geo_lep1", ), migrations.RemoveField( - model_name='organisation', - name='geo_lep2', + model_name="organisation", + name="geo_lep2", ), migrations.RemoveField( - model_name='organisation', - name='geo_long', + model_name="organisation", + name="geo_long", ), migrations.RemoveField( - model_name='organisation', - name='geo_lsoa11', + model_name="organisation", + name="geo_lsoa11", ), migrations.RemoveField( - model_name='organisation', - name='geo_msoa11', + model_name="organisation", + name="geo_msoa11", ), migrations.RemoveField( - model_name='organisation', - name='geo_oa11', + model_name="organisation", + name="geo_oa11", ), migrations.RemoveField( - model_name='organisation', - name='geo_pcon', + model_name="organisation", + name="geo_pcon", ), migrations.RemoveField( - model_name='organisation', - name='geo_rgn', + model_name="organisation", + name="geo_rgn", ), migrations.RemoveField( - model_name='organisation', - name='geo_ttwa', + model_name="organisation", + name="geo_ttwa", ), migrations.RemoveField( - model_name='organisation', - name='geo_ward', + model_name="organisation", + name="geo_ward", ), ] diff --git a/ftc/models/organisation.py b/ftc/models/organisation.py index 2245b022..be50cd19 100644 --- a/ftc/models/organisation.py +++ b/ftc/models/organisation.py @@ -8,9 +8,9 @@ from django_better_admin_arrayfield.models.fields import ArrayField from .organisation_link import OrganisationLink +from .organisation_location import OrganisationLocation from .orgid import OrgidField from .orgid_scheme import OrgidScheme -from .organisation_location import OrganisationLocation EXTERNAL_LINKS = { "GB-CHC": [ @@ -376,7 +376,9 @@ def locations_group(self): locations = defaultdict(lambda: defaultdict(set)) for location in self.locations.all(): - location_type = OrganisationLocation.LocationTypes(location.locationType).label + location_type = OrganisationLocation.LocationTypes( + location.locationType + ).label if location.geoCodeType == OrganisationLocation.GeoCodeTypes.POSTCODE: locations[location_type][location.geo_iso].add(location.geo_laua) else: @@ -391,12 +393,14 @@ def location(self): geocode = location.geo_laua else: geocode = location.geoCode - locations.append({ - "id": location.geoCode, - "name": location.name, - "geoCode": geocode, - "type": location.locationType, - }) + locations.append( + { + "id": location.geoCode, + "name": location.name, + "geoCode": geocode, + "type": location.locationType, + } + ) return locations @property @@ -404,6 +408,10 @@ def lat_lngs(self): return_lat_lngs = [] for location in self.locations.all(): if location.geo_lat and location.geo_long: - location_type = OrganisationLocation.LocationTypes(location.locationType).label - return_lat_lngs.append((location.geo_lat, location.geo_long, location_type, location.name)) + location_type = OrganisationLocation.LocationTypes( + location.locationType + ).label + return_lat_lngs.append( + (location.geo_lat, location.geo_long, location_type, location.name) + ) return return_lat_lngs diff --git a/ftc/models/organisation_location.py b/ftc/models/organisation_location.py index 0bbc866f..854a5b9e 100644 --- a/ftc/models/organisation_location.py +++ b/ftc/models/organisation_location.py @@ -120,6 +120,4 @@ class GeoCodeTypes(models.TextChoices): geo_long = models.FloatField(null=True, blank=True, verbose_name="Longitude") def __repr__(self): - return "".format( - self.geoCode - ) + return "".format(self.geoCode) diff --git a/geo/management/commands/import_geolookups.py b/geo/management/commands/import_geolookups.py index c2342ded..1473a255 100644 --- a/geo/management/commands/import_geolookups.py +++ b/geo/management/commands/import_geolookups.py @@ -175,12 +175,14 @@ def run_scraper(self, *args, **options): # Import countries from pycountry for country in pycountry.countries: - self.add_record(**{ - "geoCode": country.alpha_2, - "geoCodeType": "iso", - "geo_iso": country.alpha_2, - "name": country.name - }) + self.add_record( + **{ + "geoCode": country.alpha_2, + "geoCodeType": "iso", + "geo_iso": country.alpha_2, + "name": country.name, + } + ) # create the manual records for m in self.MANUAL_RECORDS: From f6f0d37ac7a5b08d00cf19e88ac259349f278a9d Mon Sep 17 00:00:00 2001 From: David Kane Date: Sun, 4 Apr 2021 23:49:58 +0100 Subject: [PATCH 11/19] use new charity commission download --- charity/admin.py | 1 - charity/feeds.py | 45 -- charity/management/commands/_ccew_sql.py | 278 ++++--- .../management/commands/ccew_data_fetch.py | 21 - charity/management/commands/charity_setup.py | 29 +- charity/management/commands/import_ccew.py | 258 ++----- charity/migrations/0010_auto_20210403_1729.py | 40 + ..._1729_squashed_0016_delete_ccewdatafile.py | 369 ++++++++++ charity/migrations/0011_ccew_updated_files.py | 308 ++++++++ charity/migrations/0012_auto_20210403_1943.py | 18 + charity/migrations/0013_auto_20210404_2125.py | 32 + charity/migrations/0014_auto_20210404_2145.py | 18 + charity/migrations/0015_auto_20210404_2147.py | 18 + .../migrations/0016_delete_ccewdatafile.py | 16 + charity/models.py | 469 ------------ charity/models/__init__.py | 286 ++++++++ charity/models/ccew.py | 689 ++++++++++++++++++ charity/urls.py | 4 +- 18 files changed, 1998 insertions(+), 901 deletions(-) delete mode 100644 charity/feeds.py delete mode 100644 charity/management/commands/ccew_data_fetch.py create mode 100644 charity/migrations/0010_auto_20210403_1729.py create mode 100644 charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py create mode 100644 charity/migrations/0011_ccew_updated_files.py create mode 100644 charity/migrations/0012_auto_20210403_1943.py create mode 100644 charity/migrations/0013_auto_20210404_2125.py create mode 100644 charity/migrations/0014_auto_20210404_2145.py create mode 100644 charity/migrations/0015_auto_20210404_2147.py create mode 100644 charity/migrations/0016_delete_ccewdatafile.py delete mode 100644 charity/models.py create mode 100644 charity/models/__init__.py create mode 100644 charity/models/ccew.py diff --git a/charity/admin.py b/charity/admin.py index d229c39b..607c495a 100644 --- a/charity/admin.py +++ b/charity/admin.py @@ -105,4 +105,3 @@ def code_(self, obj): admin.site.register(charity.AreaOfOperation, JSONFieldAdmin) admin.site.register(charity.Vocabulary, VocabularyAdmin) admin.site.register(charity.VocabularyEntries, VocabularyEntriesAdmin) -admin.site.register(charity.CcewDataFile, JSONFieldAdmin) diff --git a/charity/feeds.py b/charity/feeds.py deleted file mode 100644 index 2c82d342..00000000 --- a/charity/feeds.py +++ /dev/null @@ -1,45 +0,0 @@ -from django.contrib.syndication.views import Feed -from django.utils.feedgenerator import Atom1Feed -from requests_html import HTMLSession - -from charity.models import CcewDataFile - -CCEW_DATA_URL = "https://register-of-charities.charitycommission.gov.uk/register/full-register-download" - - -def fetch_ccew_data(): - session = HTMLSession() - r = session.get(CCEW_DATA_URL) - for d in r.html.find("blockquote.download"): - for p in d.find("p"): - if "Charity register extract" in p.text: - links = p.absolute_links - for link in links: - f, created = CcewDataFile.objects.update_or_create( - title=d.find("h4", first=True).text, - defaults=dict(url=link, description=p.text), - ) - - -class CcewDataFeedRSS(Feed): - title = "Charity data download" - link = CCEW_DATA_URL - description = "RSS feed for updates to the Data download service provided by the Charity Commission for registered charities in England and Wales." - - def items(self): - fetch_ccew_data() - return CcewDataFile.objects.order_by("-first_added")[:10] - - def item_title(self, item): - return item.title - - def item_description(self, item): - return item.description - - def item_pubdate(self, item): - return item.first_added - - -class CcewDataFeedAtom(CcewDataFeedRSS): - feed_type = Atom1Feed - subtitle = CcewDataFeedRSS.description diff --git a/charity/management/commands/_ccew_sql.py b/charity/management/commands/_ccew_sql.py index 47e46235..a6dccd40 100644 --- a/charity/management/commands/_ccew_sql.py +++ b/charity/management/commands/_ccew_sql.py @@ -5,44 +5,43 @@ ] = """ insert into charity_charity as cc (id, name, constitution , geographical_spread, address, postcode, phone, active, date_registered, date_removed, removal_reason, web, email, - company_number, source, first_added, last_updated ) -select distinct on(c.regno) - CONCAT('GB-CHC-', c.regno) as "id", - c.name as "name", - c.gd as "constitution", - c.aob as "geographical_spread", + company_number, activities, source, first_added, last_updated, income, spending, latest_fye) +select distinct on(c.registered_charity_number) + CONCAT('GB-CHC-', c.registered_charity_number) as "id", + c.charity_name as "name", + cgd.governing_document_description as "constitution", + cgd.area_of_benefit as "geographical_spread", NULLIF(concat_ws(', ', - c."add1", - c."add2", - c."add3", - c."add4", - c."add5" + c."charity_contact_address1", + c."charity_contact_address2", + c."charity_contact_address3", + c."charity_contact_address4", + c."charity_contact_address5" ), '') as "address", - c.postcode as postcode, - c.phone as "phone", - c.orgtype = 'R' as active, - to_date(reg.data->0->>'regdate', 'YYYY-MM-DD') as "date_registered", - to_date(reg.data->-1->>'remdate', 'YYYY-MM-DD') as "date_removed", - reg.data->-1->>'remcode' as "removal_reason", - cm.web as "web", - cm.email as "email", - lpad(cm.coyno, 8, '0') as "company_number", + c.charity_contact_postcode as postcode, + c.charity_contact_phone as "phone", + c.charity_registration_status = 'Registered' as active, + c.date_of_registration as "date_registered", + c.date_of_removal as "date_removed", + ceh.reason as "removal_reason", + c.charity_contact_web as "web", + c.charity_contact_email as "email", + lpad(c.charity_company_registration_number, 8, '0') as "company_number", + coalesce(c.charity_activities, cgd.charitable_objects) as activities, '{source}' as "source", NOW() as "first_added", - NOW() as "last_updated" + NOW() as "last_updated", + c.latest_income as income, + c.latest_expenditure as spending, + c.latest_acc_fin_period_end_date as latest_fye from charity_ccewcharity c - left outer join charity_ccewmaincharity cm - on c.regno = cm.regno - left outer join ( - select regno, - subno, - jsonb_agg(cr order by regdate desc) as data - from charity_ccewregistration cr - group by regno, subno - ) as reg - on c.regno = reg.regno - and c.subno = reg.subno -where c.subno = 0 + left outer join charity_ccewcharitygoverningdocument cgd + on c.organisation_number = cgd.organisation_number + left outer join charity_ccewcharityeventhistory ceh + on c.organisation_number = ceh.organisation_number + and ceh.date_of_event = c.date_of_removal + and ceh.event_type = 'Removed' +where c.linked_charity_number = 0 on conflict (id) do update set "name" = EXCLUDED.name, constitution = COALESCE(EXCLUDED.constitution, cc.constitution), @@ -57,30 +56,36 @@ web = COALESCE(EXCLUDED.web, cc.web), email = COALESCE(EXCLUDED.email, cc.email), company_number = COALESCE(EXCLUDED.company_number, cc.company_number), + activities = COALESCE(EXCLUDED.activities, cc.activities), source = EXCLUDED.source, first_added = cc.first_added, - last_updated = EXCLUDED.last_updated; + last_updated = EXCLUDED.last_updated, + income = COALESCE(EXCLUDED.income, cc.income), + spending = COALESCE(EXCLUDED.spending, cc.spending), + latest_fye = COALESCE(EXCLUDED.latest_fye, cc.latest_fye) """ UPDATE_CCEW[ "Insert into charity financial" ] = """ -insert into charity_charityfinancial as cf (charity_id, fyend, fystart, income, spending, account_type) +insert into charity_charityfinancial as cf (charity_id, fyend, fystart, income, spending, volunteers, account_type) select DISTINCT ON ("org_id", "fyend") a.* from ( - select CONCAT('GB-CHC-', c.regno) as org_id, - c."fyend", - c."fystart", - c."income", - c."expend" as "spending", + select CONCAT('GB-CHC-', c.registered_charity_number) as org_id, + c.fin_period_end_date as "fyend", + c.fin_period_start_date as "fystart", + c.total_gross_income as "income", + c.total_gross_expenditure as "spending", + c.count_volunteers as "volunteers", 'basic' as "account_type" - from charity_ccewfinancial c - where "income" is not null + from charity_ccewcharityarparta c + where "total_gross_income" is not null ) as a on conflict (charity_id, fyend) do update set fystart = EXCLUDED.fystart, income = EXCLUDED.income, spending = EXCLUDED.spending, + volunteers = EXCLUDED.volunteers, account_type = cf.account_type; """ @@ -89,79 +94,59 @@ ] = """ update charity_charityfinancial cf set - inc_leg = a.inc_leg, - inc_end = a.inc_end, - inc_vol = a.inc_vol, - inc_fr = a.inc_fr, - inc_char = a.inc_char, - inc_invest = a.inc_invest, - inc_other = a.inc_other, - inc_total = a.inc_total, - invest_gain = a.invest_gain, - asset_gain = a.asset_gain, - pension_gain = a.pension_gain, - exp_vol = a.exp_vol, - exp_trade = a.exp_trade, - exp_invest = a.exp_invest, - exp_grant = a.exp_grant, - exp_charble = a.exp_charble, - exp_gov = a.exp_gov, - exp_other = a.exp_other, - exp_total = a.exp_total, - exp_support = a.exp_support, - exp_dep = a.exp_dep, + inc_leg = a.income_legacies, + inc_end = a.income_endowments, + inc_vol = a.income_donations_and_legacies, + inc_fr = a.income_other_trading_activities, + inc_char = a.income_charitable_activities, + inc_invest = a.income_investments, + inc_other = a.income_other, + inc_total = a.income_total_income_and_endowments, + invest_gain = a.gain_loss_investment, + asset_gain = a.gain_loss_pension_fund, + pension_gain = a.gain_loss_pension_fund, + exp_vol = a.expenditure_raising_funds, + exp_trade = null, + exp_invest = null, + exp_grant = a.expenditure_grants_institution, + exp_charble = a.expenditure_charitable_expenditure, + exp_gov = a.expenditure_governance, + exp_other = a.expenditure_other, + exp_total = a.expenditure_total, + exp_support = a.expenditure_support_costs, + exp_dep = a.expenditure_depreciation, reserves = a.reserves, - asset_open = a.asset_open, - asset_close = a.asset_close, - fixed_assets = a.fixed_assets, - open_assets = a.open_assets, - invest_assets = a.invest_assets, - cash_assets = a.cash_assets, - current_assets = a.current_assets, - credit_1 = a.credit_1, - credit_long = a.credit_long, - pension_assets = a.pension_assets, - total_assets = a.total_assets, - funds_end = a.funds_end, - funds_restrict = a.funds_restrict, - funds_unrestrict = a.funds_unrestrict, + asset_open = null, + asset_close = a.assets_total_fixed, -- Total fixed assets + fixed_assets = a.assets_long_term_investment, -- Fixed Investments Assets + open_assets = null, + invest_assets = a.assets_current_investment, -- Current Investment Assets + cash_assets = a.assets_cash, -- Cash + current_assets = null, -- Total Current Assets + credit_1 = a.creditors_one_year_total_current, + credit_long = a.creditors_falling_due_after_one_year, + pension_assets = a.defined_benefit_pension_scheme, + total_assets = a.assets_total_assets_and_liabilities, + funds_end = a.funds_endowment, + funds_restrict = a.funds_restricted, + funds_unrestrict = a.funds_unrestricted, funds_total = a.funds_total, - employees = a.employees, - volunteers = a.volunteers, - account_type = case when "cons_acc" = 'Yes' then 'consolidated' else 'charity' end -from charity_ccewpartb a -where cf.charity_id = CONCAT('GB-CHC-', a.regno) - and cf.fyend = a.fyend; -""" - -UPDATE_CCEW[ - "Update charity table with latest income" -] = """ -update charity_charity -set latest_fye = cf.fyend, - income = cf.income, - spending = cf.spending -from ( - select charity_id, max(fyend) as max_fyend - from charity_charityfinancial - group by charity_id -) as cf_max - inner join charity_charityfinancial cf - on cf_max.charity_id = cf.charity_id - and cf_max.max_fyend = cf.fyend -where cf.charity_id = charity_charity.id; + employees = a.count_employees, + account_type = case when "consolidated_accounts" then 'consolidated' else 'charity' end +from charity_ccewcharityarpartb a +where cf.charity_id = CONCAT('GB-CHC-', a.registered_charity_number) + and cf.fyend = a.fin_period_end_date; """ UPDATE_CCEW[ "Insert into charity areas of operation" ] = """ insert into charity_charity_areas_of_operation as ca (charity_id, areaofoperation_id ) -select CONCAT('GB-CHC-', c.regno) as charity_id, +select CONCAT('GB-CHC-', c.registered_charity_number) as charity_id, aoo.id as "areaofoperation_id" -from charity_ccewcharityaoo c +from charity_ccewcharityareaofoperation c inner join charity_areaofoperation aoo - on c.aookey = aoo.aookey - and c.aootype = aoo.aootype + on c.geographic_area_description = aoo.aooname on conflict (charity_id, areaofoperation_id) do nothing; """ @@ -169,36 +154,38 @@ "Insert into charity names" ] = """ insert into charity_charityname as cn (charity_id, name, "normalisedName", name_type) -select CONCAT('GB-CHC-', c.regno) as charity_id, - c."name", - c."name" as "normalisedName", - case when c.subno != '0' then 'Subsidiary' - when c.name = ccc."name" then 'Primary' - else 'Alternative' end as "name_type" -from charity_ccewname c +select CONCAT('GB-CHC-', c.registered_charity_number) as charity_id, + c.charity_name as "name", + c.charity_name as "normalisedName", + case when c.linked_charity_number != '0' then 'Subsidiary' + when c.charity_name = ccc."charity_name" then 'Primary' + else c.charity_name_type end as "name_type" +from charity_ccewcharityothernames c left outer join charity_ccewcharity ccc - on c.regno = ccc.regno - and c.subno = ccc.subno + on c.registered_charity_number = ccc.registered_charity_number + and c.linked_charity_number = ccc.linked_charity_number +where c.charity_name is not null on conflict (charity_id, name) do nothing; """ UPDATE_CCEW[ - "Update charity objects" + "Insert charity classification into vocabs" ] = """ -update charity_charity -set activities = "objects" -from ( - select CONCAT('GB-CHC-', a.regno) as charity_id, - string_agg(regexp_replace("object", "next_seqno" || '$', ''), '' order by seqno) as "objects" - from ( - select *, - to_char(seqno::int + 1, 'fm0000') as "next_seqno" - from charity_ccewobjects cc - where cc.subno = '0' - ) as a - group by charity_id -) as a -where charity_charity.id = a.charity_id; +insert into charity_vocabularyentries (code, title, vocabulary_id) +select c.classification_code as "code", + c.classification_description as "title", + cv.id +from charity_ccewcharityclassification c + inner join charity_vocabulary cv + on case when c.classification_type = 'What' then 'ccew_theme' + when c.classification_type = 'How' then 'ccew_activities' + when c.classification_type = 'Who' then 'ccew_beneficiaries' + else null end = cv.title +group by c.classification_code, + c.classification_description, + cv.id +on conflict (code, vocabulary_id) do update +set title = EXCLUDED.title; """ UPDATE_CCEW[ @@ -208,9 +195,9 @@ select org_id, ve.id from ( - select CONCAT('GB-CHC-', c.regno) as org_id, - c.class as "c_class" - from charity_ccewclass c + select CONCAT('GB-CHC-', c.registered_charity_number) as org_id, + cast(c.classification_code as varchar) as "c_class" + from charity_ccewcharityclassification c ) as c_class inner join charity_vocabularyentries ve on c_class.c_class = ve.code @@ -245,10 +232,10 @@ cn."alternateName" as "alternateName", regexp_replace(cc.id, 'GB\-(SC|NIC|COH|CHC)\-', '') as "charityNumber", cc.company_number as "companyNumber", - concat_ws(', ', ccew.add1, ccew.add2) as "streetAddress", - ccew.add3 as "addressLocality", - ccew.add4 as "addressRegion", - ccew.add5 as "addressCountry", + concat_ws(', ', ccew.charity_contact_address1, ccew.charity_contact_address2) as "streetAddress", + ccew.charity_contact_address3 as "addressLocality", + ccew.charity_contact_address4 as "addressRegion", + ccew.charity_contact_address5 as "addressCountry", cc.postcode as "postalCode", cc.phone as telephone, cc.email as email, @@ -267,13 +254,15 @@ when cc."source" = 'oscr' then array['registered-charity-scotland'] when cc."source" = 'ccni' then array['registered-charity-northern-ireland'] else array[]::text[] end || - case when cc.company_number is not null then array['registered-company', 'incorporated-charity'] + case when cc.company_number is not null or ccew.charity_type = 'Charitable company' then array['registered-company', 'incorporated-charity'] else array[]::text[] end || - case when cc.constitution ilike 'CIO - %' then array['charitable-incorporated-organisation', 'incorporated-charity'] + case when cc.constitution ilike 'CIO - %' or ccew.charity_type = 'CIO' then array['charitable-incorporated-organisation', 'incorporated-charity'] else array[]::text[] end || case when cc.constitution ilike 'cio - association %' then array['charitable-incorporated-organisation-association'] else array[]::text[] end || case when cc.constitution ilike 'cio - foundation %' then array['charitable-incorporated-organisation-foundation'] + else array[]::text[] end || + case when ccew.charity_type = 'Trust' then array['trust'] else array[]::text[] end as "organisationType", cc."source" as spider, @@ -293,8 +282,8 @@ ) as cn on cc.id = cn.charity_id left outer join charity_ccewcharity ccew - on cc.id = CONCAT('GB-CHC-', ccew.regno) - and ccew.subno = '0', + on cc.id = CONCAT('GB-CHC-', ccew.registered_charity_number) + and ccew.linked_charity_number = '0', ftc_organisationtype ot where ot.title = 'Registered Charity' and cc.source = '{source}' @@ -335,7 +324,7 @@ "scrape_id" ) select fo.id as organisation_id, - CONCAT('GB-CHC-', cc.regno) as org_id, + CONCAT('GB-CHC-', cc.registered_charity_number) as org_id, aooname as name, coalesce(ca."GSS", ca."ISO3166_1") as "geoCode", case when ca."GSS" is not null then 'ONS' @@ -345,9 +334,10 @@ ca."ISO3166_1" as geo_iso, fo.source_id as source_id, {scrape_id} as scrape_id -from charity_ccewcharityaoo cc +from charity_ccewcharityareaofoperation cc inner join charity_areaofoperation ca - on cc.aookey = ca.aookey and cc.aootype = ca.aootype + on cc.geographic_area_description = ca.aooname inner join ftc_organisation fo - on fo.org_id = CONCAT('GB-CHC-', cc.regno) + on fo.org_id = CONCAT('GB-CHC-', cc.registered_charity_number) +where ca."GSS" is not null or ca."ISO3166_1" is not null """ diff --git a/charity/management/commands/ccew_data_fetch.py b/charity/management/commands/ccew_data_fetch.py deleted file mode 100644 index 9de0de5c..00000000 --- a/charity/management/commands/ccew_data_fetch.py +++ /dev/null @@ -1,21 +0,0 @@ -import requests_html -from django.core.management.base import BaseCommand - -from charity.feeds import CCEW_DATA_URL -from charity.models import CcewDataFile - - -class Command(BaseCommand): - def handle(self, *args, **options): - session = requests_html.HTMLSession() - r = session.get(CCEW_DATA_URL) - for d in r.html.find("blockquote.download"): - for p in d.find("p"): - if "Charity register extract" in p.text: - links = p.absolute_links - for link in links: - f, created = CcewDataFile.objects.update_or_create( - title=d.find("h4", first=True).text, - defaults=dict(url=link, description=p.text), - ) - print("{} ({})".format(f, "created" if created else "updated")) diff --git a/charity/management/commands/charity_setup.py b/charity/management/commands/charity_setup.py index 37196a4e..6a9b7e0e 100644 --- a/charity/management/commands/charity_setup.py +++ b/charity/management/commands/charity_setup.py @@ -1,6 +1,5 @@ import csv import io -import string import requests from django.core.management.base import BaseCommand @@ -17,6 +16,7 @@ def handle(self, *args, **options): self.fetch_aoo() def fetch_cc_classifications(self): + print("Fetching CC classifications") ccew = "https://github.com/drkane/charity-lookups/raw/master/classification/ccew.csv" for r in self.get_csv(ccew): @@ -28,6 +28,7 @@ def fetch_cc_classifications(self): ) def fetch_icnpo(self): + print("Fetching ICNPO") icnpo = "https://github.com/drkane/charity-lookups/raw/master/classification/icnpo.csv" v, _ = Vocabulary.objects.update_or_create( title="International Classification of Non Profit Organisations (ICNPO)", @@ -57,6 +58,7 @@ def fetch_icnpo(self): ) def fetch_icnptso(self): + print("Fetching ICNPTSO") icnptso = "https://github.com/drkane/charity-lookups/raw/master/classification/icnptso.csv" v, _ = Vocabulary.objects.update_or_create( title="International Classification of Non-profit and Third Sector Organizations (ICNP/TSO)", @@ -84,31 +86,17 @@ def fetch_icnptso(self): # ntee = 'https://github.com/drkane/charity-lookups/raw/master/classification/ntee.csv' def fetch_aoo(self): - aoo = "https://github.com/drkane/charity-lookups/raw/master/cc-aoo-gss-iso.csv" - cache = {} - aoos = [r for r in self.get_csv(aoo)] - aoos.reverse() - for r in aoos: - code = (r["aootype"], r["aookey"]) - master = None - if r.get("master"): - master = cache[ - ( - string.ascii_letters[string.ascii_letters.index(code[0]) + 1], - r["master"], - ) - ] + print("Fetching AOO lookups") + aoo = "https://github.com/drkane/charity-lookups/raw/master/cc-aoo-gss-iso-new.csv" + for r in self.get_csv(aoo): for k, v in r.items(): if v == "": r[k] = None ve, _ = AreaOfOperation.objects.update_or_create( - aootype=r["aootype"], - aookey=r["aookey"], + aooname=r["geographic_area_description"], defaults={ - "aooname": r.get("aooname"), - "aoosort": r.get("aoosort"), + "aoosort": r.get("geographic_area_description"), "welsh": r.get("welsh") == "Y", - "master": master, "GSS": r.get("GSS"), "ISO3166_1": r.get("ISO3166-1"), "ISO3166_1_3": r.get("ISO3166-1:3"), @@ -116,7 +104,6 @@ def fetch_aoo(self): "ContinentCode": r.get("ContinentCode"), }, ) - cache[code] = ve def get_csv(self, url, encoding="utf8"): response = requests.get(url) diff --git a/charity/management/commands/import_ccew.py b/charity/management/commands/import_ccew.py index c972f2a8..3c3ca242 100644 --- a/charity/management/commands/import_ccew.py +++ b/charity/management/commands/import_ccew.py @@ -1,45 +1,44 @@ # -*- coding: utf-8 -*- -import os +import io +import csv import re -import tempfile import zipfile -import bcp import psycopg2 import tqdm -from django.core.management.base import CommandError from django.db import connection -from charity.feeds import CCEW_DATA_URL from charity.management.commands._ccew_sql import UPDATE_CCEW from charity.models import ( CCEWCharity, - CCEWCharityAOO, - CCEWClass, - CCEWFinancial, - CCEWMainCharity, - CCEWName, - CCEWObjects, - CCEWPartB, - CCEWRegistration, + CCEWCharityAnnualReturnHistory, + CCEWCharityAreaOfOperation, + CCEWCharityARPartA, + CCEWCharityARPartB, + CCEWCharityClassification, + CCEWCharityEventHistory, + CCEWCharityGoverningDocument, + CCEWCharityOtherNames, + CCEWCharityOtherRegulators, + CCEWCharityPolicy, + CCEWCharityPublishedReport, + CCEWCharityTrustee, ) -from ftc.management.commands._base_scraper import HTMLScraper +from ftc.management.commands._base_scraper import BaseScraper from ftc.models import Organisation, OrganisationLink, Scrape -class Command(HTMLScraper): +class Command(BaseScraper): name = "ccew" allowed_domains = ["charitycommission.gov.uk"] - start_urls = [ - CCEW_DATA_URL, - # "https://raw.githubusercontent.com/drkane/charity-lookups/master/cc-aoo-gss-iso.csv", - ] + start_urls = [] encoding = "cp858" org_id_prefix = "GB-CHC" id_field = "regno" date_fields = [] date_format = "%Y-%m-%d %H:%M:%S" zip_regex = re.compile(r".*/RegPlusExtract.*?\.zip.*?") + base_url = "https://ccewuksprdoneregsadata1.blob.core.windows.net/data/txt/publicextract.{}.zip" source = { "title": "Registered charities in England and Wales", "description": "Data download service provided by the Charity Commission", @@ -61,133 +60,19 @@ class Command(HTMLScraper): ], } ccew_file_to_object = { - "extract_charity": CCEWCharity, - "extract_main_charity": CCEWMainCharity, - "extract_name": CCEWName, - "extract_registration": CCEWRegistration, - "extract_charity_aoo": CCEWCharityAOO, - "extract_objects": CCEWObjects, - "extract_financial": CCEWFinancial, - "extract_class": CCEWClass, - "extract_partb": CCEWPartB, - } - ccew_files = { - "extract_charity": [ - "regno", # integer registered number of a charity - "subno", # integer subsidiary number of a charity (may be 0 for main/group charity) - "name", # varchar(150) main name of the charity - "orgtype", # varchar(2) R (registered) or RM (removed) - "gd", # varchar(250) Description of Governing Document - "aob", # varchar(175) area of benefit - may not be defined - "aob_defined", # char(1) area of benefit defined by Governing Document (T/F) - "nhs", # char(1) NHS charity (T/F) - "ha_no", # varchar(20) Housing Association number - "corr", # varchar(70) Charity correspondent name - "add1", # varchar(35) address line of charity's correspondent - "add2", # varchar(35) address line of charity's correspondent - "add3", # varchar(35) address line of charity's correspondent - "add4", # varchar(35) address line of charity's correspondent - "add5", # varchar(35) address line of charity's correspondent - "postcode", # varchar(8) postcode of charity's correspondent - "phone", # varchar(23) telephone of charity's correspondent - "fax", # varchar(23) fax of charity's correspondent - ], - "extract_main_charity": [ - "regno", # integer registered number of a charity - "coyno", # integer company registration number - "trustees", # char(1) trustees incorporated (T/F) - "fyend", # char(4) Financial year end - "welsh", # char(1) requires correspondence in both Welsh & English (T/F) - "incomedate", # datetime date for latest gross income (blank if income is an estimate) - "income", # integer - "grouptype", # varchar(4) may be blank - "email", # varchar(255) email address - "web", # varchar(255) website address - ], - "extract_name": [ - "regno", # integer registered number of a charity - "subno", # integer subsidiary number of a charity (may be 0 for main/group charity) - "nameno", # integer number identifying a charity name - "name", # varchar(150) name of a charity (multiple occurrences possible) - ], - "extract_registration": [ - "regno", # integer registered number of a charity - "subno", # integer subsidiary number of a charity (may be 0 for main/group charity) - "regdate", # datetime date of registration for a charity - "remdate", # datetime Removal date of a charity - Blank for Registered Charities - "remcode", # varchar(3) Register removal reason code - ], - "extract_charity_aoo": [ - "regno", # integer registered number of a charity - "aootype", # char(1) A B or D - "aookey", # integer up to three digits - "welsh", # char(1) Flag: Y or blank - "master", # integer may be blank. If aootype=D then holds continent; if aootype=B then holds GLA/met county - ], - "extract_objects": [ - "regno", # integer registered number of a charity - "subno", # integer subsidiary number of a charity (may be 0 for main/group charity) - "seqno", # char(4) sequence number (in practice 0-20) - "object", # varchar(255) Description of objects of a charity - ], - "extract_financial": [ - "regno", # integer registered number of a charity - "fystart", # datetime Charity's financial year start date - "fyend", # datetime Charity's financial year end date - "income", # integer - "expend", # integer - ], - "extract_class": [ - "regno", # integer registered number of a charity - "class", # integer classification code for a charity(multiple occurrences possible) - ], - "extract_partb": [ - "regno", - "artype", - "fystart", - "fyend", - "inc_leg", - "inc_end", - "inc_vol", - "inc_fr", - "inc_char", - "inc_invest", - "inc_other", - "inc_total", - "invest_gain", - "asset_gain", - "pension_gain", - "exp_vol", - "exp_trade", - "exp_invest", - "exp_grant", - "exp_charble", - "exp_gov", - "exp_other", - "exp_total", - "exp_support", - "exp_dep", - "reserves", - "asset_open", - "asset_close", - "fixed_assets", - "open_assets", - "invest_assets", - "cash_assets", - "current_assets", - "credit_1", - "credit_long", - "pension_assets", - "total_assets", - "funds_end", - "funds_restrict", - "funds_unrestrict", - "funds_total", - "employees", - "volunteers", - "cons_acc", - "charity_acc", - ], + "charity": CCEWCharity, + "charity_annual_return_history": CCEWCharityAnnualReturnHistory, + "charity_annual_return_parta": CCEWCharityARPartA, + "charity_annual_return_partb": CCEWCharityARPartB, + "charity_area_of_operation": CCEWCharityAreaOfOperation, + "charity_classification": CCEWCharityClassification, + "charity_event_history": CCEWCharityEventHistory, + "charity_governing_document": CCEWCharityGoverningDocument, + "charity_other_names": CCEWCharityOtherNames, + "charity_other_regulators": CCEWCharityOtherRegulators, + "charity_policy": CCEWCharityPolicy, + "charity_published_report": CCEWCharityPublishedReport, + "charity_trustee": CCEWCharityTrustee, } orgtypes = [ "Registered Charity", @@ -197,72 +82,51 @@ class Command(HTMLScraper): "Charitable Incorporated Organisation", "Charitable Incorporated Organisation - Association", "Charitable Incorporated Organisation - Foundation", + "Trust", ] - def parse_file(self, response, source_urls): - zip_found = False - for link in response.html.absolute_links: - if ".zip" not in link or "TableBuildScripts" in link: - continue - self.set_download_url(link) - self.logger.info("Using file: {}".format(link)) - zip_found = True - r = self.session.get(link) + def fetch_file(self): + self.files = {} + for filename in self.ccew_file_to_object: + url = self.base_url.format(filename) + self.set_download_url(url) + r = self.session.get(url) r.raise_for_status() - self.process_zip(r) - break - - if not zip_found: - raise CommandError("No zip file found") - - def process_zip(self, response): - self.logger.info("File size: {}".format(len(response.content))) - - with tempfile.TemporaryDirectory() as tmpdirname: - cczip_name = os.path.join(tmpdirname, "ccew.zip") - files = {} - - with open(cczip_name, "wb") as cczip: - self.logger.info("Saving ZIP to disk") - cczip.write(response.content) + self.files[filename] = r + + def parse_file(self, response, filename): + try: + z = zipfile.ZipFile(io.BytesIO(response.content)) + except zipfile.BadZipFile: + self.logger.info(response.content[0:1000]) + raise + for f in z.infolist(): + self.logger.info("Opening: {}".format(f.filename)) + with z.open(f) as csvfile: + self.process_file(csvfile, filename) + z.close() + + def process_file(self, csvfile, filename): - with zipfile.ZipFile(cczip_name, "r") as z: - for f in z.infolist(): - filename = f.filename.replace(".bcp", "") - filepath = os.path.join(tmpdirname, f.filename) - if filename not in self.ccew_files.keys(): - self.logger.debug("Skipping: {}".format(f.filename)) - continue - self.logger.info("Saving {} to disk".format(f.filename)) - z.extract(f, path=tmpdirname) - files[filename] = filepath - - for filename, filepath in files.items(): - with open(filepath, "r", encoding=self.encoding) as bcpfile: - self.logger.info("Processing: {}".format(filename)) - self.process_bcp(bcpfile, filename) - - def process_bcp(self, bcpfile, filename): - - fields = self.ccew_files.get(filename) db_table = self.ccew_file_to_object.get(filename) page_size = 1000 - self.date_fields = [f for f in fields if f.endswith("date")] def convert_encoding(row): for k in row: if isinstance(row[k], str): row[k] = row[k].decode(self.encoding).encode("utf8") - def get_data(bcpreader): - for k, row in tqdm.tqdm(enumerate(bcpreader)): + def get_data(reader): + for k, row in tqdm.tqdm(enumerate(reader)): row = self.clean_fields(row) - if not row.get("regno"): - continue yield [k] + list(row.values()) with connection.cursor() as cursor: - bcpreader = bcp.DictReader(bcpfile, fieldnames=fields) + reader = csv.DictReader( + io.TextIOWrapper(csvfile, encoding="utf8"), + delimiter="\t", + escapechar="\\", + ) self.logger.info( "Starting table insert [{}]".format(db_table._meta.db_table) ) @@ -270,7 +134,7 @@ def get_data(bcpreader): psycopg2.extras.execute_values( cursor, """INSERT INTO {} VALUES %s;""".format(db_table._meta.db_table), - get_data(bcpreader), + get_data(reader), page_size=page_size, ) self.logger.info( diff --git a/charity/migrations/0010_auto_20210403_1729.py b/charity/migrations/0010_auto_20210403_1729.py new file mode 100644 index 00000000..2c3a9500 --- /dev/null +++ b/charity/migrations/0010_auto_20210403_1729.py @@ -0,0 +1,40 @@ +# Generated by Django 3.1.1 on 2021-04-03 16:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('charity', '0009_auto_20200921_1156'), + ] + + operations = [ + migrations.DeleteModel( + name='CCEWCharity', + ), + migrations.DeleteModel( + name='CCEWCharityAOO', + ), + migrations.DeleteModel( + name='CCEWClass', + ), + migrations.DeleteModel( + name='CCEWFinancial', + ), + migrations.DeleteModel( + name='CCEWMainCharity', + ), + migrations.DeleteModel( + name='CCEWName', + ), + migrations.DeleteModel( + name='CCEWObjects', + ), + migrations.DeleteModel( + name='CCEWPartB', + ), + migrations.DeleteModel( + name='CCEWRegistration', + ), + ] diff --git a/charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py b/charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py new file mode 100644 index 00000000..16f019dd --- /dev/null +++ b/charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py @@ -0,0 +1,369 @@ +# Generated by Django 3.1.1 on 2021-04-04 21:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + replaces = [('charity', '0010_auto_20210403_1729'), ('charity', '0011_ccew_updated_files'), ('charity', '0012_auto_20210403_1943'), ('charity', '0013_auto_20210404_2125'), ('charity', '0014_auto_20210404_2145'), ('charity', '0015_auto_20210404_2147'), ('charity', '0016_delete_ccewdatafile')] + + dependencies = [ + ('charity', '0009_auto_20200921_1156'), + ] + + operations = [ + migrations.DeleteModel( + name='CCEWCharity', + ), + migrations.DeleteModel( + name='CCEWCharityAOO', + ), + migrations.DeleteModel( + name='CCEWClass', + ), + migrations.DeleteModel( + name='CCEWFinancial', + ), + migrations.DeleteModel( + name='CCEWMainCharity', + ), + migrations.DeleteModel( + name='CCEWName', + ), + migrations.DeleteModel( + name='CCEWObjects', + ), + migrations.DeleteModel( + name='CCEWPartB', + ), + migrations.DeleteModel( + name='CCEWRegistration', + ), + migrations.CreateModel( + name='CCEWCharity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(db_index=True)), + ('charity_name', models.CharField(blank=True, max_length=255, null=True)), + ('charity_type', models.CharField(blank=True, max_length=255, null=True)), + ('charity_registration_status', models.CharField(blank=True, max_length=255, null=True)), + ('date_of_registration', models.DateField(blank=True, null=True)), + ('date_of_removal', models.DateField(blank=True, null=True)), + ('charity_reporting_status', models.CharField(blank=True, max_length=255, null=True)), + ('latest_acc_fin_period_start_date', models.DateField(blank=True, null=True)), + ('latest_acc_fin_period_end_date', models.DateField(blank=True, null=True)), + ('latest_income', models.FloatField(blank=True, null=True)), + ('latest_expenditure', models.FloatField(blank=True, null=True)), + ('charity_contact_address1', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address2', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address3', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address4', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address5', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_postcode', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_phone', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_email', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_web', models.CharField(blank=True, max_length=255, null=True)), + ('charity_company_registration_number', models.CharField(blank=True, max_length=255, null=True)), + ('charity_insolvent', models.BooleanField(blank=True, null=True)), + ('charity_in_administration', models.BooleanField(blank=True, null=True)), + ('charity_previously_excepted', models.BooleanField(blank=True, null=True)), + ('charity_is_cdf_or_cif', models.CharField(blank=True, max_length=255, null=True)), + ('charity_is_cio', models.BooleanField(blank=True, null=True)), + ('cio_is_dissolved', models.BooleanField(blank=True, null=True)), + ('date_cio_dissolution_notice', models.DateField(blank=True, null=True)), + ('charity_activities', models.TextField(blank=True, null=True)), + ('charity_gift_aid', models.BooleanField(blank=True, null=True)), + ('charity_has_land', models.BooleanField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityAnnualReturnHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('fin_period_start_date', models.DateField(blank=True, null=True)), + ('fin_period_end_date', models.DateField(blank=True, null=True)), + ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), + ('reporting_due_date', models.DateField(blank=True, null=True)), + ('date_annual_return_received', models.DateField(blank=True, null=True)), + ('date_accounts_received', models.DateField(blank=True, null=True)), + ('total_gross_income', models.BigIntegerField(blank=True, null=True)), + ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), + ('accounts_qualified', models.BooleanField(blank=True, null=True)), + ('suppression_ind', models.BooleanField(blank=True, null=True)), + ('suppression_type', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityAreaOfOperation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), + ('geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), + ('parent_geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), + ('parent_geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), + ('welsh_ind', models.BooleanField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityARPartA', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), + ('fin_period_order_number', models.IntegerField(blank=True, null=True)), + ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), + ('fin_period_start_date', models.DateField(blank=True, null=True)), + ('fin_period_end_date', models.DateField(blank=True, null=True)), + ('ar_due_date', models.DateField(blank=True, null=True)), + ('ar_received_date', models.DateField(blank=True, null=True)), + ('total_gross_income', models.BigIntegerField(blank=True, null=True)), + ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), + ('charity_raises_funds_from_public', models.BooleanField(blank=True, null=True)), + ('charity_professional_fundraiser', models.BooleanField(blank=True, null=True)), + ('charity_agreement_professional_fundraiser', models.BooleanField(blank=True, null=True)), + ('charity_commercial_participator', models.BooleanField(blank=True, null=True)), + ('charity_agreement_commerical_participator', models.BooleanField(blank=True, null=True)), + ('grant_making_is_main_activity', models.BooleanField(blank=True, null=True)), + ('charity_receives_govt_funding_contracts', models.BooleanField(blank=True, null=True)), + ('count_govt_contracts', models.IntegerField(blank=True, null=True)), + ('charity_receives_govt_funding_grants', models.BooleanField(blank=True, null=True)), + ('count_govt_grants', models.IntegerField(blank=True, null=True)), + ('income_from_government_contracts', models.BigIntegerField(blank=True, null=True)), + ('income_from_government_grants', models.BigIntegerField(blank=True, null=True)), + ('charity_has_trading_subsidiary', models.BooleanField(blank=True, null=True)), + ('trustee_also_director_of_subsidiary', models.BooleanField(blank=True, null=True)), + ('does_trustee_receive_any_benefit', models.BooleanField(blank=True, null=True)), + ('trustee_payments_acting_as_trustee', models.BooleanField(blank=True, null=True)), + ('trustee_receives_payments_services', models.BooleanField(blank=True, null=True)), + ('trustee_receives_other_benefit', models.BooleanField(blank=True, null=True)), + ('trustee_resigned_employment', models.BooleanField(blank=True, null=True)), + ('employees_salary_over_60k', models.BooleanField(blank=True, null=True)), + ('count_salary_band_60001_70000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_70001_80000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_80001_90000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_90001_100000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_100001_110000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_110001_120000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_120001_130000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_130001_140000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_140001_150000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_150001_200000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_200001_250000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_250001_300000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_300001_350000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_350001_400000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_400001_450000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_450001_500000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_over_500000', models.IntegerField(blank=True, null=True)), + ('count_volunteers', models.IntegerField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityARPartB', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), + ('fin_period_order_number', models.IntegerField(blank=True, null=True)), + ('ar_cycle_reference', models.CharField(max_length=255)), + ('fin_period_start_date', models.DateField(blank=True, null=True)), + ('fin_period_end_date', models.DateField(blank=True, null=True)), + ('ar_due_date', models.DateField(blank=True, null=True)), + ('ar_received_date', models.DateField(blank=True, null=True)), + ('income_donations_and_legacies', models.BigIntegerField(blank=True, null=True)), + ('income_charitable_activities', models.BigIntegerField(blank=True, null=True)), + ('income_other_trading_activities', models.BigIntegerField(blank=True, null=True)), + ('income_investments', models.BigIntegerField(blank=True, null=True)), + ('income_other', models.BigIntegerField(blank=True, null=True)), + ('income_total_income_and_endowments', models.BigIntegerField(blank=True, null=True)), + ('income_legacies', models.BigIntegerField(blank=True, null=True)), + ('income_endowments', models.BigIntegerField(blank=True, null=True)), + ('expenditure_raising_funds', models.BigIntegerField(blank=True, null=True)), + ('expenditure_charitable_expenditure', models.BigIntegerField(blank=True, null=True)), + ('expenditure_other', models.BigIntegerField(blank=True, null=True)), + ('expenditure_total', models.BigIntegerField(blank=True, null=True)), + ('expenditure_investment_management', models.BigIntegerField(blank=True, null=True)), + ('expenditure_grants_institution', models.BigIntegerField(blank=True, null=True)), + ('expenditure_governance', models.BigIntegerField(blank=True, null=True)), + ('expenditure_support_costs', models.BigIntegerField(blank=True, null=True)), + ('expenditure_depreciation', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_investment', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_pension_fund', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_revaluation_fixed_investment', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_other', models.BigIntegerField(blank=True, null=True)), + ('reserves', models.BigIntegerField(blank=True, null=True)), + ('assets_total_fixed', models.BigIntegerField(blank=True, null=True)), + ('assets_own_use', models.BigIntegerField(blank=True, null=True)), + ('assets_long_term_investment', models.BigIntegerField(blank=True, null=True)), + ('defined_benefit_pension_scheme', models.BigIntegerField(blank=True, null=True)), + ('assets_other_assets', models.BigIntegerField(blank=True, null=True)), + ('assets_total_liabilities', models.BigIntegerField(blank=True, null=True)), + ('assets_current_investment', models.BigIntegerField(blank=True, null=True)), + ('assets_total_assets_and_liabilities', models.BigIntegerField(blank=True, null=True)), + ('creditors_one_year_total_current', models.BigIntegerField(blank=True, null=True)), + ('creditors_falling_due_after_one_year', models.BigIntegerField(blank=True, null=True)), + ('assets_cash', models.BigIntegerField(blank=True, null=True)), + ('funds_endowment', models.BigIntegerField(blank=True, null=True)), + ('funds_unrestricted', models.BigIntegerField(blank=True, null=True)), + ('funds_restricted', models.BigIntegerField(blank=True, null=True)), + ('funds_total', models.BigIntegerField(blank=True, null=True)), + ('count_employees', models.IntegerField(blank=True, null=True)), + ('charity_only_accounts', models.BooleanField(blank=True, null=True)), + ('consolidated_accounts', models.BooleanField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityClassification', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('classification_code', models.IntegerField(blank=True, null=True)), + ('classification_type', models.CharField(blank=True, max_length=255, null=True)), + ('classification_description', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityGoverningDocument', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('governing_document_description', models.TextField(blank=True, null=True)), + ('charitable_objects', models.TextField(blank=True, null=True)), + ('area_of_benefit', models.TextField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityOtherNames', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('charity_name_id', models.IntegerField(blank=True, null=True)), + ('charity_name_type', models.CharField(blank=True, max_length=255, null=True)), + ('charity_name', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityOtherRegulators', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('regulator_order', models.IntegerField(blank=True, null=True)), + ('regulator_name', models.CharField(blank=True, max_length=255, null=True)), + ('regulator_web_url', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityPolicy', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField()), + ('policy_name', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityPublishedReport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('charity_id', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('report_name', models.CharField(blank=True, max_length=255, null=True)), + ('report_location', models.CharField(blank=True, max_length=255, null=True)), + ('date_published', models.DateField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityTrustee', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField()), + ('trustee_id', models.IntegerField(blank=True, null=True)), + ('trustee_name', models.CharField(blank=True, max_length=255, null=True)), + ('trustee_is_chair', models.BooleanField(blank=True, null=True)), + ('individual_or_organisation', models.CharField(blank=True, choices=[('P', 'Individual'), ('O', 'Organisation')], max_length=1, null=True)), + ('trustee_date_of_appointment', models.DateField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityEventHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('charity_name', models.CharField(blank=True, max_length=255, null=True)), + ('charity_event_order', models.IntegerField(blank=True, null=True)), + ('event_type', models.CharField(blank=True, max_length=255, null=True)), + ('date_of_event', models.DateField(blank=True, null=True)), + ('reason', models.CharField(blank=True, max_length=255, null=True)), + ('assoc_organisation_number', models.IntegerField(blank=True, null=True)), + ('assoc_registered_charity_number', models.IntegerField(blank=True, db_index=True, null=True)), + ('assoc_charity_name', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.AlterField( + model_name='areaofoperation', + name='aookey', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='areaofoperation', + name='aootype', + field=models.CharField(blank=True, max_length=1, null=True), + ), + migrations.AlterField( + model_name='areaofoperation', + name='welsh', + field=models.BooleanField(blank=True, null=True, verbose_name='In Wales'), + ), + migrations.AlterUniqueTogether( + name='areaofoperation', + unique_together=set(), + ), + migrations.AlterField( + model_name='charityname', + name='name', + field=models.CharField(db_index=True, max_length=255), + ), + migrations.AlterField( + model_name='charityname', + name='normalisedName', + field=models.CharField(blank=True, db_index=True, max_length=255, null=True), + ), + migrations.DeleteModel( + name='CcewDataFile', + ), + ] diff --git a/charity/migrations/0011_ccew_updated_files.py b/charity/migrations/0011_ccew_updated_files.py new file mode 100644 index 00000000..a76f1e25 --- /dev/null +++ b/charity/migrations/0011_ccew_updated_files.py @@ -0,0 +1,308 @@ +# Generated by Django 3.1.1 on 2021-04-03 18:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('charity', '0010_auto_20210403_1729'), + ] + + operations = [ + migrations.CreateModel( + name='CCEWCharity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(db_index=True)), + ('charity_name', models.CharField(blank=True, max_length=255, null=True)), + ('charity_type', models.CharField(blank=True, max_length=255, null=True)), + ('charity_registration_status', models.CharField(blank=True, max_length=255, null=True)), + ('date_of_registration', models.DateField(blank=True, null=True)), + ('date_of_removal', models.DateField(blank=True, null=True)), + ('charity_reporting_status', models.CharField(blank=True, max_length=255, null=True)), + ('latest_acc_fin_period_start_date', models.DateField(blank=True, null=True)), + ('latest_acc_fin_period_end_date', models.DateField(blank=True, null=True)), + ('latest_income', models.FloatField(blank=True, null=True)), + ('latest_expenditure', models.FloatField(blank=True, null=True)), + ('charity_contact_address1', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address2', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address3', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address4', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_address5', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_postcode', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_phone', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_email', models.CharField(blank=True, max_length=255, null=True)), + ('charity_contact_web', models.CharField(blank=True, max_length=255, null=True)), + ('charity_company_registration_number', models.CharField(blank=True, max_length=255, null=True)), + ('charity_insolvent', models.BooleanField(blank=True, null=True)), + ('charity_in_administration', models.BooleanField(blank=True, null=True)), + ('charity_previously_excepted', models.BooleanField(blank=True, null=True)), + ('charity_is_cdf_or_cif', models.CharField(blank=True, max_length=255, null=True)), + ('charity_is_cio', models.BooleanField(blank=True, null=True)), + ('cio_is_dissolved', models.BooleanField(blank=True, null=True)), + ('date_cio_dissolution_notice', models.DateField(blank=True, null=True)), + ('charity_activities', models.TextField(blank=True, null=True)), + ('charity_gift_aid', models.BooleanField(blank=True, null=True)), + ('charity_has_land', models.BooleanField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityAnnualReturnHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('fin_period_start_date', models.DateField(blank=True, null=True)), + ('fin_period_end_date', models.DateField(blank=True, null=True)), + ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), + ('reporting_due_date', models.DateField(blank=True, null=True)), + ('date_annual_return_received', models.DateField(blank=True, null=True)), + ('date_accounts_received', models.DateField(blank=True, null=True)), + ('total_gross_income', models.BigIntegerField(blank=True, null=True)), + ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), + ('accounts_qualified', models.BooleanField(blank=True, null=True)), + ('suppression_ind', models.BooleanField(blank=True, null=True)), + ('suppression_type', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityAreaOfOperation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), + ('geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), + ('parent_geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), + ('parent_geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), + ('welsh_ind', models.BooleanField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityARPartA', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), + ('fin_period_order_number', models.IntegerField(blank=True, null=True)), + ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), + ('fin_period_start_date', models.DateField(blank=True, null=True)), + ('fin_period_end_date', models.DateField(blank=True, null=True)), + ('ar_due_date', models.DateField(blank=True, null=True)), + ('ar_received_date', models.DateField(blank=True, null=True)), + ('total_gross_income', models.BigIntegerField(blank=True, null=True)), + ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), + ('charity_raises_funds_from_public', models.BooleanField(blank=True, null=True)), + ('charity_professional_fundraiser', models.BooleanField(blank=True, null=True)), + ('charity_agreement_professional_fundraiser', models.BooleanField(blank=True, null=True)), + ('charity_commercial_participator', models.BooleanField(blank=True, null=True)), + ('charity_agreement_commerical_participator', models.BooleanField(blank=True, null=True)), + ('grant_making_is_main_activity', models.BooleanField(blank=True, null=True)), + ('charity_receives_govt_funding_contracts', models.BooleanField(blank=True, null=True)), + ('count_govt_contracts', models.IntegerField(blank=True, null=True)), + ('charity_receives_govt_funding_grants', models.BooleanField(blank=True, null=True)), + ('count_govt_grants', models.IntegerField(blank=True, null=True)), + ('income_from_government_contracts', models.BigIntegerField(blank=True, null=True)), + ('income_from_government_grants', models.BigIntegerField(blank=True, null=True)), + ('charity_has_trading_subsidiary', models.BooleanField(blank=True, null=True)), + ('trustee_also_director_of_subsidiary', models.BooleanField(blank=True, null=True)), + ('does_trustee_receive_any_benefit', models.BooleanField(blank=True, null=True)), + ('trustee_payments_acting_as_trustee', models.BooleanField(blank=True, null=True)), + ('trustee_receives_payments_services', models.BooleanField(blank=True, null=True)), + ('trustee_receives_other_benefit', models.BooleanField(blank=True, null=True)), + ('trustee_resigned_employment', models.BooleanField(blank=True, null=True)), + ('employees_salary_over_60k', models.BooleanField(blank=True, null=True)), + ('count_salary_band_60001_70000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_70001_80000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_80001_90000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_90001_100000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_100001_110000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_110001_120000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_120001_130000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_130001_140000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_140001_150000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_150001_200000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_200001_250000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_250001_300000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_300001_350000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_350001_400000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_400001_450000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_450001_500000', models.IntegerField(blank=True, null=True)), + ('count_salary_band_over_500000', models.IntegerField(blank=True, null=True)), + ('count_volunteers', models.IntegerField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityARPartB', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), + ('fin_period_order_number', models.IntegerField(blank=True, null=True)), + ('ar_cycle_reference', models.CharField(max_length=255)), + ('fin_period_start_date', models.DateField(blank=True, null=True)), + ('fin_period_end_date', models.DateField(blank=True, null=True)), + ('ar_due_date', models.DateField(blank=True, null=True)), + ('ar_received_date', models.DateField(blank=True, null=True)), + ('income_donations_and_legacies', models.BigIntegerField(blank=True, null=True)), + ('income_charitable_activities', models.BigIntegerField(blank=True, null=True)), + ('income_other_trading_activities', models.BigIntegerField(blank=True, null=True)), + ('income_investments', models.BigIntegerField(blank=True, null=True)), + ('income_other', models.BigIntegerField(blank=True, null=True)), + ('income_total_income_and_endowments', models.BigIntegerField(blank=True, null=True)), + ('income_legacies', models.BigIntegerField(blank=True, null=True)), + ('income_endowments', models.BigIntegerField(blank=True, null=True)), + ('expenditure_raising_funds', models.BigIntegerField(blank=True, null=True)), + ('expenditure_charitable_expenditure', models.BigIntegerField(blank=True, null=True)), + ('expenditure_other', models.BigIntegerField(blank=True, null=True)), + ('expenditure_total', models.BigIntegerField(blank=True, null=True)), + ('expenditure_investment_management', models.BigIntegerField(blank=True, null=True)), + ('expenditure_grants_institution', models.BigIntegerField(blank=True, null=True)), + ('expenditure_governance', models.BigIntegerField(blank=True, null=True)), + ('expenditure_support_costs', models.BigIntegerField(blank=True, null=True)), + ('expenditure_depreciation', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_investment', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_pension_fund', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_revaluation_fixed_investment', models.BigIntegerField(blank=True, null=True)), + ('gain_loss_other', models.BigIntegerField(blank=True, null=True)), + ('reserves', models.BigIntegerField(blank=True, null=True)), + ('assets_total_fixed', models.BigIntegerField(blank=True, null=True)), + ('assets_own_use', models.BigIntegerField(blank=True, null=True)), + ('assets_long_term_investment', models.BigIntegerField(blank=True, null=True)), + ('defined_benefit_pension_scheme', models.BigIntegerField(blank=True, null=True)), + ('assets_other_assets', models.BigIntegerField(blank=True, null=True)), + ('assets_total_liabilities', models.BigIntegerField(blank=True, null=True)), + ('assets_current_investment', models.BigIntegerField(blank=True, null=True)), + ('assets_total_assets_and_liabilities', models.BigIntegerField(blank=True, null=True)), + ('creditors_one_year_total_current', models.BigIntegerField(blank=True, null=True)), + ('creditors_falling_due_after_one_year', models.BigIntegerField(blank=True, null=True)), + ('assets_cash', models.BigIntegerField(blank=True, null=True)), + ('funds_endowment', models.BigIntegerField(blank=True, null=True)), + ('funds_unrestricted', models.BigIntegerField(blank=True, null=True)), + ('funds_restricted', models.BigIntegerField(blank=True, null=True)), + ('funds_total', models.BigIntegerField(blank=True, null=True)), + ('count_employees', models.IntegerField(blank=True, null=True)), + ('charity_only_accounts', models.BooleanField(blank=True, null=True)), + ('consolidated_accounts', models.BooleanField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityClassification', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('classification_code', models.IntegerField(blank=True, null=True)), + ('classification_type', models.CharField(blank=True, max_length=255, null=True)), + ('classification_description', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityEventHistory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('charity_name', models.CharField(max_length=255)), + ('charity_event_order', models.IntegerField(blank=True, null=True)), + ('event_type', models.CharField(blank=True, max_length=255, null=True)), + ('date_of_event', models.DateField(blank=True, null=True)), + ('reason', models.CharField(blank=True, max_length=255, null=True)), + ('assoc_organisation_number', models.IntegerField(blank=True, null=True)), + ('assoc_registered_charity_number', models.IntegerField(blank=True, db_index=True, null=True)), + ('assoc_charity_name', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityGoverningDocument', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('governing_document_description', models.TextField(blank=True, null=True)), + ('charitable_objects', models.TextField(blank=True, null=True)), + ('area_of_benefit', models.TextField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityOtherNames', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('charity_name_id', models.IntegerField(blank=True, null=True)), + ('charity_name_type', models.CharField(blank=True, max_length=255, null=True)), + ('charity_name', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityOtherRegulators', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('regulator_order', models.IntegerField(blank=True, null=True)), + ('regulator_name', models.CharField(blank=True, max_length=255, null=True)), + ('regulator_web_url', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityPolicy', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField()), + ('policy_name', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityPublishedReport', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('charity_id', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField(blank=True, null=True)), + ('report_name', models.CharField(blank=True, max_length=255, null=True)), + ('report_location', models.CharField(blank=True, max_length=255, null=True)), + ('date_published', models.DateField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name='CCEWCharityTrustee', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_of_extract', models.DateField(blank=True, null=True)), + ('organisation_number', models.IntegerField()), + ('registered_charity_number', models.IntegerField(db_index=True)), + ('linked_charity_number', models.IntegerField()), + ('trustee_id', models.IntegerField(blank=True, null=True)), + ('trustee_name', models.CharField(blank=True, max_length=255, null=True)), + ('trustee_is_chair', models.BooleanField(blank=True, null=True)), + ('individual_or_organisation', models.CharField(blank=True, choices=[('P', 'Individual'), ('O', 'Organisation')], max_length=1, null=True)), + ('trustee_date_of_appointment', models.DateField(blank=True, null=True)), + ], + ), + ] diff --git a/charity/migrations/0012_auto_20210403_1943.py b/charity/migrations/0012_auto_20210403_1943.py new file mode 100644 index 00000000..f69752d6 --- /dev/null +++ b/charity/migrations/0012_auto_20210403_1943.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2021-04-03 18:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('charity', '0011_ccew_updated_files'), + ] + + operations = [ + migrations.AlterField( + model_name='ccewcharityeventhistory', + name='charity_name', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/charity/migrations/0013_auto_20210404_2125.py b/charity/migrations/0013_auto_20210404_2125.py new file mode 100644 index 00000000..ad361a62 --- /dev/null +++ b/charity/migrations/0013_auto_20210404_2125.py @@ -0,0 +1,32 @@ +# Generated by Django 3.1.1 on 2021-04-04 20:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('charity', '0012_auto_20210403_1943'), + ] + + operations = [ + migrations.AlterField( + model_name='areaofoperation', + name='aookey', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='areaofoperation', + name='aootype', + field=models.CharField(blank=True, max_length=1, null=True), + ), + migrations.AlterField( + model_name='areaofoperation', + name='welsh', + field=models.BooleanField(blank=True, null=True, verbose_name='In Wales'), + ), + migrations.AlterUniqueTogether( + name='areaofoperation', + unique_together=set(), + ), + ] diff --git a/charity/migrations/0014_auto_20210404_2145.py b/charity/migrations/0014_auto_20210404_2145.py new file mode 100644 index 00000000..06d06f9a --- /dev/null +++ b/charity/migrations/0014_auto_20210404_2145.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2021-04-04 20:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('charity', '0013_auto_20210404_2125'), + ] + + operations = [ + migrations.AlterField( + model_name='charityname', + name='name', + field=models.CharField(db_index=True, max_length=255), + ), + ] diff --git a/charity/migrations/0015_auto_20210404_2147.py b/charity/migrations/0015_auto_20210404_2147.py new file mode 100644 index 00000000..053554a4 --- /dev/null +++ b/charity/migrations/0015_auto_20210404_2147.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2021-04-04 20:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('charity', '0014_auto_20210404_2145'), + ] + + operations = [ + migrations.AlterField( + model_name='charityname', + name='normalisedName', + field=models.CharField(blank=True, db_index=True, max_length=255, null=True), + ), + ] diff --git a/charity/migrations/0016_delete_ccewdatafile.py b/charity/migrations/0016_delete_ccewdatafile.py new file mode 100644 index 00000000..70e3e0d9 --- /dev/null +++ b/charity/migrations/0016_delete_ccewdatafile.py @@ -0,0 +1,16 @@ +# Generated by Django 3.1.1 on 2021-04-04 21:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('charity', '0015_auto_20210404_2147'), + ] + + operations = [ + migrations.DeleteModel( + name='CcewDataFile', + ), + ] diff --git a/charity/models.py b/charity/models.py deleted file mode 100644 index 342269d4..00000000 --- a/charity/models.py +++ /dev/null @@ -1,469 +0,0 @@ -from django.core.serializers.json import DjangoJSONEncoder -from django.db import models -from django.forms.models import model_to_dict -from django.utils.text import slugify - - -class Charity(models.Model): - id = models.CharField(max_length=200, primary_key=True) - name = models.CharField(max_length=200, db_index=True) - constitution = models.TextField(null=True, blank=True) - geographical_spread = models.TextField(null=True, blank=True) - address = models.TextField(null=True, blank=True) - postcode = models.CharField(max_length=200, null=True, blank=True) - phone = models.CharField(max_length=200, null=True, blank=True) - active = models.BooleanField(db_index=True) - date_registered = models.DateField(null=True, blank=True, db_index=True) - date_removed = models.DateField(null=True, blank=True, db_index=True) - removal_reason = models.CharField(max_length=200, null=True, blank=True) - web = models.URLField(null=True, blank=True) - email = models.CharField(max_length=200, null=True, blank=True) - company_number = models.CharField(max_length=200, null=True, blank=True) - activities = models.TextField(null=True, blank=True) - source = models.CharField(max_length=200, null=True, blank=True, db_index=True) - first_added = models.DateTimeField(auto_now_add=True) - last_updated = models.DateTimeField(auto_now=True) - income = models.BigIntegerField(null=True, blank=True, db_index=True) - spending = models.BigIntegerField(null=True, blank=True) - latest_fye = models.DateField(null=True, blank=True) - dual_registered = models.BooleanField(null=True, blank=True) - - areas_of_operation = models.ManyToManyField("AreaOfOperation") - classification = models.ManyToManyField("VocabularyEntries") - - class Meta: - verbose_name_plural = "Charities" - - def __str__(self): - return "{} [{}]".format(self.name, self.id) - - def financial_json(self): - return [ - { - **model_to_dict(f), - "exp_gen": f.exp_gen, - "reserves_months": f.reserves_months, - "fyend": f.fyend.isoformat(), - "fystart": f.fystart.isoformat() if f.fystart else None, - } - for f in self.financial.order_by("fyend").all() - ] - - @property - def has_ccew_partb(self): - for f in self.financial.all(): - if f.has_ccew_partb: - return True - return False - - -class CharityName(models.Model): - charity = models.ForeignKey( - "Charity", on_delete=models.CASCADE, related_name="other_names" - ) - name = models.CharField(max_length=200, db_index=True) - normalisedName = models.CharField( - max_length=200, db_index=True, blank=True, null=True - ) - name_type = models.CharField(max_length=200, db_index=True) - - class Meta: - unique_together = ( - "charity", - "name", - ) - - def __str__(self): - return "{} [{}]".format(self.name, self.charity.id) - - -class CharityFinancial(models.Model): - class AccountType(models.TextChoices): - BASIC = "basic", "Basic" - CONSOLIDATED = "consolidated", "Consolidated" - CHARITY = "charity", "Charity" - BASIC_OSCR = "basic_oscr", "Basic (OSCR)" - DETAILED_OSCR = "detailed_oscr", "Detailed (OSCR)" - BASIC_CCNI = "basic_ccni", "Basic (CCNI)" - DETAILED_CCNI = "detailed_ccni", "Detailed (CCNI)" - - charity = models.ForeignKey( - "Charity", on_delete=models.CASCADE, related_name="financial" - ) - fyend = models.DateField(db_index=True) - fystart = models.DateField(null=True, blank=True) - income = models.BigIntegerField(null=True, blank=True) - spending = models.BigIntegerField(null=True, blank=True) - inc_leg = models.BigIntegerField(null=True, blank=True) - inc_end = models.BigIntegerField(null=True, blank=True) - inc_vol = models.BigIntegerField(null=True, blank=True) - inc_fr = models.BigIntegerField(null=True, blank=True) - inc_char = models.BigIntegerField(null=True, blank=True) - inc_invest = models.BigIntegerField(null=True, blank=True) - inc_other = models.BigIntegerField(null=True, blank=True) - inc_total = models.BigIntegerField(null=True, blank=True) - invest_gain = models.BigIntegerField(null=True, blank=True) - asset_gain = models.BigIntegerField(null=True, blank=True) - pension_gain = models.BigIntegerField(null=True, blank=True) - exp_vol = models.BigIntegerField(null=True, blank=True) - exp_trade = models.BigIntegerField(null=True, blank=True) - exp_invest = models.BigIntegerField(null=True, blank=True) - exp_grant = models.BigIntegerField(null=True, blank=True) - exp_charble = models.BigIntegerField(null=True, blank=True) - exp_gov = models.BigIntegerField(null=True, blank=True) - exp_other = models.BigIntegerField(null=True, blank=True) - exp_total = models.BigIntegerField(null=True, blank=True) - exp_support = models.BigIntegerField(null=True, blank=True) - exp_dep = models.BigIntegerField(null=True, blank=True) - reserves = models.BigIntegerField(null=True, blank=True) - asset_open = models.BigIntegerField(null=True, blank=True) - asset_close = models.BigIntegerField(null=True, blank=True) - fixed_assets = models.BigIntegerField(null=True, blank=True) - open_assets = models.BigIntegerField(null=True, blank=True) - invest_assets = models.BigIntegerField(null=True, blank=True) - cash_assets = models.BigIntegerField(null=True, blank=True) - current_assets = models.BigIntegerField(null=True, blank=True) - credit_1 = models.BigIntegerField(null=True, blank=True) - credit_long = models.BigIntegerField(null=True, blank=True) - pension_assets = models.BigIntegerField(null=True, blank=True) - total_assets = models.BigIntegerField(null=True, blank=True) - funds_end = models.BigIntegerField(null=True, blank=True) - funds_restrict = models.BigIntegerField(null=True, blank=True) - funds_unrestrict = models.BigIntegerField(null=True, blank=True) - funds_total = models.BigIntegerField(null=True, blank=True) - employees = models.BigIntegerField(null=True, blank=True) - volunteers = models.BigIntegerField(null=True, blank=True) - account_type = models.CharField( - max_length=50, default=AccountType.BASIC, choices=AccountType.choices - ) - - class Meta: - unique_together = ( - "charity", - "fyend", - ) - - def __str__(self): - return "{} {}".format(self.charity.name, self.fyend) - - @property - def has_ccew_partb(self): - return self.account_type in ( - CharityFinancial.AccountType.CHARITY, - CharityFinancial.AccountType.CONSOLIDATED, - ) - - @property - def exp_gen(self): - """Expenditure on generating funds""" - if self.exp_total and self.has_ccew_partb: - return self.exp_total - (self.exp_other + self.exp_gov + self.exp_charble) - - @property - def reserves_months(self): - if self.exp_total and self.reserves: - return (self.reserves / self.exp_total) * 12 - - -class CharityRaw(models.Model): - org_id = models.CharField(max_length=200, db_index=True) - spider = models.CharField(max_length=200, db_index=True) - scrape = models.ForeignKey( - "ftc.Scrape", - on_delete=models.CASCADE, - ) - data = models.JSONField(encoder=DjangoJSONEncoder) - - class Meta: - verbose_name = "Raw charity data" - verbose_name_plural = "Raw charity data" - - def __str__(self): - return "{} {}".format(self.spider, self.org_id) - - -class AreaOfOperation(models.Model): - aootype = models.CharField(max_length=1) - aookey = models.IntegerField() - aooname = models.CharField(max_length=200, db_index=True) - aoosort = models.CharField(max_length=200, db_index=True) - welsh = models.BooleanField(verbose_name="In Wales") - master = models.ForeignKey( - "self", - on_delete=models.CASCADE, - verbose_name="Parent area", - null=True, - blank=True, - ) - GSS = models.CharField( - verbose_name="ONS Geocode for Local Authority", - max_length=10, - null=True, - blank=True, - db_index=True, - ) - ISO3166_1 = models.CharField( - verbose_name="ISO3166-1 country code (2 character)", - max_length=2, - null=True, - blank=True, - db_index=True, - ) - ISO3166_1_3 = models.CharField( - verbose_name="ISO3166-1 country code (3 character)", - max_length=3, - null=True, - blank=True, - db_index=True, - ) - ISO3166_2_GB = models.CharField( - verbose_name="ISO3166-2 region code (GB only)", - max_length=6, - null=True, - blank=True, - db_index=True, - ) - ContinentCode = models.CharField( - verbose_name="Continent", max_length=2, null=True, blank=True, db_index=True - ) - - class Meta: - unique_together = ( - "aootype", - "aookey", - ) - verbose_name = "Area of operation" - verbose_name_plural = "Areas of operation" - - def __str__(self): - return "{}-{} {}".format(self.aootype, self.aookey, self.aooname) - - -class Vocabulary(models.Model): - title = models.CharField(max_length=200, db_index=True, unique=True) - single = models.BooleanField() - - def __str__(self): - return self.title - - class Meta: - verbose_name = "Vocabulary" - verbose_name_plural = "Vocabularies" - - -class VocabularyEntries(models.Model): - vocabulary = models.ForeignKey( - "Vocabulary", on_delete=models.CASCADE, related_name="entries" - ) - code = models.CharField(max_length=500, db_index=True) - title = models.CharField(max_length=500, db_index=True) - parent = models.ForeignKey("self", on_delete=models.CASCADE, null=True, blank=True) - - class Meta: - unique_together = ( - "vocabulary", - "code", - ) - verbose_name = "Vocabulary Entry" - verbose_name_plural = "Vocabulary Entries" - - def __str__(self): - if slugify(self.title) == slugify(self.code): - return self.title - return "[{}] {}".format(self.code, self.title) - - -class CcewDataFile(models.Model): - title = models.CharField(max_length=500, db_index=True, unique=True) - url = models.URLField() - description = models.CharField(max_length=500, db_index=True) - first_added = models.DateTimeField(auto_now_add=True) - last_updated = models.DateTimeField(auto_now=True) - - def __str__(self): - return "Charity Commission data file: {}".format(self.title) - - def get_absolute_url(self): - return self.url - - class Meta: - verbose_name = "Charity Commission data file" - verbose_name_plural = "Charity Commission data files" - - -class CCEWCharity(models.Model): - regno = models.CharField( - db_index=True, max_length=255 - ) # integer registered number of a charity - # integer subsidiary number of a charity (may be 0 for main/group charity) - subno = models.IntegerField(db_index=True) - # varchar(150) main name of the charity - name = models.CharField(max_length=255, null=True, blank=True) - # varchar(2) R (registered) or RM (removed) - orgtype = models.CharField(max_length=255, null=True, blank=True) - # varchar(250) Description of Governing Document - gd = models.TextField(null=True, blank=True) - # varchar(175) area of benefit - may not be defined - aob = models.TextField(null=True, blank=True) - # char(1) area of benefit defined by Governing Document (T/F) - aob_defined = models.CharField(max_length=255, null=True, blank=True) - # char(1) NHS charity (T/F) - nhs = models.CharField(max_length=255, null=True, blank=True) - # varchar(20) Housing Association number - ha_no = models.CharField(max_length=255, null=True, blank=True) - corr = models.CharField( - max_length=255, null=True, blank=True - ) # varchar(70) Charity correspondent name - add1 = models.CharField( - max_length=255, null=True, blank=True - ) # varchar(35) address line of charity's correspondent - add2 = models.CharField( - max_length=255, null=True, blank=True - ) # varchar(35) address line of charity's correspondent - add3 = models.CharField( - max_length=255, null=True, blank=True - ) # varchar(35) address line of charity's correspondent - add4 = models.CharField( - max_length=255, null=True, blank=True - ) # varchar(35) address line of charity's correspondent - add5 = models.CharField( - max_length=255, null=True, blank=True - ) # varchar(35) address line of charity's correspondent - # varchar(8) postcode of charity's correspondent - postcode = models.CharField(max_length=255, null=True, blank=True) - # varchar(23) telephone of charity's correspondent - phone = models.CharField(max_length=255, null=True, blank=True) - # varchar(23) fax of charity's correspondent - fax = models.CharField(max_length=255, null=True, blank=True) - - -class CCEWMainCharity(models.Model): - # integer registered number of a charity - regno = models.CharField(db_index=True, max_length=255) - # integer company registration number - coyno = models.CharField(max_length=255, null=True, blank=True) - # char(1) trustees incorporated (T/F) - trustees = models.CharField(max_length=255, null=True, blank=True) - # char(4) Financial year end - fyend = models.CharField(max_length=255, null=True, blank=True) - # char(1) requires correspondence in both Welsh & English (T/F) - welsh = models.CharField(max_length=255, null=True, blank=True) - # datetime date for latest gross income (blank if income is an estimate) - incomedate = models.DateField(null=True, blank=True) - income = models.BigIntegerField(null=True, blank=True) # integer - # varchar(4) may be blank - grouptype = models.CharField(max_length=255, null=True, blank=True) - # varchar(255) email address - email = models.CharField(max_length=255, null=True, blank=True) - # varchar(255) website address - web = models.CharField(max_length=255, null=True, blank=True) - - -class CCEWName(models.Model): - # integer registered number of a charity - regno = models.CharField(db_index=True, max_length=255) - # integer subsidiary number of a charity (may be 0 for main/group charity) - subno = models.IntegerField(db_index=True) - nameno = models.IntegerField() # integer number identifying a charity name - name = models.CharField( - max_length=255 - ) # varchar(150) name of a charity (multiple occurrences possible) - - -class CCEWRegistration(models.Model): - # integer registered number of a charity - regno = models.CharField(db_index=True, max_length=255) - # integer subsidiary number of a charity (may be 0 for main/group charity) - subno = models.IntegerField(db_index=True) - regdate = models.DateField() # datetime date of registration for a charity - # datetime Removal date of a charity - Blank for Registered Charities - remdate = models.DateField(null=True, blank=True) - # varchar(3) Register removal reason code - remcode = models.CharField(max_length=255, null=True, blank=True) - - -class CCEWCharityAOO(models.Model): - # integer registered number of a charity - regno = models.CharField(db_index=True, max_length=255) - aootype = models.CharField(max_length=255) # char(1) A B or D - aookey = models.IntegerField() # integer up to three digits - # char(1) Flag: Y or blank - welsh = models.CharField(max_length=255, null=True, blank=True) - # integer may be blank. If aootype=D then holds continent; if aootype=B then holds GLA/met county - master = models.IntegerField(null=True, blank=True) - - -class CCEWObjects(models.Model): - # integer registered number of a charity - regno = models.CharField(db_index=True, max_length=255) - # integer subsidiary number of a charity (may be 0 for main/group charity) - subno = models.IntegerField(db_index=True) - # char(4) sequence number (in practice 0-20) - seqno = models.CharField(db_index=True, max_length=255) - # varchar(255) Description of objects of a charity - object_text = models.TextField(db_column="object") - - -class CCEWFinancial(models.Model): - # integer registered number of a charity - regno = models.CharField(db_index=True, max_length=255) - fystart = models.DateField() # datetime Charity's financial year start date - # datetime Charity's financial year end date - fyend = models.DateField(db_index=True) - income = models.BigIntegerField(null=True, blank=True) # integer - expend = models.BigIntegerField(null=True, blank=True) # integer - - -class CCEWClass(models.Model): - # integer registered number of a charity - regno = models.CharField(db_index=True, max_length=255) - # integer classification code for a charity(multiple occurrences possible) - classification = models.CharField(db_column="class", max_length=255) - - -class CCEWPartB(models.Model): - regno = models.CharField(db_index=True, max_length=255) - artype = models.CharField(max_length=255) - fystart = models.DateField() - fyend = models.DateField(db_index=True) - inc_leg = models.BigIntegerField(null=True, blank=True) - inc_end = models.BigIntegerField(null=True, blank=True) - inc_vol = models.BigIntegerField(null=True, blank=True) - inc_fr = models.BigIntegerField(null=True, blank=True) - inc_char = models.BigIntegerField(null=True, blank=True) - inc_invest = models.BigIntegerField(null=True, blank=True) - inc_other = models.BigIntegerField(null=True, blank=True) - inc_total = models.BigIntegerField(null=True, blank=True) - invest_gain = models.BigIntegerField(null=True, blank=True) - asset_gain = models.BigIntegerField(null=True, blank=True) - pension_gain = models.BigIntegerField(null=True, blank=True) - exp_vol = models.BigIntegerField(null=True, blank=True) - exp_trade = models.BigIntegerField(null=True, blank=True) - exp_invest = models.BigIntegerField(null=True, blank=True) - exp_grant = models.BigIntegerField(null=True, blank=True) - exp_charble = models.BigIntegerField(null=True, blank=True) - exp_gov = models.BigIntegerField(null=True, blank=True) - exp_other = models.BigIntegerField(null=True, blank=True) - exp_total = models.BigIntegerField(null=True, blank=True) - exp_support = models.BigIntegerField(null=True, blank=True) - exp_dep = models.BigIntegerField(null=True, blank=True) - reserves = models.BigIntegerField(null=True, blank=True) - asset_open = models.BigIntegerField(null=True, blank=True) - asset_close = models.BigIntegerField(null=True, blank=True) - fixed_assets = models.BigIntegerField(null=True, blank=True) - open_assets = models.BigIntegerField(null=True, blank=True) - invest_assets = models.BigIntegerField(null=True, blank=True) - cash_assets = models.BigIntegerField(null=True, blank=True) - current_assets = models.BigIntegerField(null=True, blank=True) - credit_1 = models.BigIntegerField(null=True, blank=True) - credit_long = models.BigIntegerField(null=True, blank=True) - pension_assets = models.BigIntegerField(null=True, blank=True) - total_assets = models.BigIntegerField(null=True, blank=True) - funds_end = models.BigIntegerField(null=True, blank=True) - funds_restrict = models.BigIntegerField(null=True, blank=True) - funds_unrestrict = models.BigIntegerField(null=True, blank=True) - funds_total = models.BigIntegerField(null=True, blank=True) - employees = models.BigIntegerField(null=True, blank=True) - volunteers = models.BigIntegerField(null=True, blank=True) - cons_acc = models.CharField(max_length=255, null=True, blank=True) - charity_acc = models.CharField(max_length=255, null=True, blank=True) diff --git a/charity/models/__init__.py b/charity/models/__init__.py new file mode 100644 index 00000000..747ce127 --- /dev/null +++ b/charity/models/__init__.py @@ -0,0 +1,286 @@ +from django.core.serializers.json import DjangoJSONEncoder +from django.db import models +from django.forms.models import model_to_dict +from django.utils.text import slugify + + +from .ccew import ( + CCEWCharity, + CCEWCharityAnnualReturnHistory, + CCEWCharityAreaOfOperation, + CCEWCharityARPartA, + CCEWCharityARPartB, + CCEWCharityClassification, + CCEWCharityEventHistory, + CCEWCharityGoverningDocument, + CCEWCharityOtherNames, + CCEWCharityOtherRegulators, + CCEWCharityPolicy, + CCEWCharityPublishedReport, + CCEWCharityTrustee, +) + + +class Charity(models.Model): + id = models.CharField(max_length=200, primary_key=True) + name = models.CharField(max_length=200, db_index=True) + constitution = models.TextField(null=True, blank=True) + geographical_spread = models.TextField(null=True, blank=True) + address = models.TextField(null=True, blank=True) + postcode = models.CharField(max_length=200, null=True, blank=True) + phone = models.CharField(max_length=200, null=True, blank=True) + active = models.BooleanField(db_index=True) + date_registered = models.DateField(null=True, blank=True, db_index=True) + date_removed = models.DateField(null=True, blank=True, db_index=True) + removal_reason = models.CharField(max_length=200, null=True, blank=True) + web = models.URLField(null=True, blank=True) + email = models.CharField(max_length=200, null=True, blank=True) + company_number = models.CharField(max_length=200, null=True, blank=True) + activities = models.TextField(null=True, blank=True) + source = models.CharField(max_length=200, null=True, blank=True, db_index=True) + first_added = models.DateTimeField(auto_now_add=True) + last_updated = models.DateTimeField(auto_now=True) + income = models.BigIntegerField(null=True, blank=True, db_index=True) + spending = models.BigIntegerField(null=True, blank=True) + latest_fye = models.DateField(null=True, blank=True) + dual_registered = models.BooleanField(null=True, blank=True) + + areas_of_operation = models.ManyToManyField("AreaOfOperation") + classification = models.ManyToManyField("VocabularyEntries") + + class Meta: + verbose_name_plural = "Charities" + + def __str__(self): + return "{} [{}]".format(self.name, self.id) + + def financial_json(self): + return [ + { + **model_to_dict(f), + "exp_gen": f.exp_gen, + "reserves_months": f.reserves_months, + "fyend": f.fyend.isoformat(), + "fystart": f.fystart.isoformat() if f.fystart else None, + } + for f in self.financial.order_by("fyend").all() + ] + + @property + def has_ccew_partb(self): + for f in self.financial.all(): + if f.has_ccew_partb: + return True + return False + + +class CharityName(models.Model): + charity = models.ForeignKey( + "Charity", on_delete=models.CASCADE, related_name="other_names" + ) + name = models.CharField(max_length=255, db_index=True) + normalisedName = models.CharField( + max_length=255, db_index=True, blank=True, null=True + ) + name_type = models.CharField(max_length=200, db_index=True) + + class Meta: + unique_together = ( + "charity", + "name", + ) + + def __str__(self): + return "{} [{}]".format(self.name, self.charity.id) + + +class CharityFinancial(models.Model): + class AccountType(models.TextChoices): + BASIC = "basic", "Basic" + CONSOLIDATED = "consolidated", "Consolidated" + CHARITY = "charity", "Charity" + BASIC_OSCR = "basic_oscr", "Basic (OSCR)" + DETAILED_OSCR = "detailed_oscr", "Detailed (OSCR)" + BASIC_CCNI = "basic_ccni", "Basic (CCNI)" + DETAILED_CCNI = "detailed_ccni", "Detailed (CCNI)" + + charity = models.ForeignKey( + "Charity", on_delete=models.CASCADE, related_name="financial" + ) + fyend = models.DateField(db_index=True) + fystart = models.DateField(null=True, blank=True) + income = models.BigIntegerField(null=True, blank=True) + spending = models.BigIntegerField(null=True, blank=True) + inc_leg = models.BigIntegerField(null=True, blank=True) + inc_end = models.BigIntegerField(null=True, blank=True) + inc_vol = models.BigIntegerField(null=True, blank=True) + inc_fr = models.BigIntegerField(null=True, blank=True) + inc_char = models.BigIntegerField(null=True, blank=True) + inc_invest = models.BigIntegerField(null=True, blank=True) + inc_other = models.BigIntegerField(null=True, blank=True) + inc_total = models.BigIntegerField(null=True, blank=True) + invest_gain = models.BigIntegerField(null=True, blank=True) + asset_gain = models.BigIntegerField(null=True, blank=True) + pension_gain = models.BigIntegerField(null=True, blank=True) + exp_vol = models.BigIntegerField(null=True, blank=True) + exp_trade = models.BigIntegerField(null=True, blank=True) + exp_invest = models.BigIntegerField(null=True, blank=True) + exp_grant = models.BigIntegerField(null=True, blank=True) + exp_charble = models.BigIntegerField(null=True, blank=True) + exp_gov = models.BigIntegerField(null=True, blank=True) + exp_other = models.BigIntegerField(null=True, blank=True) + exp_total = models.BigIntegerField(null=True, blank=True) + exp_support = models.BigIntegerField(null=True, blank=True) + exp_dep = models.BigIntegerField(null=True, blank=True) + reserves = models.BigIntegerField(null=True, blank=True) + asset_open = models.BigIntegerField(null=True, blank=True) + asset_close = models.BigIntegerField(null=True, blank=True) + fixed_assets = models.BigIntegerField(null=True, blank=True) + open_assets = models.BigIntegerField(null=True, blank=True) + invest_assets = models.BigIntegerField(null=True, blank=True) + cash_assets = models.BigIntegerField(null=True, blank=True) + current_assets = models.BigIntegerField(null=True, blank=True) + credit_1 = models.BigIntegerField(null=True, blank=True) + credit_long = models.BigIntegerField(null=True, blank=True) + pension_assets = models.BigIntegerField(null=True, blank=True) + total_assets = models.BigIntegerField(null=True, blank=True) + funds_end = models.BigIntegerField(null=True, blank=True) + funds_restrict = models.BigIntegerField(null=True, blank=True) + funds_unrestrict = models.BigIntegerField(null=True, blank=True) + funds_total = models.BigIntegerField(null=True, blank=True) + employees = models.BigIntegerField(null=True, blank=True) + volunteers = models.BigIntegerField(null=True, blank=True) + account_type = models.CharField( + max_length=50, default=AccountType.BASIC, choices=AccountType.choices + ) + + class Meta: + unique_together = ( + "charity", + "fyend", + ) + + def __str__(self): + return "{} {}".format(self.charity.name, self.fyend) + + @property + def has_ccew_partb(self): + return self.account_type in ( + CharityFinancial.AccountType.CHARITY, + CharityFinancial.AccountType.CONSOLIDATED, + ) + + @property + def exp_gen(self): + """Expenditure on generating funds""" + if self.exp_total and self.has_ccew_partb: + return self.exp_total - (self.exp_other + self.exp_gov + self.exp_charble) + + @property + def reserves_months(self): + if self.exp_total and self.reserves: + return (self.reserves / self.exp_total) * 12 + + +class CharityRaw(models.Model): + org_id = models.CharField(max_length=200, db_index=True) + spider = models.CharField(max_length=200, db_index=True) + scrape = models.ForeignKey( + "ftc.Scrape", + on_delete=models.CASCADE, + ) + data = models.JSONField(encoder=DjangoJSONEncoder) + + class Meta: + verbose_name = "Raw charity data" + verbose_name_plural = "Raw charity data" + + def __str__(self): + return "{} {}".format(self.spider, self.org_id) + + +class AreaOfOperation(models.Model): + aootype = models.CharField(max_length=1, null=True, blank=True) + aookey = models.IntegerField(null=True, blank=True) + aooname = models.CharField(max_length=200, db_index=True) + aoosort = models.CharField(max_length=200, db_index=True) + welsh = models.BooleanField(verbose_name="In Wales", null=True, blank=True) + master = models.ForeignKey( + "self", + on_delete=models.CASCADE, + verbose_name="Parent area", + null=True, + blank=True, + ) + GSS = models.CharField( + verbose_name="ONS Geocode for Local Authority", + max_length=10, + null=True, + blank=True, + db_index=True, + ) + ISO3166_1 = models.CharField( + verbose_name="ISO3166-1 country code (2 character)", + max_length=2, + null=True, + blank=True, + db_index=True, + ) + ISO3166_1_3 = models.CharField( + verbose_name="ISO3166-1 country code (3 character)", + max_length=3, + null=True, + blank=True, + db_index=True, + ) + ISO3166_2_GB = models.CharField( + verbose_name="ISO3166-2 region code (GB only)", + max_length=6, + null=True, + blank=True, + db_index=True, + ) + ContinentCode = models.CharField( + verbose_name="Continent", max_length=2, null=True, blank=True, db_index=True + ) + + class Meta: + verbose_name = "Area of operation" + verbose_name_plural = "Areas of operation" + + def __str__(self): + return self.aooname + + +class Vocabulary(models.Model): + title = models.CharField(max_length=200, db_index=True, unique=True) + single = models.BooleanField() + + def __str__(self): + return self.title + + class Meta: + verbose_name = "Vocabulary" + verbose_name_plural = "Vocabularies" + + +class VocabularyEntries(models.Model): + vocabulary = models.ForeignKey( + "Vocabulary", on_delete=models.CASCADE, related_name="entries" + ) + code = models.CharField(max_length=500, db_index=True) + title = models.CharField(max_length=500, db_index=True) + parent = models.ForeignKey("self", on_delete=models.CASCADE, null=True, blank=True) + + class Meta: + unique_together = ( + "vocabulary", + "code", + ) + verbose_name = "Vocabulary Entry" + verbose_name_plural = "Vocabulary Entries" + + def __str__(self): + if slugify(self.title) == slugify(self.code): + return self.title + return "[{}] {}".format(self.code, self.title) diff --git a/charity/models/ccew.py b/charity/models/ccew.py new file mode 100644 index 00000000..832d609b --- /dev/null +++ b/charity/models/ccew.py @@ -0,0 +1,689 @@ +from django.db import models + + +class CCEWCharity(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = models.IntegerField( + db_index=True + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + charity_name = models.CharField( + max_length=255, null=True, blank=True + ) # The Main Name of the Charity + charity_type = models.CharField( + max_length=255, null=True, blank=True + ) # The type of the charity displayed on the public register of charities. Only the main parent charity will have a value for this field (i.e. linked_charity_number=0). + charity_registration_status = models.CharField( + max_length=255, null=True, blank=True + ) # The charity registration status indicates whether a charity is registered or removed + date_of_registration = models.DateField( + null=True, blank=True + ) # The date the charity was registered with the Charity Commission. + date_of_removal = models.DateField( + null=True, blank=True + ) # This is the date the charity was removed from the Register of Charities. This will not necessarily be the same date that the charity ceased to exist or ceased to operate. For non-removed charities the field is NULL. + charity_reporting_status = models.CharField( + max_length=255, null=True, blank=True + ) # The current reporting status of the charity + latest_acc_fin_period_start_date = models.DateField( + null=True, blank=True + ) # The start date of the latest financial period for which the charity has made a submission. + latest_acc_fin_period_end_date = models.DateField( + null=True, blank=True + ) # The end date of the latest financial period for which the charity has made a submission. + latest_income = models.FloatField( + null=True, blank=True + ) # The latest income submitted by the charity. This is the total gross income submitted on part A of the annual return submission. + latest_expenditure = models.FloatField( + null=True, blank=True + ) # The latest expenditure submitted by a charity. This is the expenditure submitted on part A of the annual return submission. + charity_contact_address1 = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Address Line 1 + charity_contact_address2 = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Address Line 2 + charity_contact_address3 = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Address Line 3 + charity_contact_address4 = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Address Line 4 + charity_contact_address5 = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Address Line 5 + charity_contact_postcode = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Postcode + charity_contact_phone = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Public Telephone Number + charity_contact_email = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Public email address + charity_contact_web = models.CharField( + max_length=255, null=True, blank=True + ) # Charity Website Address + charity_company_registration_number = models.CharField( + max_length=255, null=True, blank=True + ) # Registered Company Number of the Charity as assigned by Companies House. Integer returned as string + charity_insolvent = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity is insolvent. + charity_in_administration = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity is in administration. + charity_previously_excepted = models.BooleanField( + null=True, blank=True + ) # Indicates the charity was previously an excepted charity. + charity_is_cdf_or_cif = models.CharField( + max_length=255, null=True, blank=True + ) # Indicates whether the charity is a Common Investment Fund or Common Deposit Fund. + charity_is_cio = models.BooleanField( + null=True, blank=True + ) # Indicates whether the charity is a Charitable Incorporated Organisation. + cio_is_dissolved = models.BooleanField( + null=True, blank=True + ) # Indicates the CIO is to be dissolved. + date_cio_dissolution_notice = models.DateField( + null=True, blank=True + ) # Date the CIO dissolution notice expires + charity_activities = models.TextField( + null=True, blank=True + ) # The charity activities, the trustees’ description of what they do and who they help. + charity_gift_aid = models.BooleanField( + null=True, blank=True + ) # Indicates whether the charity is registered for gift aid with HMRC. True, False, NULL (not known) + charity_has_land = models.BooleanField( + null=True, blank=True + ) # Indicates whether the charity owns or leases any land or buildings. True, False, NULL (not known) + + +class CCEWCharityAnnualReturnHistory(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + fin_period_start_date = models.DateField( + null=True, blank=True + ) # The start date of the financial period which is detailed for the charity. + fin_period_end_date = models.DateField( + null=True, blank=True + ) # The end date of the financial period which is detailed for the charity. + ar_cycle_reference = models.CharField( + max_length=255, null=True, blank=True + ) # The annual return cycle to which the submission details relate. + reporting_due_date = models.DateField( + null=True, blank=True + ) # The due date of the financial period which is detailed for the charity. + date_annual_return_received = models.DateField( + null=True, blank=True + ) # The date the annual return was received for the financial period which is detailed for the charity. + date_accounts_received = models.DateField( + null=True, blank=True + ) # The date the charity accounts were received for the financial period which is detailed for the charity. + total_gross_income = models.BigIntegerField( + null=True, blank=True + ) # The total gross income reported on Part A of the annual return for the financial period detailed. + total_gross_expenditure = models.BigIntegerField( + null=True, blank=True + ) # The total gross expenditure reported on Part A of the annual return for the financial period detailed. + accounts_qualified = models.BooleanField( + null=True, blank=True + ) # Indicates whether the accounts have a qualified opinion. (True or NULL) + suppression_ind = models.BooleanField( + null=True, blank=True + ) # An indicator of whether the finances for this year are currently suppressed. 1 = Supressed, 0 = not supressed. + suppression_type = models.CharField(max_length=255, null=True, blank=True) + + +class CCEWCharityARPartA(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + latest_fin_period_submitted_ind = models.BooleanField( + null=True, blank=True + ) # Indicates whether the financial data on this line relates to the latest financial data submitted by the charity. (True or False) + fin_period_order_number = models.IntegerField( + null=True, blank=True + ) # A field to aid ordering of the financial data for each charity. (1=Most recent data in the table, 5=Least recent data in the table) + ar_cycle_reference = models.CharField( + max_length=255, null=True, blank=True + ) # The annual return cycle to which the submission details relate. + fin_period_start_date = models.DateField( + null=True, blank=True + ) # The start date of the financial period which is detailed for the charity. + fin_period_end_date = models.DateField( + null=True, blank=True + ) # The end date of the financial period which is detailed for the charity. + ar_due_date = models.DateField( + null=True, blank=True + ) # The due date of the annual return which is detailed for the charity. + ar_received_date = models.DateField( + null=True, blank=True + ) # The date the annual return was received for the financial period which is detailed for the charity. + total_gross_income = models.BigIntegerField( + null=True, blank=True + ) # The total gross income reported on Part A of the annual return for the financial period detailed. + total_gross_expenditure = models.BigIntegerField( + null=True, blank=True + ) # The total gross expenditure reported on Part A of the annual return for the financial period detailed. + charity_raises_funds_from_public = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity raised funds from the public for the financial period which is detailed for the charity. (True, False or NULL) + charity_professional_fundraiser = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity worked with professional fundraisers for the financial period which is detailed for the charity. (True, False or NULL) + charity_agreement_professional_fundraiser = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity had an agreement with its professional fundraisers for the financial period which is detailed. (True, False or NULL) + charity_commercial_participator = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity worked with commercial participators for the financial period detailed. (True, False or NULL) + charity_agreement_commerical_participator = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity had an agreement with its commercial participators for the financial period detailed. (True, False or NULL) + grant_making_is_main_activity = models.BooleanField( + null=True, blank=True + ) # Indicates if grant making was the main way the charity carried out its purposes for the financial period detailed. (True, False or NULL) + charity_receives_govt_funding_contracts = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity received any income from government contracts for the financial period detailed. (True, False or NULL) + count_govt_contracts = models.IntegerField( + null=True, blank=True + ) # The number of government contracts the charity had for the financial period detailed. + charity_receives_govt_funding_grants = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity received any income from government grants for the financial period detailed. (True, False or NULL) + count_govt_grants = models.IntegerField( + null=True, blank=True + ) # The number of government grants the charity received for the financial period detailed. + income_from_government_contracts = models.BigIntegerField( + null=True, blank=True + ) # The income the charity received from government contracts for the financial period detailed. + income_from_government_grants = models.BigIntegerField( + null=True, blank=True + ) # The income the charity received from government grants for the financial period detailed. + charity_has_trading_subsidiary = models.BooleanField( + null=True, blank=True + ) # Indicates if the charity had a trading subsidiary for the financial period detailed. (True, False or NULL) + trustee_also_director_of_subsidiary = models.BooleanField( + null=True, blank=True + ) # Indicates if a trustee was also a director of a trading subsidiary for the financial period detailed. (True, False or NULL) + does_trustee_receive_any_benefit = models.BooleanField( + null=True, blank=True + ) # Indicates if any of the trustees received any remuneration, payments or benefits from the charity other than refunds of legitimate trustee expenses for the financial period detailed. (True, False or NULL) + trustee_payments_acting_as_trustee = models.BooleanField( + null=True, blank=True + ) # Indicates if any trustees received payments for acting as a trustee for the financial period detailed. (True, False or NULL) + trustee_receives_payments_services = models.BooleanField( + null=True, blank=True + ) # Indicates if any trustees received payments for providing services to the charity for the financial period detailed. (True, False or NULL) + trustee_receives_other_benefit = models.BooleanField( + null=True, blank=True + ) # Indicates if any trustees received any other benefit from the charity for the financial period detailed. (True, False or NULL) + trustee_resigned_employment = models.BooleanField( + null=True, blank=True + ) # Indicates if any of the trustees resigned and took up employment with the charity during the financial period detailed. (True, False or NULL) + employees_salary_over_60k = models.BooleanField( + null=True, blank=True + ) # Indicates if any of the charity's staff received total employee benefits of £60,000 or more. (True, False or NULL) + count_salary_band_60001_70000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_70001_80000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_80001_90000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_90001_100000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_100001_110000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_110001_120000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_120001_130000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_130001_140000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_140001_150000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_150001_200000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_200001_250000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_250001_300000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_300001_350000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_350001_400000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_400001_450000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_450001_500000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_salary_band_over_500000 = models.IntegerField( + null=True, blank=True + ) # Number of staff whose total employment benefits were in this band for the financial period detailed. + count_volunteers = models.IntegerField( + null=True, blank=True + ) # Number of Volunteers. The trustees' estimate of the number of people who undertook voluntary work in the UK for the charity during the year. The number shown is a head count and not expressed as full-time equivalents. Charities are invited to provide an estimate of volunteer numbers in their Annual Return but are not obliged to do so. Where a number is provided by the charity, including zero, that number is displayed. + + +class CCEWCharityARPartB(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + latest_fin_period_submitted_ind = models.BooleanField( + null=True, blank=True + ) # Indicates whether the financial data on this line relates to the latest financial data submitted by the charity. (True or False) + fin_period_order_number = models.IntegerField( + null=True, blank=True + ) # A field to aid ordering of the financial data for each charity. (1=Most recent data in the table, 5=Least recent data in the table) + ar_cycle_reference = models.CharField( + max_length=255 + ) # The annual return cycle to which the submission details relate. + fin_period_start_date = models.DateField( + null=True, blank=True + ) # The start date of the financial period which is detailed for the charity. + fin_period_end_date = models.DateField( + null=True, blank=True + ) # The end date of the financial period which is detailed for the charity. + ar_due_date = models.DateField( + null=True, blank=True + ) # The due date of the annual return which is detailed for the charity. + ar_received_date = models.DateField( + null=True, blank=True + ) # The date the annual return was received for the financial period which is detailed for the charity. + income_donations_and_legacies = models.BigIntegerField( + null=True, blank=True + ) # Income from donations and legacies as entered on the Annual Return form for the financial period detailed. + income_charitable_activities = models.BigIntegerField( + null=True, blank=True + ) # Income received as fees or grants specifically for goods and services supplied by the charity to meet the needs of its beneficiaries for the financial period detailed. + income_other_trading_activities = models.BigIntegerField( + null=True, blank=True + ) # Income from other trading activity as entered on the Annual Return form for the financial period detailed. + income_investments = models.BigIntegerField( + null=True, blank=True + ) # Income from investments including dividends, interest and rents but excluding changes (realised and unrealised gains) in the capital value of the investment portfolio for the financial period detailed. + income_other = models.BigIntegerField( + null=True, blank=True + ) # Other income. This category includes gains on the disposal of own use assets (i.e. fixed assets not held as investments), but otherwise is only used exceptionally for very unusual transactions that cannot be accounted for in the categories above for the financial period detailed. + income_total_income_and_endowments = models.BigIntegerField( + null=True, blank=True + ) # Total income including endowments for the financial period detailed. + income_legacies = models.BigIntegerField( + null=True, blank=True + ) # Income from legacies as entered on the Annual Return form for the financial period detailed. + income_endowments = models.BigIntegerField( + null=True, blank=True + ) # Income from endowments as entered on the Annual Return form for the financial period detailed. + expenditure_raising_funds = models.BigIntegerField( + null=True, blank=True + ) # Costs associated with providing goods and services to the public, where the main motive is to raise funds for the charity rather than providing goods or services to meet the needs of its beneficiaries for the financial period detailed. (eg charity shops, fundraising dinners etc.). + expenditure_charitable_expenditure = models.BigIntegerField( + null=True, blank=True + ) # Costs incurred by the charity in supplying goods or services to meet the needs of its beneficiaries. Grants made to meet the needs of the charity’s beneficiaries for the financial period detailed. + expenditure_other = models.BigIntegerField( + null=True, blank=True + ) # Other expenditure for the financial period detailed. This category is only used very exceptionally for items that don’t fit within one of the categories above. + expenditure_total = models.BigIntegerField( + null=True, blank=True + ) # Total expenditure for the financial period detailed on the Part B of the annual return. + expenditure_investment_management = models.BigIntegerField( + null=True, blank=True + ) # Expenditure managing investments for the financial period detailed. + expenditure_grants_institution = models.BigIntegerField( + null=True, blank=True + ) # Any grants that the charity has awarded to other institutions to further their charitable work. + expenditure_governance = models.BigIntegerField( + null=True, blank=True + ) # Costs associated with running the charity itself for the financial period. (e.g. costs of trustee meetings, internal and external audit costs and legal advice relating to governance matters). + expenditure_support_costs = models.BigIntegerField( + null=True, blank=True + ) # Support costs should be allocated across activities and are those costs which, while necessary to deliver an activity, do not themselves produce the activity. They include the central office functions of the charity and are often apportioned to activities. The amount shown here is the total amount of support costs (for charitable, fundraising and governance activities) included in resources expended. + expenditure_depreciation = models.BigIntegerField( + null=True, blank=True + ) # Depreciation charge for the year can be found in the fixed asset analysis notes to the accounts. This is the amount of depreciation on tangible fixed assets (including impairment charges, if any), which will be shown as the charge for the year in the tangible fixed asset note to the accounts. + gain_loss_investment = models.BigIntegerField( + null=True, blank=True + ) # The gain or loss associated with the charity’s investments + gain_loss_pension_fund = models.BigIntegerField( + null=True, blank=True + ) # The gain or loss associated with the charity’s pension fund + gain_loss_revaluation_fixed_investment = models.BigIntegerField( + null=True, blank=True + ) # The gain or loss associated with any revaluation of fixed assets + gain_loss_other = models.BigIntegerField( + null=True, blank=True + ) # The gain or loss associated with any other assets + reserves = models.BigIntegerField( + null=True, blank=True + ) # The level of reserves is those unrestricted funds which are freely available for the charity to spend and can be found in the Financial Review in the Trustees Annual Report and will exclude endowments. + assets_total_fixed = models.BigIntegerField( + null=True, blank=True + ) # Total fixed assets. Fixed assets are those held for continuing use and include tangible fixed assets such as land, buildings, equipment and vehicles, and any investments held on a long-term basis to generate income or gains. + assets_own_use = models.BigIntegerField( + null=True, blank=True + ) # Total own use assets. This is a calculated field. assets_own_use = assets_total_fixed – assets_long_term_investment + assets_long_term_investment = models.BigIntegerField( + null=True, blank=True + ) # Fixed Asset Investment are held for the long term to generate income or gains and may include quoted and unquoted shares, bonds, gilts, common investment funds, investment property and term deposits held as part of an investment portfolio. + defined_benefit_pension_scheme = models.BigIntegerField( + null=True, blank=True + ) # This is surplus or deficit in any defined benefit pension scheme operated and represents a potential long-term asset or liability. + assets_other_assets = models.BigIntegerField( + null=True, blank=True + ) # The value of any other assets + assets_total_liabilities = models.BigIntegerField( + null=True, blank=True + ) # The value of the total liabilities for the charity. This is a calculated field. assets_total_liabilities = creditors_one_year_total_current + creditors_falling_due_after_one_year + assets_current_investment = models.BigIntegerField( + null=True, blank=True + ) # Total Current Assets are a separate class of Total Current Asset and they are held with intention of disposing of them within 12 months. + assets_total_assets_and_liabilities = models.BigIntegerField( + null=True, blank=True + ) # Total Net assets or liabilities can be found on the Balance Sheet. This is the total of all assets shown less all liabilities. This should be the same as the Total funds of the charity. + creditors_one_year_total_current = models.BigIntegerField( + null=True, blank=True + ) # Creditors due within one year are the amounts owed to creditors and include loans and overdrafts, trade creditors, accruals and deferred income and they are payable within one year. + creditors_falling_due_after_one_year = models.BigIntegerField( + null=True, blank=True + ) # These are the amounts owed to creditors payable after more than one year, with provisions for liabilities and charges. + assets_cash = models.BigIntegerField( + null=True, blank=True + ) # Cash at bank and in hand are a separate class of Total Current Assets. This amount includes deposits with banks and other financial institutions, which are repayable on demand, but excludes bank overdrafts. + funds_endowment = models.BigIntegerField( + null=True, blank=True + ) # Endowment funds include the amount of all permanent and expendable endowment funds. + funds_unrestricted = models.BigIntegerField( + null=True, blank=True + ) # Unrestricted funds include the amount of all funds held for the general purposes of the charity. This will include unrestricted income funds, designated funds, revaluation reserves and any pension reserve. + funds_restricted = models.BigIntegerField( + null=True, blank=True + ) # Restricted funds include the amount of all funds held that must be spent on the purposes of the charity. + funds_total = models.BigIntegerField( + null=True, blank=True + ) # Total funds can be found on the Balance Sheet and should be the same as Total net assets/(liabilities). + count_employees = models.IntegerField( + null=True, blank=True + ) # The number of people that the charity employs + charity_only_accounts = models.BooleanField( + null=True, blank=True + ) # Indicates if the accounts represent only the charity accounts + consolidated_accounts = models.BooleanField( + null=True, blank=True + ) # Consolidated accounts bring together the resources of the charity and the subsidiaries under its control in one statement. These subsidiaries may be non-charitable and to exist for purposes that benefit the parent charity e.g. fund-raising. If set to 1 the accounts are consolidated. + + +class CCEWCharityAreaOfOperation(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = models.IntegerField( + null=True, blank=True + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + geographic_area_type = models.CharField( + max_length=255, null=True, blank=True + ) # The area type for this row + geographic_area_description = models.CharField( + max_length=255, null=True, blank=True + ) # The area descriptor for this row + parent_geographic_area_type = models.CharField( + max_length=255, null=True, blank=True + ) # The parent area type. For example, if the area type is a country this indicator will be continent + parent_geographic_area_description = models.CharField( + max_length=255, null=True, blank=True + ) # The descriptor for the parent area type + welsh_ind = models.BooleanField(null=True, blank=True) # Indicates Welsh areas + + +class CCEWCharityClassification(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = models.IntegerField( + null=True, blank=True + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + classification_code = models.IntegerField( + null=True, blank=True + ) # The code of the classification described in the row + classification_type = models.CharField( + max_length=255, null=True, blank=True + ) # The type of the classification. What - What the charity does. How - How the charity helps. Who - Who the charity helps + classification_description = models.CharField( + max_length=255, null=True, blank=True + ) # The descriptor of the classification code. + + +class CCEWCharityEventHistory(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = models.IntegerField( + null=True, blank=True + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + charity_name = models.CharField(max_length=255, null=True, blank=True) # The Main Name of the Charity + charity_event_order = models.IntegerField( + null=True, blank=True + ) # The order of the event in the charity history. 1 is the earliest event. + event_type = models.CharField( + max_length=255, null=True, blank=True + ) # The type of charity event that has occurred. + date_of_event = models.DateField( + null=True, blank=True + ) # The date that the event occurred. + reason = models.CharField( + max_length=255, null=True, blank=True + ) # The reason that the event occurred. A registration event will not have a reason available. + assoc_organisation_number = models.IntegerField( + null=True, blank=True + ) # The charity id for the charity associated with the charity event. For example, in the case of asset transfer in this is the charity that has transferred the funds into the charity. + assoc_registered_charity_number = models.IntegerField( + db_index=True, null=True, blank=True + ) # The registered charity number for the charity associated with the charity event. For example, in the case of asset transfer in this is the charity that has transferred the funds into the charity. + assoc_charity_name = models.CharField( + max_length=255, null=True, blank=True + ) # The charity name of the charity associated with the charity event. For example, in the case of asset transfer in this is the charity that has transferred the funds into the charity. + + +class CCEWCharityGoverningDocument(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = models.IntegerField( + null=True, blank=True + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + governing_document_description = models.TextField( + null=True, blank=True + ) # The description of the governing document. Note that this is not the governing document itself but the details of the original document and any subsequent amendments. + charitable_objects = models.TextField( + null=True, blank=True + ) # The charitable objects of the charity. + area_of_benefit = models.TextField( + null=True, blank=True + ) # The area of benefit of the charity as defined in the governing document. This field can be blank as a charity does not have to define an area of benefit in the governing document. + + +class CCEWCharityOtherNames(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = models.IntegerField( + null=True, blank=True + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + charity_name_id = models.IntegerField( + null=True, blank=True + ) # An id for the other charity name + charity_name_type = models.CharField( + max_length=255, null=True, blank=True + ) # The type of other charity name. This can be working name or previous name. + charity_name = models.CharField( + max_length=255, null=True, blank=True + ) # The Main Name of the Charity + + +class CCEWCharityOtherRegulators(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + regulator_order = models.IntegerField( + null=True, blank=True + ) # A field to aid the ordering of the regulators for the charity. + regulator_name = models.CharField( + max_length=255, null=True, blank=True + ) # The name of the regulator. + regulator_web_url = models.CharField( + max_length=255, null=True, blank=True + ) # The web URL for the regulator. + + +class CCEWCharityPolicy(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = ( + models.IntegerField() + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + policy_name = models.CharField( + max_length=255, null=True, blank=True + ) # The name of the policy that the charity has in place. + + +class CCEWCharityPublishedReport(models.Model): + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + charity_id = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = models.IntegerField( + null=True, blank=True + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + report_name = models.CharField( + max_length=255, null=True, blank=True + ) # The type of report that has been published in relation to the charity. + report_location = models.CharField( + max_length=255, null=True, blank=True + ) # The web URL for the location on the charity commission .gov site where the published report can be located. + date_published = models.DateField( + null=True, blank=True + ) # The date that the message on the public register of charities to the report was published. + + +class CCEWCharityTrustee(models.Model): + class IndividualOrOrganisation(models.TextChoices): + INDIVIDUAL = "P" + ORGANISATION = "O" + + date_of_extract = models.DateField( + null=True, blank=True + ) # The date that the extract was taken from the main dataset. + organisation_number = ( + models.IntegerField() + ) # The organisation number for the charity. This is the index value for the charity. + registered_charity_number = models.IntegerField( + db_index=True + ) # The registration number of the registered organisation allocated by the Commission. Note that a main charity and all its linked charities will share the same registered_charity_number. + linked_charity_number = ( + models.IntegerField() + ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. + trustee_id = models.IntegerField( + null=True, blank=True + ) # The id number of the trustee. + trustee_name = models.CharField( + max_length=255, null=True, blank=True + ) # The name of the trustee. + trustee_is_chair = models.BooleanField( + null=True, blank=True + ) # TRUE if the trustee is the Chair. FALSE otherwise. + individual_or_organisation = models.CharField( + max_length=1, choices=IndividualOrOrganisation.choices, null=True, blank=True + ) # A flag to denote whether the trustee is an individual or an organisation. O for organisation or P for an individual. + trustee_date_of_appointment = models.DateField( + null=True, blank=True + ) # The date of appointment of the trustee for that charity. diff --git a/charity/urls.py b/charity/urls.py index b2acd20c..c2f52abf 100644 --- a/charity/urls.py +++ b/charity/urls.py @@ -1,10 +1,8 @@ from django.urls import path -from . import feeds, views +from . import views urlpatterns = [ - path("ccew/feed.rss", feeds.CcewDataFeedRSS()), - path("ccew/feed.atom", feeds.CcewDataFeedAtom()), path(".json", views.get_charity, {"filetype": "json"}), path(".html", views.get_charity, {"filetype": "html"}), path("", views.get_charity, {"filetype": "html"}, name="charity_html"), From 80400a122ca9c8f7964d574a23e85b63efe1d618 Mon Sep 17 00:00:00 2001 From: David Kane Date: Sun, 11 Apr 2021 23:10:42 +0100 Subject: [PATCH 12/19] add organisation location to header --- ftc/jinja2/panels/org-header.html.j2 | 7 ++++++- ftc/models/organisation.py | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ftc/jinja2/panels/org-header.html.j2 b/ftc/jinja2/panels/org-header.html.j2 index 6b678cf4..063e3100 100644 --- a/ftc/jinja2/panels/org-header.html.j2 +++ b/ftc/jinja2/panels/org-header.html.j2 @@ -12,7 +12,12 @@ {% else %} A {% endif %} - {{ orgtype_link(org.organisationTypePrimary, large=true) }}based in the UK + {{ orgtype_link(org.organisationTypePrimary, large=true) }}based in + {% if org.hq.geo_ctry %} + {{ org.hq.geo_ctry|get_geoname }} + {% else %} + the UK + {% endif %} {%- if org.dateRemoved %}. Removed on {{ "{:%d %B %Y}".format(org.dateRemoved) }}.{% endif %}

diff --git a/ftc/models/organisation.py b/ftc/models/organisation.py index be50cd19..686df8e6 100644 --- a/ftc/models/organisation.py +++ b/ftc/models/organisation.py @@ -415,3 +415,9 @@ def lat_lngs(self): (location.geo_lat, location.geo_long, location_type, location.name) ) return return_lat_lngs + + @property + def hq(self): + for location in self.locations.all(): + if location.locationType == OrganisationLocation.LocationTypes.REGISTERED_OFFICE: + return location From c991c08edb0e5f4ba8e56bd3d50ccde09e28d1c7 Mon Sep 17 00:00:00 2001 From: David Kane Date: Mon, 12 Apr 2021 00:08:15 +0100 Subject: [PATCH 13/19] add location search to db query --- ftc/query.py | 26 ++++++++++++++++++-------- ftc/views.py | 5 ++++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/ftc/query.py b/ftc/query.py index a96ac2de..be938cb6 100644 --- a/ftc/query.py +++ b/ftc/query.py @@ -1,7 +1,7 @@ import copy from django.core.paginator import Paginator -from django.db.models import Count, F, Func +from django.db.models import Count, F, Func, Q from django.shortcuts import Http404 from elasticsearch_dsl import A @@ -187,7 +187,7 @@ def run_es(self, with_pagination=False, with_aggregation=False): if self.source: filter_.append({"terms": {"source": self.source}}) - # check for source + # check for location if self.location: filter_.append({"terms": {"location": self.location}}) @@ -249,17 +249,27 @@ def run_es(self, with_pagination=False, with_aggregation=False): self.aggregation["by_active"]["inactive"] = b["doc_count"] def run_db(self, with_pagination=False, with_aggregation=False): - db_filter = {} + db_filter = [] if self.base_orgtype: - db_filter["organisationType__contains"] = self.base_orgtype + db_filter.append(Q(organisationType__contains=self.base_orgtype)) if self.source: - db_filter["source__id__in"] = self.source + db_filter.append(Q(source__id__in=self.source)) if self.term: - db_filter["name__search"] = self.term + db_filter.append(Q(name__search=self.term)) if self.active is True or self.active is False: - db_filter["active"] = self.active + db_filter.append(Q(active=self.active)) + + # check for location + if self.location: + db_filter.append( + Q(locations__geo_laua__in=self.location) + | Q(locations__geo_rgn__in=self.location) + | Q(locations__geo_ctry__in=self.location) + | Q(locations__geoCode__in=self.location) + ) - self.query = Organisation.objects.filter(**{k: v for k, v in db_filter.items()}) + # self.query = Organisation.objects.filter(**{k: v for k, v in db_filter.items()}) + self.query = Organisation.objects.filter(*db_filter) if with_pagination: self.paginator = Paginator( diff --git a/ftc/views.py b/ftc/views.py index 2b7f3102..03178e9a 100644 --- a/ftc/views.py +++ b/ftc/views.py @@ -156,8 +156,11 @@ def stream(): yield writer.writerow(columns.values()) s.run_db() res = s.query.values_list(*columns.keys()).order_by("org_id") + prev_id = None for r in res: - yield writer.writerow(r) + if r[0] != prev_id: + yield writer.writerow(r) + prev_id = r[0] response = StreamingHttpResponse(stream(), content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="{}.csv"'.format( From 4f05392c298fbef8340038b8750f8f9fbdabbcc1 Mon Sep 17 00:00:00 2001 From: David Kane Date: Mon, 12 Apr 2021 16:33:51 +0100 Subject: [PATCH 14/19] better search faceting --- ftc/jinja2/components/search_facets.html.j2 | 96 +++++++++++---------- ftc/jinja2/components/search_form.html.j2 | 52 ++++++----- ftc/jinja2/index.html.j2 | 2 + ftc/jinja2/orgtype.html.j2 | 57 ++++++------ ftc/query.py | 13 ++- ftc/static/css/style.css | 12 ++- ftc/static/js/choices.js | 13 ++- 7 files changed, 139 insertions(+), 106 deletions(-) diff --git a/ftc/jinja2/components/search_facets.html.j2 b/ftc/jinja2/components/search_facets.html.j2 index 0a58a5a2..32dc920f 100644 --- a/ftc/jinja2/components/search_facets.html.j2 +++ b/ftc/jinja2/components/search_facets.html.j2 @@ -26,65 +26,71 @@ {% endif %}

Organisation types

-{# + - {% for t in search.aggregation.by_orgtype if orgtypes[t.orgtype].is_keytype() %} - {% endfor %} - {% for t in search.aggregation.by_orgtype if not orgtypes[t.orgtype].is_keytype() %} - {% endfor %} -#} -
    -{% for t in search.aggregation.by_orgtype if orgtypes[t.orgtype].is_keytype() %} -
  • -
    - {{ orgtypes[t.orgtype].title }} - {{ "{:,.0f}".format(t.records) }} -
    -
    • -{% endfor %} -{% for t in search.aggregation.by_orgtype if not orgtypes[t.orgtype].is_keytype() %} -
  • -
    - {{ orgtypes[t.orgtype].title }} - {{ "{:,.0f}".format(t.records) }} -
    -
    • -{% endfor %} -
+ +

Data sources

-
    -{% for t in search.aggregation.by_source %} -
  • -
    - {{ sources[t.source].title }} - {{ "{:,.0f}".format(t.records) }} -
    -
    • + + +

    UK areas

    + + + +

    Country

    + + + +Clear filters \ No newline at end of file diff --git a/ftc/jinja2/components/search_form.html.j2 b/ftc/jinja2/components/search_form.html.j2 index 9e371ecc..e96a7682 100644 --- a/ftc/jinja2/components/search_form.html.j2 +++ b/ftc/jinja2/components/search_form.html.j2 @@ -1,28 +1,26 @@ - -
    -
    - - {% if not search %} - - {% endif %} -
    - +
    +
    + + {% if not search %} + + {% endif %}
    -
    - -
    - \ No newline at end of file + +
    +
    + +
    \ No newline at end of file diff --git a/ftc/jinja2/index.html.j2 b/ftc/jinja2/index.html.j2 index a997e8c3..14e23b33 100644 --- a/ftc/jinja2/index.html.j2 +++ b/ftc/jinja2/index.html.j2 @@ -10,7 +10,9 @@

    Find non profit organisations working in the UK

    +
    {% include 'components/search_form.html.j2' %} +

    Find charities

    diff --git a/ftc/jinja2/orgtype.html.j2 b/ftc/jinja2/orgtype.html.j2 index 2d1ef5ae..2168cd5a 100644 --- a/ftc/jinja2/orgtype.html.j2 +++ b/ftc/jinja2/orgtype.html.j2 @@ -15,15 +15,25 @@ {% set subtitle = "Showing all {:,.0f} organisations".format(res.paginator.count) %} {% endif %} -{#% block headscripts %} -{{ super() }} +{% block headscripts %} + +{{ super() }} {% endblock headscripts %} {% block bodyscripts %} {{ super() }} -{% endblock bodyscripts %#} +{% endblock bodyscripts %} + +{% macro filter_tag(key, value) %} +
  • + {{ key }}{{ value }} +
    • +{% endmacro %} {% block content %}
    @@ -33,30 +43,23 @@ {% if search.other_orgtypes or search.source or search.location %}
    Filters: - {% if search.other_orgtypes %} - {% for r in search.other_orgtypes if r in orgtypes and base_query.slug != r %} - - Type: - {{ orgtypes[r].title }} - - {% endfor %} - {% endif %} - {% if search.source %} - {% for r in search.source if r in sources and base_query.slug != r %} - - Source: - {{ sources[r].title }} - - {% endfor %} - {% endif %} - {% if search.location %} - {% for l in search.location if base_query.slug != l %} - - Location: - {{ l|get_geoname }} - - {% endfor %} - {% endif %} +
      + {% if search.other_orgtypes %} + {% for r in search.other_orgtypes if r in orgtypes and base_query.slug != r %} + {{ filter_tag('Type', orgtypes[r].title) }} + {% endfor %} + {% endif %} + {% if search.source %} + {% for r in search.source if r in sources and base_query.slug != r %} + {{ filter_tag('Source', sources[r].title) }} + {% endfor %} + {% endif %} + {% if search.location %} + {% for l in search.location if base_query.slug != l %} + {{ filter_tag('Location', l|get_geoname) }} + {% endfor %} + {% endif %} +
    Clear filters
    {% endif %} diff --git a/ftc/query.py b/ftc/query.py index be938cb6..260b7639 100644 --- a/ftc/query.py +++ b/ftc/query.py @@ -138,6 +138,15 @@ def set_criteria_from_request(self, request): self.set_criteria(active=True) elif request.GET.get("active", "").lower().startswith("f"): self.set_criteria(active=False) + + @property + def orgtypes(self): + orgtypes = [] + if isinstance(self.base_orgtype, list): + orgtypes.extend(self.base_orgtype) + if isinstance(self.other_orgtypes, list): + orgtypes.extend(self.other_orgtypes) + return orgtypes def run_es(self, with_pagination=False, with_aggregation=False): """ @@ -215,7 +224,7 @@ def run_es(self, with_pagination=False, with_aggregation=False): by_source = A("terms", field="source", size=150) by_orgtype = A("terms", field="organisationType", size=150) by_active = A("terms", field="active", size=150) - by_location = A("terms", field="location", size=150) + by_location = A("terms", field="location", size=1000) q.aggs.bucket("by_source", by_source) q.aggs.bucket("by_orgtype", by_orgtype) q.aggs.bucket("by_active", by_active) @@ -252,6 +261,8 @@ def run_db(self, with_pagination=False, with_aggregation=False): db_filter = [] if self.base_orgtype: db_filter.append(Q(organisationType__contains=self.base_orgtype)) + if self.other_orgtypes: + db_filter.append(Q(organisationType__contains=self.other_orgtypes)) if self.source: db_filter.append(Q(source__id__in=self.source)) if self.term: diff --git a/ftc/static/css/style.css b/ftc/static/css/style.css index 9c627a34..89be0257 100644 --- a/ftc/static/css/style.css +++ b/ftc/static/css/style.css @@ -46,12 +46,16 @@ .choices__list.choices__list--dropdown { display: none; } +/* .choices__input.choices__input--cloned { + background-color: inherit; + margin-bottom: 0px; +} */ .choices__list.choices__list--dropdown.is-active { display: block; position: absolute; background-color: white; border: 1px solid lightgray; - padding: .5rem; + /* padding: .5rem; */ max-width: 100%; box-shadow: 2px 2px 4px 2px rgba( 0, 0, 0, 0.2 ); } @@ -65,13 +69,17 @@ border-top: 0px solid lightgray; } .choices__list.choices__list--dropdown.is-active .choices__group .choices__heading { - display: none; + /* display: none; */ } .choices__list.choices__list--dropdown.is-active .choices__item { padding-top: 0.25rem; padding-bottom: 0.25rem; cursor: pointer; + word-break: normal; } .choices__list.choices__list--dropdown.is-active .choices__item.is-highlighted { background-color: yellow; +} +.choices__list.choices__list--dropdown.is-active .choices__item--selectable { + padding-right: 0; } \ No newline at end of file diff --git a/ftc/static/js/choices.js b/ftc/static/js/choices.js index 2ef6113c..246e6bdb 100644 --- a/ftc/static/js/choices.js +++ b/ftc/static/js/choices.js @@ -1,9 +1,14 @@ document.querySelectorAll('.js-choices').forEach((el) => { const choices = new Choices(el, { + placeholder: true, + shouldSort: false, + removeItemButton: true, classNames: { - containerOuter: 'choices w-100 f6', - containerInner: 'choices__inner w-100 pa2 ba bw1 b--gray', - inputCloned: 'choices__input choices__input--cloned bn base-font w-100', - } + containerOuter: 'choices w-100 f6 mt2', + containerInner: 'w-100 pa2 ba bw1 b--gray', + input: 'base-font', + inputCloned: 'choices__input--cloned bn w-100', + }, + itemSelectText: '', }); }); \ No newline at end of file From e770b3bea2feee7c3184a52f92e78e03a05ee372 Mon Sep 17 00:00:00 2001 From: David Kane Date: Mon, 12 Apr 2021 16:37:40 +0100 Subject: [PATCH 15/19] linting --- charity/management/commands/import_ccew.py | 2 +- charity/migrations/0010_auto_20210403_1729.py | 20 +- ..._1729_squashed_0016_delete_ccewdatafile.py | 967 +++++++++++++----- charity/migrations/0011_ccew_updated_files.py | 906 +++++++++++----- charity/migrations/0012_auto_20210403_1943.py | 6 +- charity/migrations/0013_auto_20210404_2125.py | 18 +- charity/migrations/0014_auto_20210404_2145.py | 6 +- charity/migrations/0015_auto_20210404_2147.py | 10 +- .../migrations/0016_delete_ccewdatafile.py | 4 +- charity/models/__init__.py | 1 - charity/models/ccew.py | 4 +- ftc/models/organisation.py | 5 +- ftc/query.py | 2 +- 13 files changed, 1403 insertions(+), 548 deletions(-) diff --git a/charity/management/commands/import_ccew.py b/charity/management/commands/import_ccew.py index 3c3ca242..13f15917 100644 --- a/charity/management/commands/import_ccew.py +++ b/charity/management/commands/import_ccew.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import io import csv +import io import re import zipfile diff --git a/charity/migrations/0010_auto_20210403_1729.py b/charity/migrations/0010_auto_20210403_1729.py index 2c3a9500..dfb649df 100644 --- a/charity/migrations/0010_auto_20210403_1729.py +++ b/charity/migrations/0010_auto_20210403_1729.py @@ -6,35 +6,35 @@ class Migration(migrations.Migration): dependencies = [ - ('charity', '0009_auto_20200921_1156'), + ("charity", "0009_auto_20200921_1156"), ] operations = [ migrations.DeleteModel( - name='CCEWCharity', + name="CCEWCharity", ), migrations.DeleteModel( - name='CCEWCharityAOO', + name="CCEWCharityAOO", ), migrations.DeleteModel( - name='CCEWClass', + name="CCEWClass", ), migrations.DeleteModel( - name='CCEWFinancial', + name="CCEWFinancial", ), migrations.DeleteModel( - name='CCEWMainCharity', + name="CCEWMainCharity", ), migrations.DeleteModel( - name='CCEWName', + name="CCEWName", ), migrations.DeleteModel( - name='CCEWObjects', + name="CCEWObjects", ), migrations.DeleteModel( - name='CCEWPartB', + name="CCEWPartB", ), migrations.DeleteModel( - name='CCEWRegistration', + name="CCEWRegistration", ), ] diff --git a/charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py b/charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py index 16f019dd..34c3df6e 100644 --- a/charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py +++ b/charity/migrations/0010_auto_20210403_1729_squashed_0016_delete_ccewdatafile.py @@ -5,365 +5,796 @@ class Migration(migrations.Migration): - replaces = [('charity', '0010_auto_20210403_1729'), ('charity', '0011_ccew_updated_files'), ('charity', '0012_auto_20210403_1943'), ('charity', '0013_auto_20210404_2125'), ('charity', '0014_auto_20210404_2145'), ('charity', '0015_auto_20210404_2147'), ('charity', '0016_delete_ccewdatafile')] + replaces = [ + ("charity", "0010_auto_20210403_1729"), + ("charity", "0011_ccew_updated_files"), + ("charity", "0012_auto_20210403_1943"), + ("charity", "0013_auto_20210404_2125"), + ("charity", "0014_auto_20210404_2145"), + ("charity", "0015_auto_20210404_2147"), + ("charity", "0016_delete_ccewdatafile"), + ] dependencies = [ - ('charity', '0009_auto_20200921_1156'), + ("charity", "0009_auto_20200921_1156"), ] operations = [ migrations.DeleteModel( - name='CCEWCharity', + name="CCEWCharity", ), migrations.DeleteModel( - name='CCEWCharityAOO', + name="CCEWCharityAOO", ), migrations.DeleteModel( - name='CCEWClass', + name="CCEWClass", ), migrations.DeleteModel( - name='CCEWFinancial', + name="CCEWFinancial", ), migrations.DeleteModel( - name='CCEWMainCharity', + name="CCEWMainCharity", ), migrations.DeleteModel( - name='CCEWName', + name="CCEWName", ), migrations.DeleteModel( - name='CCEWObjects', + name="CCEWObjects", ), migrations.DeleteModel( - name='CCEWPartB', + name="CCEWPartB", ), migrations.DeleteModel( - name='CCEWRegistration', + name="CCEWRegistration", ), migrations.CreateModel( - name='CCEWCharity', + name="CCEWCharity", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(db_index=True)), - ('charity_name', models.CharField(blank=True, max_length=255, null=True)), - ('charity_type', models.CharField(blank=True, max_length=255, null=True)), - ('charity_registration_status', models.CharField(blank=True, max_length=255, null=True)), - ('date_of_registration', models.DateField(blank=True, null=True)), - ('date_of_removal', models.DateField(blank=True, null=True)), - ('charity_reporting_status', models.CharField(blank=True, max_length=255, null=True)), - ('latest_acc_fin_period_start_date', models.DateField(blank=True, null=True)), - ('latest_acc_fin_period_end_date', models.DateField(blank=True, null=True)), - ('latest_income', models.FloatField(blank=True, null=True)), - ('latest_expenditure', models.FloatField(blank=True, null=True)), - ('charity_contact_address1', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address2', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address3', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address4', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address5', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_postcode', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_phone', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_email', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_web', models.CharField(blank=True, max_length=255, null=True)), - ('charity_company_registration_number', models.CharField(blank=True, max_length=255, null=True)), - ('charity_insolvent', models.BooleanField(blank=True, null=True)), - ('charity_in_administration', models.BooleanField(blank=True, null=True)), - ('charity_previously_excepted', models.BooleanField(blank=True, null=True)), - ('charity_is_cdf_or_cif', models.CharField(blank=True, max_length=255, null=True)), - ('charity_is_cio', models.BooleanField(blank=True, null=True)), - ('cio_is_dissolved', models.BooleanField(blank=True, null=True)), - ('date_cio_dissolution_notice', models.DateField(blank=True, null=True)), - ('charity_activities', models.TextField(blank=True, null=True)), - ('charity_gift_aid', models.BooleanField(blank=True, null=True)), - ('charity_has_land', models.BooleanField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(db_index=True)), + ( + "charity_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_registration_status", + models.CharField(blank=True, max_length=255, null=True), + ), + ("date_of_registration", models.DateField(blank=True, null=True)), + ("date_of_removal", models.DateField(blank=True, null=True)), + ( + "charity_reporting_status", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "latest_acc_fin_period_start_date", + models.DateField(blank=True, null=True), + ), + ( + "latest_acc_fin_period_end_date", + models.DateField(blank=True, null=True), + ), + ("latest_income", models.FloatField(blank=True, null=True)), + ("latest_expenditure", models.FloatField(blank=True, null=True)), + ( + "charity_contact_address1", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address2", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address3", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address4", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address5", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_postcode", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_phone", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_email", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_web", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_company_registration_number", + models.CharField(blank=True, max_length=255, null=True), + ), + ("charity_insolvent", models.BooleanField(blank=True, null=True)), + ( + "charity_in_administration", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_previously_excepted", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_is_cdf_or_cif", + models.CharField(blank=True, max_length=255, null=True), + ), + ("charity_is_cio", models.BooleanField(blank=True, null=True)), + ("cio_is_dissolved", models.BooleanField(blank=True, null=True)), + ( + "date_cio_dissolution_notice", + models.DateField(blank=True, null=True), + ), + ("charity_activities", models.TextField(blank=True, null=True)), + ("charity_gift_aid", models.BooleanField(blank=True, null=True)), + ("charity_has_land", models.BooleanField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityAnnualReturnHistory', + name="CCEWCharityAnnualReturnHistory", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('fin_period_start_date', models.DateField(blank=True, null=True)), - ('fin_period_end_date', models.DateField(blank=True, null=True)), - ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), - ('reporting_due_date', models.DateField(blank=True, null=True)), - ('date_annual_return_received', models.DateField(blank=True, null=True)), - ('date_accounts_received', models.DateField(blank=True, null=True)), - ('total_gross_income', models.BigIntegerField(blank=True, null=True)), - ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), - ('accounts_qualified', models.BooleanField(blank=True, null=True)), - ('suppression_ind', models.BooleanField(blank=True, null=True)), - ('suppression_type', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("fin_period_start_date", models.DateField(blank=True, null=True)), + ("fin_period_end_date", models.DateField(blank=True, null=True)), + ( + "ar_cycle_reference", + models.CharField(blank=True, max_length=255, null=True), + ), + ("reporting_due_date", models.DateField(blank=True, null=True)), + ( + "date_annual_return_received", + models.DateField(blank=True, null=True), + ), + ("date_accounts_received", models.DateField(blank=True, null=True)), + ("total_gross_income", models.BigIntegerField(blank=True, null=True)), + ( + "total_gross_expenditure", + models.BigIntegerField(blank=True, null=True), + ), + ("accounts_qualified", models.BooleanField(blank=True, null=True)), + ("suppression_ind", models.BooleanField(blank=True, null=True)), + ( + "suppression_type", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityAreaOfOperation', + name="CCEWCharityAreaOfOperation", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), - ('geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), - ('parent_geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), - ('parent_geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), - ('welsh_ind', models.BooleanField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ( + "geographic_area_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "geographic_area_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "parent_geographic_area_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "parent_geographic_area_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ("welsh_ind", models.BooleanField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityARPartA', + name="CCEWCharityARPartA", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), - ('fin_period_order_number', models.IntegerField(blank=True, null=True)), - ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), - ('fin_period_start_date', models.DateField(blank=True, null=True)), - ('fin_period_end_date', models.DateField(blank=True, null=True)), - ('ar_due_date', models.DateField(blank=True, null=True)), - ('ar_received_date', models.DateField(blank=True, null=True)), - ('total_gross_income', models.BigIntegerField(blank=True, null=True)), - ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), - ('charity_raises_funds_from_public', models.BooleanField(blank=True, null=True)), - ('charity_professional_fundraiser', models.BooleanField(blank=True, null=True)), - ('charity_agreement_professional_fundraiser', models.BooleanField(blank=True, null=True)), - ('charity_commercial_participator', models.BooleanField(blank=True, null=True)), - ('charity_agreement_commerical_participator', models.BooleanField(blank=True, null=True)), - ('grant_making_is_main_activity', models.BooleanField(blank=True, null=True)), - ('charity_receives_govt_funding_contracts', models.BooleanField(blank=True, null=True)), - ('count_govt_contracts', models.IntegerField(blank=True, null=True)), - ('charity_receives_govt_funding_grants', models.BooleanField(blank=True, null=True)), - ('count_govt_grants', models.IntegerField(blank=True, null=True)), - ('income_from_government_contracts', models.BigIntegerField(blank=True, null=True)), - ('income_from_government_grants', models.BigIntegerField(blank=True, null=True)), - ('charity_has_trading_subsidiary', models.BooleanField(blank=True, null=True)), - ('trustee_also_director_of_subsidiary', models.BooleanField(blank=True, null=True)), - ('does_trustee_receive_any_benefit', models.BooleanField(blank=True, null=True)), - ('trustee_payments_acting_as_trustee', models.BooleanField(blank=True, null=True)), - ('trustee_receives_payments_services', models.BooleanField(blank=True, null=True)), - ('trustee_receives_other_benefit', models.BooleanField(blank=True, null=True)), - ('trustee_resigned_employment', models.BooleanField(blank=True, null=True)), - ('employees_salary_over_60k', models.BooleanField(blank=True, null=True)), - ('count_salary_band_60001_70000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_70001_80000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_80001_90000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_90001_100000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_100001_110000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_110001_120000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_120001_130000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_130001_140000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_140001_150000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_150001_200000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_200001_250000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_250001_300000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_300001_350000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_350001_400000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_400001_450000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_450001_500000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_over_500000', models.IntegerField(blank=True, null=True)), - ('count_volunteers', models.IntegerField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ( + "latest_fin_period_submitted_ind", + models.BooleanField(blank=True, null=True), + ), + ("fin_period_order_number", models.IntegerField(blank=True, null=True)), + ( + "ar_cycle_reference", + models.CharField(blank=True, max_length=255, null=True), + ), + ("fin_period_start_date", models.DateField(blank=True, null=True)), + ("fin_period_end_date", models.DateField(blank=True, null=True)), + ("ar_due_date", models.DateField(blank=True, null=True)), + ("ar_received_date", models.DateField(blank=True, null=True)), + ("total_gross_income", models.BigIntegerField(blank=True, null=True)), + ( + "total_gross_expenditure", + models.BigIntegerField(blank=True, null=True), + ), + ( + "charity_raises_funds_from_public", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_professional_fundraiser", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_agreement_professional_fundraiser", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_commercial_participator", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_agreement_commerical_participator", + models.BooleanField(blank=True, null=True), + ), + ( + "grant_making_is_main_activity", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_receives_govt_funding_contracts", + models.BooleanField(blank=True, null=True), + ), + ("count_govt_contracts", models.IntegerField(blank=True, null=True)), + ( + "charity_receives_govt_funding_grants", + models.BooleanField(blank=True, null=True), + ), + ("count_govt_grants", models.IntegerField(blank=True, null=True)), + ( + "income_from_government_contracts", + models.BigIntegerField(blank=True, null=True), + ), + ( + "income_from_government_grants", + models.BigIntegerField(blank=True, null=True), + ), + ( + "charity_has_trading_subsidiary", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_also_director_of_subsidiary", + models.BooleanField(blank=True, null=True), + ), + ( + "does_trustee_receive_any_benefit", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_payments_acting_as_trustee", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_receives_payments_services", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_receives_other_benefit", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_resigned_employment", + models.BooleanField(blank=True, null=True), + ), + ( + "employees_salary_over_60k", + models.BooleanField(blank=True, null=True), + ), + ( + "count_salary_band_60001_70000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_70001_80000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_80001_90000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_90001_100000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_100001_110000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_110001_120000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_120001_130000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_130001_140000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_140001_150000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_150001_200000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_200001_250000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_250001_300000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_300001_350000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_350001_400000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_400001_450000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_450001_500000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_over_500000", + models.IntegerField(blank=True, null=True), + ), + ("count_volunteers", models.IntegerField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityARPartB', + name="CCEWCharityARPartB", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), - ('fin_period_order_number', models.IntegerField(blank=True, null=True)), - ('ar_cycle_reference', models.CharField(max_length=255)), - ('fin_period_start_date', models.DateField(blank=True, null=True)), - ('fin_period_end_date', models.DateField(blank=True, null=True)), - ('ar_due_date', models.DateField(blank=True, null=True)), - ('ar_received_date', models.DateField(blank=True, null=True)), - ('income_donations_and_legacies', models.BigIntegerField(blank=True, null=True)), - ('income_charitable_activities', models.BigIntegerField(blank=True, null=True)), - ('income_other_trading_activities', models.BigIntegerField(blank=True, null=True)), - ('income_investments', models.BigIntegerField(blank=True, null=True)), - ('income_other', models.BigIntegerField(blank=True, null=True)), - ('income_total_income_and_endowments', models.BigIntegerField(blank=True, null=True)), - ('income_legacies', models.BigIntegerField(blank=True, null=True)), - ('income_endowments', models.BigIntegerField(blank=True, null=True)), - ('expenditure_raising_funds', models.BigIntegerField(blank=True, null=True)), - ('expenditure_charitable_expenditure', models.BigIntegerField(blank=True, null=True)), - ('expenditure_other', models.BigIntegerField(blank=True, null=True)), - ('expenditure_total', models.BigIntegerField(blank=True, null=True)), - ('expenditure_investment_management', models.BigIntegerField(blank=True, null=True)), - ('expenditure_grants_institution', models.BigIntegerField(blank=True, null=True)), - ('expenditure_governance', models.BigIntegerField(blank=True, null=True)), - ('expenditure_support_costs', models.BigIntegerField(blank=True, null=True)), - ('expenditure_depreciation', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_investment', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_pension_fund', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_revaluation_fixed_investment', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_other', models.BigIntegerField(blank=True, null=True)), - ('reserves', models.BigIntegerField(blank=True, null=True)), - ('assets_total_fixed', models.BigIntegerField(blank=True, null=True)), - ('assets_own_use', models.BigIntegerField(blank=True, null=True)), - ('assets_long_term_investment', models.BigIntegerField(blank=True, null=True)), - ('defined_benefit_pension_scheme', models.BigIntegerField(blank=True, null=True)), - ('assets_other_assets', models.BigIntegerField(blank=True, null=True)), - ('assets_total_liabilities', models.BigIntegerField(blank=True, null=True)), - ('assets_current_investment', models.BigIntegerField(blank=True, null=True)), - ('assets_total_assets_and_liabilities', models.BigIntegerField(blank=True, null=True)), - ('creditors_one_year_total_current', models.BigIntegerField(blank=True, null=True)), - ('creditors_falling_due_after_one_year', models.BigIntegerField(blank=True, null=True)), - ('assets_cash', models.BigIntegerField(blank=True, null=True)), - ('funds_endowment', models.BigIntegerField(blank=True, null=True)), - ('funds_unrestricted', models.BigIntegerField(blank=True, null=True)), - ('funds_restricted', models.BigIntegerField(blank=True, null=True)), - ('funds_total', models.BigIntegerField(blank=True, null=True)), - ('count_employees', models.IntegerField(blank=True, null=True)), - ('charity_only_accounts', models.BooleanField(blank=True, null=True)), - ('consolidated_accounts', models.BooleanField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ( + "latest_fin_period_submitted_ind", + models.BooleanField(blank=True, null=True), + ), + ("fin_period_order_number", models.IntegerField(blank=True, null=True)), + ("ar_cycle_reference", models.CharField(max_length=255)), + ("fin_period_start_date", models.DateField(blank=True, null=True)), + ("fin_period_end_date", models.DateField(blank=True, null=True)), + ("ar_due_date", models.DateField(blank=True, null=True)), + ("ar_received_date", models.DateField(blank=True, null=True)), + ( + "income_donations_and_legacies", + models.BigIntegerField(blank=True, null=True), + ), + ( + "income_charitable_activities", + models.BigIntegerField(blank=True, null=True), + ), + ( + "income_other_trading_activities", + models.BigIntegerField(blank=True, null=True), + ), + ("income_investments", models.BigIntegerField(blank=True, null=True)), + ("income_other", models.BigIntegerField(blank=True, null=True)), + ( + "income_total_income_and_endowments", + models.BigIntegerField(blank=True, null=True), + ), + ("income_legacies", models.BigIntegerField(blank=True, null=True)), + ("income_endowments", models.BigIntegerField(blank=True, null=True)), + ( + "expenditure_raising_funds", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_charitable_expenditure", + models.BigIntegerField(blank=True, null=True), + ), + ("expenditure_other", models.BigIntegerField(blank=True, null=True)), + ("expenditure_total", models.BigIntegerField(blank=True, null=True)), + ( + "expenditure_investment_management", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_grants_institution", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_governance", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_support_costs", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_depreciation", + models.BigIntegerField(blank=True, null=True), + ), + ("gain_loss_investment", models.BigIntegerField(blank=True, null=True)), + ( + "gain_loss_pension_fund", + models.BigIntegerField(blank=True, null=True), + ), + ( + "gain_loss_revaluation_fixed_investment", + models.BigIntegerField(blank=True, null=True), + ), + ("gain_loss_other", models.BigIntegerField(blank=True, null=True)), + ("reserves", models.BigIntegerField(blank=True, null=True)), + ("assets_total_fixed", models.BigIntegerField(blank=True, null=True)), + ("assets_own_use", models.BigIntegerField(blank=True, null=True)), + ( + "assets_long_term_investment", + models.BigIntegerField(blank=True, null=True), + ), + ( + "defined_benefit_pension_scheme", + models.BigIntegerField(blank=True, null=True), + ), + ("assets_other_assets", models.BigIntegerField(blank=True, null=True)), + ( + "assets_total_liabilities", + models.BigIntegerField(blank=True, null=True), + ), + ( + "assets_current_investment", + models.BigIntegerField(blank=True, null=True), + ), + ( + "assets_total_assets_and_liabilities", + models.BigIntegerField(blank=True, null=True), + ), + ( + "creditors_one_year_total_current", + models.BigIntegerField(blank=True, null=True), + ), + ( + "creditors_falling_due_after_one_year", + models.BigIntegerField(blank=True, null=True), + ), + ("assets_cash", models.BigIntegerField(blank=True, null=True)), + ("funds_endowment", models.BigIntegerField(blank=True, null=True)), + ("funds_unrestricted", models.BigIntegerField(blank=True, null=True)), + ("funds_restricted", models.BigIntegerField(blank=True, null=True)), + ("funds_total", models.BigIntegerField(blank=True, null=True)), + ("count_employees", models.IntegerField(blank=True, null=True)), + ("charity_only_accounts", models.BooleanField(blank=True, null=True)), + ("consolidated_accounts", models.BooleanField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityClassification', + name="CCEWCharityClassification", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('classification_code', models.IntegerField(blank=True, null=True)), - ('classification_type', models.CharField(blank=True, max_length=255, null=True)), - ('classification_description', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ("classification_code", models.IntegerField(blank=True, null=True)), + ( + "classification_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "classification_description", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityGoverningDocument', + name="CCEWCharityGoverningDocument", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('governing_document_description', models.TextField(blank=True, null=True)), - ('charitable_objects', models.TextField(blank=True, null=True)), - ('area_of_benefit', models.TextField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ( + "governing_document_description", + models.TextField(blank=True, null=True), + ), + ("charitable_objects", models.TextField(blank=True, null=True)), + ("area_of_benefit", models.TextField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityOtherNames', + name="CCEWCharityOtherNames", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('charity_name_id', models.IntegerField(blank=True, null=True)), - ('charity_name_type', models.CharField(blank=True, max_length=255, null=True)), - ('charity_name', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ("charity_name_id", models.IntegerField(blank=True, null=True)), + ( + "charity_name_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_name", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityOtherRegulators', + name="CCEWCharityOtherRegulators", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('regulator_order', models.IntegerField(blank=True, null=True)), - ('regulator_name', models.CharField(blank=True, max_length=255, null=True)), - ('regulator_web_url', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("regulator_order", models.IntegerField(blank=True, null=True)), + ( + "regulator_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "regulator_web_url", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityPolicy', + name="CCEWCharityPolicy", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField()), - ('policy_name', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField()), + ( + "policy_name", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityPublishedReport', + name="CCEWCharityPublishedReport", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('charity_id', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('report_name', models.CharField(blank=True, max_length=255, null=True)), - ('report_location', models.CharField(blank=True, max_length=255, null=True)), - ('date_published', models.DateField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("charity_id", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ( + "report_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "report_location", + models.CharField(blank=True, max_length=255, null=True), + ), + ("date_published", models.DateField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityTrustee', + name="CCEWCharityTrustee", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField()), - ('trustee_id', models.IntegerField(blank=True, null=True)), - ('trustee_name', models.CharField(blank=True, max_length=255, null=True)), - ('trustee_is_chair', models.BooleanField(blank=True, null=True)), - ('individual_or_organisation', models.CharField(blank=True, choices=[('P', 'Individual'), ('O', 'Organisation')], max_length=1, null=True)), - ('trustee_date_of_appointment', models.DateField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField()), + ("trustee_id", models.IntegerField(blank=True, null=True)), + ( + "trustee_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ("trustee_is_chair", models.BooleanField(blank=True, null=True)), + ( + "individual_or_organisation", + models.CharField( + blank=True, + choices=[("P", "Individual"), ("O", "Organisation")], + max_length=1, + null=True, + ), + ), + ( + "trustee_date_of_appointment", + models.DateField(blank=True, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityEventHistory', + name="CCEWCharityEventHistory", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('charity_name', models.CharField(blank=True, max_length=255, null=True)), - ('charity_event_order', models.IntegerField(blank=True, null=True)), - ('event_type', models.CharField(blank=True, max_length=255, null=True)), - ('date_of_event', models.DateField(blank=True, null=True)), - ('reason', models.CharField(blank=True, max_length=255, null=True)), - ('assoc_organisation_number', models.IntegerField(blank=True, null=True)), - ('assoc_registered_charity_number', models.IntegerField(blank=True, db_index=True, null=True)), - ('assoc_charity_name', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ( + "charity_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ("charity_event_order", models.IntegerField(blank=True, null=True)), + ("event_type", models.CharField(blank=True, max_length=255, null=True)), + ("date_of_event", models.DateField(blank=True, null=True)), + ("reason", models.CharField(blank=True, max_length=255, null=True)), + ( + "assoc_organisation_number", + models.IntegerField(blank=True, null=True), + ), + ( + "assoc_registered_charity_number", + models.IntegerField(blank=True, db_index=True, null=True), + ), + ( + "assoc_charity_name", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.AlterField( - model_name='areaofoperation', - name='aookey', + model_name="areaofoperation", + name="aookey", field=models.IntegerField(blank=True, null=True), ), migrations.AlterField( - model_name='areaofoperation', - name='aootype', + model_name="areaofoperation", + name="aootype", field=models.CharField(blank=True, max_length=1, null=True), ), migrations.AlterField( - model_name='areaofoperation', - name='welsh', - field=models.BooleanField(blank=True, null=True, verbose_name='In Wales'), + model_name="areaofoperation", + name="welsh", + field=models.BooleanField(blank=True, null=True, verbose_name="In Wales"), ), migrations.AlterUniqueTogether( - name='areaofoperation', + name="areaofoperation", unique_together=set(), ), migrations.AlterField( - model_name='charityname', - name='name', + model_name="charityname", + name="name", field=models.CharField(db_index=True, max_length=255), ), migrations.AlterField( - model_name='charityname', - name='normalisedName', - field=models.CharField(blank=True, db_index=True, max_length=255, null=True), + model_name="charityname", + name="normalisedName", + field=models.CharField( + blank=True, db_index=True, max_length=255, null=True + ), ), migrations.DeleteModel( - name='CcewDataFile', + name="CcewDataFile", ), ] diff --git a/charity/migrations/0011_ccew_updated_files.py b/charity/migrations/0011_ccew_updated_files.py index a76f1e25..28f8d37d 100644 --- a/charity/migrations/0011_ccew_updated_files.py +++ b/charity/migrations/0011_ccew_updated_files.py @@ -6,303 +6,721 @@ class Migration(migrations.Migration): dependencies = [ - ('charity', '0010_auto_20210403_1729'), + ("charity", "0010_auto_20210403_1729"), ] operations = [ migrations.CreateModel( - name='CCEWCharity', + name="CCEWCharity", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(db_index=True)), - ('charity_name', models.CharField(blank=True, max_length=255, null=True)), - ('charity_type', models.CharField(blank=True, max_length=255, null=True)), - ('charity_registration_status', models.CharField(blank=True, max_length=255, null=True)), - ('date_of_registration', models.DateField(blank=True, null=True)), - ('date_of_removal', models.DateField(blank=True, null=True)), - ('charity_reporting_status', models.CharField(blank=True, max_length=255, null=True)), - ('latest_acc_fin_period_start_date', models.DateField(blank=True, null=True)), - ('latest_acc_fin_period_end_date', models.DateField(blank=True, null=True)), - ('latest_income', models.FloatField(blank=True, null=True)), - ('latest_expenditure', models.FloatField(blank=True, null=True)), - ('charity_contact_address1', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address2', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address3', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address4', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_address5', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_postcode', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_phone', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_email', models.CharField(blank=True, max_length=255, null=True)), - ('charity_contact_web', models.CharField(blank=True, max_length=255, null=True)), - ('charity_company_registration_number', models.CharField(blank=True, max_length=255, null=True)), - ('charity_insolvent', models.BooleanField(blank=True, null=True)), - ('charity_in_administration', models.BooleanField(blank=True, null=True)), - ('charity_previously_excepted', models.BooleanField(blank=True, null=True)), - ('charity_is_cdf_or_cif', models.CharField(blank=True, max_length=255, null=True)), - ('charity_is_cio', models.BooleanField(blank=True, null=True)), - ('cio_is_dissolved', models.BooleanField(blank=True, null=True)), - ('date_cio_dissolution_notice', models.DateField(blank=True, null=True)), - ('charity_activities', models.TextField(blank=True, null=True)), - ('charity_gift_aid', models.BooleanField(blank=True, null=True)), - ('charity_has_land', models.BooleanField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(db_index=True)), + ( + "charity_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_registration_status", + models.CharField(blank=True, max_length=255, null=True), + ), + ("date_of_registration", models.DateField(blank=True, null=True)), + ("date_of_removal", models.DateField(blank=True, null=True)), + ( + "charity_reporting_status", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "latest_acc_fin_period_start_date", + models.DateField(blank=True, null=True), + ), + ( + "latest_acc_fin_period_end_date", + models.DateField(blank=True, null=True), + ), + ("latest_income", models.FloatField(blank=True, null=True)), + ("latest_expenditure", models.FloatField(blank=True, null=True)), + ( + "charity_contact_address1", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address2", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address3", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address4", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_address5", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_postcode", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_phone", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_email", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_contact_web", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_company_registration_number", + models.CharField(blank=True, max_length=255, null=True), + ), + ("charity_insolvent", models.BooleanField(blank=True, null=True)), + ( + "charity_in_administration", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_previously_excepted", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_is_cdf_or_cif", + models.CharField(blank=True, max_length=255, null=True), + ), + ("charity_is_cio", models.BooleanField(blank=True, null=True)), + ("cio_is_dissolved", models.BooleanField(blank=True, null=True)), + ( + "date_cio_dissolution_notice", + models.DateField(blank=True, null=True), + ), + ("charity_activities", models.TextField(blank=True, null=True)), + ("charity_gift_aid", models.BooleanField(blank=True, null=True)), + ("charity_has_land", models.BooleanField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityAnnualReturnHistory', + name="CCEWCharityAnnualReturnHistory", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('fin_period_start_date', models.DateField(blank=True, null=True)), - ('fin_period_end_date', models.DateField(blank=True, null=True)), - ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), - ('reporting_due_date', models.DateField(blank=True, null=True)), - ('date_annual_return_received', models.DateField(blank=True, null=True)), - ('date_accounts_received', models.DateField(blank=True, null=True)), - ('total_gross_income', models.BigIntegerField(blank=True, null=True)), - ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), - ('accounts_qualified', models.BooleanField(blank=True, null=True)), - ('suppression_ind', models.BooleanField(blank=True, null=True)), - ('suppression_type', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("fin_period_start_date", models.DateField(blank=True, null=True)), + ("fin_period_end_date", models.DateField(blank=True, null=True)), + ( + "ar_cycle_reference", + models.CharField(blank=True, max_length=255, null=True), + ), + ("reporting_due_date", models.DateField(blank=True, null=True)), + ( + "date_annual_return_received", + models.DateField(blank=True, null=True), + ), + ("date_accounts_received", models.DateField(blank=True, null=True)), + ("total_gross_income", models.BigIntegerField(blank=True, null=True)), + ( + "total_gross_expenditure", + models.BigIntegerField(blank=True, null=True), + ), + ("accounts_qualified", models.BooleanField(blank=True, null=True)), + ("suppression_ind", models.BooleanField(blank=True, null=True)), + ( + "suppression_type", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityAreaOfOperation', + name="CCEWCharityAreaOfOperation", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), - ('geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), - ('parent_geographic_area_type', models.CharField(blank=True, max_length=255, null=True)), - ('parent_geographic_area_description', models.CharField(blank=True, max_length=255, null=True)), - ('welsh_ind', models.BooleanField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ( + "geographic_area_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "geographic_area_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "parent_geographic_area_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "parent_geographic_area_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ("welsh_ind", models.BooleanField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityARPartA', + name="CCEWCharityARPartA", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), - ('fin_period_order_number', models.IntegerField(blank=True, null=True)), - ('ar_cycle_reference', models.CharField(blank=True, max_length=255, null=True)), - ('fin_period_start_date', models.DateField(blank=True, null=True)), - ('fin_period_end_date', models.DateField(blank=True, null=True)), - ('ar_due_date', models.DateField(blank=True, null=True)), - ('ar_received_date', models.DateField(blank=True, null=True)), - ('total_gross_income', models.BigIntegerField(blank=True, null=True)), - ('total_gross_expenditure', models.BigIntegerField(blank=True, null=True)), - ('charity_raises_funds_from_public', models.BooleanField(blank=True, null=True)), - ('charity_professional_fundraiser', models.BooleanField(blank=True, null=True)), - ('charity_agreement_professional_fundraiser', models.BooleanField(blank=True, null=True)), - ('charity_commercial_participator', models.BooleanField(blank=True, null=True)), - ('charity_agreement_commerical_participator', models.BooleanField(blank=True, null=True)), - ('grant_making_is_main_activity', models.BooleanField(blank=True, null=True)), - ('charity_receives_govt_funding_contracts', models.BooleanField(blank=True, null=True)), - ('count_govt_contracts', models.IntegerField(blank=True, null=True)), - ('charity_receives_govt_funding_grants', models.BooleanField(blank=True, null=True)), - ('count_govt_grants', models.IntegerField(blank=True, null=True)), - ('income_from_government_contracts', models.BigIntegerField(blank=True, null=True)), - ('income_from_government_grants', models.BigIntegerField(blank=True, null=True)), - ('charity_has_trading_subsidiary', models.BooleanField(blank=True, null=True)), - ('trustee_also_director_of_subsidiary', models.BooleanField(blank=True, null=True)), - ('does_trustee_receive_any_benefit', models.BooleanField(blank=True, null=True)), - ('trustee_payments_acting_as_trustee', models.BooleanField(blank=True, null=True)), - ('trustee_receives_payments_services', models.BooleanField(blank=True, null=True)), - ('trustee_receives_other_benefit', models.BooleanField(blank=True, null=True)), - ('trustee_resigned_employment', models.BooleanField(blank=True, null=True)), - ('employees_salary_over_60k', models.BooleanField(blank=True, null=True)), - ('count_salary_band_60001_70000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_70001_80000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_80001_90000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_90001_100000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_100001_110000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_110001_120000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_120001_130000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_130001_140000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_140001_150000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_150001_200000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_200001_250000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_250001_300000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_300001_350000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_350001_400000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_400001_450000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_450001_500000', models.IntegerField(blank=True, null=True)), - ('count_salary_band_over_500000', models.IntegerField(blank=True, null=True)), - ('count_volunteers', models.IntegerField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ( + "latest_fin_period_submitted_ind", + models.BooleanField(blank=True, null=True), + ), + ("fin_period_order_number", models.IntegerField(blank=True, null=True)), + ( + "ar_cycle_reference", + models.CharField(blank=True, max_length=255, null=True), + ), + ("fin_period_start_date", models.DateField(blank=True, null=True)), + ("fin_period_end_date", models.DateField(blank=True, null=True)), + ("ar_due_date", models.DateField(blank=True, null=True)), + ("ar_received_date", models.DateField(blank=True, null=True)), + ("total_gross_income", models.BigIntegerField(blank=True, null=True)), + ( + "total_gross_expenditure", + models.BigIntegerField(blank=True, null=True), + ), + ( + "charity_raises_funds_from_public", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_professional_fundraiser", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_agreement_professional_fundraiser", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_commercial_participator", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_agreement_commerical_participator", + models.BooleanField(blank=True, null=True), + ), + ( + "grant_making_is_main_activity", + models.BooleanField(blank=True, null=True), + ), + ( + "charity_receives_govt_funding_contracts", + models.BooleanField(blank=True, null=True), + ), + ("count_govt_contracts", models.IntegerField(blank=True, null=True)), + ( + "charity_receives_govt_funding_grants", + models.BooleanField(blank=True, null=True), + ), + ("count_govt_grants", models.IntegerField(blank=True, null=True)), + ( + "income_from_government_contracts", + models.BigIntegerField(blank=True, null=True), + ), + ( + "income_from_government_grants", + models.BigIntegerField(blank=True, null=True), + ), + ( + "charity_has_trading_subsidiary", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_also_director_of_subsidiary", + models.BooleanField(blank=True, null=True), + ), + ( + "does_trustee_receive_any_benefit", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_payments_acting_as_trustee", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_receives_payments_services", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_receives_other_benefit", + models.BooleanField(blank=True, null=True), + ), + ( + "trustee_resigned_employment", + models.BooleanField(blank=True, null=True), + ), + ( + "employees_salary_over_60k", + models.BooleanField(blank=True, null=True), + ), + ( + "count_salary_band_60001_70000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_70001_80000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_80001_90000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_90001_100000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_100001_110000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_110001_120000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_120001_130000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_130001_140000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_140001_150000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_150001_200000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_200001_250000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_250001_300000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_300001_350000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_350001_400000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_400001_450000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_450001_500000", + models.IntegerField(blank=True, null=True), + ), + ( + "count_salary_band_over_500000", + models.IntegerField(blank=True, null=True), + ), + ("count_volunteers", models.IntegerField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityARPartB', + name="CCEWCharityARPartB", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('latest_fin_period_submitted_ind', models.BooleanField(blank=True, null=True)), - ('fin_period_order_number', models.IntegerField(blank=True, null=True)), - ('ar_cycle_reference', models.CharField(max_length=255)), - ('fin_period_start_date', models.DateField(blank=True, null=True)), - ('fin_period_end_date', models.DateField(blank=True, null=True)), - ('ar_due_date', models.DateField(blank=True, null=True)), - ('ar_received_date', models.DateField(blank=True, null=True)), - ('income_donations_and_legacies', models.BigIntegerField(blank=True, null=True)), - ('income_charitable_activities', models.BigIntegerField(blank=True, null=True)), - ('income_other_trading_activities', models.BigIntegerField(blank=True, null=True)), - ('income_investments', models.BigIntegerField(blank=True, null=True)), - ('income_other', models.BigIntegerField(blank=True, null=True)), - ('income_total_income_and_endowments', models.BigIntegerField(blank=True, null=True)), - ('income_legacies', models.BigIntegerField(blank=True, null=True)), - ('income_endowments', models.BigIntegerField(blank=True, null=True)), - ('expenditure_raising_funds', models.BigIntegerField(blank=True, null=True)), - ('expenditure_charitable_expenditure', models.BigIntegerField(blank=True, null=True)), - ('expenditure_other', models.BigIntegerField(blank=True, null=True)), - ('expenditure_total', models.BigIntegerField(blank=True, null=True)), - ('expenditure_investment_management', models.BigIntegerField(blank=True, null=True)), - ('expenditure_grants_institution', models.BigIntegerField(blank=True, null=True)), - ('expenditure_governance', models.BigIntegerField(blank=True, null=True)), - ('expenditure_support_costs', models.BigIntegerField(blank=True, null=True)), - ('expenditure_depreciation', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_investment', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_pension_fund', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_revaluation_fixed_investment', models.BigIntegerField(blank=True, null=True)), - ('gain_loss_other', models.BigIntegerField(blank=True, null=True)), - ('reserves', models.BigIntegerField(blank=True, null=True)), - ('assets_total_fixed', models.BigIntegerField(blank=True, null=True)), - ('assets_own_use', models.BigIntegerField(blank=True, null=True)), - ('assets_long_term_investment', models.BigIntegerField(blank=True, null=True)), - ('defined_benefit_pension_scheme', models.BigIntegerField(blank=True, null=True)), - ('assets_other_assets', models.BigIntegerField(blank=True, null=True)), - ('assets_total_liabilities', models.BigIntegerField(blank=True, null=True)), - ('assets_current_investment', models.BigIntegerField(blank=True, null=True)), - ('assets_total_assets_and_liabilities', models.BigIntegerField(blank=True, null=True)), - ('creditors_one_year_total_current', models.BigIntegerField(blank=True, null=True)), - ('creditors_falling_due_after_one_year', models.BigIntegerField(blank=True, null=True)), - ('assets_cash', models.BigIntegerField(blank=True, null=True)), - ('funds_endowment', models.BigIntegerField(blank=True, null=True)), - ('funds_unrestricted', models.BigIntegerField(blank=True, null=True)), - ('funds_restricted', models.BigIntegerField(blank=True, null=True)), - ('funds_total', models.BigIntegerField(blank=True, null=True)), - ('count_employees', models.IntegerField(blank=True, null=True)), - ('charity_only_accounts', models.BooleanField(blank=True, null=True)), - ('consolidated_accounts', models.BooleanField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ( + "latest_fin_period_submitted_ind", + models.BooleanField(blank=True, null=True), + ), + ("fin_period_order_number", models.IntegerField(blank=True, null=True)), + ("ar_cycle_reference", models.CharField(max_length=255)), + ("fin_period_start_date", models.DateField(blank=True, null=True)), + ("fin_period_end_date", models.DateField(blank=True, null=True)), + ("ar_due_date", models.DateField(blank=True, null=True)), + ("ar_received_date", models.DateField(blank=True, null=True)), + ( + "income_donations_and_legacies", + models.BigIntegerField(blank=True, null=True), + ), + ( + "income_charitable_activities", + models.BigIntegerField(blank=True, null=True), + ), + ( + "income_other_trading_activities", + models.BigIntegerField(blank=True, null=True), + ), + ("income_investments", models.BigIntegerField(blank=True, null=True)), + ("income_other", models.BigIntegerField(blank=True, null=True)), + ( + "income_total_income_and_endowments", + models.BigIntegerField(blank=True, null=True), + ), + ("income_legacies", models.BigIntegerField(blank=True, null=True)), + ("income_endowments", models.BigIntegerField(blank=True, null=True)), + ( + "expenditure_raising_funds", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_charitable_expenditure", + models.BigIntegerField(blank=True, null=True), + ), + ("expenditure_other", models.BigIntegerField(blank=True, null=True)), + ("expenditure_total", models.BigIntegerField(blank=True, null=True)), + ( + "expenditure_investment_management", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_grants_institution", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_governance", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_support_costs", + models.BigIntegerField(blank=True, null=True), + ), + ( + "expenditure_depreciation", + models.BigIntegerField(blank=True, null=True), + ), + ("gain_loss_investment", models.BigIntegerField(blank=True, null=True)), + ( + "gain_loss_pension_fund", + models.BigIntegerField(blank=True, null=True), + ), + ( + "gain_loss_revaluation_fixed_investment", + models.BigIntegerField(blank=True, null=True), + ), + ("gain_loss_other", models.BigIntegerField(blank=True, null=True)), + ("reserves", models.BigIntegerField(blank=True, null=True)), + ("assets_total_fixed", models.BigIntegerField(blank=True, null=True)), + ("assets_own_use", models.BigIntegerField(blank=True, null=True)), + ( + "assets_long_term_investment", + models.BigIntegerField(blank=True, null=True), + ), + ( + "defined_benefit_pension_scheme", + models.BigIntegerField(blank=True, null=True), + ), + ("assets_other_assets", models.BigIntegerField(blank=True, null=True)), + ( + "assets_total_liabilities", + models.BigIntegerField(blank=True, null=True), + ), + ( + "assets_current_investment", + models.BigIntegerField(blank=True, null=True), + ), + ( + "assets_total_assets_and_liabilities", + models.BigIntegerField(blank=True, null=True), + ), + ( + "creditors_one_year_total_current", + models.BigIntegerField(blank=True, null=True), + ), + ( + "creditors_falling_due_after_one_year", + models.BigIntegerField(blank=True, null=True), + ), + ("assets_cash", models.BigIntegerField(blank=True, null=True)), + ("funds_endowment", models.BigIntegerField(blank=True, null=True)), + ("funds_unrestricted", models.BigIntegerField(blank=True, null=True)), + ("funds_restricted", models.BigIntegerField(blank=True, null=True)), + ("funds_total", models.BigIntegerField(blank=True, null=True)), + ("count_employees", models.IntegerField(blank=True, null=True)), + ("charity_only_accounts", models.BooleanField(blank=True, null=True)), + ("consolidated_accounts", models.BooleanField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityClassification', + name="CCEWCharityClassification", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('classification_code', models.IntegerField(blank=True, null=True)), - ('classification_type', models.CharField(blank=True, max_length=255, null=True)), - ('classification_description', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ("classification_code", models.IntegerField(blank=True, null=True)), + ( + "classification_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "classification_description", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityEventHistory', + name="CCEWCharityEventHistory", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('charity_name', models.CharField(max_length=255)), - ('charity_event_order', models.IntegerField(blank=True, null=True)), - ('event_type', models.CharField(blank=True, max_length=255, null=True)), - ('date_of_event', models.DateField(blank=True, null=True)), - ('reason', models.CharField(blank=True, max_length=255, null=True)), - ('assoc_organisation_number', models.IntegerField(blank=True, null=True)), - ('assoc_registered_charity_number', models.IntegerField(blank=True, db_index=True, null=True)), - ('assoc_charity_name', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ("charity_name", models.CharField(max_length=255)), + ("charity_event_order", models.IntegerField(blank=True, null=True)), + ("event_type", models.CharField(blank=True, max_length=255, null=True)), + ("date_of_event", models.DateField(blank=True, null=True)), + ("reason", models.CharField(blank=True, max_length=255, null=True)), + ( + "assoc_organisation_number", + models.IntegerField(blank=True, null=True), + ), + ( + "assoc_registered_charity_number", + models.IntegerField(blank=True, db_index=True, null=True), + ), + ( + "assoc_charity_name", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityGoverningDocument', + name="CCEWCharityGoverningDocument", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('governing_document_description', models.TextField(blank=True, null=True)), - ('charitable_objects', models.TextField(blank=True, null=True)), - ('area_of_benefit', models.TextField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ( + "governing_document_description", + models.TextField(blank=True, null=True), + ), + ("charitable_objects", models.TextField(blank=True, null=True)), + ("area_of_benefit", models.TextField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityOtherNames', + name="CCEWCharityOtherNames", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('charity_name_id', models.IntegerField(blank=True, null=True)), - ('charity_name_type', models.CharField(blank=True, max_length=255, null=True)), - ('charity_name', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ("charity_name_id", models.IntegerField(blank=True, null=True)), + ( + "charity_name_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "charity_name", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityOtherRegulators', + name="CCEWCharityOtherRegulators", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('regulator_order', models.IntegerField(blank=True, null=True)), - ('regulator_name', models.CharField(blank=True, max_length=255, null=True)), - ('regulator_web_url', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("regulator_order", models.IntegerField(blank=True, null=True)), + ( + "regulator_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "regulator_web_url", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityPolicy', + name="CCEWCharityPolicy", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField()), - ('policy_name', models.CharField(blank=True, max_length=255, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField()), + ( + "policy_name", + models.CharField(blank=True, max_length=255, null=True), + ), ], ), migrations.CreateModel( - name='CCEWCharityPublishedReport', + name="CCEWCharityPublishedReport", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('charity_id', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField(blank=True, null=True)), - ('report_name', models.CharField(blank=True, max_length=255, null=True)), - ('report_location', models.CharField(blank=True, max_length=255, null=True)), - ('date_published', models.DateField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("charity_id", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField(blank=True, null=True)), + ( + "report_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "report_location", + models.CharField(blank=True, max_length=255, null=True), + ), + ("date_published", models.DateField(blank=True, null=True)), ], ), migrations.CreateModel( - name='CCEWCharityTrustee', + name="CCEWCharityTrustee", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('date_of_extract', models.DateField(blank=True, null=True)), - ('organisation_number', models.IntegerField()), - ('registered_charity_number', models.IntegerField(db_index=True)), - ('linked_charity_number', models.IntegerField()), - ('trustee_id', models.IntegerField(blank=True, null=True)), - ('trustee_name', models.CharField(blank=True, max_length=255, null=True)), - ('trustee_is_chair', models.BooleanField(blank=True, null=True)), - ('individual_or_organisation', models.CharField(blank=True, choices=[('P', 'Individual'), ('O', 'Organisation')], max_length=1, null=True)), - ('trustee_date_of_appointment', models.DateField(blank=True, null=True)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_of_extract", models.DateField(blank=True, null=True)), + ("organisation_number", models.IntegerField()), + ("registered_charity_number", models.IntegerField(db_index=True)), + ("linked_charity_number", models.IntegerField()), + ("trustee_id", models.IntegerField(blank=True, null=True)), + ( + "trustee_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ("trustee_is_chair", models.BooleanField(blank=True, null=True)), + ( + "individual_or_organisation", + models.CharField( + blank=True, + choices=[("P", "Individual"), ("O", "Organisation")], + max_length=1, + null=True, + ), + ), + ( + "trustee_date_of_appointment", + models.DateField(blank=True, null=True), + ), ], ), ] diff --git a/charity/migrations/0012_auto_20210403_1943.py b/charity/migrations/0012_auto_20210403_1943.py index f69752d6..1ccba7d8 100644 --- a/charity/migrations/0012_auto_20210403_1943.py +++ b/charity/migrations/0012_auto_20210403_1943.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('charity', '0011_ccew_updated_files'), + ("charity", "0011_ccew_updated_files"), ] operations = [ migrations.AlterField( - model_name='ccewcharityeventhistory', - name='charity_name', + model_name="ccewcharityeventhistory", + name="charity_name", field=models.CharField(blank=True, max_length=255, null=True), ), ] diff --git a/charity/migrations/0013_auto_20210404_2125.py b/charity/migrations/0013_auto_20210404_2125.py index ad361a62..1455347c 100644 --- a/charity/migrations/0013_auto_20210404_2125.py +++ b/charity/migrations/0013_auto_20210404_2125.py @@ -6,27 +6,27 @@ class Migration(migrations.Migration): dependencies = [ - ('charity', '0012_auto_20210403_1943'), + ("charity", "0012_auto_20210403_1943"), ] operations = [ migrations.AlterField( - model_name='areaofoperation', - name='aookey', + model_name="areaofoperation", + name="aookey", field=models.IntegerField(blank=True, null=True), ), migrations.AlterField( - model_name='areaofoperation', - name='aootype', + model_name="areaofoperation", + name="aootype", field=models.CharField(blank=True, max_length=1, null=True), ), migrations.AlterField( - model_name='areaofoperation', - name='welsh', - field=models.BooleanField(blank=True, null=True, verbose_name='In Wales'), + model_name="areaofoperation", + name="welsh", + field=models.BooleanField(blank=True, null=True, verbose_name="In Wales"), ), migrations.AlterUniqueTogether( - name='areaofoperation', + name="areaofoperation", unique_together=set(), ), ] diff --git a/charity/migrations/0014_auto_20210404_2145.py b/charity/migrations/0014_auto_20210404_2145.py index 06d06f9a..640e5026 100644 --- a/charity/migrations/0014_auto_20210404_2145.py +++ b/charity/migrations/0014_auto_20210404_2145.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('charity', '0013_auto_20210404_2125'), + ("charity", "0013_auto_20210404_2125"), ] operations = [ migrations.AlterField( - model_name='charityname', - name='name', + model_name="charityname", + name="name", field=models.CharField(db_index=True, max_length=255), ), ] diff --git a/charity/migrations/0015_auto_20210404_2147.py b/charity/migrations/0015_auto_20210404_2147.py index 053554a4..6d52c8f3 100644 --- a/charity/migrations/0015_auto_20210404_2147.py +++ b/charity/migrations/0015_auto_20210404_2147.py @@ -6,13 +6,15 @@ class Migration(migrations.Migration): dependencies = [ - ('charity', '0014_auto_20210404_2145'), + ("charity", "0014_auto_20210404_2145"), ] operations = [ migrations.AlterField( - model_name='charityname', - name='normalisedName', - field=models.CharField(blank=True, db_index=True, max_length=255, null=True), + model_name="charityname", + name="normalisedName", + field=models.CharField( + blank=True, db_index=True, max_length=255, null=True + ), ), ] diff --git a/charity/migrations/0016_delete_ccewdatafile.py b/charity/migrations/0016_delete_ccewdatafile.py index 70e3e0d9..6fda6a6a 100644 --- a/charity/migrations/0016_delete_ccewdatafile.py +++ b/charity/migrations/0016_delete_ccewdatafile.py @@ -6,11 +6,11 @@ class Migration(migrations.Migration): dependencies = [ - ('charity', '0015_auto_20210404_2147'), + ("charity", "0015_auto_20210404_2147"), ] operations = [ migrations.DeleteModel( - name='CcewDataFile', + name="CcewDataFile", ), ] diff --git a/charity/models/__init__.py b/charity/models/__init__.py index 747ce127..070cf23e 100644 --- a/charity/models/__init__.py +++ b/charity/models/__init__.py @@ -3,7 +3,6 @@ from django.forms.models import model_to_dict from django.utils.text import slugify - from .ccew import ( CCEWCharity, CCEWCharityAnnualReturnHistory, diff --git a/charity/models/ccew.py b/charity/models/ccew.py index 832d609b..ad89e902 100644 --- a/charity/models/ccew.py +++ b/charity/models/ccew.py @@ -520,7 +520,9 @@ class CCEWCharityEventHistory(models.Model): linked_charity_number = models.IntegerField( null=True, blank=True ) # A number that uniquely identifies the subsidiary or group member associated with a registered charity. Used for user identification purposes where the subsidiary is known by the parent registration number and the subsidiary number. The main parent charity has a linked_charity_number of 0. - charity_name = models.CharField(max_length=255, null=True, blank=True) # The Main Name of the Charity + charity_name = models.CharField( + max_length=255, null=True, blank=True + ) # The Main Name of the Charity charity_event_order = models.IntegerField( null=True, blank=True ) # The order of the event in the charity history. 1 is the earliest event. diff --git a/ftc/models/organisation.py b/ftc/models/organisation.py index 686df8e6..1aed8bd8 100644 --- a/ftc/models/organisation.py +++ b/ftc/models/organisation.py @@ -419,5 +419,8 @@ def lat_lngs(self): @property def hq(self): for location in self.locations.all(): - if location.locationType == OrganisationLocation.LocationTypes.REGISTERED_OFFICE: + if ( + location.locationType + == OrganisationLocation.LocationTypes.REGISTERED_OFFICE + ): return location diff --git a/ftc/query.py b/ftc/query.py index 260b7639..47f09f87 100644 --- a/ftc/query.py +++ b/ftc/query.py @@ -138,7 +138,7 @@ def set_criteria_from_request(self, request): self.set_criteria(active=True) elif request.GET.get("active", "").lower().startswith("f"): self.set_criteria(active=False) - + @property def orgtypes(self): orgtypes = [] From e18b116cfbc6eaa3e13ed35ff9b9969416c4c4ad Mon Sep 17 00:00:00 2001 From: David Kane Date: Sat, 17 Apr 2021 09:26:03 +0100 Subject: [PATCH 16/19] update requirements --- dev-requirements.txt | 87 ++++++++++++++++++++++---------------------- requirements.in | 2 +- requirements.txt | 76 +++++++++++++++++++------------------- 3 files changed, 84 insertions(+), 81 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fdebf961..28b013d5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,76 +5,77 @@ # pip-compile dev-requirements.in # appdirs==1.4.4 # via -r requirements.txt, black, pyppeteer -asgiref==3.2.10 # via -r requirements.txt, django +asgiref==3.3.4 # via -r requirements.txt, django bcp-reader==0.1.1 # via -r requirements.txt -beautifulsoup4==4.9.1 # via -r requirements.txt, bs4 +beautifulsoup4==4.9.3 # via -r requirements.txt, bs4 black==20.8b1 # via -r dev-requirements.in bs4==0.0.1 # via -r requirements.txt, requests-html -certifi==2020.4.5.1 # via -r requirements.txt, elasticsearch, requests -chardet==3.0.4 # via -r requirements.txt, requests +certifi==2020.12.5 # via -r requirements.txt, elasticsearch, requests +chardet==4.0.0 # via -r requirements.txt, requests click==7.1.2 # via black cssselect==1.1.0 # via -r requirements.txt, pyquery -decorator==4.4.2 # via -r requirements.txt, validators +decorator==5.0.7 # via -r requirements.txt, validators dj-database-url==0.5.0 # via -r requirements.txt -django-better-admin-arrayfield==1.4.0 # via -r requirements.txt -django-cors-headers==3.5.0 # via -r requirements.txt +django-better-admin-arrayfield==1.4.2 # via -r requirements.txt +django-cors-headers==3.7.0 # via -r requirements.txt django-elasticsearch-dsl==7.1.4 # via -r requirements.txt django-environ==0.4.5 # via -r requirements.txt django-filter==2.4.0 # via -r requirements.txt -django-ninja==0.9.1 # via -r requirements.txt +django-ninja==0.12.2 # via -r requirements.txt django-prettyjson==0.4.1 # via -r requirements.txt -django==3.1.1 # via -r requirements.txt, django-cors-headers, django-filter, django-ninja, django-prettyjson +django==3.2 # via -r requirements.txt, django-cors-headers, django-filter, django-ninja, django-prettyjson elasticsearch-dsl==7.3.0 # via -r requirements.txt, django-elasticsearch-dsl -elasticsearch==7.10.0 # via -r requirements.txt, elasticsearch-dsl +elasticsearch==7.12.0 # via -r requirements.txt, elasticsearch-dsl et-xmlfile==1.0.1 # via -r requirements.txt, openpyxl fake-useragent==0.1.11 # via -r requirements.txt, requests-html -flake8==3.8.4 # via -r dev-requirements.in -gunicorn==20.0.4 # via -r requirements.txt -humanize==2.4.0 # via -r requirements.txt -idna==2.9 # via -r requirements.txt, requests -inflect==4.1.0 # via -r requirements.txt -isort==5.6.4 # via -r dev-requirements.in -jdcal==1.4.1 # via -r requirements.txt, openpyxl -jinja2==2.11.2 # via -r requirements.txt -lml==0.0.9 # via -r requirements.txt, pyexcel-io -lxml==4.6.2 # via -r requirements.txt, pyexcel-ezodf, pyexcel-ods3, pyquery +flake8==3.9.1 # via -r dev-requirements.in +gunicorn==20.1.0 # via -r requirements.txt +humanize==3.4.1 # via -r requirements.txt +idna==2.10 # via -r requirements.txt, requests +inflect==5.3.0 # via -r requirements.txt +isort==5.8.0 # via -r dev-requirements.in +itsdangerous==1.1.0 # via -r requirements.txt, requests-cache +jinja2==2.11.3 # via -r requirements.txt +lml==0.1.0 # via -r requirements.txt, pyexcel-io +lxml==4.6.3 # via -r requirements.txt, pyexcel-ezodf, pyexcel-ods3, pyquery markupsafe==1.1.1 # via -r requirements.txt, jinja2 mccabe==0.6.1 # via flake8 mypy-extensions==0.4.3 # via black -openpyxl==3.0.3 # via -r requirements.txt -parse==1.15.0 # via -r requirements.txt, requests-html +openpyxl==3.0.7 # via -r requirements.txt +parse==1.19.0 # via -r requirements.txt, requests-html pathspec==0.8.1 # via black psycopg2==2.8.6 # via -r requirements.txt -pycodestyle==2.6.0 # via flake8 +pycodestyle==2.7.0 # via flake8 pycountry==20.7.3 # via -r requirements.txt pydantic==1.7.3 # via -r requirements.txt, django-ninja -pyee==7.0.2 # via -r requirements.txt, pyppeteer +pyee==8.1.0 # via -r requirements.txt, pyppeteer pyexcel-ezodf==0.3.4 # via -r requirements.txt, pyexcel-ods3 -pyexcel-io==0.5.20 # via -r requirements.txt, pyexcel-ods3 -pyexcel-ods3==0.5.3 # via -r requirements.txt -pyflakes==2.2.0 # via flake8 -pyppeteer==0.2.2 # via -r requirements.txt, requests-html -pyquery==1.4.1 # via -r requirements.txt, requests-html +pyexcel-io==0.6.4 # via -r requirements.txt, pyexcel-ods3 +pyexcel-ods3==0.6.0 # via -r requirements.txt +pyflakes==2.3.1 # via flake8 +pyppeteer==0.2.5 # via -r requirements.txt, requests-html +pyquery==1.4.3 # via -r requirements.txt, requests-html python-dateutil==2.8.1 # via -r requirements.txt, elasticsearch-dsl -python-dotenv==0.13.0 # via -r requirements.txt -pytz==2020.1 # via -r requirements.txt, django -pyyaml==5.3.1 # via -r requirements.txt -regex==2020.11.13 # via black -requests-cache==0.5.2 # via -r requirements.txt +python-dotenv==0.17.0 # via -r requirements.txt +pytz==2021.1 # via -r requirements.txt, django +pyyaml==5.4.1 # via -r requirements.txt +regex==2021.4.4 # via -r requirements.txt, black, titlecase +requests-cache==0.6.2 # via -r requirements.txt requests-html==0.10.0 # via -r requirements.txt requests-mock==1.8.0 # via -r dev-requirements.in -requests==2.24.0 # via -r requirements.txt, requests-cache, requests-html, requests-mock -six==1.15.0 # via -r requirements.txt, django-elasticsearch-dsl, django-prettyjson, elasticsearch-dsl, python-dateutil, requests-mock, validators, w3lib -soupsieve==2.0.1 # via -r requirements.txt, beautifulsoup4 -sqlparse==0.3.1 # via -r requirements.txt, django +requests==2.25.1 # via -r requirements.txt, requests-cache, requests-html, requests-mock +six==1.15.0 # via -r requirements.txt, django-elasticsearch-dsl, django-prettyjson, elasticsearch-dsl, python-dateutil, requests-mock, url-normalize, validators, w3lib +soupsieve==2.2.1 # via -r requirements.txt, beautifulsoup4 +sqlparse==0.4.1 # via -r requirements.txt, django standardjson==0.3.1 # via -r requirements.txt, django-prettyjson -titlecase==0.12.0 # via -r requirements.txt +titlecase==2.0.0 # via -r requirements.txt toml==0.10.2 # via black -tqdm==4.46.0 # via -r requirements.txt, pyppeteer -typed-ast==1.4.1 # via black +tqdm==4.60.0 # via -r requirements.txt, pyppeteer +typed-ast==1.4.3 # via black typing-extensions==3.7.4.3 # via black -urllib3==1.25.9 # via -r requirements.txt, elasticsearch, pyppeteer, requests -validators==0.18.1 # via -r requirements.txt +url-normalize==1.4.3 # via -r requirements.txt, requests-cache +urllib3==1.26.4 # via -r requirements.txt, elasticsearch, pyppeteer, requests +validators==0.18.2 # via -r requirements.txt w3lib==1.22.0 # via -r requirements.txt, requests-html websockets==8.1 # via -r requirements.txt, pyppeteer whitenoise==5.2.0 # via -r requirements.txt diff --git a/requirements.in b/requirements.in index d5d6a30e..041b8e2d 100644 --- a/requirements.in +++ b/requirements.in @@ -24,5 +24,5 @@ titlecase psycopg2 bcp-reader PyYAML -lxml==4.6.2 +lxml==4.6.3 pycountry \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 613444b6..87e6423f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,62 +5,64 @@ # pip-compile # appdirs==1.4.4 # via pyppeteer -asgiref==3.2.10 # via django +asgiref==3.3.4 # via django bcp-reader==0.1.1 # via -r requirements.in -beautifulsoup4==4.9.1 # via bs4 +beautifulsoup4==4.9.3 # via bs4 bs4==0.0.1 # via requests-html -certifi==2020.4.5.1 # via elasticsearch, requests -chardet==3.0.4 # via requests +certifi==2020.12.5 # via elasticsearch, requests +chardet==4.0.0 # via requests cssselect==1.1.0 # via pyquery -decorator==4.4.2 # via validators +decorator==5.0.7 # via validators dj-database-url==0.5.0 # via -r requirements.in -django-better-admin-arrayfield==1.4.0 # via -r requirements.in -django-cors-headers==3.5.0 # via -r requirements.in +django-better-admin-arrayfield==1.4.2 # via -r requirements.in +django-cors-headers==3.7.0 # via -r requirements.in django-elasticsearch-dsl==7.1.4 # via -r requirements.in django-environ==0.4.5 # via -r requirements.in django-filter==2.4.0 # via -r requirements.in -django-ninja==0.9.1 # via -r requirements.in +django-ninja==0.12.2 # via -r requirements.in django-prettyjson==0.4.1 # via -r requirements.in -django==3.1.1 # via -r requirements.in, django-cors-headers, django-filter, django-ninja, django-prettyjson +django==3.2 # via -r requirements.in, django-cors-headers, django-filter, django-ninja, django-prettyjson elasticsearch-dsl==7.3.0 # via django-elasticsearch-dsl -elasticsearch==7.10.0 # via elasticsearch-dsl +elasticsearch==7.12.0 # via elasticsearch-dsl et-xmlfile==1.0.1 # via openpyxl fake-useragent==0.1.11 # via requests-html -gunicorn==20.0.4 # via -r requirements.in -humanize==2.4.0 # via -r requirements.in -idna==2.9 # via requests -inflect==4.1.0 # via -r requirements.in -jdcal==1.4.1 # via openpyxl -jinja2==2.11.2 # via -r requirements.in -lml==0.0.9 # via pyexcel-io -lxml==4.6.2 # via -r requirements.in, pyexcel-ezodf, pyexcel-ods3, pyquery +gunicorn==20.1.0 # via -r requirements.in +humanize==3.4.1 # via -r requirements.in +idna==2.10 # via requests +inflect==5.3.0 # via -r requirements.in +itsdangerous==1.1.0 # via requests-cache +jinja2==2.11.3 # via -r requirements.in +lml==0.1.0 # via pyexcel-io +lxml==4.6.3 # via -r requirements.in, pyexcel-ezodf, pyexcel-ods3, pyquery markupsafe==1.1.1 # via jinja2 -openpyxl==3.0.3 # via -r requirements.in -parse==1.15.0 # via requests-html +openpyxl==3.0.7 # via -r requirements.in +parse==1.19.0 # via requests-html psycopg2==2.8.6 # via -r requirements.in pycountry==20.7.3 # via -r requirements.in pydantic==1.7.3 # via django-ninja -pyee==7.0.2 # via pyppeteer +pyee==8.1.0 # via pyppeteer pyexcel-ezodf==0.3.4 # via pyexcel-ods3 -pyexcel-io==0.5.20 # via pyexcel-ods3 -pyexcel-ods3==0.5.3 # via -r requirements.in -pyppeteer==0.2.2 # via requests-html -pyquery==1.4.1 # via requests-html +pyexcel-io==0.6.4 # via pyexcel-ods3 +pyexcel-ods3==0.6.0 # via -r requirements.in +pyppeteer==0.2.5 # via requests-html +pyquery==1.4.3 # via requests-html python-dateutil==2.8.1 # via elasticsearch-dsl -python-dotenv==0.13.0 # via -r requirements.in -pytz==2020.1 # via django -pyyaml==5.3.1 # via -r requirements.in -requests-cache==0.5.2 # via -r requirements.in +python-dotenv==0.17.0 # via -r requirements.in +pytz==2021.1 # via django +pyyaml==5.4.1 # via -r requirements.in +regex==2021.4.4 # via titlecase +requests-cache==0.6.2 # via -r requirements.in requests-html==0.10.0 # via -r requirements.in -requests==2.24.0 # via -r requirements.in, requests-cache, requests-html -six==1.15.0 # via django-elasticsearch-dsl, django-prettyjson, elasticsearch-dsl, python-dateutil, validators, w3lib -soupsieve==2.0.1 # via beautifulsoup4 -sqlparse==0.3.1 # via django +requests==2.25.1 # via -r requirements.in, requests-cache, requests-html +six==1.15.0 # via django-elasticsearch-dsl, django-prettyjson, elasticsearch-dsl, python-dateutil, url-normalize, validators, w3lib +soupsieve==2.2.1 # via beautifulsoup4 +sqlparse==0.4.1 # via django standardjson==0.3.1 # via django-prettyjson -titlecase==0.12.0 # via -r requirements.in -tqdm==4.46.0 # via -r requirements.in, pyppeteer -urllib3==1.25.9 # via elasticsearch, pyppeteer, requests -validators==0.18.1 # via -r requirements.in +titlecase==2.0.0 # via -r requirements.in +tqdm==4.60.0 # via -r requirements.in, pyppeteer +url-normalize==1.4.3 # via requests-cache +urllib3==1.26.4 # via elasticsearch, pyppeteer, requests +validators==0.18.2 # via -r requirements.in w3lib==1.22.0 # via requests-html websockets==8.1 # via pyppeteer whitenoise==5.2.0 # via -r requirements.in From 1a2cfc9ff57a417ce50317a65f0acb3791807c40 Mon Sep 17 00:00:00 2001 From: David Kane Date: Sat, 17 Apr 2021 09:35:21 +0100 Subject: [PATCH 17/19] change search box font - fixes issue #88 --- ftc/jinja2/components/search_form.html.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftc/jinja2/components/search_form.html.j2 b/ftc/jinja2/components/search_form.html.j2 index e96a7682..e765b126 100644 --- a/ftc/jinja2/components/search_form.html.j2 +++ b/ftc/jinja2/components/search_form.html.j2 @@ -2,14 +2,14 @@
    {% if not search %}