From eb42b2d6fc2a446e05daabdd3d0da2fde3ec3231 Mon Sep 17 00:00:00 2001 From: Struan Donald Date: Tue, 5 Sep 2023 13:44:40 +0100 Subject: [PATCH] negative points and evidence for national points Handle importing reponses that have negative points. Also adds evidence to national data --- .../fixtures/negative_audit_responses.json | 1 + crowdsourcer/fixtures/negative_questions.json | 2 +- .../commands/import_national_data.py | 343 ++++++++++++++++-- crowdsourcer/scoring.py | 6 + crowdsourcer/views/stats.py | 9 +- 5 files changed, 323 insertions(+), 38 deletions(-) diff --git a/crowdsourcer/fixtures/negative_audit_responses.json b/crowdsourcer/fixtures/negative_audit_responses.json index 21313a0b..176dc96f 100644 --- a/crowdsourcer/fixtures/negative_audit_responses.json +++ b/crowdsourcer/fixtures/negative_audit_responses.json @@ -17,6 +17,7 @@ "revision_notes": null, "created": "2023-03-15T17:22:10+0000", "last_update": "2023-03-15T17:22:10+0000", + "points": -2, "multi_option": [] } } diff --git a/crowdsourcer/fixtures/negative_questions.json b/crowdsourcer/fixtures/negative_questions.json index 0f272a1a..ae1be709 100644 --- a/crowdsourcer/fixtures/negative_questions.json +++ b/crowdsourcer/fixtures/negative_questions.json @@ -36,7 +36,7 @@ "pk": 204, "fields": { "question": 355, - "score": -2, + "score": 0, "description": "Yes", "ordering": 100 } diff --git a/crowdsourcer/management/commands/import_national_data.py b/crowdsourcer/management/commands/import_national_data.py index ac183e83..2811020e 100644 --- a/crowdsourcer/management/commands/import_national_data.py +++ b/crowdsourcer/management/commands/import_national_data.py @@ -49,11 +49,24 @@ class Command(BaseCommand): }, # XXX - negative points { - "sheet": "Planning Q11 - Fossil Fuel infr", + "sheet": "Planning Q11 Fossil Fuel infras", "section": "Planning & Land Use", "number": 11, - "default_if_missing": 0, + "default_if_missing": "No", "missing_filter": {"type__in": ["CTY", "UTA"]}, + "evidence": "Link to Evidence", + "evidence_detail": "Status (explanation)", + "negative": True, + "gss_col": "local authority code", + "score_col": "Score", + "type": "yes_no", + "points_map": { + "default": { + "-20%": -4.4, + "-0.2": -4.4, + "-6.0": -6, + }, + }, }, { "sheet": "Recycling", @@ -62,6 +75,7 @@ class Command(BaseCommand): "header_row": 6, "council_col": "Local Authority 2020/21", "score_col": "Mark", + "evidence": "Evidence", # XXX - tiered "type": "select_one", "options": [ @@ -79,6 +93,7 @@ class Command(BaseCommand): "header_row": 6, "council_col": "Local Authority 2020/21", "score_col": "Mark", + "evidence": "Source", # XXX - tiered "type": "select_one", "options": [ @@ -93,6 +108,7 @@ class Command(BaseCommand): "number": 4, "council_col": "Council name", "score_col": "Award point - only 1 tier", + "evidence": "Evidence link", "default_if_missing": 0, "missing_filter": {"type__in": ["CTY", "UTA"], "country": "england"}, "type": "yes_no", @@ -104,6 +120,7 @@ class Command(BaseCommand): "header_row": 1, "council_col": "Local Authority", "score_col": "Unweighted scores", + "evidence": "Evidence link", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -121,6 +138,7 @@ class Command(BaseCommand): "header_row": 2, "council_col": "Local Authority", "score_col": "Unweighted Question scores", + "evidence": "Evidence link", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -135,6 +153,7 @@ class Command(BaseCommand): "header_row": 1, "council_col": "Local Authority / Region Name", "score_col": "Unweighted Question Scores", + "evidence": "Evidence link", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -154,6 +173,39 @@ class Command(BaseCommand): "section": "Transport", "number": 12, "number_part": "a", + "header_row": 1, + "council_col": "Local authority name", + "score_col": "Award negative points", + "negative": True, + "type": "select_one", + "options": [ + { + "desc": "None", + "score": 0, + }, + { + "desc": "-2%", + "score": 0, + }, + { + "desc": "-6%", + "score": 0, + }, + ], + "points_map": { + "UTA": { + "-2%": -0.5, + "-6%": -1.5, + }, + "CTY": { + "-2%": -0.5, + "-6%": -1.5, + }, + "default": { + "-2%": -0.16, + "-6%": -0.48, + }, + }, }, # XXX - negative points { @@ -161,6 +213,71 @@ class Command(BaseCommand): "section": "Transport", "number": 12, "number_part": "b", + "header_row": 1, + "council_col": "Local authority name", + "score_col": "Award negative points", + "negative": True, + "type": "select_one", + "options": [ + { + "desc": "None", + "score": 0, + }, + { + "desc": "-2%", + "score": 0, + }, + { + "desc": "-4%", + "score": 0, + }, + ], + "points_map": { + "UTA": { + "-2%": -0.5, + "-4%": -1, + }, + "CTY": { + "-2%": -0.5, + "-4%": -1, + }, + "default": { + "-2%": -0.16, + "-4%": -0.32, + }, + }, + }, + { + "sheet": "Transport Q11 (negative)", + "section": "Transport", + "number": 11, + "number_part": None, + "council_col": "authority", + "score_col": "score", + "negative": True, + "skip_clear_existing": False, + "update_points_only": True, + "type": "multi_select", + "points_map": { + "UTA": { + "-": 0, + "-0.05": -1.25, + "-0.15": -3.75, + "-0.2": -5, + }, + "CTY": { + "-": 0, + "-0.05": -1.25, + "-0.15": -3.75, + "-0.2": -5, + }, + "default": { + "-": 0, + "-0.05": -0.4, + "-0.15": -1.2, + "-0.2": -1.6, + }, + }, }, { "sheet": "Biodiversity Q2 - Pesticides", @@ -170,6 +287,8 @@ class Command(BaseCommand): "council_col": "Council", "score_col": "Unweighted points", "default_if_missing": 0, + "evidence": "Link to data", + "evidence_detail": "Banned Pesticides?", "type": "yes_no", }, { @@ -179,16 +298,18 @@ class Command(BaseCommand): "header_row": 2, "council_col": "Council", "score_col": "Point awarded", + "evidence": "Evidence link", "type": "yes_no", }, { - "sheet": "Sheet20", + "sheet": "Sheet23", "section": "Biodiversity", "number": 7, "council_col": "local authority council code", "score_col": "Unweighted score", "type": "select_one", "default_if_missing": 0, + "evidence": "Evidence link", "options": [ {"desc": "Criteria not met", "score": 0}, {"desc": "1-3 Green Flag accredited parks", "score": 1}, @@ -202,6 +323,7 @@ class Command(BaseCommand): "number_part": "a", "council_col": "Council", "score_col": "Score", + "evidence": "Evidence", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -216,6 +338,7 @@ class Command(BaseCommand): "number_part": "b", "council_col": "Council", "score_col": "Score", + "evidence": "Evidence", "default_if_missing": 0, "type": "select_one", "options": [ @@ -232,6 +355,7 @@ class Command(BaseCommand): "gss_col": "Local Authority Code", "score_col": "Score", "skip_check": {"col": "Calendar Year", "val": 2019}, + "evidence_link": "https://www.gov.uk/government/statistics/uk-local-authority-and-regional-greenhouse-gas-emissions-national-statistics-2005-to-2021", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -246,6 +370,7 @@ class Command(BaseCommand): "number": 5, "header_row": 6, "gss_col": "local-authority-code", + "evidence_link": "https://www.gov.uk/government/statistics/uk-local-authority-and-regional-greenhouse-gas-emissions-national-statistics-2005-to-2021", "score_col": "Score", "type": "select_one", "options": [ @@ -262,6 +387,7 @@ class Command(BaseCommand): "gss_col": "local-authority-code", "score_col": "Score", "type": "select_one", + "skip_clear_existing": False, "options": [ {"desc": "Criteria not met", "score": 0}, {"desc": "more than 0.5% of staff working on climate", "score": 1}, @@ -277,6 +403,7 @@ class Command(BaseCommand): "gss_col": "local-authority-code", "score_col": "Score", "type": "select_one", + "skip_clear_existing": False, "options": [ {"desc": "Criteria not met", "score": 0}, {"desc": "more than 0.5% of staff working on climate", "score": 1}, @@ -284,6 +411,25 @@ class Command(BaseCommand): {"desc": "more than 2% of staff working on climate", "score": 3}, ], }, + { + "sheet": "Gov&Fin Q12 (negative)", + "section": "Governance & Finance", + "number": 12, + "number_part": None, + "negative": True, + "gss_col": "local-authority-code", + "score_col": "score", + "type": "yes_no", + "update_points_only": True, + "skip_clear_existing": False, + "update": True, + "points_map": { + "default": { + "-15": -4.35, + "-0.15": -4.35, + }, + }, + }, { "sheet": "EPC - England & Wales", "section": "Buildings & Heating", @@ -291,6 +437,7 @@ class Command(BaseCommand): "header_row": 5, "gss_col": "Local Authority Code", "score_col": "Tiered mark", + "evidence": "Evidence", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -306,6 +453,7 @@ class Command(BaseCommand): "header_row": 5, "council_col": "Local authority", "score_col": "Tiered mark", + "evidence": "Evidence", "type": "select_one", "skip_clear_existing": True, "options": [ @@ -322,6 +470,7 @@ class Command(BaseCommand): "header_row": 2, "gss_col": "Northern Irish Council GSS code", "score_col": "Tiered mark", + "evidence": "Evidence", "type": "select_one", "skip_clear_existing": True, "options": [ @@ -338,6 +487,7 @@ class Command(BaseCommand): "header_row": 1, "council_col": "Council", "score_col": "Unweighted Points", + "evidence": "Evidence link", "type": "select_one", "default_if_missing": 0, "options": [ @@ -359,6 +509,7 @@ class Command(BaseCommand): "council_col": "Council", "score_col": "Score", "type": "select_one", + "evidence": "Evidence", "options": [ {"desc": "Criteria not met", "score": 0}, {"desc": "Divestment of council's investments", "score": 1}, @@ -372,6 +523,7 @@ class Command(BaseCommand): "number_part": "b", "council_col": "Council", "score_col": "Score", + "evidence": "Evidence", "default_if_missing": 0, "type": "select_one", "options": [ @@ -387,6 +539,7 @@ class Command(BaseCommand): "header_row": 1, "council_col": "Local Authority", "score_col": "Unweighted scores", + "evidence": "Evidence link", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -404,6 +557,7 @@ class Command(BaseCommand): "header_row": 2, "council_col": "Local Authority", "score_col": "Unweighted Question scores", + "evidence": "Evidence link", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -418,6 +572,7 @@ class Command(BaseCommand): "header_row": 1, "council_col": "Local Authority / Region Name", "score_col": "Unweighted Question Scores", + "evidence": "Evidence link", "type": "select_one", "options": [ {"desc": "Criteria not met", "score": 0}, @@ -431,6 +586,38 @@ class Command(BaseCommand): }, ], }, + { + "sheet": "Transport 1b CA (negative)", + "section": "Transport (CA)", + "number": 1, + "number_part": "b", + "council_col": "authority", + "score_col": "score", + "negative": True, + "update_points_only": True, + "skip_clear_existing": False, + "type": "select_one", + "options": [ + { + "desc": "None", + "score": 0, + }, + { + "desc": "-5%", + "score": 0, + }, + { + "desc": "-20%", + "score": 0, + }, + ], + "points_map": { + "default": { + "-0.05": -1.05, + "-0.2": -3.15, + }, + }, + }, ] def add_arguments(self, parser): @@ -438,6 +625,11 @@ def add_arguments(self, parser): "-q", "--quiet", action="store_true", help="Silence debug data." ) parser.add_argument("--only_sheet", help="only process this sheet") + parser.add_argument( + "--negative_only", + action="store_true", + help="only process Qs with negative points", + ) def add_options(self, q, details): if details.get("type", None) is not None: @@ -500,7 +692,7 @@ def get_question(self, details): return q - def get_score(self, q, row, details): + def get_score(self, q, row, details, authority): q_type = details.get("type", "") score = row[details["score_col"]] @@ -508,8 +700,27 @@ def get_score(self, q, row, details): match = re.match(r"\"?(\d) out of \d", score) if match: score = int(match.group(1)) - - if q_type == "yes_no": + match = re.match(r".*-(\d%).*", score) + if match: + score = match.group(1) + + if details.get("negative", False): + if score != 0: + desc = score + if type(score) is str and score.find("-") == -1: + desc = f"-{score}" + points_map = details["points_map"].get( + authority.type, details["points_map"]["default"] + ) + score = points_map[f"{desc}"] + if details["type"] == "yes_no": + desc = "Yes" + else: + if details["type"] == "yes_no": + desc = "No" + else: + desc = "None" + elif q_type == "yes_no": if score == "Yes": desc = "Yes" score = 1 @@ -579,8 +790,8 @@ def import_answers(self, user, rt, df, q, details): if authority.questiongroup not in q.questiongroup.all(): continue - score_desc, score = self.get_score(q, row, details) - # self.print_info(authority.name, score) + score_desc, score = self.get_score(q, row, details, authority) + # self.print_info(f"{authority.name}, {score_desc}, {score}", 1) if not isinstance(score, numbers.Number) or math.isnan(score): self.print_info( @@ -597,37 +808,78 @@ def import_answers(self, user, rt, df, q, details): ) option = None - try: - option = Option.objects.get(question=q, description=score_desc) - except Option.DoesNotExist: - self.print_info( - f"No option found for {q.number}, {score_desc}, {authority.name}", - 1, - ) - except Option.MultipleObjectsReturned: - self.print_info( - f"Multiple options returned for score {q.number}, {score_desc}", - 1, + if not details.get("update_points_only", False): + try: + option = Option.objects.get(question=q, description=score_desc) + except Option.DoesNotExist: + self.print_info( + f"No option found for {q.number}, {score_desc}, {authority.name}", + 1, + ) + except Option.MultipleObjectsReturned: + self.print_info( + f"Multiple options returned for score {q.number}, {score_desc}", + 1, + ) + except ValueError: + self.print_info(f"Bad score: {score_desc}", 1) + + if option is None: + continue + + if not self.quiet: + self.print_info(f"{authority.name}: {option}") + + if details.get("update_points_only", False): + try: + r = Response.objects.get( + question=q, + authority=authority, + response_type=rt, + ) + except Response.DoesNotExist: + self.print_info( + f"{YELLOW}No matching response for {q.number}, {authority.name}{NOBOLD}", + 1, + ) + continue + else: + r, _ = Response.objects.update_or_create( + question=q, + authority=authority, + user=user, + response_type=rt, + defaults={"option": option}, ) - except ValueError: - self.print_info(f"Bad score: {score_desc}", 1) + if score != 0 and details.get("evidence", None) is not None: + r.public_notes = row[details["evidence"]] + r.page_number = 0 + r.save() - if option is None: - continue + if score != 0 and details.get("evidence_link", None) is not None: + r.public_notes = details["evidence_link"] + r.page_number = 0 + r.save() - if not self.quiet: - self.print_info(f"{authority.name}: {option}") + if score != 0 and details.get("evidence_detail", None) is not None: + r.evidence = row[details["evidence_detail"]] + r.save() + + if score != 0 and details.get("evidence_text", None) is not None: + r.evidence = details["evidence_text"] + r.save() + + if details.get("negative", False): + r.points = score + r.save() - r, _ = Response.objects.update_or_create( - question=q, - authority=authority, - user=user, - response_type=rt, - defaults={"option": option}, - ) count += 1 if details.get("default_if_missing", None) is not None: - option = Option.objects.get(question=q, score=0) + default = details["default_if_missing"] + if type(default) is int: + option = Option.objects.get(question=q, score=default) + else: + option = Option.objects.get(question=q, description=default) groups = q.questiongroup.all() answered = Response.objects.filter(response_type=rt, question=q).values( "authority" @@ -648,7 +900,7 @@ def import_answers(self, user, rt, df, q, details): ) auto_zero += 1 - message = f"Added {count} responses, {auto_zero} default 0 responses, bad authorities {bad_authority_count}" + message = f"{GREEN}Added {count} responses, {auto_zero} default 0 responses, bad authorities {bad_authority_count}{NOBOLD}" if count == 0: message = f"{YELLOW}{message}{NOBOLD}" @@ -657,6 +909,7 @@ def import_answers(self, user, rt, df, q, details): def handle_sheet(self, sheet, details, user): self.print_info("") self.print_info("--", 1) + self.print_info(sheet, 1) self.print_info( f"{details['section']}, {details['number']}, {details.get('number_part', '')}", 1, @@ -664,6 +917,13 @@ def handle_sheet(self, sheet, details, user): self.print_info("") df = self.get_df(sheet, details) q = self.get_question(details) + if q is None: + self.print_info( + f"{RED}Could not find questions {details['section']}, {details['number']}, {details.get('number_part', '')}{NOBOLD}", + 1, + ) + return + self.clear_existing_answers(q, details) self.add_options(q, details) self.import_answers(user, self.rt, df, q, details) @@ -674,7 +934,14 @@ def print_info(self, message, level=2): print(message) - def handle(self, quiet: bool = False, only_sheet: str = None, *args, **kwargs): + def handle( + self, + quiet: bool = False, + only_sheet: str = None, + negative_only: bool = False, + *args, + **kwargs, + ): self.quiet = quiet self.popuplate_council_lookup() @@ -685,6 +952,9 @@ def handle(self, quiet: bool = False, only_sheet: str = None, *args, **kwargs): if only_sheet is not None and sheet != only_sheet: continue + if negative_only and not details.get("negative", False): + continue + self.handle_sheet(sheet, details, user) for details in self.ca_sheets: @@ -692,4 +962,7 @@ def handle(self, quiet: bool = False, only_sheet: str = None, *args, **kwargs): if only_sheet is not None and sheet != only_sheet: continue + if negative_only and not details.get("negative", False): + continue + self.handle_sheet(sheet, details, user) diff --git a/crowdsourcer/scoring.py b/crowdsourcer/scoring.py index 2333de6a..ced45ce7 100644 --- a/crowdsourcer/scoring.py +++ b/crowdsourcer/scoring.py @@ -338,6 +338,7 @@ def get_section_scores(scoring): options.select_related("questions") .annotate(score=Sum("score")) .values( + "points", "score", "authority__name", "question__number", @@ -390,6 +391,11 @@ def get_section_scores(scoring): continue q_score = score["score"] + if scoring["negative_q"][section.title].get(q, None) is not None: + if score["points"] is not None: + q_score = score["points"] + else: + q_score = 0 if ( SCORE_EXCEPTIONS.get(section.title, None) is not None and SCORE_EXCEPTIONS[section.title].get(q, None) is not None diff --git a/crowdsourcer/views/stats.py b/crowdsourcer/views/stats.py index a198c023..9fefd597 100644 --- a/crowdsourcer/views/stats.py +++ b/crowdsourcer/views/stats.py @@ -45,7 +45,9 @@ def get_queryset(self): def get_response_score(self, response): score = 0 - if response.multi_count > 0: + if response.question.question_type == "negative": + score = response.points + elif response.multi_count > 0: for opt in response.multi_option.all(): score += opt.score elif response.option is not None: @@ -256,7 +258,10 @@ def get_response_data(self, response): score = 0 answer = "" - if response.multi_count > 0: + if response.question.question_type == "negative": + answer = response.option.description + score = response.points + elif response.multi_count > 0: descs = [] for opt in response.multi_option.all(): descs.append(opt.description)