diff --git a/tests/integration/test_current_cases.py b/tests/integration/test_current_cases.py new file mode 100644 index 0000000000..5e90955208 --- /dev/null +++ b/tests/integration/test_current_cases.py @@ -0,0 +1,350 @@ +import subprocess +from unittest.mock import patch +from datetime import datetime +from decimal import Decimal +import pytest + + +from webservices import rest +from webservices.legal_docs.current_cases import get_cases + +from tests.common import TEST_CONN, BaseTestCase + +@pytest.mark.usefixtures("migrate_db") +class TestLoadCurrentMURs(BaseTestCase): + + case_type = 'MUR' + + def setUp(self): + self.connection = rest.db.engine.connect() + subprocess.check_call( + ['psql', TEST_CONN, '-f', 'data/load_base_mur_data.sql']) + + def tearDown(self): + self.clear_test_data() + self.connection.close() + rest.db.session.remove() + + @patch('webservices.legal_docs.current_cases.get_bucket') + def test_simple_mur(self, get_bucket): + mur_subject = 'Fraudulent misrepresentation' + expected_mur = { + 'no': '1', + 'name': 'Simple MUR', + 'mur_type': 'current', + 'election_cycles': [2016], + 'doc_id': 'mur_1', + 'participants': [], + 'subjects': [mur_subject], + 'respondents': [], + 'documents': [], + 'commission_votes': [], + 'dispositions': [], + 'close_date': None, + 'open_date': None, + 'url': '/legal/matter-under-review/1/', + 'sort1': -1, + 'sort2': None + } + self.create_mur(1, expected_mur['no'], expected_mur['name'], mur_subject) + actual_mur = next(get_cases(self.case_type)) + + assert actual_mur == expected_mur + + @patch('webservices.env.env.get_credential', return_value='BUCKET_NAME') + @patch('webservices.legal_docs.current_cases.get_bucket') + def test_mur_with_participants_and_documents(self, get_bucket, get_credential): + case_id = 1 + mur_subject = 'Fraudulent misrepresentation' + expected_mur = { + 'no': '1', + 'name': 'MUR with participants', + 'mur_type': 'current', + 'election_cycles': [2016], + 'doc_id': 'mur_1', + 'subjects': [mur_subject], + 'respondents': ["Bilbo Baggins", "Thorin Oakenshield"] + } + participants = [ + ("Complainant", "Gollum"), + ("Respondent", "Bilbo Baggins"), + ("Respondent", "Thorin Oakenshield") + ] + filename = "Some File.pdf" + documents = [ + ('A Category', 'Some text', 'legal/murs/{0}/{1}'.format('1', + filename.replace(' ', '-'))), + ('Another Category', 'Different text', 'legal/murs/{0}/{1}'.format('1', + filename.replace(' ', '-'))), + ] + + self.create_mur(case_id, expected_mur['no'], expected_mur['name'], mur_subject) + for entity_id, participant in enumerate(participants): + role, name = participant + self.create_participant(case_id, entity_id, role, name) + for document_id, document in enumerate(documents): + category, ocrtext, url = document + self.create_document(case_id, document_id, category, ocrtext, filename) + + actual_mur = next(get_cases(self.case_type)) + + for key in expected_mur: + assert actual_mur[key] == expected_mur[key] + + assert participants == [(p['role'], p['name']) + for p in actual_mur['participants']] + + assert [(d[0], d[1], len(d[1])) for d in documents] == [ + (d['category'], d['text'], d['length']) for d in actual_mur['documents']] + + @patch('webservices.env.env.get_credential', return_value='BUCKET_NAME') + @patch('webservices.legal_docs.current_cases.get_bucket') + def test_mur_with_disposition(self, get_bucket, get_credential): + case_id = 1 + case_no = '1' + name = 'Open Elections LLC' + mur_subject = 'Fraudulent misrepresentation' + pg_date = '2016-10-08' + self.create_mur(case_id, case_no, name, mur_subject) + + entity_id = 1 + event_date = '2005-01-01' + event_id = 1 + self.create_calendar_event(entity_id, event_date, event_id, case_id) + + entity_id = 1 + event_date = '2008-01-01' + event_id = 2 + self.create_calendar_event(entity_id, event_date, event_id, case_id) + + parent_event = 0 + event_name = 'Conciliation-PPC' + path = '' + is_key_date = 0 + check_primary_respondent = 0 + pg_date = '2016-01-01' + self.create_event(event_id, parent_event, event_name, path, is_key_date, + check_primary_respondent, pg_date) + + first_name = "Commander" + last_name = "Data" + middle_name, prefix, suffix, type = ('', '', '', '') + self.create_entity(entity_id, first_name, last_name, middle_name, prefix, suffix, type, name, pg_date) + + master_key = 1 + detail_key = 1 + relation_id = 1 + self.create_relatedobjects(master_key, detail_key, relation_id) + + settlement_id = 1 + initial_amount = 0 + final_amount = 50000 + amount_received, settlement_type = (0, '') + self.create_settlement(settlement_id, case_id, initial_amount, final_amount, + amount_received, settlement_type, pg_date) + + stage = 'Closed' + statutory_citation = '431' + regulatory_citation = '456' + self.create_violation(case_id, entity_id, stage, statutory_citation, regulatory_citation) + + commission_id = 1 + agenda_date = event_date + vote_date = event_date + action = 'Conciliation Reached.' + self.create_commission(commission_id, agenda_date, vote_date, action, case_id, pg_date) + + actual_mur = next(get_cases(self.case_type)) + + expected_mur = { + 'commission_votes': [{'action': 'Conciliation Reached.', 'vote_date': datetime(2008, 1, 1, 0, 0)}], + 'dispositions': [{ + 'disposition': 'Conciliation-PPC', + 'respondent': 'Open Elections LLC', 'penalty': Decimal('50000.00'), + 'citations': [ + {'text': '431', + 'title': '2', + 'type': 'statute', + 'url': 'https://api.fdsys.gov/link?collection=uscode&year=mostrecent&link-type=html&title=52' + '§ion=30101'}, + {'text': '456', + 'title': '11', + 'type': 'regulation', + 'url': '/regulations/456/CURRENT'} + ] + }], + 'subjects': ['Fraudulent misrepresentation'], + 'respondents': [], + 'documents': [], 'participants': [], 'no': '1', 'doc_id': 'mur_1', + 'mur_type': 'current', 'name': 'Open Elections LLC', 'open_date': datetime(2005, 1, 1, 0, 0), + 'election_cycles': [2016], + 'close_date': datetime(2008, 1, 1, 0, 0), + 'url': '/legal/matter-under-review/1/', + 'sort1': -1, + 'sort2': None + } + assert actual_mur == expected_mur + + @patch('webservices.legal_docs.current_cases.get_bucket') + def test_mur_offsets(self, get_bucket): + mur_subject = 'Fraudulent misrepresentation' + expected_mur1 = { + 'no': '1', + 'name': 'Simple MUR1', + 'mur_type': 'current', + 'election_cycles': [2016], + 'doc_id': 'mur_1', + 'participants': [], + 'subjects': [mur_subject], + 'respondents': [], + 'documents': [], + 'commission_votes': [], + 'dispositions': [], + 'close_date': None, + 'open_date': None, + 'url': '/legal/matter-under-review/1/', + 'sort1': -1, + 'sort2': None + } + expected_mur2 = { + 'no': '2', + 'name': 'Simple MUR2', + 'mur_type': 'current', + 'election_cycles': [2016], + 'doc_id': 'mur_2', + 'participants': [], + 'subjects': [mur_subject], + 'respondents': [], + 'documents': [], + 'commission_votes': [], + 'dispositions': [], + 'close_date': None, + 'open_date': None, + 'url': '/legal/matter-under-review/2/', + 'sort1': -2, + 'sort2': None + } + expected_mur3 = { + 'no': '3', + 'name': 'Simple MUR', + 'mur_type': 'current', + 'election_cycles': [2016], + 'doc_id': 'mur_3', + 'participants': [], + 'subjects': [mur_subject], + 'respondents': [], + 'documents': [], + 'commission_votes': [], + 'dispositions': [], + 'close_date': None, + 'open_date': None, + 'url': '/legal/matter-under-review/3/', + 'sort1': -3, + 'sort2': None + } + self.create_mur(1, expected_mur1['no'], expected_mur1['name'], mur_subject) + self.create_mur(2, expected_mur2['no'], expected_mur2['name'], mur_subject) + self.create_mur(3, expected_mur3['no'], expected_mur3['name'], mur_subject) + + gen = get_cases(self.case_type) + assert(next(gen)) == expected_mur1 + assert(next(gen)) == expected_mur2 + assert(next(gen)) == expected_mur3 + + actual_murs = [mur for mur in get_cases(self.case_type, '2')] + assert actual_murs == [expected_mur2] + + def create_mur(self, case_id, case_no, name, subject_description): + subject_id = self.connection.execute( + "SELECT subject_id FROM fecmur.subject " + " WHERE description = %s ", subject_description).scalar() + self.connection.execute( + "INSERT INTO fecmur.case (case_id, case_no, name, case_type) " + "VALUES (%s, %s, %s, 'MUR')", case_id, case_no, name) + self.connection.execute( + "INSERT INTO fecmur.case_subject (case_id, subject_id, relatedsubject_id) " + "VALUES (%s, %s, -1)", case_id, subject_id) + self.connection.execute( + "INSERT INTO fecmur.electioncycle (case_id, election_cycle) " + "VALUES (%s, 2016)", case_id) + + def create_participant(self, case_id, entity_id, role, name, + stage=None, statutory_citation=None, regulatory_citation=None): + role_id = self.connection.execute( + "SELECT role_id FROM fecmur.role " + " WHERE description = %s ", role).scalar() + self.connection.execute( + "INSERT INTO fecmur.entity (entity_id, name) " + "VALUES (%s, %s)", entity_id, name) + self.connection.execute( + "INSERT INTO fecmur.players (player_id, entity_id, case_id, role_id) " + "VALUES (%s, %s, %s, %s)", entity_id, entity_id, case_id, role_id) + if stage: + self.create_violation(case_id, entity_id, stage, statutory_citation, regulatory_citation) + + def create_violation(self, case_id, entity_id, stage, statutory_citation, regulatory_citation): + self.connection.execute( + "INSERT INTO fecmur.violations (case_id, entity_id, stage, statutory_citation, regulatory_citation) " + "VALUES (%s, %s, %s, %s, %s)", case_id, entity_id, stage, statutory_citation, regulatory_citation) + + def create_document(self, case_id, document_id, category, ocrtext, filename='129812.pdf'): + self.connection.execute( + "INSERT INTO fecmur.document (document_id, doc_order_id, case_id, category, ocrtext, fileimage, filename) " + "VALUES (%s, %s, %s, %s, %s, %s, %s)", + document_id, document_id, case_id, category, ocrtext, ocrtext, filename) + + def create_calendar_event(self, entity_id, event_date, event_id, case_id): + self.connection.execute( + "INSERT INTO fecmur.calendar (entity_id, event_date, event_id, case_id) " + "VALUES (%s, %s, %s, %s)", entity_id, event_date, event_id, case_id) + + def create_entity(self, entity_id, first_name, last_name, middle_name, prefix, suffix, type, name, pg_date): + self.connection.execute( + "INSERT INTO fecmur.entity (entity_id, first_name, last_name, middle_name, " + "prefix, suffix, type, name, pg_date) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", entity_id, first_name, + last_name, middle_name, prefix, suffix, type, name, pg_date) + + def create_event(self, event_id, parent_event, event_name, path, is_key_date, check_primary_respondent, pg_date): + self.connection.execute( + "INSERT INTO fecmur.event (event_id, parent_event, event_name, path, is_key_date, " + "check_primary_respondent, pg_date) " + "VALUES (%s, %s, %s, %s, %s, %s, %s)", event_id, parent_event, + event_name, path, is_key_date, check_primary_respondent, pg_date) + + def create_relatedobjects(self, master_key, detail_key, relation_id): + self.connection.execute( + "INSERT INTO fecmur.relatedobjects (master_key, detail_key, relation_id) " + "VALUES (%s, %s, %s)", master_key, detail_key, relation_id) + + def create_settlement(self, settlement_id, case_id, initial_amount, final_amount, + amount_received, settlement_type, pg_date): + self.connection.execute( + "INSERT INTO fecmur.settlement (settlement_id, case_id, initial_amount, " + "final_amount, amount_received, settlement_type, pg_date) " + "VALUES (%s, %s, %s, %s, %s, %s, %s)", settlement_id, case_id, initial_amount, final_amount, + amount_received, settlement_type, pg_date) + + def create_commission(self, commission_id, agenda_date, vote_date, action, case_id, pg_date): + self.connection.execute( + "INSERT INTO fecmur.commission (commission_id, agenda_date, vote_date, action, case_id, pg_date) " + "VALUES (%s, %s, %s, %s, %s, %s)", commission_id, agenda_date, vote_date, action, case_id, pg_date) + + def clear_test_data(self): + tables = [ + "violations", + "document", + "players", + "entity", + "case_subject", + "electioncycle", + "case", + "calendar", + "settlement", + "event", + "commission", + "subject", + "role" + ] + for table in tables: + self.connection.execute("DELETE FROM fecmur.{}".format(table)) diff --git a/webservices/legal_docs/current_cases.py b/webservices/legal_docs/current_cases.py index c00f2feeb2..100dd76ea5 100644 --- a/webservices/legal_docs/current_cases.py +++ b/webservices/legal_docs/current_cases.py @@ -30,8 +30,8 @@ name, case_type FROM fecmur.cases_with_parsed_case_serial_numbers_vw - WHERE case_no = %s - AND case_type = %s + WHERE case_type = %s + AND case_no = %s """ AF_SPECIFIC_FIELDS = """ @@ -155,51 +155,54 @@ def load_current_murs(specific_mur_no=None): - load_cases(specific_mur_no, 'MUR') + load_cases('MUR', specific_mur_no) def load_adrs(specific_adr_no=None): - load_cases(specific_adr_no, 'ADR') + load_cases('ADR', specific_adr_no) def load_admin_fines(specific_af_no=None): - load_cases(specific_af_no, 'AF') + load_cases('AF', specific_af_no) def get_es_type(case_type): - if case_type.upper() == 'AF': + case_type = case_type.upper() + if case_type == 'AF': return 'admin_fines' - elif case_type.upper() == 'ADR': + elif case_type == 'ADR': return 'adrs' else: return 'murs' def get_full_name(case_type): - if case_type.upper() == 'AF': + case_type = case_type.upper() + if case_type == 'AF': return 'administrative-fine' - elif case_type.upper() == 'ADR': + elif case_type == 'ADR': return 'alternative-dispute-resolution' else: return 'matter-under-review' -#TODO: How to handle if there's no case_type specified? -def load_cases(case_no=None, case_type=None): +def load_cases(case_type, case_no=None): """ Reads data for current MURs, AFs, and ADRs from a Postgres database, assembles a JSON document corresponding to the case, and indexes this document in Elasticsearch in the index `docs_index` with doc_type corresponding to case_type. In addition, all documents attached to the case are uploaded to an S3 bucket under the _directory_ `legal///`. """ - es = get_elasticsearch_connection() - logger.info("Loading {0}(s)".format(case_type)) - case_count = 0 - for case in get_cases(case_no, case_type): - if case is not None: - logger.info("Loading {0}: {1}".format(case_type, case['no'])) - es.index('docs_index', get_es_type(case_type), case, id=case['doc_id']) - case_count += 1 - logger.info("{0} {1}(s) loaded".format(case_count, case_type)) - - -def get_cases(case_no=None, case_type=None): + if case_type in ('MUR', 'ADR', 'AF'): + es = get_elasticsearch_connection() + logger.info("Loading {0}(s)".format(case_type)) + case_count = 0 + for case in get_cases(case_type, case_no): + if case is not None: + logger.info("Loading {0}: {1}".format(case_type, case['no'])) + es.index('docs_index', get_es_type(case_type), case, id=case['doc_id']) + case_count += 1 + logger.info("{0} {1}(s) loaded".format(case_count, case_type)) + else: + logger.error("Invalid case_type: must be 'MUR', 'ADR', or 'AF'.") + +def get_cases(case_type, case_no=None): """ Takes a specific case to load. If none are specified, all cases are reloaded @@ -209,17 +212,17 @@ def get_cases(case_no=None, case_type=None): with db.engine.connect() as conn: rs = conn.execute(ALL_CASES, case_type) for row in rs: - yield get_single_case(row['case_no'], case_type) + yield get_single_case(case_type, row['case_no']) else: - yield get_single_case(case_no, case_type) + yield get_single_case(case_type, case_no) -def get_single_case(case_no, case_type): +def get_single_case(case_type, case_no): bucket = get_bucket() bucket_name = env.get_credential('bucket') with db.engine.connect() as conn: - rs = conn.execute(SINGLE_CASE, case_no, case_type) + rs = conn.execute(SINGLE_CASE, case_type, case_no) row = rs.first() if row is not None: case_id = row['case_id'] @@ -231,7 +234,7 @@ def get_single_case(case_no, case_type): 'sort1': sort1, 'sort2': sort2, } - case['commission_votes'] = get_commission_votes(case_id, case_type) + case['commission_votes'] = get_commission_votes(case_type, case_id) case['documents'] = get_documents(case_id, bucket, bucket_name) case['url'] = '/legal/{0}/{1}/'.format(get_full_name(case_type), row['case_no']) if case_type == 'AF': @@ -305,7 +308,7 @@ def get_dispositions(case_id): return disposition_data -def get_commission_votes(case_id, case_type): +def get_commission_votes(case_type, case_id): with db.engine.connect() as conn: if case_type == 'AF': rs = conn.execute(AF_COMMISSION_VOTES, case_id)