Add ADRs and AFs to API

README.md

@@ -585,16 +585,25 @@ This command requires that the environment variable `FEC_EREGS_API` is set to th
 python manage.py load_advisory_opinions [-f FROM_AO_NO]
 ```
 
-#### Loading current MURs [only one MUR_NO]]
+#### Loading current MURs [only one MUR_NO]
 ```
-python manage.py load_current_murs [-m MUR_NO]
+python manage.py load_current_murs [-s MUR_NO]
 ```
 
 #### Loading archived MURs (This takes a very long time)
 ```
 python manage.py load_archived_murs [-s MUR_NO] or [-f FROM_MUR_NO]
 ```
 
+#### Loading ADRs [only one ADR_NO]
+```
+python manage.py load_adrs [-s ADR_NO]
+```
+
+#### Loading Admin Fines [only one AF_NO]
+```
+python manage.py load_admin_fines [-s AF_NO]
+```
 
 #### Reloading all current legal documents with no downtime (excludes archived MURs)
 ```

data/migrations/V0105__add_adr_af.sql

@@ -0,0 +1,23 @@
+/*
+Issue #3359
+Remove case_type = 'MUR' so we can include ADR and AF
+*/
+
+SET search_path = fecmur, pg_catalog;
+
+CREATE OR REPLACE VIEW cases_with_parsed_case_serial_numbers_vw AS
+    SELECT "case".case_id,
+        "case".case_no,
+        (regexp_replace(("case".case_no)::text, '(\d+).*'::text, '\1'::text))::integer AS case_serial,
+        "case".name,
+        "case".case_type,
+        "case".pg_date
+    FROM "case"
+    WHERE case_type IN ('MUR', 'AF', 'ADR');
+
+ALTER TABLE cases_with_parsed_case_serial_numbers_vw OWNER TO fec;
+GRANT ALL ON TABLE cases_with_parsed_case_serial_numbers_vw TO fec;
+GRANT SELECT ON TABLE cases_with_parsed_case_serial_numbers_vw TO fec_read;
+GRANT SELECT ON TABLE cases_with_parsed_case_serial_numbers_vw TO openfec_read;
+
+DROP VIEW cases_with_parsed_case_serial_numbers;

locustfile.py

@@ -194,14 +194,28 @@ def load_legal_documents_search(self, term=None):
         self.client.get('legal/search', name='legal_search', params=params)
 
     @locust.task
-    def get_mur(self, term=None):
+    def get_mur(self):
         params = {
             'api_key': API_KEY,
         }
         self.client.get('legal/docs/murs/7074', name='legal_get_mur', params=params)
 
     @locust.task
-    def get_ao(self, term=None):
+    def get_adr(self):
+        params = {
+            'api_key': API_KEY,
+        }
+        self.client.get('legal/docs/adrs/668', name='legal_get_adr', params=params)
+
+    @locust.task
+    def get_admin_fine(self):
+        params = {
+            'api_key': API_KEY,
+        }
+        self.client.get('legal/docs/admin_fines/2274', name='legal_get_admin_fine', params=params)
+
+    @locust.task
+    def get_ao(self):
         params = {
             'api_key': API_KEY,
         }

manage.py

@@ -35,6 +35,8 @@
 manager.command(legal_docs.load_archived_murs)
 manager.command(legal_docs.load_advisory_opinions)
 manager.command(legal_docs.load_current_murs)
+manager.command(legal_docs.load_adrs)
+manager.command(legal_docs.load_admin_fines)
 manager.command(legal_docs.create_docs_index)
 manager.command(legal_docs.create_archived_murs_index)
 manager.command(legal_docs.create_staging_index)

tests/integration/test_current_murs.py → tests/integration/test_current_cases.py

@@ -6,11 +6,13 @@
 
 
 from webservices import rest
-from webservices.legal_docs.current_murs import get_murs
+from webservices.legal_docs.current_cases import get_cases
+
 from tests.common import TEST_CONN, BaseTestCase
 
 @pytest.mark.usefixtures("migrate_db")
-class TestLoadCurrentMURs(BaseTestCase):
+class TestLoadCurrentCases(BaseTestCase):
+
     def setUp(self):
         self.connection = rest.db.engine.connect()
         subprocess.check_call(
@@ -21,7 +23,7 @@ def tearDown(self):
         self.connection.close()
         rest.db.session.remove()
 
-    @patch('webservices.legal_docs.current_murs.get_bucket')
+    @patch('webservices.legal_docs.current_cases.get_bucket')
     def test_simple_mur(self, get_bucket):
         mur_subject = 'Fraudulent misrepresentation'
         expected_mur = {
@@ -42,13 +44,85 @@ def test_simple_mur(self, get_bucket):
             'sort1': -1,
             'sort2': None
         }
-        self.create_mur(1, expected_mur['no'], expected_mur['name'], mur_subject)
-        actual_mur = next(get_murs(None))
+        self.create_case(1, expected_mur['no'], expected_mur['name'], mur_subject)
+        actual_mur = next(get_cases('MUR'))
 
         assert actual_mur == expected_mur
 
+    @patch('webservices.legal_docs.current_cases.get_bucket')
+    def test_simple_adr(self, get_bucket):
+        adr_subject = 'Personal use'
+        expected_adr = {
+            'no': '1',
+            'name': 'Simple ADR',
+            'election_cycles': [2016],
+            'doc_id': 'adr_1',
+            'participants': [],
+            'subjects': [adr_subject],
+            'respondents': [],
+            'documents': [],
+            'commission_votes': [],
+            'dispositions': [],
+            'close_date': None,
+            'open_date': None,
+            'url': '/legal/alternative-dispute-resolution/1/',
+            'sort1': -1,
+            'sort2': None
+        }
+        self.create_case(1, expected_adr['no'], expected_adr['name'], adr_subject, 'ADR')
+        actual_adr = next(get_cases('ADR'))
+
+        assert actual_adr == expected_adr
+
+    @patch('webservices.legal_docs.current_cases.get_bucket')
+    def test_admin_fine(self, get_bucket):
+        dummy_subject = 'Personal use'
+        expected_admin_fine = {
+            'no': '1',
+            'name': 'Big Admin Fine',
+            'doc_id': 'af_1',
+            'documents': [],
+            'commission_votes': [{'action': None, 'vote_date': None}],
+            'committee_id': 'C001',
+            'report_year': '2016',
+            'report_type': '30G',
+            'reason_to_believe_action_date': None,
+            'reason_to_believe_fine_amount': 5000,
+            'challenge_receipt_date': None,
+            'challenge_outcome': '',
+            'final_determination_date': None,
+            'final_determination_amount': 5000,
+            'check_amount': 5000,
+            'treasury_referral_date': None,
+            'treasury_referral_amount': 0,
+            'petition_court_filing_date': None,
+            'petition_court_decision_date': None,
+            'url': '/legal/administrative-fine/1/',
+            'sort1': -1,
+            'sort2': None
+        }
+        self.create_case(1, expected_admin_fine['no'], expected_admin_fine['name'], dummy_subject, 'AF')
+        self.create_admin_fine(1,
+            expected_admin_fine['committee_id'],
+            expected_admin_fine['report_year'],
+            expected_admin_fine['report_type'],
+            expected_admin_fine['reason_to_believe_action_date'],
+            expected_admin_fine['reason_to_believe_fine_amount'],
+            expected_admin_fine['challenge_receipt_date'],
+            expected_admin_fine['challenge_outcome'],
+            expected_admin_fine['final_determination_date'],
+            expected_admin_fine['final_determination_amount'],
+            expected_admin_fine['check_amount'],
+            expected_admin_fine['treasury_referral_date'],
+            expected_admin_fine['treasury_referral_amount'],
+            expected_admin_fine['petition_court_filing_date'],
+            expected_admin_fine['petition_court_decision_date'])
+        actual_admin_fine = next(get_cases('AF'))
+
+        assert actual_admin_fine == expected_admin_fine
+
     @patch('webservices.env.env.get_credential', return_value='BUCKET_NAME')
-    @patch('webservices.legal_docs.current_murs.get_bucket')
+    @patch('webservices.legal_docs.current_cases.get_bucket')
     def test_mur_with_participants_and_documents(self, get_bucket, get_credential):
         case_id = 1
         mur_subject = 'Fraudulent misrepresentation'
@@ -74,15 +148,15 @@ def test_mur_with_participants_and_documents(self, get_bucket, get_credential):
                     filename.replace(' ', '-'))),
         ]
 
-        self.create_mur(case_id, expected_mur['no'], expected_mur['name'], mur_subject)
+        self.create_case(case_id, expected_mur['no'], expected_mur['name'], mur_subject)
         for entity_id, participant in enumerate(participants):
             role, name = participant
             self.create_participant(case_id, entity_id, role, name)
         for document_id, document in enumerate(documents):
             category, ocrtext, url = document
             self.create_document(case_id, document_id, category, ocrtext, filename)
 
-        actual_mur = next(get_murs(None))
+        actual_mur = next(get_cases('MUR'))
 
         for key in expected_mur:
             assert actual_mur[key] == expected_mur[key]
@@ -94,14 +168,14 @@ def test_mur_with_participants_and_documents(self, get_bucket, get_credential):
             (d['category'], d['text'], d['length']) for d in actual_mur['documents']]
 
     @patch('webservices.env.env.get_credential', return_value='BUCKET_NAME')
-    @patch('webservices.legal_docs.current_murs.get_bucket')
+    @patch('webservices.legal_docs.current_cases.get_bucket')
     def test_mur_with_disposition(self, get_bucket, get_credential):
         case_id = 1
         case_no = '1'
         name = 'Open Elections LLC'
         mur_subject = 'Fraudulent misrepresentation'
         pg_date = '2016-10-08'
-        self.create_mur(case_id, case_no, name, mur_subject)
+        self.create_case(case_id, case_no, name, mur_subject)
 
         entity_id = 1
         event_date = '2005-01-01'
@@ -150,7 +224,7 @@ def test_mur_with_disposition(self, get_bucket, get_credential):
         action = 'Conciliation Reached.'
         self.create_commission(commission_id, agenda_date, vote_date, action, case_id, pg_date)
 
-        actual_mur = next(get_murs(None))
+        actual_mur = next(get_cases('MUR'))
 
         expected_mur = {
             'commission_votes': [{'action': 'Conciliation Reached.', 'vote_date': datetime(2008, 1, 1, 0, 0)}],
@@ -181,7 +255,7 @@ def test_mur_with_disposition(self, get_bucket, get_credential):
         }
         assert actual_mur == expected_mur
 
-    @patch('webservices.legal_docs.current_murs.get_bucket')
+    @patch('webservices.legal_docs.current_cases.get_bucket')
     def test_mur_offsets(self, get_bucket):
         mur_subject = 'Fraudulent misrepresentation'
         expected_mur1 = {
@@ -238,31 +312,44 @@ def test_mur_offsets(self, get_bucket):
             'sort1': -3,
             'sort2': None
         }
-        self.create_mur(1, expected_mur1['no'], expected_mur1['name'], mur_subject)
-        self.create_mur(2, expected_mur2['no'], expected_mur2['name'], mur_subject)
-        self.create_mur(3, expected_mur3['no'], expected_mur3['name'], mur_subject)
+        self.create_case(1, expected_mur1['no'], expected_mur1['name'], mur_subject)
+        self.create_case(2, expected_mur2['no'], expected_mur2['name'], mur_subject)
+        self.create_case(3, expected_mur3['no'], expected_mur3['name'], mur_subject)
 
-        gen = get_murs(None)
+        gen = get_cases('MUR')
         assert(next(gen)) == expected_mur1
         assert(next(gen)) == expected_mur2
         assert(next(gen)) == expected_mur3
 
-        actual_murs = [mur for mur in get_murs('2')]
+        actual_murs = [mur for mur in get_cases('MUR', '2')]
         assert actual_murs == [expected_mur2]
 
-    def create_mur(self, case_id, case_no, name, subject_description):
+    def create_case(self, case_id, case_no, name, subject_description, case_type='MUR'):
         subject_id = self.connection.execute(
             "SELECT subject_id FROM fecmur.subject "
             " WHERE description = %s ", subject_description).scalar()
         self.connection.execute(
             "INSERT INTO fecmur.case (case_id, case_no, name, case_type) "
-            "VALUES (%s, %s, %s, 'MUR')", case_id, case_no, name)
-        self.connection.execute(
-            "INSERT INTO fecmur.case_subject (case_id, subject_id, relatedsubject_id) "
-            "VALUES (%s, %s, -1)", case_id, subject_id)
+            "VALUES (%s, %s, %s, %s)", case_id, case_no, name, case_type)
+        if case_type != 'AF':
+            self.connection.execute(
+                "INSERT INTO fecmur.case_subject (case_id, subject_id, relatedsubject_id) "
+                "VALUES (%s, %s, -1)", case_id, subject_id)
+            self.connection.execute(
+                "INSERT INTO fecmur.electioncycle (case_id, election_cycle) "
+                "VALUES (%s, 2016)", case_id)
+
+    def create_admin_fine(
+            self, case_id, committee_id, report_year, report_type,
+            reason_to_believe_action_date, reason_to_believe_fine_amount,
+            challenge_receipt_date, challenge_outcome, final_determination_date,
+            final_determination_amount, check_amount, treasury_referral_date,
+            treasury_referral_amount, petition_court_filing_date,
+            petition_court_decision_date):
+
         self.connection.execute(
-            "INSERT INTO fecmur.electioncycle (case_id, election_cycle) "
-            "VALUES (%s, 2016)", case_id)
+            "INSERT INTO fecmur.af_case (case_id, committee_id, report_year, report_type, rtb_action_date, rtb_fine_amount, chal_receipt_date, chal_outcome_code_desc, fd_date, fd_final_fine_amount, check_amount, treasury_date, treasury_amount, petition_court_filing_date, petition_court_decision_date) "
+            "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", case_id, committee_id, report_year, report_type, reason_to_believe_action_date, reason_to_believe_fine_amount,  challenge_receipt_date, challenge_outcome, final_determination_date, final_determination_amount, check_amount, treasury_referral_date, treasury_referral_amount, petition_court_filing_date, petition_court_decision_date)
 
     def create_participant(self, case_id, entity_id, role, name,
             stage=None, statutory_citation=None, regulatory_citation=None):

tests/test_legal.py

@@ -53,6 +53,14 @@ def es_search(**kwargs):
         return {'hits': {'hits': [{'highlight': {'text': ['f']},
                                     '_source': {}, '_type': 'murs'}], 'total': 4}}
 
+    if _type == 'adrs':
+        return {'hits': {'hits': [{'highlight': {'text': ['f']},
+                                    '_source': {}, '_type': 'adrs'}], 'total': 2}}
+
+    if _type == 'admin_fines':
+        return {'hits': {'hits': [{'highlight': {'text': ['f']},
+                                    '_source': {}, '_type': 'admin_fines'}], 'total': 5}}
+
 
 class CanonicalPageTest(unittest.TestCase):
     @patch('webservices.rest.legal.es.search', es_advisory_opinion)
@@ -109,12 +117,20 @@ def test_default_search(self):
                 {'highlights': ['f'], 'document_highlights': {}}
             ],
             'total_murs': 4,
+            'adrs': [
+                {'highlights': ['f'], 'document_highlights': {}}
+            ],
+            'total_adrs': 2,
+            'admin_fines': [
+                {'highlights': ['f'], 'document_highlights': {}}
+            ],
+            'total_admin_fines': 5,
             'advisory_opinions': [
                 {'highlights': ['a', 'b'], 'document_highlights': {}},
                 {'highlights': ['c', 'd'], 'document_highlights': {}}
             ],
             'total_advisory_opinions': 2,
-            'total_all': 10}
+            'total_all': 17}
 
     @patch('webservices.rest.legal.es.search', es_search)
     def test_type_search(self):

tests/test_parse_mur_citations.py

@@ -1,6 +1,6 @@
 import pytest
 
-from webservices.legal_docs.current_murs import (
+from webservices.legal_docs.current_cases import (
     parse_regulatory_citations,
     parse_statutory_citations,
 )

webservices/args.py

@@ -191,6 +191,29 @@ def make_seek_args(field=fields.Int, description=None):
         description='Filter MURs by earliest date closed'),
     'mur_max_close_date': fields.Date(required=False,
         description='Filter MURs by latest date closed'),
+    'case_no': fields.List(IStr, required=False, description='Enforcement matter case number'),
+    'case_document_category': fields.List(IStr, required=False,
+        description='Filter cases by category of associated documents'),
+    'case_respondents': fields.Str(IStr, required=False, description='Filter cases by respondents'),
+    'case_dispositions': fields.List(IStr, required=False, description='Filter cases by dispositions'),
+    'case_election_cycles': fields.Int(IStr, required=False, description='Filter cases by election cycles'),
+    'case_min_open_date': fields.Date(required=False, description='Filter cases by earliest date opened'),
+    'case_max_open_date': fields.Date(required=False, description='Filter cases by latest date opened'),
+    'case_min_close_date': fields.Date(required=False,
+        description='Filter cases by earliest date closed'),
+    'case_max_close_date': fields.Date(required=False,
+        description='Filter cases by latest date closed'),
+    'af_name': fields.List(IStr, required=False, description='Admin fine committee name'),
+    'af_committee_id': fields.Str(IStr, required=False, description='Admin fine committee ID'),
+    'af_report_year': fields.Str(IStr, required=False, description='Admin fine report year'),
+    'af_min_rtb_date': fields.Date(required=False, description='Filter cases by earliest Reason to Believe date'),
+    'af_max_rtb_date': fields.Date(required=False, description='Filter cases by latest Reason to Believe date'),
+    'af_rtb_fine_amount': fields.Int(IStr, required=False, description='Filter cases by Reason to Believe fine amount'),
+    'af_min_fd_date': fields.Date(required=False,
+        description='Filter cases by earliest Final Determination date'),
+    'af_max_fd_date': fields.Date(required=False,
+        description='Filter cases by latest Final Determination date'),
+    'af_fd_fine_amount': fields.Int(IStr, required=False, description='Filter cases by Final Determination fine amount')
 }
 
 candidate_detail = {