Included amended F3* records.

* Include F3* records with non-null `expire_date`
* Add "amended" filter to committee reports views
* Index `expire_date`

Note: this requires a quick patch to the frontend so that only current
reports are requested for the committee detail view. This will be
submitted shortly.

[Resolves fecgov/openFEC-web-app#874]

data/sql_updates/create_reports_house_senate_view.sql

@@ -97,12 +97,12 @@ from
     left join dimdates end_date on cvg_end_dt_sk = end_date.date_sk and cvg_end_dt_sk != 1
 where
     two_yr_period_sk >= :START_YEAR
-    and f3.expire_date is null
 ;
 
 create unique index on ofec_reports_house_senate_mv_tmp(idx);
 
 create index on ofec_reports_house_senate_mv_tmp(cycle);
+create index on ofec_reports_house_senate_mv_tmp(expire_date);
 create index on ofec_reports_house_senate_mv_tmp(report_type);
 create index on ofec_reports_house_senate_mv_tmp(report_year);
 create index on ofec_reports_house_senate_mv_tmp(committee_id);

data/sql_updates/create_reports_pacs_parties_view.sql

@@ -119,12 +119,12 @@ from
     left join dimdates end_date on cvg_end_dt_sk = end_date.date_sk and cvg_end_dt_sk != 1
 where
     two_yr_period_sk >= :START_YEAR
-    and f3x.expire_date is null
 ;
 
 create unique index on ofec_reports_pacs_parties_mv_tmp(idx);
 
 create index on ofec_reports_pacs_parties_mv_tmp(cycle);
+create index on ofec_reports_pacs_parties_mv_tmp(expire_date);
 create index on ofec_reports_pacs_parties_mv_tmp(report_type);
 create index on ofec_reports_pacs_parties_mv_tmp(report_year);
 create index on ofec_reports_pacs_parties_mv_tmp(committee_id);

data/sql_updates/create_reports_presidential_view.sql

@@ -98,12 +98,12 @@ from
     left join dimdates end_date on cvg_end_dt_sk = end_date.date_sk and cvg_end_dt_sk != 1
 where
     two_yr_period_sk >= :START_YEAR
-    and f3p.expire_date is null
 ;
 
 create unique index on ofec_reports_presidential_mv_tmp(idx);
 
 create index on ofec_reports_presidential_mv_tmp(cycle);
+create index on ofec_reports_presidential_mv_tmp(expire_date);
 create index on ofec_reports_presidential_mv_tmp(report_type);
 create index on ofec_reports_presidential_mv_tmp(report_year);
 create index on ofec_reports_presidential_mv_tmp(committee_id);

tests/test_integration.py

@@ -99,11 +99,6 @@ def test_totals_year_filter(self):
         for model in TOTALS_MODELS:
             self._check_financial_model(model)
 
-    def test_keeps_only_non_expired_reports(self):
-        for model in REPORTS_MODELS:
-            with self.subTest(report_model=model):
-                self.assertFalse(model.query.filter(model.expire_date != None).count())
-
     def _check_financial_model(self, model):
         count = model.query.filter(
             model.cycle < manage.SQL_CONFIG['START_YEAR']

tests/test_reports.py

@@ -69,6 +69,23 @@ def test_reports_by_committee_type_and_cycle(self):
             [presidential_report_2012, house_report_2016],
         )
 
+    def test_reports_by_amended(self):
+        reports = [
+            factories.ReportsPresidentialFactory(expire_date=None),
+            factories.ReportsPresidentialFactory(expire_date=datetime.date(2014, 1, 1)),
+        ]
+
+        results = self._results(api.url_for(ReportsView, committee_type='presidential'))
+        self.assertEqual(len(results), 2)
+
+        results = self._results(api.url_for(ReportsView, committee_type='presidential', amended='false'))
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['committee_id'], reports[0].committee_id)
+
+        results = self._results(api.url_for(ReportsView, committee_type='presidential', amended='true'))
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['committee_id'], reports[1].committee_id)
+
     def test_reports_by_committee_type_and_year(self):
         presidential_report_2012 = factories.ReportsPresidentialFactory(report_year=2012)
         presidential_report_2016 = factories.ReportsPresidentialFactory(report_year=2016)

webservices/args.py

@@ -256,6 +256,7 @@ def make_seek_args(field=fields.Int, description=None):
     'cycle': fields.List(fields.Int, description=docs.RECORD_CYCLE),
     'beginning_image_number': fields.List(fields.Int, description=docs.BEGINNING_IMAGE_NUMBER),
     'report_type': fields.List(fields.Str, description='Report type; prefix with "-" to exclude'),
+    'amended': fields.Bool(description='Report has been amended'),
 }
 
 

webservices/resources/reports.py

@@ -92,6 +92,10 @@ def get_reports(self, committee_id, committee_type, kwargs):
             elif exclude:
                 query = query.filter(sa.not_(reports_class.report_type.in_(exclude)))
 
+        if kwargs.get('amended') is not None:
+            column = reports_class.expire_date
+            query = query.filter(column != None if kwargs['amended'] else column == None)  # noqa
+
         return query, reports_class, reports_schema
 
     def _resolve_committee_type(self, committee_id, committee_type, kwargs):