diff --git a/data/migrations/V0086__add_future_ofec_candidate_totals_mv.sql b/data/migrations/V0086__add_future_ofec_candidate_totals_mv.sql new file mode 100644 index 000000000..81cff55bb --- /dev/null +++ b/data/migrations/V0086__add_future_ofec_candidate_totals_mv.sql @@ -0,0 +1,383 @@ +/* +Addresses #3214: Fixes missing 2020 presidential data (and other future elections) + +- Add ofec_candidate_history_with_future_election_mv which builds off + ofec_candidate_history but adds a row for future cycle candidates +- Recreate ofec_candidate_totals_mv to include future cycle totals + +*/ + +SET search_path = public; + +CREATE MATERIALIZED VIEW ofec_candidate_history_with_future_election_mv AS + +WITH combined AS ( + SELECT + load_date, + two_year_period, + candidate_election_year + candidate_election_year % 2 as candidate_election_year, + candidate_id, + name, + address_state, + address_city, + address_street_1, + address_street_2, + address_zip, + incumbent_challenge, + incumbent_challenge_full, + candidate_status, + candidate_inactive, + office, + office_full, + state, + district, + district_number, + party, + party_full, + cycles, + first_file_date, + last_file_date, + last_f2_date, + election_years, + election_districts, + active_through + from ofec_candidate_history_mv + UNION --Add a row for current office sought (adds future elections) + SELECT DISTINCT ON (ofec_candidate_history_mv.candidate_id) + load_date, + candidate_election_year + candidate_election_year % 2 as two_year_period, + candidate_election_year + candidate_election_year % 2 as candidate_election_year, + candidate_id, + name, + address_state, + address_city, + address_street_1, + address_street_2, + address_zip, + incumbent_challenge, + incumbent_challenge_full, + candidate_status, + candidate_inactive, + office, + office_full, + state, + district, + district_number, + party, + party_full, + cycles, + first_file_date, + last_file_date, + last_f2_date, + election_years, + election_districts, + active_through + FROM ofec_candidate_history_mv + --Only bring in an extra row for elections two years away or more + --(keeps past elections from doubling) + WHERE candidate_election_year - date_part('year', CURRENT_DATE) >= 2) +SELECT row_number() OVER () AS idx, + load_date, + two_year_period, + candidate_election_year + candidate_election_year % 2 as candidate_election_year, + candidate_id, + name, + address_state, + address_city, + address_street_1, + address_street_2, + address_zip, + incumbent_challenge, + incumbent_challenge_full, + candidate_status, + candidate_inactive, + office, + office_full, + state, + district, + district_number, + party, + party_full, + cycles, + first_file_date, + last_file_date, + last_f2_date, + election_years, + election_districts, + active_through +FROM combined; + +--Add permissions + +ALTER TABLE ofec_candidate_history_with_future_election_mv OWNER TO fec; +GRANT SELECT ON TABLE ofec_candidate_history_with_future_election_mv TO fec_read; + +--Indexes: Should this be a unique index on ID, two year period? + +CREATE UNIQUE INDEX ofec_candidate_history_with_future_election_mv_idx_idx + ON ofec_candidate_history_with_future_election_mv USING btree (idx); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_candidate_id_idx + ON ofec_candidate_history_with_future_election_mv USING btree (candidate_id); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_district_idx + ON ofec_candidate_history_with_future_election_mv USING btree (district); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_district_number_idx + ON ofec_candidate_history_with_future_election_mv USING btree (district_number); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_first_file_date_idx + ON ofec_candidate_history_with_future_election_mv USING btree (first_file_date); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_load_date_idx + ON ofec_candidate_history_with_future_election_mv USING btree (load_date); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_office_idx + ON ofec_candidate_history_with_future_election_mv USING btree (office); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_state_idx + ON ofec_candidate_history_with_future_election_mv USING btree (state); + +CREATE INDEX ofec_candidate_history_with_future_election_mv_two_year_period_candidate_id_idx + ON ofec_candidate_history_with_future_election_mv USING btree (two_year_period, candidate_id); + +/* + +Recreate ofec_candidate_totals_mv to use ofec_candidate_history_with_future_election_mv +instead of ofec_candidate_history_mv as base table - this will include future election cycles + +*/ + +DROP MATERIALIZED VIEW IF EXISTS ofec_candidate_totals_mv_tmp; + +CREATE MATERIALIZED VIEW ofec_candidate_totals_mv_tmp AS + WITH totals AS ( + SELECT ofec_totals_house_senate_mv.committee_id, + ofec_totals_house_senate_mv.cycle, + ofec_totals_house_senate_mv.receipts, + ofec_totals_house_senate_mv.disbursements, + ofec_totals_house_senate_mv.last_cash_on_hand_end_period, + ofec_totals_house_senate_mv.last_debts_owed_by_committee, + ofec_totals_house_senate_mv.coverage_start_date, + ofec_totals_house_senate_mv.coverage_end_date, + false AS federal_funds_flag + FROM ofec_totals_house_senate_mv + UNION ALL + SELECT ofec_totals_presidential_mv.committee_id, + ofec_totals_presidential_mv.cycle, + ofec_totals_presidential_mv.receipts, + ofec_totals_presidential_mv.disbursements, + ofec_totals_presidential_mv.last_cash_on_hand_end_period, + ofec_totals_presidential_mv.last_debts_owed_by_committee, + ofec_totals_presidential_mv.coverage_start_date, + ofec_totals_presidential_mv.coverage_end_date, + ofec_totals_presidential_mv.federal_funds_flag + FROM ofec_totals_presidential_mv + ), link AS ( --Off-year special election candidates have > 1 row/cycle + SELECT DISTINCT cand_id, + cand_election_yr + cand_election_yr % 2 as rounded_election_yr, + fec_election_yr, + cmte_id, + cmte_dsgn + FROM ofec_cand_cmte_linkage_mv + WHERE cmte_dsgn IN ('P', 'A') + ), cycle_totals AS ( + SELECT DISTINCT ON (link.cand_id, totals.cycle) link.cand_id AS candidate_id, + max(link.rounded_election_yr) AS election_year, + totals.cycle, + false AS is_election, + sum(totals.receipts) AS receipts, + sum(totals.disbursements) AS disbursements, + (sum(totals.receipts) > (0)::numeric) AS has_raised_funds, + sum(totals.last_cash_on_hand_end_period) AS cash_on_hand_end_period, + sum(totals.last_debts_owed_by_committee) AS debts_owed_by_committee, + min(totals.coverage_start_date) AS coverage_start_date, + max(totals.coverage_end_date) AS coverage_end_date, + (array_agg(totals.federal_funds_flag) @> ARRAY[true]) AS federal_funds_flag + FROM link + INNER JOIN totals + ON link.cmte_id = totals.committee_id + AND link.fec_election_yr = totals.cycle + GROUP BY link.cand_id, totals.cycle + ), election_aggregates AS ( + SELECT cycle_totals.candidate_id, + cycle_totals.election_year, + sum(cycle_totals.receipts) AS receipts, + sum(cycle_totals.disbursements) AS disbursements, + (sum(cycle_totals.receipts) > (0)::numeric) AS has_raised_funds, + min(cycle_totals.coverage_start_date) AS coverage_start_date, + max(cycle_totals.coverage_end_date) AS coverage_end_date, + (array_agg(cycle_totals.federal_funds_flag) @> ARRAY[true]) AS federal_funds_flag + FROM cycle_totals + GROUP BY cycle_totals.candidate_id, cycle_totals.election_year + ), election_latest AS ( + SELECT DISTINCT ON (totals.candidate_id, totals.election_year) totals.candidate_id, + totals.election_year, + totals.cash_on_hand_end_period, + totals.debts_owed_by_committee, + totals.federal_funds_flag + FROM cycle_totals totals + ORDER BY totals.candidate_id, totals.election_year, totals.cycle DESC + ), election_totals AS ( + SELECT totals.candidate_id, + totals.election_year, + totals.election_year AS cycle, + true AS is_election, + totals.receipts, + totals.disbursements, + totals.has_raised_funds, + latest.cash_on_hand_end_period, + latest.debts_owed_by_committee, + totals.coverage_start_date, + totals.coverage_end_date, + totals.federal_funds_flag + FROM (election_aggregates totals + JOIN election_latest latest USING (candidate_id, election_year)) + ), combined_totals as ( + SELECT cycle_totals.candidate_id, + cycle_totals.election_year, + cycle_totals.cycle, + cycle_totals.is_election, + cycle_totals.receipts, + cycle_totals.disbursements, + cycle_totals.has_raised_funds, + cycle_totals.cash_on_hand_end_period, + cycle_totals.debts_owed_by_committee, + cycle_totals.coverage_start_date, + cycle_totals.coverage_end_date, + cycle_totals.federal_funds_flag + FROM cycle_totals +UNION ALL + SELECT election_totals.candidate_id, + election_totals.election_year, + election_totals.cycle, + election_totals.is_election, + election_totals.receipts, + election_totals.disbursements, + election_totals.has_raised_funds, + election_totals.cash_on_hand_end_period, + election_totals.debts_owed_by_committee, + election_totals.coverage_start_date, + election_totals.coverage_end_date, + election_totals.federal_funds_flag + FROM election_totals) +SELECT cand.candidate_id AS candidate_id, + cand.candidate_election_year AS election_year, + cand.two_year_period AS cycle, + COALESCE (totals.is_election, + CASE WHEN cand.candidate_election_year = cand.two_year_period + THEN true ELSE false END) AS is_election, + COALESCE (totals.receipts, 0) AS receipts, + COALESCE (totals.disbursements, 0) AS disbursements, + COALESCE (totals.has_raised_funds, false) AS has_raised_funds, + COALESCE (totals.cash_on_hand_end_period, 0) AS cash_on_hand_end_period, + COALESCE (totals.debts_owed_by_committee, 0) AS debts_owed_by_committee, + totals.coverage_start_date, + totals.coverage_end_date, + COALESCE (totals.federal_funds_flag, false) AS federal_funds_flag + FROM ofec_candidate_history_with_future_election_mv cand + LEFT JOIN combined_totals totals + ON cand.candidate_id = totals.candidate_id + AND cand.two_year_period = totals.cycle; + + +--Permissions + +ALTER TABLE ofec_candidate_totals_mv_tmp OWNER TO fec; +GRANT SELECT ON TABLE ofec_candidate_totals_mv_tmp TO fec_read; + +--Indexes-------------- + +/* +Filters on this model for TotalsCandidateView: + +- filter_multi_fields = election_year, cycle +- filter_range_fields = receipts, disbursements, cash_on_hand_end_period, debts_owed_by_committee +- filter_match_fields = has_raised_funds, federal_funds_flag, is_election + +*/ + +CREATE UNIQUE INDEX ofec_candidate_totals_mv_candidate_id_cycle_is_election_idx_tmp ON ofec_candidate_totals_mv_tmp USING btree (candidate_id, cycle, is_election); + +CREATE INDEX ofec_candidate_totals_mv_candidate_id_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (candidate_id); + +CREATE INDEX ofec_candidate_totals_mv_cycle_candidate_id_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (cycle, candidate_id); + +CREATE INDEX ofec_candidate_totals_mv_cycle_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (cycle); + +CREATE INDEX ofec_candidate_totals_mv_receipts_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (receipts); + +CREATE INDEX ofec_candidate_totals_mv_disbursements_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (disbursements); + +CREATE INDEX ofec_candidate_totals_mv_election_year_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (election_year); + +CREATE INDEX ofec_candidate_totals_mv_federal_funds_flag_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (federal_funds_flag); + +CREATE INDEX ofec_candidate_totals_mv_has_raised_funds_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (has_raised_funds); + +CREATE INDEX ofec_candidate_totals_mv_is_election_idx_tmp + ON ofec_candidate_totals_mv_tmp USING btree (is_election); + +/* +Drop dependent MV +*/ + +DROP MATERIALIZED VIEW IF EXISTS ofec_candidate_flag_mv; + +/* +Don't recreate dependent MV ofec_candidate_totals_with_0s_mv +Supersedes migration V0076, V0085 +*/ + +DROP MATERIALIZED VIEW IF EXISTS ofec_candidate_totals_with_0s_mv; + +/* +Drop old ofec_candidate_totals_mv, rename _tmp MV and indexes +*/ + +DROP MATERIALIZED VIEW IF EXISTS ofec_candidate_totals_mv; + +ALTER MATERIALIZED VIEW IF EXISTS ofec_candidate_totals_mv_tmp RENAME TO ofec_candidate_totals_mv; + +SELECT rename_indexes('ofec_candidate_totals_mv'); + +/* +Recreate ofec_candidate_flag_mv + +Supersedes migrations V0026, V0034, V0038, V0059.2, V0059, V0069 +*/ + +CREATE MATERIALIZED VIEW ofec_candidate_flag_mv AS + SELECT row_number() OVER () AS idx, + ofec_candidate_history_mv.candidate_id, + (array_agg(oct.has_raised_funds) @> ARRAY[true]) AS has_raised_funds, + (array_agg(oct.federal_funds_flag) @> ARRAY[true]) AS federal_funds_flag + FROM ofec_candidate_history_mv + LEFT JOIN ofec_candidate_totals_mv oct USING (candidate_id) + GROUP BY ofec_candidate_history_mv.candidate_id; + + +ALTER TABLE ofec_candidate_flag_mv OWNER TO fec; +GRANT SELECT ON TABLE ofec_candidate_flag_mv TO fec_read; + + +CREATE UNIQUE INDEX ofec_candidate_flag_mv_idx_idx + ON ofec_candidate_flag_mv USING btree (idx); + +CREATE INDEX ofec_candidate_flag_mv_candidate_id_idx + ON ofec_candidate_flag_mv USING btree (candidate_id); + +CREATE INDEX ofec_candidate_flag_mv_federal_funds_flag_idx + ON ofec_candidate_flag_mv USING btree (federal_funds_flag); + +CREATE INDEX ofec_candidate_flag_mv_has_raised_funds_idx + ON ofec_candidate_flag_mv USING btree (has_raised_funds); + diff --git a/manage.py b/manage.py index 552797139..69aa7842c 100755 --- a/manage.py +++ b/manage.py @@ -1,9 +1,7 @@ #!/usr/bin/env python -import os import glob import logging -import shlex import subprocess import multiprocessing @@ -192,8 +190,6 @@ def load_election_dates(): logger.info('Finished loading election dates.') - - @manager.command def refresh_itemized(): """These are run nightly to refresh the itemized schedule A and B data.""" @@ -283,13 +279,13 @@ def refresh_materialized(concurrent=True): 'ofec_committee_fulltext_audit_mv', 'ofec_candidate_fulltext_audit_mv'], 'cand_cmte_linkage': ['ofec_cand_cmte_linkage_mv'], - 'candidate_aggregates': ['ofec_candidate_totals_mv', - 'ofec_candidate_totals_with_0s_mv'], + 'candidate_aggregates': ['ofec_candidate_totals_mv'], 'candidate_detail': ['ofec_candidate_detail_mv'], 'candidate_election': ['ofec_candidate_election_mv'], 'candidate_flags': ['ofec_candidate_flag_mv'], 'candidate_fulltext': ['ofec_candidate_fulltext_mv'], 'candidate_history': ['ofec_candidate_history_mv'], + 'candidate_history_future': ['ofec_candidate_history_with_future_election_mv'], 'committee_detail': ['ofec_committee_detail_mv'], 'committee_fulltext': ['ofec_committee_fulltext_mv'], 'committee_history': ['ofec_committee_history_mv'], diff --git a/tests/factories.py b/tests/factories.py index 9747f5741..fd628fa54 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -57,6 +57,11 @@ class Meta: two_year_period = 2016 candidate_inactive = False +class CandidateHistoryFutureFactory(BaseCandidateFactory): + class Meta: + model = models.CandidateHistoryWithFuture + two_year_period = 2016 + candidate_inactive = False class CandidateElectionFactory(BaseCandidateFactory): class Meta: diff --git a/tests/test_aggregates.py b/tests/test_aggregates.py index 9889315b2..215e189dc 100644 --- a/tests/test_aggregates.py +++ b/tests/test_aggregates.py @@ -1,6 +1,8 @@ from tests import factories from tests.common import ApiBaseTest, assert_dicts_subset +from webservices.utils import get_current_cycle + from webservices import schemas from webservices.rest import db, api from webservices.resources.aggregates import ( @@ -168,9 +170,12 @@ def test_candidate_aggregates_by_election(self): class TestCandidateAggregates(ApiBaseTest): + current_cycle = get_current_cycle() + next_cycle = current_cycle + 2 + def setUp(self): super().setUp() - self.candidate = factories.CandidateHistoryFactory( + self.candidate = factories.CandidateHistoryFutureFactory( candidate_id='S123', two_year_period=2012, candidate_election_year=2012, @@ -234,7 +239,7 @@ def setUp(self): fec_election_year=2010, ) # Create a candidate_zero without a committee and $0 in CandidateTotal - self.candidate_zero = factories.CandidateHistoryFactory( + self.candidate_zero = factories.CandidateHistoryFutureFactory( candidate_id='H321', two_year_period=2018, candidate_election_year=2018, @@ -251,7 +256,7 @@ def setUp(self): ) # Create data for a candidate who ran in 2017 and 2018 - self.candidate_17_18 = factories.CandidateHistoryFactory( + self.candidate_17_18 = factories.CandidateHistoryFutureFactory( candidate_id='S456', two_year_period=2018, candidate_election_year=2018, @@ -308,7 +313,7 @@ def setUp(self): fec_election_year=2018, ) # Create data for a candidate who ran just in 2017 - self.candidate_17_only = factories.CandidateHistoryFactory( + self.candidate_17_only = factories.CandidateHistoryFutureFactory( candidate_id='H456', two_year_period=2018, candidate_election_year=2017, @@ -357,6 +362,76 @@ def setUp(self): fec_election_year=2018, ) + # Create data for future presidential - next_cycle. Use formula for future + + # Test full next_cycle and current_cycle 2-year totals + + self.candidate_20 = factories.CandidateHistoryFutureFactory( + candidate_id='P456', + two_year_period=self.current_cycle, + candidate_election_year=self.next_cycle, + ) + self.candidate_20 = factories.CandidateHistoryFutureFactory( + candidate_id='P456', + two_year_period=self.next_cycle, + candidate_election_year=self.next_cycle, + ) + #Candidate history won't have next_cycle yet + self.committees_20 = [ + factories.CommitteeHistoryFactory(cycle=self.current_cycle, designation='P'), + ] + factories.CandidateDetailFactory( + candidate_id=self.candidate_20.candidate_id, + election_years=[self.next_cycle], + ) + [ + factories.CandidateElectionFactory( + candidate_id=self.candidate_20.candidate_id, + cand_election_year=election_year + ) + for election_year in [self.next_cycle - 4, self.next_cycle] + ] + [ + factories.CommitteeDetailFactory(committee_id=each.committee_id) + for each in self.committees_20 + ] + #Full next_cycle + factories.CandidateTotalFactory( + candidate_id=self.candidate_20.candidate_id, + cycle=self.next_cycle, + is_election=True, + receipts=55000, + ) + #current_cycle 2-year + factories.CandidateTotalFactory( + candidate_id=self.candidate_20.candidate_id, + cycle=self.current_cycle, + is_election=False, + receipts=25000, + ) + factories.CandidateFlagsFactory( + candidate_id=self.candidate_20.candidate_id + ) + db.session.flush() + + factories.CandidateCommitteeLinkFactory( + candidate_id=self.candidate_20.candidate_id, + committee_id=self.committees_20[0].committee_id, + committee_designation='P', + committee_type='P', + cand_election_year=self.next_cycle, + fec_election_year=self.current_cycle, + ) + + factories.CandidateCommitteeLinkFactory( + candidate_id=self.candidate_20.candidate_id, + committee_id=self.committees_20[0].committee_id, + committee_designation='P', + committee_type='P', + cand_election_year=self.next_cycle, + fec_election_year=self.next_cycle, + ) + def test_by_size(self): [ factories.ScheduleABySizeFactory( @@ -466,6 +541,17 @@ def test_totals(self): assert len(results) == 1 assert_dicts_subset(results[0], {'cycle': 2018, 'receipts': 150}) + # candidate_20 + results = self._results( + api.url_for( + TotalsCandidateView, + candidate_id=self.candidate_20.candidate_id, + cycle=self.current_cycle, + ) + ) + assert len(results) == 1 + assert_dicts_subset(results[0], {'cycle': self.current_cycle, 'receipts': 25000}) + def test_totals_full(self): results = self._results( api.url_for( @@ -477,3 +563,15 @@ def test_totals_full(self): ) assert len(results) == 1 assert_dicts_subset(results[0], {'cycle': 2012, 'receipts': 100}) + + # candidate_20 + results = self._results( + api.url_for( + TotalsCandidateView, + candidate_id=self.candidate_20.candidate_id, + cycle=self.next_cycle, + election_full='true', + ) + ) + assert len(results) == 1 + assert_dicts_subset(results[0], {'cycle': self.next_cycle, 'receipts': 55000}) diff --git a/tests/test_utils.py b/tests/test_utils.py index 754733a2f..e24c0cdd6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -82,8 +82,8 @@ def test_hide_null_candidate_totals(self): ] candidateHistory = [ - factories.CandidateHistoryFactory(candidate_id='C1234', two_year_period=2016, election_years=[2016], cycles=[2016],candidate_election_year=2016), - factories.CandidateHistoryFactory(candidate_id='C5678', two_year_period=2016, election_years=[2016], cycles=[2016],candidate_election_year=2016) + factories.CandidateHistoryFutureFactory(candidate_id='C1234', two_year_period=2016, election_years=[2016], cycles=[2016],candidate_election_year=2016), + factories.CandidateHistoryFutureFactory(candidate_id='C5678', two_year_period=2016, election_years=[2016], cycles=[2016],candidate_election_year=2016) ] candidateTotals = [ factories.CandidateTotalFactory(candidate_id='C1234', is_election=False, cycle=2016), @@ -188,8 +188,8 @@ def test_ignore_case(self): """ format the URL when the API_KEY is all uppecase """ - url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?API_KEY=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100" + url_before_format = "https://api.open.fec.gov/v1/schedules/schedule_e/by_candidate/?API_KEY=DEMO_KEY&candidate_id=S0AL00156&cycle=2018&election_full=false&per_page=100" url_after_format = utils.format_url(url_before_format) print('Formatted URL::', url_after_format) expected_url = "schedules/schedule_e/by_candidate/API_KEY-DEMO_KEY-candidate_id-S0AL00156-cycle-2018-election_full-false-per_page-100" - self.assertEqual(url_after_format, expected_url) \ No newline at end of file + self.assertEqual(url_after_format, expected_url) diff --git a/webservices/common/models/candidates.py b/webservices/common/models/candidates.py index 49ef185b1..dee020e5b 100644 --- a/webservices/common/models/candidates.py +++ b/webservices/common/models/candidates.py @@ -114,9 +114,23 @@ class CandidateHistory(BaseCandidate): candidate_inactive = db.Column(db.Boolean, doc='True indicates that a candidate is inactive.') active_through = db.Column(db.Integer, doc=docs.ACTIVE_THROUGH) +class CandidateHistoryWithFuture(BaseCandidate): + __tablename__ = 'ofec_candidate_history_with_future_election_mv' + + candidate_id = db.Column(db.String, primary_key=True, index=True, doc=docs.CANDIDATE_ID) + two_year_period = db.Column(db.Integer, primary_key=True, index=True, doc=docs.CANDIDATE_CYCLE) + candidate_election_year = db.Column(db.Integer, doc="The last year of the cycle for this election.") + address_city = db.Column(db.String(100), doc='City of candidate\'s address, as reported on their Form 2.') + address_state = db.Column(db.String(2), doc='State of candidate\'s address, as reported on their Form 2.') + address_street_1 = db.Column(db.String(200), doc='Street of candidate\'s address, as reported on their Form 2.') + address_street_2 = db.Column(db.String(200), doc='Additional street information of candidate\'s address, as reported on their Form 2.') + address_zip = db.Column(db.String(10), doc='Zip code of candidate\'s address, as reported on their Form 2.') + candidate_inactive = db.Column(db.Boolean, doc='True indicates that a candidate is inactive.') + active_through = db.Column(db.Integer, doc=docs.ACTIVE_THROUGH) + class CandidateTotal(db.Model): - __tablename__ = 'ofec_candidate_totals_with_0s_mv' + __tablename__ = 'ofec_candidate_totals_mv' candidate_id = db.Column(db.String, index=True, primary_key=True) election_year = db.Column(db.Integer, index=True, primary_key=True, autoincrement=True) cycle = db.Column(db.Integer, index=True, primary_key=True) diff --git a/webservices/flow.py b/webservices/flow.py index 4c3853977..862aef0c1 100644 --- a/webservices/flow.py +++ b/webservices/flow.py @@ -16,6 +16,7 @@ def get_graph(): 'candidate_flags', 'candidate_fulltext', 'candidate_history', + 'candidate_history_future', 'committee_detail', 'committee_fulltext', 'committee_history', @@ -51,7 +52,12 @@ def get_graph(): graph.add_nodes_from(MATERIALIZED_VIEWS) graph.add_edge('candidate_history', 'candidate_detail') + graph.add_edge('candidate_history', 'candidate_history_future') + graph.add_edge('candidate_detail', 'candidate_election') + graph.add_edge('candidate_detail', 'candidate_history_future') + + graph.add_edge('candidate_aggregates', 'candidate_history_future') graph.add_edge('committee_history', 'committee_detail') diff --git a/webservices/resources/candidate_aggregates.py b/webservices/resources/candidate_aggregates.py index 967d0728d..4d9ff16d4 100644 --- a/webservices/resources/candidate_aggregates.py +++ b/webservices/resources/candidate_aggregates.py @@ -162,7 +162,7 @@ def filter_range_fields(self, model): ] def build_query(self, **kwargs): - history = models.CandidateHistory + history = models.CandidateHistoryWithFuture query = db.session.query( history.__table__, models.CandidateTotal.__table__ diff --git a/webservices/utils.py b/webservices/utils.py index da9579622..cf68f74fa 100644 --- a/webservices/utils.py +++ b/webservices/utils.py @@ -394,6 +394,11 @@ def get_election_duration(column): else_=2, ) +def get_current_cycle(): + year = date.today().year + return year + year % 2 + + def get_elasticsearch_connection(): es_conn = env.get_service(name='fec-api-search56') if es_conn: