From 20404db72b04dee0bc578404bdb97225c50430d2 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 17:41:26 -0400 Subject: [PATCH 01/21] Add data_source to default file test data --- webapp/apps/test_assets/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/webapp/apps/test_assets/utils.py b/webapp/apps/test_assets/utils.py index 6a01fb5d..71bdd1c7 100644 --- a/webapp/apps/test_assets/utils.py +++ b/webapp/apps/test_assets/utils.py @@ -129,6 +129,7 @@ def get_file_post_data(start_year, reform_text, assumptions_text=None, quick_cal data = {u'docfile': tc_file, u'has_errors': [u'False'], u'start_year': unicode(start_year), + 'data_source': 'PUF', u'quick_calc': quick_calc, 'csrfmiddlewaretoken':'abc123'} From 172352d1b0cb23e4056b7d271ce55da985037252 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 17:47:15 -0400 Subject: [PATCH 02/21] Refactor taxbrain static test class so pytest can parametrize methods --- webapp/apps/taxbrain/tests/test_views.py | 152 +++++++++++------------ 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index 124af7f1..dbe2f24c 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -1,4 +1,3 @@ -from django.test import TestCase from django.test import Client from django.core.files.uploadedfile import SimpleUploadedFile from django.test.client import RequestFactory @@ -23,24 +22,23 @@ get_dropq_compute_from_module, get_taxbrain_model) +CLIENT = Client() + START_YEAR = 2016 @pytest.mark.usefixtures("r1", "assumptions_text", "warning_reform", "bad_reform", "test_coverage_fields") -class TaxBrainViewsTests(TestCase): +@pytest.mark.django_db +class TestTaxBrainViews(object): ''' Test the views of this app. ''' - def setUp(self): - # Every test needs a client. - self.client = Client() - def test_taxbrain_get(self): # Issue a GET request. - response = self.client.get('/taxbrain/') + response = CLIENT.get('/taxbrain/') # Check that the response is 200 OK. - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 def test_taxbrain_post(self): """ @@ -49,7 +47,7 @@ def test_taxbrain_post(self): data = get_post_data(START_YEAR) data[u'II_em'] = [u'4333'] data['data_source'] = ['PUF'] - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) truth_mods = {} @@ -63,7 +61,7 @@ def test_taxbrain_post_cps(self): data = get_post_data(START_YEAR) data[u'II_em'] = [u'4333'] data['data_source'] = ['CPS'] - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) truth_mods = {} @@ -82,13 +80,13 @@ def test_taxbrain_quick_calc_post(self): wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) current_dropq_worker_offset = wnc.current_offset - result = do_micro_sim(self.client, data, compute_count=1) + result = do_micro_sim(CLIENT, data, compute_count=1) wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) next_dropq_worker_offset = wnc.current_offset # Check that quick calc does not advance the counter - self.assertEqual(current_dropq_worker_offset, next_dropq_worker_offset) + assert current_dropq_worker_offset == next_dropq_worker_offset # Check that data was saved properly truth_mods = {START_YEAR: {"_ID_BenefitSurtax_Switch": @@ -106,7 +104,7 @@ def test_taxbrain_quick_calc_post(self): submit_data = {'csrfmiddlewaretoken':'abc123'} result = do_micro_sim( - self.client, + CLIENT, submit_data, compute_count=NUM_BUDGET_YEARS, post_url=post_url @@ -132,13 +130,13 @@ def test_taxbrain_quick_calc_post_cps(self): wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) current_dropq_worker_offset = wnc.current_offset - result = do_micro_sim(self.client, data, compute_count=1) + result = do_micro_sim(CLIENT, data, compute_count=1) wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) next_dropq_worker_offset = wnc.current_offset # Check that quick calc does not advance the counter - self.assertEqual(current_dropq_worker_offset, next_dropq_worker_offset) + assert current_dropq_worker_offset == next_dropq_worker_offset # Check that data was saved properly truth_mods = {START_YEAR: {"_ID_BenefitSurtax_Switch": @@ -156,7 +154,7 @@ def test_taxbrain_quick_calc_post_cps(self): submit_data = {'csrfmiddlewaretoken':'abc123'} result = do_micro_sim( - self.client, + CLIENT, submit_data, compute_count=NUM_BUDGET_YEARS, post_url=post_url @@ -166,20 +164,22 @@ def test_taxbrain_quick_calc_post_cps(self): check_posted_params(result['tb_dropq_compute'], truth_mods, str(START_YEAR), use_puf_not_cps=False) - def test_taxbrain_file_post_quick_calc(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_file_post_quick_calc(self, data_source): """ Using file-upload interface, test quick calculation post and full post from quick_calc page """ data = get_file_post_data(START_YEAR, self.r1, quick_calc=False) - + data['data_source'] = data_source + use_puf_not_cps = True if data_source == 'PUF' else False wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) current_dropq_worker_offset = wnc.current_offset post_url = '/taxbrain/file/' result = do_micro_sim( - self.client, + CLIENT, data, compute_count=1, post_url=post_url @@ -189,7 +189,7 @@ def test_taxbrain_file_post_quick_calc(self): next_dropq_worker_offset = wnc.current_offset # Check that quick calc does not advance the counter - self.assertEqual(current_dropq_worker_offset, next_dropq_worker_offset) + assert current_dropq_worker_offset == next_dropq_worker_offset # Check that data was saved properly truth_mods = taxcalc.Calculator.read_json_param_objects( @@ -206,7 +206,7 @@ def test_taxbrain_file_post_quick_calc(self): submit_data = {'csrfmiddlewaretoken':'abc123'} result = do_micro_sim( - self.client, + CLIENT, submit_data, compute_count=NUM_BUDGET_YEARS, post_url=post_url @@ -226,7 +226,7 @@ def test_back_to_back_quickcalc(self): data[u'ID_BenefitSurtax_Switch_6'] = ['0.0'] data[u'II_em'] = [u'4333'] - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was saved properly truth_mods = {START_YEAR: {"_ID_BenefitSurtax_Switch": @@ -238,12 +238,12 @@ def test_back_to_back_quickcalc(self): edit_micro = '/taxbrain/edit/{0}/?start_year={1}'.format(result["pk"], START_YEAR) - edit_page = self.client.get(edit_micro) - self.assertEqual(edit_page.status_code, 200) + edit_page = CLIENT.get(edit_micro) + assert edit_page.status_code == 200 next_csrf = str(edit_page.context['csrf_token']) data['csrfmiddlewaretoken'] = next_csrf - result2 = do_micro_sim(self.client, data) + result2 = do_micro_sim(CLIENT, data) check_posted_params(result2['tb_dropq_compute'], truth_mods, str(START_YEAR)) @@ -260,9 +260,9 @@ def test_taxbrain_post_no_behavior_entries(self): data = get_post_data(START_YEAR) data[u'BE_inc'] = [u'0.1'] - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that we get a 400 - self.assertEqual(response.status_code, 400) + assert response.status_code == 400 def test_taxbrain_nodes_down(self): @@ -276,7 +276,7 @@ def test_taxbrain_nodes_down(self): data[u'II_em'] = [u'4333'] result = do_micro_sim( - self.client, + CLIENT, data, tb_dropq_compute=dropq_compute ) @@ -300,14 +300,14 @@ def test_taxbrain_failed_job(self): data = get_post_data(START_YEAR) data[u'II_em'] = [u'4333'] - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 302) + assert response.status_code == 302 link_idx = response.url[:-1].rfind('/') - self.failUnless(response.url[:link_idx+1].endswith("taxbrain/")) - response = self.client.get(response.url) + assert response.url[:link_idx+1].endswith("taxbrain/") + response = CLIENT.get(response.url) # Make sure the failure message is in the response - self.failUnless("Your calculation failed" in str(response)) + assert "Your calculation failed" in str(response) @pytest.mark.xfail def test_taxbrain_has_growth_params(self): @@ -318,7 +318,7 @@ def test_taxbrain_has_growth_params(self): 'has_errors': [u'False'], 'growth_choice': u'factor_adjustment' } - do_micro_sim(self.client, reform) + do_micro_sim(CLIENT, reform) def test_taxbrain_edit_cpi_flags_show_correctly(self): @@ -328,15 +328,15 @@ def test_taxbrain_edit_cpi_flags_show_correctly(self): data[u'AMT_CG_brk2_cpi'] = u'False' data[u'AMEDT_ec_cpi'] = u'True' - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) edit_micro = '/taxbrain/edit/{0}/?start_year={1}'.format(result["pk"], START_YEAR) - edit_page = self.client.get(edit_micro) - self.assertEqual(edit_page.status_code, 200) + edit_page = CLIENT.get(edit_micro) + assert edit_page.status_code == 200 cpi_flag = edit_page.context['form']['AMT_CG_brk2_cpi'].field.widget.attrs['placeholder'] - self.assertEqual(cpi_flag, False) + assert cpi_flag == False cpi_flag = edit_page.context['form']['AMEDT_ec_cpi'].field.widget.attrs['placeholder'] - self.assertEqual(cpi_flag, True) + assert cpi_flag == True @@ -347,7 +347,7 @@ def test_taxbrain_edit_benefitsurtax_switch_show_correctly(self): data[u'II_em'] = [u'4333'] data['ID_BenefitSurtax_Switch_3'] = [u'True'] - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) out = OutputUrl.objects.get(pk=result["pk"]) tsi = TaxSaveInputs.objects.get(pk=out.model_pk) @@ -359,8 +359,8 @@ def test_taxbrain_edit_benefitsurtax_switch_show_correctly(self): # Now edit this page edit_micro = '/taxbrain/edit/{0}/?start_year={1}'.format(result["pk"], START_YEAR) - edit_page = self.client.get(edit_micro) - self.assertEqual(edit_page.status_code, 200) + edit_page = CLIENT.get(edit_micro) + assert edit_page.status_code == 200 # post some more data from the edit parameters page. Posting the # same data (switch_0) again looks a little funny, but this @@ -374,7 +374,7 @@ def test_taxbrain_edit_benefitsurtax_switch_show_correctly(self): 'csrfmiddlewaretoken': next_csrf} data2.update(mod) - result2 = do_micro_sim(self.client, data2) + result2 = do_micro_sim(CLIENT, data2) out2 = OutputUrl.objects.get(pk=result2["pk"]) tsi2 = TaxSaveInputs.objects.get(pk=out2.model_pk) @@ -393,7 +393,7 @@ def test_taxbrain_wildcard_params_with_validation_is_OK(self): mod = {u'II_brk1_0': [u'*, *, 15000'], u'II_brk2_cpi': u'False'} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was saved properly truth_mods = { @@ -421,9 +421,9 @@ def test_taxbrain_wildcard_params_with_validation_gives_error(self): u'II_brk2_cpi': u'False'} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True @@ -439,7 +439,7 @@ def test_taxbrain_spec_operators_in_validation_params_OK(self): u'cpi_offset': [u'<,-0.0025'], u'FICA_ss_trt': [u'< ,0.1,*,0.15,0.2']} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) truth_mods = { START_YEAR - 1: { @@ -462,7 +462,7 @@ def test_taxbrain_warning_on_widow_param(self): """ data = get_post_data(START_YEAR, _ID_BenefitSurtax_Switches=False) data[u'STD_3'] = ['10000'] - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) assert response.status_code == 200 @@ -482,9 +482,9 @@ def test_taxbrain_wildcard_in_validation_params_gives_error(self): u'II_brk2_cpi': u'False'} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True @@ -499,9 +499,9 @@ def test_taxbrain_improper_reverse_gives_error1(self): mod = {u'cpi_offset': [u'<,']} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True def test_taxbrain_improper_reverse_gives_error2(self): @@ -515,9 +515,9 @@ def test_taxbrain_improper_reverse_gives_error2(self): mod = {u'cpi_offset': [u'-0.002,<,-0.001']} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True def test_taxbrain_bool_separated_values(self): @@ -528,7 +528,7 @@ def test_taxbrain_bool_separated_values(self): data = get_post_data(2018, _ID_BenefitSurtax_Switches=False) data['DependentCredit_before_CTC'] = [u'True,*, FALSE,tRUe,*,0'] - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was submitted properly truth_mods = { @@ -555,7 +555,7 @@ def test_taxbrain_rt_capital_gain_goes_to_amt(self): 'CG_brk2_2': [u'243475.0'], 'CG_brk2_3': [u'451000.0']} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) out2 = OutputUrl.objects.get(pk=result["pk"]) tsi2 = TaxSaveInputs.objects.get(pk=out2.model_pk) @@ -637,11 +637,11 @@ def test_taxbrain_rt_to_passthrough(self): data.update(values6) data.update(values7) #TODO: check how values are saved - do_micro_sim(self.client, data) + do_micro_sim(CLIENT, data) def test_taxbrain_file_post_only_reform(self): data = get_file_post_data(START_YEAR, self.r1) - do_micro_sim(self.client, data, post_url="/taxbrain/file/") + do_micro_sim(CLIENT, data, post_url="/taxbrain/file/") def test_taxbrain_file_post_reform_and_assumptions(self): @@ -649,7 +649,7 @@ def test_taxbrain_file_post_reform_and_assumptions(self): self.r1, self.assumptions_text) - do_micro_sim(self.client, data, post_url="/taxbrain/file/") + do_micro_sim(CLIENT, data, post_url="/taxbrain/file/") def test_taxbrain_view_old_data_model(self): @@ -690,8 +690,8 @@ def test_taxbrain_bad_expression(self): u'II_brk2_0': [u'*, *, 39500']} data.update(mod) - response = self.client.post('/taxbrain/', data) - self.assertEqual(response.status_code, 200) + response = CLIENT.post('/taxbrain/', data) + assert response.status_code == 200 assert response.context['has_errors'] is True @@ -707,9 +707,9 @@ def test_taxbrain_error_reform_file(self): data = get_file_post_data(START_YEAR, self.bad_reform) #TODO: make sure still not allowed to submit on second submission - response = self.client.post('/taxbrain/file/', data) + response = CLIENT.post('/taxbrain/file/', data) # Check that no redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True assert any(['_II_brk1_4' in msg and '2024' in msg for msg in response.context['errors']]) @@ -728,7 +728,7 @@ def test_taxbrain_error_reform_file(self): 'start_year': START_YEAR } - response = self.client.post('/taxbrain/file/', data2) + response = CLIENT.post('/taxbrain/file/', data2) assert response.status_code == 200 @@ -743,9 +743,9 @@ def test_taxbrain_warning_reform_file(self): data = get_file_post_data(START_YEAR, self.warning_reform) - response = self.client.post('/taxbrain/file/', data) + response = CLIENT.post('/taxbrain/file/', data) # Check that no redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True assert any(['_STD_0' in msg and '2023' in msg for msg in response.context['errors']]) @@ -764,7 +764,7 @@ def test_taxbrain_warning_reform_file(self): 'start_year': START_YEAR } - result = do_micro_sim(self.client, data2, post_url='/taxbrain/file/') + result = do_micro_sim(CLIENT, data2, post_url='/taxbrain/file/') truth_mods = { 2020: { @@ -786,9 +786,9 @@ def test_taxbrain_reform_file_file_swap(self): data = get_file_post_data(start_year, self.warning_reform) - response = self.client.post('/taxbrain/file/', data) + response = CLIENT.post('/taxbrain/file/', data) # Check that no redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True assert any(['_STD_0' in msg and '2023' in msg for msg in response.context['errors']]) @@ -812,7 +812,7 @@ def test_taxbrain_reform_file_file_swap(self): data2['docfile'] = data_file['docfile'] data2['assumpfile'] = data_file['assumpfile'] - result = do_micro_sim(self.client, data2, post_url='/taxbrain/file/') + result = do_micro_sim(CLIENT, data2, post_url='/taxbrain/file/') dropq_compute = result['tb_dropq_compute'] user_mods = json.loads(dropq_compute.last_posted["user_mods"]) @@ -833,9 +833,9 @@ def test_taxbrain_reform_file_file_swap_no_assump(self): data = get_file_post_data(start_year, self.warning_reform) - response = self.client.post('/taxbrain/file/', data) + response = CLIENT.post('/taxbrain/file/', data) # Check that no redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True assert any(['_STD_0' in msg and '2023' in msg for msg in response.context['errors']]) @@ -856,7 +856,7 @@ def test_taxbrain_reform_file_file_swap_no_assump(self): data_file = get_file_post_data(START_YEAR, self.r1) data2['docfile'] = data_file['docfile'] - result = do_micro_sim(self.client, data2, post_url='/taxbrain/file/') + result = do_micro_sim(CLIENT, data2, post_url='/taxbrain/file/') dropq_compute = result['tb_dropq_compute'] user_mods = json.loads(dropq_compute.last_posted["user_mods"]) @@ -870,7 +870,7 @@ def test_taxbrain_up_to_2018(self): mod = {u'II_brk1_0': [u'*, *, 15000'], u'II_brk2_cpi': u'False'} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was saved properly truth_mods = { @@ -887,7 +887,7 @@ def test_taxbrain_file_up_to_2018(self): post_url = '/taxbrain/file/' result = do_micro_sim( - self.client, + CLIENT, data, post_url=post_url ) @@ -927,7 +927,7 @@ def test_taxbrain_old_data_gives_deprecation_errors(self): pk = unique_url.pk edit_micro = "/taxbrain/edit/{}/?start_year={}".format(pk, start_year) - response = self.client.get(edit_micro) + response = CLIENT.get(edit_micro) assert response.status_code == 200 assert response.context['has_errors'] is False From 6d263262b68b25ef05326dc3d7ff33ed7e5a902c Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 17:54:03 -0400 Subject: [PATCH 03/21] Add data_source option for test utils check parameter function --- webapp/apps/test_assets/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webapp/apps/test_assets/utils.py b/webapp/apps/test_assets/utils.py index 71bdd1c7..ddaa72f0 100644 --- a/webapp/apps/test_assets/utils.py +++ b/webapp/apps/test_assets/utils.py @@ -2,6 +2,7 @@ import os import sys import ast +import six from ..taxbrain.compute import MockCompute @@ -74,7 +75,7 @@ def do_micro_sim(client, data, tb_dropq_compute=None, dyn_dropq_compute=None, def check_posted_params(mock_compute, params_to_check, start_year, - use_puf_not_cps=True): + use_puf_not_cps=True, data_source=None): """ Make sure posted params match expected results user_mods: parameters that are actually passed to taxcalc @@ -84,6 +85,8 @@ def check_posted_params(mock_compute, params_to_check, start_year, last_posted = mock_compute.last_posted user_mods = json.loads(last_posted["user_mods"]) assert last_posted["first_budget_year"] == int(start_year) + if data_source is not None: + use_puf_not_cps = True if data_source == 'PUF' else False assert last_posted["use_puf_not_cps"] == use_puf_not_cps for year in params_to_check: for param in params_to_check[year]: From 9371b5c3d4e57c73bad93e0a72d2d5bad3dcb0b5 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 17:54:28 -0400 Subject: [PATCH 04/21] Parametrize test_taxbrain_quick_calc_post --- webapp/apps/taxbrain/tests/test_views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index dbe2f24c..a4c006a7 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -68,7 +68,8 @@ def test_taxbrain_post_cps(self): check_posted_params(result['tb_dropq_compute'], truth_mods, str(START_YEAR), use_puf_not_cps=False) - def test_taxbrain_quick_calc_post(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_quick_calc_post(self, data_source): "Test quick calculation post and full post from quick_calc page" # switches 0, 4, 6 are False data = get_post_data(START_YEAR, quick_calc=True) @@ -77,6 +78,7 @@ def test_taxbrain_quick_calc_post(self): data[u'ID_BenefitSurtax_Switch_6'] = ['0.0'] data[u'II_em'] = [u'4333'] data[u'ID_AmountCap_Switch_0'] = [u'0'] + data['data_source'] = data_source wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) current_dropq_worker_offset = wnc.current_offset @@ -96,7 +98,7 @@ def test_taxbrain_quick_calc_post(self): "_II_em": [4333.0]} } check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) # reset worker node count without clearing MockCompute session result['tb_dropq_compute'].reset_count() @@ -112,7 +114,7 @@ def test_taxbrain_quick_calc_post(self): # Check that data was saved properly check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) def test_taxbrain_quick_calc_post_cps(self): """ From 08304eb8bd315fb09e364fbf873c01299c86d3b0 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 18:18:38 -0400 Subject: [PATCH 05/21] Parameterize taxbrain tests to use both data sources --- webapp/apps/taxbrain/tests/test_views.py | 147 ++++------------------- 1 file changed, 25 insertions(+), 122 deletions(-) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index a4c006a7..17894e1b 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -40,33 +40,20 @@ def test_taxbrain_get(self): # Check that the response is 200 OK. assert response.status_code == 200 - def test_taxbrain_post(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_post(self, data_source): """ submit simple reform """ data = get_post_data(START_YEAR) data[u'II_em'] = [u'4333'] - data['data_source'] = ['PUF'] - result = do_micro_sim(CLIENT, data) - - truth_mods = {} - - check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=True) - - def test_taxbrain_post_cps(self): - """ - submit simple reform with CPS as data source - """ - data = get_post_data(START_YEAR) - data[u'II_em'] = [u'4333'] - data['data_source'] = ['CPS'] + data['data_source'] = data_source result = do_micro_sim(CLIENT, data) truth_mods = {} check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=False) + str(START_YEAR), data_source=data_source) @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) def test_taxbrain_quick_calc_post(self, data_source): @@ -116,56 +103,6 @@ def test_taxbrain_quick_calc_post(self, data_source): check_posted_params(result['tb_dropq_compute'], truth_mods, str(START_YEAR), data_source=data_source) - def test_taxbrain_quick_calc_post_cps(self): - """ - Test quick calculation post and full post from quick_calc page using - cps data - """ - # switches 0, 4, 6 are False - data = get_post_data(START_YEAR, quick_calc=True) - data[u'ID_BenefitSurtax_Switch_0'] = ['False'] - data[u'ID_BenefitSurtax_Switch_4'] = ['0'] - data[u'ID_BenefitSurtax_Switch_6'] = ['0.0'] - data[u'II_em'] = [u'4333'] - data[u'ID_AmountCap_Switch_0'] = [u'0'] - data['data_source'] = ['CPS'] - wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) - current_dropq_worker_offset = wnc.current_offset - - result = do_micro_sim(CLIENT, data, compute_count=1) - - wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) - next_dropq_worker_offset = wnc.current_offset - - # Check that quick calc does not advance the counter - assert current_dropq_worker_offset == next_dropq_worker_offset - - # Check that data was saved properly - truth_mods = {START_YEAR: {"_ID_BenefitSurtax_Switch": - [[0.0, 1.0, 1.0, 1.0, 0.0, 1.0, 0.0]], - "_ID_AmountCap_Switch": - [[0, 1, 1, 1, 1, 1, True]], - "_II_em": [4333.0]} - } - check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=False) - - # reset worker node count without clearing MockCompute session - result['tb_dropq_compute'].reset_count() - post_url = '/taxbrain/submit/{0}/'.format(result['pk']) - submit_data = {'csrfmiddlewaretoken':'abc123'} - - result = do_micro_sim( - CLIENT, - submit_data, - compute_count=NUM_BUDGET_YEARS, - post_url=post_url - ) - - # Check that data was saved properly - check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=False) - @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) def test_taxbrain_file_post_quick_calc(self, data_source): """ @@ -218,8 +155,8 @@ def test_taxbrain_file_post_quick_calc(self, data_source): check_posted_params(result['tb_dropq_compute'], truth_mods, str(START_YEAR)) - - def test_back_to_back_quickcalc(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_back_to_back_quickcalc(self, data_source): "Test back to back quick calc posts" # switches 0, 4, 6 are False data = get_post_data(START_YEAR, quick_calc=True) @@ -227,6 +164,7 @@ def test_back_to_back_quickcalc(self): data[u'ID_BenefitSurtax_Switch_4'] = ['0'] data[u'ID_BenefitSurtax_Switch_6'] = ['0.0'] data[u'II_em'] = [u'4333'] + data['data_source'] = data_source result = do_micro_sim(CLIENT, data) @@ -236,7 +174,7 @@ def test_back_to_back_quickcalc(self): "_II_em": [4333.0]} } check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) edit_micro = '/taxbrain/edit/{0}/?start_year={1}'.format(result["pk"], START_YEAR) @@ -248,7 +186,7 @@ def test_back_to_back_quickcalc(self): result2 = do_micro_sim(CLIENT, data) check_posted_params(result2['tb_dropq_compute'], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) @pytest.mark.xfail def test_taxbrain_post_no_behavior_entries(self): @@ -775,8 +713,11 @@ def test_taxbrain_warning_reform_file(self): } check_posted_params(result['tb_dropq_compute'], truth_mods, START_YEAR) - - def test_taxbrain_reform_file_file_swap(self): + @pytest.mark.parametrize( + 'data_source,use_assumptions', + [('PUF', True), ('CPS', False)] + ) + def test_taxbrain_reform_file_file_swap(self, data_source, use_assumptions): """ POST a reform file that causes warnings, swap files, and make sure swapped files are used. See PB issue #630 and #761 @@ -785,8 +726,13 @@ def test_taxbrain_reform_file_file_swap(self): from webapp.apps.taxbrain.models import JSONReformTaxCalculator as js #Monkey patch to mock out running of compute jobs get_dropq_compute_from_module('webapp.apps.taxbrain.views') + if use_assumptions: + assumptions_text = self.assumptions_text + else: + assumptions_text = None - data = get_file_post_data(start_year, self.warning_reform) + data = get_file_post_data(start_year, self.warning_reform, + assumptions_text) response = CLIENT.post('/taxbrain/file/', data) # Check that no redirect happens @@ -810,61 +756,18 @@ def test_taxbrain_reform_file_file_swap(self): } data_file = get_file_post_data(START_YEAR, self.r1, - self.assumptions_text) - data2['docfile'] = data_file['docfile'] - data2['assumpfile'] = data_file['assumpfile'] - - result = do_micro_sim(CLIENT, data2, post_url='/taxbrain/file/') - - dropq_compute = result['tb_dropq_compute'] - user_mods = json.loads(dropq_compute.last_posted["user_mods"]) - assert user_mods["behavior"][str(2018)]["_BE_sub"][0] == 1.0 - truth_mods = {2018: {'_II_em': [8000.0]}} - check_posted_params(dropq_compute, truth_mods, start_year) - - - def test_taxbrain_reform_file_file_swap_no_assump(self): - """ - POST a reform file that causes warnings, swap files, and make sure - swapped files are used. See PB issue #630 and #761 - """ - start_year = 2017 - from webapp.apps.taxbrain.models import JSONReformTaxCalculator as js - #Monkey patch to mock out running of compute jobs - get_dropq_compute_from_module('webapp.apps.taxbrain.views') - - data = get_file_post_data(start_year, self.warning_reform) - - response = CLIENT.post('/taxbrain/file/', data) - # Check that no redirect happens - assert response.status_code == 200 - assert response.context['has_errors'] is True - assert any(['_STD_0' in msg and '2023' in msg - for msg in response.context['errors']]) - - # get most recent object - objects = js.objects.order_by('id') - obj = objects[len(objects) - 1] - - next_token = str(response.context['csrf_token']) - - form_id = obj.id - data2 = { - 'csrfmiddlewaretoken': next_token, - 'form_id': form_id, - 'has_errors': [u'True'], - 'start_year': start_year - } - data_file = get_file_post_data(START_YEAR, self.r1) + assumptions_text) data2['docfile'] = data_file['docfile'] result = do_micro_sim(CLIENT, data2, post_url='/taxbrain/file/') dropq_compute = result['tb_dropq_compute'] user_mods = json.loads(dropq_compute.last_posted["user_mods"]) + if use_assumptions: + assert user_mods["behavior"][str(2018)]["_BE_sub"][0] == 1.0 truth_mods = {2018: {'_II_em': [8000.0]}} - check_posted_params(dropq_compute, truth_mods, start_year) - + check_posted_params(dropq_compute, truth_mods, start_year, + data_source=data_source) def test_taxbrain_up_to_2018(self): start_year = 2018 From 9fec0bb76a12d469264d6df3e6b26a300348935e Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 18:26:41 -0400 Subject: [PATCH 06/21] Use GET parameters for regular post --- webapp/apps/taxbrain/tests/test_views.py | 32 ++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index 17894e1b..b98d9e60 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -47,8 +47,10 @@ def test_taxbrain_post(self, data_source): """ data = get_post_data(START_YEAR) data[u'II_em'] = [u'4333'] - data['data_source'] = data_source - result = do_micro_sim(CLIENT, data) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + result = do_micro_sim(CLIENT, data, post_url=url) truth_mods = {} @@ -579,17 +581,27 @@ def test_taxbrain_rt_to_passthrough(self): #TODO: check how values are saved do_micro_sim(CLIENT, data) - def test_taxbrain_file_post_only_reform(self): - data = get_file_post_data(START_YEAR, self.r1) - do_micro_sim(CLIENT, data, post_url="/taxbrain/file/") - - - def test_taxbrain_file_post_reform_and_assumptions(self): + @pytest.mark.parametrize( + 'data_source,use_assumptions', + [('PUF', True), ('CPS', False)] + ) + def test_taxbrain_post_file(self, data_source, use_assumptions): + if use_assumptions: + assumptions_text = self.assumptions_text + else: + assumptions_text = None data = get_file_post_data(START_YEAR, self.r1, - self.assumptions_text) + assumptions_text) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + result = do_micro_sim(CLIENT, data, post_url=url) - do_micro_sim(CLIENT, data, post_url="/taxbrain/file/") + truth_mods = {} + + check_posted_params(result['tb_dropq_compute'], truth_mods, + str(START_YEAR), data_source=data_source) def test_taxbrain_view_old_data_model(self): From 74d646d969e1290192011a74464ca34abb9a3f99 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 18:53:52 -0400 Subject: [PATCH 07/21] Update more tests to use GET params for start year and data source --- webapp/apps/taxbrain/tests/test_views.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index b98d9e60..528f45d5 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -112,12 +112,12 @@ def test_taxbrain_file_post_quick_calc(self, data_source): post from quick_calc page """ data = get_file_post_data(START_YEAR, self.r1, quick_calc=False) - data['data_source'] = data_source - use_puf_not_cps = True if data_source == 'PUF' else False + data.pop('data_source') + data.pop('start_year') wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) current_dropq_worker_offset = wnc.current_offset - post_url = '/taxbrain/file/' + post_url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) result = do_micro_sim( CLIENT, @@ -139,7 +139,7 @@ def test_taxbrain_file_post_quick_calc(self, data_source): ) truth_mods = truth_mods["policy"] check_posted_params(result["tb_dropq_compute"], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) # reset worker node count without clearing MockCompute session result['tb_dropq_compute'].reset_count() @@ -155,7 +155,7 @@ def test_taxbrain_file_post_quick_calc(self, data_source): # Check that data was saved properly check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) def test_back_to_back_quickcalc(self, data_source): @@ -745,11 +745,15 @@ def test_taxbrain_reform_file_file_swap(self, data_source, use_assumptions): data = get_file_post_data(start_year, self.warning_reform, assumptions_text) - - response = CLIENT.post('/taxbrain/file/', data) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(start_year, data_source) + response = CLIENT.post(url, data) # Check that no redirect happens assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['data_source'] == data_source + assert response.context['start_year'] == str(start_year) assert any(['_STD_0' in msg and '2023' in msg for msg in response.context['errors']]) @@ -764,14 +768,13 @@ def test_taxbrain_reform_file_file_swap(self, data_source, use_assumptions): 'csrfmiddlewaretoken': next_token, 'form_id': form_id, 'has_errors': [u'True'], - 'start_year': start_year } data_file = get_file_post_data(START_YEAR, self.r1, assumptions_text) data2['docfile'] = data_file['docfile'] - result = do_micro_sim(CLIENT, data2, post_url='/taxbrain/file/') + result = do_micro_sim(CLIENT, data2, post_url=url) dropq_compute = result['tb_dropq_compute'] user_mods = json.loads(dropq_compute.last_posted["user_mods"]) From 4539ddeee71517a341d6f6cf3a9a355156843557 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 18:54:24 -0400 Subject: [PATCH 08/21] Add data source to file upload page --- webapp/apps/taxbrain/views.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/webapp/apps/taxbrain/views.py b/webapp/apps/taxbrain/views.py index 7c92aabe..0f293a97 100644 --- a/webapp/apps/taxbrain/views.py +++ b/webapp/apps/taxbrain/views.py @@ -97,7 +97,7 @@ def normalize(x): def save_model(url, request, model, json_reform, has_errors, start_year, do_full_calc, is_file, reform_dict, assumptions_dict, reform_text, assumptions_text, submitted_ids, - max_q_length, user): + max_q_length, user, data_source): """ Save user input data returns OutputUrl object @@ -108,6 +108,7 @@ def save_model(url, request, model, json_reform, has_errors, start_year, model.job_ids = denormalize(submitted_ids) model.json_text = json_reform model.first_year = int(start_year) + model.data_source = data_source model.quick_calc = not do_full_calc model.save() @@ -158,7 +159,8 @@ def submit_reform(request, user=None, json_reform_id=None): request_files = request.FILES # which file to use, front-end not yet implemented - if fields.get('data_source', 'PUF') == 'PUF': + data_source = fields.get('data_source', 'PUF') + if data_source == 'PUF': use_puf_not_cps = True else: use_puf_not_cps = False @@ -322,6 +324,7 @@ def submit_reform(request, user=None, json_reform_id=None): no_inputs, not is_valid]), 'errors_warnings': errors_warnings, 'start_year': start_year, + 'data_source': data_source, 'do_full_calc': do_full_calc, 'is_file': is_file, 'reform_dict': reform_dict, @@ -371,16 +374,21 @@ def file_input(request): form_id = None start_year = START_YEAR + data_source = DEFAULT_SOURCE errors = [] has_errors = False - print('post', request.POST) - print('get', request.GET) print('files', request.FILES) if request.method == 'POST': + print('method=POST get', request.GET) + print('method=POST post', request.POST) # save start_year start_year = (request.GET.get('start_year', None) or request.POST.get('start_year', None)) assert start_year is not None + data_source = (request.GET.get('data_source', None) or + request.POST.get('start_year', None)) + assert data_source is not None + # File is not submitted if 'docfile' not in dict(request.FILES) and form_id is None: errors = ["Please specify a tax-law change before submitting."] @@ -412,11 +420,16 @@ def file_input(request): else: return redirect(unique_url) else: - # GET request, load a default form + # Probably a GET request, load a default form + print('method=GET get', request.GET) + print('method=GET post', request.POST) params = parse_qs(urlparse(request.build_absolute_uri()).query) if 'start_year' in params and params['start_year'][0] in START_YEARS: start_year = params['start_year'][0] + if 'data_source' in params and params['data_source'][0] in DATA_SOURCES: + data_source = params['data_source'][0] + json_reform = None init_context = { @@ -428,6 +441,8 @@ def file_input(request): 'params': None, 'start_years': START_YEARS, 'start_year': start_year, + 'data_sources': DATA_SOURCES, + 'data_source': data_source, 'enable_quick_calc': ENABLE_QUICK_CALC, 'input_type': "file" } @@ -546,6 +561,7 @@ def submit_micro(request, pk): assumptions_dict = json.loads(model.json_text.assumption_text) user_mods = dict({'policy': reform_dict}, **assumptions_dict) + print('data source', model.data_source) data = {'user_mods': json.dumps(user_mods), 'first_budget_year': int(start_year), 'start_budget_year': 0, @@ -563,6 +579,7 @@ def submit_micro(request, pk): 'json_reform': json_reform, 'has_errors': False, 'start_year': start_year, + 'data_source': model.data_source, 'do_full_calc': True, 'is_file': is_file, 'reform_dict': reform_dict, From e85931453af5701e05b8b9ec45823ca5a8017838 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 19:00:48 -0400 Subject: [PATCH 09/21] Make sure start year is saved when warnings are shown --- webapp/apps/taxbrain/tests/test_views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index 528f45d5..8bd7a939 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -683,8 +683,8 @@ def test_taxbrain_error_reform_file(self): response = CLIENT.post('/taxbrain/file/', data2) assert response.status_code == 200 - - def test_taxbrain_warning_reform_file(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_warning_reform_file(self, data_source): """ POST a reform file that causes warnings and check that re-submission is allowed. See PB issue #630 and #761 @@ -694,11 +694,15 @@ def test_taxbrain_warning_reform_file(self): get_dropq_compute_from_module('webapp.apps.taxbrain.views') data = get_file_post_data(START_YEAR, self.warning_reform) - - response = CLIENT.post('/taxbrain/file/', data) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + response = CLIENT.post(url, data) # Check that no redirect happens assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['start_year'] == str(START_YEAR) + assert response.context['data_source'] == data_source assert any(['_STD_0' in msg and '2023' in msg for msg in response.context['errors']]) @@ -713,17 +717,17 @@ def test_taxbrain_warning_reform_file(self): 'csrfmiddlewaretoken': next_token, 'form_id': form_id, 'has_errors': [u'True'], - 'start_year': START_YEAR } - result = do_micro_sim(CLIENT, data2, post_url='/taxbrain/file/') + result = do_micro_sim(CLIENT, data2, post_url=url) truth_mods = { 2020: { "_STD": [[1000, 24981.84, 12490.92, 18736.38, 24981.84]] } } - check_posted_params(result['tb_dropq_compute'], truth_mods, START_YEAR) + check_posted_params(result['tb_dropq_compute'], truth_mods, START_YEAR, + data_source=data_source) @pytest.mark.parametrize( 'data_source,use_assumptions', From c708e4c7688256ae96ab3faa487561076b59704f Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Tue, 13 Mar 2018 19:03:46 -0400 Subject: [PATCH 10/21] Make sure start year is saved when errors are shown --- webapp/apps/taxbrain/tests/test_views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index 8bd7a939..599bd23b 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -646,8 +646,8 @@ def test_taxbrain_bad_expression(self): assert response.status_code == 200 assert response.context['has_errors'] is True - - def test_taxbrain_error_reform_file(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_error_reform_file(self, data_source): """ POST a reform file that causes errors. See PB issue #630 """ @@ -657,12 +657,15 @@ def test_taxbrain_error_reform_file(self): get_dropq_compute_from_module('webapp.apps.taxbrain.views') data = get_file_post_data(START_YEAR, self.bad_reform) - - #TODO: make sure still not allowed to submit on second submission - response = CLIENT.post('/taxbrain/file/', data) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + response = CLIENT.post(url, data) # Check that no redirect happens assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['data_source'] == data_source + assert response.context['start_year'] == str(START_YEAR) assert any(['_II_brk1_4' in msg and '2024' in msg for msg in response.context['errors']]) @@ -677,11 +680,12 @@ def test_taxbrain_error_reform_file(self): 'csrfmiddlewaretoken': next_token, 'form_id': form_id, 'has_errors': [u'True'], - 'start_year': START_YEAR } - response = CLIENT.post('/taxbrain/file/', data2) + response = CLIENT.post(url, data2) assert response.status_code == 200 + assert response.context['data_source'] == data_source + assert response.context['start_year'] == str(START_YEAR) @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) def test_taxbrain_warning_reform_file(self, data_source): From 0827fa83f9af5760e08a5e63a5c2df175057dc22 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 10:01:18 -0400 Subject: [PATCH 11/21] Add backend for data source in file input and refactor to use namedtuples in process_reform --- webapp/apps/taxbrain/forms.py | 2 + webapp/apps/taxbrain/tests/test_views.py | 17 +- webapp/apps/taxbrain/views.py | 199 +++++++++++------------ 3 files changed, 113 insertions(+), 105 deletions(-) diff --git a/webapp/apps/taxbrain/forms.py b/webapp/apps/taxbrain/forms.py index 61a779c0..7c512570 100644 --- a/webapp/apps/taxbrain/forms.py +++ b/webapp/apps/taxbrain/forms.py @@ -199,6 +199,8 @@ def set_form(defaults): class TaxBrainForm(PolicyBrainForm, ModelForm): def __init__(self, first_year, use_puf_not_cps, *args, **kwargs): + # the start year and the data source form object for later access. + # This should be refactored into `process_model` if first_year is None: first_year = START_YEAR self._first_year = int(first_year) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index 599bd23b..d46e0dae 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -403,12 +403,18 @@ def test_taxbrain_warning_on_widow_param(self): Test case where error is added on undisplayed parameter """ data = get_post_data(START_YEAR, _ID_BenefitSurtax_Switches=False) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/?start_year={0}&data_source={1}'.format(START_YEAR, 'PUF') data[u'STD_3'] = ['10000'] - response = CLIENT.post('/taxbrain/', data) + response = CLIENT.post(url, data) assert response.status_code == 200 + assert response.context['start_year'] == str(START_YEAR) + assert response.context['data_source'] == 'PUF' - def test_taxbrain_wildcard_in_validation_params_gives_error(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_wildcard_in_validation_params_gives_error(self, data_source): """ Set upper threshold for income tax bracket 1 to *, 38000 Set upper threshold for income tax bracket 2 to *, *, 39500 @@ -419,15 +425,20 @@ def test_taxbrain_wildcard_in_validation_params_gives_error(self): get_dropq_compute_from_module('webapp.apps.taxbrain.views') data = get_post_data(START_YEAR, _ID_BenefitSurtax_Switches=False) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) mod = {u'II_brk1_0': [u'*, 38000'], u'II_brk2_0': [u'*, *, 39500'], u'II_brk2_cpi': u'False'} data.update(mod) - response = CLIENT.post('/taxbrain/', data) + response = CLIENT.post(url, data) # Check that redirect happens assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['start_year'] == str(START_YEAR) + assert response.context['data_source'] == data_source def test_taxbrain_improper_reverse_gives_error1(self): diff --git a/webapp/apps/taxbrain/views.py b/webapp/apps/taxbrain/views.py index 0f293a97..bb995580 100644 --- a/webapp/apps/taxbrain/views.py +++ b/webapp/apps/taxbrain/views.py @@ -56,6 +56,7 @@ from ..formatters import get_version from .param_formatters import (get_reform_from_file, get_reform_from_gui, to_json_reform, append_errors_warnings) +from submit_data import PostMeta, BadPost from django.conf import settings WEBAPP_VERSION = settings.WEBAPP_VERSION @@ -94,33 +95,31 @@ def normalize(x): ans = [i.split('#') for i in x] return ans -def save_model(url, request, model, json_reform, has_errors, start_year, - do_full_calc, is_file, reform_dict, assumptions_dict, - reform_text, assumptions_text, submitted_ids, - max_q_length, user, data_source): +def save_model(post_meta): """ Save user input data returns OutputUrl object """ + model = post_meta.model # create model for file_input case if model is None: model = TaxSaveInputs() - model.job_ids = denormalize(submitted_ids) - model.json_text = json_reform - model.first_year = int(start_year) - model.data_source = data_source - model.quick_calc = not do_full_calc + model.job_ids = denormalize(post_meta.submitted_ids) + model.json_text = post_meta.json_reform + model.first_year = int(post_meta.start_year) + model.data_source = post_meta.data_source + model.quick_calc = not post_meta.do_full_calc model.save() # create OutputUrl object - if url is None: + if post_meta.url is None: unique_url = OutputUrl() else: - unique_url = url - if user: - unique_url.user = user - elif request and request.user.is_authenticated(): - current_user = User.objects.get(pk=request.user.id) + unique_url = post_meta.url + if post_meta.user: + unique_url.user = post_meta.user + elif post_meta.request and post_meta.request.user.is_authenticated(): + current_user = User.objects.get(pk=post_meta.request.user.id) unique_url.user = current_user if unique_url.taxcalc_vers is None: @@ -131,7 +130,7 @@ def save_model(url, request, model, json_reform, has_errors, start_year, unique_url.unique_inputs = model unique_url.model_pk = model.pk cur_dt = datetime.datetime.utcnow() - future_offset_seconds = ((2 + max_q_length) * JOB_PROC_TIME_IN_SECONDS) + future_offset_seconds = ((2 + post_meta.max_q_length) * JOB_PROC_TIME_IN_SECONDS) future_offset = datetime.timedelta(seconds=future_offset_seconds) expected_completion = cur_dt + future_offset unique_url.exp_comp_datetime = expected_completion @@ -193,10 +192,10 @@ def submit_reform(request, user=None, json_reform_id=None): json_reform = JSONReformTaxCalculator.objects.get(id=int(json_reform_id)) except Exception as e: msg = "ID {} is not in JSON reform database".format(json_reform_id) - http_response = HttpResponse(msg, status=400) - return {'HttpResponse': http_response, - 'has_errors': True} - + return BadPost( + http_response_404=HttpResponse(msg, status=400), + has_errors= True + ) reform_dict = json.loads(json_reform.reform_text) assumptions_dict = json.loads(json_reform.assumption_text) reform_text = json_reform.raw_reform_text @@ -233,8 +232,10 @@ def submit_reform(request, user=None, json_reform_id=None): # If an attempt is made to post data we don't accept # raise a 400 if personal_inputs.non_field_errors(): - return {'HttpResponse': HttpResponse("Bad Input!", status=400), - 'has_errors': True} + return BadPost( + http_response_404=HttpResponse("Bad Input!", status=400), + has_errors= True + ) is_valid = personal_inputs.is_valid() if is_valid: model = personal_inputs.save(commit=False) @@ -316,23 +317,28 @@ def submit_reform(request, user=None, json_reform_id=None): data ) - return {'personal_inputs': personal_inputs, - 'json_reform': json_reform, - 'model': model, - 'stop_submission': stop_submission, - 'has_errors': any([taxcalc_errors, taxcalc_warnings, + return PostMeta( + request=request, + personal_inputs=personal_inputs, + json_reform=json_reform, + model=model, + stop_submission=stop_submission, + has_errors=any([taxcalc_errors, taxcalc_warnings, no_inputs, not is_valid]), - 'errors_warnings': errors_warnings, - 'start_year': start_year, - 'data_source': data_source, - 'do_full_calc': do_full_calc, - 'is_file': is_file, - 'reform_dict': reform_dict, - 'assumptions_dict': assumptions_dict, - 'reform_text': reform_text, - 'assumptions_text': assumptions_text, - 'submitted_ids': submitted_ids, - 'max_q_length': max_q_length} + errors_warnings=errors_warnings, + start_year=start_year, + data_source=data_source, + do_full_calc=do_full_calc, + is_file=is_file, + reform_dict=reform_dict, + assumptions_dict=assumptions_dict, + reform_text=reform_text, + assumptions_text=assumptions_text, + submitted_ids=submitted_ids, + max_q_length=max_q_length, + user=user, + url=None + ) def process_reform(request, user=None, **kwargs): @@ -346,22 +352,16 @@ def process_reform(request, user=None, **kwargs): boolean variable indicating whether this reform has errors or not """ - args = submit_reform(request, user=user, **kwargs) - if 'HttpResponse' in args: - return args['HttpResponse'], None, args['has_errors'], None - - args['request'] = request - args['user'] = user - args['url'] = None - errors_warnings = args.pop("errors_warnings") - if args['stop_submission']: - return (args['personal_inputs'], args['json_reform'], args['has_errors'], - errors_warnings) + post_meta = submit_reform(request, user=user, **kwargs) + if isinstance(post_meta, BadPost): + return None, post_meta + + if post_meta.stop_submission: + return None, post_meta#(args['personal_inputs'], args['json_reform'], args['has_errors'], + #errors_warnings) else: - del args['stop_submission'] - del args['personal_inputs'] - url = save_model(**args) - return url, args['json_reform'], args['has_errors'], errors_warnings + url = save_model(post_meta) + return url, post_meta def file_input(request): @@ -394,24 +394,16 @@ def file_input(request): errors = ["Please specify a tax-law change before submitting."] json_reform = None else: - (obj, json_reform, has_errors, - errors_warnings) = process_reform( - request, - json_reform_id=form_id - ) - if has_errors and isinstance(obj, HttpResponse): - return obj + obj, post_meta = process_reform(request, json_reform_id=form_id) + if isinstance(post_meta, BadPost): + return post_meta.http_response_404 else: unique_url = obj - # Case 1: has_errors is True and there are warning/error messages - # --> display errors - # Case 2: has_errors is False and there are warning messages - # --> run model (user has already seen warning messages) - # Case 3: has_errors is False and there are no warning/error messages - # --> run model - if (has_errors and - (errors_warnings['warnings'] or errors_warnings['errors'])): + if post_meta.stop_submission: + errors_warnings = post_meta.errors_warnings + json_reform = post_meta.json_reform + has_errors = post_meta.has_errors errors.append(OUT_OF_RANGE_ERROR_MSG) append_errors_warnings( errors_warnings, @@ -457,24 +449,30 @@ def personal_results(request): """ start_year = START_YEAR has_errors = False - use_puf_not_cps = True + data_source = DEFAULT_SOURCE if request.method=='POST': print('method=POST get', request.GET) print('method=POST post', request.POST) - obj, _, has_errors, _ = process_reform(request) - + obj, post_meta = process_reform(request) + print(type(obj), post_meta.__dict__) # case where validation failed in forms.TaxBrainForm # TODO: assert HttpResponse status is 404 - if has_errors and isinstance(obj, HttpResponse): - return obj + if isinstance(post_meta, BadPost): + return post_meta.http_response_404 # No errors--submit to model - if not has_errors: + if not post_meta.stop_submission: return redirect(obj) # Errors from taxcalc.tbi.reform_warnings_errors else: personal_inputs = obj - start_year = personal_inputs._first_year + start_year = post_meta.start_year + data_source = post_meta.data_source + if data_source == 'PUF': + use_puf_not_cps = True + else: + use_puf_not_cps = False + has_errors = post_meta.has_errors else: # Probably a GET request, load a default form @@ -484,22 +482,16 @@ def personal_results(request): if 'start_year' in params and params['start_year'][0] in START_YEARS: start_year = params['start_year'][0] + # use puf by default + use_puf_not_cps = True if 'data_source' in params and params['data_source'][0] in DATA_SOURCES: data_source = params['data_source'][0] - print('we got the data source', data_source, 'now what...') - if data_source == 'PUF': - use_puf_not_cps = True - else: + if data_source != 'PUF': use_puf_not_cps = False personal_inputs = TaxBrainForm(first_year=start_year, use_puf_not_cps=use_puf_not_cps) - if use_puf_not_cps: - data_source = 'PUF' - else: - data_source = 'CPS' - print(data_source, DATA_SOURCES) init_context = { 'form': personal_inputs, 'params': nested_form_parameters(int(start_year), use_puf_not_cps), @@ -573,26 +565,29 @@ def submit_micro(request, pk): data ) - args = {'url': url, - 'request': request, - 'model': model, - 'json_reform': json_reform, - 'has_errors': False, - 'start_year': start_year, - 'data_source': model.data_source, - 'do_full_calc': True, - 'is_file': is_file, - 'reform_dict': reform_dict, - 'assumptions_dict': assumptions_dict, - 'reform_text': (model.json_text.raw_reform_text + post_meta = PostMeta(url=url, + request=request, + model=model, + json_reform=json_reform, + has_errors=False, + start_year=start_year, + data_source=model.data_source, + do_full_calc=True, + is_file=is_file, + reform_dict=reform_dict, + assumptions_dict=assumptions_dict, + reform_text=(model.json_text.raw_reform_text if model.json_text else ""), - 'assumptions_text': (model.json_text.raw_assumption_text + assumptions_text=(model.json_text.raw_assumption_text if model.json_text else ""), - 'submitted_ids': submitted_ids, - 'max_q_length': max_q_length, - 'user': None} - - url = save_model(**args) + submitted_ids=submitted_ids, + max_q_length=max_q_length, + user=None, + personal_inputs=None, + stop_submission=False, + errors_warnings=None) + + url = save_model(post_meta) return redirect(url) From 929fde8c922c51d98192b186d29020e0ccbf7464 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 10:27:34 -0400 Subject: [PATCH 12/21] Remove debugging print statement --- webapp/apps/taxbrain/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/webapp/apps/taxbrain/views.py b/webapp/apps/taxbrain/views.py index bb995580..285d9528 100644 --- a/webapp/apps/taxbrain/views.py +++ b/webapp/apps/taxbrain/views.py @@ -454,7 +454,6 @@ def personal_results(request): print('method=POST get', request.GET) print('method=POST post', request.POST) obj, post_meta = process_reform(request) - print(type(obj), post_meta.__dict__) # case where validation failed in forms.TaxBrainForm # TODO: assert HttpResponse status is 404 if isinstance(post_meta, BadPost): From 18b773df966f73157c29b549b73b74acac1fbe5b Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 10:28:11 -0400 Subject: [PATCH 13/21] Add front-end for file upload select datasource --- static/js/filetb.js | 13 +++++++++- templates/taxbrain/input_file.html | 39 ++++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/static/js/filetb.js b/static/js/filetb.js index c2d6ecba..c06602d9 100644 --- a/static/js/filetb.js +++ b/static/js/filetb.js @@ -78,7 +78,7 @@ $('#id_factor_adjustment, #id_factor_target').focus(function() { var currentYear = $('#start-year-select').val(); $('#start-year-select').change(function(e) { - $('#current-year-link').attr('href', '/taxbrain/file/?start_year=' + $(this).val()); + $('#current-year-link').attr('href', '/taxbrain/file/?start_year=' + $(this).val() + '&data_source=' + $('#data-source-select').val()); $('#current-year-modal').modal('show'); }); @@ -86,3 +86,14 @@ $('#current-year-modal').on('hide.bs.modal', function (e) { $('#start-year-select option').removeAttr("selected"); $('#start-year-select option[value="' + currentYear + '"]').attr("selected", "selected"); }); + +var dataSource = $('#data-source-select').val(); +$('#data-source-select').change(function(e) { + $('#data-source-link').attr('href', '/taxbrain/file/?start_year=' + $('#start-year-select').val() + '&data_source=' + $(this).val()); + $('#data-source-modal').modal('show'); +}); + +$('#data-choice-modal').on('hide.bs.modal', function (e) { + $('#data-source option').removeAttr("selected"); + $('#data-source option[value="' + dataSource + '"]').attr("selected", "selected"); +}); diff --git a/templates/taxbrain/input_file.html b/templates/taxbrain/input_file.html index 8ea2749f..87a38200 100644 --- a/templates/taxbrain/input_file.html +++ b/templates/taxbrain/input_file.html @@ -124,7 +124,7 @@

Tax-Calculator Code Build

-
+ {% csrf_token %} @@ -133,13 +133,20 @@

Tax-Calculator Code Build

-
+ + {% endblock %} From 6cebdb800877ca0d226a7909020cc59fad84258a Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 10:40:37 -0400 Subject: [PATCH 14/21] Update RELEASES for PR 847 --- RELEASES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index c319be35..3b06eeb3 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,7 +4,7 @@ Go [here](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pulls?q=is%3Apr+is%3Aclosed) for a complete commit history. -Release 1.5.0 on 2018-03-13 +Release 1.5.0 on 2018-03-14 ---------------------------- **Major Changes** - None @@ -12,6 +12,7 @@ Release 1.5.0 on 2018-03-13 **Minor Changes** - [#835](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/835/) - Add button to allow CPS as input data source - Sean Wang, Anderson Frailey, and Hank Doupe - [#842](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/842/) - Gray out fields based on data-source selection - Hank Doupe and Sean Wang +- [#847](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/847) - Adds data source to file input page, refactors reform submission, and improves tests - Hank Doupe **Bug Fixes** - [#844](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/842/) - Pass start year to dynamic behavioral - Hank Doupe From 6a79076b0068444ee7d7194fa7cf1f3c8218bfac Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 13:24:50 -0400 Subject: [PATCH 15/21] Add dateutil dependency for travis install bug --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 60247c71..57508712 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ boto django-storages django-htmlmin pyparsing +dateutil From 400576c8161330fb72b5f24ca1120f1b9e6eeecc Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 13:49:48 -0400 Subject: [PATCH 16/21] Try conda clean before running install --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2b8af934..3348899b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ install: - export PATH="$HOME/miniconda/bin:$PATH" - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' - pushd deploy + - conda clean --all - ./install_taxbrain_server.sh - popd - source activate aei_dropq From 7ea3031d24317697bd6f55abf4da83666aaeb88d Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 13:51:41 -0400 Subject: [PATCH 17/21] Yes by default for conda clean --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3348899b..230ef22b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - export PATH="$HOME/miniconda/bin:$PATH" - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' - pushd deploy - - conda clean --all + - conda clean --all -y - ./install_taxbrain_server.sh - popd - source activate aei_dropq From 8965a80610d79d133f5c97c3f1deb4b64826f257 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 14:31:27 -0400 Subject: [PATCH 18/21] Remove dateutil --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 57508712..60247c71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,4 +25,3 @@ boto django-storages django-htmlmin pyparsing -dateutil From d867e82a75ab366969a81b8dac0e77a8d4567d7c Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 14:37:33 -0400 Subject: [PATCH 19/21] Dateutil back in requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 60247c71..a1b7d540 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ boto django-storages django-htmlmin pyparsing +python-dateutil From cde3fcd53ab1e077fe5ef5b51ce060280d49d851 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 14:43:52 -0400 Subject: [PATCH 20/21] Use relative import --- webapp/apps/taxbrain/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/apps/taxbrain/views.py b/webapp/apps/taxbrain/views.py index 285d9528..36e78457 100644 --- a/webapp/apps/taxbrain/views.py +++ b/webapp/apps/taxbrain/views.py @@ -56,7 +56,7 @@ from ..formatters import get_version from .param_formatters import (get_reform_from_file, get_reform_from_gui, to_json_reform, append_errors_warnings) -from submit_data import PostMeta, BadPost +from .submit_data import PostMeta, BadPost from django.conf import settings WEBAPP_VERSION = settings.WEBAPP_VERSION From df2e85d7b449e3a5b8af0d3f74ca8aa7c927d149 Mon Sep 17 00:00:00 2001 From: Henry Doupe Date: Wed, 14 Mar 2018 16:16:15 -0400 Subject: [PATCH 21/21] Actually add the submit_data.py file --- webapp/apps/taxbrain/submit_data.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 webapp/apps/taxbrain/submit_data.py diff --git a/webapp/apps/taxbrain/submit_data.py b/webapp/apps/taxbrain/submit_data.py new file mode 100644 index 00000000..9ba341f4 --- /dev/null +++ b/webapp/apps/taxbrain/submit_data.py @@ -0,0 +1,26 @@ +from collections import namedtuple + +PostMeta = namedtuple( + 'PostMeta', + ['request', + 'personal_inputs', + 'json_reform', + 'model', + 'stop_submission', + 'has_errors', + 'errors_warnings', + 'start_year', + 'data_source', + 'do_full_calc', + 'is_file', + 'reform_dict', + 'assumptions_dict', + 'reform_text', + 'assumptions_text', + 'submitted_ids', + 'max_q_length', + 'user', + 'url'] +) + +BadPost = namedtuple('BadPost', ['http_response_404', 'has_errors'])