diff --git a/.travis.yml b/.travis.yml index 2b8af934..230ef22b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ install: - export PATH="$HOME/miniconda/bin:$PATH" - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' - pushd deploy + - conda clean --all -y - ./install_taxbrain_server.sh - popd - source activate aei_dropq diff --git a/RELEASES.md b/RELEASES.md index c319be35..3b06eeb3 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,7 +4,7 @@ Go [here](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pulls?q=is%3Apr+is%3Aclosed) for a complete commit history. -Release 1.5.0 on 2018-03-13 +Release 1.5.0 on 2018-03-14 ---------------------------- **Major Changes** - None @@ -12,6 +12,7 @@ Release 1.5.0 on 2018-03-13 **Minor Changes** - [#835](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/835/) - Add button to allow CPS as input data source - Sean Wang, Anderson Frailey, and Hank Doupe - [#842](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/842/) - Gray out fields based on data-source selection - Hank Doupe and Sean Wang +- [#847](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/847) - Adds data source to file input page, refactors reform submission, and improves tests - Hank Doupe **Bug Fixes** - [#844](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/842/) - Pass start year to dynamic behavioral - Hank Doupe diff --git a/requirements.txt b/requirements.txt index 60247c71..a1b7d540 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ boto django-storages django-htmlmin pyparsing +python-dateutil diff --git a/static/js/filetb.js b/static/js/filetb.js index c2d6ecba..c06602d9 100644 --- a/static/js/filetb.js +++ b/static/js/filetb.js @@ -78,7 +78,7 @@ $('#id_factor_adjustment, #id_factor_target').focus(function() { var currentYear = $('#start-year-select').val(); $('#start-year-select').change(function(e) { - $('#current-year-link').attr('href', '/taxbrain/file/?start_year=' + $(this).val()); + $('#current-year-link').attr('href', '/taxbrain/file/?start_year=' + $(this).val() + '&data_source=' + $('#data-source-select').val()); $('#current-year-modal').modal('show'); }); @@ -86,3 +86,14 @@ $('#current-year-modal').on('hide.bs.modal', function (e) { $('#start-year-select option').removeAttr("selected"); $('#start-year-select option[value="' + currentYear + '"]').attr("selected", "selected"); }); + +var dataSource = $('#data-source-select').val(); +$('#data-source-select').change(function(e) { + $('#data-source-link').attr('href', '/taxbrain/file/?start_year=' + $('#start-year-select').val() + '&data_source=' + $(this).val()); + $('#data-source-modal').modal('show'); +}); + +$('#data-choice-modal').on('hide.bs.modal', function (e) { + $('#data-source option').removeAttr("selected"); + $('#data-source option[value="' + dataSource + '"]').attr("selected", "selected"); +}); diff --git a/templates/taxbrain/input_file.html b/templates/taxbrain/input_file.html index 8ea2749f..87a38200 100644 --- a/templates/taxbrain/input_file.html +++ b/templates/taxbrain/input_file.html @@ -124,7 +124,7 @@

Tax-Calculator Code Build

-
+ {% csrf_token %} @@ -133,13 +133,20 @@

Tax-Calculator Code Build

-
+ + {% endblock %} diff --git a/webapp/apps/taxbrain/forms.py b/webapp/apps/taxbrain/forms.py index 61a779c0..7c512570 100644 --- a/webapp/apps/taxbrain/forms.py +++ b/webapp/apps/taxbrain/forms.py @@ -199,6 +199,8 @@ def set_form(defaults): class TaxBrainForm(PolicyBrainForm, ModelForm): def __init__(self, first_year, use_puf_not_cps, *args, **kwargs): + # the start year and the data source form object for later access. + # This should be refactored into `process_model` if first_year is None: first_year = START_YEAR self._first_year = int(first_year) diff --git a/webapp/apps/taxbrain/submit_data.py b/webapp/apps/taxbrain/submit_data.py new file mode 100644 index 00000000..9ba341f4 --- /dev/null +++ b/webapp/apps/taxbrain/submit_data.py @@ -0,0 +1,26 @@ +from collections import namedtuple + +PostMeta = namedtuple( + 'PostMeta', + ['request', + 'personal_inputs', + 'json_reform', + 'model', + 'stop_submission', + 'has_errors', + 'errors_warnings', + 'start_year', + 'data_source', + 'do_full_calc', + 'is_file', + 'reform_dict', + 'assumptions_dict', + 'reform_text', + 'assumptions_text', + 'submitted_ids', + 'max_q_length', + 'user', + 'url'] +) + +BadPost = namedtuple('BadPost', ['http_response_404', 'has_errors']) diff --git a/webapp/apps/taxbrain/tests/test_views.py b/webapp/apps/taxbrain/tests/test_views.py index 124af7f1..d46e0dae 100644 --- a/webapp/apps/taxbrain/tests/test_views.py +++ b/webapp/apps/taxbrain/tests/test_views.py @@ -1,4 +1,3 @@ -from django.test import TestCase from django.test import Client from django.core.files.uploadedfile import SimpleUploadedFile from django.test.client import RequestFactory @@ -23,54 +22,43 @@ get_dropq_compute_from_module, get_taxbrain_model) +CLIENT = Client() + START_YEAR = 2016 @pytest.mark.usefixtures("r1", "assumptions_text", "warning_reform", "bad_reform", "test_coverage_fields") -class TaxBrainViewsTests(TestCase): +@pytest.mark.django_db +class TestTaxBrainViews(object): ''' Test the views of this app. ''' - def setUp(self): - # Every test needs a client. - self.client = Client() - def test_taxbrain_get(self): # Issue a GET request. - response = self.client.get('/taxbrain/') + response = CLIENT.get('/taxbrain/') # Check that the response is 200 OK. - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 - def test_taxbrain_post(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_post(self, data_source): """ submit simple reform """ data = get_post_data(START_YEAR) data[u'II_em'] = [u'4333'] - data['data_source'] = ['PUF'] - result = do_micro_sim(self.client, data) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + result = do_micro_sim(CLIENT, data, post_url=url) truth_mods = {} check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=True) - - def test_taxbrain_post_cps(self): - """ - submit simple reform with CPS as data source - """ - data = get_post_data(START_YEAR) - data[u'II_em'] = [u'4333'] - data['data_source'] = ['CPS'] - result = do_micro_sim(self.client, data) + str(START_YEAR), data_source=data_source) - truth_mods = {} - - check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=False) - - def test_taxbrain_quick_calc_post(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_quick_calc_post(self, data_source): "Test quick calculation post and full post from quick_calc page" # switches 0, 4, 6 are False data = get_post_data(START_YEAR, quick_calc=True) @@ -79,66 +67,17 @@ def test_taxbrain_quick_calc_post(self): data[u'ID_BenefitSurtax_Switch_6'] = ['0.0'] data[u'II_em'] = [u'4333'] data[u'ID_AmountCap_Switch_0'] = [u'0'] + data['data_source'] = data_source wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) current_dropq_worker_offset = wnc.current_offset - result = do_micro_sim(self.client, data, compute_count=1) - - wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) - next_dropq_worker_offset = wnc.current_offset - - # Check that quick calc does not advance the counter - self.assertEqual(current_dropq_worker_offset, next_dropq_worker_offset) - - # Check that data was saved properly - truth_mods = {START_YEAR: {"_ID_BenefitSurtax_Switch": - [[0.0, 1.0, 1.0, 1.0, 0.0, 1.0, 0.0]], - "_ID_AmountCap_Switch": - [[0, 1, 1, 1, 1, 1, True]], - "_II_em": [4333.0]} - } - check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) - - # reset worker node count without clearing MockCompute session - result['tb_dropq_compute'].reset_count() - post_url = '/taxbrain/submit/{0}/'.format(result['pk']) - submit_data = {'csrfmiddlewaretoken':'abc123'} - - result = do_micro_sim( - self.client, - submit_data, - compute_count=NUM_BUDGET_YEARS, - post_url=post_url - ) - - # Check that data was saved properly - check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) - - def test_taxbrain_quick_calc_post_cps(self): - """ - Test quick calculation post and full post from quick_calc page using - cps data - """ - # switches 0, 4, 6 are False - data = get_post_data(START_YEAR, quick_calc=True) - data[u'ID_BenefitSurtax_Switch_0'] = ['False'] - data[u'ID_BenefitSurtax_Switch_4'] = ['0'] - data[u'ID_BenefitSurtax_Switch_6'] = ['0.0'] - data[u'II_em'] = [u'4333'] - data[u'ID_AmountCap_Switch_0'] = [u'0'] - data['data_source'] = ['CPS'] - wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) - current_dropq_worker_offset = wnc.current_offset - - result = do_micro_sim(self.client, data, compute_count=1) + result = do_micro_sim(CLIENT, data, compute_count=1) wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) next_dropq_worker_offset = wnc.current_offset # Check that quick calc does not advance the counter - self.assertEqual(current_dropq_worker_offset, next_dropq_worker_offset) + assert current_dropq_worker_offset == next_dropq_worker_offset # Check that data was saved properly truth_mods = {START_YEAR: {"_ID_BenefitSurtax_Switch": @@ -148,7 +87,7 @@ def test_taxbrain_quick_calc_post_cps(self): "_II_em": [4333.0]} } check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=False) + str(START_YEAR), data_source=data_source) # reset worker node count without clearing MockCompute session result['tb_dropq_compute'].reset_count() @@ -156,7 +95,7 @@ def test_taxbrain_quick_calc_post_cps(self): submit_data = {'csrfmiddlewaretoken':'abc123'} result = do_micro_sim( - self.client, + CLIENT, submit_data, compute_count=NUM_BUDGET_YEARS, post_url=post_url @@ -164,22 +103,24 @@ def test_taxbrain_quick_calc_post_cps(self): # Check that data was saved properly check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR), use_puf_not_cps=False) + str(START_YEAR), data_source=data_source) - def test_taxbrain_file_post_quick_calc(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_file_post_quick_calc(self, data_source): """ Using file-upload interface, test quick calculation post and full post from quick_calc page """ data = get_file_post_data(START_YEAR, self.r1, quick_calc=False) - + data.pop('data_source') + data.pop('start_year') wnc, created = WorkerNodesCounter.objects.get_or_create(singleton_enforce=1) current_dropq_worker_offset = wnc.current_offset - post_url = '/taxbrain/file/' + post_url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) result = do_micro_sim( - self.client, + CLIENT, data, compute_count=1, post_url=post_url @@ -189,7 +130,7 @@ def test_taxbrain_file_post_quick_calc(self): next_dropq_worker_offset = wnc.current_offset # Check that quick calc does not advance the counter - self.assertEqual(current_dropq_worker_offset, next_dropq_worker_offset) + assert current_dropq_worker_offset == next_dropq_worker_offset # Check that data was saved properly truth_mods = taxcalc.Calculator.read_json_param_objects( @@ -198,7 +139,7 @@ def test_taxbrain_file_post_quick_calc(self): ) truth_mods = truth_mods["policy"] check_posted_params(result["tb_dropq_compute"], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) # reset worker node count without clearing MockCompute session result['tb_dropq_compute'].reset_count() @@ -206,7 +147,7 @@ def test_taxbrain_file_post_quick_calc(self): submit_data = {'csrfmiddlewaretoken':'abc123'} result = do_micro_sim( - self.client, + CLIENT, submit_data, compute_count=NUM_BUDGET_YEARS, post_url=post_url @@ -214,10 +155,10 @@ def test_taxbrain_file_post_quick_calc(self): # Check that data was saved properly check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) - + str(START_YEAR), data_source=data_source) - def test_back_to_back_quickcalc(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_back_to_back_quickcalc(self, data_source): "Test back to back quick calc posts" # switches 0, 4, 6 are False data = get_post_data(START_YEAR, quick_calc=True) @@ -225,8 +166,9 @@ def test_back_to_back_quickcalc(self): data[u'ID_BenefitSurtax_Switch_4'] = ['0'] data[u'ID_BenefitSurtax_Switch_6'] = ['0.0'] data[u'II_em'] = [u'4333'] + data['data_source'] = data_source - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was saved properly truth_mods = {START_YEAR: {"_ID_BenefitSurtax_Switch": @@ -234,19 +176,19 @@ def test_back_to_back_quickcalc(self): "_II_em": [4333.0]} } check_posted_params(result['tb_dropq_compute'], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) edit_micro = '/taxbrain/edit/{0}/?start_year={1}'.format(result["pk"], START_YEAR) - edit_page = self.client.get(edit_micro) - self.assertEqual(edit_page.status_code, 200) + edit_page = CLIENT.get(edit_micro) + assert edit_page.status_code == 200 next_csrf = str(edit_page.context['csrf_token']) data['csrfmiddlewaretoken'] = next_csrf - result2 = do_micro_sim(self.client, data) + result2 = do_micro_sim(CLIENT, data) check_posted_params(result2['tb_dropq_compute'], truth_mods, - str(START_YEAR)) + str(START_YEAR), data_source=data_source) @pytest.mark.xfail def test_taxbrain_post_no_behavior_entries(self): @@ -260,9 +202,9 @@ def test_taxbrain_post_no_behavior_entries(self): data = get_post_data(START_YEAR) data[u'BE_inc'] = [u'0.1'] - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that we get a 400 - self.assertEqual(response.status_code, 400) + assert response.status_code == 400 def test_taxbrain_nodes_down(self): @@ -276,7 +218,7 @@ def test_taxbrain_nodes_down(self): data[u'II_em'] = [u'4333'] result = do_micro_sim( - self.client, + CLIENT, data, tb_dropq_compute=dropq_compute ) @@ -300,14 +242,14 @@ def test_taxbrain_failed_job(self): data = get_post_data(START_YEAR) data[u'II_em'] = [u'4333'] - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 302) + assert response.status_code == 302 link_idx = response.url[:-1].rfind('/') - self.failUnless(response.url[:link_idx+1].endswith("taxbrain/")) - response = self.client.get(response.url) + assert response.url[:link_idx+1].endswith("taxbrain/") + response = CLIENT.get(response.url) # Make sure the failure message is in the response - self.failUnless("Your calculation failed" in str(response)) + assert "Your calculation failed" in str(response) @pytest.mark.xfail def test_taxbrain_has_growth_params(self): @@ -318,7 +260,7 @@ def test_taxbrain_has_growth_params(self): 'has_errors': [u'False'], 'growth_choice': u'factor_adjustment' } - do_micro_sim(self.client, reform) + do_micro_sim(CLIENT, reform) def test_taxbrain_edit_cpi_flags_show_correctly(self): @@ -328,15 +270,15 @@ def test_taxbrain_edit_cpi_flags_show_correctly(self): data[u'AMT_CG_brk2_cpi'] = u'False' data[u'AMEDT_ec_cpi'] = u'True' - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) edit_micro = '/taxbrain/edit/{0}/?start_year={1}'.format(result["pk"], START_YEAR) - edit_page = self.client.get(edit_micro) - self.assertEqual(edit_page.status_code, 200) + edit_page = CLIENT.get(edit_micro) + assert edit_page.status_code == 200 cpi_flag = edit_page.context['form']['AMT_CG_brk2_cpi'].field.widget.attrs['placeholder'] - self.assertEqual(cpi_flag, False) + assert cpi_flag == False cpi_flag = edit_page.context['form']['AMEDT_ec_cpi'].field.widget.attrs['placeholder'] - self.assertEqual(cpi_flag, True) + assert cpi_flag == True @@ -347,7 +289,7 @@ def test_taxbrain_edit_benefitsurtax_switch_show_correctly(self): data[u'II_em'] = [u'4333'] data['ID_BenefitSurtax_Switch_3'] = [u'True'] - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) out = OutputUrl.objects.get(pk=result["pk"]) tsi = TaxSaveInputs.objects.get(pk=out.model_pk) @@ -359,8 +301,8 @@ def test_taxbrain_edit_benefitsurtax_switch_show_correctly(self): # Now edit this page edit_micro = '/taxbrain/edit/{0}/?start_year={1}'.format(result["pk"], START_YEAR) - edit_page = self.client.get(edit_micro) - self.assertEqual(edit_page.status_code, 200) + edit_page = CLIENT.get(edit_micro) + assert edit_page.status_code == 200 # post some more data from the edit parameters page. Posting the # same data (switch_0) again looks a little funny, but this @@ -374,7 +316,7 @@ def test_taxbrain_edit_benefitsurtax_switch_show_correctly(self): 'csrfmiddlewaretoken': next_csrf} data2.update(mod) - result2 = do_micro_sim(self.client, data2) + result2 = do_micro_sim(CLIENT, data2) out2 = OutputUrl.objects.get(pk=result2["pk"]) tsi2 = TaxSaveInputs.objects.get(pk=out2.model_pk) @@ -393,7 +335,7 @@ def test_taxbrain_wildcard_params_with_validation_is_OK(self): mod = {u'II_brk1_0': [u'*, *, 15000'], u'II_brk2_cpi': u'False'} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was saved properly truth_mods = { @@ -421,9 +363,9 @@ def test_taxbrain_wildcard_params_with_validation_gives_error(self): u'II_brk2_cpi': u'False'} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True @@ -439,7 +381,7 @@ def test_taxbrain_spec_operators_in_validation_params_OK(self): u'cpi_offset': [u'<,-0.0025'], u'FICA_ss_trt': [u'< ,0.1,*,0.15,0.2']} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) truth_mods = { START_YEAR - 1: { @@ -461,12 +403,18 @@ def test_taxbrain_warning_on_widow_param(self): Test case where error is added on undisplayed parameter """ data = get_post_data(START_YEAR, _ID_BenefitSurtax_Switches=False) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/?start_year={0}&data_source={1}'.format(START_YEAR, 'PUF') data[u'STD_3'] = ['10000'] - response = self.client.post('/taxbrain/', data) + response = CLIENT.post(url, data) assert response.status_code == 200 + assert response.context['start_year'] == str(START_YEAR) + assert response.context['data_source'] == 'PUF' - def test_taxbrain_wildcard_in_validation_params_gives_error(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_wildcard_in_validation_params_gives_error(self, data_source): """ Set upper threshold for income tax bracket 1 to *, 38000 Set upper threshold for income tax bracket 2 to *, *, 39500 @@ -477,15 +425,20 @@ def test_taxbrain_wildcard_in_validation_params_gives_error(self): get_dropq_compute_from_module('webapp.apps.taxbrain.views') data = get_post_data(START_YEAR, _ID_BenefitSurtax_Switches=False) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) mod = {u'II_brk1_0': [u'*, 38000'], u'II_brk2_0': [u'*, *, 39500'], u'II_brk2_cpi': u'False'} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post(url, data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['start_year'] == str(START_YEAR) + assert response.context['data_source'] == data_source def test_taxbrain_improper_reverse_gives_error1(self): @@ -499,9 +452,9 @@ def test_taxbrain_improper_reverse_gives_error1(self): mod = {u'cpi_offset': [u'<,']} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True def test_taxbrain_improper_reverse_gives_error2(self): @@ -515,9 +468,9 @@ def test_taxbrain_improper_reverse_gives_error2(self): mod = {u'cpi_offset': [u'-0.002,<,-0.001']} data.update(mod) - response = self.client.post('/taxbrain/', data) + response = CLIENT.post('/taxbrain/', data) # Check that redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True def test_taxbrain_bool_separated_values(self): @@ -528,7 +481,7 @@ def test_taxbrain_bool_separated_values(self): data = get_post_data(2018, _ID_BenefitSurtax_Switches=False) data['DependentCredit_before_CTC'] = [u'True,*, FALSE,tRUe,*,0'] - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was submitted properly truth_mods = { @@ -555,7 +508,7 @@ def test_taxbrain_rt_capital_gain_goes_to_amt(self): 'CG_brk2_2': [u'243475.0'], 'CG_brk2_3': [u'451000.0']} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) out2 = OutputUrl.objects.get(pk=result["pk"]) tsi2 = TaxSaveInputs.objects.get(pk=out2.model_pk) @@ -637,19 +590,29 @@ def test_taxbrain_rt_to_passthrough(self): data.update(values6) data.update(values7) #TODO: check how values are saved - do_micro_sim(self.client, data) - - def test_taxbrain_file_post_only_reform(self): - data = get_file_post_data(START_YEAR, self.r1) - do_micro_sim(self.client, data, post_url="/taxbrain/file/") - - - def test_taxbrain_file_post_reform_and_assumptions(self): + do_micro_sim(CLIENT, data) + + @pytest.mark.parametrize( + 'data_source,use_assumptions', + [('PUF', True), ('CPS', False)] + ) + def test_taxbrain_post_file(self, data_source, use_assumptions): + if use_assumptions: + assumptions_text = self.assumptions_text + else: + assumptions_text = None data = get_file_post_data(START_YEAR, self.r1, - self.assumptions_text) + assumptions_text) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + result = do_micro_sim(CLIENT, data, post_url=url) + + truth_mods = {} - do_micro_sim(self.client, data, post_url="/taxbrain/file/") + check_posted_params(result['tb_dropq_compute'], truth_mods, + str(START_YEAR), data_source=data_source) def test_taxbrain_view_old_data_model(self): @@ -690,12 +653,12 @@ def test_taxbrain_bad_expression(self): u'II_brk2_0': [u'*, *, 39500']} data.update(mod) - response = self.client.post('/taxbrain/', data) - self.assertEqual(response.status_code, 200) + response = CLIENT.post('/taxbrain/', data) + assert response.status_code == 200 assert response.context['has_errors'] is True - - def test_taxbrain_error_reform_file(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_error_reform_file(self, data_source): """ POST a reform file that causes errors. See PB issue #630 """ @@ -705,12 +668,15 @@ def test_taxbrain_error_reform_file(self): get_dropq_compute_from_module('webapp.apps.taxbrain.views') data = get_file_post_data(START_YEAR, self.bad_reform) - - #TODO: make sure still not allowed to submit on second submission - response = self.client.post('/taxbrain/file/', data) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + response = CLIENT.post(url, data) # Check that no redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['data_source'] == data_source + assert response.context['start_year'] == str(START_YEAR) assert any(['_II_brk1_4' in msg and '2024' in msg for msg in response.context['errors']]) @@ -725,14 +691,15 @@ def test_taxbrain_error_reform_file(self): 'csrfmiddlewaretoken': next_token, 'form_id': form_id, 'has_errors': [u'True'], - 'start_year': START_YEAR } - response = self.client.post('/taxbrain/file/', data2) + response = CLIENT.post(url, data2) assert response.status_code == 200 + assert response.context['data_source'] == data_source + assert response.context['start_year'] == str(START_YEAR) - - def test_taxbrain_warning_reform_file(self): + @pytest.mark.parametrize('data_source', ['PUF', 'CPS']) + def test_taxbrain_warning_reform_file(self, data_source): """ POST a reform file that causes warnings and check that re-submission is allowed. See PB issue #630 and #761 @@ -742,11 +709,15 @@ def test_taxbrain_warning_reform_file(self): get_dropq_compute_from_module('webapp.apps.taxbrain.views') data = get_file_post_data(START_YEAR, self.warning_reform) - - response = self.client.post('/taxbrain/file/', data) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(START_YEAR, data_source) + response = CLIENT.post(url, data) # Check that no redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['start_year'] == str(START_YEAR) + assert response.context['data_source'] == data_source assert any(['_STD_0' in msg and '2023' in msg for msg in response.context['errors']]) @@ -761,20 +732,23 @@ def test_taxbrain_warning_reform_file(self): 'csrfmiddlewaretoken': next_token, 'form_id': form_id, 'has_errors': [u'True'], - 'start_year': START_YEAR } - result = do_micro_sim(self.client, data2, post_url='/taxbrain/file/') + result = do_micro_sim(CLIENT, data2, post_url=url) truth_mods = { 2020: { "_STD": [[1000, 24981.84, 12490.92, 18736.38, 24981.84]] } } - check_posted_params(result['tb_dropq_compute'], truth_mods, START_YEAR) - + check_posted_params(result['tb_dropq_compute'], truth_mods, START_YEAR, + data_source=data_source) - def test_taxbrain_reform_file_file_swap(self): + @pytest.mark.parametrize( + 'data_source,use_assumptions', + [('PUF', True), ('CPS', False)] + ) + def test_taxbrain_reform_file_file_swap(self, data_source, use_assumptions): """ POST a reform file that causes warnings, swap files, and make sure swapped files are used. See PB issue #630 and #761 @@ -783,13 +757,22 @@ def test_taxbrain_reform_file_file_swap(self): from webapp.apps.taxbrain.models import JSONReformTaxCalculator as js #Monkey patch to mock out running of compute jobs get_dropq_compute_from_module('webapp.apps.taxbrain.views') - - data = get_file_post_data(start_year, self.warning_reform) - - response = self.client.post('/taxbrain/file/', data) + if use_assumptions: + assumptions_text = self.assumptions_text + else: + assumptions_text = None + + data = get_file_post_data(start_year, self.warning_reform, + assumptions_text) + data.pop('start_year') + data.pop('data_source') + url = '/taxbrain/file/?start_year={0}&data_source={1}'.format(start_year, data_source) + response = CLIENT.post(url, data) # Check that no redirect happens - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 assert response.context['has_errors'] is True + assert response.context['data_source'] == data_source + assert response.context['start_year'] == str(start_year) assert any(['_STD_0' in msg and '2023' in msg for msg in response.context['errors']]) @@ -804,65 +787,21 @@ def test_taxbrain_reform_file_file_swap(self): 'csrfmiddlewaretoken': next_token, 'form_id': form_id, 'has_errors': [u'True'], - 'start_year': start_year } data_file = get_file_post_data(START_YEAR, self.r1, - self.assumptions_text) - data2['docfile'] = data_file['docfile'] - data2['assumpfile'] = data_file['assumpfile'] - - result = do_micro_sim(self.client, data2, post_url='/taxbrain/file/') - - dropq_compute = result['tb_dropq_compute'] - user_mods = json.loads(dropq_compute.last_posted["user_mods"]) - assert user_mods["behavior"][str(2018)]["_BE_sub"][0] == 1.0 - truth_mods = {2018: {'_II_em': [8000.0]}} - check_posted_params(dropq_compute, truth_mods, start_year) - - - def test_taxbrain_reform_file_file_swap_no_assump(self): - """ - POST a reform file that causes warnings, swap files, and make sure - swapped files are used. See PB issue #630 and #761 - """ - start_year = 2017 - from webapp.apps.taxbrain.models import JSONReformTaxCalculator as js - #Monkey patch to mock out running of compute jobs - get_dropq_compute_from_module('webapp.apps.taxbrain.views') - - data = get_file_post_data(start_year, self.warning_reform) - - response = self.client.post('/taxbrain/file/', data) - # Check that no redirect happens - self.assertEqual(response.status_code, 200) - assert response.context['has_errors'] is True - assert any(['_STD_0' in msg and '2023' in msg - for msg in response.context['errors']]) - - # get most recent object - objects = js.objects.order_by('id') - obj = objects[len(objects) - 1] - - next_token = str(response.context['csrf_token']) - - form_id = obj.id - data2 = { - 'csrfmiddlewaretoken': next_token, - 'form_id': form_id, - 'has_errors': [u'True'], - 'start_year': start_year - } - data_file = get_file_post_data(START_YEAR, self.r1) + assumptions_text) data2['docfile'] = data_file['docfile'] - result = do_micro_sim(self.client, data2, post_url='/taxbrain/file/') + result = do_micro_sim(CLIENT, data2, post_url=url) dropq_compute = result['tb_dropq_compute'] user_mods = json.loads(dropq_compute.last_posted["user_mods"]) + if use_assumptions: + assert user_mods["behavior"][str(2018)]["_BE_sub"][0] == 1.0 truth_mods = {2018: {'_II_em': [8000.0]}} - check_posted_params(dropq_compute, truth_mods, start_year) - + check_posted_params(dropq_compute, truth_mods, start_year, + data_source=data_source) def test_taxbrain_up_to_2018(self): start_year = 2018 @@ -870,7 +809,7 @@ def test_taxbrain_up_to_2018(self): mod = {u'II_brk1_0': [u'*, *, 15000'], u'II_brk2_cpi': u'False'} data.update(mod) - result = do_micro_sim(self.client, data) + result = do_micro_sim(CLIENT, data) # Check that data was saved properly truth_mods = { @@ -887,7 +826,7 @@ def test_taxbrain_file_up_to_2018(self): post_url = '/taxbrain/file/' result = do_micro_sim( - self.client, + CLIENT, data, post_url=post_url ) @@ -927,7 +866,7 @@ def test_taxbrain_old_data_gives_deprecation_errors(self): pk = unique_url.pk edit_micro = "/taxbrain/edit/{}/?start_year={}".format(pk, start_year) - response = self.client.get(edit_micro) + response = CLIENT.get(edit_micro) assert response.status_code == 200 assert response.context['has_errors'] is False diff --git a/webapp/apps/taxbrain/views.py b/webapp/apps/taxbrain/views.py index 7c92aabe..36e78457 100644 --- a/webapp/apps/taxbrain/views.py +++ b/webapp/apps/taxbrain/views.py @@ -56,6 +56,7 @@ from ..formatters import get_version from .param_formatters import (get_reform_from_file, get_reform_from_gui, to_json_reform, append_errors_warnings) +from .submit_data import PostMeta, BadPost from django.conf import settings WEBAPP_VERSION = settings.WEBAPP_VERSION @@ -94,32 +95,31 @@ def normalize(x): ans = [i.split('#') for i in x] return ans -def save_model(url, request, model, json_reform, has_errors, start_year, - do_full_calc, is_file, reform_dict, assumptions_dict, - reform_text, assumptions_text, submitted_ids, - max_q_length, user): +def save_model(post_meta): """ Save user input data returns OutputUrl object """ + model = post_meta.model # create model for file_input case if model is None: model = TaxSaveInputs() - model.job_ids = denormalize(submitted_ids) - model.json_text = json_reform - model.first_year = int(start_year) - model.quick_calc = not do_full_calc + model.job_ids = denormalize(post_meta.submitted_ids) + model.json_text = post_meta.json_reform + model.first_year = int(post_meta.start_year) + model.data_source = post_meta.data_source + model.quick_calc = not post_meta.do_full_calc model.save() # create OutputUrl object - if url is None: + if post_meta.url is None: unique_url = OutputUrl() else: - unique_url = url - if user: - unique_url.user = user - elif request and request.user.is_authenticated(): - current_user = User.objects.get(pk=request.user.id) + unique_url = post_meta.url + if post_meta.user: + unique_url.user = post_meta.user + elif post_meta.request and post_meta.request.user.is_authenticated(): + current_user = User.objects.get(pk=post_meta.request.user.id) unique_url.user = current_user if unique_url.taxcalc_vers is None: @@ -130,7 +130,7 @@ def save_model(url, request, model, json_reform, has_errors, start_year, unique_url.unique_inputs = model unique_url.model_pk = model.pk cur_dt = datetime.datetime.utcnow() - future_offset_seconds = ((2 + max_q_length) * JOB_PROC_TIME_IN_SECONDS) + future_offset_seconds = ((2 + post_meta.max_q_length) * JOB_PROC_TIME_IN_SECONDS) future_offset = datetime.timedelta(seconds=future_offset_seconds) expected_completion = cur_dt + future_offset unique_url.exp_comp_datetime = expected_completion @@ -158,7 +158,8 @@ def submit_reform(request, user=None, json_reform_id=None): request_files = request.FILES # which file to use, front-end not yet implemented - if fields.get('data_source', 'PUF') == 'PUF': + data_source = fields.get('data_source', 'PUF') + if data_source == 'PUF': use_puf_not_cps = True else: use_puf_not_cps = False @@ -191,10 +192,10 @@ def submit_reform(request, user=None, json_reform_id=None): json_reform = JSONReformTaxCalculator.objects.get(id=int(json_reform_id)) except Exception as e: msg = "ID {} is not in JSON reform database".format(json_reform_id) - http_response = HttpResponse(msg, status=400) - return {'HttpResponse': http_response, - 'has_errors': True} - + return BadPost( + http_response_404=HttpResponse(msg, status=400), + has_errors= True + ) reform_dict = json.loads(json_reform.reform_text) assumptions_dict = json.loads(json_reform.assumption_text) reform_text = json_reform.raw_reform_text @@ -231,8 +232,10 @@ def submit_reform(request, user=None, json_reform_id=None): # If an attempt is made to post data we don't accept # raise a 400 if personal_inputs.non_field_errors(): - return {'HttpResponse': HttpResponse("Bad Input!", status=400), - 'has_errors': True} + return BadPost( + http_response_404=HttpResponse("Bad Input!", status=400), + has_errors= True + ) is_valid = personal_inputs.is_valid() if is_valid: model = personal_inputs.save(commit=False) @@ -314,22 +317,28 @@ def submit_reform(request, user=None, json_reform_id=None): data ) - return {'personal_inputs': personal_inputs, - 'json_reform': json_reform, - 'model': model, - 'stop_submission': stop_submission, - 'has_errors': any([taxcalc_errors, taxcalc_warnings, + return PostMeta( + request=request, + personal_inputs=personal_inputs, + json_reform=json_reform, + model=model, + stop_submission=stop_submission, + has_errors=any([taxcalc_errors, taxcalc_warnings, no_inputs, not is_valid]), - 'errors_warnings': errors_warnings, - 'start_year': start_year, - 'do_full_calc': do_full_calc, - 'is_file': is_file, - 'reform_dict': reform_dict, - 'assumptions_dict': assumptions_dict, - 'reform_text': reform_text, - 'assumptions_text': assumptions_text, - 'submitted_ids': submitted_ids, - 'max_q_length': max_q_length} + errors_warnings=errors_warnings, + start_year=start_year, + data_source=data_source, + do_full_calc=do_full_calc, + is_file=is_file, + reform_dict=reform_dict, + assumptions_dict=assumptions_dict, + reform_text=reform_text, + assumptions_text=assumptions_text, + submitted_ids=submitted_ids, + max_q_length=max_q_length, + user=user, + url=None + ) def process_reform(request, user=None, **kwargs): @@ -343,22 +352,16 @@ def process_reform(request, user=None, **kwargs): boolean variable indicating whether this reform has errors or not """ - args = submit_reform(request, user=user, **kwargs) - if 'HttpResponse' in args: - return args['HttpResponse'], None, args['has_errors'], None - - args['request'] = request - args['user'] = user - args['url'] = None - errors_warnings = args.pop("errors_warnings") - if args['stop_submission']: - return (args['personal_inputs'], args['json_reform'], args['has_errors'], - errors_warnings) + post_meta = submit_reform(request, user=user, **kwargs) + if isinstance(post_meta, BadPost): + return None, post_meta + + if post_meta.stop_submission: + return None, post_meta#(args['personal_inputs'], args['json_reform'], args['has_errors'], + #errors_warnings) else: - del args['stop_submission'] - del args['personal_inputs'] - url = save_model(**args) - return url, args['json_reform'], args['has_errors'], errors_warnings + url = save_model(post_meta) + return url, post_meta def file_input(request): @@ -371,39 +374,36 @@ def file_input(request): form_id = None start_year = START_YEAR + data_source = DEFAULT_SOURCE errors = [] has_errors = False - print('post', request.POST) - print('get', request.GET) print('files', request.FILES) if request.method == 'POST': + print('method=POST get', request.GET) + print('method=POST post', request.POST) # save start_year start_year = (request.GET.get('start_year', None) or request.POST.get('start_year', None)) assert start_year is not None + data_source = (request.GET.get('data_source', None) or + request.POST.get('start_year', None)) + assert data_source is not None + # File is not submitted if 'docfile' not in dict(request.FILES) and form_id is None: errors = ["Please specify a tax-law change before submitting."] json_reform = None else: - (obj, json_reform, has_errors, - errors_warnings) = process_reform( - request, - json_reform_id=form_id - ) - if has_errors and isinstance(obj, HttpResponse): - return obj + obj, post_meta = process_reform(request, json_reform_id=form_id) + if isinstance(post_meta, BadPost): + return post_meta.http_response_404 else: unique_url = obj - # Case 1: has_errors is True and there are warning/error messages - # --> display errors - # Case 2: has_errors is False and there are warning messages - # --> run model (user has already seen warning messages) - # Case 3: has_errors is False and there are no warning/error messages - # --> run model - if (has_errors and - (errors_warnings['warnings'] or errors_warnings['errors'])): + if post_meta.stop_submission: + errors_warnings = post_meta.errors_warnings + json_reform = post_meta.json_reform + has_errors = post_meta.has_errors errors.append(OUT_OF_RANGE_ERROR_MSG) append_errors_warnings( errors_warnings, @@ -412,11 +412,16 @@ def file_input(request): else: return redirect(unique_url) else: - # GET request, load a default form + # Probably a GET request, load a default form + print('method=GET get', request.GET) + print('method=GET post', request.POST) params = parse_qs(urlparse(request.build_absolute_uri()).query) if 'start_year' in params and params['start_year'][0] in START_YEARS: start_year = params['start_year'][0] + if 'data_source' in params and params['data_source'][0] in DATA_SOURCES: + data_source = params['data_source'][0] + json_reform = None init_context = { @@ -428,6 +433,8 @@ def file_input(request): 'params': None, 'start_years': START_YEARS, 'start_year': start_year, + 'data_sources': DATA_SOURCES, + 'data_source': data_source, 'enable_quick_calc': ENABLE_QUICK_CALC, 'input_type': "file" } @@ -442,24 +449,29 @@ def personal_results(request): """ start_year = START_YEAR has_errors = False - use_puf_not_cps = True + data_source = DEFAULT_SOURCE if request.method=='POST': print('method=POST get', request.GET) print('method=POST post', request.POST) - obj, _, has_errors, _ = process_reform(request) - + obj, post_meta = process_reform(request) # case where validation failed in forms.TaxBrainForm # TODO: assert HttpResponse status is 404 - if has_errors and isinstance(obj, HttpResponse): - return obj + if isinstance(post_meta, BadPost): + return post_meta.http_response_404 # No errors--submit to model - if not has_errors: + if not post_meta.stop_submission: return redirect(obj) # Errors from taxcalc.tbi.reform_warnings_errors else: personal_inputs = obj - start_year = personal_inputs._first_year + start_year = post_meta.start_year + data_source = post_meta.data_source + if data_source == 'PUF': + use_puf_not_cps = True + else: + use_puf_not_cps = False + has_errors = post_meta.has_errors else: # Probably a GET request, load a default form @@ -469,22 +481,16 @@ def personal_results(request): if 'start_year' in params and params['start_year'][0] in START_YEARS: start_year = params['start_year'][0] + # use puf by default + use_puf_not_cps = True if 'data_source' in params and params['data_source'][0] in DATA_SOURCES: data_source = params['data_source'][0] - print('we got the data source', data_source, 'now what...') - if data_source == 'PUF': - use_puf_not_cps = True - else: + if data_source != 'PUF': use_puf_not_cps = False personal_inputs = TaxBrainForm(first_year=start_year, use_puf_not_cps=use_puf_not_cps) - if use_puf_not_cps: - data_source = 'PUF' - else: - data_source = 'CPS' - print(data_source, DATA_SOURCES) init_context = { 'form': personal_inputs, 'params': nested_form_parameters(int(start_year), use_puf_not_cps), @@ -546,6 +552,7 @@ def submit_micro(request, pk): assumptions_dict = json.loads(model.json_text.assumption_text) user_mods = dict({'policy': reform_dict}, **assumptions_dict) + print('data source', model.data_source) data = {'user_mods': json.dumps(user_mods), 'first_budget_year': int(start_year), 'start_budget_year': 0, @@ -557,25 +564,29 @@ def submit_micro(request, pk): data ) - args = {'url': url, - 'request': request, - 'model': model, - 'json_reform': json_reform, - 'has_errors': False, - 'start_year': start_year, - 'do_full_calc': True, - 'is_file': is_file, - 'reform_dict': reform_dict, - 'assumptions_dict': assumptions_dict, - 'reform_text': (model.json_text.raw_reform_text + post_meta = PostMeta(url=url, + request=request, + model=model, + json_reform=json_reform, + has_errors=False, + start_year=start_year, + data_source=model.data_source, + do_full_calc=True, + is_file=is_file, + reform_dict=reform_dict, + assumptions_dict=assumptions_dict, + reform_text=(model.json_text.raw_reform_text if model.json_text else ""), - 'assumptions_text': (model.json_text.raw_assumption_text + assumptions_text=(model.json_text.raw_assumption_text if model.json_text else ""), - 'submitted_ids': submitted_ids, - 'max_q_length': max_q_length, - 'user': None} - - url = save_model(**args) + submitted_ids=submitted_ids, + max_q_length=max_q_length, + user=None, + personal_inputs=None, + stop_submission=False, + errors_warnings=None) + + url = save_model(post_meta) return redirect(url) diff --git a/webapp/apps/test_assets/utils.py b/webapp/apps/test_assets/utils.py index 6a01fb5d..ddaa72f0 100644 --- a/webapp/apps/test_assets/utils.py +++ b/webapp/apps/test_assets/utils.py @@ -2,6 +2,7 @@ import os import sys import ast +import six from ..taxbrain.compute import MockCompute @@ -74,7 +75,7 @@ def do_micro_sim(client, data, tb_dropq_compute=None, dyn_dropq_compute=None, def check_posted_params(mock_compute, params_to_check, start_year, - use_puf_not_cps=True): + use_puf_not_cps=True, data_source=None): """ Make sure posted params match expected results user_mods: parameters that are actually passed to taxcalc @@ -84,6 +85,8 @@ def check_posted_params(mock_compute, params_to_check, start_year, last_posted = mock_compute.last_posted user_mods = json.loads(last_posted["user_mods"]) assert last_posted["first_budget_year"] == int(start_year) + if data_source is not None: + use_puf_not_cps = True if data_source == 'PUF' else False assert last_posted["use_puf_not_cps"] == use_puf_not_cps for year in params_to_check: for param in params_to_check[year]: @@ -129,6 +132,7 @@ def get_file_post_data(start_year, reform_text, assumptions_text=None, quick_cal data = {u'docfile': tc_file, u'has_errors': [u'False'], u'start_year': unicode(start_year), + 'data_source': 'PUF', u'quick_calc': quick_calc, 'csrfmiddlewaretoken':'abc123'}