Fix #788 Data import should skip blank, non-required attribute fields (…

…#790)

cadasta/config/settings/default.py

@@ -337,5 +337,5 @@
 }
 
 FILE_UPLOAD_HANDLERS = [
-    'organization.views.handlers.S3FileUploadHandler',
+    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
 ]

cadasta/organization/importers/csv.py

@@ -33,7 +33,7 @@ def __init__(self, project=None, path=None,
 
     def get_headers(self):
         headers = []
-        with open(self.path, newline='') as csvfile:
+        with open(self.path, 'r', newline='') as csvfile:
             reader = csv.reader(
                 csvfile, delimiter=self.delimiter, quotechar=self.quotechar
             )
@@ -84,7 +84,7 @@ def import_data(self, config_dict, **kwargs):
         path = config_dict['file']
         try:
             with transaction.atomic():
-                with open(path) as csvfile:
+                with open(path, 'r', newline='') as csvfile:
                     reader = csv.reader(
                         csvfile, delimiter=self.delimiter,
                         quotechar=self.quotechar
@@ -130,8 +130,11 @@ def import_data(self, config_dict, **kwargs):
                             attribute, content_type, name = attr_map.get(attr)
                             if (attribute is not None):
                                 val = row[csv_headers.index(attr)]
+                                if (not attribute.required and val == ""):
+                                    continue
                                 content_types[content_type][
-                                    'attributes'][attribute.name] = val
+                                    'attributes'][
+                                        attribute.name] = val
                         party = Party.objects.create(
                             **content_types['party.party']
                         )

cadasta/organization/tests/files/test.csv

@@ -1,11 +1,11 @@
 name_mouza,j_l,tenure_type,name_of_HH,name_father_hus,present_add,village_name,Mobile_No,Occupation_HH,educational_qualification,amount_agriland,amount_othersland,class_HH,How_aquire_landw,How_aquire_landwh,How_aquire_landm,How_aquire_landd,How_aquire_landt,How_aquire_landp,Land_Calculation,deed_of_land,Khatain_of_land,Mutation_of_land,Dakhal_on_land,Everything,female_member,male_member,ownership_conflict,boundary_conflict,others_conflict,conflicts_resoulation,howconflict_resoulation,nid_number,geo_type,location_geometry,_geopoint_hh_latitude,_geopoint_hh_longitude,_geopoint_hh_altitude,_geopoint_hh_precision,image_hh,audio_hh,start,end,today,deviceid,subscriberid,sim_serial,phonenumber,meta/instanceID,_id,_uuid,_submission_time,_index,_parent_table_name,_parent_index,_tags,_notes,_version,_duration,_submitted_by
-chanderhowara,90,FH,সুরুজ্জামান,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৮০.০০ অারঙ্গহাটি ৩৬০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,yes,no,yes,no,4,5,yes,no,কিছু বেখলে অাছে জোর করে দখল করে ভোগ করে অন্য জমির মালক।,yes,local_justice,3913647224045,,,,,,,,,2016-02-08T10:43:19.861+06,2016-02-09T09:03:56.526+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:c2b46f0e-dd3c-4c0b-858e-edd582047f6c,5349241,c2b46f0e-dd3c-4c0b-858e-edd582047f6c,2016-02-09T11:30:07,1,,-1,,,201602060307,80437,arifuttaran
-chanderhowara,90,FH,অাব্দুল বারেক মন্ডল,মৃত বঙ্গু মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭৯৮০৮১৯৯৪,farmer,Illiterate,৩০.০০ আরঙ্গহাটি ১২০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,no,yes,no,5,4,no,no,,,,3913647224033,,,,,,,,,2016-02-08T11:13:52.453+06,2016-02-09T13:58:53.495+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,5349244,4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,2016-02-09T11:30:17,2,,-1,,,201602060307,96301,arifuttaran
-chanderhowara,90,FH,হানিফ উদ্দিন,মৃত মুগল মন্ডল,অারঙ্গহাটি,পশ্চিম অারঙ্গহাটি,০১৭৯৮০৮১৯৯৪,farmer,litteracy,১৫.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,yes,yes,no,3,4,no,no,,,,3913647225965,,,,,,,,,2016-02-08T11:21:20.859+06,2016-02-09T13:58:16.633+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:405a9d1c-3c54-42ca-929d-ab4f261ce59a,5349247,405a9d1c-3c54-42ca-929d-ab4f261ce59a,2016-02-09T11:30:28,3,,-1,,,201602060307,95816,arifuttaran
-chanderhowara,90,FH,মো: অামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,service,educated,১৫.০০ অারঙ্গহাটি ৩০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,no,no,yes,no,2,1,no,no,,,,3913647224032,,24.850375 89.832815 17.3 4.8,24.850375,89.832815,17.3,4.8,,,2016-02-08T11:27:32.805+06,2016-02-08T11:55:08.868+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:6925434e-37d3-4f09-b308-1e6e9ccf1d11,5349250,6925434e-37d3-4f09-b308-1e6e9ccf1d11,2016-02-09T11:30:38,4,,-1,,,201602060307,1656,arifuttaran
-chanderhowara,90,FH,সোজা মন্ডল,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,৭০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,no,no,yes,no,0,1,no,no,,,,3913647224043,,,,,,,,,2016-02-08T11:32:28.205+06,2016-02-09T13:58:42.516+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:b2cdbeba-1ced-4740-9605-9430d1ed7ead,5349251,b2cdbeba-1ced-4740-9605-9430d1ed7ead,2016-02-09T11:30:40,5,,-1,,,201602060307,95174,arifuttaran
-chanderhowara,90,FH,মিজানুর রহমান,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,15,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,3,2,no,no,,,,3913647224044,,24.85034333333333 89.83295000000001 42.3 7.1,24.8503433333,89.83295,42.3,7.1,,,2016-02-08T11:37:42.708+06,2016-02-08T11:47:20.509+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:d16a03fa-5009-490e-bc4f-c3812e704b46,5349252,d16a03fa-5009-490e-bc4f-c3812e704b46,2016-02-09T11:30:42,6,,-1,,,201602060307,578,arifuttaran
-chanderhowara,90,FH, জামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৫.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,2,2,no,no,,no,,3913647224031,,24.850428333333337 89.83293166666665 13.2 4.9,24.8504283333,89.8329316667,13.2,4.9,,,2016-02-08T11:47:29.749+06,2016-02-08T11:51:46.250+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,5349254,73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,2016-02-09T11:30:50,7,,-1,,,201602060307,257,arifuttaran
-chanderhowara,90,FH,মো: গোলজার হোসেন,মৃত মো: শাহাজান অালী,মেম্বার বাড়ি,চান্দের হাওড়া,০১৭৪৩০৪৮৪২৩,doctor,educated,0,১৫.০০,poor,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,no,no,no,yes,no,4,3,no,no,,,,3913647224185,,24.849339999999998 89.83242166666666 -25.9 4.7,24.84934,89.8324216667,-25.9,4.7,,,2016-02-08T12:14:23.738+06,2016-02-08T12:21:48.113+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:35bb0314-bdc4-4e1d-9994-66bd745d0c0b,5349255,35bb0314-bdc4-4e1d-9994-66bd745d0c0b,2016-02-09T11:30:52,8,,-1,,,201602060307,445,arifuttaran
-chanderhowara,90,FH, সুমন টিকাদার,মৃত নওয়াব অালি টিকাদার,টিকাদার বাড়ি,চান্দের হাওড়া,০১৭৩৪২৫১৩৩৮,farmer,litteracy,১০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,no,no,no,yes,no,1,4,no,no,,,,3913647224283,,24.848106666666663 89.83239833333333 -115.7 5.4,24.8481066667,89.8323983333,-115.7,5.4,,,2016-02-08T12:41:22.328+06,2016-02-08T12:45:05.345+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:2454181a-4d9a-4797-b908-0c1c82c6b92a,5349256,2454181a-4d9a-4797-b908-0c1c82c6b92a,2016-02-09T11:30:53,9,,-1,,,201602060307,223,arifuttaran
-chanderhowara,90,FH,অাব্দুল জলিল মন্ডল ,মৃত কুব্বাত মন্ডল ,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭৭২৫৬০১৯১,farmer,Illiterate,২৭.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,4,8,no,no,,,,3913647224167,,24.848364999999998 89.83268500000001 -50.4 4.9,24.848365,89.832685,-50.4,4.9,,,2016-02-08T12:45:10.187+06,2016-02-08T12:53:02.738+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:68e07841-0eb3-4960-bb5e-bbfffa0b609d,5349258,68e07841-0eb3-4960-bb5e-bbfffa0b609d,2016-02-09T11:30:57,10,,-1,,,201602060307,472,arifuttaran
+chanderhowara,90,FH,সুরুজ্জামান,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৮০.০০ অারঙ্গহাটি ৩৬০.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,yes,no,yes,no,4,5,yes,no,কিছু বেখলে অাছে জোর করে দখল করে ভোগ করে অন্য জমির মালক।,yes,local_justice,3913647224045,,,,,,,,,2016-02-08T10:43:19.861+06,2016-02-09T09:03:56.526+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:c2b46f0e-dd3c-4c0b-858e-edd582047f6c,5349241,c2b46f0e-dd3c-4c0b-858e-edd582047f6c,2016-02-09T11:30:07,1,,-1,,,201602060307,80437,arifuttaran
+chanderhowara,90,FH,অাব্দুল বারেক মন্ডল,মৃত বঙ্গু মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭৯৮০৮১৯৯৪,farmer,Illiterate,৩০.০০ আরঙ্গহাটি ১২০.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,no,yes,no,5,4,no,no,,,,3913647224033,,,,,,,,,2016-02-08T11:13:52.453+06,2016-02-09T13:58:53.495+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,5349244,4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,2016-02-09T11:30:17,2,,-1,,,201602060307,96301,arifuttaran
+chanderhowara,90,FH,হানিফ উদ্দিন,মৃত মুগল মন্ডল,অারঙ্গহাটি,পশ্চিম অারঙ্গহাটি,০১৭৯৮০৮১৯৯৪,farmer,litteracy,১৫.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,yes,yes,no,3,4,no,no,,,,3913647225965,,,,,,,,,2016-02-08T11:21:20.859+06,2016-02-09T13:58:16.633+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:405a9d1c-3c54-42ca-929d-ab4f261ce59a,5349247,405a9d1c-3c54-42ca-929d-ab4f261ce59a,2016-02-09T11:30:28,3,,-1,,,201602060307,95816,arifuttaran
+chanderhowara,90,FH,মো: অামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,service,educated,১৫.০০ অারঙ্গহাটি ৩০.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,no,no,yes,no,2,1,no,no,,,,3913647224032,,24.850375 89.832815 17.3 4.8,24.850375,89.832815,17.3,4.8,,,2016-02-08T11:27:32.805+06,2016-02-08T11:55:08.868+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:6925434e-37d3-4f09-b308-1e6e9ccf1d11,5349250,6925434e-37d3-4f09-b308-1e6e9ccf1d11,2016-02-09T11:30:38,4,,-1,,,201602060307,1656,arifuttaran
+chanderhowara,90,FH,সোজা মন্ডল,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,৭০.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,no,no,yes,no,0,1,no,no,,,,3913647224043,,,,,,,,,2016-02-08T11:32:28.205+06,2016-02-09T13:58:42.516+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:b2cdbeba-1ced-4740-9605-9430d1ed7ead,5349251,b2cdbeba-1ced-4740-9605-9430d1ed7ead,2016-02-09T11:30:40,5,,-1,,,201602060307,95174,arifuttaran
+chanderhowara,90,FH,মিজানুর রহমান,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,15,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,3,2,no,no,,,,3913647224044,,24.85034333333333 89.83295000000001 42.3 7.1,24.8503433333,89.83295,42.3,7.1,,,2016-02-08T11:37:42.708+06,2016-02-08T11:47:20.509+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:d16a03fa-5009-490e-bc4f-c3812e704b46,5349252,d16a03fa-5009-490e-bc4f-c3812e704b46,2016-02-09T11:30:42,6,,-1,,,201602060307,578,arifuttaran
+chanderhowara,90,FH, জামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৫.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,2,2,no,no,,no,,3913647224031,,24.850428333333337 89.83293166666665 13.2 4.9,24.8504283333,89.8329316667,13.2,4.9,,,2016-02-08T11:47:29.749+06,2016-02-08T11:51:46.250+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,5349254,73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,2016-02-09T11:30:50,7,,-1,,,201602060307,257,arifuttaran
+chanderhowara,90,FH,মো: গোলজার হোসেন,মৃত মো: শাহাজান অালী,মেম্বার বাড়ি,চান্দের হাওড়া,০১৭৪৩০৪৮৪২৩,doctor,educated,0,১৫.০০,poor,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,no,no,no,no,yes,no,4,3,no,no,,,,3913647224185,,24.849339999999998 89.83242166666666 -25.9 4.7,24.84934,89.8324216667,-25.9,4.7,,,2016-02-08T12:14:23.738+06,2016-02-08T12:21:48.113+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:35bb0314-bdc4-4e1d-9994-66bd745d0c0b,5349255,35bb0314-bdc4-4e1d-9994-66bd745d0c0b,2016-02-09T11:30:52,8,,-1,,,201602060307,445,arifuttaran
+chanderhowara,90,FH, সুমন টিকাদার,মৃত নওয়াব অালি টিকাদার,টিকাদার বাড়ি,চান্দের হাওড়া,০১৭৩৪২৫১৩৩৮,farmer,litteracy,১০.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,no,no,no,no,yes,no,1,4,no,no,,,,3913647224283,,24.848106666666663 89.83239833333333 -115.7 5.4,24.8481066667,89.8323983333,-115.7,5.4,,,2016-02-08T12:41:22.328+06,2016-02-08T12:45:05.345+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:2454181a-4d9a-4797-b908-0c1c82c6b92a,5349256,2454181a-4d9a-4797-b908-0c1c82c6b92a,2016-02-09T11:30:53,9,,-1,,,201602060307,223,arifuttaran
+chanderhowara,90,FH,অাব্দুল জলিল মন্ডল ,মৃত কুব্বাত মন্ডল ,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭৭২৫৬০১৯১,farmer,Illiterate,২৭.০০,১৫.০০,others,inheritance,,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,4,8,no,no,,,,3913647224167,,24.848364999999998 89.83268500000001 -50.4 4.9,24.848365,89.832685,-50.4,4.9,,,2016-02-08T12:45:10.187+06,2016-02-08T12:53:02.738+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:68e07841-0eb3-4960-bb5e-bbfffa0b609d,5349258,68e07841-0eb3-4960-bb5e-bbfffa0b609d,2016-02-09T11:30:57,10,,-1,,,201602060307,472,arifuttaran

cadasta/organization/tests/test_forms.py

@@ -1013,7 +1013,7 @@ def test_get_all_download(self):
             assert 'data-shp.zip' in testzip.namelist()
 
 
-class SelectImportFormTest(UserTestCase):
+class SelectImportFormTest(UserTestCase, TestCase):
 
     def setUp(self):
         super().setUp()

cadasta/organization/tests/test_importers.py

@@ -124,6 +124,11 @@ def test_import_data(self):
             if su.geometry is not None:
                 assert type(su.geometry) is Point
 
+        # test attribute creation
+        su = SpatialUnit.objects.first()
+        assert 'how_aquire_landwh' not in su.attributes.keys()
+        assert 'how_aquire_landw' in su.attributes.keys()
+
     def test_import_with_geoshape(self):
         importer = csv.CSVImporter(
             project=self.project, path=self.path + self.geoshape_csv)