Fix utf-8 encoding issues for all Python CSV exports

kolibri/core/auth/csv_utils.py

@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
 import csv
-import io
 import logging
 import os
-import sys
 from collections import OrderedDict
 from functools import partial
 
@@ -18,6 +16,7 @@
 from kolibri.core.auth.models import Facility
 from kolibri.core.auth.models import FacilityUser
 from kolibri.core.query import SQCount
+from kolibri.core.utils.csv import open_csv_for_writing
 
 
 logger = logging.getLogger(__name__)
@@ -186,10 +185,7 @@ def csv_file_generator(facility, filepath, overwrite=True, demographic=False):
         column for column in db_columns if demographic or column not in DEMO_FIELDS
     )
 
-    if sys.version_info[0] < 3:
-        csv_file = io.open(filepath, "wb")
-    else:
-        csv_file = io.open(filepath, "w", newline="")
+    csv_file = open_csv_for_writing(filepath)
 
     with csv_file as f:
         writer = csv.DictWriter(f, header_labels)

kolibri/core/auth/management/commands/bulkexportusers.py

@@ -1,9 +1,7 @@
 import csv
-import io
 import logging
 import ntpath
 import os
-import sys
 from collections import OrderedDict
 from functools import partial
 from tempfile import mkstemp
@@ -28,6 +26,7 @@
 from kolibri.core.query import GroupConcatSubquery
 from kolibri.core.tasks.management.commands.base import AsyncCommand
 from kolibri.core.tasks.utils import get_current_job
+from kolibri.core.utils.csv import open_csv_for_writing
 from kolibri.utils import conf
 
 try:
@@ -161,10 +160,7 @@ def csv_file_generator(facility, filepath, overwrite=True):
 
     header_labels = labels.values()
 
-    if sys.version_info[0] < 3:
-        csv_file = io.open(filepath, "wb")
-    else:
-        csv_file = io.open(filepath, "w", newline="")
+    csv_file = open_csv_for_writing(filepath)
 
     with csv_file as f:
         writer = csv.DictWriter(f, header_labels)

kolibri/core/auth/test/test_bulk_import.py

@@ -1,5 +1,4 @@
 import csv
-import io
 import sys
 import tempfile
 from uuid import uuid4
@@ -15,6 +14,7 @@
 from kolibri.core.auth.constants import role_kinds
 from kolibri.core.auth.models import Classroom
 from kolibri.core.auth.models import FacilityUser
+from kolibri.core.utils.csv import open_csv_for_writing
 
 if sys.version_info[0] < 3:
     from cStringIO import StringIO
@@ -118,10 +118,7 @@ def setUp(self):
     def create_csv(self, filepath, rows, remove_uuid=False):
         header_labels = list(labels.values())
 
-        if sys.version_info[0] < 3:
-            csv_file = io.open(filepath, "wb")
-        else:
-            csv_file = io.open(filepath, "w", newline="")
+        csv_file = open_csv_for_writing(filepath)
 
         with csv_file as f:
             writer = csv.writer(f)

kolibri/core/logger/csv_export.py

@@ -6,7 +6,6 @@
 import logging
 import math
 import os
-import sys
 from collections import OrderedDict
 
 from django.core.cache import cache
@@ -23,6 +22,7 @@
 from kolibri.core.auth.models import Facility
 from kolibri.core.content.models import ChannelMetadata
 from kolibri.core.content.models import ContentNode
+from kolibri.core.utils.csv import open_csv_for_writing
 from kolibri.utils import conf
 
 
@@ -151,10 +151,7 @@ def csv_file_generator(facility, log_type, filepath, overwrite=False):
         if log_type == "summary" or label != "completion_timestamp"
     )
 
-    if sys.version_info[0] < 3:
-        csv_file = io.open(filepath, "wb")
-    else:
-        csv_file = io.open(filepath, "w", newline="", encoding="utf-8")
+    csv_file = open_csv_for_writing(filepath)
 
     with csv_file as f:
         writer = csv.DictWriter(f, header_labels)

kolibri/core/utils/csv.py

@@ -0,0 +1,8 @@
+import io
+import sys
+
+
+def open_csv_for_writing(filepath):
+    if sys.version_info[0] < 3:
+        return io.open(filepath, "wb")
+    return io.open(filepath, "w", newline="", encoding="utf-8-sig")