Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add file storage option #12590

Draft
wants to merge 10 commits into
base: release-v0.17.x
Choose a base branch
from
27 changes: 21 additions & 6 deletions kolibri/core/auth/management/commands/bulkexportusers.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import csv
import io
import logging
import ntpath
import os
from collections import OrderedDict
from functools import partial

from django.conf import settings
from django.core.files.storage import DefaultStorage
from django.core.management.base import CommandError
from django.db.models import OuterRef
from django.db.models import Subquery
Expand All @@ -26,7 +28,6 @@
from kolibri.core.query import GroupConcatSubquery
from kolibri.core.tasks.management.commands.base import AsyncCommand
from kolibri.core.tasks.utils import get_current_job
from kolibri.core.utils.csv import open_csv_for_writing
from kolibri.core.utils.csv import output_mapper
from kolibri.utils import conf

Expand Down Expand Up @@ -153,17 +154,20 @@ def translate_labels():


def csv_file_generator(facility, filepath, overwrite=True):
if not overwrite and os.path.exists(filepath):
raise ValueError("{} already exists".format(filepath))
file_storage = DefaultStorage()
filename = file_storage.generate_filename(filepath.split("/")[-1])

if not overwrite and file_storage.exists(filename):
raise ValueError("{} already exists".format(filename))
queryset = FacilityUser.objects.filter(facility=facility)

header_labels = translate_labels().values()

csv_file = open_csv_for_writing(filepath)
csv_file = io.BytesIO()

with csv_file as f:
writer = csv.DictWriter(f, header_labels)
logger.info("Creating csv file {filename}".format(filename=filepath))
writer = csv.DictWriter(io.TextIOWrapper(f, encoding="utf-8"), header_labels)
logger.info("Creating users csv file {filename}".format(filename=filepath))
writer.writeheader()
usernames = set()

Expand Down Expand Up @@ -203,6 +207,17 @@ def csv_file_generator(facility, filepath, overwrite=True):
usernames.add(item["username"])
yield item

f.seek(0)
file = file_storage.save(filename, f)

try:
# If the file is local, we can get the path
logger.info("File saved - Path: {}".format(file_storage.path(file)))
except NotImplementedError:
# But if path is not implemented, we assume we can get the URL
logger.info("File saved - Path: {}".format(file_storage.url(file)))
logger.info("File saved - Size: {}".format(file_storage.size(file)))
Comment on lines +218 to +219
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jredrejo I've tried several things around here, but no matter what I do I the file always shows size 0... I've confirmed that there are users (usernames is full of data, for example) -- so I'm not sure why the writer isn't updating the file object here...



class Command(AsyncCommand):
def add_arguments(self, parser):
Expand Down
Loading