Skip to content

Commit

Permalink
Zip all files at final step as celery task, #1398
Browse files Browse the repository at this point in the history
  • Loading branch information
@njkim
njkim committed Nov 1, 2023
1 parent 982b40e commit 252afca
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 129 deletions.
3 changes: 2 additions & 1 deletion arches_for_science/media/js/views/components/plugins/project-report-workflow.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ define([
templates: "['select-report-template']['select-template']['templates']",
projectId: "['select-project']['select-project']['project']",
physicalThingIds: "['select-project']['select-project']['physicalThings']",
annotationStepData: "['add-annotations']['add-annotations']"
annotationStepData: "['add-annotations']['add-annotations']",
projectFiles: "['download-project-files']['download-project-files']['files']",
}
},
],
Expand Down
46 changes: 1 addition & 45 deletions ...nce/media/js/views/components/workflows/project-report-workflow/download-project-files.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
define([
'jquery',
'js-cookie',
'knockout',
'arches',
'viewmodels/alert-json',
'templates/views/components/workflows/project-report-workflow/download-project-files.htm',
], function($, Cookies, ko, arches, JsonErrorAlertViewModel, downloadFilesTemplate) {
], function(ko, arches, downloadFilesTemplate) {
function viewModel(params) {
var self = this;

Expand Down Expand Up @@ -134,47 +131,6 @@ define([
};

this.getFilesFromObservation();

// this.downloadFiles = () => {
// const files = self.relatedObservations().reduce(
// (acc, observation) => acc.concat(observation.relatedFiles().filter(
// file => file.selected())), [])
// .map(file => {
// return {'name': file.name, 'fileid': file.file_id, 'project': self.projectName};
// });
// const formData = new window.FormData();

// formData.append('files', JSON.stringify(files));
// window.fetch(arches.urls.download_project_files, {
// method: 'POST',
// credentials: 'include',
// body: formData,
// headers: {
// "X-CSRFToken": Cookies.get('csrftoken')
// }
// })
// .then(response => {
// if (response.ok) {
// return response.json();
// } else {
// throw response;
// }
// })
// .then((json) => self.message(json.message))
// .catch((response) => {
// response.json().then(
// error => {
// params.pageVm.alert(
// new JsonErrorAlertViewModel(
// 'ep-alert-red',
// error,
// null,
// function(){}
// )
// );
// });
// });
// };
}

ko.components.register('download-project-files', {
Expand Down
76 changes: 40 additions & 36 deletions ...or_science/media/js/views/components/workflows/project-report-workflow/download-report.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ define([
'underscore',
'arches',
'knockout',
'knockout-mapping',
'js-cookie',
'utils/report',
'viewmodels/alert-json',
'templates/views/components/workflows/project-report-workflow/download-report.htm'
], function(_, arches, ko, koMapping, cookies, reportUtils, downloadReportTemplate) {
], function(_, arches, ko, cookies, reportUtils, JsonErrorAlertViewModel, downloadReportTemplate) {
function viewModel(params) {
const observationGraphId = "615b11ee-c457-11e9-910c-a4d18cec433a";
const collectionGraphId = "1b210ef3-b25c-11e9-a037-a4d18cec433a";
Expand All @@ -17,6 +17,8 @@ define([
const self = this;
const projectId = params.projectId;
const physicalThingFromPreviousStep = params.physicalThingIds;
const projectFiles = params.projectFiles;
this.message = ko.observable();
this.templates = ko.observableArray(params.templates);
const screenshots = params.annotationStepData ? params.annotationStepData.screenshots : [];
const lbgApiEndpoint = `${arches.urls.api_bulk_disambiguated_resource_instance}?v=beta&resource_ids=`;
Expand Down Expand Up @@ -46,7 +48,7 @@ define([
};
getProjectName();

const generateReport = async(template) => {
this.generateReport = async() => {
const relatedObjects = await getRelatedResources(projectId);
const collections = relatedObjects.related_resources.filter(rr => rr.graph_id == collectionGraphId);
const allPhysicalThingsResponse = collections.map(async(collection) => {
Expand Down Expand Up @@ -79,60 +81,62 @@ define([
const today = new Date();
const options = { year: 'numeric', month: 'long', day: 'numeric' };
const reportDate = today.toLocaleDateString('en-US', options);
const filename = reportUtils.slugify(`${self.projectName()}_${template.name}_${reportDate}`);
const physicalThingsDetailsArray = [...Object.values(physicalThingsDetails)];
const objectOfStudyDetailsArray = physicalThingsDetailsArray.filter(thing => physicalThingFromPreviousStep.includes(thing.resourceinstanceid));
const analysisAreas = physicalThingsDetailsArray.filter(physicalThing => physicalThing.resource?.type?.["@display_value"] == 'analysis areas');
const annotationScreenshots = screenshots?.map((screenshot) => {
const url = `${window.location.origin}/temp_file/${screenshot.fileId}`;
return {...screenshot, url};
});
const data = {
projectId,
const files = projectFiles;

const templates = self.templates().map((template) => ({
templateId: template.id,
filename,
filename: reportUtils.slugify(`${self.projectName()}_${template.name}_${reportDate}`)
}));

data = {
projectId,
templates,
annotationScreenshots,
reportDate,
analysisAreas,
observationDetails: [...Object.values(observationDetails)],
projectDetails: [...Object.values(projectDetails)],
physicalThingsDetails: physicalThingsDetailsArray,
objectOfStudyDetails: objectOfStudyDetailsArray
objectOfStudyDetails: objectOfStudyDetailsArray,
files: files
};

const result = await fetch(arches.urls.reports(template.id), {
window.fetch(arches.urls.download_project_files, {
method: 'POST',
credentials: 'include',
body: JSON.stringify(data),
headers: {
'Content-Type': 'application/json',
"X-CSRFToken": cookies.get('csrftoken')
},
body: JSON.stringify(data)
}
}).then(response => {
if (response.ok) {
return response.json();
} else {
throw response;
}
})
.then((json) => self.message(json.message))
.catch((response) => {
response.json().then(
error => {
params.pageVm.alert(
new JsonErrorAlertViewModel(
'ep-alert-red',
error,
null,
function(){}
)
);
});
});

if(result.ok){
const blobResult = await result.blob();
let downloadName;
result.headers.forEach(header => {
if (header.match(regex)) {
downloadName = header.match(regex)[1];
}
});

this.downloadInfo.push({
downloadLink: URL.createObjectURL(blobResult),
downloadName: downloadName,
templateName: template.name,
});
} else {
this.errorInfo.push({
templateName: template.name,
})
}
};

this.templates().forEach(template => {
generateReport(template);
});
}

ko.components.register('download-report', {
Expand Down
6 changes: 3 additions & 3 deletions arches_for_science/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,17 @@


@shared_task
def download_project_files_task(userid, files, project_name):
def download_project_files_task(userid, files, temp_files, project_name):
from arches_for_science.views.download_project_files import FileDownloader

downloader = FileDownloader()
user = User.objects.get(pk=userid)
exportid, files = downloader.create_download_zipfile(user, files, project_name)
exportid, files = downloader.create_download_zipfile(user, files, temp_files, project_name)

search_history_obj = SearchExportHistory.objects.get(pk=exportid) if exportid else None

msg = _(
"The related files for the project '{}' are ready for download. You have 24 hours to download your files before they are automatically removed."
"The report(s) and the related file(s) for the project '{}' are ready to download. You have 24 hours to download your files before they are automatically removed."
).format(project_name)
notiftype_name = "Search Export Download Ready"
context = dict(
Expand Down
39 changes: 7 additions & 32 deletions ..._science/templates/views/components/workflows/project-report-workflow/download-report.htm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,11 @@
<span class="h5 file-list-label" data-bind="text: projectName() + $root.translations.reportDownload"></span>
</div>

<!-- ko if: (downloadInfo().length + errorInfo().length) !== templates().length -->
<div class="download-report-container">
<div class="loading-mask" style="background-color: transparent;"></div>

<!-- ko if: templates().length <= 1 -->
<div>{% trans "A report is being generated. A link will appear when the report is ready to be downloaded." %}</div>
<!-- /ko -->

<!-- ko if: templates().length > 1 -->
<div>{% trans "The reports are being generated. A link for each report will appear when the report is ready to be downloaded." %}</div>
<!-- /ko -->
</div>
<!-- /ko -->

<!-- ko if: (downloadInfo().length + errorInfo().length) === templates().length -->
<!-- ko foreach: downloadInfo -->
<div class="download-report-button-container">
<div class="file-download-button btn btn-success" role="link">
<a style="color: #fff;" target="_blank"
data-bind="
attr:{ href: downloadLink, download: downloadName },
text: $root.translations.download + ' ' + templateName"
></a><i class="fa fa-download" style="margin-inline-start: 0.5em; margin-block-start: 0.25em"></i>
</div>
</div>
<!-- /ko -->
<!-- ko foreach: errorInfo -->
<div class="download-report-container">
<div>{% trans "There was a problem generating a report for " %}<span data-bind="text: templateName"></span>{% trans ". You can try again (refresh this page) or contact an administrator for help." %}</div>
</div>
<!-- /ko -->
<!-- /ko -->
<div class="download-report-button-container">
<button class="file-download-button btn btn-success" data-bind="click: generateReport">
<span style="color: #fff;" data-bind="text: $root.translations.download"></span>
<i class="fa fa-download" style="margin-inline-start: 0.5em; margin-block-start: 0.25em"></i>
</button>
<div style="padding-top: 10px;" data-bind="text: message"></div>
</div>
</div>
62 changes: 50 additions & 12 deletions arches_for_science/views/download_project_files.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,56 @@
import arches.app.utils.zip as zip_utils
import arches.app.utils.task_management as task_management
import arches_for_science.tasks as tasks
from arches_templating.views.template import TemplateView


class FileDownloader(View):
def post(self, request):
files = json.loads(request.POST.get("files"))
if len(files) == 0:
message = _("No files were selected for download.")
return JSONResponse({"success": False, "message": message})
json_data = json.loads(request.body)

# get reports and save them
templates = json_data.pop("templates")
generated_reports = []
for template in templates:
request.POST = request.POST.copy()
json_data["templateId"] = template["templateId"]
json_data["filename"] = template["filename"]
request._body = json.dumps(json_data) # TODO: there should be a better way
report_template = TemplateView()
response = report_template.post(request, template["templateId"])

name = response.headers["Content-Disposition"].split("=")[1] #TODO: need more robust way to do this
content = response.content
f = BytesIO(content)
file = File(f)

report_file = models.TempFile()
report_file.source = "project-report-workflow"
report_file.path.save(name, file)

generated_reports.append({"name": name, "fileid": report_file.fileid})

# get attached files and zip them
files = json_data["files"]
screenshots = [{"fileid": i["fileId"], "name": i["imageName"]} for i in json_data["annotationScreenshots"]]
temp_files = generated_reports + screenshots

project_name = files[0]["project"]
userid = request.user.id

if len(files) + len(temp_files) == 0:
message = _(
"The report(s) are submitted for download. When completed, the bell icon at the top of the page will update with links to download your files."
).format(len(files))
return JSONResponse({"success": True, "message": message})

if task_management.check_if_celery_available():
result = tasks.download_project_files_task.apply_async(
(userid, files, project_name),
(userid, files, temp_files, project_name),
)
message = _(
"{} file(s) submitted for download. When completed, the bell icon at the top of the page will update with links to download your files."
).format(len(files))
"The report(s) and {} additional file(s) are submitted for download. When completed, the bell icon at the top of the page will update with links to download your files."
).format(len(files) + len(screenshots))
return JSONResponse({"success": True, "message": message})
else:
response = {
Expand All @@ -40,17 +71,24 @@ def post(self, request):
}
return JSONErrorResponse(content=response)

def create_download_zipfile(self, user, files, project_name):
def create_download_zipfile(self, user, files, temp_files, project_name):
file_ids = [file["fileid"] for file in files]
file_objects = list(models.File.objects.filter(pk__in=file_ids))
for file in files:
for file_object in file_objects:
if str(file_object.fileid) == file["fileid"]:

temp_file_ids = [temp_file["fileid"] for temp_file in temp_files]
temp_file_objects = list(models.TempFile.objects.filter(pk__in=temp_file_ids))

all_files = files + temp_files
all_file_objects = file_objects + temp_file_objects

for file in all_files:
for file_object in all_file_objects:
if str(file_object.fileid) == str(file["fileid"]):
file["file"] = file_object.path
download_files = []
skipped_files = []
size_limit = 104857600 # 100MByte
for file in files:
for file in all_files:
if file["file"].size >= size_limit:
skipped_files.append({"name": file["name"], "fileid": file["fileid"]})
else:
Expand Down

0 comments on commit 252afca

Please sign in to comment.