diff --git a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts index a82653b946..9a09166530 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts @@ -9,6 +9,7 @@ import { import type { APIDataset } from "types/api_flow_types"; import * as api from "admin/admin_rest_api"; import test from "ava"; +import fs from "node:fs"; async function getFirstDataset(): Promise { const datasets = await api.getActiveDatasetsOfMyOrganization(); @@ -108,3 +109,100 @@ test("Zarr 3 streaming", async (t) => { const base64 = btoa(String.fromCharCode(...new Uint8Array(bytes.slice(-128)))); t.snapshot(base64); }); + +test("Dataset upload", async (t) => { + const uploadId = "test-dataset-upload-" + Date.now(); + + await fetch("/data/datasets/reserveUpload", { + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + }), + body: JSON.stringify({ + filePaths: ["test-dataset-upload.zip"], + folderId: "570b9f4e4bb848d0885ea917", + initialTeams: [], + layersToLink: [], + name: "test-dataset-upload", + organization: "Organization_X", + totalFileCount: 1, + uploadId: uploadId, + }), + }); + + const filePath = "test/dataset/test-dataset.zip"; + const testDataset = fs.readFileSync(filePath); + + let formData = new FormData(); + formData.append("resumableChunkNumber", "1"); + formData.append("resumableChunkSize", "10485760"); + formData.append("resumableCurrentChunkSize", "71988"); + formData.append("resumableTotalSize", "71988"); + formData.append("resumableType", "application/zip"); + formData.append("resumableIdentifier", uploadId + "/test-dataset.zip"); + formData.append("resumableFilename", "test-dataset.zip"); + formData.append("resumableRelativePath", "test-dataset.zip"); + formData.append("resumableTotalChunks", "1"); + + // Setting the correct content type header automatically does not work (the boundary is not included) + // We can not extract the boundary from the FormData object + // Thus we have to set the content type header ourselves and create the body manually + + const boundary = "----WebKitFormBoundaryAqTsFa4N9FW7zF7I"; + let bodyString = `--${boundary}\r\n`; + // @ts-ignore + for (const [key, value] of formData.entries()) { + bodyString += `Content-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`; + bodyString += `--${boundary}\r\n`; + } + bodyString += `Content-Disposition: form-data; name="file"; filename="test-dataset.zip"\r\n`; + bodyString += "Content-Type: application/octet-stream\r\n\r\n"; + + // We have to send the file as bytes, otherwise JS does some encoding, resulting in erroneous bytes + + const formBytes = new TextEncoder().encode(bodyString); + const fileBytes = new Uint8Array(testDataset); + const endBytes = new TextEncoder().encode(`\r\n--${boundary}--`); + const body = new Uint8Array(formBytes.length + fileBytes.length + endBytes.length); + body.set(formBytes, 0); + body.set(fileBytes, formBytes.length); + body.set(endBytes, formBytes.length + fileBytes.length); + + let content_type = `multipart/form-data; boundary=${boundary}`; + + const uploadResult = await fetch("/data/datasets", { + method: "POST", + headers: new Headers({ + "Content-Type": content_type, + }), + body: body, + }); + + if (uploadResult.status !== 200) { + t.fail("Dataset upload failed"); + } + + const finishResult = await fetch("/data/datasets/finishUpload", { + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + }), + body: JSON.stringify({ + uploadId: uploadId, + needsConversion: false, + }), + }); + + if (finishResult.status !== 200) { + t.fail("Dataset upload failed at finish"); + } + + const result = await fetch("/api/datasets/Organization_X/test-dataset-upload/health", { + headers: new Headers(), + }); + + if (result.status !== 200) { + t.fail("Dataset health check after upload failed"); + } + t.pass(); +}); diff --git a/frontend/javascripts/test/e2e-setup.ts b/frontend/javascripts/test/e2e-setup.ts index b1330eb25e..e8d6ed720d 100644 --- a/frontend/javascripts/test/e2e-setup.ts +++ b/frontend/javascripts/test/e2e-setup.ts @@ -3,7 +3,7 @@ import _ from "lodash"; // @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'deep... Remove this comment to see the full error message import deepForEach from "deep-for-each"; // @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'node... Remove this comment to see the full error message -import fetch, { Headers, Request, Response, FetchError } from "node-fetch"; +import fetch, { Headers, FormData, Request, Response, FetchError, File } from "node-fetch"; import fs from "node:fs"; // @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'shel... Remove this comment to see the full error message import shell from "shelljs"; @@ -67,7 +67,7 @@ global.fetch = function fetchWrapper(url, options) { let newUrl = url; // @ts-expect-error ts-migrate(2339) FIXME: Property 'indexOf' does not exist on type 'Request... Remove this comment to see the full error message - if (url.indexOf("http:") === -1) { + if (url.indexOf("http:") === -1 && url.indexOf("https:") === -1) { newUrl = `http://localhost:9000${url}`; } @@ -84,6 +84,8 @@ global.Request = Request; global.Response = Response; // @ts-ignore FIXME: Element implicitly has an 'any' type because type ... Remove this comment to see the full error message global.FetchError = FetchError; +global.FormData = FormData; +global.File = File; const { JSDOM } = require("jsdom"); diff --git a/test/e2e/End2EndSpec.scala b/test/e2e/End2EndSpec.scala index 1de30f63de..dc61e6c5d3 100644 --- a/test/e2e/End2EndSpec.scala +++ b/test/e2e/End2EndSpec.scala @@ -1,6 +1,6 @@ package e2e -import com.scalableminds.util.io.ZipIO +import com.scalableminds.util.io.{PathUtils, ZipIO} import com.typesafe.scalalogging.LazyLogging import org.scalatestplus.play.guice._ import org.specs2.main.Arguments @@ -51,9 +51,11 @@ class End2EndSpec(arguments: Arguments) extends Specification with GuiceFakeAppl private def ensureTestDataset(): Unit = { val testDatasetPath = "test/dataset/test-dataset.zip" val dataDirectory = new File("binaryData/Organization_X") - if (!dataDirectory.exists()) { - dataDirectory.mkdirs() + if (dataDirectory.exists()) { + println("Deleting existing data directory Organization_X") + PathUtils.deleteDirectoryRecursively(dataDirectory.toPath) } + dataDirectory.mkdirs() val testDatasetZip = new File(testDatasetPath) if (!testDatasetZip.exists()) { throw new Exception("Test dataset zip file does not exist.")