From 10c90ceaf4da4e9f5a3b9dc4a96e866f23cc19b9 Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 11 Nov 2024 11:28:53 +0100 Subject: [PATCH 1/7] Add dataset upload test without using frontend --- .../backend-snapshot-tests/datasets.e2e.ts | 105 ++++++++++++++++++ frontend/javascripts/test/e2e-setup.ts | 6 +- 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts index a82653b946..14f1db9694 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts @@ -9,6 +9,7 @@ import { import type { APIDataset } from "types/api_flow_types"; import * as api from "admin/admin_rest_api"; import test from "ava"; +import fs from "node:fs"; async function getFirstDataset(): Promise { const datasets = await api.getActiveDatasetsOfMyOrganization(); @@ -108,3 +109,107 @@ test("Zarr 3 streaming", async (t) => { const base64 = btoa(String.fromCharCode(...new Uint8Array(bytes.slice(-128)))); t.snapshot(base64); }); + +test("Dataset upload", async (t) => { + + const uploadId = "test-dataset-upload-" + Date.now(); + + await fetch("/data/datasets/reserveUpload", + { + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + }), + body: JSON.stringify({ + filePaths: ["test-dataset-upload.zip"], + folderId: "570b9f4e4bb848d0885ea917", + initialTeams: [], + layersToLink: [], + name: "test-dataset-upload", + organization: "Organization_X", + totalFileCount: 1, + uploadId: uploadId, + }), + }) + + const filePath = "test/dataset/test-dataset.zip"; + const testDataset = fs.readFileSync(filePath); + + let formData = new FormData(); + formData.append("resumableChunkNumber", "1"); + formData.append("resumableChunkSize", "10485760"); + formData.append("resumableCurrentChunkSize", "71988"); + formData.append("resumableTotalSize", "71988"); + formData.append("resumableType", "application/zip"); + formData.append("resumableIdentifier", uploadId + "/test-dataset.zip"); + formData.append("resumableFilename", "test-dataset.zip"); + formData.append("resumableRelativePath", "test-dataset.zip"); + formData.append("resumableTotalChunks", "1"); + + // Setting the correct content type header automatically does not work (the boundary is not included) + // We can not extract the boundary from the FormData object + // Thus we have to set the content type header ourselves and create the body manually + + const boundary = "----WebKitFormBoundaryAqTsFa4N9FW7zF7I"; + let bodyString = `--${boundary}\r\n`; + for (const [key, value] of formData.entries()) { + bodyString += `Content-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`; + bodyString += `--${boundary}\r\n`; + } + bodyString += `Content-Disposition: form-data; name="file"; filename="test-dataset.zip"\r\n`; + bodyString += `Content-Type: application/octet-stream\r\n\r\n`; + + // We have to send the file as bytes, otherwise JS does some encoding, resulting in erroneous bytes + + const formBytes = new TextEncoder().encode(bodyString); + const fileBytes =new Uint8Array(testDataset); + const endBytes = new TextEncoder().encode(`\r\n--${boundary}--`); + const body = new Uint8Array(formBytes.length + fileBytes.length + endBytes.length); + body.set(formBytes, 0); + body.set(fileBytes, formBytes.length); + body.set(endBytes, formBytes.length + fileBytes.length); + + let content_type = `multipart/form-data; boundary=${boundary}`; + + const uploadResult = await fetch("/data/datasets", + { + method: "POST", + headers: new Headers( + { + "Content-Type": content_type, + } + ), + body: body + }) + + if (uploadResult.status !== 200) { + t.fail("Dataset upload failed"); + } + + const finishResult = await fetch("/data/datasets/finishUpload", + { + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + }), + body: JSON.stringify({ + uploadId: uploadId, + needsConversion: false + }) + }); + + if (finishResult.status !== 200) { + t.fail("Dataset upload failed at finish"); + } + + const result = await fetch("/api/datasets/Organization_X/test-dataset-upload/health", + { + headers: new Headers(), + }) + + if (result.status !== 200) { + t.fail("Dataset upload failed"); + } + t.pass(); + +}); diff --git a/frontend/javascripts/test/e2e-setup.ts b/frontend/javascripts/test/e2e-setup.ts index b1330eb25e..e8d6ed720d 100644 --- a/frontend/javascripts/test/e2e-setup.ts +++ b/frontend/javascripts/test/e2e-setup.ts @@ -3,7 +3,7 @@ import _ from "lodash"; // @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'deep... Remove this comment to see the full error message import deepForEach from "deep-for-each"; // @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'node... Remove this comment to see the full error message -import fetch, { Headers, Request, Response, FetchError } from "node-fetch"; +import fetch, { Headers, FormData, Request, Response, FetchError, File } from "node-fetch"; import fs from "node:fs"; // @ts-expect-error ts-migrate(7016) FIXME: Could not find a declaration file for module 'shel... Remove this comment to see the full error message import shell from "shelljs"; @@ -67,7 +67,7 @@ global.fetch = function fetchWrapper(url, options) { let newUrl = url; // @ts-expect-error ts-migrate(2339) FIXME: Property 'indexOf' does not exist on type 'Request... Remove this comment to see the full error message - if (url.indexOf("http:") === -1) { + if (url.indexOf("http:") === -1 && url.indexOf("https:") === -1) { newUrl = `http://localhost:9000${url}`; } @@ -84,6 +84,8 @@ global.Request = Request; global.Response = Response; // @ts-ignore FIXME: Element implicitly has an 'any' type because type ... Remove this comment to see the full error message global.FetchError = FetchError; +global.FormData = FormData; +global.File = File; const { JSDOM } = require("jsdom"); From d708efe8838d03893eec0f92061659a74bac0386 Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 11 Nov 2024 11:37:48 +0100 Subject: [PATCH 2/7] Delete test dataset directory before test --- test/e2e/End2EndSpec.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e/End2EndSpec.scala b/test/e2e/End2EndSpec.scala index 1de30f63de..dc61e6c5d3 100644 --- a/test/e2e/End2EndSpec.scala +++ b/test/e2e/End2EndSpec.scala @@ -1,6 +1,6 @@ package e2e -import com.scalableminds.util.io.ZipIO +import com.scalableminds.util.io.{PathUtils, ZipIO} import com.typesafe.scalalogging.LazyLogging import org.scalatestplus.play.guice._ import org.specs2.main.Arguments @@ -51,9 +51,11 @@ class End2EndSpec(arguments: Arguments) extends Specification with GuiceFakeAppl private def ensureTestDataset(): Unit = { val testDatasetPath = "test/dataset/test-dataset.zip" val dataDirectory = new File("binaryData/Organization_X") - if (!dataDirectory.exists()) { - dataDirectory.mkdirs() + if (dataDirectory.exists()) { + println("Deleting existing data directory Organization_X") + PathUtils.deleteDirectoryRecursively(dataDirectory.toPath) } + dataDirectory.mkdirs() val testDatasetZip = new File(testDatasetPath) if (!testDatasetZip.exists()) { throw new Exception("Test dataset zip file does not exist.") From 2bb90db885485a3336662fad1ab264dbb34cd38c Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 11 Nov 2024 11:46:28 +0100 Subject: [PATCH 3/7] Fix frontend lint --- .../backend-snapshot-tests/datasets.e2e.ts | 82 +++++++++---------- 1 file changed, 37 insertions(+), 45 deletions(-) diff --git a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts index 14f1db9694..6f7674ed2f 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts @@ -111,26 +111,24 @@ test("Zarr 3 streaming", async (t) => { }); test("Dataset upload", async (t) => { - const uploadId = "test-dataset-upload-" + Date.now(); - await fetch("/data/datasets/reserveUpload", - { - method: "POST", - headers: new Headers({ - "Content-Type": "application/json", - }), - body: JSON.stringify({ - filePaths: ["test-dataset-upload.zip"], - folderId: "570b9f4e4bb848d0885ea917", - initialTeams: [], - layersToLink: [], - name: "test-dataset-upload", - organization: "Organization_X", - totalFileCount: 1, - uploadId: uploadId, - }), - }) + await fetch("/data/datasets/reserveUpload", { + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + }), + body: JSON.stringify({ + filePaths: ["test-dataset-upload.zip"], + folderId: "570b9f4e4bb848d0885ea917", + initialTeams: [], + layersToLink: [], + name: "test-dataset-upload", + organization: "Organization_X", + totalFileCount: 1, + uploadId: uploadId, + }), + }); const filePath = "test/dataset/test-dataset.zip"; const testDataset = fs.readFileSync(filePath); @@ -162,7 +160,7 @@ test("Dataset upload", async (t) => { // We have to send the file as bytes, otherwise JS does some encoding, resulting in erroneous bytes const formBytes = new TextEncoder().encode(bodyString); - const fileBytes =new Uint8Array(testDataset); + const fileBytes = new Uint8Array(testDataset); const endBytes = new TextEncoder().encode(`\r\n--${boundary}--`); const body = new Uint8Array(formBytes.length + fileBytes.length + endBytes.length); body.set(formBytes, 0); @@ -171,45 +169,39 @@ test("Dataset upload", async (t) => { let content_type = `multipart/form-data; boundary=${boundary}`; - const uploadResult = await fetch("/data/datasets", - { - method: "POST", - headers: new Headers( - { - "Content-Type": content_type, - } - ), - body: body - }) + const uploadResult = await fetch("/data/datasets", { + method: "POST", + headers: new Headers({ + "Content-Type": content_type, + }), + body: body, + }); if (uploadResult.status !== 200) { t.fail("Dataset upload failed"); } - const finishResult = await fetch("/data/datasets/finishUpload", - { - method: "POST", - headers: new Headers({ - "Content-Type": "application/json", - }), - body: JSON.stringify({ - uploadId: uploadId, - needsConversion: false - }) - }); + const finishResult = await fetch("/data/datasets/finishUpload", { + method: "POST", + headers: new Headers({ + "Content-Type": "application/json", + }), + body: JSON.stringify({ + uploadId: uploadId, + needsConversion: false, + }), + }); if (finishResult.status !== 200) { t.fail("Dataset upload failed at finish"); } - const result = await fetch("/api/datasets/Organization_X/test-dataset-upload/health", - { - headers: new Headers(), - }) + const result = await fetch("/api/datasets/Organization_X/test-dataset-upload/health", { + headers: new Headers(), + }); if (result.status !== 200) { t.fail("Dataset upload failed"); } t.pass(); - }); From b198e84ec0e81d02a8838d477da660fd13d4973c Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 11 Nov 2024 11:55:40 +0100 Subject: [PATCH 4/7] Fix frontend lint again --- .../javascripts/test/backend-snapshot-tests/datasets.e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts index 6f7674ed2f..df867c1bc6 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts @@ -155,7 +155,7 @@ test("Dataset upload", async (t) => { bodyString += `--${boundary}\r\n`; } bodyString += `Content-Disposition: form-data; name="file"; filename="test-dataset.zip"\r\n`; - bodyString += `Content-Type: application/octet-stream\r\n\r\n`; + bodyString += "Content-Type: application/octet-stream\r\n\r\n"; // We have to send the file as bytes, otherwise JS does some encoding, resulting in erroneous bytes From db2bc6cf4e4905f2eb2a1aefdf8c34717a4b8bcc Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 11 Nov 2024 13:00:07 +0100 Subject: [PATCH 5/7] Clean up config.yml --- .circleci/config.yml | 1 - frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a70616eb2d..f35317647f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,6 @@ jobs: - run: name: Run end-to-end tests command: | - mkdir -p binaryData/Organization_X && chmod 777 binaryData/Organization_X for i in {1..3}; do # retry .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$? done diff --git a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts index df867c1bc6..c4305a7885 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts @@ -150,6 +150,7 @@ test("Dataset upload", async (t) => { const boundary = "----WebKitFormBoundaryAqTsFa4N9FW7zF7I"; let bodyString = `--${boundary}\r\n`; + // @ts-ignore for (const [key, value] of formData.entries()) { bodyString += `Content-Disposition: form-data; name="${key}"\r\n\r\n${value}\r\n`; bodyString += `--${boundary}\r\n`; From 84eed1ea3bb26d4d71b30778b4b5edede783fbe0 Mon Sep 17 00:00:00 2001 From: frcroth Date: Mon, 11 Nov 2024 13:15:34 +0100 Subject: [PATCH 6/7] Recreate Org x directory in circle ci config --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f35317647f..a70616eb2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,6 +118,7 @@ jobs: - run: name: Run end-to-end tests command: | + mkdir -p binaryData/Organization_X && chmod 777 binaryData/Organization_X for i in {1..3}; do # retry .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$? done From 03ece1c07247d4056bc4961387a3aaac47f95c87 Mon Sep 17 00:00:00 2001 From: frcroth Date: Wed, 13 Nov 2024 10:09:29 +0100 Subject: [PATCH 7/7] Update frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts Co-authored-by: Daniel --- .../javascripts/test/backend-snapshot-tests/datasets.e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts index c4305a7885..9a09166530 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts @@ -202,7 +202,7 @@ test("Dataset upload", async (t) => { }); if (result.status !== 200) { - t.fail("Dataset upload failed"); + t.fail("Dataset health check after upload failed"); } t.pass(); });