Zarr streaming e2e test (#8137)

.circleci/config.yml

@@ -118,6 +118,7 @@ jobs:
       - run:
           name: Run end-to-end tests
           command: |
+            mkdir -p binaryData/Organization_X && chmod 777 binaryData/Organization_X
             for i in {1..3}; do # retry
               .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$?
             done

conf/messages

@@ -73,10 +73,6 @@ oidc.disabled=OIDC is disabled
 oidc.configuration.invalid=OIDC configuration is invalid
 oidc.authentication.failed=Failed to register / log in via Single-Sign-On (SSO with OIDC)
 
-braintracing.new=An account on braintracing.org was created for you. You can use the same credentials as on WEBKNOSSOS to login.
-braintracing.error=We could not automatically create an account for you on braintracing.org. Please do it on your own.
-braintracing.exists=Great, you already have an account on braintracing.org. Please double check that you have uploaded all requested information.
-
 dataset=Dataset
 dataset.notFound=Dataset {0} does not exist or could not be accessed
 dataset.notFoundConsiderLogin=Dataset {0} does not exist or could not be accessed. You may need to log in.

docker-compose.yml

@@ -201,7 +201,7 @@ services:
           -Ddatastore.redis.address=redis
           -Ddatastore.watchFileSystem.enabled=false"
     volumes:
-      - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X
+      - ./binaryData/Organization_X:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X
 
   screenshot-tests:
     image: scalableminds/puppeteer:master

frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts

@@ -1,5 +1,11 @@
 import _ from "lodash";
-import { tokenUserA, setCurrToken, resetDatabase, writeTypeCheckingFile } from "test/e2e-setup";
+import {
+  tokenUserA,
+  setCurrToken,
+  resetDatabase,
+  writeTypeCheckingFile,
+  replaceVolatileValues,
+} from "test/e2e-setup";
 import type { APIDataset } from "types/api_flow_types";
 import * as api from "admin/admin_rest_api";
 import test from "ava";
@@ -15,6 +21,7 @@ async function getFirstDataset(): Promise<APIDataset> {
 test.before("Reset database and change token", async () => {
   resetDatabase();
   setCurrToken(tokenUserA);
+  await api.triggerDatasetCheck("http://localhost:9000");
 });
 test.serial("getDatasets", async (t) => {
   let datasets = await api.getDatasets();
@@ -29,27 +36,27 @@ test.serial("getDatasets", async (t) => {
   writeTypeCheckingFile(datasets, "dataset", "APIDatasetCompact", {
     isArray: true,
   });
-  t.snapshot(datasets);
+  t.snapshot(replaceVolatileValues(datasets));
 });
 test("getActiveDatasets", async (t) => {
   let datasets = await api.getActiveDatasetsOfMyOrganization();
   datasets = _.sortBy(datasets, (d) => d.name);
-  t.snapshot(datasets);
+  t.snapshot(replaceVolatileValues(datasets));
 });
 test("getDatasetAccessList", async (t) => {
   const dataset = await getFirstDataset();
 
   const accessList = _.sortBy(await api.getDatasetAccessList(dataset), (user) => user.id);
 
-  t.snapshot(accessList);
+  t.snapshot(replaceVolatileValues(accessList));
 });
 test("updateDatasetTeams", async (t) => {
   const [dataset, newTeams] = await Promise.all([getFirstDataset(), api.getEditableTeams()]);
   const updatedDataset = await api.updateDatasetTeams(
     dataset,
     newTeams.map((team) => team.id),
   );
-  t.snapshot(updatedDataset);
+  t.snapshot(replaceVolatileValues(updatedDataset));
   // undo the Change
   await api.updateDatasetTeams(
     dataset,
@@ -62,3 +69,42 @@ test("updateDatasetTeams", async (t) => {
 //   await api.revokeDatasetSharingToken(dataset.name);
 //   t.pass();
 // });
+
+test("Zarr streaming", async (t) => {
+  const zattrsResp = await fetch("/data/zarr/Organization_X/test-dataset/segmentation/.zattrs", {
+    headers: new Headers(),
+  });
+  const zattrs = await zattrsResp.text();
+  t.snapshot(zattrs);
+
+  const rawDataResponse = await fetch(
+    "/data/zarr/Organization_X/test-dataset/segmentation/1/0.1.1.0",
+    {
+      headers: new Headers(),
+    },
+  );
+  const bytes = await rawDataResponse.arrayBuffer();
+  const base64 = btoa(String.fromCharCode(...new Uint8Array(bytes.slice(-128))));
+  t.snapshot(base64);
+});
+
+test("Zarr 3 streaming", async (t) => {
+  const zarrJsonResp = await fetch(
+    "/data/zarr3_experimental/Organization_X/test-dataset/segmentation/zarr.json",
+    {
+      headers: new Headers(),
+    },
+  );
+  const zarrJson = await zarrJsonResp.text();
+  t.snapshot(zarrJson);
+
+  const rawDataResponse = await fetch(
+    "/data/zarr3_experimental/Organization_X/test-dataset/segmentation/1/0.1.1.0",
+    {
+      headers: new Headers(),
+    },
+  );
+  const bytes = await rawDataResponse.arrayBuffer();
+  const base64 = btoa(String.fromCharCode(...new Uint8Array(bytes.slice(-128))));
+  t.snapshot(base64);
+});

frontend/javascripts/test/e2e-setup.ts

@@ -39,6 +39,7 @@ const volatileKeys: Array<string | number | symbol> = [
   "lastActivity",
   "tracingTime",
   "tracingId",
+  "sortingKey",
 ];
 export function replaceVolatileValues(obj: ArbitraryObject | null | undefined) {
   if (obj == null) return obj;
@@ -130,7 +131,7 @@ export async function writeTypeCheckingFile(
   const fullTypeAnnotation = options.isArray ? `Array<${typeString}>` : typeString;
   fs.writeFileSync(
     `frontend/javascripts/test/snapshots/type-check/test-type-checking-${name}.ts`,
-    ` 
+    `
 import type { ${typeString} } from "types/api_flow_types";
 const a: ${fullTypeAnnotation} = ${JSON.stringify(object)}`,
   );