feat(website): support excel metadata files in data upload form

docs/src/content/docs/for-users/submit-sequences.md

@@ -9,7 +9,7 @@ Before you begin this process, you should ensure your data is in the correct for
 Loculus expects:
 
 -   Sequence data in `fasta` format with a unique submissionID per sequence.
--   Metadata in `tsv` format for each sequence.If you need help formatting metadata, there is a metadata template for each organism on the submission page.
+-   Metadata in `tsv` format for each sequence. If you upload through the Website, you can also use Excel files (`xls` or `xlsx` format). If you need help formatting metadata, there is a metadata template for each organism on the submission page.
 
 ![Metadata template.](../../../assets/MetadataTemplate.png)
 
@@ -60,3 +60,7 @@ curl -X 'POST' \
 Further information can be found in the API documentation of the instance.
 
 As with the website, data will now be processed, and you will have to approve your submission before it is finalized. You can see how to do this [here](../approve-submissions/).
+
+## Compressing files
+
+For both sequence and metadata files, compression is supported. The supported formats are: `zip`, `gz` (gzip), `zst` (ZStandard) and `xz` (LZMA). (Note that Excel file uploads with `xz` compression are currently not supported.)

website/package.json

@@ -35,8 +35,11 @@
         "axios": "^1.7.9",
         "change-case": "~5.3.0",
         "chart.js": "^4.4.7",
+        "fflate": "^0.8.2",
         "flowbite-react": "^0.10.2",
+        "fzstd": "^0.1.1",
         "jsonwebtoken": "^9.0.2",
+        "jszip": "^3.10.1",
         "just-kebab-case": "^4.2.0",
         "jwks-rsa": "^3.1.0",
         "luxon": "^3.5.0",
@@ -51,6 +54,7 @@
         "rsuite": "^5.76.2",
         "unplugin-icons": "^0.22.0",
         "winston": "^3.17.0",
+        "xlsx": "^0.18.5",
         "zod": "^3.24.1"
     },
     "devDependencies": {

website/src/components/Submission/DataUploadForm.tsx

@@ -1,10 +1,11 @@
 import { isErrorFromAlias } from '@zodios/core';
 import type { AxiosError } from 'axios';
 import { DateTime } from 'luxon';
-import { type ElementType, type FormEvent, useCallback, useEffect, useRef, useState } from 'react';
+import { type FormEvent, useState } from 'react';
 
 import { dataUploadDocsUrl } from './dataUploadDocsUrl.ts';
 import { getClientLogger } from '../../clientLogger.ts';
+import { UploadComponent } from './FileUpload/UploadComponent.tsx';
 import DataUseTermsSelector from '../../components/DataUseTerms/DataUseTermsSelector';
 import useClientFlag from '../../hooks/isClient.ts';
 import { routes } from '../../routes/routes.ts';
@@ -22,9 +23,7 @@ import { dateTimeInMonths } from '../../utils/DateTimeInMonths.tsx';
 import { createAuthorizationHeader } from '../../utils/createAuthorizationHeader.ts';
 import { stringifyMaybeAxiosError } from '../../utils/stringifyMaybeAxiosError.ts';
 import { withQueryProvider } from '../common/withQueryProvider.tsx';
-import MaterialSymbolsInfoOutline from '~icons/material-symbols/info-outline';
-import MaterialSymbolsLightDataTableOutline from '~icons/material-symbols-light/data-table-outline';
-import PhDnaLight from '~icons/ph/dna-light';
+import { FASTA_FILE_KIND, METADATA_FILE_KIND } from './FileUpload/fileProcessing.ts';
 
 export type UploadAction = 'submit' | 'revise';
 
@@ -103,9 +102,9 @@ const DevExampleData = ({
                 type='number'
                 value={exampleEntries ?? ''}
                 onChange={(event) => setExampleEntries(parseInt(event.target.value, 10))}
-                className='w-32'
+                className='w-32 h-6 rounded'
             />
-            <button type='button' onClick={handleLoadExampleData} className='border rounded px-2 py-1 '>
+            <button type='button' onClick={handleLoadExampleData} className='border rounded px-2 py-1 ml-2 h-6'>
                 Load Example Data
             </button>{' '}
             <br />
@@ -114,142 +113,6 @@ const DevExampleData = ({
     );
 };
 
-const UploadComponent = ({
-    setFile,
-    name,
-    title,
-    // eslint-disable-next-line @typescript-eslint/naming-convention
-    Icon,
-    fileType,
-}: {
-    setFile: (file: File | null) => void;
-    name: string;
-    title: string;
-    Icon: ElementType; // eslint-disable-line @typescript-eslint/naming-convention
-    fileType: string;
-}) => {
-    const [myFile, rawSetMyFile] = useState<File | null>(null);
-    const [isDragOver, setIsDragOver] = useState(false);
-    const isClient = useClientFlag();
-
-    const setMyFile = useCallback(
-        (file: File | null) => {
-            setFile(file);
-            rawSetMyFile(file);
-        },
-        [setFile, rawSetMyFile],
-    );
-    const fileInputRef = useRef<HTMLInputElement>(null);
-    const handleUpload = () => {
-        document.getElementById(name)?.click();
-    };
-
-    const handleDragOver = (e: React.DragEvent<HTMLDivElement>) => {
-        e.preventDefault();
-        setIsDragOver(true);
-    };
-
-    const handleDragLeave = (e: React.DragEvent<HTMLDivElement>) => {
-        e.preventDefault();
-        setIsDragOver(false);
-    };
-
-    const handleDrop = (e: React.DragEvent<HTMLDivElement>) => {
-        e.preventDefault();
-        setIsDragOver(false);
-        const file = e.dataTransfer.files[0];
-        setMyFile(file);
-    };
-
-    useEffect(() => {
-        const interval = setInterval(() => {
-            // Check if the file is no longer readable - which generally indicates the file has been edited since being
-            // selected in the UI - and if so clear it.
-            myFile
-                ?.slice(0, 1)
-                .arrayBuffer()
-                .catch(() => {
-                    setMyFile(null);
-                    if (fileInputRef.current) {
-                        fileInputRef.current.value = '';
-                    }
-                });
-        }, 500);
-
-        return () => clearInterval(interval);
-    }, [myFile, setMyFile]);
-    return (
-        <div className='sm:col-span-4'>
-            <label className='text-gray-900 font-medium text-sm block'>{title}</label>
-            {name === 'metadata_file' && (
-                <div>
-                    <span className='text-gray-500 text-xs'>
-                        The documentation pages contain more details on the required
-                    </span>
-                    <a href='/docs/concepts/metadataformat' className='text-primary-700 text-xs'>
-                        {' '}
-                        metadata format{' '}
-                    </a>
-                </div>
-            )}
-            <div
-                className={`mt-2 flex flex-col h-40 rounded-lg border ${myFile ? 'border-hidden' : 'border-dashed border-gray-900/25'} ${isDragOver && !myFile ? 'bg-green-100' : ''}`}
-                onDragOver={handleDragOver}
-                onDragLeave={handleDragLeave}
-                onDrop={handleDrop}
-            >
-                <div className='flex items-center justify-center'>
-                    <Icon className='mx-auto mt-4 mb-0 h-12 w-12 text-gray-300' aria-hidden='true' />
-                </div>
-                {!myFile ? (
-                    <div className='flex flex-col items-center justify-center flex-1 px-4 py-2'>
-                        <div className='text-center'>
-                            <label className='inline relative cursor-pointer rounded-md bg-white font-semibold text-primary-600 focus-within:outline-none focus-within:ring-2 focus-within:ring-primary-600 focus-within:ring-offset-2 hover:text-primary-500'>
-                                <span
-                                    onClick={(e) => {
-                                        e.preventDefault();
-                                        handleUpload();
-                                    }}
-                                >
-                                    Upload
-                                </span>
-                                {isClient && (
-                                    <input
-                                        id={name}
-                                        name={name}
-                                        type='file'
-                                        className='sr-only'
-                                        aria-label={title}
-                                        data-testid={name}
-                                        onChange={(event) => {
-                                            const file = event.target.files?.[0] ?? null;
-                                            setMyFile(file);
-                                        }}
-                                        ref={fileInputRef}
-                                    />
-                                )}
-                            </label>
-                            <span className='pl-1'>or drag and drop</span>
-                        </div>
-                        <p className='text-sm pb+2 leading-5 text-gray-600'>{fileType}</p>
-                    </div>
-                ) : (
-                    <div className='flex flex-col items-center justify-center text-center flex-1 px-4 py-2'>
-                        <div className='text-sm text-gray-500 mb-1'>{myFile.name}</div>
-                        <button
-                            onClick={() => setMyFile(null)}
-                            data-testid={`discard_${name}`}
-                            className='text-xs break-words text-gray-700 py-1.5 px-4 border border-gray-300 rounded-md hover:bg-gray-50'
-                        >
-                            Discard file
-                        </button>
-                    </div>
-                )}
-            </div>
-        </div>
-    );
-};
-
 const InnerDataUploadForm = ({
     accessToken,
     organism,
@@ -340,9 +203,7 @@ const InnerDataUploadForm = ({
                     <div className=''>
                         <h2 className='font-medium text-lg'>Sequences and metadata</h2>
                         <p className='text-gray-500 text-sm'>Select your sequence data and metadata files</p>
-
                         <p className='text-gray-400 text-xs mt-5'>
-                            <MaterialSymbolsInfoOutline className='w-5 h-5 inline-block mr-2' />
                             {action === 'revise' && (
                                 <span>
                                     <strong>
@@ -351,11 +212,12 @@ const InnerDataUploadForm = ({
                                     </strong>
                                 </span>
                             )}
-                            You can download{' '}
-                            <a
-                                href={routes.metadataTemplate(organism, action)}
-                                className='text-primary-700  opacity-90'
-                            >
+                            The documentation pages contain more details on the required{' '}
+                            <a href='/docs/concepts/metadataformat' className='text-primary-700 opacity-90'>
+                                metadata format
+                            </a>
+                            . You can download{' '}
+                            <a href={routes.metadataTemplate(organism, action)} className='text-primary-700 opacity-90'>
                                 a template
                             </a>{' '}
                             for the TSV metadata file with column headings.
@@ -399,27 +261,25 @@ const InnerDataUploadForm = ({
                                 />
                             )}
                     </div>
-                    <form className='sm:col-span-2 '>
-                        <div className='px-8'>
-                            <div className='flex flex-col gap-6 max-w-64'>
-                                <div className='sm:col-span-3'>
-                                    <UploadComponent
-                                        setFile={setSequenceFile}
-                                        name='sequence_file'
-                                        title='Sequence file'
-                                        Icon={PhDnaLight}
-                                        fileType='FASTA file'
-                                    />
-                                </div>
-                                <div className='sm:col-span-3'>
-                                    <UploadComponent
-                                        setFile={setMetadataFile}
-                                        name='metadata_file'
-                                        title='Metadata file'
-                                        Icon={MaterialSymbolsLightDataTableOutline}
-                                        fileType='TSV file'
-                                    />
-                                </div>
+                    <form className='sm:col-span-2'>
+                        <div className='flex flex-col lg:flex-row gap-6'>
+                            <div className='w-60 space-y-2'>
+                                <label className='text-gray-900 font-medium text-sm block'>Sequence File</label>
+                                <UploadComponent
+                                    setFile={setSequenceFile}
+                                    name='sequence_file'
+                                    ariaLabel='Sequence File'
+                                    fileKind={FASTA_FILE_KIND}
+                                />
+                            </div>
+                            <div className='w-60 space-y-2'>
+                                <label className='text-gray-900 font-medium text-sm block'>Metadata File</label>
+                                <UploadComponent
+                                    setFile={setMetadataFile}
+                                    name='metadata_file'
+                                    ariaLabel='Metadata File'
+                                    fileKind={METADATA_FILE_KIND}
+                                />
                             </div>
                         </div>
                     </form>