Upload files by drag and drop (#8214)

The first part of #8158. No progress indication for the user implemented.


https://github.com/enso-org/enso/assets/6566674/2d8157d4-748f-4442-a9c3-a96ba0029056

# Important Notes
A few notable changes:
- A fix for `DataServer/writeBytes`
- Using `@noble/hashes` instead of `sha3` because the latter only works with node. I also tried using `js-sha3`, but it does not work well with Vite (see emn178/js-sha3#36)
- Fixed initialization of the ID map for new nodes. Also, all new nodes are prepended by four spaces to have proper indentation inside the `main` function.
- Fixed random pattern name generation because the previous approach sometimes produces identifiers starting with a number.

app/gui2/package.json

@@ -37,6 +37,7 @@
     "@fast-check/vitest": "^0.0.8",
     "@lezer/common": "^1.1.0",
     "@lezer/highlight": "^1.1.6",
+    "@noble/hashes": "^1.3.2",
     "@open-rpc/client-js": "^1.8.1",
     "@vueuse/core": "^10.4.1",
     "codemirror": "^6.0.1",
@@ -53,7 +54,6 @@
     "pinia": "^2.1.6",
     "postcss-inline-svg": "^6.0.0",
     "postcss-nesting": "^12.0.1",
-    "sha3": "^2.1.4",
     "sucrase": "^3.34.0",
     "vue": "^3.3.4",
     "ws": "^8.13.0",

app/gui2/shared/dataServer.ts

@@ -1,5 +1,6 @@
 import { ObservableV2 } from 'lib0/observable'
 import * as random from 'lib0/random'
+import type { Path as LSPath } from 'shared/languageServerTypes'
 import {
   Builder,
   ByteBuffer,
@@ -15,6 +16,7 @@ import {
   None,
   OutboundMessage,
   OutboundPayload,
+  Path,
   ReadBytesCommand,
   ReadBytesReply,
   ReadFileCommand,
@@ -161,14 +163,17 @@ export class DataServer extends ObservableV2<DataServerEvents> {
   }
 
   async writeBytes(
-    path: string,
+    path: LSPath,
     index: bigint,
     overwriteExisting: boolean,
     contents: string | ArrayBuffer | Uint8Array,
   ): Promise<WriteBytesReply> {
     const builder = new Builder()
     const bytesOffset = builder.createString(contents)
-    const pathOffset = builder.createString(path)
+    const segmentOffsets = [...path.segments].map((segment) => builder.createString(segment))
+    const segmentsOffset = Path.createSegmentsVector(builder, segmentOffsets)
+    const rootIdOffset = this.createUUID(builder, path.rootId)
+    const pathOffset = Path.createPath(builder, rootIdOffset, segmentsOffset)
     const command = WriteBytesCommand.createWriteBytesCommand(
       builder,
       pathOffset,

app/gui2/shared/languageServer.ts

@@ -1,7 +1,8 @@
+import { sha3_224 as SHA3 } from '@noble/hashes/sha3'
+import { bytesToHex } from '@noble/hashes/utils'
 import { Client } from '@open-rpc/client-js'
 import { ObservableV2 } from 'lib0/observable'
 import { uuidv4 } from 'lib0/random'
-import { SHA3 } from 'sha3'
 import { z } from 'zod'
 import type {
   Checksum,
@@ -383,5 +384,5 @@ export class LanguageServer extends ObservableV2<Notifications> {
 }
 
 export function computeTextChecksum(text: string): Checksum {
-  return new SHA3(224).update(text).digest('hex') as Checksum
+  return bytesToHex(SHA3.create().update(text).digest()) as Checksum
 }

app/gui2/shared/yjsModel.ts

@@ -127,8 +127,11 @@ export class DistributedModule {
     this.undoManager = new Y.UndoManager([this.doc.contents, this.doc.idMap, this.doc.metadata])
   }
 
-  insertNewNode(offset: number, content: string, meta: NodeMetadata): ExprId {
-    const range = [offset, offset + content.length] as const
+  insertNewNode(offset: number, pattern: string, expression: string, meta: NodeMetadata): ExprId {
+    // Spaces at the beginning are needed to place the new node in scope of the `main` function with proper indentation.
+    const lhs = `    ${pattern} = `
+    const content = lhs + expression
+    const range = [offset + lhs.length, offset + content.length] as const
     const newId = random.uuidv4() as ExprId
     this.transact(() => {
       this.doc.contents.insert(offset, content + '\n')

app/gui2/src/components/GraphEditor.vue

@@ -2,6 +2,7 @@
 import { codeEditorBindings, graphBindings, interactionBindings } from '@/bindings'
 import CodeEditor from '@/components/CodeEditor.vue'
 import ComponentBrowser from '@/components/ComponentBrowser.vue'
+import { Uploader, uploadedExpression } from '@/components/GraphEditor/upload'
 import SelectionBrush from '@/components/SelectionBrush.vue'
 import TopBar from '@/components/TopBar.vue'
 import { provideGraphNavigator } from '@/providers/graphNavigator'
@@ -180,6 +181,32 @@ watch(componentBrowserVisible, (visible) => {
     interactionEnded(editingNode)
   }
 })
+
+async function handleFileDrop(event: DragEvent) {
+  try {
+    if (event.dataTransfer && event.dataTransfer.items) {
+      ;[...event.dataTransfer.items].forEach(async (item) => {
+        if (item.kind === 'file') {
+          const file = item.getAsFile()
+          if (file) {
+            const clientPos = new Vec2(event.clientX, event.clientY)
+            const pos = navigator.clientToScenePos(clientPos)
+            const uploader = await Uploader.create(
+              projectStore.lsRpcConnection,
+              projectStore.dataConnection,
+              projectStore.contentRoots,
+              file,
+            )
+            const name = await uploader.upload()
+            graphStore.createNode(pos, uploadedExpression(name))
+          }
+        }
+      })
+    }
+  } catch (err) {
+    console.error(`Uploading file failed. ${err}`)
+  }
+}
 </script>
 
 <template>
@@ -191,6 +218,8 @@ watch(componentBrowserVisible, (visible) => {
     @click="graphBindingsHandler"
     v-on.="navigator.events"
     v-on..="nodeSelection.events"
+    @dragover.prevent
+    @drop.prevent="handleFileDrop($event)"
   >
     <svg :viewBox="navigator.viewBox">
       <GraphEdges @startInteraction="setCurrentInteraction" @endInteraction="interactionEnded" />

app/gui2/src/components/GraphEditor/upload.ts

@@ -0,0 +1,134 @@
+import { Keccak, sha3_224 as SHA3 } from '@noble/hashes/sha3'
+import type { Hash } from '@noble/hashes/utils'
+import { bytesToHex } from '@noble/hashes/utils'
+import type { DataServer } from 'shared/dataServer'
+import type { LanguageServer } from 'shared/languageServer'
+import { ErrorCode, RemoteRpcError } from 'shared/languageServer'
+import type { ContentRoot, Path, Uuid } from 'shared/languageServerTypes'
+
+export const uploadedExpression = (name: string) => `enso_project.data/"${name}" . read`
+const DATA_DIR_NAME = 'data'
+
+export class Uploader {
+  private rpc: LanguageServer
+  private binary: DataServer
+  private file: File
+  private projectRootId: Uuid
+  private checksum: Hash<Keccak>
+  private uploadedBytes: bigint
+
+  private constructor(rpc: LanguageServer, binary: DataServer, file: File, projectRootId: Uuid) {
+    this.rpc = rpc
+    this.binary = binary
+    this.file = file
+    this.projectRootId = projectRootId
+    this.checksum = SHA3.create()
+    this.uploadedBytes = BigInt(0)
+  }
+
+  static async create(
+    rpc: Promise<LanguageServer>,
+    binary: Promise<DataServer>,
+    contentRoots: Promise<ContentRoot[]>,
+    file: File,
+  ): Promise<Uploader> {
+    const projectRootId = await contentRoots.then((roots) =>
+      roots.find((root) => root.type == 'Project'),
+    )
+    if (!projectRootId) throw new Error('Unable to find project root, uploading not possible.')
+    const instance = new Uploader(await rpc, await binary, file, projectRootId.id)
+    return instance
+  }
+
+  async upload(): Promise<string> {
+    await this.ensureDataDirExists()
+    const name = await this.pickUniqueName(this.file.name)
+    const remotePath: Path = { rootId: this.projectRootId, segments: [DATA_DIR_NAME, name] }
+    const uploader = this
+    const writableStream = new WritableStream<Uint8Array>({
+      async write(chunk: Uint8Array) {
+        await uploader.binary.writeBytes(remotePath, uploader.uploadedBytes, false, chunk)
+        uploader.checksum.update(chunk)
+        uploader.uploadedBytes += BigInt(chunk.length)
+      },
+      async close() {
+        // Disabled until https://github.com/enso-org/enso/issues/6691 is fixed.
+        // uploader.assertChecksum(remotePath)
+      },
+      async abort(reason: string) {
+        await uploader.rpc.deleteFile(remotePath)
+        throw new Error(`Uploading process aborted. ${reason}`)
+      },
+    })
+    await this.file.stream().pipeTo(writableStream)
+    return name
+  }
+
+  private async assertChecksum(path: Path) {
+    const engineChecksum = await this.rpc.fileChecksum(path)
+    const hexChecksum = bytesToHex(this.checksum.digest())
+    if (hexChecksum != engineChecksum.checksum) {
+      throw new Error(
+        `Uploading file failed, checksum does not match. ${hexChecksum} != ${engineChecksum.checksum}`,
+      )
+    }
+  }
+
+  private dataDirPath(): Path {
+    return { rootId: this.projectRootId, segments: [DATA_DIR_NAME] }
+  }
+
+  private async ensureDataDirExists() {
+    const exists = await this.dataDirExists()
+    if (!exists) {
+      await this.rpc.createFile({
+        type: 'Directory',
+        name: DATA_DIR_NAME,
+        path: { rootId: this.projectRootId, segments: [] },
+      })
+    }
+  }
+
+  private async dataDirExists(): Promise<boolean> {
+    try {
+      const info = await this.rpc.fileInfo(this.dataDirPath())
+      return info.attributes.kind.type == 'Directory'
+    } catch (err: any) {
+      if (err.cause && err.cause instanceof RemoteRpcError) {
+        if ([ErrorCode.FILE_NOT_FOUND, ErrorCode.CONTENT_ROOT_NOT_FOUND].includes(err.cause.code)) {
+          return false
+        }
+      }
+      throw err
+    }
+  }
+
+  private async pickUniqueName(suggestedName: string): Promise<string> {
+    const files = await this.rpc.listFiles(this.dataDirPath())
+    const existingNames = new Set(files.paths.map((path) => path.name))
+    const [stem, maybeExtension] = splitFilename(suggestedName)
+    const extension = maybeExtension ?? ''
+    let candidate = suggestedName
+    let num = 1
+    while (existingNames.has(candidate)) {
+      candidate = `${stem}_${num}.${extension}`
+      num += 1
+    }
+    return candidate
+  }
+}
+
+/**
+ * Split filename into stem and (optional) extension.
+ */
+function splitFilename(filename: string): [string, string | null] {
+  const dotIndex = filename.lastIndexOf('.')
+
+  if (dotIndex !== -1 && dotIndex !== 0) {
+    const stem = filename.substring(0, dotIndex)
+    const extension = filename.substring(dotIndex + 1)
+    return [stem, extension]
+  }
+
+  return [filename, null]
+}

app/gui2/src/stores/graph.ts

@@ -250,8 +250,7 @@ export const useGraphStore = defineStore('graph', () => {
       vis: null,
     }
     const ident = generateUniqueIdent()
-    const content = `${ident} = ${expression}`
-    return mod.insertNewNode(mod.doc.contents.length, content, meta)
+    return mod.insertNewNode(mod.doc.contents.length, ident, expression, meta)
   }
 
   function deleteNode(id: ExprId) {
@@ -358,7 +357,7 @@ export const useGraphStore = defineStore('graph', () => {
 })
 
 function randomString() {
-  return Math.random().toString(36).substring(2, 10)
+  return 'operator' + Math.round(Math.random() * 100000)
 }
 
 export interface Node {

app/gui2/src/util/navigator.ts

@@ -130,5 +130,6 @@ export function useNavigator(viewportNode: Ref<Element | undefined>) {
     /** Use this transform instead, if the element should not be scaled. */
     prescaledTransform,
     sceneMousePos,
+    clientToScenePos,
   })
 }

app/gui2/src/util/vec2.ts

@@ -67,4 +67,8 @@ export class Vec2 {
   lerp(to: Vec2, t: number): Vec2 {
     return new Vec2(this.x + (to.x - this.x) * t, this.y + (to.y - this.y) * t)
   }
+
+  toString(): string {
+    return `(${this.x}, ${this.y})`
+  }
 }