Implement byte-based file operations (#1795)

RELEASES.md

@@ -36,6 +36,9 @@
   verify the integrity of files that it has transferred. The checksum is
   calculated in a streaming fashion so the checksummed file need not be resident
   in memory all at once.
+- Added support for reading and writing byte ranges in files remotely
+  ([#1795](https://github.com/enso-org/enso/pull/1795)). This allows the IDE to
+  transfer files to a remote back-end in a streaming fashion.
 
 ## Libraries
 

docs/language-server/protocol-language-server.md

@@ -1644,8 +1644,12 @@ This method will create a file if no file is present at `path`.
   length of the file.
 - The `byteOffset` property is zero-indexed. To append to the file you begin
   writing at index `file.length`.
+- If `byteOffset` is less than the length of the file and `overwriteExisting` is
+  set, it will truncate the file to length `byteOffset + bytes.length`.
 - If `byteOffset > file.length`, the bytes in the range
-  `[file.length, byteOffset)` will be filled with null bytes.
+  `[file.length, byteOffset)` will be filled with null bytes. Please note that,
+  in this case, the checksum in the response will also be calculated on the null
+  bytes.
 
 #### Parameters
 
@@ -4055,7 +4059,7 @@ Signals that the requested file read was out of bounds for the file's size.
   "code" : 1009
   "message" : "Read is out of bounds for the file"
   "data" : {
-    fileLength : 0
+    "fileLength" : 0
   }
 }
 ```

docs/language-server/streaming-file-transfer.md

@@ -23,6 +23,7 @@ A few key requirements:
 <!-- MarkdownTOC levels="2,3" autolink="true" indent="    " -->
 
 - [Control](#control)
+  - [Concurrency](#concurrency)
 - [UX](#ux)
 
 <!-- /MarkdownTOC -->
@@ -50,6 +51,19 @@ used.
 Resumption of transfers is also handled by the IDE, which may keep track of what
 portions of a file have been written or read.
 
+### Concurrency
+
+The language server natively supports running these file operations in parallel
+as it spawns a separate request-handler actor for each operation. It does,
+however, not provide any _intrinsic_ guarantees to its operation. As _all_ file
+operations are evaluated in parallel, coordinating them for consistency is up to
+the IDE.
+
+For example, if you want to write bytes to a file `f1` and then checksum the
+resulting file, you need to wait for the `WriteBytesReply` to come back before
+sending `file/checksum(f1)`. Otherwise, there is no guarantee that the write has
+completed by the time the checksum is calculated.
+
 ## UX
 
 The IDE wants to be able to provide two major UX benefits to users as part of

engine/language-server/src/main/scala/org/enso/languageserver/filemanager/FileManager.scala

@@ -4,6 +4,7 @@ import akka.actor.{Actor, Props}
 import akka.routing.SmallestMailboxPool
 import akka.pattern.pipe
 import com.typesafe.scalalogging.LazyLogging
+import org.bouncycastle.util.encoders.Hex
 import org.enso.languageserver.effect._
 import org.enso.languageserver.data.Config
 import org.enso.languageserver.monitoring.MonitoringProtocol.{Ping, Pong}
@@ -195,14 +196,48 @@ class FileManager(
         .pipeTo(sender())
       ()
 
-    case FileManagerProtocol.ChecksumRequest(path) =>
+    case FileManagerProtocol.ChecksumFileRequest(path) =>
       val getChecksum = for {
         rootPath <- IO.fromEither(config.findContentRoot(path.rootId))
         checksum <- fs.digest(path.toFile(rootPath))
       } yield checksum
       exec
         .execTimed(config.fileManager.timeout, getChecksum)
-        .map(FileManagerProtocol.ChecksumResponse)
+        .map(x =>
+          FileManagerProtocol.ChecksumFileResponse(
+            x.map(digest => Hex.toHexString(digest.bytes))
+          )
+        )
+        .pipeTo(sender())
+
+    case FileManagerProtocol.ChecksumBytesRequest(segment) =>
+      val getChecksum = for {
+        rootPath <- IO.fromEither(config.findContentRoot(segment.path.rootId))
+        checksum <- fs.digestBytes(segment.toApiSegment(rootPath))
+      } yield checksum
+      exec
+        .execTimed(config.fileManager.timeout, getChecksum)
+        .map(x => FileManagerProtocol.ChecksumBytesResponse(x.map(_.bytes)))
+        .pipeTo(sender())
+
+    case FileManagerProtocol.WriteBytesRequest(path, off, overwrite, bytes) =>
+      val doWrite = for {
+        rootPath <- IO.fromEither(config.findContentRoot(path.rootId))
+        response <- fs.writeBytes(path.toFile(rootPath), off, overwrite, bytes)
+      } yield response
+      exec
+        .execTimed(config.fileManager.timeout, doWrite)
+        .map(x => FileManagerProtocol.WriteBytesResponse(x.map(_.bytes)))
+        .pipeTo(sender())
+
+    case FileManagerProtocol.ReadBytesRequest(segment) =>
+      val doRead = for {
+        rootPath <- IO.fromEither(config.findContentRoot(segment.path.rootId))
+        response <- fs.readBytes(segment.toApiSegment(rootPath))
+      } yield response
+      exec
+        .execTimed(config.fileManager.timeout, doRead)
+        .map(FileManagerProtocol.ReadBytesResponse)
         .pipeTo(sender())
   }
 }

engine/language-server/src/main/scala/org/enso/languageserver/filemanager/FileManagerApi.scala

@@ -1,5 +1,7 @@
 package org.enso.languageserver.filemanager
 
+import io.circe.Json
+import io.circe.literal.JsonStringContext
 import org.enso.jsonrpc.{Error, HasParams, HasResult, Method, Unused}
 
 /** The file manager JSON RPC API provided by the language server.
@@ -177,6 +179,19 @@ object FileManagerApi {
 
   case object NotFileError extends Error(1007, "Path is not a file")
 
+  case object CannotOverwriteError
+      extends Error(
+        1008,
+        "Cannot overwrite the file without `overwriteExisting` set"
+      )
+
+  case class ReadOutOfBoundsError(length: Long)
+      extends Error(1009, "Read is out of bounds for the file") {
+    override def payload: Option[Json] = Some(
+      json""" { "fileLength" : $length }"""
+    )
+  }
+
   case object CannotDecodeError
       extends Error(1010, "Cannot decode the project configuration")
 

engine/language-server/src/main/scala/org/enso/languageserver/filemanager/FileManagerProtocol.scala

@@ -182,12 +182,82 @@ object FileManagerProtocol {
     *
     * @param path to the file system object
     */
-  case class ChecksumRequest(path: Path)
+  case class ChecksumFileRequest(path: Path)
 
   /** Returns the checksum of the file system object in question.
     *
     * @param checksum either a FS failure or the checksum as a base64-encoded
     *                 string
     */
-  case class ChecksumResponse(checksum: Either[FileSystemFailure, String])
+  case class ChecksumFileResponse(checksum: Either[FileSystemFailure, String])
+
+  /** Requests that the file manager provide the checksum of the specified bytes
+    * in a file.
+    *
+    * @param segment a description of the bytes in a file to checksum.
+    */
+  case class ChecksumBytesRequest(segment: Data.FileSegment)
+
+  /** Returns the checksum of the bytes in question.
+    *
+    * @param checksum either a FS failure or the checksum as an array of bytes
+    */
+  case class ChecksumBytesResponse(
+    checksum: Either[FileSystemFailure, Array[Byte]]
+  )
+
+  /** Requests that the file manager writes the provided `bytes` to the file at
+    * `path`.
+    *
+    * @param path the file to write to
+    * @param byteOffset the offset in the file to begin writing from
+    * @param overwriteExisting whether or not the request can overwrite existing
+    *                          data
+    * @param bytes the bytes to write
+    */
+  case class WriteBytesRequest(
+    path: Path,
+    byteOffset: Long,
+    overwriteExisting: Boolean,
+    bytes: Array[Byte]
+  )
+
+  /** Returns the checksum of the bytes that were written to disk.
+    *
+    * @param checksum either a FS failure or the checksum as an array of bytes
+    */
+  case class WriteBytesResponse(
+    checksum: Either[FileSystemFailure, Array[Byte]]
+  )
+
+  /** Requests to read the bytes in the file identified by `segment`.
+    *
+    * @param segment an identification of where the bytes should be read from
+    */
+  case class ReadBytesRequest(segment: Data.FileSegment)
+
+  /** Returns the requested bytes and their checksum.
+    *
+    * @param result either a FS failure or the checksum and corresponding bytes
+    *               that were read
+    */
+  case class ReadBytesResponse(
+    result: Either[FileSystemFailure, FileSystemApi.ReadBytesResult]
+  )
+
+  /** Data types for the protocol. */
+  object Data {
+
+    /** A representation of a segment in the file.
+      *
+      * @param path the path to the file in question
+      * @param byteOffset the byte offset in the file to start from
+      * @param length the number of bytes in the segment
+      */
+    case class FileSegment(path: Path, byteOffset: Long, length: Long) {
+      def toApiSegment(rootPath: File): FileSystemApi.FileSegment = {
+        FileSystemApi.FileSegment(path.toFile(rootPath), byteOffset, length)
+      }
+    }
+  }
 }