Run ydoc-server with GraalVM

app/gui2/env.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="vite/client" />
 
 declare const PROJECT_MANAGER_URL: string
+declare const YDOC_SERVER_URL: string
 declare const RUNNING_VITEST: boolean
 declare const IS_CLOUD_BUILD: boolean
 

app/gui2/package.json

@@ -19,6 +19,7 @@
     "story:build": "histoire build",
     "story:preview": "histoire preview",
     "build-only": "vite build",
+    "build-ydoc-server-polyglot": "vite build --config vite.ydoc-server-polyglot.config.ts",
     "compile-server": "tsc -p tsconfig.server.json",
     "typecheck": "vue-tsc --noEmit -p tsconfig.app.json --composite false",
     "lint": "eslint .",

app/gui2/shared/ast/ffi.ts

@@ -1,3 +1,9 @@
+/**
+ * @file Provides the Rust ffi interface. The interface should be kept in sync with polyglot ffi inteface {@link module:ffiPolyglot}.
+ *
+ * @module ffi
+ */
+
 import { createXXHash128 } from 'hash-wasm'
 import type { IDataType } from 'hash-wasm/dist/lib/util'
 import init, { is_ident_or_operator, parse, parse_doc_to_json } from '../../rust-ffi/pkg/rust_ffi'

app/gui2/shared/ast/ffiPolyglot.ts

@@ -0,0 +1,21 @@
+/**
+ * @file This file is used as ffi {@link module:ffi} interface for building the polyglot ydoc server.
+ * All the exported methods are provided by the ydoc server implementation.
+ * The interface should be kept in sync with Rust ffi inteface {@link module:ffi}.
+ *
+ * @module ffiPolyglot
+ */
+
+import type { IDataType } from 'hash-wasm/dist/lib/util'
+
+declare global {
+  function parse_tree(code: string): Uint8Array
+  function parse_doc_to_json(docs: string): string
+  function is_ident_or_operator(code: string): number
+  function xxHash128(input: IDataType): string
+}
+
+export async function initializeFFI(path?: string | undefined) {}
+
+/* eslint-disable-next-line camelcase */
+export const { is_ident_or_operator, parse_doc_to_json, parse_tree, xxHash128 } = globalThis

app/gui2/src/config.json

@@ -46,6 +46,11 @@
           "value": "",
           "primary": false
         },
+        "ydocUrl": {
+          "description": "The address of the Ydoc Server endpoint.",
+          "value": "",
+          "primary": false
+        },
         "namespace": {
           "description": "The namespace of the opened project. It can be used when connecting to an existing Language Server process.",
           "value": "local",

app/gui2/src/entrypoint.ts

@@ -68,6 +68,7 @@ function main() {
   const supportsVibrancy = config.window.vibrancy
   const shouldUseAuthentication = config.authentication.enabled
   const projectManagerUrl = config.engine.projectManagerUrl || PROJECT_MANAGER_URL
+  const ydocUrl = config.engine.ydocUrl === '' ? YDOC_SERVER_URL : config.engine.ydocUrl
   const initialProjectName = config.startup.project || null
 
   dashboard.run({
@@ -77,6 +78,7 @@ function main() {
     supportsLocalBackend: !IS_CLOUD_BUILD,
     supportsDeepLinks: !isDevMode && !isOnLinux(),
     projectManagerUrl,
+    ydocUrl,
     isAuthenticationDisabled: !shouldUseAuthentication,
     shouldShowDashboard: true,
     initialProjectName,

app/gui2/src/stores/project/index.ts

@@ -43,15 +43,29 @@ import * as Y from 'yjs'
 interface LsUrls {
   rpcUrl: string
   dataUrl: string
+  ydocUrl: URL
 }
 
 function resolveLsUrl(config: GuiConfig): LsUrls {
   const engine = config.engine
   if (engine == null) throw new Error('Missing engine configuration')
   if (engine.rpcUrl != null && engine.dataUrl != null) {
+    let dataUrl = engine.dataUrl
+    let rpcUrl = engine.rpcUrl
+    let ydocUrl
+    if (engine.ydocUrl == null || engine.ydocUrl === '') {
+      ydocUrl = new URL(location.origin)
+      ydocUrl.protocol = location.protocol.replace(/^http/, 'ws')
+    } else {
+      ydocUrl = new URL(engine.rpcUrl)
+      ydocUrl.port = '1234'
+    }
+    ydocUrl.pathname = '/project'
+
     return {
-      rpcUrl: engine.rpcUrl,
-      dataUrl: engine.dataUrl,
+      rpcUrl,
+      dataUrl,
+      ydocUrl,
     }
   }
   throw new Error('Incomplete engine configuration')
@@ -132,13 +146,8 @@ export const useProjectStore = defineStore('project', () => {
 
   let yDocsProvider: ReturnType<typeof attachProvider> | undefined
   watchEffect((onCleanup) => {
-    // For now, let's assume that the websocket server is running on the same host as the web server.
-    // Eventually, we can make this configurable, or even runtime variable.
-    const socketUrl = new URL(location.origin)
-    socketUrl.protocol = location.protocol.replace(/^http/, 'ws')
-    socketUrl.pathname = '/project'
     yDocsProvider = attachProvider(
-      socketUrl.href,
+      lsUrls.ydocUrl.href,
       'index',
       { ls: lsUrls.rpcUrl },
       doc,

app/gui2/vite.config.ts

@@ -2,6 +2,8 @@
 
 import vue from '@vitejs/plugin-vue'
 import { getDefines, readEnvironmentFromFile } from 'enso-common/src/appConfig'
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
+import { existsSync } from 'node:fs'
 import { fileURLToPath } from 'node:url'
 import postcssNesting from 'postcss-nesting'
 import tailwindcss from 'tailwindcss'
@@ -14,6 +16,9 @@ const localServerPort = 8080
 const projectManagerUrl = 'ws://127.0.0.1:30535'
 
 const IS_CLOUD_BUILD = process.env.CLOUD_BUILD === 'true'
+const IS_POLYGLOT_YDOC_SERVER_DEBUG = process.env.POLYGLOT_YDOC_SERVER_DEBUG === 'true'
+const IS_POLYGLOT_YDOC_SERVER =
+  process.env.POLYGLOT_YDOC_SERVER === 'true' || IS_POLYGLOT_YDOC_SERVER_DEBUG
 
 await readEnvironmentFromFile()
 
@@ -59,6 +64,7 @@ export default defineConfig({
     ...getDefines(localServerPort),
     IS_CLOUD_BUILD: JSON.stringify(IS_CLOUD_BUILD),
     PROJECT_MANAGER_URL: JSON.stringify(projectManagerUrl),
+    YDOC_SERVER_URL: IS_POLYGLOT_YDOC_SERVER ? JSON.stringify('defined') : undefined,
     RUNNING_VITEST: false,
     'import.meta.vitest': false,
     // Single hardcoded usage of `global` in aws-amplify.
@@ -94,13 +100,48 @@ export default defineConfig({
   },
 })
 
+let ydocServer: ChildProcessWithoutNullStreams | null
 function gatewayServer(): Plugin {
   return {
     name: 'gateway-server',
     configureServer(server) {
       if (server.httpServer == null) return
 
-      createGatewayServer(server.httpServer, undefined)
+      if (IS_POLYGLOT_YDOC_SERVER) {
+        const ydocServerJar = fileURLToPath(
+          new URL(
+            '../../lib/java/ydoc-server/target/ydoc-server-assembly-0.1.0-SNAPSHOT.jar',
+            import.meta.url,
+          ),
+        )
+        const runYdocServer = () => {
+          let args = []
+          if (IS_POLYGLOT_YDOC_SERVER_DEBUG) {
+            args.push('-DinspectPort=34567')
+          }
+          args.push('-jar', ydocServerJar)
+          ydocServer = spawn('java', args)
+          if (IS_POLYGLOT_YDOC_SERVER_DEBUG) {
+            ydocServer.stdout.on('data', (data) => console.log(`ydoc: ${data}`))
+          }
+          ydocServer.stderr.on('data', (data) => console.log(`ydoc: ${data}`))
+        }
+        if (!existsSync(ydocServerJar)) {
+          const cwd = fileURLToPath(new URL('../..', import.meta.url))
+          const sbt = spawn('sbt', ['ydoc-server/assembly'], { cwd })
+          sbt.stdout.on('data', (data) => console.log(`sbt: ${data}`))
+          sbt.on('exit', runYdocServer)
+        } else {
+          runYdocServer()
+        }
+      } else {
+        createGatewayServer(server.httpServer, undefined)
+      }
+    },
+    buildEnd() {
+      if (ydocServer == null) return
+
+      ydocServer.kill('SIGTERM')
     },
   }
 }

app/gui2/vite.ydoc-server-polyglot.config.ts

@@ -0,0 +1,72 @@
+/// <reference types="histoire" />
+
+import { getDefines, readEnvironmentFromFile } from 'enso-common/src/appConfig'
+import * as fs from 'node:fs'
+import { fileURLToPath } from 'node:url'
+import { defineConfig, type Plugin } from 'vite'
+
+const localServerPort = 8080
+const projectManagerUrl = 'ws://127.0.0.1:30535'
+
+const IS_CLOUD_BUILD = process.env.CLOUD_BUILD === 'true'
+
+await readEnvironmentFromFile()
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  root: fileURLToPath(new URL('.', import.meta.url)),
+  cacheDir: fileURLToPath(new URL('../../node_modules/.cache/vite', import.meta.url)),
+  publicDir: fileURLToPath(new URL('./public', import.meta.url)),
+  envDir: fileURLToPath(new URL('.', import.meta.url)),
+  plugins: [usePolyglotFfi()],
+  resolve: {
+    alias: {
+      shared: fileURLToPath(new URL('./shared', import.meta.url)),
+      '@': fileURLToPath(new URL('./src', import.meta.url)),
+    },
+  },
+  define: {
+    ...getDefines(localServerPort),
+    IS_CLOUD_BUILD: JSON.stringify(IS_CLOUD_BUILD),
+    PROJECT_MANAGER_URL: JSON.stringify(projectManagerUrl),
+    RUNNING_VITEST: false,
+    'import.meta.vitest': false,
+    // Single hardcoded usage of `global` in aws-amplify.
+    'global.TYPED_ARRAY_SUPPORT': true,
+    // One of the libraries refers to self in `self.fetch.bind(self)`
+    self: 'globalThis',
+  },
+  build: {
+    minify: false, // For debugging
+    emptyOutDir: true,
+    outDir: '../../lib/java/ydoc-server/target/classes/dist',
+    rollupOptions: {
+      input: {
+        ydocServer: fileURLToPath(new URL('ydoc-server/indexPolyglot.ts', import.meta.url)),
+      },
+      output: {
+        entryFileNames: `assets/[name].js`,
+      },
+    },
+  },
+})
+
+/**
+ * Use `ffiPolyglot` module as `ffi` interface during the build.
+ */
+function usePolyglotFfi(): Plugin {
+  const ffiPolyglot = fileURLToPath(new URL('./shared/ast/ffiPolyglot.ts', import.meta.url))
+  const ffiBackup = fileURLToPath(new URL('./shared/ast/ffiBackup.ts', import.meta.url))
+  const ffi = fileURLToPath(new URL('./shared/ast/ffi.ts', import.meta.url))
+
+  return {
+    name: 'use-polyglot-ffi',
+    options: () => {
+      fs.renameSync(ffi, ffiBackup)
+      fs.copyFileSync(ffiPolyglot, ffi)
+    },
+    buildEnd: () => {
+      fs.renameSync(ffiBackup, ffi)
+    },
+  }
+}

app/gui2/ydoc-server/auth.ts

@@ -0,0 +1,22 @@
+/**
+ * @file Utility methods for ydoc server authentication.
+ */
+
+export type ConnectionData = {
+  lsUrl: string
+  doc: string
+  user: string
+}
+
+const docNameRegex = /^[a-z0-9/-]+$/i
+
+export function docName(pathname: string) {
+  const prefix = '/project/'
+  if (pathname != null && pathname.startsWith(prefix)) {
+    const docName = pathname.slice(prefix.length)
+    if (docNameRegex.test(docName)) {
+      return docName
+    }
+  }
+  return null
+}

app/gui2/ydoc-server/index.ts

@@ -13,14 +13,9 @@ import { IncomingMessage } from 'node:http'
 import { parse } from 'url'
 import { WebSocket, WebSocketServer } from 'ws'
 import { initializeFFI } from '../shared/ast/ffi'
+import { ConnectionData, docName } from './auth'
 import { setupGatewayClient } from './ydoc'
 
-type ConnectionData = {
-  lsUrl: string
-  doc: string
-  user: string
-}
-
 export async function createGatewayServer(httpServer: Server, rustFFIPath: string | undefined) {
   await initializeFFI(rustFFIPath)
   const wss = new WebSocketServer({ noServer: true })
@@ -71,15 +66,3 @@ function authenticate(
   const data = doc != null && typeof lsUrl === 'string' ? { lsUrl, doc, user } : null
   callback(null, data)
 }
-
-const docNameRegex = /^[a-z0-9/-]+$/i
-function docName(pathname: string) {
-  const prefix = '/project/'
-  if (pathname != null && pathname.startsWith(prefix)) {
-    const docName = pathname.slice(prefix.length)
-    if (docNameRegex.test(docName)) {
-      return docName
-    }
-  }
-  return null
-}

app/gui2/ydoc-server/indexPolyglot.ts

@@ -0,0 +1,29 @@
+/**
+ * @file An entry point for polyglot Yjs gateway server.
+ */
+
+import { docName } from './auth'
+import { setupGatewayClient } from './ydoc'
+
+declare global {
+  class WebSocketServer {
+    constructor(config: any)
+    onconnect: ((socket: any, url: any) => any) | null
+    start(): void
+  }
+}
+
+const wss = new WebSocketServer({ host: 'localhost', port: 1234 })
+
+wss.onconnect = (socket, url) => {
+  const doc = docName(url.pathname)
+  const ls = url.searchParams.get('ls')
+  if (doc != null && ls != null) {
+    console.log('setupGatewayClient', ls, doc)
+    setupGatewayClient(socket, ls, doc)
+  } else {
+    console.log('Failed to authenticate user', ls, doc)
+  }
+}
+
+wss.start()

app/gui2/ydoc-server/ydoc.ts

@@ -7,10 +7,10 @@ import {
 import { readSyncMessage, writeSyncStep1, writeUpdate } from 'y-protocols/sync'
 import * as Y from 'yjs'
 
+import WebSocket from 'isomorphic-ws'
 import * as decoding from 'lib0/decoding'
 import * as encoding from 'lib0/encoding'
 import { ObservableV2 } from 'lib0/observable'
-import { WebSocket } from 'ws'
 import { LanguageServerSession } from './languageServerSession'
 
 const pingTimeout = 30000

app/ide-desktop/lib/dashboard/src/App.tsx

@@ -139,6 +139,7 @@ export interface AppProps {
   readonly initialProjectName: string | null
   readonly onAuthenticated: (accessToken: string | null) => void
   readonly projectManagerUrl: string | null
+  readonly ydocUrl: string | null
   readonly appRunner: AppRunner
 }
 

app/ide-desktop/lib/dashboard/src/entrypoint.ts

@@ -24,6 +24,7 @@ main.run({
   },
   /** The cloud frontend is not capable of running a Project Manager. */
   projectManagerUrl: null,
+  ydocUrl: null,
   // This cannot be `appRunner: window.enso` as `window.enso` is set to a new value
   // every time a new project is opened.
   appRunner: {