Setup AppImage (#258)

* Added husky

* Fixed csvconf image path

* Removed fixtures folder

* Added electron

* Migrated ts on react-jsx

* Setup electron-vite

* Maximize electron window

* Removed beta

* Implemented AppImage build

* Updated version

* Fixed conflict

* Bootstrapped build

* Added server build

* Added example build

* Implemented runner build

* Removed desktop build

* Improved build

* Fixed ignored folders

* Rebase on exact deps for js

* Rebase on exact deps for py

* Add runner build to build command

* Found python in dist

* Bootstrapped desktop scripts

* Stick to opendataeditor

* Updated app home dir

* Improved desktop

* Improved desktop scripts

* Simplifed runner structure

* Ensured python/libraries

* Updted python deps

* Improved logging

* Bootstrapped start server

* Fixed running server

* Fixed server

* Added project/check endpoint

* Fixed server

* Fixed tests

.gitignore

@@ -156,3 +156,4 @@ tmp/
 .vim
 .vercel
 client/coverage
+.opendataeditor

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all build client components docs format install lint release server start test version write
+.PHONY: all build client dist docs format install lint release server start test version
 
 
 VERSION := $(shell node -p -e "require('./package.json').version")
@@ -8,13 +8,14 @@ all:
 	@grep '^\.PHONY' Makefile | cut -d' ' -f2- | tr ' ' '\n'
 
 build:
+	hatch run build
 	npm run build
 
 client:
 	npm run start
 
-components:
-	npm run component
+dist:
+	npm run dist:linux
 
 docs:
 	cd portal && npm start
@@ -49,6 +50,3 @@ test:
 
 version:
 	@echo $(VERSION)
-
-write:
-	hatch run write

client/components/Application/Header.tsx

@@ -1,7 +1,6 @@
 import { useTheme } from '@mui/material/styles'
 import Box from '@mui/material/Box'
 import Grid from '@mui/material/Grid'
-import Chip from '@mui/material/Chip'
 import AppBar from '@mui/material/AppBar'
 import Toolbar from '@mui/material/Toolbar'
 import Button from '@mui/material/Button'
@@ -37,15 +36,7 @@ export default function Header() {
                   }}
                   onClick={() => closeFile()}
                 >
-                  <strong>
-                    Open Data Editor{' '}
-                    <Chip
-                      size="small"
-                      label="beta"
-                      variant="outlined"
-                      sx={{ color: 'white', borderRadius: 1 }}
-                    />
-                  </strong>
+                  <strong>Open Data Editor</strong>
                 </Typography>
               </LightTooltip>
             </Grid>

desktop/index.ts

@@ -1,60 +1,48 @@
-import { app, shell, BrowserWindow } from 'electron'
-import { resolve } from 'path'
-import { electronApp, optimizer, is } from '@electron-toolkit/utils'
-
-function createWindow(): void {
-  // Create the browser window.
-  const mainWindow = new BrowserWindow({
-    width: 900,
-    height: 670,
-    show: false,
-    autoHideMenuBar: true,
-    // ...(process.platform === 'linux' ? { icon } : {}),
-    // webPreferences: {
-    // preload: join(__dirname, '../preload/index.js'),
-    // sandbox: false,
-    // },
-  })
-
-  mainWindow.on('ready-to-show', () => {
-    mainWindow.show()
-  })
-
-  mainWindow.webContents.setWindowOpenHandler((details) => {
-    shell.openExternal(details.url)
-    return { action: 'deny' }
-  })
-
-  // HMR for renderer base on electron-vite cli.
-  // Load the remote URL for development or the local html file for production.
-  if (is.dev && process.env['ELECTRON_RENDERER_URL']) {
-    mainWindow.loadURL(process.env['ELECTRON_RENDERER_URL'])
-  } else {
-    mainWindow.loadFile(resolve(__dirname, '../client/index.html'))
-  }
-}
+import { app, dialog, BrowserWindow } from 'electron'
+import { electronApp, optimizer } from '@electron-toolkit/utils'
+import { createWindow } from './window'
+import { is } from '@electron-toolkit/utils'
+import log from 'electron-log'
+import * as server from './server'
+import * as python from './python'
+import * as settings from './settings'
+import * as resources from './resources'
 
 // This method will be called when Electron has finished
 // initialization and is ready to create browser windows.
 // Some APIs can only be used after this event occurs.
-app.whenReady().then(() => {
-  // Set app user model id for windows
-  electronApp.setAppUserModelId('com.electron')
+app.whenReady().then(async () => {
+  log.info('# Start application')
+  electronApp.setAppUserModelId(settings.APP_USER_MODEL_ID)
 
-  // Default open or close DevTools by F12 in development
-  // and ignore CommandOrControl + R in production.
-  // see https://github.com/alex8088/electron-toolkit/tree/master/packages/utils
-  app.on('browser-window-created', (_, window) => {
-    optimizer.watchWindowShortcuts(window)
-  })
+  if (!is.dev) {
+    log.info('## Ensure resources')
+    await resources.ensureExample()
+    await resources.ensureRunner()
 
+    log.info('## Prepare python')
+    await python.ensurePython()
+    await python.ensureLibraries()
+
+    log.info('## Start server')
+    await server.startServer()
+  }
+
+  log.info('## Create window')
   createWindow()
+})
 
-  app.on('activate', function () {
-    // On macOS it's common to re-create a window in the app when the
-    // dock icon is clicked and there are no other windows open.
-    if (BrowserWindow.getAllWindows().length === 0) createWindow()
-  })
+// Default open or close DevTools by F12 in development
+// and ignore CommandOrControl + R in production.
+// see https://github.com/alex8088/electron-toolkit/tree/master/packages/utils
+app.on('browser-window-created', (_, window) => {
+  optimizer.watchWindowShortcuts(window)
+})
+
+// On macOS it's common to re-create a window in the app when the
+// dock icon is clicked and there are no other windows open.
+app.on('activate', function () {
+  if (BrowserWindow.getAllWindows().length === 0) createWindow()
 })
 
 // Quit when all windows are closed, except on macOS. There, it's common
@@ -66,5 +54,15 @@ app.on('window-all-closed', () => {
   }
 })
 
-// In this file you can include the rest of your app"s specific main process
-// code. You can also put them in separate files and require them here.
+// For convinience, we catch all unhandled rejections here
+// instead of wrapping all individual async functions with try/catch
+process.on('unhandledRejection', async (error: any) => {
+  log.error(error)
+  await dialog.showMessageBox({
+    type: 'error',
+    title: 'Open Data Editor',
+    message: 'Error during the application startup',
+    detail: error.toString(),
+  })
+  app.quit()
+})

desktop/python.ts

@@ -0,0 +1,64 @@
+import fs from 'fs'
+import fsp from 'fs/promises'
+import * as settings from './settings'
+import { join } from 'path'
+import log from 'electron-log'
+import toml from 'toml'
+import * as system from './system'
+
+export async function ensurePython() {
+  log.info('[ensurePython]', { path: settings.APP_PYTHON })
+
+  let message = 'existed'
+  if (!fs.existsSync(settings.APP_PYTHON)) {
+    await system.execFile(settings.ORIGINAL_PYTHON, ['-m', 'venv', settings.APP_PYTHON])
+    message = 'created'
+  }
+
+  log.info('[ensurePython]', { message })
+}
+
+export async function ensureLibraries() {
+  log.info('[ensureLibraries]')
+
+  const required = await readRequiredLibraries()
+  const installed = await readInstalledLibraries()
+
+  for (const spec of required) {
+    if (installed.includes(spec)) continue
+    await system.execFile(settings.PIP, [
+      'install',
+      spec,
+      '--upgrade',
+      '--disable-pip-version-check',
+    ])
+  }
+
+  log.info('[ensureLibraries]', { message: 'done' })
+}
+
+export async function readRequiredLibraries() {
+  log.info('[readRequiredLibraries]')
+
+  const path = join(settings.DIST, 'pyproject.toml')
+  const text = await fsp.readFile(path, 'utf-8')
+  const data = toml.parse(text).project.dependencies
+
+  log.info('[readRequiredLibraries]', { data })
+  return data
+}
+
+export async function readInstalledLibraries() {
+  log.info('[readInstalledLibraries]')
+
+  const text = await system.execFile(settings.PIP, [
+    'list',
+    '--format',
+    'freeze',
+    '--disable-pip-version-check',
+  ])
+  const data = text.split(/\r?\n/).map((line) => line.trim().toLowerCase())
+
+  log.info('[readInstalledLibraries]', { data })
+  return data
+}

desktop/resources.ts

@@ -0,0 +1,36 @@
+import fs from 'fs'
+import fsp from 'fs/promises'
+import * as settings from './settings'
+import log from 'electron-log'
+
+export async function ensureExample() {
+  log.info('[ensureExample]', { path: settings.APP_RUNNER })
+
+  let message = 'existed'
+  if (!fs.existsSync(settings.APP_EXAMPLE)) {
+    await fsp.mkdir(settings.APP_EXAMPLE, { recursive: true })
+    await fsp.cp(settings.DIST_EXAMPLE, settings.APP_EXAMPLE, {
+      recursive: true,
+      verbatimSymlinks: true,
+    })
+    message = 'created'
+  }
+
+  log.info('[ensureExample]', { message })
+}
+
+export async function ensureRunner() {
+  log.info('[ensureRunner]', { path: settings.APP_RUNNER })
+
+  let message = 'existed'
+  if (!fs.existsSync(settings.APP_RUNNER)) {
+    await fsp.mkdir(settings.APP_RUNNER, { recursive: true })
+    await fsp.cp(settings.DIST_RUNNER, settings.APP_RUNNER, {
+      recursive: true,
+      verbatimSymlinks: true,
+    })
+    message = 'created'
+  }
+
+  log.info('[ensureRunner]', { message })
+}

desktop/server.ts

@@ -0,0 +1,38 @@
+import { spawnFile } from './system'
+import timersp from 'timers/promises'
+import portfinder from 'portfinder'
+import * as settings from './settings'
+import log from 'electron-log'
+
+export async function startServer() {
+  const port = await portfinder.getPortPromise({ port: 4040 })
+  const url = `http://localhost:${port}`
+  log.info('[startServer]', { url })
+
+  // Start server
+  const proc = spawnFile(
+    settings.PYTHON,
+    ['-m', 'server', settings.APP_EXAMPLE, '--port', port.toString()],
+    process.resourcesPath
+  )
+
+  // Wait for server
+  let ready = false
+  let attempt = 0
+  const maxAttempts = 10
+  const delaySeconds = 0.5
+  const checkUrl = `${url}/project/check`
+  while (!ready) {
+    try {
+      const response = await fetch(checkUrl, { method: 'POST' })
+      if (response.status !== 200) throw new Error()
+      ready = true
+    } catch {
+      attempt += 1
+      if (attempt >= maxAttempts) throw new Error('Server is not responding')
+      await timersp.setTimeout(delaySeconds * 1000)
+    }
+  }
+
+  return { port, proc }
+}

desktop/settings.ts

@@ -0,0 +1,20 @@
+import os from 'os'
+import { join } from 'path'
+
+export const HOME = os.homedir()
+
+export const DIST = process.resourcesPath
+export const DIST_EXAMPLE = join(DIST, 'example')
+export const DIST_RUNNER = join(DIST, 'runner')
+export const DIST_SERVER = join(DIST, 'server')
+
+export const APP_NAME = 'opendataeditor'
+export const APP_USER_MODEL_ID = 'org.opendataeditor'
+export const APP_HOME = join(HOME, `.${APP_NAME}`)
+export const APP_RUNNER = join(APP_HOME, 'runner')
+export const APP_PYTHON = join(APP_HOME, 'python')
+export const APP_EXAMPLE = join(APP_HOME, 'example')
+
+export const ORIGINAL_PYTHON = join(APP_RUNNER, 'bin', 'python3')
+export const PYTHON = join(APP_PYTHON, 'bin', 'python3')
+export const PIP = join(APP_PYTHON, 'bin', 'pip3')

desktop/system.ts

@@ -0,0 +1,25 @@
+import cp from 'child_process'
+import util from 'util'
+import log from 'electron-log'
+const execFilePromise = util.promisify(cp.execFile)
+
+export async function execFile(path: string, args: string[], cwd?: string) {
+  log.info('[execFile]', { path, args, cwd })
+  const { stdout } = await execFilePromise(path, args, { cwd })
+  return stdout
+}
+
+export async function spawnFile(path: string, args: string[], cwd?: string) {
+  log.info('[spawnFile]', { path, args, cwd })
+  const proc = cp.spawn(path, args, { cwd })
+  proc.stdout.on('data', (data) => log.info(data.toString().trim()))
+  proc.stderr.on('data', (data) => log.error(data.toString().trim()))
+  proc.on('close', (code) => {
+    log.info('[spawnFile]', { message: `child process exited with code ${code}` })
+  })
+  process.on('exit', () => {
+    log.info('[spawnFile]', { message: `exiting child process on node exit` })
+    proc.kill()
+  })
+  return proc
+}