diff --git a/.gitignore b/.gitignore index 5fa6840cd..75843e928 100644 --- a/.gitignore +++ b/.gitignore @@ -156,3 +156,4 @@ tmp/ .vim .vercel client/coverage +.opendataeditor diff --git a/Makefile b/Makefile index c1e633820..7bae1911d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build client components docs format install lint release server start test version write +.PHONY: all build client dist docs format install lint release server start test version VERSION := $(shell node -p -e "require('./package.json').version") @@ -8,13 +8,14 @@ all: @grep '^\.PHONY' Makefile | cut -d' ' -f2- | tr ' ' '\n' build: + hatch run build npm run build client: npm run start -components: - npm run component +dist: + npm run dist:linux docs: cd portal && npm start @@ -49,6 +50,3 @@ test: version: @echo $(VERSION) - -write: - hatch run write diff --git a/client/components/Application/Header.tsx b/client/components/Application/Header.tsx index 5f5b9bb3d..fe6d32d55 100644 --- a/client/components/Application/Header.tsx +++ b/client/components/Application/Header.tsx @@ -1,7 +1,6 @@ import { useTheme } from '@mui/material/styles' import Box from '@mui/material/Box' import Grid from '@mui/material/Grid' -import Chip from '@mui/material/Chip' import AppBar from '@mui/material/AppBar' import Toolbar from '@mui/material/Toolbar' import Button from '@mui/material/Button' @@ -37,15 +36,7 @@ export default function Header() { }} onClick={() => closeFile()} > - - Open Data Editor{' '} - - + Open Data Editor diff --git a/desktop/index.ts b/desktop/index.ts index a63b0544f..5344e9c58 100644 --- a/desktop/index.ts +++ b/desktop/index.ts @@ -1,60 +1,48 @@ -import { app, shell, BrowserWindow } from 'electron' -import { resolve } from 'path' -import { electronApp, optimizer, is } from '@electron-toolkit/utils' - -function createWindow(): void { - // Create the browser window. - const mainWindow = new BrowserWindow({ - width: 900, - height: 670, - show: false, - autoHideMenuBar: true, - // ...(process.platform === 'linux' ? { icon } : {}), - // webPreferences: { - // preload: join(__dirname, '../preload/index.js'), - // sandbox: false, - // }, - }) - - mainWindow.on('ready-to-show', () => { - mainWindow.show() - }) - - mainWindow.webContents.setWindowOpenHandler((details) => { - shell.openExternal(details.url) - return { action: 'deny' } - }) - - // HMR for renderer base on electron-vite cli. - // Load the remote URL for development or the local html file for production. - if (is.dev && process.env['ELECTRON_RENDERER_URL']) { - mainWindow.loadURL(process.env['ELECTRON_RENDERER_URL']) - } else { - mainWindow.loadFile(resolve(__dirname, '../client/index.html')) - } -} +import { app, dialog, BrowserWindow } from 'electron' +import { electronApp, optimizer } from '@electron-toolkit/utils' +import { createWindow } from './window' +import { is } from '@electron-toolkit/utils' +import log from 'electron-log' +import * as server from './server' +import * as python from './python' +import * as settings from './settings' +import * as resources from './resources' // This method will be called when Electron has finished // initialization and is ready to create browser windows. // Some APIs can only be used after this event occurs. -app.whenReady().then(() => { - // Set app user model id for windows - electronApp.setAppUserModelId('com.electron') +app.whenReady().then(async () => { + log.info('# Start application') + electronApp.setAppUserModelId(settings.APP_USER_MODEL_ID) - // Default open or close DevTools by F12 in development - // and ignore CommandOrControl + R in production. - // see https://github.com/alex8088/electron-toolkit/tree/master/packages/utils - app.on('browser-window-created', (_, window) => { - optimizer.watchWindowShortcuts(window) - }) + if (!is.dev) { + log.info('## Ensure resources') + await resources.ensureExample() + await resources.ensureRunner() + log.info('## Prepare python') + await python.ensurePython() + await python.ensureLibraries() + + log.info('## Start server') + await server.startServer() + } + + log.info('## Create window') createWindow() +}) - app.on('activate', function () { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (BrowserWindow.getAllWindows().length === 0) createWindow() - }) +// Default open or close DevTools by F12 in development +// and ignore CommandOrControl + R in production. +// see https://github.com/alex8088/electron-toolkit/tree/master/packages/utils +app.on('browser-window-created', (_, window) => { + optimizer.watchWindowShortcuts(window) +}) + +// On macOS it's common to re-create a window in the app when the +// dock icon is clicked and there are no other windows open. +app.on('activate', function () { + if (BrowserWindow.getAllWindows().length === 0) createWindow() }) // Quit when all windows are closed, except on macOS. There, it's common @@ -66,5 +54,15 @@ app.on('window-all-closed', () => { } }) -// In this file you can include the rest of your app"s specific main process -// code. You can also put them in separate files and require them here. +// For convinience, we catch all unhandled rejections here +// instead of wrapping all individual async functions with try/catch +process.on('unhandledRejection', async (error: any) => { + log.error(error) + await dialog.showMessageBox({ + type: 'error', + title: 'Open Data Editor', + message: 'Error during the application startup', + detail: error.toString(), + }) + app.quit() +}) diff --git a/desktop/python.ts b/desktop/python.ts new file mode 100644 index 000000000..32a0c3667 --- /dev/null +++ b/desktop/python.ts @@ -0,0 +1,64 @@ +import fs from 'fs' +import fsp from 'fs/promises' +import * as settings from './settings' +import { join } from 'path' +import log from 'electron-log' +import toml from 'toml' +import * as system from './system' + +export async function ensurePython() { + log.info('[ensurePython]', { path: settings.APP_PYTHON }) + + let message = 'existed' + if (!fs.existsSync(settings.APP_PYTHON)) { + await system.execFile(settings.ORIGINAL_PYTHON, ['-m', 'venv', settings.APP_PYTHON]) + message = 'created' + } + + log.info('[ensurePython]', { message }) +} + +export async function ensureLibraries() { + log.info('[ensureLibraries]') + + const required = await readRequiredLibraries() + const installed = await readInstalledLibraries() + + for (const spec of required) { + if (installed.includes(spec)) continue + await system.execFile(settings.PIP, [ + 'install', + spec, + '--upgrade', + '--disable-pip-version-check', + ]) + } + + log.info('[ensureLibraries]', { message: 'done' }) +} + +export async function readRequiredLibraries() { + log.info('[readRequiredLibraries]') + + const path = join(settings.DIST, 'pyproject.toml') + const text = await fsp.readFile(path, 'utf-8') + const data = toml.parse(text).project.dependencies + + log.info('[readRequiredLibraries]', { data }) + return data +} + +export async function readInstalledLibraries() { + log.info('[readInstalledLibraries]') + + const text = await system.execFile(settings.PIP, [ + 'list', + '--format', + 'freeze', + '--disable-pip-version-check', + ]) + const data = text.split(/\r?\n/).map((line) => line.trim().toLowerCase()) + + log.info('[readInstalledLibraries]', { data }) + return data +} diff --git a/desktop/resources.ts b/desktop/resources.ts new file mode 100644 index 000000000..a66565639 --- /dev/null +++ b/desktop/resources.ts @@ -0,0 +1,36 @@ +import fs from 'fs' +import fsp from 'fs/promises' +import * as settings from './settings' +import log from 'electron-log' + +export async function ensureExample() { + log.info('[ensureExample]', { path: settings.APP_RUNNER }) + + let message = 'existed' + if (!fs.existsSync(settings.APP_EXAMPLE)) { + await fsp.mkdir(settings.APP_EXAMPLE, { recursive: true }) + await fsp.cp(settings.DIST_EXAMPLE, settings.APP_EXAMPLE, { + recursive: true, + verbatimSymlinks: true, + }) + message = 'created' + } + + log.info('[ensureExample]', { message }) +} + +export async function ensureRunner() { + log.info('[ensureRunner]', { path: settings.APP_RUNNER }) + + let message = 'existed' + if (!fs.existsSync(settings.APP_RUNNER)) { + await fsp.mkdir(settings.APP_RUNNER, { recursive: true }) + await fsp.cp(settings.DIST_RUNNER, settings.APP_RUNNER, { + recursive: true, + verbatimSymlinks: true, + }) + message = 'created' + } + + log.info('[ensureRunner]', { message }) +} diff --git a/desktop/server.ts b/desktop/server.ts new file mode 100644 index 000000000..f365dfdcc --- /dev/null +++ b/desktop/server.ts @@ -0,0 +1,38 @@ +import { spawnFile } from './system' +import timersp from 'timers/promises' +import portfinder from 'portfinder' +import * as settings from './settings' +import log from 'electron-log' + +export async function startServer() { + const port = await portfinder.getPortPromise({ port: 4040 }) + const url = `http://localhost:${port}` + log.info('[startServer]', { url }) + + // Start server + const proc = spawnFile( + settings.PYTHON, + ['-m', 'server', settings.APP_EXAMPLE, '--port', port.toString()], + process.resourcesPath + ) + + // Wait for server + let ready = false + let attempt = 0 + const maxAttempts = 10 + const delaySeconds = 0.5 + const checkUrl = `${url}/project/check` + while (!ready) { + try { + const response = await fetch(checkUrl, { method: 'POST' }) + if (response.status !== 200) throw new Error() + ready = true + } catch { + attempt += 1 + if (attempt >= maxAttempts) throw new Error('Server is not responding') + await timersp.setTimeout(delaySeconds * 1000) + } + } + + return { port, proc } +} diff --git a/desktop/settings.ts b/desktop/settings.ts new file mode 100644 index 000000000..0e80fb18c --- /dev/null +++ b/desktop/settings.ts @@ -0,0 +1,20 @@ +import os from 'os' +import { join } from 'path' + +export const HOME = os.homedir() + +export const DIST = process.resourcesPath +export const DIST_EXAMPLE = join(DIST, 'example') +export const DIST_RUNNER = join(DIST, 'runner') +export const DIST_SERVER = join(DIST, 'server') + +export const APP_NAME = 'opendataeditor' +export const APP_USER_MODEL_ID = 'org.opendataeditor' +export const APP_HOME = join(HOME, `.${APP_NAME}`) +export const APP_RUNNER = join(APP_HOME, 'runner') +export const APP_PYTHON = join(APP_HOME, 'python') +export const APP_EXAMPLE = join(APP_HOME, 'example') + +export const ORIGINAL_PYTHON = join(APP_RUNNER, 'bin', 'python3') +export const PYTHON = join(APP_PYTHON, 'bin', 'python3') +export const PIP = join(APP_PYTHON, 'bin', 'pip3') diff --git a/desktop/system.ts b/desktop/system.ts new file mode 100644 index 000000000..f8ee974ec --- /dev/null +++ b/desktop/system.ts @@ -0,0 +1,25 @@ +import cp from 'child_process' +import util from 'util' +import log from 'electron-log' +const execFilePromise = util.promisify(cp.execFile) + +export async function execFile(path: string, args: string[], cwd?: string) { + log.info('[execFile]', { path, args, cwd }) + const { stdout } = await execFilePromise(path, args, { cwd }) + return stdout +} + +export async function spawnFile(path: string, args: string[], cwd?: string) { + log.info('[spawnFile]', { path, args, cwd }) + const proc = cp.spawn(path, args, { cwd }) + proc.stdout.on('data', (data) => log.info(data.toString().trim())) + proc.stderr.on('data', (data) => log.error(data.toString().trim())) + proc.on('close', (code) => { + log.info('[spawnFile]', { message: `child process exited with code ${code}` }) + }) + process.on('exit', () => { + log.info('[spawnFile]', { message: `exiting child process on node exit` }) + proc.kill() + }) + return proc +} diff --git a/desktop/window.ts b/desktop/window.ts new file mode 100644 index 000000000..d750daff8 --- /dev/null +++ b/desktop/window.ts @@ -0,0 +1,37 @@ +import { shell, BrowserWindow } from 'electron' +import { resolve } from 'path' +import { is } from '@electron-toolkit/utils' + +export function createWindow(): void { + // Create the browser window. + const mainWindow = new BrowserWindow({ + // width: 900, + // height: 670, + show: false, + autoHideMenuBar: true, + titleBarStyle: 'hidden', + // ...(process.platform === 'linux' ? { icon } : {}), + // webPreferences: { + // preload: join(__dirname, '../preload/index.js'), + // sandbox: false, + // }, + }) + + mainWindow.on('ready-to-show', () => { + mainWindow.maximize() + mainWindow.show() + }) + + mainWindow.webContents.setWindowOpenHandler((details) => { + shell.openExternal(details.url) + return { action: 'deny' } + }) + + // HMR for renderer base on electron-vite cli. + // Load the remote URL for development or the local html file for production. + if (is.dev && process.env['ELECTRON_RENDERER_URL']) { + mainWindow.loadURL(process.env['ELECTRON_RENDERER_URL']) + } else { + mainWindow.loadFile(resolve(__dirname, '../client/index.html')) + } +} diff --git a/electron-builder.yaml b/electron-builder.yaml new file mode 100644 index 000000000..cc1ce074a --- /dev/null +++ b/electron-builder.yaml @@ -0,0 +1,46 @@ +appId: org.opendataeditor.app +copyright: Copyright © 2023 ${author} +files: + - 'build/client/**/*' + - 'build/desktop/**/*' +extraResources: + - from: 'pyproject.toml' + to: 'pyproject.toml' + - from: 'build/runner' + to: 'runner' + - from: 'build/example' + to: 'example' + filter: + - '**/*' + - '!**/.opendataeditor' + - from: 'build/server' + to: 'server' + filter: + - '**/*' + - '!**/__pycache__' +win: + executableName: ${name} +nsis: + artifactName: ${name}-${version}-setup.${ext} + shortcutName: ${productName} + uninstallDisplayName: ${productName} + createDesktopShortcut: always +mac: + extendInfo: + - NSCameraUsageDescription: Application requests access to the device's camera. + - NSMicrophoneUsageDescription: Application requests access to the device's microphone. + - NSDocumentsFolderUsageDescription: Application requests access to the user's Documents folder. + - NSDownloadsFolderUsageDescription: Application requests access to the user's Downloads folder. + notarize: false +dmg: + artifactName: ${name}-${version}.${ext} +linux: + target: + - AppImage + # - deb + # - snap + maintainer: okfn.org + category: Utility +appImage: + artifactName: ${name}-${version}.${ext} +npmRebuild: false diff --git a/example/__init__.py b/example/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/example/build.py b/example/build.py new file mode 100644 index 000000000..f93c90557 --- /dev/null +++ b/example/build.py @@ -0,0 +1,12 @@ +import os +import shutil + +source = "example" +target = "build/example" + +shutil.rmtree(target, ignore_errors=True) +shutil.copytree(source, target) +os.remove(f"{target}/__init__.py") +os.remove(f"{target}/build.py") + +print(f"[example] Copied '{source}' to '{target}'") diff --git a/example/table.csv b/example/table.csv new file mode 100644 index 000000000..6d09540e9 --- /dev/null +++ b/example/table.csv @@ -0,0 +1,3 @@ +id,name +1,english +2,中国人 diff --git a/package.json b/package.json index 3c26380b9..97d7c9bad 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,11 @@ { - "name": "odet", - "version": "0.2.0", + "name": "opendataeditor", + "license": "MIT", + "version": "1.0.0-rc.1", + "productName": "Open Data Editor", + "author": "Open Knowledge Foundation", + "description": "Data management for humans", + "homepage": "https://opendataeditor.org", "main": "build/desktop/index.js", "engines": { "node": "^18.0.0" @@ -11,13 +16,17 @@ }, "repository": { "type": "git", - "url": "git+https://github.com/frictionlessdata/application.git" + "url": "git+https://github.com/okfn/opendataeditor.git" }, "bugs": { - "url": "https://github.com/frictionlessdata/application/issues" + "url": "https://github.com/okfn/opendataeditor/issues" }, "scripts": { "build": "electron-vite build", + "dist": "npm run dist:linux && npm run dist:mac && npm run dist:win", + "dist:linux": "electron-builder --linux", + "dist:mac": "electron-builder --mac", + "dist:win": "electron-builder --win", "check": "tsc --noEmit", "coverage": "sensible-browser coverage/index.html", "format": "prettier --write 'client/**/*.ts*' && eslint --fix 'client/**/*.ts*'", @@ -30,68 +39,72 @@ "update": "ncu -u" }, "dependencies": { - "@electron-toolkit/utils": "^2.0.1", - "@emotion/react": "^11.11.1", - "@emotion/styled": "^11.11.0", - "@fontsource/roboto": "^5.0.5", - "@fontsource/roboto-mono": "^5.0.5", - "@inovua/reactdatagrid-community": "^5.10.1", - "@monaco-editor/react": "^4.5.1", - "@mui/icons-material": "^5.14.1", - "@mui/lab": "^5.0.0-alpha.145", - "@mui/material": "^5.14.2", - "@mui/system": "^5.14.1", - "@mui/x-date-pickers": "^6.10.1", - "ahooks": "^3.7.8", - "classnames": "^2.3.2", - "dangerously-set-html-content": "^1.0.13", - "dayjs": "^1.11.9", - "delay": "^6.0.0", - "dirty-json": "^0.9.2", - "electron": "^26.2.1", - "fast-deep-equal": "^3.1.3", - "js-yaml": "^4.1.0", - "jsonschema": "^1.4.1", - "leaflet": "^1.9.4", - "lodash": "^4.17.21", - "marked": "^4.3.0", - "react": "^18.2.0", - "react-dom": "^18.2.0", - "react-leaflet": "^4.2.1", - "react-vega": "^7.6.0", - "reselect": "^4.1.8", - "topojson-client": "^3.1.0", - "ts-essentials": "^9.3.2", - "validator": "^13.9.0", - "vega": "^5.25.0", - "vega-lite": "^5.14.1", - "zustand": "^4.3.9" + "@electron-toolkit/utils": "2.0.1", + "electron-log": "4.4.8", + "portfinder": "1.0.32", + "toml": "3.0.0" }, "devDependencies": { - "@modyfi/vite-plugin-yaml": "^1.0.4", - "@types/js-yaml": "^4.0.5", - "@types/leaflet": "^1.9.3", - "@types/lodash": "^4.14.195", - "@types/marked": "^4.3.1", - "@types/react": "^18.2.16", - "@types/react-dom": "^18.2.7", - "@types/topojson-client": "^3.1.1", - "@types/validator": "^13.7.17", - "@typescript-eslint/eslint-plugin": "^6.2.0", - "@typescript-eslint/parser": "^6.2.0", - "@vitejs/plugin-react": "^4.0.4", - "@vitest/coverage-v8": "^0.34.4", - "concurrently": "^8.2.0", - "electron-builder": "^24.6.4", - "electron-updater": "^6.1.4", - "electron-vite": "^1.0.28", - "eslint": "^8.45.0", - "husky": "^8.0.3", - "npm-check-updates": "^16.10.16", - "prettier": "^3.0.0", - "typescript": "^5.2.2", - "vite": "^4.4.9", - "vitest": "^0.34.4" + "@emotion/react": "11.11.1", + "@emotion/styled": "11.11.0", + "@fontsource/roboto": "5.0.5", + "@fontsource/roboto-mono": "5.0.5", + "@inovua/reactdatagrid-community": "5.10.1", + "@modyfi/vite-plugin-yaml": "1.0.4", + "@monaco-editor/react": "4.5.1", + "@mui/icons-material": "5.14.1", + "@mui/lab": "5.0.0-alpha.145", + "@mui/material": "5.14.2", + "@mui/system": "5.14.1", + "@mui/x-date-pickers": "6.10.1", + "@types/js-yaml": "4.0.5", + "@types/leaflet": "1.9.3", + "@types/lodash": "4.14.195", + "@types/marked": "4.3.1", + "@types/react": "18.2.16", + "@types/react-dom": "18.2.7", + "@types/shelljs": "0.8.12", + "@types/topojson-client": "3.1.1", + "@types/validator": "13.7.17", + "@typescript-eslint/eslint-plugin": "6.2.0", + "@typescript-eslint/parser": "6.2.0", + "@vitejs/plugin-react": "4.0.4", + "@vitest/coverage-v8": "0.34.4", + "ahooks": "3.7.8", + "classnames": "2.3.2", + "concurrently": "8.2.0", + "dangerously-set-html-content": "1.0.13", + "dayjs": "1.11.9", + "delay": "6.0.0", + "dirty-json": "0.9.2", + "electron": "26.2.1", + "electron-builder": "24.6.4", + "electron-updater": "6.1.4", + "electron-vite": "1.0.28", + "eslint": "8.45.0", + "fast-deep-equal": "3.1.3", + "husky": "8.0.3", + "js-yaml": "4.1.0", + "jsonschema": "1.4.1", + "leaflet": "1.9.4", + "lodash": "4.17.21", + "marked": "4.3.0", + "npm-check-updates": "16.10.16", + "prettier": "3.0.0", + "react": "18.2.0", + "react-dom": "18.2.0", + "react-leaflet": "4.2.1", + "react-vega": "7.6.0", + "reselect": "4.1.8", + "topojson-client": "3.1.0", + "ts-essentials": "9.3.2", + "typescript": "5.2.2", + "validator": "13.9.0", + "vega": "5.25.0", + "vega-lite": "5.14.1", + "vite": "4.4.9", + "vitest": "0.34.4", + "zustand": "4.3.9" }, "prettier": { "semi": false, diff --git a/portal/src/content/docs/contributing/development.md b/portal/src/content/docs/contributing/development.md index 82e44b8dd..6d2e3c1e3 100644 --- a/portal/src/content/docs/contributing/development.md +++ b/portal/src/content/docs/contributing/development.md @@ -37,7 +37,7 @@ hatch shell # Enter the venv Now you can setup you IDE to use a proper Python path: ```bash -.python/odet/bin/python +.python/opendataeditor/bin/python ``` ## Installation diff --git a/pyproject.toml b/pyproject.toml index deef448af..87ef9e4d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,30 @@ [project] -name = "odet" +name = "opendataeditor" dynamic = ["version"] requires-python = ">=3.10" dependencies = [ - "typer>=0.9", - "marko>=1.0", - "jinja2>=3.1", - "tinydb>=4.7", - "openai>=0.27", - "pydantic>=2.0", - "fastapi>=0.78", - "uvicorn>=0.17", - "sqlalchemy>=2.0", - "platformdirs>=3.8", - "python-multipart>=0.0", - "typing-extensions>=4.3", - "frictionless[ckan,csv,excel,json,github,pandas,sql,zenodo]>=5.15.9", - "gitignore-parser>=0.1", + "fastapi==0.103.1", + "frictionless-ckan-mapper==1.0.9", + "frictionless==5.15.10", + "gitignore-parser==0.1.6", + "ijson==3.2.3", + "jinja2==3.1.2", + "jsonlines==4.0.0", + "marko==2.0.0", + "openai==0.28.0", + "openpyxl==3.1.2", + "pandas==2.1.1", + "pydantic==2.3.0", + "pygithub==1.59.1", + "python-multipart==0.0.6", + "pyzenodo3==1.0.2", + "sqlalchemy==2.0.20", + "tinydb==4.8.0", + "typer==0.9.0", + "typing_extensions==4.8.0", + "uvicorn==0.23.2", + "xlrd==2.0.1", + "xlwt==1.3.0", ] [tool.setuptools] @@ -35,6 +43,7 @@ dependencies = [ "black", "pytest", "neovim", + "fsspec", "pyright==1.1.317", "ipython", "pytest-cov", @@ -47,7 +56,11 @@ dependencies = [ ] [tool.hatch.envs.default.scripts] -# TODO: support autoreloading and providing a custom folder/port +build = [ + "python -m example.build", + "python -m runner.build", + "python -m server.build", +] check = [ "pyright server", ] @@ -64,6 +77,7 @@ lint = [ spec = [ "pytest --cov server --cov-report term-missing --cov-report html:coverage --cov-fail-under 70 --timeout=300", ] +# TODO: support autoreloading and providing a custom folder/port start = [ "python -m server", ] diff --git a/runner/build.py b/runner/build.py new file mode 100644 index 000000000..e517ad2e6 --- /dev/null +++ b/runner/build.py @@ -0,0 +1,27 @@ +import os +import shutil +import tarfile + +import fsspec + +cache = ".cache" +target = "build/runner" +datemark = "20230826" +basepath = "https://github.com/indygreg/python-build-standalone/releases/download" +filename = f"cpython-3.10.13+{datemark}-x86_64-unknown-linux-gnu-install_only.tar.gz" + +os.makedirs(cache, exist_ok=True) +shutil.rmtree(target, ignore_errors=True) + +if not os.path.exists(f"{cache}/{filename}"): + local = fsspec.filesystem("file") + remote = fsspec.filesystem("http") + with local.open(f"{cache}/{filename}", "wb") as file_to: + with remote.open(f"{basepath}/{datemark}/{filename}", "rb") as file_from: + file_to.write(file_from.read()) + +with tarfile.open(f"{cache}/{filename}", "r:gz") as tar: + tar.extractall(cache) + shutil.move(f"{cache}/python", target) + +print(f"[runner] Downloaded runner and extracted into '{target}'") diff --git a/server/build.py b/server/build.py new file mode 100644 index 000000000..62502e445 --- /dev/null +++ b/server/build.py @@ -0,0 +1,11 @@ +import os +import shutil + +source = "server" +target = "build/server" + +shutil.rmtree(target, ignore_errors=True) +shutil.copytree(source, target) +os.remove(f"{target}/build.py") + +print(f"[server] Copied '{source}' to '{target}'") diff --git a/server/endpoints/file/list.py b/server/endpoints/file/list.py index 820eacbbd..e8a6c60b5 100644 --- a/server/endpoints/file/list.py +++ b/server/endpoints/file/list.py @@ -70,7 +70,7 @@ def action(project: Project, props: Optional[Props] = None) -> Result: item.errors = errors_by_path[path] items.append(item) for folder in list(folders): - if folder.startswith(".") or folder in settings.IGNORED_FOLDERS: + if folder in settings.IGNORED_FOLDERS: folders.remove(folder) continue path = fs.get_path(root / folder) diff --git a/server/endpoints/project/__init__.py b/server/endpoints/project/__init__.py index bcc7421d9..d067a891e 100644 --- a/server/endpoints/project/__init__.py +++ b/server/endpoints/project/__init__.py @@ -1,2 +1,2 @@ # Register modules -from . import sync +from . import check, sync diff --git a/server/endpoints/project/check.py b/server/endpoints/project/check.py new file mode 100644 index 000000000..623f25be4 --- /dev/null +++ b/server/endpoints/project/check.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from fastapi import Request +from pydantic import BaseModel + +from ...project import Project +from ...router import router + + +class Result(BaseModel, extra="forbid"): + healthy: bool + + +@router.post("/project/check") +def endpoint(request: Request) -> Result: + return action(request.app.get_project()) + + +def action(project: Project) -> Result: + return Result(healthy=True) diff --git a/server/project.py b/server/project.py index 99ffc9183..982e4e996 100644 --- a/server/project.py +++ b/server/project.py @@ -3,7 +3,6 @@ from pathlib import Path from typing import Optional -import platformdirs from frictionless.resources import TextResource from . import settings @@ -20,9 +19,9 @@ class Project: def __init__(self, basepath: Optional[str] = None): # Ensure structure - self.system = platformdirs.user_config_path(appname=settings.APPNAME) + self.system = Path(settings.APP_HOME) self.public = Path(basepath or "") - self.private = self.public / ".frictionless" + self.private = self.public / f".{settings.APP_NAME}" self.system.mkdir(parents=True, exist_ok=True) self.public.mkdir(parents=True, exist_ok=True) self.private.mkdir(parents=True, exist_ok=True) diff --git a/server/settings.py b/server/settings.py index 5a38731b9..588ed2ea2 100644 --- a/server/settings.py +++ b/server/settings.py @@ -1,61 +1,21 @@ +from pathlib import Path + # General -APPNAME = "odet" +HOME = str(Path.home()) + +APP_NAME = "opendataeditor" +APP_HOME = f"{HOME}/.{APP_NAME}" + ARTIFACTS_IDENTIFIER = "_artifacts" BUFFER_SIZE = 1000 DEFAULT_HTTP_PORT = 4040 - IGNORED_FOLDERS = [ + ".venv", + ".python", "node_modules", - "logs" "*.logs", - ".pyc", - ".idea/", - ".vscode/", - "*.sublime*", - ".DS_STORE", - "npm-debug.log*", - "package-lock.json", - "/.cache", - "*.sqlite", - # Byte-compiled - ".pytest_cache/", - ".ruff_cache/", - "__pycache__/", - # Unit test / coverage - ".coverage", - ".coverage.*", - "coverage.xml", - "*.py[cod]", - ".pytest_cache/", - ".tox/", - ".nox/", - "cover/", - "*.whl", - # C - "*.so" - # Distribution - "bin/", - "build/", - "develop-eggs/", - "dist/", - "downloads/", - "eggs/", - ".eggs/", - "lib/", - "lib64/", - "parts", - "sdist/", - "var/", - "wheels/", - "share/python-wheels/", - "*.egg-info/", - ".installed.cfg", - "*.egg", - "MANIFEST" - # Jupyter - ".ipynb_checkpoints", - # mypy - ".mypy_cache/", - ".dmypy.json", - "dmypy.json", + ".pytest_cache", + ".ruff_cache", + "__pycache__", + ".opendataeditor", ]