diff --git a/packages/filesystem/package.json b/packages/filesystem/package.json index ff69bc191bfa9..50d87ce690a65 100644 --- a/packages/filesystem/package.json +++ b/packages/filesystem/package.json @@ -4,7 +4,6 @@ "description": "Theia - FileSystem Extension", "dependencies": { "@theia/core": "^0.6.0", - "@types/base64-js": "^1.2.5", "@types/body-parser": "^1.17.0", "@types/fs-extra": "^4.0.2", "@types/mime-types": "^2.1.0", @@ -12,7 +11,6 @@ "@types/tar-fs": "^1.16.1", "@types/touch": "0.0.1", "@types/uuid": "^3.4.3", - "base64-js": "^1.2.1", "body-parser": "^1.18.3", "drivelist": "^6.4.3", "fs-extra": "^4.0.2", diff --git a/packages/filesystem/src/browser/file-upload-service.ts b/packages/filesystem/src/browser/file-upload-service.ts index 02836fd839546..d4629e6195cbe 100644 --- a/packages/filesystem/src/browser/file-upload-service.ts +++ b/packages/filesystem/src/browser/file-upload-service.ts @@ -16,14 +16,13 @@ // tslint:disable:no-any -import * as base64 from 'base64-js'; import { injectable, inject, postConstruct } from 'inversify'; import URI from '@theia/core/lib/common/uri'; import { CancellationTokenSource, CancellationToken, checkCancelled } from '@theia/core/lib/common/cancellation'; import { Deferred } from '@theia/core/lib/common/promise-util'; import { MessageService } from '@theia/core/lib/common/message-service'; -import { Progress } from '@theia/core/src/common/message-service-protocol'; -import { FileUploadServer } from '../common/file-upload-server'; +import { Progress } from '@theia/core/lib/common/message-service-protocol'; +import { Endpoint } from '@theia/core/lib/browser/endpoint'; const maxChunkSize = 64 * 1024; @@ -48,9 +47,6 @@ export class FileUploadService { @inject(MessageService) protected readonly messageService: MessageService; - @inject(FileUploadServer) - protected readonly uploadServer: FileUploadServer; - protected uploadForm: FileUploadService.Form; @postConstruct() @@ -116,69 +112,63 @@ export class FileUploadService { if (!entries.length) { return result; } - const total = totalSize; - let done = 0; - for (const entry of entries) { - progress.report({ work: { done, total } }); - const { file } = entry; - let id: string | undefined; - let readBytes = 0; - let someAppendFailed: Error | undefined; + const deferredUpload = new Deferred(); + const endpoint = new Endpoint({ path: '/file-upload' }); + const socket = new WebSocket(endpoint.getWebSocketUrl().toString()); + socket.onerror = deferredUpload.reject; + socket.onclose = ({ code, reason }) => deferredUpload.reject(new Error(String(reason || code))); + socket.onmessage = ({ data }) => { + const response = JSON.parse(data); + if (response.progress) { + const { done, total } = response.progress; + progress.report({ work: { done, total } }); + return; + } + if (response.ok) { + deferredUpload.resolve(result); + } else if (response.error) { + deferredUpload.reject(new Error(response.error)); + } else { + console.error('unknown upload response: ' + response); + } + socket.close(); + }; + socket.onopen = async () => { try { - const promises: Promise[] = []; - do { - const fileSlice = await this.readFileSlice(file, readBytes); - if (someAppendFailed) { - throw someAppendFailed; - } - checkCancelled(token); - readBytes = fileSlice.read; - if (id === undefined) { - id = await this.uploadServer.open(entry.uri.toString(), fileSlice.content, readBytes >= file.size); - checkCancelled(token); - progress.report({ - work: { - done: done + fileSlice.read, - total - } - }); - } else { - promises.push(this.uploadServer.append(id, fileSlice.content, readBytes >= file.size).then(() => { + const total = totalSize; + socket.send(JSON.stringify({ total })); + for (const entry of entries) { + const { file } = entry; + let readBytes = 0; + socket.send(JSON.stringify({ uri: entry.uri.toString(), size: file.size })); + if (file.size) { + do { + const fileSlice = await this.readFileSlice(file, readBytes); checkCancelled(token); - progress.report({ - work: { - done: done + fileSlice.read, - total - } - }); - }, appendError => { - someAppendFailed = appendError; - throw appendError; - })); + readBytes = fileSlice.read; + socket.send(fileSlice.content); + while (socket.bufferedAmount > maxChunkSize * 2) { + await new Promise(resolve => setTimeout(resolve)); + checkCancelled(token); + } + } while (readBytes < file.size); } - } while (readBytes < file.size); - await Promise.all(promises); - done += file.size; - progress.report({ work: { done, total } }); - } finally { - if (id !== undefined) { - this.uploadServer.close(id); + } + } catch (e) { + deferredUpload.reject(e); + if (socket.readyState === 1) { + socket.close(); } } - } - progress.report({ work: { done: total, total } }); - return result; + }; + return deferredUpload.promise; } protected readFileSlice(file: File, read: number): Promise<{ - content: string + content: ArrayBuffer read: number }> { return new Promise((resolve, reject) => { - if (file.size === 0 && read === 0) { - resolve({ content: '', read }); - return; - } const bytesLeft = file.size - read; if (!bytesLeft) { reject(new Error('nothing to read')); @@ -189,8 +179,7 @@ export class FileUploadService { const reader = new FileReader(); reader.onload = () => { read += size; - const buffer = reader.result as ArrayBuffer; - const content = base64.fromByteArray(new Uint8Array(buffer)); + const content = reader.result as ArrayBuffer; resolve({ content, read }); }; reader.onerror = reject; @@ -321,22 +310,10 @@ export namespace FileUploadService { token: CancellationToken entries: UploadEntry[] totalSize: number - } export interface Form { targetInput: HTMLInputElement fileInput: HTMLInputElement progress?: FileUploadProgressParams } - export interface SubmitOptions { - body: FormData - token: CancellationToken - onDidProgress: (event: ProgressEvent) => void - } - export interface SubmitError extends Error { - status: number; - } - export function isSubmitError(e: any): e is SubmitError { - return !!e && 'status' in e; - } } diff --git a/packages/filesystem/src/browser/filesystem-frontend-module.ts b/packages/filesystem/src/browser/filesystem-frontend-module.ts index 46ea43b97744e..618fefa59c3ef 100644 --- a/packages/filesystem/src/browser/filesystem-frontend-module.ts +++ b/packages/filesystem/src/browser/filesystem-frontend-module.ts @@ -30,7 +30,6 @@ import { FileSystemWatcher } from './filesystem-watcher'; import { FileSystemFrontendContribution } from './filesystem-frontend-contribution'; import { FileSystemProxyFactory } from './filesystem-proxy-factory'; import { FileUploadService } from './file-upload-service'; -import { fileUploadPath, FileUploadServer } from '../common/file-upload-server'; export default new ContainerModule(bind => { bindFileSystemPreferences(bind); @@ -63,11 +62,6 @@ export default new ContainerModule(bind => { bind(FileSystemFrontendContribution).toSelf().inSingletonScope(); bind(CommandContribution).toService(FileSystemFrontendContribution); bind(FrontendApplicationContribution).toService(FileSystemFrontendContribution); - - bind(FileUploadServer).toDynamicValue(ctx => { - const provider = ctx.container.get(WebSocketConnectionProvider); - return provider.createProxy(fileUploadPath); - }).inSingletonScope(); }); export function bindFileResource(bind: interfaces.Bind): void { diff --git a/packages/filesystem/src/common/file-upload-server.ts b/packages/filesystem/src/common/file-upload-server.ts deleted file mode 100644 index 51c764e4f5e51..0000000000000 --- a/packages/filesystem/src/common/file-upload-server.ts +++ /dev/null @@ -1,27 +0,0 @@ -/******************************************************************************** - * Copyright (C) 2019 TypeFox and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * This Source Code may also be made available under the following Secondary - * Licenses when the conditions for such availability set forth in the Eclipse - * Public License v. 2.0 are satisfied: GNU General Public License, version 2 - * with the GNU Classpath Exception which is available at - * https://www.gnu.org/software/classpath/license.html. - * - * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 - ********************************************************************************/ - -import { Disposable } from '@theia/core/lib/common/disposable'; - -export const fileUploadPath = '/services/file-upload'; - -export const FileUploadServer = Symbol('FileUploadServer'); -/** content in base64 encoding */ -export interface FileUploadServer extends Disposable { - open(uri: string, content: string, done: boolean): Promise; - append(id: string, content: string, done: boolean): Promise; - close(id: string): Promise; -} diff --git a/packages/filesystem/src/node/filesystem-backend-module.ts b/packages/filesystem/src/node/filesystem-backend-module.ts index 5309465adb2d2..4715f912153fd 100644 --- a/packages/filesystem/src/node/filesystem-backend-module.ts +++ b/packages/filesystem/src/node/filesystem-backend-module.ts @@ -21,8 +21,8 @@ import { FileSystem, FileSystemClient, fileSystemPath, DispatchingFileSystemClie import { FileSystemWatcherServer, FileSystemWatcherClient, fileSystemWatcherPath } from '../common/filesystem-watcher-protocol'; import { FileSystemWatcherServerClient } from './filesystem-watcher-client'; import { NsfwFileSystemWatcherServer } from './nsfw-watcher/nsfw-filesystem-watcher'; -import { fileUploadPath, FileUploadServer } from '../common/file-upload-server'; -import { NodeFileUploadServer } from './node-file-upload-server'; +import { MessagingService } from '@theia/core/lib/node/messaging/messaging-service'; +import { NodeFileUploadService } from './node-file-upload-service'; const SINGLE_THREADED = process.argv.indexOf('--no-cluster') !== -1; @@ -82,13 +82,6 @@ export default new ContainerModule(bind => { }) ).inSingletonScope(); - bind(NodeFileUploadServer).toSelf().inTransientScope(); - bind(FileUploadServer).toService(NodeFileUploadServer); - bind(ConnectionHandler).toDynamicValue(ctx => - new JsonRpcConnectionHandler(fileUploadPath, client => { - const server = ctx.container.get(FileUploadServer); - client.onDidCloseConnection(() => server.dispose()); - return server; - }) - ).inSingletonScope(); + bind(NodeFileUploadService).toSelf().inSingletonScope(); + bind(MessagingService.Contribution).toService(NodeFileUploadService); }); diff --git a/packages/filesystem/src/node/node-file-upload-server.ts b/packages/filesystem/src/node/node-file-upload-server.ts deleted file mode 100644 index a2845d17aaab9..0000000000000 --- a/packages/filesystem/src/node/node-file-upload-server.ts +++ /dev/null @@ -1,98 +0,0 @@ -/******************************************************************************** - * Copyright (C) 2019 TypeFox and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License v. 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * This Source Code may also be made available under the following Secondary - * Licenses when the conditions for such availability set forth in the Eclipse - * Public License v. 2.0 are satisfied: GNU General Public License, version 2 - * with the GNU Classpath Exception which is available at - * https://www.gnu.org/software/classpath/license.html. - * - * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 - ********************************************************************************/ - -import * as path from 'path'; -import * as crypto from 'crypto'; -import * as fs from 'fs-extra'; -import { injectable } from 'inversify'; -import { FileUri } from '@theia/core/lib/node/file-uri'; -import { Disposable, DisposableCollection } from '@theia/core/lib/common/disposable'; -import { FileUploadServer } from '../common/file-upload-server'; - -@injectable() -export class NodeFileUploadServer implements FileUploadServer { - - protected readonly toDispose = new DisposableCollection(); - protected readonly uploads = new Map(); - - dispose(): void { - this.toDispose.dispose(); - } - - async open(uri: string, content: string, done: boolean): Promise { - const upload = new NodeFileUpload(FileUri.fsPath(uri)); - this.toDispose.push(upload); - this.uploads.set(upload.id, upload); - this.toDispose.push(Disposable.create(() => this.uploads.delete(upload.id))); - await upload.create(content); - if (done) { - await upload.rename(); - await this.close(upload.id); - } - return upload.id; - } - - async append(id: string, content: string, done: boolean): Promise { - const upload = this.uploads.get(id); - if (!upload) { - throw new Error(`upload '${id}' does not exist`); - } - await upload.append(content); - if (done) { - await upload.rename(); - await this.close(upload.id); - } - } - - async close(id: string): Promise { - const upload = this.uploads.get(id); - if (upload) { - upload.dispose(); - } - } - -} - -export class NodeFileUpload implements Disposable { - - readonly id: string; - readonly uploadPath: string; - - constructor( - readonly fsPath: string - ) { - this.id = 'upload_' + crypto.randomBytes(16).toString('hex'); - this.uploadPath = path.join(path.dirname(fsPath), this.id); - } - - async create(content: string): Promise { - await fs.outputFile(this.uploadPath, content, 'base64'); - } - - async append(content: string): Promise { - await fs.appendFile(this.uploadPath, content, { encoding: 'base64' }); - } - - async rename(): Promise { - await fs.move(this.uploadPath, this.fsPath, { overwrite: true }); - this.dispose = () => Promise.resolve(); - } - - dispose(): void { - fs.unlink(this.uploadPath).catch(() => {/*no-op*/ }); - } - -} diff --git a/packages/filesystem/src/node/node-file-upload-service.ts b/packages/filesystem/src/node/node-file-upload-service.ts new file mode 100644 index 0000000000000..65b9178152277 --- /dev/null +++ b/packages/filesystem/src/node/node-file-upload-service.ts @@ -0,0 +1,91 @@ +/******************************************************************************** + * Copyright (C) 2019 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +// tslint:disable-next-line +import * as ws from 'ws'; +import { injectable } from 'inversify'; +import { FileUri } from '@theia/core/lib/node/file-uri'; +import { MessagingService } from '@theia/core/lib/node/messaging/messaging-service'; +import { NodeFileUpload } from './node-file-upload'; + +@injectable() +export class NodeFileUploadService implements MessagingService.Contribution { + + static wsPath = '/file-upload'; + + configure(service: MessagingService): void { + service.ws(NodeFileUploadService.wsPath, (_, socket) => this.handleFileUpload(socket)); + } + + protected handleFileUpload(socket: ws): void { + let total = 0; + let done = 0; + let upload: NodeFileUpload | undefined; + let queue = Promise.resolve(); + socket.on('message', data => queue = queue.then(async () => { + try { + if (upload) { + if (await upload.append(data as ArrayBuffer)) { + done += upload.size; + upload = undefined; + } + if (socket.readyState !== 1) { + return; + } + if (done < total) { + socket.send(JSON.stringify({ + progress: { + done: done + (upload ? upload.uploadedBytes : 0), + total + } + })); + } else { + socket.send(JSON.stringify({ ok: true })); + socket.close(); + } + return; + } + const request = JSON.parse(data.toString()); + if (request.total) { + total = request.total; + return; + } + if (request.uri) { + upload = new NodeFileUpload(FileUri.fsPath(request.uri), request.size); + await upload.create(); + return; + } + console.error('unknown upload request', data); + throw new Error('unknown upload request, see backend logs'); + } catch (e) { + console.error(e); + if (socket.readyState === 1) { + socket.send(JSON.stringify({ + error: 'upload failed (see backend logs for details), reason: ' + e.message + })); + socket.close(); + } + } + })); + socket.on('error', console.error); + socket.on('close', () => { + if (upload) { + upload.dispose(); + } + }); + } + +} diff --git a/packages/filesystem/src/node/node-file-upload.ts b/packages/filesystem/src/node/node-file-upload.ts new file mode 100644 index 0000000000000..d1cd881cf6c8b --- /dev/null +++ b/packages/filesystem/src/node/node-file-upload.ts @@ -0,0 +1,63 @@ +/******************************************************************************** + * Copyright (C) 2019 TypeFox and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * This Source Code may also be made available under the following Secondary + * Licenses when the conditions for such availability set forth in the Eclipse + * Public License v. 2.0 are satisfied: GNU General Public License, version 2 + * with the GNU Classpath Exception which is available at + * https://www.gnu.org/software/classpath/license.html. + * + * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + ********************************************************************************/ + +import * as path from 'path'; +import * as crypto from 'crypto'; +import * as fs from 'fs-extra'; +import { Buffer } from 'buffer'; +import { Disposable } from '@theia/core/lib/common/disposable'; + +export class NodeFileUpload implements Disposable { + + readonly id: string; + readonly uploadPath: string; + protected _uploadedBytes = 0; + get uploadedBytes(): number { + return this._uploadedBytes; + } + + constructor( + readonly fsPath: string, + readonly size: number + ) { + this.id = 'upload_' + crypto.randomBytes(16).toString('hex'); + this.uploadPath = path.join(path.dirname(fsPath), this.id); + } + + async create(): Promise { + await fs.outputFile(this.uploadPath, ''); + } + + async append(chunk: ArrayBuffer): Promise { + await fs.appendFile(this.uploadPath, Buffer.from(chunk)); + this._uploadedBytes += chunk.byteLength; + if (this._uploadedBytes >= this.size) { + await this.rename(); + return true; + } + return false; + } + + protected async rename(): Promise { + await fs.move(this.uploadPath, this.fsPath, { overwrite: true }); + this.dispose = () => Promise.resolve(); + } + + dispose(): void { + fs.unlink(this.uploadPath).catch(() => {/*no-op*/ }); + } + +} diff --git a/yarn.lock b/yarn.lock index 280844793069d..e0eaa0a62987d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -133,10 +133,6 @@ version "0.1.0" resolved "https://registry.yarnpkg.com/@types/base64-arraybuffer/-/base64-arraybuffer-0.1.0.tgz#739eea0a974d13ae831f96d97d882ceb0b187543" -"@types/base64-js@^1.2.5": - version "1.2.5" - resolved "https://registry.yarnpkg.com/@types/base64-js/-/base64-js-1.2.5.tgz#582b2476169a6cba460a214d476c744441d873d5" - "@types/body-parser@*", "@types/body-parser@^1.16.4", "@types/body-parser@^1.17.0": version "1.17.0" resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.17.0.tgz#9f5c9d9bd04bb54be32d5eb9fc0d8c974e6cf58c" @@ -1586,7 +1582,7 @@ base64-js@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-0.0.8.tgz#1101e9544f4a76b1bc3b26d452ca96d7a35e7978" -base64-js@^1.0.2, base64-js@^1.2.1: +base64-js@^1.0.2: version "1.3.0" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.0.tgz#cab1e6118f051095e58b5281aea8c1cd22bfc0e3"