diff --git a/contrib/pai_vscode/CHANGELOG.md b/contrib/pai_vscode/CHANGELOG.md index 7a1d5e20e1..2c90cb1dc7 100644 --- a/contrib/pai_vscode/CHANGELOG.md +++ b/contrib/pai_vscode/CHANGELOG.md @@ -4,8 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.0.1] - 2018-01-09 +## [0.1.0] - 2019-01 ### Added - Open job pages and dashboard pages in VS Code - Submit job to PAI cluster from VS Code - Open PAI's hdfs as a workspace folder + +## [0.2.0] - 2019-03 +### Added +- Generate jsonc job config by default +- Add a PAI view container (sidebar), includes + - Job list view + - Auto refresh enabled + - HDFS explorer + - You can choose where hdfs explorer will be shown (view container or workspace folder) diff --git a/contrib/pai_vscode/README.md b/contrib/pai_vscode/README.md index 7ae088bc48..f224328a4d 100644 --- a/contrib/pai_vscode/README.md +++ b/contrib/pai_vscode/README.md @@ -41,6 +41,21 @@ The extension has a useful feature called "Simulate Job Running". It enables the ![](https://raw.githubusercontent.com/Microsoft/pai/master/contrib/pai_vscode/assets/simulate-job.gif) +## Sidebar +HDFS Explorer and Job List view will be shown in the extension's view container (sidebar). + +### HDFS Explorer +1. You are able to connect to an OpenPAI cluster's HDFS by double click its corresponding tree node. +2. The HDFS directory structure will be expended in the tree view. +3. Right click the folder or file node to perform file system operations. + +### Job List +1. Browse latest jobs in the OpenPAI cluster in job list tree view. +2. Browse recent submitted jobs by local (extension) in job list tree view. +3. Job list will auto refresh every 10 seconds. +4. Double click job node to browse Job details in external browser. + +![](https://raw.githubusercontent.com/Microsoft/pai/master/contrib/pai_vscode/assets/job-list.png) ## Commands ### Command Pallete @@ -61,7 +76,7 @@ The extension has a useful feature called "Simulate Job Running". It enables the |Submit Job...|Submit job to selected cluster| |Simulate Job Running...|Generate Dockerfile to simulate PAI job running| |Edit Configuration...|Edit cluster configuration| -|Open HDFS...|Open selected cluster's HDFS as a VS Code workspace folder| +|Open HDFS...|Open HDFS explorer of selected cluster| ## Settings |ID|Description| @@ -70,6 +85,10 @@ The extension has a useful feature called "Simulate Job Running". It enables the |pai.job.upload.exclude|Glob pattern for excluding files and folders| |pai.job.upload.include|Glob pattern for including files and folders| |pai.job.generateJobName.enabled|Controls whether the extension will add a random suffix to your job name when submitting job| +|pai.job.jobList.recentJobsLength|Controls the number of recently submitted jobs to keep in history for each PAI cluster| +|pai.job.jobList.allJobsPageSize|Controls the page size of list when listing jobs for each PAI cluster| +|pai.job.jobList.refreshInterval|Controls the refresh interval of job list (in seconds)| +|pai.hdfs.location|Location where hdfs explorer will be shown| ## Requirements PAI Cluster Version >= 0.8.0 diff --git a/contrib/pai_vscode/VSCodeExt.md b/contrib/pai_vscode/VSCodeExt.md index 2f05d80622..eabbe9267f 100644 --- a/contrib/pai_vscode/VSCodeExt.md +++ b/contrib/pai_vscode/VSCodeExt.md @@ -1,23 +1,22 @@ # OpenPAI VS Code Client -OpenPAI VS Code Client can submit AI jobs, simulate job running locally, manage HDFS files, and etc. It's an extension of Visual Studio Code. - -Visual Studio Code is a popular, free, lightweight but powerful source code editor which runs on your desktop and is available for Windows, macOS and Linux. - ## Installation -1. Download and install [Visual Studio Code](https://code.visualstudio.com/) by several clicks. +Visual Studio Code is a lightweight but powerful source code editor which runs on your desktop and is available for Windows, macOS and Linux. + +More information please refer to the [Visual Studio Code Official Site](https://code.visualstudio.com/). -2. Install **OpenPAI Client**. +OpenPAI Client is a VS Code extension to connect PAI clusters, submit AI jobs, and manage files on HDFS, etc. You need to install the extension in VS code before using it. - 1. Launch VS Code. - 2. Click the "Extensions" icon in Activity Bar or press **Ctrl+Shift+X** to bring up the Extensions view. - 3. Input **openpai** in the text box, the OpenPAI VS Code Client will appear in the result list. - 4. Click the **Install** button. The extension will be installed. - 5. After a successful installation, you will see an introduction page. Follow the instructions there and try the PAI client. +To install the OpenPAI Client: +1. Launch VS Code. +2. Click the "Extensions" icon in Activity Bar or press **Ctrl+Shift+X** to bring up the Extensions view. +3. Input **openpai** in the text box, the OpenPAI VS Code Client will appear in the result list. +4. Click the **Install** button. The extension will be installed. +5. After a successful installation, you will see an introduction page. Follow the instructions there and try the PAI client. - ![Extension](./assets/ext-install-1.png) +![Extension](./assets/ext-install-1.png) -## Next step +## Next steps Learn how to [use OpenPAI VS Code Client](./README.md) diff --git a/contrib/pai_vscode/assets/job-list.png b/contrib/pai_vscode/assets/job-list.png new file mode 100644 index 0000000000..19c7d43c9c Binary files /dev/null and b/contrib/pai_vscode/assets/job-list.png differ diff --git a/contrib/pai_vscode/i18n/common.json b/contrib/pai_vscode/i18n/common.json index f7ad857859..057d4202b4 100644 --- a/contrib/pai_vscode/i18n/common.json +++ b/contrib/pai_vscode/i18n/common.json @@ -23,10 +23,18 @@ "treeview.node.edit": "Edit Configuration...", "treeview.node.openhdfs": "Open HDFS...", "treeview.node.openPortal": "Open Web Portal...", - "treeview.node.listjob": "List Jobs...", + "treeview.node.listjob": "List Jobs Externally...", "treeview.node.create-config": "Create Job Config...", "treeview.node.submitjob": "Submit Job...", "treeview.node.simulate": "Simulate Job Running...", + "treeview.hdfs.select-cluster.label": "Double click to connect to a PAI cluster's HDFS...", + "treeview.joblist.recent": "Recent Submitted Jobs from VS Code", + "treeview.joblist.all": "All Jobs", + "treeview.joblist.view": "View Job Detail", + "treeview.joblist.more": "View More...", + "treeview.joblist.error": "Failed to load job list: {0}", + "container.hdfs.mkdir.prompt": "Please enter a folder name", + "container.hdfs.mkdir.cancelled": "Cancelled creating new folder", "hdfs.workspace.title": "HDFS Explorer - {0}", "hdfs.progress": "Transfering file - {0}% ({1} bytes / {2} bytes)", "hdfs.downloading": "Downloading {0}", @@ -47,6 +55,7 @@ "job.prepare.status": "PAI: Preparing for job submission", "job.prepare.cluster.cancelled": "No cluster selected, job submission cancelled.", "job.prepare.config.prompt": "Please select a PAI job config json file", + "job.prepare.config.invalid": "Invalid job config json file, job submission cancelled.", "job.prepare.config.cancelled": "No job config selected, job submission cancelled.", "job.prepare.upload.prompt": "Enable auto uploading of code?", "job.prepare.upload.yes.detail": "The extension will upload your project files to PAI job config's code dir automatically.", @@ -57,6 +66,7 @@ "job.upload.status": "PAI: Uploading code", "job.upload.progress": "PAI: Uploading code - {0} / {1}", "job.upload.error": "Error occurred while uploading code: {0}", + "job.upload.invalid-code-dir": "Auto uploading doesn't support code dir with url scheme hdfs:// or webhdfs://. Please use environment variable $PAI_DEFAULT_FS_URI instead.", "job.request.status": "PAI: Submitting job", "job.submission.error": "Error occurred while submitting job: {0}", "job.submission.success": "Successfully submitted job.", @@ -93,10 +103,18 @@ "treeview.node.edit": "编辑配置...", "treeview.node.openhdfs": "打开 HDFS...", "treeview.node.openPortal": "打开 OpenPAI 门户...", - "treeview.node.listjob": "打开任务列表...", + "treeview.node.listjob": "在浏览器里打开任务列表...", "treeview.node.create-config": "创建任务配置文件...", "treeview.node.submitjob": "提交任务...", "treeview.node.simulate": "模拟任务执行...", + "treeview.hdfs.select-cluster.label": "双击以连接到 PAI 集群的 HDFS...", + "treeview.joblist.recent": "近期从 VS Code 提交的任务", + "treeview.joblist.all": "所有任务", + "treeview.joblist.view": "查看任务详情", + "treeview.joblist.more": "显示更多...", + "treeview.joblist.error": "载入任务列表时发生错误：{0}", + "container.hdfs.mkdir.prompt": "请输入文件夹名", + "container.hdfs.mkdir.cancelled": "新建文件夹操作已取消", "hdfs.workspace.title": "HDFS 浏览器 - {0}", "hdfs.progress": "正在传输 - {0}% ({1} 字节 / {2} 字节)", "hdfs.downloading": "正在下载 {0}", @@ -118,6 +136,7 @@ "job.prepare.status": "PAI: 正在准备提交任务", "job.prepare.cluster.cancelled": "未选择集群，任务提交已被取消。", "job.prepare.config.prompt": "请选择一个 PAI 任务配置 JSON", + "job.prepare.config.invalid": "任务配置文件不合法，任务提交已被取消。", "job.prepare.config.cancelled": "未选择任务配置文件，任务提交已被取消。", "job.prepare.upload.prompt": "是否启用代码自动上传功能？", "job.prepare.upload.yes.detail": "插件将会自动上传你的项目文件至 PAI 任务配置中的 code dir", @@ -128,6 +147,7 @@ "job.upload.status": "PAI: 正在上传代码", "job.upload.progress": "PAI: 代码上传 - {0} / {1}", "job.upload.error": "代码上传时发生错误：{0}", + "job.upload.invalid-code-dir": "自动上传不支持 hdfs:// 及 webhdfs:// 形式的 code dir, 请使用环境变量 $PAI_DEFAULT_FS_URI", "job.submission.name-exist": "提交失败，已存在同名任务，是否启用自动生成任务名称后缀功能?", "job.submission.name-exist.enable": "启用并重新提交", "job.submission.error": "提交任务时发生错误：{0}", diff --git a/contrib/pai_vscode/icons/ellipsis.svg b/contrib/pai_vscode/icons/ellipsis.svg new file mode 100644 index 0000000000..9a76b9fab8 --- /dev/null +++ b/contrib/pai_vscode/icons/ellipsis.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/error.svg b/contrib/pai_vscode/icons/error.svg new file mode 100644 index 0000000000..8e08d84186 --- /dev/null +++ b/contrib/pai_vscode/icons/error.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/history.svg b/contrib/pai_vscode/icons/history.svg new file mode 100644 index 0000000000..9ef41c37cb --- /dev/null +++ b/contrib/pai_vscode/icons/history.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/latest.svg b/contrib/pai_vscode/icons/latest.svg new file mode 100644 index 0000000000..3d882c1878 --- /dev/null +++ b/contrib/pai_vscode/icons/latest.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/loading.svg b/contrib/pai_vscode/icons/loading.svg new file mode 100644 index 0000000000..e762f06d5e --- /dev/null +++ b/contrib/pai_vscode/icons/loading.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/loading_dark.svg b/contrib/pai_vscode/icons/loading_dark.svg new file mode 100644 index 0000000000..7dc1ebd8cf --- /dev/null +++ b/contrib/pai_vscode/icons/loading_dark.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + diff --git a/contrib/pai_vscode/icons/octicon/file.svg b/contrib/pai_vscode/icons/octicon/file.svg index 1bb5df6e03..f4efbc238b 100644 --- a/contrib/pai_vscode/icons/octicon/file.svg +++ b/contrib/pai_vscode/icons/octicon/file.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/file_dark.svg b/contrib/pai_vscode/icons/octicon/file_dark.svg index 4dc8aa60dd..6abbe9f171 100644 --- a/contrib/pai_vscode/icons/octicon/file_dark.svg +++ b/contrib/pai_vscode/icons/octicon/file_dark.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/home.svg b/contrib/pai_vscode/icons/octicon/home.svg new file mode 100644 index 0000000000..8799a322c1 --- /dev/null +++ b/contrib/pai_vscode/icons/octicon/home.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/home_dark.svg b/contrib/pai_vscode/icons/octicon/home_dark.svg new file mode 100644 index 0000000000..1d033a9d85 --- /dev/null +++ b/contrib/pai_vscode/icons/octicon/home_dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/terminal.svg b/contrib/pai_vscode/icons/octicon/terminal.svg index d1aaf41897..b6df312df4 100644 --- a/contrib/pai_vscode/icons/octicon/terminal.svg +++ b/contrib/pai_vscode/icons/octicon/terminal.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/octicon/terminal_dark.svg b/contrib/pai_vscode/icons/octicon/terminal_dark.svg index 5fa29ec4ad..fb45d72a62 100644 --- a/contrib/pai_vscode/icons/octicon/terminal_dark.svg +++ b/contrib/pai_vscode/icons/octicon/terminal_dark.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/ok.svg b/contrib/pai_vscode/icons/ok.svg new file mode 100644 index 0000000000..3efeb56727 --- /dev/null +++ b/contrib/pai_vscode/icons/ok.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/pai_container.png b/contrib/pai_vscode/icons/pai_container.png new file mode 100644 index 0000000000..b071bcc917 Binary files /dev/null and b/contrib/pai_vscode/icons/pai_container.png differ diff --git a/contrib/pai_vscode/icons/queue.svg b/contrib/pai_vscode/icons/queue.svg new file mode 100644 index 0000000000..a0577b519e --- /dev/null +++ b/contrib/pai_vscode/icons/queue.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/icons/run.svg b/contrib/pai_vscode/icons/run.svg new file mode 100644 index 0000000000..a37ceb2579 --- /dev/null +++ b/contrib/pai_vscode/icons/run.svg @@ -0,0 +1 @@ +ProgressBar_16x \ No newline at end of file diff --git a/contrib/pai_vscode/icons/stop.svg b/contrib/pai_vscode/icons/stop.svg new file mode 100644 index 0000000000..6b6668df9c --- /dev/null +++ b/contrib/pai_vscode/icons/stop.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/contrib/pai_vscode/package.json b/contrib/pai_vscode/package.json index 5467611c62..99348c0f93 100644 --- a/contrib/pai_vscode/package.json +++ b/contrib/pai_vscode/package.json @@ -2,7 +2,7 @@ "name": "pai-vscode", "displayName": "OpenPAI VS Code Client", "description": "Interact with Open Platform for AI (OpenPAI) from inside your editor", - "version": "0.1.0", + "version": "0.2.0", "publisher": "OpenPAIVSCodeClient", "preview": true, "icon": "assets/pai_logo.png", @@ -85,9 +85,23 @@ "title": "%paiext.hdfs.download%", "category": "PAI" }, + { + "command": "paiext.container.joblist.refresh", + "title": "%paiext.common.refresh%", + "icon": { + "light": "icons/refresh_light.svg", + "dark": "icons/refresh_dark.svg" + }, + "category": "PAI" + }, + { + "command": "paiext.container.joblist.more", + "title": "%paiext.cluster.job.more%", + "category": "PAI" + }, { "command": "paiext.cluster.refresh", - "title": "%paiext.cluster.refresh%", + "title": "%paiext.common.refresh%", "icon": { "light": "icons/refresh_light.svg", "dark": "icons/refresh_dark.svg" @@ -102,14 +116,58 @@ "dark": "icons/add_dark.svg" }, "category": "PAI" + }, + { + "command": "paiext.container.hdfs.refresh", + "title": "%paiext.common.refresh%", + "icon": { + "light": "icons/refresh_light.svg", + "dark": "icons/refresh_dark.svg" + }, + "category": "PAI" + }, + { + "command": "paiext.container.hdfs.back", + "title": "%paiext.container.hdfs.back%", + "icon": { + "light": "icons/octicon/home.svg", + "dark": "icons/octicon/home_dark.svg" + } + }, + { + "command": "paiext.container.hdfs.delete", + "title": "%paiext.container.hdfs.delete%" + }, + { + "command": "paiext.container.hdfs.mkdir", + "title": "%paiext.container.hdfs.mkdir%" } ], + "viewsContainers": { + "activitybar": [ + { + "id": "PAIContainer", + "title": "%container.title%", + "icon": "icons/pai_container.png" + } + ] + }, "views": { "explorer": [ { "id": "PAIExplorer", "name": "%explorer.paiClusterExplorer%" } + ], + "PAIContainer": [ + { + "id": "PAIContainerHDFS", + "name": "%container.hdfs.title%" + }, + { + "id": "PAIContainerJobList", + "name": "%container.joblist.title%" + } ] }, "menus": { @@ -131,7 +189,11 @@ "when": "false" }, { - "command": "paiext.hdfs.open", + "command": "paiext.container.joblist.refresh", + "when": "false" + }, + { + "command": "paiext.container.joblist.more", "when": "false" }, { @@ -145,6 +207,22 @@ { "command": "paiext.hdfs.download", "when": "false" + }, + { + "command": "paiext.container.hdfs.refresh", + "when": "false" + }, + { + "command": "paiext.container.hdfs.back", + "when": "false" + }, + { + "command": "paiext.container.hdfs.delete", + "when": "false" + }, + { + "command": "paiext.container.hdfs.mkdir", + "when": "false" } ], "view/title": [ @@ -157,6 +235,21 @@ "command": "paiext.cluster.add", "when": "view == PAIExplorer", "group": "navigation" + }, + { + "command": "paiext.container.hdfs.refresh", + "when": "view == PAIContainerHDFS", + "group": "navigation" + }, + { + "command": "paiext.container.hdfs.back", + "when": "view == PAIContainerHDFS", + "group": "navigation" + }, + { + "command": "paiext.container.joblist.refresh", + "when": "view == PAIContainerJobList", + "group": "navigation" } ], "explorer/context": [ @@ -207,6 +300,35 @@ { "command": "paiext.cluster.delete", "when": "view == PAIExplorer && viewItem == PAIConfiguration" + }, + { + "command": "paiext.container.hdfs.mkdir", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsFile", + "group": "1@1" + }, + { + "command": "paiext.hdfs.download", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsRoot", + "group": "2@1" + }, + { + "command": "paiext.hdfs.upload.files", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsFile", + "group": "2@2" + }, + { + "command": "paiext.hdfs.upload.folders", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsFile", + "group": "2@3" + }, + { + "command": "paiext.container.hdfs.delete", + "when": "view == PAIContainerHDFS && viewItem && viewItem != PAIHdfsRoot", + "group": "3@1" + }, + { + "command": "paiext.cluster.job.list", + "when": "view == PAIContainerJobList && viewItem && viewItem == PAIJobListCluster" } ] }, @@ -244,6 +366,30 @@ "type": "boolean", "description": "%config.job.generateJobName.enabled%", "default": null + }, + "pai.job.jobList.recentJobsLength": { + "type": "number", + "description": "%config.job.jobList.recentJobsLength%", + "default": 5 + }, + "pai.job.jobList.allJobsPageSize": { + "type": "number", + "description": "%config.job.jobList.allJobsPageSize%", + "default": 20 + }, + "pai.job.jobList.refreshInterval": { + "type": "number", + "description": "%config.job.jobList.refreshInterval%", + "default": 10 + }, + "pai.hdfs.location": { + "type": "string", + "enum": [ + "sidebar", + "explorer" + ], + "default": "sidebar", + "description": "%config.hdfs.location%" } } } diff --git a/contrib/pai_vscode/package.nls.json b/contrib/pai_vscode/package.nls.json index 73cc7308c5..eabb2210c0 100644 --- a/contrib/pai_vscode/package.nls.json +++ b/contrib/pai_vscode/package.nls.json @@ -1,21 +1,33 @@ { "explorer.paiClusterExplorer": "PAI Cluster Explorer", + "container.title": "Open Platform for AI", + "container.hdfs.title": "HDFS Explorer", + "container.joblist.title": "PAI Job List", + "paiext.common.refresh": "Refresh", "paiext.cluster.add": "Add PAI Cluster", "paiext.cluster.edit": "Edit PAI Cluster Configuration", "paiext.cluster.delete": "Delete PAI Cluster Configuration", - "paiext.cluster.refresh": "Refresh", "paiext.hdfs.open": "Open HDFS", - "paiext.hdfs.upload.files": "Upload Files...", - "paiext.hdfs.upload.folders": "Upload Folders...", - "paiext.hdfs.download": "Download...", + "paiext.hdfs.upload.files": "Upload Files", + "paiext.hdfs.upload.folders": "Upload Folders", + "paiext.hdfs.download": "Download", + "paiext.container.hdfs.back": "Go back to cluster selection", + "paiext.container.hdfs.delete": "Delete", + "paiext.container.hdfs.mkdir": "New Folder", "paiext.cluster.dashboard.open": "Open Dashboard", - "paiext.cluster.job.list": "Open Job List", + "paiext.cluster.job.list": "Open Job List Externally", "paiext.cluster.job.submit": "Submit Job to PAI Cluster", "paiext.cluster.job.create-config": "Create PAI Job Config JSON", "paiext.cluster.job.simulate": "Simulate PAI Job Running", + "paiext.cluster.job.view": "View Job Detail", + "paiext.cluster.job.more": "View More...", "config.title": "OpenPAI VS Code Client Settings", "config.job.upload.enabled": "Controls whether the extension will upload your project files to PAI job config's code dir automatically", "config.job.upload.exclude": "Glob pattern for excluding files and folders", "config.job.upload.include": "Glob pattern for including files and folders", - "config.job.generateJobName.enabled": "Controls whether the extension will add a random suffix to your job name when submitting job" + "config.job.generateJobName.enabled": "Controls whether the extension will add a random suffix to your job name when submitting job", + "config.job.jobList.recentJobsLength": "Controls the number of recently submitted jobs to keep in history for each PAI cluster", + "config.job.jobList.allJobsPageSize": "Controls the page size of list when listing jobs for each PAI cluster", + "config.job.jobList.refreshInterval": "Controls the refresh interval of job list (in seconds)", + "config.hdfs.location": "Location where hdfs explorer will be shown" } \ No newline at end of file diff --git a/contrib/pai_vscode/package.nls.zh-cn.json b/contrib/pai_vscode/package.nls.zh-cn.json index 31135fbb12..7a6ec1fb92 100644 --- a/contrib/pai_vscode/package.nls.zh-cn.json +++ b/contrib/pai_vscode/package.nls.zh-cn.json @@ -1,21 +1,33 @@ { "explorer.paiClusterExplorer": "PAI 集群浏览器", + "container.title": "AI 开发平台 (PAI)", + "container.hdfs.title": "HDFS 浏览器", + "container.joblist.title": "PAI 任务列表", + "paiext.common.refresh": "刷新", "paiext.cluster.add": "添加 PAI 集群", "paiext.cluster.edit": "编辑 PAI 集群配置", "paiext.cluster.delete": "删除 PAI 集群配置", - "paiext.cluster.refresh": "刷新", "paiext.hdfs.open": "打开 HDFS", - "paiext.hdfs.upload.files": "上传文件...", - "paiext.hdfs.upload.folders": "上传文件夹...", - "paiext.hdfs.download": "下载...", + "paiext.hdfs.upload.files": "上传文件", + "paiext.hdfs.upload.folders": "上传文件夹", + "paiext.hdfs.download": "下载", + "paiext.container.hdfs.back": "后退至集群选择", + "paiext.container.hdfs.delete": "删除", + "paiext.container.hdfs.mkdir": "新建文件夹", "paiext.cluster.dashboard.open": "打开仪表板", - "paiext.cluster.job.list": "打开任务列表", + "paiext.cluster.job.list": "在浏览器里打开任务列表", "paiext.cluster.job.submit": "在 PAI 集群上提交任务", "paiext.cluster.job.create-config": "创建 PAI 任务配置 JSON 文件", "paiext.cluster.job.simulate": "模拟 PAI 任务执行", + "paiext.cluster.job.view": "查看任务详情", + "paiext.cluster.job.more": "显示更多...", "config.title": "OpenPAI VS Code 客户端设置", "config.job.upload.enabled": "控制插件是否会自动将项目源代码上传至 PAI 任务配置文件的 CodeDir", "config.job.upload.exclude": "控制排除文件、文件夹的 Glob 模式", "config.job.upload.include": "控制包括文件、文件夹的 Glob 模式", - "config.job.generateJobName.enabled": "控制插件是否会在提交任务时，自动在任务名称后添加随机字符串，以避免重复" + "config.job.generateJobName.enabled": "控制插件是否会在提交任务时，自动在任务名称后添加随机字符串，以避免重复", + "config.job.jobList.recentJobsLength": "控制每个 PAI 集群保留最近提交任务的数量", + "config.job.jobList.allJobsPageSize": "控制 PAI 集群任务列表的分页大小", + "config.job.jobList.refreshInterval": "控制任务列表的刷新间隔（秒）", + "config.hdfs.location": "HDFS 浏览器显示的位置" } \ No newline at end of file diff --git a/contrib/pai_vscode/schemas/pai_job_config.schema.json b/contrib/pai_vscode/schemas/pai_job_config.schema.json index 0e6e2a996d..989c9d36e7 100644 --- a/contrib/pai_vscode/schemas/pai_job_config.schema.json +++ b/contrib/pai_vscode/schemas/pai_job_config.schema.json @@ -1,6 +1,6 @@ { "type": "object", - "description": "PAI job config", + "description": "PAI job config\nThis file can be submitted directly on PAI web portal.", "properties": { "jobName": { "type": "string", diff --git a/contrib/pai_vscode/src/common/constants.ts b/contrib/pai_vscode/src/common/constants.ts index c610763221..49d38dca0b 100644 --- a/contrib/pai_vscode/src/common/constants.ts +++ b/contrib/pai_vscode/src/common/constants.ts @@ -15,21 +15,46 @@ export const COMMAND_HDFS_UPLOAD_FOLDERS = 'paiext.hdfs.upload.folders'; export const COMMAND_HDFS_DOWNLOAD = 'paiext.hdfs.download'; export const COMMAND_OPEN_DASHBOARD = 'paiext.cluster.dashboard.open'; export const COMMAND_LIST_JOB = 'paiext.cluster.job.list'; +export const COMMAND_VIEW_JOB = 'paiext.cluster.job.view'; export const COMMAND_TREEVIEW_OPEN_PORTAL = 'paiext.treeview.openPortal'; export const COMMAND_TREEVIEW_DOUBLECLICK = 'paiext.treeview.doubleclick'; export const COMMAND_SUBMIT_JOB = 'paiext.cluster.job.submit'; export const COMMAND_SIMULATE_JOB = 'paiext.cluster.job.simulate'; export const COMMAND_CREATE_JOB_CONFIG = 'paiext.cluster.job.create-config'; +export const COMMAND_CONTAINER_HDFS_BACK = 'paiext.container.hdfs.back'; +export const COMMAND_CONTAINER_HDFS_REFRESH = 'paiext.container.hdfs.refresh'; +export const COMMAND_CONTAINER_HDFS_DELETE = 'paiext.container.hdfs.delete'; +export const COMMAND_CONTAINER_HDFS_MKDIR = 'paiext.container.hdfs.mkdir'; +export const COMMAND_CONTAINER_JOBLIST_REFRESH = 'paiext.container.joblist.refresh'; +export const COMMAND_CONTAINER_JOBLIST_MORE = 'paiext.container.joblist.more'; export const VIEW_CONFIGURATION_TREE = 'PAIExplorer'; export const CONTEXT_CONFIGURATION_ITEM = 'PAIConfiguration'; -export const CONTEXT_CONFIGURATION_ITEM_WEBPAGE = 'PAIWebpage'; +export const VIEW_CONTAINER_HDFS = 'PAIContainerHDFS'; +export const CONTEXT_HDFS_FILE = 'PAIHdfsFile'; +export const CONTEXT_HDFS_FOLDER = 'PAIHdfsFolder'; +export const CONTEXT_HDFS_ROOT = 'PAIHdfsRoot'; +export const CONTEXT_HDFS_SELECT_CLUSTER_ROOT = 'PAIHdfsSelectRoot'; +export const CONTEXT_HDFS_SELECT_CLUSTER = 'PAIHdfsSelect'; + +export const VIEW_CONTAINER_JOBLIST = 'PAIContainerJobList'; +export const CONTEXT_JOBLIST_CLUSTER = 'PAIJobListCluster'; + +export const SETTING_SECTION_HDFS = 'pai.hdfs'; +export const SETTING_HDFS_EXPLORER_LOCATION = 'location'; +export const ENUM_HDFS_EXPLORER_LOCATION = { + sidebar: 'sidebar', + explorer: 'explorer' +}; export const SETTING_SECTION_JOB = 'pai.job'; export const SETTING_JOB_UPLOAD_ENABLED = 'upload.enabled'; export const SETTING_JOB_UPLOAD_EXCLUDE = 'upload.exclude'; export const SETTING_JOB_UPLOAD_INCLUDE = 'upload.include'; export const SETTING_JOB_GENERATEJOBNAME_ENABLED = 'generateJobName.enabled'; +export const SETTING_JOB_JOBLIST_RECENTJOBSLENGTH = 'jobList.recentJobsLength'; +export const SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE = 'jobList.allJobsPageSize'; +export const SETTING_JOB_JOBLIST_REFERSHINTERVAL = 'jobList.refreshInterval'; export const ICON_PAI = { light: 'icons/PAI_light.png', @@ -57,6 +82,18 @@ export const ICON_CREATE_CONFIG = { light: 'icons/octicon/file.svg', dark: 'icons/octicon/file_dark.svg' }; +export const ICON_QUEUE = 'icons/queue.svg'; +export const ICON_STOP = 'icons/stop.svg'; +export const ICON_ERROR = 'icons/error.svg'; +export const ICON_RUN = 'icons/run.svg'; +export const ICON_OK = 'icons/ok.svg'; +export const ICON_HISTORY = 'icons/history.svg'; +export const ICON_LATEST = 'icons/latest.svg'; +export const ICON_ELLIPSIS = 'icons/ellipsis.svg'; +export const ICON_LOADING = { + light: 'icons/loading.svg', + dark: 'icons/loading_dark.svg' +}; export const OCTICON_CLOUDUPLOAD = '$(cloud-upload)'; diff --git a/contrib/pai_vscode/src/common/singleton.ts b/contrib/pai_vscode/src/common/singleton.ts index 2240d862c8..d3cc09b07c 100644 --- a/contrib/pai_vscode/src/common/singleton.ts +++ b/contrib/pai_vscode/src/common/singleton.ts @@ -6,7 +6,7 @@ import 'reflect-metadata'; // tslint:disable-line -import { Container, injectable } from 'inversify'; +import { injectable, Container } from 'inversify'; import * as vscode from 'vscode'; import { __ } from './i18n'; diff --git a/contrib/pai_vscode/src/common/treeViewHelper.ts b/contrib/pai_vscode/src/common/treeViewHelper.ts new file mode 100644 index 0000000000..525bf8d4f0 --- /dev/null +++ b/contrib/pai_vscode/src/common/treeViewHelper.ts @@ -0,0 +1,47 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ + +import { injectable } from 'inversify'; +import { isNil } from 'lodash'; +import { commands, workspace } from 'vscode'; + +import { COMMAND_TREEVIEW_DOUBLECLICK } from '../common/constants'; +import { __ } from '../common/i18n'; +import { Singleton } from '../common/singleton'; + +/** + * Supports double click commands for tree view + */ +@injectable() +export class TreeViewHelper extends Singleton { + + private lastClick?: { command: string, time: number }; + private readonly doubleClickInterval: number = 300; + + constructor() { + super(); + this.context.subscriptions.push( + commands.registerCommand(COMMAND_TREEVIEW_DOUBLECLICK, (command: string, ...args: string[]) => { + const mode: string | undefined = workspace.getConfiguration('workbench.list').get('openMode'); + if (mode === 'doubleClick') { + void commands.executeCommand(command, ...args); + } else { + // Single Click + if ( + !isNil(this.lastClick) && + this.lastClick.command === command && + Date.now() - this.lastClick.time < this.doubleClickInterval + ) { + this.lastClick = undefined; + void commands.executeCommand(command, ...args); + } else { + this.lastClick = { command, time: Date.now() }; + } + } + }) + ); + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/extension.ts b/contrib/pai_vscode/src/extension.ts index dcc3bc919c..138e3b37de 100644 --- a/contrib/pai_vscode/src/extension.ts +++ b/contrib/pai_vscode/src/extension.ts @@ -5,6 +5,7 @@ */ import * as vscode from 'vscode'; + import * as Singleton from './common/singleton'; import { allSingletonClasses } from './root'; diff --git a/contrib/pai_vscode/src/pai/clusterManager.ts b/contrib/pai_vscode/src/pai/clusterManager.ts index 21382072bc..38dcaa48e4 100644 --- a/contrib/pai_vscode/src/pai/clusterManager.ts +++ b/contrib/pai_vscode/src/pai/clusterManager.ts @@ -10,20 +10,28 @@ import * as request from 'request-promise-native'; import * as vscode from 'vscode'; import { - COMMAND_ADD_CLUSTER, COMMAND_DELETE_CLUSTER, COMMAND_EDIT_CLUSTER -} from '../common/constants'; + COMMAND_ADD_CLUSTER, COMMAND_DELETE_CLUSTER, COMMAND_EDIT_CLUSTER} from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ConfigurationNode, ConfigurationTreeDataProvider } from './configurationTreeDataProvider'; + +import { ClusterExplorerChildNode, ConfigurationTreeDataProvider, ITreeData } from './configurationTreeDataProvider'; import { IPAICluster } from './paiInterface'; import semverCompare = require('semver-compare'); // tslint:disable-line + export interface IConfiguration { readonly version: string; pais: IPAICluster[]; } +export type IClusterModification = { + index: number; + type: 'EDIT' | 'REMOVE'; +} | { + type: 'RESET'; +}; + export function getClusterIdentifier(conf: IPAICluster): string { return `${conf.username}@${conf.rest_server_uri}`; } @@ -32,10 +40,6 @@ export function getClusterName(conf: IPAICluster): string { return conf.name || getClusterIdentifier(conf); } -export function getClusterWebPortalUri(conf: IPAICluster): string { - return conf.web_portal_uri || conf.rest_server_uri.split(':')[0]; -} - /** * Manager class for cluster configurations */ @@ -57,6 +61,9 @@ export class ClusterManager extends Singleton { k8s_dashboard_uri: '127.0.0.1:9090' }; + private onDidChangeEmitter: vscode.EventEmitter = new vscode.EventEmitter(); + public onDidChange: vscode.Event = this.onDidChangeEmitter.event; // tslint:disable-line + private readonly EDIT: string = __('common.edit'); private readonly DISCARD: string = __('cluster.activate.fix.discard'); @@ -65,8 +72,20 @@ export class ClusterManager extends Singleton { public async onActivate(): Promise { this.context.subscriptions.push( vscode.commands.registerCommand(COMMAND_ADD_CLUSTER, () => this.add()), - vscode.commands.registerCommand(COMMAND_EDIT_CLUSTER, (node: ConfigurationNode) => this.edit(node.index)), - vscode.commands.registerCommand(COMMAND_DELETE_CLUSTER, (node: ConfigurationNode) => this.delete(node.index)) + vscode.commands.registerCommand(COMMAND_EDIT_CLUSTER, async (node: ClusterExplorerChildNode | ITreeData) => { + if (node instanceof ClusterExplorerChildNode) { + await this.edit(node.index); + } else { + await this.edit(node.clusterIndex); + } + }), + vscode.commands.registerCommand(COMMAND_DELETE_CLUSTER, async (node: ClusterExplorerChildNode | ITreeData) => { + if (node instanceof ClusterExplorerChildNode) { + await this.delete(node.index); + } else { + await this.delete(node.clusterIndex); + } + }) ); this.configuration = this.context.globalState.get(ClusterManager.CONF_KEY) || ClusterManager.default; try { @@ -126,7 +145,7 @@ export class ClusterManager extends Singleton { cluster.rest_server_uri = `${host}/rest-server`; cluster.k8s_dashboard_uri = `${host}/kubernetes-dashboard`; cluster.grafana_uri = `${host}/grafana`; - cluster.web_portal_uri = `${host}/`; + cluster.web_portal_uri = `${host}`; cluster.hdfs_uri = `hdfs://${host}:9000`; cluster.webhdfs_uri = `${host}/webhdfs/api/v1`; } catch { @@ -138,39 +157,40 @@ export class ClusterManager extends Singleton { cluster.hdfs_uri = `hdfs://${host}:9000`; cluster.k8s_dashboard_uri = `${host}:9090`; } - return this.edit(this.configuration!.pais.length, cluster); + return this.edit(this.allConfigurations.length, cluster); } public async edit(index: number, defaultConfiguration: IPAICluster = ClusterManager.paiDefault): Promise { - const original: IPAICluster = this.configuration!.pais[index] || defaultConfiguration; + const original: IPAICluster = this.allConfigurations[index] || defaultConfiguration; const editResult: IPAICluster | undefined = await Util.editJSON( original, `pai_cluster_${original.rest_server_uri}.json`, 'pai_cluster.schema.json' ); if (editResult) { - this.configuration!.pais[index] = editResult; - await (await getSingleton(ConfigurationTreeDataProvider)).refresh(index); + this.allConfigurations[index] = editResult; + this.onDidChangeEmitter.fire({ type: 'EDIT', index }); await this.save(); } } public async delete(index: number): Promise { - this.configuration!.pais.splice(index, 1); - await (await getSingleton(ConfigurationTreeDataProvider)).refresh(); + this.allConfigurations.splice(index, 1); + this.onDidChangeEmitter.fire({ type: 'REMOVE', index }); await this.save(); } - public save(): PromiseLike { - return this.context.globalState.update(ClusterManager.CONF_KEY, this.configuration!); + public async save(): Promise { + await this.context.globalState.update(ClusterManager.CONF_KEY, this.configuration!); + await (await getSingleton(ConfigurationTreeDataProvider)).refresh(); } public async pick(): Promise { - if (this.configuration!.pais.length === 1) { + if (this.allConfigurations.length === 1) { return 0; } - return await Util.pick(range(this.configuration!.pais.length), __('cluster.pick.prompt'), (index: number) => { - const conf: IPAICluster = this.allConfigurations![index]; + return await Util.pick(range(this.allConfigurations.length), __('cluster.pick.prompt'), (index: number) => { + const conf: IPAICluster = this.allConfigurations[index]; return { label: getClusterName(conf), detail: getClusterIdentifier(conf) @@ -195,7 +215,6 @@ export class ClusterManager extends Singleton { ); if (editResult) { this.configuration = editResult; - await (await getSingleton(ConfigurationTreeDataProvider)).refresh(); await this.save(); } break; @@ -205,5 +224,6 @@ export class ClusterManager extends Singleton { default: break; } + this.onDidChangeEmitter.fire({ type: 'RESET' }); } } \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts b/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts index 90b2c73f42..f9b8e2a87e 100644 --- a/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts +++ b/contrib/pai_vscode/src/pai/configurationTreeDataProvider.ts @@ -5,69 +5,43 @@ */ import { injectable } from 'inversify'; -import { isNil } from 'lodash'; import { - commands, Event, EventEmitter, TreeDataProvider, TreeItem, TreeItemCollapsibleState, window, - workspace + commands, window, + Event, EventEmitter, TreeDataProvider, TreeItem, TreeItemCollapsibleState } from 'vscode'; import { COMMAND_CREATE_JOB_CONFIG, COMMAND_EDIT_CLUSTER, COMMAND_LIST_JOB, COMMAND_OPEN_HDFS, COMMAND_REFRESH_CLUSTER, COMMAND_SIMULATE_JOB, COMMAND_SUBMIT_JOB, COMMAND_TREEVIEW_DOUBLECLICK, COMMAND_TREEVIEW_OPEN_PORTAL, - CONTEXT_CONFIGURATION_ITEM, CONTEXT_CONFIGURATION_ITEM_WEBPAGE, + CONTEXT_CONFIGURATION_ITEM, ICON_CREATE_CONFIG, ICON_DASHBOARD, ICON_EDIT, ICON_HDFS, ICON_LIST_JOB, ICON_PAI, ICON_SIMULATE_JOB, ICON_SUBMIT_JOB, VIEW_CONFIGURATION_TREE } from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ClusterManager, getClusterName } from './clusterManager'; + +import { getClusterName, ClusterManager } from './clusterManager'; import { IPAICluster } from './paiInterface'; interface IChildNodeDefinition { title: string; command: string; icon: string | { light: string, dark: string }; - type?: { new(...args: any[]): TreeNode }; condition?(conf: IPAICluster): boolean; } -/** - * General tree node recording its parent - */ -class TreeNode extends TreeItem { - constructor(title: string, public readonly parent?: TreeNode) { - super(title, parent ? TreeItemCollapsibleState.None : TreeItemCollapsibleState.Expanded); - } -} - -/** - * Tree node representing external link - */ -class TreeNodeWithLink extends TreeNode { - constructor(title: string, parent?: TreeNode) { - super(title, parent); - this.contextValue = CONTEXT_CONFIGURATION_ITEM_WEBPAGE; - } - - public get realCommand(): string | undefined { - return this.command && this.command.arguments && this.command.arguments[1]; - } -} - const childNodeDefinitions: IChildNodeDefinition[] = [ { title: 'treeview.node.openPortal', command: COMMAND_TREEVIEW_OPEN_PORTAL, - icon: ICON_DASHBOARD, - type: TreeNodeWithLink + icon: ICON_DASHBOARD }, { title: 'treeview.node.listjob', command: COMMAND_LIST_JOB, - icon: ICON_LIST_JOB, - type: TreeNodeWithLink + icon: ICON_LIST_JOB }, { title: 'treeview.node.create-config', @@ -97,43 +71,40 @@ const childNodeDefinitions: IChildNodeDefinition[] = [ } ]; +export interface ITreeData { + clusterIndex: number; + childDef?: IChildNodeDefinition; +} + /** * Root nodes representing cluster configuration */ -export class ConfigurationNode extends TreeNode { - public children: TreeNode[] = []; +export class ClusterExplorerRootNode extends TreeItem { + public readonly index: number; - public constructor(private _configuration: IPAICluster, public readonly index: number) { - super('...'); + public constructor(configuration: IPAICluster, index: number) { + super(getClusterName(configuration), TreeItemCollapsibleState.Expanded); this.iconPath = Util.resolvePath(ICON_PAI); - this.configuration = this._configuration; + this.index = index; this.contextValue = CONTEXT_CONFIGURATION_ITEM; } +} - public get configuration(): IPAICluster { - return this._configuration; - } - public set configuration(to: IPAICluster) { - this.label = getClusterName(to); - this._configuration = to; - this.initializeChildren(); - } - - private initializeChildren(): void { - this.children = []; - for (const def of childNodeDefinitions) { - if (def.condition && !def.condition(this.configuration)) { - continue; - } - const node: TreeNode = new (def.type || TreeNode)(__(def.title), this); - node.command = { - title: __(def.title), - command: COMMAND_TREEVIEW_DOUBLECLICK, - arguments: [this, def.command] - }; - node.iconPath = Util.resolvePath(def.icon); - this.children.push(node); - } +/** + * Child nodes representing operation + */ +export class ClusterExplorerChildNode extends TreeItem { + public readonly index: number; + + public constructor(clusterIndex: number, def: IChildNodeDefinition) { + super(__(def.title), TreeItemCollapsibleState.None); + this.iconPath = Util.resolvePath(def.icon); + this.index = clusterIndex; + this.command = { + title: __(def.title), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [def.command, this] + }; } } @@ -141,71 +112,54 @@ export class ConfigurationNode extends TreeNode { * Contributes to the tree view of cluster configurations */ @injectable() -export class ConfigurationTreeDataProvider extends Singleton implements TreeDataProvider { - private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); - public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line - - private configurationNodes: ConfigurationNode[] = []; - private lastClick?: { command: string, time: number }; - private readonly doubleClickInterval: number = 300; +export class ConfigurationTreeDataProvider extends Singleton implements TreeDataProvider { + private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); + public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line - constructor() { - super(); - this.context.subscriptions.push( - commands.registerCommand(COMMAND_REFRESH_CLUSTER, index => this.refresh(index)), - commands.registerCommand(COMMAND_TREEVIEW_DOUBLECLICK, (node: TreeNode, command: string) => { - const mode: string | undefined = workspace.getConfiguration('workbench.list').get('openMode'); - if (mode === 'doubleClick') { - void commands.executeCommand(command, node); - } else { - // Single Click - if ( - !isNil(this.lastClick) && - this.lastClick.command === command && - Date.now() - this.lastClick.time < this.doubleClickInterval - ) { - this.lastClick = undefined; - void commands.executeCommand(command, node); - } else { - this.lastClick = { command, time: Date.now() }; - } - } - }), - window.registerTreeDataProvider(VIEW_CONFIGURATION_TREE, this) - ); + public async refresh(): Promise { + this.onDidChangeTreeDataEmitter.fire(); } - public async refresh(index: number = -1): Promise { - const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; - if (index === -1 || !this.configurationNodes[index]) { - this.configurationNodes = allConfigurations.map((conf, i) => new ConfigurationNode(conf, i)); - this.onDidChangeTreeDataEmitter.fire(); + public async getTreeItem(data: ITreeData): Promise { + if (!data.childDef) { + const cluster: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations[data.clusterIndex]; + return new ClusterExplorerRootNode(cluster, data.clusterIndex); } else { - this.configurationNodes[index].configuration = allConfigurations[index]; - this.onDidChangeTreeDataEmitter.fire(this.configurationNodes[index]); + return new ClusterExplorerChildNode(data.clusterIndex, data.childDef); } } - public getTreeItem(element: TreeNode): TreeNode { - return element; - } - - public getChildren(element?: TreeNode): TreeNode[] | undefined { - if (!element) { - // Root nodes: configurations - return this.configurationNodes; - } - if (element instanceof ConfigurationNode) { - return element.children; + public async getChildren(data?: ITreeData): Promise { + if (!data) { + const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; + return allConfigurations.map((c, i) => ({ + clusterIndex: i + })); + } else if (!data.childDef) { + const cluster: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations[data.clusterIndex]; + return childNodeDefinitions.filter((def) => !def.condition || def.condition(cluster)).map(def => ({ + clusterIndex: data.clusterIndex, + childDef: def + })); + } else { + return undefined; } - return; } - public getParent(element: TreeNode): TreeNode | undefined { - return element.parent; + public getParent(data: ITreeData): ITreeData | undefined { + if (data.childDef) { + return { + clusterIndex: data.clusterIndex + }; + } else { + return undefined; + } } - public onActivate(): Promise { - return this.refresh(); + public async onActivate(): Promise { + this.context.subscriptions.push( + commands.registerCommand(COMMAND_REFRESH_CLUSTER, () => this.refresh()), + window.registerTreeDataProvider(VIEW_CONFIGURATION_TREE, this) + ); } } \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/container/hdfsTreeView.ts b/contrib/pai_vscode/src/pai/container/hdfsTreeView.ts new file mode 100644 index 0000000000..6639da7912 --- /dev/null +++ b/contrib/pai_vscode/src/pai/container/hdfsTreeView.ts @@ -0,0 +1,218 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ +/* tslint:disable:max-classes-per-file */ + +import { injectable } from 'inversify'; +import { + commands, window, Event, EventEmitter, FileType, + TreeDataProvider, TreeItem, TreeItemCollapsibleState, + TreeView, Uri +} from 'vscode'; + +import { + COMMAND_CONTAINER_HDFS_BACK, COMMAND_CONTAINER_HDFS_DELETE, COMMAND_CONTAINER_HDFS_MKDIR, COMMAND_CONTAINER_HDFS_REFRESH, + COMMAND_OPEN_HDFS, COMMAND_TREEVIEW_DOUBLECLICK, + CONTEXT_HDFS_FILE, CONTEXT_HDFS_FOLDER, CONTEXT_HDFS_ROOT, CONTEXT_HDFS_SELECT_CLUSTER, CONTEXT_HDFS_SELECT_CLUSTER_ROOT, + ICON_PAI, VIEW_CONTAINER_HDFS +} from '../../common/constants'; +import { __ } from '../../common/i18n'; +import { getSingleton, Singleton } from '../../common/singleton'; +import { Util } from '../../common/util'; + +import { getClusterName, ClusterManager } from '../clusterManager'; +import { HDFS, HDFSFileSystemProvider } from '../hdfs'; +import { IPAICluster } from '../paiInterface'; + +type IFileList = [string, FileType][]; + +/** + * Abstract tree node + */ +abstract class TreeNode extends TreeItem { + public parent?: TreeNode; +} + +/** + * File node + */ +class FileNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_FILE; + constructor(uri: Uri, parent: TreeNode) { + super(uri, TreeItemCollapsibleState.None); + this.parent = parent; + } +} + +/** + * Folder node + */ +class FolderNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_FOLDER; + constructor(uri: Uri, parent: TreeNode) { + super(uri, TreeItemCollapsibleState.Collapsed); + this.parent = parent; + } +} + +/** + * Root node + */ +class RootNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_ROOT; + constructor(uri: Uri) { + super(uri, TreeItemCollapsibleState.Expanded); + this.label = uri.toString(); + this.iconPath = Util.resolvePath(ICON_PAI); + } +} + +/** + * Cluster root node + */ +class SelectClusterRootNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_SELECT_CLUSTER_ROOT; + constructor() { + super(__('treeview.hdfs.select-cluster.label'), TreeItemCollapsibleState.Expanded); + } +} + +/** + * Cluster node (when no cluster is selected) + */ +class SelectClusterNode extends TreeNode { + public readonly contextValue: string = CONTEXT_HDFS_SELECT_CLUSTER; + public readonly cluster: IPAICluster; + constructor(cluster: IPAICluster, parent: TreeNode) { + super(getClusterName(cluster)); + this.cluster = cluster; + this.parent = parent; + this.command = { + title: __('treeview.node.openhdfs'), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [COMMAND_OPEN_HDFS, this.cluster] + }; + this.iconPath = Util.resolvePath(ICON_PAI); + } +} + +/** + * Contributes to the tree view of cluster configurations + */ +@injectable() +export class HDFSTreeDataProvider extends Singleton implements TreeDataProvider { + public readonly view: TreeView; + public root: TreeNode; + + private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); + public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line + + private uri?: Uri; + + constructor() { + super(); + this.root = new SelectClusterRootNode(); + this.view = window.createTreeView(VIEW_CONTAINER_HDFS, { treeDataProvider: this }); + } + + public async onActivate(): Promise { + this.context.subscriptions.push( + commands.registerCommand(COMMAND_CONTAINER_HDFS_REFRESH, () => this.refresh()), + commands.registerCommand(COMMAND_CONTAINER_HDFS_BACK, () => this.reset()), + commands.registerCommand(COMMAND_CONTAINER_HDFS_DELETE, async (node: TreeItem) => { + await (await getSingleton(HDFS)).provider!.delete(node.resourceUri!, { recursive: true }); + }), + commands.registerCommand(COMMAND_CONTAINER_HDFS_MKDIR, async (node: TreeItem) => { + const res: string | undefined = await window.showInputBox({ + prompt: __('container.hdfs.mkdir.prompt') + }); + if (res === undefined) { + Util.warn('container.hdfs.mkdir.cancelled'); + } else { + await (await getSingleton(HDFS)).provider!.createDirectory(Util.uriPathAppend(node.resourceUri!, res)); + } + }) + ); + this.refresh(); + (await getSingleton(HDFS)).provider!.onDidChangeFile(() => this.refresh()); + } + + public reset(): void { + this.uri = undefined; + this.root = new SelectClusterRootNode(); + this.refresh(); + void this.view.reveal(this.root); + } + + public setUri(uri?: Uri): void { + if (uri === undefined) { + this.reset(); + } else { + this.uri = uri; + this.root = new RootNode(this.uri); + this.refresh(); + void this.view.reveal(this.root); + } + } + + public refresh(node?: TreeNode): void { + this.onDidChangeTreeDataEmitter.fire(node); + } + + public getTreeItem(element: TreeNode): TreeNode { + return element; + } + + public async getChildren(element?: TreeNode): Promise { + if (!this.uri) { + if (!element) { + return [this.root]; + } else if (element.contextValue === CONTEXT_HDFS_SELECT_CLUSTER_ROOT) { + const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; + return allConfigurations.filter( + conf => !!conf.webhdfs_uri + ).map( + conf => new SelectClusterNode(conf, element) + ); + } else { + return; + } + } else { + if (!element) { + return [this.root]; + } else if (element.contextValue === CONTEXT_HDFS_FOLDER || element.contextValue === CONTEXT_HDFS_ROOT) { + const provider: HDFSFileSystemProvider | undefined = (await getSingleton(HDFS)).provider; + if (!provider) { + return; + } + const uri: Uri = element.resourceUri!; + const res: IFileList = await provider.readDirectory(uri); + return [ + ...res.filter( + ([name, type]) => type === FileType.Directory + ).sort( + ([name1, type1], [name2, type2]) => name1.localeCompare(name2) + ).map( + ([name, type]) => new FolderNode(Util.uriPathAppend(uri, name), element) + ), + ...res.filter( + ([name, type]) => type === FileType.File + ).sort( + ([name1, type1], [name2, type2]) => name1.localeCompare(name2) + ).map( + ([name, type]) => new FileNode(Util.uriPathAppend(uri, name), element) + ) + ]; + } else { + return; + } + + } + } + + public getParent(element: TreeNode): TreeNode | undefined { + return element.parent; + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/container/jobListTreeView.ts b/contrib/pai_vscode/src/pai/container/jobListTreeView.ts new file mode 100644 index 0000000000..60f8390d5b --- /dev/null +++ b/contrib/pai_vscode/src/pai/container/jobListTreeView.ts @@ -0,0 +1,360 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ +/* tslint:disable:max-classes-per-file */ + +import { injectable } from 'inversify'; +import * as request from 'request-promise-native'; +import { + commands, window, workspace, Event, EventEmitter, TreeDataProvider, + TreeItem, TreeItemCollapsibleState, TreeView, WorkspaceConfiguration +} from 'vscode'; + +import { + COMMAND_CONTAINER_JOBLIST_MORE, COMMAND_CONTAINER_JOBLIST_REFRESH, + COMMAND_TREEVIEW_DOUBLECLICK, COMMAND_VIEW_JOB, + CONTEXT_JOBLIST_CLUSTER, + ICON_ELLIPSIS, + ICON_ERROR, + ICON_HISTORY, + ICON_LATEST, + ICON_LOADING, + ICON_OK, + ICON_PAI, + ICON_QUEUE, + ICON_RUN, + ICON_STOP, + SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE, + SETTING_JOB_JOBLIST_RECENTJOBSLENGTH, + SETTING_JOB_JOBLIST_REFERSHINTERVAL, + SETTING_SECTION_JOB, + VIEW_CONTAINER_JOBLIST +} from '../../common/constants'; +import { __ } from '../../common/i18n'; +import { getSingleton, Singleton } from '../../common/singleton'; +import { Util } from '../../common/util'; +import { getClusterName, ClusterManager } from '../clusterManager'; +import { IPAICluster, IPAIJobInfo } from '../paiInterface'; +import { PAIRestUri } from '../paiUri'; +import { RecentJobManager } from '../recentJobManager'; + +enum FilterType { + Recent = 0, + All = 1 +} + +enum LoadingState { + Finished = 0, + Loading = 1, + Error = 2 +} + +enum TreeDataType { + Cluster = 0, + Filter = 1, + Job = 2, + More = 3 +} + +/** + * Leaf node representing job on PAI + */ +export class JobNode extends TreeItem { + private static statusIcons: { [status in IPAIJobInfo['state']]: string | undefined } = { + SUCCEEDED: ICON_OK, + FAILED: ICON_ERROR, + WAITING: ICON_QUEUE, + STOPPED: ICON_STOP, + RUNNING: ICON_RUN, + UNKNOWN: undefined + }; + + public constructor(jobInfo: IPAIJobInfo, config: IPAICluster) { + super(jobInfo.name); + this.command = { + title: __('treeview.joblist.view'), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [COMMAND_VIEW_JOB, jobInfo, config] + }; + const icon: string | undefined = JobNode.statusIcons[jobInfo.state]; + if (icon) { + this.iconPath = Util.resolvePath(icon); + } + } +} + +/** + * Expand job list when chosen + */ +class ShowMoreNode extends TreeItem { + public constructor(cluster: IClusterData) { + super(__('treeview.joblist.more')); + this.command = { + title: __('treeview.joblist.more'), + command: COMMAND_TREEVIEW_DOUBLECLICK, + arguments: [COMMAND_CONTAINER_JOBLIST_MORE, cluster] + }; + this.iconPath = Util.resolvePath(ICON_ELLIPSIS); + } +} + +/** + * Secondary node containing filtered job list + */ +class FilterNode extends TreeItem { + public constructor(type: FilterType, loadingState: LoadingState) { + if (type === FilterType.Recent) { + super(__('treeview.joblist.recent'), TreeItemCollapsibleState.Expanded); + } else { + super(__('treeview.joblist.all'), TreeItemCollapsibleState.Collapsed); + } + this.iconPath = Util.resolvePath( + loadingState === LoadingState.Loading ? ICON_LOADING : + loadingState === LoadingState.Error ? ICON_ERROR : + type === FilterType.Recent ? ICON_LATEST : ICON_HISTORY); + } +} + +/** + * Root node representing PAI cluster + */ +export class ClusterNode extends TreeItem { + public readonly index: number; + public constructor(configuration: IPAICluster, index: number) { + super(getClusterName(configuration), TreeItemCollapsibleState.Collapsed); + this.index = index; + this.iconPath = Util.resolvePath(ICON_PAI); + this.contextValue = CONTEXT_JOBLIST_CLUSTER; + } +} + +interface IClusterData { + type: TreeDataType.Cluster; + config: IPAICluster; + index: number; + shownAmount: number; + loadingState: LoadingState; + jobs: IPAIJobInfo[]; + lastShownAmount?: number; +} + +interface IFilterData { + type: TreeDataType.Filter; + filterType: FilterType; + clusterIndex: number; +} + +interface IJobData { + type: TreeDataType.Job; + job: IPAIJobInfo; + clusterIndex: number; + filterType: FilterType; +} + +interface IMoreData { + type: TreeDataType.More; + clusterIndex: number; +} + +type ITreeData = IClusterData | IFilterData | IJobData | IMoreData; + +/** + * Contributes to the tree view of cluster job list + */ +@injectable() +export class JobListTreeDataProvider extends Singleton implements TreeDataProvider { + private onDidChangeTreeDataEmitter: EventEmitter = new EventEmitter(); + public onDidChangeTreeData: Event = this.onDidChangeTreeDataEmitter.event; // tslint:disable-line + + private clusters: IClusterData[] = []; + private readonly treeView: TreeView; + private refreshTimer: NodeJS.Timer | undefined; + + constructor() { + super(); + this.treeView = window.createTreeView(VIEW_CONTAINER_JOBLIST, { treeDataProvider: this }); + this.context.subscriptions.push( + commands.registerCommand(COMMAND_CONTAINER_JOBLIST_REFRESH, () => this.refresh()), + commands.registerCommand( + COMMAND_CONTAINER_JOBLIST_MORE, + (cluster: IClusterData) => { + if (cluster.jobs.length <= cluster.shownAmount) { + return; + } + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + cluster.lastShownAmount = cluster.shownAmount; + cluster.shownAmount += settings.get(SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE)!; + void this.refresh(cluster.index, false); + } + ) + ); + } + + public async refresh(index: number = -1, reload: boolean = true): Promise { + if (index === -1 || !this.clusters[index]) { + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + const allConfigurations: IPAICluster[] = (await getSingleton(ClusterManager)).allConfigurations; + this.clusters = allConfigurations.map((config, i) => ({ + type: TreeDataType.Cluster, + index: i, + config, + loadingState: LoadingState.Finished, + jobs: [], + shownAmount: settings.get(SETTING_JOB_JOBLIST_ALLJOBSPAGESIZE)! + })); + this.onDidChangeTreeDataEmitter.fire(); + if (reload) { + void this.reloadJobs(); + } + } else { + this.onDidChangeTreeDataEmitter.fire(this.clusters[index]); + if (reload) { + void this.reloadJobs(index); + } + } + } + + public getTreeItem(element: ITreeData): TreeItem { + switch (element.type) { + case TreeDataType.Cluster: + return new ClusterNode(element.config, element.index); + case TreeDataType.Filter: + return new FilterNode(element.filterType, this.clusters[element.clusterIndex].loadingState); + case TreeDataType.Job: + return new JobNode(element.job, this.clusters[element.clusterIndex].config); + case TreeDataType.More: + return new ShowMoreNode(this.clusters[element.clusterIndex]); + default: + throw new Error('Unexpected node type'); + } + } + + public async getChildren(element?: ITreeData): Promise { + if (!element) { + // Root nodes: configurations + return this.clusters; + } + switch (element.type) { + case TreeDataType.Cluster: + { + return [ + { type: TreeDataType.Filter, filterType: FilterType.Recent, clusterIndex: element.index }, + { type: TreeDataType.Filter, filterType: FilterType.All, clusterIndex: element.index } + ]; + } + case TreeDataType.Filter: + if (element.filterType === FilterType.Recent) { + const cluster: IClusterData = this.clusters[element.clusterIndex]; + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + const recentMaxLen: number = settings.get(SETTING_JOB_JOBLIST_RECENTJOBSLENGTH)!; + const recentJobs: string[] | undefined = (await getSingleton(RecentJobManager)).allRecentJobs[cluster.index] || []; + const result: IJobData[] = []; + for (const name of recentJobs.slice(0, recentMaxLen)) { + const foundJob: IPAIJobInfo | undefined = cluster.jobs.find(job => job.name === name); + if (foundJob) { + result.push({ + type: TreeDataType.Job, + job: foundJob, + clusterIndex: element.clusterIndex, + filterType: element.filterType + }); + } + } + return result; + } else { + const cluster: IClusterData = this.clusters[element.clusterIndex]; + const result: (IJobData | IMoreData)[] = cluster.jobs.slice(0, cluster.shownAmount).map( + job => ({ + type: TreeDataType.Job, + job, + clusterIndex: element.clusterIndex, + filterType: element.filterType + }) + ); + if (cluster.lastShownAmount && cluster.lastShownAmount !== cluster.shownAmount) { + setImmediate(i => this.treeView.reveal(result[i]), cluster.lastShownAmount - 1); + cluster.lastShownAmount = cluster.shownAmount; + } + if (cluster.jobs.length > cluster.shownAmount) { + result.push({ type: TreeDataType.More, clusterIndex: element.clusterIndex }); + } + return result; + } + case TreeDataType.Job: + case TreeDataType.More: + return undefined; + default: + } + } + + public getParent(element: ITreeData): ITreeData | undefined { + if (element.type === TreeDataType.Job) { + return { + type: TreeDataType.Filter, + filterType: element.filterType, + clusterIndex: element.clusterIndex + }; + } + if (element.type === TreeDataType.Filter) { + return this.clusters[element.clusterIndex]; + } + } + + public async revealLatestJob(clusterIndex: number, jobName: string): Promise { + const job: IPAIJobInfo | undefined = this.clusters[clusterIndex].jobs.find(j => j.name === jobName); + if (job) { + /** + * Note: treeView.reveal() will obtain the node's parent and + * traverse to ancestors (upwards) until a expanded node has been found, + * then go back down and expand nodes via getChildren() on demand + */ + await this.treeView.reveal( + { + type: TreeDataType.Job, + job: job, + clusterIndex: clusterIndex, + filterType: FilterType.Recent + }, + { focus: true } + ); + } + } + + public onActivate(): Promise { + return this.refresh(); + } + + public async onDeactivate(): Promise { + if (this.refreshTimer) { + clearTimeout(this.refreshTimer); + } + this.treeView.dispose(); + } + + private async reloadJobs(index: number = -1): Promise { + if (this.refreshTimer) { + clearTimeout(this.refreshTimer); + } + const clusters: IClusterData[] = index !== -1 ? [this.clusters[index]] : this.clusters; + await Promise.all(clusters.map(async cluster => { + cluster.loadingState = LoadingState.Loading; + this.onDidChangeTreeDataEmitter.fire(cluster); + try { + cluster.jobs = await request.get( + PAIRestUri.jobs(cluster.config), + { json: true } + ); + cluster.loadingState = LoadingState.Finished; + } catch (e) { + Util.err('treeview.joblist.error', [e.message || e]); + cluster.loadingState = LoadingState.Error; + } + this.onDidChangeTreeDataEmitter.fire(cluster); + })); + const settings: WorkspaceConfiguration = workspace.getConfiguration(SETTING_SECTION_JOB); + const interval: number = settings.get(SETTING_JOB_JOBLIST_REFERSHINTERVAL)!; + this.refreshTimer = setTimeout(this.reloadJobs.bind(this), interval * 1000); + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/hdfs.ts b/contrib/pai_vscode/src/pai/hdfs.ts index 0ce4a71dac..6161b28c84 100644 --- a/contrib/pai_vscode/src/pai/hdfs.ts +++ b/contrib/pai_vscode/src/pai/hdfs.ts @@ -15,13 +15,16 @@ import { promisify } from 'util'; import * as vscode from 'vscode'; import { - COMMAND_HDFS_DOWNLOAD, COMMAND_HDFS_UPLOAD_FILES, COMMAND_HDFS_UPLOAD_FOLDERS, COMMAND_OPEN_HDFS, OCTICON_CLOUDUPLOAD + COMMAND_HDFS_DOWNLOAD, COMMAND_HDFS_UPLOAD_FILES, COMMAND_HDFS_UPLOAD_FOLDERS, COMMAND_OPEN_HDFS, + ENUM_HDFS_EXPLORER_LOCATION, OCTICON_CLOUDUPLOAD, SETTING_HDFS_EXPLORER_LOCATION, SETTING_SECTION_HDFS } from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; + import { ClusterManager } from './clusterManager'; -import { ConfigurationNode } from './configurationTreeDataProvider'; +import { ClusterExplorerChildNode } from './configurationTreeDataProvider'; +import { HDFSTreeDataProvider } from './container/hdfsTreeView'; import { IPAICluster } from './paiInterface'; import { createWebHDFSClient, IHDFSClient, IHDFSStatResult } from './webhdfs-workaround'; @@ -94,11 +97,17 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { statResult = await this.stat(uri); if (statResult.type !== vscode.FileType.Directory) { throw vscode.FileSystemError.FileExists(uri); + } else { + // pass + } + } catch (e) { + if (uri.path === '/') { + throw e; } - } catch { await this.createDirectory(Util.uriPathPop(uri)); try { await (await this.getClient(uri)).mkdir(path.join('/', uri.path)); + this.onDidChangeFileEmitter.fire([{ type: vscode.FileChangeType.Created, uri }]); } catch (ex) { throw new vscode.FileSystemError(ex); } @@ -108,6 +117,7 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { public async delete(uri: vscode.Uri, options: {recursive: boolean}): Promise { try { await (await this.getClient(uri)).unlink(path.join('/', uri.path), options.recursive); + this.onDidChangeFileEmitter.fire([{ type: vscode.FileChangeType.Deleted, uri }]); } catch (ex) { throw new vscode.FileSystemError(ex); } @@ -248,79 +258,81 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { const client: IHDFSClient = (await this.getClient(uri)); const filePath: string = path.join('/', uri.path); await vscode.window.withProgress( - { - location: vscode.ProgressLocation.Notification, - title: __('hdfs.uploading', [filePath]), - cancellable: true - }, - (progress, cancellationToken) => new Promise(async (resolve, reject) => { - try { - if ((await client.stat(filePath)).type === 'DIRECTORY') { - reject(vscode.FileSystemError.FileIsADirectory(uri)); - return; - } - if (!options.overwrite) { - reject(vscode.FileSystemError.FileExists(uri)); - return; - } - } catch { - if (!options.create) { - reject(vscode.FileSystemError.FileNotFound(uri)); - return; + { + location: vscode.ProgressLocation.Notification, + title: __('hdfs.uploading', [filePath]), + cancellable: true + }, + (progress, cancellationToken) => new Promise(async (resolve, reject) => { + try { + if ((await client.stat(filePath)).type === 'DIRECTORY') { + reject(vscode.FileSystemError.FileIsADirectory(uri)); + return; + } + if (!options.overwrite) { + reject(vscode.FileSystemError.FileExists(uri)); + return; + } + } catch { + if (!options.create) { + reject(vscode.FileSystemError.FileNotFound(uri)); + return; + } } - } - const local: fs.ReadStream = streamifier.createReadStream(content); - let writeAmount: number = 0; - const transform: Transform = new Transform({ - transform: (chunk: string | Buffer, encoding: string, callback: Function) => { - writeAmount += chunk.length; - progress.report({ - message: __('hdfs.progress', [ - (writeAmount / content.length * 100).toFixed(0), writeAmount, content.length - ]), - increment: chunk.length / content.length * 100 - }); - callback(null, chunk); + const local: fs.ReadStream = streamifier.createReadStream(content); + let writeAmount: number = 0; + const transform: Transform = new Transform({ + transform: (chunk: string | Buffer, encoding: string, callback: Function) => { + writeAmount += chunk.length; + progress.report({ + message: __('hdfs.progress', [ + (writeAmount / content.length * 100).toFixed(0), writeAmount, content.length + ]), + increment: chunk.length / content.length * 100 + }); + callback(null, chunk); + } + }); + const stream: Request = await client.createRobustWriteStream(filePath); + let error: any; + + function cleanup(): void { + local.unpipe(); + transform.unpipe(); + local.destroy(); + transform.destroy(); + stream.destroy(); } - }); - const stream: Request = await client.createRobustWriteStream(filePath); - let error: any; - - function cleanup(): void { - local.unpipe(); - transform.unpipe(); - local.destroy(); - transform.destroy(); - stream.destroy(); - } - cancellationToken.onCancellationRequested(() => { - error = true; - cleanup(); - reject(__('hdfs.write.cancelled')); - }); + cancellationToken.onCancellationRequested(() => { + error = true; + cleanup(); + reject(__('hdfs.write.cancelled')); + }); - local.once('error', err => { - error = err; - cleanup(); - reject(new vscode.FileSystemError(err)); - }); - stream.once('error', err => { - error = err; - cleanup(); - reject(new vscode.FileSystemError(err)); - }); + local.once('error', err => { + error = err; + cleanup(); + reject(new vscode.FileSystemError(err)); + }); + stream.once('error', err => { + error = err; + cleanup(); + reject(new vscode.FileSystemError(err)); + }); - stream.once('finish', () => { - cleanup(); - if (!error) { - resolve(); - } - }); + stream.once('finish', () => { + cleanup(); + if (!error) { + resolve(); + } + }); - // TODO: local.pipe(transform).pipe(stream); is not working due to unknown reason...maybe a bug in node-webhdfs? - local.pipe(transform).pipe(stream); - })); + // TODO: local.pipe(transform).pipe(stream); is not working due to unknown reason...maybe a bug in node-webhdfs? + local.pipe(transform).pipe(stream); + }) + ); + this.onDidChangeFileEmitter.fire([{ type: vscode.FileChangeType.Created, uri }]); } public async copy(source: vscode.Uri, destination: vscode.Uri, options: { overwrite: boolean }): Promise { @@ -460,65 +472,81 @@ export class HDFSFileSystemProvider implements vscode.FileSystemProvider { */ @injectable() export class HDFS extends Singleton { + public readonly provider: HDFSFileSystemProvider; private UPLOADFILES: string = __('hdfs.dialog.label.upload-files'); private UPLOADFOLDER: string = __('hdfs.dialog.label.upload-folders'); private DOWNLOADHERE: string = __('hdfs.dialog.label.download'); - private _provider: HDFSFileSystemProvider | undefined; - public get provider(): HDFSFileSystemProvider | undefined { - return this._provider; - } - constructor() { super(); console.log('Registering HDFS...'); - try { - this._provider = new HDFSFileSystemProvider(); - this.context.subscriptions.push( - vscode.workspace.registerFileSystemProvider('webhdfs', this._provider, { isCaseSensitive: true }) - ); - console.log('HDFS registered as webhdfs:/...'); - } catch (ex) { - Util.err('hdfs.initialization.error', [ex]); - } + this.provider = new HDFSFileSystemProvider(); + this.context.subscriptions.push( + vscode.workspace.registerFileSystemProvider('webhdfs', this.provider, { isCaseSensitive: true }) + ); + console.log('HDFS registered as webhdfs:/...'); this.context.subscriptions.push( vscode.commands.registerCommand( COMMAND_OPEN_HDFS, - (node: ConfigurationNode) => this.open(node.index) + async (node?: ClusterExplorerChildNode | IPAICluster) => { + if (!node) { + const manager: ClusterManager = await getSingleton(ClusterManager); + const index: number | undefined = await manager.pick(); + if (index === undefined) { + return; + } + await this.open(manager.allConfigurations[index]); + } else if (node instanceof ClusterExplorerChildNode) { + await this.open((await getSingleton(ClusterManager)).allConfigurations[node.index]); + } else { + await this.open(node); + } + } ), - vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FILES, this.uploadFiles.bind(this)), - vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FOLDERS, this.uploadFolders.bind(this)), - vscode.commands.registerCommand(COMMAND_HDFS_DOWNLOAD, this.download.bind(this)) + vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FILES, async (param: vscode.Uri | vscode.TreeItem) => { + await this.uploadFiles(this.unpackParam(param)); + }), + vscode.commands.registerCommand(COMMAND_HDFS_UPLOAD_FOLDERS, async (param: vscode.Uri | vscode.TreeItem) => { + await this.uploadFolders(this.unpackParam(param)); + }), + vscode.commands.registerCommand(COMMAND_HDFS_DOWNLOAD, async (param: vscode.Uri | vscode.TreeItem) => { + await this.download(this.unpackParam(param)); + }) ); } - public async open(index: number): Promise { - const configuration: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations[index]; - if (!configuration.webhdfs_uri) { + public async open(conf: IPAICluster): Promise { + if (!conf.webhdfs_uri) { Util.err('hdfs.initialization.missingconfiguration'); return; } - let start: number = 0; - let deleteCount: number = 0; - if (vscode.workspace.workspaceFolders) { - start = vscode.workspace.workspaceFolders.findIndex(folder => folder.uri.scheme === 'webhdfs'); - if (start >= 0) { - deleteCount = 1; - } else { - start = vscode.workspace.workspaceFolders.length; + const setting: string | undefined = vscode.workspace.getConfiguration(SETTING_SECTION_HDFS).get(SETTING_HDFS_EXPLORER_LOCATION); + if (setting === ENUM_HDFS_EXPLORER_LOCATION.explorer) { + let start: number = 0; + let deleteCount: number = 0; + if (vscode.workspace.workspaceFolders) { + start = vscode.workspace.workspaceFolders.findIndex(folder => folder.uri.scheme === 'webhdfs'); + if (start >= 0) { + deleteCount = 1; + } else { + start = vscode.workspace.workspaceFolders.length; + } } - } - try { - // this._provider!.addClient(getHDFSUriAuthority(configuration)); - await vscode.commands.executeCommand('workbench.view.explorer'); - void vscode.window.showInformationMessage(__('hdfs.open.prompt', [configuration.webhdfs_uri])); - vscode.workspace.updateWorkspaceFolders(start, deleteCount, { - uri: vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(configuration)}/`), - name: __('hdfs.workspace.title', [configuration.webhdfs_uri]) - }); - // Extension may be reloaded at this point due to workspace changes - } catch (ex) { - Util.err('hdfs.open.error', [ex]); + try { + // this.provider!.addClient(getHDFSUriAuthority(configuration)); + await vscode.commands.executeCommand('workbench.view.explorer'); + void vscode.window.showInformationMessage(__('hdfs.open.prompt', [conf.webhdfs_uri])); + vscode.workspace.updateWorkspaceFolders(start, deleteCount, { + uri: vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(conf)}/`), + name: __('hdfs.workspace.title', [conf.webhdfs_uri]) + }); + // Extension may be reloaded at this point due to workspace changes + } catch (ex) { + Util.err('hdfs.open.error', [ex]); + } + } else { + const provider: HDFSTreeDataProvider = await getSingleton(HDFSTreeDataProvider); + provider.setUri(vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(conf)}/`)); } } @@ -534,7 +562,19 @@ export class HDFS extends Singleton { } } - public async download(from: vscode.Uri): Promise { + private unpackParam(param: vscode.Uri | vscode.TreeItem): vscode.Uri { + if (param instanceof vscode.TreeItem) { + if (param.resourceUri !== undefined) { + return param.resourceUri; + } else { + throw new Error('Invalid HDFS operation param'); + } + } else { + return param; + } + } + + private async download(from: vscode.Uri): Promise { const files: vscode.Uri[] | undefined = await vscode.window.showOpenDialog({ canSelectFiles: false, canSelectFolders: true, @@ -547,7 +587,7 @@ export class HDFS extends Singleton { await this.provider!.copy(from, files[0], { overwrite: true }); } - public async uploadFiles(target: vscode.Uri): Promise { + private async uploadFiles(target: vscode.Uri): Promise { const files: vscode.Uri[] | undefined = await vscode.window.showOpenDialog({ canSelectFiles: true, canSelectMany: true, @@ -559,7 +599,7 @@ export class HDFS extends Singleton { return this.upload(files, target); } - public async uploadFolders(target: vscode.Uri): Promise { + private async uploadFolders(target: vscode.Uri): Promise { const folders: vscode.Uri[] | undefined = await vscode.window.showOpenDialog({ canSelectFiles: false, canSelectFolders: true, diff --git a/contrib/pai_vscode/src/pai/paiInterface.ts b/contrib/pai_vscode/src/pai/paiInterface.ts index 5ca4a2faf3..c0ef9fe468 100644 --- a/contrib/pai_vscode/src/pai/paiInterface.ts +++ b/contrib/pai_vscode/src/pai/paiInterface.ts @@ -33,4 +33,17 @@ export interface IPAIJobConfig { codeDir: string; outputDir: string; taskRoles: IPAITaskRole[]; +} + +export interface IPAIJobInfo { + name: string; + username: string; + state: 'SUCCEEDED' | 'FAILED' | 'WAITING' | 'STOPPED' | 'RUNNING' | 'UNKNOWN'; + subState: 'FRAMEWORK_COMPLETED' | 'FRAMEWORK_WAITING'; + executionType: 'START' | 'STOP'; + retries: number; + createdTime: number; + completedTime: number; + appExitCode: number; + virtualCluster: string; } \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/paiJobManager.ts b/contrib/pai_vscode/src/pai/paiJobManager.ts index 52b6dac74f..0cdbfb5cea 100644 --- a/contrib/pai_vscode/src/pai/paiJobManager.ts +++ b/contrib/pai_vscode/src/pai/paiJobManager.ts @@ -8,10 +8,9 @@ import * as fs from 'fs-extra'; import * as globby from 'globby'; import { injectable } from 'inversify'; import * as JSONC from 'jsonc-parser'; -import { isEmpty } from 'lodash'; +import { isEmpty, isNil } from 'lodash'; import * as os from 'os'; import * as path from 'path'; -import * as querystring from 'querystring'; import * as request from 'request-promise-native'; import * as uuid from 'uuid'; import * as vscode from 'vscode'; @@ -27,13 +26,16 @@ import { import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ClusterManager, getClusterIdentifier, getClusterWebPortalUri } from './clusterManager'; -import { ConfigurationNode } from './configurationTreeDataProvider'; + +import { getClusterIdentifier, ClusterManager } from './clusterManager'; +import { ClusterExplorerChildNode } from './configurationTreeDataProvider'; import { getHDFSUriAuthority, HDFS, HDFSFileSystemProvider } from './hdfs'; import { IPAICluster, IPAIJobConfig, IPAITaskRole } from './paiInterface'; import opn = require('opn'); // tslint:disable-line import unixify = require('unixify'); // tslint:disable-line +import { PAIRestUri, PAIWebPortalUri } from './paiUri'; +import { RecentJobManager } from './recentJobManager'; interface ITokenItem { token: string; expireTime: number; @@ -60,7 +62,6 @@ interface IJobInput { */ @injectable() export class PAIJobManager extends Singleton { - private static readonly API_PREFIX: string = 'api/v1'; private static readonly TIMEOUT: number = 60 * 1000; private static readonly SIMULATION_DOCKERFILE_FOLDER: string = '.pai_simulator'; private static readonly propertiesToBeReplaced: (keyof IPAIJobConfig)[] = [ @@ -84,7 +85,7 @@ export class PAIJobManager extends Singleton { this.context.subscriptions.push( vscode.commands.registerCommand( COMMAND_CREATE_JOB_CONFIG, - async (input?: ConfigurationNode | vscode.Uri) => { + async (input?: ClusterExplorerChildNode | vscode.Uri) => { if (input instanceof vscode.Uri) { await PAIJobManager.generateJobConfig(input.fsPath); } else { @@ -94,10 +95,10 @@ export class PAIJobManager extends Singleton { ), vscode.commands.registerCommand( COMMAND_SIMULATE_JOB, - async (input?: ConfigurationNode | vscode.Uri) => { + async (input?: ClusterExplorerChildNode | vscode.Uri) => { if (input instanceof vscode.Uri) { await this.simulate({ jobConfigPath: input.fsPath }); - } else if (input instanceof ConfigurationNode) { + } else if (input instanceof ClusterExplorerChildNode) { await this.simulate({ clusterIndex: input.index }); } else { await this.simulate(); @@ -106,10 +107,10 @@ export class PAIJobManager extends Singleton { ), vscode.commands.registerCommand( COMMAND_SUBMIT_JOB, - async (input?: ConfigurationNode | vscode.Uri) => { + async (input?: ClusterExplorerChildNode | vscode.Uri) => { if (input instanceof vscode.Uri) { await this.submitJob({ jobConfigPath: input.fsPath }); - } else if (input instanceof ConfigurationNode) { + } else if (input instanceof ClusterExplorerChildNode) { await this.submitJob({ clusterIndex: input.index }); } else { await this.submitJob(); @@ -132,7 +133,7 @@ export class PAIJobManager extends Singleton { parent = fileFolders[0].uri.fsPath; } } - defaultSaveDir = path.join(parent, `${name}.pai.json`); + defaultSaveDir = path.join(parent, `${name}.pai.jsonc`); config = { jobName: '', image: 'aiplatform/pai.build.base', @@ -160,7 +161,7 @@ export class PAIJobManager extends Singleton { } script = path.relative(parent, script); const jobName: string = path.basename(script, path.extname(script)); - defaultSaveDir = path.join(parent, `${jobName}.pai.json`); + defaultSaveDir = path.join(parent, `${jobName}.pai.jsonc`); config = { jobName, image: 'aiplatform/pai.build.base', @@ -181,10 +182,17 @@ export class PAIJobManager extends Singleton { } const saveDir: vscode.Uri | undefined = await vscode.window.showSaveDialog({ - defaultUri: vscode.Uri.file(defaultSaveDir) + defaultUri: vscode.Uri.file(defaultSaveDir), + filters: { + JSON: ['json', 'jsonc'] + } }); if (saveDir) { - await fs.writeJSON(saveDir.fsPath, config, { spaces: 4 }); + if (saveDir.fsPath.endsWith('.jsonc')) { + await fs.writeFile(saveDir.fsPath, await Util.generateCommentedJSON(config, 'pai_job_config.schema.json')); + } else { + await fs.writeJSON(saveDir.fsPath, config, { spaces: 4 }); + } await vscode.window.showTextDocument(saveDir); } } @@ -282,7 +290,7 @@ export class PAIJobManager extends Singleton { param.config.jobName = `${param.config.jobName}_${uuid().substring(0, 8)}`; } else { try { - await request.get(`${this.getRestUrl(param.cluster)}/user/${param.cluster.username}/jobs/${param.config.jobName}`, { + await request.get(PAIRestUri.jobDetail(param.cluster, param.cluster.username, param.config.jobName), { headers: { Authorization: `Bearer ${await this.getToken(param.cluster)}` }, timeout: PAIJobManager.TIMEOUT, json: true @@ -325,21 +333,19 @@ export class PAIJobManager extends Singleton { // send job submission request statusBarItem.text = `${OCTICON_CLOUDUPLOAD} ${__('job.request.status')}`; try { - await request.post(`${this.getRestUrl(param.cluster)}/user/${param.cluster.username}/jobs`, { + await request.post(PAIRestUri.jobs(param.cluster, param.cluster.username), { headers: { Authorization: `Bearer ${await this.getToken(param.cluster)}` }, form: param.config, timeout: PAIJobManager.TIMEOUT, json: true }); + void (await getSingleton(RecentJobManager)).enqueueRecentJobs(param.cluster, param.config.jobName); const open: string = __('job.submission.success.open'); void vscode.window.showInformationMessage( __('job.submission.success'), open ).then(async res => { - const url: string = `${getClusterWebPortalUri(param.cluster!)}/view.html?${querystring.stringify({ - username: param.cluster!.username, - jobName: param.config.jobName - })}`; + const url: string = await PAIWebPortalUri.jobDetail(param.cluster!, param.cluster!.username, param.config.jobName); if (res === open) { await Util.openExternally(url); } @@ -542,6 +548,9 @@ export class PAIJobManager extends Singleton { jobConfigPath = jobConfigUrl![0].fsPath; } const config: IPAIJobConfig = JSONC.parse(await fs.readFile(jobConfigPath, 'utf8')); + if (isNil(config)) { + Util.err('job.prepare.config.invalid'); + } const error: string | undefined = await Util.validateJSON(config, SCHEMA_JOB_CONFIG); if (error) { throw new Error(error); @@ -571,21 +580,11 @@ export class PAIJobManager extends Singleton { return result; } - private getRestUrl(cluster: IPAICluster): string { - let url: string = cluster.rest_server_uri; - if (!url.endsWith('/')) { - url += '/'; - } - url += PAIJobManager.API_PREFIX; - - return Util.fixURL(url); - } - private async getToken(cluster: IPAICluster): Promise { const id: string = getClusterIdentifier(cluster); let item: ITokenItem | undefined = this.cachedTokens.get(id); if (!item || Date.now() > item.expireTime) { - const result: any = await request.post(`${this.getRestUrl(cluster)}/token`, { + const result: any = await request.post(PAIRestUri.token(cluster), { form: { username: cluster.username, password: cluster.password, @@ -618,9 +617,15 @@ export class PAIJobManager extends Singleton { }); const fsProvider: HDFSFileSystemProvider = (await getSingleton(HDFS)).provider!; let codeDir: string = param.config.codeDir; - if (codeDir.startsWith('$PAI_DEFAULT_FS_URI')) { - codeDir = codeDir.substring('$PAI_DEFAULT_FS_URI'.length); + if (codeDir.startsWith('hdfs://') || codeDir.startsWith('webhdfs://')) { + throw new Error(__('job.upload.invalid-code-dir')); + } else { + if (codeDir.startsWith('$PAI_DEFAULT_FS_URI')) { + codeDir = codeDir.substring('$PAI_DEFAULT_FS_URI'.length); + } + codeDir = path.posix.resolve('/', codeDir); } + const codeUri: vscode.Uri = vscode.Uri.parse(`webhdfs://${getHDFSUriAuthority(param.cluster!)}${codeDir}`); const total: number = projectFiles.length; @@ -654,7 +659,7 @@ export class PAIJobManager extends Singleton { return true; } catch (e) { - Util.err('job.upload.error', [e]); + Util.err('job.upload.error', [e.message]); return false; } } diff --git a/contrib/pai_vscode/src/pai/paiUri.ts b/contrib/pai_vscode/src/pai/paiUri.ts new file mode 100644 index 0000000000..72b4ea6003 --- /dev/null +++ b/contrib/pai_vscode/src/pai/paiUri.ts @@ -0,0 +1,70 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ + +import * as querystring from 'querystring'; +import * as request from 'request-promise-native'; + +import { Util } from '../common/util'; + +import { IPAICluster } from './paiInterface'; + +export namespace PAIRestUri { + const API_PREFIX: string = 'api/v1'; + + export function getRestUrl(cluster: IPAICluster): string { + let url: string = cluster.rest_server_uri; + if (!url.endsWith('/')) { + url += '/'; + } + url += API_PREFIX; + + return Util.fixURL(url); + } + + export function token(cluster: IPAICluster): string { + return `${getRestUrl(cluster)}/token`; + } + + export function jobDetail(cluster: IPAICluster, username: string, jobName: string): string { + return `${getRestUrl(cluster)}/user/${username}/jobs/${jobName}`; + } + + export function jobs(cluster: IPAICluster, username?: string): string { + if (username) { + return `${getRestUrl(cluster)}/user/${username}/jobs`; + } + return `${getRestUrl(cluster)}/jobs`; + } +} + +export namespace PAIWebPortalUri { + export function getClusterWebPortalUri(conf: IPAICluster): string { + return conf.web_portal_uri || conf.rest_server_uri.split(':')[0]; + } + + export async function isOldJobLinkAvailable(cluster: IPAICluster): Promise { + const link: string = `${getClusterWebPortalUri(cluster)}/view.html`; + try { + await request.get(Util.fixURL(link)); + return true; + } catch { + return false; + } + } + + export async function jobDetail(cluster: IPAICluster, username: string, jobName: string): Promise { + const oldLink: boolean = await isOldJobLinkAvailable(cluster); + return `${getClusterWebPortalUri(cluster)}/${oldLink ? 'view' : 'job-detail'}.html?${querystring.stringify({ + username, + jobName + })}`; + } + + export async function jobs(cluster: IPAICluster): Promise { + const oldLink: boolean = await isOldJobLinkAvailable(cluster); + return `${getClusterWebPortalUri(cluster)}/${oldLink ? 'view' : 'job-list'}.html`; + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/pai/paiWebpages.ts b/contrib/pai_vscode/src/pai/paiWebpages.ts index 4e10233c2d..cc366cdbf7 100644 --- a/contrib/pai_vscode/src/pai/paiWebpages.ts +++ b/contrib/pai_vscode/src/pai/paiWebpages.ts @@ -8,14 +8,17 @@ import { injectable } from 'inversify'; import * as vscode from 'vscode'; import { - COMMAND_LIST_JOB, COMMAND_OPEN_DASHBOARD, COMMAND_TREEVIEW_OPEN_PORTAL + COMMAND_LIST_JOB, COMMAND_OPEN_DASHBOARD, COMMAND_TREEVIEW_OPEN_PORTAL, COMMAND_VIEW_JOB } from '../common/constants'; import { __ } from '../common/i18n'; import { getSingleton, Singleton } from '../common/singleton'; import { Util } from '../common/util'; -import { ClusterManager, getClusterName, getClusterWebPortalUri } from './clusterManager'; -import { ConfigurationNode } from './configurationTreeDataProvider'; -import { IPAICluster } from './paiInterface'; + +import { getClusterName, ClusterManager } from './clusterManager'; +import { ClusterExplorerChildNode } from './configurationTreeDataProvider'; +import { ClusterNode } from './container/jobListTreeView'; +import { IPAICluster, IPAIJobInfo } from './paiInterface'; +import { PAIWebPortalUri } from './paiUri'; const paiDashboardPropertyLabelMapping: { [propertyName: string]: string } = { grafana_uri: 'Grafana', @@ -38,11 +41,15 @@ export class PAIWebpages extends Singleton { ), vscode.commands.registerCommand( COMMAND_TREEVIEW_OPEN_PORTAL, - (node: ConfigurationNode) => this.openDashboardFromTreeView(node.index) + (node: ClusterExplorerChildNode) => this.openDashboardFromTreeView(node.index) ), vscode.commands.registerCommand( COMMAND_LIST_JOB, - (node: ConfigurationNode) => this.listJobs(node.index) + (node: ClusterExplorerChildNode | ClusterNode) => this.listJobs(node.index) + ), + vscode.commands.registerCommand( + COMMAND_VIEW_JOB, + this.viewJob.bind(this) ) ); } @@ -55,7 +62,7 @@ export class PAIWebpages extends Singleton { const config: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations![index]; const options: vscode.QuickPickItem[] = []; - const paiUrl: string = getClusterWebPortalUri(config); + const paiUrl: string = PAIWebPortalUri.getClusterWebPortalUri(config); options.push({ label: __('webpage.dashboard.webportal', [getClusterName(config)]), detail: paiUrl @@ -83,13 +90,18 @@ export class PAIWebpages extends Singleton { public async openDashboardFromTreeView(index: number): Promise { const config: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations![index]; - const url: string = getClusterWebPortalUri(config); + const url: string = PAIWebPortalUri.getClusterWebPortalUri(config); await Util.openExternally(url); } public async listJobs(index: number): Promise { const config: IPAICluster = (await getSingleton(ClusterManager)).allConfigurations![index]; - const url: string = getClusterWebPortalUri(config) + '/view.html'; + const url: string = await PAIWebPortalUri.jobs(config); + await Util.openExternally(url); + } + + public async viewJob(jobInfo: IPAIJobInfo, config: IPAICluster): Promise { + const url: string = await PAIWebPortalUri.jobDetail(config, jobInfo.username, jobInfo.name); await Util.openExternally(url); } } diff --git a/contrib/pai_vscode/src/pai/recentJobManager.ts b/contrib/pai_vscode/src/pai/recentJobManager.ts new file mode 100644 index 0000000000..502f0e1b33 --- /dev/null +++ b/contrib/pai_vscode/src/pai/recentJobManager.ts @@ -0,0 +1,84 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License in the project root for license information. + * @author Microsoft + */ + +import { injectable } from 'inversify'; +import * as vscode from 'vscode'; + +import { + SETTING_JOB_JOBLIST_RECENTJOBSLENGTH, SETTING_SECTION_JOB +} from '../common/constants'; +import { __ } from '../common/i18n'; +import { getSingleton, Singleton } from '../common/singleton'; + +import { IPAICluster } from './paiInterface'; + +import { ClusterManager } from './clusterManager'; +import { JobListTreeDataProvider } from './container/jobListTreeView'; + +/** + * Manager class for cluster configurations + */ +@injectable() +export class RecentJobManager extends Singleton { + private static readonly RECENT_JOBS_KEY: string = 'openpai.recentJobs'; + + private onClusterChangeDisposable: vscode.Disposable | undefined; + private recentJobs: (string[] | undefined)[] | undefined; + + public async onActivate(): Promise { + this.onClusterChangeDisposable = (await getSingleton(ClusterManager)).onDidChange(modification => { + switch (modification.type) { + case 'EDIT': + this.allRecentJobs[modification.index] = []; + break; + case 'REMOVE': + this.allRecentJobs.splice(modification.index, 1); + break; + case 'RESET': + this.recentJobs = []; + break; + default: + } + void this.saveRecentJobs(); + }); + this.recentJobs = this.context.globalState.get(RecentJobManager.RECENT_JOBS_KEY) || []; + } + + public get allRecentJobs(): (string[] | undefined)[] { + return this.recentJobs!; + } + + public async saveRecentJobs(index: number = -1): Promise { + await this.context.globalState.update(RecentJobManager.RECENT_JOBS_KEY, this.allRecentJobs); + const provider: JobListTreeDataProvider = await getSingleton(JobListTreeDataProvider); + await provider.refresh(index); + if (index !== -1) { + const latestJobName: string | undefined = (this.allRecentJobs[index] || [])[0]; + if (latestJobName) { + await provider.revealLatestJob(index, latestJobName); + } + } + } + + public async enqueueRecentJobs(cluster: IPAICluster, jobName: string): Promise { + const index: number = (await getSingleton(ClusterManager)).allConfigurations.findIndex(c => c === cluster); + if (index === -1) { + return; + } + const list: string[] = this.allRecentJobs[index] = this.allRecentJobs[index] || []; + list.unshift(jobName); + const settings: vscode.WorkspaceConfiguration = vscode.workspace.getConfiguration(SETTING_SECTION_JOB); + const maxLen: number = settings.get(SETTING_JOB_JOBLIST_RECENTJOBSLENGTH)!; + list.splice(maxLen); // Make sure not longer than maxLen + await this.saveRecentJobs(index); + } + + public onDeactivate(): void { + if (this.onClusterChangeDisposable) { + this.onClusterChangeDisposable.dispose(); + } + } +} \ No newline at end of file diff --git a/contrib/pai_vscode/src/root.ts b/contrib/pai_vscode/src/root.ts index 954f6d76ba..c1ca6856e4 100644 --- a/contrib/pai_vscode/src/root.ts +++ b/contrib/pai_vscode/src/root.ts @@ -5,18 +5,26 @@ */ import { Singleton } from './common/singleton'; +import { TreeViewHelper } from './common/treeViewHelper'; import { UtilClass } from './common/util'; import { ClusterManager } from './pai/clusterManager'; import { ConfigurationTreeDataProvider } from './pai/configurationTreeDataProvider'; +import { HDFSTreeDataProvider } from './pai/container/hdfsTreeView'; +import { JobListTreeDataProvider } from './pai/container/jobListTreeView'; import { HDFS } from './pai/hdfs'; import { PAIJobManager } from './pai/paiJobManager'; import { PAIWebpages } from './pai/paiWebpages'; +import { RecentJobManager } from './pai/recentJobManager'; export const allSingletonClasses: { new(...arg: any[]): Singleton }[] = [ UtilClass, ClusterManager, + RecentJobManager, + TreeViewHelper, ConfigurationTreeDataProvider, PAIJobManager, PAIWebpages, - HDFS + HDFS, + HDFSTreeDataProvider, + JobListTreeDataProvider ]; \ No newline at end of file diff --git a/contrib/pai_vscode/src/test/clusterConfiguration.test.ts b/contrib/pai_vscode/src/test/clusterConfiguration.test.ts index f855c6d668..631f87a125 100644 --- a/contrib/pai_vscode/src/test/clusterConfiguration.test.ts +++ b/contrib/pai_vscode/src/test/clusterConfiguration.test.ts @@ -5,6 +5,7 @@ */ // tslint:disable:align import * as assert from 'assert'; + import { getSingleton } from '../common/singleton'; import { ClusterManager } from '../pai/clusterManager'; diff --git a/contrib/pai_vscode/tslint.json b/contrib/pai_vscode/tslint.json index 690b0fc847..1ee738e7f3 100644 --- a/contrib/pai_vscode/tslint.json +++ b/contrib/pai_vscode/tslint.json @@ -141,7 +141,7 @@ "no-inferrable-types": false, "no-multiline-string": true, "no-null-keyword": false, - "no-parameter-properties": false, + "no-parameter-properties": true, "no-relative-imports": false, "no-require-imports": true, "no-shadowed-variable": true, @@ -158,7 +158,13 @@ "object-literal-sort-keys": false, "one-variable-per-declaration": true, "only-arrow-functions": false, - "ordered-imports": true, + "ordered-imports": [ + true, + { + "named-imports-order": "lowercase-first", + "grouped-imports": true + } + ], "prefer-array-literal": false, "prefer-const": true, "prefer-for-of": true, @@ -201,7 +207,8 @@ "import-spacing": true, "indent": [ true, - "spaces" + "spaces", + 4 ], "linebreak-style": true, "newline-before-return": false,