From 0e077a85652bb7d6b1e9d8df6d21e9ff28ea8e53 Mon Sep 17 00:00:00 2001 From: Qixiang Cheng Date: Tue, 19 Mar 2019 22:15:36 +0800 Subject: [PATCH 1/8] Add simple NFS job --- .travis.yml | 6 + contrib/simple-nfs-job/.editorconfig | 14 + contrib/simple-nfs-job/.gitignore | 64 + contrib/simple-nfs-job/README.md | 26 + contrib/simple-nfs-job/package.json | 34 + contrib/simple-nfs-job/src/App/Context.ts | 3 + contrib/simple-nfs-job/src/App/Job.ts | 6 + .../src/App/MountDirectories.tsx | 219 + .../simple-nfs-job/src/App/SimpleNFSJob.tsx | 125 + .../src/App/TensorflowDistributedJob.tsx | 262 ++ .../src/App/TensorflowSingleNodeJob.tsx | 125 + contrib/simple-nfs-job/src/App/hooks.ts | 40 + contrib/simple-nfs-job/src/App/index.tsx | 186 + contrib/simple-nfs-job/src/App/utils.ts | 19 + contrib/simple-nfs-job/src/index.ts | 46 + contrib/simple-nfs-job/tsconfig.json | 60 + contrib/simple-nfs-job/tslint.json | 12 + contrib/simple-nfs-job/webpack.config.ts | 31 + contrib/simple-nfs-job/yarn.lock | 4140 +++++++++++++++++ 19 files changed, 5418 insertions(+) create mode 100644 contrib/simple-nfs-job/.editorconfig create mode 100644 contrib/simple-nfs-job/.gitignore create mode 100644 contrib/simple-nfs-job/README.md create mode 100644 contrib/simple-nfs-job/package.json create mode 100644 contrib/simple-nfs-job/src/App/Context.ts create mode 100644 contrib/simple-nfs-job/src/App/Job.ts create mode 100644 contrib/simple-nfs-job/src/App/MountDirectories.tsx create mode 100644 contrib/simple-nfs-job/src/App/SimpleNFSJob.tsx create mode 100644 contrib/simple-nfs-job/src/App/TensorflowDistributedJob.tsx create mode 100644 contrib/simple-nfs-job/src/App/TensorflowSingleNodeJob.tsx create mode 100644 contrib/simple-nfs-job/src/App/hooks.ts create mode 100644 contrib/simple-nfs-job/src/App/index.tsx create mode 100644 contrib/simple-nfs-job/src/App/utils.ts create mode 100644 contrib/simple-nfs-job/src/index.ts create mode 100644 contrib/simple-nfs-job/tsconfig.json create mode 100644 contrib/simple-nfs-job/tslint.json create mode 100644 contrib/simple-nfs-job/webpack.config.ts create mode 100644 contrib/simple-nfs-job/yarn.lock diff --git a/.travis.yml b/.travis.yml index d5619997cf..a51d21e8c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -131,3 +131,9 @@ matrix: before_install: cd contrib/submit-simple-job install: npm install script: npm test + + - language: node_js + node_js: node + before_install: cd contrib/simple-nfs-job + install: yarn --frozen-lockfiles + script: yarn build diff --git a/contrib/simple-nfs-job/.editorconfig b/contrib/simple-nfs-job/.editorconfig new file mode 100644 index 0000000000..68e74812a3 --- /dev/null +++ b/contrib/simple-nfs-job/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +max_line_length = 80 + +[*.md] +indent_size = 4 +trim_trailing_whitespace = false diff --git a/contrib/simple-nfs-job/.gitignore b/contrib/simple-nfs-job/.gitignore new file mode 100644 index 0000000000..cc0da56291 --- /dev/null +++ b/contrib/simple-nfs-job/.gitignore @@ -0,0 +1,64 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# TypeScript v1 declaration files +typings/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# next.js build output +.next + +# distributed files +/dist diff --git a/contrib/simple-nfs-job/README.md b/contrib/simple-nfs-job/README.md new file mode 100644 index 0000000000..8d4745c2f1 --- /dev/null +++ b/contrib/simple-nfs-job/README.md @@ -0,0 +1,26 @@ +# DLWorkspace Plugin + +PAI web portal plugin for submit DLWorkspace jobs. + +## Getting Started + +TODO: Guide users through getting your code up and running on their own system. In this section you can talk about: + +1. Installation process +2. Software dependencies +3. Latest releases +4. API references + +## Build and Test + +TODO: Describe and show how to build your code and run the tests. + +## Contribute + +TODO: Explain how other users and developers can contribute to make your code better. + +If you want to learn more about creating good readme files then refer the following [guidelines](https://www.visualstudio.com/en-us/docs/git/create-a-readme). You can also seek inspiration from the below readme files: + +- [ASP.NET Core](https://github.com/aspnet/Home) +- [Visual Studio Code](https://github.com/Microsoft/vscode) +- [Chakra Core](https://github.com/Microsoft/ChakraCore) diff --git a/contrib/simple-nfs-job/package.json b/contrib/simple-nfs-job/package.json new file mode 100644 index 0000000000..1da181b0dc --- /dev/null +++ b/contrib/simple-nfs-job/package.json @@ -0,0 +1,34 @@ +{ + "name": "dlworkspace-plugin", + "version": "1.0.0", + "description": "PAI web portal plugin for submit DLWorkspace jobs.", + "main": "src/index.ts", + "scripts": { + "start": "webpack-dev-server --mode=development", + "prebuild": "tslint --project .", + "build": "webpack --mode=production" + }, + "repository": "openpai@vs-ssh.visualstudio.com:v3/openpai/OpenPAI/DLWorkspace-Plugin", + "author": "Microsoft (https://microsoft.com/)", + "license": "UNLICENSED", + "private": true, + "dependencies": { + "react": "^16.8.4", + "react-dom": "^16.8.4", + "whatwg-fetch": "^3.0.0" + }, + "devDependencies": { + "@types/react": "^16.8.7", + "@types/react-dom": "^16.8.2", + "@types/webpack": "^4.4.25", + "@types/webpack-dev-server": "^3.1.4", + "ts-loader": "^5.3.3", + "ts-node": "^8.0.3", + "tslint": "^5.13.1", + "tslint-react": "^3.6.0", + "typescript": "^3.3.3333", + "webpack": "^4.29.6", + "webpack-cli": "^3.2.3", + "webpack-dev-server": "^3.2.1" + } +} diff --git a/contrib/simple-nfs-job/src/App/Context.ts b/contrib/simple-nfs-job/src/App/Context.ts new file mode 100644 index 0000000000..de630ce458 --- /dev/null +++ b/contrib/simple-nfs-job/src/App/Context.ts @@ -0,0 +1,3 @@ +import { createContext } from "react"; + +export default createContext({ user: "", api: "", token: "" }); diff --git a/contrib/simple-nfs-job/src/App/Job.ts b/contrib/simple-nfs-job/src/App/Job.ts new file mode 100644 index 0000000000..a9c098bf49 --- /dev/null +++ b/contrib/simple-nfs-job/src/App/Job.ts @@ -0,0 +1,6 @@ +import MountDirectories from "./MountDirectories"; + +export default abstract class Job { + public readonly mountDirectories: MountDirectories | null = null; + public abstract convert(): any; +} diff --git a/contrib/simple-nfs-job/src/App/MountDirectories.tsx b/contrib/simple-nfs-job/src/App/MountDirectories.tsx new file mode 100644 index 0000000000..54966e750a --- /dev/null +++ b/contrib/simple-nfs-job/src/App/MountDirectories.tsx @@ -0,0 +1,219 @@ +import React, { useContext, useEffect, useMemo } from "react"; + +import { join } from "path"; + +import Context from "./Context"; +import { usePromise, useValue } from "./hooks"; +import { getProp } from "./utils"; + +export interface IMountDirectoriesObject { + readonly workPath: string; + readonly dataPath: string; + readonly jobPath: string; +} + +export default class MountDirectories { + constructor( + private readonly nfsRoot: string, + private readonly user: string, + private readonly jobName: string, + + private readonly workPath: string, + private readonly dataPath: string, + private readonly jobPath: string, + ) {} + + private get normalizedWorkPath() { + return join("/", this.workPath, "/"); + } + private get normalizedDataPath() { + return join("/", this.dataPath, "/"); + } + private get normalizedJobPath() { + return join("/", this.jobPath, "/"); + } + + public get nfsUserRoot() { + return join(this.nfsRoot, "users", this.user, "/"); + } + public get nfsDataRoot() { + return join(this.nfsRoot, "data", "/"); + } + + public getPaiCommand() { + return [ + // Install NFS + "apt-get update", + "apt-get install --assume-yes nfs-common", + + // Make local directories. + "mkdir --parents /work /data /job /mnt/nfs", + + // Make remove (NFS) work & job directories + `mount -t nfs4 ${this.nfsUserRoot} /mnt/nfs`, + `mkdir --parents ${join("/mnt/nfs", this.normalizedWorkPath)}`, + `mkdir --parents ${join("/mnt/nfs", this.normalizedJobPath, this.jobName)}`, + "umount /mnt/nfs", + + // Mount all directories + `mount -t nfs4 ${join(this.nfsUserRoot, this.normalizedWorkPath)} /work`, + `mount -t nfs4 ${join(this.nfsDataRoot, this.normalizedDataPath)} /data`, + `mount -t nfs4 ${join(this.nfsUserRoot, this.normalizedJobPath, this.jobName)} /job`, + ].join(" && "); + } + + public applyJSON({ workPath, dataPath, jobPath }: IMountDirectoriesObject) { + Object.assign(this, { workPath, dataPath, jobPath }); + } + + public toJSON(): IMountDirectoriesObject { + const { workPath, dataPath, jobPath } = this; + return { workPath, dataPath, jobPath } ; + } +} + +interface IProps { + jobName: string; + defaultValue: IMountDirectoriesObject | null; + onChange(mountDirectories: MountDirectories): void; +} + +export function MountDirectoriesForm({ + jobName, + defaultValue, + onChange, +}: IProps) { + const [workPath, onWorkPathChanged] = useValue(getProp(defaultValue, "workPath", "work")); + const [dataPath, onDataPathChanged] = useValue(getProp(defaultValue, "dataPath", "")); + const [jobPath, onJobPathChanged] = useValue(getProp(defaultValue, "jobPath", "jobs")); + + const { api, user } = useContext(Context); + + const [nfsRoot, nfsRootError] = usePromise(() => { + const responseToData = (response: Response) => { + if (response.ok) { + return response.json().then((responseData) => responseData.data); + } else { + throw Error(`HTTP ${response.status}`); + } + }; + + const storageExternalUrl = `${api}/api/v1/kubernetes/api/v1/namespaces/default/configmaps/storage-external`; + const storageUserUrl = `${api}/api/v1/kubernetes/api/v1/namespaces/default/configmaps/storage-user`; + return Promise.all([ + fetch(storageExternalUrl).then(responseToData), + fetch(storageUserUrl).then(responseToData), + ]).then(([storageExternalData, storageUserData]) => { + let storageKey = "default.json"; + try { + const content = storageUserData[`${user}.json`]; + const { defaultStorage } = JSON.parse(content); + if (typeof defaultStorage === "string" && defaultStorage.length > 0) { + storageKey = defaultStorage; + } + } catch (e) { + // ignored + } + + const storageContent = storageExternalData[storageKey]; + const { type, address, rootPath } = JSON.parse(storageContent); + if (type !== "nfs") { + throw Error(`Unknown storage type ${type}`); + } + + return `${address}:${rootPath}`; + }); + }, []); + + const mountDirectories = useMemo(() => { + if (nfsRoot === undefined) { + return null; + } + return new MountDirectories(nfsRoot, user, jobName, workPath, dataPath, jobPath); + }, [nfsRoot, user, jobName, workPath, dataPath, jobPath]); + + useEffect(() => { + if (mountDirectories !== null) { + onChange(mountDirectories); + } + }, [mountDirectories]); + + if (nfsRootError !== undefined) { + const link = "https://github.com/Microsoft/pai/wiki/Simplified-Job-Submission-for-OpenPAI-with-NFS-deployment"; + return ( +
+ NFS had not yet configured, please contact your IT Admin to set up the NFS.
+ Refer to the wiki for more information:
+ {link} +
+ ); + } + + if (mountDirectories === null) { return null; } + + return ( +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ Path inside containerPath on host machine (or storage server)
/work +
+ {mountDirectories.nfsUserRoot} + +
+
/data +
+ {mountDirectories.nfsDataRoot} + +
+
/job +
+ {mountDirectories.nfsUserRoot} + + {join("/", jobName)} +
+
+
+ ); +} diff --git a/contrib/simple-nfs-job/src/App/SimpleNFSJob.tsx b/contrib/simple-nfs-job/src/App/SimpleNFSJob.tsx new file mode 100644 index 0000000000..6323742414 --- /dev/null +++ b/contrib/simple-nfs-job/src/App/SimpleNFSJob.tsx @@ -0,0 +1,125 @@ +import React, { useEffect, useState } from "react"; + +import { useNumericValue, useValue } from "./hooks"; +import Job from "./Job"; +import MountDirectories, { IMountDirectoriesObject, MountDirectoriesForm } from "./MountDirectories"; +import { getProp } from "./utils"; + +const CPU_PER_GPU = 5; +const MEMORY_PER_GPU = 50 * 1024; + +interface ISimpleNFSJobObject { + readonly type: "simple-nfs"; + readonly gpuNumber: number; + readonly command: string; + readonly mountDirectories: IMountDirectoriesObject | null; +} + +export default class SimpleNFSJob extends Job { + public constructor( + private readonly name: string, + private readonly gpuNumber: number, + private readonly command: string, + public readonly mountDirectories: MountDirectories | null, + ) { + super(); + } + + public convert() { + const paiTaskRole = Object.create(null); + paiTaskRole.name = "master"; + paiTaskRole.taskNumber = 1; + paiTaskRole.cpuNumber = this.gpuNumber * CPU_PER_GPU; + paiTaskRole.memoryMB = this.gpuNumber * MEMORY_PER_GPU; + paiTaskRole.gpuNumber = this.gpuNumber; + paiTaskRole.command = this.getPaiCommand(); + + const paiJob = Object.create(null); + paiJob.jobName = this.name; + paiJob.image = "openpai/pai.build.base:hadoop2.7.2-cuda9.0-cudnn7-devel-ubuntu16.04"; + paiJob.virtualCluster = "default"; + paiJob.taskRoles = [paiTaskRole]; + + return paiJob; + } + + public toJSON(): ISimpleNFSJobObject { + const { gpuNumber, command, mountDirectories } = this; + return { + type: "simple-nfs", + gpuNumber, + command, + mountDirectories: mountDirectories !== null ? mountDirectories.toJSON() : null, + }; + } + + private getPaiCommand() { + const commands: string[] = []; + + if (this.mountDirectories !== null) { + commands.push(this.mountDirectories.getPaiCommand()); + } + + commands.push(this.command.split("\n").join(" && ")); + + return commands.join(" && "); + } +} + +interface IProps { + name: string; + defaultValue: ISimpleNFSJobObject | null; + onChange(job: SimpleNFSJob): void; +} + +export function SimpleNFSJobForm({ name, defaultValue, onChange }: IProps) { + const [gpuNumber, onGpuNumberChanged] = useNumericValue(getProp(defaultValue, "gpuNumber", 1)); + const [command, onCommandChanged] = useValue(getProp(defaultValue, "command", "echo \"Hello OpenPAI!\"")); + const [mountDirectories, setMountDirectories] = useState(null); + + useEffect(() => { + onChange(new SimpleNFSJob(name, gpuNumber, command, mountDirectories)); + }, [name, gpuNumber, command, mountDirectories]); + + return ( + <> +
+ +
+
+ +
+
+
+
+
+ +