Add simple NFS job

.travis.yml

@@ -131,3 +131,9 @@ matrix:
       before_install: cd contrib/submit-simple-job
       install: npm install
       script: npm test
+
+    - language: node_js
+      node_js: node
+      before_install: cd contrib/simple-nfs-job
+      install: yarn --frozen-lockfiles
+      script: yarn build

contrib/simple-nfs-job/.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+max_line_length = 80
+
+[*.md]
+indent_size = 4
+trim_trailing_whitespace = false

contrib/simple-nfs-job/.gitignore

@@ -0,0 +1,64 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# next.js build output
+.next
+
+# distributed files
+/dist

contrib/simple-nfs-job/README.md

@@ -0,0 +1,26 @@
+# DLWorkspace Plugin
+
+PAI web portal plugin for submit DLWorkspace jobs.
+
+## Getting Started
+
+TODO: Guide users through getting your code up and running on their own system. In this section you can talk about:
+
+1. Installation process
+2. Software dependencies
+3. Latest releases
+4. API references
+
+## Build and Test
+
+TODO: Describe and show how to build your code and run the tests. 
+
+## Contribute
+
+TODO: Explain how other users and developers can contribute to make your code better. 
+
+If you want to learn more about creating good readme files then refer the following [guidelines](https://www.visualstudio.com/en-us/docs/git/create-a-readme). You can also seek inspiration from the below readme files:
+
+- [ASP.NET Core](https://github.com/aspnet/Home)
+- [Visual Studio Code](https://github.com/Microsoft/vscode)
+- [Chakra Core](https://github.com/Microsoft/ChakraCore)

contrib/simple-nfs-job/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "dlworkspace-plugin",
+  "version": "1.0.0",
+  "description": "PAI web portal plugin for submit DLWorkspace jobs.",
+  "main": "src/index.ts",
+  "scripts": {
+    "start": "webpack-dev-server --mode=development",
+    "prebuild": "tslint --project .",
+    "build": "webpack --mode=production"
+  },
+  "repository": "[email protected]:v3/openpai/OpenPAI/DLWorkspace-Plugin",
+  "author": "Microsoft (https://microsoft.com/)",
+  "license": "UNLICENSED",
+  "private": true,
+  "dependencies": {
+    "react": "^16.8.4",
+    "react-dom": "^16.8.4",
+    "whatwg-fetch": "^3.0.0"
+  },
+  "devDependencies": {
+    "@types/react": "^16.8.7",
+    "@types/react-dom": "^16.8.2",
+    "@types/webpack": "^4.4.25",
+    "@types/webpack-dev-server": "^3.1.4",
+    "ts-loader": "^5.3.3",
+    "ts-node": "^8.0.3",
+    "tslint": "^5.13.1",
+    "tslint-react": "^3.6.0",
+    "typescript": "^3.3.3333",
+    "webpack": "^4.29.6",
+    "webpack-cli": "^3.2.3",
+    "webpack-dev-server": "^3.2.1"
+  }
+}

contrib/simple-nfs-job/src/App/Context.ts

@@ -0,0 +1,3 @@
+import { createContext } from "react";
+
+export default createContext({ user: "", api: "", token: "" });

contrib/simple-nfs-job/src/App/Job.ts

@@ -0,0 +1,6 @@
+import MountDirectories from "./MountDirectories";
+
+export default abstract class Job {
+  public readonly mountDirectories: MountDirectories | null = null;
+  public abstract convert(): any;
+}

contrib/simple-nfs-job/src/App/MountDirectories.tsx

@@ -0,0 +1,219 @@
+import React, { useContext, useEffect, useMemo } from "react";
+
+import { join } from "path";
+
+import Context from "./Context";
+import { usePromise, useValue } from "./hooks";
+import { getProp } from "./utils";
+
+export interface IMountDirectoriesObject {
+  readonly workPath: string;
+  readonly dataPath: string;
+  readonly jobPath: string;
+}
+
+export default class MountDirectories {
+  constructor(
+    private readonly nfsRoot: string,
+    private readonly user: string,
+    private readonly jobName: string,
+
+    private readonly workPath: string,
+    private readonly dataPath: string,
+    private readonly jobPath: string,
+  ) {}
+
+  private get normalizedWorkPath() {
+    return join("/", this.workPath, "/");
+  }
+  private get normalizedDataPath() {
+    return join("/", this.dataPath, "/");
+  }
+  private get normalizedJobPath() {
+    return join("/", this.jobPath, "/");
+  }
+
+  public get nfsUserRoot() {
+    return join(this.nfsRoot, "users", this.user, "/");
+  }
+  public get nfsDataRoot() {
+    return join(this.nfsRoot, "data", "/");
+  }
+
+  public getPaiCommand() {
+    return [
+      // Install NFS
+      "apt-get update",
+      "apt-get install --assume-yes nfs-common",
+
+      // Make local directories.
+      "mkdir --parents /work /data /job /mnt/nfs",
+
+      // Make remove (NFS) work & job directories
+      `mount -t nfs4 ${this.nfsUserRoot} /mnt/nfs`,
+      `mkdir --parents ${join("/mnt/nfs", this.normalizedWorkPath)}`,
+      `mkdir --parents ${join("/mnt/nfs", this.normalizedJobPath, this.jobName)}`,
+      "umount /mnt/nfs",
+
+      // Mount all directories
+      `mount -t nfs4 ${join(this.nfsUserRoot, this.normalizedWorkPath)} /work`,
+      `mount -t nfs4 ${join(this.nfsDataRoot, this.normalizedDataPath)} /data`,
+      `mount -t nfs4 ${join(this.nfsUserRoot, this.normalizedJobPath, this.jobName)} /job`,
+    ].join(" && ");
+  }
+
+  public applyJSON({ workPath, dataPath, jobPath }: IMountDirectoriesObject) {
+    Object.assign(this, { workPath, dataPath, jobPath });
+  }
+
+  public toJSON(): IMountDirectoriesObject {
+    const { workPath, dataPath, jobPath } = this;
+    return { workPath, dataPath, jobPath } ;
+  }
+}
+
+interface IProps {
+  jobName: string;
+  defaultValue: IMountDirectoriesObject | null;
+  onChange(mountDirectories: MountDirectories): void;
+}
+
+export function MountDirectoriesForm({
+  jobName,
+  defaultValue,
+  onChange,
+}: IProps) {
+  const [workPath, onWorkPathChanged] = useValue(getProp(defaultValue, "workPath", "work"));
+  const [dataPath, onDataPathChanged] = useValue(getProp(defaultValue, "dataPath", ""));
+  const [jobPath, onJobPathChanged] = useValue(getProp(defaultValue, "jobPath", "jobs"));
+
+  const { api, user } = useContext(Context);
+
+  const [nfsRoot, nfsRootError] = usePromise<string>(() => {
+    const responseToData = (response: Response) => {
+      if (response.ok) {
+        return response.json().then((responseData) => responseData.data);
+      } else {
+        throw Error(`HTTP ${response.status}`);
+      }
+    };
+
+    const storageExternalUrl = `${api}/api/v1/kubernetes/api/v1/namespaces/default/configmaps/storage-external`;
+    const storageUserUrl = `${api}/api/v1/kubernetes/api/v1/namespaces/default/configmaps/storage-user`;
+    return Promise.all([
+      fetch(storageExternalUrl).then(responseToData),
+      fetch(storageUserUrl).then(responseToData),
+    ]).then(([storageExternalData, storageUserData]) => {
+      let storageKey = "default.json";
+      try {
+        const content = storageUserData[`${user}.json`];
+        const { defaultStorage } = JSON.parse(content);
+        if (typeof defaultStorage === "string" && defaultStorage.length > 0) {
+          storageKey = defaultStorage;
+        }
+      } catch (e) {
+        // ignored
+      }
+
+      const storageContent = storageExternalData[storageKey];
+      const { type, address, rootPath } = JSON.parse(storageContent);
+      if (type !== "nfs") {
+        throw Error(`Unknown storage type ${type}`);
+      }
+
+      return `${address}:${rootPath}`;
+    });
+  }, []);
+
+  const mountDirectories = useMemo(() => {
+    if (nfsRoot === undefined) {
+      return null;
+    }
+    return new MountDirectories(nfsRoot, user, jobName, workPath, dataPath, jobPath);
+  }, [nfsRoot, user, jobName, workPath, dataPath, jobPath]);
+
+  useEffect(() => {
+    if (mountDirectories !== null) {
+      onChange(mountDirectories);
+    }
+  }, [mountDirectories]);
+
+  if (nfsRootError !== undefined) {
+    const link = "https://github.com/Microsoft/pai/wiki/Simplified-Job-Submission-for-OpenPAI-with-NFS-deployment";
+    return (
+      <div className="alert alert-warning" role="alert">
+        NFS had not yet configured, please contact your IT Admin to set up the NFS.<br/>
+        Refer to the wiki for more information:<br/>
+        <a href={link} target="_blank">{link}</a>
+      </div>
+    );
+  }
+
+  if (mountDirectories === null) { return null; }
+
+  return (
+    <div className="form-group">
+      <label>Mount Directories</label>
+      <table className="table">
+        <thead>
+          <tr>
+            <th/>
+            <th>Path inside container</th>
+            <th>Path on host machine (or storage server)</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr>
+            <th><label htmlFor="work-path">Work Path</label></th>
+            <td><code>/work</code></td>
+            <td>
+              <div className="input-group">
+                <span className="input-group-addon">{mountDirectories.nfsUserRoot}</span>
+                <input
+                  type="text"
+                  className="form-control"
+                  id="work-path"
+                  value={workPath}
+                  onChange={onWorkPathChanged}
+                />
+              </div>
+            </td>
+          </tr>
+          <tr>
+            <th><label htmlFor="data-path">Data Path</label></th>
+            <td><code>/data</code></td>
+            <td>
+              <div className="input-group">
+                <span className="input-group-addon">{mountDirectories.nfsDataRoot}</span>
+                <input
+                  type="text"
+                  className="form-control"
+                  id="data-path"
+                  value={dataPath}
+                  onChange={onDataPathChanged}
+                />
+              </div>
+            </td>
+          </tr>
+          <tr>
+            <th><label htmlFor="job-path">Job Path</label></th>
+            <td><code>/job</code></td>
+            <td>
+              <div className="input-group">
+                <span className="input-group-addon">{mountDirectories.nfsUserRoot}</span>
+                <input
+                  type="text"
+                  className="form-control"
+                  id="job-path"
+                  value={jobPath}
+                  onChange={onJobPathChanged}
+                />
+                <span className="input-group-addon">{join("/", jobName)}</span>
+              </div>
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+  );
+}