2.4 release! (#75)

* add uat base urls and buckets

* config-based urls: maapsec

* dps_magic readme typo

* config based urls: submit_jobs

* config based urls: dps_info

* config based urls: pull_projects

* revert timeout setting change

* config based urls: ssh_info

* add default_host field for explicitness

* remove hard coded url from dps_info client

* clean up hard coded url references

* remove unnecessary state param

* Update dps_magic to use maap_environments.json

* send presigned s3 url req to maap api

* fix ci build error

* Update entrypoint.sh

* Replace statedb environment with server lookup

* replace env statedb with server lookup

* set ade_server in client once during startup

* fix keycloak callback error

* submit jobs convert to tabs to spaces

* fix submit jobs getMetrics result key

* fix dps_info ref to state

* Use maap-py for MAAP API (#73)

* use maap-py for list jobs
* use maap-py for desribe process
* use maap-py for get job result
* use maap-py for get job metrics
* use maap-py for get job status
* use maap-py for get execute inputs
* use maap-py for delete job
* use maap-py for dismiss job
* use maap-py for get capabilities
* use maap-py for publish algorithm
* use maap-py for delete algorithm
* use maap-py for execute job
* fix mismatching keys in submit_jobs to maap-py
* use maap-py for register algorithm

* move edsc url to configuration file

* Registration updates (#74)

* fix mismatching keys in submit_jobs to maap-py
* use maap-py for register algorithm
* pass username to defaultvalues helper
* prepend "demo-${username}-" to tutorial hello world algorithm name
* use regex to match hello-world repo under any group or user
* add dynamic queue selection for algo registration
* properly re-populate algo config with memory selection and add instructions for disk size

* make maap-users org folder read-only

* Fix IStateDB bug

* update hello-world url regex to allow git token in url

* fix org bucket mount line

* update ops search port to 30052

Co-authored-by: bsatoriu <[email protected]>
Co-authored-by: echyam <[email protected]>
Co-authored-by: Elizabeth Yam <[email protected]>

dps_info/src/funcs.ts

@@ -4,12 +4,23 @@ import { INotification } from 'jupyterlab_toastify';
 import { getUserInfo } from "./getKeycloak";
 import { request, RequestResult } from './request';
 
-export function getUsernameToken(state: IStateDB, profileId:string) {
+let ade_server = '';
+var valuesUrl = new URL(PageConfig.getBaseUrl() + 'maapsec/environment');
+
+request('get', valuesUrl.href).then((res: RequestResult) => {
+  if (res.ok) {
+    let environment = JSON.parse(res.data);
+    ade_server = environment['ade_server'];
+  }
+});
+
+export async function getUsernameToken(state: IStateDB, profileId:string) {
     let uname:string = 'anonymous';
     let ticket:string = '';
     let result:string[] = [uname, ticket];
+
     return new Promise<string[]> ((resolve,reject) => {
-        if (["https://che-k8s.maap.xyz","https://ade.maap-project.org"].includes(document.location.origin)) {
+        if ("https://" + ade_server === document.location.origin) {
             getUserInfo(function(profile: any) {
                 if (profile['cas:username'] === undefined) {
                     INotification.error("Get profile failed.");
@@ -40,73 +51,66 @@ export function getUsernameToken(state: IStateDB, profileId:string) {
     });
   }
 
-export async function DPSCall(endpoint:string, keywords:string[], kwargs:{[k:string]:string}) {
+export async function DPSCall(state: IStateDB, endpoint:string, keywords:string[], kwargs:{[k:string]:string}) {
     let requestUrl = new URL(PageConfig.getBaseUrl() + 'hysds/' + endpoint);
     for (let k of keywords) {
         requestUrl.searchParams.append(k,kwargs[k]);
     }
     console.log(requestUrl.href);
-
     const res = await request('get', requestUrl.href);
     return res; 
 }
 
-export async function getAlgoList(username:string): Promise<RequestResult> {
+export async function getAlgoList(state: IStateDB, username:string): Promise<RequestResult> {
     let requestUrl = new URL(PageConfig.getBaseUrl() + 'hysds/listAlgorithms');
     requestUrl.searchParams.append('username', username);
     console.log(requestUrl.href);
-
     const res = await request('get', requestUrl.href);
     return res;
 }
 
-export async function describeAlgo(algo:string, version:string, username:string): Promise<RequestResult> {
+export async function describeAlgo(state: IStateDB, algo:string, version:string, username:string): Promise<RequestResult> {
     let requestUrl = new URL(PageConfig.getBaseUrl() + 'hysds/describeProcess');
     requestUrl.searchParams.append('algo_id', algo);
     requestUrl.searchParams.append('version', version)
     requestUrl.searchParams.append('username', username);
     console.log(requestUrl.href);
-
     const res = await request('get', requestUrl.href);
     return res;
 }
 
-export async function executeInputs(algo:string, version:string, username:string): Promise<RequestResult> {
+export async function executeInputs(state: IStateDB, algo:string, version:string, username:string): Promise<RequestResult> {
     let requestUrl = new URL(PageConfig.getBaseUrl() + 'hysds/executeInputs');
     requestUrl.searchParams.append('algo_id', algo);
     requestUrl.searchParams.append('version', version)
     requestUrl.searchParams.append('username', username);
     console.log(requestUrl.href);
-
     const res = await request('get', requestUrl.href);
     return res;
 }
 
-export async function getJobs(username:string): Promise<RequestResult> {
+export async function getJobs(state: IStateDB, username:string): Promise<RequestResult> {
     let requestUrl = new URL(PageConfig.getBaseUrl() + 'hysds/listJobs');
     requestUrl.searchParams.append('username', username);
     console.log(requestUrl.href);
-
     const res = await request('get', requestUrl.href);
     return res;
 }
 
-export async function getResults(job_id: string, username:string): Promise<RequestResult> {
+export async function getResults(state: IStateDB, job_id: string, username:string): Promise<RequestResult> {
     let requestUrl = new URL(PageConfig.getBaseUrl() + 'hysds/getResult');
     requestUrl.searchParams.append('username', username);
     requestUrl.searchParams.append('job_id', job_id);
     console.log(requestUrl.href);
-
     const res = await request('get', requestUrl.href);
     return res;
 }
 
-export async function getMetrics(job_id: string, username:string): Promise<RequestResult> {
+export async function getMetrics(state: IStateDB, job_id: string, username:string): Promise<RequestResult> {
     let requestUrl = new URL(PageConfig.getBaseUrl() + 'hysds/getMetrics');
     requestUrl.searchParams.append('username', username);
     requestUrl.searchParams.append('job_id', job_id);
     console.log(requestUrl.href);
-
     const res = await request('get', requestUrl.href);
     return res;
 }

dps_info/src/index.ts

@@ -18,9 +18,10 @@ const extensionPanel: JupyterFrontEndPlugin<void> = {
     autoStart: true,
     requires: [ICommandPalette, IStateDB],
     activate: (app: JupyterFrontEnd, palette: ICommandPalette, state: IStateDB) => {
+
         getUsernameToken(state,profileId).then((res) => {
             let username = res[0];
-            let jPanel = new JobTable(username);
+            let jPanel = new JobTable(username, state);
             jPanel.id = 'job-cache-display';
 
             jPanel.title.label = 'Jobs';
@@ -49,7 +50,7 @@ const extensionTabUI: JupyterFrontEndPlugin<void> = {
     activate: (app: JupyterFrontEnd, palette: ICommandPalette, state: IStateDB) => {
         getUsernameToken(state,profileId).then((res) => {
             let username = res[0];
-            let content = new JobWidget(username);
+            let content = new JobWidget(username, state);
             const jobsWidget: MainAreaWidget<JobWidget> = new MainAreaWidget({content});
 
             app.commands.addCommand(jWidget_command, {

dps_info/src/jobinfo.ts

@@ -4,6 +4,7 @@ import { INotification } from 'jupyterlab_toastify';
 import { RequestResult } from './request';
 import { getAlgoList, describeAlgo, executeInputs, getJobs, getResults, getMetrics, onRowClick, DPSCall } from './funcs';
 import '../style/index.css';
+import { IStateDB } from '@jupyterlab/statedb';
 
 export const WIDGET_CLASS = 'jp-Widget';
 export const CONTENT_CLASS = 'jp-Inspector-content';
@@ -19,20 +20,22 @@ export class JobTable extends Widget {
     _table: string;
     _results: string;
     _metrics: string;
+    _state: IStateDB;
 
     JOBS: {[k:string]:string};
     DISPLAYS: {[k:string]:string};
 
     _resultsTableName: string;
     _metricsTableName: string;
 
-    constructor(uname:string) {
+    constructor(uname:string, state:IStateDB) {
         super();
         this._username = uname;
         this._job_id = '';
         this._table = '';
         this._results = '';
         this._metrics = '';
+        this._state = state;
 
         this.addClass(CONTENT_CLASS);
         this._resultsTableName = 'job-result-display';
@@ -139,7 +142,7 @@ export class JobTable extends Widget {
 
     async _getJobList(me:JobTable) {
         // console.log(this._username);
-        const res:RequestResult = await getJobs(me._username);
+        const res:RequestResult = await getJobs(me._state, me._username);
         if(res.ok){
             let json_response:any = res.json();
             INotification.success("Get user jobs success.");
@@ -217,7 +220,7 @@ export class JobTable extends Widget {
             console.log('job not complete');
         } else {
             console.log('looking up job results');
-            const res:RequestResult = await getResults(this._job_id,this._username);
+            const res:RequestResult = await getResults(this._state, this._job_id,this._username);
             // console.log(res);
             if (res.ok) {
                 let json_response:any = res.json();
@@ -284,7 +287,7 @@ export class JobTable extends Widget {
             console.log('job not complete');
         } else {
             console.log('looking up job metrics');
-            const res:RequestResult = await getMetrics(this._job_id,this._username);
+            const res:RequestResult = await getMetrics(this._state, this._job_id,this._username);
             // console.log(res);
             if (res.ok) {
                 let json_response:any = res.json();
@@ -350,10 +353,12 @@ export class JobWidget extends Widget {
     _execute_params_id: string;
     _resultsTableName: string;
     _metricsTableName: string;
+    _state: IStateDB;
 
-    constructor(uname:string) {
+    constructor(uname:string, state: IStateDB) {
         super();
         this._username = uname;
+        this._state = state;
         this._algorithm = '';
         this._version = '';
         this._job_id = '';
@@ -403,6 +408,7 @@ export class JobWidget extends Widget {
     // RUN JOBS TAB ==============================================
 
     async _updateListCol(): Promise<void> {
+
         return new Promise<void>(async (resolve, reject) => {
             let listCell = document.getElementById('cell-algolist') as HTMLTableCellElement;
             if (listCell) {
@@ -454,7 +460,7 @@ export class JobWidget extends Widget {
                 }
 
                 // gets algorithm list and calls helper function
-                const res:RequestResult =  await getAlgoList(this._username);
+                const res:RequestResult =  await getAlgoList(this._state, this._username);
                 if (res.ok) {
                     let json_response:any = res.json();
                     let algo_set = json_response['algo_set'];
@@ -524,7 +530,7 @@ export class JobWidget extends Widget {
             paramdiv.appendChild(inputsp);
         }
         // get params and populate table
-        const res:RequestResult = await executeInputs(this._algorithm, this._version, this._username);
+        const res:RequestResult = await executeInputs(this._state, this._algorithm, this._version, this._username);
         if (res.ok) {
             let json_response: any = res.json();
             // format [[param1,type1],[param2,type2]]
@@ -578,7 +584,7 @@ export class JobWidget extends Widget {
                 kwargs['username'] = me._username;
 
                 // send execute request
-                const res:RequestResult = await DPSCall('execute', keywords, kwargs);
+                const res:RequestResult = await DPSCall(me._state, 'execute', keywords, kwargs);
                 if (res.ok){
                     let json_response:any = res.json();
                     p = p.concat(json_response['result']);
@@ -675,7 +681,7 @@ export class JobWidget extends Widget {
                         pre.innerText = 'No algorithm selected.';
                         resolve();
                     } else {
-                        const res: RequestResult = await describeAlgo(this._algorithm, this._version, this._username);
+                        const res: RequestResult = await describeAlgo(this._state, this._algorithm, this._version, this._username);
                         if (res.ok) {
                             let json_response:any = res.json();
                             let describe = json_response['result'];
@@ -697,7 +703,7 @@ export class JobWidget extends Widget {
                         resolve();
                     } else {
                         // populate div with describe call
-                        const res: RequestResult = await describeAlgo(this._algorithm, this._version, this._username);
+                        const res: RequestResult = await describeAlgo(this._state, this._algorithm, this._version, this._username);
                         if (res.ok) {
                             let json_response:any = res.json();
                             let describe = json_response['result'];
@@ -789,11 +795,12 @@ export class JobWidget extends Widget {
 
             let job_id = this._job_id;
             let uname = this._username;
+            let state = this._state;
 
             // Delete Job Button
             // redefine delete btn with current job_id
             let delete_fn = async function() {
-                const res:RequestResult = await DPSCall('delete',['job_id','username'],{'job_id':job_id,'username':uname});
+                const res:RequestResult = await DPSCall(state, 'delete',['job_id','username'],{'job_id':job_id,'username':uname});
                 if (res.ok) {
                     let json_response:any = res.json();
                     let result = json_response['result'];
@@ -840,7 +847,7 @@ export class JobWidget extends Widget {
             // Dismiss Job Button
             // redefine dismiss btn with current job_id
             let dismiss_fn = async function() {
-                const res:RequestResult = await DPSCall('dismiss',['job_id','username'],{'job_id':job_id,'username':uname});
+                const res:RequestResult = await DPSCall(state, 'dismiss',['job_id','username'],{'job_id':job_id,'username':uname});
                 if (res.ok) {
                     let json_response:any = res.json();
                     let result = json_response['result'];
@@ -906,7 +913,7 @@ export class JobWidget extends Widget {
             console.log('job not complete');
         } else {
             console.log('looking up job results');
-            const res:RequestResult = await getResults(this._job_id,this._username);
+            const res:RequestResult = await getResults(this._state, this._job_id,this._username);
             // console.log(res);
             if (res.ok) {
                 let json_response:any = res.json();
@@ -981,7 +988,7 @@ export class JobWidget extends Widget {
             console.log('job not complete');
         } else {
             console.log('looking up job metrics');
-            const res:RequestResult = await getMetrics(this._job_id,this._username);
+            const res:RequestResult = await getMetrics(this._state, this._job_id,this._username);
             // console.log(res);
             if (res.ok) {
                 let json_response:any = res.json();
@@ -1120,7 +1127,7 @@ export class JobWidget extends Widget {
     }
 
     async _getJobList(me:JobWidget) {
-        const res:RequestResult = await getJobs(this._username);
+        const res:RequestResult = await getJobs(this._state, this._username);
             if(res.ok){
                 let json_response:any = res.json();
                 INotification.success("Get user jobs success.");

dps_info/src/request.ts

@@ -13,13 +13,14 @@ export interface RequestOptions {
 export const DEFAULT_REQUEST_OPTIONS = {
   ignoreCache: false,
   headers: {
-    Accept: 'application/json, text/javascript, text/plain',
+    Accept: 'application/json, text/javascript, text/plain'
   },
   // default max duration for a request in ms
   // currently set to 120s = 2min
   timeout: 60000,
 };
 
+
 export interface RequestResult {
   ok: boolean;
   status: number;

dps_magic/README.md

@@ -62,7 +62,7 @@ cd maap-jupyter-ide/dps_magic
 npm install
 npm run build
 pip install .
-jupyter nbextension install --symlink --py --dps_magic --sys-prefix
+jupyter nbextension install --symlink --py dps_magic --sys-prefix
 jupyter nbextension enable --py dps_magic --sys-prefix
 ```
 

dps_magic/dps_magic/hysds.py

@@ -8,23 +8,18 @@
 logger = logging.getLogger()
 # logger.setLevel(logging.DEBUG)
 
-# set base url based on ops/dev environment
-CHE_BASE_URL = "https://che-k8s.maap.xyz"
-if 'ENVIRONMENT' in os.environ.keys() and os.environ['ENVIRONMENT'] == 'OPS':
-    CHE_BASE_URL = "https://ade.maap-project.org"
-
-
 @magics_class
 class HysdsMagic(Magics):
-    # '{workspace_id}'.format(workspace_id=workspace_id)
 
     def __init__(self):
         super(HysdsMagic, self).__init__()
-        PREVIEW_URL = os.environ['PREVIEW_URL']
-        # self.JUPYTER_SERVER_URL = CHE_BASE_URL+PREVIEW_URL
-        self.lk = CHE_BASE_URL+PREVIEW_URL
-        # self.lk = 'http://localhost:8888'
-        self.bucket = 'maap-mount-dev'
+        PREVIEW_URL = os.environ['PREVIEW_URL'] if 'PREVIEW_URL' in os.environ else ''
+
+        self.config_helper = ConfigHelper()
+        self.env = self.config_helper.get_maap_config()
+
+        self.lk = 'https://{}{}'.format(self.env['ade_server'], PREVIEW_URL) 
+        self.bucket = 'maap-{}-dataset'.format(self.env['environment']) 
 
     def html(self,txt1,txt2=None):
         if txt2 == None:
@@ -387,3 +382,25 @@ def s3_url(self, line):
         resp = json.loads(r.text)
         self.html(resp['url'])
         return
+
+class ConfigHelper(object):
+    def get_maap_config(self):
+        # Reaching into the maapsec extension here because this is a server-only extension with no reference to the statedb
+        path_to_json = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../..', 'maap_environments.json')
+
+        with open(path_to_json) as f:
+            data = json.load(f)
+
+        env = self._get_environment(data)
+
+        return env
+
+    def _get_environment(self, envs):
+        CHE_API = os.environ['CHE_API'].replace('/api', '') if 'CHE_API' in os.environ else 'NON-ADE-HOST'
+
+        match = next((x for x in envs if CHE_API in x['ade_server']), None)
+
+        if match is None:
+            return next((x for x in envs if x['default_host'] == True), None)
+        else:
+            return match