Merge branch 'test' into 'prod'

Test

See merge request !91

src/app/jobs.service.ts

@@ -4,9 +4,9 @@ import { ComputesitesService } from './computesites.service';
 import { AuthorisationService } from './authorisation.service';
 import { Job } from './job';
 import { Identity } from './identity';
-import { Subscription, interval, pipe, Observable } from 'rxjs';
+import { Subscription, interval, pipe, Observable, throwError, of } from 'rxjs';
 import { BehaviorSubject, timer } from 'rxjs';
-import { repeat } from 'rxjs/operators';
+import { repeatWhen, delay, tap, catchError } from 'rxjs/operators';
 import { NotificationsService } from './notifications.service';
 
 
@@ -24,45 +24,61 @@ export class JobsService {
 
   public setId(id: Identity) {
     this.id = id;
-    if (this.tsub !== undefined) {
-      this.tsub.unsubscribe()
-    }
-    this.tsub = timer(5000).pipe(repeat()).subscribe(() => this.refreshJobs())
+  //  if (this.tsub !== undefined) {
+  //   this.tsub.unsubscribe()
+  //  }
+    //this.tsub = timer(5000).pipe(repeat()).subscribe(() => this.refreshJobs())
+    this.refreshJobs()
   }
 
   public refreshJobs() {
     if (this.id !== undefined && this.id !== null) {
       var query$: Observable<Job[]>;
-      console.log('refreshJobs');
       query$ = this.tes.runCommand(this.id, this.id.site.statcmd)
-      query$.subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs),
+      //query$.subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs),
+      //                 (error) => this.getJobsError(error,this.id))
+      query$.pipe(
+          repeatWhen(x => x.pipe(delay(5000))),
+          catchError((err) => this.getJobsErrorHandler(err, this.id.site.statcmd, this.id)),
+        ).subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs),
                        (error) => this.getJobsError(error,this.id))
     }
 
   }
 
-  public getJobsError(error,identity: Identity) {
-    if (error.hasOwnProperty('status') && error.status == 401) {
-      this.notifications.notify("Your login appears to have expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } );
-      return;
+  public getJobsErrorHandler(error, cmd: string, identity: Identity): Observable<any> {
+    var emsg: string = "";
+    if (error.hasOwnProperty('error') && error.error.hasOwnProperty('message') ) {
+      emsg = error.error.message;
+    } else {
+      emsg = error.message;
+    }
+    if (identity.expiry < Date.now() || (error.hasOwnProperty("status") && error.status == 401)) {
+      this.notifications.notify("Your login has expired. Please log in again",   () => { this.authService.updateAgentContents().subscribe( () => {return} ) }  );
+      return throwError(error);
+    }
+  
+    if (error.hasOwnProperty("status") && error.status == 500) {
+      this.notifications.notify("The Strudel2 API server had an error.\n Please report this via the contact us link.\nThe error message was"+emsg);
+      return of([]);
     }
-    this.tsub.unsubscribe();
-    if (identity.expiry < Date.now()) {
-      this.notifications.notify("Your login has expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } );
-      return;
+    if (error.hasOwnProperty("status") && error.status == 400) {
+      this.notifications.notify(cmd + " unexpectedly returned an error message.\n Please report this vai the contact us link.\nThe error message was\n" + emsg);
+      return throwError(error);
     }
-    console.error('getJobsError id', identity);
-    if (error.hasOwnProperty("error") && error.error.hasOwnProperty("message")) {
-      if (error.error.message.indexOf("Permission denied") != -1) {
-        this.notifications.notify("Your login appears to have expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } );
-        return;
-      } 
-      this.notifications.notify("Unable to retrieve a list of running jobs.\nThe error messge was " + error.error.message);
-      console.log(error);
-      return;
+    if (error.hasOwnProperty("status") && error.status == 504) {
+      this.notifications.notify("The Server timed out while retrieveing a list of running jobs. Is the application server OK?");
+      return of([]);
     }
+    if (error.error instanceof ErrorEvent) {
+      this.notifications.notify("A networking error occured. Is your internet connection OK?")
+      return throwError(error);
+    } 
+  }
+
+  public getJobsError(error,identity: Identity) {
     console.error(error);
-    this.notifications.notify("Unable to retrieve a list of running jobs.\nThe error wasn't specified\nPlease report what you were doign via the contact us link", () => { this.authService.updateAgentContents().subscribe((_) => {return}) });
+    this.notifications.notify("Unable to retrieve a list of running jobs.\nThis is probably an error on our end.\nPlease report what you were doing via the contact us link", () => { this.authService.updateAgentContents().subscribe((_) => {return}) });
     //this.authService.updateAgentContents().subscribe((_) => {return});
   }
 

src/app/tes.service.ts

@@ -16,6 +16,7 @@ import { BackendSelectionService } from './backend-selection.service';
 import {BrowserWindowService} from './browser-window.service';
 import { NotificationsService } from './notifications.service';
 import { APIServer } from './apiserver';
+import { throwToolbarMixedModesError } from '@angular/material/toolbar';
 
 /** The TES service contains ways to start Tunnels, and Execute programs
 Its also responsible for querying a compute site for running jobs */
@@ -104,8 +105,8 @@ public getCachetIncidents(identity: Identity) {
   let headers = new HttpHeaders();
   let options = { headers: headers, withCredentials: false};
   let params = new URLSearchParams();
-
   if (identity.site.cacheturis === undefined ||  identity.site.cacheturis.length == 0) {
+    console.log('no uris');
     identity.systemalerts.next([]);
     return
   }
@@ -243,7 +244,7 @@ private addUserHealth(identity,resp) {
    return this.http.get<string>(apiserver.tes+'/appinstance/'+username+'/'+loginhost+'/'+batchhost+'/'+jobid+'?'+paramstr, options)
  }
 
- public createTunnel(job: Job, appinst: any, action: AppAction, apiserver: APIServer) {
+ public createTunnel(job: Job, appinst: any, action: AppAction, apiserver: APIServer): Observable<string> {
    let username = job.identity.username;
    let loginhost = job.identity.site.host;
    let batchhost = job.batch_host;
@@ -252,7 +253,7 @@ private addUserHealth(identity,resp) {
    job.connectionState = 2;
 
    if ((action.notunnel != undefined) && (action.notunnel == true)) {
-     return of([])
+     return of('')
    }
    let options = { headers: headers, withCredentials: true};
    let paramstr = params.toString();
@@ -286,36 +287,82 @@ private addUserHealth(identity,resp) {
    )
  }
 
- public connect(job: Job, action: AppAction, appinst?: any) {
-   let headers = new HttpHeaders();
-   let options = { headers: headers, withCredentials: true};
+//  public connect(job: Job, action: AppAction, appinst?: any) {
+//    let headers = new HttpHeaders();
+//    let options = { headers: headers, withCredentials: true};
+
+//    job.connectionState=1
+//    // Can no connect till we know which api server we are using. Threfore the apiserver
+//    // is the start of the pipe
+//    // the API server is behaviorsubject so it should always fire immediately
+//    this.backendSelectionService.apiserver.pipe(
+//      filter((v) => v !== undefined && v !== null),
+//      switchMap((apiserver) => this.getAppInstance(job,action,apiserver),
+//        (apiserver,appinst,i1,i2) => {return [apiserver,appinst]}),
+//      tap((_) => job.connectionState=2),
+//      switchMap(([apiserver,appinst]) => { return this.createTunnel(job,appinst,action,<APIServer>apiserver)},
+//        (data,tunnelresp) => { data[1]['localport'] = (<any> tunnelresp)['localport'] ; return data}), // we don't care about data from the tunnel as long as it was successful.`
+//      tap((_) => job.connectionState=3),
+//      switchMap(([apiserver,appinst]) => this.getAppUrl(job,appinst,action,<APIServer>apiserver),
+//        ([apiserver,appinst],url) => {return [apiserver,appinst,url]}),
+//      tap((_) => job.connectionState=4),
+//    ).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
+//                                           this.openWindow$.next( {'job':job, 
+//                                                 'url': <string> url, 
+//                                                 'usebasicauth':action.client.usebasicauth,
+//                                                 'apiserver':apiserver,
+//                                             'action':action,'appinst':appinst});
+//                                       }
+//                                     job.connectionState=null},
+//      (err) => { job.connectionState = 0; this.handleError(job.identity, err)})
+//  }
 
-   job.connectionState=1
-   // Can no connect till we know which api server we are using. Threfore the apiserver
-   // is the start of the pipe
-   // the API server is behaviorsubject so it should always fire immediately
-   this.backendSelectionService.apiserver.pipe(
-     filter((v) => v !== undefined && v !== null),
-     switchMap((apiserver) => this.getAppInstance(job,action,apiserver),
-       (apiserver,appinst,i1,i2) => {return [apiserver,appinst]}),
-     tap((_) => job.connectionState=2),
-     switchMap(([apiserver,appinst]) => { return this.createTunnel(job,appinst,action,<APIServer>apiserver)},
-       (data,tunnelresp) => { data[1]['localport'] = (<any> tunnelresp)['localport'] ; return data}), // we don't care about data from the tunnel as long as it was successful.`
-     tap((_) => job.connectionState=3),
-     switchMap(([apiserver,appinst]) => this.getAppUrl(job,appinst,action,<APIServer>apiserver),
-       ([apiserver,appinst],url) => {return [apiserver,appinst,url]}),
-     tap((_) => job.connectionState=4),
-   ).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
-                                          this.openWindow$.next( {'job':job, 
-                                                'url': <string> url, 
-                                                'usebasicauth':action.client.usebasicauth,
-                                                'apiserver':apiserver,
-                                            'action':action,'appinst':appinst});
-                                      }
-                                    job.connectionState=null},
-     (err) => { job.connectionState = 0; this.handleError(job.identity, err)})
- }
 
+ public connect(job: Job, action: AppAction, appinst?: any) {
+  let headers = new HttpHeaders();
+  let options = { headers: headers, withCredentials: true};
+
+  job.connectionState=1
+  // Can no connect till we know which api server we are using. Threfore the apiserver
+  // is the start of the pipe
+  // the API server is behaviorsubject so it should always fire immediately
+  this.backendSelectionService.apiserver.pipe(
+    filter((v) => v !== undefined && v !== null),
+    switchMap((apiserver): Observable<[APIServer, string]> => 
+            { return this.getAppInstance(job,action,apiserver)
+              .pipe(
+                catchError(err => this.handleAppInstanceError(job,action,err)),
+                tap((v) => console.log('getappinstance succeeded',v)),
+                map((v)=>[apiserver,v]))
+            } ),
+    tap((_) => job.connectionState=2),
+    switchMap(([apiserver,appinst]) => 
+            { return this.createTunnel(job,appinst,action,<APIServer>apiserver)
+              .pipe(
+                tap((_) => console.log('called create tunnel')),
+                catchError(err => this.handleTunnelError(job,action,err)),
+                map((v) => 
+                { 
+                  appinst['localport'] = v['localport']; 
+                  return [apiserver, appinst];
+                }))
+            } ),
+    tap((_) => job.connectionState=3),
+    switchMap(([apiserver,appinst]) => 
+            { return this.getAppUrl(job,appinst,action,<APIServer>apiserver)
+              .pipe(map((url) => [apiserver, appinst, url]))
+            }),
+    tap((_) => job.connectionState=4),
+  ).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
+                                         this.openWindow$.next( {'job':job, 
+                                               'url': <string> url, 
+                                               'usebasicauth':action.client.usebasicauth,
+                                               'apiserver':apiserver,
+                                           'action':action,'appinst':appinst});
+                                     }
+                                   job.connectionState=null},
+    (err) => { job.connectionState = 0; this.handleError(job.identity, err)})
+}
 
   public runCommand(identity: Identity, command: string): Observable<any> {
   let headers = new HttpHeaders();
@@ -360,27 +407,71 @@ public getSftpData(id: Identity, path: string, cd: string ) {
 private handleError(identity: Identity, error: HttpErrorResponse) {
   console.error(error);
   console.log('in handle error');
-  if (identity.expiry < Date.now()) {
+  // if (identity.expiry < Date.now()) {
+  //   this.notifications.notify("Your login has expired. Please log in again",   () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) }  );
+  //   return;
+  // }
+  // if (error.error instanceof ErrorEvent) {
+  //   this.notifications.notify("A networking error occured.")
+  //   return
+  // } 
+  // if (error.hasOwnProperty("status") && error.status == 500) {
+  //   this.notifications.notify(error);
+  //   return
+  // }
+  // if (error.hasOwnProperty("status") && error.status == 400) {
+  //   this.notifications.notify(error);
+  //   return
+  // }
+  // this.notifications.notify(error);
+}
+
+private handleAppInstanceError(job: Job, action: AppAction, error: HttpErrorResponse): Observable<any> {
+  console.error(error);
+  const msg: string = "The command " + action.paramscmd + " on host " + job.batch_host + " "
+  return this.handleGenericError(job,action,error,msg);
+}
+
+private handleTunnelError(job: Job, action: AppAction, error: HttpErrorResponse): Observable<any> {
+  console.error(error);
+  const msg: string = "Attempting to create an SSH tunnel to" + job.batch_host + " via " + job.identity.site.host;
+  return this.handleGenericError(job,action,error,msg);
+}
+
+private handleGenericError(job: Job, action: AppAction, error: HttpErrorResponse, msg: string): Observable<any> {
+  console.log('in handleGenericError');
+  var emsg: string = "";
+  if (error.hasOwnProperty('error') && error.error.hasOwnProperty('message') ) {
+    emsg = error.error.message;
+  } else {
+    emsg = error.message;
+  }
+  if (job.identity.expiry < Date.now() || (error.hasOwnProperty("status") && error.status == 401)) {
     this.notifications.notify("Your login has expired. Please log in again",   () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) }  );
-    return;
+    return throwError(error);
   }
-  if (error.error instanceof ErrorEvent) {
-    this.notifications.notify("A networking error occured.")
-    return
-  } 
+
   if (error.hasOwnProperty("status") && error.status == 500) {
-    this.notifications.notify(error);
-    return
+    this.notifications.notify("The Strudel2 API server had an error.\n Please report this via the contact us link.\nThe error message was"+emsg);
+    return throwError(error);
   }
   if (error.hasOwnProperty("status") && error.status == 400) {
-    this.notifications.notify(error);
-    return
+    this.notifications.notify(msg + " unexpectedly returned an error message.\n Please report this vai the contact us link.\nThe error message was\n" + emsg);
+    return throwError(error);
+  }
+  if (error.hasOwnProperty("status") && error.status == 504) {
+    this.notifications.notify(msg + " timed out. Is the application server OK?");
+    return throwError(error);
   }
+  if (error.error instanceof ErrorEvent) {
+    this.notifications.notify("A networking error occured. Is your internet connection OK?")
+    return throwError(error);
+  } 
+  console.log('performing raw notification');
   this.notifications.notify(error);
+  return throwError(error);
 }
 
 
 
-
-
 }

src/assets/config/computesites.dev.json

@@ -2,15 +2,15 @@
   {
     "url": "https://strudel2-api-dev.cloud.cvl.org.au/m3/",
     "name": "M3",
-    "host": "m3-login2.massive.org.au",
+    "host": "m3.massive.org.au",
     "dtn": "m3-dtn1.massive.org.au",
     "cafingerprint": "ECDSA SHA256:6wVXdokvvlTNcXPMc9KyvIXA8a8XNfLuhBfNOYeeMdg",
     "appCatalog": [],
     "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps-dev",
     "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
     "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
-    "userhealth": "/usr/local/bin/uijson",
-    "cacheturis": [],
+    "userhealth": "/home/chines/strudel2_desktop_user_info.py",
+    "cacheturis": ["/assets/config/incidents.json"],
     "contact": "<MASSIVE Support> help@massive.org.au"
   },
   {

src/assets/config/computesites.json

@@ -9,8 +9,8 @@
     "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps",
     "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
     "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
-    "userhealth": "/usr/local/bin/uijson",
-    "cacheturis": [],
+    "userhealth": "/home/chines/clusterinfo_venv/bin/show_cluster --jsondesktopsummary",
+    "cacheturis": ["/assets/config/incidents.json"],
     "contact": "<MASSIVE Support> help@massive.org.au"
   }
 ]

src/assets/config/computesites.prod.json

@@ -9,6 +9,7 @@
     "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps",
     "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
     "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
+    "cacheturis": ["/assets/config/incidents.json"],
     "userhealth": "/usr/local/bin/uijson"
   }
 ]

src/assets/config/computesites.test.json

@@ -9,6 +9,7 @@
     "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps",
     "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
     "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
+    "cacheturis": ["/assets/config/incidents.json"],
     "userhealth": "/usr/local/bin/uijson"
   },
   {