Skip to content
Snippets Groups Projects
Commit 8f7426b2 authored by Chris Hines's avatar Chris Hines
Browse files

revamp error handling to be more informative

parent d7f24f0b
No related branches found
No related tags found
4 merge requests!106if stat fails, display the error instead of immediately refreshing...,!99Dev,!91Test,!90Dev
Pipeline #15929 passed
Stage: build
    Stage: deploy
      Hide whitespace changes
      Inline Side-by-side
      src/app/jobs.service.ts
      + 43
      − 25
      View file @ 8f7426b2
      ...@@ -4,9 +4,9 @@ import { ComputesitesService } from './computesites.service'; ...@@ -4,9 +4,9 @@ import { ComputesitesService } from './computesites.service';
      import { AuthorisationService } from './authorisation.service'; import { AuthorisationService } from './authorisation.service';
      import { Job } from './job'; import { Job } from './job';
      import { Identity } from './identity'; import { Identity } from './identity';
      import { Subscription, interval, pipe, Observable } from 'rxjs'; import { Subscription, interval, pipe, Observable, throwError, of } from 'rxjs';
      import { BehaviorSubject, timer } from 'rxjs'; import { BehaviorSubject, timer } from 'rxjs';
      import { repeat } from 'rxjs/operators'; import { repeatWhen, delay, tap, catchError } from 'rxjs/operators';
      import { NotificationsService } from './notifications.service'; import { NotificationsService } from './notifications.service';
      ...@@ -24,10 +24,11 @@ export class JobsService { ...@@ -24,10 +24,11 @@ export class JobsService {
      public setId(id: Identity) { public setId(id: Identity) {
      this.id = id; this.id = id;
      if (this.tsub !== undefined) { // if (this.tsub !== undefined) {
      this.tsub.unsubscribe() // this.tsub.unsubscribe()
      } // }
      this.tsub = timer(5000).pipe(repeat()).subscribe(() => this.refreshJobs()) //this.tsub = timer(5000).pipe(repeat()).subscribe(() => this.refreshJobs())
      this.refreshJobs()
      } }
      public refreshJobs() { public refreshJobs() {
      ...@@ -35,34 +36,51 @@ export class JobsService { ...@@ -35,34 +36,51 @@ export class JobsService {
      var query$: Observable<Job[]>; var query$: Observable<Job[]>;
      console.log('refreshJobs'); console.log('refreshJobs');
      query$ = this.tes.runCommand(this.id, this.id.site.statcmd) query$ = this.tes.runCommand(this.id, this.id.site.statcmd)
      query$.subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs), //query$.subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs),
      // (error) => this.getJobsError(error,this.id))
      query$.pipe(
      repeatWhen(x => x.pipe(delay(5000))),
      catchError((err) => this.getJobsErrorHandler(err, this.id.site.statcmd, this.id)),
      tap((_) => console.log('in refreshJobs' + Date())),
      ).subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs),
      (error) => this.getJobsError(error,this.id)) (error) => this.getJobsError(error,this.id))
      } }
      } }
      public getJobsError(error,identity: Identity) { public getJobsErrorHandler(error, cmd: string, identity: Identity): Observable<any> {
      if (error.hasOwnProperty('status') && error.status == 401) { var emsg: string = "";
      this.notifications.notify("Your login appears to have expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } ); if (error.hasOwnProperty('error') && error.error.hasOwnProperty('message') ) {
      return; emsg = error.error.message;
      } else {
      emsg = error.message;
      }
      if (identity.expiry < Date.now() || (error.hasOwnProperty("status") && error.status == 401)) {
      this.notifications.notify("Your login has expired. Please log in again", () => { this.authService.updateAgentContents().subscribe( () => {return} ) } );
      return throwError(error);
      }
      if (error.hasOwnProperty("status") && error.status == 500) {
      this.notifications.notify("The Strudel2 API server had an error.\n Please report this via the contact us link.\nThe error message was"+emsg);
      return of([]);
      } }
      this.tsub.unsubscribe(); if (error.hasOwnProperty("status") && error.status == 400) {
      if (identity.expiry < Date.now()) { this.notifications.notify(cmd + " unexpectedly returned an error message.\n Please report this vai the contact us link.\nThe error message was\n" + emsg);
      this.notifications.notify("Your login has expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } ); return throwError(error);
      return;
      } }
      console.error('getJobsError id', identity); if (error.hasOwnProperty("status") && error.status == 504) {
      if (error.hasOwnProperty("error") && error.error.hasOwnProperty("message")) { this.notifications.notify("The Server timed out while retrieveing a list of running jobs. Is the application server OK?");
      if (error.error.message.indexOf("Permission denied") != -1) { return of([]);
      this.notifications.notify("Your login appears to have expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } );
      return;
      }
      this.notifications.notify("Unable to retrieve a list of running jobs.\nThe error messge was " + error.error.message);
      console.log(error);
      return;
      } }
      if (error.error instanceof ErrorEvent) {
      this.notifications.notify("A networking error occured. Is your internet connection OK?")
      return throwError(error);
      }
      }
      public getJobsError(error,identity: Identity) {
      console.error(error); console.error(error);
      this.notifications.notify("Unable to retrieve a list of running jobs.\nThe error wasn't specified\nPlease report what you were doign via the contact us link", () => { this.authService.updateAgentContents().subscribe((_) => {return}) }); this.notifications.notify("Unable to retrieve a list of running jobs.\nThis is probably an error on our end.\nPlease report what you were doing via the contact us link", () => { this.authService.updateAgentContents().subscribe((_) => {return}) });
      //this.authService.updateAgentContents().subscribe((_) => {return}); //this.authService.updateAgentContents().subscribe((_) => {return});
      } }
      ......
      src/app/tes.service.ts
      + 133
      − 42
      View file @ 8f7426b2
      ...@@ -16,6 +16,7 @@ import { BackendSelectionService } from './backend-selection.service'; ...@@ -16,6 +16,7 @@ import { BackendSelectionService } from './backend-selection.service';
      import {BrowserWindowService} from './browser-window.service'; import {BrowserWindowService} from './browser-window.service';
      import { NotificationsService } from './notifications.service'; import { NotificationsService } from './notifications.service';
      import { APIServer } from './apiserver'; import { APIServer } from './apiserver';
      import { throwToolbarMixedModesError } from '@angular/material/toolbar';
      /** The TES service contains ways to start Tunnels, and Execute programs /** The TES service contains ways to start Tunnels, and Execute programs
      Its also responsible for querying a compute site for running jobs */ Its also responsible for querying a compute site for running jobs */
      ...@@ -243,7 +244,7 @@ private addUserHealth(identity,resp) { ...@@ -243,7 +244,7 @@ private addUserHealth(identity,resp) {
      return this.http.get<string>(apiserver.tes+'/appinstance/'+username+'/'+loginhost+'/'+batchhost+'/'+jobid+'?'+paramstr, options) return this.http.get<string>(apiserver.tes+'/appinstance/'+username+'/'+loginhost+'/'+batchhost+'/'+jobid+'?'+paramstr, options)
      } }
      public createTunnel(job: Job, appinst: any, action: AppAction, apiserver: APIServer) { public createTunnel(job: Job, appinst: any, action: AppAction, apiserver: APIServer): Observable<string> {
      let username = job.identity.username; let username = job.identity.username;
      let loginhost = job.identity.site.host; let loginhost = job.identity.site.host;
      let batchhost = job.batch_host; let batchhost = job.batch_host;
      ...@@ -252,7 +253,7 @@ private addUserHealth(identity,resp) { ...@@ -252,7 +253,7 @@ private addUserHealth(identity,resp) {
      job.connectionState = 2; job.connectionState = 2;
      if ((action.notunnel != undefined) && (action.notunnel == true)) { if ((action.notunnel != undefined) && (action.notunnel == true)) {
      return of([]) return of('')
      } }
      let options = { headers: headers, withCredentials: true}; let options = { headers: headers, withCredentials: true};
      let paramstr = params.toString(); let paramstr = params.toString();
      ...@@ -286,36 +287,82 @@ private addUserHealth(identity,resp) { ...@@ -286,36 +287,82 @@ private addUserHealth(identity,resp) {
      ) )
      } }
      public connect(job: Job, action: AppAction, appinst?: any) { // public connect(job: Job, action: AppAction, appinst?: any) {
      let headers = new HttpHeaders(); // let headers = new HttpHeaders();
      let options = { headers: headers, withCredentials: true}; // let options = { headers: headers, withCredentials: true};
      // job.connectionState=1
      // // Can no connect till we know which api server we are using. Threfore the apiserver
      // // is the start of the pipe
      // // the API server is behaviorsubject so it should always fire immediately
      // this.backendSelectionService.apiserver.pipe(
      // filter((v) => v !== undefined && v !== null),
      // switchMap((apiserver) => this.getAppInstance(job,action,apiserver),
      // (apiserver,appinst,i1,i2) => {return [apiserver,appinst]}),
      // tap((_) => job.connectionState=2),
      // switchMap(([apiserver,appinst]) => { return this.createTunnel(job,appinst,action,<APIServer>apiserver)},
      // (data,tunnelresp) => { data[1]['localport'] = (<any> tunnelresp)['localport'] ; return data}), // we don't care about data from the tunnel as long as it was successful.`
      // tap((_) => job.connectionState=3),
      // switchMap(([apiserver,appinst]) => this.getAppUrl(job,appinst,action,<APIServer>apiserver),
      // ([apiserver,appinst],url) => {return [apiserver,appinst,url]}),
      // tap((_) => job.connectionState=4),
      // ).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
      // this.openWindow$.next( {'job':job,
      // 'url': <string> url,
      // 'usebasicauth':action.client.usebasicauth,
      // 'apiserver':apiserver,
      // 'action':action,'appinst':appinst});
      // }
      // job.connectionState=null},
      // (err) => { job.connectionState = 0; this.handleError(job.identity, err)})
      // }
      job.connectionState=1
      // Can no connect till we know which api server we are using. Threfore the apiserver
      // is the start of the pipe
      // the API server is behaviorsubject so it should always fire immediately
      this.backendSelectionService.apiserver.pipe(
      filter((v) => v !== undefined && v !== null),
      switchMap((apiserver) => this.getAppInstance(job,action,apiserver),
      (apiserver,appinst,i1,i2) => {return [apiserver,appinst]}),
      tap((_) => job.connectionState=2),
      switchMap(([apiserver,appinst]) => { return this.createTunnel(job,appinst,action,<APIServer>apiserver)},
      (data,tunnelresp) => { data[1]['localport'] = (<any> tunnelresp)['localport'] ; return data}), // we don't care about data from the tunnel as long as it was successful.`
      tap((_) => job.connectionState=3),
      switchMap(([apiserver,appinst]) => this.getAppUrl(job,appinst,action,<APIServer>apiserver),
      ([apiserver,appinst],url) => {return [apiserver,appinst,url]}),
      tap((_) => job.connectionState=4),
      ).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
      this.openWindow$.next( {'job':job,
      'url': <string> url,
      'usebasicauth':action.client.usebasicauth,
      'apiserver':apiserver,
      'action':action,'appinst':appinst});
      }
      job.connectionState=null},
      (err) => { job.connectionState = 0; this.handleError(job.identity, err)})
      }
      public connect(job: Job, action: AppAction, appinst?: any) {
      let headers = new HttpHeaders();
      let options = { headers: headers, withCredentials: true};
      job.connectionState=1
      // Can no connect till we know which api server we are using. Threfore the apiserver
      // is the start of the pipe
      // the API server is behaviorsubject so it should always fire immediately
      this.backendSelectionService.apiserver.pipe(
      filter((v) => v !== undefined && v !== null),
      switchMap((apiserver): Observable<[APIServer, string]> =>
      { return this.getAppInstance(job,action,apiserver)
      .pipe(
      catchError(err => this.handleAppInstanceError(job,action,err)),
      tap((v) => console.log('getappinstance succeeded',v)),
      map((v)=>[apiserver,v]))
      } ),
      tap((_) => job.connectionState=2),
      switchMap(([apiserver,appinst]) =>
      { return this.createTunnel(job,appinst,action,<APIServer>apiserver)
      .pipe(
      tap((_) => console.log('called create tunnel')),
      catchError(err => this.handleTunnelError(job,action,err)),
      map((v) =>
      {
      appinst['localport'] = v['localport'];
      return [apiserver, appinst];
      }))
      } ),
      tap((_) => job.connectionState=3),
      switchMap(([apiserver,appinst]) =>
      { return this.getAppUrl(job,appinst,action,<APIServer>apiserver)
      .pipe(map((url) => [apiserver, appinst, url]))
      }),
      tap((_) => job.connectionState=4),
      ).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
      this.openWindow$.next( {'job':job,
      'url': <string> url,
      'usebasicauth':action.client.usebasicauth,
      'apiserver':apiserver,
      'action':action,'appinst':appinst});
      }
      job.connectionState=null},
      (err) => { job.connectionState = 0; this.handleError(job.identity, err)})
      }
      public runCommand(identity: Identity, command: string): Observable<any> { public runCommand(identity: Identity, command: string): Observable<any> {
      let headers = new HttpHeaders(); let headers = new HttpHeaders();
      ...@@ -360,27 +407,71 @@ public getSftpData(id: Identity, path: string, cd: string ) { ...@@ -360,27 +407,71 @@ public getSftpData(id: Identity, path: string, cd: string ) {
      private handleError(identity: Identity, error: HttpErrorResponse) { private handleError(identity: Identity, error: HttpErrorResponse) {
      console.error(error); console.error(error);
      console.log('in handle error'); console.log('in handle error');
      if (identity.expiry < Date.now()) { // if (identity.expiry < Date.now()) {
      // this.notifications.notify("Your login has expired. Please log in again", () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) } );
      // return;
      // }
      // if (error.error instanceof ErrorEvent) {
      // this.notifications.notify("A networking error occured.")
      // return
      // }
      // if (error.hasOwnProperty("status") && error.status == 500) {
      // this.notifications.notify(error);
      // return
      // }
      // if (error.hasOwnProperty("status") && error.status == 400) {
      // this.notifications.notify(error);
      // return
      // }
      // this.notifications.notify(error);
      }
      private handleAppInstanceError(job: Job, action: AppAction, error: HttpErrorResponse): Observable<any> {
      console.error(error);
      const msg: string = "The command " + action.paramscmd + " on host " + job.batch_host + " "
      return this.handleGenericError(job,action,error,msg);
      }
      private handleTunnelError(job: Job, action: AppAction, error: HttpErrorResponse): Observable<any> {
      console.error(error);
      const msg: string = "Attempting to create an SSH tunnel to" + job.batch_host + " via " + job.identity.site.host;
      return this.handleGenericError(job,action,error,msg);
      }
      private handleGenericError(job: Job, action: AppAction, error: HttpErrorResponse, msg: string): Observable<any> {
      console.log('in handleGenericError');
      var emsg: string = "";
      if (error.hasOwnProperty('error') && error.error.hasOwnProperty('message') ) {
      emsg = error.error.message;
      } else {
      emsg = error.message;
      }
      if (job.identity.expiry < Date.now() || (error.hasOwnProperty("status") && error.status == 401)) {
      this.notifications.notify("Your login has expired. Please log in again", () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) } ); this.notifications.notify("Your login has expired. Please log in again", () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) } );
      return; return throwError(error);
      } }
      if (error.error instanceof ErrorEvent) {
      this.notifications.notify("A networking error occured.")
      return
      }
      if (error.hasOwnProperty("status") && error.status == 500) { if (error.hasOwnProperty("status") && error.status == 500) {
      this.notifications.notify(error); this.notifications.notify("The Strudel2 API server had an error.\n Please report this via the contact us link.\nThe error message was"+emsg);
      return return throwError(error);
      } }
      if (error.hasOwnProperty("status") && error.status == 400) { if (error.hasOwnProperty("status") && error.status == 400) {
      this.notifications.notify(error); this.notifications.notify(msg + " unexpectedly returned an error message.\n Please report this vai the contact us link.\nThe error message was\n" + emsg);
      return return throwError(error);
      }
      if (error.hasOwnProperty("status") && error.status == 504) {
      this.notifications.notify(msg + " timed out. Is the application server OK?");
      return throwError(error);
      } }
      if (error.error instanceof ErrorEvent) {
      this.notifications.notify("A networking error occured. Is your internet connection OK?")
      return throwError(error);
      }
      console.log('performing raw notification');
      this.notifications.notify(error); this.notifications.notify(error);
      return throwError(error);
      } }
      } }
      0% or .
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment