Commit 3eb6db02 authored by Lance Wilson's avatar Lance Wilson
Browse files

Merge branch 'test' into 'prod'

Test

See merge request !91
parents ac23fd20 2933b914
Pipeline #16274 passed with stages
in 10 minutes and 6 seconds
...@@ -4,9 +4,9 @@ import { ComputesitesService } from './computesites.service'; ...@@ -4,9 +4,9 @@ import { ComputesitesService } from './computesites.service';
import { AuthorisationService } from './authorisation.service'; import { AuthorisationService } from './authorisation.service';
import { Job } from './job'; import { Job } from './job';
import { Identity } from './identity'; import { Identity } from './identity';
import { Subscription, interval, pipe, Observable } from 'rxjs'; import { Subscription, interval, pipe, Observable, throwError, of } from 'rxjs';
import { BehaviorSubject, timer } from 'rxjs'; import { BehaviorSubject, timer } from 'rxjs';
import { repeat } from 'rxjs/operators'; import { repeatWhen, delay, tap, catchError } from 'rxjs/operators';
import { NotificationsService } from './notifications.service'; import { NotificationsService } from './notifications.service';
...@@ -24,45 +24,61 @@ export class JobsService { ...@@ -24,45 +24,61 @@ export class JobsService {
public setId(id: Identity) { public setId(id: Identity) {
this.id = id; this.id = id;
if (this.tsub !== undefined) { // if (this.tsub !== undefined) {
this.tsub.unsubscribe() // this.tsub.unsubscribe()
} // }
this.tsub = timer(5000).pipe(repeat()).subscribe(() => this.refreshJobs()) //this.tsub = timer(5000).pipe(repeat()).subscribe(() => this.refreshJobs())
this.refreshJobs()
} }
public refreshJobs() { public refreshJobs() {
if (this.id !== undefined && this.id !== null) { if (this.id !== undefined && this.id !== null) {
var query$: Observable<Job[]>; var query$: Observable<Job[]>;
console.log('refreshJobs');
query$ = this.tes.runCommand(this.id, this.id.site.statcmd) query$ = this.tes.runCommand(this.id, this.id.site.statcmd)
query$.subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs), //query$.subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs),
// (error) => this.getJobsError(error,this.id))
query$.pipe(
repeatWhen(x => x.pipe(delay(5000))),
catchError((err) => this.getJobsErrorHandler(err, this.id.site.statcmd, this.id)),
).subscribe((qjobs) => this.jobs$.next(<Job[]>qjobs),
(error) => this.getJobsError(error,this.id)) (error) => this.getJobsError(error,this.id))
} }
} }
public getJobsError(error,identity: Identity) { public getJobsErrorHandler(error, cmd: string, identity: Identity): Observable<any> {
if (error.hasOwnProperty('status') && error.status == 401) { var emsg: string = "";
this.notifications.notify("Your login appears to have expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } ); if (error.hasOwnProperty('error') && error.error.hasOwnProperty('message') ) {
return; emsg = error.error.message;
} else {
emsg = error.message;
}
if (identity.expiry < Date.now() || (error.hasOwnProperty("status") && error.status == 401)) {
this.notifications.notify("Your login has expired. Please log in again", () => { this.authService.updateAgentContents().subscribe( () => {return} ) } );
return throwError(error);
}
if (error.hasOwnProperty("status") && error.status == 500) {
this.notifications.notify("The Strudel2 API server had an error.\n Please report this via the contact us link.\nThe error message was"+emsg);
return of([]);
} }
this.tsub.unsubscribe(); if (error.hasOwnProperty("status") && error.status == 400) {
if (identity.expiry < Date.now()) { this.notifications.notify(cmd + " unexpectedly returned an error message.\n Please report this vai the contact us link.\nThe error message was\n" + emsg);
this.notifications.notify("Your login has expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } ); return throwError(error);
return;
} }
console.error('getJobsError id', identity); if (error.hasOwnProperty("status") && error.status == 504) {
if (error.hasOwnProperty("error") && error.error.hasOwnProperty("message")) { this.notifications.notify("The Server timed out while retrieveing a list of running jobs. Is the application server OK?");
if (error.error.message.indexOf("Permission denied") != -1) { return of([]);
this.notifications.notify("Your login appears to have expired. Please log in again", () => { this.authService.updateAgentContents().subscribe((_) => {return}) } );
return;
}
this.notifications.notify("Unable to retrieve a list of running jobs.\nThe error messge was " + error.error.message);
console.log(error);
return;
} }
if (error.error instanceof ErrorEvent) {
this.notifications.notify("A networking error occured. Is your internet connection OK?")
return throwError(error);
}
}
public getJobsError(error,identity: Identity) {
console.error(error); console.error(error);
this.notifications.notify("Unable to retrieve a list of running jobs.\nThe error wasn't specified\nPlease report what you were doign via the contact us link", () => { this.authService.updateAgentContents().subscribe((_) => {return}) }); this.notifications.notify("Unable to retrieve a list of running jobs.\nThis is probably an error on our end.\nPlease report what you were doing via the contact us link", () => { this.authService.updateAgentContents().subscribe((_) => {return}) });
//this.authService.updateAgentContents().subscribe((_) => {return}); //this.authService.updateAgentContents().subscribe((_) => {return});
} }
......
...@@ -16,6 +16,7 @@ import { BackendSelectionService } from './backend-selection.service'; ...@@ -16,6 +16,7 @@ import { BackendSelectionService } from './backend-selection.service';
import {BrowserWindowService} from './browser-window.service'; import {BrowserWindowService} from './browser-window.service';
import { NotificationsService } from './notifications.service'; import { NotificationsService } from './notifications.service';
import { APIServer } from './apiserver'; import { APIServer } from './apiserver';
import { throwToolbarMixedModesError } from '@angular/material/toolbar';
/** The TES service contains ways to start Tunnels, and Execute programs /** The TES service contains ways to start Tunnels, and Execute programs
Its also responsible for querying a compute site for running jobs */ Its also responsible for querying a compute site for running jobs */
...@@ -104,8 +105,8 @@ public getCachetIncidents(identity: Identity) { ...@@ -104,8 +105,8 @@ public getCachetIncidents(identity: Identity) {
let headers = new HttpHeaders(); let headers = new HttpHeaders();
let options = { headers: headers, withCredentials: false}; let options = { headers: headers, withCredentials: false};
let params = new URLSearchParams(); let params = new URLSearchParams();
if (identity.site.cacheturis === undefined || identity.site.cacheturis.length == 0) { if (identity.site.cacheturis === undefined || identity.site.cacheturis.length == 0) {
console.log('no uris');
identity.systemalerts.next([]); identity.systemalerts.next([]);
return return
} }
...@@ -243,7 +244,7 @@ private addUserHealth(identity,resp) { ...@@ -243,7 +244,7 @@ private addUserHealth(identity,resp) {
return this.http.get<string>(apiserver.tes+'/appinstance/'+username+'/'+loginhost+'/'+batchhost+'/'+jobid+'?'+paramstr, options) return this.http.get<string>(apiserver.tes+'/appinstance/'+username+'/'+loginhost+'/'+batchhost+'/'+jobid+'?'+paramstr, options)
} }
public createTunnel(job: Job, appinst: any, action: AppAction, apiserver: APIServer) { public createTunnel(job: Job, appinst: any, action: AppAction, apiserver: APIServer): Observable<string> {
let username = job.identity.username; let username = job.identity.username;
let loginhost = job.identity.site.host; let loginhost = job.identity.site.host;
let batchhost = job.batch_host; let batchhost = job.batch_host;
...@@ -252,7 +253,7 @@ private addUserHealth(identity,resp) { ...@@ -252,7 +253,7 @@ private addUserHealth(identity,resp) {
job.connectionState = 2; job.connectionState = 2;
if ((action.notunnel != undefined) && (action.notunnel == true)) { if ((action.notunnel != undefined) && (action.notunnel == true)) {
return of([]) return of('')
} }
let options = { headers: headers, withCredentials: true}; let options = { headers: headers, withCredentials: true};
let paramstr = params.toString(); let paramstr = params.toString();
...@@ -286,36 +287,82 @@ private addUserHealth(identity,resp) { ...@@ -286,36 +287,82 @@ private addUserHealth(identity,resp) {
) )
} }
public connect(job: Job, action: AppAction, appinst?: any) { // public connect(job: Job, action: AppAction, appinst?: any) {
let headers = new HttpHeaders(); // let headers = new HttpHeaders();
let options = { headers: headers, withCredentials: true}; // let options = { headers: headers, withCredentials: true};
// job.connectionState=1
// // Can no connect till we know which api server we are using. Threfore the apiserver
// // is the start of the pipe
// // the API server is behaviorsubject so it should always fire immediately
// this.backendSelectionService.apiserver.pipe(
// filter((v) => v !== undefined && v !== null),
// switchMap((apiserver) => this.getAppInstance(job,action,apiserver),
// (apiserver,appinst,i1,i2) => {return [apiserver,appinst]}),
// tap((_) => job.connectionState=2),
// switchMap(([apiserver,appinst]) => { return this.createTunnel(job,appinst,action,<APIServer>apiserver)},
// (data,tunnelresp) => { data[1]['localport'] = (<any> tunnelresp)['localport'] ; return data}), // we don't care about data from the tunnel as long as it was successful.`
// tap((_) => job.connectionState=3),
// switchMap(([apiserver,appinst]) => this.getAppUrl(job,appinst,action,<APIServer>apiserver),
// ([apiserver,appinst],url) => {return [apiserver,appinst,url]}),
// tap((_) => job.connectionState=4),
// ).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
// this.openWindow$.next( {'job':job,
// 'url': <string> url,
// 'usebasicauth':action.client.usebasicauth,
// 'apiserver':apiserver,
// 'action':action,'appinst':appinst});
// }
// job.connectionState=null},
// (err) => { job.connectionState = 0; this.handleError(job.identity, err)})
// }
job.connectionState=1
// Can no connect till we know which api server we are using. Threfore the apiserver
// is the start of the pipe
// the API server is behaviorsubject so it should always fire immediately
this.backendSelectionService.apiserver.pipe(
filter((v) => v !== undefined && v !== null),
switchMap((apiserver) => this.getAppInstance(job,action,apiserver),
(apiserver,appinst,i1,i2) => {return [apiserver,appinst]}),
tap((_) => job.connectionState=2),
switchMap(([apiserver,appinst]) => { return this.createTunnel(job,appinst,action,<APIServer>apiserver)},
(data,tunnelresp) => { data[1]['localport'] = (<any> tunnelresp)['localport'] ; return data}), // we don't care about data from the tunnel as long as it was successful.`
tap((_) => job.connectionState=3),
switchMap(([apiserver,appinst]) => this.getAppUrl(job,appinst,action,<APIServer>apiserver),
([apiserver,appinst],url) => {return [apiserver,appinst,url]}),
tap((_) => job.connectionState=4),
).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
this.openWindow$.next( {'job':job,
'url': <string> url,
'usebasicauth':action.client.usebasicauth,
'apiserver':apiserver,
'action':action,'appinst':appinst});
}
job.connectionState=null},
(err) => { job.connectionState = 0; this.handleError(job.identity, err)})
}
public connect(job: Job, action: AppAction, appinst?: any) {
let headers = new HttpHeaders();
let options = { headers: headers, withCredentials: true};
job.connectionState=1
// Can no connect till we know which api server we are using. Threfore the apiserver
// is the start of the pipe
// the API server is behaviorsubject so it should always fire immediately
this.backendSelectionService.apiserver.pipe(
filter((v) => v !== undefined && v !== null),
switchMap((apiserver): Observable<[APIServer, string]> =>
{ return this.getAppInstance(job,action,apiserver)
.pipe(
catchError(err => this.handleAppInstanceError(job,action,err)),
tap((v) => console.log('getappinstance succeeded',v)),
map((v)=>[apiserver,v]))
} ),
tap((_) => job.connectionState=2),
switchMap(([apiserver,appinst]) =>
{ return this.createTunnel(job,appinst,action,<APIServer>apiserver)
.pipe(
tap((_) => console.log('called create tunnel')),
catchError(err => this.handleTunnelError(job,action,err)),
map((v) =>
{
appinst['localport'] = v['localport'];
return [apiserver, appinst];
}))
} ),
tap((_) => job.connectionState=3),
switchMap(([apiserver,appinst]) =>
{ return this.getAppUrl(job,appinst,action,<APIServer>apiserver)
.pipe(map((url) => [apiserver, appinst, url]))
}),
tap((_) => job.connectionState=4),
).subscribe(([apiserver,appinst,url]) => { if (url !== null) {
this.openWindow$.next( {'job':job,
'url': <string> url,
'usebasicauth':action.client.usebasicauth,
'apiserver':apiserver,
'action':action,'appinst':appinst});
}
job.connectionState=null},
(err) => { job.connectionState = 0; this.handleError(job.identity, err)})
}
public runCommand(identity: Identity, command: string): Observable<any> { public runCommand(identity: Identity, command: string): Observable<any> {
let headers = new HttpHeaders(); let headers = new HttpHeaders();
...@@ -360,27 +407,71 @@ public getSftpData(id: Identity, path: string, cd: string ) { ...@@ -360,27 +407,71 @@ public getSftpData(id: Identity, path: string, cd: string ) {
private handleError(identity: Identity, error: HttpErrorResponse) { private handleError(identity: Identity, error: HttpErrorResponse) {
console.error(error); console.error(error);
console.log('in handle error'); console.log('in handle error');
if (identity.expiry < Date.now()) { // if (identity.expiry < Date.now()) {
// this.notifications.notify("Your login has expired. Please log in again", () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) } );
// return;
// }
// if (error.error instanceof ErrorEvent) {
// this.notifications.notify("A networking error occured.")
// return
// }
// if (error.hasOwnProperty("status") && error.status == 500) {
// this.notifications.notify(error);
// return
// }
// if (error.hasOwnProperty("status") && error.status == 400) {
// this.notifications.notify(error);
// return
// }
// this.notifications.notify(error);
}
private handleAppInstanceError(job: Job, action: AppAction, error: HttpErrorResponse): Observable<any> {
console.error(error);
const msg: string = "The command " + action.paramscmd + " on host " + job.batch_host + " "
return this.handleGenericError(job,action,error,msg);
}
private handleTunnelError(job: Job, action: AppAction, error: HttpErrorResponse): Observable<any> {
console.error(error);
const msg: string = "Attempting to create an SSH tunnel to" + job.batch_host + " via " + job.identity.site.host;
return this.handleGenericError(job,action,error,msg);
}
private handleGenericError(job: Job, action: AppAction, error: HttpErrorResponse, msg: string): Observable<any> {
console.log('in handleGenericError');
var emsg: string = "";
if (error.hasOwnProperty('error') && error.error.hasOwnProperty('message') ) {
emsg = error.error.message;
} else {
emsg = error.message;
}
if (job.identity.expiry < Date.now() || (error.hasOwnProperty("status") && error.status == 401)) {
this.notifications.notify("Your login has expired. Please log in again", () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) } ); this.notifications.notify("Your login has expired. Please log in again", () => { this.authorisationService.updateAgentContents().subscribe( () => {return} ) } );
return; return throwError(error);
} }
if (error.error instanceof ErrorEvent) {
this.notifications.notify("A networking error occured.")
return
}
if (error.hasOwnProperty("status") && error.status == 500) { if (error.hasOwnProperty("status") && error.status == 500) {
this.notifications.notify(error); this.notifications.notify("The Strudel2 API server had an error.\n Please report this via the contact us link.\nThe error message was"+emsg);
return return throwError(error);
} }
if (error.hasOwnProperty("status") && error.status == 400) { if (error.hasOwnProperty("status") && error.status == 400) {
this.notifications.notify(error); this.notifications.notify(msg + " unexpectedly returned an error message.\n Please report this vai the contact us link.\nThe error message was\n" + emsg);
return return throwError(error);
}
if (error.hasOwnProperty("status") && error.status == 504) {
this.notifications.notify(msg + " timed out. Is the application server OK?");
return throwError(error);
} }
if (error.error instanceof ErrorEvent) {
this.notifications.notify("A networking error occured. Is your internet connection OK?")
return throwError(error);
}
console.log('performing raw notification');
this.notifications.notify(error); this.notifications.notify(error);
return throwError(error);
} }
} }
...@@ -2,15 +2,15 @@ ...@@ -2,15 +2,15 @@
{ {
"url": "https://strudel2-api-dev.cloud.cvl.org.au/m3/", "url": "https://strudel2-api-dev.cloud.cvl.org.au/m3/",
"name": "M3", "name": "M3",
"host": "m3-login2.massive.org.au", "host": "m3.massive.org.au",
"dtn": "m3-dtn1.massive.org.au", "dtn": "m3-dtn1.massive.org.au",
"cafingerprint": "ECDSA SHA256:6wVXdokvvlTNcXPMc9KyvIXA8a8XNfLuhBfNOYeeMdg", "cafingerprint": "ECDSA SHA256:6wVXdokvvlTNcXPMc9KyvIXA8a8XNfLuhBfNOYeeMdg",
"appCatalog": [], "appCatalog": [],
"appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps-dev", "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps-dev",
"cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}", "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
"statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat", "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
"userhealth": "/usr/local/bin/uijson", "userhealth": "/home/chines/strudel2_desktop_user_info.py",
"cacheturis": [], "cacheturis": ["/assets/config/incidents.json"],
"contact": "<MASSIVE Support> help@massive.org.au" "contact": "<MASSIVE Support> help@massive.org.au"
}, },
{ {
......
...@@ -9,8 +9,8 @@ ...@@ -9,8 +9,8 @@
"appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps", "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps",
"cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}", "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
"statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat", "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
"userhealth": "/usr/local/bin/uijson", "userhealth": "/home/chines/clusterinfo_venv/bin/show_cluster --jsondesktopsummary",
"cacheturis": [], "cacheturis": ["/assets/config/incidents.json"],
"contact": "<MASSIVE Support> help@massive.org.au" "contact": "<MASSIVE Support> help@massive.org.au"
} }
] ]
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
"appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps", "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps",
"cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}", "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
"statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat", "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
"cacheturis": ["/assets/config/incidents.json"],
"userhealth": "/usr/local/bin/uijson" "userhealth": "/usr/local/bin/uijson"
} }
] ]
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
"appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps", "appCatalogCmd": "/usr/local/strudel2_cluster/latest/bin/getapps",
"cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}", "cancelcmd": "/usr/local/strudel2_cluster/latest/bin/s2cancel {jobid}",
"statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat", "statcmd": "/usr/local/strudel2_cluster/latest/bin/s2stat",
"cacheturis": ["/assets/config/incidents.json"],
"userhealth": "/usr/local/bin/uijson" "userhealth": "/usr/local/bin/uijson"
}, },
{ {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment