From 2a260a4fffc3be6b7b24042c9310ae2fa6b7633a Mon Sep 17 00:00:00 2001 From: Chris Hines <chris.hines@monash.edu> Date: Tue, 3 Jul 2018 16:57:30 +1000 Subject: [PATCH] rework the way the SPA specifies apps, identities and batch scheduler interfaces --- src/app/app-routing.module.ts | 4 +- src/app/app.component.html | 2 +- src/app/app.component.ts | 2 +- src/app/batchinterface.ts | 6 + src/app/computesite.ts | 10 +- src/app/identity.ts | 7 + src/app/job.ts | 5 + src/app/job/job.component.ts | 9 +- src/app/joblist/joblist.component.html | 4 +- src/app/joblist/joblist.component.ts | 2 +- src/app/launcher/launcher.component.ts | 7 +- src/app/mock-compute-site.ts | 5 +- .../siteselection/siteselection.component.ts | 4 +- src/app/strudelapp.ts | 14 +- src/app/tes.service.ts | 211 +++++++++++++----- src/app/teserrors/teserrors.component.html | 2 +- src/app/teserrors/teserrors.component.ts | 6 +- .../tokenextractor.component.ts | 2 +- 18 files changed, 214 insertions(+), 88 deletions(-) create mode 100644 src/app/batchinterface.ts create mode 100644 src/app/identity.ts diff --git a/src/app/app-routing.module.ts b/src/app/app-routing.module.ts index 9ffa6d8..bc5e5a3 100644 --- a/src/app/app-routing.module.ts +++ b/src/app/app-routing.module.ts @@ -7,7 +7,9 @@ import { TokenextractorComponent } from './tokenextractor/tokenextractor.compone const routes: Routes = [ { path: '', redirectTo: 'launch', pathMatch: 'full'}, { path: 'launch', component: LauncherComponent}, - { path: 'sshauthz_callback', component: TokenextractorComponent} + // { path: 'sshauthz_callback', component: TokenextractorComponent} + { path: 'sshauthz_callback', component: LauncherComponent} + ]; diff --git a/src/app/app.component.html b/src/app/app.component.html index 1fc1028..10e69ef 100644 --- a/src/app/app.component.html +++ b/src/app/app.component.html @@ -3,7 +3,7 @@ <span>{{title}}</span> <span class="fill-remaining-space"></span> <span class="fill-remaining-space"></span> - <app-siteselection></app-siteselection> + <!-- <app-siteselection></app-siteselection> --> <button mat-button *ngIf="authorised" (click)="logout()">Logout</button> <button mat-button *ngIf="!authorised" (click)="login()">Login</button> </mat-toolbar-row> diff --git a/src/app/app.component.ts b/src/app/app.component.ts index 69710d1..963b7ef 100644 --- a/src/app/app.component.ts +++ b/src/app/app.component.ts @@ -10,7 +10,7 @@ export class AppComponent { title = 'Strudelv2'; private loggedin: Boolean = false; private errmsg: string=''; - private authorised: boolean; + public authorised: boolean; // private testingAuth: boolean; constructor(private tesService: TesService,) { diff --git a/src/app/batchinterface.ts b/src/app/batchinterface.ts new file mode 100644 index 0000000..68f2435 --- /dev/null +++ b/src/app/batchinterface.ts @@ -0,0 +1,6 @@ +export class BatchInterface { + cancelcmd: string; + submitcmd: string; + statcmd: string; + internalfirewall: boolean; +} diff --git a/src/app/computesite.ts b/src/app/computesite.ts index 807c020..567f55a 100644 --- a/src/app/computesite.ts +++ b/src/app/computesite.ts @@ -1,4 +1,10 @@ export class Computesite { - url: string; - name: string; + url: string; // The URL runs a web service to help construct sbatch commands + // Infact the URL should return the entire interface for + // submit stat and cancel + name: string; // Human readable name + host: string; // Login host + cafingerprint: string; // Certificates contain a CA fingerprint. We use this + // to figure out which compute site a certificate is valid + // for } diff --git a/src/app/identity.ts b/src/app/identity.ts new file mode 100644 index 0000000..94fd05a --- /dev/null +++ b/src/app/identity.ts @@ -0,0 +1,7 @@ +import {Computesite} from './computesite'; +// Identities are defined by a username on a computer, but rather than just +// DNS entry, there is extra info in the Computesite +export class Identity { + username: string; + site: Computesite; +} diff --git a/src/app/job.ts b/src/app/job.ts index 2e26957..c4673f9 100644 --- a/src/app/job.ts +++ b/src/app/job.ts @@ -1,3 +1,6 @@ +import {Computesite} from './computesite'; +import {Identity} from './identity'; +import {Strudelapp} from './strudelapp'; export class Job { public name: string; public jobid: string; @@ -5,4 +8,6 @@ export class Job { public state: string; public time: number; public batch_host: string; + public identity: Identity; + public app: Strudelapp; } diff --git a/src/app/job/job.component.ts b/src/app/job/job.component.ts index aac09fe..cfcdbd3 100644 --- a/src/app/job/job.component.ts +++ b/src/app/job/job.component.ts @@ -11,7 +11,7 @@ import { StrudelappsService } from '../strudelapps.service'; export class JobComponent implements OnInit { @Input() jobdata: Job; - private available: Boolean; + public available: Boolean; private busy: Boolean; constructor(private tesService: TesService, private strudelAppsService: StrudelappsService) { @@ -28,15 +28,14 @@ export class JobComponent implements OnInit { } onCancel() { - this.tesService.cancel(this.jobdata.jobid); + this.tesService.cancel(this.jobdata); } onConnect() { console.log('attempting connect'); // Before connecting we must resolve what type of app we are connecting to - let app = this.strudelAppsService.getApp(this.jobdata.name); - console.log('app definition is ',app); - this.tesService.connect(this.jobdata.jobid,this.jobdata.batch_host,app); + this.jobdata.app = this.strudelAppsService.getApp(this.jobdata.name); + this.tesService.connect(this.jobdata); } } diff --git a/src/app/joblist/joblist.component.html b/src/app/joblist/joblist.component.html index 552ce46..7f6ea40 100644 --- a/src/app/joblist/joblist.component.html +++ b/src/app/joblist/joblist.component.html @@ -1,5 +1,7 @@ -<div *ngIf="jobs.lenght == 0"> +<div *ngIf="jobs.length == 0"> + <mat-card> You don't appear to have any jobs +</mat-card> </div> <div *ngFor="let job of jobs"> <app-job [jobdata]=job></app-job> diff --git a/src/app/joblist/joblist.component.ts b/src/app/joblist/joblist.component.ts index 0c85dac..21b0007 100644 --- a/src/app/joblist/joblist.component.ts +++ b/src/app/joblist/joblist.component.ts @@ -11,7 +11,7 @@ import { Observable } from 'rxjs/Observable'; styleUrls: ['./joblist.component.css'] }) export class JoblistComponent implements OnInit { - private jobs: any[] = []; + public jobs: any[] = []; private displayedColumns = ['id']; private jobsSubscription: any; diff --git a/src/app/launcher/launcher.component.ts b/src/app/launcher/launcher.component.ts index deaf186..bbd4c4c 100644 --- a/src/app/launcher/launcher.component.ts +++ b/src/app/launcher/launcher.component.ts @@ -3,6 +3,7 @@ import {Strudelapp} from '../strudelapp'; import { StrudelappsService } from '../strudelapps.service'; import { MatDialog, MatDialogRef, MAT_DIALOG_DATA, MatDialogModule } from '@angular/material'; import { TesService } from '../tes.service'; +import {Identity } from '../identity'; @@ -15,7 +16,8 @@ export class LauncherComponent implements OnInit { private strudelapps: Strudelapp[]; private app: Strudelapp; - private authorised: boolean; + public authorised: boolean; + private identity: Identity; constructor(private strudelappsService: StrudelappsService, public dialog: MatDialog, private tesService: TesService) { } @@ -36,7 +38,8 @@ export class LauncherComponent implements OnInit { submitApp() { console.log('submitting app for',this.app.name); - this.tesService.submit(this.app); + this.identity = this.tesService.identities[0]; + this.tesService.submit(this.app, this.identity); // this.openDialog('This is where we should start the app'); } diff --git a/src/app/mock-compute-site.ts b/src/app/mock-compute-site.ts index 29078a9..c2a9b36 100644 --- a/src/app/mock-compute-site.ts +++ b/src/app/mock-compute-site.ts @@ -1,6 +1,7 @@ import { Computesite } from './computesite'; export const COMPUTESITES: Computesite[] = [ - { url: 'http://localhost:8080/config', name: 'M3' }, - {url: '', name: 'Other' }, + { url: 'http://localhost:8080/config', name: 'M3', + host: 'm3.massive.org.au', + cafingerprint: 'RSA SHA256:cmDxHrZQSPlBMUUcI/BWmruXho1XOzfXPDHSqVTwV2I' }, ]; diff --git a/src/app/siteselection/siteselection.component.ts b/src/app/siteselection/siteselection.component.ts index bfb204f..0f4408b 100644 --- a/src/app/siteselection/siteselection.component.ts +++ b/src/app/siteselection/siteselection.component.ts @@ -8,8 +8,8 @@ import { ComputesitesService} from '../computesites.service'; styleUrls: ['./siteselection.component.css'] }) export class SiteselectionComponent implements OnInit { - private computesites: Computesite[]; - private siteurl: String; + public computesites: Computesite[]; + public siteurl: String; constructor(private computesiteSerivce: ComputesitesService) { } ngOnInit() { diff --git a/src/app/strudelapp.ts b/src/app/strudelapp.ts index b5214d8..8cc0a73 100644 --- a/src/app/strudelapp.ts +++ b/src/app/strudelapp.ts @@ -1,8 +1,12 @@ export class Strudelapp { - url: string; - name: string; - startscript: string; - paramscmd: string; + url: string; // A url used to retrieve extra config options. May be null + name: string; // Human readable name + startscript: string; // batch script ... should NOT include resource directives + // resource directives like #SBATCH belong in the batchinterface + paramscmd: string; // command to return extra data such as passwords and tokens + // values returned here can be used in the client strings client: {cmd: string[], redir: string}; - localbind: boolean; + localbind: boolean; // does the application bind to a port on the localhost + // interface or on all interfaces. This behaviour determins + /// how we create tunnels } diff --git a/src/app/tes.service.ts b/src/app/tes.service.ts index 0f44f50..66f8ca3 100644 --- a/src/app/tes.service.ts +++ b/src/app/tes.service.ts @@ -6,11 +6,19 @@ import { catchError, map, tap } from 'rxjs/operators'; import { Job } from './job'; import {BehaviorSubject} from 'rxjs/BehaviorSubject'; import { Strudelapp } from './strudelapp'; +import { Computesite } from './computesite'; +import { Identity } from './identity'; +import { BatchInterface} from './batchinterface'; +import {ComputesitesService} from './computesites.service'; +import { StrudelappsService } from './strudelapps.service'; import { timer } from 'rxjs/observable/timer'; import { repeat } from 'rxjs/operators'; +import {LocationStrategy} from '@angular/common'; // import { keypair } from 'keypair'; import * as keypair from 'keypair'; import * as forge from "node-forge"; +import { ActivatedRoute } from '@angular/router'; + // import keypair = require('keypair'); // import forge = require('node-forge'); @@ -18,9 +26,10 @@ import * as forge from "node-forge"; @Injectable() export class TesService { private Base='http://localhost:5000' +//private Base='https://vm-118-138-240-255.erc.monash.edu.au/tes' public authorised: BehaviorSubject<boolean>; public tesSelected: BehaviorSubject<boolean>; -public errMsg: BehaviorSubject<any>; +public statusMsg: BehaviorSubject<any>; public jobs: any[]; public busy: BehaviorSubject<boolean> ; public testingAuth: BehaviorSubject<boolean>; @@ -34,11 +43,17 @@ private client_id: string; private keypair: any; private ssh: any; private sshcert: any; +private identities: Identity[]; +private batchinterface: BatchInterface; - constructor(private http: HttpClient) { - this.errMsg = new BehaviorSubject<any>(''); + constructor(private http: HttpClient, + private locationStrategy: LocationStrategy, + private route: ActivatedRoute, + private computesite: ComputesitesService, + private strudelappsService: StrudelappsService) { + this.statusMsg = new BehaviorSubject<any>(''); this.authorised = new BehaviorSubject<boolean>(false); this.tesSelected = new BehaviorSubject<boolean>(true); this.testingAuth = new BehaviorSubject<boolean>(false); @@ -50,57 +65,101 @@ private sshcert: any; this.sshauthz = this.sshauthzbase + 'oauth/authorize'; this.signing_endpoint = this.sshauthzbase + 'api/v1/sign_key'; this.client_id = "86c06039-e589-4b39-9d1f-9eca431be18f"; + this.route.fragment.subscribe(frag => this.getCert(frag)); + this.batchinterface = new BatchInterface(); + this.identities=[]; + this.batchinterface = { cancelcmd: 'scancel {jobid}', statcmd: '/home/chines/jsonstat.py', + submitcmd: 'sbatch --partition=m3f', internalfirewall: false } + } + + updateJoblist(resp) { + let joblist = <Job[]>resp; + for (let j of joblist) { + j.app = this.strudelappsService.getApp(j.name) + } + this.joblist.next(joblist); + this.statusMsg.next(null); + } + submitted(resp: any ) { + this.busy.next(false); + this.statusMsg.next('Updating job list'); + this.getJobs(); + } + + buildParams(app: Strudelapp, identity: Identity, batchinterface: BatchInterface): string { + let params = new URLSearchParams(); + let appstr = JSON.stringify(app); + params.set('app',appstr); + let interfacestr = JSON.stringify(batchinterface); + params.set('interface',interfacestr); + let identitystr = JSON.stringify(identity); + params.set('identity',identitystr); + return params.toString(); } getJobs() { let headers = new HttpHeaders(); let options = { headers: headers, withCredentials: true}; - this.errMsg.next(null); - this.http.get<Job[]>(this.Base+'/stat',options) + this.statusMsg.next(null); + for (let identity of this.identities) { + let paramstr = this.buildParams(null,identity,this.batchinterface); + this.http.get<Job[]>(this.Base+'/stat'+'?'+paramstr,options) .pipe(catchError(this.handleError('getJobs',[]))) - .subscribe(resp => this.joblist.next(resp)); + .subscribe(resp => this.updateJoblist(resp)); + } } - submit(app: Strudelapp) { + + submit(app: Strudelapp, identity: Identity) { console.log("In tes submit url "+this.Base+'/submit'); console.log("starting app"+app.name); let headers = new HttpHeaders(); let options = { headers: headers, withCredentials: true}; - this.errMsg.next(null); + this.statusMsg.next('Submitting job'); this.busy.next(true); - this.http.post<any>(this.Base+'/appsetup',app, options) - .pipe(catchError(this.handleError('submit',[]))) - .subscribe(resp => this.http.post<any>(this.Base+'/submit',{}, options) - .pipe(catchError(this.handleError('submit',[]))) - .subscribe(resp => this.busy.next(false))); + let paramstr = this.buildParams(app,identity,this.batchinterface); + this.http.post<any>(this.Base+'/submit'+'?'+paramstr,{}, options) + .pipe(catchError(this.handleError('submit',[]))) + .subscribe(resp => this.busy.next(false)); + } - cancel(jobid: string) { + cancel(job: Job) { console.log("In tes cancel"); let headers = new HttpHeaders(); let options = { headers: headers, withCredentials: true}; - this.errMsg.next(null); + this.statusMsg.next(null); let data = {}; - this.http.delete<any>(this.Base+'/cancel/'+jobid, options) + let paramstr = this.buildParams(job.app,job.identity,this.batchinterface); + this.http.delete<any>(this.Base+'/cancel/'+job.jobid+'?'+paramstr, options) .pipe(catchError(this.handleError('cancel',[]))) .subscribe(resp => this.submitted(resp)); } - submitted(resp: any ) { + public connect(job: Job) { + this.statusMsg.next(null); + let headers = new HttpHeaders(); + let options = { headers: headers, withCredentials: true}; + this.busy.next(true); + let paramstr = this.buildParams(job.app,job.identity,this.batchinterface); + let appwindow = window.open(this.Base+"/connect/"+job.jobid+"/"+job.batch_host+"?"+paramstr); + appwindow.focus(); this.busy.next(false); } - getCert(frag: any) { + + +getCert(frag: string) { // Given a URL fragment, extract the authentication token, and use it to get // an SSH certificate. SSH cert to be used on subsequent calls to the TES - console.log('in tesservice getCert'); + console.log('in tesservice getCert argument is',frag); + console.log('fragment.getvalue is') + console.log(frag) let match = frag.match(/access_token\=([\S\s]*?)[&|$]/); if (match == null) { - this.errMsg.next("No access token: It looks like your login to SSHAuthZ didn't work."); - this.testingAuth.next(false); - this.authorised.next(false); + return; } this.authorised.next(true); console.log(match); @@ -111,9 +170,13 @@ private sshcert: any; // let data = {'token': this.token, 'sshauthz_endpoint': this.signing_endpoint} // console.log(data); + console.log("Generating key matter") + let starttime = new Date(); this.keypair = keypair(); let publicKey = forge.pki.publicKeyFromPem(this.keypair.public); this.ssh = forge.ssh.publicKeyToOpenSSH(publicKey, 'sv2@monash.edu'); + let endtime = new Date(); + console.log("generating new keys took", endtime.valueOf() - starttime.valueOf()) let headers = new HttpHeaders(); let options = { headers: headers, withCredentials: true}; @@ -121,6 +184,7 @@ private sshcert: any; let data = {'token': this.token, 'pubkey': this.ssh, 'signing_url': this.signing_endpoint}; this.busy.next(true); + this.statusMsg.next("Generating Certificates ...") console.log('posting to getcert',this.Base); this.http.post<any>(this.Base+'/getcert',data, options) .pipe(catchError(this.handleError('getCert',[]))) @@ -135,63 +199,92 @@ private storeCert(resp) { console.log(resp['cert']); this.sshcert = resp['cert']; + let headers = new HttpHeaders(); + let options = { headers: headers, withCredentials: true}; + + this.statusMsg.next("Authorising ...") + let data = {'key': this.keypair.private, 'cert': this.sshcert}; + + this.http.post<any>(this.Base+'/sshagent',data,options) + .pipe(catchError(this.handleError('storeCert',[]))) + .subscribe(resp => this.getIdentities(resp)); +} + +private getIdentities(resp) { + this.statusMsg.next("Updating the list of available accounts") + this + let headers = new HttpHeaders(); + let options = { headers: headers, withCredentials: true}; + this.http.get<any>(this.Base+'/sshagent',options) + .pipe(catchError(this.handleError('updateIdentities',[]))) + .subscribe(resp => this.updateIdentities(resp)); +} + +private killAgent() { + this.statusMsg.next("Updating the list of available accounts") + this + let headers = new HttpHeaders(); + let options = { headers: headers, withCredentials: true}; + this.http.delete<any>(this.Base+'/sshagent',options) + .pipe(catchError(this.handleError('updateIdentities',[]))) + .subscribe(resp => this.updateIdentities(resp)); +} + +private updateIdentities(resp) { + let certcontents = resp; + console.log(certcontents); + this.identities = []; + console.log('identities updated'); + let sites = this.computesite.getComputeSites(); + for (let i in certcontents) { + let id = new Identity() + id.username = certcontents[i].Principals[0]; + id.site = sites[0] + this.identities.push(id) + } + if (this.identities.length == 0) { + this.statusMsg.next(null); + return; + } + console.log(this.identities); + this.statusMsg.next('Updating Job list'); + this.startPolling(); +} + +private startPolling() { + this.statusMsg.next(null); if (!(this.timerSubscription === null)) { + console.log('unsubscribing timer'); this.timerSubscription.unsubscribe() } console.log('creating timer to poll for jobs'); + // this.timerSubscription = timer(100,5000).subscribe(() => this.getJobs()); + this.timerSubscription = timer(5000).pipe(repeat()).subscribe(() => this.getJobs()); this.getJobs(); - } public login() { - - - let redirect_uri = window.location.origin+"/sshauthz_callback"; + let redirect_uri = window.location.origin+this.locationStrategy.getBaseHref()+"sshauthz_callback"; let state="asdfzxcv"; window.location.assign(this.sshauthz+"?response_type=token&redirect_uri="+redirect_uri+"&state="+state+"&client_id="+this.client_id); } -// public testAuth() { -// if (this.sshcert === null) { -// this.authorised.next(false); -// } else { -// this.authorised.next(true); -// } -// this.testingAuth.next(false); -// // this.tesSelected.next(true); -// // this.Base = 'http://localhost:5000' -// // this.errMsg.next(null); -// // -// // let headers = new HttpHeaders(); -// // this.testingAuth.next(true); -// // let options = { headers: headers, withCredentials: true}; -// // this.http.get(this.Base+'/testauth',options).pipe(catchError(this.handleError('submit',[]))) -// // .subscribe(resp => this.testAuthComplete(resp)); -// } - public logout(): Boolean { this.token = null; this.sshcert = null; - this.errMsg.next(null); + this.statusMsg.next(null); this.authorised.next(false); + this.killAgent(); if (!(this.timerSubscription === null)) { this.timerSubscription.unsubscribe() } return true; } -public connect(jobid: string,exechost: string,app: Strudelapp) { - this.errMsg.next(null); - let headers = new HttpHeaders(); - let options = { headers: headers, withCredentials: true}; - this.busy.next(true); - this.http.post<any>(this.Base+'/appsetup',app, options) - .pipe(catchError(this.handleError('submit',[]))) - .subscribe(resp => { window.open(this.Base+"/connect/"+jobid+"/"+exechost); this.busy.next(false)}); -} -public getErrmsgSubject(): BehaviorSubject<any> { - return this.errMsg; + +public getstatusMsgSubject(): BehaviorSubject<any> { + return this.statusMsg; } @@ -204,14 +297,12 @@ private handleError<T> (operation = 'operation', result?: T) { this.sshcert = null; if ( operation == 'getJobs') { - this.errMsg.next("Hmm, that didn't work. If you're using a local connection, please make sure Strudel-TES is running."); + this.statusMsg.next("Hmm, that didn't work. If you're using a local connection, please make sure Strudel-TES is running."); } if (operation == 'submit') { - this.errMsg.next("Hmm, I couldn't submit that job") + this.statusMsg.next("Hmm, I couldn't submit that job") } return of(result as T); }; } - - } diff --git a/src/app/teserrors/teserrors.component.html b/src/app/teserrors/teserrors.component.html index 8b4f737..d8a160b 100644 --- a/src/app/teserrors/teserrors.component.html +++ b/src/app/teserrors/teserrors.component.html @@ -1,3 +1,3 @@ <p> -{{ errmsg }} +{{ statusMsg }} </p> diff --git a/src/app/teserrors/teserrors.component.ts b/src/app/teserrors/teserrors.component.ts index 89a3a56..068b8cd 100644 --- a/src/app/teserrors/teserrors.component.ts +++ b/src/app/teserrors/teserrors.component.ts @@ -8,16 +8,16 @@ import { TesService } from '../tes.service'; }) export class TeserrorsComponent implements OnInit { - private errmsg: string; + public statusMsg: string; constructor(private tesService: TesService,) { } ngOnInit() { - this.tesService.getErrmsgSubject().subscribe( msg => this.getNewMessage(msg)); + this.tesService.getstatusMsgSubject().subscribe( msg => this.getNewMessage(msg)); } getNewMessage(msg) { - this.errmsg = msg; + this.statusMsg = msg; } } diff --git a/src/app/tokenextractor/tokenextractor.component.ts b/src/app/tokenextractor/tokenextractor.component.ts index 7366e36..0376d04 100644 --- a/src/app/tokenextractor/tokenextractor.component.ts +++ b/src/app/tokenextractor/tokenextractor.component.ts @@ -16,7 +16,7 @@ export class TokenextractorComponent implements OnInit { } ngOnInit() { - this.route.fragment.subscribe( frag => this.tesService.getCert(frag)); + // this.route.fragment.subscribe( frag => this.tesService.setFragment(frag)); this.router.navigate(['/']); } -- GitLab