-
Chris Hines authoredChris Hines authored
apiendpoints.py 26.88 KiB
"""
All the API endpoints for controling processes and tunnels via SSH
"""
import json
from flask import session, redirect, request, Response, make_response
from flask_restful import Resource
from flask import render_template
import flask_restful
from . import api, app
from .sshwrapper import Ssh, SshAgentException, SftpPermissionException, SftpException, SshCtrlException, SshExecException
from .tunnelstat import SSHSession
import subprocess
class AppParamsException(Exception):
pass
class GetCert(Resource):
"""
This class is necessary because I'm not reconfiguring
SSHAuthZ to support CORS, but the TES does support CORS
"""
def post(self):
"""
takes a public key, returns to the SPA a certificate
"""
import logging
logger = logging.getLogger()
try:
data = request.get_json()
if type(data) is not dict:
return apiabort(400, message="necessary data not provided in post")
response = {'cert':GetCert.get_cert(data['token'], data['pubkey'], data['signing_url'])}
return response
except:
import traceback
logger.error('Failed to get certificate')
logger.error(traceback.format_exc())
flask_restful.abort(400,message="an error occured generating your certificate")
@staticmethod
def get_cert(access_token, pub_key, url):
"""
Sign a pub key into a cert
"""
import requests
import logging
logger = logging.getLogger()
sess = requests.Session()
headers = {"Authorization":"Bearer %s"%access_token}
data = {"public_key":pub_key}
resp = sess.post(url, json=data, headers=headers, verify=False)
data = resp.json()
return data['certificate']
class SSHAgent(Resource):
def post(self):
import logging
logger = logging.getLogger()
try:
session.permanent = True
from .tunnelstat import SSHSession
sshsess = SSHSession.get_sshsession()
data = request.get_json()
if type(data) is not dict:
return apiabort(400, message="necessary data not provided in post")
sshsess.add_keycert(key=data['key'],cert=data['cert'])
sshsess.refresh()
return "OK"
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
logger.error('failed to add ssh key to the agent')
flask_restful.abort(500,message="failed to add the ssh key to the agent")
def get(self):
from .tunnelstat import SSHSession
import logging
logger = logging.getLogger()
try:
sshsess = SSHSession.get_sshsession()
if sshsess.socket == None:
return [] # The agent hasn't even been started yet
return sshsess.get_cert_contents()
except Exception as e:
import traceback
logger.error('SSHAgent.get: Exception {}'.format(e))
logger.error(traceback.format_exc())
flask_restful.abort(500,message="failed to query the ssh-agent")
def delete(self):
import logging
logger = logging.getLogger()
from .tunnelstat import SSHSession
try:
sshsess = SSHSession.get_sshsession()
sshsess.kill()
SSHSession.remove_sshsession()
session.pop('sshsessid', None)
return []
except Exception as e:
import traceback
logger.error(traceback.format_exc())
flask_restful.abort(500,message="failed to shut down ssh-agent")
def get_conn_params():
"""
Return parameters relating to the backend compute service
Retrieve them from the query string
"""
import logging
logger = logging.getLogger()
identitystr = request.args.get('identity')
if type(identitystr) is str:
identityparams = json.loads(identitystr)
else:
identityparams = {}
try:
interfacestr = request.args.get('interface')
if type(interfacestr) is str:
interfaceparams = json.loads(interfacestr)
else:
interfaceparams = {}
except:
interfaceparams = {}
pass
try:
pathstr = request.args.get('path')
if type(pathstr) is str:
pathparams = json.loads(pathstr)
else:
pathparams = None
cdstr = request.args.get('cd')
if type(cdstr) is str:
cdparams = json.loads(cdstr)
else:
cdparams = None
except:
pathparams = None
cdparams = None
try:
appstr = request.args.get('app')
if type(appstr) is str:
appparams = json.loads(appstr)
else:
appparams = {}
except:
appparams = {}
try:
appinstancestr = request.args.get('appinstance')
if type(appinstancestr) is str:
appinstanceparams = json.loads(appinstancestr)
else:
appinstanceparams = {}
except:
appinstanceparams = {}
params = {}
params['identity'] = identityparams
params['interface'] = interfaceparams
params['app'] = appparams
params['appinstance'] = appinstanceparams
params.update(interfaceparams)
params['user'] = identityparams['username']
params['host'] = identityparams['site']['host']
params['path'] = pathparams
params['cd'] = cdparams
return params
class TunnelstatEP(Resource):
"""
Endpoints used by the WS proxy
The Frontend should never call this
"""
def get(self, authtok):
"""
given an authtoken, return the port of the tunnels
"""
import logging
logger = logging.getLogger()
from . import sshsessions
port = None
try:
for (sessid,sshsess) in sshsessions.items():
for (tok,port) in sshsess.port.items():
if tok == authtok:
return port
except:
logger.error("exception in TunnelstatEP.get")
import traceback
logger.error(traceback.format_exc())
return None
class ContactUs(Resource):
def post(self):
import tempfile
data = request.get_json()
f = tempfile.NamedTemporaryFile(mode='w+b',dir=app.config['MESSAGES'],delete=False)
f.write(json.dumps(data).encode())
f.close()
return
def wrap_execute(sess, host, user, cmd, bastion=None,stdin=None, sshport="22", bastionsshport="22"):
"""
This function is supposed to interpret all the possible exceptions from Ssh.execute and generate
appropriate HTTP errors (both status codes and messages)
There are a range of situations:
it might raise an SshExecException (non-zero return code)
it might return valid data or not
it might return stderr or not
the data on stderr might be valid json or not.
"""
import logging
logger = logging.getLogger()
try:
res = Ssh.execute(sess, host, user, cmd, bastion, stdin, sshport, bastionsshport)
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
logger.error(res['stderr'])
#flask_restful.abort(400, message=res['stderr'].decode())
return apiabort(400, message=res['stderr'].decode(), sess=sess)
try:
if res['stdout'].decode() != '':
data = json.loads(res['stdout'].decode())
return data
else:
return None
except json.decoder.JSONDecodeError:
return None
#return apiabort(500, data = {'stdout':res['stdout'].decode(),'stderr':res['stderr'].decode()})
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
return apiabort(400, message=e, sess=sess)
except SshCtrlException as e:
return apiabort(401,message="{} {}".format(host,e), sess=sess, logdata = json.dumps({'user':user,'cmd':cmd,'host':host}))
# This exception is raised if the remove command had a non-zero return code
except SshExecException as e:
return apiabort(400, message="{}".format(e), sess=sess)
# Any other exceptions. This should never be reached.
except subprocess.TimeoutExpired as e:
return apiabort(504, message="{}".format(e), sess=sess)
except Exception as e:
import traceback
logger.error('JobStat.get: Exception {}'.format(e)
)
logger.error(traceback.format_exc())
return apiabort(500,message="{}".format(e), sess=sess)
class JobStat(Resource):
"""
endpoints to return info on jobs on the backend compute Resource
"""
def get(self):
"""
get info on the job from the backend
"""
import logging
logger = logging.getLogger()
logger.debug('entered jobstat')
try:
sshsess = SSHSession.get_sshsession()
except Exception as e:
return apiabort(500, message="{}".format(e))
try:
s = request.args.get('statcmd')
if type(s) is str:
cmd = json.loads(s)
else:
cmd = None
s = request.args.get('host')
if type(s) is str:
host = json.loads(s)
else:
host = None
s = request.args.get('username')
if type(s) is str:
user = json.loads(s)
else:
user = None
except (TypeError, KeyError) as e:
import traceback
return apiabort(400, message="Missing required parameter {}\n{}".format(e,traceback.format_exc()), sess=sshsess)
rv = wrap_execute(sshsess, host=host, user=user, cmd=cmd)
return rv
class MkDir(Resource):
def post(self):
import logging
logger = logging.getLogger()
try:
data = request.get_json()
params = get_conn_params()
sshsess = SSHSession.get_sshsession()
site = params['identity']['site']
if type(data) is not dict:
return apiabort(400, message="necessary parameters not provided in the post information")
if 'dtnport' in site:
sshport = site['dtnport']
else:
sshport = "22"
try:
Ssh.sftpmkdir(sshsess, host=params['identity']['site']['host'],
user=params['identity']['username'], path=params['path'],name=data['name'], sshport=sshport)
except SftpPermissionException as e:
#return json.dumps({'message':"You don't have permission to make a directory there"}), 403
#flask_restful.abort(403,message="You don't have permission to make a directory there")
return apiabort(403,message="You don't have permission to make a directory there")
except SftpException as e:
#return json.dumps({'message':"Something went wrong making that directory"}), 500
#flask_restful.abort(500,message="Something went wrong making that directory")
return apiabort(500,message="Something went wrong making that directory")
except Exception as e:
import traceback
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="Something went wrong creating that directory, probably a bug")
return apiabort(500,message="Something went wrong creating that directory, probably a bug")
return
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="mkdir failed in some unexpected way")
return apiabort(500,message="mkdir failed in some unexpected way")
class DirList(Resource):
def get(self):
import logging
logger = logging.getLogger()
try:
params = get_conn_params()
sshsess = SSHSession.get_sshsession()
site = params['identity']['site']
if 'dtnport' in site:
sshport = site['dtnport']
else:
sshport = "22"
path = params['path']
cd = params['cd']
if path == "":
path = "."
if cd == "":
cd = "."
try:
if 'lscmd' in site and site['lscmd'] is not None and site['lscmd'] != "":
res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
sshport=sshport,
cmd="{} {} {}".format(site['lscmd'],path,cd))
try:
dirls = json.loads(res['stdout'].decode())
except:
#return json.dumps({'message':"You don't have permission to view that directory"}), 401
#flask_restful.abort(401,message="You don't have permission to view that directory")
return apiabort(403,message="You don't have permission to view that directory")
else:
dirls = Ssh.sftpls(sshsess, host=params['identity']['site']['host'],
user=params['identity']['username'], path=params['path'],changepath=params['cd'], sshport=sshport)
return dirls
except SshCtrlException as e:
if ("{}".format(e) != ''):
return apiabort(400,message="{}".format(e))
else:
# This is the most common error, if the user disconnected from the network and the api server cleaned up all the agents and control sockets
return apiabort(401)
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="dirlist failed in some unexpected way")
return apiabort(500,message="dirlist failed in some unexpected way")
class JobCancel(Resource):
"""
Terminate a job on the compute backend
"""
def delete(self, jobid):
import logging
logger = logging.getLogger()
"""
Terminate a job on the backend
"""
sshsess = None
try:
params = get_conn_params()
sshsess = SSHSession.get_sshsession()
res = wrap_execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
cmd=params['interface']['cancelcmd'].format(jobid=jobid))
return res
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
#return json.dumps({'message':"jobcancel failed in some unexpected way"}), 500
#flask_restful.abort(500,message="jobcancel failed in some unexpected way")
return apiabort(500,message="jobcancel failed in some unexpected way", sess=sshsess)
class RemoteCommand(Resource):
# jobsubmit res = wrap_execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'], cmd=params['interface']['submitcmd'], stdin=script)
# appinstance rv = wrap_execute(sshsess, host=batchhost, bastion=loginhost, user=username, cmd=paramscmd) ... part of path
# jobscancel res = wrap_execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'], cmd=params['interface']['cancelcmd'].format(jobid=jobid))
def post(self):
import logging
logger=logging.getLogger()
try:
sshsess = SSHSession.get_sshsession()
data = request.get_json()
cmd = None
bastion = None
if type(data) is dict:
user = data['user']
host = data['host']
if 'bastion' in data:
bastion = data['bastion']
else:
bastion = None
cmd = data['cmd']
else:
return apiabort(400, message="necessary data not provided")
try:
if 'input' in data:
input = data['input'].format(**data)
else:
input = None
res = wrap_execute(sshsess, host, user, cmd, bastion, stdin=input)
return res
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
return apiabort(500,message="Remote Command failed in some unexpected way", sess=sshsess)
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
return apiabort(500,message="Remote command failed in some unexpected way")
class JobSubmit(Resource):
"""
Class dealing the starting a new job on the compute backend
"""
def post(self):
"""starting a job is a post, since it changes the state of the backend"""
import logging
logger=logging.getLogger()
sshsess = None
try:
params = get_conn_params()
sshsess = SSHSession.get_sshsession()
data = request.get_json()
if type(data) is not dict:
return apiabort(400, message="necessary data not provided in the post information")
try:
script = data['app']['startscript'].format(**data)
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
logger.error('formating data')
logger.error(data)
logger.error('end formating data')
logger.error('body')
logger.error(request.data)
logger.error('end body')
#return json.dumps({'message':'Incomplete job information was passed to the backend.'}), 400
#flask_restful.abort(400, message='Incomplete job information was passed to the backend.')
return apiabort(400, message='Incomplete job information was passed to the backend.', sess=sshsess)
res = wrap_execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
cmd=params['interface']['submitcmd'], stdin=script)
return res
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
#return json.dumps({'message':"JobSubmission failed in some unexpected way"}), 500
#flask_restful.abort(500,message="JobSubmission failed in some unexpected way")
return apiabort(500,message="JobSubmission failed in some unexpected way", sess=sshsess)
def gen_authtok():
"""
generate a random string suitable for an auth token stored in a cookie
"""
import random
import string
import logging
logger=logging.getLogger()
return ''.join(random.SystemRandom().choice(string.ascii_uppercase +
string.digits) for _ in range(16))
def apiabort(code, message=None, data = None, sess = None, logdata = None):
import logging
import traceback
from flask import g
import time
logger = logging.getLogger('apiabort')
logger.error("request time {}".format(time.time() - g.start))
if sess is not None:
logger.error("certs {}".format(sess.get_certs()))
logger.error(logdata)
logger.error("{} {} {}".format(code,data,message))
logger.error("{}".format(traceback.format_exc()))
if data is not None:
return data, code
else:
return {'message':message}, code
class AppUrl(Resource):
def get(self):
import logging
logger = logging.getLogger()
try:
s = request.args.get('app')
if type(s) is str:
appdef = json.loads(s)
else:
return apiabort(400, message="Failed to provide information on the application")
s = request.args.get('appinst')
if type(s) is str:
inst = json.loads(s)
else:
return apiabort(400, message="failed to provide information in the application instance")
inst['twsproxy']='{twsproxy}'
inst['twsproxyauth'] = request.cookies.get('twsproxyauth')
logger.debug('twsproxyauth {} {}'.format(inst['twsproxyauth'],request.cookies.get('twsproxyauth')))
url = "{}/{}".format("{twsproxy}",appdef['client']['redir'].format(**inst))
return url
except Exception as e:
import traceback
logger.error('AppUrl failed')
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="AppUrl failed in some unexpected way")
return apiabort(500,message="AppUrl failed in some unexpected way")
class SetTWSProxyAuth(Resource):
def get(self, authtok):
import urllib.parse
import logging
import datetime
logger = logging.getLogger()
s = request.args.get('redirect')
if type(s) is str:
url = urllib.parse.unquote(s)
else:
return apiabort(400, message="no redirect provided")
logger.debug('SetTWSProxyAuth will redirect to {}'.format(url))
response = make_response(redirect(url))
response.set_cookie('twsproxyauth', authtok, secure=True, httponly=True, expires = datetime.datetime.now()+datetime.timedelta(days=30))
return response
class Ping(Resource):
def get(self):
return None, 201
class AppInstance(Resource):
def get(self, username, loginhost, batchhost, jobid):
"""Run a command to get things like password and port number
command is passed as a query string"""
import logging
import hashlib
logger=logging.getLogger()
events = logging.getLogger('connection_events')
sshsess = SSHSession.get_sshsession()
sessionId = hashlib.sha1(sshsess.sshsessid.encode()).hexdigest()
s = request.args.get('cmd')
if type(s) is str:
paramscmd = json.loads(s).format(jobid=jobid)
else:
return apiabort(400, message="no command provided to find information about the application instance")
try:
rv = wrap_execute(sshsess, host=batchhost, bastion=loginhost, user=username, cmd=paramscmd)
events.info('connect {}'.format(json.dumps({'user':username, 'jobid':jobid, 'loginhost': loginhost, 'sessid':sessionId})))
return rv
except Exception as e:
import traceback
logger.error('AppInstance failed')
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="AppUrl failed in some unexpected way")
return apiabort(500,message="AppInstance failed in some unexpected way")
class CreateTunnel(Resource):
@staticmethod
def validate_connect_params(port, username, host, batchhost):
try:
intport = int(port)
except Exception as e:
return False
if ' ' in username or '\n' in username: # This really needs more validation
return False
if ' ' in host or '\n' in host: # This really needs more validation
return False
return True
def post(self,username,loginhost,batchhost):
"""
Create a tunnel using established keys
parameters for the tunnel (host username port etc)
will be passed in the body
"""
import logging
import json
logger = logging.getLogger()
try:
data = request.get_json()
if type(data) is not dict:
return apiabort(400, message="necessary data not provided in the post information")
try:
port = data['port']
except KeyError as missingdata:
raise AppParamsException("missing value for {} data contains {}".format(missingdata,data))
if not CreateTunnel.validate_connect_params(port, username, loginhost, batchhost):
raise AppParamsException("Invalid value: {} {} {} {}".
format(username, loginhost, batchhost, port))
if 'internalfirewall' in data:
firewall = data['internalfirewall']
else:
firewall = False
if 'localbind' in data:
localbind = data['localbind']
else:
localbind = True
sshsess = SSHSession.get_sshsession()
if 'token' in data:
authtok = data['token']
logger.info('using token for authenticating tunnel')
else:
authtok = gen_authtok()
port,pids = Ssh.tunnel(sshsess, port=port, batchhost=batchhost,
user=username, host=loginhost,
internalfirewall=firewall,
localbind=localbind, authtok=authtok)
response = make_response(json.dumps({'localport':port}),200)
response.mimetype = 'application/json'
response.set_cookie('twsproxyauth', authtok, secure=True, samesite='None')
return response
except AppParamsException as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
return apiabort(400,message="Unable to create the secure tunnel. Please try again.")
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="CreateTunnel failed in some unexpected way")
return apiabort(500,message="CreateTunnel failed in some unexpected way")
api.add_resource(TunnelstatEP, '/tunnelstat/<string:authtok>')
api.add_resource(GetCert, '/getcert')
api.add_resource(JobStat, '/stat') # trying to deprecate
api.add_resource(JobSubmit, '/submit') # trying to deprecate
api.add_resource(AppInstance, '/appinstance/<string:username>/<string:loginhost>/<string:batchhost>/<string:jobid>') # trying to deprecate
api.add_resource(JobCancel, '/cancel/<string:jobid>') # trying to deprecate
api.add_resource(RemoteCommand,'/remotecommand')
api.add_resource(Ping, '/ping')
api.add_resource(SetTWSProxyAuth, '/settwsproxyauth/<string:authtok>')
api.add_resource(CreateTunnel, '/createtunnel/<string:username>/<string:loginhost>/<string:batchhost>')
api.add_resource(AppUrl, '/appurl')
api.add_resource(SSHAgent,'/sshagent')
api.add_resource(DirList,'/ls')
api.add_resource(MkDir,'/mkdir')
api.add_resource(ContactUs,'/contactus')