Skip to content
Snippets Groups Projects
Commit 5e51f73f authored by Chris Hines's avatar Chris Hines
Browse files

Merge branch 'dev' into 'test' 

Dev

See merge request !62
parents 93f956ff 0dcb8a3a
No related branches found
No related tags found
2 merge requests!64Test,!62Dev
Pipeline #16277 passed
Stage: build
Stage: deploy
Hide whitespace changes
Inline Side-by-side
tes/apiendpoints.py
+ 82
− 100
View file @ 5e51f73f
......@@ -10,6 +10,7 @@ import flask_restful
from . import api, app
from .sshwrapper import Ssh, SshAgentException, SftpPermissionException, SftpException, SshCtrlException, SshExecException
from .tunnelstat import SSHSession
import subprocess
class AppParamsException(Exception):
pass
......@@ -93,6 +94,7 @@ class SSHAgent(Resource):
sshsess = SSHSession.get_sshsession()
sshsess.kill()
SSHSession.remove_sshsession()
session.pop('sshsessid', None)
return []
except Exception as e:
import traceback
......@@ -186,6 +188,57 @@ class ContactUs(Resource):
f.close()
return
def wrap_execute(sess, host, user, cmd, bastion=None,stdin=None, sshport="22", bastionsshport="22"):
"""
This function is supposed to interpret all the possible exceptions from Ssh.execute and generate
approrparite HTTP errors (both status codes and messages)
There are a range of situations:
it might raise an SshExecException (non-zero return code)
it might return valid data or not
it might return stderr or not
the data on stderr might be valid json or not.
"""
try:
res = Ssh.execute(sess, host, user, cmd, bastion, stdin, sshport, bastionsshport)
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
logger.error(res['stderr'])
#flask_restful.abort(400, message=res['stderr'].decode())
return apiabort(400, message=res['stderr'].decode())
try:
if res['stdout'].decode() != '':
data = json.loads(res['stdout'].decode())
return data
else:
return None
except json.decoder.JSONDecodeError:
return None
#return apiabort(500, data = {'stdout':res['stdout'].decode(),'stderr':res['stderr'].decode()})
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
return apiabort(400, message=e)
except SshCtrlException as e:
if ("{}".format(e) != ''):
return apiabort(400,message="{}".format(host,e))
else:
# This is the most common error, if the user disconnected from the network and the api server cleaned up all the agents and control sockets
return apiabort(401,message="".format(host))
# This exception is raised if the remove command had a non-zero return code
except SshExecException as e:
return apiabort(400, message="{}".format(e))
# Any other exceptions. This should never be reached.
except subprocess.TimedoutExpired as e:
return apiabort(504, message="{}".format(e))
except Exception as e:
import traceback
logger.error('JobStat.get: Exception {}'.format(e))
logger.error(traceback.format_exc())
return apiabort(500,message="{}".format(e))
class JobStat(Resource):
"""
endpoints to return info on jobs on the backend compute Resource
......@@ -199,48 +252,18 @@ class JobStat(Resource):
try:
sshsess = SSHSession.get_sshsession()
sshsess.refresh()
except:
#flask_restful.abort(500, message="Error relating to the ssh sessions")
return apiabort(500, message="Error relating to the ssh sessions")
except Exception as e:
return apiabort(500, message="{}".format(e))
try:
cmd = json.loads(request.args.get('statcmd'))
host = json.loads(request.args.get('host'))
user = json.loads(request.args.get('username'))
except (TypeError, KeyError) as e:
#flask_restful.abort(400, message="Missing required parameter {}".format(e))
import traceback
return apiabort(400, message="Missing required parameter {}\n{}".format(e,traceback.format_exc()))
try:
res = Ssh.execute(sshsess, host=host, user=user, cmd=cmd)
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
logger.error(res['stderr'])
#flask_restful.abort(400, message=res['stderr'].decode())
return apiabort(400, message=res['stderr'].decode())
try:
jobs = json.loads(res['stdout'].decode())
return jobs
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(400, message=e)
return apiabort(400, message=e)
except SshAgentException as e:
logger.error(e)
#flask_restful.abort(401, message="Identity error {}".format(e))
return apiabort(401, message="Identity error {}".format(e))
except SshCtrlException as e:
#flask_restful.abort(400,message="We're having difficultly contacting {}. We failed with the message: {}".format(host,e))
return apiabort(400,message="We're having difficultly contacting {}. We failed with the message: {}".format(host,e))
except SshExecException as e:
return apiabort(400,message="{}".format(e))
except Exception as e:
import traceback
logger.error('JobStat.get: Exception {}'.format(e))
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="SSH failed in an unexpected way")
return apiabort(500,message="SSH failed in an unexpected way")
rv = wrap_execute(sshsess, host=host, user=user, cmd=cmd)
return rv
class MkDir(Resource):
def post(self):
......@@ -308,14 +331,17 @@ class DirList(Resource):
except:
#return json.dumps({'message':"You don't have permission to view that directory"}), 401
#flask_restful.abort(401,message="You don't have permission to view that directory")
return apiabort(401,message="You don't have permission to view that directory")
return apiabort(403,message="You don't have permission to view that directory")
else:
dirls = Ssh.sftpls(sshsess, host=params['identity']['site']['host'],
user=params['identity']['username'], path=params['path'],changepath=params['cd'], sshport=sshport)
return dirls
except SshCtrlException as e:
#flask_restful.abort(400,message="We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e))
return apiabort(400,message="We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e))
if ("{}".format(e) != ''):
return apiabort(400,message="{}".format(e))
else:
# This is the most common error, if the user disconnected from the network and the api server cleaned up all the agents and control sockets
return apiabort(401)
except Exception as e:
import traceback
logger.error(e)
......@@ -334,18 +360,10 @@ class JobCancel(Resource):
try:
params = get_conn_params()
sshsess = SSHSession.get_sshsession()
try:
res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
res = wrap_execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
cmd=params['interface']['cancelcmd'].format(jobid=jobid))
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
#return json.dumps({'message':res['stderr'].decode()}), 400
#flask_restful.abort(400, message=res['stderr'].decode())
return apiabort(400, message=res['stderr'].decode())
return res['stdout'].decode()
except SshCtrlException as e:
#return json.dumps({'message':"We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e)}), 400
#flask_restful.abort(400,message="We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e))
return apiabort(400,message="We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e))
return res
except Exception as e:
import traceback
logger.error(e)
......@@ -384,20 +402,9 @@ class JobSubmit(Resource):
#flask_restful.abort(400, message='Incomplete job information was passed to the backend.')
return apiabort(400, message='Incomplete job information was passed to the backend.')
try:
res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
cmd=params['interface']['submitcmd'], stdin=script)
except SshCtrlException as e:
#return json.dumps({'message':"We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e)}), 400
#flask_restful.abort(400,message="We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e))
return apiabort(400, message="We're having difficultly contacting {}. We failed with the message {}".format(params['identity']['site']['host'],e))
except SshExecException as e:
return apiabort(400, message="{}".format(e))
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
#return json.dumps({'message':res['stderr'].decode()}), 400
#flask_restful.abort(400, message=res['stderr'].decode())
return apiabort(400, message=res['stderr'].decode())
return res['stdout'].decode()
res = wrap_execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
cmd=params['interface']['submitcmd'], stdin=script)
return res
except Exception as e:
import traceback
logger.error(e)
......@@ -417,8 +424,11 @@ def gen_authtok():
return ''.join(random.SystemRandom().choice(string.ascii_uppercase +
string.digits) for _ in range(16))
def apiabort(code, message):
return {'message':message}, code
def apiabort(code, message=None, data = None):
if data is not None:
return data, code
else:
return {'message':message}, code
class AppUrl(Resource):
def get(self):
......@@ -470,7 +480,7 @@ class AppLaunch(Resource):
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="AppUrl failed in some unexpected way")
return apiabort(500,message="AppUrl failed in some unexpected way")
return apiabort(500,message="AppLaunc failed in some unexpected way")
logger.debug('application launched, returning')
return None, 200
......@@ -480,48 +490,20 @@ class AppInstance(Resource):
command is passed as a query string"""
import logging
logger=logging.getLogger()
sshsess = SSHSession.get_sshsession()
try:
sshsess = SSHSession.get_sshsession()
paramscmd = json.loads(request.args.get('cmd')).format(jobid=jobid)
import logging
logger = logging.getLogger()
#cmd = 'ssh -o StrictHostKeyChecking=no -o CheckHostIP=no {batchhost} '.format(batchhost=batchhost) + paramscmd
try:
res = Ssh.execute(sshsess, host=batchhost, bastion=loginhost, user=username, cmd=paramscmd)
except:
message = "The server couldn't execute to {} to get the necessary info".format(batchhost)
#flask_restful.abort(400, message=message)
import traceback
logger.error(traceback.format_exc())
return apiabort(400, message=message)
try:
data = json.loads(res['stdout'].decode())
if 'error' in data:
return data, 400
return data
except json.decoder.JSONDecodeError as e:
logger.error(res['stderr']+res['stdout'])
message="I'm having trouble using ssh to find out about that application"
#flask_restful.abort(400, message=message)
return apiabort(400, message=message)
#raise AppParamsException(res['stderr']+res['stdout'])
if len(res['stderr']) > 0:
logger.error(res['stderr']+res['stdout'])
#flask_restful.abort(400, message="The command {} on {} didn't work".format(paramscmd,batchhost))
return apiabort(400, message="The command {} on {} didn't work".format(paramscmd,batchhost))
#raise AppParamsException(res['stderr'])
if 'error' in data:
raise AppParamsException(data['error'])
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
#flask_restful.abort(400, message=res['stderr'].decode())
return apiabort(400, message=res['stderr'].decode())
return data
except json.decoder.JSONDecodeError:
return apiabort(400, message="No command was sent to the API server")
try:
rv = wrap_execute(sshsess, host=batchhost, bastion=loginhost, user=username, cmd=paramscmd)
return rv
except Exception as e:
import traceback
logger.error(e)
logger.error(traceback.format_exc())
#flask_restful.abort(500,message="AppUrl failed in some unexpected way")
return apiabort(500,message="AppUrl failed in some unexpected way")
return apiabort(500,message="AppInstance failed in some unexpected way")
class CreateTunnel(Resource):
@staticmethod
......
tes/sshwrapper/__init__.py
+ 10
− 3
View file @ 5e51f73f
......@@ -337,7 +337,8 @@ class Ssh:
except subprocess.TimeoutExpired as e:
logger.debug('cmd timed out')
logger.debug("exception execute")
raise SshExecException(message="A program timed out on the backend server")
raise e
#raise SshExecException(message="A program timed out on the backend server")
if exec_p.returncode == 255:
logger.error(stderr)
sess.kill()
......@@ -360,15 +361,21 @@ class Ssh:
logger.error('ran command {}'.format(" ".join(sshcmd)))
logger.error('got return code {} stderr {} stdout {}'.format(exec_p.returncode,stderr,stdout))
logger.debug("exception execute")
raise SshExecException(message="A program failed to execute on the backend server.\n It gave an error message\n{}.\nHopefully this is helpful".format(stderr))
raise SshExecException(message="{}".format(stderr.decode()))
#sess.kill()
#raise SshCtrlException(message="ssh execute failed, killing off the agent. Log in again")
# If the return code is non-zero, its likely something went wrong logging in. We should perhaps consider killing the ssh session, or at least the ctrl socket processes
# And allowing the client to retry them
# I haven't decided yet whether we should kill the whole agent or just the ctrl process
# Because people often put stuff in their bashrc which casuses stderr to be non-empty, we will swallow errors in the bashrc without comment
if exec_p.returncode != 0:
if stderr == b'':
msg = "The program {} on {} failed".format(cmd,host)
else:
msg = "The program {} on {} failed. The error message was {}".format(cmd,host,stderr.decode())
raise SshExecException(message=msg)
logger.debug("leaving execute")
# Because people often put stuff in their bashrc which casuses stderr to be non-empty, we will swallow errors in the bashrc without comment
if b'bashrc' in stderr:
return {'stdout':stdout,'stderr':b''}
else:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment