Skip to content
Snippets Groups Projects
Commit a1f06226 authored by Ubuntu's avatar Ubuntu
Browse files

manual merge

parents e9e14669 2024cbb9
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
runserver.py
+ 1
− 1
View file @ a1f06226
from tes import app
import logging
#logging.basicConfig(filename="tes.log",format="%(asctime)s %(levelname)s:%(process)s: %(message)s")
logging.basicConfig(filename="tes.log",format="%(asctime)s %(levelname)s:%(process)s: %(message)s")
logger=logging.getLogger()
logger.setLevel(logging.DEBUG)
......
tes/apiendpoints.py
+ 125
− 107
View file @ a1f06226
......@@ -84,6 +84,7 @@ class SSHAgent(Resource):
try:
sshsess = SSHSession.get_sshsession()
if sshsess.socket == None:
logger.debug('trying to get the agent contents, but the agent isn\'t started yet')
return [] # The agent hasn't even been started yet
return sshsess.get_cert_contents()
except Exception as e:
......@@ -129,11 +130,18 @@ def get_conn_params():
appparams = json.loads(appstr)
except:
appparams = {}
try:
appinstancestr = request.args.get('appinstance')
appinstanceparams = json.loads(appinstancestr)
except:
appinstanceparams = {}
params = {}
params['identity'] = identityparams
params['interface'] = interfaceparams
params['app'] = appparams
params['appinstance'] = appinstanceparams
params.update(interfaceparams)
params['user'] = identityparams['username']
......@@ -146,14 +154,6 @@ def get_conn_params():
return params
def get_app_params():
"""
Return the parameters for the application retrieved from the Session
"""
keys = ['startscript', 'paramscmd', 'client','localbind']
appstr = request.args.get('app')
returnvalue = json.loads(appstr)
return returnvalue
class TunnelstatEP(Resource):
"""
......@@ -204,6 +204,7 @@ class JobStat(Resource):
"""
import logging
logger = logging.getLogger()
logger.debug('enteringing JobStat')
try:
params = get_conn_params()
except:
......@@ -225,7 +226,10 @@ class JobStat(Resource):
cmd = params['interface']['statcmd']
except (TypeError, KeyError) as e:
flask_restful.abort(400, message="stat: definition of batch interface incomplete")
logger.debug('ssh sess socket is {}'.format(sshsess.socket))
try:
logger.debug('attempting ssh execute {} {} {}'.format(host,user,cmd))
res = Ssh.execute(sshsess, host=host, user=user, cmd=cmd)
except SshAgentException as e:
logger.error(e)
......@@ -240,6 +244,7 @@ class JobStat(Resource):
for j in jobs:
j['identity'] = params['identity']
logger.debug('leaving jobstat gracefully')
return jobs
except Exception as e:
import traceback
......@@ -247,12 +252,46 @@ class JobStat(Resource):
logger.error(traceback.format_exc())
flask_resful.abort(400, message=e)
class MkDir(Resource):
def post(self):
import logging
logger = logging.getLogger()
data = request.get_json()
logger.debug('mkdir data')
logger.debug(data)
params = get_conn_params()
sshsess = SSHSession.get_sshsession()
Ssh.sftpmkdir(sshsess, host=params['identity']['site']['host'],
user=params['identity']['username'], path=params['path'],name=data['name'])
return
class DirList(Resource):
def get(self):
params = get_conn_params()
sshsess = SSHSession.get_sshsession()
dirls = Ssh.sftpls(sshsess, host=params['identity']['site']['host'],
user=params['identity']['username'], path=params['path'],changepath=params['cd'])
site = params['identity']['site']
if 'dtnport' in site:
sshport = site['dtnport']
else:
sshport = "22"
path = params['path']
cd = params['cd']
if path == "":
path = "."
if cd == "":
cd = "."
print('sshport is {}'.format(sshport))
if 'lscmd' in site and site['lscmd'] is not None and site['lscmd'] is not "":
res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
sshport=sshport,
cmd="{} {} {}".format(site['lscmd'],path,cd))
dirls = json.loads(res['stdout'].decode())
print(dirls)
else:
dirls = Ssh.sftpls(sshsess, host=params['identity']['site']['host'],
user=params['identity']['username'], path=params['path'],changepath=params['cd'], sshport=sshport)
return dirls
class JobCancel(Resource):
......@@ -278,6 +317,7 @@ class JobSubmit(Resource):
"""
def post(self):
"""starting a job is a post, since it changes the state of the backend"""
print("entering jobsubmit")
import logging
logger=logging.getLogger()
params = get_conn_params()
......@@ -288,7 +328,7 @@ class JobSubmit(Resource):
try:
script = data['app']['startscript'].format(**data)
except:
flask_restful.abort(400,message='Incomplete job information was passed to the backend.')
flask_restful.abort(400, message='Incomplete job information was passed to the backend.')
logger.debug('script formated to {}'.format(script))
res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
......@@ -311,62 +351,46 @@ def gen_authtok():
return ''.join(random.SystemRandom().choice(string.ascii_uppercase +
string.digits) for _ in range(16))
class JobConnect(Resource):
"""
endpoints for connecting to an existing JobCancel
"""
def create_tunnel(self, username, loginhost, appparams, batchhost, firewall, data):
class AppUrl(Resource):
def get(self):
import logging
logger=logging.getLogger()
logger.debug('entering JobConnect.create_tunnel {} {}'.format(username,batchhost))
connectparams = {}
logger = logging.getLogger()
appdef = json.loads(request.args.get('app'))
logger.debug('appdef {}'.format(appdef))
inst = json.loads(request.args.get('appinst'))
logger.debug('appinst {}'.format(inst))
url = "{}{}".format(app.config['TWSPROXY'],appdef['client']['redir'].format(**inst))
return url
class AppInstance(Resource):
def get(self, username, loginhost, batchhost):
"""Run a command to get things like password and port number
command is passed as a query string"""
sshsess = SSHSession.get_sshsession()
paramscmd = request.args.get('cmd')
import logging
logger = logging.getLogger()
logger.debug('getting appinstance {} {} {}'.format(username,loginhost,batchhost))
logger.debug('ssh sess socket is {}'.format(sshsess.socket))
cmd = 'ssh -o StrictHostKeyChecking=no -o CheckHostIP=no {batchhost} '.format(batchhost=batchhost) + paramscmd
res = Ssh.execute(sshsess, host=loginhost, user=username, cmd=cmd)
try:
data = json.loads(res['stdout'].decode())
except json.decoder.JSONDecodeError as e:
raise AppParamsException(res['stderr']+res['stdout'])
if len(res['stderr']) > 0:
raise AppParamsException(res['stderr'])
if 'error' in data:
raise AppParamsException(data['error'])
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
flask_restful.abort(400, message=res['stderr'].decode())
return data
if 'paramscmd' in appparams and appparams['paramscmd'] is not None:
connectparams['batchhost'] = batchhost
paramcmd = 'ssh -o StrictHostKeyChecking=no -o CheckHostIP=no {batchhost} '.format(batchhost=batchhost) + appparams['paramscmd']
logger.debug('JobCreate.create_tunnel: using ssh to extract connection parameters')
res = Ssh.execute(sshsess, host=loginhost, user=username, cmd=paramcmd.format(data))
try:
data = json.loads(res['stdout'])
except json.decoder.JSONDecodeError as e:
raise AppParamsException(res['stderr']+res['stdout'])
if len(res['stderr']) > 0:
raise AppParamsException(res['stderr'])
if 'error' in data:
raise AppParamsException(data['error'])
try:
connectparams.update(json.loads(res['stdout']))
except json.decoder.JSONDecodeError as e:
logger.error(res['stdout'])
logger.error(res['stderr'])
if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
flask_restful.abort(400, message=res['stderr'].decode())
if self.validate_connect_params(connectparams, username, loginhost):
authtok = gen_authtok()
logger.debug('JobCreate.create_tunnel: creating a tunnel for authtok {}'.format(authtok))
tunnelport, pids = Ssh.tunnel(sshsess, port=connectparams['port'],
batchhost=connectparams['batchhost'],
user=username, host=loginhost,
internalfirewall=firewall,
localbind=appparams['localbind'], authtok=authtok)
connectparams['localtunnelport'] = tunnelport
connectparams['authtok'] = authtok
logger.debug('JobCreate.create_tunnel: created a tunnel for authtok {} port {}'.format(authtok,tunnelport))
else:
raise AppParamsException("connection parameters invalid {} {} {}".format(connectparams,username,loginhost))
return connectparams
def validate_connect_params(self, connectparams, username, host):
if not 'port' in connectparams:
return False
if not 'batchhost' in connectparams:
return False
class CreateTunnel(Resource):
@staticmethod
def validate_connect_params(port, username, host, batchhost):
try:
intport = int(connectparams['port'])
intport = int(port)
except Exception as e:
return False
if ' ' in username or '\n' in username: # This really needs more validation
......@@ -375,52 +399,45 @@ class JobConnect(Resource):
return False
return True
def get(self, jobid, batchhost):
def post(self,username,loginhost,batchhost):
"""
Connecting to a job is a get operation (i.e. it does not make modifications)
Create a tunnel using established keys
parameters for the tunnel (host username port etc)
will be passed in the body
"""
import logging
logger=logging.getLogger()
logger.debug('entering JobConnect.get for jobid {} {}'.format(jobid,batchhost))
params = get_conn_params()
appparams = get_app_params()
logger = logging.getLogger()
logger.debug("Createing tunnel")
logger.debug("recieved data {}".format(request.data))
data = request.get_json()
try:
connectparams = self.create_tunnel(params['identity']['username'],params['identity']['site']['host'],
appparams, batchhost, params['interface']['internalfirewall'],
data)
except AppParamsException as e:
return make_response(render_template('appparams.html.j2',data = "{}".format(e)))
logger.debug('JobConnect.get tunnels created, moving to redirect'.format(jobid,batchhost))
return self.connect(appparams, connectparams)
def connect(self, appparams, connectparams):
"""
perform the connection either by forking a local client or returning a redirect
"""
import subprocess
import logging
import json
logger=logging.getLogger()
if 'cmd' in appparams['client'] and appparams['client']['cmd'] is not None:
# We need for fork a local process such as vncviewer or a terminal
# We may need a wrapper for local processes to find the correct
# process on all OS
cmdlist = []
for cmdarg in appparams['client']['cmd']:
cmdlist.append(cmdarg.format(**connectparams))
app_process = subprocess.Popen(cmdlist, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
elif 'redir' in appparams['client'] and appparams['client']['redir'] is not None:
twsproxy = app.config['TWSPROXY']
data = json.dumps({'location': twsproxy+appparams['client']['redir'].
format(**connectparams) })
response = make_response(data)
response.mime_type = 'application/json'
response.set_cookie('twsproxyauth', connectparams['authtok'])
logger.debug('JobConnect.connect: connecting via redirect with cookie authtok set to {}'.format(connectparams['authtok']))
return response
return "Connecting with cmd {}".format(cmdlist)
port = data['port']
except KeyError as missingdata:
raise AppParamsException("missing value for {}".format(missingdata))
if not CreateTunnel.validate_connect_params(port, username, loginhost, batchhost):
raise AppParamsException("Invalid value: {} {} {} {}".
format(username, loginhost, batchhost, port))
if 'internalfirewall' in data:
firewall = data['internalfirewall']
else:
firewall = False
if 'localbind' in data:
localbind = data['localbind']
else:
localbind = True
sshsess = SSHSession.get_sshsession()
authtok = gen_authtok()
# logger.debug('JobCreate.create_tunnel: creating a tunnel for authtok {}'.format(authtok))
Ssh.tunnel(sshsess, port=port, batchhost=batchhost,
user=username, host=loginhost,
internalfirewall=firewall,
localbind=localbind, authtok=authtok)
response = make_response("")
response.mime_type = 'application/json'
response.set_cookie('twsproxyauth', authtok)
logger.debug('JobConnect.connect: connecting via redirect with cookie authtok set to {}'.format(authtok))
return response
api.add_resource(TunnelstatEP, '/tunnelstat/<string:authtok>')
......@@ -428,8 +445,9 @@ api.add_resource(GetCert, '/getcert')
api.add_resource(JobStat, '/stat')
api.add_resource(JobCancel, '/cancel/<int:jobid>')
api.add_resource(JobSubmit, '/submit')
api.add_resource(JobConnect, '/connect/<int:jobid>/<string:batchhost>')
# api.add_resource(SessionTest,'/sesstest')
#api.add_resource(StartAgent,'/startagent')
api.add_resource(CreateTunnel, '/createtunnel/<string:username>/<string:loginhost>/<string:batchhost>')
api.add_resource(AppInstance, '/appinstance/<string:username>/<string:loginhost>/<string:batchhost>')
api.add_resource(AppUrl, '/appurl')
api.add_resource(SSHAgent,'/sshagent')
api.add_resource(DirList,'/ls')
api.add_resource(MkDir,'/mkdir')
tes/sshwrapper/__init__.py
+ 66
− 17
View file @ a1f06226
......@@ -68,7 +68,7 @@ class Ssh:
pass
@staticmethod
def get_ctrl_master_socket(sess, host, user):
def get_ctrl_master_socket(sess, host, user, sshport):
"""
Returns the socket for the control master process
Opens it if needed.
......@@ -87,16 +87,18 @@ class Ssh:
sshcmd = ["ssh", '-o', 'StrictHostKeyChecking=no',
"-S", ctrlsocket,
"-M", '-o', 'ControlPersist=10m',
'-N','-l', user, host]
'-p', sshport, '-N','-l', user, host]
env = os.environ.copy()
if sess.socket is None:
raise SshAgentException("No ssh-agent yet")
env['SSH_AUTH_SOCK'] = sess.socket
logger.debug("socket not found, starting new master")
ctrl_p = subprocess.Popen(sshcmd,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=None,
env=env)
stdout = ctrl_p.communicate()
logger.debug("master spawned, attempt communicate")
stdout, stderr = ctrl_p.communicate()
logger.debug('communicate on the control port complete')
sess.pids.append(ctrl_p.pid)
......@@ -108,6 +110,7 @@ class Ssh:
logger.error(ctrl_p.stderr.read())
raise SshCtrlException()
if not stat.S_ISSOCK(mode):
logger.error(ctrl_p.stderr.read())
raise SshCtrlException()
return ctrlsocket
......@@ -119,7 +122,7 @@ class Ssh:
pwd = None
import dateutil.parser
for l in output.splitlines():
fields = l.split(None,8)
fd = {}
if len(fields) == 9:
......@@ -138,9 +141,51 @@ class Ssh:
pwd = l[len(remotestr):].rstrip()
return ({'pwd':pwd,'files':rv})
@staticmethod
def sftpmkdir(sess, host, user, path, name, sshport):
"""
Use sftp to run mkdir.
"""
import os
import logging
logger = logging.getLogger()
env = os.environ.copy()
if path is None:
path = "."
if sess.socket is None:
raise SshAgentException("No ssh-agent yet")
env['SSH_AUTH_SOCK'] = sess.socket
Ssh.validate_username(user)
Ssh.validate_hostname(host)
ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user, sshport)
exec_p = subprocess.Popen(['sftp', '-b', '-','-o', 'Stricthostkeychecking=no',
'-P', sshport, '-o', 'ControlPath={}'.format(ctrlsocket),
'{}@{}'.format(user, host)],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE, env=env)
sftpcmd="cd {}\n mkdir \"{}\"\n".format(path,name)
(stdout, stderr) = exec_p.communicate(sftpcmd.encode())
if stderr is not None:
if not (stderr == b''):
logger.error('sftp failure')
logger.error(stdout.decode())
logger.error(stderr.decode())
if ('Couldn\'t create directory: Failure' in stderr.decode()):
return
if ('Couldn\'t canonicalize: No such file or directory' in stderr.decode()):
logger.error('can\'t change to that directory')
return
if ('Permission denied' in stderr.decode()):
logger.error('can\'t change to that directory')
return
raise SshCtrlException()
return
@staticmethod
def sftpls(sess, host, user, path=".",changepath="."):
def sftpls(sess, host, user, sshport, path=".",changepath="."):
"""
Use sftp to run an ls on the given path.
Return the directory listing (as a list) and the cwd
......@@ -154,13 +199,13 @@ class Ssh:
env['SSH_AUTH_SOCK'] = sess.socket
Ssh.validate_username(user)
Ssh.validate_hostname(host)
ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user)
ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user, sshport)
if (path is None or path == ""):
path="."
if (changepath is None or changepath == ""):
changepath="."
exec_p = subprocess.Popen(['sftp', '-b', '-','-o', 'Stricthostkeychecking=no',
'-o', 'ControlPath={}'.format(ctrlsocket),
'-P', sshport, '-o', 'ControlPath={}'.format(ctrlsocket),
'{}@{}'.format(user, host)],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE, env=env)
......@@ -173,10 +218,10 @@ class Ssh:
logger.error('sftp failure')
if ('Couldn\'t canonicalize: No such file or directory' in stderr.decode()):
logger.error('can\'t change to that directory')
return Ssh.sftpls(sess,host,user,path,changepath='.')
return Ssh.sftpls(sess,host,user,sshport, path,changepath='.')
if ('Permission denied' in stderr.decode()):
logger.error('can\'t change to that directory')
return Ssh.sftpls(sess,host,user,path,changepath='.')
return Ssh.sftpls(sess,host,user,sshport, path,changepath='.')
logger.error(stderr.decode())
logger.error(('Permission denied' in stderr.decode()))
......@@ -188,13 +233,15 @@ class Ssh:
@staticmethod
def execute(sess, host, user, cmd, stdin=None):
def execute(sess, host, user, cmd, stdin=None, sshport="22"):
"""
execute the command cmd on the host via ssh
# assume the environment is already setup with an
# SSH_AUTH_SOCK that allows login
"""
import os
import logging
logger = logging.getLogger()
if cmd is None and stdin is None:
return {'stdout': b'', 'stderr': b'No command given to execute'}
env = os.environ.copy()
......@@ -204,10 +251,12 @@ class Ssh:
Ssh.validate_username(user)
Ssh.validate_hostname(host)
Ssh.validate_command(cmd)
ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user)
logger.debug('getting ctrl_master')
ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user, sshport)
logger.debug('ssh.execute: got ctrlsocket {}'.format(ctrlsocket))
exec_p = subprocess.Popen(['ssh', '-A', '-o', 'Stricthostkeychecking=no',
'-S', ctrlsocket,
'-l', user, host, cmd],
'-p', sshport, '-l', user, host, cmd],
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE, env=env)
if stdin is not None:
......@@ -217,7 +266,7 @@ class Ssh:
return {'stdout':stdout, 'stderr':stderr}
@staticmethod
def tunnel(sess, port, batchhost, user, host, internalfirewall = True, localbind = True, authtok = None):
def tunnel(sess, port, batchhost, user, host, internalfirewall = True, localbind = True, authtok = None, sshport="22"):
"""
the ProxyCommand is used if the server we run on the batch host is only
addressable on localhost
......@@ -237,7 +286,7 @@ class Ssh:
Ssh.validate_hostname(batchhost)
Ssh.validate_username(user)
Ssh.validate_hostname(host)
ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user)
ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user, sshport)
logger.debug('Ssh.tunnel, got ctrlsocket')
localport = Ssh.get_free_port()
......@@ -248,7 +297,7 @@ class Ssh:
'-L', '{localport}:{batchhost}:{port}'.
format(port=port, localport=localport, batchhost=batchhost),
'-O', 'forward', '-S', ctrlsocket,
'-l', user, host]
'-p', sshport, '-l', user, host]
else:
# Create an ssh tunnel to the batch node using a proxycommand.
# The proxy command should utilise
......@@ -261,7 +310,7 @@ class Ssh:
'-L', '{localport}:localhost:{port}'.
format(port=port, localport=localport),
'-o', "ProxyCommand={}".format(proxycmd),
'-l', user, batchhost]
'-p', sshport, '-l', user, batchhost]
logger.debug('Ssh.tunnel: attempting command {}'.format(sshcmd))
tunnel_p = subprocess.Popen(sshcmd, env=env)
......
tes/tunnelstat/__init__.py
+ 3
− 0
View file @ a1f06226
......@@ -41,7 +41,9 @@ class SSHSession:
import logging
logger = logging.getLogger()
if self.socket is None:
logger.debug('adding a cert, start the agent first')
self.start_agent()
logger.debug('agent socket is now {}'.format(self.socket))
keyf = tempfile.NamedTemporaryFile(mode='w',delete=False)
keyname = keyf.name
keyf.write(key)
......@@ -160,6 +162,7 @@ class SSHSession:
session['sshsessid'] = sshsessid
if sshsessid not in sshsessions:
sshsessions[sshsessid] = SSHSession()
return sshsessions[sshsessid]
@staticmethod
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment