throw more useful error messages when the remote server is down

TES/tes/apiendpoints.py

@@ -8,7 +8,7 @@ from flask import render_template
 import flask_restful
 
 from . import api, app
-from .sshwrapper import Ssh, SshAgentException, SftpPermissionException, SftpException
+from .sshwrapper import Ssh, SshAgentException, SftpPermissionException, SftpException, SshCtrlException
 from .tunnelstat import SSHSession
 
 
@@ -224,6 +224,8 @@ class JobStat(Resource):
         except SshAgentException as e:
             logger.error(e)
             flask_restful.abort(400, message="Identity error {}".format(e))
+        except SshCtrlException as e:
+            flask_restful.abort(400,message="We're having difficultly contacting {}. Could there be an outage in effect?".format(params['identity']['site']['host']))
         if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
             logger.error(res['stderr'])
             flask_restful.abort(400, message=res['stderr'].decode())
@@ -284,21 +286,24 @@ class DirList(Resource):
             path = "."
         if cd == "":
             cd = "."
-        if 'lscmd' in site and site['lscmd'] is not None and site['lscmd'] is not "":
-            logger.debug('using ssh.execute with lscmd')
-            res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
-                                sshport=sshport,
-                                cmd="{} {} {}".format(site['lscmd'],path,cd))
-            try:
-                dirls = json.loads(res['stdout'].decode())
-            except:
-                flask_restful.abort(404,message="You don't have permission to view that directory")
-        else:
-            logger.debug('using ssh.sftpls')
-
-            dirls = Ssh.sftpls(sshsess, host=params['identity']['site']['host'],
-                               user=params['identity']['username'], path=params['path'],changepath=params['cd'], sshport=sshport)
-        return dirls
+        try:
+            if 'lscmd' in site and site['lscmd'] is not None and site['lscmd'] is not "":
+                logger.debug('using ssh.execute with lscmd')
+                res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
+                                    sshport=sshport,
+                                    cmd="{} {} {}".format(site['lscmd'],path,cd))
+                try:
+                    dirls = json.loads(res['stdout'].decode())
+                except:
+                    flask_restful.abort(404,message="You don't have permission to view that directory")
+            else:
+                logger.debug('using ssh.sftpls')
+
+                dirls = Ssh.sftpls(sshsess, host=params['identity']['site']['host'],
+                                   user=params['identity']['username'], path=params['path'],changepath=params['cd'], sshport=sshport)
+            return dirls
+        except SshCtrlException as e:
+            flask_restful.abort(400,message="We're having difficultly contacting {}. Could there be an outage in effect?".format(params['identity']['site']['host']))
 
 class JobCancel(Resource):
     """
@@ -310,11 +315,14 @@ class JobCancel(Resource):
         """
         params = get_conn_params()
         sshsess = SSHSession.get_sshsession()
-        res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
-                          cmd=params['interface']['cancelcmd'].format(jobid=jobid))
-        if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
-            flask_restful.abort(400, message=res['stderr'].decode())
-        return res['stdout'].decode()
+        try:
+            res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
+                              cmd=params['interface']['cancelcmd'].format(jobid=jobid))
+            if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
+                flask_restful.abort(400, message=res['stderr'].decode())
+            return res['stdout'].decode()
+        except SshCtrlException as e:
+            flask_restful.abort(400,message="We're having difficultly contacting {}. Could there be an outage in effect?".format(params['identity']['site']['host']))
 
 
 class JobSubmit(Resource):
@@ -345,8 +353,11 @@ class JobSubmit(Resource):
             flask_restful.abort(400, message='Incomplete job information was passed to the backend.')
         logger.debug('script formated to {}'.format(script))
 
-        res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
+        try:
+            res = Ssh.execute(sshsess, host=params['identity']['site']['host'], user=params['identity']['username'],
                           cmd=params['interface']['submitcmd'], stdin=script)
+        except SshCtrlException as e:
+            flask_restful.abort(400,message="We're having difficultly contacting {}. Could there be an outage in effect?".format(params['identity']['site']['host']))
         if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):
             logger.debug('failed to submit job')
             logger.debug(res['stderr'])