From 2c17e33567fd8b9d76fd3ef41eb884e13b77cce1 Mon Sep 17 00:00:00 2001
From: Chris Hines <chris.hines@monash.edu>
Date: Wed, 28 Apr 2021 11:58:51 +1000
Subject: [PATCH 1/2] more logging into APIAbort, change a 500 error to a 401

---
 tes/apiendpoints.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tes/apiendpoints.py b/tes/apiendpoints.py
index a86a950..7d41fe2 100644
--- a/tes/apiendpoints.py
+++ b/tes/apiendpoints.py
@@ -224,7 +224,7 @@ def wrap_execute(sess, host, user, cmd, bastion=None,stdin=None, sshport="22", b
             return apiabort(400, message=e, sess=sess)
 
     except SshCtrlException as e:
-        return apiabort(401,message="{} {}".format(host,e), sess=sess)
+        return apiabort(401,message="{} {}".format(host,e), sess=sess, logdata = json.dumps({'user':user,'cmd':cmd,'host':host}))
     # This exception is raised if the remove command had a non-zero return code
     except SshExecException as e:
         return apiabort(400, message="{}".format(e), sess=sess)
@@ -422,7 +422,7 @@ def gen_authtok():
     return ''.join(random.SystemRandom().choice(string.ascii_uppercase +
                                                 string.digits) for _ in range(16))
 
-def apiabort(code, message=None, data = None, sess = None):
+def apiabort(code, message=None, data = None, sess = None, logdata = None):
     import logging
     import traceback
     from flask import g
@@ -431,6 +431,7 @@ def apiabort(code, message=None, data = None, sess = None):
     logger.error("request time {}".format(time.time() - g.start))
     if sess is not None:
         logger.error("certs {}".format(sess.get_certs()))
+    logger.error(logdata)
     logger.error("{} {} {}".format(code,data,message))
     logger.error("{}".format(traceback.format_exc()))
     if data is not None:
@@ -560,7 +561,7 @@ class CreateTunnel(Resource):
             try:
                 port = data['port']
             except KeyError as missingdata:
-                raise AppParamsException("missing value for {}".format(missingdata))
+                raise AppParamsException("missing value for {} data contains {}".format(missingdata,data))
             if not CreateTunnel.validate_connect_params(port, username, loginhost, batchhost):
                 raise AppParamsException("Invalid value: {} {} {} {}".
                                          format(username, loginhost, batchhost, port))
@@ -586,6 +587,13 @@ class CreateTunnel(Resource):
             response.mime_type = 'application/json'
             response.set_cookie('twsproxyauth', authtok, secure=True)
             return response
+        except AppParamsException as e:
+            import traceback
+            logger.error(e)
+            logger.error(traceback.format_exc())
+            
+            return apiabort(400,message="Unable to create the secure tunnel. Please try again.")
+
         except Exception as e:
             import traceback
             logger.error(e)
-- 
GitLab


From 5a04244bfece6dd88e9c35fffb55a11b623e9277 Mon Sep 17 00:00:00 2001
From: Chris Hines <chris.hines@monash.edu>
Date: Wed, 28 Apr 2021 15:13:24 +1000
Subject: [PATCH 2/2] a bit more logging to resolve random logout errors

---
 tes/sshwrapper/__init__.py | 1 +
 tes/tunnelstat/__init__.py | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/tes/sshwrapper/__init__.py b/tes/sshwrapper/__init__.py
index b2ad59b..c81a7a9 100644
--- a/tes/sshwrapper/__init__.py
+++ b/tes/sshwrapper/__init__.py
@@ -344,6 +344,7 @@ class Ssh:
             if b'client_loop' in stderr:
                 raise SshExecException(message="It looks like your job died for some reason. Please check the log files")
             else:
+                logger.error('ssh execute returned error code 255, killing the session')
                 sess.kill()
                 raise SshCtrlException(message="Ssh was unable to login. It's likely you're credentials need to be renewed by loggin in again")
 # this error condition should now be handled in  get_control_socket
diff --git a/tes/tunnelstat/__init__.py b/tes/tunnelstat/__init__.py
index a242ac4..e8cb6ff 100644
--- a/tes/tunnelstat/__init__.py
+++ b/tes/tunnelstat/__init__.py
@@ -232,6 +232,7 @@ class SSHSession:
         import string
         from .. import sshsessions
         from flask import session, request
+        import logging
         sshsessid = session.get('sshsessid', None)
         N = 8
         while sshsessid is None:
@@ -246,6 +247,8 @@ class SSHSession:
         try:
             SSHSession.test_sshsession(sshsession)
         except SshAgentException:
+            logger = logging.getLogger()
+            logger.error('SSHAgentException, killing the agent and restarted. This is highly unexpected')
             sshsession.kill()
             sshsessions[sshsessid] = SSHSession(sshsessid)
             sshsession = sshsessions[sshsessid]
-- 
GitLab