From 2c17e33567fd8b9d76fd3ef41eb884e13b77cce1 Mon Sep 17 00:00:00 2001 From: Chris Hines <chris.hines@monash.edu> Date: Wed, 28 Apr 2021 11:58:51 +1000 Subject: [PATCH 1/2] more logging into APIAbort, change a 500 error to a 401 --- tes/apiendpoints.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tes/apiendpoints.py b/tes/apiendpoints.py index a86a950..7d41fe2 100644 --- a/tes/apiendpoints.py +++ b/tes/apiendpoints.py @@ -224,7 +224,7 @@ def wrap_execute(sess, host, user, cmd, bastion=None,stdin=None, sshport="22", b return apiabort(400, message=e, sess=sess) except SshCtrlException as e: - return apiabort(401,message="{} {}".format(host,e), sess=sess) + return apiabort(401,message="{} {}".format(host,e), sess=sess, logdata = json.dumps({'user':user,'cmd':cmd,'host':host})) # This exception is raised if the remove command had a non-zero return code except SshExecException as e: return apiabort(400, message="{}".format(e), sess=sess) @@ -422,7 +422,7 @@ def gen_authtok(): return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(16)) -def apiabort(code, message=None, data = None, sess = None): +def apiabort(code, message=None, data = None, sess = None, logdata = None): import logging import traceback from flask import g @@ -431,6 +431,7 @@ def apiabort(code, message=None, data = None, sess = None): logger.error("request time {}".format(time.time() - g.start)) if sess is not None: logger.error("certs {}".format(sess.get_certs())) + logger.error(logdata) logger.error("{} {} {}".format(code,data,message)) logger.error("{}".format(traceback.format_exc())) if data is not None: @@ -560,7 +561,7 @@ class CreateTunnel(Resource): try: port = data['port'] except KeyError as missingdata: - raise AppParamsException("missing value for {}".format(missingdata)) + raise AppParamsException("missing value for {} data contains {}".format(missingdata,data)) if not CreateTunnel.validate_connect_params(port, username, loginhost, batchhost): raise AppParamsException("Invalid value: {} {} {} {}". format(username, loginhost, batchhost, port)) @@ -586,6 +587,13 @@ class CreateTunnel(Resource): response.mime_type = 'application/json' response.set_cookie('twsproxyauth', authtok, secure=True) return response + except AppParamsException as e: + import traceback + logger.error(e) + logger.error(traceback.format_exc()) + + return apiabort(400,message="Unable to create the secure tunnel. Please try again.") + except Exception as e: import traceback logger.error(e) -- GitLab From 5a04244bfece6dd88e9c35fffb55a11b623e9277 Mon Sep 17 00:00:00 2001 From: Chris Hines <chris.hines@monash.edu> Date: Wed, 28 Apr 2021 15:13:24 +1000 Subject: [PATCH 2/2] a bit more logging to resolve random logout errors --- tes/sshwrapper/__init__.py | 1 + tes/tunnelstat/__init__.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tes/sshwrapper/__init__.py b/tes/sshwrapper/__init__.py index b2ad59b..c81a7a9 100644 --- a/tes/sshwrapper/__init__.py +++ b/tes/sshwrapper/__init__.py @@ -344,6 +344,7 @@ class Ssh: if b'client_loop' in stderr: raise SshExecException(message="It looks like your job died for some reason. Please check the log files") else: + logger.error('ssh execute returned error code 255, killing the session') sess.kill() raise SshCtrlException(message="Ssh was unable to login. It's likely you're credentials need to be renewed by loggin in again") # this error condition should now be handled in get_control_socket diff --git a/tes/tunnelstat/__init__.py b/tes/tunnelstat/__init__.py index a242ac4..e8cb6ff 100644 --- a/tes/tunnelstat/__init__.py +++ b/tes/tunnelstat/__init__.py @@ -232,6 +232,7 @@ class SSHSession: import string from .. import sshsessions from flask import session, request + import logging sshsessid = session.get('sshsessid', None) N = 8 while sshsessid is None: @@ -246,6 +247,8 @@ class SSHSession: try: SSHSession.test_sshsession(sshsession) except SshAgentException: + logger = logging.getLogger() + logger.error('SSHAgentException, killing the agent and restarted. This is highly unexpected') sshsession.kill() sshsessions[sshsessid] = SSHSession(sshsessid) sshsession = sshsessions[sshsessid] -- GitLab