Merge branch 'master' of gitlab.erc.monash.edu.au:hpc-team/strudelv2_tes

tes/apiendpoints.py

@@ -28,15 +28,16 @@ class GetCert(Resource):
         import logging
         logger = logging.getLogger()
         logger.debug('entering GetCert.post')
-        data = request.get_json()
         try:
+            data = request.get_json()
             response = {'cert':GetCert.get_cert(data['token'], data['pubkey'], data['signing_url'])}
+            return response
         except:
             import traceback
             logger.error('Failed to get certificate')
             logger.error(traceback.format_exc())
-            response = ("Unable to get a signed certificate",500)
-        return response
+            flask_restful.abort(500,message="an error occured generating your certificate")
+
     @staticmethod
     def get_cert(access_token, pub_key, url):
         """
@@ -384,14 +385,26 @@ class AppInstance(Resource):
         logger = logging.getLogger()
         logger.debug('getting appinstance {} {} {}'.format(username,loginhost,batchhost))
         logger.debug('ssh sess socket is {}'.format(sshsess.socket))
-        cmd = 'ssh -o StrictHostKeyChecking=no -o CheckHostIP=no {batchhost} '.format(batchhost=batchhost) + paramscmd
-        res = Ssh.execute(sshsess, host=loginhost, user=username, cmd=cmd)
+        #cmd = 'ssh -o StrictHostKeyChecking=no -o CheckHostIP=no {batchhost} '.format(batchhost=batchhost) + paramscmd
+        try:
+            res = Ssh.execute(sshsess, host=batchhost, bastion=loginhost, user=username, cmd=paramscmd)
+        except:
+            message = "The server couldn't execute to {} to get the necessary info".format(batchhost)
+            flask_restful.abort(500, message=message)
+            import traceback
+            logger.error(traceback.format_exc())
         try:
             data = json.loads(res['stdout'].decode())
+            return data
         except json.decoder.JSONDecodeError as e:
-            raise AppParamsException(res['stderr']+res['stdout'])
+            logger.error(res['stderr']+res['stdout'])
+            message="I'm having trouble using ssh to find out about that application"
+            flask_restful.abort(500, message=message)
+            #raise AppParamsException(res['stderr']+res['stdout'])
         if len(res['stderr']) > 0:
-            raise AppParamsException(res['stderr'])
+            logger.error(res['stderr']+res['stdout'])
+            flask_restful.abort(500, message="The command {} on {} didn't work".format(paramscmd,batchhost))
+            #raise AppParamsException(res['stderr'])
         if 'error' in data:
             raise AppParamsException(data['error'])
         if not (res['stderr'] == '' or res['stderr'] is None or res['stderr'] == b''):

tes/sshwrapper/__init__.py

@@ -245,7 +245,7 @@ class Ssh:
 
 
     @staticmethod
-    def execute(sess, host, user, cmd, stdin=None, sshport="22"):
+    def execute(sess,  host, user, cmd, bastion=None, stdin=None, sshport="22", bastionsshport="22"):
         """
         execute the command cmd on the host via ssh
         # assume the environment is already setup with an
@@ -264,18 +264,33 @@ class Ssh:
         Ssh.validate_hostname(host)
         Ssh.validate_command(cmd)
         logger.debug('getting ctrl_master')
-        ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user, sshport)
+        if bastion == None:
+            ctrlsocket = Ssh.get_ctrl_master_socket(sess, host, user, sshport)
+        else:
+            ctrlsocket = Ssh.get_ctrl_master_socket(sess, bastion, user, bastionsshport)
         logger.debug('ssh.execute: got ctrlsocket {}'.format(ctrlsocket))
-        exec_p = subprocess.Popen(['ssh', '-A', '-o', 'Stricthostkeychecking=no',
-                                   '-S', ctrlsocket,
-                                   '-p', sshport, '-l', user, host, cmd],
-                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  stdin=subprocess.PIPE, env=env)
-        try:
-            if stdin is not None:
-                (stdout, stderr) = exec_p.communicate(stdin.encode(),timeout=TIMEOUT)
-            else:
-                (stdout, stderr) = exec_p.communicate(timeout=TIMEOUT)
+        if bastion == None:
+            # we are executing this command on the login node, so no more magic is needed
+            sshcmd = ['ssh', '-o', 'Stricthostkeychecking=no', 
+                      '-S', ctrlsocket, '-p', sshport,
+                      '-l', user, host, cmd]
+        else:
+            # we are executing on a node (e.g. a compute/batch node) using a bastion (e.g. login node)
+            # at the moment I'll assume the ssh port for the batch host is the same as the ssh port for the bastion/login host
+            proxycmd = "ssh -o Stricthostkeychecking=no {user}@{bastion} -W {host}:{sshport} -S {ctrlsocket}".format(
+                user=user, host=host,
+                ctrlsocket=ctrlsocket,
+                sshport=sshport,
+                bastion=bastion)
+            sshcmd = ['ssh', '-o', 'Stricthostkeychecking=no',
+                      '-o', "ProxyCommand={}".format(proxycmd),
+                      '-p', bastionsshport, '-l', user, bastion, cmd]
+        exec_p = subprocess.Popen(sshcmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,stdin=subprocess.PIPE,env=env)
+
+        if stdin is not None:
+            (stdout, stderr) = exec_p.communicate(stdin.encode(),timeout=TIMEOUT)
+        else:
+            (stdout, stderr) = exec_p.communicate(timeout=TIMEOUT)
         except subprocess.TimeoutExpired:
             exec_p.kill()
             (stdout, stderr) = exec_p.communicate()