Test

This is to be done 10pm Mondayt 21 Sept 2020 to minimise interuption and allow Chris to recover if things go sideways.

tes/tunnelstat/__init__.py

@@ -3,6 +3,7 @@ This module persistently stores informion on tunnels in an in memory structure.
 """
 import datetime
 import yaml
+import threading
 
 class SshAgentException(Exception):
     pass
@@ -25,6 +26,8 @@ class SSHSession:
         self.sshadd = '/usr/bin/ssh-add'
         self.sshkeygen = 'ssh-keygen'
         self.ctrl_processes = {}
+        self.lock = threading.Lock()
+        self.tunnels = []
 
     def start_agent(self):
         import subprocess
@@ -33,7 +36,7 @@ class SSHSession:
         import os
         logger = logging.getLogger()
         logger.debug('starting agent')
-        if app.config['ENABLELAUNCH'] and os.environ['SSH_AUTH_SOCK']:
+        if app.config['ENABLELAUNCH'] and 'SSH_AUTH_SOCK' in os.environ and os.environ['SSH_AUTH_SOCK']:
             logger.debug('using existing agent')
             self.socket = os.environ['SSH_AUTH_SOCK']
             return
@@ -53,8 +56,10 @@ class SSHSession:
         import subprocess
         import logging
         logger = logging.getLogger()
+        self.lock.acquire()
         if self.socket is None:
             self.start_agent()
+        self.lock.release()
         keyf = tempfile.NamedTemporaryFile(mode='w',delete=False)
         keyname = keyf.name
         keyf.write(key)
@@ -64,6 +69,7 @@ class SSHSession:
         certf.close()
         p = subprocess.Popen([self.sshkeygen,'-L','-f','-'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
         keygenout,keygenerr = p.communicate(cert.encode())
+        logger.debug('inspecting keycert pid {}'.format(p.pid))
         # Examine the cert to determine its expiry. Use the -t flag to automatically remove from the ssh-agent when the cert expires
         certcontents = SSHSession.parse_cert_contents(keygenout.decode().splitlines())
         endtime = datetime.datetime.strptime(certcontents['Valid'][0].split()[3],"%Y-%m-%dT%H:%M:%S")
@@ -73,6 +79,7 @@ class SSHSession:
         cmd = [self.sshadd,'-t',"{}".format(int(delta.total_seconds()))]
         cmd.append(keyname)
         p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,env=env)
+        logger.debug('adding keycert pid {}'.format(p.pid))
         (stdout,stderr) = p.communicate()
         if p.returncode != 0:
             logger.error("Couldn't add key and cert")
@@ -95,10 +102,12 @@ class SSHSession:
         cmd = [self.sshadd,'-L']
         p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,env=env)
         (stdout,stderr) = p.communicate()
+        logger.debug('query agent pid {}'.format(p.pid))
         for l in stdout.splitlines():
             if b'cert' in l:
                 p = subprocess.Popen([self.sshkeygen,'-L','-f','-'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
                 keygenout,keygenerr = p.communicate(l)
+                logger.debug('inspecting agent contents pid {}'.format(p.pid))
                 certcontents = SSHSession.parse_cert_contents(keygenout.decode().splitlines())
                 res.append(certcontents)
         return res
@@ -140,6 +149,7 @@ class SSHSession:
         logger=logging.getLogger()
         logger.debug("shuting down ssh session for {} last seen at {}".format(self.authtok,self.last))
         for pid in self.pids:
+            logger.debug("killing pid {}".format(pid))
             try:
                 os.killpg(int(pid), signal.SIGTERM) # Sometimes this fails and I don't know why
                 try:
@@ -153,6 +163,7 @@ class SSHSession:
             except ProcessLookupError as e:
                 logger.debug("process {} not found".format(pid))
         for ctrl in self.ctrl_processes.items():
+            logger.debug("killing ctrl pid {}".format(ctrl[1]))
             try:
                 ctrl[1].kill()
             except:
@@ -165,6 +176,9 @@ class SSHSession:
                 os.unlink(ctrl[0])
             except: 
                 pass
+        for tunnel in self.tunnels:
+            tunnel.kill()
+            (stdout, stderr) = tunnel.communicate()
 
     @staticmethod
     def test_sshsession(sess):
@@ -173,11 +187,14 @@ class SSHSession:
         import logging
         logger=logging.getLogger()
         env = os.environ.copy()
+        sess.lock.acquire()
         if sess.socket is None:
             sess.start_agent()
+        sess.lock.release()
         env['SSH_AUTH_SOCK'] = sess.socket
         cmd = [sess.sshadd,'-l']
         p = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,env=env)
+        logger.debug('test sshsession with pid {}'.format(p.pid))
         (stdout,stderr) = p.communicate()
         if p.returncode != 0:
             """