Test

This is to be done 10pm Mondayt 21 Sept 2020 to minimise interuption and allow Chris to recover if things go sideways.

tes/sshwrapper/__init__.py

@@ -109,6 +109,10 @@ class Ssh:
         sess.lock.acquire()
         try:
             mode = os.stat(ctrlsocket).st_mode
+            # If the control process died (docker restarted, or otherwise exited) but the socket was not removed:
+            if mode is not None and ctrlsocket not in sess.ctrl_processes:
+                os.unlink(ctrlsocket)
+                mode = None
         except FileNotFoundError as e:
             mode = None
             logger.error('attempting to access control master socket raised exception')
@@ -374,18 +378,19 @@ class Ssh:
             logger.error(stderr)
             sess.kill()
             raise SshCtrlException(message="Ssh was unable to login. It's likely you're credentials need to be renewed by loggin in again")
-        if exec_p.returncode != 0 or b'Control socket connect' in stderr:
-            try:
-                ctrl_p = sess.ctrl_processes[ctrlsocket]
-            except KeyError as e:
-                # If the process is restarted and the agents all get restarted, but the control socket /tmp/cm-* is not removed
-                # This can happen
-                import os
-                try:
-                    os.unlink(ctrlsocket)
-                except FileNotFoundError as e:
-                    pass
-                raise SshCtrlException(message="ssh execute failed, there was a problem with the control socket. Did the docker backend get restartd?")
+# this error condition should now be handled in  get_control_socket
+#        if exec_p.returncode != 0 or b'Control socket connect' in stderr:
+#            try:
+#                ctrl_p = sess.ctrl_processes[ctrlsocket]
+#            except KeyError as e:
+#                # If the process is restarted and the agents all get restarted, but the control socket /tmp/cm-* is not removed
+#                # This can happen
+#                import os
+#                try:
+#                    os.unlink(ctrlsocket)
+#                except FileNotFoundError as e:
+#                    pass
+#                raise SshCtrlException(message="ssh execute failed, there was a problem with the control socket. Did the docker backend get restartd?")
 
             logger.error('ran command {}'.format(" ".join(sshcmd)))
             logger.error('got return code {} stderr {} stdout {}'.format(exec_p.returncode,stderr,stdout))