Changes to use Fabric Connection properly, and change from cpio to tar to support >2GB files

asci_to_vault.py

@@ -53,6 +53,35 @@ import textwrap
 from fabric import Connection
 
 
+def escape_parens(path):
+    """ Explicitly escape parentheses. This is required to work around a bug in Fabric's
+    Invoke module. See my question on Stackoverflow:
+    https://stackoverflow.com/q/63225018/607587
+    The recommended workaround, until Fabric fixes the bug, is to just "manually escape
+    the parentheses"
+
+    I used this method: https://stackoverflow.com/a/23563806/607587
+
+    """
+    replacements = {"(":"\(", ")":"\)"}
+    escaped_path = "".join([replacements.get(c, c) for c in path])
+    return escaped_path
+
+
+def escape_path(path):
+    """ Explicitly escape parentheses AND spaces.
+    I used this method: https://stackoverflow.com/a/23563806/607587
+
+    """
+    # kludge; first (un)escape any already escaped characters
+    unreplacements = {"\(":"(", "\)":")", "\ ":" "}
+    unescaped_path = "".join([unreplacements.get(c, c) for c in path])
+    # Now escape unescaped spaces, plus any unescaped parens
+    replacements = {"(":"\(", ")":"\)", " ":"\ "}
+    escaped_path = "".join([replacements.get(c, c) for c in unescaped_path])
+    return escaped_path
+
+
 @dataclass
 class Node:
     """A directory tree node"""
@@ -101,44 +130,47 @@ def send_directory(node, remote_login, src_path):
     """
     # Check if there are any files in the node.
     with Connection(remote_login) as c:
-        files = c.run(
-            rf"cd {node.src}; nice find -maxdepth 1 -type f -printf '%f\n'",
-            echo=True
-        )
-        files = files.stdout.strip()
+        with c.cd(escape_parens(node.src)):
+            result = c.run(r"nice find -maxdepth 1 -type f -printf '%f\n'", echo=True)
+
+    files = result.stdout.strip()
     node.count = len(files.splitlines())
 
     print(f"Node:{node.src}, file count:{node.count}")
     if node.count == 0:
         # No files at this node, just return
         print("No files to transfer")
-    elif node.count == 1:
-        # Only one file. Just copy unchanged.
-        output = subprocess.run(
-            f"scp -q {remote_login}:{node.src}/{files} {node.dest}",
-            shell=True,
-            check=True
-        )
-        print("stdout:", output.stdout)
-        print("stderr:", output.stderr)
-        # os.chmod(f"{node.dest}/{files}", 0o550)
-        print(f"Transferred single file {node.src} -> {node.dest}")
+    # elif node.count == 1:
+    #     # Only one file. Just copy unchanged.
+    #     output = subprocess.run(
+    #         f"scp -q {remote_login}:{node.src}/{files} {node.dest}",
+    #         shell=True,
+    #         check=True
+    #     )
+    #     print("stdout:", output.stdout)
+    #     print("stderr:", output.stderr)
+    #     # os.chmod(f"{node.dest}/{files}", 0o550)
+    #     print(f"Transferred single file {node.src} -> {node.dest}")
     else:
-        # More than one file. Transfer all files to a tarball.
+        # At least one file. Transfer all files to a tarball.
         if node.src == src_path:
             filename = os.path.basename(node.src)
         else:
-            filename = node.src.replace(src_path+"/", "").replace("/", "_")
+            filename = node.src.replace(src_path + "/", "").replace("/", "_")
 
+        cmd_src = escape_path(node.src)
+        cmd_dest = escape_path(node.dest)
+        cmd_filename = escape_path(filename)
         output = subprocess.run(
-            f"ssh {remote_login} 'cd {node.src};"
+            f"ssh {remote_login} 'cd {cmd_src};"
             f"nice find -maxdepth 1 -type f -print0 |"
-            f"cpio -o -H ustar -0' | cat > {node.dest}/{filename}.tar",
+            f"xargs -0 tar -cf - ' | cat > {cmd_dest}/{cmd_filename}.tar",
             shell=True,
             check=True
         )
         print("stdout:", output.stdout)
         print("stderr:", output.stderr)
+
         # os.chmod(f"{node.dest}/{filename}.tar", 0o550)
         print(f"Transferred {node.count} files {node.src} -> {node.dest}")
 
@@ -185,7 +217,7 @@ def main(
     file to True.
 
     """
-    assert 5 <= len(experiment_name) <= 6
+    assert 4 <= len(experiment_name) <= 6
     if pickle_filename is None:
         pickle_filename = experiment_name+".pickle"