Commit 65a9f305 authored by Gary Ruben (Monash University)'s avatar Gary Ruben (Monash University)
Browse files

Changes to use Fabric Connection properly, and change from cpio to tar to support >2GB files

parent 1b239383
......@@ -53,6 +53,35 @@ import textwrap
from fabric import Connection
def escape_parens(path):
""" Explicitly escape parentheses. This is required to work around a bug in Fabric's
Invoke module. See my question on Stackoverflow:
https://stackoverflow.com/q/63225018/607587
The recommended workaround, until Fabric fixes the bug, is to just "manually escape
the parentheses"
I used this method: https://stackoverflow.com/a/23563806/607587
"""
replacements = {"(":"\(", ")":"\)"}
escaped_path = "".join([replacements.get(c, c) for c in path])
return escaped_path
def escape_path(path):
""" Explicitly escape parentheses AND spaces.
I used this method: https://stackoverflow.com/a/23563806/607587
"""
# kludge; first (un)escape any already escaped characters
unreplacements = {"\(":"(", "\)":")", "\ ":" "}
unescaped_path = "".join([unreplacements.get(c, c) for c in path])
# Now escape unescaped spaces, plus any unescaped parens
replacements = {"(":"\(", ")":"\)", " ":"\ "}
escaped_path = "".join([replacements.get(c, c) for c in unescaped_path])
return escaped_path
@dataclass
class Node:
"""A directory tree node"""
......@@ -101,44 +130,47 @@ def send_directory(node, remote_login, src_path):
"""
# Check if there are any files in the node.
with Connection(remote_login) as c:
files = c.run(
rf"cd {node.src}; nice find -maxdepth 1 -type f -printf '%f\n'",
echo=True
)
files = files.stdout.strip()
with c.cd(escape_parens(node.src)):
result = c.run(r"nice find -maxdepth 1 -type f -printf '%f\n'", echo=True)
files = result.stdout.strip()
node.count = len(files.splitlines())
print(f"Node:{node.src}, file count:{node.count}")
if node.count == 0:
# No files at this node, just return
print("No files to transfer")
elif node.count == 1:
# Only one file. Just copy unchanged.
output = subprocess.run(
f"scp -q {remote_login}:{node.src}/{files} {node.dest}",
shell=True,
check=True
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
# os.chmod(f"{node.dest}/{files}", 0o550)
print(f"Transferred single file {node.src} -> {node.dest}")
# elif node.count == 1:
# # Only one file. Just copy unchanged.
# output = subprocess.run(
# f"scp -q {remote_login}:{node.src}/{files} {node.dest}",
# shell=True,
# check=True
# )
# print("stdout:", output.stdout)
# print("stderr:", output.stderr)
# # os.chmod(f"{node.dest}/{files}", 0o550)
# print(f"Transferred single file {node.src} -> {node.dest}")
else:
# More than one file. Transfer all files to a tarball.
# At least one file. Transfer all files to a tarball.
if node.src == src_path:
filename = os.path.basename(node.src)
else:
filename = node.src.replace(src_path+"/", "").replace("/", "_")
filename = node.src.replace(src_path + "/", "").replace("/", "_")
cmd_src = escape_path(node.src)
cmd_dest = escape_path(node.dest)
cmd_filename = escape_path(filename)
output = subprocess.run(
f"ssh {remote_login} 'cd {node.src};"
f"ssh {remote_login} 'cd {cmd_src};"
f"nice find -maxdepth 1 -type f -print0 |"
f"cpio -o -H ustar -0' | cat > {node.dest}/{filename}.tar",
f"xargs -0 tar -cf - ' | cat > {cmd_dest}/{cmd_filename}.tar",
shell=True,
check=True
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
# os.chmod(f"{node.dest}/{filename}.tar", 0o550)
print(f"Transferred {node.count} files {node.src} -> {node.dest}")
......@@ -185,7 +217,7 @@ def main(
file to True.
"""
assert 5 <= len(experiment_name) <= 6
assert 4 <= len(experiment_name) <= 6
if pickle_filename is None:
pickle_filename = experiment_name+".pickle"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment