Skip to content
Snippets Groups Projects
Commit 65a9f305 authored by Gary Ruben (Monash University)'s avatar Gary Ruben (Monash University)
Browse files

Changes to use Fabric Connection properly, and change from cpio to tar to support >2GB files

parent 1b239383
No related branches found
No related tags found
No related merge requests found
......@@ -53,6 +53,35 @@ import textwrap
from fabric import Connection
def escape_parens(path):
""" Explicitly escape parentheses. This is required to work around a bug in Fabric's
Invoke module. See my question on Stackoverflow:
https://stackoverflow.com/q/63225018/607587
The recommended workaround, until Fabric fixes the bug, is to just "manually escape
the parentheses"
I used this method: https://stackoverflow.com/a/23563806/607587
"""
replacements = {"(":"\(", ")":"\)"}
escaped_path = "".join([replacements.get(c, c) for c in path])
return escaped_path
def escape_path(path):
""" Explicitly escape parentheses AND spaces.
I used this method: https://stackoverflow.com/a/23563806/607587
"""
# kludge; first (un)escape any already escaped characters
unreplacements = {"\(":"(", "\)":")", "\ ":" "}
unescaped_path = "".join([unreplacements.get(c, c) for c in path])
# Now escape unescaped spaces, plus any unescaped parens
replacements = {"(":"\(", ")":"\)", " ":"\ "}
escaped_path = "".join([replacements.get(c, c) for c in unescaped_path])
return escaped_path
@dataclass
class Node:
"""A directory tree node"""
......@@ -101,44 +130,47 @@ def send_directory(node, remote_login, src_path):
"""
# Check if there are any files in the node.
with Connection(remote_login) as c:
files = c.run(
rf"cd {node.src}; nice find -maxdepth 1 -type f -printf '%f\n'",
echo=True
)
files = files.stdout.strip()
with c.cd(escape_parens(node.src)):
result = c.run(r"nice find -maxdepth 1 -type f -printf '%f\n'", echo=True)
files = result.stdout.strip()
node.count = len(files.splitlines())
print(f"Node:{node.src}, file count:{node.count}")
if node.count == 0:
# No files at this node, just return
print("No files to transfer")
elif node.count == 1:
# Only one file. Just copy unchanged.
output = subprocess.run(
f"scp -q {remote_login}:{node.src}/{files} {node.dest}",
shell=True,
check=True
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
# os.chmod(f"{node.dest}/{files}", 0o550)
print(f"Transferred single file {node.src} -> {node.dest}")
# elif node.count == 1:
# # Only one file. Just copy unchanged.
# output = subprocess.run(
# f"scp -q {remote_login}:{node.src}/{files} {node.dest}",
# shell=True,
# check=True
# )
# print("stdout:", output.stdout)
# print("stderr:", output.stderr)
# # os.chmod(f"{node.dest}/{files}", 0o550)
# print(f"Transferred single file {node.src} -> {node.dest}")
else:
# More than one file. Transfer all files to a tarball.
# At least one file. Transfer all files to a tarball.
if node.src == src_path:
filename = os.path.basename(node.src)
else:
filename = node.src.replace(src_path+"/", "").replace("/", "_")
filename = node.src.replace(src_path + "/", "").replace("/", "_")
cmd_src = escape_path(node.src)
cmd_dest = escape_path(node.dest)
cmd_filename = escape_path(filename)
output = subprocess.run(
f"ssh {remote_login} 'cd {node.src};"
f"ssh {remote_login} 'cd {cmd_src};"
f"nice find -maxdepth 1 -type f -print0 |"
f"cpio -o -H ustar -0' | cat > {node.dest}/{filename}.tar",
f"xargs -0 tar -cf - ' | cat > {cmd_dest}/{cmd_filename}.tar",
shell=True,
check=True
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
# os.chmod(f"{node.dest}/{filename}.tar", 0o550)
print(f"Transferred {node.count} files {node.src} -> {node.dest}")
......@@ -185,7 +217,7 @@ def main(
file to True.
"""
assert 5 <= len(experiment_name) <= 6
assert 4 <= len(experiment_name) <= 6
if pickle_filename is None:
pickle_filename = experiment_name+".pickle"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment