Commit 1b239383 authored by Gary Ruben (Monash University)'s avatar Gary Ruben (Monash University)
Browse files

Heavy rewrite to use Fabric's context managers as intended, but requiring a bugfix.

parent 4ebbc3c3
......@@ -37,9 +37,21 @@ import pprint
import time
import click
import textwrap
from shlex import quote
from fabric import Connection
def escape_path(path):
""" Explicitly escape parentheses. This is required to work around a bug in Fabric's
Invoke module. See my question on Stackoverflow:
https://stackoverflow.com/q/63225018/607587
The recommended workaround, until Fabric fixes the bug, is to just "manually escape
the parentheses"
"""
escaped_path = path.replace('(', '\(').replace(')', '\)')
return escaped_path
@dataclass
class Node:
"""A directory tree node"""
......@@ -83,11 +95,10 @@ def send_checksum(node, remote_login, src_path):
"""
# Check if there are any files in the node
with Connection(remote_login) as c:
files = c.run(
rf"cd {node.src}; find -maxdepth 1 -type f -printf '%f\n'",
echo=True
)
files = files.stdout.strip()
with c.cd(escape_path(node.src)):
result = c.run(r"nice find -maxdepth 1 -type f -printf '%f\n'", echo=True)
files = result.stdout.strip()
node.count = len(files.splitlines())
print(f"Node:{node.src}, file count:{node.count}")
......@@ -101,16 +112,18 @@ def send_checksum(node, remote_login, src_path):
else:
filename = node.src.replace(src_path+'/', '').replace('/', '_')
output = subprocess.run(
f"ssh {remote_login} 'cd {node.src}; nice md5sum *'"
f"| cat > {node.dest}/{filename}.md5",
shell=True,
check=True
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
with Connection(remote_login) as c:
with c.cd(escape_path(node.src)):
# checksum all files including hidden files
result = c.run("nice md5sum {*,.[^.],.??*}", echo=True)
md5s = result.stdout
md5_filename = pathlib.Path(node.dest) / (filename + '.md5')
with open(md5_filename, 'w') as f:
f.write(md5s)
print(f"Checksummed {node.count} files {node.src} -> {md5_filename}")
# os.chmod(f"{node.dest}/{filename}.md5", 0o550)
print(f"Checksummed {node.count} files {node.src} -> {node.dest}")
node.processed = True
......@@ -155,7 +168,7 @@ def main(
flag.
"""
assert 5 <= len(experiment_name) <= 6
assert 4 <= len(experiment_name) <= 6
if pickle_filename is None:
pickle_filename = experiment_name + "md5_state.pickle"
......@@ -224,7 +237,7 @@ def main(
else:
# Get the directory tree from the remote server as a list.
with Connection(remote_login) as c:
result = c.run(f"find {src_path} -type d")
result = c.run(f"nice find {quote(src_path)} -type d")
remote_dirs = result.stdout.strip().splitlines()
# Create a tree data structure that represents both source and
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment