Skip to content
Snippets Groups Projects
Commit 1b239383 authored by Gary Ruben (Monash University)'s avatar Gary Ruben (Monash University)
Browse files

Heavy rewrite to use Fabric's context managers as intended, but requiring a bugfix.

parent 4ebbc3c3
No related branches found
No related tags found
No related merge requests found
......@@ -37,9 +37,21 @@ import pprint
import time
import click
import textwrap
from shlex import quote
from fabric import Connection
def escape_path(path):
""" Explicitly escape parentheses. This is required to work around a bug in Fabric's
Invoke module. See my question on Stackoverflow:
https://stackoverflow.com/q/63225018/607587
The recommended workaround, until Fabric fixes the bug, is to just "manually escape
the parentheses"
"""
escaped_path = path.replace('(', '\(').replace(')', '\)')
return escaped_path
@dataclass
class Node:
"""A directory tree node"""
......@@ -83,11 +95,10 @@ def send_checksum(node, remote_login, src_path):
"""
# Check if there are any files in the node
with Connection(remote_login) as c:
files = c.run(
rf"cd {node.src}; find -maxdepth 1 -type f -printf '%f\n'",
echo=True
)
files = files.stdout.strip()
with c.cd(escape_path(node.src)):
result = c.run(r"nice find -maxdepth 1 -type f -printf '%f\n'", echo=True)
files = result.stdout.strip()
node.count = len(files.splitlines())
print(f"Node:{node.src}, file count:{node.count}")
......@@ -101,16 +112,18 @@ def send_checksum(node, remote_login, src_path):
else:
filename = node.src.replace(src_path+'/', '').replace('/', '_')
output = subprocess.run(
f"ssh {remote_login} 'cd {node.src}; nice md5sum *'"
f"| cat > {node.dest}/{filename}.md5",
shell=True,
check=True
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
with Connection(remote_login) as c:
with c.cd(escape_path(node.src)):
# checksum all files including hidden files
result = c.run("nice md5sum {*,.[^.],.??*}", echo=True)
md5s = result.stdout
md5_filename = pathlib.Path(node.dest) / (filename + '.md5')
with open(md5_filename, 'w') as f:
f.write(md5s)
print(f"Checksummed {node.count} files {node.src} -> {md5_filename}")
# os.chmod(f"{node.dest}/{filename}.md5", 0o550)
print(f"Checksummed {node.count} files {node.src} -> {node.dest}")
node.processed = True
......@@ -155,7 +168,7 @@ def main(
flag.
"""
assert 5 <= len(experiment_name) <= 6
assert 4 <= len(experiment_name) <= 6
if pickle_filename is None:
pickle_filename = experiment_name + "md5_state.pickle"
......@@ -224,7 +237,7 @@ def main(
else:
# Get the directory tree from the remote server as a list.
with Connection(remote_login) as c:
result = c.run(f"find {src_path} -type d")
result = c.run(f"nice find {quote(src_path)} -type d")
remote_dirs = result.stdout.strip().splitlines()
# Create a tree data structure that represents both source and
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment