Skip to content
Snippets Groups Projects
Commit 0a64bf40 authored by Gary Ruben (Monash University)'s avatar Gary Ruben (Monash University)
Browse files

Added a display pickle file option

parent 0e20caa0
No related branches found
No related tags found
No related merge requests found
......@@ -17,7 +17,7 @@ Note that current version creates two files in the same directory as this script
1. A .log file named based on the start-time timestamp which is a capture of all
stdout activity.
2. A Python pickle file named tree_state.pickle that contains the transfer state
from which failed transfers can be restarted by setting the read_pickle_file
from which failed transfers can be restarted by setting the resume
file to True.
Known issues
......@@ -79,7 +79,7 @@ class Logger(object):
self.log.flush()
def send_directory(node):
def send_directory(node, remote_login, src_path):
"""Sends all files in the node.src directory to the node.dest directory
across an ssh connection.
......@@ -96,6 +96,10 @@ def send_directory(node):
dest: full path to a destination node
e.g. /home/grub0002/bapcxi/vault/imbl2018
count: number of files at the remote node
remote_login: str
remote login username@url
src_path: str
asci src top-level directory
"""
# Check if there are any files in the node.
......@@ -120,7 +124,7 @@ def send_directory(node):
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
os.chmod(f"{node.dest}/{files}", 0o550)
# os.chmod(f"{node.dest}/{files}", 0o550)
print(f"Transferred single file {node.src} -> {node.dest}")
else:
# More than one file. Transfer all files to a tarball.
......@@ -138,7 +142,7 @@ def send_directory(node):
)
print("stdout:", output.stdout)
print("stderr:", output.stderr)
os.chmod(f"{node.dest}/{filename}.tar", 0o550)
# os.chmod(f"{node.dest}/{filename}.tar", 0o550)
print(f"Transferred {node.count} files {node.src} -> {node.dest}")
node.processed = True
......@@ -150,14 +154,16 @@ def send_directory(node):
@click.argument("src_path", type=click.Path())
@click.argument("dest_path", type=click.Path())
@click.option("-p","pickle_filename", help="Pickle filename, e.g. 'foo.pickle' (default = experiment_name.pickle")
@click.option("-r","read_pickle_file",is_flag=True, help="If True, continue from current pickle state")
@click.option("-r","resume",is_flag=True, help="If True, continue from current pickle state")
@click.option("-d","display_pickle_file",is_flag=True, help="If True, just show the pickle file state")
def main(
remote_login,
experiment_name,
src_path,
dest_path,
pickle_filename,
read_pickle_file
resume,
display_pickle_file
):
"""
\b
......@@ -178,7 +184,7 @@ def main(
1. A .log file named based on the start-time timestamp which is a capture of all
stdout activity.
2. A Python pickle file named tree_state.pickle that contains the transfer state
from which failed transfers can be restarted by setting the read_pickle_file
from which failed transfers can be restarted by setting the resume
file to True.
"""
......@@ -233,25 +239,30 @@ def main(
src_path = {src_path}
dest_path = {dest_path}
pickle_filename = {pickle_filename}
read_pickle_file = {read_pickle_file}
resume = {resume}
display_pickle_file = {display_pickle_file}
"""))
# If the read_pickle_file flag is set, resume the transfer.
if read_pickle_file:
# If the resume flag is set, resume the transfer.
if resume or display_pickle_file:
# Read the saved transfer state from the locally pickled tree object.
with open(pickle_filename, "rb") as f:
tree = pickle.load(f)
print("tree:")
pprint.pprint(tree)
# Reset nodes at the end of the list with count==0 to unprocessed
# This is done because we observed a failure that mistakenly reported
# source tree nodes to have 0 files, so force a recheck of those.
for node in reversed(tree):
if node.count == 0:
node.processed = False
else:
break
if display_pickle_file:
sys.exit()
if resume:
# Reset nodes at the end of the list with count==0 to unprocessed
# This is done because we observed a failure that mistakenly reported
# source tree nodes to have 0 files, so force a recheck of those.
for node in reversed(tree):
if node.count == 0:
node.processed = False
else:
break
else:
# Get the directory tree from the remote server as a list.
with Connection(remote_login) as c:
......@@ -268,11 +279,9 @@ def main(
# Transfer all directory tree nodes.
for i, node in enumerate(tree):
if not node.processed:
print(pathlib.Path(node.dest))
1/0
pathlib.Path(node.dest).mkdir(mode=0o770, parents=True, exist_ok=True)
os.chmod(node.dest, 0o770)
send_directory(node)
# os.chmod(node.dest, 0o770)
send_directory(node, remote_login, src_path)
# pickle the tree to keep a record of the processed state.
with open(pickle_filename, "wb") as f:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment