diff --git a/asci_to_vault.py b/asci_to_vault.py index feb726f20b4650f7f9b9d3aecdf967f01863487b..555ec60a8bd95fe93bde02da7834a80c1776b0e4 100644 --- a/asci_to_vault.py +++ b/asci_to_vault.py @@ -17,7 +17,7 @@ Note that current version creates two files in the same directory as this script 1. A .log file named based on the start-time timestamp which is a capture of all stdout activity. 2. A Python pickle file named tree_state.pickle that contains the transfer state - from which failed transfers can be restarted by setting the read_pickle_file + from which failed transfers can be restarted by setting the resume file to True. Known issues @@ -79,7 +79,7 @@ class Logger(object): self.log.flush() -def send_directory(node): +def send_directory(node, remote_login, src_path): """Sends all files in the node.src directory to the node.dest directory across an ssh connection. @@ -96,6 +96,10 @@ def send_directory(node): dest: full path to a destination node e.g. /home/grub0002/bapcxi/vault/imbl2018 count: number of files at the remote node + remote_login: str + remote login username@url + src_path: str + asci src top-level directory """ # Check if there are any files in the node. @@ -120,7 +124,7 @@ def send_directory(node): ) print("stdout:", output.stdout) print("stderr:", output.stderr) - os.chmod(f"{node.dest}/{files}", 0o550) + # os.chmod(f"{node.dest}/{files}", 0o550) print(f"Transferred single file {node.src} -> {node.dest}") else: # More than one file. Transfer all files to a tarball. @@ -138,7 +142,7 @@ def send_directory(node): ) print("stdout:", output.stdout) print("stderr:", output.stderr) - os.chmod(f"{node.dest}/{filename}.tar", 0o550) + # os.chmod(f"{node.dest}/{filename}.tar", 0o550) print(f"Transferred {node.count} files {node.src} -> {node.dest}") node.processed = True @@ -150,14 +154,16 @@ def send_directory(node): @click.argument("src_path", type=click.Path()) @click.argument("dest_path", type=click.Path()) @click.option("-p","pickle_filename", help="Pickle filename, e.g. 'foo.pickle' (default = experiment_name.pickle") -@click.option("-r","read_pickle_file",is_flag=True, help="If True, continue from current pickle state") +@click.option("-r","resume",is_flag=True, help="If True, continue from current pickle state") +@click.option("-d","display_pickle_file",is_flag=True, help="If True, just show the pickle file state") def main( remote_login, experiment_name, src_path, dest_path, pickle_filename, - read_pickle_file + resume, + display_pickle_file ): """ \b @@ -178,7 +184,7 @@ def main( 1. A .log file named based on the start-time timestamp which is a capture of all stdout activity. 2. A Python pickle file named tree_state.pickle that contains the transfer state - from which failed transfers can be restarted by setting the read_pickle_file + from which failed transfers can be restarted by setting the resume file to True. """ @@ -233,25 +239,30 @@ def main( src_path = {src_path} dest_path = {dest_path} pickle_filename = {pickle_filename} - read_pickle_file = {read_pickle_file} + resume = {resume} + display_pickle_file = {display_pickle_file} """)) - # If the read_pickle_file flag is set, resume the transfer. - if read_pickle_file: + # If the resume flag is set, resume the transfer. + if resume or display_pickle_file: # Read the saved transfer state from the locally pickled tree object. with open(pickle_filename, "rb") as f: tree = pickle.load(f) print("tree:") pprint.pprint(tree) - # Reset nodes at the end of the list with count==0 to unprocessed - # This is done because we observed a failure that mistakenly reported - # source tree nodes to have 0 files, so force a recheck of those. - for node in reversed(tree): - if node.count == 0: - node.processed = False - else: - break + if display_pickle_file: + sys.exit() + + if resume: + # Reset nodes at the end of the list with count==0 to unprocessed + # This is done because we observed a failure that mistakenly reported + # source tree nodes to have 0 files, so force a recheck of those. + for node in reversed(tree): + if node.count == 0: + node.processed = False + else: + break else: # Get the directory tree from the remote server as a list. with Connection(remote_login) as c: @@ -268,11 +279,9 @@ def main( # Transfer all directory tree nodes. for i, node in enumerate(tree): if not node.processed: - print(pathlib.Path(node.dest)) - 1/0 pathlib.Path(node.dest).mkdir(mode=0o770, parents=True, exist_ok=True) - os.chmod(node.dest, 0o770) - send_directory(node) + # os.chmod(node.dest, 0o770) + send_directory(node, remote_login, src_path) # pickle the tree to keep a record of the processed state. with open(pickle_filename, "wb") as f: