diff --git a/asci_to_vault.py b/asci_to_vault.py index d62d911083806dc162029d1edc13ae1217397542..d1b863d3258fdc27d542bd0a7c24faaa5c042327 100644 --- a/asci_to_vault.py +++ b/asci_to_vault.py @@ -91,21 +91,24 @@ class Logger(object): def send_directory(node): """Sends all files in the node.src directory to the node.dest directory - across an ssh connection using the cpio command to tar the files into a + across an ssh connection. + + Different methods are used for single versus multiple files. For single + files, scp is used. For multiple files cpio is used to tar the files into a single tarball. The destination tarball is named after the directories - trailing the SRC_PATH. Permissions are set to r_x for group and owner. + trailing SRC_PATH. Permissions are set to r_x for group and owner. Args: node: Node object - Contains source and destination directories as follows: + Contains source and destination directory information as follows: src: full path to a remote node e.g. /data/13660a/asci/input dest: full path to a destination node e.g. /home/grub0002/bapcxi/vault/imbl2018 + count: number of files at the remote node """ - # Check if there are any files in the node - # add check to make sure connection is working + # Check if there are any files in the node. with Connection(REMOTE_LOGIN) as c: files = c.run( rf"cd {node.src}; find -maxdepth 1 -type f -printf '%f\n'", @@ -155,8 +158,7 @@ if __name__ == "__main__": sys.stdout = Logger() # Log all stdout to a log file # A hacky way to restart an interrupted transfer is to set - # READ_PICKLE_FILE = True above so that the transfer state is retrieved. By - # default the tree is built from scratch from the remote file system. + # READ_PICKLE_FILE = True above so that the transfer state is retrieved. if READ_PICKLE_FILE: # Read the saved transfer state from the locally pickled tree object. with open(PICKLE_FILENAME, 'rb') as f: @@ -173,7 +175,7 @@ if __name__ == "__main__": else: break else: - # Get the directory tree from remote server as a list + # Get the directory tree from the remote server as a list. with Connection(REMOTE_LOGIN) as c: result = c.run(f'find {SRC_PATH} -type d') remote_dirs = result.stdout.strip().splitlines() @@ -185,14 +187,14 @@ if __name__ == "__main__": dest = src.replace(SRC_PATH, DEST_PATH) tree.append(Node(src, dest)) - # Transfer all directory tree nodes + # Transfer all directory tree nodes. for i, node in enumerate(tree): if not node.processed: pathlib.Path(node.dest).mkdir(parents=True, exist_ok=True) os.chmod(node.dest, 0o770) send_directory(node) - # pickle the tree to keep a record of the processed state + # pickle the tree to keep a record of the processed state. with open(PICKLE_FILENAME, 'wb') as f: pickle.dump(tree, f)