Commit 0b4176ab authored by Gary Ruben's avatar Gary Ruben
Browse files

documentation improvements

parent d36a9b3f
......@@ -91,21 +91,24 @@ class Logger(object):
def send_directory(node):
"""Sends all files in the node.src directory to the node.dest directory
across an ssh connection using the cpio command to tar the files into a
across an ssh connection.
Different methods are used for single versus multiple files. For single
files, scp is used. For multiple files cpio is used to tar the files into a
single tarball. The destination tarball is named after the directories
trailing the SRC_PATH. Permissions are set to r_x for group and owner.
trailing SRC_PATH. Permissions are set to r_x for group and owner.
Args:
node: Node object
Contains source and destination directories as follows:
Contains source and destination directory information as follows:
src: full path to a remote node
e.g. /data/13660a/asci/input
dest: full path to a destination node
e.g. /home/grub0002/bapcxi/vault/imbl2018
count: number of files at the remote node
"""
# Check if there are any files in the node
# add check to make sure connection is working
# Check if there are any files in the node.
with Connection(REMOTE_LOGIN) as c:
files = c.run(
rf"cd {node.src}; find -maxdepth 1 -type f -printf '%f\n'",
......@@ -155,8 +158,7 @@ if __name__ == "__main__":
sys.stdout = Logger() # Log all stdout to a log file
# A hacky way to restart an interrupted transfer is to set
# READ_PICKLE_FILE = True above so that the transfer state is retrieved. By
# default the tree is built from scratch from the remote file system.
# READ_PICKLE_FILE = True above so that the transfer state is retrieved.
if READ_PICKLE_FILE:
# Read the saved transfer state from the locally pickled tree object.
with open(PICKLE_FILENAME, 'rb') as f:
......@@ -173,7 +175,7 @@ if __name__ == "__main__":
else:
break
else:
# Get the directory tree from remote server as a list
# Get the directory tree from the remote server as a list.
with Connection(REMOTE_LOGIN) as c:
result = c.run(f'find {SRC_PATH} -type d')
remote_dirs = result.stdout.strip().splitlines()
......@@ -185,14 +187,14 @@ if __name__ == "__main__":
dest = src.replace(SRC_PATH, DEST_PATH)
tree.append(Node(src, dest))
# Transfer all directory tree nodes
# Transfer all directory tree nodes.
for i, node in enumerate(tree):
if not node.processed:
pathlib.Path(node.dest).mkdir(parents=True, exist_ok=True)
os.chmod(node.dest, 0o770)
send_directory(node)
# pickle the tree to keep a record of the processed state
# pickle the tree to keep a record of the processed state.
with open(PICKLE_FILENAME, 'wb') as f:
pickle.dump(tree, f)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment