Replaced os.system with subprocess.run and added node.processed flag

asci_to_vault.py

@@ -10,6 +10,7 @@ import os
 import warnings
 from dataclasses import dataclass
 import pathlib
+import subprocess
 
 # This isn't suppressing the warnings that fabric is generating; we need to
 # investigate further
@@ -26,6 +27,7 @@ KERNEL_CHECKSUM=$(cpio --to-stdout -i kernel.fat16 < archive.cpio  | sha256sum |
 """
 
 REMOTE_LOGIN = "gary.ruben@monash.edu@sftp.synchrotron.org.au"
+# SRC_PATH = "/data/13660a/asci/input"
 SRC_PATH = "/data/13660a/asci/output/tar_test"
 DEST_PATH = "/home/grub0002/bapcxi/vault/rubbish"
 
@@ -34,6 +36,7 @@ DEST_PATH = "/home/grub0002/bapcxi/vault/rubbish"
 class Node:
     src: str
     dest: str
+    processed: bool = False
 
 
 def tar_and_send_directory(node):
@@ -57,16 +60,21 @@ def tar_and_send_directory(node):
         files = files.stdout.strip()
     count = files.count('/')
 
+    print(f'Node:{node.src}, file count:{count}')
     if count == 0:
         # No files at this node, just return
-        return
+        pass
     elif count == 1:
         # Only one file. No need to tar. Just copy unchanged.
-        os.system(
+        output = subprocess.run(
             f"ssh {REMOTE_LOGIN} 'cd {node.src};"
             f"find -type f -maxdepth 1 | cpio -o' |"
-            f"cat > {node.dest}/{files}"
+            f"cat > {node.dest}/{files}",
+            shell=True,
+            check=True
         )
+        print('stdout:', output.stdout)
+        print('stderr:', output.stderr)
         os.chmod(f'{node.dest}/{files}', 0o550)
         print(f'transfer {node.src} -> {node.dest}')
     else:
@@ -76,14 +84,19 @@ def tar_and_send_directory(node):
         else:
             filename = node.src.replace(SRC_PATH+'/', '').replace('/', '_')
 
-        os.system(
+        output = subprocess.run(
             f"ssh {REMOTE_LOGIN} 'cd {node.src};"
             f"find -type f -maxdepth 1 -print0 | cpio -o -H ustar -0' |"
-            f"cat > {node.dest}/{filename}.tar"
+            f"cat > {node.dest}/{filename}.tar",
+            shell=True,
+            check=True
         )
+        print('stdout:', output.stdout)
+        print('stderr:', output.stderr)
         os.chmod(f'{node.dest}/{filename}.tar', 0o550)
         print(f'transfer {node.src} -> {node.dest}')
 
+    node.processed = True
 
 if __name__ == "__main__":
     # Get the directory tree from remote server as a list