Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
R
remote_tree_to_local_tars
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Gary Ruben
remote_tree_to_local_tars
Commits
0a64bf40
Commit
0a64bf40
authored
4 years ago
by
Gary Ruben (Monash University)
Browse files
Options
Downloads
Patches
Plain Diff
Added a display pickle file option
parent
0e20caa0
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
asci_to_vault.py
+31
-22
31 additions, 22 deletions
asci_to_vault.py
with
31 additions
and
22 deletions
asci_to_vault.py
+
31
−
22
View file @
0a64bf40
...
...
@@ -17,7 +17,7 @@ Note that current version creates two files in the same directory as this script
1. A .log file named based on the start-time timestamp which is a capture of all
stdout activity.
2. A Python pickle file named tree_state.pickle that contains the transfer state
from which failed transfers can be restarted by setting the re
ad_pickle_fil
e
from which failed transfers can be restarted by setting the re
sum
e
file to True.
Known issues
...
...
@@ -79,7 +79,7 @@ class Logger(object):
self
.
log
.
flush
()
def
send_directory
(
node
):
def
send_directory
(
node
,
remote_login
,
src_path
):
"""
Sends all files in the node.src directory to the node.dest directory
across an ssh connection.
...
...
@@ -96,6 +96,10 @@ def send_directory(node):
dest: full path to a destination node
e.g. /home/grub0002/bapcxi/vault/imbl2018
count: number of files at the remote node
remote_login: str
remote login username@url
src_path: str
asci src top-level directory
"""
# Check if there are any files in the node.
...
...
@@ -120,7 +124,7 @@ def send_directory(node):
)
print
(
"
stdout:
"
,
output
.
stdout
)
print
(
"
stderr:
"
,
output
.
stderr
)
os
.
chmod
(
f
"
{
node
.
dest
}
/
{
files
}
"
,
0o550
)
#
os.chmod(f"{node.dest}/{files}", 0o550)
print
(
f
"
Transferred single file
{
node
.
src
}
->
{
node
.
dest
}
"
)
else
:
# More than one file. Transfer all files to a tarball.
...
...
@@ -138,7 +142,7 @@ def send_directory(node):
)
print
(
"
stdout:
"
,
output
.
stdout
)
print
(
"
stderr:
"
,
output
.
stderr
)
os
.
chmod
(
f
"
{
node
.
dest
}
/
{
filename
}
.tar
"
,
0o550
)
#
os.chmod(f"{node.dest}/{filename}.tar", 0o550)
print
(
f
"
Transferred
{
node
.
count
}
files
{
node
.
src
}
->
{
node
.
dest
}
"
)
node
.
processed
=
True
...
...
@@ -150,14 +154,16 @@ def send_directory(node):
@click.argument
(
"
src_path
"
,
type
=
click
.
Path
())
@click.argument
(
"
dest_path
"
,
type
=
click
.
Path
())
@click.option
(
"
-p
"
,
"
pickle_filename
"
,
help
=
"
Pickle filename, e.g.
'
foo.pickle
'
(default = experiment_name.pickle
"
)
@click.option
(
"
-r
"
,
"
read_pickle_file
"
,
is_flag
=
True
,
help
=
"
If True, continue from current pickle state
"
)
@click.option
(
"
-r
"
,
"
resume
"
,
is_flag
=
True
,
help
=
"
If True, continue from current pickle state
"
)
@click.option
(
"
-d
"
,
"
display_pickle_file
"
,
is_flag
=
True
,
help
=
"
If True, just show the pickle file state
"
)
def
main
(
remote_login
,
experiment_name
,
src_path
,
dest_path
,
pickle_filename
,
read_pickle_file
resume
,
display_pickle_file
):
"""
\b
...
...
@@ -178,7 +184,7 @@ def main(
1. A .log file named based on the start-time timestamp which is a capture of all
stdout activity.
2. A Python pickle file named tree_state.pickle that contains the transfer state
from which failed transfers can be restarted by setting the re
ad_pickle_fil
e
from which failed transfers can be restarted by setting the re
sum
e
file to True.
"""
...
...
@@ -233,25 +239,30 @@ def main(
src_path =
{
src_path
}
dest_path =
{
dest_path
}
pickle_filename =
{
pickle_filename
}
read_pickle_file =
{
read_pickle_file
}
resume =
{
resume
}
display_pickle_file =
{
display_pickle_file
}
"""
))
# If the re
ad_pickle_fil
e flag is set, resume the transfer.
if
re
ad
_pickle_file
:
# If the re
sum
e flag is set, resume the transfer.
if
re
sume
or
display
_pickle_file
:
# Read the saved transfer state from the locally pickled tree object.
with
open
(
pickle_filename
,
"
rb
"
)
as
f
:
tree
=
pickle
.
load
(
f
)
print
(
"
tree:
"
)
pprint
.
pprint
(
tree
)
# Reset nodes at the end of the list with count==0 to unprocessed
# This is done because we observed a failure that mistakenly reported
# source tree nodes to have 0 files, so force a recheck of those.
for
node
in
reversed
(
tree
):
if
node
.
count
==
0
:
node
.
processed
=
False
else
:
break
if
display_pickle_file
:
sys
.
exit
()
if
resume
:
# Reset nodes at the end of the list with count==0 to unprocessed
# This is done because we observed a failure that mistakenly reported
# source tree nodes to have 0 files, so force a recheck of those.
for
node
in
reversed
(
tree
):
if
node
.
count
==
0
:
node
.
processed
=
False
else
:
break
else
:
# Get the directory tree from the remote server as a list.
with
Connection
(
remote_login
)
as
c
:
...
...
@@ -268,11 +279,9 @@ def main(
# Transfer all directory tree nodes.
for
i
,
node
in
enumerate
(
tree
):
if
not
node
.
processed
:
print
(
pathlib
.
Path
(
node
.
dest
))
1
/
0
pathlib
.
Path
(
node
.
dest
).
mkdir
(
mode
=
0o770
,
parents
=
True
,
exist_ok
=
True
)
os
.
chmod
(
node
.
dest
,
0o770
)
send_directory
(
node
)
#
os.chmod(node.dest, 0o770)
send_directory
(
node
,
remote_login
,
src_path
)
# pickle the tree to keep a record of the processed state.
with
open
(
pickle_filename
,
"
wb
"
)
as
f
:
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment