Added new role to enable Slurm DB SQL to be backed up and then ssh to a node.

Location of node, backup dir, and dummy user account are contained in defaults/main.yml
Both SQL nodes and Management node need to have this role applied, with 'server' parameter determining the different

roles/slurm_sql_bk/defaults/main.yml

+---
+# for slurm mysql backup
+SQL_BK_DIR: "/mnt/db_backup"
+SQL_BK_DEST_HOST: "m3-mgmt1"
+SQL_BK_DEST_DIR: "/mnt/home/slurm_db_backup"
+SQL_USER: "slurmsqlbk"

roles/slurm_sql_bk/files/id_rsa

+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAp6+p7UzisLHTb7P4b2V3tvAtVDNuDV5gMiu+vyNPT/ihjgMo
+BXgz5fuBMlv+1fsmiBAvZBzJD7m5+PkeQ2vXKGywGvUX6OHAppY6AtYf6T9b7gU6
+R1JgXTwu3HTNgtUZ87NvkxbS3jggL40Q6v+MJyXWfS76oBKp8lDVzS27wFDlrMYd
+OdFKeV+KNV68rKziBhTIVQ2B5bq3OhDXISR1jOKHPPEeZVtX9Qmkm10n+0UstyQ6
+UpzOMabY4BiHJh6lrOrnsJ7nAB1sezz8tqLSKvD9Z9i9VlI2aBXgPmNEdqario7F
+v5afAsGxVyngXnXqxtKU4e7xGSQ2JlEeYJBGgQIDAQABAoIBAQCLWgluQknVK0KZ
+/scD1CADsF8LLV5Uz8xDhWUUNMTRob6oZOkaeoUup677acZCj1+7DTBFE0LnxaHO
+TUCgKPt6BbDoBiyjhRBzoD5/YvDMBrJgAIopAUVB1iTfYlXzVyMlCkY7x2jf8WaV
+/bZEhLA1TFWOXL88zaXb0N+MjmgL7hY3yP/i8+6dbDUkPsqnatLPYCNfqxzp+rOZ
+XXCg8DQnfKu32ZyEiBfPqa/DOrLV0f4J+dSA3n/ygjisyzvUe3165AYHGH7ORfyS
+yCPJnIge1Nf1wzmpB9vOf5/Rw+cICBAhzx78rgYqFJ1uTHNuiL3KLC+9LCkGojmY
+hWgVwHzRAoGBANJ4NGBe/SHHc1bQja4iv9gEqaSkxhdHjic9/nW/fp/9p9T7GuDv
+5Qjlos5GdFZp9nMURpeyJstDL/Mq6hWYnT24qS/qVr5CSLsgkE/mF+/0shiF4Fe/
+GPCPUwdwLMSNAz3r9Uir6QSpqfJFqPmmdoLtE12P/E0rkj04HFhDlRVFAoGBAMv2
+HkStzKUeYHChBl1s3ZZxvsG2q1XuSRPI3qbvzwOq5GZmJ9aEv/LxLiSragYju1C6
+LQTKzD3q0iCTcmF6hNee48GotEnbcc909yVpBfOogQ3BPraibVhWKX+Pp/NN1R7X
+PRkDzgD3rpEi1twEEs6783KiHUDrPzvzVsnBW4oNAoGAFYeWVAVjf4EXm1naJGYJ
+md9gOydf3CHFuN3umbztBxC7il3lcf0YQMC81rz+SZitTThVwE7JCSHmVcmeCHeE
+Hjw1kmE2XmAJcJr0KDgxWqp9TWTkfhVeR71hkPE4oyKJolvwy25Kd9+jMtVNY7xT
+gzEdOgTwmjRK0VhcDetb5MECgYBteFjAlePbz9UFPnfzFuZLxBsVT5zgTzerxckz
+Ik0i88sI5tWE/tUjaK1Sgyoxx2XXK0XUZ6B9rxY+48SlayP3stEddRXm+kA4j9fC
+nX7TS2KQ5mJplf5IRl6Zl5oWLXUwy2eAkJk1zbpFLcTSa+FIUnqU0UwQNvNGkxHG
+0d/21QKBgQDMX+rScehbymOj7Lq+eBKoaDPC4cIzxBBXCKZFDxdmd14rGkvceRGN
+BYYo6iWReDhX6P5BnSSaq4ZHCYL1fSVxnVRTuk6F2IOMU7ZeYlxPQDFvqpB8R+RB
+gp/bUuCFRYNpSlEb/GSPvVsl7M6irmdgsAPQTyq5VAxUb9Z0DtrcGw==
+-----END RSA PRIVATE KEY-----

roles/slurm_sql_bk/files/id_rsa.pub

+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCnr6ntTOKwsdNvs/hvZXe28C1UM24NXmAyK76/I09P+KGOAygFeDPl+4EyW/7V+yaIEC9kHMkPubn4+R5Da9cobLAa9Rfo4cCmljoC1h/pP1vuBTpHUmBdPC7cdM2C1Rnzs2+TFtLeOCAvjRDq/4wnJdZ9LvqgEqnyUNXNLbvAUOWsxh050Up5X4o1XrysrOIGFMhVDYHlurc6ENchJHWM4oc88R5lW1f1CaSbXSf7RSy3JDpSnM4xptjgGIcmHqWs6uewnucAHWx7PPy2otIq8P1n2L1WUjZoFeA+Y0R2pquKjsW/lp8CwbFXKeBederG0pTh7vEZJDYmUR5gkEaB root@m3-sql0

roles/slurm_sql_bk/tasks/main.yml

+---
+# this code is for the sql server only
+  - name: template sql backupscript to /etc/cron.daily
+    template: src="backup_mysql_for_slurm.sh.j2" dest="/etc/cron.daily/backup_mysql_for_slurm.sh"  mode="700"
+    sudo: true
+    when: server == 'True'
+  - name: Create directory {{ SQL_BK_DIR }}
+    file: path={{ SQL_BK_DIR }} state=directory
+    sudo: true
+    when: server == 'True'
+  - name: template mysql config file to server
+    template: src="mysql.conf.j2" dest="{{ SQL_BK_DIR }}/mysql.conf" mode="600"
+    sudo: true
+    when: server == 'True'
+  - name: copy ssh pub key to .ssh if it does not exist already
+    copy: src="id_rsa.pub" dest="/root/.ssh/id_rsa.pub"
+    sudo: true
+    when: server == 'True'
+  - name: copy ssh private key to .ssh if it does not exist already
+    copy: src="id_rsa" dest="/root/.ssh/id_rsa" mode="600"
+    sudo: true
+    when: server == 'True'
+    
+#this code is for the Destination Node only
+  - name: create dummy user account
+    user: name="{{ SQL_USER }}" comment="Account for scp of slurm sql backups" 
+    sudo: true
+    when: server == 'False'
+  - name: Add MySQL server ssh key to authorised_files on management nodes"
+    authorized_key: user="{{ SQL_USER }}" state=present  key="{{ lookup('file', 'id_rsa.pub') }}" #"
+    sudo: true
+    when: server == 'False'
+  - name: ensure the dest directory exists (for backups to be copied too)
+    file: path={{ SQL_BK_DEST_DIR }} state=directory owner={{ SQL_USER }}
+    sudo: true
+    when: server == 'False'
+  - name: setup cron job to delete old slurm logs
+    template: src="delete_old_mysql_bk.sh.j2" dest="/etc/cron.daily/delete_old_mysql_bk.sh"  mode="700"
+    sudo: true
+    when: server == 'False'

roles/slurm_sql_bk/templates/backup_mysql_for_slurm.sh.j2

+#!/bin/sh
+#
+# mysql dump for slurm. 
+# S.Michnowicz
+# 20/Jan/2016
+#
+
+TIME=$(date '+%y-%m-%d')
+DIR={{ SQL_BK_DIR }}
+NAME="$DIR/mysql_dump_20${TIME}.sql"
+
+sudo mysqldump  --defaults-file=$DIR/mysql.conf   slurm_acct_db > $NAME
+sudo chmod go-r $NAME
+sudo gzip -f $NAME
+#scp file to   dummy user @ Destination Node and Directory
+scp ${NAME}.gz  {{ SQL_USER }}@{{ SQL_BK_DEST_HOST }}:{{ SQL_BK_DEST_DIR }}
+rm -f  ${NAME}.gz 

roles/slurm_sql_bk/templates/delete_old_mysql_bk.sh.j2

+#!/bin/sh
+#
+# delete old backups from slurm mysql
+# S.Michnowicz
+#
+
+#  2 day timeframe specified by -mtime +2
+#  also includes email check
+#find  {{ SQL_BK_DEST_DIR }} -type f -mtime +2 -name 'mysql_dump_*.gz' -execdir   mail -s "Deleting {}" simon.michnowicz@monash.edu < /dev/null  \; -execdir rm -f {} \;
+find  {{ SQL_BK_DEST_DIR }} -type f -mtime +7 -name 'mysql_dump_*.gz' -execdir rm -f {} \;
+

roles/slurm_sql_bk/templates/mysql.conf.j2

+[client]
+password="{{ sqlrootPasswd }}"
+user=root