Merge branch 'sql_backup' into 'master'

Sql backup

See merge request hpc-team/ansible_cluster_in_a_box!229

Former-commit-id: a8de5ee7

roles/slurm_sql_bk/README.md

+This role sets up a cronjob on one  Management machine to perform a mysql dump.(via a remote mysql dump to SQL machine)
+This file is then gzipped and a 'scp' done to transfer it to the destination machine for storage.(SQL_BK_DEST_HOST)
+
+As this script is meant to be portable across clusters, some variables need to defined when called.
+
+Architecture:
+	As SQL machine may not be able to access SQL_BK_DEST_HOST, a management machine must do a dump. Then this is scp'd to SQL_BK_DEST_HOST
+	MySQL password is stored in mysql.conf in the BIN dir of Management Machine
+	Only one Management should do the dump for performance reasons
+        A cron job on SQL_BK_DEST_HOST deletes any backups > 7 days
+
+
+EXAMPLE USAGE:
+
+*ansible-playbook -i static --limit=monarch-mgmt1 slurm_bk.yml*
+
+where slurm_bk.yml contains
+
+~~~~
+- hosts: 'ManagementNodes'
+  gather_facts: True
+  roles:
+  - { role: slurm_sql_bk, create_key: "True", SQL_BK_DEST_HOST: "118.138.234.186", SQL_BK_DEST_DIR: "/mnt/backup/monarch", SQL_IP: "172.16.226.88" , sqlUsername: "slurmdb", sqlPasswd: "{{ slurmdb_passwd }}" }
+~~~~
+
+Meaning:
+* **create_key**: If defined to True then ssh-keygen is called on local machine and then:
+	* private key copied to ~/.ssh on management node
+	* public key inserted into authroized_keys on SQL_BK_DEST_HOST
+* **SQL_BK_DEST_HOST:** IP number of Machine where we store mysql dumps
+* **SQL_BK_DEST_DIR:** Directory on backup machine for mysql dumps. Note this is owned by ec2-user and is manually created. It should be cluster-specific, i.e. different for each cluster
+* **SQL_IP:** IP number of slurm mysql machine
+* **sqlUsername,sqlPasswd** Mysql username/password, same as in slurmdbd.conf
+
+Other VARIABLES:
+
+defined in default/main.yml
+* **BIN_DIR:**          "/root/bin" #where the backup shell script and mysql.conf exists on Management Node
+* **SQL_BK_DATA_DIR:**  "/tmp" # where the inital dump exists on the management node. The file is deleted after a scp so it should not fill up disk
+* **SQL_USER: "ec2-user"**  The user account on SQL_BK_DEST_HOST. Normally ec2-user but could be something else for security reasons

roles/slurm_sql_bk/defaults/main.yml

 ---
 # for slurm mysql backup
-SQL_BK_DIR: "/mnt/db_backup"
-SQL_BK_DEST_HOST: "m3-mgmt1"
-SQL_BK_DEST_DIR: "/mnt/home/slurm_db_backup"
-SQL_USER: "slurmsqlbk"
+MGMT_BIN_DIR: "/root/bin"
+SQL_BK_DATA_DIR: "/tmp"
+SQL_USER: "ec2-user"

roles/slurm_sql_bk/tasks/main.yml

 ---
-# this code is for the sql server only
-  - name: template sql backupscript to /etc/cron.daily
-    template: src="backup_mysql_for_slurm.sh.j2" dest="/etc/cron.daily/backup_mysql_for_slurm.sh"  mode="700"
-    sudo: true
-    when: server == 'True'
-  - name: Create directory {{ SQL_BK_DIR }}
-    file: path={{ SQL_BK_DIR }} state=directory
-    sudo: true
-    when: server == 'True'
+  #
+  #first generate ssh keys Gif the variable "create_key" is defined.
+  #
+  - name: delete any existing local private key
+    local_action: command rm -f  ./slm_db_backup
+    when: create_key is defined and create_key=="True"
+  - name: delete any existing local public keys
+    local_action: command rm -f ./slm_db_backup.pub
+    when: create_key is defined and create_key=="True"
+  - name: generate ssh keys if necessary
+    #this command will create a two files "slm_db_backup" and "slm_db_backup.pub"
+    local_action: command ssh-keygen -t rsa -f slm_db_backup -P ""
+    when: create_key is defined and create_key=="True"
+  - name: copy private key to management node
+    copy:
+      src: "./slm_db_backup"
+      dest: "/root/.ssh"
+      owner: root 
+      group: root
+      mode: '600'
+    become: True
+    become_user: root
+    when: create_key is defined and create_key=="True"
+  - name: copy public key to authorised key file of backup volume machine
+    local_action: command  ssh-copy-id -i ./slm_db_backup.pub  {{ SQL_BK_DEST_HOST }}
+    when: create_key is defined and create_key=="True"
+  #
+  # now setup cronjob on management node
+  #
+  - name: ensure {{ MGMT_BIN_DIR }} exists
+    file:
+      path: "{{ MGMT_BIN_DIR }}"
+      state: directory
+    become: true
+    become_user: root 
+  - name: "template sql backupscript to  {{ MGMT_BIN_DIR }}"
+    template: src="backup_mysql_for_slurm.sh.j2" dest="{{ MGMT_BIN_DIR }}/backup_mysql_for_slurm.sh"  mode="700"
+    become: true
+    become_user: root 
+  - name: Make a daily crontab entry
+    cron:
+      name: "Backup of MySQL Database for Slurm"
+      job: "{{ MGMT_BIN_DIR }}/backup_mysql_for_slurm.sh"  
+      hour: 23
+      minute: 55
+    become: true
+    become_user: root 
+  - name: Create directory {{ SQL_BK_DATA_DIR }} to store initial mysql dump
+    file: path={{ SQL_BK_DATA_DIR }} state=directory
+    become: true
+    become_user: root 
   - name: template mysql config file to server
-    template: src="mysql.conf.j2" dest="{{ SQL_BK_DIR }}/mysql.conf" mode="600"
-    sudo: true
-    when: server == 'True'
-  - name: copy ssh pub key to .ssh if it does not exist already
-    copy: src="id_rsa.pub" dest="/root/.ssh/id_rsa.pub"
-    sudo: true
-    when: server == 'True'
-  - name: copy ssh private key to .ssh if it does not exist already
-    copy: src="id_rsa" dest="/root/.ssh/id_rsa" mode="600"
-    sudo: true
-    when: server == 'True'
-    
-#this code is for the Destination Node only
-  - name: create dummy user account
-    user: name="{{ SQL_USER }}" comment="Account for scp of slurm sql backups" 
-    sudo: true
-    when: server == 'False'
-  - name: Add MySQL server ssh key to authorised_files on management nodes"
-    authorized_key: user="{{ SQL_USER }}" state=present  key="{{ lookup('file', 'id_rsa.pub') }}" #"
-    sudo: true
-    when: server == 'False'
-  - name: ensure the dest directory exists (for backups to be copied too)
-    file: path={{ SQL_BK_DEST_DIR }} state=directory owner={{ SQL_USER }}
-    sudo: true
-    when: server == 'False'
-  - name: setup cron job to delete old slurm logs
-    template: src="delete_old_mysql_bk.sh.j2" dest="/etc/cron.daily/delete_old_mysql_bk.sh"  mode="700"
-    sudo: true
-    when: server == 'False'
+    template: src="mysql.conf.j2" dest="{{ MGMT_BIN_DIR }}/mysql.conf" mode="600"
+    become: true
+    become_user: root
+  #
+  # template delete file to localhost.then copy to remote host
+  #
+  - name: make a unique name for the backup script 
+    set_fact: 
+        unique_name: "delete_old_mysql_bk_{{ SQL_BK_DEST_DIR | basename }}.sh"
+  - name: Unique filename is
+    debug: var=unique_name
+  - name: delete local del file
+    local_action: command rm -f  ./{{ unique_name }}
+  - name: template delete script to local dir
+    local_action: template src=delete_old_mysql_bk.sh.j2 dest=./{{ unique_name }}
+  - name: copy backup script to server ec2-user@{{ SQL_BK_DEST_HOST }}
+    local_action: command scp -i ./slm_db_backup ./{{ unique_name }}  "ec2-user@{{ SQL_BK_DEST_HOST }}:"
+  - name: insert delete cron job entry on remote server
+    local_action: command ssh -i ./slm_db_backup ec2-user@{{ SQL_BK_DEST_HOST }}  "{ crontab -l ; echo '#delete old slurm backups' ; echo '00 23 * * * /home/ec2-user/{{ unique_name }}' ; } | crontab  -  "

roles/slurm_sql_bk/templates/backup_mysql_for_slurm.sh.j2

 #!/bin/sh
 #
 # mysql dump for slurm. 
-# S.Michnowicz
-# 20/Jan/2016
 #
 
 TIME=$(date '+%y-%m-%d')
-DIR={{ SQL_BK_DIR }}
-NAME="$DIR/mysql_dump_20${TIME}.sql"
+BIN_DIR={{ MGMT_BIN_DIR }}
+DATA_DIR={{ SQL_BK_DATA_DIR }}
+NAME="$DATA_DIR/mysql_dump_20${TIME}.sql"
 
-sudo mysqldump  --defaults-file=$DIR/mysql.conf   slurm_acct_db > $NAME
+cd $DATA_DIR
+sudo mysqldump --defaults-file=$BIN_DIR/mysql.conf --host={{ SQL_IP }}   slurm_acct_db > $NAME
 sudo chmod go-r $NAME
 sudo gzip -f $NAME
 #scp file to   dummy user @ Destination Node and Directory
-scp ${NAME}.gz  {{ SQL_USER }}@{{ SQL_BK_DEST_HOST }}:{{ SQL_BK_DEST_DIR }}
+scp -i ~/.ssh/slm_db_backup ${NAME}.gz  {{ SQL_USER }}@{{ SQL_BK_DEST_HOST }}:{{ SQL_BK_DEST_DIR }}
 rm -f  ${NAME}.gz 

roles/slurm_sql_bk/templates/mysql.conf.j2

 [client]
-password="{{ sqlrootPasswd }}"
-user=root
+password="{{ sqlPasswd }}"
+user="{{ sqlUsername }}"