do not start slurmctld if gluster is still healing

roles/slurm-start/defaults/main.yml

+use_glusterfs: true
+#EMAIL_DEST I am not providing a default to force users to override.
\ No newline at end of file

roles/slurm-start/tasks/main.yml

@@ -32,6 +32,12 @@
   when: use_systemd is defined and start_slurmd is defined
   register: slurmd_service_installed
 
+- name: deploy glusterfsheltest
+  template: dest=/etc/systemd/system/glusterfsheltest.sh src=glusterfsheltest.j2 mode=744
+  become: true
+  when: use_systemd is defined and start_slurmctld is defined
+  register: slurmctld_service_installed
+
 - name: slurmctld.service
   template: dest=/etc/systemd/system/slurmctld.service src=slurmctld.service.j2 mode=644
   become: true

roles/slurm-start/templates/glusterfsheltest.sh.j2

+#!/bin/bash
+exitCode=[[ "3" == `gluster volume heal gv info | grep -c "Number of entries: 0"` ]]
+
+HEADER="glusterfs preventing start of slurmctld on `hostname`"
+MAILTO="{{ EMAIL_DEST }}"
+
+echo $exitCode
+if [ $exitCode -eq  0 ]
+then
+    EmailBody="glusterfs preventing start of slurmctld on `hostname` on `date` \ncheck via gluster volume heal gv info"
+    echo -e "$EmailBody" | mail -s "$HEADER" "$MAILTO"
+fi
+return $exitCode
\ No newline at end of file

roles/slurm-start/templates/slurmctld.service.j2

@@ -6,6 +6,7 @@ ConditionPathExists={{ slurm_dir }}/etc/slurm.conf
 [Service]
 Type=forking
 #EnvironmentFile=/etc/default/slurmctld
+ExecStartPre=/etc/systemd/system/glusterfsheltest.sh
 ExecStart={{ slurm_dir }}/sbin/slurmctld $SLURMCTLD_OPTIONS
 PIDFile={{ slurmpiddir }}/slurmctld.pid
 [Install]