Merge branch 'master' of https://github.com/monash-merc/ansible_cluster_in_a_box

roles/slurm-from-source/tasks/installMungeFromSource.yml

+- name: clean up system installation
+  shell: rpm -e munge-libs slurm-munge munge slurm-plugins munge-devel slurm slurm-slurmdbd slurm-perlapi slurm-devel slurm-pam_slurm slurm-sjstat slurm-perlapi slurm-pam_slurm slurm-slurmdb-direct slurm-torque slurm-sjobexit
+  ignore_errors: true
+  sudo: true
+  when: ansible_os_family == 'RedHat'
+
 - name: get munge
   shell: wget https://munge.googlecode.com/files/munge-{{ munge_version }}.tar.bz2
   args:

roles/slurm-from-source/tasks/main.yml

 ---
+- name: clean up system installation
+  shell: rpm -e munge munge-libs munge-devel slurm slurm-sql
+  ignore_errors: true
+  sudo: true
+  when: ansible_os_family == 'RedHat' 
+
 - name: create munge group
   group: name=munge system=yes gid=498
   sudo: true
@@ -105,3 +111,9 @@
   template: src=slurm-vpn.conf.j2 dest={{ slurm_dir }}/etc/slurm.conf
   sudo: true
   when: slurm_use_vpn==True
+
+- name: setup envirnment variables 
+  template: src=slurm_setup.sh.j2 dest=/etc/profile.d/slurm_setup.sh
+  sudo: true
+  when: ansible_os_family == 'RedHat' 
+

roles/slurm-from-source/templates/munge.initd.j2

@@ -34,7 +34,7 @@ localstatedir="${prefix}/var"
 
 SERVICE_NAME="MUNGE"
 DAEMON_EXEC="$sbindir/munged"
-#DAEMON_ARGS=
+DAEMON_ARGS="-S ${localstatedir}/run/munge/munge.socket.2"
 #CONFIG=#_NOT_SUPPORTED_#
 PIDFILE="$localstatedir/run/munge/munged.pid"
 #NICE=

roles/slurm-from-source/templates/slurm.conf.j2

@@ -98,6 +98,11 @@ SlurmSchedLogFile={{ slurmschedlog.log }}
 JobCompType=jobcomp/none
 #JobCompLoc=
 #
+{% if slurmjob %}
+Prolog={{ slurmjob.prolog }}
+Epilog={{ slurmjob.epilog }}
+{% endif %}
+#
 # ACCOUNTING
 #JobAcctGatherType=jobacct_gather/linux
 #JobAcctGatherFrequency=30
@@ -110,7 +115,7 @@ AccountingStorageHost={{ slurmctrl }}
 #AccountingStorageUser=
 #
 #GRES
-GresTypes=gpu
+#GresTypes=gpu
 
 # Fair share
 {% if slurmfairshare.def %}
@@ -128,7 +133,7 @@ MpiParams=ports=12000-12999
 {% endfor %}
 {% endfor %}
 {% for node in nodelist|unique %}
-NodeName={{ node }} Procs={{ hostvars[node]['ansible_processor_vcpus'] }} State=UNKNOWN
+NodeName={{ node }} Procs={{ hostvars[node]['ansible_processor_vcpus'] }} 
 {% endfor %}
 
 {% for queue in slurmqueues %}

roles/slurm-from-source/templates/slurm.initd.j2

@@ -24,12 +24,12 @@
 # Short-Description: slurm daemon management
 # Description:       Start slurm to provide resource management
 ### END INIT INFO
-
+munge_lib="{{ munge_dir }}/lib"
 exec_prefix="{{ slurm_dir }}"
 prefix="{{ slurm_dir }}"
 BINDIR="${exec_prefix}/bin"
 CONFDIR="${prefix}/etc"
-LIBDIR="${exec_prefix}/lib"
+LIBDIR="${exec_prefix}/lib:${munge_lib}"
 SBINDIR="${exec_prefix}/sbin"
 
 # Source function library.

roles/slurm-from-source/templates/slurm_setup.sh.j2

+
+export PATH={{ munge_dir }}/bin:{{ slurm_dir }}/bin:{{ slurm_dir }}/sbin:$PATH
+
+export LD_LIBRARY_PATH={{ munge_dir }}/lib:{{ slurm_dir }}/lib:{{ slurm_dir }}/lib/slurm:$LD_LIBRARY_PATH
+

roles/slurm-provision/templates/slurm_provision.sh.j2

 #!/bin/sh
 
-CVL_HOME="/cvl/home"
+HOME_DIR="/cvl/home"
 user_list=($(getent passwd | cut -d ":" -f1))
 log_file="/root/slurm.log"
 
 for user in ${user_list[*]}; do
-    uid=$(id -u ${user})
-    gid=$(id -g ${user})
-    user_home=${CVL_HOME}/${user}
-    if [[ ! -d ${user_home} && ${uid} -gt 1000 ]]; then
-        mkdir -p ${user_home}
-        cp -r /etc/skel/* ${user_home}
-        chown -R ${uid}:${gid} ${user_home}
-        chmod 700 ${user_home}
+    {% if project_check is defined %}
+        run=$(id ${user} | grep {{ project_check }})
+    {% else %}
+        run="1"
+    {% endif %}
+    if [ ! -z "${run}" ]; then
+        uid=$(id -u ${user})
+        gid=$(id -g ${user})
+        user_home=${HOME_DIR}/${user}
+        if [[ ! -d ${user_home} && ${uid} -gt 1000 ]]; then
+            cp -r /etc/skel ${user_home}
+            chown -R ${uid}:${gid} ${user_home}
+            chmod 700 ${user_home}
 
-        account={{ projectname }}
-        cluster={{ clustername }}
+            account={{ projectname }}
+            cluster={{ clustername }}
 
-        find=$(sacctmgr list cluster ${cluster} | grep ${cluster})
-        if [ -z "${find}" ]; then
-            su slurm -c "sacctmgr -i add cluster ${cluster}" || { echo "error to create cluster ${cluster}" >> ${log_file} && exit 1; }
-        fi
-        find=$(sacctmgr list account ${account} | grep ${account})
-        if [ -z "${find}" ]; then
-            su slurm -c "sacctmgr -i add account ${account} Description=CVL Organization=monash cluster=${cluster}" || { echo "error to create account ${account}" >> ${log_file} && exit 1; }
-        fi
-        find=$(sacctmgr list user ${username} | grep ${username})
-        if [ -z "${find}" ]; then
-            su slurm -c "sacctmgr -i add user ${username} account=${account} cluster=${cluster}" || { echo "error to create user ${username}" >> ${log_file} && exit 1; }
+            find=$(sacctmgr list cluster ${cluster} | grep ${cluster})
+            if [ -z "${find}" ]; then
+                su slurm -c "sacctmgr -i add cluster ${cluster}" || { echo "error to create cluster ${cluster}" >> ${log_file} && exit 1; }
+            fi
+            find=$(sacctmgr list account ${account} | grep ${account})
+            if [ -z "${find}" ]; then
+                su slurm -c "sacctmgr -i add account ${account} Description=CVL Organization=monash cluster=${cluster}" || { echo "error to create account ${account}" >> ${log_file} && exit 1; }
+            fi
+            find=$(sacctmgr list user ${user} | grep ${user})
+            if [ -z "${find}" ]; then
+                su slurm -c "sacctmgr -i add user ${user} account=${account} cluster=${cluster}" || { echo "error to create user ${user}" >> ${log_file} && exit 1; }
+            fi
         fi
     fi
 done

roles/slurmdb/templates/slurmdbd.initd.j2

@@ -21,10 +21,11 @@
 # Description:       Start slurm to provide database server for SLURM
 ### END INIT INFO
 
+munge_lib="{{ munge_dir }}/lib"
 exec_prefix="{{ slurm_dir }}"
 prefix="{{ slurm_dir }}"
 CONFDIR="${prefix}/etc"
-LIBDIR="${exec_prefix}/lib"
+LIBDIR="${exec_prefix}/lib:${munge_lib}"
 SBINDIR="${exec_prefix}/sbin"
 
 #Source function library.