diff --git a/roles/cvlExtraFiles/tasks/main.yml b/roles/cvlExtraFiles/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4dff080397f5cdf7d9897f912b77cc2e1da90832
--- /dev/null
+++ b/roles/cvlExtraFiles/tasks/main.yml
@@ -0,0 +1,7 @@
+---
+- name: extra file symbolic links
+ file: src={{ item.src }} path={{ item.dest }} state={{ item.type }} force=yes
+ with_items:
+ extraFiles
+ sudo: true
+ when: extraFiles is defined
diff --git a/roles/cvlExtraFiles/vars/readme.txt b/roles/cvlExtraFiles/vars/readme.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6aae5abbb4a3f6cdd16a8082efc38473f74fc6ea
--- /dev/null
+++ b/roles/cvlExtraFiles/vars/readme.txt
@@ -0,0 +1,9 @@
+---
+extraFiles:
+ - {src: '/usr/local/etc/sshfs_default_sites.cfg', dest: '/etc/sshfs_default_sites.cfg', type: 'link'}
+ - {src: '/usr/local/etc/vnc/turbovncserver-auth.conf', dest: '/etc/turbovncserver-auth.conf', type: 'link'}
+ - {src: '/usr/local/etc/vnc/turbovncserver.conf', dest: '/etc/turbovncserver.conf', type: 'link'}
+ - {src: '/usr/local/etc/profile.d/modules.sh', dest: '/etc/profile.d/modules.sh', type: 'link'}
+ - {src: '/usr/local/etc/profile.d/modules.csh', dest: '/etc/profile.d/modules.csh', type: 'link'}
+ - {src: '/usr/local/etc/profile.d/cvl.sh', dest: '/etc/profile.d/cvl.sh', type: 'link'}
+
diff --git a/roles/cvlFijiCronJob/tasks/main.yml b/roles/cvlFijiCronJob/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..f3f586d0dd7ee60957f4c292e9b0b6d86a7a94b7
--- /dev/null
+++ b/roles/cvlFijiCronJob/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+- name: Check CVL cron job
+ shell: ls /cvl/local/bin/fiji_daily_update.sh
+ ignore_errors: true
+ register: fijiupdatescript
+
+- name: CVL fiji cron job
+ cron: name=fijiupdate job=/cvl/local/bin/fiji_daily_update.sh user=root hour=5 state=present
+ sudo: true
+ when: fijiupdatescript | success
diff --git a/roles/extra_rpms/vars/main.yml b/roles/extra_rpms/vars/main.yml
index b02a910e9059ec0650b2bb85cb717c77912c2968..a8a78add72c210a0c7d2140a7ce5eef32577b980 100644
--- a/roles/extra_rpms/vars/main.yml
+++ b/roles/extra_rpms/vars/main.yml
@@ -145,6 +145,8 @@ pkgs:
- perl-devel
- perl-ExtUtils-MakeMaker
- perl-ExtUtils-ParseXS
+ - perl-HTML-Parser
+ - perl-HTML-Tagset
- perl-Test-Harness
- perl-Time-HiRes
- pexpect
@@ -159,6 +161,8 @@ pkgs:
- plymouth-system-theme
- polkit-gnome
- postgresql
+ - postgresql-contrib
+ - postgresql-devel
- postgresql-libs
- postgresql-server
- PyGreSQL
@@ -179,10 +183,12 @@ pkgs:
- qt-sqlite
- qt-x11
- rhino
+ - rsync
- scipy
- spice-vdagent
- suitesparse
- system-gnome-theme
+ - tcl
- tcsh
- Terminal
- texlive-texmf-errata-fonts
@@ -235,8 +241,3 @@ pkgs:
- xvattr
- yum-utils
- zip
- - perl-HTML-Parser
- - perl-HTML-Tagset
- - postgresql-contrib
- - postgresql-devel
- - environment-modules
diff --git a/roles/gpu/tasks/main.yml b/roles/gpu/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a08e9fb7fd3c74c0b63d023e4efc6a1ab220f10a
--- /dev/null
+++ b/roles/gpu/tasks/main.yml
@@ -0,0 +1,98 @@
+---
+- name: install deps
+ yum: name={{ item }} state=installed
+ sudo: true
+ with_items:
+ - gcc
+ - perl
+ - wget
+ - pciutils
+ - kernel-headers
+ - kernel-devel
+ - xterm
+ - libX11-common
+ - libX11-devel
+ - libX11
+ - xorg-x11-server-common
+ - xorg-x11-util-macros
+ - xorg-x11-server-utils
+ - xorg-x11-font-utils
+ - xorg-x11-server-Xorg
+ - xorg-x11-glamor
+ - xorg-x11-xinit
+ - xorg-x11-utils
+ - xorg-x11-xauth
+ - xorg-x11-proto-devel
+ - xorg-x11-xkb-utils
+
+- name: Add nouveau from blacklist
+ lineinfile:
+ args:
+ dest: /etc/modprobe.d/blacklist.conf
+ line: "blacklist nouveau"
+ state: present
+ sudo: true
+
+- name: Template disable-nouvear.conf
+ template: dest=/etc/modprobe.d/disable-nouveau.conf src=disable-nouveau.conf.j2
+ sudo: true
+
+- name: Template nvidia.conf
+ template: dest=/etc/modprobe.d/nvidia.conf src=nvidia.conf.j2
+ sudo: true
+
+- name: check nvidia driver
+ shell: ls /usr/lib64/libnvidia-opencl.so.{{ nvidia_version }}
+ register: has_been_compiled
+ ignore_errors: true
+
+- name: Copy boot file
+ template: src=grub.conf.j2 dest=/boot/grub/grub.conf
+ sudo: true
+
+- name: Copy X config file
+ template: src=xorg.conf.j2 dest=/etc/X11/xorg.conf
+ sudo: true
+
+- name: Copy xserver file
+ template: src=xserver.j2 dest=/etc/pam.d/xserver
+ sudo: true
+
+- name: restart_host
+ command: shutdown -r now "Reboot triggered by Ansible"
+ async: 0
+ poll: 0
+ sudo: true
+ ignore_errors: true
+ when: has_been_compiled | failed
+
+- name: wait_restart
+ local_action: wait_for host="{{ inventory_hostname }}" port=22 delay=5 timeout=600
+ sudo: true
+ when: has_been_compiled | failed
+
+- name: get nvidia driver
+ shell: wget http://us.download.nvidia.com/XFree86/Linux-x86_64/{{ nvidia_version }}/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run
+ args:
+ chdir: /tmp
+ creates: /tmp/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run
+ sudo: true
+ when: has_been_compiled | failed
+
+- name: build nvidia driver
+ shell: chmod 755 /tmp/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run; /tmp/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run --silent --kernel-source-path /usr/src/kernels/{{ kernel_version }}.el6.x86_64
+ sudo: true
+ when: has_been_compiled | failed
+
+- name: set persistence mode
+ lineinfile:
+ args:
+ dest: /etc/rc.d/rc.local
+ line: "nvidia-smi --persistence-mode=1"
+ state: present
+ sudo: true
+
+- name: Load module
+ shell: modprobe nvidia
+ sudo: true
+
diff --git a/roles/gpu/templates/disable-nouveau.conf.j2 b/roles/gpu/templates/disable-nouveau.conf.j2
new file mode 100644
index 0000000000000000000000000000000000000000..96993361f370408d6cb1e6fe227b69e63e0f4cab
--- /dev/null
+++ b/roles/gpu/templates/disable-nouveau.conf.j2
@@ -0,0 +1,3 @@
+blacklist nouveau
+options nouveau modeset=0
+
diff --git a/roles/gpu/templates/grub.conf.j2 b/roles/gpu/templates/grub.conf.j2
new file mode 100644
index 0000000000000000000000000000000000000000..26b822130ddcf46ec8b43965e41ea02ab3352d2d
--- /dev/null
+++ b/roles/gpu/templates/grub.conf.j2
@@ -0,0 +1,17 @@
+# grub.conf generated by anaconda
+#
+# Note that you do not have to rerun grub after making changes to this file
+# NOTICE: You do not have a /boot partition. This means that
+# all kernel and initrd paths are relative to /, eg.
+# root (hd0,0)
+# kernel /boot/vmlinuz-version ro root=/dev/vda1
+# initrd /boot/initrd-[generic-]version.img
+#boot=/dev/vda
+default=0
+timeout=5
+splashimage=(hd0,0)/boot/grub/splash.xpm.gz
+hiddenmenu
+title CentOS (2.6.32-504.el6.x86_64)
+ root (hd0,0)
+ kernel /boot/vmlinuz-2.6.32-504.el6.x86_64 ro root=/dev/vda1 rd_NO_LUKS KEYBOARDTYPE=pc KEYTABLE=us LANG=en_US.UTF-8 rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=auto elevator=noop biosdevname=0 console=ttyS0 rdblacklist=nouveau nouveau.modeset=0 rd_NO_LVM rd_NO_DM rhgb quiet
+ initrd /boot/initramfs-2.6.32-504.el6.x86_64.img
diff --git a/roles/gpu/templates/nvidia.conf.j2 b/roles/gpu/templates/nvidia.conf.j2
new file mode 100644
index 0000000000000000000000000000000000000000..d5c81c1f576c20be851dc2061c395f11238aad27
--- /dev/null
+++ b/roles/gpu/templates/nvidia.conf.j2
@@ -0,0 +1,2 @@
+options nvidia NVreg_DeviceFileMode=0666
+
diff --git a/roles/gpu/templates/xorg.conf.j2 b/roles/gpu/templates/xorg.conf.j2
new file mode 100644
index 0000000000000000000000000000000000000000..b6df768fe69c46466f2f8e51c52279746df7f067
--- /dev/null
+++ b/roles/gpu/templates/xorg.conf.j2
@@ -0,0 +1,62 @@
+# nvidia-xconfig: X configuration file generated by nvidia-xconfig
+# nvidia-xconfig: version 340.58 (buildmeister@swio-display-x86-rhel47-09) Fri Oct 31 17:40:05 PDT 2014
+Section "DRI"
+ Mode 0660
+ Group "vglusers"
+EndSection
+
+Section "ServerLayout"
+ Identifier "Layout0"
+ Screen 0 "Screen0"
+ InputDevice "Keyboard0" "CoreKeyboard"
+ InputDevice "Mouse0" "CorePointer"
+EndSection
+
+Section "Files"
+ FontPath "/usr/share/fonts/default/Type1"
+EndSection
+
+Section "InputDevice"
+ # generated from default
+ Identifier "Mouse0"
+ Driver "mouse"
+ Option "Protocol" "auto"
+ Option "Device" "/dev/input/mice"
+ Option "Emulate3Buttons" "no"
+ Option "ZAxisMapping" "4 5"
+EndSection
+
+Section "InputDevice"
+ # generated from data in "/etc/sysconfig/keyboard"
+ Identifier "Keyboard0"
+ Driver "kbd"
+ Option "XkbLayout" "us"
+ Option "XkbModel" "pc105"
+EndSection
+
+Section "Monitor"
+ Identifier "Monitor0"
+ VendorName "Unknown"
+ ModelName "Unknown"
+ HorizSync 28.0 - 33.0
+ VertRefresh 43.0 - 72.0
+ Option "DPMS"
+EndSection
+
+Section "Device"
+ Identifier "Device0"
+ Driver "nvidia"
+ VendorName "NVIDIA Corporation"
+ BusID "PCI:00:06:0"
+EndSection
+
+Section "Screen"
+ Identifier "Screen0"
+ Device "Device0"
+ Monitor "Monitor0"
+ DefaultDepth 24
+ SubSection "Display"
+ Depth 24
+ EndSubSection
+EndSection
+
diff --git a/roles/gpu/templates/xserver.j2 b/roles/gpu/templates/xserver.j2
new file mode 100644
index 0000000000000000000000000000000000000000..737d03ba940650104a96e56b861bdaf255c517f4
--- /dev/null
+++ b/roles/gpu/templates/xserver.j2
@@ -0,0 +1,5 @@
+#%PAM-1.0
+auth sufficient pam_rootok.so
+auth sufficient pam_permit.so
+account required pam_permit.so
+session optional pam_keyinit.so force revoke
diff --git a/roles/openLdapClient/tasks/configLdapClient.yml b/roles/openLdapClient/tasks/configLdapClient.yml
index c401d4b4d96dc784805d2887b283898f9301ad1c..e50aab3a95f95c14d5251a07617200f72b39fb3d 100644
--- a/roles/openLdapClient/tasks/configLdapClient.yml
+++ b/roles/openLdapClient/tasks/configLdapClient.yml
@@ -27,15 +27,15 @@
sudo: true
when: ldapCaCertContents is defined
-- name: "Copy pam config to ldap client"
- template: src=system-auth-ac.j2 dest=/etc/pam.d/system-auth
+- name: "Copy system auth"
+ template: src=system-auth.j2 dest=/etc/pam.d/system-auth
sudo: true
-- name: "Copy pam config to ldap client"
+- name: "Copy password auth"
template: src=password-auth.j2 dest=/etc/pam.d/password-auth
sudo: true
-- name: "Copy system auth to ldap client"
+- name: "Copy authconfig"
template: src=authconfig.j2 dest=/etc/sysconfig/authconfig
sudo: true
@@ -43,6 +43,10 @@
template: src=ldap.conf.j2 dest=/etc/openldap/ldap.conf
sudo: true
+- name: "Copy pam xserver file "
+ template: src=xserver.j2 dest=/etc/pam.d/xserver
+ sudo: true
+
- name: "Add LDAP server IP address to /etc/hosts"
lineinfile: dest=/etc/hosts line="{{ ldapServerHostIpLine }}" state=present insertafter=EOF
sudo: true
diff --git a/roles/openLdapClient/templates/system-auth-ac.j2 b/roles/openLdapClient/templates/system-auth.j2
similarity index 100%
rename from roles/openLdapClient/templates/system-auth-ac.j2
rename to roles/openLdapClient/templates/system-auth.j2
diff --git a/roles/setupKnownHosts/tasks/main.yml b/roles/setupKnownHosts/tasks/main.yml
new file mode 100644
index 0000000000000000000000000000000000000000..56872722f40b8aa2daa2ca14a50d1f716a5f87b7
--- /dev/null
+++ b/roles/setupKnownHosts/tasks/main.yml
@@ -0,0 +1,13 @@
+- name: "Templating /etc/ssh/known_hosts"
+ template: src=known_hosts.j2 dest=/etc/ssh/known_hosts owner=root group=root mode=600
+ delegate_to: "{{ slurmlogin }}"
+ run_once: true
+ sudo: true
+ register: sshknowhost
+
+- name: ecrypt the hosts file
+ shell: ssh-keygen -H -f /etc/ssh/known_hosts
+ sudo: true
+ run_once: true
+ when: sshknowhost.changed
+
diff --git a/roles/setupKnownHosts/templates/known_hosts.j2 b/roles/setupKnownHosts/templates/known_hosts.j2
new file mode 100644
index 0000000000000000000000000000000000000000..c5df8c27755da771d9fc950e19351c48f0f30a2d
--- /dev/null
+++ b/roles/setupKnownHosts/templates/known_hosts.j2
@@ -0,0 +1,15 @@
+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% set host = {'name': node, 'ip': hostvars[node]['ansible_'+desktop.interface]['ipv4']['address'], 'rsa': hostvars[node]['ansible_ssh_host_key_rsa_public']} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+{{ host.name }} {{ host.rsa }}
+{{ host.ip }} {{ host.rsa }}
+{% endfor %}
diff --git a/roles/setupKnownHosts/templates/known_hosts.j2.v1 b/roles/setupKnownHosts/templates/known_hosts.j2.v1
new file mode 100644
index 0000000000000000000000000000000000000000..ed9688799f614aa527e856640fab46fabde6fb2a
--- /dev/null
+++ b/roles/setupKnownHosts/templates/known_hosts.j2.v1
@@ -0,0 +1,17 @@
+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% if not ssh-keygen -F hostvars[node] -f /etc/ssh/known_hosts and not ssh-keygen -F hostvars[node]['ansible_'+desktop.interface] -f /etc/ssh/known_hosts %}
+{% set host = {name: node, ip: hostvars[node]['ansible_'+desktop.interface]} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+ssh-keyscan -t rsa {{ host.name }} >> /etc/ssh/known_hosts
+ssh-keyscan -t rsa {{ host.ip }} >> /etc/ssh/known_hosts
+{% endfor %}
diff --git a/roles/setupKnownHosts/templates/known_hosts.j2.v2 b/roles/setupKnownHosts/templates/known_hosts.j2.v2
new file mode 100644
index 0000000000000000000000000000000000000000..3fa6710a8da171e959e43dde50470e656421554a
--- /dev/null
+++ b/roles/setupKnownHosts/templates/known_hosts.j2.v2
@@ -0,0 +1,18 @@
+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% if not ssh-keygen -F node -f /etc/ssh/known_hosts and not ssh-keygen -F hostvars[node]['ansible_'+desktop.interface]['ipv4']['address'] -f /etc/ssh/known_hosts %}
+{% set host = {name: node, ip: hostvars[node]['ansible_'+desktop.interface]['ipv4']['address']} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+echo "{{ host.name }} {{ host.ip }}"
+#ssh-keyscan -t rsa {{ host.name }} >> /etc/ssh/known_hosts
+#ssh-keyscan -t rsa {{ host.ip }} >> /etc/ssh/known_hosts
+{% endfor %}
diff --git a/roles/setupKnownHosts/templates/known_hosts.j2.v3 b/roles/setupKnownHosts/templates/known_hosts.j2.v3
new file mode 100644
index 0000000000000000000000000000000000000000..b3898e0351824a69793583fb84578018cfeb9f8c
--- /dev/null
+++ b/roles/setupKnownHosts/templates/known_hosts.j2.v3
@@ -0,0 +1,16 @@
+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% set host = {'name': node, 'ip': hostvars[node]['ansible_'+desktop.interface]['ipv4']['address']} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+echo "{{ host.name }} {{ host.ip }}"
+#ssh-keyscan -t rsa {{ host.name }} >> /etc/ssh/known_hosts
+#ssh-keyscan -t rsa {{ host.ip }} >> /etc/ssh/known_hosts
+{% endfor %}
diff --git a/roles/slurm/defaults/main.yml b/roles/slurm/defaults/main.yml
index 2b482d25ec92db781dfa5ef8e1b56cc33d47940a..362d9cf4ae92e58c49950d6a09dcaaf9df92cba6 100644
--- a/roles/slurm/defaults/main.yml
+++ b/roles/slurm/defaults/main.yml
@@ -1,8 +1,12 @@
---
-clustername: "CIAB"
-slurmctlddebug: {}
-slurmddebug: {}
-slurmschedlog: {}
-slurmdbdlog: {}
+slurm_use_vpn: False
+slurmctlddebug: {level: 9, log: '/var/log/slurm/slurmctld.log'}
+slurmddebug: {level: 9, log: '/var/log/slurm/slurmd.log'}
+slurmschedlog: {level: 9, log: '/var/log/slurm/slurmsched.log'}
+slurmdbdlog: {level: 9, log: '/var/log/slurm/slurmdbd.log'}
slurmfairshare: {def: false, val: 10000}
slurmdatadir: "/var/spool/slurm"
+slurmselecttype: "select/linear"
+slurmfastschedule: "1"
+slurmschedulertype: "sched/backfill"
+
diff --git a/roles/slurm/templates/slurm.conf.j2 b/roles/slurm/templates/slurm.conf.j2
index d389c913844d7123a646cf1ec4f229cbabdf6114..a1535220bb43b6ee403e772872dd19ae38c7423c 100644
--- a/roles/slurm/templates/slurm.conf.j2
+++ b/roles/slurm/templates/slurm.conf.j2
@@ -58,12 +58,12 @@ KillWait=30
Waittime=0
#
# SCHEDULING
-SchedulerType=sched/backfill
+SchedulerType={{ slurmschedulertype }}
#SchedulerAuth=
#SchedulerPort=
#SchedulerRootFilter=
-SelectType=select/linear
-FastSchedule=1
+SelectType={{ slurmselecttype }}
+FastSchedule={{ slurmfastschedule }}
#PriorityType=priority/multifactor
#PriorityDecayHalfLife=14-0
#PriorityUsageResetPeriod=14-0
@@ -109,6 +109,8 @@ AccountingStorageHost={{ slurmctrl }}
#AccountingStoragePass=
#AccountingStorageUser=
#
+#GRES
+GresTypes=gpu
# Fair share
{% if slurmfairshare.def %}
diff --git a/roles/slurm/vars/readme.txt b/roles/slurm/vars/readme.txt
deleted file mode 100644
index c038c3c6ce63b81f4fd674cd0b76d9292471b5f6..0000000000000000000000000000000000000000
--- a/roles/slurm/vars/readme.txt
+++ /dev/null
@@ -1,3 +0,0 @@
----
-slurm_use_vpn: True
-clustername: "CIAB"