Merge pull request #67 from CVL-GitHub/m2cvlbranch

M2cvlbranch

roles/cvlExtraFiles/tasks/main.yml

+---
+- name: extra file symbolic links 
+  file: src={{ item.src }} path={{ item.dest }} state={{ item.type }} force=yes
+  with_items: 
+    extraFiles
+  sudo: true
+  when: extraFiles is defined

roles/cvlExtraFiles/vars/readme.txt

+---
+extraFiles:
+  - {src: '/usr/local/etc/sshfs_default_sites.cfg', dest: '/etc/sshfs_default_sites.cfg', type: 'link'}
+  - {src: '/usr/local/etc/vnc/turbovncserver-auth.conf', dest: '/etc/turbovncserver-auth.conf', type: 'link'}
+  - {src: '/usr/local/etc/vnc/turbovncserver.conf', dest: '/etc/turbovncserver.conf', type: 'link'}
+  - {src: '/usr/local/etc/profile.d/modules.sh', dest: '/etc/profile.d/modules.sh', type: 'link'}
+  - {src: '/usr/local/etc/profile.d/modules.csh', dest: '/etc/profile.d/modules.csh', type: 'link'}
+  - {src: '/usr/local/etc/profile.d/cvl.sh', dest: '/etc/profile.d/cvl.sh', type: 'link'}
+

roles/cvlFijiCronJob/tasks/main.yml

+---
+- name: Check CVL cron job
+  shell: ls /cvl/local/bin/fiji_daily_update.sh
+  ignore_errors: true
+  register: fijiupdatescript
+
+- name: CVL fiji cron job 
+  cron: name=fijiupdate job=/cvl/local/bin/fiji_daily_update.sh user=root hour=5 state=present
+  sudo: true
+  when: fijiupdatescript | success

roles/extra_rpms/vars/main.yml

@@ -145,6 +145,8 @@ pkgs:
  - perl-devel
  - perl-ExtUtils-MakeMaker
  - perl-ExtUtils-ParseXS
+ - perl-HTML-Parser
+ - perl-HTML-Tagset 
  - perl-Test-Harness
  - perl-Time-HiRes
  - pexpect
@@ -159,6 +161,8 @@ pkgs:
  - plymouth-system-theme
  - polkit-gnome
  - postgresql
+ - postgresql-contrib
+ - postgresql-devel
  - postgresql-libs
  - postgresql-server
  - PyGreSQL
@@ -179,10 +183,12 @@ pkgs:
  - qt-sqlite
  - qt-x11
  - rhino
+ - rsync 
  - scipy
  - spice-vdagent
  - suitesparse
  - system-gnome-theme
+ - tcl
  - tcsh
  - Terminal
  - texlive-texmf-errata-fonts
@@ -235,8 +241,3 @@ pkgs:
  - xvattr
  - yum-utils
  - zip
- - perl-HTML-Parser
- - perl-HTML-Tagset 
- - postgresql-contrib
- - postgresql-devel
- - environment-modules

roles/gpu/tasks/main.yml

+---
+- name: install deps 
+  yum: name={{ item }} state=installed
+  sudo: true
+  with_items:
+    - gcc
+    - perl
+    - wget
+    - pciutils
+    - kernel-headers
+    - kernel-devel 
+    - xterm
+    - libX11-common
+    - libX11-devel
+    - libX11
+    - xorg-x11-server-common
+    - xorg-x11-util-macros
+    - xorg-x11-server-utils
+    - xorg-x11-font-utils
+    - xorg-x11-server-Xorg
+    - xorg-x11-glamor
+    - xorg-x11-xinit
+    - xorg-x11-utils
+    - xorg-x11-xauth
+    - xorg-x11-proto-devel
+    - xorg-x11-xkb-utils
+
+- name: Add nouveau from blacklist
+  lineinfile: 
+  args:
+    dest: /etc/modprobe.d/blacklist.conf 
+    line: "blacklist nouveau"  
+    state: present
+  sudo: true
+
+- name: Template disable-nouvear.conf 
+  template: dest=/etc/modprobe.d/disable-nouveau.conf src=disable-nouveau.conf.j2
+  sudo: true
+
+- name: Template nvidia.conf 
+  template: dest=/etc/modprobe.d/nvidia.conf src=nvidia.conf.j2
+  sudo: true
+
+- name: check nvidia driver
+  shell: ls /usr/lib64/libnvidia-opencl.so.{{ nvidia_version }} 
+  register: has_been_compiled
+  ignore_errors: true
+
+- name: Copy boot file
+  template: src=grub.conf.j2 dest=/boot/grub/grub.conf 
+  sudo: true
+
+- name: Copy X config file
+  template: src=xorg.conf.j2 dest=/etc/X11/xorg.conf 
+  sudo: true
+
+- name: Copy xserver file
+  template: src=xserver.j2 dest=/etc/pam.d/xserver
+  sudo: true
+  
+- name: restart_host
+  command: shutdown -r now "Reboot triggered by Ansible"
+  async: 0
+  poll: 0
+  sudo: true
+  ignore_errors: true
+  when: has_been_compiled | failed
+
+- name: wait_restart
+  local_action: wait_for host="{{ inventory_hostname }}" port=22 delay=5 timeout=600
+  sudo: true 
+  when: has_been_compiled | failed
+ 
+- name: get nvidia driver 
+  shell: wget http://us.download.nvidia.com/XFree86/Linux-x86_64/{{ nvidia_version }}/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run
+  args:
+    chdir: /tmp
+    creates: /tmp/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run
+  sudo: true 
+  when: has_been_compiled | failed
+
+- name: build nvidia driver 
+  shell: chmod 755 /tmp/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run; /tmp/NVIDIA-Linux-x86_64-{{ nvidia_version }}.run --silent --kernel-source-path /usr/src/kernels/{{ kernel_version }}.el6.x86_64
+  sudo: true
+  when: has_been_compiled | failed
+
+- name: set persistence mode 
+  lineinfile: 
+  args:
+    dest: /etc/rc.d/rc.local
+    line: "nvidia-smi --persistence-mode=1"  
+    state: present
+  sudo: true
+
+- name: Load module
+  shell: modprobe nvidia
+  sudo: true
+

roles/gpu/templates/disable-nouveau.conf.j2

+blacklist nouveau
+options nouveau modeset=0
+

roles/gpu/templates/grub.conf.j2

+# grub.conf generated by anaconda
+#
+# Note that you do not have to rerun grub after making changes to this file
+# NOTICE:  You do not have a /boot partition.  This means that
+#          all kernel and initrd paths are relative to /, eg.
+#          root (hd0,0)
+#          kernel /boot/vmlinuz-version ro root=/dev/vda1
+#          initrd /boot/initrd-[generic-]version.img
+#boot=/dev/vda
+default=0
+timeout=5
+splashimage=(hd0,0)/boot/grub/splash.xpm.gz
+hiddenmenu
+title CentOS (2.6.32-504.el6.x86_64)
+	root (hd0,0)
+	kernel /boot/vmlinuz-2.6.32-504.el6.x86_64 ro root=/dev/vda1 rd_NO_LUKS  KEYBOARDTYPE=pc KEYTABLE=us LANG=en_US.UTF-8 rd_NO_MD SYSFONT=latarcyrheb-sun16 crashkernel=auto elevator=noop biosdevname=0 console=ttyS0 rdblacklist=nouveau nouveau.modeset=0 rd_NO_LVM rd_NO_DM rhgb quiet 
+	initrd /boot/initramfs-2.6.32-504.el6.x86_64.img

roles/gpu/templates/nvidia.conf.j2

+options nvidia NVreg_DeviceFileMode=0666
+

roles/gpu/templates/xorg.conf.j2

+# nvidia-xconfig: X configuration file generated by nvidia-xconfig
+# nvidia-xconfig:  version 340.58  (buildmeister@swio-display-x86-rhel47-09)  Fri Oct 31 17:40:05 PDT 2014
+Section "DRI"
+	Mode 0660
+	Group "vglusers"
+EndSection
+
+Section "ServerLayout"
+    Identifier     "Layout0"
+    Screen      0  "Screen0"
+    InputDevice    "Keyboard0" "CoreKeyboard"
+    InputDevice    "Mouse0" "CorePointer"
+EndSection
+
+Section "Files"
+    FontPath        "/usr/share/fonts/default/Type1"
+EndSection
+
+Section "InputDevice"
+    # generated from default
+    Identifier     "Mouse0"
+    Driver         "mouse"
+    Option         "Protocol" "auto"
+    Option         "Device" "/dev/input/mice"
+    Option         "Emulate3Buttons" "no"
+    Option         "ZAxisMapping" "4 5"
+EndSection
+
+Section "InputDevice"
+    # generated from data in "/etc/sysconfig/keyboard"
+    Identifier     "Keyboard0"
+    Driver         "kbd"
+    Option         "XkbLayout" "us"
+    Option         "XkbModel" "pc105"
+EndSection
+
+Section "Monitor"
+    Identifier     "Monitor0"
+    VendorName     "Unknown"
+    ModelName      "Unknown"
+    HorizSync       28.0 - 33.0
+    VertRefresh     43.0 - 72.0
+    Option         "DPMS"
+EndSection
+
+Section "Device"
+    Identifier     "Device0"
+    Driver         "nvidia"
+    VendorName     "NVIDIA Corporation"
+    BusID          "PCI:00:06:0"
+EndSection
+
+Section "Screen"
+    Identifier     "Screen0"
+    Device         "Device0"
+    Monitor        "Monitor0"
+    DefaultDepth    24
+    SubSection     "Display"
+        Depth       24
+    EndSubSection
+EndSection
+

roles/gpu/templates/xserver.j2

+#%PAM-1.0
+auth       sufficient	pam_rootok.so
+auth sufficient pam_permit.so
+account    required	pam_permit.so
+session    optional	pam_keyinit.so force revoke

roles/openLdapClient/tasks/configLdapClient.yml

@@ -27,15 +27,15 @@
   sudo: true
   when: ldapCaCertContents is defined
 
-- name: "Copy pam config to ldap client"
-  template: src=system-auth-ac.j2 dest=/etc/pam.d/system-auth
+- name: "Copy system auth"
+  template: src=system-auth.j2 dest=/etc/pam.d/system-auth
   sudo: true
 
-- name: "Copy pam config to ldap client"
+- name: "Copy password auth"
   template: src=password-auth.j2 dest=/etc/pam.d/password-auth
   sudo: true
 
-- name: "Copy system auth to ldap client"
+- name: "Copy authconfig"
   template: src=authconfig.j2 dest=/etc/sysconfig/authconfig
   sudo: true
 
@@ -43,6 +43,10 @@
   template: src=ldap.conf.j2 dest=/etc/openldap/ldap.conf
   sudo: true
 
+- name: "Copy pam xserver file "
+  template: src=xserver.j2 dest=/etc/pam.d/xserver
+  sudo: true
+
 - name: "Add LDAP server IP address to /etc/hosts"
   lineinfile: dest=/etc/hosts line="{{ ldapServerHostIpLine }}" state=present insertafter=EOF
   sudo: true

roles/setupKnownHosts/tasks/main.yml

+- name: "Templating /etc/ssh/known_hosts"
+  template: src=known_hosts.j2 dest=/etc/ssh/known_hosts owner=root group=root mode=600
+  delegate_to: "{{ slurmlogin }}"
+  run_once: true
+  sudo: true
+  register: sshknowhost 
+
+- name: ecrypt the hosts file
+  shell: ssh-keygen -H -f /etc/ssh/known_hosts
+  sudo: true
+  run_once: true
+  when: sshknowhost.changed
+

roles/setupKnownHosts/templates/known_hosts.j2

+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% set host = {'name': node, 'ip': hostvars[node]['ansible_'+desktop.interface]['ipv4']['address'], 'rsa': hostvars[node]['ansible_ssh_host_key_rsa_public']} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+{{ host.name }} {{ host.rsa }}
+{{ host.ip }} {{ host.rsa }}
+{% endfor %}

roles/setupKnownHosts/templates/known_hosts.j2.v1

+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% if not ssh-keygen -F hostvars[node] -f /etc/ssh/known_hosts and not ssh-keygen -F hostvars[node]['ansible_'+desktop.interface] -f /etc/ssh/known_hosts %}
+{% set host = {name: node, ip: hostvars[node]['ansible_'+desktop.interface]} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+ssh-keyscan -t rsa {{ host.name }} >> /etc/ssh/known_hosts
+ssh-keyscan -t rsa {{ host.ip }} >> /etc/ssh/known_hosts
+{% endfor %}

roles/setupKnownHosts/templates/known_hosts.j2.v2

+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% if not ssh-keygen -F node -f /etc/ssh/known_hosts and not ssh-keygen -F hostvars[node]['ansible_'+desktop.interface]['ipv4']['address'] -f /etc/ssh/known_hosts %}
+{% set host = {name: node, ip: hostvars[node]['ansible_'+desktop.interface]['ipv4']['address']} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+echo "{{ host.name }} {{ host.ip }}"
+#ssh-keyscan -t rsa {{ host.name }} >> /etc/ssh/known_hosts
+#ssh-keyscan -t rsa {{ host.ip }} >> /etc/ssh/known_hosts
+{% endfor %}

roles/setupKnownHosts/templates/known_hosts.j2.v3

+{% set nodelist = [] %}
+{% for desktop in desktopNodeList %}
+{% for node in groups[desktop.name] %}
+{% if hostvars[node]['ansible_'+desktop.interface] is defined %}
+{% set host = {'name': node, 'ip': hostvars[node]['ansible_'+desktop.interface]['ipv4']['address']} %}
+{% if nodelist.append(host) %}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+{% for host in nodelist|unique %}
+echo "{{ host.name }} {{ host.ip }}"
+#ssh-keyscan -t rsa {{ host.name }} >> /etc/ssh/known_hosts
+#ssh-keyscan -t rsa {{ host.ip }} >> /etc/ssh/known_hosts
+{% endfor %}

roles/slurm/defaults/main.yml

 ---
-clustername: "CIAB" 
-slurmctlddebug: {} 
-slurmddebug: {} 
-slurmschedlog: {} 
-slurmdbdlog: {}
+slurm_use_vpn: False 
+slurmctlddebug: {level: 9, log: '/var/log/slurm/slurmctld.log'}
+slurmddebug: {level: 9, log: '/var/log/slurm/slurmd.log'}
+slurmschedlog: {level: 9, log: '/var/log/slurm/slurmsched.log'}
+slurmdbdlog: {level: 9, log: '/var/log/slurm/slurmdbd.log'}
 slurmfairshare: {def: false, val: 10000}
 slurmdatadir: "/var/spool/slurm"
+slurmselecttype: "select/linear"
+slurmfastschedule: "1"
+slurmschedulertype: "sched/backfill"
+

roles/slurm/templates/slurm.conf.j2

@@ -58,12 +58,12 @@ KillWait=30
 Waittime=0
 #
 # SCHEDULING
-SchedulerType=sched/backfill
+SchedulerType={{ slurmschedulertype }}
 #SchedulerAuth=
 #SchedulerPort=
 #SchedulerRootFilter=
-SelectType=select/linear
-FastSchedule=1
+SelectType={{ slurmselecttype }}
+FastSchedule={{ slurmfastschedule }}
 #PriorityType=priority/multifactor
 #PriorityDecayHalfLife=14-0
 #PriorityUsageResetPeriod=14-0
@@ -109,6 +109,8 @@ AccountingStorageHost={{ slurmctrl }}
 #AccountingStoragePass=
 #AccountingStorageUser=
 #
+#GRES
+GresTypes=gpu
 
 # Fair share
 {% if slurmfairshare.def %}

roles/slurm/vars/readme.txt

----
-slurm_use_vpn: True
-clustername: "CIAB"