playbook/massive_var/main.yml

----
-ldapServerHostIpLine: "130.220.209.234 m2-w.massive.org.au"
-ldapCaCertSrc: "/tmp/m1-w-ca.pem" 
-countryName: "AU"
-reginalName: "Victoria"
-cityName: "Melbourne"
-organizationName: "Monash University"
-emailAddress: "help@massive.org.au"
-organizationUnit: "MASSIVE"
-nfsServerIpAddress: m2-login3.massive.org.au
-
-x509_cert_file: "/etc/openvpn/certs/{{ x509_ca_server }}.crt"
-x509_key_file: "/etc/openvpn/private/{{ x509_ca_server }}.key"
-x509_cacert_file: "/etc/ssl/certs/ca_{{ x509_ca_server }}.crt"
-###x509_common_name: "{{ x509_ca_server }}CommonName"
-x509_common_name: "{{ inventory_hostname }}"
-x509_csr_args: "--server"
-x509_sign_args: "{{ x509_csr_args }}"
-dhparms_file: "/etc/openvpn/private/dh.pem"
-server_network: "10.8.0.0"
-server_netmask: "255.255.255.0"
-
-slurm_version: 14.11.2
-munge_version: 0.5.11
-userRelocationName: "ec2-user"
-userNewHome: "/local_home"
-#nfs_type: "nfs4"
-#nfs_options: "defaults"
-#nfs_server: "m2-login3.massive.org.au"
-ldapServerHost: "130.220.209.234 m2-w.massive.org.au"
-ldapDomain: "massive.org.au"
-ldapURI: "ldaps://m2-w.massive.org.au:1637/"
-ldapBindDN: "cn=ldapbind,cn=users,dc=massive,dc=org,dc=au" 
-ldapBase: "cn=users,dc=massive,dc=org,dc=au"
-ldapUserClass: "user"
-ldapUserHomeDirectory: "unixHomeDirectory"
-ldapUserPricipal: "userPrincipalName"
-ldapGroupBase: "ou=groups,dc=massive,dc=org,dc=au"
-tlsCaCertDirectory: "/etc/openldap/certs"
-ldapCaCertFile: "/etc/openldap/certs/m1-w-ca.pem"
-ldapCaCertFileSource: "/tmp/cvl2server/m1-w-ca.pem"
-cacertFile: "cacert.pem"
-#domain: "cvl.massive.org.au"
-domain: "massive.org.au"
-ldapRfc2307: |
-  ldap_schema = rfc2307
-  ldap_search_base = cn=users,dc=massive,dc=org,dc=au
-  ldap_user_search_base = cn=users,dc=massive,dc=org,dc=au
-  ldap_user_object_class = user
-  ldap_user_home_directory = unixHomeDirectory
-  ldap_user_principal = userPrincipalName
-  ldap_user_name = uid
-  ldap_group_search_base = ou=groups,dc=massive,dc=org,dc=au
-  ldap_group_object_class = group
-  ldap_access_order = expire
-  ldap_account_expire_policy = ad
-
-ldapRfc2307Pam: |
-  scope sub
-  nss_base_passwd cn=users,dc=massive,dc=org,dc=au?sub
-  nss_base_shadow cn=users,dc=massive,dc=org,dc=au?sub
-  nss_base_group cn=users,dc=massive,dc=org,dc=au?sub
-  nss_map_objectclass posixAccount user
-  nss_map_objectclass shadowAccount user
-  nss_map_objectclass posixGroup group
-  nss_map_attribute homeDirectory unixHomeDirectory
-  nss_map_attribute uniqueMember member
-  nss_map_attribute shadowLastChange pwdLastSet
-  pam_login_attribute sAMAccountName
-  pam_filter objectClass=User
-  pam_password ad

playbook/massive_var/package.yml

----
-importRepo: { command: "wget http://cvlrepo.massive.org.au/repo/cvl.repo -O", destination: "/etc/yum.repos.d/cvl.repo" }
-#yumGroupPackageList:
-#  - CVL Pre-installation
-#  - CVL Base Packages
-#  - CVL System
-#  - CVL System Extension
-#  - CVL General Imaging Tools

playbook/readme.txt

-Files in the playbook directory should be used as examples for the reference only.

provisionCluster.yaml

---- 
-description: " A simple template to boot a 3 node cluster"
-heat_template_version: 2013-05-23
-parameters:
- image_id:
-  type: string
-  label: Image ID
-  description: Image to be used for compute instance
-  default: a5e74703-f343-415a-aa23-bd0f0aacfc9e
- key_name:
-  type: string
-  label: Key Name
-  description: Name of key-pair to be used for compute instance
-  default: shahaan
- availability_z:
-  type: string
-  label: Availability Zone
-  description: Availability Zone to be used for launching compute instance
-  default: monash-01
-resources:
-  computeNodes:
-   type: "OS::Heat::ResourceGroup"
-   properties:
-    count: 2
-    resource_def:
-     type: "OS::Nova::Server"
-     properties:
-      availability_zone: { get_param: availability_z }
-      flavor: m1.small
-      image: { get_param: image_id }
-      key_name: { get_param: key_name }
-      metadata:
-       ansible_host_group: computeNodes
-       ansible_ssh_user: ec2-user
-       ansible_ssh_private_key_file: /home/sgeadmin/.ssh/shahaan.pem
-  headNodes:
-    type: "OS::Heat::ResourceGroup"
-    properties:
-      count: 1
-      resource_def: 
-       type: headNode.yaml

qa.yml

+#- hosts: 'all'
+  #gather_facts: false # not sure if false is clever here
+  #tasks:
+  #- include_vars: vars/ldapConfig.yml
+  #- include_vars: vars/filesystems.yml
+  #- include_vars: vars/slurm.yml
+  #- include_vars: vars/vars.yml
+  #- { name: set use shared state, set_fact: usesharedstatedir=False }
+  #tags: [ always ]
+
+# this playbook is roughly sorted by
+# - hostgroupstopics like ComputeNodes or ComputeNodes,LoginNodes, last VisNodes
+# - "tag_groups" each starting after a #comment see #misc or misc tag
+- hosts: 'ComputeNodes'
+  gather_facts: false
+  tasks:
+  # these are just templates.
+  #Note the tag never! Everything with never is only executed if called explicitly aka ansible-playbook --tags=foo,bar OR -tags=tag_group
+  - { name: template_shell, shell: ls, tags: [never,tag_group,uniquetag_foo] }
+  - { name: template_command, command: uname chdir=/bin, tags: [never,tag_group,uniquetag_bar] }
+  - { name: template_scipt, script: ./scripts/qa/test.sh, tags: [never,tag_group,uniquetag_script] }
+#mpi stuff
+  - { name: run mpi on one computenode, command: ls, args: {chdir: "/tmp"} , failed_when: "TODO is TRUE", tags: [never,mpi,mpi_local,TODO] }
+  - { name: run mpi on two computenode, command: ls, args: {chdir: "/tmp"} , failed_when: "TODO is TRUE", tags: [never,mpi,mpi_local_two,TODO] }
+  #- { name: run mpi via sbatch, command: cmd=ls chdir="/tmp" , failed_when: "TODO is TRUE", tags: [never,mpi,slurm_mpi,TODO] }
+  #- { name: mpi_pinging,        command: cmd=ls chdir="/tmp" , failed_when: "TODO is TRUE", tags: [never,mpi,mpi_ping,TODO] }
+   #module load openmpi/3.1.6-ucx;mpirun --mca btl self --mca pml ucx -x UCX_TLS=mm  -n 24 /projects/pMOSP/mpi/parallel_mandelbrot/parallel/mandelbrot
+   #module load openmpi/3.1.6-ucx;srun mpirun --mca btl self --mca pml ucx -x UCX_TLS=mm  -n 24 /projects/pMOSP/mpi/parallel_mandelbrot/parallel/mandelbrot
+
+#slurm
+  - { name: slurmd should be running, service: name=slurmd state=started, tags: [never,slurm,slurmd] }
+  - { name: munged should be running, service: name=munged state=started, tags: [never,slurm,munged] }
+  - { name: ensure connectivity to the controller, shell: scontrol ping, tags: [never,slurm,scontrol_ping] }
+  - { name: the most simple srun test, shell: srun --reservation=AWX hostname, tags: [never,slurm,srun_hostname] }
+#nhc, manually run nhc because it contains many tests
+  - { name: run nhc explicitly, command: /opt/nhc-1.4.2/sbin/nhc -c /opt/nhc-1.4.2/etc/nhc/nhc.conf, become: true , tags: [never,slurm,nhc] }
+# networking
+  - { name: ping license server, shell: ls, tags: [never,network,ping_license] }
+  - { name: ping something outside monash, command: ping -c 1 8.8.8.8, tags: [never,network,ping_external] }
+#mounts
+- hosts: 'ComputeNodes,LoginNodes'
+  gather_facts: false
+  tasks:
+  - { name: check mount for usr_local, shell: "mount | grep -q local", tags: [never,mountpoints,mountpoints_local] }
+  - { name: check mount for projects, shell: "lfs df -h", tags: [never,mountpoints_projects] }
+  - { name: check mount for home, shell: "mount | grep -q home", tags: [never,mountpoints,mountpoints_home] }
+  - { name: check mount for scratch, shell: "mount | grep -q scratch" , tags: [never,mountpoints_scratch] }
+#misc
+  - { name: check singularity, shell: module load octave && octave --version, tags: [never,misc,singularity3] }
+  - { name: module test, shell: cmd="module load gcc" executable="/bin/bash", tags: [never,misc,modulecmd] }
+  - { name: contact ldap, shell: maybe test ldapsearch, failed_when: "TODO is TRUE", tags: [never,misc,ldap,TODO] }
+#gpu
+- hosts: 'VisNodes'
+  gather_facts: false
+  tasks:
+  - { name: run nvida-smi to see if a gpu driver is present, command: "/bin/nvidia-smi", tags: [never,gpu,smi] }
+  - { name: run gpu burn defaults to 30 seconds, command: "/usr/local/gpu_burn/1.0/run_silent.sh", tags: [never,gpu,long,gpuburn] }
+
+
+# extended time-consuming tests
+# relion see https://docs.massive.org.au/communities/cryo-em/tuning/tuning.html
+# linpack
+
+#module load openmpi/1.10.7-mlx;ldd /usr/local/openmpi/1.10.7-mlx/bin/* | grep -ic found

roles/MonashBioinformaticsPlatform_node_allocation/tasks/main.yml

 ---
 - name: make sure /usr/local/bin exists
   file: path=/usr/local/bin state=directory mode=755 owner=root
-  sudo: true
+  become: true
 
 - name: install get_node.py
   copy: src=get_node.py dest=/usr/local/bin/get_node.py mode=755 owner=root
-  sudo: true
+  become: true
 
 - name: install mbp_node
   copy: src=mbp_node dest=/usr/local/bin/mbp_node mode=755 owner=root
-  sudo: true
+  become: true

roles/OpenVPN-Client/handlers/main.yml

 ---
   - name: restart openvpn
     service: name=openvpn state=restarted
-    sudo: true
+    become: true

roles/OpenVPN-Client/tasks/main.yml

@@ -3,6 +3,6 @@
  include: installOpenVPN.yml
 
 - name: "Start OpenVPN"
-  service: name=openvpn state=started
-  sudo: true
+  service: name=openvpn state=started enabled=yes
+  become: true
 

roles/additional_paths/templates/additional_paths.sh.j2

+export PATH=$PATH:{{ additional_paths|join(":") }}