Merge pull request #16 from l1ll1/build_ldap_and_karaage

Build ldap and karaage

roles/ldapserver/templates/accounts_ldif.j2

+dn: ou=Accounts,{{ ldapDomain }}
+objectClass: organizationalUnit

roles/ldapserver/templates/acls_ldif.j2

+dn: olcDatabase={2}bdb,cn=config
+changetype: modify
+add: olcAccess
+olcAccess: {0}to attrs=userPassword by dn="cn=Manager,{{ ldapDomain }}" write by self write by * auth
+olcAccess: {1}to attrs=shadowLastChange by dn="cn=Manager,{{ ldapDomain }}" write by self write by * read
+olcAccess: {2}to * by users read by anonymous auth

roles/ldapserver/templates/binddn_ldif.j2

+dn: cn=binddn,ou=Accounts,{{ ldapDomain }}
+objectClass: inetOrgPerson
+cn: binddn
+sn: binddn
+userPassword: {{ ldapBindDNHash.stdout }}

roles/ldapserver/templates/default_ppolicy_ldif.j2

+dn: cn=default,ou=pwpolicies,{{ ldapDomain }}
+objectClass: top
+objectClass: device
+objectClass: pwdPolicy
+pwdAttribute: 2.5.4.35
+cn: default

roles/ldapserver/templates/groups_ldif.j2

+dn: ou=Groups,{{ ldapDomain }}
+objectClass: organizationalUnit

roles/ldapserver/templates/manager_ldif.j2

+dn: olcDatabase={2}bdb,cn=config
+changetype: modify
+replace: olcSuffix
+olcSuffix: {{ ldapDomain }}
+-
+replace: olcRootDN
+olcRootDN: cn=Manager,{{ ldapDomain }}
+-
+add: olcRootPW
+olcRootPW: {{ ldapManagerHash.stdout }}

roles/ldapserver/templates/ppolicy_moduleload_ldif.j2

+dn: cn=module,cn=config
+objectClass: olcModuleList
+cn: module
+olcModulePath: /usr/lib64/openldap/
+olcModuleLoad: ppolicy.la

roles/ldapserver/templates/ppolicy_overlay_ldif.j2

+dn: olcOverlay=ppolicy,olcDatabase={2}bdb,cn=config
+olcOverlay: ppolicy
+objectClass: olcOverlayConfig
+objectClass: olcPPolicyConfig
+olcPPolicyHashCleartext: TRUE
+olcPPolicyUseLockout: FALSE
+olcPPolicyDefault: cn=default,ou=pwpolicies,{{ ldapDomain }}

roles/ldapserver/templates/pwpolicies_ldif.j2

+dn: ou=pwpolicies,{{ ldapDomain }}
+objectClass: organizationalUnit
+objectClass: top
+ou: pwpolicies

roles/ldapserver/templates/root_ldif.j2

+dn: {{ ldapDomain }}
+objectClass: dcObject
+objectClass: organization
+o: {{ ansible_domain }}
+description: root

roles/ldapserver/templates/ssl_ldif.j2

+dn: cn=config
+replace: olcTLSCACertificateFile
+olcTLSCACertificateFile: /etc/openldap/certs/cacert.pem
+-
+replace: olcTLSCertificateFile
+olcTLSCertificateFile: /etc/openldap/certs/ldapcert.pem
+-
+replace: olcTLSCertificateKeyFile
+olcTLSCertificateKeyFile: /etc/openldap/certs/ldapkey.pem

roles/ldapserver/vars/CentOS_6.5_x86_64.yml

+---
+ system_packages:
+  - openldap-servers
+  - openldap-clients
+  - openssl

roles/ldapserver/vars/main.yml

+---
+ ldapuser:  ldap
+ ldapgroup: ldap

roles/slurm/tasks/main.yml

@@ -43,6 +43,13 @@
   template: src=slurm.conf.j2 dest=/etc/slurm/slurm.conf
   sudo: true
   notify: restart slurm
+  when: slurm_use_vpn==False
+
+- name: install slurm.conf
+  template: src=slurm-vpn.conf.j2 dest=/etc/slurm/slurm.conf
+  sudo: true
+  notify: restart slurm
+  when: slurm_use_vpn==True
 
 - name: start slurm
   service: name=slurm state=started

roles/slurm/templates/slurm-vpn.conf.j2

+#
+# Example slurm.conf file. Please run configurator.html
+# (in doc/html) to build a configuration file customized
+# for your environment.
+#
+#
+# slurm.conf file generated by configurator.html.
+#
+# See the slurm.conf man page for more information.
+#
+ClusterName=CIAB
+ControlMachine={{ slurmctrl }}
+ControlAddr={{ slurmctrl }}-vpn
+#BackupController=
+#BackupAddr=
+#
+SlurmUser=slurm
+#SlurmdUser=root
+SlurmctldPort=6817
+SlurmdPort=6818
+AuthType=auth/munge
+#JobCredentialPrivateKey=
+#JobCredentialPublicCertificate=
+StateSaveLocation=/tmp
+SlurmdSpoolDir=/tmp/slurmd
+SwitchType=switch/none
+MpiDefault=none
+SlurmctldPidFile=/var/run/slurmctld.pid
+SlurmdPidFile=/var/run/slurmd.pid
+ProctrackType=proctrack/pgid
+#PluginDir=
+CacheGroups=0
+#FirstJobId=
+ReturnToService=0
+#MaxJobCount=
+#PlugStackConfig=
+#PropagatePrioProcess=
+#PropagateResourceLimits=
+#PropagateResourceLimitsExcept=
+#Prolog=
+#Epilog=
+#SrunProlog=
+#SrunEpilog=
+#TaskProlog=
+#TaskEpilog=
+#TaskPlugin=
+#TrackWCKey=no
+#TreeWidth=50
+#TmpFS=
+#UsePAM=
+#
+# TIMERS
+SlurmctldTimeout=300
+SlurmdTimeout=300
+InactiveLimit=0
+MinJobAge=300
+KillWait=30
+Waittime=0
+#
+# SCHEDULING
+SchedulerType=sched/backfill
+#SchedulerAuth=
+#SchedulerPort=
+#SchedulerRootFilter=
+SelectType=select/linear
+FastSchedule=1
+#PriorityType=priority/multifactor
+#PriorityDecayHalfLife=14-0
+#PriorityUsageResetPeriod=14-0
+#PriorityWeightFairshare=100000
+#PriorityWeightAge=1000
+#PriorityWeightPartition=10000
+#PriorityWeightJobSize=1000
+#PriorityMaxAge=1-0
+#
+# LOGGING
+SlurmctldDebug=3
+#SlurmctldLogFile=
+SlurmdDebug=3
+#SlurmdLogFile=
+JobCompType=jobcomp/none
+#JobCompLoc=
+#
+# ACCOUNTING
+#JobAcctGatherType=jobacct_gather/linux
+#JobAcctGatherFrequency=30
+#
+#AccountingStorageType=accounting_storage/slurmdbd
+#AccountingStorageHost=
+#AccountingStorageLoc=
+#AccountingStoragePass=
+#AccountingStorageUser=
+#
+MpiParams=ports=12000-12999
+# COMPUTE NODES
+{% set nodelist = [] %}
+{% for queue in slurmqueues %}
+{% for node in groups[queue.group] %}
+{% if nodelist.append(node) %}
+{% endif %}
+{% endfor %}
+{% endfor %}
+{% for node in nodelist|unique %}
+NodeName={{ node }} NodeAddr={{ node }}-vpn Procs={{ hostvars[node]['ansible_processor_cores'] }} State=UNKNOWN
+{% endfor %}
+
+{% for queue in slurmqueues %}
+PartitionName={{ queue.name }} Nodes={{ groups[queue.group]|join(',') }}
+{% endfor %}

roles/slurm/templates/slurm.conf.j2

@@ -103,6 +103,7 @@ MpiParams=ports=12000-12999
 {% for node in nodelist|unique %}
 NodeName={{ node }} Procs={{ hostvars[node]['ansible_processor_cores'] }} State=UNKNOWN
 {% endfor %}
+
 {% for queue in slurmqueues %}
 PartitionName={{ queue.name }} Nodes={{ groups[queue.group]|join(',') }}
 {% endfor %}

scripts/make_passwords.py

+# This program writes a yaml varaible file where each varible is suitable as a password
+# If a variable is not defined it will pick a new random varaible for you
+# If a variable is already defined it will not change
+import random
+import sys
+import string
+import yaml
+
+def new_pass(length):
+    return ''.join(random.choice(string.ascii_uppercase + string.digits+string.ascii_lowercase) for _ in range(length))
+
+# required_passwords is a dictionay consisting of variable names and the length of random password you would like to associate with that variable
+required_passwords={}
+# Passwords for munge and slurm
+required_passwords['mungekey']=32
+# Passwords for karaage and ldap
+required_passwords['ldapManagerPassword']=8
+required_passwords['ldapBindDNPassword']=8
+required_passwords['karaageSqlPassword']=8
+required_passwords['sqlrootPasswd']=8
+
+changed=False
+pwpath='./passwords.yml'
+try:
+    f=open(pwpath,'r')
+    data=yaml.load(f.read())
+    f.close()
+except Exception as e:
+    pass
+if data==None:
+    data={}
+
+print data
+
+for pw in required_passwords.keys():
+    if data.has_key(pw):
+        pass
+    else:
+        data[pw]=new_pass(required_passwords[pw])
+        changed=True
+if changed:
+    f=open(pwpath,'w+')
+    f.write(yaml.dump(data,default_flow_style=False,explicit_start=True))
+    f.close()
+

scripts/makehosts.py

@@ -20,3 +20,10 @@ for h in hosts.keys():
         for name in hosts[h]:
             string=string+" %s"%(name)
         print string
+
+for h in hosts.keys():
+    if d['hostvars'].has_key(h):
+        if d['hostvars'][h].has_key('ansible_tun0'):
+            string="%s"%(d['hostvars'][h]['ansible_tun0']['ipv4']['address'])
+            string=string+" %s-vpn"%h
+            print string

topplay.yml

 ---
 
+# First we need hostname -f to return the correct domain
+
 - hosts: '*'
   roles:
-  - etcHosts
-
-<<<<<<< HEAD
-#- hosts: 'x509_ca'
-#  vars:
-#  roles:
-#  - { role: easy-rsa-CA }
-#
-#- hosts: 'OpenVPN-Server'
-#  vars:
-#    x509_ca_server: "{{ groups['x509_ca'][0] }}"
-#  roles:
-#  - { role: OpenVPN-Server }
-#
-#- hosts: 'OpenVPN-Client'
-#  vars:
-#    x509_ca_server: "{{ groups['x509_ca'][0] }}"
-#    openvpn_servers: "{{ groups['OpenVPN-Server'] }}"
-#  roles:
-#  - { role: OpenVPN-Client }
-#
-#
-#- hosts: 'SubmitHost'
-#  roles:
-#      - { role: slurm-build, slurm_version: 14.11.0, munge_version: 0.5.11 }
-
-- hosts: ComputeNode
-  vars:
-    openmpi_version: 1.8.3
-    build_node: aciab-comp-0
-  roles:
+  - {role: etcHosts, domain: 'test.massive.org.au' }
 
-- hosts: 'x509_ca'
+# Create an x509 CA. Certificates are used for the VPN, the ldap server and the web server (karaage)
+
+- hosts: ManagmentNodes[0]
   vars:
   roles:
   - { role: easy-rsa-CA }
 
-- hosts: 'OpenVPN-Server'
+- hosts: 'ManagmentNodes'
   vars:
-    x509_ca_server: "{{ groups['x509_ca'][0] }}"
+    x509_ca_server: "{{ groups['ManagmentNodes'][0] }}"
+    ldapDomain: "dc=test,dc=massive,dc=org,dc=au"
   roles:
   - { role: OpenVPN-Server }
+  - { role: ldapserver }
 
-- hosts: 'OpenVPN-Client'
+- hosts: 'karaage'
   vars:
-    x509_ca_server: "{{ groups['x509_ca'][0] }}"
-    openvpn_servers: "{{ groups['OpenVPN-Server'] }}"
+    x509_ca_server: "{{ groups['ManagmentNodes'][0] }}"
+    smtp_smarthost: "does_not_exist.massive.org.au"
+    ldapDomain: "dc=test,dc=massive,dc=org,dc=au"
+    ldapURL: "ldaps://{{ hostvars[groups['ManagmentNodes'][0]]['ansible_fqdn'] }}"
   roles:
-  - { role: OpenVPN-Client }
+  - { role: karaage2.7 }
+
+# The next two tasks, apply the VPN and recalcuate /etc/hosts. Comment them if you are on a secure network
 
-- hosts: 'SubmitHost'
+- hosts: ComputeNodes
+  vars:
+    x509_ca_server: "{{ groups['ManagmentNodes'][0] }}"
+    openvpn_servers: "{{ groups['ManagmentNodes'] }}"
   roles:
-      - { role: slurm-build, slurm_version: 14.11.0, munge_version: 0.5.11 }
+    - { role: OpenVPN-Client }
 
 - hosts: '*'
+  roles:
+  - {role: etcHosts, domain: 'test.massive.org.au' }
+
+# These tasks build some software (slurm, munge and openmpi) and copy them back to localhost. Skip if you have some other way of getting 
+# copyies of these software
+
+- hosts: ComputeNodes[0]
+  vars:
+    slurm_version: 14.11.0
+    munge_version: 0.5.11
+  roles:
+    - { role: slurm-build }
+
+# Slurm and openmpi tasks assume copies of the software in /tmp under a very specific directory strucutre (created by the build tasks above) Modifiy 
+# the roles if you have a different way of installing the software
+
+- hosts: ComputeNodes:ManagmentNodes:LoginNodes
   vars:
     slurm_version: 14.11.0
     munge_version: 0.5.11
     slurmqueues: 
-      - {name: DEFAULT, group: ComputeNode}
-      - {name: batch, group: ComputeNode}
-      - {name: gpu, group: ComputeNode}
-    slurmctrl: "{{ groups['SubmitHost'][0] }}"
+      - {name: batch, group: ComputeNodes}
+    slurmctrl: "{{ groups['ManagmentNodes'][0] }}"
   roles:
-      - { role: slurm }
+      - { role: slurm, slurm_use_vpn: true} # change to false if you are on a secure network
       - { role: test_user }
-      - { role: openmpi-build }
+
+# These tasks build some software (slurm, munge and openmpi) and copy them back to localhost. Skip if you have some other way of getting 
+# copyies of these software
+
+- hosts: ComputeNodes[0]
+  vars:
+    openmpi_version: 1.8.3
+  roles:
+    - { role: openmpi-build }
+
+# Slurm and openmpi tasks assume copies of the software in /tmp under a very specific directory strucutre (created by the build tasks above) Modifiy 
+# the roles if you have a different way of installing the software
+
+- hosts: ComputeNodes:ManagmentNodes:LoginNodes
+  vars:
+    openmpi_version: 1.8.3
+  roles:
+      - { role: openmpi }