Merge pull request #48 from l1ll1/master

dynamic inventories and fixes for other roles

dynamicInventory

@@ -3,8 +3,13 @@ import sys, os, string, subprocess, socket, re
 import copy, shlex,uuid, random, multiprocessing, time, shutil, json
 import novaclient.v1_1.client as nvclient
 import novaclient.exceptions as nvexceptions
+from keystoneclient.auth.identity import v2 as v2_auth
+from heatclient import client as heat_client
 
-class Authenticate:
+from keystoneclient import session as kssession
+
+
+class OpenStackConnection:
 	
 	def __init__(self, username, passwd):
 		self.username=username
@@ -12,39 +17,188 @@ class Authenticate:
 		self.tenantName= os.environ['OS_TENANT_NAME']
 		self.tenantID= os.environ['OS_TENANT_ID']
 		self.authUrl="https://keystone.rc.nectar.org.au:5000/v2.0"
-	
-	def gatherInfo(self):
 
-		## Fetch the Nova Object
+        def _get_keystone_v2_auth(self, v2_auth_url, **kwargs):
+            auth_token = kwargs.pop('auth_token', None)
+            tenant_id = kwargs.pop('project_id', None)
+            tenant_name = kwargs.pop('project_name', None)
+            if auth_token:
+                return v2_auth.Token(v2_auth_url, auth_token,
+                                     tenant_id=tenant_id,
+                                     tenant_name=tenant_name)
+            else:
+                return v2_auth.Password(v2_auth_url,
+                                        username=kwargs.pop('username', None),
+                                        password=kwargs.pop('password', None),
+                                        tenant_id=tenant_id,
+                                        tenant_name=tenant_name)
+
+
+        def _get_keystone_session(self, **kwargs):
+            # first create a Keystone session
+            cacert = kwargs.pop('cacert', None)
+            cert = kwargs.pop('cert', None)
+            key = kwargs.pop('key', None)
+            insecure = kwargs.pop('insecure', False)
+            timeout = kwargs.pop('timeout', None)
+            verify = kwargs.pop('verify', None)
+
+            # FIXME(gyee): this code should come from keystoneclient
+            if verify is None:
+                if insecure:
+                    verify = False
+                else:
+                    # TODO(gyee): should we do
+                    # heatclient.common.http.get_system_ca_fle()?
+                    verify = cacert or True
+            if cert and key:
+                # passing cert and key together is deprecated in favour of the
+                # requests lib form of having the cert and key as a tuple
+                cert = (cert, key)
+            return kssession.Session(verify=verify, cert=cert, timeout=timeout)
+
+        def _get_keystone_auth(self, session, auth_url, **kwargs):
+            # FIXME(dhu): this code should come from keystoneclient
+
+            # discover the supported keystone versions using the given url
+            v2_auth_url=auth_url
+            v3_auth_url=None
+
+            # Determine which authentication plugin to use. First inspect the
+            # auth_url to see the supported version. If both v3 and v2 are
+            # supported, then use the highest version if possible.
+            auth = None
+            if v3_auth_url and v2_auth_url:
+                user_domain_name = kwargs.get('user_domain_name', None)
+                user_domain_id = kwargs.get('user_domain_id', None)
+                project_domain_name = kwargs.get('project_domain_name', None)
+                project_domain_id = kwargs.get('project_domain_id', None)
+
+                # support both v2 and v3 auth. Use v3 if domain information is
+                # provided.
+                if (user_domain_name or user_domain_id or project_domain_name or
+                        project_domain_id):
+                    auth = self._get_keystone_v3_auth(v3_auth_url, **kwargs)
+                else:
+                    auth = self._get_keystone_v2_auth(v2_auth_url, **kwargs)
+            elif v3_auth_url:
+                # support only v3
+                auth = self._get_keystone_v3_auth(v3_auth_url, **kwargs)
+            elif v2_auth_url:
+                # support only v2
+                auth = self._get_keystone_v2_auth(v2_auth_url, **kwargs)
+            else:
+                raise exc.CommandError(_('Unable to determine the Keystone '
+                                         'version to authenticate with using the '
+                                         'given auth_url.'))
+
+            return auth
+
+        def get_stack_name(self,stack):
+            stacks=[]
+            for s in self.hc.stacks.list():
+                stacks.append(s.stack_name)
+            if stack in stacks:
+                return stack
+            elif len(stacks)==1:
+                return stacks[0]
+            elif len(stacks)==0:
+                raise Exception("You do not have any heat stacks in your OpenStack Project")
+            else:
+                raise Exception("You have multiple heat stacks in your OpenStack Project and I'm not sure which one to use.\n You can select a stack by symlinking to a stack, for example if you have a stack called mycluster do ln -s %s mycluster\n"%stack)
 
-		nc = nvclient.Client(	auth_url=self.authUrl,
+        def auth(self):
+		self.nc = nvclient.Client(	auth_url=self.authUrl,
 			username=self.username,
 			api_key=self.passwd,
 			project_id=self.tenantName,
 			tenant_id=self.tenantID,
 			service_type="compute"
 			)
+                kwargs = {
+                    'insecure': False,
+                }
+                keystone_session = self._get_keystone_session(**kwargs)
+
+                kwargs = {
+                    'username': self.username,
+                    'password': self.passwd,
+                    'project_id': self.tenantID,
+                    'project_name': self.tenantName 
+                }
+
+                keystone_auth = self._get_keystone_auth(keystone_session,
+                                                    self.authUrl,
+                                                    **kwargs)
+
+                endpoint = keystone_auth.get_endpoint(keystone_session,service_type='orchestration', region_name=None)
+
+
+                kwargs = {
+                    'username': self.username,
+                    'include_pass': False,
+                    'session': keystone_session,
+                    'auth_url': self.authUrl,
+                    'region_name': '',
+                    'endpoint_type': 'publicURL',
+                    'service_type': 'orchestration',
+                    'password': self.passwd,
+                    'auth': keystone_auth,
+                }
+                api_version=1
+
+                self.hc = heat_client.Client(api_version, endpoint, **kwargs)
+
+	
+        def recurse_resources(self,stack,resource):
+            result=[]
+            if 'OS::Nova::Server' in resource.resource_type:
+                result.append(resource.physical_resource_id)
+            if 'OS::Heat::ResourceGroup' in resource.resource_type:
+                for r in self.hc.resources.list(resource.physical_resource_id):
+                    result.extend(self.recurse_resources(stack,r))
+                    
+            return result
+
+	def gatherInfo(self,stack_name):
+
+		## Fetch the Nova Object
+                instance_ids=[]
+                for i in self.hc.stacks.list():
+                    if i.stack_name == stack_name:
+                        for r in self.hc.resources.list(i.stack_name):
+                            instance_ids.extend(self.recurse_resources(stack=i,resource=r))
+
+                nc=self.nc
 		inventory = {}
 		inventory['_meta'] = { 'hostvars': {} }
 		for server in nc.servers.list():
-			if server.metadata:
-				hostname = socket.gethostbyaddr(server.networks.values()[0][0])[0]
-				# Set Ansible Host Group
-				if server.metadata['ansible_host_group'] in inventory:
-					inventory[server.metadata['ansible_host_group']].append(hostname)
-				else:
-					inventory[server.metadata['ansible_host_group']] = [hostname]
-				# Set the other host variables
-				inventory['_meta']['hostvars'][hostname] = {}
-				inventory['_meta']['hostvars'][hostname]['ansible_ssh_user'] =  server.metadata['ansible_ssh_user']
-				inventory['_meta']['hostvars'][hostname]['ansible_ssh_private_key_file'] = server.metadata['ansible_ssh_private_key_file']	
-				
-			else:
-				continue
+                        if server.id in instance_ids:
+                            if server.metadata and 'ansible_host_group' in server.metadata:
+                                    #hostname = socket.gethostbyaddr(server.networks.values()[0][0])[0]
+                                    hostname = server.name
+                                    # Set Ansible Host Group
+                                    if server.metadata['ansible_host_group'] in inventory:
+                                            inventory[server.metadata['ansible_host_group']].append(hostname)
+                                    else:
+                                            inventory[server.metadata['ansible_host_group']] = [hostname]
+                                    # Set the other host variables
+                                    inventory['_meta']['hostvars'][hostname] = {}
+                                    inventory['_meta']['hostvars'][hostname]['ansible_ssh_host'] = server.networks.values()[0][0]
+                                    inventory['_meta']['hostvars'][hostname]['ansible_remote_tmp'] = '/tmp/ansible'
+                                    for key in server.metadata.keys():
+                                        if 'ansible_ssh' in key:
+                                            inventory['_meta']['hostvars'][hostname][key] = server.metadata[key]
+                                    
+                            else:
+                                    continue
 		print json.dumps(inventory)
 
 if __name__ == "__main__":
+        stack_name=os.path.basename(sys.argv[0])
 	username = os.environ['OS_USERNAME']
 	passwd = os.environ['OS_PASSWORD']
-	auth = Authenticate(username, passwd)
-	auth.gatherInfo()
+	openstack = OpenStackConnection(username, passwd)
+        openstack.auth()
+        stack_name=openstack.get_stack_name(stack_name)
+	openstack.gatherInfo(stack_name)

roles/dump_ldap_config/tasks/main.yml

+---
+- name: grab cacert
+  shell: cat /etc/openldap/certs/cacert.pem
+  register: ldapCaCertContents
+
+- name: dump vars
+  template: src=ldapConfig.j2 dest=/tmp/ldapConfig.out
+
+- name: fetch vars
+  fetch: src=/tmp/ldapConfig.out dest=/tmp/ldapConfig.out flat=yes
+

roles/dump_ldap_config/templates/ldapConfig.j2

+---
+ldapServerHostIpLine: "{{ ansible_eth0.ipv4.address }} {{ ansible_fqdn }}"
+ldapCaCertContents: |
+{% for l in  ldapCaCertContents.stdout_lines %}
+  {{ l }}
+{% endfor %}
+ldapCaCertFile: /etc/ssl/certs/cacert.crt
+ldapDomain: "{{ ldapDomain }}"
+ldapURI: "ldaps://{{ ansible_fqdn }}:636"
+ldapBindDN: "{{ ldapBindDN }}"
+ldapBindDNPassword: "{{ ldapBindDNPassword }}"
+ldapBase: "{{ ldapBase }}"
+ldapGroupBase: "{{ ldapGroupBase }}"
+ldapRfc2307Pam: ""
+ldap_access_filter: "(objectClass=posixAccount)"

roles/easy-rsa-certificate/tasks/buildCert.yml

@@ -2,6 +2,7 @@
 - name: "Check client ca certificate"
   register: ca_cert
   stat: "path={{ x509_cacert_file }}"
+  sudo: true
 
 - name: "Check certificate and key"
   shell: (openssl x509 -noout -modulus -in {{ x509_cert_file }}  | openssl md5 ; openssl rsa -noout -modulus -in {{ x509_key_file }} | openssl md5) | uniq | wc -l
@@ -46,7 +47,7 @@
   when: needcert
 
 - name: "Creating CSR"
-  shell: " cd /etc/easy-rsa/2.0; . ./vars; export EASY_RSA=\"${EASY_RSA:-.}\"; \"$EASY_RSA\"/pkitool --csr {{ x509_csr_args }} {{ x509_common_name }}"
+  shell: "cd /etc/easy-rsa/2.0; . ./vars; export EASY_RSA=\"${EASY_RSA:-.}\"; \"$EASY_RSA\"/pkitool --csr {{ x509_csr_args }} {{ x509_common_name }}"
   when: needcert
   sudo: true
 

roles/easy-rsa-common/tasks/installEasyRsa.yml

@@ -6,7 +6,7 @@
   when: ansible_os_family == 'RedHat'
 - 
   name: "Installing easy-rsa"
-  apt: "name=openvpn state=present"
+  apt: "name=openvpn state=present update_cache=yes"
   sudo: True
   when: ansible_os_family == 'Debian'
 - 

roles/easy-rsa-common/tasks/main.yml

@@ -3,6 +3,3 @@
   include: installEasyRsa.yml
 -
   include: copyConfigurationFile.yml
--
-  include: yumList.yml
-

roles/karaage2.7/meta/main.yml

 ---
 dependencies:
-    - { role: easy-rsa-certificate, x509_csr_args: "--server" }
+    - { role: easy-rsa-certificate, x509_csr_args: "", x509_sign_args: "--server", x509_cacert_file: "/etc/ssl/certs/ca.crt", x509_key_file: "/etc/ssl/private/server.key", x509_cert_file: "/etc/ssl/certs/server.crt", x509_common_name: "{{ ansible_fqdn }}" }

roles/karaage2.7/tasks/main.yml

 ---
-- include_vars: "{{ hostvars[ansible_hostname]['ansible_distribution'] }}_{{ hostvars[ansible_hostname]['ansible_distribution_version'] }}_{{ ansible_architecture }}.yml"
+- include_vars: "{{ ansible_distribution }}_{{ ansible_distribution_version }}_{{ ansible_architecture }}.yml"
 
 - name: install system packages apt
   apt: name={{ item }} state=installed update_cache=true

roles/ldapserver/meta/main.yml

 ---
 dependencies:
-  - { role: easy-rsa-certificate, x509_csr_args: "--server" }
+    - { role: easy-rsa-certificate, x509_csr_args: "", x509_sign_args: "--server", x509_cacert_file: "/etc/ssl/certs/ca.crt", x509_key_file: "/etc/ssl/private/server.key", x509_cert_file: "/etc/ssl/certs/server.crt", x509_common_name: "{{ ansible_fqdn }}" }

roles/ldapserver/tasks/main.yml

 ---
 
-- include_vars: "{{ hostvars[ansible_hostname]['ansible_distribution'] }}_{{ hostvars[ansible_hostname]['ansible_distribution_version'] }}_{{ ansible_architecture }}.yml"
+- include_vars: "{{ ansible_distribution }}_{{ ansible_distribution_version }}_{{ ansible_architecture }}.yml"
 
 - name: install system packages apt
   apt: name={{ item }} state=installed update_cache=true

roles/nfs-client/tasks/mountFileSystem.yml

 --- 
+- name: "stop fail2ban"
+  service: name=fail2ban state=stopped
+  sudo: true
+
+- name: restart idmap 
+  service: name={{ item }} state=restarted
+  with_items:
+    - rpcbind
+    - rpcidmapd
+  sudo: true
+
 - name: "Mounting NFS mounts"
-  mount: name={{ item.name }} src={{ hostvars[nfs_server]['ansible_'+item.interface]['ipv4']['address'] }}:{{ item.src }} fstype={{ item.fstype }} opts={{ item.opts }} state=mounted
+  mount: name={{ item.src }} src={{ item.ipv4 }}:{{ item.name }} fstype={{ item.fstype }} opts={{ item.opts }} state=mounted
   with_items: exportList 
   notify: "restart authentication"
   notify: "restart idmap"
   sudo: true 
+  ignore_errors: true
+  register: firstMount
   when: exportList is defined 
+
+- name: "Wait for nfs to stabailse"
+  command: sleep 60
+  delegate_to: 127.0.0.1
+  when: firstMount | failed
+
+- name: "Mounting NFS mounts"
+  mount: name={{ item.src }} src={{ item.ipv4 }}:{{ item.name }} fstype={{ item.fstype }} opts={{ item.opts }} state=mounted
+  with_items: exportList 
+  notify: "restart authentication"
+  notify: "restart idmap"
+  sudo: true 
+  when: exportList is defined and firstMount | failed
+
+- name: "restart fail2ban"
+  service: name=fail2ban state=started
+  sudo: true
+

roles/openLdapClient/tasks/configLdapClient.yml

@@ -6,22 +6,41 @@
     - nsswitch.conf
   sudo: true
 
+- name: "get cert dir"
+  shell: "dirname {{ ldapCaCertFile }}"
+  delegate_to: localhost
+  run_once: true
+  register: ldapCaCertDir
+
+- name: "make basedir"
+  file: path={{ ldapCaCertDir.stdout }} state=directory owner=root
+  sudo: true
+
 - name: "Copy the CA cert"
   copy: src={{ ldapCaCertSrc }} dest={{ ldapCaCertFile }} owner=root mode=644
   sudo: true
   when: ldapCaCertSrc is defined
 
+- name: "Template CA cert"
+  template: src=ldapCaCert.j2 dest={{ ldapCaCertFile }} owner=root mode=644
+  sudo: true
+  when: ldapCaCertContents is defined
+
+- name: "Copy pam config to ldap client"
+  template: src=system-auth-ac.j2 dest=/etc/pam.d/system-auth
+  sudo: true
+
 - name: "Copy pam config to ldap client"
-  template: src=system-auth-ac.j2 dest=/etc/pam.d/system-auth-ac
+  template: src=password-auth.j2 dest=/etc/pam.d/password-auth
   sudo: true
 
 - name: "Copy system auth to ldap client"
   template: src=authconfig.j2 dest=/etc/sysconfig/authconfig
   sudo: true
 
-- name: "Copy ldap.conf file "
-  template: src=ldap.conf.j2 dest=/etc/openldap/ldap.conf
-  sudo: true
+#- name: "Copy ldap.conf file "
+#  template: src=ldap.conf.j2 dest=/etc/openldap/ldap.conf
+#  sudo: true
 
 - name: "Add LDAP server IP address to /etc/hosts"
   lineinfile: dest=/etc/hosts line="{{ ldapServerHostIpLine }}" state=present insertafter=EOF
@@ -33,4 +52,8 @@
   sudo: true
   notify: restart sssd
 
+- name: "start sssd"
+  service: name=sssd state=started
+  sudo: true
+
 

roles/openLdapClient/tasks/installOpenLdap.yml

@@ -2,15 +2,15 @@
 - name: "Install open ldap package yum"
   action: yum pkg={{ item }} state=installed 
   with_items:
-    - openldap
-    - openldap-clients
+      #    - openldap
+      #    - openldap-clients
     - sssd
     - sssd-common
     - sssd-client
     - nss
     - nss-tools
-    - nss-pam-ldapd
-    - pam_ldap
+      #    - nss-pam-ldapd
+      #    - pam_ldap
   sudo: true
   when: ansible_os_family == 'RedHat'
 

roles/openLdapClient/templates/authconfig.j2

@@ -2,7 +2,7 @@ IPADOMAINJOINED=no
 USEMKHOMEDIR=no
 USEPAMACCESS=no
 CACHECREDENTIALS=yes
-USESSSDAUTH=no
+USESSSDAUTH=yes
 USESHADOW=yes
 USEWINBIND=no
 USEDB=no
@@ -10,7 +10,7 @@ FORCELEGACY=no
 USEFPRINTD=yes
 FORCESMARTCARD=no
 PASSWDALGORITHM=sha512
-USELDAPAUTH=yes
+USELDAPAUTH=no
 USEPASSWDQC=no
 IPAV2NONTP=no
 USELOCAUTHORIZE=yes
@@ -18,9 +18,9 @@ USECRACKLIB=yes
 USEIPAV2=no
 USEWINBINDAUTH=no
 USESMARTCARD=no
-USELDAP=yes
+USELDAP=no
 USENIS=no
 USEKERBEROS=no
 USESYSNETAUTH=no
-USESSSD=no
+USESSSD=yes
 USEHESIOD=no

roles/openLdapClient/templates/ldapCaCert.j2

+{{ ldapCaCertContents }}

roles/openLdapClient/templates/password-auth.j2

+# This file is auto-generated.
+# User changes will be destroyed the next time authconfig is run.
+auth        required      pam_env.so
+auth        sufficient    pam_unix.so nullok try_first_pass
+auth        requisite     pam_succeed_if.so uid >= 500 quiet
+auth        sufficient    pam_sss.so use_first_pass
+auth        required      pam_deny.so
+
+account     required      pam_unix.so
+account     sufficient    pam_localuser.so
+account     sufficient    pam_succeed_if.so uid < 500 quiet
+account     [default=bad success=ok user_unknown=ignore] pam_sss.so
+account     required      pam_permit.so
+
+password    requisite     pam_cracklib.so try_first_pass retry=3
+password    sufficient    pam_unix.so md5 shadow nullok try_first_pass use_authtok
+password    sufficient    pam_sss.so use_authtok
+password    required      pam_deny.so
+
+session     optional      pam_keyinit.so revoke
+session     required      pam_limits.so
+session     [success=1 default=ignore] pam_succeed_if.so service in crond quiet use_uid
+session     required      pam_unix.so
+session     optional      pam_sss.so
+

roles/openLdapClient/templates/sssd.j2

@@ -27,6 +27,7 @@ ldap_tls_cacert = {{ ldapCaCertFile }}
 ldap_default_bind_dn = {{ ldapBindDN }} 
 ldap_default_authtok_type = password
 ldap_default_authtok = {{ ldapBindDNPassword }} 
+ldap_access_filter = {{ ldap_access_filter }}
 
 {{ ldapRfc2307 }}
 

roles/openLdapClient/templates/system-auth-ac.j2

@@ -4,21 +4,21 @@
 auth        required      pam_env.so
 auth        sufficient    pam_unix.so nullok try_first_pass
 auth        requisite     pam_succeed_if.so uid >= 500 quiet
-auth        sufficient    pam_ldap.so use_first_pass
+auth        sufficient    pam_sss.so use_first_pass
 auth        required      pam_deny.so
 
 account     required      pam_unix.so broken_shadow
 account     sufficient    pam_succeed_if.so uid < 500 quiet
-account     [default=bad success=ok user_unknown=ignore] pam_ldap.so
+account     [default=bad success=ok user_unknown=ignore] pam_sss.so
 account     required      pam_permit.so
 
 password    requisite     pam_cracklib.so try_first_pass retry=3
 password    sufficient    pam_unix.so md5 shadow nullok try_first_pass use_authtok
-password    sufficient    pam_ldap.so use_authtok
+password    sufficient    pam_sss.so use_authtok
 password    required      pam_deny.so
 
 session     optional      pam_keyinit.so revoke
 session     required      pam_limits.so
 session     [success=1 default=ignore] pam_succeed_if.so service in crond quiet use_uid
 session     required      pam_unix.so
-session     optional      pam_ldap.so
+session     optional      pam_sss.so

roles/slurm/tasks/main.yml

@@ -16,11 +16,11 @@
   sudo: true
 
 - name: create slurm group
-  group: name=slurm
+  group: name=slurm system=yes
   sudo: true
 
 - name: create slurm user
-  user: name=slurm group=slurm createhome=no 
+  user: name=slurm group=slurm system=yes createhome=no
   sudo: true
 
 - name: install slurm rpms

roles/ssh-password-login/handlers/main.yml

+- name: "restart sshd"
+  service: name=sshd state=restarted
+  sudo: true