From 5d9bd257818ec6b3bda6f8c54d65502fcca611d4 Mon Sep 17 00:00:00 2001
From: Chris Hines <chris.hines@monash.edu>
Date: Wed, 13 Jul 2016 04:44:31 +0000
Subject: [PATCH] role to configure routes and mellanox device name. Add the
drivers to this role latter
---
roles/m3_networking/files/ibdev2netdev.sh | 222 ++++++++++++++++++
roles/m3_networking/tasks/main.yml | 164 +++++++++++++
roles/m3_networking/templates/90-mlx.rules.j2 | 7 +
.../templates/Ethernet connection 1 | 17 ++
.../templates/EthernetConnection | 15 ++
.../templates/MellanoxConnection | 18 ++
roles/m3_networking/templates/ifcfg-conf.j2 | 8 +
roles/m3_networking/vars/mellanoxVars.yml | 5 +
8 files changed, 456 insertions(+)
create mode 100755 roles/m3_networking/files/ibdev2netdev.sh
create mode 100644 roles/m3_networking/tasks/main.yml
create mode 100644 roles/m3_networking/templates/90-mlx.rules.j2
create mode 100644 roles/m3_networking/templates/Ethernet connection 1
create mode 100644 roles/m3_networking/templates/EthernetConnection
create mode 100644 roles/m3_networking/templates/MellanoxConnection
create mode 100644 roles/m3_networking/templates/ifcfg-conf.j2
create mode 100644 roles/m3_networking/vars/mellanoxVars.yml
diff --git a/roles/m3_networking/files/ibdev2netdev.sh b/roles/m3_networking/files/ibdev2netdev.sh
new file mode 100755
index 00000000..4fb4708a
--- /dev/null
+++ b/roles/m3_networking/files/ibdev2netdev.sh
@@ -0,0 +1,222 @@
+#! /bin/bash
+#
+#modified by simon to print device name
+#
+
+
+usage()
+{
+ echo "$(basename $0) <options>"
+ echo "-h, --help print help message"
+ echo "-v, --verbose print more info"
+}
+
+function find_pdev()
+{
+ pdevlist=$(ls /sys/bus/pci/devices)
+
+ for pdev in $pdevlist; do
+ if [ -d /sys/bus/pci/devices/$pdev/infiniband ]; then
+ ibd=$(ls /sys/bus/pci/devices/$pdev/infiniband/)
+ if [ "x$ibd" == "x$1" ]; then
+ echo -n $pdev
+ fi
+ fi
+ done
+}
+
+case $1 in
+ "-h" | "--help")
+ usage
+ exit 0
+ ;;
+esac
+
+if (( $# > 1 )); then
+ usage
+ exit -1
+fi
+
+if (( $# == 1 )) && [ "$1" != "-v" ]; then
+ usage
+ exit -1
+fi
+
+ibdevs=$(ls /sys/class/infiniband/)
+
+devs=
+for netpath in /sys/class/net/*
+do
+ if (grep 0x15b3 ${netpath}/device/vendor > /dev/null 2>&1); then
+ devs="$devs ${netpath##*/}"
+ fi
+done
+
+if [ "x$devs" == "x" ]; then
+ # no relevant devices - quit immediately
+ exit
+fi
+
+for d in $devs; do
+ if [ -f /sys/class/net/$d/dev_id ]; then
+ oldstyle=n
+ break
+ fi
+done
+
+if [ "x$oldstyle" == "xn" ]; then
+ for d in $ibdevs; do
+ ibrsc=$(cat /sys/class/infiniband/$d/device/resource)
+ eths=$(ls /sys/class/net/)
+ for eth in $eths; do
+ filepath_resource=/sys/class/net/$eth/device/resource
+
+ if [ -f $filepath_resource ]; then
+ ethrsc=$(cat $filepath_resource)
+ if [ "x$ethrsc" == "x$ibrsc" ]; then
+ filepath_devid=/sys/class/net/$eth/dev_id
+ filepath_devport=/sys/class/net/$eth/dev_port
+ if [ -f $filepath_devid ]; then
+ port1=0
+ if [ -f $filepath_devport ]; then
+ port1=$(cat $filepath_devport)
+ port1=$(printf "%d" $port1)
+ fi
+
+ port=$(cat $filepath_devid)
+ port=$(printf "%d" $port)
+ if [ $port1 -gt $port ]; then
+ port=$port1
+ fi
+
+ port=$(( port + 1 ))
+
+ filepath_carrier=/sys/class/net/$eth/carrier
+
+ if [ -f $filepath_carrier ]; then
+ link_state=$(cat $filepath_carrier 2> /dev/null)
+ if (( link_state == 1 )); then
+ link_state="Up"
+ else
+ link_state="Down"
+ fi
+ else
+ link_state="NA"
+ fi
+
+ x=$(find_pdev $d)
+ if [ "$1" == "-v" ]; then
+ filepath_portstate=/sys/class/infiniband/$d/ports/$port/state
+ filepath_deviceid=/sys/class/infiniband/$d/device/device
+ filepath_fwver=/sys/class/infiniband/$d/fw_ver
+ filepath_vpd=/sys/class/infiniband/$d/device/vpd
+
+ # read port state
+ if [ -f $filepath_portstate ]; then
+ ibstate=$(printf "%-6s" $(cat $filepath_portstate | gawk '{print $2}'))
+ else
+ ibstate="NA"
+ fi
+
+ # read device
+ if [ -f $filepath_deviceid ]; then
+ devid=$(printf "MT%d" $(cat $filepath_deviceid))
+ else
+ devid="NA"
+ fi
+
+ # read FW version
+ if [ -f $filepath_fwver ]; then
+ fwver=$(cat $filepath_fwver)
+ else
+ fwver="NA"
+ fi
+
+ # read device description and part ID from the VPD
+ if [ -f $filepath_vpd ]; then
+ tmp=$IFS
+ IFS=":"
+ vpd_content=`cat $filepath_vpd`
+ devdesc=$(printf "%-15s" $(echo $vpd_content | strings | head -1))
+ partid=$(printf "%-11s" $(echo $vpd_content | strings | head -4 | tail -1 | gawk '{print $1}'))
+ IFS=$tmp
+ else
+ devdesc=""
+ partid="NA"
+ fi
+ #echo "$x $d ($devid - $partid) $devdesc fw $fwver port $port ($ibstate) ==> $eth ($link_state)"
+ echo -n "$eth"
+ else
+ echo -n "$eth"
+ #echo "$d port $port ==> $eth ($link_state)"
+ fi
+ fi
+ fi
+ fi
+ done
+ done
+else
+##########################
+### old style
+##########################
+
+function print_line()
+{
+ echo -n "$eth"
+# echo "$1 port $2 <===> $3"
+
+}
+
+function find_guid()
+{
+ ibdevs=$(ls /sys/class/infiniband/)
+ for ibdev in $ibdevs; do
+ ports=$(ls /sys/class/infiniband/$ibdev/ports/)
+ for port in $ports; do
+ gids=$(ls /sys/class/infiniband/$ibdev/ports/$port/gids)
+ for gid in $gids; do
+ pguid=$(cat /sys/class/infiniband/$ibdev/ports/$port/gids/$gid | cut -b 21- | sed -e 's/://g')
+ if [ x$pguid == x$1 ]; then
+ print_line $ibdev $port $2
+ fi
+ done
+ done
+ done
+}
+
+function find_mac()
+{
+ ibdevs=$(ls /sys/class/infiniband/)
+ for ibdev in $ibdevs; do
+ ports=$(ls /sys/class/infiniband/$ibdev/ports/)
+ for port in $ports; do
+ gids=$(ls /sys/class/infiniband/$ibdev/ports/$port/gids)
+ for gid in $gids; do
+ first=$(cat /sys/class/infiniband/$ibdev/ports/$port/gids/$gid | cut -b 21-22)
+ first=$(( first ^ 2 ))
+ first=$(printf "%02x" $first)
+ second=$(cat /sys/class/infiniband/$ibdev/ports/$port/gids/$gid | cut -b 21- | sed -e 's/://g' | cut -b 3-6)
+ third=$(cat /sys/class/infiniband/$ibdev/ports/$port/gids/$gid | cut -b 21- | sed -e 's/://g' | cut -b 11-)
+ pmac=$first$second$third
+ if [ x$pmac == x$1 ]; then
+ print_line $ibdev $port $2
+ fi
+ done
+ done
+ done
+}
+
+ifcs=$(ifconfig -a | egrep '^eth|^ib' | gawk '{print $1}')
+
+for ifc in $ifcs; do
+ len=$(cat /sys/class/net/$ifc/addr_len)
+ if (( len == 20 )); then
+ guid=$(cat /sys/class/net/$ifc/address | cut -b 37- | sed -e 's/://g')
+ find_guid $guid $ifc
+ elif (( len == 6)); then
+ mac=$(cat /sys/class/net/$ifc/address | sed -e 's/://g')
+ find_mac $mac $ifc
+ fi
+done
+fi
+
diff --git a/roles/m3_networking/tasks/main.yml b/roles/m3_networking/tasks/main.yml
new file mode 100644
index 00000000..e7caa626
--- /dev/null
+++ b/roles/m3_networking/tasks/main.yml
@@ -0,0 +1,164 @@
+---
+- include_vars: mellanoxVars.yml
+
+- name: install rules to make sure the Mellanox card is mlx0
+ template: src="90-mlx.rules.j2" dest="/etc/udev/rules.d/90-mlx.rules" mode=644 owner=root
+ become: true
+ become_user: root
+
+- name: set NetworkManager to use keyfiles
+ lineinfile: dest=/etc/NetworkManager/NetworkManager.conf regexp="plugins=.*" line="plugins=keyfile" backrefs=True
+ become: true
+ become_user: root
+
+- name: set configs
+ template: src=EthernetConnection dest=/etc/NetworkManager/system-connections/EthernetConnection mode=600
+ become: true
+ become_user: root
+#
+#- name: set configs
+# template: src=MellanoxConnection dest=/etc/NetworkManager/system-connections/MellanoxConnection
+# become: true
+# become_user: root
+
+
+#- name: yum install dependencies
+# yum: name=perl,pciutils,gtk2,atk,cairo,gcc-gfortran,libxml2-python,tcsh,libnl,lsof,tcl,tk
+# sudo: true
+# ignore_errors: true
+# when: ansible_os_family == "RedHat"
+#
+#- name: test for existing installation of drivers
+# command: ibv_devinfo
+# sudo: true
+# register: drivers_installed
+# ignore_errors: true
+#
+#
+## This is NASTY. Don't upgrade production systems without taking them out of the queue first.
+##- name: yum update to upgrade kernel
+## shell: "yum update -y"
+## sudo: true
+## ignore_errors: true
+## when: ansible_os_family == "RedHat" and drivers_installed|failed
+#
+##
+## A REBOOT IS NEEDED AFTER a KERNEL UPDATE
+##
+##- name: restart machine
+## shell: sleep 5; sudo shutdown -r now "Ansible updates triggered"
+## async: 2
+## poll: 0
+## ignore_errors: true
+## sudo: true
+## when: ansible_os_family == "RedHat" and drivers_installed|failed
+##
+##- name: waiting for server to come back
+## local_action: wait_for host={{ ansible_host }} state=started port=22 delay=10 search_regex=OpenSSH
+## sudo: false
+##
+##- name: waiting for server to come back number 2
+## local_action: wait_for host={{ ansible_host }} state=started port=22 delay=10 search_regex=OpenSSH
+## sudo: false
+#
+#
+#- name: copy driver source
+# #make this a variable
+# unarchive: copy=yes src="files/{{ MELLANOX_DRIVER_SRC }}.tgz" dest=/tmp
+# sudo: true
+# #when: drivers_installed|failed and ansible_os_family=="RedHat" and ansible_distribution_major_version == "7"
+# when: drivers_installed|failed
+#
+#- name: install drivers
+# shell: ./mlnxofedinstall -q
+# args:
+# #more changes
+# chdir: "/tmp/{{ MELLANOX_DRIVER_SRC }}"
+# sudo: true
+# when: drivers_installed|failed
+#
+##
+## get IP address before reboot
+##
+#- name: get IP address
+# local_action: command ./scripts/map_ib_ip.pl {{ inventory_hostname }}
+# register: ip_address
+# sudo: false
+# #when: drivers_installed|failed
+#
+##get the interface name
+#- name: get devide name of Mellanox Interface from ibdev2netdev.sh
+# script: ibdev2netdev.sh
+# register: MELLANOX_DEVICE_NAME
+# sudo: true
+#
+#- name: create /etc/sysconfig/network-scripts/ifcfg-<device name>
+# template: dest=/etc/sysconfig/network-scripts/ifcfg-{{ MELLANOX_DEVICE_NAME.stdout }} src=ifcfg-conf.j2 owner=root group=root
+# sudo: true
+# when: ansible_os_family=="RedHat"
+#
+#
+##ubuntu equivalent of previous command
+#- name: Ubuntu network interfaces - line 1
+# lineinfile:
+# args:
+# dest: /etc/network/interfaces
+# line: auto {{ MELLANOX_DEVICE_NAME.stdout }}
+# state: present
+# sudo: true
+# when: ansible_os_family=="Debian" and drivers_installed|failed
+#
+#- name: Ubuntu network interfaces - line 2
+# lineinfile:
+# args:
+# dest: /etc/network/interfaces
+# line: iface {{ MELLANOX_DEVICE_NAME.stdout }} inet static
+# state: present
+# insertafter: "auto {{ MELLANOX_DEVICE_NAME.stdout }}"
+# sudo: true
+# when: ansible_os_family=="Debian" and drivers_installed|failed
+#
+#- name: Ubuntu network interfaces - line 3
+# lineinfile:
+# args:
+# dest: /etc/network/interfaces
+# line: address {{ ip_address.stdout }}
+# state: present
+# insertafter: "iface {{ MELLANOX_DEVICE_NAME.stdout }} inet static"
+# sudo: true
+# when: ansible_os_family=="Debian" and drivers_installed|failed
+##
+## A REBOOT IS NEEDED AFTER SUCCESSFUL INSTALL
+##
+#- name: restart machine
+# shell: "sleep 5; sudo shutdown -r now"
+# async: 2
+# poll: 1
+# ignore_errors: true
+# sudo: true
+# when: ansible_os_family=="RedHat" and drivers_installed|failed
+#
+#- name: restart machine for Ubuntu -cos it is 'special'
+# shell: "sleep 5; sudo shutdown -r now"
+# async: 2
+# poll: 1
+# ignore_errors: true
+# sudo: true
+# when: ansible_os_family=="Debian" and drivers_installed|failed
+#
+#- name: waiting for server to come back
+# local_action: wait_for host={{ ansible_host }} state=started port=22 delay=10 search_regex=OpenSSH
+# sudo: false
+# when: drivers_installed|failed
+#
+#- name: waiting for server to come back 2
+# local_action: wait_for host={{ ansible_host }} state=started port=22 delay=10 search_regex=OpenSSH
+# when: drivers_installed|failed
+#
+#- name: bring up interface
+# #variable=eth0 or ens6 or eth1
+# command: ifup {{ MELLANOX_DEVICE_NAME.stdout }}
+# sudo: true
+# when: ansible_os_family=="RedHat" and ansible_distribution_major_version == "7"
+#
+#
diff --git a/roles/m3_networking/templates/90-mlx.rules.j2 b/roles/m3_networking/templates/90-mlx.rules.j2
new file mode 100644
index 00000000..09fe9a36
--- /dev/null
+++ b/roles/m3_networking/templates/90-mlx.rules.j2
@@ -0,0 +1,7 @@
+KERNEL=="umad*", NAME="infiniband/%k" MODE="0666"
+KERNEL=="issm*", NAME="infiniband/%k"
+KERNEL=="ucm*", NAME="infiniband/%k", MODE="0666"
+KERNEL=="uverbs*", NAME="infiniband/%k", MODE="0666"
+KERNEL=="ucma", NAME="infiniband/%k", MODE="0666"
+KERNEL=="rdma_cm", NAME="infiniband/%k", MODE="0666"
+DRIVERS=="*mlx*", SUBSYSTEM=="net", ACTION=="add", NAME="mlx0", RUN+="/usr/bin/systemctl --no-block start mlnx_interface_mgr@%k.service"
diff --git a/roles/m3_networking/templates/Ethernet connection 1 b/roles/m3_networking/templates/Ethernet connection 1
new file mode 100644
index 00000000..36946f70
--- /dev/null
+++ b/roles/m3_networking/templates/Ethernet connection 1
@@ -0,0 +1,17 @@
+[connection]
+id=Ethernet connection 1
+uuid=346d20c3-1fc9-4157-9d7f-dae14c4bccce
+type=ethernet
+permissions=
+secondaries=
+interface-name=mlx0
+
+
+[ipv4]
+address1=172.16.193.12/32
+dns-search=
+method=manual
+
+[ipv6]
+dns-search=
+method=auto
diff --git a/roles/m3_networking/templates/EthernetConnection b/roles/m3_networking/templates/EthernetConnection
new file mode 100644
index 00000000..2f72ae4e
--- /dev/null
+++ b/roles/m3_networking/templates/EthernetConnection
@@ -0,0 +1,15 @@
+[connection]
+id=EthernetConnection
+type=ethernet
+permissions=
+secondaries=
+
+[ipv4]
+dns-search=
+method=auto
+route1=118.138.240.0/21,172.16.207.254
+route2=130.194.0.0/16,172.16.207.254
+
+[ipv6]
+dns-search=
+method=auto
diff --git a/roles/m3_networking/templates/MellanoxConnection b/roles/m3_networking/templates/MellanoxConnection
new file mode 100644
index 00000000..eb62ff24
--- /dev/null
+++ b/roles/m3_networking/templates/MellanoxConnection
@@ -0,0 +1,18 @@
+[connection]
+id=MellanoxConnection
+type=infiniband
+interface-name=mlx0
+permissions=
+secondaries=
+
+[infiniband]
+transport-mode=datagram
+
+[ipv4]
+address1=172.16.192.12/32
+dns-search=
+method=manual
+
+[ipv6]
+dns-search=
+method=auto
diff --git a/roles/m3_networking/templates/ifcfg-conf.j2 b/roles/m3_networking/templates/ifcfg-conf.j2
new file mode 100644
index 00000000..ecd3c4f5
--- /dev/null
+++ b/roles/m3_networking/templates/ifcfg-conf.j2
@@ -0,0 +1,8 @@
+DEVICE={{ MELLANOX_DEVICE_NAME.stdout }}
+ONBOOT=yes
+NM_CONTROLLED=no
+BOOTPROTO=none
+IPADDR={{ ip_address.stdout }}
+PREFIX=22
+MTU=9000
+
diff --git a/roles/m3_networking/vars/mellanoxVars.yml b/roles/m3_networking/vars/mellanoxVars.yml
new file mode 100644
index 00000000..5b53fe8f
--- /dev/null
+++ b/roles/m3_networking/vars/mellanoxVars.yml
@@ -0,0 +1,5 @@
+---
+ #note. do not add '.tgz' to driver src. done in playbook
+ #MELLANOX_DRIVER_SRC: "{% if ansible_os_family == 'RedHat' %}MLNX_OFED_LINUX-3.1-1.0.3-rhel7.1-x86_64-ext{% elif ansible_os_family == 'Debian' %}MLNX_OFED_LINUX-3.1-1.0.3-ubuntu14.04-x86_64{% endif %}"
+ #MELLANOX_DRIVER_SRC: "{% if ansible_os_family == 'RedHat' %}MLNX_OFED_LINUX-3.1-1.0.3-rhel7.2-x86_64-ext{% elif ansible_os_family == 'Debian' %}MLNX_OFED_LINUX-3.1-1.0.3-ubuntu14.04-x86_64{% endif %}"
+ MELLANOX_DRIVER_SRC: "{% if ansible_os_family == 'RedHat' %}MLNX_OFED_LINUX-3.2-2.0.0.0-rhel7.2-x86_64{% elif ansible_os_family == 'Debian' %}MLNX_OFED_LINUX-3.1-1.0.3-ubuntu14.04-x86_64{% endif %}"
--
GitLab