pfc-ecn.sh.j2 2.57 KB
Newer Older
Andreas Hamacher's avatar
Andreas Hamacher committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#!/bin/bash

# Get the list of Mellanox interfaces (p1p1, p1p2, etc...)
declare -a mellanox_interface_array

for f in /sys/class/net/*; do
    int=$(basename $f)
    driver=$(readlink $f/device/driver/module)
    if [ $driver ]; then
        driver=$(basename $driver)
        operstate=$(cat $f/operstate)
    fi
    if [[ "$driver" == *"mlx"* ]] && [[ "$operstate" == "up"  ]]; then
      mellanox_interface_array+=($int)
    fi
done

# Get the list of Mellanox devices (mlx5_0, mlx5_1, etc...)
declare -a mellanox_device_array

for int in "${mellanox_interface_array[@]}"; do
    dev=`\`which ibdev2netdev\` | grep $int | awk '{print $1}'`
    mellanox_device_array+=($dev)
done

if [ -f /proc/net/bonding/bond0 ]; then
    dev=`\`which ibdev2netdev\` | grep bond | awk '{print $1}'`
    mellanox_device_array+=($dev)
fi

# Set DSCP (L3) as trust mode for the NIC
# Syntax: mlnx_qos -i <interface> --trust dscp
# Example: /bin/mlnx_qos -i p5p1 --trust dscp

Andreas Hamacher's avatar
Andreas Hamacher committed
35
{% if "OpenStack Nova" not in ansible_product_name %}
Andreas Hamacher's avatar
Andreas Hamacher committed
36
37
38
39
for int in "${mellanox_interface_array[@]}"; do
   printf "`which mlnx_qos` -i $int --trust dscp \n"
   `which mlnx_qos` -i $int --trust dscp
done
Andreas Hamacher's avatar
Andreas Hamacher committed
40
{% endif %}
Andreas Hamacher's avatar
Andreas Hamacher committed
41
42
43
44

# Set ToS to 106 (DSCP 26) for ALL RoCE traffic
# Syntax: echo 106 > /sys/class/infiniband/<mlx-device>/tc/1/traffic_class
# Example: echo 106 > /sys/class/infiniband/mlx5_0/tc/1/traffic_class
Andreas Hamacher's avatar
Andreas Hamacher committed
45
{% if "OpenStack Nova" not in ansible_product_name %}
Andreas Hamacher's avatar
Andreas Hamacher committed
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
for dev in "${mellanox_device_array[@]}"; do
   printf "echo 106 > /sys/class/infiniband/$dev/tc/1/traffic_class \n"
   echo 106 > /sys/class/infiniband/$dev/tc/1/traffic_class
done

# Set the RDMA-CM ToS to 106 (DSCP 26)
# Syntax: cma_roce_tos -d <mlx-device> -t 106
# Example: /sbin/cma_roce_tos -d mlx5_0 -t 106
for dev in "${mellanox_device_array[@]}"; do
   printf "`which cma_roce_tos` -d $dev -t 106 \n"
   `which cma_roce_tos` -d $dev -t 106
done


# Enable PFC on RoCE prioritry - Activate PFC on priority 3
# Syntax: mlnx_qos -i <interface> --pfc 0,0,0,1,0,0,0,0
# Example: mlnx_qos -i p5p1 --pfc 0,0,0,1,0,0,0,0

for int in "${mellanox_interface_array[@]}"; do
   printf "`which mlnx_qos` -i $int --pfc 0,0,0,1,0,0,0,0 \n"
   `which mlnx_qos` -i $int --pfc 0,0,0,1,0,0,0,0
Andreas Hamacher's avatar
Andreas Hamacher committed
67
68
done

69
70
71
/sbin/ethtool -A p1p1 rx off tx off || /bin/true
/sbin/ethtool -A p1p2 rx off tx off || /bin/true

Andreas Hamacher's avatar
Andreas Hamacher committed
72
73
74
75
76
77
78
79
80
81
82
83
{% endif %}

{% if "OpenStack Nova" in ansible_product_name %}
{% for device in qibdevicenames.stdout_lines %}
printf "echo 106 > /sys/class/infiniband/{{ device }}/tc/1/traffic_class \n"
`which cma_roce_tos` -d  {{ device }} -t 106
{% endfor %}
{% endif %}


# Enable ECN for TCP traffic
/sbin/sysctl -w net.ipv4.tcp_ecn=1