#!/bin/bash

# Get the list of Mellanox interfaces (p1p1, p1p2, etc...)
declare -a mellanox_interface_array

for f in /sys/class/net/*; do
    int=$(basename $f)
    driver=$(readlink $f/device/driver/module)
    if [ $driver ]; then
        driver=$(basename $driver)
        operstate=$(cat $f/operstate)
    fi
    if [[ "$driver" == *"mlx"* ]] && [[ "$operstate" == "up"  ]]; then
      mellanox_interface_array+=($int)
    fi
done

# Get the list of Mellanox devices (mlx5_0, mlx5_1, etc...)
declare -a mellanox_device_array

for int in "${mellanox_interface_array[@]}"; do
    dev=`\`which ibdev2netdev\` | grep $int | awk '{print $1}'`
    mellanox_device_array+=($dev)
done

if [ -f /proc/net/bonding/bond0 ]; then
    dev=`\`which ibdev2netdev\` | grep bond | awk '{print $1}'`
    mellanox_device_array+=($dev)
fi

# Set DSCP (L3) as trust mode for the NIC
# Syntax: mlnx_qos -i <interface> --trust dscp
# Example: /bin/mlnx_qos -i p5p1 --trust dscp

{% if "OpenStack Nova" not in ansible_product_name %}
for int in "${mellanox_interface_array[@]}"; do
   printf "`which mlnx_qos` -i $int --trust dscp \n"
   `which mlnx_qos` -i $int --trust dscp
done
{% endif %}

# Set ToS to 106 (DSCP 26) for ALL RoCE traffic
# Syntax: echo 106 > /sys/class/infiniband/<mlx-device>/tc/1/traffic_class
# Example: echo 106 > /sys/class/infiniband/mlx5_0/tc/1/traffic_class
{% if "OpenStack Nova" not in ansible_product_name %}
for dev in "${mellanox_device_array[@]}"; do
   printf "echo 106 > /sys/class/infiniband/$dev/tc/1/traffic_class \n"
   echo 106 > /sys/class/infiniband/$dev/tc/1/traffic_class
done

# Set the RDMA-CM ToS to 106 (DSCP 26)
# Syntax: cma_roce_tos -d <mlx-device> -t 106
# Example: /sbin/cma_roce_tos -d mlx5_0 -t 106
for dev in "${mellanox_device_array[@]}"; do
   printf "`which cma_roce_tos` -d $dev -t 106 \n"
   `which cma_roce_tos` -d $dev -t 106
done


# Enable PFC on RoCE prioritry - Activate PFC on priority 3
# Syntax: mlnx_qos -i <interface> --pfc 0,0,0,1,0,0,0,0
# Example: mlnx_qos -i p5p1 --pfc 0,0,0,1,0,0,0,0

for int in "${mellanox_interface_array[@]}"; do
   printf "`which mlnx_qos` -i $int --pfc 0,0,0,1,0,0,0,0 \n"
   `which mlnx_qos` -i $int --pfc 0,0,0,1,0,0,0,0
done

/sbin/ethtool -A p1p1 rx off tx off || /bin/true
/sbin/ethtool -A p1p2 rx off tx off || /bin/true

{% endif %}

{% if "OpenStack Nova" in ansible_product_name %}
{% for device in qibdevicenames.stdout_lines %}
printf "echo 106 > /sys/class/infiniband/{{ device }}/tc/1/traffic_class \n"
echo 106 > /sys/class/infiniband/{{ device }}/tc/1/traffic_class
printf "`which cma_roce_tos` -d $dev -t 106 \n"
`which cma_roce_tos` -d  {{ device }} -t 106
{% endfor %}
{% endif %}


# Enable ECN for TCP traffic
/sbin/sysctl -w net.ipv4.tcp_ecn=1