Skip to content

Commit

Permalink
Identify whether the prepare.sh script on host or on bluefield runs
Browse files Browse the repository at this point in the history
The Bluefield card is visible to the host x86 system where it is
connected and the prepare script also runs on the host system.
(not the dpservice but just the prepare script)

Therefore the script needs to be able to differentiate between
the cases running on the bluefield directly and on the host where
the bluefield is connected.

Signed-off-by: Guvenc Gulce <[email protected]>
  • Loading branch information
guvenc committed Mar 1, 2024
1 parent 04bccb1 commit 797d9c6
Showing 1 changed file with 64 additions and 17 deletions.
81 changes: 64 additions & 17 deletions hack/prepare.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ set -Eeuo pipefail
BLUEFIELD_IDENTIFIERS=("MT_0000000543", "MT_0000000541")
NUMVFS=126
CONFIG="/tmp/dp_service.conf"
IS_BLUEFIELD=false
IS_X86_WITH_BLUEFIELD=false
IS_ARM_WITH_BLUEFIELD=false
IS_X86_WITH_MLX=false

function log() {
echo "$(date +"%Y-%m-%d_%H-%M-%S-%3N") $1"
Expand All @@ -27,19 +29,49 @@ function get_pf() {
readarray -t devs < <(devlink dev | awk -F/ '{print $2}')
}

function detect_card_type() {
function detect_card_and_arch_type() {
local pf="${devs[0]}"
local is_bluefield=false
for id in "${BLUEFIELD_IDENTIFIERS[@]}"; do
if devlink dev info pci/$pf | grep -q "$id"; then
IS_BLUEFIELD=true
is_bluefield=true
log "Detected BlueField card with identifier $id on pf: $pf"
break
fi
done

if ! $IS_BLUEFIELD; then
local arch=$(uname -m)
case $arch in
x86_64)
log "Architecture is AMD/Intel 64-bit"
;;
aarch64)
log "Architecture is ARM 64-bit"
;;
*)
err "Unsupported architecture: $arch"
exit
;;
esac

if ! $is_bluefield; then
log "Detected Mellanox card on pf: $pf"
fi

if [[ "$arch" = "aarch64" ]] && [[ "$is_bluefield" = "true" ]]; then
log "Detected system is ARM architecture with Bluefield card"
IS_ARM_WITH_BLUEFIELD=true
fi

if [[ "$arch" = "x86_64" ]] && [[ "$is_bluefield" = "false" ]]; then
log "Detected system is AMD/Intel 64-bit architecture with Mellanox card"
IS_X86_WITH_MLX=true
fi

if [[ "$arch" = "x86_64" ]] && [[ "$is_bluefield" = "true" ]]; then
log "Detected system is AMD/Intel 64-bit architecture with Bluefield card"
IS_X86_WITH_BLUEFIELD=true
fi
}

function validate() {
Expand All @@ -50,8 +82,8 @@ function validate() {
}

function validate_pf() {
if $IS_BLUEFIELD; then
log "Skipping PF validation for BlueField card"
if [[ "$IS_ARM_WITH_BLUEFIELD" == "true" ]]; then
log "Skipping PF validation for BlueField card on ARM"
return
fi

Expand All @@ -73,14 +105,29 @@ function validate_pf() {
devs=("${valid_devs[@]}")
}

process_switchdev_mode() {
local pf=$1

log "enabling switchdev for $pf"
if ! devlink dev eswitch set pci/$pf mode switchdev; then
log "can't set eswitch mode, setting VFs to 0"
echo 0 > /sys/bus/pci/devices/$pf/sriov_numvfs
fi
log "now waiting for everything to settle"
udevadm settle
}

function create_vf() {
if $IS_BLUEFIELD; then
local pf="${devs[0]}"

if [[ "$IS_ARM_WITH_BLUEFIELD" == "true" ]]; then
actualvfs=$NUMVFS
log "Skipping VF creation for BlueField card"
log "Skipping VF creation for BlueField card on ARM"
# enable switchdev mode, this operation takes most time
process_switchdev_mode "$pf"
return
fi

local pf="${devs[0]}"
# we disable automatic binding so that VFs don't get created, saves a lot of time
# plus we don't need to unbind them before enabling switchdev mode
log "disabling automatic binding of VFs on pf: $pf"
Expand All @@ -92,14 +139,10 @@ function create_vf() {
log "creating $actualvfs virtual functions"
echo $actualvfs > /sys/bus/pci/devices/$pf/sriov_numvfs

# enable switchdev mode, this operation takes most time
log "enabling switchdev for $pf"
if ! devlink dev eswitch set pci/$pf mode switchdev; then
log "can't set eswitch mode, setting VFs to 0"
echo 0 > /sys/bus/pci/devices/$pf/sriov_numvfs
if [[ "$IS_X86_WITH_MLX" == "true" ]]; then
# enable switchdev mode, this operation takes most time
process_switchdev_mode "$pf"
fi
log "now waiting for everything to settle"
udevadm settle
}

function get_pattern() {
Expand Down Expand Up @@ -129,6 +172,10 @@ function get_ipv6() {
}

function make_config() {
if [[ "$IS_X86_WITH_BLUEFIELD" == "true" ]]; then
log "Skipping config file creation on AMD/Intel 64-bit host with Bluefield"
return
fi
: > "$CONFIG"
{ echo "# This has been generated by prepare.sh"
echo "no-stats";
Expand All @@ -148,7 +195,7 @@ fi

validate
get_pf
detect_card_type
detect_card_and_arch_type
validate_pf
create_vf
make_config
Expand Down

0 comments on commit 797d9c6

Please sign in to comment.