tools/testing/selftests/net/fcnal-test.sh

#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (c) 2019 David Ahern <dsahern@gmail.com>. All rights reserved.
#
# IPv4 and IPv6 functional tests focusing on VRF and routing lookups
# for various permutations:
#   1. icmp, tcp, udp and netfilter
#   2. client, server, no-server
#   3. global address on interface
#   4. global address on 'lo'
#   5. remote and local traffic
#   6. VRF and non-VRF permutations
#
# Setup:
#                     ns-A     |     ns-B
# No VRF case:
#    [ lo ]         [ eth1 ]---|---[ eth1 ]      [ lo ]
#                                                remote address
# VRF case:
#         [ red ]---[ eth1 ]---|---[ eth1 ]      [ lo ]
#
# ns-A:
#     eth1: 172.16.1.1/24, 2001:db8:1::1/64
#       lo: 127.0.0.1/8, ::1/128
#           172.16.2.1/32, 2001:db8:2::1/128
#      red: 127.0.0.1/8, ::1/128
#           172.16.3.1/32, 2001:db8:3::1/128
#
# ns-B:
#     eth1: 172.16.1.2/24, 2001:db8:1::2/64
#      lo2: 127.0.0.1/8, ::1/128
#           172.16.2.2/32, 2001:db8:2::2/128
#
# server / client nomenclature relative to ns-A

VERBOSE=0

NSA_DEV=eth1
NSB_DEV=eth1
VRF=red
VRF_TABLE=1101

# IPv4 config
NSA_IP=172.16.1.1
NSB_IP=172.16.1.2
VRF_IP=172.16.3.1

# IPv6 config
NSA_IP6=2001:db8:1::1
NSB_IP6=2001:db8:1::2
VRF_IP6=2001:db8:3::1

NSA_LO_IP=172.16.2.1
NSB_LO_IP=172.16.2.2
NSA_LO_IP6=2001:db8:2::1
NSB_LO_IP6=2001:db8:2::2

MCAST=ff02::1
# set after namespace create
NSA_LINKIP6=
NSB_LINKIP6=

NSA=ns-A
NSB=ns-B

NSA_CMD="ip netns exec ${NSA}"
NSB_CMD="ip netns exec ${NSB}"

which ping6 > /dev/null 2>&1 && ping6=$(which ping6) || ping6=$(which ping)

################################################################################
# utilities

log_test()
{
	local rc=$1
	local expected=$2
	local msg="$3"

	[ "${VERBOSE}" = "1" ] && echo

	if [ ${rc} -eq ${expected} ]; then
		nsuccess=$((nsuccess+1))
		printf "TEST: %-70s  [ OK ]\n" "${msg}"
	else
		nfail=$((nfail+1))
		printf "TEST: %-70s  [FAIL]\n" "${msg}"
		if [ "${PAUSE_ON_FAIL}" = "yes" ]; then
			echo
			echo "hit enter to continue, 'q' to quit"
			read a
			[ "$a" = "q" ] && exit 1
		fi
	fi

	if [ "${PAUSE}" = "yes" ]; then
		echo
		echo "hit enter to continue, 'q' to quit"
		read a
		[ "$a" = "q" ] && exit 1
	fi

	kill_procs
}

log_test_addr()
{
	local addr=$1
	local rc=$2
	local expected=$3
	local msg="$4"
	local astr

	astr=$(addr2str ${addr})
	log_test $rc $expected "$msg - ${astr}"
}

log_section()
{
	echo
	echo "###########################################################################"
	echo "$*"
	echo "###########################################################################"
	echo
}

log_subsection()
{
	echo
	echo "#################################################################"
	echo "$*"
	echo
}

log_start()
{
	# make sure we have no test instances running
	kill_procs

	if [ "${VERBOSE}" = "1" ]; then
		echo
		echo "#######################################################"
	fi
}

log_debug()
{
	if [ "${VERBOSE}" = "1" ]; then
		echo
		echo "$*"
		echo
	fi
}

show_hint()
{
	if [ "${VERBOSE}" = "1" ]; then
		echo "HINT: $*"
		echo
	fi
}

kill_procs()
{
	killall nettest ping ping6 >/dev/null 2>&1
	sleep 1
}

do_run_cmd()
{
	local cmd="$*"
	local out

	if [ "$VERBOSE" = "1" ]; then
		echo "COMMAND: ${cmd}"
	fi

	out=$($cmd 2>&1)
	rc=$?
	if [ "$VERBOSE" = "1" -a -n "$out" ]; then
		echo "$out"
	fi

	return $rc
}

run_cmd()
{
	do_run_cmd ${NSA_CMD} $*
}

run_cmd_nsb()
{
	do_run_cmd ${NSB_CMD} $*
}

setup_cmd()
{
	local cmd="$*"
	local rc

	run_cmd ${cmd}
	rc=$?
	if [ $rc -ne 0 ]; then
		# show user the command if not done so already
		if [ "$VERBOSE" = "0" ]; then
			echo "setup command: $cmd"
		fi
		echo "failed. stopping tests"
		if [ "${PAUSE_ON_FAIL}" = "yes" ]; then
			echo
			echo "hit enter to continue"
			read a
		fi
		exit $rc
	fi
}

setup_cmd_nsb()
{
	local cmd="$*"
	local rc

	run_cmd_nsb ${cmd}
	rc=$?
	if [ $rc -ne 0 ]; then
		# show user the command if not done so already
		if [ "$VERBOSE" = "0" ]; then
			echo "setup command: $cmd"
		fi
		echo "failed. stopping tests"
		if [ "${PAUSE_ON_FAIL}" = "yes" ]; then
			echo
			echo "hit enter to continue"
			read a
		fi
		exit $rc
	fi
}

# set sysctl values in NS-A
set_sysctl()
{
	echo "SYSCTL: $*"
	echo
	run_cmd sysctl -q -w $*
}

################################################################################
# Setup for tests

addr2str()
{
	case "$1" in
	127.0.0.1) echo "loopback";;
	::1) echo "IPv6 loopback";;

	${NSA_IP})	echo "ns-A IP";;
	${NSA_IP6})	echo "ns-A IPv6";;
	${NSA_LO_IP})	echo "ns-A loopback IP";;
	${NSA_LO_IP6})	echo "ns-A loopback IPv6";;
	${NSA_LINKIP6}|${NSA_LINKIP6}%*) echo "ns-A IPv6 LLA";;

	${NSB_IP})	echo "ns-B IP";;
	${NSB_IP6})	echo "ns-B IPv6";;
	${NSB_LO_IP})	echo "ns-B loopback IP";;
	${NSB_LO_IP6})	echo "ns-B loopback IPv6";;
	${NSB_LINKIP6}|${NSB_LINKIP6}%*) echo "ns-B IPv6 LLA";;

	${VRF_IP})	echo "VRF IP";;
	${VRF_IP6})	echo "VRF IPv6";;

	${MCAST}%*)	echo "multicast IP";;

	*) echo "unknown";;
	esac
}

get_linklocal()
{
	local ns=$1
	local dev=$2
	local addr

	addr=$(ip -netns ${ns} -6 -br addr show dev ${dev} | \
	awk '{
		for (i = 3; i <= NF; ++i) {
			if ($i ~ /^fe80/)
				print $i
		}
	}'
	)
	addr=${addr/\/*}

	[ -z "$addr" ] && return 1

	echo $addr

	return 0
}

################################################################################
# create namespaces and vrf

create_vrf()
{
	local ns=$1
	local vrf=$2
	local table=$3
	local addr=$4
	local addr6=$5

	ip -netns ${ns} link add ${vrf} type vrf table ${table}
	ip -netns ${ns} link set ${vrf} up
	ip -netns ${ns} route add vrf ${vrf} unreachable default metric 8192
	ip -netns ${ns} -6 route add vrf ${vrf} unreachable default metric 8192

	ip -netns ${ns} addr add 127.0.0.1/8 dev ${vrf}
	ip -netns ${ns} -6 addr add ::1 dev ${vrf} nodad
	if [ "${addr}" != "-" ]; then
		ip -netns ${ns} addr add dev ${vrf} ${addr}
	fi
	if [ "${addr6}" != "-" ]; then
		ip -netns ${ns} -6 addr add dev ${vrf} ${addr6}
	fi

	ip -netns ${ns} ru del pref 0
	ip -netns ${ns} ru add pref 32765 from all lookup local
	ip -netns ${ns} -6 ru del pref 0
	ip -netns ${ns} -6 ru add pref 32765 from all lookup local
}

create_ns()
{
	local ns=$1
	local addr=$2
	local addr6=$3

	ip netns add ${ns}

	ip -netns ${ns} link set lo up
	if [ "${addr}" != "-" ]; then
		ip -netns ${ns} addr add dev lo ${addr}
	fi
	if [ "${addr6}" != "-" ]; then
		ip -netns ${ns} -6 addr add dev lo ${addr6}
	fi

	ip -netns ${ns} ro add unreachable default metric 8192
	ip -netns ${ns} -6 ro add unreachable default metric 8192

	ip netns exec ${ns} sysctl -qw net.ipv4.ip_forward=1
	ip netns exec ${ns} sysctl -qw net.ipv6.conf.all.keep_addr_on_down=1
	ip netns exec ${ns} sysctl -qw net.ipv6.conf.all.forwarding=1
	ip netns exec ${ns} sysctl -qw net.ipv6.conf.default.forwarding=1
}

# create veth pair to connect namespaces and apply addresses.
connect_ns()
{
	local ns1=$1
	local ns1_dev=$2
	local ns1_addr=$3
	local ns1_addr6=$4
	local ns2=$5
	local ns2_dev=$6
	local ns2_addr=$7
	local ns2_addr6=$8

	ip -netns ${ns1} li add ${ns1_dev} type veth peer name tmp
	ip -netns ${ns1} li set ${ns1_dev} up
	ip -netns ${ns1} li set tmp netns ${ns2} name ${ns2_dev}
	ip -netns ${ns2} li set ${ns2_dev} up

	if [ "${ns1_addr}" != "-" ]; then
		ip -netns ${ns1} addr add dev ${ns1_dev} ${ns1_addr}
		ip -netns ${ns2} addr add dev ${ns2_dev} ${ns2_addr}
	fi

	if [ "${ns1_addr6}" != "-" ]; then
		ip -netns ${ns1} addr add dev ${ns1_dev} ${ns1_addr6}
		ip -netns ${ns2} addr add dev ${ns2_dev} ${ns2_addr6}
	fi
}

cleanup()
{
	# explicit cleanups to check those code paths
	ip netns | grep -q ${NSA}
	if [ $? -eq 0 ]; then
		ip -netns ${NSA} link delete ${VRF}
		ip -netns ${NSA} ro flush table ${VRF_TABLE}

		ip -netns ${NSA} addr flush dev ${NSA_DEV}
		ip -netns ${NSA} -6 addr flush dev ${NSA_DEV}
		ip -netns ${NSA} link set dev ${NSA_DEV} down
		ip -netns ${NSA} link del dev ${NSA_DEV}

		ip netns del ${NSA}
	fi

	ip netns del ${NSB}
}

setup()
{
	local with_vrf=${1}

	# make sure we are starting with a clean slate
	kill_procs
	cleanup 2>/dev/null

	log_debug "Configuring network namespaces"
	set -e

	create_ns ${NSA} ${NSA_LO_IP}/32 ${NSA_LO_IP6}/128
	create_ns ${NSB} ${NSB_LO_IP}/32 ${NSB_LO_IP6}/128
	connect_ns ${NSA} ${NSA_DEV} ${NSA_IP}/24 ${NSA_IP6}/64 \
		   ${NSB} ${NSB_DEV} ${NSB_IP}/24 ${NSB_IP6}/64

	NSA_LINKIP6=$(get_linklocal ${NSA} ${NSA_DEV})
	NSB_LINKIP6=$(get_linklocal ${NSB} ${NSB_DEV})

	# tell ns-A how to get to remote addresses of ns-B
	if [ "${with_vrf}" = "yes" ]; then
		create_vrf ${NSA} ${VRF} ${VRF_TABLE} ${VRF_IP} ${VRF_IP6}

		ip -netns ${NSA} link set dev ${NSA_DEV} vrf ${VRF}
		ip -netns ${NSA} ro add vrf ${VRF} ${NSB_LO_IP}/32 via ${NSB_IP} dev ${NSA_DEV}
		ip -netns ${NSA} -6 ro add vrf ${VRF} ${NSB_LO_IP6}/128 via ${NSB_IP6} dev ${NSA_DEV}

		ip -netns ${NSB} ro add ${VRF_IP}/32 via ${NSA_IP} dev ${NSB_DEV}
		ip -netns ${NSB} -6 ro add ${VRF_IP6}/128 via ${NSA_IP6} dev ${NSB_DEV}
	else
		ip -netns ${NSA} ro add ${NSB_LO_IP}/32 via ${NSB_IP} dev ${NSA_DEV}
		ip -netns ${NSA} ro add ${NSB_LO_IP6}/128 via ${NSB_IP6} dev ${NSA_DEV}
	fi


	# tell ns-B how to get to remote addresses of ns-A
	ip -netns ${NSB} ro add ${NSA_LO_IP}/32 via ${NSA_IP} dev ${NSB_DEV}
	ip -netns ${NSB} ro add ${NSA_LO_IP6}/128 via ${NSA_IP6} dev ${NSB_DEV}

	set +e

	sleep 1
}

################################################################################
# IPv4

ipv4_ping_novrf()
{
	local a

	#
	# out
	#
	for a in ${NSB_IP} ${NSB_LO_IP}
	do
		log_start
		run_cmd ping -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping out"

		log_start
		run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 0 "ping out, device bind"

		log_start
		run_cmd ping -c1 -w1 -I ${NSA_LO_IP} ${a}
		log_test_addr ${a} $? 0 "ping out, address bind"
	done

	#
	# in
	#
	for a in ${NSA_IP} ${NSA_LO_IP}
	do
		log_start
		run_cmd_nsb ping -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping in"
	done

	#
	# local traffic
	#
	for a in ${NSA_IP} ${NSA_LO_IP} 127.0.0.1
	do
		log_start
		run_cmd ping -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping local"
	done

	#
	# local traffic, socket bound to device
	#
	# address on device
	a=${NSA_IP}
	log_start
	run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 0 "ping local, device bind"

	# loopback addresses not reachable from device bind
	# fails in a really weird way though because ipv4 special cases
	# route lookups with oif set.
	for a in ${NSA_LO_IP} 127.0.0.1
	do
		log_start
		show_hint "Fails since address on loopback device is out of device scope"
		run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 1 "ping local, device bind"
	done

	#
	# ip rule blocks reachability to remote address
	#
	log_start
	setup_cmd ip rule add pref 32765 from all lookup local
	setup_cmd ip rule del pref 0 from all lookup local
	setup_cmd ip rule add pref 50 to ${NSB_LO_IP} prohibit
	setup_cmd ip rule add pref 51 from ${NSB_IP} prohibit

	a=${NSB_LO_IP}
	run_cmd ping -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, blocked by rule"

	# NOTE: ipv4 actually allows the lookup to fail and yet still create
	# a viable rtable if the oif (e.g., bind to device) is set, so this
	# case succeeds despite the rule
	# run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}

	a=${NSA_LO_IP}
	log_start
	show_hint "Response generates ICMP (or arp request is ignored) due to ip rule"
	run_cmd_nsb ping -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in, blocked by rule"

	[ "$VERBOSE" = "1" ] && echo
	setup_cmd ip rule del pref 32765 from all lookup local
	setup_cmd ip rule add pref 0 from all lookup local
	setup_cmd ip rule del pref 50 to ${NSB_LO_IP} prohibit
	setup_cmd ip rule del pref 51 from ${NSB_IP} prohibit

	#
	# route blocks reachability to remote address
	#
	log_start
	setup_cmd ip route replace unreachable ${NSB_LO_IP}
	setup_cmd ip route replace unreachable ${NSB_IP}

	a=${NSB_LO_IP}
	run_cmd ping -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, blocked by route"

	# NOTE: ipv4 actually allows the lookup to fail and yet still create
	# a viable rtable if the oif (e.g., bind to device) is set, so this
	# case succeeds despite not having a route for the address
	# run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}

	a=${NSA_LO_IP}
	log_start
	show_hint "Response is dropped (or arp request is ignored) due to ip route"
	run_cmd_nsb ping -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in, blocked by route"

	#
	# remove 'remote' routes; fallback to default
	#
	log_start
	setup_cmd ip ro del ${NSB_LO_IP}

	a=${NSB_LO_IP}
	run_cmd ping -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, unreachable default route"

	# NOTE: ipv4 actually allows the lookup to fail and yet still create
	# a viable rtable if the oif (e.g., bind to device) is set, so this
	# case succeeds despite not having a route for the address
	# run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
}

ipv4_ping_vrf()
{
	local a

	# should default on; does not exist on older kernels
	set_sysctl net.ipv4.raw_l3mdev_accept=1 2>/dev/null

	#
	# out
	#
	for a in ${NSB_IP} ${NSB_LO_IP}
	do
		log_start
		run_cmd ping -c1 -w1 -I ${VRF} ${a}
		log_test_addr ${a} $? 0 "ping out, VRF bind"

		log_start
		run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 0 "ping out, device bind"

		log_start
		run_cmd ip vrf exec ${VRF} ping -c1 -w1 -I ${NSA_IP} ${a}
		log_test_addr ${a} $? 0 "ping out, vrf device + dev address bind"

		log_start
		run_cmd ip vrf exec ${VRF} ping -c1 -w1 -I ${VRF_IP} ${a}
		log_test_addr ${a} $? 0 "ping out, vrf device + vrf address bind"
	done

	#
	# in
	#
	for a in ${NSA_IP} ${VRF_IP}
	do
		log_start
		run_cmd_nsb ping -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping in"
	done

	#
	# local traffic, local address
	#
	for a in ${NSA_IP} ${VRF_IP} 127.0.0.1
	do
		log_start
		show_hint "Source address should be ${a}"
		run_cmd ping -c1 -w1 -I ${VRF} ${a}
		log_test_addr ${a} $? 0 "ping local, VRF bind"
	done

	#
	# local traffic, socket bound to device
	#
	# address on device
	a=${NSA_IP}
	log_start
	run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 0 "ping local, device bind"

	# vrf device is out of scope
	for a in ${VRF_IP} 127.0.0.1
	do
		log_start
		show_hint "Fails since address on vrf device is out of device scope"
		run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 1 "ping local, device bind"
	done

	#
	# ip rule blocks address
	#
	log_start
	setup_cmd ip rule add pref 50 to ${NSB_LO_IP} prohibit
	setup_cmd ip rule add pref 51 from ${NSB_IP} prohibit

	a=${NSB_LO_IP}
	run_cmd ping -c1 -w1 -I ${VRF} ${a}
	log_test_addr ${a} $? 2 "ping out, vrf bind, blocked by rule"

	log_start
	run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 2 "ping out, device bind, blocked by rule"

	a=${NSA_LO_IP}
	log_start
	show_hint "Response lost due to ip rule"
	run_cmd_nsb ping -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in, blocked by rule"

	[ "$VERBOSE" = "1" ] && echo
	setup_cmd ip rule del pref 50 to ${NSB_LO_IP} prohibit
	setup_cmd ip rule del pref 51 from ${NSB_IP} prohibit

	#
	# remove 'remote' routes; fallback to default
	#
	log_start
	setup_cmd ip ro del vrf ${VRF} ${NSB_LO_IP}

	a=${NSB_LO_IP}
	run_cmd ping -c1 -w1 -I ${VRF} ${a}
	log_test_addr ${a} $? 2 "ping out, vrf bind, unreachable route"

	log_start
	run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 2 "ping out, device bind, unreachable route"

	a=${NSA_LO_IP}
	log_start
	show_hint "Response lost by unreachable route"
	run_cmd_nsb ping -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in, unreachable route"
}

ipv4_ping()
{
	log_section "IPv4 ping"

	log_subsection "No VRF"
	setup
	set_sysctl net.ipv4.raw_l3mdev_accept=0 2>/dev/null
	ipv4_ping_novrf
	setup
	set_sysctl net.ipv4.raw_l3mdev_accept=1 2>/dev/null
	ipv4_ping_novrf

	log_subsection "With VRF"
	setup "yes"
	ipv4_ping_vrf
}

################################################################################
# IPv4 TCP

ipv4_tcp_novrf()
{
	local a

	#
	# server tests
	#
	for a in ${NSA_IP} ${NSA_LO_IP}
	do
		log_start
		run_cmd nettest -s &
		sleep 1
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 0 "Global server"
	done

	a=${NSA_IP}
	log_start
	run_cmd nettest -s -d ${NSA_DEV} &
	sleep 1
	run_cmd_nsb nettest -r ${a}
	log_test_addr ${a} $? 0 "Device server"

	# verify TCP reset sent and received
	for a in ${NSA_IP} ${NSA_LO_IP}
	do
		log_start
		show_hint "Should fail 'Connection refused' since there is no server"
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 1 "No server"
	done

	#
	# client
	#
	for a in ${NSB_IP} ${NSB_LO_IP}
	do
		log_start
		run_cmd_nsb nettest -s &
		sleep 1
		run_cmd nettest -r ${a} -0 ${NSA_IP}
		log_test_addr ${a} $? 0 "Client"

		log_start
		run_cmd_nsb nettest -s &
		sleep 1
		run_cmd nettest -r ${a} -d ${NSA_DEV}
		log_test_addr ${a} $? 0 "Client, device bind"

		log_start
		show_hint "Should fail 'Connection refused'"
		run_cmd nettest -r ${a}
		log_test_addr ${a} $? 1 "No server, unbound client"

		log_start
		show_hint "Should fail 'Connection refused'"
		run_cmd nettest -r ${a} -d ${NSA_DEV}
		log_test_addr ${a} $? 1 "No server, device client"
	done

	#
	# local address tests
	#
	for a in ${NSA_IP} ${NSA_LO_IP} 127.0.0.1
	do
		log_start
		run_cmd nettest -s &
		sleep 1
		run_cmd nettest -r ${a} -0 ${a} -1 ${a}
		log_test_addr ${a} $? 0 "Global server, local connection"
	done

	a=${NSA_IP}
	log_start
	run_cmd nettest -s -d ${NSA_DEV} &
	sleep 1
	run_cmd nettest -r ${a} -0 ${a}
	log_test_addr ${a} $? 0 "Device server, unbound client, local connection"

	for a in ${NSA_LO_IP} 127.0.0.1
	do
		log_start
		show_hint "Should fail 'Connection refused' since addresses on loopback are out of device scope"
		run_cmd nettest -s -d ${NSA_DEV} &
		sleep 1
		run_cmd nettest -r ${a}
		log_test_addr ${a} $? 1 "Device server, unbound client, local connection"
	done

	a=${NSA_IP}
	log_start
	run_cmd nettest -s &
	sleep 1
	run_cmd nettest -r ${a} -0 ${a} -d ${NSA_DEV}
	log_test_addr ${a} $? 0 "Global server, device client, local connection"

	for a in ${NSA_LO_IP} 127.0.0.1
	do
		log_start
		show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope"
		run_cmd nettest -s &
		sleep 1
		run_cmd nettest -r ${a} -d ${NSA_DEV}
		log_test_addr ${a} $? 1 "Global server, device client, local connection"
	done

	a=${NSA_IP}
	log_start
	run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} &
	sleep 1
	run_cmd nettest  -d ${NSA_DEV} -r ${a} -0 ${a}
	log_test_addr ${a} $? 0 "Device server, device client, local connection"

	log_start
	show_hint "Should fail 'Connection refused'"
	run_cmd nettest -d ${NSA_DEV} -r ${a}
	log_test_addr ${a} $? 1 "No server, device client, local conn"
}

ipv4_tcp_vrf()
{
	local a

	# disable global server
	log_subsection "Global server disabled"

	set_sysctl net.ipv4.tcp_l3mdev_accept=0

	#
	# server tests
	#
	for a in ${NSA_IP} ${VRF_IP}
	do
		log_start
		show_hint "Should fail 'Connection refused' since global server with VRF is disabled"
		run_cmd nettest -s &
		sleep 1
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 1 "Global server"

		log_start
		run_cmd nettest -s -d ${VRF} -2 ${VRF} &
		sleep 1
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 0 "VRF server"

		log_start
		run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} &
		sleep 1
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 0 "Device server"

		# verify TCP reset received
		log_start
		show_hint "Should fail 'Connection refused' since there is no server"
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 1 "No server"
	done

	# local address tests
	# (${VRF_IP} and 127.0.0.1 both timeout)
	a=${NSA_IP}
	log_start
	show_hint "Should fail 'Connection refused' since global server with VRF is disabled"
	run_cmd nettest -s &
	sleep 1
	run_cmd nettest -r ${a} -d ${NSA_DEV}
	log_test_addr ${a} $? 1 "Global server, local connection"

	#
	# enable VRF global server
	#
	log_subsection "VRF Global server enabled"
	set_sysctl net.ipv4.tcp_l3mdev_accept=1

	for a in ${NSA_IP} ${VRF_IP}
	do
		log_start
		show_hint "client socket should be bound to VRF"
		run_cmd nettest -s -2 ${VRF} &
		sleep 1
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 0 "Global server"

		log_start
		show_hint "client socket should be bound to VRF"
		run_cmd nettest -s -d ${VRF} -2 ${VRF} &
		sleep 1
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 0 "VRF server"

		# verify TCP reset received
		log_start
		show_hint "Should fail 'Connection refused'"
		run_cmd_nsb nettest -r ${a}
		log_test_addr ${a} $? 1 "No server"
	done

	a=${NSA_IP}
	log_start
	show_hint "client socket should be bound to device"
	run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} &
	sleep 1
	run_cmd_nsb nettest -r ${a}
	log_test_addr ${a} $? 0 "Device server"

	# local address tests
	for a in ${NSA_IP} ${VRF_IP}
	do
		log_start
		show_hint "Should fail 'No route to host' since client is not bound to VRF"
		run_cmd nettest -s -2 ${VRF} &
		sleep 1
		run_cmd nettest -r ${a}
		log_test_addr ${a} $? 1 "Global server, local connection"
	done

	#
	# client
	#
	for a in ${NSB_IP} ${NSB_LO_IP}
	do
		log_start
		run_cmd_nsb nettest -s &
		sleep 1
		run_cmd nettest -r ${a} -d ${VRF}
		log_test_addr ${a} $? 0 "Client, VRF bind"

		log_start
		run_cmd_nsb nettest -s &
		sleep 1
		run_cmd nettest -r ${a} -d ${NSA_DEV}
		log_test_addr ${a} $? 0 "Client, device bind"

		log_start
		show_hint "Should fail 'Connection refused'"
		run_cmd nettest -r ${a} -d ${VRF}
		log_test_addr ${a} $? 1 "No server, VRF client"

		log_start
		show_hint "Should fail 'Connection refused'"
		run_cmd nettest -r ${a} -d ${NSA_DEV}
		log_test_addr ${a} $? 1 "No server, device client"
	done

	for a in ${NSA_IP} ${VRF_IP} 127.0.0.1
	do
		log_start
		run_cmd nettest -s -d ${VRF} -2 ${VRF} &
		sleep 1
		run_cmd nettest -r ${a} -d ${VRF} -0 ${a}
		log_test_addr ${a} $? 0 "VRF server, VRF client, local connection"
	done

	a=${NSA_IP}
	log_start
	run_cmd nettest -s -d ${VRF} -2 ${VRF} &
	sleep 1
	run_cmd nettest -r ${a} -d ${NSA_DEV} -0 ${a}
	log_test_addr ${a} $? 0 "VRF server, device client, local connection"

	log_start
	show_hint "Should fail 'No route to host' since client is out of VRF scope"
	run_cmd nettest -s -d ${VRF} &
	sleep 1
	run_cmd nettest -r ${a}
	log_test_addr ${a} $? 1 "VRF server, unbound client, local connection"

	log_start
	run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} &
	sleep 1
	run_cmd nettest -r ${a} -d ${VRF} -0 ${a}
	log_test_addr ${a} $? 0 "Device server, VRF client, local connection"

	log_start
	run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} &
	sleep 1
	run_cmd nettest -r ${a} -d ${NSA_DEV} -0 ${a}
	log_test_addr ${a} $? 0 "Device server, device client, local connection"
}

ipv4_tcp()
{
	log_section "IPv4/TCP"

	which nettest >/dev/null
	if [ $? -ne 0 ]; then
		log_error "nettest not found; skipping tests"
		return
	fi

	log_subsection "No VRF"
	setup

	# tcp_l3mdev_accept should have no affect without VRF;
	# run tests with it enabled and disabled to verify
	log_subsection "tcp_l3mdev_accept disabled"
	set_sysctl net.ipv4.tcp_l3mdev_accept=0
	ipv4_tcp_novrf
	log_subsection "tcp_l3mdev_accept enabled"
	set_sysctl net.ipv4.tcp_l3mdev_accept=1
	ipv4_tcp_novrf

	log_subsection "With VRF"
	setup "yes"
	ipv4_tcp_vrf
}

################################################################################
# IPv6

ipv6_ping_novrf()
{
	local a

	# should not have an impact, but make a known state
	set_sysctl net.ipv4.raw_l3mdev_accept=0 2>/dev/null

	#
	# out
	#
	for a in ${NSB_IP6} ${NSB_LO_IP6} ${NSB_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV}
	do
		log_start
		run_cmd ${ping6} -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping out"
	done

	for a in ${NSB_IP6} ${NSB_LO_IP6}
	do
		log_start
		run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 0 "ping out, device bind"

		log_start
		run_cmd ${ping6} -c1 -w1 -I ${NSA_LO_IP6} ${a}
		log_test_addr ${a} $? 0 "ping out, loopback address bind"
	done

	#
	# in
	#
	for a in ${NSA_IP6} ${NSA_LO_IP6} ${NSA_LINKIP6}%${NSB_DEV} ${MCAST}%${NSB_DEV}
	do
		log_start
		run_cmd_nsb ${ping6} -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping in"
	done

	#
	# local traffic, local address
	#
	for a in ${NSA_IP6} ${NSA_LO_IP6} ::1 ${NSA_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV}
	do
		log_start
		run_cmd ${ping6} -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping local, no bind"
	done

	for a in ${NSA_IP6} ${NSA_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV}
	do
		log_start
		run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 0 "ping local, device bind"
	done

	for a in ${NSA_LO_IP6} ::1
	do
		log_start
		show_hint "Fails since address on loopback is out of device scope"
		run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 2 "ping local, device bind"
	done

	#
	# ip rule blocks address
	#
	log_start
	setup_cmd ip -6 rule add pref 32765 from all lookup local
	setup_cmd ip -6 rule del pref 0 from all lookup local
	setup_cmd ip -6 rule add pref 50 to ${NSB_LO_IP6} prohibit
	setup_cmd ip -6 rule add pref 51 from ${NSB_IP6} prohibit

	a=${NSB_LO_IP6}
	run_cmd ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, blocked by rule"

	log_start
	run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 2 "ping out, device bind, blocked by rule"

	a=${NSA_LO_IP6}
	log_start
	show_hint "Response lost due to ip rule"
	run_cmd_nsb ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in, blocked by rule"

	setup_cmd ip -6 rule add pref 0 from all lookup local
	setup_cmd ip -6 rule del pref 32765 from all lookup local
	setup_cmd ip -6 rule del pref 50 to ${NSB_LO_IP6} prohibit
	setup_cmd ip -6 rule del pref 51 from ${NSB_IP6} prohibit

	#
	# route blocks reachability to remote address
	#
	log_start
	setup_cmd ip -6 route del ${NSB_LO_IP6}
	setup_cmd ip -6 route add unreachable ${NSB_LO_IP6} metric 10
	setup_cmd ip -6 route add unreachable ${NSB_IP6} metric 10

	a=${NSB_LO_IP6}
	run_cmd ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, blocked by route"

	log_start
	run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 2 "ping out, device bind, blocked by route"

	a=${NSA_LO_IP6}
	log_start
	show_hint "Response lost due to ip route"
	run_cmd_nsb ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in, blocked by route"


	#
	# remove 'remote' routes; fallback to default
	#
	log_start
	setup_cmd ip -6 ro del unreachable ${NSB_LO_IP6}
	setup_cmd ip -6 ro del unreachable ${NSB_IP6}

	a=${NSB_LO_IP6}
	run_cmd ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, unreachable route"

	log_start
	run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 2 "ping out, device bind, unreachable route"
}

ipv6_ping_vrf()
{
	local a

	# should default on; does not exist on older kernels
	set_sysctl net.ipv4.raw_l3mdev_accept=1 2>/dev/null

	#
	# out
	#
	for a in ${NSB_IP6} ${NSB_LO_IP6}
	do
		log_start
		run_cmd ${ping6} -c1 -w1 -I ${VRF} ${a}
		log_test_addr ${a} $? 0 "ping out, VRF bind"
	done

	for a in ${NSB_LINKIP6}%${VRF} ${MCAST}%${VRF}
	do
		log_start
		show_hint "Fails since VRF device does not support linklocal or multicast"
		run_cmd ${ping6} -c1 -w1 ${a}
		log_test_addr ${a} $? 2 "ping out, VRF bind"
	done

	for a in ${NSB_IP6} ${NSB_LO_IP6} ${NSB_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV}
	do
		log_start
		run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 0 "ping out, device bind"
	done

	for a in ${NSB_IP6} ${NSB_LO_IP6} ${NSB_LINKIP6}%${NSA_DEV}
	do
		log_start
		run_cmd ip vrf exec ${VRF} ${ping6} -c1 -w1 -I ${VRF_IP6} ${a}
		log_test_addr ${a} $? 0 "ping out, vrf device+address bind"
	done

	#
	# in
	#
	for a in ${NSA_IP6} ${VRF_IP6} ${NSA_LINKIP6}%${NSB_DEV} ${MCAST}%${NSB_DEV}
	do
		log_start
		run_cmd_nsb ${ping6} -c1 -w1 ${a}
		log_test_addr ${a} $? 0 "ping in"
	done

	a=${NSA_LO_IP6}
	log_start
	show_hint "Fails since loopback address is out of VRF scope"
	run_cmd_nsb ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in"

	#
	# local traffic, local address
	#
	for a in ${NSA_IP6} ${VRF_IP6} ::1
	do
		log_start
		show_hint "Source address should be ${a}"
		run_cmd ${ping6} -c1 -w1 -I ${VRF} ${a}
		log_test_addr ${a} $? 0 "ping local, VRF bind"
	done

	for a in ${NSA_IP6} ${NSA_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV}
	do
		log_start
		run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
		log_test_addr ${a} $? 0 "ping local, device bind"
	done

	# LLA to GUA - remove ipv6 global addresses from ns-B
	setup_cmd_nsb ip -6 addr del ${NSB_IP6}/64 dev ${NSB_DEV}
	setup_cmd_nsb ip -6 addr del ${NSB_LO_IP6}/128 dev lo
	setup_cmd_nsb ip -6 ro add ${NSA_IP6}/128 via ${NSA_LINKIP6} dev ${NSB_DEV}

	for a in ${NSA_IP6} ${VRF_IP6}
	do
		log_start
		run_cmd_nsb ${ping6} -c1 -w1 ${NSA_IP6}
		log_test_addr ${a} $? 0 "ping in, LLA to GUA"
	done

	setup_cmd_nsb ip -6 ro del ${NSA_IP6}/128 via ${NSA_LINKIP6} dev ${NSB_DEV}
	setup_cmd_nsb ip -6 addr add ${NSB_IP6}/64 dev ${NSB_DEV}
	setup_cmd_nsb ip -6 addr add ${NSB_LO_IP6}/128 dev lo

	#
	# ip rule blocks address
	#
	log_start
	setup_cmd ip -6 rule add pref 50 to ${NSB_LO_IP6} prohibit
	setup_cmd ip -6 rule add pref 51 from ${NSB_IP6} prohibit

	a=${NSB_LO_IP6}
	run_cmd ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, blocked by rule"

	log_start
	run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 2 "ping out, device bind, blocked by rule"

	a=${NSA_LO_IP6}
	log_start
	show_hint "Response lost due to ip rule"
	run_cmd_nsb ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 1 "ping in, blocked by rule"

	log_start
	setup_cmd ip -6 rule del pref 50 to ${NSB_LO_IP6} prohibit
	setup_cmd ip -6 rule del pref 51 from ${NSB_IP6} prohibit

	#
	# remove 'remote' routes; fallback to default
	#
	log_start
	setup_cmd ip -6 ro del ${NSB_LO_IP6} vrf ${VRF}

	a=${NSB_LO_IP6}
	run_cmd ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping out, unreachable route"

	log_start
	run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a}
	log_test_addr ${a} $? 2 "ping out, device bind, unreachable route"

	ip -netns ${NSB} -6 ro del ${NSA_LO_IP6}
	a=${NSA_LO_IP6}
	log_start
	run_cmd_nsb ${ping6} -c1 -w1 ${a}
	log_test_addr ${a} $? 2 "ping in, unreachable route"
}

ipv6_ping()
{
	log_section "IPv6 ping"

	log_subsection "No VRF"
	setup
	ipv6_ping_novrf

	log_subsection "With VRF"
	setup "yes"
	ipv6_ping_vrf
}

################################################################################
# usage

usage()
{
	cat <<EOF
usage: ${0##*/} OPTS

	-4          IPv4 tests only
	-6          IPv6 tests only
	-t <test>   Test name/set to run
	-p          Pause on fail
	-P          Pause after each test
	-v          Be verbose
EOF
}

################################################################################
# main

TESTS_IPV4="ipv4_ping ipv4_tcp"
TESTS_IPV6="ipv6_ping"
PAUSE_ON_FAIL=no
PAUSE=no

while getopts :46t:pPvh o
do
	case $o in
		4) TESTS=ipv4;;
		6) TESTS=ipv6;;
		t) TESTS=$OPTARG;;
		p) PAUSE_ON_FAIL=yes;;
		P) PAUSE=yes;;
		v) VERBOSE=1;;
		h) usage; exit 0;;
		*) usage; exit 1;;
	esac
done

# make sure we don't pause twice
[ "${PAUSE}" = "yes" ] && PAUSE_ON_FAIL=no

#
# show user test config
#
if [ -z "$TESTS" ]; then
	TESTS="$TESTS_IPV4 $TESTS_IPV6 $TESTS_OTHER"
elif [ "$TESTS" = "ipv4" ]; then
	TESTS="$TESTS_IPV4"
elif [ "$TESTS" = "ipv6" ]; then
	TESTS="$TESTS_IPV6"
fi

declare -i nfail=0
declare -i nsuccess=0

for t in $TESTS
do
	case $t in
	ipv4_ping|ping)  ipv4_ping;;
	ipv4_tcp|tcp)    ipv4_tcp;;

	ipv6_ping|ping6) ipv6_ping;;

	# setup namespaces and config, but do not run any tests
	setup)		 setup; exit 0;;
	vrf_setup)	 setup "yes"; exit 0;;

	help)            echo "Test names: $TESTS"; exit 0;;
	esac
done

cleanup 2>/dev/null

printf "\nTests passed: %3d\n" ${nsuccess}
printf "Tests failed: %3d\n"   ${nfail}