| #!/bin/bash |
| # SPDX-License-Identifier: GPL-2.0 |
| # |
| # Copyright (c) 2019 David Ahern <dsahern@gmail.com>. All rights reserved. |
| # |
| # IPv4 and IPv6 functional tests focusing on VRF and routing lookups |
| # for various permutations: |
| # 1. icmp, tcp, udp and netfilter |
| # 2. client, server, no-server |
| # 3. global address on interface |
| # 4. global address on 'lo' |
| # 5. remote and local traffic |
| # 6. VRF and non-VRF permutations |
| # |
| # Setup: |
| # ns-A | ns-B |
| # No VRF case: |
| # [ lo ] [ eth1 ]---|---[ eth1 ] [ lo ] |
| # remote address |
| # VRF case: |
| # [ red ]---[ eth1 ]---|---[ eth1 ] [ lo ] |
| # |
| # ns-A: |
| # eth1: 172.16.1.1/24, 2001:db8:1::1/64 |
| # lo: 127.0.0.1/8, ::1/128 |
| # 172.16.2.1/32, 2001:db8:2::1/128 |
| # red: 127.0.0.1/8, ::1/128 |
| # 172.16.3.1/32, 2001:db8:3::1/128 |
| # |
| # ns-B: |
| # eth1: 172.16.1.2/24, 2001:db8:1::2/64 |
| # lo2: 127.0.0.1/8, ::1/128 |
| # 172.16.2.2/32, 2001:db8:2::2/128 |
| # |
| # server / client nomenclature relative to ns-A |
| |
| VERBOSE=0 |
| |
| NSA_DEV=eth1 |
| NSB_DEV=eth1 |
| VRF=red |
| VRF_TABLE=1101 |
| |
| # IPv4 config |
| NSA_IP=172.16.1.1 |
| NSB_IP=172.16.1.2 |
| VRF_IP=172.16.3.1 |
| |
| # IPv6 config |
| NSA_IP6=2001:db8:1::1 |
| NSB_IP6=2001:db8:1::2 |
| VRF_IP6=2001:db8:3::1 |
| |
| NSA_LO_IP=172.16.2.1 |
| NSB_LO_IP=172.16.2.2 |
| NSA_LO_IP6=2001:db8:2::1 |
| NSB_LO_IP6=2001:db8:2::2 |
| |
| MCAST=ff02::1 |
| # set after namespace create |
| NSA_LINKIP6= |
| NSB_LINKIP6= |
| |
| NSA=ns-A |
| NSB=ns-B |
| |
| NSA_CMD="ip netns exec ${NSA}" |
| NSB_CMD="ip netns exec ${NSB}" |
| |
| which ping6 > /dev/null 2>&1 && ping6=$(which ping6) || ping6=$(which ping) |
| |
| ################################################################################ |
| # utilities |
| |
| log_test() |
| { |
| local rc=$1 |
| local expected=$2 |
| local msg="$3" |
| |
| [ "${VERBOSE}" = "1" ] && echo |
| |
| if [ ${rc} -eq ${expected} ]; then |
| nsuccess=$((nsuccess+1)) |
| printf "TEST: %-70s [ OK ]\n" "${msg}" |
| else |
| nfail=$((nfail+1)) |
| printf "TEST: %-70s [FAIL]\n" "${msg}" |
| if [ "${PAUSE_ON_FAIL}" = "yes" ]; then |
| echo |
| echo "hit enter to continue, 'q' to quit" |
| read a |
| [ "$a" = "q" ] && exit 1 |
| fi |
| fi |
| |
| if [ "${PAUSE}" = "yes" ]; then |
| echo |
| echo "hit enter to continue, 'q' to quit" |
| read a |
| [ "$a" = "q" ] && exit 1 |
| fi |
| |
| kill_procs |
| } |
| |
| log_test_addr() |
| { |
| local addr=$1 |
| local rc=$2 |
| local expected=$3 |
| local msg="$4" |
| local astr |
| |
| astr=$(addr2str ${addr}) |
| log_test $rc $expected "$msg - ${astr}" |
| } |
| |
| log_section() |
| { |
| echo |
| echo "###########################################################################" |
| echo "$*" |
| echo "###########################################################################" |
| echo |
| } |
| |
| log_subsection() |
| { |
| echo |
| echo "#################################################################" |
| echo "$*" |
| echo |
| } |
| |
| log_start() |
| { |
| # make sure we have no test instances running |
| kill_procs |
| |
| if [ "${VERBOSE}" = "1" ]; then |
| echo |
| echo "#######################################################" |
| fi |
| } |
| |
| log_debug() |
| { |
| if [ "${VERBOSE}" = "1" ]; then |
| echo |
| echo "$*" |
| echo |
| fi |
| } |
| |
| show_hint() |
| { |
| if [ "${VERBOSE}" = "1" ]; then |
| echo "HINT: $*" |
| echo |
| fi |
| } |
| |
| kill_procs() |
| { |
| killall nettest ping ping6 >/dev/null 2>&1 |
| sleep 1 |
| } |
| |
| do_run_cmd() |
| { |
| local cmd="$*" |
| local out |
| |
| if [ "$VERBOSE" = "1" ]; then |
| echo "COMMAND: ${cmd}" |
| fi |
| |
| out=$($cmd 2>&1) |
| rc=$? |
| if [ "$VERBOSE" = "1" -a -n "$out" ]; then |
| echo "$out" |
| fi |
| |
| return $rc |
| } |
| |
| run_cmd() |
| { |
| do_run_cmd ${NSA_CMD} $* |
| } |
| |
| run_cmd_nsb() |
| { |
| do_run_cmd ${NSB_CMD} $* |
| } |
| |
| setup_cmd() |
| { |
| local cmd="$*" |
| local rc |
| |
| run_cmd ${cmd} |
| rc=$? |
| if [ $rc -ne 0 ]; then |
| # show user the command if not done so already |
| if [ "$VERBOSE" = "0" ]; then |
| echo "setup command: $cmd" |
| fi |
| echo "failed. stopping tests" |
| if [ "${PAUSE_ON_FAIL}" = "yes" ]; then |
| echo |
| echo "hit enter to continue" |
| read a |
| fi |
| exit $rc |
| fi |
| } |
| |
| setup_cmd_nsb() |
| { |
| local cmd="$*" |
| local rc |
| |
| run_cmd_nsb ${cmd} |
| rc=$? |
| if [ $rc -ne 0 ]; then |
| # show user the command if not done so already |
| if [ "$VERBOSE" = "0" ]; then |
| echo "setup command: $cmd" |
| fi |
| echo "failed. stopping tests" |
| if [ "${PAUSE_ON_FAIL}" = "yes" ]; then |
| echo |
| echo "hit enter to continue" |
| read a |
| fi |
| exit $rc |
| fi |
| } |
| |
| # set sysctl values in NS-A |
| set_sysctl() |
| { |
| echo "SYSCTL: $*" |
| echo |
| run_cmd sysctl -q -w $* |
| } |
| |
| ################################################################################ |
| # Setup for tests |
| |
| addr2str() |
| { |
| case "$1" in |
| 127.0.0.1) echo "loopback";; |
| ::1) echo "IPv6 loopback";; |
| |
| ${NSA_IP}) echo "ns-A IP";; |
| ${NSA_IP6}) echo "ns-A IPv6";; |
| ${NSA_LO_IP}) echo "ns-A loopback IP";; |
| ${NSA_LO_IP6}) echo "ns-A loopback IPv6";; |
| ${NSA_LINKIP6}|${NSA_LINKIP6}%*) echo "ns-A IPv6 LLA";; |
| |
| ${NSB_IP}) echo "ns-B IP";; |
| ${NSB_IP6}) echo "ns-B IPv6";; |
| ${NSB_LO_IP}) echo "ns-B loopback IP";; |
| ${NSB_LO_IP6}) echo "ns-B loopback IPv6";; |
| ${NSB_LINKIP6}|${NSB_LINKIP6}%*) echo "ns-B IPv6 LLA";; |
| |
| ${VRF_IP}) echo "VRF IP";; |
| ${VRF_IP6}) echo "VRF IPv6";; |
| |
| ${MCAST}%*) echo "multicast IP";; |
| |
| *) echo "unknown";; |
| esac |
| } |
| |
| get_linklocal() |
| { |
| local ns=$1 |
| local dev=$2 |
| local addr |
| |
| addr=$(ip -netns ${ns} -6 -br addr show dev ${dev} | \ |
| awk '{ |
| for (i = 3; i <= NF; ++i) { |
| if ($i ~ /^fe80/) |
| print $i |
| } |
| }' |
| ) |
| addr=${addr/\/*} |
| |
| [ -z "$addr" ] && return 1 |
| |
| echo $addr |
| |
| return 0 |
| } |
| |
| ################################################################################ |
| # create namespaces and vrf |
| |
| create_vrf() |
| { |
| local ns=$1 |
| local vrf=$2 |
| local table=$3 |
| local addr=$4 |
| local addr6=$5 |
| |
| ip -netns ${ns} link add ${vrf} type vrf table ${table} |
| ip -netns ${ns} link set ${vrf} up |
| ip -netns ${ns} route add vrf ${vrf} unreachable default metric 8192 |
| ip -netns ${ns} -6 route add vrf ${vrf} unreachable default metric 8192 |
| |
| ip -netns ${ns} addr add 127.0.0.1/8 dev ${vrf} |
| ip -netns ${ns} -6 addr add ::1 dev ${vrf} nodad |
| if [ "${addr}" != "-" ]; then |
| ip -netns ${ns} addr add dev ${vrf} ${addr} |
| fi |
| if [ "${addr6}" != "-" ]; then |
| ip -netns ${ns} -6 addr add dev ${vrf} ${addr6} |
| fi |
| |
| ip -netns ${ns} ru del pref 0 |
| ip -netns ${ns} ru add pref 32765 from all lookup local |
| ip -netns ${ns} -6 ru del pref 0 |
| ip -netns ${ns} -6 ru add pref 32765 from all lookup local |
| } |
| |
| create_ns() |
| { |
| local ns=$1 |
| local addr=$2 |
| local addr6=$3 |
| |
| ip netns add ${ns} |
| |
| ip -netns ${ns} link set lo up |
| if [ "${addr}" != "-" ]; then |
| ip -netns ${ns} addr add dev lo ${addr} |
| fi |
| if [ "${addr6}" != "-" ]; then |
| ip -netns ${ns} -6 addr add dev lo ${addr6} |
| fi |
| |
| ip -netns ${ns} ro add unreachable default metric 8192 |
| ip -netns ${ns} -6 ro add unreachable default metric 8192 |
| |
| ip netns exec ${ns} sysctl -qw net.ipv4.ip_forward=1 |
| ip netns exec ${ns} sysctl -qw net.ipv6.conf.all.keep_addr_on_down=1 |
| ip netns exec ${ns} sysctl -qw net.ipv6.conf.all.forwarding=1 |
| ip netns exec ${ns} sysctl -qw net.ipv6.conf.default.forwarding=1 |
| } |
| |
| # create veth pair to connect namespaces and apply addresses. |
| connect_ns() |
| { |
| local ns1=$1 |
| local ns1_dev=$2 |
| local ns1_addr=$3 |
| local ns1_addr6=$4 |
| local ns2=$5 |
| local ns2_dev=$6 |
| local ns2_addr=$7 |
| local ns2_addr6=$8 |
| |
| ip -netns ${ns1} li add ${ns1_dev} type veth peer name tmp |
| ip -netns ${ns1} li set ${ns1_dev} up |
| ip -netns ${ns1} li set tmp netns ${ns2} name ${ns2_dev} |
| ip -netns ${ns2} li set ${ns2_dev} up |
| |
| if [ "${ns1_addr}" != "-" ]; then |
| ip -netns ${ns1} addr add dev ${ns1_dev} ${ns1_addr} |
| ip -netns ${ns2} addr add dev ${ns2_dev} ${ns2_addr} |
| fi |
| |
| if [ "${ns1_addr6}" != "-" ]; then |
| ip -netns ${ns1} addr add dev ${ns1_dev} ${ns1_addr6} |
| ip -netns ${ns2} addr add dev ${ns2_dev} ${ns2_addr6} |
| fi |
| } |
| |
| cleanup() |
| { |
| # explicit cleanups to check those code paths |
| ip netns | grep -q ${NSA} |
| if [ $? -eq 0 ]; then |
| ip -netns ${NSA} link delete ${VRF} |
| ip -netns ${NSA} ro flush table ${VRF_TABLE} |
| |
| ip -netns ${NSA} addr flush dev ${NSA_DEV} |
| ip -netns ${NSA} -6 addr flush dev ${NSA_DEV} |
| ip -netns ${NSA} link set dev ${NSA_DEV} down |
| ip -netns ${NSA} link del dev ${NSA_DEV} |
| |
| ip netns del ${NSA} |
| fi |
| |
| ip netns del ${NSB} |
| } |
| |
| setup() |
| { |
| local with_vrf=${1} |
| |
| # make sure we are starting with a clean slate |
| kill_procs |
| cleanup 2>/dev/null |
| |
| log_debug "Configuring network namespaces" |
| set -e |
| |
| create_ns ${NSA} ${NSA_LO_IP}/32 ${NSA_LO_IP6}/128 |
| create_ns ${NSB} ${NSB_LO_IP}/32 ${NSB_LO_IP6}/128 |
| connect_ns ${NSA} ${NSA_DEV} ${NSA_IP}/24 ${NSA_IP6}/64 \ |
| ${NSB} ${NSB_DEV} ${NSB_IP}/24 ${NSB_IP6}/64 |
| |
| NSA_LINKIP6=$(get_linklocal ${NSA} ${NSA_DEV}) |
| NSB_LINKIP6=$(get_linklocal ${NSB} ${NSB_DEV}) |
| |
| # tell ns-A how to get to remote addresses of ns-B |
| if [ "${with_vrf}" = "yes" ]; then |
| create_vrf ${NSA} ${VRF} ${VRF_TABLE} ${VRF_IP} ${VRF_IP6} |
| |
| ip -netns ${NSA} link set dev ${NSA_DEV} vrf ${VRF} |
| ip -netns ${NSA} ro add vrf ${VRF} ${NSB_LO_IP}/32 via ${NSB_IP} dev ${NSA_DEV} |
| ip -netns ${NSA} -6 ro add vrf ${VRF} ${NSB_LO_IP6}/128 via ${NSB_IP6} dev ${NSA_DEV} |
| |
| ip -netns ${NSB} ro add ${VRF_IP}/32 via ${NSA_IP} dev ${NSB_DEV} |
| ip -netns ${NSB} -6 ro add ${VRF_IP6}/128 via ${NSA_IP6} dev ${NSB_DEV} |
| else |
| ip -netns ${NSA} ro add ${NSB_LO_IP}/32 via ${NSB_IP} dev ${NSA_DEV} |
| ip -netns ${NSA} ro add ${NSB_LO_IP6}/128 via ${NSB_IP6} dev ${NSA_DEV} |
| fi |
| |
| |
| # tell ns-B how to get to remote addresses of ns-A |
| ip -netns ${NSB} ro add ${NSA_LO_IP}/32 via ${NSA_IP} dev ${NSB_DEV} |
| ip -netns ${NSB} ro add ${NSA_LO_IP6}/128 via ${NSA_IP6} dev ${NSB_DEV} |
| |
| set +e |
| |
| sleep 1 |
| } |
| |
| ################################################################################ |
| # IPv4 |
| |
| ipv4_ping_novrf() |
| { |
| local a |
| |
| # |
| # out |
| # |
| for a in ${NSB_IP} ${NSB_LO_IP} |
| do |
| log_start |
| run_cmd ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping out" |
| |
| log_start |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping out, device bind" |
| |
| log_start |
| run_cmd ping -c1 -w1 -I ${NSA_LO_IP} ${a} |
| log_test_addr ${a} $? 0 "ping out, address bind" |
| done |
| |
| # |
| # in |
| # |
| for a in ${NSA_IP} ${NSA_LO_IP} |
| do |
| log_start |
| run_cmd_nsb ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping in" |
| done |
| |
| # |
| # local traffic |
| # |
| for a in ${NSA_IP} ${NSA_LO_IP} 127.0.0.1 |
| do |
| log_start |
| run_cmd ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping local" |
| done |
| |
| # |
| # local traffic, socket bound to device |
| # |
| # address on device |
| a=${NSA_IP} |
| log_start |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping local, device bind" |
| |
| # loopback addresses not reachable from device bind |
| # fails in a really weird way though because ipv4 special cases |
| # route lookups with oif set. |
| for a in ${NSA_LO_IP} 127.0.0.1 |
| do |
| log_start |
| show_hint "Fails since address on loopback device is out of device scope" |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 1 "ping local, device bind" |
| done |
| |
| # |
| # ip rule blocks reachability to remote address |
| # |
| log_start |
| setup_cmd ip rule add pref 32765 from all lookup local |
| setup_cmd ip rule del pref 0 from all lookup local |
| setup_cmd ip rule add pref 50 to ${NSB_LO_IP} prohibit |
| setup_cmd ip rule add pref 51 from ${NSB_IP} prohibit |
| |
| a=${NSB_LO_IP} |
| run_cmd ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, blocked by rule" |
| |
| # NOTE: ipv4 actually allows the lookup to fail and yet still create |
| # a viable rtable if the oif (e.g., bind to device) is set, so this |
| # case succeeds despite the rule |
| # run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| |
| a=${NSA_LO_IP} |
| log_start |
| show_hint "Response generates ICMP (or arp request is ignored) due to ip rule" |
| run_cmd_nsb ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in, blocked by rule" |
| |
| [ "$VERBOSE" = "1" ] && echo |
| setup_cmd ip rule del pref 32765 from all lookup local |
| setup_cmd ip rule add pref 0 from all lookup local |
| setup_cmd ip rule del pref 50 to ${NSB_LO_IP} prohibit |
| setup_cmd ip rule del pref 51 from ${NSB_IP} prohibit |
| |
| # |
| # route blocks reachability to remote address |
| # |
| log_start |
| setup_cmd ip route replace unreachable ${NSB_LO_IP} |
| setup_cmd ip route replace unreachable ${NSB_IP} |
| |
| a=${NSB_LO_IP} |
| run_cmd ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, blocked by route" |
| |
| # NOTE: ipv4 actually allows the lookup to fail and yet still create |
| # a viable rtable if the oif (e.g., bind to device) is set, so this |
| # case succeeds despite not having a route for the address |
| # run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| |
| a=${NSA_LO_IP} |
| log_start |
| show_hint "Response is dropped (or arp request is ignored) due to ip route" |
| run_cmd_nsb ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in, blocked by route" |
| |
| # |
| # remove 'remote' routes; fallback to default |
| # |
| log_start |
| setup_cmd ip ro del ${NSB_LO_IP} |
| |
| a=${NSB_LO_IP} |
| run_cmd ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, unreachable default route" |
| |
| # NOTE: ipv4 actually allows the lookup to fail and yet still create |
| # a viable rtable if the oif (e.g., bind to device) is set, so this |
| # case succeeds despite not having a route for the address |
| # run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| } |
| |
| ipv4_ping_vrf() |
| { |
| local a |
| |
| # should default on; does not exist on older kernels |
| set_sysctl net.ipv4.raw_l3mdev_accept=1 2>/dev/null |
| |
| # |
| # out |
| # |
| for a in ${NSB_IP} ${NSB_LO_IP} |
| do |
| log_start |
| run_cmd ping -c1 -w1 -I ${VRF} ${a} |
| log_test_addr ${a} $? 0 "ping out, VRF bind" |
| |
| log_start |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping out, device bind" |
| |
| log_start |
| run_cmd ip vrf exec ${VRF} ping -c1 -w1 -I ${NSA_IP} ${a} |
| log_test_addr ${a} $? 0 "ping out, vrf device + dev address bind" |
| |
| log_start |
| run_cmd ip vrf exec ${VRF} ping -c1 -w1 -I ${VRF_IP} ${a} |
| log_test_addr ${a} $? 0 "ping out, vrf device + vrf address bind" |
| done |
| |
| # |
| # in |
| # |
| for a in ${NSA_IP} ${VRF_IP} |
| do |
| log_start |
| run_cmd_nsb ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping in" |
| done |
| |
| # |
| # local traffic, local address |
| # |
| for a in ${NSA_IP} ${VRF_IP} 127.0.0.1 |
| do |
| log_start |
| show_hint "Source address should be ${a}" |
| run_cmd ping -c1 -w1 -I ${VRF} ${a} |
| log_test_addr ${a} $? 0 "ping local, VRF bind" |
| done |
| |
| # |
| # local traffic, socket bound to device |
| # |
| # address on device |
| a=${NSA_IP} |
| log_start |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping local, device bind" |
| |
| # vrf device is out of scope |
| for a in ${VRF_IP} 127.0.0.1 |
| do |
| log_start |
| show_hint "Fails since address on vrf device is out of device scope" |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 1 "ping local, device bind" |
| done |
| |
| # |
| # ip rule blocks address |
| # |
| log_start |
| setup_cmd ip rule add pref 50 to ${NSB_LO_IP} prohibit |
| setup_cmd ip rule add pref 51 from ${NSB_IP} prohibit |
| |
| a=${NSB_LO_IP} |
| run_cmd ping -c1 -w1 -I ${VRF} ${a} |
| log_test_addr ${a} $? 2 "ping out, vrf bind, blocked by rule" |
| |
| log_start |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping out, device bind, blocked by rule" |
| |
| a=${NSA_LO_IP} |
| log_start |
| show_hint "Response lost due to ip rule" |
| run_cmd_nsb ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in, blocked by rule" |
| |
| [ "$VERBOSE" = "1" ] && echo |
| setup_cmd ip rule del pref 50 to ${NSB_LO_IP} prohibit |
| setup_cmd ip rule del pref 51 from ${NSB_IP} prohibit |
| |
| # |
| # remove 'remote' routes; fallback to default |
| # |
| log_start |
| setup_cmd ip ro del vrf ${VRF} ${NSB_LO_IP} |
| |
| a=${NSB_LO_IP} |
| run_cmd ping -c1 -w1 -I ${VRF} ${a} |
| log_test_addr ${a} $? 2 "ping out, vrf bind, unreachable route" |
| |
| log_start |
| run_cmd ping -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping out, device bind, unreachable route" |
| |
| a=${NSA_LO_IP} |
| log_start |
| show_hint "Response lost by unreachable route" |
| run_cmd_nsb ping -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in, unreachable route" |
| } |
| |
| ipv4_ping() |
| { |
| log_section "IPv4 ping" |
| |
| log_subsection "No VRF" |
| setup |
| set_sysctl net.ipv4.raw_l3mdev_accept=0 2>/dev/null |
| ipv4_ping_novrf |
| setup |
| set_sysctl net.ipv4.raw_l3mdev_accept=1 2>/dev/null |
| ipv4_ping_novrf |
| |
| log_subsection "With VRF" |
| setup "yes" |
| ipv4_ping_vrf |
| } |
| |
| ################################################################################ |
| # IPv4 TCP |
| |
| ipv4_tcp_novrf() |
| { |
| local a |
| |
| # |
| # server tests |
| # |
| for a in ${NSA_IP} ${NSA_LO_IP} |
| do |
| log_start |
| run_cmd nettest -s & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 0 "Global server" |
| done |
| |
| a=${NSA_IP} |
| log_start |
| run_cmd nettest -s -d ${NSA_DEV} & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 0 "Device server" |
| |
| # verify TCP reset sent and received |
| for a in ${NSA_IP} ${NSA_LO_IP} |
| do |
| log_start |
| show_hint "Should fail 'Connection refused' since there is no server" |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 1 "No server" |
| done |
| |
| # |
| # client |
| # |
| for a in ${NSB_IP} ${NSB_LO_IP} |
| do |
| log_start |
| run_cmd_nsb nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -0 ${NSA_IP} |
| log_test_addr ${a} $? 0 "Client" |
| |
| log_start |
| run_cmd_nsb nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${NSA_DEV} |
| log_test_addr ${a} $? 0 "Client, device bind" |
| |
| log_start |
| show_hint "Should fail 'Connection refused'" |
| run_cmd nettest -r ${a} |
| log_test_addr ${a} $? 1 "No server, unbound client" |
| |
| log_start |
| show_hint "Should fail 'Connection refused'" |
| run_cmd nettest -r ${a} -d ${NSA_DEV} |
| log_test_addr ${a} $? 1 "No server, device client" |
| done |
| |
| # |
| # local address tests |
| # |
| for a in ${NSA_IP} ${NSA_LO_IP} 127.0.0.1 |
| do |
| log_start |
| run_cmd nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -0 ${a} -1 ${a} |
| log_test_addr ${a} $? 0 "Global server, local connection" |
| done |
| |
| a=${NSA_IP} |
| log_start |
| run_cmd nettest -s -d ${NSA_DEV} & |
| sleep 1 |
| run_cmd nettest -r ${a} -0 ${a} |
| log_test_addr ${a} $? 0 "Device server, unbound client, local connection" |
| |
| for a in ${NSA_LO_IP} 127.0.0.1 |
| do |
| log_start |
| show_hint "Should fail 'Connection refused' since addresses on loopback are out of device scope" |
| run_cmd nettest -s -d ${NSA_DEV} & |
| sleep 1 |
| run_cmd nettest -r ${a} |
| log_test_addr ${a} $? 1 "Device server, unbound client, local connection" |
| done |
| |
| a=${NSA_IP} |
| log_start |
| run_cmd nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -0 ${a} -d ${NSA_DEV} |
| log_test_addr ${a} $? 0 "Global server, device client, local connection" |
| |
| for a in ${NSA_LO_IP} 127.0.0.1 |
| do |
| log_start |
| show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope" |
| run_cmd nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${NSA_DEV} |
| log_test_addr ${a} $? 1 "Global server, device client, local connection" |
| done |
| |
| a=${NSA_IP} |
| log_start |
| run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} & |
| sleep 1 |
| run_cmd nettest -d ${NSA_DEV} -r ${a} -0 ${a} |
| log_test_addr ${a} $? 0 "Device server, device client, local connection" |
| |
| log_start |
| show_hint "Should fail 'Connection refused'" |
| run_cmd nettest -d ${NSA_DEV} -r ${a} |
| log_test_addr ${a} $? 1 "No server, device client, local conn" |
| } |
| |
| ipv4_tcp_vrf() |
| { |
| local a |
| |
| # disable global server |
| log_subsection "Global server disabled" |
| |
| set_sysctl net.ipv4.tcp_l3mdev_accept=0 |
| |
| # |
| # server tests |
| # |
| for a in ${NSA_IP} ${VRF_IP} |
| do |
| log_start |
| show_hint "Should fail 'Connection refused' since global server with VRF is disabled" |
| run_cmd nettest -s & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 1 "Global server" |
| |
| log_start |
| run_cmd nettest -s -d ${VRF} -2 ${VRF} & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 0 "VRF server" |
| |
| log_start |
| run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 0 "Device server" |
| |
| # verify TCP reset received |
| log_start |
| show_hint "Should fail 'Connection refused' since there is no server" |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 1 "No server" |
| done |
| |
| # local address tests |
| # (${VRF_IP} and 127.0.0.1 both timeout) |
| a=${NSA_IP} |
| log_start |
| show_hint "Should fail 'Connection refused' since global server with VRF is disabled" |
| run_cmd nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${NSA_DEV} |
| log_test_addr ${a} $? 1 "Global server, local connection" |
| |
| # |
| # enable VRF global server |
| # |
| log_subsection "VRF Global server enabled" |
| set_sysctl net.ipv4.tcp_l3mdev_accept=1 |
| |
| for a in ${NSA_IP} ${VRF_IP} |
| do |
| log_start |
| show_hint "client socket should be bound to VRF" |
| run_cmd nettest -s -2 ${VRF} & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 0 "Global server" |
| |
| log_start |
| show_hint "client socket should be bound to VRF" |
| run_cmd nettest -s -d ${VRF} -2 ${VRF} & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 0 "VRF server" |
| |
| # verify TCP reset received |
| log_start |
| show_hint "Should fail 'Connection refused'" |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 1 "No server" |
| done |
| |
| a=${NSA_IP} |
| log_start |
| show_hint "client socket should be bound to device" |
| run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} & |
| sleep 1 |
| run_cmd_nsb nettest -r ${a} |
| log_test_addr ${a} $? 0 "Device server" |
| |
| # local address tests |
| for a in ${NSA_IP} ${VRF_IP} |
| do |
| log_start |
| show_hint "Should fail 'No route to host' since client is not bound to VRF" |
| run_cmd nettest -s -2 ${VRF} & |
| sleep 1 |
| run_cmd nettest -r ${a} |
| log_test_addr ${a} $? 1 "Global server, local connection" |
| done |
| |
| # |
| # client |
| # |
| for a in ${NSB_IP} ${NSB_LO_IP} |
| do |
| log_start |
| run_cmd_nsb nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${VRF} |
| log_test_addr ${a} $? 0 "Client, VRF bind" |
| |
| log_start |
| run_cmd_nsb nettest -s & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${NSA_DEV} |
| log_test_addr ${a} $? 0 "Client, device bind" |
| |
| log_start |
| show_hint "Should fail 'Connection refused'" |
| run_cmd nettest -r ${a} -d ${VRF} |
| log_test_addr ${a} $? 1 "No server, VRF client" |
| |
| log_start |
| show_hint "Should fail 'Connection refused'" |
| run_cmd nettest -r ${a} -d ${NSA_DEV} |
| log_test_addr ${a} $? 1 "No server, device client" |
| done |
| |
| for a in ${NSA_IP} ${VRF_IP} 127.0.0.1 |
| do |
| log_start |
| run_cmd nettest -s -d ${VRF} -2 ${VRF} & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${VRF} -0 ${a} |
| log_test_addr ${a} $? 0 "VRF server, VRF client, local connection" |
| done |
| |
| a=${NSA_IP} |
| log_start |
| run_cmd nettest -s -d ${VRF} -2 ${VRF} & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${NSA_DEV} -0 ${a} |
| log_test_addr ${a} $? 0 "VRF server, device client, local connection" |
| |
| log_start |
| show_hint "Should fail 'No route to host' since client is out of VRF scope" |
| run_cmd nettest -s -d ${VRF} & |
| sleep 1 |
| run_cmd nettest -r ${a} |
| log_test_addr ${a} $? 1 "VRF server, unbound client, local connection" |
| |
| log_start |
| run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${VRF} -0 ${a} |
| log_test_addr ${a} $? 0 "Device server, VRF client, local connection" |
| |
| log_start |
| run_cmd nettest -s -d ${NSA_DEV} -2 ${NSA_DEV} & |
| sleep 1 |
| run_cmd nettest -r ${a} -d ${NSA_DEV} -0 ${a} |
| log_test_addr ${a} $? 0 "Device server, device client, local connection" |
| } |
| |
| ipv4_tcp() |
| { |
| log_section "IPv4/TCP" |
| |
| which nettest >/dev/null |
| if [ $? -ne 0 ]; then |
| log_error "nettest not found; skipping tests" |
| return |
| fi |
| |
| log_subsection "No VRF" |
| setup |
| |
| # tcp_l3mdev_accept should have no affect without VRF; |
| # run tests with it enabled and disabled to verify |
| log_subsection "tcp_l3mdev_accept disabled" |
| set_sysctl net.ipv4.tcp_l3mdev_accept=0 |
| ipv4_tcp_novrf |
| log_subsection "tcp_l3mdev_accept enabled" |
| set_sysctl net.ipv4.tcp_l3mdev_accept=1 |
| ipv4_tcp_novrf |
| |
| log_subsection "With VRF" |
| setup "yes" |
| ipv4_tcp_vrf |
| } |
| |
| ################################################################################ |
| # IPv6 |
| |
| ipv6_ping_novrf() |
| { |
| local a |
| |
| # should not have an impact, but make a known state |
| set_sysctl net.ipv4.raw_l3mdev_accept=0 2>/dev/null |
| |
| # |
| # out |
| # |
| for a in ${NSB_IP6} ${NSB_LO_IP6} ${NSB_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV} |
| do |
| log_start |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping out" |
| done |
| |
| for a in ${NSB_IP6} ${NSB_LO_IP6} |
| do |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping out, device bind" |
| |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_LO_IP6} ${a} |
| log_test_addr ${a} $? 0 "ping out, loopback address bind" |
| done |
| |
| # |
| # in |
| # |
| for a in ${NSA_IP6} ${NSA_LO_IP6} ${NSA_LINKIP6}%${NSB_DEV} ${MCAST}%${NSB_DEV} |
| do |
| log_start |
| run_cmd_nsb ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping in" |
| done |
| |
| # |
| # local traffic, local address |
| # |
| for a in ${NSA_IP6} ${NSA_LO_IP6} ::1 ${NSA_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV} |
| do |
| log_start |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping local, no bind" |
| done |
| |
| for a in ${NSA_IP6} ${NSA_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV} |
| do |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping local, device bind" |
| done |
| |
| for a in ${NSA_LO_IP6} ::1 |
| do |
| log_start |
| show_hint "Fails since address on loopback is out of device scope" |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping local, device bind" |
| done |
| |
| # |
| # ip rule blocks address |
| # |
| log_start |
| setup_cmd ip -6 rule add pref 32765 from all lookup local |
| setup_cmd ip -6 rule del pref 0 from all lookup local |
| setup_cmd ip -6 rule add pref 50 to ${NSB_LO_IP6} prohibit |
| setup_cmd ip -6 rule add pref 51 from ${NSB_IP6} prohibit |
| |
| a=${NSB_LO_IP6} |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, blocked by rule" |
| |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping out, device bind, blocked by rule" |
| |
| a=${NSA_LO_IP6} |
| log_start |
| show_hint "Response lost due to ip rule" |
| run_cmd_nsb ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in, blocked by rule" |
| |
| setup_cmd ip -6 rule add pref 0 from all lookup local |
| setup_cmd ip -6 rule del pref 32765 from all lookup local |
| setup_cmd ip -6 rule del pref 50 to ${NSB_LO_IP6} prohibit |
| setup_cmd ip -6 rule del pref 51 from ${NSB_IP6} prohibit |
| |
| # |
| # route blocks reachability to remote address |
| # |
| log_start |
| setup_cmd ip -6 route del ${NSB_LO_IP6} |
| setup_cmd ip -6 route add unreachable ${NSB_LO_IP6} metric 10 |
| setup_cmd ip -6 route add unreachable ${NSB_IP6} metric 10 |
| |
| a=${NSB_LO_IP6} |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, blocked by route" |
| |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping out, device bind, blocked by route" |
| |
| a=${NSA_LO_IP6} |
| log_start |
| show_hint "Response lost due to ip route" |
| run_cmd_nsb ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in, blocked by route" |
| |
| |
| # |
| # remove 'remote' routes; fallback to default |
| # |
| log_start |
| setup_cmd ip -6 ro del unreachable ${NSB_LO_IP6} |
| setup_cmd ip -6 ro del unreachable ${NSB_IP6} |
| |
| a=${NSB_LO_IP6} |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, unreachable route" |
| |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping out, device bind, unreachable route" |
| } |
| |
| ipv6_ping_vrf() |
| { |
| local a |
| |
| # should default on; does not exist on older kernels |
| set_sysctl net.ipv4.raw_l3mdev_accept=1 2>/dev/null |
| |
| # |
| # out |
| # |
| for a in ${NSB_IP6} ${NSB_LO_IP6} |
| do |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${VRF} ${a} |
| log_test_addr ${a} $? 0 "ping out, VRF bind" |
| done |
| |
| for a in ${NSB_LINKIP6}%${VRF} ${MCAST}%${VRF} |
| do |
| log_start |
| show_hint "Fails since VRF device does not support linklocal or multicast" |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, VRF bind" |
| done |
| |
| for a in ${NSB_IP6} ${NSB_LO_IP6} ${NSB_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV} |
| do |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping out, device bind" |
| done |
| |
| for a in ${NSB_IP6} ${NSB_LO_IP6} ${NSB_LINKIP6}%${NSA_DEV} |
| do |
| log_start |
| run_cmd ip vrf exec ${VRF} ${ping6} -c1 -w1 -I ${VRF_IP6} ${a} |
| log_test_addr ${a} $? 0 "ping out, vrf device+address bind" |
| done |
| |
| # |
| # in |
| # |
| for a in ${NSA_IP6} ${VRF_IP6} ${NSA_LINKIP6}%${NSB_DEV} ${MCAST}%${NSB_DEV} |
| do |
| log_start |
| run_cmd_nsb ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 0 "ping in" |
| done |
| |
| a=${NSA_LO_IP6} |
| log_start |
| show_hint "Fails since loopback address is out of VRF scope" |
| run_cmd_nsb ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in" |
| |
| # |
| # local traffic, local address |
| # |
| for a in ${NSA_IP6} ${VRF_IP6} ::1 |
| do |
| log_start |
| show_hint "Source address should be ${a}" |
| run_cmd ${ping6} -c1 -w1 -I ${VRF} ${a} |
| log_test_addr ${a} $? 0 "ping local, VRF bind" |
| done |
| |
| for a in ${NSA_IP6} ${NSA_LINKIP6}%${NSA_DEV} ${MCAST}%${NSA_DEV} |
| do |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 0 "ping local, device bind" |
| done |
| |
| # LLA to GUA - remove ipv6 global addresses from ns-B |
| setup_cmd_nsb ip -6 addr del ${NSB_IP6}/64 dev ${NSB_DEV} |
| setup_cmd_nsb ip -6 addr del ${NSB_LO_IP6}/128 dev lo |
| setup_cmd_nsb ip -6 ro add ${NSA_IP6}/128 via ${NSA_LINKIP6} dev ${NSB_DEV} |
| |
| for a in ${NSA_IP6} ${VRF_IP6} |
| do |
| log_start |
| run_cmd_nsb ${ping6} -c1 -w1 ${NSA_IP6} |
| log_test_addr ${a} $? 0 "ping in, LLA to GUA" |
| done |
| |
| setup_cmd_nsb ip -6 ro del ${NSA_IP6}/128 via ${NSA_LINKIP6} dev ${NSB_DEV} |
| setup_cmd_nsb ip -6 addr add ${NSB_IP6}/64 dev ${NSB_DEV} |
| setup_cmd_nsb ip -6 addr add ${NSB_LO_IP6}/128 dev lo |
| |
| # |
| # ip rule blocks address |
| # |
| log_start |
| setup_cmd ip -6 rule add pref 50 to ${NSB_LO_IP6} prohibit |
| setup_cmd ip -6 rule add pref 51 from ${NSB_IP6} prohibit |
| |
| a=${NSB_LO_IP6} |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, blocked by rule" |
| |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping out, device bind, blocked by rule" |
| |
| a=${NSA_LO_IP6} |
| log_start |
| show_hint "Response lost due to ip rule" |
| run_cmd_nsb ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 1 "ping in, blocked by rule" |
| |
| log_start |
| setup_cmd ip -6 rule del pref 50 to ${NSB_LO_IP6} prohibit |
| setup_cmd ip -6 rule del pref 51 from ${NSB_IP6} prohibit |
| |
| # |
| # remove 'remote' routes; fallback to default |
| # |
| log_start |
| setup_cmd ip -6 ro del ${NSB_LO_IP6} vrf ${VRF} |
| |
| a=${NSB_LO_IP6} |
| run_cmd ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping out, unreachable route" |
| |
| log_start |
| run_cmd ${ping6} -c1 -w1 -I ${NSA_DEV} ${a} |
| log_test_addr ${a} $? 2 "ping out, device bind, unreachable route" |
| |
| ip -netns ${NSB} -6 ro del ${NSA_LO_IP6} |
| a=${NSA_LO_IP6} |
| log_start |
| run_cmd_nsb ${ping6} -c1 -w1 ${a} |
| log_test_addr ${a} $? 2 "ping in, unreachable route" |
| } |
| |
| ipv6_ping() |
| { |
| log_section "IPv6 ping" |
| |
| log_subsection "No VRF" |
| setup |
| ipv6_ping_novrf |
| |
| log_subsection "With VRF" |
| setup "yes" |
| ipv6_ping_vrf |
| } |
| |
| ################################################################################ |
| # usage |
| |
| usage() |
| { |
| cat <<EOF |
| usage: ${0##*/} OPTS |
| |
| -4 IPv4 tests only |
| -6 IPv6 tests only |
| -t <test> Test name/set to run |
| -p Pause on fail |
| -P Pause after each test |
| -v Be verbose |
| EOF |
| } |
| |
| ################################################################################ |
| # main |
| |
| TESTS_IPV4="ipv4_ping ipv4_tcp" |
| TESTS_IPV6="ipv6_ping" |
| PAUSE_ON_FAIL=no |
| PAUSE=no |
| |
| while getopts :46t:pPvh o |
| do |
| case $o in |
| 4) TESTS=ipv4;; |
| 6) TESTS=ipv6;; |
| t) TESTS=$OPTARG;; |
| p) PAUSE_ON_FAIL=yes;; |
| P) PAUSE=yes;; |
| v) VERBOSE=1;; |
| h) usage; exit 0;; |
| *) usage; exit 1;; |
| esac |
| done |
| |
| # make sure we don't pause twice |
| [ "${PAUSE}" = "yes" ] && PAUSE_ON_FAIL=no |
| |
| # |
| # show user test config |
| # |
| if [ -z "$TESTS" ]; then |
| TESTS="$TESTS_IPV4 $TESTS_IPV6 $TESTS_OTHER" |
| elif [ "$TESTS" = "ipv4" ]; then |
| TESTS="$TESTS_IPV4" |
| elif [ "$TESTS" = "ipv6" ]; then |
| TESTS="$TESTS_IPV6" |
| fi |
| |
| declare -i nfail=0 |
| declare -i nsuccess=0 |
| |
| for t in $TESTS |
| do |
| case $t in |
| ipv4_ping|ping) ipv4_ping;; |
| ipv4_tcp|tcp) ipv4_tcp;; |
| |
| ipv6_ping|ping6) ipv6_ping;; |
| |
| # setup namespaces and config, but do not run any tests |
| setup) setup; exit 0;; |
| vrf_setup) setup "yes"; exit 0;; |
| |
| help) echo "Test names: $TESTS"; exit 0;; |
| esac |
| done |
| |
| cleanup 2>/dev/null |
| |
| printf "\nTests passed: %3d\n" ${nsuccess} |
| printf "Tests failed: %3d\n" ${nfail} |