Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 1 | /* |
| 2 | * Low Latency Sockets |
| 3 | * Copyright(c) 2013 Intel Corporation. |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify it |
| 6 | * under the terms and conditions of the GNU General Public License, |
| 7 | * version 2, as published by the Free Software Foundation. |
| 8 | * |
| 9 | * This program is distributed in the hope it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License along with |
| 15 | * this program; if not, write to the Free Software Foundation, Inc., |
| 16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. |
| 17 | * |
| 18 | * Author: Eliezer Tamir |
| 19 | * |
| 20 | * Contact Information: |
| 21 | * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> |
| 22 | */ |
| 23 | |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 24 | #ifndef _LINUX_NET_LL_POLL_H |
| 25 | #define _LINUX_NET_LL_POLL_H |
| 26 | |
| 27 | #include <linux/netdevice.h> |
| 28 | #include <net/ip.h> |
| 29 | |
| 30 | #ifdef CONFIG_NET_LL_RX_POLL |
| 31 | |
| 32 | struct napi_struct; |
Eliezer Tamir | 2d48d67 | 2013-06-24 10:28:03 +0300 | [diff] [blame] | 33 | extern unsigned int sysctl_net_ll_read __read_mostly; |
Eliezer Tamir | eb6db62 | 2013-06-14 16:33:25 +0300 | [diff] [blame] | 34 | extern unsigned int sysctl_net_ll_poll __read_mostly; |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 35 | |
| 36 | /* return values from ndo_ll_poll */ |
| 37 | #define LL_FLUSH_FAILED -1 |
| 38 | #define LL_FLUSH_BUSY -2 |
| 39 | |
Eliezer Tamir | 91e2fd33 | 2013-06-28 15:59:35 +0300 | [diff] [blame] | 40 | static inline unsigned int ll_get_flag(void) |
| 41 | { |
| 42 | return sysctl_net_ll_poll ? POLL_LL : 0; |
| 43 | } |
| 44 | |
Eliezer Tamir | ad6276e | 2013-06-28 15:59:26 +0300 | [diff] [blame] | 45 | /* a wrapper to make debug_smp_processor_id() happy |
| 46 | * we can use sched_clock() because we don't care much about precision |
Eliezer Tamir | 9a3c71a | 2013-06-14 16:33:35 +0300 | [diff] [blame] | 47 | * we only care that the average is bounded |
Eliezer Tamir | ad6276e | 2013-06-28 15:59:26 +0300 | [diff] [blame] | 48 | */ |
| 49 | #ifdef CONFIG_DEBUG_PREEMPT |
| 50 | static inline u64 ll_sched_clock(void) |
| 51 | { |
| 52 | u64 rc; |
| 53 | |
| 54 | preempt_disable_notrace(); |
| 55 | rc = sched_clock(); |
| 56 | preempt_enable_no_resched_notrace(); |
| 57 | |
| 58 | return rc; |
| 59 | } |
| 60 | #else /* CONFIG_DEBUG_PREEMPT */ |
| 61 | static inline u64 ll_sched_clock(void) |
| 62 | { |
| 63 | return sched_clock(); |
| 64 | } |
| 65 | #endif /* CONFIG_DEBUG_PREEMPT */ |
| 66 | |
| 67 | /* we don't mind a ~2.5% imprecision so <<10 instead of *1000 |
Eliezer Tamir | 2d48d67 | 2013-06-24 10:28:03 +0300 | [diff] [blame] | 68 | * sk->sk_ll_usec is a u_int so this can't overflow |
Eliezer Tamir | 9a3c71a | 2013-06-14 16:33:35 +0300 | [diff] [blame] | 69 | */ |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 70 | static inline u64 ll_sk_run_time(struct sock *sk) |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 71 | { |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 72 | return (u64)ACCESS_ONCE(sk->sk_ll_usec) << 10; |
Eliezer Tamir | 2d48d67 | 2013-06-24 10:28:03 +0300 | [diff] [blame] | 73 | } |
Eliezer Tamir | 9a3c71a | 2013-06-14 16:33:35 +0300 | [diff] [blame] | 74 | |
Eliezer Tamir | 91e2fd33 | 2013-06-28 15:59:35 +0300 | [diff] [blame] | 75 | /* in poll/select we use the global sysctl_net_ll_poll value |
| 76 | * only call sched_clock() if enabled |
| 77 | */ |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 78 | static inline u64 ll_run_time(void) |
Eliezer Tamir | 2d48d67 | 2013-06-24 10:28:03 +0300 | [diff] [blame] | 79 | { |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 80 | return (u64)ACCESS_ONCE(sysctl_net_ll_poll) << 10; |
| 81 | } |
Eliezer Tamir | 91e2fd33 | 2013-06-28 15:59:35 +0300 | [diff] [blame] | 82 | |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 83 | /* if flag is not set we don't need to know the time */ |
| 84 | static inline u64 ll_start_time(unsigned int flag) |
| 85 | { |
| 86 | return flag ? ll_sched_clock() : 0; |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 87 | } |
| 88 | |
| 89 | static inline bool sk_valid_ll(struct sock *sk) |
| 90 | { |
Eliezer Tamir | dafcc43 | 2013-06-14 16:33:57 +0300 | [diff] [blame] | 91 | return sk->sk_ll_usec && sk->sk_napi_id && |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 92 | !need_resched() && !signal_pending(current); |
| 93 | } |
| 94 | |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 95 | /* careful! time_in_range64 will evaluate now twice */ |
| 96 | static inline bool can_poll_ll(u64 start_time, u64 run_time) |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 97 | { |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 98 | u64 now = ll_sched_clock(); |
| 99 | |
| 100 | return time_in_range64(now, start_time, start_time + run_time); |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 101 | } |
| 102 | |
Eliezer Tamir | 2d48d67 | 2013-06-24 10:28:03 +0300 | [diff] [blame] | 103 | /* when used in sock_poll() nonblock is known at compile time to be true |
| 104 | * so the loop and end_time will be optimized out |
| 105 | */ |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 106 | static inline bool sk_poll_ll(struct sock *sk, int nonblock) |
| 107 | { |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 108 | u64 start_time = ll_start_time(!nonblock); |
| 109 | u64 run_time = ll_sk_run_time(sk); |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 110 | const struct net_device_ops *ops; |
| 111 | struct napi_struct *napi; |
| 112 | int rc = false; |
| 113 | |
| 114 | /* |
| 115 | * rcu read lock for napi hash |
| 116 | * bh so we don't race with net_rx_action |
| 117 | */ |
| 118 | rcu_read_lock_bh(); |
| 119 | |
| 120 | napi = napi_by_id(sk->sk_napi_id); |
| 121 | if (!napi) |
| 122 | goto out; |
| 123 | |
| 124 | ops = napi->dev->netdev_ops; |
| 125 | if (!ops->ndo_ll_poll) |
| 126 | goto out; |
| 127 | |
| 128 | do { |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 129 | rc = ops->ndo_ll_poll(napi); |
| 130 | |
| 131 | if (rc == LL_FLUSH_FAILED) |
| 132 | break; /* permanent failure */ |
| 133 | |
| 134 | if (rc > 0) |
| 135 | /* local bh are disabled so it is ok to use _BH */ |
| 136 | NET_ADD_STATS_BH(sock_net(sk), |
| 137 | LINUX_MIB_LOWLATENCYRXPACKETS, rc); |
| 138 | |
Eliezer Tamir | 2d48d67 | 2013-06-24 10:28:03 +0300 | [diff] [blame] | 139 | } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) && |
Eliezer Tamir | 1bc2774 | 2013-07-02 23:22:47 +0300 | [diff] [blame^] | 140 | can_poll_ll(start_time, run_time)); |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 141 | |
| 142 | rc = !skb_queue_empty(&sk->sk_receive_queue); |
| 143 | out: |
| 144 | rcu_read_unlock_bh(); |
| 145 | return rc; |
| 146 | } |
| 147 | |
| 148 | /* used in the NIC receive handler to mark the skb */ |
| 149 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) |
| 150 | { |
| 151 | skb->napi_id = napi->napi_id; |
| 152 | } |
| 153 | |
| 154 | /* used in the protocol hanlder to propagate the napi_id to the socket */ |
| 155 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) |
| 156 | { |
| 157 | sk->sk_napi_id = skb->napi_id; |
| 158 | } |
| 159 | |
| 160 | #else /* CONFIG_NET_LL_RX_POLL */ |
Eliezer Tamir | 91e2fd33 | 2013-06-28 15:59:35 +0300 | [diff] [blame] | 161 | static inline unsigned long ll_get_flag(void) |
| 162 | { |
| 163 | return 0; |
| 164 | } |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 165 | |
Eliezer Tamir | 2d48d67 | 2013-06-24 10:28:03 +0300 | [diff] [blame] | 166 | static inline u64 sk_ll_end_time(struct sock *sk) |
| 167 | { |
| 168 | return 0; |
| 169 | } |
| 170 | |
| 171 | static inline u64 ll_end_time(void) |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 172 | { |
| 173 | return 0; |
| 174 | } |
| 175 | |
| 176 | static inline bool sk_valid_ll(struct sock *sk) |
| 177 | { |
| 178 | return false; |
| 179 | } |
| 180 | |
| 181 | static inline bool sk_poll_ll(struct sock *sk, int nonblock) |
| 182 | { |
| 183 | return false; |
| 184 | } |
| 185 | |
| 186 | static inline void skb_mark_ll(struct sk_buff *skb, struct napi_struct *napi) |
| 187 | { |
| 188 | } |
| 189 | |
| 190 | static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb) |
| 191 | { |
| 192 | } |
| 193 | |
Eliezer Tamir | 9a3c71a | 2013-06-14 16:33:35 +0300 | [diff] [blame] | 194 | static inline bool can_poll_ll(u64 end_time) |
Eliezer Tamir | 0602129 | 2013-06-10 11:39:50 +0300 | [diff] [blame] | 195 | { |
| 196 | return false; |
| 197 | } |
| 198 | |
| 199 | #endif /* CONFIG_NET_LL_RX_POLL */ |
| 200 | #endif /* _LINUX_NET_LL_POLL_H */ |