Chuck Lever | a0ce85f | 2015-03-30 14:34:21 -0400 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Oracle. All rights reserved. |
| 3 | * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. |
| 4 | */ |
| 5 | |
| 6 | /* Lightweight memory registration using Fast Memory Regions (FMR). |
| 7 | * Referred to sometimes as MTHCAFMR mode. |
| 8 | * |
| 9 | * FMR uses synchronous memory registration and deregistration. |
| 10 | * FMR registration is known to be fast, but FMR deregistration |
| 11 | * can take tens of usecs to complete. |
| 12 | */ |
| 13 | |
| 14 | #include "xprt_rdma.h" |
| 15 | |
| 16 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
| 17 | # define RPCDBG_FACILITY RPCDBG_TRANS |
| 18 | #endif |
| 19 | |
Chuck Lever | 1c9351e | 2015-03-30 14:34:30 -0400 | [diff] [blame] | 20 | /* Maximum scatter/gather per FMR */ |
| 21 | #define RPCRDMA_MAX_FMR_SGES (64) |
| 22 | |
Chuck Lever | 3968cb5 | 2015-03-30 14:35:26 -0400 | [diff] [blame] | 23 | static int |
| 24 | fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, |
| 25 | struct rpcrdma_create_data_internal *cdata) |
| 26 | { |
| 27 | return 0; |
| 28 | } |
| 29 | |
Chuck Lever | 1c9351e | 2015-03-30 14:34:30 -0400 | [diff] [blame] | 30 | /* FMR mode conveys up to 64 pages of payload per chunk segment. |
| 31 | */ |
| 32 | static size_t |
| 33 | fmr_op_maxpages(struct rpcrdma_xprt *r_xprt) |
| 34 | { |
| 35 | return min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS, |
| 36 | rpcrdma_max_segments(r_xprt) * RPCRDMA_MAX_FMR_SGES); |
| 37 | } |
| 38 | |
Chuck Lever | 91e70e7 | 2015-03-30 14:34:58 -0400 | [diff] [blame] | 39 | static int |
| 40 | fmr_op_init(struct rpcrdma_xprt *r_xprt) |
| 41 | { |
| 42 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
| 43 | int mr_access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ; |
| 44 | struct ib_fmr_attr fmr_attr = { |
| 45 | .max_pages = RPCRDMA_MAX_FMR_SGES, |
| 46 | .max_maps = 1, |
| 47 | .page_shift = PAGE_SHIFT |
| 48 | }; |
| 49 | struct ib_pd *pd = r_xprt->rx_ia.ri_pd; |
| 50 | struct rpcrdma_mw *r; |
| 51 | int i, rc; |
| 52 | |
| 53 | INIT_LIST_HEAD(&buf->rb_mws); |
| 54 | INIT_LIST_HEAD(&buf->rb_all); |
| 55 | |
| 56 | i = (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS; |
Linus Torvalds | 59953fb | 2015-04-26 17:33:59 -0700 | [diff] [blame] | 57 | dprintk("RPC: %s: initializing %d FMRs\n", __func__, i); |
Chuck Lever | 91e70e7 | 2015-03-30 14:34:58 -0400 | [diff] [blame] | 58 | |
| 59 | while (i--) { |
| 60 | r = kzalloc(sizeof(*r), GFP_KERNEL); |
| 61 | if (!r) |
| 62 | return -ENOMEM; |
| 63 | |
| 64 | r->r.fmr = ib_alloc_fmr(pd, mr_access_flags, &fmr_attr); |
| 65 | if (IS_ERR(r->r.fmr)) |
| 66 | goto out_fmr_err; |
| 67 | |
| 68 | list_add(&r->mw_list, &buf->rb_mws); |
| 69 | list_add(&r->mw_all, &buf->rb_all); |
| 70 | } |
| 71 | return 0; |
| 72 | |
| 73 | out_fmr_err: |
| 74 | rc = PTR_ERR(r->r.fmr); |
| 75 | dprintk("RPC: %s: ib_alloc_fmr status %i\n", __func__, rc); |
| 76 | kfree(r); |
| 77 | return rc; |
| 78 | } |
| 79 | |
Chuck Lever | 9c1b4d7 | 2015-03-30 14:34:39 -0400 | [diff] [blame] | 80 | /* Use the ib_map_phys_fmr() verb to register a memory region |
| 81 | * for remote access via RDMA READ or RDMA WRITE. |
| 82 | */ |
| 83 | static int |
| 84 | fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, |
| 85 | int nsegs, bool writing) |
| 86 | { |
| 87 | struct rpcrdma_ia *ia = &r_xprt->rx_ia; |
Chuck Lever | d654788 | 2015-03-30 14:35:44 -0400 | [diff] [blame] | 88 | struct ib_device *device = ia->ri_id->device; |
| 89 | enum dma_data_direction direction = rpcrdma_data_dir(writing); |
Chuck Lever | 9c1b4d7 | 2015-03-30 14:34:39 -0400 | [diff] [blame] | 90 | struct rpcrdma_mr_seg *seg1 = seg; |
| 91 | struct rpcrdma_mw *mw = seg1->rl_mw; |
| 92 | u64 physaddrs[RPCRDMA_MAX_DATA_SEGS]; |
| 93 | int len, pageoff, i, rc; |
| 94 | |
| 95 | pageoff = offset_in_page(seg1->mr_offset); |
| 96 | seg1->mr_offset -= pageoff; /* start of page */ |
| 97 | seg1->mr_len += pageoff; |
| 98 | len = -pageoff; |
| 99 | if (nsegs > RPCRDMA_MAX_FMR_SGES) |
| 100 | nsegs = RPCRDMA_MAX_FMR_SGES; |
| 101 | for (i = 0; i < nsegs;) { |
Chuck Lever | d654788 | 2015-03-30 14:35:44 -0400 | [diff] [blame] | 102 | rpcrdma_map_one(device, seg, direction); |
Chuck Lever | 9c1b4d7 | 2015-03-30 14:34:39 -0400 | [diff] [blame] | 103 | physaddrs[i] = seg->mr_dma; |
| 104 | len += seg->mr_len; |
| 105 | ++seg; |
| 106 | ++i; |
| 107 | /* Check for holes */ |
| 108 | if ((i < nsegs && offset_in_page(seg->mr_offset)) || |
| 109 | offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) |
| 110 | break; |
| 111 | } |
| 112 | |
| 113 | rc = ib_map_phys_fmr(mw->r.fmr, physaddrs, i, seg1->mr_dma); |
| 114 | if (rc) |
| 115 | goto out_maperr; |
| 116 | |
| 117 | seg1->mr_rkey = mw->r.fmr->rkey; |
| 118 | seg1->mr_base = seg1->mr_dma + pageoff; |
| 119 | seg1->mr_nsegs = i; |
| 120 | seg1->mr_len = len; |
| 121 | return i; |
| 122 | |
| 123 | out_maperr: |
| 124 | dprintk("RPC: %s: ib_map_phys_fmr %u@0x%llx+%i (%d) status %i\n", |
| 125 | __func__, len, (unsigned long long)seg1->mr_dma, |
| 126 | pageoff, i, rc); |
| 127 | while (i--) |
Chuck Lever | d654788 | 2015-03-30 14:35:44 -0400 | [diff] [blame] | 128 | rpcrdma_unmap_one(device, --seg); |
Chuck Lever | 9c1b4d7 | 2015-03-30 14:34:39 -0400 | [diff] [blame] | 129 | return rc; |
| 130 | } |
| 131 | |
Chuck Lever | 6814bae | 2015-03-30 14:34:48 -0400 | [diff] [blame] | 132 | /* Use the ib_unmap_fmr() verb to prevent further remote |
| 133 | * access via RDMA READ or RDMA WRITE. |
| 134 | */ |
| 135 | static int |
| 136 | fmr_op_unmap(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg) |
| 137 | { |
| 138 | struct rpcrdma_ia *ia = &r_xprt->rx_ia; |
| 139 | struct rpcrdma_mr_seg *seg1 = seg; |
Chuck Lever | d654788 | 2015-03-30 14:35:44 -0400 | [diff] [blame] | 140 | struct ib_device *device; |
Chuck Lever | 6814bae | 2015-03-30 14:34:48 -0400 | [diff] [blame] | 141 | int rc, nsegs = seg->mr_nsegs; |
| 142 | LIST_HEAD(l); |
| 143 | |
| 144 | list_add(&seg1->rl_mw->r.fmr->list, &l); |
| 145 | rc = ib_unmap_fmr(&l); |
| 146 | read_lock(&ia->ri_qplock); |
Chuck Lever | d654788 | 2015-03-30 14:35:44 -0400 | [diff] [blame] | 147 | device = ia->ri_id->device; |
Chuck Lever | 6814bae | 2015-03-30 14:34:48 -0400 | [diff] [blame] | 148 | while (seg1->mr_nsegs--) |
Chuck Lever | d654788 | 2015-03-30 14:35:44 -0400 | [diff] [blame] | 149 | rpcrdma_unmap_one(device, seg++); |
Chuck Lever | 6814bae | 2015-03-30 14:34:48 -0400 | [diff] [blame] | 150 | read_unlock(&ia->ri_qplock); |
| 151 | if (rc) |
| 152 | goto out_err; |
| 153 | return nsegs; |
| 154 | |
| 155 | out_err: |
| 156 | dprintk("RPC: %s: ib_unmap_fmr status %i\n", __func__, rc); |
| 157 | return nsegs; |
| 158 | } |
| 159 | |
Chuck Lever | 31a701a | 2015-03-30 14:35:07 -0400 | [diff] [blame] | 160 | /* After a disconnect, unmap all FMRs. |
| 161 | * |
| 162 | * This is invoked only in the transport connect worker in order |
| 163 | * to serialize with rpcrdma_register_fmr_external(). |
| 164 | */ |
| 165 | static void |
| 166 | fmr_op_reset(struct rpcrdma_xprt *r_xprt) |
| 167 | { |
| 168 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
| 169 | struct rpcrdma_mw *r; |
| 170 | LIST_HEAD(list); |
| 171 | int rc; |
| 172 | |
| 173 | list_for_each_entry(r, &buf->rb_all, mw_all) |
| 174 | list_add(&r->r.fmr->list, &list); |
| 175 | |
| 176 | rc = ib_unmap_fmr(&list); |
| 177 | if (rc) |
| 178 | dprintk("RPC: %s: ib_unmap_fmr failed %i\n", |
| 179 | __func__, rc); |
| 180 | } |
| 181 | |
Chuck Lever | 4561f34 | 2015-03-30 14:35:17 -0400 | [diff] [blame] | 182 | static void |
| 183 | fmr_op_destroy(struct rpcrdma_buffer *buf) |
| 184 | { |
| 185 | struct rpcrdma_mw *r; |
| 186 | int rc; |
| 187 | |
| 188 | while (!list_empty(&buf->rb_all)) { |
| 189 | r = list_entry(buf->rb_all.next, struct rpcrdma_mw, mw_all); |
| 190 | list_del(&r->mw_all); |
| 191 | rc = ib_dealloc_fmr(r->r.fmr); |
| 192 | if (rc) |
| 193 | dprintk("RPC: %s: ib_dealloc_fmr failed %i\n", |
| 194 | __func__, rc); |
| 195 | kfree(r); |
| 196 | } |
| 197 | } |
| 198 | |
Chuck Lever | a0ce85f | 2015-03-30 14:34:21 -0400 | [diff] [blame] | 199 | const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = { |
Chuck Lever | 9c1b4d7 | 2015-03-30 14:34:39 -0400 | [diff] [blame] | 200 | .ro_map = fmr_op_map, |
Chuck Lever | 6814bae | 2015-03-30 14:34:48 -0400 | [diff] [blame] | 201 | .ro_unmap = fmr_op_unmap, |
Chuck Lever | 3968cb5 | 2015-03-30 14:35:26 -0400 | [diff] [blame] | 202 | .ro_open = fmr_op_open, |
Chuck Lever | 1c9351e | 2015-03-30 14:34:30 -0400 | [diff] [blame] | 203 | .ro_maxpages = fmr_op_maxpages, |
Chuck Lever | 91e70e7 | 2015-03-30 14:34:58 -0400 | [diff] [blame] | 204 | .ro_init = fmr_op_init, |
Chuck Lever | 31a701a | 2015-03-30 14:35:07 -0400 | [diff] [blame] | 205 | .ro_reset = fmr_op_reset, |
Chuck Lever | 4561f34 | 2015-03-30 14:35:17 -0400 | [diff] [blame] | 206 | .ro_destroy = fmr_op_destroy, |
Chuck Lever | a0ce85f | 2015-03-30 14:34:21 -0400 | [diff] [blame] | 207 | .ro_displayname = "fmr", |
| 208 | }; |