RDMA/nes: Improve use of PBLs
Two level 256 byte PBLs was not implemented so the driver could report
out of memory when in fact there were PBLs still available.
This solution prefers to use 4KB PBLs over two level 256B PBLs until
the number of 4KB PBLs falls below a threshold. At this point the 4KB
PBL structure is converted to use 256B PBLs which prevents the driver
from running out of 4KB PBLs too quickly.
Signed-off-by: Don Wood <donald.e.wood@intel.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index b42b17a..96d9535 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -1887,21 +1887,75 @@
return ret;
}
+/**
+ * root_256
+ */
+static u32 root_256(struct nes_device *nesdev,
+ struct nes_root_vpbl *root_vpbl,
+ struct nes_root_vpbl *new_root,
+ u16 pbl_count_4k,
+ u16 pbl_count_256)
+{
+ u64 leaf_pbl;
+ int i, j, k;
+
+ if (pbl_count_4k == 1) {
+ new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev,
+ 512, &new_root->pbl_pbase);
+
+ if (new_root->pbl_vbase == NULL)
+ return 0;
+
+ leaf_pbl = (u64)root_vpbl->pbl_pbase;
+ for (i = 0; i < 16; i++) {
+ new_root->pbl_vbase[i].pa_low =
+ cpu_to_le32((u32)leaf_pbl);
+ new_root->pbl_vbase[i].pa_high =
+ cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
+ leaf_pbl += 256;
+ }
+ } else {
+ for (i = 3; i >= 0; i--) {
+ j = i * 16;
+ root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i];
+ leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) +
+ (((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high))
+ << 32);
+ for (k = 1; k < 16; k++) {
+ leaf_pbl += 256;
+ root_vpbl->pbl_vbase[j + k].pa_low =
+ cpu_to_le32((u32)leaf_pbl);
+ root_vpbl->pbl_vbase[j + k].pa_high =
+ cpu_to_le32((u32)((((u64)leaf_pbl) >> 32)));
+ }
+ }
+ }
+
+ return 1;
+}
+
/**
* nes_reg_mr
*/
static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl,
- dma_addr_t single_buffer, u16 pbl_count, u16 residual_page_count,
- int acc, u64 *iova_start)
+ dma_addr_t single_buffer, u16 pbl_count_4k,
+ u16 residual_page_count_4k, int acc, u64 *iova_start,
+ u16 *actual_pbl_cnt, u8 *used_4k_pbls)
{
struct nes_hw_cqp_wqe *cqp_wqe;
struct nes_cqp_request *cqp_request;
unsigned long flags;
int ret;
struct nes_adapter *nesadapter = nesdev->nesadapter;
- /* int count; */
+ uint pg_cnt = 0;
+ u16 pbl_count_256;
+ u16 pbl_count = 0;
+ u8 use_256_pbls = 0;
+ u8 use_4k_pbls = 0;
+ u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0;
+ struct nes_root_vpbl new_root = {0, 0, 0};
u32 opcode = 0;
u16 major_code;
@@ -1914,41 +1968,70 @@
cqp_request->waiting = 1;
cqp_wqe = &cqp_request->cqp_wqe;
- spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- /* track PBL resources */
- if (pbl_count != 0) {
- if (pbl_count > 1) {
- /* Two level PBL */
- if ((pbl_count+1) > nesadapter->free_4kpbl) {
- nes_debug(NES_DBG_MR, "Out of 4KB Pbls for two level request.\n");
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- nes_free_cqp_request(nesdev, cqp_request);
- return -ENOMEM;
- } else {
- nesadapter->free_4kpbl -= pbl_count+1;
- }
- } else if (residual_page_count > 32) {
- if (pbl_count > nesadapter->free_4kpbl) {
- nes_debug(NES_DBG_MR, "Out of 4KB Pbls.\n");
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- nes_free_cqp_request(nesdev, cqp_request);
- return -ENOMEM;
- } else {
- nesadapter->free_4kpbl -= pbl_count;
+ if (pbl_count_4k) {
+ spin_lock_irqsave(&nesadapter->pbl_lock, flags);
+
+ pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k;
+ pbl_count_256 = (pg_cnt + 31) / 32;
+ if (pg_cnt <= 32) {
+ if (pbl_count_256 <= nesadapter->free_256pbl)
+ use_256_pbls = 1;
+ else if (pbl_count_4k <= nesadapter->free_4kpbl)
+ use_4k_pbls = 1;
+ } else if (pg_cnt <= 2048) {
+ if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) &&
+ (nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) {
+ use_4k_pbls = 1;
+ } else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) {
+ use_256_pbls = 1;
+ use_two_level = 1;
+ } else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
+ use_4k_pbls = 1;
}
} else {
- if (pbl_count > nesadapter->free_256pbl) {
- nes_debug(NES_DBG_MR, "Out of 256B Pbls.\n");
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
- nes_free_cqp_request(nesdev, cqp_request);
- return -ENOMEM;
- } else {
- nesadapter->free_256pbl -= pbl_count;
- }
+ if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl)
+ use_4k_pbls = 1;
}
+
+ if (use_256_pbls) {
+ pbl_count = pbl_count_256;
+ nesadapter->free_256pbl -= pbl_count + use_two_level;
+ } else if (use_4k_pbls) {
+ pbl_count = pbl_count_4k;
+ nesadapter->free_4kpbl -= pbl_count + use_two_level;
+ } else {
+ spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
+ nes_debug(NES_DBG_MR, "Out of Pbls\n");
+ nes_free_cqp_request(nesdev, cqp_request);
+ return -ENOMEM;
+ }
+
+ spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
}
- spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
+ if (use_256_pbls && use_two_level) {
+ if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k, pbl_count_256) == 1) {
+ if (new_root.pbl_pbase != 0)
+ root_vpbl = &new_root;
+ } else {
+ spin_lock_irqsave(&nesadapter->pbl_lock, flags);
+ nesadapter->free_256pbl += pbl_count_256 + use_two_level;
+ use_256_pbls = 0;
+
+ if (pbl_count_4k == 1)
+ use_two_level = 0;
+ pbl_count = pbl_count_4k;
+
+ if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) {
+ nesadapter->free_4kpbl -= pbl_count + use_two_level;
+ use_4k_pbls = 1;
+ }
+ spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
+
+ if (use_4k_pbls == 0)
+ return -ENOMEM;
+ }
+ }
opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ |
NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR;
@@ -1977,10 +2060,9 @@
} else {
set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase);
set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count);
- set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX,
- (((pbl_count - 1) * 4096) + (residual_page_count*8)));
+ set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8));
- if ((pbl_count > 1) || (residual_page_count > 32))
+ if (use_4k_pbls)
cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
}
barrier();
@@ -1996,23 +2078,26 @@
stag, ret, cqp_request->major_code, cqp_request->minor_code);
major_code = cqp_request->major_code;
nes_put_cqp_request(nesdev, cqp_request);
+
if ((!ret || major_code) && pbl_count != 0) {
spin_lock_irqsave(&nesadapter->pbl_lock, flags);
- if (pbl_count > 1)
- nesadapter->free_4kpbl += pbl_count+1;
- else if (residual_page_count > 32)
- nesadapter->free_4kpbl += pbl_count;
- else
- nesadapter->free_256pbl += pbl_count;
+ if (use_256_pbls)
+ nesadapter->free_256pbl += pbl_count + use_two_level;
+ else if (use_4k_pbls)
+ nesadapter->free_4kpbl += pbl_count + use_two_level;
spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
}
+ if (new_root.pbl_pbase)
+ pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase,
+ new_root.pbl_pbase);
+
if (!ret)
return -ETIME;
else if (major_code)
return -EIO;
- else
- return 0;
+ *actual_pbl_cnt = pbl_count + use_two_level;
+ *used_4k_pbls = use_4k_pbls;
return 0;
}
@@ -2177,18 +2262,14 @@
pbl_count = root_pbl_index;
}
ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl,
- buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start);
+ buffer_list[0].addr, pbl_count, (u16)cur_pbl_index, acc, iova_start,
+ &nesmr->pbls_used, &nesmr->pbl_4k);
if (ret == 0) {
nesmr->ibmr.rkey = stag;
nesmr->ibmr.lkey = stag;
nesmr->mode = IWNES_MEMREG_TYPE_MEM;
ibmr = &nesmr->ibmr;
- nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0;
- nesmr->pbls_used = pbl_count;
- if (pbl_count > 1) {
- nesmr->pbls_used++;
- }
} else {
kfree(nesmr);
ibmr = ERR_PTR(-ENOMEM);
@@ -2466,8 +2547,9 @@
stag, (unsigned int)iova_start,
(unsigned int)region_length, stag_index,
(unsigned long long)region->length, pbl_count);
- ret = nes_reg_mr( nesdev, nespd, stag, region->length, &root_vpbl,
- first_dma_addr, pbl_count, (u16)cur_pbl_index, acc, &iova_start);
+ ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl,
+ first_dma_addr, pbl_count, (u16)cur_pbl_index, acc,
+ &iova_start, &nesmr->pbls_used, &nesmr->pbl_4k);
nes_debug(NES_DBG_MR, "ret=%d\n", ret);
@@ -2476,11 +2558,6 @@
nesmr->ibmr.lkey = stag;
nesmr->mode = IWNES_MEMREG_TYPE_MEM;
ibmr = &nesmr->ibmr;
- nesmr->pbl_4k = ((pbl_count > 1) || (cur_pbl_index > 32)) ? 1 : 0;
- nesmr->pbls_used = pbl_count;
- if (pbl_count > 1) {
- nesmr->pbls_used++;
- }
} else {
ib_umem_release(region);
kfree(nesmr);