RDMA-MR内核接口-rxe_alloc_mr能否实现超长尾词的内存区域分配？

本文共计1322个文字，预计阅读时间需要6分钟。

描述：register_mr函数用于快速内存注册，静态结构ib_mr *rxe_alloc_mr（struct ib_pd *ibpd，enum ib_mr_type mr_type，u32 max_num_sg，struct ib_udata *udata）

参数1：pd ibv_alloc_pd（），返回的pd参数

描述：register stag for fast memory registration

static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata)

Value

Description

参数1

pd

ibv_alloc_pd ()返回的 pd

参数2

mr_type

指定类型IB_MR_TYPE_MEM_REG

参数3

max_num_sg

最大页数mem->max_buf

参数4

udata

元数据，在rxe_alloc_mr未使用

返回ok

struct ib_mr *

指向新分配的mr的指针

返回err

NULL

流程：

a) 将ib类型转换为rxe类型

b) 验证mr_type

c) 申请rxe_mem： mr = rxe_alloc(&rxe->mr_pool);

d) 为mr分配索引：rxe_add_index(mr)

e) 对对象进行引用：rxe_add_ref(pd)

f) 构成mr：rxe_mem_init_fast(pd, max_num_sg, mr);

1) int rxe_mem_init_fast(struct rxe_pd *pd,

int max_pages, struct rxe_mem *mem)

2) 初始化mr：rxe_mem_init(0, mem)->生成lkey和rkey

3) 申请mr：rxe_mem_alloc(mem, max_pages)->申请对应大小的mr

g) 返回新分配的mr

用户申请一块长度为len的虚拟内存，执行RDMA注册函数，将会得到一个映射表，包含两部分，

一个是sg_table记录着物理地址到pci域地址的转换关系，

另一个是VA->PA映射表，记录着VA到PA的转换关系。

调用链：在ib_create_qp()中使用

static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,

struct ib_qp_init_attr *qp_init_attr)

{

return ib_create_qp_user(pd, qp_init_attr, NULL);

}

for user qp use ib_create_qp_user with valid udata：

struct ib_qp *ib_create_qp_user(struct ib_pd *pd,

struct ib_qp_init_attr *qp_init_attr,

struct ib_udata *udata);

在ib_creat_qp_user()中

if (qp_init_attr->cap.max_rdma_ctxs) {

ret = rdma_rw_init_mrs(qp, qp_init_attr);

if (ret)

goto err;

在rdma_rw_init_mrs()中-> ib_mr_pool_init()

int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr)

{

u32 nr_mrs = 0, nr_sig_mrs = 0, max_num_sg = 0;

if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) {

nr_sig_mrs = attr->cap.max_rdma_ctxs;

nr_mrs = attr->cap.max_rdma_ctxs;

max_num_sg = rdma_rw_fr_page_list_len(dev, true);

} else if (rdma_rw_can_use_mr(dev, attr->port_num)) {

nr_mrs = attr->cap.max_rdma_ctxs;

max_num_sg = rdma_rw_fr_page_list_len(dev, false);

if (nr_mrs) {

ret = ib_mr_pool_init(qp, &qp->rdma_mrs, nr_mrs,

IB_MR_TYPE_MEM_REG,

max_num_sg, 0);

if (ret) {

pr_err("%s: failed to allocated %d MRs\n",

__func__, nr_mrs);

return ret;

if (nr_sig_mrs) {

ret = ib_mr_pool_init(qp, &qp->sig_mrs, nr_sig_mrs,

IB_MR_TYPE_INTEGRITY, max_num_sg, max_num_sg);

if (ret) {

pr_err("%s: failed to allocated %d SIG MRs\n",

__func__, nr_sig_mrs);

goto out_free_rdma_mrs;

int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr,

enum ib_mr_type type, u32 max_num_sg, u32 max_num_meta_sg)

{

for (i = 0; i < nr; i++) {

if (type == IB_MR_TYPE_INTEGRITY)

mr = ib_alloc_mr_integrity(qp->pd, max_num_sg,

max_num_meta_sg);

else

mr = ib_alloc_mr(qp->pd, type, max_num_sg);

}

static inline struct ib_mr *ib_alloc_mr(struct ib_pd *pd,

enum ib_mr_type mr_type, u32 max_num_sg)

{

return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL);

}

struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,u32 max_num_sg, struct ib_udata *udata)

{

mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);

if (!IS_ERR(mr)) {

mr->device = pd->device;

mr->pd = pd;

mr->dm = NULL;

mr->uobject = NULL;

atomic_inc(&pd->usecnt);

mr->need_inval = false;

mr->res.type = RDMA_RESTRACK_MR;

rdma_restrack_kadd(&mr->res);

mr->type = mr_type;

mr->sig_attrs = NULL;

return mr;

}

在pd->device->ops.alloc_mr调用到.alloc_mr

总结：在申请QP的时候会使用到.alloc_mr，申请快速内存注册

Ops_rxe:

int rxe_mem_init_fast(struct rxe_pd *pd,

int max_pages, struct rxe_mem *mem)

{

int err;

rxe_mem_init(0, mem);

/* In fastreg, we also set the rkey */

mem->ibmr.rkey = mem->ibmr.lkey;

err = rxe_mem_alloc(mem, max_pages);

if (err)

goto err1;

mem->pd = pd;

mem->max_buf = max_pages;

mem->state = RXE_MEM_STATE_FREE;

mem->type = RXE_MEM_TYPE_MR;

return 0;

err1:

return err;

}

Ops_i40iw:

/**

* i40iw_alloc_mr - register stag for fast memory registration

* @pd: ibpd pointer

* @mr_type: memory for stag registrion

* @max_num_sg: man number of pages

* @udata: user data or NULL for kernel objects

*/

static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,

u32 max_num_sg, struct ib_udata *udata)

{

struct i40iw_pd *iwpd = to_iwpd(pd);

struct i40iw_device *iwdev = to_iwdev(pd->device);

struct i40iw_pble_alloc *palloc;

struct i40iw_pbl *iwpbl;

struct i40iw_mr *iwmr;

enum i40iw_status_code status;

u32 stag;

int err_code = -ENOMEM;

iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);

if (!iwmr)

return ERR_PTR(-ENOMEM);

stag = i40iw_create_stag(iwdev);

if (!stag) {

err_code = -EOVERFLOW;

goto err;

stag &= ~I40IW_CQPSQ_STAG_KEY_MASK;

iwmr->stag = stag;

iwmr->ibmr.rkey = stag;

iwmr->ibmr.lkey = stag;

iwmr->ibmr.pd = pd;

iwmr->ibmr.device = pd->device;

iwpbl = &iwmr->iwpbl;

iwpbl->iwmr = iwmr;

iwmr->type =

palloc = &iwpbl->pble_alloc;

iwmr->page_cnt = max_num_sg;

mutex_lock(&iwdev->pbl_mutex);

status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);

mutex_unlock(&iwdev->pbl_mutex);

if (status)

goto err1;

if (palloc->level != I40IW_LEVEL_1)

goto err2;

err_code = i40iw_hw_alloc_stag(iwdev, iwmr);

if (err_code)

goto err2;

iwpbl->pbl_allocated = true;

i40iw_add_pdusecount(iwpd);

return &iwmr->ibmr;

err2:

i40iw_free_pble(iwdev->pble_rsrc, palloc);

err1:

i40iw_free_stag(iwdev, stag);

err:

kfree(iwmr);

return ERR_PTR(err_code);

}