xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Tükrözés 0
Kód Hibajegyek Egyesítési kérések Actions Packages Projektek Kiadások Wiki Tevékenység

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Forráskód böngészése 
Pull rdma updates from Jason Gunthorpe:
 "A more active cycle than most of the recent past, with a few large,
  long discussed works this time.

  The RNBD block driver has been posted for nearly two years now, and
  flowing through RDMA due to it also introducing a new ULP.

  The removal of FMR has been a recurring discussion theme for a long
  time.

  And the usual smattering of features and bug fixes.

  Summary:

   - Various small driver bugs fixes in rxe, mlx5, hfi1, and efa

   - Continuing driver cleanups in bnxt_re, hns

   - Big cleanup of mlx5 QP creation flows

   - More consistent use of src port and flow label when LAG is used and
     a mlx5 implementation

   - Additional set of cleanups for IB CM

   - 'RNBD' network block driver and target. This is a network block
     RDMA device specific to ionos's cloud environment. It brings strong
     multipath and resiliency capabilities.

   - Accelerated IPoIB for HFI1

   - QP/WQ/SRQ ioctl migration for uverbs, and support for multiple
     async fds

   - Support for exchanging the new IBTA defiend ECE data during RDMA CM
     exchanges

   - Removal of the very old and insecure FMR interface from all ULPs
     and drivers. FRWR should be preferred for at least a decade now"

* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (247 commits)
  RDMA/cm: Spurious WARNING triggered in cm_destroy_id()
  RDMA/mlx5: Return ECE DC support
  RDMA/mlx5: Don't rely on FW to set zeros in ECE response
  RDMA/mlx5: Return an error if copy_to_user fails
  IB/hfi1: Use free_netdev() in hfi1_netdev_free()
  RDMA/hns: Uninitialized variable in modify_qp_init_to_rtr()
  RDMA/core: Move and rename trace_cm_id_create()
  IB/hfi1: Fix hfi1_netdev_rx_init() error handling
  RDMA: Remove 'max_map_per_fmr'
  RDMA: Remove 'max_fmr'
  RDMA/core: Remove FMR device ops
  RDMA/rdmavt: Remove FMR memory registration
  RDMA/mthca: Remove FMR support for memory registration
  RDMA/mlx4: Remove FMR support for memory registration
  RDMA/i40iw: Remove FMR leftovers
  RDMA/bnxt_re: Remove FMR leftovers
  RDMA/mlx5: Remove FMR leftovers
  RDMA/core: Remove FMR pool API
  RDMA/rds: Remove FMR support for memory registration
  RDMA/srp: Remove support for FMR memory registration
  ...
This commit is contained in:
Linus Torvalds
2020-06-05 14:05:57 -07:00
szülő 3f7e82379f fba97dc7fc
commit 242b233198
244 fájl változott, egészen pontosan 23842 új sor hozzáadva és 10837 régi sor törölve
Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/infiniband/ulp/Makefile
Fájl megtekintése

@@ -5,3 +5,4 @@ obj-$(CONFIG_INFINIBAND_SRPT) += srpt/
obj-$(CONFIG_INFINIBAND_ISER) += iser/
obj-$(CONFIG_INFINIBAND_ISERT) += isert/
obj-$(CONFIG_INFINIBAND_OPA_VNIC) += opa_vnic/
obj-$(CONFIG_INFINIBAND_RTRS) += rtrs/

37
drivers/infiniband/ulp/ipoib/ipoib_main.c
Fájl megtekintése

@@ -86,7 +86,7 @@ struct workqueue_struct *ipoib_workqueue;
struct ib_sa_client ipoib_sa_client;
static void ipoib_add_one(struct ib_device *device);
static int ipoib_add_one(struct ib_device *device);
static void ipoib_remove_one(struct ib_device *device, void *client_data);
static void ipoib_neigh_reclaim(struct rcu_head *rp);
static struct net_device *ipoib_get_net_dev_by_params(
@@ -479,9 +479,6 @@ static struct net_device *ipoib_get_net_dev_by_params(
if (ret)
return NULL;
if (!dev_list)
return NULL;
/* See if we can find a unique device matching the L2 parameters */
matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
gid, NULL, &net_dev);
@@ -529,6 +526,7 @@ int ipoib_set_mode(struct net_device *dev, const char *buf)
"will cause multicast packet drops\n");
netdev_update_features(dev);
dev_set_mtu(dev, ipoib_cm_max_mtu(dev));
netif_set_real_num_tx_queues(dev, 1);
rtnl_unlock();
priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
@@ -540,6 +538,7 @@ int ipoib_set_mode(struct net_device *dev, const char *buf)
clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
netdev_update_features(dev);
dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
netif_set_real_num_tx_queues(dev, dev->num_tx_queues);
rtnl_unlock();
ipoib_flush_paths(dev);
return (!rtnl_trylock()) ? -EBUSY : 0;
@@ -1860,7 +1859,7 @@ static int ipoib_parent_init(struct net_device *ndev)
priv->port);
return result;
}
priv->max_ib_mtu = ib_mtu_enum_to_int(attr.max_mtu);
priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);
result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
if (result) {
@@ -1901,6 +1900,7 @@ static int ipoib_ndo_init(struct net_device *ndev)
{
struct ipoib_dev_priv *priv = ipoib_priv(ndev);
int rc;
struct rdma_netdev *rn = netdev_priv(ndev);
if (priv->parent) {
ipoib_child_init(ndev);
@@ -1913,6 +1913,7 @@ static int ipoib_ndo_init(struct net_device *ndev)
/* MTU will be reset when mcast join happens */
ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
rn->mtu = priv->mcast_mtu;
ndev->max_mtu = IPOIB_CM_MTU;
ndev->neigh_priv_len = sizeof(struct ipoib_neigh);
@@ -2074,9 +2075,17 @@ static const struct net_device_ops ipoib_netdev_ops_vf = {
.ndo_do_ioctl = ipoib_ioctl,
};
static const struct net_device_ops ipoib_netdev_default_pf = {
.ndo_init = ipoib_dev_init_default,
.ndo_uninit = ipoib_dev_uninit_default,
.ndo_open = ipoib_ib_dev_open_default,
.ndo_stop = ipoib_ib_dev_stop_default,
};
void ipoib_setup_common(struct net_device *dev)
{
dev->header_ops = &ipoib_header_ops;
dev->netdev_ops = &ipoib_netdev_default_pf;
ipoib_set_ethtool_ops(dev);
@@ -2126,13 +2135,6 @@ static void ipoib_build_priv(struct net_device *dev)
INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
}
static const struct net_device_ops ipoib_netdev_default_pf = {
.ndo_init = ipoib_dev_init_default,
.ndo_uninit = ipoib_dev_uninit_default,
.ndo_open = ipoib_ib_dev_open_default,
.ndo_stop = ipoib_ib_dev_stop_default,
};
static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u8 port,
const char *name)
{
@@ -2170,7 +2172,6 @@ int ipoib_intf_init(struct ib_device *hca, u8 port, const char *name,
if (rc != -EOPNOTSUPP)
goto out;
dev->netdev_ops = &ipoib_netdev_default_pf;
rn->send = ipoib_send;
rn->attach_mcast = ipoib_mcast_attach;
rn->detach_mcast = ipoib_mcast_detach;
@@ -2516,7 +2517,7 @@ sysfs_failed:
return ERR_PTR(-ENOMEM);
}
static void ipoib_add_one(struct ib_device *device)
static int ipoib_add_one(struct ib_device *device)
{
struct list_head *dev_list;
struct net_device *dev;
@@ -2526,7 +2527,7 @@ static void ipoib_add_one(struct ib_device *device)
dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);
if (!dev_list)
return;
return -ENOMEM;
INIT_LIST_HEAD(dev_list);
@@ -2543,10 +2544,11 @@ static void ipoib_add_one(struct ib_device *device)
if (!count) {
kfree(dev_list);
return;
return -EOPNOTSUPP;
}
ib_set_client_data(device, &ipoib_client, dev_list);
return 0;
}
static void ipoib_remove_one(struct ib_device *device, void *client_data)
@@ -2554,9 +2556,6 @@ static void ipoib_remove_one(struct ib_device *device, void *client_data)
struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
struct list_head *dev_list = client_data;
if (!dev_list)
return;
list_for_each_entry_safe(priv, tmp, dev_list, list) {
LIST_HEAD(head);
ipoib_parent_unregister_pre(priv->dev);

23
drivers/infiniband/ulp/ipoib/ipoib_multicast.c
Fájl megtekintése

@@ -135,12 +135,11 @@ static void ipoib_mcast_free(struct ipoib_mcast *mcast)
kfree(mcast);
}
static struct ipoib_mcast *ipoib_mcast_alloc(struct net_device *dev,
int can_sleep)
static struct ipoib_mcast *ipoib_mcast_alloc(struct net_device *dev)
{
struct ipoib_mcast *mcast;
mcast = kzalloc(sizeof(*mcast), can_sleep ? GFP_KERNEL : GFP_ATOMIC);
mcast = kzalloc(sizeof(*mcast), GFP_ATOMIC);
if (!mcast)
return NULL;
@@ -218,6 +217,7 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
struct rdma_ah_attr av;
int ret;
int set_qkey = 0;
int mtu;
mcast->mcmember = *mcmember;
@@ -240,13 +240,12 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
priv->broadcast->mcmember.flow_label = mcmember->flow_label;
priv->broadcast->mcmember.hop_limit = mcmember->hop_limit;
/* assume if the admin and the mcast are the same both can be changed */
mtu = rdma_mtu_enum_to_int(priv->ca, priv->port,
priv->broadcast->mcmember.mtu);
if (priv->mcast_mtu == priv->admin_mtu)
priv->admin_mtu =
priv->mcast_mtu =
IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu));
else
priv->mcast_mtu =
IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu));
priv->admin_mtu = IPOIB_UD_MTU(mtu);
priv->mcast_mtu = IPOIB_UD_MTU(mtu);
rn->mtu = priv->mcast_mtu;
priv->qkey = be32_to_cpu(priv->broadcast->mcmember.qkey);
spin_unlock_irq(&priv->lock);
@@ -599,7 +598,7 @@ void ipoib_mcast_join_task(struct work_struct *work)
if (!priv->broadcast) {
struct ipoib_mcast *broadcast;
broadcast = ipoib_mcast_alloc(dev, 0);
broadcast = ipoib_mcast_alloc(dev);
if (!broadcast) {
ipoib_warn(priv, "failed to allocate broadcast group\n");
/*
@@ -782,7 +781,7 @@ void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb)
ipoib_dbg_mcast(priv, "setting up send only multicast group for %pI6\n",
mgid);
mcast = ipoib_mcast_alloc(dev, 0);
mcast = ipoib_mcast_alloc(dev);
if (!mcast) {
ipoib_warn(priv, "unable to allocate memory "
"for multicast structure\n");
@@ -936,7 +935,7 @@ void ipoib_mcast_restart_task(struct work_struct *work)
ipoib_dbg_mcast(priv, "adding multicast entry for mgid %pI6\n",
mgid.raw);
nmcast = ipoib_mcast_alloc(dev, 0);
nmcast = ipoib_mcast_alloc(dev);
if (!nmcast) {
ipoib_warn(priv, "unable to allocate memory for multicast structure\n");
continue;

3
drivers/infiniband/ulp/ipoib/ipoib_verbs.c
Fájl megtekintése

@@ -206,6 +206,9 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
if (priv->hca_caps & IB_DEVICE_MANAGED_FLOW_STEERING)
init_attr.create_flags |= IB_QP_CREATE_NETIF_QP;
if (priv->hca_caps & IB_DEVICE_RDMA_NETDEV_OPA)
init_attr.create_flags |= IB_QP_CREATE_NETDEV_USE;
priv->qp = ib_create_qp(priv->pd, &init_attr);
if (IS_ERR(priv->qp)) {
pr_warn("%s: failed to create QP\n", ca->name);

3
drivers/infiniband/ulp/ipoib/ipoib_vlan.c
Fájl megtekintése

@@ -97,6 +97,7 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv,
{
struct net_device *ndev = priv->dev;
int result;
struct rdma_netdev *rn = netdev_priv(ndev);
ASSERT_RTNL();
@@ -117,6 +118,8 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv,
goto out_early;
}
rn->mtu = priv->mcast_mtu;
priv->parent = ppriv->dev;
priv->pkey = pkey;
priv->child_type = type;

79
drivers/infiniband/ulp/iser/iscsi_iser.h
Fájl megtekintése

@@ -65,7 +65,6 @@
#include <linux/in6.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_fmr_pool.h>
#include <rdma/rdma_cm.h>
#define DRV_NAME "iser"
@@ -312,33 +311,6 @@ struct iser_comp {
int active_qps;
};
/**
* struct iser_reg_ops - Memory registration operations
* per-device registration schemes
*
* @alloc_reg_res: Allocate registration resources
* @free_reg_res: Free registration resources
* @reg_mem: Register memory buffers
* @unreg_mem: Un-register memory buffers
* @reg_desc_get: Get a registration descriptor for pool
* @reg_desc_put: Get a registration descriptor to pool
*/
struct iser_reg_ops {
int (*alloc_reg_res)(struct ib_conn *ib_conn,
unsigned cmds_max,
unsigned int size);
void (*free_reg_res)(struct ib_conn *ib_conn);
int (*reg_mem)(struct iscsi_iser_task *iser_task,
struct iser_data_buf *mem,
struct iser_reg_resources *rsc,
struct iser_mem_reg *reg);
void (*unreg_mem)(struct iscsi_iser_task *iser_task,
enum iser_data_dir cmd_dir);
struct iser_fr_desc * (*reg_desc_get)(struct ib_conn *ib_conn);
void (*reg_desc_put)(struct ib_conn *ib_conn,
struct iser_fr_desc *desc);
};
/**
* struct iser_device - iSER device handle
*
@@ -351,8 +323,6 @@ struct iser_reg_ops {
* @comps_used: Number of completion contexts used, Min between online
* cpus and device max completion vectors
* @comps: Dinamically allocated array of completion handlers
* @reg_ops: Registration ops
* @remote_inv_sup: Remote invalidate is supported on this device
*/
struct iser_device {
struct ib_device *ib_device;
@@ -362,26 +332,18 @@ struct iser_device {
int refcount;
int comps_used;
struct iser_comp *comps;
const struct iser_reg_ops *reg_ops;
bool remote_inv_sup;
};
/**
* struct iser_reg_resources - Fast registration resources
*
* @mr: memory region
* @fmr_pool: pool of fmrs
* @sig_mr: signature memory region
* @page_vec: fast reg page list used by fmr pool
* @mr_valid: is mr valid indicator
*/
struct iser_reg_resources {
union {
struct ib_mr *mr;
struct ib_fmr_pool *fmr_pool;
};
struct ib_mr *mr;
struct ib_mr *sig_mr;
struct iser_page_vec *page_vec;
u8 mr_valid:1;
};
@@ -403,7 +365,7 @@ struct iser_fr_desc {
* struct iser_fr_pool - connection fast registration pool
*
* @list: list of fastreg descriptors
* @lock: protects fmr/fastreg pool
* @lock: protects fastreg pool
* @size: size of the pool
*/
struct iser_fr_pool {
@@ -518,12 +480,6 @@ struct iscsi_iser_task {
struct iser_data_buf prot[ISER_DIRS_NUM];
};
struct iser_page_vec {
u64 *pages;
int npages;
struct ib_mr fake_mr;
};
/**
* struct iser_global - iSER global context
*
@@ -548,8 +504,6 @@ extern int iser_pi_guard;
extern unsigned int iser_max_sectors;
extern bool iser_always_reg;
int iser_assign_reg_ops(struct iser_device *device);
int iser_send_control(struct iscsi_conn *conn,
struct iscsi_task *task);
@@ -591,22 +545,17 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
struct iser_data_buf *mem,
enum iser_data_dir cmd_dir);
int iser_reg_rdma_mem(struct iscsi_iser_task *task,
enum iser_data_dir dir,
bool all_imm);
void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
enum iser_data_dir dir);
int iser_reg_mem_fastreg(struct iscsi_iser_task *task,
enum iser_data_dir dir,
bool all_imm);
void iser_unreg_mem_fastreg(struct iscsi_iser_task *task,
enum iser_data_dir dir);
int iser_connect(struct iser_conn *iser_conn,
struct sockaddr *src_addr,
struct sockaddr *dst_addr,
int non_blocking);
void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
enum iser_data_dir cmd_dir);
void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
enum iser_data_dir cmd_dir);
int iser_post_recvl(struct iser_conn *iser_conn);
int iser_post_recvm(struct iser_conn *iser_conn, int count);
int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
@@ -625,26 +574,12 @@ int iser_initialize_task_headers(struct iscsi_task *task,
struct iser_tx_desc *tx_desc);
int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
struct iscsi_session *session);
int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
unsigned cmds_max,
unsigned int size);
void iser_free_fmr_pool(struct ib_conn *ib_conn);
int iser_alloc_fastreg_pool(struct ib_conn *ib_conn,
unsigned cmds_max,
unsigned int size);
void iser_free_fastreg_pool(struct ib_conn *ib_conn);
u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
enum iser_data_dir cmd_dir, sector_t *sector);
struct iser_fr_desc *
iser_reg_desc_get_fr(struct ib_conn *ib_conn);
void
iser_reg_desc_put_fr(struct ib_conn *ib_conn,
struct iser_fr_desc *desc);
struct iser_fr_desc *
iser_reg_desc_get_fmr(struct ib_conn *ib_conn);
void
iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
struct iser_fr_desc *desc);
static inline struct iser_conn *
to_iser_conn(struct ib_conn *ib_conn)

19
drivers/infiniband/ulp/iser/iser_initiator.c
Fájl megtekintése

@@ -72,7 +72,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task)
return err;
}
err = iser_reg_rdma_mem(iser_task, ISER_DIR_IN, false);
err = iser_reg_mem_fastreg(iser_task, ISER_DIR_IN, false);
if (err) {
iser_err("Failed to set up Data-IN RDMA\n");
return err;
@@ -126,8 +126,8 @@ iser_prepare_write_cmd(struct iscsi_task *task,
return err;
}
err = iser_reg_rdma_mem(iser_task, ISER_DIR_OUT,
buf_out->data_len == imm_sz);
err = iser_reg_mem_fastreg(iser_task, ISER_DIR_OUT,
buf_out->data_len == imm_sz);
if (err != 0) {
iser_err("Failed to register write cmd RDMA mem\n");
return err;
@@ -250,8 +250,8 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
iser_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */
iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;
if (device->reg_ops->alloc_reg_res(ib_conn, session->scsi_cmds_max,
iser_conn->pages_per_mr))
if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max,
iser_conn->pages_per_mr))
goto create_rdma_reg_res_failed;
if (iser_alloc_login_buf(iser_conn))
@@ -293,7 +293,7 @@ rx_desc_dma_map_failed:
rx_desc_alloc_fail:
iser_free_login_buf(iser_conn);
alloc_login_buf_fail:
device->reg_ops->free_reg_res(ib_conn);
iser_free_fastreg_pool(ib_conn);
create_rdma_reg_res_failed:
iser_err("failed allocating rx descriptors / data buffers\n");
return -ENOMEM;
@@ -306,8 +306,7 @@ void iser_free_rx_descriptors(struct iser_conn *iser_conn)
struct ib_conn *ib_conn = &iser_conn->ib_conn;
struct iser_device *device = ib_conn->device;
if (device->reg_ops->free_reg_res)
device->reg_ops->free_reg_res(ib_conn);
iser_free_fastreg_pool(ib_conn);
rx_desc = iser_conn->rx_descs;
for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)
@@ -768,7 +767,7 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
int prot_count = scsi_prot_sg_count(iser_task->sc);
if (iser_task->dir[ISER_DIR_IN]) {
iser_unreg_rdma_mem(iser_task, ISER_DIR_IN);
iser_unreg_mem_fastreg(iser_task, ISER_DIR_IN);
iser_dma_unmap_task_data(iser_task,
&iser_task->data[ISER_DIR_IN],
DMA_FROM_DEVICE);
@@ -779,7 +778,7 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
}
if (iser_task->dir[ISER_DIR_OUT]) {
iser_unreg_rdma_mem(iser_task, ISER_DIR_OUT);
iser_unreg_mem_fastreg(iser_task, ISER_DIR_OUT);
iser_dma_unmap_task_data(iser_task,
&iser_task->data[ISER_DIR_OUT],
DMA_TO_DEVICE);

188
drivers/infiniband/ulp/iser/iser_memory.c
Fájl megtekintése

@@ -38,62 +38,13 @@
#include <linux/scatterlist.h>
#include "iscsi_iser.h"
static
int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
struct iser_data_buf *mem,
struct iser_reg_resources *rsc,
struct iser_mem_reg *mem_reg);
static
int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
struct iser_data_buf *mem,
struct iser_reg_resources *rsc,
struct iser_mem_reg *mem_reg);
static const struct iser_reg_ops fastreg_ops = {
.alloc_reg_res = iser_alloc_fastreg_pool,
.free_reg_res = iser_free_fastreg_pool,
.reg_mem = iser_fast_reg_mr,
.unreg_mem = iser_unreg_mem_fastreg,
.reg_desc_get = iser_reg_desc_get_fr,
.reg_desc_put = iser_reg_desc_put_fr,
};
static const struct iser_reg_ops fmr_ops = {
.alloc_reg_res = iser_alloc_fmr_pool,
.free_reg_res = iser_free_fmr_pool,
.reg_mem = iser_fast_reg_fmr,
.unreg_mem = iser_unreg_mem_fmr,
.reg_desc_get = iser_reg_desc_get_fmr,
.reg_desc_put = iser_reg_desc_put_fmr,
};
void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc)
{
iser_err_comp(wc, "memreg");
}
int iser_assign_reg_ops(struct iser_device *device)
{
struct ib_device *ib_dev = device->ib_device;
/* Assign function handles - based on FMR support */
if (ib_dev->ops.alloc_fmr && ib_dev->ops.dealloc_fmr &&
ib_dev->ops.map_phys_fmr && ib_dev->ops.unmap_fmr) {
iser_info("FMR supported, using FMR for registration\n");
device->reg_ops = &fmr_ops;
} else if (ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
iser_info("FastReg supported, using FastReg for registration\n");
device->reg_ops = &fastreg_ops;
device->remote_inv_sup = iser_always_reg;
} else {
iser_err("IB device does not support FMRs nor FastRegs, can't register memory\n");
return -1;
}
return 0;
}
struct iser_fr_desc *
static struct iser_fr_desc *
iser_reg_desc_get_fr(struct ib_conn *ib_conn)
{
struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
@@ -109,7 +60,7 @@ iser_reg_desc_get_fr(struct ib_conn *ib_conn)
return desc;
}
void
static void
iser_reg_desc_put_fr(struct ib_conn *ib_conn,
struct iser_fr_desc *desc)
{
@@ -121,44 +72,6 @@ iser_reg_desc_put_fr(struct ib_conn *ib_conn,
spin_unlock_irqrestore(&fr_pool->lock, flags);
}
struct iser_fr_desc *
iser_reg_desc_get_fmr(struct ib_conn *ib_conn)
{
struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
return list_first_entry(&fr_pool->list,
struct iser_fr_desc, list);
}
void
iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
struct iser_fr_desc *desc)
{
}
static void iser_data_buf_dump(struct iser_data_buf *data,
struct ib_device *ibdev)
{
struct scatterlist *sg;
int i;
for_each_sg(data->sg, sg, data->dma_nents, i)
iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p "
"off:0x%x sz:0x%x dma_len:0x%x\n",
i, (unsigned long)sg_dma_address(sg),
sg_page(sg), sg->offset, sg->length, sg_dma_len(sg));
}
static void iser_dump_page_vec(struct iser_page_vec *page_vec)
{
int i;
iser_err("page vec npages %d data length %lld\n",
page_vec->npages, page_vec->fake_mr.length);
for (i = 0; i < page_vec->npages; i++)
iser_err("vec[%d]: %llx\n", i, page_vec->pages[i]);
}
int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
struct iser_data_buf *data,
enum iser_data_dir iser_dir,
@@ -213,84 +126,9 @@ iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
return 0;
}
static int iser_set_page(struct ib_mr *mr, u64 addr)
{
struct iser_page_vec *page_vec =
container_of(mr, struct iser_page_vec, fake_mr);
page_vec->pages[page_vec->npages++] = addr;
return 0;
}
static
int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
struct iser_data_buf *mem,
struct iser_reg_resources *rsc,
struct iser_mem_reg *reg)
{
struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
struct iser_device *device = ib_conn->device;
struct iser_page_vec *page_vec = rsc->page_vec;
struct ib_fmr_pool *fmr_pool = rsc->fmr_pool;
struct ib_pool_fmr *fmr;
int ret, plen;
page_vec->npages = 0;
page_vec->fake_mr.page_size = SZ_4K;
plen = ib_sg_to_pages(&page_vec->fake_mr, mem->sg,
mem->dma_nents, NULL, iser_set_page);
if (unlikely(plen < mem->dma_nents)) {
iser_err("page vec too short to hold this SG\n");
iser_data_buf_dump(mem, device->ib_device);
iser_dump_page_vec(page_vec);
return -EINVAL;
}
fmr = ib_fmr_pool_map_phys(fmr_pool, page_vec->pages,
page_vec->npages, page_vec->pages[0]);
if (IS_ERR(fmr)) {
ret = PTR_ERR(fmr);
iser_err("ib_fmr_pool_map_phys failed: %d\n", ret);
return ret;
}
reg->sge.lkey = fmr->fmr->lkey;
reg->rkey = fmr->fmr->rkey;
reg->sge.addr = page_vec->fake_mr.iova;
reg->sge.length = page_vec->fake_mr.length;
reg->mem_h = fmr;
iser_dbg("fmr reg: lkey=0x%x, rkey=0x%x, addr=0x%llx,"
" length=0x%x\n", reg->sge.lkey, reg->rkey,
reg->sge.addr, reg->sge.length);
return 0;
}
/**
* Unregister (previosuly registered using FMR) memory.
* If memory is non-FMR does nothing.
*/
void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
enum iser_data_dir cmd_dir)
{
struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
if (!reg->mem_h)
return;
iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n", reg->mem_h);
ib_fmr_pool_unmap((struct ib_pool_fmr *)reg->mem_h);
reg->mem_h = NULL;
}
void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
enum iser_data_dir cmd_dir)
{
struct iser_device *device = iser_task->iser_conn->ib_conn.device;
struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
struct iser_fr_desc *desc;
struct ib_mr_status mr_status;
@@ -312,7 +150,7 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS,
&mr_status);
}
device->reg_ops->reg_desc_put(&iser_task->iser_conn->ib_conn, desc);
iser_reg_desc_put_fr(&iser_task->iser_conn->ib_conn, reg->mem_h);
reg->mem_h = NULL;
}
@@ -509,15 +347,14 @@ iser_reg_data_sg(struct iscsi_iser_task *task,
if (use_dma_key)
return iser_reg_dma(device, mem, reg);
return device->reg_ops->reg_mem(task, mem, &desc->rsc, reg);
return iser_fast_reg_mr(task, mem, &desc->rsc, reg);
}
int iser_reg_rdma_mem(struct iscsi_iser_task *task,
enum iser_data_dir dir,
bool all_imm)
int iser_reg_mem_fastreg(struct iscsi_iser_task *task,
enum iser_data_dir dir,
bool all_imm)
{
struct ib_conn *ib_conn = &task->iser_conn->ib_conn;
struct iser_device *device = ib_conn->device;
struct iser_data_buf *mem = &task->data[dir];
struct iser_mem_reg *reg = &task->rdma_reg[dir];
struct iser_fr_desc *desc = NULL;
@@ -528,7 +365,7 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task,
scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL;
if (!use_dma_key) {
desc = device->reg_ops->reg_desc_get(ib_conn);
desc = iser_reg_desc_get_fr(ib_conn);
reg->mem_h = desc;
}
@@ -549,15 +386,8 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *task,
err_reg:
if (desc)
device->reg_ops->reg_desc_put(ib_conn, desc);
iser_reg_desc_put_fr(ib_conn, desc);
return err;
}
void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
enum iser_data_dir dir)
{
struct iser_device *device = task->iser_conn->ib_conn.device;
device->reg_ops->unreg_mem(task, dir);
}

126
drivers/infiniband/ulp/iser/iser_verbs.c
Fájl megtekintése

@@ -68,11 +68,12 @@ static void iser_event_handler(struct ib_event_handler *handler,
static int iser_create_device_ib_res(struct iser_device *device)
{
struct ib_device *ib_dev = device->ib_device;
int ret, i, max_cqe;
int i, max_cqe;
ret = iser_assign_reg_ops(device);
if (ret)
return ret;
if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {
iser_err("IB device does not support memory registrations\n");
return -1;
}
device->comps_used = min_t(int, num_online_cpus(),
ib_dev->num_comp_vectors);
@@ -147,96 +148,6 @@ static void iser_free_device_ib_res(struct iser_device *device)
device->pd = NULL;
}
/**
* iser_alloc_fmr_pool - Creates FMR pool and page_vector
* @ib_conn: connection RDMA resources
* @cmds_max: max number of SCSI commands for this connection
* @size: max number of pages per map request
*
* Return: 0 on success, or errno code on failure
*/
int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
unsigned cmds_max,
unsigned int size)
{
struct iser_device *device = ib_conn->device;
struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
struct iser_page_vec *page_vec;
struct iser_fr_desc *desc;
struct ib_fmr_pool *fmr_pool;
struct ib_fmr_pool_param params;
int ret;
INIT_LIST_HEAD(&fr_pool->list);
spin_lock_init(&fr_pool->lock);
desc = kzalloc(sizeof(*desc), GFP_KERNEL);
if (!desc)
return -ENOMEM;
page_vec = kmalloc(sizeof(*page_vec) + (sizeof(u64) * size),
GFP_KERNEL);
if (!page_vec) {
ret = -ENOMEM;
goto err_frpl;
}
page_vec->pages = (u64 *)(page_vec + 1);
params.page_shift = ilog2(SZ_4K);
params.max_pages_per_fmr = size;
/* make the pool size twice the max number of SCSI commands *
* the ML is expected to queue, watermark for unmap at 50% */
params.pool_size = cmds_max * 2;
params.dirty_watermark = cmds_max;
params.cache = 0;
params.flush_function = NULL;
params.access = (IB_ACCESS_LOCAL_WRITE |
IB_ACCESS_REMOTE_WRITE |
IB_ACCESS_REMOTE_READ);
fmr_pool = ib_create_fmr_pool(device->pd, &params);
if (IS_ERR(fmr_pool)) {
ret = PTR_ERR(fmr_pool);
iser_err("FMR allocation failed, err %d\n", ret);
goto err_fmr;
}
desc->rsc.page_vec = page_vec;
desc->rsc.fmr_pool = fmr_pool;
list_add(&desc->list, &fr_pool->list);
return 0;
err_fmr:
kfree(page_vec);
err_frpl:
kfree(desc);
return ret;
}
/**
* iser_free_fmr_pool - releases the FMR pool and page vec
* @ib_conn: connection RDMA resources
*/
void iser_free_fmr_pool(struct ib_conn *ib_conn)
{
struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
struct iser_fr_desc *desc;
desc = list_first_entry(&fr_pool->list,
struct iser_fr_desc, list);
list_del(&desc->list);
iser_info("freeing conn %p fmr pool %p\n",
ib_conn, desc->rsc.fmr_pool);
ib_destroy_fmr_pool(desc->rsc.fmr_pool);
kfree(desc->rsc.page_vec);
kfree(desc);
}
static struct iser_fr_desc *
iser_create_fastreg_desc(struct iser_device *device,
struct ib_pd *pd,
@@ -667,13 +578,12 @@ iser_calc_scsi_params(struct iser_conn *iser_conn,
u32 max_num_sg;
/*
* FRs without SG_GAPS or FMRs can only map up to a (device) page per
* entry, but if the first entry is misaligned we'll end up using two
* entries (head and tail) for a single page worth data, so one
* additional entry is required.
* FRs without SG_GAPS can only map up to a (device) page per entry,
* but if the first entry is misaligned we'll end up using two entries
* (head and tail) for a single page worth data, so one additional
* entry is required.
*/
if ((attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) &&
(attr->device_cap_flags & IB_DEVICE_SG_GAPS_REG))
if (attr->device_cap_flags & IB_DEVICE_SG_GAPS_REG)
reserved_mr_pages = 0;
else
reserved_mr_pages = 1;
@@ -684,14 +594,8 @@ iser_calc_scsi_params(struct iser_conn *iser_conn,
max_num_sg = attr->max_fast_reg_page_list_len;
sg_tablesize = DIV_ROUND_UP(max_sectors * SECTOR_SIZE, SZ_4K);
if (attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)
sup_sg_tablesize =
min_t(
uint, ISCSI_ISER_MAX_SG_TABLESIZE,
max_num_sg - reserved_mr_pages);
else
sup_sg_tablesize = ISCSI_ISER_MAX_SG_TABLESIZE;
sup_sg_tablesize = min_t(uint, ISCSI_ISER_MAX_SG_TABLESIZE,
max_num_sg - reserved_mr_pages);
iser_conn->scsi_sg_tablesize = min(sg_tablesize, sup_sg_tablesize);
iser_conn->pages_per_mr =
iser_conn->scsi_sg_tablesize + reserved_mr_pages;
@@ -755,7 +659,7 @@ static void iser_route_handler(struct rdma_cm_id *cma_id)
struct iser_cm_hdr req_hdr;
struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
struct ib_conn *ib_conn = &iser_conn->ib_conn;
struct iser_device *device = ib_conn->device;
struct ib_device *ib_dev = ib_conn->device->ib_device;
if (iser_conn->state != ISER_CONN_PENDING)
/* bailout */
@@ -766,14 +670,14 @@ static void iser_route_handler(struct rdma_cm_id *cma_id)
goto failure;
memset(&conn_param, 0, sizeof conn_param);
conn_param.responder_resources = device->ib_device->attrs.max_qp_rd_atom;
conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom;
conn_param.initiator_depth = 1;
conn_param.retry_count = 7;
conn_param.rnr_retry_count = 6;
memset(&req_hdr, 0, sizeof(req_hdr));
req_hdr.flags = ISER_ZBVA_NOT_SUP;
if (!device->remote_inv_sup)
if (!iser_always_reg)
req_hdr.flags |= ISER_SEND_W_INV_NOT_SUP;
conn_param.private_data = (void *)&req_hdr;
conn_param.private_data_len = sizeof(struct iser_cm_hdr);

5
drivers/infiniband/ulp/isert/ib_isert.c
Fájl megtekintése

@@ -15,6 +15,7 @@
#include <linux/in.h>
#include <linux/in6.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_cm.h>
#include <rdma/rdma_cm.h>
#include <target/target_core_base.h>
#include <target/target_core_fabric.h>
@@ -502,7 +503,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
if (!np->enabled) {
spin_unlock_bh(&np->np_thread_lock);
isert_dbg("iscsi_np is not enabled, reject connect request\n");
return rdma_reject(cma_id, NULL, 0);
return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
}
spin_unlock_bh(&np->np_thread_lock);
@@ -553,7 +554,7 @@ out_rsp_dma_map:
isert_free_login_buf(isert_conn);
out:
kfree(isert_conn);
rdma_reject(cma_id, NULL, 0);
rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
return ret;
}

12
drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
Fájl megtekintése

@@ -113,7 +113,7 @@ struct opa_vnic_vema_port {
struct mutex lock;
};
static void opa_vnic_vema_add_one(struct ib_device *device);
static int opa_vnic_vema_add_one(struct ib_device *device);
static void opa_vnic_vema_rem_one(struct ib_device *device,
void *client_data);
@@ -989,18 +989,18 @@ static void opa_vnic_ctrl_config_dev(struct opa_vnic_ctrl_port *cport, bool en)
*
* Allocate the vnic control port and initialize it.
*/
static void opa_vnic_vema_add_one(struct ib_device *device)
static int opa_vnic_vema_add_one(struct ib_device *device)
{
struct opa_vnic_ctrl_port *cport;
int rc, size = sizeof(*cport);
if (!rdma_cap_opa_vnic(device))
return;
return -EOPNOTSUPP;
size += device->phys_port_cnt * sizeof(struct opa_vnic_vema_port);
cport = kzalloc(size, GFP_KERNEL);
if (!cport)
return;
return -ENOMEM;
cport->num_ports = device->phys_port_cnt;
cport->ibdev = device;
@@ -1012,6 +1012,7 @@ static void opa_vnic_vema_add_one(struct ib_device *device)
ib_set_client_data(device, &opa_vnic_client, cport);
opa_vnic_ctrl_config_dev(cport, true);
return 0;
}
/**
@@ -1026,9 +1027,6 @@ static void opa_vnic_vema_rem_one(struct ib_device *device,
{
struct opa_vnic_ctrl_port *cport = client_data;
if (!cport)
return;
c_info("removing VNIC client\n");
opa_vnic_ctrl_config_dev(cport, false);
vema_unregister(cport);

27
drivers/infiniband/ulp/rtrs/Kconfig Normal file
Fájl megtekintése

@@ -0,0 +1,27 @@
# SPDX-License-Identifier: GPL-2.0-or-later
config INFINIBAND_RTRS
tristate
depends on INFINIBAND_ADDR_TRANS
config INFINIBAND_RTRS_CLIENT
tristate "RTRS client module"
depends on INFINIBAND_ADDR_TRANS
select INFINIBAND_RTRS
help
RDMA transport client module.
RDMA Transport (RTRS) client implements a reliable transport layer
and also multipathing functionality and that it is intended to be
the base layer for a block storage initiator over RDMA.
config INFINIBAND_RTRS_SERVER
tristate "RTRS server module"
depends on INFINIBAND_ADDR_TRANS
select INFINIBAND_RTRS
help
RDMA transport server module.
RDMA Transport (RTRS) server module processing connection and IO
requests received from the RTRS client module, it will pass the
IO requests to its user eg. RNBD_server.

15
drivers/infiniband/ulp/rtrs/Makefile Normal file
Fájl megtekintése

@@ -0,0 +1,15 @@
# SPDX-License-Identifier: GPL-2.0-or-later
rtrs-client-y := rtrs-clt.o \
rtrs-clt-stats.o \
rtrs-clt-sysfs.o
rtrs-server-y := rtrs-srv.o \
rtrs-srv-stats.o \
rtrs-srv-sysfs.o
rtrs-core-y := rtrs.o
obj-$(CONFIG_INFINIBAND_RTRS) += rtrs-core.o
obj-$(CONFIG_INFINIBAND_RTRS_CLIENT) += rtrs-client.o
obj-$(CONFIG_INFINIBAND_RTRS_SERVER) += rtrs-server.o

213
drivers/infiniband/ulp/rtrs/README Normal file
Fájl megtekintése

@@ -0,0 +1,213 @@
****************************
RDMA Transport (RTRS)
****************************
RTRS (RDMA Transport) is a reliable high speed transport library
which provides support to establish optimal number of connections
between client and server machines using RDMA (InfiniBand, RoCE, iWarp)
transport. It is optimized to transfer (read/write) IO blocks.
In its core interface it follows the BIO semantics of providing the
possibility to either write data from an sg list to the remote side
or to request ("read") data transfer from the remote side into a given
sg list.
RTRS provides I/O fail-over and load-balancing capabilities by using
multipath I/O (see "add_path" and "mp_policy" configuration entries in
Documentation/ABI/testing/sysfs-class-rtrs-client).
RTRS is used by the RNBD (RDMA Network Block Device) modules.
==================
Transport protocol
==================
Overview
--------
An established connection between a client and a server is called rtrs
session. A session is associated with a set of memory chunks reserved on the
server side for a given client for rdma transfer. A session
consists of multiple paths, each representing a separate physical link
between client and server. Those are used for load balancing and failover.
Each path consists of as many connections (QPs) as there are cpus on
the client.
When processing an incoming write or read request, rtrs client uses memory
chunks reserved for him on the server side. Their number, size and addresses
need to be exchanged between client and server during the connection
establishment phase. Apart from the memory related information client needs to
inform the server about the session name and identify each path and connection
individually.
On an established session client sends to server write or read messages.
Server uses immediate field to tell the client which request is being
acknowledged and for errno. Client uses immediate field to tell the server
which of the memory chunks has been accessed and at which offset the message
can be found.
Module parameter always_invalidate is introduced for the security problem
discussed in LPC RDMA MC 2019. When always_invalidate=Y, on the server side we
invalidate each rdma buffer before we hand it over to RNBD server and
then pass it to the block layer. A new rkey is generated and registered for the
buffer after it returns back from the block layer and RNBD server.
The new rkey is sent back to the client along with the IO result.
The procedure is the default behaviour of the driver. This invalidation and
registration on each IO causes performance drop of up to 20%. A user of the
driver may choose to load the modules with this mechanism switched off
(always_invalidate=N), if he understands and can take the risk of a malicious
client being able to corrupt memory of a server it is connected to. This might
be a reasonable option in a scenario where all the clients and all the servers
are located within a secure datacenter.
Connection establishment
------------------------
1. Client starts establishing connections belonging to a path of a session one
by one via attaching RTRS_MSG_CON_REQ messages to the rdma_connect requests.
Those include uuid of the session and uuid of the path to be
established. They are used by the server to find a persisting session/path or
to create a new one when necessary. The message also contains the protocol
version and magic for compatibility, total number of connections per session
(as many as cpus on the client), the id of the current connection and
the reconnect counter, which is used to resolve the situations where
client is trying to reconnect a path, while server is still destroying the old
one.
2. Server accepts the connection requests one by one and attaches
RTRS_MSG_CONN_RSP messages to the rdma_accept. Apart from magic and
protocol version, the messages include error code, queue depth supported by
the server (number of memory chunks which are going to be allocated for that
session) and the maximum size of one io, RTRS_MSG_NEW_RKEY_F flags is set
when always_invalidate=Y.
3. After all connections of a path are established client sends to server the
RTRS_MSG_INFO_REQ message, containing the name of the session. This message
requests the address information from the server.
4. Server replies to the session info request message with RTRS_MSG_INFO_RSP,
which contains the addresses and keys of the RDMA buffers allocated for that
session.
5. Session becomes connected after all paths to be established are connected
(i.e. steps 1-4 finished for all paths requested for a session)
6. Server and client exchange periodically heartbeat messages (empty rdma
messages with an immediate field) which are used to detect a crash on remote
side or network outage in an absence of IO.
7. On any RDMA related error or in the case of a heartbeat timeout, the
corresponding path is disconnected, all the inflight IO are failed over to a
healthy path, if any, and the reconnect mechanism is triggered.
CLT SRV
*for each connection belonging to a path and for each path:
RTRS_MSG_CON_REQ ------------------->
<------------------- RTRS_MSG_CON_RSP
...
*after all connections are established:
RTRS_MSG_INFO_REQ ------------------->
<------------------- RTRS_MSG_INFO_RSP
*heartbeat is started from both sides:
-------------------> [RTRS_HB_MSG_IMM]
[RTRS_HB_MSG_ACK] <-------------------
[RTRS_HB_MSG_IMM] <-------------------
-------------------> [RTRS_HB_MSG_ACK]
IO path
-------
* Write (always_invalidate=N) *
1. When processing a write request client selects one of the memory chunks
on the server side and rdma writes there the user data, user header and the
RTRS_MSG_RDMA_WRITE message. Apart from the type (write), the message only
contains size of the user header. The client tells the server which chunk has
been accessed and at what offset the RTRS_MSG_RDMA_WRITE can be found by
using the IMM field.
2. When confirming a write request server sends an "empty" rdma message with
an immediate field. The 32 bit field is used to specify the outstanding
inflight IO and for the error code.
CLT SRV
usr_data + usr_hdr + rtrs_msg_rdma_write -----------------> [RTRS_IO_REQ_IMM]
[RTRS_IO_RSP_IMM] <----------------- (id + errno)
* Write (always_invalidate=Y) *
1. When processing a write request client selects one of the memory chunks
on the server side and rdma writes there the user data, user header and the
RTRS_MSG_RDMA_WRITE message. Apart from the type (write), the message only
contains size of the user header. The client tells the server which chunk has
been accessed and at what offset the RTRS_MSG_RDMA_WRITE can be found by
using the IMM field, Server invalidate rkey associated to the memory chunks
first, when it finishes, pass the IO to RNBD server module.
2. When confirming a write request server sends an "empty" rdma message with
an immediate field. The 32 bit field is used to specify the outstanding
inflight IO and for the error code. The new rkey is sent back using
SEND_WITH_IMM WR, client When it recived new rkey message, it validates
the message and finished IO after update rkey for the rbuffer, then post
back the recv buffer for later use.
CLT SRV
usr_data + usr_hdr + rtrs_msg_rdma_write -----------------> [RTRS_IO_REQ_IMM]
[RTRS_MSG_RKEY_RSP] <----------------- (RTRS_MSG_RKEY_RSP)
[RTRS_IO_RSP_IMM] <----------------- (id + errno)
* Read (always_invalidate=N)*
1. When processing a read request client selects one of the memory chunks
on the server side and rdma writes there the user header and the
RTRS_MSG_RDMA_READ message. This message contains the type (read), size of
the user header, flags (specifying if memory invalidation is necessary) and the
list of addresses along with keys for the data to be read into.
2. When confirming a read request server transfers the requested data first,
attaches an invalidation message if requested and finally an "empty" rdma
message with an immediate field. The 32 bit field is used to specify the
outstanding inflight IO and the error code.
CLT SRV
usr_hdr + rtrs_msg_rdma_read --------------> [RTRS_IO_REQ_IMM]
[RTRS_IO_RSP_IMM] <-------------- usr_data + (id + errno)
or in case client requested invalidation:
[RTRS_IO_RSP_IMM_W_INV] <-------------- usr_data + (INV) + (id + errno)
* Read (always_invalidate=Y)*
1. When processing a read request client selects one of the memory chunks
on the server side and rdma writes there the user header and the
RTRS_MSG_RDMA_READ message. This message contains the type (read), size of
the user header, flags (specifying if memory invalidation is necessary) and the
list of addresses along with keys for the data to be read into.
Server invalidate rkey associated to the memory chunks first, when it finishes,
passes the IO to RNBD server module.
2. When confirming a read request server transfers the requested data first,
attaches an invalidation message if requested and finally an "empty" rdma
message with an immediate field. The 32 bit field is used to specify the
outstanding inflight IO and the error code. The new rkey is sent back using
SEND_WITH_IMM WR, client When it recived new rkey message, it validates
the message and finished IO after update rkey for the rbuffer, then post
back the recv buffer for later use.
CLT SRV
usr_hdr + rtrs_msg_rdma_read --------------> [RTRS_IO_REQ_IMM]
[RTRS_IO_RSP_IMM] <-------------- usr_data + (id + errno)
[RTRS_MSG_RKEY_RSP] <----------------- (RTRS_MSG_RKEY_RSP)
or in case client requested invalidation:
[RTRS_IO_RSP_IMM_W_INV] <-------------- usr_data + (INV) + (id + errno)
=========================================
Contributors List(in alphabetical order)
=========================================
Danil Kipnis <danil.kipnis@profitbricks.com>
Fabian Holler <mail@fholler.de>
Guoqing Jiang <guoqing.jiang@cloud.ionos.com>
Jack Wang <jinpu.wang@profitbricks.com>
Kleber Souza <kleber.souza@profitbricks.com>
Lutz Pogrell <lutz.pogrell@cloud.ionos.com>
Milind Dumbare <Milind.dumbare@gmail.com>
Roman Penyaev <roman.penyaev@profitbricks.com>

200
drivers/infiniband/ulp/rtrs/rtrs-clt-stats.c Normal file
Fájl megtekintése

@@ -0,0 +1,200 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#undef pr_fmt
#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
#include "rtrs-clt.h"
void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con)
{
struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
struct rtrs_clt_stats *stats = sess->stats;
struct rtrs_clt_stats_pcpu *s;
int cpu;
cpu = raw_smp_processor_id();
s = this_cpu_ptr(stats->pcpu_stats);
if (unlikely(con->cpu != cpu)) {
s->cpu_migr.to++;
/* Careful here, override s pointer */
s = per_cpu_ptr(stats->pcpu_stats, con->cpu);
atomic_inc(&s->cpu_migr.from);
}
}
void rtrs_clt_inc_failover_cnt(struct rtrs_clt_stats *stats)
{
struct rtrs_clt_stats_pcpu *s;
s = this_cpu_ptr(stats->pcpu_stats);
s->rdma.failover_cnt++;
}
int rtrs_clt_stats_migration_cnt_to_str(struct rtrs_clt_stats *stats,
char *buf, size_t len)
{
struct rtrs_clt_stats_pcpu *s;
size_t used;
int cpu;
used = scnprintf(buf, len, " ");
for_each_possible_cpu(cpu)
used += scnprintf(buf + used, len - used, " CPU%u", cpu);
used += scnprintf(buf + used, len - used, "\nfrom:");
for_each_possible_cpu(cpu) {
s = per_cpu_ptr(stats->pcpu_stats, cpu);
used += scnprintf(buf + used, len - used, " %d",
atomic_read(&s->cpu_migr.from));
}
used += scnprintf(buf + used, len - used, "\nto :");
for_each_possible_cpu(cpu) {
s = per_cpu_ptr(stats->pcpu_stats, cpu);
used += scnprintf(buf + used, len - used, " %d",
s->cpu_migr.to);
}
used += scnprintf(buf + used, len - used, "\n");
return used;
}
int rtrs_clt_stats_reconnects_to_str(struct rtrs_clt_stats *stats, char *buf,
size_t len)
{
return scnprintf(buf, len, "%d %d\n",
stats->reconnects.successful_cnt,
stats->reconnects.fail_cnt);
}
ssize_t rtrs_clt_stats_rdma_to_str(struct rtrs_clt_stats *stats,
char *page, size_t len)
{
struct rtrs_clt_stats_rdma sum;
struct rtrs_clt_stats_rdma *r;
int cpu;
memset(&sum, 0, sizeof(sum));
for_each_possible_cpu(cpu) {
r = &per_cpu_ptr(stats->pcpu_stats, cpu)->rdma;
sum.dir[READ].cnt += r->dir[READ].cnt;
sum.dir[READ].size_total += r->dir[READ].size_total;
sum.dir[WRITE].cnt += r->dir[WRITE].cnt;
sum.dir[WRITE].size_total += r->dir[WRITE].size_total;
sum.failover_cnt += r->failover_cnt;
}
return scnprintf(page, len, "%llu %llu %llu %llu %u %llu\n",
sum.dir[READ].cnt, sum.dir[READ].size_total,
sum.dir[WRITE].cnt, sum.dir[WRITE].size_total,
atomic_read(&stats->inflight), sum.failover_cnt);
}
ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *s,
char *page, size_t len)
{
return scnprintf(page, len, "echo 1 to reset all statistics\n");
}
int rtrs_clt_reset_rdma_stats(struct rtrs_clt_stats *stats, bool enable)
{
struct rtrs_clt_stats_pcpu *s;
int cpu;
if (!enable)
return -EINVAL;
for_each_possible_cpu(cpu) {
s = per_cpu_ptr(stats->pcpu_stats, cpu);
memset(&s->rdma, 0, sizeof(s->rdma));
}
return 0;
}
int rtrs_clt_reset_cpu_migr_stats(struct rtrs_clt_stats *stats, bool enable)
{
struct rtrs_clt_stats_pcpu *s;
int cpu;
if (!enable)
return -EINVAL;
for_each_possible_cpu(cpu) {
s = per_cpu_ptr(stats->pcpu_stats, cpu);
memset(&s->cpu_migr, 0, sizeof(s->cpu_migr));
}
return 0;
}
int rtrs_clt_reset_reconnects_stat(struct rtrs_clt_stats *stats, bool enable)
{
if (!enable)
return -EINVAL;
memset(&stats->reconnects, 0, sizeof(stats->reconnects));
return 0;
}
int rtrs_clt_reset_all_stats(struct rtrs_clt_stats *s, bool enable)
{
if (enable) {
rtrs_clt_reset_rdma_stats(s, enable);
rtrs_clt_reset_cpu_migr_stats(s, enable);
rtrs_clt_reset_reconnects_stat(s, enable);
atomic_set(&s->inflight, 0);
return 0;
}
return -EINVAL;
}
static inline void rtrs_clt_update_rdma_stats(struct rtrs_clt_stats *stats,
size_t size, int d)
{
struct rtrs_clt_stats_pcpu *s;
s = this_cpu_ptr(stats->pcpu_stats);
s->rdma.dir[d].cnt++;
s->rdma.dir[d].size_total += size;
}
void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir)
{
struct rtrs_clt_con *con = req->con;
struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess);
struct rtrs_clt_stats *stats = sess->stats;
unsigned int len;
len = req->usr_len + req->data_len;
rtrs_clt_update_rdma_stats(stats, len, dir);
if (sess->clt->mp_policy == MP_POLICY_MIN_INFLIGHT)
atomic_inc(&stats->inflight);
}
int rtrs_clt_init_stats(struct rtrs_clt_stats *stats)
{
stats->pcpu_stats = alloc_percpu(typeof(*stats->pcpu_stats));
if (!stats->pcpu_stats)
return -ENOMEM;
/*
* successful_cnt will be set to 0 after session
* is established for the first time
*/
stats->reconnects.successful_cnt = -1;
return 0;
}

483
drivers/infiniband/ulp/rtrs/rtrs-clt-sysfs.c Normal file
Fájl megtekintése

@@ -0,0 +1,483 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#undef pr_fmt
#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
#include "rtrs-pri.h"
#include "rtrs-clt.h"
#include "rtrs-log.h"
#define MIN_MAX_RECONN_ATT -1
#define MAX_MAX_RECONN_ATT 9999
static void rtrs_clt_sess_release(struct kobject *kobj)
{
struct rtrs_clt_sess *sess;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
free_sess(sess);
}
static struct kobj_type ktype_sess = {
.sysfs_ops = &kobj_sysfs_ops,
.release = rtrs_clt_sess_release
};
static void rtrs_clt_sess_stats_release(struct kobject *kobj)
{
struct rtrs_clt_stats *stats;
stats = container_of(kobj, struct rtrs_clt_stats, kobj_stats);
free_percpu(stats->pcpu_stats);
kfree(stats);
}
static struct kobj_type ktype_stats = {
.sysfs_ops = &kobj_sysfs_ops,
.release = rtrs_clt_sess_stats_release,
};
static ssize_t max_reconnect_attempts_show(struct device *dev,
struct device_attribute *attr,
char *page)
{
struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);
return sprintf(page, "%d\n", rtrs_clt_get_max_reconnect_attempts(clt));
}
static ssize_t max_reconnect_attempts_store(struct device *dev,
struct device_attribute *attr,
const char *buf,
size_t count)
{
int value;
int ret;
struct rtrs_clt *clt = container_of(dev, struct rtrs_clt, dev);
ret = kstrtoint(buf, 10, &value);
if (ret) {
rtrs_err(clt, "%s: failed to convert string '%s' to int\n",
attr->attr.name, buf);
return ret;
}
if (value > MAX_MAX_RECONN_ATT ||
value < MIN_MAX_RECONN_ATT) {
rtrs_err(clt,
"%s: invalid range (provided: '%s', accepted: min: %d, max: %d)\n",
attr->attr.name, buf, MIN_MAX_RECONN_ATT,
MAX_MAX_RECONN_ATT);
return -EINVAL;
}
rtrs_clt_set_max_reconnect_attempts(clt, value);
return count;
}
static DEVICE_ATTR_RW(max_reconnect_attempts);
static ssize_t mpath_policy_show(struct device *dev,
struct device_attribute *attr,
char *page)
{
struct rtrs_clt *clt;
clt = container_of(dev, struct rtrs_clt, dev);
switch (clt->mp_policy) {
case MP_POLICY_RR:
return sprintf(page, "round-robin (RR: %d)\n", clt->mp_policy);
case MP_POLICY_MIN_INFLIGHT:
return sprintf(page, "min-inflight (MI: %d)\n", clt->mp_policy);
default:
return sprintf(page, "Unknown (%d)\n", clt->mp_policy);
}
}
static ssize_t mpath_policy_store(struct device *dev,
struct device_attribute *attr,
const char *buf,
size_t count)
{
struct rtrs_clt *clt;
int value;
int ret;
clt = container_of(dev, struct rtrs_clt, dev);
ret = kstrtoint(buf, 10, &value);
if (!ret && (value == MP_POLICY_RR ||
value == MP_POLICY_MIN_INFLIGHT)) {
clt->mp_policy = value;
return count;
}
if (!strncasecmp(buf, "round-robin", 11) ||
!strncasecmp(buf, "rr", 2))
clt->mp_policy = MP_POLICY_RR;
else if (!strncasecmp(buf, "min-inflight", 12) ||
!strncasecmp(buf, "mi", 2))
clt->mp_policy = MP_POLICY_MIN_INFLIGHT;
else
return -EINVAL;
return count;
}
static DEVICE_ATTR_RW(mpath_policy);
static ssize_t add_path_show(struct device *dev,
struct device_attribute *attr, char *page)
{
return scnprintf(page, PAGE_SIZE,
"Usage: echo [<source addr>@]<destination addr> > %s\n\n*addr ::= [ ip:<ipv4|ipv6> | gid:<gid> ]\n",
attr->attr.name);
}
static ssize_t add_path_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct sockaddr_storage srcaddr, dstaddr;
struct rtrs_addr addr = {
.src = &srcaddr,
.dst = &dstaddr
};
struct rtrs_clt *clt;
const char *nl;
size_t len;
int err;
clt = container_of(dev, struct rtrs_clt, dev);
nl = strchr(buf, '\n');
if (nl)
len = nl - buf;
else
len = count;
err = rtrs_addr_to_sockaddr(buf, len, clt->port, &addr);
if (err)
return -EINVAL;
err = rtrs_clt_create_path_from_sysfs(clt, &addr);
if (err)
return err;
return count;
}
static DEVICE_ATTR_RW(add_path);
static ssize_t rtrs_clt_state_show(struct kobject *kobj,
struct kobj_attribute *attr, char *page)
{
struct rtrs_clt_sess *sess;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
if (sess->state == RTRS_CLT_CONNECTED)
return sprintf(page, "connected\n");
return sprintf(page, "disconnected\n");
}
static struct kobj_attribute rtrs_clt_state_attr =
__ATTR(state, 0444, rtrs_clt_state_show, NULL);
static ssize_t rtrs_clt_reconnect_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
attr->attr.name);
}
static ssize_t rtrs_clt_reconnect_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
struct rtrs_clt_sess *sess;
int ret;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
if (!sysfs_streq(buf, "1")) {
rtrs_err(sess->clt, "%s: unknown value: '%s'\n",
attr->attr.name, buf);
return -EINVAL;
}
ret = rtrs_clt_reconnect_from_sysfs(sess);
if (ret)
return ret;
return count;
}
static struct kobj_attribute rtrs_clt_reconnect_attr =
__ATTR(reconnect, 0644, rtrs_clt_reconnect_show,
rtrs_clt_reconnect_store);
static ssize_t rtrs_clt_disconnect_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
attr->attr.name);
}
static ssize_t rtrs_clt_disconnect_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
struct rtrs_clt_sess *sess;
int ret;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
if (!sysfs_streq(buf, "1")) {
rtrs_err(sess->clt, "%s: unknown value: '%s'\n",
attr->attr.name, buf);
return -EINVAL;
}
ret = rtrs_clt_disconnect_from_sysfs(sess);
if (ret)
return ret;
return count;
}
static struct kobj_attribute rtrs_clt_disconnect_attr =
__ATTR(disconnect, 0644, rtrs_clt_disconnect_show,
rtrs_clt_disconnect_store);
static ssize_t rtrs_clt_remove_path_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
attr->attr.name);
}
static ssize_t rtrs_clt_remove_path_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
struct rtrs_clt_sess *sess;
int ret;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
if (!sysfs_streq(buf, "1")) {
rtrs_err(sess->clt, "%s: unknown value: '%s'\n",
attr->attr.name, buf);
return -EINVAL;
}
ret = rtrs_clt_remove_path_from_sysfs(sess, &attr->attr);
if (ret)
return ret;
return count;
}
static struct kobj_attribute rtrs_clt_remove_path_attr =
__ATTR(remove_path, 0644, rtrs_clt_remove_path_show,
rtrs_clt_remove_path_store);
STAT_ATTR(struct rtrs_clt_stats, cpu_migration,
rtrs_clt_stats_migration_cnt_to_str,
rtrs_clt_reset_cpu_migr_stats);
STAT_ATTR(struct rtrs_clt_stats, reconnects,
rtrs_clt_stats_reconnects_to_str,
rtrs_clt_reset_reconnects_stat);
STAT_ATTR(struct rtrs_clt_stats, rdma,
rtrs_clt_stats_rdma_to_str,
rtrs_clt_reset_rdma_stats);
STAT_ATTR(struct rtrs_clt_stats, reset_all,
rtrs_clt_reset_all_help,
rtrs_clt_reset_all_stats);
static struct attribute *rtrs_clt_stats_attrs[] = {
&cpu_migration_attr.attr,
&reconnects_attr.attr,
&rdma_attr.attr,
&reset_all_attr.attr,
NULL
};
static struct attribute_group rtrs_clt_stats_attr_group = {
.attrs = rtrs_clt_stats_attrs,
};
static ssize_t rtrs_clt_hca_port_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_clt_sess *sess;
sess = container_of(kobj, typeof(*sess), kobj);
return scnprintf(page, PAGE_SIZE, "%u\n", sess->hca_port);
}
static struct kobj_attribute rtrs_clt_hca_port_attr =
__ATTR(hca_port, 0444, rtrs_clt_hca_port_show, NULL);
static ssize_t rtrs_clt_hca_name_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_clt_sess *sess;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
return scnprintf(page, PAGE_SIZE, "%s\n", sess->hca_name);
}
static struct kobj_attribute rtrs_clt_hca_name_attr =
__ATTR(hca_name, 0444, rtrs_clt_hca_name_show, NULL);
static ssize_t rtrs_clt_src_addr_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_clt_sess *sess;
int cnt;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
page, PAGE_SIZE);
return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
}
static struct kobj_attribute rtrs_clt_src_addr_attr =
__ATTR(src_addr, 0444, rtrs_clt_src_addr_show, NULL);
static ssize_t rtrs_clt_dst_addr_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_clt_sess *sess;
int cnt;
sess = container_of(kobj, struct rtrs_clt_sess, kobj);
cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
page, PAGE_SIZE);
return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
}
static struct kobj_attribute rtrs_clt_dst_addr_attr =
__ATTR(dst_addr, 0444, rtrs_clt_dst_addr_show, NULL);
static struct attribute *rtrs_clt_sess_attrs[] = {
&rtrs_clt_hca_name_attr.attr,
&rtrs_clt_hca_port_attr.attr,
&rtrs_clt_src_addr_attr.attr,
&rtrs_clt_dst_addr_attr.attr,
&rtrs_clt_state_attr.attr,
&rtrs_clt_reconnect_attr.attr,
&rtrs_clt_disconnect_attr.attr,
&rtrs_clt_remove_path_attr.attr,
NULL,
};
static struct attribute_group rtrs_clt_sess_attr_group = {
.attrs = rtrs_clt_sess_attrs,
};
int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess)
{
struct rtrs_clt *clt = sess->clt;
char str[NAME_MAX];
int err, cnt;
cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
str, sizeof(str));
cnt += scnprintf(str + cnt, sizeof(str) - cnt, "@");
sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
str + cnt, sizeof(str) - cnt);
err = kobject_init_and_add(&sess->kobj, &ktype_sess, clt->kobj_paths,
"%s", str);
if (err) {
pr_err("kobject_init_and_add: %d\n", err);
return err;
}
err = sysfs_create_group(&sess->kobj, &rtrs_clt_sess_attr_group);
if (err) {
pr_err("sysfs_create_group(): %d\n", err);
goto put_kobj;
}
err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats,
&sess->kobj, "stats");
if (err) {
pr_err("kobject_init_and_add: %d\n", err);
goto remove_group;
}
err = sysfs_create_group(&sess->stats->kobj_stats,
&rtrs_clt_stats_attr_group);
if (err) {
pr_err("failed to create stats sysfs group, err: %d\n", err);
goto put_kobj_stats;
}
return 0;
put_kobj_stats:
kobject_del(&sess->stats->kobj_stats);
kobject_put(&sess->stats->kobj_stats);
remove_group:
sysfs_remove_group(&sess->kobj, &rtrs_clt_sess_attr_group);
put_kobj:
kobject_del(&sess->kobj);
kobject_put(&sess->kobj);
return err;
}
void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,
const struct attribute *sysfs_self)
{
kobject_del(&sess->stats->kobj_stats);
kobject_put(&sess->stats->kobj_stats);
if (sysfs_self)
sysfs_remove_file_self(&sess->kobj, sysfs_self);
kobject_del(&sess->kobj);
}
static struct attribute *rtrs_clt_attrs[] = {
&dev_attr_max_reconnect_attempts.attr,
&dev_attr_mpath_policy.attr,
&dev_attr_add_path.attr,
NULL,
};
static struct attribute_group rtrs_clt_attr_group = {
.attrs = rtrs_clt_attrs,
};
int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt)
{
return sysfs_create_group(&clt->dev.kobj, &rtrs_clt_attr_group);
}
void rtrs_clt_destroy_sysfs_root_folders(struct rtrs_clt *clt)
{
if (clt->kobj_paths) {
kobject_del(clt->kobj_paths);
kobject_put(clt->kobj_paths);
}
}
void rtrs_clt_destroy_sysfs_root_files(struct rtrs_clt *clt)
{
sysfs_remove_group(&clt->dev.kobj, &rtrs_clt_attr_group);
}

2992
drivers/infiniband/ulp/rtrs/rtrs-clt.c Normal file
Fájl megtekintése

A különbségek nem kerülnek megjelenítésre, mivel a fájl túl nagy Load Diff

252
drivers/infiniband/ulp/rtrs/rtrs-clt.h Normal file
Fájl megtekintése

@@ -0,0 +1,252 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#ifndef RTRS_CLT_H
#define RTRS_CLT_H
#include <linux/device.h>
#include "rtrs-pri.h"
/**
* enum rtrs_clt_state - Client states.
*/
enum rtrs_clt_state {
RTRS_CLT_CONNECTING,
RTRS_CLT_CONNECTING_ERR,
RTRS_CLT_RECONNECTING,
RTRS_CLT_CONNECTED,
RTRS_CLT_CLOSING,
RTRS_CLT_CLOSED,
RTRS_CLT_DEAD,
};
enum rtrs_mp_policy {
MP_POLICY_RR,
MP_POLICY_MIN_INFLIGHT,
};
/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
struct rtrs_clt_stats_reconnects {
int successful_cnt;
int fail_cnt;
};
/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
struct rtrs_clt_stats_cpu_migr {
atomic_t from;
int to;
};
/* stats for Read and write operation.
* see Documentation/ABI/testing/sysfs-class-rtrs-client for details
*/
struct rtrs_clt_stats_rdma {
struct {
u64 cnt;
u64 size_total;
} dir[2];
u64 failover_cnt;
};
struct rtrs_clt_stats_pcpu {
struct rtrs_clt_stats_cpu_migr cpu_migr;
struct rtrs_clt_stats_rdma rdma;
};
struct rtrs_clt_stats {
struct kobject kobj_stats;
struct rtrs_clt_stats_pcpu __percpu *pcpu_stats;
struct rtrs_clt_stats_reconnects reconnects;
atomic_t inflight;
};
struct rtrs_clt_con {
struct rtrs_con c;
struct rtrs_iu *rsp_ius;
u32 queue_size;
unsigned int cpu;
atomic_t io_cnt;
int cm_err;
};
/**
* rtrs_permit - permits the memory allocation for future RDMA operation.
* Combine with irq pinning to keep IO on same CPU.
*/
struct rtrs_permit {
enum rtrs_clt_con_type con_type;
unsigned int cpu_id;
unsigned int mem_id;
unsigned int mem_off;
};
/**
* rtrs_clt_io_req - describes one inflight IO request
*/
struct rtrs_clt_io_req {
struct list_head list;
struct rtrs_iu *iu;
struct scatterlist *sglist; /* list holding user data */
unsigned int sg_cnt;
unsigned int sg_size;
unsigned int data_len;
unsigned int usr_len;
void *priv;
bool in_use;
struct rtrs_clt_con *con;
struct rtrs_sg_desc *desc;
struct ib_sge *sge;
struct rtrs_permit *permit;
enum dma_data_direction dir;
void (*conf)(void *priv, int errno);
unsigned long start_jiffies;
struct ib_mr *mr;
struct ib_cqe inv_cqe;
struct completion inv_comp;
int inv_errno;
bool need_inv_comp;
bool need_inv;
};
struct rtrs_rbuf {
u64 addr;
u32 rkey;
};
struct rtrs_clt_sess {
struct rtrs_sess s;
struct rtrs_clt *clt;
wait_queue_head_t state_wq;
enum rtrs_clt_state state;
atomic_t connected_cnt;
struct mutex init_mutex;
struct rtrs_clt_io_req *reqs;
struct delayed_work reconnect_dwork;
struct work_struct close_work;
unsigned int reconnect_attempts;
bool established;
struct rtrs_rbuf *rbufs;
size_t max_io_size;
u32 max_hdr_size;
u32 chunk_size;
size_t queue_depth;
u32 max_pages_per_mr;
int max_send_sge;
u32 flags;
struct kobject kobj;
struct rtrs_clt_stats *stats;
/* cache hca_port and hca_name to display in sysfs */
u8 hca_port;
char hca_name[IB_DEVICE_NAME_MAX];
struct list_head __percpu
*mp_skip_entry;
};
struct rtrs_clt {
struct list_head paths_list; /* rcu protected list */
size_t paths_num;
struct rtrs_clt_sess
__rcu * __percpu *pcpu_path;
uuid_t paths_uuid;
int paths_up;
struct mutex paths_mutex;
struct mutex paths_ev_mutex;
char sessname[NAME_MAX];
u16 port;
unsigned int max_reconnect_attempts;
unsigned int reconnect_delay_sec;
unsigned int max_segments;
size_t max_segment_size;
void *permits;
unsigned long *permits_map;
size_t queue_depth;
size_t max_io_size;
wait_queue_head_t permits_wait;
size_t pdu_sz;
void *priv;
void (*link_ev)(void *priv,
enum rtrs_clt_link_ev ev);
struct device dev;
struct kobject *kobj_paths;
enum rtrs_mp_policy mp_policy;
};
static inline struct rtrs_clt_con *to_clt_con(struct rtrs_con *c)
{
return container_of(c, struct rtrs_clt_con, c);
}
static inline struct rtrs_clt_sess *to_clt_sess(struct rtrs_sess *s)
{
return container_of(s, struct rtrs_clt_sess, s);
}
static inline int permit_size(struct rtrs_clt *clt)
{
return sizeof(struct rtrs_permit) + clt->pdu_sz;
}
static inline struct rtrs_permit *get_permit(struct rtrs_clt *clt, int idx)
{
return (struct rtrs_permit *)(clt->permits + permit_size(clt) * idx);
}
int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess);
int rtrs_clt_disconnect_from_sysfs(struct rtrs_clt_sess *sess);
int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,
struct rtrs_addr *addr);
int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess,
const struct attribute *sysfs_self);
void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value);
int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt);
void free_sess(struct rtrs_clt_sess *sess);
/* rtrs-clt-stats.c */
int rtrs_clt_init_stats(struct rtrs_clt_stats *stats);
void rtrs_clt_inc_failover_cnt(struct rtrs_clt_stats *s);
void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con);
void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir);
int rtrs_clt_reset_rdma_lat_distr_stats(struct rtrs_clt_stats *stats,
bool enable);
ssize_t rtrs_clt_stats_rdma_lat_distr_to_str(struct rtrs_clt_stats *stats,
char *page, size_t len);
int rtrs_clt_reset_cpu_migr_stats(struct rtrs_clt_stats *stats, bool enable);
int rtrs_clt_stats_migration_cnt_to_str(struct rtrs_clt_stats *stats, char *buf,
size_t len);
int rtrs_clt_reset_reconnects_stat(struct rtrs_clt_stats *stats, bool enable);
int rtrs_clt_stats_reconnects_to_str(struct rtrs_clt_stats *stats, char *buf,
size_t len);
int rtrs_clt_reset_wc_comp_stats(struct rtrs_clt_stats *stats, bool enable);
int rtrs_clt_stats_wc_completion_to_str(struct rtrs_clt_stats *stats, char *buf,
size_t len);
int rtrs_clt_reset_rdma_stats(struct rtrs_clt_stats *stats, bool enable);
ssize_t rtrs_clt_stats_rdma_to_str(struct rtrs_clt_stats *stats,
char *page, size_t len);
int rtrs_clt_reset_all_stats(struct rtrs_clt_stats *stats, bool enable);
ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *stats,
char *page, size_t len);
/* rtrs-clt-sysfs.c */
int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt);
void rtrs_clt_destroy_sysfs_root_folders(struct rtrs_clt *clt);
void rtrs_clt_destroy_sysfs_root_files(struct rtrs_clt *clt);
int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess);
void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,
const struct attribute *sysfs_self);
#endif /* RTRS_CLT_H */

28
drivers/infiniband/ulp/rtrs/rtrs-log.h Normal file
Fájl megtekintése

@@ -0,0 +1,28 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#ifndef RTRS_LOG_H
#define RTRS_LOG_H
#define rtrs_log(fn, obj, fmt, ...) \
fn("<%s>: " fmt, obj->sessname, ##__VA_ARGS__)
#define rtrs_err(obj, fmt, ...) \
rtrs_log(pr_err, obj, fmt, ##__VA_ARGS__)
#define rtrs_err_rl(obj, fmt, ...) \
rtrs_log(pr_err_ratelimited, obj, fmt, ##__VA_ARGS__)
#define rtrs_wrn(obj, fmt, ...) \
rtrs_log(pr_warn, obj, fmt, ##__VA_ARGS__)
#define rtrs_wrn_rl(obj, fmt, ...) \
rtrs_log(pr_warn_ratelimited, obj, fmt, ##__VA_ARGS__)
#define rtrs_info(obj, fmt, ...) \
rtrs_log(pr_info, obj, fmt, ##__VA_ARGS__)
#define rtrs_info_rl(obj, fmt, ...) \
rtrs_log(pr_info_ratelimited, obj, fmt, ##__VA_ARGS__)
#endif /* RTRS_LOG_H */

399
drivers/infiniband/ulp/rtrs/rtrs-pri.h Normal file
Fájl megtekintése

@@ -0,0 +1,399 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#ifndef RTRS_PRI_H
#define RTRS_PRI_H
#include <linux/uuid.h>
#include <rdma/rdma_cm.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib.h>
#include "rtrs.h"
#define RTRS_PROTO_VER_MAJOR 2
#define RTRS_PROTO_VER_MINOR 0
#define RTRS_PROTO_VER_STRING __stringify(RTRS_PROTO_VER_MAJOR) "." \
__stringify(RTRS_PROTO_VER_MINOR)
enum rtrs_imm_const {
MAX_IMM_TYPE_BITS = 4,
MAX_IMM_TYPE_MASK = ((1 << MAX_IMM_TYPE_BITS) - 1),
MAX_IMM_PAYL_BITS = 28,
MAX_IMM_PAYL_MASK = ((1 << MAX_IMM_PAYL_BITS) - 1),
};
enum rtrs_imm_type {
RTRS_IO_REQ_IMM = 0, /* client to server */
RTRS_IO_RSP_IMM = 1, /* server to client */
RTRS_IO_RSP_W_INV_IMM = 2, /* server to client */
RTRS_HB_MSG_IMM = 8, /* HB: HeartBeat */
RTRS_HB_ACK_IMM = 9,
RTRS_LAST_IMM,
};
enum {
SERVICE_CON_QUEUE_DEPTH = 512,
MAX_PATHS_NUM = 128,
/*
* With the size of struct rtrs_permit allocated on the client, 4K
* is the maximum number of rtrs_permits we can allocate. This number is
* also used on the client to allocate the IU for the user connection
* to receive the RDMA addresses from the server.
*/
MAX_SESS_QUEUE_DEPTH = 4096,
RTRS_HB_INTERVAL_MS = 5000,
RTRS_HB_MISSED_MAX = 5,
RTRS_MAGIC = 0x1BBD,
RTRS_PROTO_VER = (RTRS_PROTO_VER_MAJOR << 8) | RTRS_PROTO_VER_MINOR,
};
struct rtrs_ib_dev;
struct rtrs_rdma_dev_pd_ops {
struct rtrs_ib_dev *(*alloc)(void);
void (*free)(struct rtrs_ib_dev *dev);
int (*init)(struct rtrs_ib_dev *dev);
void (*deinit)(struct rtrs_ib_dev *dev);
};
struct rtrs_rdma_dev_pd {
struct mutex mutex;
struct list_head list;
enum ib_pd_flags pd_flags;
const struct rtrs_rdma_dev_pd_ops *ops;
};
struct rtrs_ib_dev {
struct ib_device *ib_dev;
struct ib_pd *ib_pd;
struct kref ref;
struct list_head entry;
struct rtrs_rdma_dev_pd *pool;
};
struct rtrs_con {
struct rtrs_sess *sess;
struct ib_qp *qp;
struct ib_cq *cq;
struct rdma_cm_id *cm_id;
unsigned int cid;
};
struct rtrs_sess {
struct list_head entry;
struct sockaddr_storage dst_addr;
struct sockaddr_storage src_addr;
char sessname[NAME_MAX];
uuid_t uuid;
struct rtrs_con **con;
unsigned int con_num;
unsigned int recon_cnt;
struct rtrs_ib_dev *dev;
int dev_ref;
struct ib_cqe *hb_cqe;
void (*hb_err_handler)(struct rtrs_con *con);
struct workqueue_struct *hb_wq;
struct delayed_work hb_dwork;
unsigned int hb_interval_ms;
unsigned int hb_missed_cnt;
unsigned int hb_missed_max;
};
/* rtrs information unit */
struct rtrs_iu {
struct list_head list;
struct ib_cqe cqe;
dma_addr_t dma_addr;
void *buf;
size_t size;
enum dma_data_direction direction;
};
/**
* enum rtrs_msg_types - RTRS message types, see also rtrs/README
* @RTRS_MSG_INFO_REQ: Client additional info request to the server
* @RTRS_MSG_INFO_RSP: Server additional info response to the client
* @RTRS_MSG_WRITE: Client writes data per RDMA to server
* @RTRS_MSG_READ: Client requests data transfer from server
* @RTRS_MSG_RKEY_RSP: Server refreshed rkey for rbuf
*/
enum rtrs_msg_types {
RTRS_MSG_INFO_REQ,
RTRS_MSG_INFO_RSP,
RTRS_MSG_WRITE,
RTRS_MSG_READ,
RTRS_MSG_RKEY_RSP,
};
/**
* enum rtrs_msg_flags - RTRS message flags.
* @RTRS_NEED_INVAL: Send invalidation in response.
* @RTRS_MSG_NEW_RKEY_F: Send refreshed rkey in response.
*/
enum rtrs_msg_flags {
RTRS_MSG_NEED_INVAL_F = 1 << 0,
RTRS_MSG_NEW_RKEY_F = 1 << 1,
};
/**
* struct rtrs_sg_desc - RDMA-Buffer entry description
* @addr: Address of RDMA destination buffer
* @key: Authorization rkey to write to the buffer
* @len: Size of the buffer
*/
struct rtrs_sg_desc {
__le64 addr;
__le32 key;
__le32 len;
};
/**
* struct rtrs_msg_conn_req - Client connection request to the server
* @magic: RTRS magic
* @version: RTRS protocol version
* @cid: Current connection id
* @cid_num: Number of connections per session
* @recon_cnt: Reconnections counter
* @sess_uuid: UUID of a session (path)
* @paths_uuid: UUID of a group of sessions (paths)
*
* NOTE: max size 56 bytes, see man rdma_connect().
*/
struct rtrs_msg_conn_req {
/* Is set to 0 by cma.c in case of AF_IB, do not touch that.
* see https://www.spinics.net/lists/linux-rdma/msg22397.html
*/
u8 __cma_version;
/* On sender side that should be set to 0, or cma_save_ip_info()
* extract garbage and will fail.
*/
u8 __ip_version;
__le16 magic;
__le16 version;
__le16 cid;
__le16 cid_num;
__le16 recon_cnt;
uuid_t sess_uuid;
uuid_t paths_uuid;
u8 reserved[12];
};
/**
* struct rtrs_msg_conn_rsp - Server connection response to the client
* @magic: RTRS magic
* @version: RTRS protocol version
* @errno: If rdma_accept() then 0, if rdma_reject() indicates error
* @queue_depth: max inflight messages (queue-depth) in this session
* @max_io_size: max io size server supports
* @max_hdr_size: max msg header size server supports
*
* NOTE: size is 56 bytes, max possible is 136 bytes, see man rdma_accept().
*/
struct rtrs_msg_conn_rsp {
__le16 magic;
__le16 version;
__le16 errno;
__le16 queue_depth;
__le32 max_io_size;
__le32 max_hdr_size;
__le32 flags;
u8 reserved[36];
};
/**
* struct rtrs_msg_info_req
* @type: @RTRS_MSG_INFO_REQ
* @sessname: Session name chosen by client
*/
struct rtrs_msg_info_req {
__le16 type;
u8 sessname[NAME_MAX];
u8 reserved[15];
};
/**
* struct rtrs_msg_info_rsp
* @type: @RTRS_MSG_INFO_RSP
* @sg_cnt: Number of @desc entries
* @desc: RDMA buffers where the client can write to server
*/
struct rtrs_msg_info_rsp {
__le16 type;
__le16 sg_cnt;
u8 reserved[4];
struct rtrs_sg_desc desc[];
};
/**
* struct rtrs_msg_rkey_rsp
* @type: @RTRS_MSG_RKEY_RSP
* @buf_id: RDMA buf_id of the new rkey
* @rkey: new remote key for RDMA buffers id from server
*/
struct rtrs_msg_rkey_rsp {
__le16 type;
__le16 buf_id;
__le32 rkey;
};
/**
* struct rtrs_msg_rdma_read - RDMA data transfer request from client
* @type: always @RTRS_MSG_READ
* @usr_len: length of user payload
* @sg_cnt: number of @desc entries
* @desc: RDMA buffers where the server can write the result to
*/
struct rtrs_msg_rdma_read {
__le16 type;
__le16 usr_len;
__le16 flags;
__le16 sg_cnt;
struct rtrs_sg_desc desc[];
};
/**
* struct_msg_rdma_write - Message transferred to server with RDMA-Write
* @type: always @RTRS_MSG_WRITE
* @usr_len: length of user payload
*/
struct rtrs_msg_rdma_write {
__le16 type;
__le16 usr_len;
};
/**
* struct_msg_rdma_hdr - header for read or write request
* @type: @RTRS_MSG_WRITE | @RTRS_MSG_READ
*/
struct rtrs_msg_rdma_hdr {
__le16 type;
};
/* rtrs.c */
struct rtrs_iu *rtrs_iu_alloc(u32 queue_size, size_t size, gfp_t t,
struct ib_device *dev, enum dma_data_direction,
void (*done)(struct ib_cq *cq, struct ib_wc *wc));
void rtrs_iu_free(struct rtrs_iu *iu, enum dma_data_direction dir,
struct ib_device *dev, u32 queue_size);
int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu);
int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,
struct ib_send_wr *head);
int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu,
struct ib_sge *sge, unsigned int num_sge,
u32 rkey, u64 rdma_addr, u32 imm_data,
enum ib_send_flags flags,
struct ib_send_wr *head);
int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe);
int rtrs_post_rdma_write_imm_empty(struct rtrs_con *con, struct ib_cqe *cqe,
u32 imm_data, enum ib_send_flags flags,
struct ib_send_wr *head);
int rtrs_cq_qp_create(struct rtrs_sess *rtrs_sess, struct rtrs_con *con,
u32 max_send_sge, int cq_vector, u16 cq_size,
u16 wr_queue_size, enum ib_poll_context poll_ctx);
void rtrs_cq_qp_destroy(struct rtrs_con *con);
void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe,
unsigned int interval_ms, unsigned int missed_max,
void (*err_handler)(struct rtrs_con *con),
struct workqueue_struct *wq);
void rtrs_start_hb(struct rtrs_sess *sess);
void rtrs_stop_hb(struct rtrs_sess *sess);
void rtrs_send_hb_ack(struct rtrs_sess *sess);
void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
struct rtrs_rdma_dev_pd *pool);
void rtrs_rdma_dev_pd_deinit(struct rtrs_rdma_dev_pd *pool);
struct rtrs_ib_dev *rtrs_ib_dev_find_or_add(struct ib_device *ib_dev,
struct rtrs_rdma_dev_pd *pool);
int rtrs_ib_dev_put(struct rtrs_ib_dev *dev);
static inline u32 rtrs_to_imm(u32 type, u32 payload)
{
BUILD_BUG_ON(MAX_IMM_PAYL_BITS + MAX_IMM_TYPE_BITS != 32);
BUILD_BUG_ON(RTRS_LAST_IMM > (1<<MAX_IMM_TYPE_BITS));
return ((type & MAX_IMM_TYPE_MASK) << MAX_IMM_PAYL_BITS) |
(payload & MAX_IMM_PAYL_MASK);
}
static inline void rtrs_from_imm(u32 imm, u32 *type, u32 *payload)
{
*payload = imm & MAX_IMM_PAYL_MASK;
*type = imm >> MAX_IMM_PAYL_BITS;
}
static inline u32 rtrs_to_io_req_imm(u32 addr)
{
return rtrs_to_imm(RTRS_IO_REQ_IMM, addr);
}
static inline u32 rtrs_to_io_rsp_imm(u32 msg_id, int errno, bool w_inval)
{
enum rtrs_imm_type type;
u32 payload;
/* 9 bits for errno, 19 bits for msg_id */
payload = (abs(errno) & 0x1ff) << 19 | (msg_id & 0x7ffff);
type = w_inval ? RTRS_IO_RSP_W_INV_IMM : RTRS_IO_RSP_IMM;
return rtrs_to_imm(type, payload);
}
static inline void rtrs_from_io_rsp_imm(u32 payload, u32 *msg_id, int *errno)
{
/* 9 bits for errno, 19 bits for msg_id */
*msg_id = payload & 0x7ffff;
*errno = -(int)((payload >> 19) & 0x1ff);
}
#define STAT_STORE_FUNC(type, set_value, reset) \
static ssize_t set_value##_store(struct kobject *kobj, \
struct kobj_attribute *attr, \
const char *buf, size_t count) \
{ \
int ret = -EINVAL; \
type *stats = container_of(kobj, type, kobj_stats); \
\
if (sysfs_streq(buf, "1")) \
ret = reset(stats, true); \
else if (sysfs_streq(buf, "0")) \
ret = reset(stats, false); \
if (ret) \
return ret; \
\
return count; \
}
#define STAT_SHOW_FUNC(type, get_value, print) \
static ssize_t get_value##_show(struct kobject *kobj, \
struct kobj_attribute *attr, \
char *page) \
{ \
type *stats = container_of(kobj, type, kobj_stats); \
\
return print(stats, page, PAGE_SIZE); \
}
#define STAT_ATTR(type, stat, print, reset) \
STAT_STORE_FUNC(type, stat, reset) \
STAT_SHOW_FUNC(type, stat, print) \
static struct kobj_attribute stat##_attr = __ATTR_RW(stat)
#endif /* RTRS_PRI_H */

38
drivers/infiniband/ulp/rtrs/rtrs-srv-stats.c Normal file
Fájl megtekintése

@@ -0,0 +1,38 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#undef pr_fmt
#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
#include "rtrs-srv.h"
int rtrs_srv_reset_rdma_stats(struct rtrs_srv_stats *stats, bool enable)
{
if (enable) {
struct rtrs_srv_stats_rdma_stats *r = &stats->rdma_stats;
memset(r, 0, sizeof(*r));
return 0;
}
return -EINVAL;
}
ssize_t rtrs_srv_stats_rdma_to_str(struct rtrs_srv_stats *stats,
char *page, size_t len)
{
struct rtrs_srv_stats_rdma_stats *r = &stats->rdma_stats;
struct rtrs_srv_sess *sess = stats->sess;
return scnprintf(page, len, "%lld %lld %lld %lld %u\n",
(s64)atomic64_read(&r->dir[READ].cnt),
(s64)atomic64_read(&r->dir[READ].size_total),
(s64)atomic64_read(&r->dir[WRITE].cnt),
(s64)atomic64_read(&r->dir[WRITE].size_total),
atomic_read(&sess->ids_inflight));
}

321
drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c Normal file
Fájl megtekintése

@@ -0,0 +1,321 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#undef pr_fmt
#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
#include "rtrs-pri.h"
#include "rtrs-srv.h"
#include "rtrs-log.h"
static void rtrs_srv_release(struct kobject *kobj)
{
struct rtrs_srv_sess *sess;
sess = container_of(kobj, struct rtrs_srv_sess, kobj);
kfree(sess);
}
static struct kobj_type ktype = {
.sysfs_ops = &kobj_sysfs_ops,
.release = rtrs_srv_release,
};
static ssize_t rtrs_srv_disconnect_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
return scnprintf(page, PAGE_SIZE, "Usage: echo 1 > %s\n",
attr->attr.name);
}
static ssize_t rtrs_srv_disconnect_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
struct rtrs_srv_sess *sess;
struct rtrs_sess *s;
char str[MAXHOSTNAMELEN];
sess = container_of(kobj, struct rtrs_srv_sess, kobj);
s = &sess->s;
if (!sysfs_streq(buf, "1")) {
rtrs_err(s, "%s: invalid value: '%s'\n",
attr->attr.name, buf);
return -EINVAL;
}
sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr, str, sizeof(str));
rtrs_info(s, "disconnect for path %s requested\n", str);
close_sess(sess);
return count;
}
static struct kobj_attribute rtrs_srv_disconnect_attr =
__ATTR(disconnect, 0644,
rtrs_srv_disconnect_show, rtrs_srv_disconnect_store);
static ssize_t rtrs_srv_hca_port_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_srv_sess *sess;
struct rtrs_con *usr_con;
sess = container_of(kobj, typeof(*sess), kobj);
usr_con = sess->s.con[0];
return scnprintf(page, PAGE_SIZE, "%u\n",
usr_con->cm_id->port_num);
}
static struct kobj_attribute rtrs_srv_hca_port_attr =
__ATTR(hca_port, 0444, rtrs_srv_hca_port_show, NULL);
static ssize_t rtrs_srv_hca_name_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_srv_sess *sess;
sess = container_of(kobj, struct rtrs_srv_sess, kobj);
return scnprintf(page, PAGE_SIZE, "%s\n",
sess->s.dev->ib_dev->name);
}
static struct kobj_attribute rtrs_srv_hca_name_attr =
__ATTR(hca_name, 0444, rtrs_srv_hca_name_show, NULL);
static ssize_t rtrs_srv_src_addr_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_srv_sess *sess;
int cnt;
sess = container_of(kobj, struct rtrs_srv_sess, kobj);
cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
page, PAGE_SIZE);
return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
}
static struct kobj_attribute rtrs_srv_src_addr_attr =
__ATTR(src_addr, 0444, rtrs_srv_src_addr_show, NULL);
static ssize_t rtrs_srv_dst_addr_show(struct kobject *kobj,
struct kobj_attribute *attr,
char *page)
{
struct rtrs_srv_sess *sess;
int cnt;
sess = container_of(kobj, struct rtrs_srv_sess, kobj);
cnt = sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
page, PAGE_SIZE);
return cnt + scnprintf(page + cnt, PAGE_SIZE - cnt, "\n");
}
static struct kobj_attribute rtrs_srv_dst_addr_attr =
__ATTR(dst_addr, 0444, rtrs_srv_dst_addr_show, NULL);
static struct attribute *rtrs_srv_sess_attrs[] = {
&rtrs_srv_hca_name_attr.attr,
&rtrs_srv_hca_port_attr.attr,
&rtrs_srv_src_addr_attr.attr,
&rtrs_srv_dst_addr_attr.attr,
&rtrs_srv_disconnect_attr.attr,
NULL,
};
static struct attribute_group rtrs_srv_sess_attr_group = {
.attrs = rtrs_srv_sess_attrs,
};
STAT_ATTR(struct rtrs_srv_stats, rdma,
rtrs_srv_stats_rdma_to_str,
rtrs_srv_reset_rdma_stats);
static struct attribute *rtrs_srv_stats_attrs[] = {
&rdma_attr.attr,
NULL,
};
static struct attribute_group rtrs_srv_stats_attr_group = {
.attrs = rtrs_srv_stats_attrs,
};
static void rtrs_srv_dev_release(struct device *dev)
{
struct rtrs_srv *srv = container_of(dev, struct rtrs_srv, dev);
kfree(srv);
}
static int rtrs_srv_create_once_sysfs_root_folders(struct rtrs_srv_sess *sess)
{
struct rtrs_srv *srv = sess->srv;
int err = 0;
mutex_lock(&srv->paths_mutex);
if (srv->dev_ref++) {
/*
* Device needs to be registered only on the first session
*/
goto unlock;
}
srv->dev.class = rtrs_dev_class;
srv->dev.release = rtrs_srv_dev_release;
err = dev_set_name(&srv->dev, "%s", sess->s.sessname);
if (err)
goto unlock;
/*
* Suppress user space notification until
* sysfs files are created
*/
dev_set_uevent_suppress(&srv->dev, true);
err = device_register(&srv->dev);
if (err) {
pr_err("device_register(): %d\n", err);
goto put;
}
srv->kobj_paths = kobject_create_and_add("paths", &srv->dev.kobj);
if (!srv->kobj_paths) {
err = -ENOMEM;
pr_err("kobject_create_and_add(): %d\n", err);
device_unregister(&srv->dev);
goto unlock;
}
dev_set_uevent_suppress(&srv->dev, false);
kobject_uevent(&srv->dev.kobj, KOBJ_ADD);
goto unlock;
put:
put_device(&srv->dev);
unlock:
mutex_unlock(&srv->paths_mutex);
return err;
}
static void
rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_sess *sess)
{
struct rtrs_srv *srv = sess->srv;
mutex_lock(&srv->paths_mutex);
if (!--srv->dev_ref) {
kobject_del(srv->kobj_paths);
kobject_put(srv->kobj_paths);
mutex_unlock(&srv->paths_mutex);
device_unregister(&srv->dev);
} else {
mutex_unlock(&srv->paths_mutex);
}
}
static void rtrs_srv_sess_stats_release(struct kobject *kobj)
{
struct rtrs_srv_stats *stats;
stats = container_of(kobj, struct rtrs_srv_stats, kobj_stats);
kfree(stats);
}
static struct kobj_type ktype_stats = {
.sysfs_ops = &kobj_sysfs_ops,
.release = rtrs_srv_sess_stats_release,
};
static int rtrs_srv_create_stats_files(struct rtrs_srv_sess *sess)
{
int err;
struct rtrs_sess *s = &sess->s;
err = kobject_init_and_add(&sess->stats->kobj_stats, &ktype_stats,
&sess->kobj, "stats");
if (err) {
rtrs_err(s, "kobject_init_and_add(): %d\n", err);
return err;
}
err = sysfs_create_group(&sess->stats->kobj_stats,
&rtrs_srv_stats_attr_group);
if (err) {
rtrs_err(s, "sysfs_create_group(): %d\n", err);
goto err;
}
return 0;
err:
kobject_del(&sess->stats->kobj_stats);
kobject_put(&sess->stats->kobj_stats);
return err;
}
int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess)
{
struct rtrs_srv *srv = sess->srv;
struct rtrs_sess *s = &sess->s;
char str[NAME_MAX];
int err, cnt;
cnt = sockaddr_to_str((struct sockaddr *)&sess->s.dst_addr,
str, sizeof(str));
cnt += scnprintf(str + cnt, sizeof(str) - cnt, "@");
sockaddr_to_str((struct sockaddr *)&sess->s.src_addr,
str + cnt, sizeof(str) - cnt);
err = rtrs_srv_create_once_sysfs_root_folders(sess);
if (err)
return err;
err = kobject_init_and_add(&sess->kobj, &ktype, srv->kobj_paths,
"%s", str);
if (err) {
rtrs_err(s, "kobject_init_and_add(): %d\n", err);
goto destroy_root;
}
err = sysfs_create_group(&sess->kobj, &rtrs_srv_sess_attr_group);
if (err) {
rtrs_err(s, "sysfs_create_group(): %d\n", err);
goto put_kobj;
}
err = rtrs_srv_create_stats_files(sess);
if (err)
goto remove_group;
return 0;
remove_group:
sysfs_remove_group(&sess->kobj, &rtrs_srv_sess_attr_group);
put_kobj:
kobject_del(&sess->kobj);
kobject_put(&sess->kobj);
destroy_root:
rtrs_srv_destroy_once_sysfs_root_folders(sess);
return err;
}
void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess)
{
if (sess->kobj.state_in_sysfs) {
kobject_del(&sess->stats->kobj_stats);
kobject_put(&sess->stats->kobj_stats);
kobject_del(&sess->kobj);
kobject_put(&sess->kobj);
rtrs_srv_destroy_once_sysfs_root_folders(sess);
}
}

2178
drivers/infiniband/ulp/rtrs/rtrs-srv.c Normal file
Fájl megtekintése

A különbségek nem kerülnek megjelenítésre, mivel a fájl túl nagy Load Diff

148
drivers/infiniband/ulp/rtrs/rtrs-srv.h Normal file
Fájl megtekintése

@@ -0,0 +1,148 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#ifndef RTRS_SRV_H
#define RTRS_SRV_H
#include <linux/device.h>
#include <linux/refcount.h>
#include "rtrs-pri.h"
/*
* enum rtrs_srv_state - Server states.
*/
enum rtrs_srv_state {
RTRS_SRV_CONNECTING,
RTRS_SRV_CONNECTED,
RTRS_SRV_CLOSING,
RTRS_SRV_CLOSED,
};
/* stats for Read and write operation.
* see Documentation/ABI/testing/sysfs-class-rtrs-server for details
*/
struct rtrs_srv_stats_rdma_stats {
struct {
atomic64_t cnt;
atomic64_t size_total;
} dir[2];
};
struct rtrs_srv_stats {
struct kobject kobj_stats;
struct rtrs_srv_stats_rdma_stats rdma_stats;
struct rtrs_srv_sess *sess;
};
struct rtrs_srv_con {
struct rtrs_con c;
atomic_t wr_cnt;
atomic_t sq_wr_avail;
struct list_head rsp_wr_wait_list;
spinlock_t rsp_wr_wait_lock;
};
/* IO context in rtrs_srv, each io has one */
struct rtrs_srv_op {
struct rtrs_srv_con *con;
u32 msg_id;
u8 dir;
struct rtrs_msg_rdma_read *rd_msg;
struct ib_rdma_wr tx_wr;
struct ib_sge tx_sg;
struct list_head wait_list;
int status;
};
/*
* server side memory region context, when always_invalidate=Y, we need
* queue_depth of memory regrion to invalidate each memory region.
*/
struct rtrs_srv_mr {
struct ib_mr *mr;
struct sg_table sgt;
struct ib_cqe inv_cqe; /* only for always_invalidate=true */
u32 msg_id; /* only for always_invalidate=true */
u32 msg_off; /* only for always_invalidate=true */
struct rtrs_iu *iu; /* send buffer for new rkey msg */
};
struct rtrs_srv_sess {
struct rtrs_sess s;
struct rtrs_srv *srv;
struct work_struct close_work;
enum rtrs_srv_state state;
spinlock_t state_lock;
int cur_cq_vector;
struct rtrs_srv_op **ops_ids;
atomic_t ids_inflight;
wait_queue_head_t ids_waitq;
struct rtrs_srv_mr *mrs;
unsigned int mrs_num;
dma_addr_t *dma_addr;
bool established;
unsigned int mem_bits;
struct kobject kobj;
struct rtrs_srv_stats *stats;
};
struct rtrs_srv {
struct list_head paths_list;
int paths_up;
struct mutex paths_ev_mutex;
size_t paths_num;
struct mutex paths_mutex;
uuid_t paths_uuid;
refcount_t refcount;
struct rtrs_srv_ctx *ctx;
struct list_head ctx_list;
void *priv;
size_t queue_depth;
struct page **chunks;
struct device dev;
unsigned int dev_ref;
struct kobject *kobj_paths;
};
struct rtrs_srv_ctx {
struct rtrs_srv_ops ops;
struct rdma_cm_id *cm_id_ip;
struct rdma_cm_id *cm_id_ib;
struct mutex srv_mutex;
struct list_head srv_list;
};
extern struct class *rtrs_dev_class;
void close_sess(struct rtrs_srv_sess *sess);
static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s,
size_t size, int d)
{
atomic64_inc(&s->rdma_stats.dir[d].cnt);
atomic64_add(size, &s->rdma_stats.dir[d].size_total);
}
/* functions which are implemented in rtrs-srv-stats.c */
int rtrs_srv_reset_rdma_stats(struct rtrs_srv_stats *stats, bool enable);
ssize_t rtrs_srv_stats_rdma_to_str(struct rtrs_srv_stats *stats,
char *page, size_t len);
int rtrs_srv_reset_wc_completion_stats(struct rtrs_srv_stats *stats,
bool enable);
int rtrs_srv_stats_wc_completion_to_str(struct rtrs_srv_stats *stats, char *buf,
size_t len);
int rtrs_srv_reset_all_stats(struct rtrs_srv_stats *stats, bool enable);
ssize_t rtrs_srv_reset_all_help(struct rtrs_srv_stats *stats,
char *page, size_t len);
/* functions which are implemented in rtrs-srv-sysfs.c */
int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess);
void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess);
#endif /* RTRS_SRV_H */

612
drivers/infiniband/ulp/rtrs/rtrs.c Normal file
Fájl megtekintése

@@ -0,0 +1,612 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#undef pr_fmt
#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
#include <linux/module.h>
#include <linux/inet.h>
#include "rtrs-pri.h"
#include "rtrs-log.h"
MODULE_DESCRIPTION("RDMA Transport Core");
MODULE_LICENSE("GPL");
struct rtrs_iu *rtrs_iu_alloc(u32 queue_size, size_t size, gfp_t gfp_mask,
struct ib_device *dma_dev,
enum dma_data_direction dir,
void (*done)(struct ib_cq *cq, struct ib_wc *wc))
{
struct rtrs_iu *ius, *iu;
int i;
ius = kcalloc(queue_size, sizeof(*ius), gfp_mask);
if (!ius)
return NULL;
for (i = 0; i < queue_size; i++) {
iu = &ius[i];
iu->buf = kzalloc(size, gfp_mask);
if (!iu->buf)
goto err;
iu->dma_addr = ib_dma_map_single(dma_dev, iu->buf, size, dir);
if (ib_dma_mapping_error(dma_dev, iu->dma_addr))
goto err;
iu->cqe.done = done;
iu->size = size;
iu->direction = dir;
}
return ius;
err:
rtrs_iu_free(ius, dir, dma_dev, i);
return NULL;
}
EXPORT_SYMBOL_GPL(rtrs_iu_alloc);
void rtrs_iu_free(struct rtrs_iu *ius, enum dma_data_direction dir,
struct ib_device *ibdev, u32 queue_size)
{
struct rtrs_iu *iu;
int i;
if (!ius)
return;
for (i = 0; i < queue_size; i++) {
iu = &ius[i];
ib_dma_unmap_single(ibdev, iu->dma_addr, iu->size, dir);
kfree(iu->buf);
}
kfree(ius);
}
EXPORT_SYMBOL_GPL(rtrs_iu_free);
int rtrs_iu_post_recv(struct rtrs_con *con, struct rtrs_iu *iu)
{
struct rtrs_sess *sess = con->sess;
struct ib_recv_wr wr;
struct ib_sge list;
list.addr = iu->dma_addr;
list.length = iu->size;
list.lkey = sess->dev->ib_pd->local_dma_lkey;
if (list.length == 0) {
rtrs_wrn(con->sess,
"Posting receive work request failed, sg list is empty\n");
return -EINVAL;
}
wr = (struct ib_recv_wr) {
.wr_cqe = &iu->cqe,
.sg_list = &list,
.num_sge = 1,
};
return ib_post_recv(con->qp, &wr, NULL);
}
EXPORT_SYMBOL_GPL(rtrs_iu_post_recv);
int rtrs_post_recv_empty(struct rtrs_con *con, struct ib_cqe *cqe)
{
struct ib_recv_wr wr;
wr = (struct ib_recv_wr) {
.wr_cqe = cqe,
};
return ib_post_recv(con->qp, &wr, NULL);
}
EXPORT_SYMBOL_GPL(rtrs_post_recv_empty);
int rtrs_iu_post_send(struct rtrs_con *con, struct rtrs_iu *iu, size_t size,
struct ib_send_wr *head)
{
struct rtrs_sess *sess = con->sess;
struct ib_send_wr wr;
struct ib_sge list;
if (WARN_ON(size == 0))
return -EINVAL;
list.addr = iu->dma_addr;
list.length = size;
list.lkey = sess->dev->ib_pd->local_dma_lkey;
wr = (struct ib_send_wr) {
.wr_cqe = &iu->cqe,
.sg_list = &list,
.num_sge = 1,
.opcode = IB_WR_SEND,
.send_flags = IB_SEND_SIGNALED,
};
if (head) {
struct ib_send_wr *tail = head;
while (tail->next)
tail = tail->next;
tail->next = &wr;
} else {
head = &wr;
}
return ib_post_send(con->qp, head, NULL);
}
EXPORT_SYMBOL_GPL(rtrs_iu_post_send);
int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu,
struct ib_sge *sge, unsigned int num_sge,
u32 rkey, u64 rdma_addr, u32 imm_data,
enum ib_send_flags flags,
struct ib_send_wr *head)
{
struct ib_rdma_wr wr;
int i;
wr = (struct ib_rdma_wr) {
.wr.wr_cqe = &iu->cqe,
.wr.sg_list = sge,
.wr.num_sge = num_sge,
.rkey = rkey,
.remote_addr = rdma_addr,
.wr.opcode = IB_WR_RDMA_WRITE_WITH_IMM,
.wr.ex.imm_data = cpu_to_be32(imm_data),
.wr.send_flags = flags,
};
/*
* If one of the sges has 0 size, the operation will fail with a
* length error
*/
for (i = 0; i < num_sge; i++)
if (WARN_ON(sge[i].length == 0))
return -EINVAL;
if (head) {
struct ib_send_wr *tail = head;
while (tail->next)
tail = tail->next;
tail->next = &wr.wr;
} else {
head = &wr.wr;
}
return ib_post_send(con->qp, head, NULL);
}
EXPORT_SYMBOL_GPL(rtrs_iu_post_rdma_write_imm);
int rtrs_post_rdma_write_imm_empty(struct rtrs_con *con, struct ib_cqe *cqe,
u32 imm_data, enum ib_send_flags flags,
struct ib_send_wr *head)
{
struct ib_send_wr wr;
wr = (struct ib_send_wr) {
.wr_cqe = cqe,
.send_flags = flags,
.opcode = IB_WR_RDMA_WRITE_WITH_IMM,
.ex.imm_data = cpu_to_be32(imm_data),
};
if (head) {
struct ib_send_wr *tail = head;
while (tail->next)
tail = tail->next;
tail->next = &wr;
} else {
head = &wr;
}
return ib_post_send(con->qp, head, NULL);
}
EXPORT_SYMBOL_GPL(rtrs_post_rdma_write_imm_empty);
static void qp_event_handler(struct ib_event *ev, void *ctx)
{
struct rtrs_con *con = ctx;
switch (ev->event) {
case IB_EVENT_COMM_EST:
rtrs_info(con->sess, "QP event %s (%d) received\n",
ib_event_msg(ev->event), ev->event);
rdma_notify(con->cm_id, IB_EVENT_COMM_EST);
break;
default:
rtrs_info(con->sess, "Unhandled QP event %s (%d) received\n",
ib_event_msg(ev->event), ev->event);
break;
}
}
static int create_cq(struct rtrs_con *con, int cq_vector, u16 cq_size,
enum ib_poll_context poll_ctx)
{
struct rdma_cm_id *cm_id = con->cm_id;
struct ib_cq *cq;
cq = ib_alloc_cq(cm_id->device, con, cq_size,
cq_vector, poll_ctx);
if (IS_ERR(cq)) {
rtrs_err(con->sess, "Creating completion queue failed, errno: %ld\n",
PTR_ERR(cq));
return PTR_ERR(cq);
}
con->cq = cq;
return 0;
}
static int create_qp(struct rtrs_con *con, struct ib_pd *pd,
u16 wr_queue_size, u32 max_sge)
{
struct ib_qp_init_attr init_attr = {NULL};
struct rdma_cm_id *cm_id = con->cm_id;
int ret;
init_attr.cap.max_send_wr = wr_queue_size;
init_attr.cap.max_recv_wr = wr_queue_size;
init_attr.cap.max_recv_sge = 1;
init_attr.event_handler = qp_event_handler;
init_attr.qp_context = con;
init_attr.cap.max_send_sge = max_sge;
init_attr.qp_type = IB_QPT_RC;
init_attr.send_cq = con->cq;
init_attr.recv_cq = con->cq;
init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
ret = rdma_create_qp(cm_id, pd, &init_attr);
if (ret) {
rtrs_err(con->sess, "Creating QP failed, err: %d\n", ret);
return ret;
}
con->qp = cm_id->qp;
return ret;
}
int rtrs_cq_qp_create(struct rtrs_sess *sess, struct rtrs_con *con,
u32 max_send_sge, int cq_vector, u16 cq_size,
u16 wr_queue_size, enum ib_poll_context poll_ctx)
{
int err;
err = create_cq(con, cq_vector, cq_size, poll_ctx);
if (err)
return err;
err = create_qp(con, sess->dev->ib_pd, wr_queue_size, max_send_sge);
if (err) {
ib_free_cq(con->cq);
con->cq = NULL;
return err;
}
con->sess = sess;
return 0;
}
EXPORT_SYMBOL_GPL(rtrs_cq_qp_create);
void rtrs_cq_qp_destroy(struct rtrs_con *con)
{
if (con->qp) {
rdma_destroy_qp(con->cm_id);
con->qp = NULL;
}
if (con->cq) {
ib_free_cq(con->cq);
con->cq = NULL;
}
}
EXPORT_SYMBOL_GPL(rtrs_cq_qp_destroy);
static void schedule_hb(struct rtrs_sess *sess)
{
queue_delayed_work(sess->hb_wq, &sess->hb_dwork,
msecs_to_jiffies(sess->hb_interval_ms));
}
void rtrs_send_hb_ack(struct rtrs_sess *sess)
{
struct rtrs_con *usr_con = sess->con[0];
u32 imm;
int err;
imm = rtrs_to_imm(RTRS_HB_ACK_IMM, 0);
err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm,
IB_SEND_SIGNALED, NULL);
if (err) {
sess->hb_err_handler(usr_con);
return;
}
}
EXPORT_SYMBOL_GPL(rtrs_send_hb_ack);
static void hb_work(struct work_struct *work)
{
struct rtrs_con *usr_con;
struct rtrs_sess *sess;
u32 imm;
int err;
sess = container_of(to_delayed_work(work), typeof(*sess), hb_dwork);
usr_con = sess->con[0];
if (sess->hb_missed_cnt > sess->hb_missed_max) {
sess->hb_err_handler(usr_con);
return;
}
if (sess->hb_missed_cnt++) {
/* Reschedule work without sending hb */
schedule_hb(sess);
return;
}
imm = rtrs_to_imm(RTRS_HB_MSG_IMM, 0);
err = rtrs_post_rdma_write_imm_empty(usr_con, sess->hb_cqe, imm,
IB_SEND_SIGNALED, NULL);
if (err) {
sess->hb_err_handler(usr_con);
return;
}
schedule_hb(sess);
}
void rtrs_init_hb(struct rtrs_sess *sess, struct ib_cqe *cqe,
unsigned int interval_ms, unsigned int missed_max,
void (*err_handler)(struct rtrs_con *con),
struct workqueue_struct *wq)
{
sess->hb_cqe = cqe;
sess->hb_interval_ms = interval_ms;
sess->hb_err_handler = err_handler;
sess->hb_wq = wq;
sess->hb_missed_max = missed_max;
sess->hb_missed_cnt = 0;
INIT_DELAYED_WORK(&sess->hb_dwork, hb_work);
}
EXPORT_SYMBOL_GPL(rtrs_init_hb);
void rtrs_start_hb(struct rtrs_sess *sess)
{
schedule_hb(sess);
}
EXPORT_SYMBOL_GPL(rtrs_start_hb);
void rtrs_stop_hb(struct rtrs_sess *sess)
{
cancel_delayed_work_sync(&sess->hb_dwork);
sess->hb_missed_cnt = 0;
sess->hb_missed_max = 0;
}
EXPORT_SYMBOL_GPL(rtrs_stop_hb);
static int rtrs_str_gid_to_sockaddr(const char *addr, size_t len,
short port, struct sockaddr_storage *dst)
{
struct sockaddr_ib *dst_ib = (struct sockaddr_ib *)dst;
int ret;
/*
* We can use some of the IPv6 functions since GID is a valid
* IPv6 address format
*/
ret = in6_pton(addr, len, dst_ib->sib_addr.sib_raw, '\0', NULL);
if (ret == 0)
return -EINVAL;
dst_ib->sib_family = AF_IB;
/*
* Use the same TCP server port number as the IB service ID
* on the IB port space range
*/
dst_ib->sib_sid = cpu_to_be64(RDMA_IB_IP_PS_IB | port);
dst_ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
dst_ib->sib_pkey = cpu_to_be16(0xffff);
return 0;
}
/**
* rtrs_str_to_sockaddr() - Convert rtrs address string to sockaddr
* @addr: String representation of an addr (IPv4, IPv6 or IB GID):
* - "ip:192.168.1.1"
* - "ip:fe80::200:5aee:feaa:20a2"
* - "gid:fe80::200:5aee:feaa:20a2"
* @len: String address length
* @port: Destination port
* @dst: Destination sockaddr structure
*
* Returns 0 if conversion successful. Non-zero on error.
*/
static int rtrs_str_to_sockaddr(const char *addr, size_t len,
u16 port, struct sockaddr_storage *dst)
{
if (strncmp(addr, "gid:", 4) == 0) {
return rtrs_str_gid_to_sockaddr(addr + 4, len - 4, port, dst);
} else if (strncmp(addr, "ip:", 3) == 0) {
char port_str[8];
char *cpy;
int err;
snprintf(port_str, sizeof(port_str), "%u", port);
cpy = kstrndup(addr + 3, len - 3, GFP_KERNEL);
err = cpy ? inet_pton_with_scope(&init_net, AF_UNSPEC,
cpy, port_str, dst) : -ENOMEM;
kfree(cpy);
return err;
}
return -EPROTONOSUPPORT;
}
/**
* sockaddr_to_str() - convert sockaddr to a string.
* @addr: the sockadddr structure to be converted.
* @buf: string containing socket addr.
* @len: string length.
*
* The return value is the number of characters written into buf not
* including the trailing '\0'. If len is == 0 the function returns 0..
*/
int sockaddr_to_str(const struct sockaddr *addr, char *buf, size_t len)
{
switch (addr->sa_family) {
case AF_IB:
return scnprintf(buf, len, "gid:%pI6",
&((struct sockaddr_ib *)addr)->sib_addr.sib_raw);
case AF_INET:
return scnprintf(buf, len, "ip:%pI4",
&((struct sockaddr_in *)addr)->sin_addr);
case AF_INET6:
return scnprintf(buf, len, "ip:%pI6c",
&((struct sockaddr_in6 *)addr)->sin6_addr);
}
return scnprintf(buf, len, "<invalid address family>");
}
EXPORT_SYMBOL(sockaddr_to_str);
/**
* rtrs_addr_to_sockaddr() - convert path string "src,dst" or "src@dst"
* to sockaddreses
* @str: string containing source and destination addr of a path
* separated by ',' or '@' I.e. "ip:1.1.1.1,ip:1.1.1.2" or
* "ip:1.1.1.1@ip:1.1.1.2". If str contains only one address it's
* considered to be destination.
* @len: string length
* @port: Destination port number.
* @addr: will be set to the source/destination address or to NULL
* if str doesn't contain any source address.
*
* Returns zero if conversion successful. Non-zero otherwise.
*/
int rtrs_addr_to_sockaddr(const char *str, size_t len, u16 port,
struct rtrs_addr *addr)
{
const char *d;
d = strchr(str, ',');
if (!d)
d = strchr(str, '@');
if (d) {
if (rtrs_str_to_sockaddr(str, d - str, 0, addr->src))
return -EINVAL;
d += 1;
len -= d - str;
str = d;
} else {
addr->src = NULL;
}
return rtrs_str_to_sockaddr(str, len, port, addr->dst);
}
EXPORT_SYMBOL(rtrs_addr_to_sockaddr);
void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
struct rtrs_rdma_dev_pd *pool)
{
WARN_ON(pool->ops && (!pool->ops->alloc ^ !pool->ops->free));
INIT_LIST_HEAD(&pool->list);
mutex_init(&pool->mutex);
pool->pd_flags = pd_flags;
}
EXPORT_SYMBOL(rtrs_rdma_dev_pd_init);
void rtrs_rdma_dev_pd_deinit(struct rtrs_rdma_dev_pd *pool)
{
mutex_destroy(&pool->mutex);
WARN_ON(!list_empty(&pool->list));
}
EXPORT_SYMBOL(rtrs_rdma_dev_pd_deinit);
static void dev_free(struct kref *ref)
{
struct rtrs_rdma_dev_pd *pool;
struct rtrs_ib_dev *dev;
dev = container_of(ref, typeof(*dev), ref);
pool = dev->pool;
mutex_lock(&pool->mutex);
list_del(&dev->entry);
mutex_unlock(&pool->mutex);
if (pool->ops && pool->ops->deinit)
pool->ops->deinit(dev);
ib_dealloc_pd(dev->ib_pd);
if (pool->ops && pool->ops->free)
pool->ops->free(dev);
else
kfree(dev);
}
int rtrs_ib_dev_put(struct rtrs_ib_dev *dev)
{
return kref_put(&dev->ref, dev_free);
}
EXPORT_SYMBOL(rtrs_ib_dev_put);
static int rtrs_ib_dev_get(struct rtrs_ib_dev *dev)
{
return kref_get_unless_zero(&dev->ref);
}
struct rtrs_ib_dev *
rtrs_ib_dev_find_or_add(struct ib_device *ib_dev,
struct rtrs_rdma_dev_pd *pool)
{
struct rtrs_ib_dev *dev;
mutex_lock(&pool->mutex);
list_for_each_entry(dev, &pool->list, entry) {
if (dev->ib_dev->node_guid == ib_dev->node_guid &&
rtrs_ib_dev_get(dev))
goto out_unlock;
}
mutex_unlock(&pool->mutex);
if (pool->ops && pool->ops->alloc)
dev = pool->ops->alloc();
else
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
if (IS_ERR_OR_NULL(dev))
goto out_err;
kref_init(&dev->ref);
dev->pool = pool;
dev->ib_dev = ib_dev;
dev->ib_pd = ib_alloc_pd(ib_dev, pool->pd_flags);
if (IS_ERR(dev->ib_pd))
goto out_free_dev;
if (pool->ops && pool->ops->init && pool->ops->init(dev))
goto out_free_pd;
mutex_lock(&pool->mutex);
list_add(&dev->entry, &pool->list);
out_unlock:
mutex_unlock(&pool->mutex);
return dev;
out_free_pd:
ib_dealloc_pd(dev->ib_pd);
out_free_dev:
if (pool->ops && pool->ops->free)
pool->ops->free(dev);
else
kfree(dev);
out_err:
return NULL;
}
EXPORT_SYMBOL(rtrs_ib_dev_find_or_add);

196
drivers/infiniband/ulp/rtrs/rtrs.h Normal file
Fájl megtekintése

@@ -0,0 +1,196 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* RDMA Transport Layer
*
* Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
* Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
* Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
*/
#ifndef RTRS_H
#define RTRS_H
#include <linux/socket.h>
#include <linux/scatterlist.h>
struct rtrs_permit;
struct rtrs_clt;
struct rtrs_srv_ctx;
struct rtrs_srv;
struct rtrs_srv_op;
/*
* RDMA transport (RTRS) client API
*/
/**
* enum rtrs_clt_link_ev - Events about connectivity state of a client
* @RTRS_CLT_LINK_EV_RECONNECTED Client was reconnected.
* @RTRS_CLT_LINK_EV_DISCONNECTED Client was disconnected.
*/
enum rtrs_clt_link_ev {
RTRS_CLT_LINK_EV_RECONNECTED,
RTRS_CLT_LINK_EV_DISCONNECTED,
};
/**
* Source and destination address of a path to be established
*/
struct rtrs_addr {
struct sockaddr_storage *src;
struct sockaddr_storage *dst;
};
/**
* rtrs_clt_ops - it holds the link event callback and private pointer.
* @priv: User supplied private data.
* @link_ev: Event notification callback function for connection state changes
* @priv: User supplied data that was passed to rtrs_clt_open()
* @ev: Occurred event
*/
struct rtrs_clt_ops {
void *priv;
void (*link_ev)(void *priv, enum rtrs_clt_link_ev ev);
};
struct rtrs_clt *rtrs_clt_open(struct rtrs_clt_ops *ops,
const char *sessname,
const struct rtrs_addr *paths,
size_t path_cnt, u16 port,
size_t pdu_sz, u8 reconnect_delay_sec,
u16 max_segments,
size_t max_segment_size,
s16 max_reconnect_attempts);
void rtrs_clt_close(struct rtrs_clt *sess);
/**
* rtrs_permit_to_pdu() - converts rtrs_permit to opaque pdu pointer
* @permit: RTRS permit pointer, it associates the memory allocation for future
* RDMA operation.
*/
void *rtrs_permit_to_pdu(struct rtrs_permit *permit);
enum {
RTRS_PERMIT_NOWAIT = 0,
RTRS_PERMIT_WAIT = 1,
};
/**
* enum rtrs_clt_con_type() type of ib connection to use with a given
* rtrs_permit
* @ADMIN_CON - use connection reserved for "service" messages
* @IO_CON - use a connection reserved for IO
*/
enum rtrs_clt_con_type {
RTRS_ADMIN_CON,
RTRS_IO_CON
};
struct rtrs_permit *rtrs_clt_get_permit(struct rtrs_clt *sess,
enum rtrs_clt_con_type con_type,
int wait);
void rtrs_clt_put_permit(struct rtrs_clt *sess, struct rtrs_permit *permit);
/**
* rtrs_clt_req_ops - it holds the request confirmation callback
* and a private pointer.
* @priv: User supplied private data.
* @conf_fn: callback function to be called as confirmation
* @priv: User provided data, passed back with corresponding
* @(conf) confirmation.
* @errno: error number.
*/
struct rtrs_clt_req_ops {
void *priv;
void (*conf_fn)(void *priv, int errno);
};
int rtrs_clt_request(int dir, struct rtrs_clt_req_ops *ops,
struct rtrs_clt *sess, struct rtrs_permit *permit,
const struct kvec *vec, size_t nr, size_t len,
struct scatterlist *sg, unsigned int sg_cnt);
/**
* rtrs_attrs - RTRS session attributes
*/
struct rtrs_attrs {
u32 queue_depth;
u32 max_io_size;
u8 sessname[NAME_MAX];
struct kobject *sess_kobj;
};
int rtrs_clt_query(struct rtrs_clt *sess, struct rtrs_attrs *attr);
/*
* Here goes RTRS server API
*/
/**
* enum rtrs_srv_link_ev - Server link events
* @RTRS_SRV_LINK_EV_CONNECTED: Connection from client established
* @RTRS_SRV_LINK_EV_DISCONNECTED: Connection was disconnected, all
* connection RTRS resources were freed.
*/
enum rtrs_srv_link_ev {
RTRS_SRV_LINK_EV_CONNECTED,
RTRS_SRV_LINK_EV_DISCONNECTED,
};
struct rtrs_srv_ops {
/**
* rdma_ev(): Event notification for RDMA operations
* If the callback returns a value != 0, an error
* message for the data transfer will be sent to
* the client.
* @sess: Session
* @priv: Private data set by rtrs_srv_set_sess_priv()
* @id: internal RTRS operation id
* @dir: READ/WRITE
* @data: Pointer to (bidirectional) rdma memory area:
* - in case of %RTRS_SRV_RDMA_EV_RECV contains
* data sent by the client
* - in case of %RTRS_SRV_RDMA_EV_WRITE_REQ points
* to the memory area where the response is to be
* written to
* @datalen: Size of the memory area in @data
* @usr: The extra user message sent by the client (%vec)
* @usrlen: Size of the user message
*/
int (*rdma_ev)(struct rtrs_srv *sess, void *priv,
struct rtrs_srv_op *id, int dir,
void *data, size_t datalen, const void *usr,
size_t usrlen);
/**
* link_ev(): Events about connectivity state changes
* If the callback returns != 0 and the event
* %RTRS_SRV_LINK_EV_CONNECTED the corresponding
* session will be destroyed.
* @sess: Session
* @ev: event
* @priv: Private data from user if previously set with
* rtrs_srv_set_sess_priv()
*/
int (*link_ev)(struct rtrs_srv *sess, enum rtrs_srv_link_ev ev,
void *priv);
};
struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port);
void rtrs_srv_close(struct rtrs_srv_ctx *ctx);
bool rtrs_srv_resp_rdma(struct rtrs_srv_op *id, int errno);
void rtrs_srv_set_sess_priv(struct rtrs_srv *sess, void *priv);
int rtrs_srv_get_sess_name(struct rtrs_srv *sess, char *sessname, size_t len);
int rtrs_srv_get_queue_depth(struct rtrs_srv *sess);
int rtrs_addr_to_sockaddr(const char *str, size_t len, u16 port,
struct rtrs_addr *addr);
int sockaddr_to_str(const struct sockaddr *addr, char *buf, size_t len);
#endif

265
drivers/infiniband/ulp/srp/ib_srp.c
Fájl megtekintése

@@ -71,7 +71,6 @@ static unsigned int srp_sg_tablesize;
static unsigned int cmd_sg_entries;
static unsigned int indirect_sg_entries;
static bool allow_ext_sg;
static bool prefer_fr = true;
static bool register_always = true;
static bool never_register;
static int topspin_workarounds = 1;
@@ -95,10 +94,6 @@ module_param(topspin_workarounds, int, 0444);
MODULE_PARM_DESC(topspin_workarounds,
"Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
module_param(prefer_fr, bool, 0444);
MODULE_PARM_DESC(prefer_fr,
"Whether to use fast registration if both FMR and fast registration are supported");
module_param(register_always, bool, 0444);
MODULE_PARM_DESC(register_always,
"Use memory registration even for contiguous memory regions");
@@ -146,7 +141,7 @@ module_param(ch_count, uint, 0444);
MODULE_PARM_DESC(ch_count,
"Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
static void srp_add_one(struct ib_device *device);
static int srp_add_one(struct ib_device *device);
static void srp_remove_one(struct ib_device *device, void *client_data);
static void srp_rename_dev(struct ib_device *device, void *client_data);
static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
@@ -388,24 +383,6 @@ static int srp_new_cm_id(struct srp_rdma_ch *ch)
srp_new_ib_cm_id(ch);
}
static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
{
struct srp_device *dev = target->srp_host->srp_dev;
struct ib_fmr_pool_param fmr_param;
memset(&fmr_param, 0, sizeof(fmr_param));
fmr_param.pool_size = target->mr_pool_size;
fmr_param.dirty_watermark = fmr_param.pool_size / 4;
fmr_param.cache = 1;
fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
fmr_param.page_shift = ilog2(dev->mr_page_size);
fmr_param.access = (IB_ACCESS_LOCAL_WRITE |
IB_ACCESS_REMOTE_WRITE |
IB_ACCESS_REMOTE_READ);
return ib_create_fmr_pool(dev->pd, &fmr_param);
}
/**
* srp_destroy_fr_pool() - free the resources owned by a pool
* @pool: Fast registration pool to be destroyed.
@@ -556,7 +533,6 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
struct ib_qp_init_attr *init_attr;
struct ib_cq *recv_cq, *send_cq;
struct ib_qp *qp;
struct ib_fmr_pool *fmr_pool = NULL;
struct srp_fr_pool *fr_pool = NULL;
const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
int ret;
@@ -619,14 +595,6 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
"FR pool allocation failed (%d)\n", ret);
goto err_qp;
}
} else if (dev->use_fmr) {
fmr_pool = srp_alloc_fmr_pool(target);
if (IS_ERR(fmr_pool)) {
ret = PTR_ERR(fmr_pool);
shost_printk(KERN_WARNING, target->scsi_host, PFX
"FMR pool allocation failed (%d)\n", ret);
goto err_qp;
}
}
if (ch->qp)
@@ -644,10 +612,6 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
if (ch->fr_pool)
srp_destroy_fr_pool(ch->fr_pool);
ch->fr_pool = fr_pool;
} else if (dev->use_fmr) {
if (ch->fmr_pool)
ib_destroy_fmr_pool(ch->fmr_pool);
ch->fmr_pool = fmr_pool;
}
kfree(init_attr);
@@ -702,9 +666,6 @@ static void srp_free_ch_ib(struct srp_target_port *target,
if (dev->use_fast_reg) {
if (ch->fr_pool)
srp_destroy_fr_pool(ch->fr_pool);
} else if (dev->use_fmr) {
if (ch->fmr_pool)
ib_destroy_fmr_pool(ch->fmr_pool);
}
srp_destroy_qp(ch);
@@ -1017,12 +978,8 @@ static void srp_free_req_data(struct srp_target_port *target,
for (i = 0; i < target->req_ring_size; ++i) {
req = &ch->req_ring[i];
if (dev->use_fast_reg) {
if (dev->use_fast_reg)
kfree(req->fr_list);
} else {
kfree(req->fmr_list);
kfree(req->map_page);
}
if (req->indirect_dma_addr) {
ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
target->indirect_size,
@@ -1056,16 +1013,8 @@ static int srp_alloc_req_data(struct srp_rdma_ch *ch)
GFP_KERNEL);
if (!mr_list)
goto out;
if (srp_dev->use_fast_reg) {
if (srp_dev->use_fast_reg)
req->fr_list = mr_list;
} else {
req->fmr_list = mr_list;
req->map_page = kmalloc_array(srp_dev->max_pages_per_mr,
sizeof(void *),
GFP_KERNEL);
if (!req->map_page)
goto out;
}
req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
if (!req->indirect_desc)
goto out;
@@ -1272,11 +1221,6 @@ static void srp_unmap_data(struct scsi_cmnd *scmnd,
if (req->nmdesc)
srp_fr_pool_put(ch->fr_pool, req->fr_list,
req->nmdesc);
} else if (dev->use_fmr) {
struct ib_pool_fmr **pfmr;
for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
ib_fmr_pool_unmap(*pfmr);
}
ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
@@ -1472,50 +1416,6 @@ static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
state->ndesc++;
}
static int srp_map_finish_fmr(struct srp_map_state *state,
struct srp_rdma_ch *ch)
{
struct srp_target_port *target = ch->target;
struct srp_device *dev = target->srp_host->srp_dev;
struct ib_pool_fmr *fmr;
u64 io_addr = 0;
if (state->fmr.next >= state->fmr.end) {
shost_printk(KERN_ERR, ch->target->scsi_host,
PFX "Out of MRs (mr_per_cmd = %d)\n",
ch->target->mr_per_cmd);
return -ENOMEM;
}
WARN_ON_ONCE(!dev->use_fmr);
if (state->npages == 0)
return 0;
if (state->npages == 1 && target->global_rkey) {
srp_map_desc(state, state->base_dma_addr, state->dma_len,
target->global_rkey);
goto reset_state;
}
fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
state->npages, io_addr);
if (IS_ERR(fmr))
return PTR_ERR(fmr);
*state->fmr.next++ = fmr;
state->nmdesc++;
srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
state->dma_len, fmr->fmr->rkey);
reset_state:
state->npages = 0;
state->dma_len = 0;
return 0;
}
static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
{
srp_handle_qp_err(cq, wc, "FAST REG");
@@ -1606,74 +1506,6 @@ static int srp_map_finish_fr(struct srp_map_state *state,
return n;
}
static int srp_map_sg_entry(struct srp_map_state *state,
struct srp_rdma_ch *ch,
struct scatterlist *sg)
{
struct srp_target_port *target = ch->target;
struct srp_device *dev = target->srp_host->srp_dev;
dma_addr_t dma_addr = sg_dma_address(sg);
unsigned int dma_len = sg_dma_len(sg);
unsigned int len = 0;
int ret;
WARN_ON_ONCE(!dma_len);
while (dma_len) {
unsigned offset = dma_addr & ~dev->mr_page_mask;
if (state->npages == dev->max_pages_per_mr ||
(state->npages > 0 && offset != 0)) {
ret = srp_map_finish_fmr(state, ch);
if (ret)
return ret;
}
len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
if (!state->npages)
state->base_dma_addr = dma_addr;
state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
state->dma_len += len;
dma_addr += len;
dma_len -= len;
}
/*
* If the end of the MR is not on a page boundary then we need to
* close it out and start a new one -- we can only merge at page
* boundaries.
*/
ret = 0;
if ((dma_addr & ~dev->mr_page_mask) != 0)
ret = srp_map_finish_fmr(state, ch);
return ret;
}
static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
struct srp_request *req, struct scatterlist *scat,
int count)
{
struct scatterlist *sg;
int i, ret;
state->pages = req->map_page;
state->fmr.next = req->fmr_list;
state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
for_each_sg(scat, sg, count, i) {
ret = srp_map_sg_entry(state, ch, sg);
if (ret)
return ret;
}
ret = srp_map_finish_fmr(state, ch);
if (ret)
return ret;
return 0;
}
static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
struct srp_request *req, struct scatterlist *scat,
int count)
@@ -1733,7 +1565,6 @@ static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
struct srp_device *dev = target->srp_host->srp_dev;
struct srp_map_state state;
struct srp_direct_buf idb_desc;
u64 idb_pages[1];
struct scatterlist idb_sg[1];
int ret;
@@ -1756,14 +1587,6 @@ static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
if (ret < 0)
return ret;
WARN_ON_ONCE(ret < 1);
} else if (dev->use_fmr) {
state.pages = idb_pages;
state.pages[0] = (req->indirect_dma_addr &
dev->mr_page_mask);
state.npages = 1;
ret = srp_map_finish_fmr(&state, ch);
if (ret < 0)
return ret;
} else {
return -EINVAL;
}
@@ -1787,9 +1610,6 @@ static void srp_check_mapping(struct srp_map_state *state,
if (dev->use_fast_reg)
for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
mr_len += (*pfr)->mr->length;
else if (dev->use_fmr)
for (i = 0; i < state->nmdesc; i++)
mr_len += be32_to_cpu(req->indirect_desc[i].len);
if (desc_len != scsi_bufflen(req->scmnd) ||
mr_len > scsi_bufflen(req->scmnd))
pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
@@ -1904,8 +1724,6 @@ static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
state.desc = req->indirect_desc;
if (dev->use_fast_reg)
ret = srp_map_sg_fr(&state, ch, req, scat, count);
else if (dev->use_fmr)
ret = srp_map_sg_fmr(&state, ch, req, scat, count);
else
ret = srp_map_sg_dma(&state, ch, req, scat, count);
req->nmdesc = state.nmdesc;
@@ -3424,6 +3242,7 @@ enum {
SRP_OPT_IP_DEST = 1 << 16,
SRP_OPT_TARGET_CAN_QUEUE= 1 << 17,
SRP_OPT_MAX_IT_IU_SIZE = 1 << 18,
SRP_OPT_CH_COUNT = 1 << 19,
};
static unsigned int srp_opt_mandatory[] = {
@@ -3457,6 +3276,7 @@ static const match_table_t srp_opt_tokens = {
{ SRP_OPT_IP_SRC, "src=%s" },
{ SRP_OPT_IP_DEST, "dest=%s" },
{ SRP_OPT_MAX_IT_IU_SIZE, "max_it_iu_size=%d" },
{ SRP_OPT_CH_COUNT, "ch_count=%u", },
{ SRP_OPT_ERR, NULL }
};
@@ -3758,6 +3578,14 @@ static int srp_parse_options(struct net *net, const char *buf,
target->max_it_iu_size = token;
break;
case SRP_OPT_CH_COUNT:
if (match_int(args, &token) || token < 1) {
pr_warn("bad channel count %s\n", p);
goto out;
}
target->ch_count = token;
break;
default:
pr_warn("unknown parameter or missing value '%s' in target creation request\n",
p);
@@ -3864,13 +3692,13 @@ static ssize_t srp_create_target(struct device *dev,
goto out;
}
if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
if (!srp_dev->has_fr && !target->allow_ext_sg &&
target->cmd_sg_cnt < target->sg_tablesize) {
pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
target->sg_tablesize = target->cmd_sg_cnt;
}
if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
if (srp_dev->use_fast_reg) {
bool gaps_reg = (ibdev->attrs.device_cap_flags &
IB_DEVICE_SG_GAPS_REG);
@@ -3878,12 +3706,12 @@ static ssize_t srp_create_target(struct device *dev,
(ilog2(srp_dev->mr_page_size) - 9);
if (!gaps_reg) {
/*
* FR and FMR can only map one HCA page per entry. If
* the start address is not aligned on a HCA page
* boundary two entries will be used for the head and
* the tail although these two entries combined
* contain at most one HCA page of data. Hence the "+
* 1" in the calculation below.
* FR can only map one HCA page per entry. If the start
* address is not aligned on a HCA page boundary two
* entries will be used for the head and the tail
* although these two entries combined contain at most
* one HCA page of data. Hence the "+ 1" in the
* calculation below.
*
* The indirect data buffer descriptor is contiguous
* so the memory for that buffer will only be
@@ -3921,11 +3749,13 @@ static ssize_t srp_create_target(struct device *dev,
goto out;
ret = -ENOMEM;
target->ch_count = max_t(unsigned, num_online_nodes(),
min(ch_count ? :
min(4 * num_online_nodes(),
ibdev->num_comp_vectors),
num_online_cpus()));
if (target->ch_count == 0)
target->ch_count =
max_t(unsigned int, num_online_nodes(),
min(ch_count ?:
min(4 * num_online_nodes(),
ibdev->num_comp_vectors),
num_online_cpus()));
target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
GFP_KERNEL);
if (!target->ch)
@@ -4132,7 +3962,7 @@ static void srp_rename_dev(struct ib_device *device, void *client_data)
}
}
static void srp_add_one(struct ib_device *device)
static int srp_add_one(struct ib_device *device)
{
struct srp_device *srp_dev;
struct ib_device_attr *attr = &device->attrs;
@@ -4144,7 +3974,7 @@ static void srp_add_one(struct ib_device *device)
srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
if (!srp_dev)
return;
return -ENOMEM;
/*
* Use the smallest page size supported by the HCA, down to a
@@ -4162,23 +3992,15 @@ static void srp_add_one(struct ib_device *device)
srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
max_pages_per_mr);
srp_dev->has_fmr = (device->ops.alloc_fmr &&
device->ops.dealloc_fmr &&
device->ops.map_phys_fmr &&
device->ops.unmap_fmr);
srp_dev->has_fr = (attr->device_cap_flags &
IB_DEVICE_MEM_MGT_EXTENSIONS);
if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
dev_warn(&device->dev, "neither FMR nor FR is supported\n");
} else if (!never_register &&
attr->max_mr_size >= 2 * srp_dev->mr_page_size) {
srp_dev->use_fast_reg = (srp_dev->has_fr &&
(!srp_dev->has_fmr || prefer_fr));
srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
}
if (!never_register && !srp_dev->has_fr)
dev_warn(&device->dev, "FR is not supported\n");
else if (!never_register &&
attr->max_mr_size >= 2 * srp_dev->mr_page_size)
srp_dev->use_fast_reg = srp_dev->has_fr;
if (never_register || !register_always ||
(!srp_dev->has_fmr && !srp_dev->has_fr))
if (never_register || !register_always || !srp_dev->has_fr)
flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
if (srp_dev->use_fast_reg) {
@@ -4197,8 +4019,12 @@ static void srp_add_one(struct ib_device *device)
srp_dev->dev = device;
srp_dev->pd = ib_alloc_pd(device, flags);
if (IS_ERR(srp_dev->pd))
goto free_dev;
if (IS_ERR(srp_dev->pd)) {
int ret = PTR_ERR(srp_dev->pd);
kfree(srp_dev);
return ret;
}
if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey;
@@ -4212,10 +4038,7 @@ static void srp_add_one(struct ib_device *device)
}
ib_set_client_data(device, &srp_client, srp_dev);
return;
free_dev:
kfree(srp_dev);
return 0;
}
static void srp_remove_one(struct ib_device *device, void *client_data)
@@ -4225,8 +4048,6 @@ static void srp_remove_one(struct ib_device *device, void *client_data)
struct srp_target_port *target;
srp_dev = client_data;
if (!srp_dev)
return;
list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
device_unregister(&host->dev);

27
drivers/infiniband/ulp/srp/ib_srp.h
Fájl megtekintése

@@ -44,7 +44,6 @@
#include <rdma/ib_verbs.h>
#include <rdma/ib_sa.h>
#include <rdma/ib_cm.h>
#include <rdma/ib_fmr_pool.h>
#include <rdma/rdma_cm.h>
enum {
@@ -95,8 +94,7 @@ enum srp_iu_type {
/*
* @mr_page_mask: HCA memory registration page mask.
* @mr_page_size: HCA memory registration page size.
* @mr_max_size: Maximum size in bytes of a single FMR / FR registration
* request.
* @mr_max_size: Maximum size in bytes of a single FR registration request.
*/
struct srp_device {
struct list_head dev_list;
@@ -107,9 +105,7 @@ struct srp_device {
int mr_page_size;
int mr_max_size;
int max_pages_per_mr;
bool has_fmr;
bool has_fr;
bool use_fmr;
bool use_fast_reg;
};
@@ -127,11 +123,7 @@ struct srp_host {
struct srp_request {
struct scsi_cmnd *scmnd;
struct srp_iu *cmd;
union {
struct ib_pool_fmr **fmr_list;
struct srp_fr_desc **fr_list;
};
u64 *map_page;
struct srp_fr_desc **fr_list;
struct srp_direct_buf *indirect_desc;
dma_addr_t indirect_dma_addr;
short nmdesc;
@@ -155,10 +147,7 @@ struct srp_rdma_ch {
struct ib_cq *send_cq;
struct ib_cq *recv_cq;
struct ib_qp *qp;
union {
struct ib_fmr_pool *fmr_pool;
struct srp_fr_pool *fr_pool;
};
struct srp_fr_pool *fr_pool;
uint32_t max_it_iu_len;
uint32_t max_ti_iu_len;
u8 max_imm_sge;
@@ -319,19 +308,15 @@ struct srp_fr_pool {
* @pages: Array with DMA addresses of pages being considered for
* memory registration.
* @base_dma_addr: DMA address of the first page that has not yet been mapped.
* @dma_len: Number of bytes that will be registered with the next
* FMR or FR memory registration call.
* @dma_len: Number of bytes that will be registered with the next FR
* memory registration call.
* @total_len: Total number of bytes in the sg-list being mapped.
* @npages: Number of page addresses in the pages[] array.
* @nmdesc: Number of FMR or FR memory descriptors used for mapping.
* @nmdesc: Number of FR memory descriptors used for mapping.
* @ndesc: Number of SRP buffer descriptors that have been filled in.
*/
struct srp_map_state {
union {
struct {
struct ib_pool_fmr **next;
struct ib_pool_fmr **end;
} fmr;
struct {
struct srp_fr_desc **next;
struct srp_fr_desc **end;

67
drivers/infiniband/ulp/srpt/ib_srpt.c
Fájl megtekintése

@@ -81,7 +81,7 @@ MODULE_PARM_DESC(srpt_srq_size,
static int srpt_get_u64_x(char *buffer, const struct kernel_param *kp)
{
return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
return sprintf(buffer, "0x%016llx\n", *(u64 *)kp->arg);
}
module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
0444);
@@ -135,14 +135,11 @@ static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
static void srpt_event_handler(struct ib_event_handler *handler,
struct ib_event *event)
{
struct srpt_device *sdev;
struct srpt_device *sdev =
container_of(handler, struct srpt_device, event_handler);
struct srpt_port *sport;
u8 port_num;
sdev = ib_get_client_data(event->device, &srpt_client);
if (!sdev || sdev->device != event->device)
return;
pr_debug("ASYNC event= %d on device= %s\n", event->event,
dev_name(&sdev->device->dev));
@@ -217,8 +214,9 @@ static const char *get_ch_state_name(enum rdma_ch_state s)
*/
static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
{
pr_debug("QP event %d on ch=%p sess_name=%s state=%d\n",
event->event, ch, ch->sess_name, ch->state);
pr_debug("QP event %d on ch=%p sess_name=%s-%d state=%s\n",
event->event, ch, ch->sess_name, ch->qp->qp_num,
get_ch_state_name(ch->state));
switch (event->event) {
case IB_EVENT_COMM_EST:
@@ -610,6 +608,11 @@ static int srpt_refresh_port(struct srpt_port *sport)
dev_name(&sport->sdev->device->dev), sport->port,
PTR_ERR(sport->mad_agent));
sport->mad_agent = NULL;
memset(&port_modify, 0, sizeof(port_modify));
port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
ib_modify_port(sport->sdev->device, sport->port, 0,
&port_modify);
}
}
@@ -633,9 +636,8 @@ static void srpt_unregister_mad_agent(struct srpt_device *sdev)
for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
sport = &sdev->port[i - 1];
WARN_ON(sport->port != i);
if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
pr_err("disabling MAD processing failed.\n");
if (sport->mad_agent) {
ib_modify_port(sdev->device, i, 0, &port_modify);
ib_unregister_mad_agent(sport->mad_agent);
sport->mad_agent = NULL;
}
@@ -1814,18 +1816,13 @@ retry:
*/
qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
qp_init->cap.max_rdma_ctxs = sq_size / 2;
qp_init->cap.max_send_sge = min(attrs->max_send_sge,
SRPT_MAX_SG_PER_WQE);
qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
SRPT_MAX_SG_PER_WQE);
qp_init->cap.max_send_sge = attrs->max_send_sge;
qp_init->cap.max_recv_sge = 1;
qp_init->port_num = ch->sport->port;
if (sdev->use_srq) {
if (sdev->use_srq)
qp_init->srq = sdev->srq;
} else {
else
qp_init->cap.max_recv_wr = ch->rq_size;
qp_init->cap.max_recv_sge = min(attrs->max_recv_sge,
SRPT_MAX_SG_PER_WQE);
}
if (ch->using_rdma_cm) {
ret = rdma_create_qp(ch->rdma_cm.cm_id, sdev->pd, qp_init);
@@ -1984,8 +1981,8 @@ static void __srpt_close_all_ch(struct srpt_port *sport)
list_for_each_entry(nexus, &sport->nexus_list, entry) {
list_for_each_entry(ch, &nexus->ch_list, list) {
if (srpt_disconnect_ch(ch) >= 0)
pr_info("Closing channel %s because target %s_%d has been disabled\n",
ch->sess_name,
pr_info("Closing channel %s-%d because target %s_%d has been disabled\n",
ch->sess_name, ch->qp->qp_num,
dev_name(&sport->sdev->device->dev),
sport->port);
srpt_close_ch(ch);
@@ -2496,7 +2493,8 @@ reject:
SRP_BUF_FORMAT_INDIRECT);
if (rdma_cm_id)
rdma_reject(rdma_cm_id, rej, sizeof(*rej));
rdma_reject(rdma_cm_id, rej, sizeof(*rej),
IB_CM_REJ_CONSUMER_DEFINED);
else
ib_send_cm_rej(ib_cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
rej, sizeof(*rej));
@@ -3104,7 +3102,7 @@ static int srpt_use_srq(struct srpt_device *sdev, bool use_srq)
* srpt_add_one - InfiniBand device addition callback function
* @device: Describes a HCA.
*/
static void srpt_add_one(struct ib_device *device)
static int srpt_add_one(struct ib_device *device)
{
struct srpt_device *sdev;
struct srpt_port *sport;
@@ -3115,14 +3113,16 @@ static void srpt_add_one(struct ib_device *device)
sdev = kzalloc(struct_size(sdev, port, device->phys_port_cnt),
GFP_KERNEL);
if (!sdev)
goto err;
return -ENOMEM;
sdev->device = device;
mutex_init(&sdev->sdev_mutex);
sdev->pd = ib_alloc_pd(device, 0);
if (IS_ERR(sdev->pd))
if (IS_ERR(sdev->pd)) {
ret = PTR_ERR(sdev->pd);
goto free_dev;
}
sdev->lkey = sdev->pd->local_dma_lkey;
@@ -3138,6 +3138,7 @@ static void srpt_add_one(struct ib_device *device)
if (IS_ERR(sdev->cm_id)) {
pr_info("ib_create_cm_id() failed: %ld\n",
PTR_ERR(sdev->cm_id));
ret = PTR_ERR(sdev->cm_id);
sdev->cm_id = NULL;
if (!rdma_cm_id)
goto err_ring;
@@ -3182,7 +3183,8 @@ static void srpt_add_one(struct ib_device *device)
mutex_init(&sport->port_gid_id.mutex);
INIT_LIST_HEAD(&sport->port_gid_id.tpg_list);
if (srpt_refresh_port(sport)) {
ret = srpt_refresh_port(sport);
if (ret) {
pr_err("MAD registration failed for %s-%d.\n",
dev_name(&sdev->device->dev), i);
goto err_event;
@@ -3193,10 +3195,9 @@ static void srpt_add_one(struct ib_device *device)
list_add_tail(&sdev->list, &srpt_dev_list);
spin_unlock(&srpt_dev_lock);
out:
ib_set_client_data(device, &srpt_client, sdev);
pr_debug("added %s.\n", dev_name(&device->dev));
return;
return 0;
err_event:
ib_unregister_event_handler(&sdev->event_handler);
@@ -3208,10 +3209,8 @@ err_ring:
ib_dealloc_pd(sdev->pd);
free_dev:
kfree(sdev);
err:
sdev = NULL;
pr_info("%s(%s) failed.\n", __func__, dev_name(&device->dev));
goto out;
return ret;
}
/**
@@ -3224,12 +3223,6 @@ static void srpt_remove_one(struct ib_device *device, void *client_data)
struct srpt_device *sdev = client_data;
int i;
if (!sdev) {
pr_info("%s(%s): nothing to do.\n", __func__,
dev_name(&device->dev));
return;
}
srpt_unregister_mad_agent(sdev);
ib_unregister_event_handler(&sdev->event_handler);

5
drivers/infiniband/ulp/srpt/ib_srpt.h
Fájl megtekintése

@@ -105,11 +105,6 @@ enum {
SRP_CMD_ACA = 0x4,
SRPT_DEF_SG_TABLESIZE = 128,
/*
* An experimentally determined value that avoids that QP creation
* fails due to "swiotlb buffer is full" on systems using the swiotlb.
*/
SRPT_MAX_SG_PER_WQE = 16,
MIN_SRPT_SQ_SIZE = 16,
DEF_SRPT_SQ_SIZE = 4096,