IB/hfi1: Rework fault injection machinery
The packet fault injection code present in the HFI1 driver had some
issues which not only fragment the code but also created user
confusion. Furthermore, it suffered from the following issues:
1. The fault_packet method only worked for received packets. This
meant that the only fault injection mode available for sent
packets is fault_opcode, which did not allow for random packet
drops on all egressing packets.
2. The mask available for the fault_opcode mode did not really work
due to the fact that the opcode values are not bits in a bitmask but
rather sequential integer values. Creating a opcode/mask pair that
would successfully capture a set of packets was nearly impossible.
3. The code was fragmented and used too many debugfs entries to
operate and control. This was confusing to users.
4. It did not allow filtering fault injection on a per direction basis -
egress vs. ingress.
In order to improve or fix the above issues, the following changes have
been made:
1. The fault injection methods have been combined into a single fault
injection facility. As such, the fault injection has been plugged
into both the send and receive code paths. Regardless of method used
the fault injection will operate on both egress and ingress packets.
2. The type of fault injection - by packet or by opcode - is now controlled
by changing the boolean value of the file "opcode_mode". When the value
is set to True, fault injection is done by opcode. Otherwise, by
packet.
2. The masking ability has been removed in favor of a bitmap that holds
opcodes of interest (one bit per opcode, a total of 256 bits). This
works in tandem with the "opcode_mode" value. When the value of
"opcode_mode" is False, this bitmap is ignored. When the value is
True, the bitmap lists all opcodes to be considered for fault injection.
By default, the bitmap is empty. When the user wants to filter by opcode,
the user sets the corresponding bit in the bitmap by echo'ing the bit
position into the 'opcodes' file. This gets around the issue that the set
of opcodes does not lend itself to effective masks and allow for extremely
fine-grained filtering by opcode.
4. fault_packet and fault_opcode methods have been combined. Hence, there
is only one debugfs directory controlling the entire operation of the
fault injection machinery. This reduces the number of debugfs entries
and provides a more unified user experience.
5. A new control files - "direction" - is provided to allow the user to
control the direction of packets, which are subject to fault injection.
6. A new control file - "skip_usec" - is added that would allow the user
to specify a "timeout" during which no fault injection will occur.
In addition, the following bug fixes have been applied:
1. The fault injection code has been split into its own header and source
files. This was done to better organize the code and support conditional
compilation without littering the code with #ifdef's.
2. The method by which the TX PIO packets were being marked for drop
conflicted with the way send contexts were being setup. As a result,
the send context was repeatedly being reset.
3. The fault injection only makes sense when the user can control it
through the debugfs entries. However, a kernel configuration can
enable fault injection but keep fault injection debugfs entries
disabled. Therefore, it makes sense that the HFI fault injection
code depends on both.
4. Error suppression did not take into account the method by which PIO
packets were being dropped. Therefore, even with error suppression
turned on, errors would still be displayed to the screen. A larger
enough packet drop percentage would case the kernel to crash because
the driver would be stuck printing errors.
Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Reviewed-by: Don Hiatt <don.hiatt@intel.com>
Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Mitko Haralanov <mitko.haralanov@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
This commit is contained in:
Mitko Haralanov
committed by
Doug Ledford
Doug Ledford
parent
8d3e71136a
commit
a74d5307ca
@@ -60,15 +60,13 @@
|
||||
#include "device.h"
|
||||
#include "qp.h"
|
||||
#include "sdma.h"
|
||||
#include "fault.h"
|
||||
|
||||
static struct dentry *hfi1_dbg_root;
|
||||
|
||||
/* wrappers to enforce srcu in seq file */
|
||||
static ssize_t hfi1_seq_read(
|
||||
struct file *file,
|
||||
char __user *buf,
|
||||
size_t size,
|
||||
loff_t *ppos)
|
||||
ssize_t hfi1_seq_read(struct file *file, char __user *buf, size_t size,
|
||||
loff_t *ppos)
|
||||
{
|
||||
struct dentry *d = file->f_path.dentry;
|
||||
ssize_t r;
|
||||
@@ -81,10 +79,7 @@ static ssize_t hfi1_seq_read(
|
||||
return r;
|
||||
}
|
||||
|
||||
static loff_t hfi1_seq_lseek(
|
||||
struct file *file,
|
||||
loff_t offset,
|
||||
int whence)
|
||||
loff_t hfi1_seq_lseek(struct file *file, loff_t offset, int whence)
|
||||
{
|
||||
struct dentry *d = file->f_path.dentry;
|
||||
loff_t r;
|
||||
@@ -100,48 +95,6 @@ static loff_t hfi1_seq_lseek(
|
||||
#define private2dd(file) (file_inode(file)->i_private)
|
||||
#define private2ppd(file) (file_inode(file)->i_private)
|
||||
|
||||
#define DEBUGFS_SEQ_FILE_OPS(name) \
|
||||
static const struct seq_operations _##name##_seq_ops = { \
|
||||
.start = _##name##_seq_start, \
|
||||
.next = _##name##_seq_next, \
|
||||
.stop = _##name##_seq_stop, \
|
||||
.show = _##name##_seq_show \
|
||||
}
|
||||
|
||||
#define DEBUGFS_SEQ_FILE_OPEN(name) \
|
||||
static int _##name##_open(struct inode *inode, struct file *s) \
|
||||
{ \
|
||||
struct seq_file *seq; \
|
||||
int ret; \
|
||||
ret = seq_open(s, &_##name##_seq_ops); \
|
||||
if (ret) \
|
||||
return ret; \
|
||||
seq = s->private_data; \
|
||||
seq->private = inode->i_private; \
|
||||
return 0; \
|
||||
}
|
||||
|
||||
#define DEBUGFS_FILE_OPS(name) \
|
||||
static const struct file_operations _##name##_file_ops = { \
|
||||
.owner = THIS_MODULE, \
|
||||
.open = _##name##_open, \
|
||||
.read = hfi1_seq_read, \
|
||||
.llseek = hfi1_seq_lseek, \
|
||||
.release = seq_release \
|
||||
}
|
||||
|
||||
#define DEBUGFS_FILE_CREATE(name, parent, data, ops, mode) \
|
||||
do { \
|
||||
struct dentry *ent; \
|
||||
ent = debugfs_create_file(name, mode, parent, \
|
||||
data, ops); \
|
||||
if (!ent) \
|
||||
pr_warn("create of %s failed\n", name); \
|
||||
} while (0)
|
||||
|
||||
#define DEBUGFS_SEQ_FILE_CREATE(name, parent, data) \
|
||||
DEBUGFS_FILE_CREATE(#name, parent, data, &_##name##_file_ops, S_IRUGO)
|
||||
|
||||
static void *_opcode_stats_seq_start(struct seq_file *s, loff_t *pos)
|
||||
{
|
||||
struct hfi1_opcode_stats_perctx *opstats;
|
||||
@@ -1160,236 +1113,6 @@ DEBUGFS_SEQ_FILE_OPS(sdma_cpu_list);
|
||||
DEBUGFS_SEQ_FILE_OPEN(sdma_cpu_list)
|
||||
DEBUGFS_FILE_OPS(sdma_cpu_list);
|
||||
|
||||
#ifdef CONFIG_FAULT_INJECTION
|
||||
static void *_fault_stats_seq_start(struct seq_file *s, loff_t *pos)
|
||||
{
|
||||
struct hfi1_opcode_stats_perctx *opstats;
|
||||
|
||||
if (*pos >= ARRAY_SIZE(opstats->stats))
|
||||
return NULL;
|
||||
return pos;
|
||||
}
|
||||
|
||||
static void *_fault_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
|
||||
{
|
||||
struct hfi1_opcode_stats_perctx *opstats;
|
||||
|
||||
++*pos;
|
||||
if (*pos >= ARRAY_SIZE(opstats->stats))
|
||||
return NULL;
|
||||
return pos;
|
||||
}
|
||||
|
||||
static void _fault_stats_seq_stop(struct seq_file *s, void *v)
|
||||
{
|
||||
}
|
||||
|
||||
static int _fault_stats_seq_show(struct seq_file *s, void *v)
|
||||
{
|
||||
loff_t *spos = v;
|
||||
loff_t i = *spos, j;
|
||||
u64 n_packets = 0, n_bytes = 0;
|
||||
struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
|
||||
struct hfi1_devdata *dd = dd_from_dev(ibd);
|
||||
struct hfi1_ctxtdata *rcd;
|
||||
|
||||
for (j = 0; j < dd->first_dyn_alloc_ctxt; j++) {
|
||||
rcd = hfi1_rcd_get_by_index(dd, j);
|
||||
if (rcd) {
|
||||
n_packets += rcd->opstats->stats[i].n_packets;
|
||||
n_bytes += rcd->opstats->stats[i].n_bytes;
|
||||
}
|
||||
hfi1_rcd_put(rcd);
|
||||
}
|
||||
for_each_possible_cpu(j) {
|
||||
struct hfi1_opcode_stats_perctx *sp =
|
||||
per_cpu_ptr(dd->tx_opstats, j);
|
||||
|
||||
n_packets += sp->stats[i].n_packets;
|
||||
n_bytes += sp->stats[i].n_bytes;
|
||||
}
|
||||
if (!n_packets && !n_bytes)
|
||||
return SEQ_SKIP;
|
||||
if (!ibd->fault_opcode->n_rxfaults[i] &&
|
||||
!ibd->fault_opcode->n_txfaults[i])
|
||||
return SEQ_SKIP;
|
||||
seq_printf(s, "%02llx %llu/%llu (faults rx:%llu faults: tx:%llu)\n", i,
|
||||
(unsigned long long)n_packets,
|
||||
(unsigned long long)n_bytes,
|
||||
(unsigned long long)ibd->fault_opcode->n_rxfaults[i],
|
||||
(unsigned long long)ibd->fault_opcode->n_txfaults[i]);
|
||||
return 0;
|
||||
}
|
||||
|
||||
DEBUGFS_SEQ_FILE_OPS(fault_stats);
|
||||
DEBUGFS_SEQ_FILE_OPEN(fault_stats);
|
||||
DEBUGFS_FILE_OPS(fault_stats);
|
||||
|
||||
static void fault_exit_opcode_debugfs(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
if (ibd->fault_opcode)
|
||||
debugfs_remove_recursive(ibd->fault_opcode->dir);
|
||||
kfree(ibd->fault_opcode);
|
||||
ibd->fault_opcode = NULL;
|
||||
}
|
||||
|
||||
static int fault_init_opcode_debugfs(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
struct dentry *parent = ibd->hfi1_ibdev_dbg;
|
||||
|
||||
ibd->fault_opcode = kzalloc(sizeof(*ibd->fault_opcode), GFP_KERNEL);
|
||||
if (!ibd->fault_opcode)
|
||||
return -ENOMEM;
|
||||
|
||||
ibd->fault_opcode->attr.interval = 1;
|
||||
ibd->fault_opcode->attr.require_end = ULONG_MAX;
|
||||
ibd->fault_opcode->attr.stacktrace_depth = 32;
|
||||
ibd->fault_opcode->attr.dname = NULL;
|
||||
ibd->fault_opcode->attr.verbose = 0;
|
||||
ibd->fault_opcode->fault_by_opcode = false;
|
||||
ibd->fault_opcode->opcode = 0;
|
||||
ibd->fault_opcode->mask = 0xff;
|
||||
|
||||
ibd->fault_opcode->dir =
|
||||
fault_create_debugfs_attr("fault_opcode",
|
||||
parent,
|
||||
&ibd->fault_opcode->attr);
|
||||
if (IS_ERR(ibd->fault_opcode->dir)) {
|
||||
kfree(ibd->fault_opcode);
|
||||
ibd->fault_opcode = NULL;
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
DEBUGFS_SEQ_FILE_CREATE(fault_stats, ibd->fault_opcode->dir, ibd);
|
||||
if (!debugfs_create_bool("fault_by_opcode", 0600,
|
||||
ibd->fault_opcode->dir,
|
||||
&ibd->fault_opcode->fault_by_opcode))
|
||||
goto fail;
|
||||
if (!debugfs_create_x8("opcode", 0600, ibd->fault_opcode->dir,
|
||||
&ibd->fault_opcode->opcode))
|
||||
goto fail;
|
||||
if (!debugfs_create_x8("mask", 0600, ibd->fault_opcode->dir,
|
||||
&ibd->fault_opcode->mask))
|
||||
goto fail;
|
||||
|
||||
return 0;
|
||||
fail:
|
||||
fault_exit_opcode_debugfs(ibd);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static void fault_exit_packet_debugfs(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
if (ibd->fault_packet)
|
||||
debugfs_remove_recursive(ibd->fault_packet->dir);
|
||||
kfree(ibd->fault_packet);
|
||||
ibd->fault_packet = NULL;
|
||||
}
|
||||
|
||||
static int fault_init_packet_debugfs(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
struct dentry *parent = ibd->hfi1_ibdev_dbg;
|
||||
|
||||
ibd->fault_packet = kzalloc(sizeof(*ibd->fault_packet), GFP_KERNEL);
|
||||
if (!ibd->fault_packet)
|
||||
return -ENOMEM;
|
||||
|
||||
ibd->fault_packet->attr.interval = 1;
|
||||
ibd->fault_packet->attr.require_end = ULONG_MAX;
|
||||
ibd->fault_packet->attr.stacktrace_depth = 32;
|
||||
ibd->fault_packet->attr.dname = NULL;
|
||||
ibd->fault_packet->attr.verbose = 0;
|
||||
ibd->fault_packet->fault_by_packet = false;
|
||||
|
||||
ibd->fault_packet->dir =
|
||||
fault_create_debugfs_attr("fault_packet",
|
||||
parent,
|
||||
&ibd->fault_opcode->attr);
|
||||
if (IS_ERR(ibd->fault_packet->dir)) {
|
||||
kfree(ibd->fault_packet);
|
||||
ibd->fault_packet = NULL;
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
if (!debugfs_create_bool("fault_by_packet", 0600,
|
||||
ibd->fault_packet->dir,
|
||||
&ibd->fault_packet->fault_by_packet))
|
||||
goto fail;
|
||||
if (!debugfs_create_u64("fault_stats", 0400,
|
||||
ibd->fault_packet->dir,
|
||||
&ibd->fault_packet->n_faults))
|
||||
goto fail;
|
||||
|
||||
return 0;
|
||||
fail:
|
||||
fault_exit_packet_debugfs(ibd);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static void fault_exit_debugfs(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
fault_exit_opcode_debugfs(ibd);
|
||||
fault_exit_packet_debugfs(ibd);
|
||||
}
|
||||
|
||||
static int fault_init_debugfs(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
ret = fault_init_opcode_debugfs(ibd);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = fault_init_packet_debugfs(ibd);
|
||||
if (ret)
|
||||
fault_exit_opcode_debugfs(ibd);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool hfi1_dbg_fault_suppress_err(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
return ibd->fault_suppress_err;
|
||||
}
|
||||
|
||||
bool hfi1_dbg_fault_opcode(struct rvt_qp *qp, u32 opcode, bool rx)
|
||||
{
|
||||
bool ret = false;
|
||||
struct hfi1_ibdev *ibd = to_idev(qp->ibqp.device);
|
||||
|
||||
if (!ibd->fault_opcode || !ibd->fault_opcode->fault_by_opcode)
|
||||
return false;
|
||||
if (ibd->fault_opcode->opcode != (opcode & ibd->fault_opcode->mask))
|
||||
return false;
|
||||
ret = should_fail(&ibd->fault_opcode->attr, 1);
|
||||
if (ret) {
|
||||
trace_hfi1_fault_opcode(qp, opcode);
|
||||
if (rx)
|
||||
ibd->fault_opcode->n_rxfaults[opcode]++;
|
||||
else
|
||||
ibd->fault_opcode->n_txfaults[opcode]++;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool hfi1_dbg_fault_packet(struct hfi1_packet *packet)
|
||||
{
|
||||
struct rvt_dev_info *rdi = &packet->rcd->ppd->dd->verbs_dev.rdi;
|
||||
struct hfi1_ibdev *ibd = dev_from_rdi(rdi);
|
||||
bool ret = false;
|
||||
|
||||
if (!ibd->fault_packet || !ibd->fault_packet->fault_by_packet)
|
||||
return false;
|
||||
|
||||
ret = should_fail(&ibd->fault_packet->attr, 1);
|
||||
if (ret) {
|
||||
++ibd->fault_packet->n_faults;
|
||||
trace_hfi1_fault_packet(packet);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
char name[sizeof("port0counters") + 1];
|
||||
@@ -1442,21 +1165,14 @@ void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
|
||||
S_IRUGO : S_IRUGO | S_IWUSR);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_FAULT_INJECTION
|
||||
debugfs_create_bool("fault_suppress_err", 0600,
|
||||
ibd->hfi1_ibdev_dbg,
|
||||
&ibd->fault_suppress_err);
|
||||
fault_init_debugfs(ibd);
|
||||
#endif
|
||||
hfi1_fault_init_debugfs(ibd);
|
||||
}
|
||||
|
||||
void hfi1_dbg_ibdev_exit(struct hfi1_ibdev *ibd)
|
||||
{
|
||||
if (!hfi1_dbg_root)
|
||||
goto out;
|
||||
#ifdef CONFIG_FAULT_INJECTION
|
||||
fault_exit_debugfs(ibd);
|
||||
#endif
|
||||
hfi1_fault_exit_debugfs(ibd);
|
||||
debugfs_remove(ibd->hfi1_ibdev_link);
|
||||
debugfs_remove_recursive(ibd->hfi1_ibdev_dbg);
|
||||
out:
|
||||
|
||||
Reference in New Issue
Block a user