samsung-sm8650/android_kernel_samsung_sm8650-modules
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge 07a41b0f00 on remote branch

Browse Source 
Change-Id: I664bd0ccc671cd0637d878716a2410ed0480b223
This commit is contained in:
Linux Build Service Account
2023-06-20 16:34:04 -07:00
parent f8e1fb5877 07a41b0f00
commit 904419bd61
7 changed files with 718 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hw_fence/include/hw_fence_drv_debug.h
View File

@@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0-only */ /* SPDX-License-Identifier: GPL-2.0-only */
/* /*
* Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
*/ */
#ifndef __HW_FENCE_DRV_DEBUG #ifndef __HW_FENCE_DRV_DEBUG
@@ -60,6 +60,9 @@ extern u32 msm_hw_fence_debug_level;
#define HWFNC_DBG_LOCK(fmt, ...) \ #define HWFNC_DBG_LOCK(fmt, ...) \
dprintk(HW_FENCE_LOCK, "[hwfence:%s:%d][dbglock]"fmt, __func__, __LINE__, ##__VA_ARGS__) dprintk(HW_FENCE_LOCK, "[hwfence:%s:%d][dbglock]"fmt, __func__, __LINE__, ##__VA_ARGS__)
#define HWFNC_DBG_DUMP(prio, fmt, ...) \
dprintk(prio, "[hwfence:%s:%d][dbgd]"fmt, __func__, __LINE__, ##__VA_ARGS__)
#define HWFNC_WARN(fmt, ...) \ #define HWFNC_WARN(fmt, ...) \
pr_warn("[hwfence:%s:%d][warn][%pS] "fmt, __func__, __LINE__, \ pr_warn("[hwfence:%s:%d][warn][%pS] "fmt, __func__, __LINE__, \
__builtin_return_address(0), ##__VA_ARGS__) __builtin_return_address(0), ##__VA_ARGS__)
@@ -70,6 +73,13 @@ int hw_fence_debug_debugfs_register(struct hw_fence_driver_data *drv_data);
int process_validation_client_loopback(struct hw_fence_driver_data *drv_data, int client_id); int process_validation_client_loopback(struct hw_fence_driver_data *drv_data, int client_id);
void hw_fence_debug_dump_queues(enum hw_fence_drv_prio prio,
struct msm_hw_fence_client *hw_fence_client);
void hw_fence_debug_dump_fence(enum hw_fence_drv_prio prio, struct msm_hw_fence *hw_fence, u64 hash,
u32 count);
void hw_fence_debug_dump_table(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data);
void hw_fence_debug_dump_events(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data);
extern const struct file_operations hw_sync_debugfs_fops; extern const struct file_operations hw_sync_debugfs_fops;
struct hw_fence_out_clients_map { struct hw_fence_out_clients_map {

16
hw_fence/include/hw_fence_drv_priv.h
View File

@@ -131,9 +131,12 @@ struct msm_hw_fence_queue {
/** /**
* enum payload_type - Enum with the queue payload types. * enum payload_type - Enum with the queue payload types.
* HW_FENCE_PAYLOAD_TYPE_1: client queue payload
* HW_FENCE_PAYLOAD_TYPE_2: ctrl queue payload for fence error; client_data stores client_id
*/ */
enum payload_type { enum payload_type {
HW_FENCE_PAYLOAD_TYPE_1 = 1 HW_FENCE_PAYLOAD_TYPE_1 = 1,
HW_FENCE_PAYLOAD_TYPE_2
}; };
/** /**
@@ -144,6 +147,10 @@ enum payload_type {
* @mem_descriptor: hfi header memory descriptor * @mem_descriptor: hfi header memory descriptor
* @queues: queues descriptor * @queues: queues descriptor
* @queues_num: number of client queues * @queues_num: number of client queues
* @fence_error_cb: function called for waiting clients that need HLOS notification of fence error
* @fence_error_cb_userdata: opaque pointer registered with fence error callback and passed to
* client during invocation of callback function
* @error_cb_lock: lock to synchronize access to fence error cb and fence error cb data
* @ipc_signal_id: id of the signal to be triggered for this client * @ipc_signal_id: id of the signal to be triggered for this client
* @ipc_client_vid: virtual id of the ipc client for this hw fence driver client * @ipc_client_vid: virtual id of the ipc client for this hw fence driver client
* @ipc_client_pid: physical id of the ipc client for this hw fence driver client * @ipc_client_pid: physical id of the ipc client for this hw fence driver client
@@ -158,6 +165,9 @@ struct msm_hw_fence_client {
struct msm_hw_fence_mem_addr mem_descriptor; struct msm_hw_fence_mem_addr mem_descriptor;
struct msm_hw_fence_queue queues[HW_FENCE_CLIENT_QUEUES]; struct msm_hw_fence_queue queues[HW_FENCE_CLIENT_QUEUES];
int queues_num; int queues_num;
msm_hw_fence_error_cb_t fence_error_cb;
void *fence_error_cb_userdata;
struct mutex error_cb_lock;
int ipc_signal_id; int ipc_signal_id;
int ipc_client_vid; int ipc_client_vid;
int ipc_client_pid; int ipc_client_pid;
@@ -505,9 +515,13 @@ int hw_fence_process_fence(struct hw_fence_driver_data *drv_data,
int hw_fence_update_queue(struct hw_fence_driver_data *drv_data, int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id, u64 seqno, u64 hash, struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id, u64 seqno, u64 hash,
u64 flags, u64 client_data, u32 error, int queue_type); u64 flags, u64 client_data, u32 error, int queue_type);
int hw_fence_update_existing_txq_payload(struct hw_fence_driver_data *drv_data,
struct msm_hw_fence_client *hw_fence_client, u64 hash, u32 error);
inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data); inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data);
int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client, int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
struct msm_hw_fence_queue_payload *payload, int queue_type); struct msm_hw_fence_queue_payload *payload, int queue_type);
int hw_fence_read_queue_helper(struct msm_hw_fence_queue *queue,
struct msm_hw_fence_queue_payload *payload);
int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data, int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
struct dma_fence *fence, struct msm_hw_fence_client *hw_fence_client, u64 context, struct dma_fence *fence, struct msm_hw_fence_client *hw_fence_client, u64 context,
u64 seqno, u64 *hash, u64 client_data); u64 seqno, u64 *hash, u64 client_data);

15
hw_fence/include/hw_fence_drv_utils.h
View File

@@ -122,6 +122,21 @@ int hw_fence_utils_cleanup_fence(struct hw_fence_driver_data *drv_data,
struct msm_hw_fence_client *hw_fence_client, struct msm_hw_fence *hw_fence, u64 hash, struct msm_hw_fence_client *hw_fence_client, struct msm_hw_fence *hw_fence, u64 hash,
u32 reset_flags); u32 reset_flags);
/**
* hw_fence_utils_fence_error_cb() - Invokes fence error callback registered by specified client
*
* @hw_fence_client: client, for which fence error callback must be invoked
* @ctxt_id: context id of the hw-fence
* @seqno: sequence number of the hw-fence
* @hash: hash of the hw-fence
* @flags: flags of the hw-fence
* @error: error of the hw-fence
*
* Returns zero if success, otherwise returns negative error code
*/
int hw_fence_utils_fence_error_cb(struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id,
u64 seqno, u64 hash, u64 flags, u32 error);
/** /**
* hw_fence_utils_get_client_id_priv() - Gets the index into clients struct within hw fence driver * hw_fence_utils_get_client_id_priv() - Gets the index into clients struct within hw fence driver
* from the client_id used externally * from the client_id used externally

276
hw_fence/src/hw_fence_drv_debug.c
View File

@@ -14,6 +14,14 @@
#define HW_FENCE_DEBUG_MAX_LOOPS 200 #define HW_FENCE_DEBUG_MAX_LOOPS 200
#define HFENCE_TBL_MSG \
"[%d]hfence[%d] v:%d err:%lu ctx:%llu seq:%llu wait:0x%llx alloc:%d f:0x%llx child_cnt:%d" \
"%s ct:%llu tt:%llu wt:%llu\n"
/* each hwfence parent includes one "32-bit" element + "," separator */
#define HW_FENCE_MAX_PARENTS_SUBLIST_DUMP (MSM_HW_FENCE_MAX_JOIN_PARENTS * 9)
#define HW_FENCE_MAX_PARENTS_DUMP (sizeof("parent_list[] ") + HW_FENCE_MAX_PARENTS_SUBLIST_DUMP)
/* event dump data includes one "32-bit" element + "|" separator */ /* event dump data includes one "32-bit" element + "|" separator */
#define HW_FENCE_MAX_DATA_PER_EVENT_DUMP (HW_FENCE_EVENT_MAX_DATA * 9) #define HW_FENCE_MAX_DATA_PER_EVENT_DUMP (HW_FENCE_EVENT_MAX_DATA * 9)
@@ -473,29 +481,82 @@ static ssize_t hw_fence_dbg_create_wr(struct file *file,
return count; return count;
} }
#define HFENCE_TBL_MSG \ static void _dump_fence_helper(enum hw_fence_drv_prio prio, struct msm_hw_fence *hw_fence,
"[%d]hfence[%d] v:%d err:%d ctx:%d seqno:%d wait:0x%llx alloc:%d f:0x%lx tt:%llu wt:%llu\n" char *parents_dump, u64 hash, u32 count)
{
char sublist[HW_FENCE_MAX_PARENTS_SUBLIST_DUMP];
u32 parents_cnt;
int i, len = 0;
if (!hw_fence || !parents_dump) {
HWFNC_ERR("invalid params hw_fence:0x%pK parents_dump:0x%pK\n", hw_fence,
parents_dump);
return;
}
memset(parents_dump, 0, sizeof(char) * HW_FENCE_MAX_PARENTS_DUMP);
if (hw_fence->parents_cnt) {
if (hw_fence->parents_cnt > MSM_HW_FENCE_MAX_JOIN_PARENTS) {
HWFNC_ERR("hfence[%d] has invalid parents_cnt:%d greater than max:%d\n",
hash, hw_fence->parents_cnt, MSM_HW_FENCE_MAX_JOIN_PARENTS);
parents_cnt = MSM_HW_FENCE_MAX_JOIN_PARENTS;
} else {
parents_cnt = hw_fence->parents_cnt;
}
memset(sublist, 0, sizeof(sublist));
for (i = 0; i < parents_cnt; i++)
len += scnprintf(sublist + len, HW_FENCE_MAX_PARENTS_SUBLIST_DUMP - len,
"%lu,", hw_fence->parent_list[i]);
scnprintf(parents_dump, HW_FENCE_MAX_PARENTS_DUMP, " p:[%s]", sublist);
}
HWFNC_DBG_DUMP(prio, HFENCE_TBL_MSG,
count, hash, hw_fence->valid, hw_fence->error, hw_fence->ctx_id, hw_fence->seq_id,
hw_fence->wait_client_mask, hw_fence->fence_allocator, hw_fence->flags,
hw_fence->pending_child_cnt, parents_dump, hw_fence->fence_create_time,
hw_fence->fence_trigger_time, hw_fence->fence_wait_time);
}
void hw_fence_debug_dump_fence(enum hw_fence_drv_prio prio, struct msm_hw_fence *hw_fence, u64 hash,
u32 count)
{
char parents_dump[HW_FENCE_MAX_PARENTS_DUMP];
return _dump_fence_helper(prio, hw_fence, parents_dump, hash, count);
}
static inline int _dump_fence(struct msm_hw_fence *hw_fence, char *buf, int len, int max_size, static inline int _dump_fence(struct msm_hw_fence *hw_fence, char *buf, int len, int max_size,
u32 index, u32 cnt) u32 index, u32 cnt)
{ {
int ret; int ret;
char parents_dump[HW_FENCE_MAX_PARENTS_DUMP];
_dump_fence_helper(HW_FENCE_INFO, hw_fence, parents_dump, index, cnt);
ret = scnprintf(buf + len, max_size - len, HFENCE_TBL_MSG, ret = scnprintf(buf + len, max_size - len, HFENCE_TBL_MSG,
cnt, index, hw_fence->valid, hw_fence->error, cnt, index, hw_fence->valid, hw_fence->error, hw_fence->ctx_id, hw_fence->seq_id,
hw_fence->ctx_id, hw_fence->seq_id, hw_fence->wait_client_mask, hw_fence->fence_allocator, hw_fence->flags,
hw_fence->wait_client_mask, hw_fence->fence_allocator, hw_fence->pending_child_cnt, parents_dump, hw_fence->fence_create_time,
hw_fence->flags, hw_fence->fence_trigger_time, hw_fence->fence_wait_time); hw_fence->fence_trigger_time, hw_fence->fence_wait_time);
HWFNC_DBG_L(HFENCE_TBL_MSG,
cnt, index, hw_fence->valid, hw_fence->error,
hw_fence->ctx_id, hw_fence->seq_id,
hw_fence->wait_client_mask, hw_fence->fence_allocator,
hw_fence->flags, hw_fence->fence_trigger_time, hw_fence->fence_wait_time);
return ret; return ret;
} }
void hw_fence_debug_dump_table(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data)
{
u32 i, cnt = 0;
struct msm_hw_fence *hw_fence;
for (i = 0; i < drv_data->hw_fences_tbl_cnt; i++) {
hw_fence = &drv_data->hw_fences_tbl[i];
if (!hw_fence->valid)
continue;
hw_fence_debug_dump_fence(prio, hw_fence, i, cnt);
cnt++;
}
}
static int dump_single_entry(struct hw_fence_driver_data *drv_data, char *buf, u32 *index, static int dump_single_entry(struct hw_fence_driver_data *drv_data, char *buf, u32 *index,
int max_size) int max_size)
{ {
@@ -545,17 +606,40 @@ static int dump_full_table(struct hw_fence_driver_data *drv_data, char *buf, u32
return len; return len;
} }
static inline int _dump_event(struct msm_hw_fence_event *event, char *buf, int len, int max_size, static void _find_earliest_event(struct hw_fence_driver_data *drv_data, u32 *start_index,
u32 index) u64 *start_time)
{
u32 i;
if (!start_index || !start_time) {
HWFNC_ERR("invalid params start_index:0x%pK start_time:0x%pK\n", start_index,
start_time);
return;
}
mb(); /* make sure data is ready before read */
for (i = 0; i < drv_data->total_events; i++) {
u64 time = drv_data->events[i].time;
if (time && (!*start_time || time < *start_time)) {
*start_time = time;
*start_index = i;
}
}
}
static void _dump_event(enum hw_fence_drv_prio prio, struct msm_hw_fence_event *event,
char *data, u32 index)
{ {
char data[HW_FENCE_MAX_DATA_PER_EVENT_DUMP];
u32 data_cnt; u32 data_cnt;
int i, tmp_len = 0, ret = 0; int i, len = 0;
if (!event->time) if (!event || !data) {
return 0; HWFNC_ERR("invalid params event:0x%pK data:0x%pK\n", event, data);
return;
}
memset(&data, 0, sizeof(data)); memset(data, 0, sizeof(char) * HW_FENCE_MAX_DATA_PER_EVENT_DUMP);
if (event->data_cnt > HW_FENCE_EVENT_MAX_DATA) { if (event->data_cnt > HW_FENCE_EVENT_MAX_DATA) {
HWFNC_ERR("event[%d] has invalid data_cnt:%lu greater than max_data_cnt:%lu\n", HWFNC_ERR("event[%d] has invalid data_cnt:%lu greater than max_data_cnt:%lu\n",
index, event->data_cnt, HW_FENCE_EVENT_MAX_DATA); index, event->data_cnt, HW_FENCE_EVENT_MAX_DATA);
@@ -565,15 +649,29 @@ static inline int _dump_event(struct msm_hw_fence_event *event, char *buf, int l
} }
for (i = 0; i < data_cnt; i++) for (i = 0; i < data_cnt; i++)
tmp_len += scnprintf(data + tmp_len, HW_FENCE_MAX_DATA_PER_EVENT_DUMP - tmp_len, len += scnprintf(data + len, HW_FENCE_MAX_DATA_PER_EVENT_DUMP - len,
"%lx|", event->data[i]); "%lx|", event->data[i]);
ret = scnprintf(buf + len, max_size - len, HFENCE_EVT_MSG, index, event->cpu, event->time, HWFNC_DBG_DUMP(prio, HFENCE_EVT_MSG, index, event->cpu, event->time, event->data_cnt, data);
event->data_cnt, data); }
HWFNC_DBG_INFO(HFENCE_EVT_MSG, index, event->cpu, event->time, event->data_cnt, data); void hw_fence_debug_dump_events(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data)
{
char data[HW_FENCE_MAX_DATA_PER_EVENT_DUMP];
u32 start_index;
u64 start_time;
int i;
return ret; if (!drv_data->events) {
HWFNC_ERR("events not supported\n");
return;
}
_find_earliest_event(drv_data, &start_index, &start_time);
for (i = start_index; i < drv_data->total_events && drv_data->events[i].time; i++)
_dump_event(prio, &drv_data->events[i], data, i);
for (i = 0; i < start_index; i++)
_dump_event(prio, &drv_data->events[i], data, i);
} }
/** /**
@@ -626,15 +724,7 @@ static ssize_t hw_fence_dbg_dump_events_rd(struct file *file, char __user *user_
/* find index of earliest event */ /* find index of earliest event */
if (!start_time) { if (!start_time) {
mb(); /* make sure data is ready before read */ _find_earliest_event(drv_data, &start_index, &start_time);
for (index = 0; index < drv_data->total_events; index++) {
u64 time = drv_data->events[index].time;
if (time && (!start_time || time < start_time)) {
start_time = time;
start_index = index;
}
}
index = start_index; index = start_index;
HWFNC_DBG_H("events:0x%pK start_index:%d start_time:%llu total_events:%d\n", HWFNC_DBG_H("events:0x%pK start_index:%d start_time:%llu total_events:%d\n",
drv_data->events, start_index, start_time, drv_data->total_events); drv_data->events, start_index, start_time, drv_data->total_events);
@@ -642,7 +732,15 @@ static ssize_t hw_fence_dbg_dump_events_rd(struct file *file, char __user *user_
HWFNC_DBG_H("++ dump_events index:%d qtime:%llu\n", index, hw_fence_get_qtime(drv_data)); HWFNC_DBG_H("++ dump_events index:%d qtime:%llu\n", index, hw_fence_get_qtime(drv_data));
while ((!wraparound || index < start_index) && len < (max_size - entry_size)) { while ((!wraparound || index < start_index) && len < (max_size - entry_size)) {
len += _dump_event(&drv_data->events[index], buf, len, max_size, index); char data[HW_FENCE_MAX_DATA_PER_EVENT_DUMP];
if (drv_data->events[index].time) {
_dump_event(HW_FENCE_INFO, &drv_data->events[index], data, index);
len += scnprintf(buf + len, max_size - len, HFENCE_EVT_MSG, index,
drv_data->events[index].cpu, drv_data->events[index].time,
drv_data->events[index].data_cnt, data);
}
index++; index++;
if (index >= drv_data->total_events) { if (index >= drv_data->total_events) {
index = 0; index = 0;
@@ -668,6 +766,63 @@ exit:
return len; return len;
} }
static void _dump_queue(enum hw_fence_drv_prio prio, struct msm_hw_fence_client *hw_fence_client,
int queue_type)
{
struct msm_hw_fence_queue *queue;
struct msm_hw_fence_hfi_queue_header *hfi_header;
struct msm_hw_fence_queue_payload *payload;
u64 timestamp;
u32 *read_ptr, queue_entries;
int i;
queue = &hw_fence_client->queues[queue_type - 1];
if ((queue_type > hw_fence_client->queues_num) || !queue || !queue->va_header
|| !queue->va_queue) {
HWFNC_ERR("Cannot dump client:%d q_type:%s q_ptr:0x%pK q_header:0x%pK q_va:0x%pK\n",
hw_fence_client->client_id,
(queue_type == HW_FENCE_TX_QUEUE) ? "TX QUEUE" : "RX QUEUE",
queue, queue ? queue->va_header : NULL, queue ? queue->va_queue : NULL);
return;
}
hfi_header = (struct msm_hw_fence_hfi_queue_header *)queue->va_header;
mb(); /* make sure data is ready before read */
HWFNC_DBG_DUMP(prio, "%s va:0x%pK rd_idx:%lu wr_idx:%lu tx_wm:%lu q_size_bytes:%lu\n",
(queue_type == HW_FENCE_TX_QUEUE) ? "TX QUEUE" : "RX QUEUE", queue->va_queue,
hfi_header->read_index, hfi_header->write_index, hfi_header->tx_wm,
queue->q_size_bytes);
queue_entries = queue->q_size_bytes / HW_FENCE_CLIENT_QUEUE_PAYLOAD;
for (i = 0; i < queue_entries; i++) {
read_ptr = ((u32 *)queue->va_queue +
(i * (sizeof(struct msm_hw_fence_queue_payload) / sizeof(u32))));
payload = (struct msm_hw_fence_queue_payload *)read_ptr;
timestamp = (u64)payload->timestamp_lo | ((u64)payload->timestamp_hi << 32);
HWFNC_DBG_DUMP(prio,
"%s[%d]: hash:%d ctx:%llu seqno:%llu f:%llu d:%llu err:%u time:%llu\n",
(queue_type == HW_FENCE_TX_QUEUE) ? "tx" : "rx", i, payload->hash,
payload->ctxt_id, payload->seqno, payload->flags, payload->client_data,
payload->error, timestamp);
}
}
void hw_fence_debug_dump_queues(enum hw_fence_drv_prio prio,
struct msm_hw_fence_client *hw_fence_client)
{
if (!hw_fence_client) {
HWFNC_ERR("Invalid params client:0x%pK\n", hw_fence_client);
return;
}
HWFNC_DBG_DUMP(prio, "Queues for client %d\n", hw_fence_client->client_id);
if (hw_fence_client->queues_num == HW_FENCE_CLIENT_QUEUES)
_dump_queue(prio, hw_fence_client, HW_FENCE_RX_QUEUE);
_dump_queue(prio, hw_fence_client, HW_FENCE_TX_QUEUE);
}
/** /**
* hw_fence_dbg_dump_queues_wr() - debugfs wr to dump the hw-fences queues. * hw_fence_dbg_dump_queues_wr() - debugfs wr to dump the hw-fences queues.
* @file: file handler. * @file: file handler.
@@ -682,12 +837,7 @@ static ssize_t hw_fence_dbg_dump_queues_wr(struct file *file, const char __user
size_t count, loff_t *ppos) size_t count, loff_t *ppos)
{ {
struct hw_fence_driver_data *drv_data; struct hw_fence_driver_data *drv_data;
struct msm_hw_fence_queue *rx_queue; int client_id;
struct msm_hw_fence_queue *tx_queue;
u64 hash, ctx_id, seqno, timestamp, flags, client_data;
u32 *read_ptr, error;
int client_id, i;
struct msm_hw_fence_queue_payload *read_ptr_payload;
if (!file || !file->private_data) { if (!file || !file->private_data) {
HWFNC_ERR("unexpected data %d\n", file); HWFNC_ERR("unexpected data %d\n", file);
@@ -699,53 +849,11 @@ static ssize_t hw_fence_dbg_dump_queues_wr(struct file *file, const char __user
if (client_id < 0) if (client_id < 0)
return -EINVAL; return -EINVAL;
if (!drv_data->clients[client_id] || if (!drv_data->clients[client_id]) {
IS_ERR_OR_NULL(&drv_data->clients[client_id]->queues[HW_FENCE_RX_QUEUE - 1]) ||
IS_ERR_OR_NULL(&drv_data->clients[client_id]->queues[HW_FENCE_TX_QUEUE - 1])) {
HWFNC_ERR("client %d not initialized\n", client_id); HWFNC_ERR("client %d not initialized\n", client_id);
return -EINVAL; return -EINVAL;
} }
hw_fence_debug_dump_queues(HW_FENCE_PRINTK, drv_data->clients[client_id]);
HWFNC_DBG_L("Queues for client %d\n", client_id);
rx_queue = &drv_data->clients[client_id]->queues[HW_FENCE_RX_QUEUE - 1];
tx_queue = &drv_data->clients[client_id]->queues[HW_FENCE_TX_QUEUE - 1];
HWFNC_DBG_L("-------RX QUEUE------\n");
for (i = 0; i < drv_data->hw_fence_queue_entries; i++) {
read_ptr = ((u32 *)rx_queue->va_queue +
(i * (sizeof(struct msm_hw_fence_queue_payload) / sizeof(u32))));
read_ptr_payload = (struct msm_hw_fence_queue_payload *)read_ptr;
ctx_id = readq_relaxed(&read_ptr_payload->ctxt_id);
seqno = readq_relaxed(&read_ptr_payload->seqno);
hash = readq_relaxed(&read_ptr_payload->hash);
flags = readq_relaxed(&read_ptr_payload->flags);
client_data = readq_relaxed(&read_ptr_payload->client_data);
error = readl_relaxed(&read_ptr_payload->error);
timestamp = (u64)readl_relaxed(&read_ptr_payload->timestamp_lo) |
((u64)readl_relaxed(&read_ptr_payload->timestamp_hi) << 32);
HWFNC_DBG_L("rx[%d]: hash:%d ctx:%llu seqno:%llu f:%llu d:%llu err:%u time:%llu\n",
i, hash, ctx_id, seqno, flags, client_data, error, timestamp);
}
HWFNC_DBG_L("-------TX QUEUE------\n");
for (i = 0; i < drv_data->hw_fence_queue_entries; i++) {
read_ptr = ((u32 *)tx_queue->va_queue +
(i * (sizeof(struct msm_hw_fence_queue_payload) / sizeof(u32))));
read_ptr_payload = (struct msm_hw_fence_queue_payload *)read_ptr;
ctx_id = readq_relaxed(&read_ptr_payload->ctxt_id);
seqno = readq_relaxed(&read_ptr_payload->seqno);
hash = readq_relaxed(&read_ptr_payload->hash);
flags = readq_relaxed(&read_ptr_payload->flags);
error = readl_relaxed(&read_ptr_payload->error);
timestamp = (u64)readl_relaxed(&read_ptr_payload->timestamp_lo) |
((u64)readl_relaxed(&read_ptr_payload->timestamp_hi) << 32);
HWFNC_DBG_L("tx[%d]: hash:%d ctx:%llu seqno:%llu f:%llu err:%u time:%llu\n",
i, hash, ctx_id, seqno, flags, error, timestamp);
}
return count; return count;
} }

236
hw_fence/src/hw_fence_drv_priv.c
View File

@@ -17,6 +17,15 @@
#define IS_HW_FENCE_TX_QUEUE(queue_type) ((queue_type) == HW_FENCE_TX_QUEUE - 1) #define IS_HW_FENCE_TX_QUEUE(queue_type) ((queue_type) == HW_FENCE_TX_QUEUE - 1)
#define REQUIRES_IDX_TRANSLATION(queue) \
((queue)->rd_wr_idx_factor && ((queue)->rd_wr_idx_start || (queue)->rd_wr_idx_factor > 1))
#define IDX_TRANSLATE_CUSTOM_TO_DEFAULT(queue, idx) \
(((idx) - (queue)->rd_wr_idx_start) * (queue)->rd_wr_idx_factor)
#define IDX_TRANSLATE_DEFAULT_TO_CUSTOM(queue, idx) \
(((idx) / (queue)->rd_wr_idx_factor) + (queue)->rd_wr_idx_start)
inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data) inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data)
{ {
#ifdef HWFENCE_USE_SLEEP_TIMER #ifdef HWFENCE_USE_SLEEP_TIMER
@@ -184,18 +193,21 @@ char *_get_queue_type(int queue_type)
return (queue_type == (HW_FENCE_RX_QUEUE - 1)) ? "RXQ" : "TXQ"; return (queue_type == (HW_FENCE_RX_QUEUE - 1)) ? "RXQ" : "TXQ";
} }
static void _translate_queue_indexes_custom_to_default(struct msm_hw_fence_queue *queue,
u32 *read_idx, u32 *write_idx)
{
if (REQUIRES_IDX_TRANSLATION(queue)) {
*read_idx = IDX_TRANSLATE_CUSTOM_TO_DEFAULT(queue, *read_idx);
*write_idx = IDX_TRANSLATE_CUSTOM_TO_DEFAULT(queue, *write_idx);
HWFNC_DBG_Q("rd_idx_u32:%lu wr_idx_u32:%lu rd_wr_idx start:%lu factor:%lu\n",
*read_idx, *write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
}
}
int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client, int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
struct msm_hw_fence_queue_payload *payload, int queue_type) struct msm_hw_fence_queue_payload *payload, int queue_type)
{ {
struct msm_hw_fence_hfi_queue_header *hfi_header;
struct msm_hw_fence_queue *queue; struct msm_hw_fence_queue *queue;
u32 read_idx;
u32 write_idx;
u32 to_read_idx;
u32 *read_ptr;
u32 payload_size_u32;
u32 q_size_u32;
struct msm_hw_fence_queue_payload *read_ptr_payload;
if (queue_type >= HW_FENCE_CLIENT_QUEUES || !hw_fence_client || !payload) { if (queue_type >= HW_FENCE_CLIENT_QUEUES || !hw_fence_client || !payload) {
HWFNC_ERR("Invalid queue type:%s hw_fence_client:0x%pK payload:0x%pK\n", queue_type, HWFNC_ERR("Invalid queue type:%s hw_fence_client:0x%pK payload:0x%pK\n", queue_type,
@@ -204,6 +216,20 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
} }
queue = &hw_fence_client->queues[queue_type]; queue = &hw_fence_client->queues[queue_type];
HWFNC_DBG_Q("read client:%lu queue:0x%pK\n", hw_fence_client->client_id, queue);
return hw_fence_read_queue_helper(queue, payload);
}
int hw_fence_read_queue_helper(struct msm_hw_fence_queue *queue,
struct msm_hw_fence_queue_payload *payload)
{
struct msm_hw_fence_hfi_queue_header *hfi_header;
u32 read_idx, write_idx, to_read_idx;
u32 *read_ptr;
u32 payload_size_u32, q_size_u32;
struct msm_hw_fence_queue_payload *read_ptr_payload;
hfi_header = queue->va_header; hfi_header = queue->va_header;
q_size_u32 = (queue->q_size_bytes / sizeof(u32)); q_size_u32 = (queue->q_size_bytes / sizeof(u32));
@@ -223,20 +249,14 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
write_idx = readl_relaxed(&hfi_header->write_index); write_idx = readl_relaxed(&hfi_header->write_index);
/* translate read and write indexes from custom indexing to dwords with no offset */ /* translate read and write indexes from custom indexing to dwords with no offset */
if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) { _translate_queue_indexes_custom_to_default(queue, &read_idx, &write_idx);
read_idx = (read_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
write_idx = (write_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
HWFNC_DBG_Q("rd_idx_u32:%lu wr_idx_u32:%lu rd_wr_idx start:%lu factor:%lu\n",
read_idx, write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
}
HWFNC_DBG_Q("read client:%d rd_ptr:0x%pK wr_ptr:0x%pK rd_idx:%d wr_idx:%d queue:0x%pK\n", HWFNC_DBG_Q("read rd_ptr:0x%pK wr_ptr:0x%pK rd_idx:%d wr_idx:%d queue:0x%pK\n",
hw_fence_client->client_id, &hfi_header->read_index, &hfi_header->write_index, &hfi_header->read_index, &hfi_header->write_index, read_idx, write_idx, queue);
read_idx, write_idx, queue);
if (read_idx == write_idx) { if (read_idx == write_idx) {
HWFNC_DBG_Q("Nothing to read!\n"); HWFNC_DBG_Q("Nothing to read!\n");
return 0; return -EINVAL;
} }
/* Move the pointer where we need to read and cast it */ /* Move the pointer where we need to read and cast it */
@@ -257,19 +277,14 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
to_read_idx = 0; to_read_idx = 0;
/* translate to_read_idx to custom indexing with offset */ /* translate to_read_idx to custom indexing with offset */
if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) { if (REQUIRES_IDX_TRANSLATION(queue)) {
to_read_idx = (to_read_idx / queue->rd_wr_idx_factor) + queue->rd_wr_idx_start; to_read_idx = IDX_TRANSLATE_DEFAULT_TO_CUSTOM(queue, to_read_idx);
HWFNC_DBG_Q("translated to_read_idx:%lu rd_wr_idx start:%lu factor:%lu\n", HWFNC_DBG_Q("translated to_read_idx:%lu rd_wr_idx start:%lu factor:%lu\n",
to_read_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor); to_read_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
} }
/* Read the Client Queue */ /* Read the Client Queue */
payload->ctxt_id = readq_relaxed(&read_ptr_payload->ctxt_id); *payload = *read_ptr_payload;
payload->seqno = readq_relaxed(&read_ptr_payload->seqno);
payload->hash = readq_relaxed(&read_ptr_payload->hash);
payload->flags = readq_relaxed(&read_ptr_payload->flags);
payload->client_data = readq_relaxed(&read_ptr_payload->client_data);
payload->error = readl_relaxed(&read_ptr_payload->error);
/* update the read index */ /* update the read index */
writel_relaxed(to_read_idx, &hfi_header->read_index); writel_relaxed(to_read_idx, &hfi_header->read_index);
@@ -281,6 +296,34 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
return to_read_idx == write_idx ? 0 : 1; return to_read_idx == write_idx ? 0 : 1;
} }
static int _get_update_queue_params(struct msm_hw_fence_queue *queue,
struct msm_hw_fence_hfi_queue_header **hfi_header, u32 *q_size_u32, u32 *payload_size,
u32 *payload_size_u32, u32 **wr_ptr)
{
if (!queue) {
HWFNC_ERR("invalid queue\n");
return -EINVAL;
}
*hfi_header = queue->va_header;
if (!*hfi_header) {
HWFNC_ERR("Invalid queue hfi_header\n");
return -EINVAL;
}
*q_size_u32 = (queue->q_size_bytes / sizeof(u32));
*payload_size = sizeof(struct msm_hw_fence_queue_payload);
*payload_size_u32 = (*payload_size / sizeof(u32));
/* if skipping update wr_index, then use hfi_header->tx_wm instead */
if (queue->skip_wr_idx)
*wr_ptr = &((*hfi_header)->tx_wm);
else
*wr_ptr = &((*hfi_header)->write_index);
return 0;
}
/* /*
* This function writes to the queue of the client. The 'queue_type' determines * This function writes to the queue of the client. The 'queue_type' determines
* if this function is writing to the rx or tx queue * if this function is writing to the rx or tx queue
@@ -312,23 +355,13 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
} }
queue = &hw_fence_client->queues[queue_type]; queue = &hw_fence_client->queues[queue_type];
hfi_header = queue->va_header; if (_get_update_queue_params(queue, &hfi_header, &q_size_u32, &payload_size,
&payload_size_u32, &wr_ptr)) {
q_size_u32 = (queue->q_size_bytes / sizeof(u32)); HWFNC_ERR("Invalid client:%d q_type:%d queue\n", hw_fence_client->client_id,
payload_size = sizeof(struct msm_hw_fence_queue_payload); queue_type);
payload_size_u32 = (payload_size / sizeof(u32));
if (!hfi_header) {
HWFNC_ERR("Invalid queue\n");
return -EINVAL; return -EINVAL;
} }
/* if skipping update wr_index, then use hfi_header->tx_wm instead */
if (queue->skip_wr_idx)
wr_ptr = &hfi_header->tx_wm;
else
wr_ptr = &hfi_header->write_index;
/* /*
* We need to lock the client if there is an Rx Queue update, since that * We need to lock the client if there is an Rx Queue update, since that
* is the only time when HW Fence driver can have a race condition updating * is the only time when HW Fence driver can have a race condition updating
@@ -361,12 +394,7 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
read_idx, write_idx, queue, queue_type, queue->skip_wr_idx ? "true" : "false"); read_idx, write_idx, queue, queue_type, queue->skip_wr_idx ? "true" : "false");
/* translate read and write indexes from custom indexing to dwords with no offset */ /* translate read and write indexes from custom indexing to dwords with no offset */
if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) { _translate_queue_indexes_custom_to_default(queue, &read_idx, &write_idx);
read_idx = (read_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
write_idx = (write_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
HWFNC_DBG_Q("rd_idx_u32:%lu wr_idx_u32:%lu rd_wr_idx start:%lu factor:%lu\n",
read_idx, write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
}
/* Check queue to make sure message will fit */ /* Check queue to make sure message will fit */
q_free_u32 = read_idx <= write_idx ? (q_size_u32 - (write_idx - read_idx)) : q_free_u32 = read_idx <= write_idx ? (q_size_u32 - (write_idx - read_idx)) :
@@ -402,8 +430,8 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
to_write_idx = 0; to_write_idx = 0;
/* translate to_write_idx to custom indexing with offset */ /* translate to_write_idx to custom indexing with offset */
if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) { if (REQUIRES_IDX_TRANSLATION(queue)) {
to_write_idx = (to_write_idx / queue->rd_wr_idx_factor) + queue->rd_wr_idx_start; to_write_idx = IDX_TRANSLATE_DEFAULT_TO_CUSTOM(queue, to_write_idx);
HWFNC_DBG_Q("translated to_write_idx:%lu rd_wr_idx start:%lu factor:%lu\n", HWFNC_DBG_Q("translated to_write_idx:%lu rd_wr_idx start:%lu factor:%lu\n",
to_write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor); to_write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
} }
@@ -438,6 +466,90 @@ exit:
return ret; return ret;
} }
int hw_fence_update_existing_txq_payload(struct hw_fence_driver_data *drv_data,
struct msm_hw_fence_client *hw_fence_client, u64 hash, u32 error)
{
u32 q_size_u32, payload_size, payload_size_u32, read_idx, write_idx, second_idx, *wr_ptr;
struct msm_hw_fence_queue_payload tmp, *first_payload, *second_payload;
struct msm_hw_fence_hfi_queue_header *hfi_header;
struct msm_hw_fence_queue *queue;
int ret = 0;
queue = &hw_fence_client->queues[HW_FENCE_TX_QUEUE - 1];
if (_get_update_queue_params(queue, &hfi_header, &q_size_u32, &payload_size,
&payload_size_u32, &wr_ptr)) {
HWFNC_ERR("Invalid client:%d tx queue\n", hw_fence_client->client_id);
return -EINVAL;
}
/* Make sure data is ready before read */
mb();
/* Get read and write index */
read_idx = hfi_header->read_index;
write_idx = *wr_ptr;
/* translate read and write indexes from custom indexing to dwords with no offset */
_translate_queue_indexes_custom_to_default(queue, &read_idx, &write_idx);
if (read_idx == write_idx) {
HWFNC_DBG_Q("Empty queue, no entry matches with hash:%llu\n", hash);
return -EINVAL;
}
first_payload = (struct msm_hw_fence_queue_payload *)((u32 *)queue->va_queue + read_idx);
HWFNC_DBG_Q("client:%d txq: va=0x%pK pa=0x%pK idx:%d ptr_payload:0x%pK\n",
hw_fence_client->client_id, queue->va_queue, queue->pa_queue, read_idx,
first_payload);
if (first_payload->hash == hash) {
/* Swap not needed, update first payload in client queue with fence error */
first_payload->error = error;
} else {
/* Check whether second entry matches hash */
second_idx = read_idx + payload_size_u32;
/* wrap-around case */
if (second_idx >= q_size_u32)
second_idx = 0;
if (second_idx == write_idx) {
HWFNC_ERR("Failed to find matching entry with hash:%llu\n", hash);
return -EINVAL;
}
second_payload = (struct msm_hw_fence_queue_payload *)
((u32 *)queue->va_queue + second_idx);
HWFNC_DBG_Q("client:%d txq: va=0x%pK pa=0x%pK idx:%d ptr_payload:0x%pK\n",
hw_fence_client->client_id, queue->va_queue, queue->pa_queue, second_idx,
second_payload);
if (second_payload->hash != hash) {
HWFNC_ERR("hash:%llu not found in first two queue payloads:%u, %u\n", hash,
read_idx, second_idx);
return -EINVAL;
}
/* swap first and second payload, updating error field in new first payload */
tmp = *first_payload;
*first_payload = *second_payload;
first_payload->error = error;
*second_payload = tmp;
HWFNC_DBG_L("client_id:%d txq move from idx:%u to idx:%u hash:%llu c:%llu s:%llu\n",
hw_fence_client->client_id, read_idx, second_idx, hash, tmp.ctxt_id,
tmp.seqno);
}
/* update memory for the messages */
wmb();
HWFNC_DBG_L("client_id:%d update tx queue index:%u hash:%llu error:%u\n",
hw_fence_client->client_id, read_idx, hash, error);
return ret;
}
static int init_global_locks(struct hw_fence_driver_data *drv_data) static int init_global_locks(struct hw_fence_driver_data *drv_data)
{ {
struct msm_hw_fence_mem_addr *mem_descriptor; struct msm_hw_fence_mem_addr *mem_descriptor;
@@ -1204,15 +1316,22 @@ static void _fence_ctl_signal(struct hw_fence_driver_data *drv_data,
HWFNC_DBG_H("We must signal the client now! hfence hash:%llu\n", hash); HWFNC_DBG_H("We must signal the client now! hfence hash:%llu\n", hash);
/* Write to Rx queue */ /* Call fence error callback */
if (hw_fence_client->update_rxq) if (error && hw_fence_client->fence_error_cb) {
hw_fence_update_queue(drv_data, hw_fence_client, hw_fence->ctx_id, hw_fence_utils_fence_error_cb(hw_fence_client, hw_fence->ctx_id, hw_fence->seq_id,
hw_fence->seq_id, hash, flags, client_data, error, HW_FENCE_RX_QUEUE - 1); hash, flags, error);
} else {
/* Write to Rx queue */
if (hw_fence_client->update_rxq)
hw_fence_update_queue(drv_data, hw_fence_client, hw_fence->ctx_id,
hw_fence->seq_id, hash, flags, client_data, error,
HW_FENCE_RX_QUEUE - 1);
/* Signal the hw fence now */ /* Signal the hw fence now */
if (hw_fence_client->send_ipc) if (hw_fence_client->send_ipc)
hw_fence_ipcc_trigger_signal(drv_data, tx_client_id, rx_client_id, hw_fence_ipcc_trigger_signal(drv_data, tx_client_id, rx_client_id,
hw_fence_client->ipc_signal_id); hw_fence_client->ipc_signal_id);
}
#if IS_ENABLED(CONFIG_DEBUG_FS) #if IS_ENABLED(CONFIG_DEBUG_FS)
if (hw_fence_client->client_id >= HW_FENCE_CLIENT_ID_VAL0 if (hw_fence_client->client_id >= HW_FENCE_CLIENT_ID_VAL0
@@ -1355,6 +1474,7 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
/* child fence is already signaled */ /* child fence is already signaled */
GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 1); /* lock */ GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 1); /* lock */
join_fence->error |= hw_fence_child->error;
if (--join_fence->pending_child_cnt == 0) if (--join_fence->pending_child_cnt == 0)
signal_join_fence = true; signal_join_fence = true;
@@ -1400,8 +1520,8 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
if (signal_join_fence) { if (signal_join_fence) {
/* signal the join hw fence */ /* signal the join hw fence */
_fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0, 0, _fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0,
client_data); client_data, join_fence->error);
set_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &array->base.flags); set_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &array->base.flags);
/* /*

160
hw_fence/src/hw_fence_drv_utils.c
View File

@@ -53,11 +53,34 @@
#define HW_FENCE_CLIENT_TYPE_MAX_VPU 32 #define HW_FENCE_CLIENT_TYPE_MAX_VPU 32
#define HW_FENCE_CLIENT_TYPE_MAX_IFE 32 #define HW_FENCE_CLIENT_TYPE_MAX_IFE 32
/* /**
* Each bit in this mask represents each of the loopback clients supported in * HW_FENCE_CTRL_QUEUE_DOORBELL:
* the enum hw_fence_client_id * Bit set in doorbell flags mask if hw fence driver should read ctrl rx queue
*/ */
#define HW_FENCE_LOOPBACK_CLIENTS_MASK 0x7fff #define HW_FENCE_CTRL_QUEUE_DOORBELL 0
/**
* HW_FENCE_DOORBELL_FLAGS_ID_LAST:
* Last doorbell flags id for which HW Fence Driver can receive doorbell
*/
#if IS_ENABLED(CONFIG_DEBUG_FS)
#define HW_FENCE_DOORBELL_FLAGS_ID_LAST HW_FENCE_CLIENT_ID_VAL6
#else
#define HW_FENCE_DOORBELL_FLAGS_ID_LAST HW_FENCE_CTRL_QUEUE_DOORBELL
#endif /* CONFIG_DEBUG_FS */
/**
* HW_FENCE_DOORBELL_MASK:
* Each bit in this mask represents possible doorbell flag ids for which hw fence driver can receive
*/
#define HW_FENCE_DOORBELL_MASK \
GENMASK(HW_FENCE_DOORBELL_FLAGS_ID_LAST, HW_FENCE_CTRL_QUEUE_DOORBELL)
/**
* HW_FENCE_MAX_ITER_READ:
* Maximum number of iterations when reading queue
*/
#define HW_FENCE_MAX_ITER_READ 100
/** /**
* HW_FENCE_MAX_EVENTS: * HW_FENCE_MAX_EVENTS:
@@ -179,12 +202,110 @@ void global_atomic_store(struct hw_fence_driver_data *drv_data, uint64_t *lock,
} }
} }
static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int client_id) int hw_fence_utils_fence_error_cb(struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id,
u64 seqno, u64 hash, u64 flags, u32 error)
{
struct msm_hw_fence_cb_data cb_data;
struct dma_fence fence;
int ret = 0;
if (IS_ERR_OR_NULL(hw_fence_client)) {
HWFNC_ERR("Invalid client:0x%pK\n", hw_fence_client);
return -EINVAL;
}
mutex_lock(&hw_fence_client->error_cb_lock);
if (!error || !hw_fence_client->fence_error_cb) {
HWFNC_ERR("Invalid error:%d fence_error_cb:0x%pK\n", error,
hw_fence_client->fence_error_cb);
ret = -EINVAL;
goto exit;
}
/* initialize cb_data info */
fence.context = ctxt_id;
fence.seqno = seqno;
fence.flags = flags;
fence.error = error;
cb_data.fence = &fence;
cb_data.data = hw_fence_client->fence_error_cb_userdata;
HWFNC_DBG_L("invoking cb for client:%d ctx:%llu seq:%llu flags:%llu e:%u data:0x%pK\n",
hw_fence_client->client_id, ctxt_id, seqno, flags, error,
hw_fence_client->fence_error_cb_userdata);
hw_fence_client->fence_error_cb(hash, error, &cb_data);
exit:
mutex_unlock(&hw_fence_client->error_cb_lock);
return ret;
}
static int _process_fence_error_client_loopback(struct hw_fence_driver_data *drv_data,
int db_flag_id)
{
struct msm_hw_fence_client *hw_fence_client;
struct msm_hw_fence_queue_payload payload;
int i, cb_ret, ret = 0, read = 1;
u32 client_id;
for (i = 0; read && i < HW_FENCE_MAX_ITER_READ; i++) {
read = hw_fence_read_queue_helper(&drv_data->ctrl_queues[HW_FENCE_RX_QUEUE - 1],
&payload);
if (read < 0) {
HWFNC_DBG_Q("unable to read ctrl rxq for db_flag_id:%d\n", db_flag_id);
return read;
}
if (payload.type != HW_FENCE_PAYLOAD_TYPE_2) {
HWFNC_ERR("unsupported payload type in ctrl rxq received:%u expected:%u\n",
payload.type, HW_FENCE_PAYLOAD_TYPE_2);
ret = -EINVAL;
continue;
}
if (payload.client_data < HW_FENCE_CLIENT_ID_CTX0 ||
payload.client_data >= drv_data->clients_num) {
HWFNC_ERR("read invalid client_id:%llu from ctrl rxq min:%u max:%u\n",
payload.client_data, HW_FENCE_CLIENT_ID_CTX0,
drv_data->clients_num);
ret = -EINVAL;
continue;
}
client_id = payload.client_data;
HWFNC_DBG_Q("ctrl rxq rd: it:%d h:%llu ctx:%llu seq:%llu f:%llu e:%u client:%u\n",
i, payload.hash, payload.ctxt_id, payload.seqno, payload.flags,
payload.error, client_id);
hw_fence_client = drv_data->clients[client_id];
if (!hw_fence_client) {
HWFNC_ERR("processing fence error cb for unregistered client_id:%u\n",
client_id);
ret = -EINVAL;
continue;
}
cb_ret = hw_fence_utils_fence_error_cb(hw_fence_client, payload.ctxt_id,
payload.seqno, payload.hash, payload.flags, payload.error);
if (cb_ret) {
HWFNC_ERR("fence_error_cb failed for client:%u ctx:%llu seq:%llu err:%u\n",
client_id, payload.ctxt_id, payload.seqno, payload.error);
ret = cb_ret;
}
}
return ret;
}
static int _process_doorbell_id(struct hw_fence_driver_data *drv_data, int db_flag_id)
{ {
int ret; int ret;
HWFNC_DBG_H("Processing doorbell client_id:%d\n", client_id); HWFNC_DBG_H("Processing doorbell mask id:%d\n", db_flag_id);
switch (client_id) { switch (db_flag_id) {
case HW_FENCE_CTRL_QUEUE_DOORBELL:
ret = _process_fence_error_client_loopback(drv_data, db_flag_id);
break;
#if IS_ENABLED(CONFIG_DEBUG_FS) #if IS_ENABLED(CONFIG_DEBUG_FS)
case HW_FENCE_CLIENT_ID_VAL0: case HW_FENCE_CLIENT_ID_VAL0:
case HW_FENCE_CLIENT_ID_VAL1: case HW_FENCE_CLIENT_ID_VAL1:
@@ -193,11 +314,11 @@ static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int c
case HW_FENCE_CLIENT_ID_VAL4: case HW_FENCE_CLIENT_ID_VAL4:
case HW_FENCE_CLIENT_ID_VAL5: case HW_FENCE_CLIENT_ID_VAL5:
case HW_FENCE_CLIENT_ID_VAL6: case HW_FENCE_CLIENT_ID_VAL6:
ret = process_validation_client_loopback(drv_data, client_id); ret = process_validation_client_loopback(drv_data, db_flag_id);
break; break;
#endif /* CONFIG_DEBUG_FS */ #endif /* CONFIG_DEBUG_FS */
default: default:
HWFNC_ERR("unknown client:%d\n", client_id); HWFNC_ERR("unknown mask id:%d\n", db_flag_id);
ret = -EINVAL; ret = -EINVAL;
} }
@@ -206,22 +327,21 @@ static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int c
void hw_fence_utils_process_doorbell_mask(struct hw_fence_driver_data *drv_data, u64 db_flags) void hw_fence_utils_process_doorbell_mask(struct hw_fence_driver_data *drv_data, u64 db_flags)
{ {
int client_id = HW_FENCE_CLIENT_ID_CTL0; int db_flag_id = HW_FENCE_CTRL_QUEUE_DOORBELL;
u64 mask; u64 mask;
for (; client_id <= HW_FENCE_CLIENT_ID_VAL6; client_id++) { for (; db_flag_id <= HW_FENCE_DOORBELL_FLAGS_ID_LAST; db_flag_id++) {
mask = 1 << client_id; mask = 1 << db_flag_id;
if (mask & db_flags) { if (mask & db_flags) {
HWFNC_DBG_H("client_id:%d signaled! flags:0x%llx\n", client_id, db_flags); HWFNC_DBG_H("db_flag:%d signaled! flags:0x%llx\n", db_flag_id, db_flags);
/* process client */ if (_process_doorbell_id(drv_data, db_flag_id))
if (_process_doorbell_client(drv_data, client_id)) HWFNC_ERR("Failed to process db_flag_id:%d\n", db_flag_id);
HWFNC_ERR("Failed to process client:%d\n", client_id);
/* clear mask for this client and if nothing else pending finish */ /* clear mask for this flag id if nothing else pending finish */
db_flags = db_flags & ~(mask); db_flags = db_flags & ~(mask);
HWFNC_DBG_H("client_id:%d cleared flags:0x%llx mask:0x%llx ~mask:0x%llx\n", HWFNC_DBG_H("db_flag_id:%d cleared flags:0x%llx mask:0x%llx ~mask:0x%llx\n",
client_id, db_flags, mask, ~(mask)); db_flag_id, db_flags, mask, ~(mask));
if (!db_flags) if (!db_flags)
break; break;
} }
@@ -232,7 +352,7 @@ void hw_fence_utils_process_doorbell_mask(struct hw_fence_driver_data *drv_data,
static void _hw_fence_cb(int irq, void *data) static void _hw_fence_cb(int irq, void *data)
{ {
struct hw_fence_driver_data *drv_data = (struct hw_fence_driver_data *)data; struct hw_fence_driver_data *drv_data = (struct hw_fence_driver_data *)data;
gh_dbl_flags_t clear_flags = HW_FENCE_LOOPBACK_CLIENTS_MASK; gh_dbl_flags_t clear_flags = HW_FENCE_DOORBELL_MASK;
int ret; int ret;
if (!drv_data) if (!drv_data)

167
hw_fence/src/msm_hw_fence.c
View File

@@ -118,6 +118,8 @@ void *msm_hw_fence_register(enum hw_fence_client_id client_id_ext,
if (ret) if (ret)
goto error; goto error;
mutex_init(&hw_fence_client->error_cb_lock);
HWFNC_DBG_INIT("Initialized ptr:0x%p client_id:%d q_num:%d ipc signal:%d vid:%d pid:%d\n", HWFNC_DBG_INIT("Initialized ptr:0x%p client_id:%d q_num:%d ipc signal:%d vid:%d pid:%d\n",
hw_fence_client, hw_fence_client->client_id, hw_fence_client->queues_num, hw_fence_client, hw_fence_client->client_id, hw_fence_client->queues_num,
hw_fence_client->ipc_signal_id, hw_fence_client->ipc_client_vid, hw_fence_client->ipc_signal_id, hw_fence_client->ipc_client_vid,
@@ -451,6 +453,35 @@ int msm_hw_fence_update_txq(void *client_handle, u64 handle, u64 flags, u32 erro
} }
EXPORT_SYMBOL(msm_hw_fence_update_txq); EXPORT_SYMBOL(msm_hw_fence_update_txq);
int msm_hw_fence_update_txq_error(void *client_handle, u64 handle, u32 error, u32 update_flags)
{
struct msm_hw_fence_client *hw_fence_client;
if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready ||
!hw_fence_drv_data->vm_ready) {
HWFNC_ERR("hw fence driver or vm not ready\n");
return -EAGAIN;
} else if (IS_ERR_OR_NULL(client_handle) ||
(handle >= hw_fence_drv_data->hw_fences_tbl_cnt) || !error) {
HWFNC_ERR("Invalid client_handle:0x%pK or fence handle:%d max:%d or error:%d\n",
client_handle, handle, hw_fence_drv_data->hw_fences_tbl_cnt, error);
return -EINVAL;
} else if (update_flags != MSM_HW_FENCE_UPDATE_ERROR_WITH_MOVE) {
HWFNC_ERR("invalid flags:0x%x expected:0x%x no support of in-place error update\n",
update_flags, MSM_HW_FENCE_UPDATE_ERROR_WITH_MOVE);
return -EINVAL;
}
hw_fence_client = (struct msm_hw_fence_client *)client_handle;
/* Write to Tx queue */
hw_fence_update_existing_txq_payload(hw_fence_drv_data, hw_fence_client,
handle, error);
return 0;
}
EXPORT_SYMBOL(msm_hw_fence_update_txq_error);
/* tx client has to be the physical, rx client virtual id*/ /* tx client has to be the physical, rx client virtual id*/
int msm_hw_fence_trigger_signal(void *client_handle, int msm_hw_fence_trigger_signal(void *client_handle,
u32 tx_client_pid, u32 rx_client_vid, u32 tx_client_pid, u32 rx_client_vid,
@@ -476,6 +507,142 @@ int msm_hw_fence_trigger_signal(void *client_handle,
} }
EXPORT_SYMBOL(msm_hw_fence_trigger_signal); EXPORT_SYMBOL(msm_hw_fence_trigger_signal);
int msm_hw_fence_register_error_cb(void *client_handle, msm_hw_fence_error_cb_t cb, void *data)
{
struct msm_hw_fence_client *hw_fence_client;
if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
HWFNC_ERR("hw fence driver not ready\n");
return -EAGAIN;
} else if (IS_ERR_OR_NULL(client_handle) || IS_ERR_OR_NULL(cb) || IS_ERR_OR_NULL(data)) {
HWFNC_ERR("Invalid params client:0x%pK cb_func:0x%pK data:0x%pK\n", client_handle,
cb, data);
return -EINVAL;
}
hw_fence_client = (struct msm_hw_fence_client *)client_handle;
if (hw_fence_client->fence_error_cb) {
HWFNC_ERR("client_id:%d client_id_ext:%d already registered cb_func:%pK data:%pK\n",
hw_fence_client->client_id, hw_fence_client->client_id_ext,
hw_fence_client->fence_error_cb, hw_fence_client->fence_error_cb_userdata);
return -EINVAL;
}
hw_fence_client->fence_error_cb_userdata = data;
hw_fence_client->fence_error_cb = cb;
return 0;
}
EXPORT_SYMBOL(msm_hw_fence_register_error_cb);
int msm_hw_fence_deregister_error_cb(void *client_handle)
{
struct msm_hw_fence_client *hw_fence_client;
int ret = 0;
if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
HWFNC_ERR("hw fence driver not ready\n");
return -EAGAIN;
} else if (IS_ERR_OR_NULL(client_handle)) {
HWFNC_ERR("Invalid client: 0x%pK\n", client_handle);
return -EINVAL;
}
hw_fence_client = (struct msm_hw_fence_client *)client_handle;
if (!mutex_trylock(&hw_fence_client->error_cb_lock)) {
HWFNC_ERR("client_id:%d is modifying or using fence_error_cb:0x%pK data:0x%pK\n",
hw_fence_client->client_id, hw_fence_client->fence_error_cb,
hw_fence_client->fence_error_cb_userdata);
return -EAGAIN;
}
if (!hw_fence_client->fence_error_cb) {
HWFNC_ERR("client_id:%d client_id_ext:%d did not register cb:%pK data:%pK\n",
hw_fence_client->client_id, hw_fence_client->client_id_ext,
hw_fence_client->fence_error_cb, hw_fence_client->fence_error_cb_userdata);
ret = -EINVAL;
goto exit;
}
hw_fence_client->fence_error_cb = NULL;
hw_fence_client->fence_error_cb_userdata = NULL;
exit:
mutex_unlock(&hw_fence_client->error_cb_lock);
return 0;
}
EXPORT_SYMBOL(msm_hw_fence_deregister_error_cb);
#if IS_ENABLED(CONFIG_DEBUG_FS)
int msm_hw_fence_dump_debug_data(void *client_handle, u32 dump_flags, u32 dump_clients_mask)
{
struct msm_hw_fence_client *hw_fence_client;
int client_id;
if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
HWFNC_ERR("hw fence driver not ready\n");
return -EAGAIN;
} else if (IS_ERR_OR_NULL(client_handle)) {
HWFNC_ERR("Invalid client handle:%d\n", IS_ERR_OR_NULL(client_handle));
return -EINVAL;
}
hw_fence_client = (struct msm_hw_fence_client *)client_handle;
if (dump_flags & MSM_HW_FENCE_DBG_DUMP_QUEUES) {
hw_fence_debug_dump_queues(HW_FENCE_PRINTK, hw_fence_client);
if (dump_clients_mask)
for (client_id = 0; client_id < HW_FENCE_CLIENT_MAX; client_id++)
if ((dump_clients_mask & (1 << client_id)) &&
hw_fence_drv_data->clients[client_id])
hw_fence_debug_dump_queues(HW_FENCE_PRINTK,
hw_fence_drv_data->clients[client_id]);
}
if (dump_flags & MSM_HW_FENCE_DBG_DUMP_TABLE)
hw_fence_debug_dump_table(HW_FENCE_PRINTK, hw_fence_drv_data);
if (dump_flags & MSM_HW_FENCE_DBG_DUMP_EVENTS)
hw_fence_debug_dump_events(HW_FENCE_PRINTK, hw_fence_drv_data);
return 0;
}
EXPORT_SYMBOL(msm_hw_fence_dump_debug_data);
int msm_hw_fence_dump_fence(void *client_handle, struct dma_fence *fence)
{
struct msm_hw_fence_client *hw_fence_client;
struct msm_hw_fence *hw_fence;
u64 hash;
if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
HWFNC_ERR("hw fence driver not ready\n");
return -EAGAIN;
} else if (IS_ERR_OR_NULL(client_handle)) {
HWFNC_ERR("Invalid client handle:%d\n", IS_ERR_OR_NULL(client_handle));
return -EINVAL;
} else if (!test_bit(MSM_HW_FENCE_FLAG_ENABLED_BIT, &fence->flags)) {
HWFNC_ERR("DMA Fence is not a HW Fence ctx:%llu seqno:%llu flags:0x%llx\n",
fence->context, fence->seqno, fence->flags);
return -EINVAL;
}
hw_fence_client = (struct msm_hw_fence_client *)client_handle;
hw_fence = msm_hw_fence_find(hw_fence_drv_data, hw_fence_client, fence->context,
fence->seqno, &hash);
if (!hw_fence) {
HWFNC_ERR("failed to find hw-fence client_id:%d fence:0x%pK ctx:%llu seqno:%llu\n",
hw_fence_client->client_id, fence, fence->context, fence->seqno);
return -EINVAL;
}
hw_fence_debug_dump_fence(HW_FENCE_PRINTK, hw_fence, hash, 0);
return 0;
}
EXPORT_SYMBOL(msm_hw_fence_dump_fence);
#endif /* CONFIG_DEBUG_FS */
/* Function used for simulation purposes only. */ /* Function used for simulation purposes only. */
int msm_hw_fence_driver_doorbell_sim(u64 db_mask) int msm_hw_fence_driver_doorbell_sim(u64 db_mask)
{ {