diff --git a/msm/Android.mk b/msm/Android.mk index ec91f145e5..fbdb5cd317 100644 --- a/msm/Android.mk +++ b/msm/Android.mk @@ -27,6 +27,7 @@ KBUILD_OPTIONS += KBUILD_EXTRA_SYMBOLS+=$(PWD)/$(call intermediates-dir-for,DLKM ifneq ($(TARGET_BOARD_PLATFORM), taro) KBUILD_OPTIONS += KBUILD_EXTRA_SYMBOLS+=$(PWD)/$(call intermediates-dir-for,DLKM,msm-ext-disp-module-symvers)/Module.symvers KBUILD_OPTIONS += KBUILD_EXTRA_SYMBOLS+=$(PWD)/$(call intermediates-dir-for,DLKM,sec-module-symvers)/Module.symvers + KBUILD_OPTIONS += KBUILD_EXTRA_SYMBOLS+=$(PWD)/$(call intermediates-dir-for,DLKM,hw-fence-module-symvers)/Module.symvers endif endif @@ -45,8 +46,10 @@ LOCAL_ADDITIONAL_DEPENDENCIES += $(call intermediates-dir-for,DLKM,mmrm-module-s ifneq ($(TARGET_BOARD_PLATFORM), taro) LOCAL_REQUIRED_MODULES += msm-ext-disp-module-symvers LOCAL_REQUIRED_MODULES += sec-module-symvers + LOCAL_REQUIRED_MODULES += hw-fence-module-symvers LOCAL_ADDITIONAL_DEPENDENCIES += $(call intermediates-dir-for,DLKM,msm-ext-disp-module-symvers)/Module.symvers LOCAL_ADDITIONAL_DEPENDENCIES += $(call intermediates-dir-for,DLKM,sec-module-symvers)/Module.symvers + LOCAL_ADDITIONAL_DEPENDENCIES += $(call intermediates-dir-for,DLKM,hw-fence-module-symvers)/Module.symvers endif endif diff --git a/msm/sde/sde_connector.c b/msm/sde/sde_connector.c index 406554be90..50c1276975 100644 --- a/msm/sde/sde_connector.c +++ b/msm/sde/sde_connector.c @@ -1637,6 +1637,7 @@ static int _sde_connector_set_prop_retire_fence(struct drm_connector *connector, struct sde_connector *c_conn; uint64_t fence_user_fd; uint64_t __user prev_user_fd; + struct sde_hw_ctl *hw_ctl = NULL; c_conn = to_sde_connector(connector); @@ -1662,8 +1663,13 @@ static int _sde_connector_set_prop_retire_fence(struct drm_connector *connector, * commit completion */ offset++; + + /* get hw_ctl for a wb connector */ + if (c_conn->connector_type == DRM_MODE_CONNECTOR_VIRTUAL) + hw_ctl = sde_encoder_get_hw_ctl(c_conn); + rc = sde_fence_create(c_conn->retire_fence, - &fence_user_fd, offset); + &fence_user_fd, offset, hw_ctl); if (rc) { SDE_ERROR("fence create failed rc:%d\n", rc); goto end; @@ -1890,19 +1896,26 @@ void sde_connector_complete_commit(struct drm_connector *connector, /* signal connector's retire fence */ sde_fence_signal(to_sde_connector(connector)->retire_fence, - ts, fence_event); + ts, fence_event, NULL); } void sde_connector_commit_reset(struct drm_connector *connector, ktime_t ts) { + struct sde_hw_ctl *hw_ctl = NULL; + struct sde_connector *c_conn; + if (!connector) { SDE_ERROR("invalid connector\n"); return; } + c_conn = to_sde_connector(connector); + + /* get hw_ctl for a wb connector */ + if (c_conn->connector_type == DRM_MODE_CONNECTOR_VIRTUAL) + hw_ctl = sde_encoder_get_hw_ctl(c_conn); /* signal connector's retire fence */ - sde_fence_signal(to_sde_connector(connector)->retire_fence, - ts, SDE_FENCE_RESET_TIMELINE); + sde_fence_signal(c_conn->retire_fence, ts, SDE_FENCE_RESET_TIMELINE, hw_ctl); } static void sde_connector_update_hdr_props(struct drm_connector *connector) @@ -2100,6 +2113,13 @@ static int _sde_connector_lm_preference(struct sde_connector *sde_conn, return ret; } +static void _sde_connector_init_hw_fence(struct sde_connector *c_conn, struct sde_kms *sde_kms) +{ + /* Enable hw-fences for wb retire-fence */ + if (c_conn->connector_type == DRM_MODE_CONNECTOR_VIRTUAL && sde_kms->catalog->hw_fence_rev) + c_conn->hwfence_wb_retire_fences_enable = true; +} + int sde_connector_get_panel_vfp(struct drm_connector *connector, struct drm_display_mode *mode) { @@ -2453,12 +2473,19 @@ static int sde_connector_init_debugfs(struct drm_connector *connector) { struct sde_connector *sde_connector; struct msm_display_info info; + struct sde_kms *sde_kms; if (!connector || !connector->debugfs_entry) { SDE_ERROR("invalid connector\n"); return -EINVAL; } + sde_kms = sde_connector_get_kms(connector); + if (!sde_kms) { + SDE_ERROR("invalid kms\n"); + return -EINVAL; + } + sde_connector = to_sde_connector(connector); sde_connector_get_info(connector, &info); @@ -2487,6 +2514,11 @@ static int sde_connector_init_debugfs(struct drm_connector *connector) } } + if (sde_connector->connector_type == DRM_MODE_CONNECTOR_VIRTUAL && + sde_kms->catalog->hw_fence_rev) + debugfs_create_bool("wb_hw_fence_enable", 0600, connector->debugfs_entry, + &sde_connector->hwfence_wb_retire_fences_enable); + return 0; } #else @@ -3353,8 +3385,11 @@ struct drm_connector *sde_connector_init(struct drm_device *dev, _sde_connector_lm_preference(c_conn, sde_kms, display_info.display_type); - SDE_DEBUG("connector %d attach encoder %d\n", - c_conn->base.base.id, encoder->base.id); + _sde_connector_init_hw_fence(c_conn, sde_kms); + + SDE_DEBUG("connector %d attach encoder %d, wb hwfences:%d\n", + DRMID(&c_conn->base), DRMID(encoder), + c_conn->hwfence_wb_retire_fences_enable); INIT_DELAYED_WORK(&c_conn->status_work, sde_connector_check_status_work); diff --git a/msm/sde/sde_connector.h b/msm/sde/sde_connector.h index 501ff6c453..f18e2584e8 100644 --- a/msm/sde/sde_connector.h +++ b/msm/sde/sde_connector.h @@ -555,6 +555,7 @@ struct sde_misr_sign { * @cached_edid: cached edid data for the connector * @misr_event_notify_enabled: Flag to indicate if misr event notify is enabled or not * @previous_misr_sign: store previous misr signature + * @hwfence_wb_retire_fences_enable: enable hw-fences for wb retire-fence */ struct sde_connector { struct drm_connector base; @@ -632,6 +633,8 @@ struct sde_connector { struct edid *cached_edid; bool misr_event_notify_enabled; struct sde_misr_sign previous_misr_sign; + + bool hwfence_wb_retire_fences_enable; }; /** diff --git a/msm/sde/sde_crtc.c b/msm/sde/sde_crtc.c index 343108cde4..32f408b398 100644 --- a/msm/sde/sde_crtc.c +++ b/msm/sde/sde_crtc.c @@ -50,6 +50,9 @@ #define SDE_PSTATES_MAX (SDE_STAGE_MAX * 4) #define SDE_MULTIRECT_PLANE_MAX (SDE_STAGE_MAX * 2) +/* Max number of planes with hw fences within one commit */ +#define MAX_HW_FENCES SDE_MULTIRECT_PLANE_MAX + struct sde_crtc_custom_events { u32 event; int (*func)(struct drm_crtc *crtc, bool en, @@ -3110,7 +3113,7 @@ static void sde_crtc_frame_event_work(struct kthread_work *work) SDE_ATRACE_BEGIN("signal_release_fence"); sde_fence_signal(sde_crtc->output_fence, fevent->ts, (fevent->event & SDE_ENCODER_FRAME_EVENT_ERROR) - ? SDE_FENCE_SIGNAL_ERROR : SDE_FENCE_SIGNAL); + ? SDE_FENCE_SIGNAL_ERROR : SDE_FENCE_SIGNAL, NULL); _sde_crtc_frame_done_notify(crtc, fevent); SDE_ATRACE_END("signal_release_fence"); } @@ -3644,40 +3647,118 @@ err: return ret; } +static struct sde_hw_ctl *_sde_crtc_get_hw_ctl(struct drm_crtc *drm_crtc) +{ + struct sde_crtc *sde_crtc = to_sde_crtc(drm_crtc); + + if (!sde_crtc || !sde_crtc->mixers[0].hw_ctl) { + DRM_ERROR("invalid crtc params %d\n", !sde_crtc); + return NULL; + } + + /* it will always return the first mixer and single CTL */ + return sde_crtc->mixers[0].hw_ctl; +} + +static struct dma_fence *_sde_plane_get_input_hw_fence(struct drm_plane *plane) +{ + struct dma_fence *fence; + struct sde_plane *psde; + struct sde_plane_state *pstate; + void *input_fence; + struct dma_fence *input_hw_fence = NULL; + + if (!plane || !plane->state) { + SDE_ERROR("invalid input %d\n", !plane); + return NULL; + } + + psde = to_sde_plane(plane); + pstate = to_sde_plane_state(plane->state); + input_fence = pstate->input_fence; + + if (input_fence) { + fence = (struct dma_fence *)pstate->input_fence; + if (fence->flags & BIT(MSM_HW_FENCE_FLAG_ENABLED_BIT)) { + input_hw_fence = fence; + + SDE_DEBUG("input hwfence ctx:%llu seqno:%llu f:0x%lx timeline:%s\n", + fence->context, fence->seqno, fence->flags, + fence->ops->get_timeline_name(fence)); + } + + SDE_EVT32_VERBOSE(DRMID(plane), fence->flags); + } + + return input_hw_fence; +} + /** - * _sde_crtc_wait_for_fences - wait for incoming framebuffer sync fences - * @crtc: Pointer to CRTC object + * _sde_crtc_fences_wait_list - wait for input sw-fences and return any hw-fences + * @crtc: Pointer to CRTC object. + * @use_hw_fences: Boolean to indicate if function should use hw-fences and skip hw-fences sw-wait. + * @dma_hw_fences: List of available hw-fences, this is populated by this function. + * @max_hw_fences: Max number of hw-fences that can be added to the dma_hw_fences list + * + * This function iterates through all crtc planes, if 'use_hw_fences' is set, for each fence: + * - If the fence is a hw-fence, it will get its dma-fence object and add it to the 'dma_hw_fences' + * list, skipping any sw-wait, since wait will happen in hw. + * - If the fence is not a hw-fence, it will wait for the sw-fence to be signaled before proceed. + * If 'use_hw_fences' is not set, function will wait on the sw-fences for all fences + * regardless if they support or not hw-fence. + * Return value is the number of hw-fences added to the 'dma_hw_fences' list. */ -static void _sde_crtc_wait_for_fences(struct drm_crtc *crtc) +static int _sde_crtc_fences_wait_list(struct drm_crtc *crtc, bool use_hw_fences, + struct dma_fence **dma_hw_fences, int max_hw_fences) { struct drm_plane *plane = NULL; - uint32_t wait_ms = 1; + u32 num_hw_fences = 0; ktime_t kt_end, kt_wait; - int rc = 0; - - SDE_DEBUG("\n"); - - if (!crtc || !crtc->state) { - SDE_ERROR("invalid crtc/state %pK\n", crtc); - return; - } + uint32_t wait_ms = 1; + int i, rc = 0; /* use monotonic timer to limit total fence wait time */ kt_end = ktime_add_ns(ktime_get(), to_sde_crtc_state(crtc->state)->input_fence_timeout_ns); - /* - * Wait for fences sequentially, as all of them need to be signalled - * before we can proceed. - * - * Limit total wait time to INPUT_FENCE_TIMEOUT, but still call - * sde_plane_wait_input_fence with wait_ms == 0 after the timeout so - * that each plane can check its fence status and react appropriately - * if its fence has timed out. Call input fence wait multiple times if - * fence wait is interrupted due to interrupt call. - */ - SDE_ATRACE_BEGIN("plane_wait_input_fence"); drm_atomic_crtc_for_each_plane(plane, crtc) { + + /* check if input-fences are hw fences and if they are, add them to the list */ + if (use_hw_fences) { + + dma_hw_fences[num_hw_fences] = _sde_plane_get_input_hw_fence(plane); + + if (dma_hw_fences[num_hw_fences] && (num_hw_fences < max_hw_fences)) { + bool repeated_fence = false; + + /* check if this fence already in the hw-fences list */ + for (i = num_hw_fences - 1; i >= 0; i--) { + if (dma_hw_fences[i] == dma_hw_fences[num_hw_fences]) { + repeated_fence = true; + break; + } + } + + if (repeated_fence) + dma_hw_fences[num_hw_fences] = NULL; /* cleanup from list */ + else + num_hw_fences++; /* keep fence in the list */ + + /* go to next, to skip sw-wait */ + continue; + } + } + + /* + * This was not a hw-fence, therefore, wait for this sw-fence to be signaled + * before proceed. + * + * Limit total wait time to INPUT_FENCE_TIMEOUT, but still call + * sde_plane_wait_input_fence with wait_ms == 0 after the timeout so + * that each plane can check its fence status and react appropriately + * if its fence has timed out. Call input fence wait multiple times if + * fence wait is interrupted due to interrupt call. + */ do { kt_wait = ktime_sub(kt_end, ktime_get()); if (ktime_compare(kt_wait, ktime_set(0, 0)) >= 0) @@ -3688,7 +3769,89 @@ static void _sde_crtc_wait_for_fences(struct drm_crtc *crtc) rc = sde_plane_wait_input_fence(plane, wait_ms); } while (wait_ms && rc == -ERESTARTSYS); } + + return num_hw_fences; +} + +static inline bool _is_vid_power_on_frame(struct drm_crtc *crtc) +{ + struct sde_crtc *sde_crtc = to_sde_crtc(crtc); + bool is_vid_mode = sde_encoder_check_curr_mode(sde_crtc->mixers[0].encoder, + MSM_DISPLAY_VIDEO_MODE); + + return is_vid_mode && crtc->state->active_changed && crtc->state->active; +} + +/** + * _sde_crtc_wait_for_fences - wait for incoming framebuffer sync fences or register hw-fences + * @crtc: Pointer to CRTC object + * + * Returns true if hw fences are used, otherwise returns false + */ +static bool _sde_crtc_wait_for_fences(struct drm_crtc *crtc) +{ + struct sde_crtc *sde_crtc = to_sde_crtc(crtc); + bool ipcc_input_signal_wait = false; + struct dma_fence *dma_hw_fences[MAX_HW_FENCES] = {0}; + int num_hw_fences = 0; + struct sde_hw_ctl *hw_ctl; + bool input_hw_fences_enable; + int ret; + + SDE_DEBUG("\n"); + + if (!crtc || !crtc->state) { + SDE_ERROR("invalid crtc/state %pK\n", crtc); + return false; + } + + hw_ctl = _sde_crtc_get_hw_ctl(crtc); + + SDE_ATRACE_BEGIN("plane_wait_input_fence"); + + /* update ctl hw to wait for ipcc input signal before fetch */ + if (test_bit(HW_FENCE_IN_FENCES_ENABLE, sde_crtc->hwfence_features_mask) && + !sde_fence_update_input_hw_fence_signal(hw_ctl)) + ipcc_input_signal_wait = true; + + /* avoid hw-fences in first frame after timing engine enable */ + input_hw_fences_enable = (ipcc_input_signal_wait && !_is_vid_power_on_frame(crtc)); + + /* wait for sw fences and get hw fences list (if any) */ + num_hw_fences = _sde_crtc_fences_wait_list(crtc, input_hw_fences_enable, &dma_hw_fences[0], + MAX_HW_FENCES); + + /* register the hw-fences for hw-wait */ + if (num_hw_fences) { + + ret = sde_fence_register_hw_fences_wait(hw_ctl, dma_hw_fences, num_hw_fences); + if (ret) { + SDE_ERROR("failed to register for hw-fence wait, will wait in sw\n"); + SDE_EVT32(SDE_EVTLOG_ERROR, num_hw_fences, + hw_ctl ? hw_ctl->idx - CTL_0 : -1); + + /* we failed to register hw-fences, wait for all fences as 'sw-fences' */ + num_hw_fences = _sde_crtc_fences_wait_list(crtc, false, &dma_hw_fences[0], + MAX_HW_FENCES); + } + } + + SDE_DEBUG("hfence_enable:%d no_override:%d ctl:%d wait_ipcc:%d num_hfences:%d\n", + input_hw_fences_enable, + test_bit(HW_FENCE_IN_FENCES_NO_OVERRIDE, sde_crtc->hwfence_features_mask), + hw_ctl ? hw_ctl->idx - CTL_0 : -1, ipcc_input_signal_wait, num_hw_fences); + SDE_EVT32(input_hw_fences_enable, + test_bit(HW_FENCE_IN_FENCES_NO_OVERRIDE, sde_crtc->hwfence_features_mask), + ipcc_input_signal_wait, num_hw_fences, hw_ctl ? hw_ctl->idx - CTL_0 : -1); + + /* if hw is waiting for ipcc signal and no hw-fences, override signal */ + if (ipcc_input_signal_wait && !num_hw_fences && hw_ctl->ops.hw_fence_trigger_sw_override && + !test_bit(HW_FENCE_IN_FENCES_NO_OVERRIDE, sde_crtc->hwfence_features_mask)) + hw_ctl->ops.hw_fence_trigger_sw_override(hw_ctl); + SDE_ATRACE_END("plane_wait_input_fence"); + + return num_hw_fences ? true : false; } static void _sde_crtc_setup_mixer_for_encoder( @@ -4100,7 +4263,7 @@ static void sde_crtc_atomic_flush_common(struct drm_crtc *crtc, sde_core_perf_crtc_update_llcc(crtc); /* wait for acquire fences before anything else is done */ - _sde_crtc_wait_for_fences(crtc); + cstate->hwfence_in_fences_set = _sde_crtc_wait_for_fences(crtc); if (!cstate->rsc_update) { drm_for_each_encoder_mask(encoder, dev, @@ -4365,6 +4528,7 @@ void sde_crtc_commit_kickoff(struct drm_crtc *crtc, unsigned long flags; enum sde_crtc_idle_pc_state idle_pc_state; struct sde_encoder_kickoff_params params = { 0 }; + bool is_vid = false; if (!crtc) { SDE_ERROR("invalid argument\n"); @@ -4410,6 +4574,9 @@ void sde_crtc_commit_kickoff(struct drm_crtc *crtc, if (idle_pc_state != IDLE_PC_NONE) sde_encoder_control_idle_pc(encoder, (idle_pc_state == IDLE_PC_ENABLE) ? true : false); + + if (sde_encoder_get_intf_mode(encoder) == INTF_MODE_VIDEO) + is_vid = true; } /* @@ -4449,6 +4616,13 @@ void sde_crtc_commit_kickoff(struct drm_crtc *crtc, _sde_crtc_blend_setup(crtc, old_state, false); } + /* + * for cmd and wb modes, update the txq for incoming fences before flush to avoid race + * condition between txq update and the hw signal during ctl-done for partial updates + */ + if (test_bit(HW_FENCE_OUT_FENCES_ENABLE, sde_crtc->hwfence_features_mask) && !is_vid) + sde_fence_update_hw_fences_txq(sde_crtc->output_fence, false); + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { if (encoder->crtc != crtc) continue; @@ -4830,6 +5004,7 @@ static void _sde_crtc_reset(struct drm_crtc *crtc) /* disable clk & bw control until clk & bw properties are set */ cstate->bw_control = false; cstate->bw_split_vote = false; + cstate->hwfence_in_fences_set = false; sde_crtc_static_img_control(crtc, CACHE_STATE_DISABLED, false); } @@ -4847,6 +5022,7 @@ static void sde_crtc_disable(struct drm_crtc *crtc) bool in_cont_splash = false; int ret, i; enum sde_intf_mode intf_mode; + struct sde_hw_ctl *hw_ctl = NULL; if (!crtc || !crtc->dev || !crtc->dev->dev_private || !crtc->state) { SDE_ERROR("invalid crtc\n"); @@ -4953,8 +5129,12 @@ static void sde_crtc_disable(struct drm_crtc *crtc) * reset the fence timeline if crtc will not be enabled for this commit */ if (!crtc->state->active || !crtc->state->enable) { + + if (test_bit(HW_FENCE_OUT_FENCES_ENABLE, sde_crtc->hwfence_features_mask)) + hw_ctl = _sde_crtc_get_hw_ctl(crtc); + sde_fence_signal(sde_crtc->output_fence, - ktime_get(), SDE_FENCE_RESET_TIMELINE); + ktime_get(), SDE_FENCE_RESET_TIMELINE, hw_ctl); for (i = 0; i < cstate->num_connectors; ++i) sde_connector_commit_reset(cstate->connectors[i], ktime_get()); @@ -6366,6 +6546,8 @@ static int _sde_crtc_get_output_fence(struct drm_crtc *crtc, uint32_t offset; bool is_vid = false; struct drm_encoder *encoder; + struct sde_hw_ctl *hw_ctl = NULL; + static u32 count; sde_crtc = to_sde_crtc(crtc); cstate = to_sde_crtc_state(state); @@ -6378,6 +6560,15 @@ static int _sde_crtc_get_output_fence(struct drm_crtc *crtc, break; } + /* + * If hw-fence is enabled, find hw_ctl and pass it to sde_fence_create, this will attempt + * to create a hw-fence for this ctl, whereas if hw_ctl is not passed to sde_fence, this + * won't use hw-fences for this output-fence. + */ + if (test_bit(HW_FENCE_OUT_FENCES_ENABLE, sde_crtc->hwfence_features_mask) && + (count++ % sde_crtc->hwfence_out_fences_skip)) + hw_ctl = _sde_crtc_get_hw_ctl(crtc); + offset = sde_crtc_get_property(cstate, CRTC_PROP_OUTPUT_FENCE_OFFSET); /* @@ -6396,7 +6587,7 @@ static int _sde_crtc_get_output_fence(struct drm_crtc *crtc, */ offset++; - return sde_fence_create(sde_crtc->output_fence, val, offset); + return sde_fence_create(sde_crtc->output_fence, val, offset, hw_ctl); } /** @@ -6878,6 +7069,72 @@ static int _sde_debugfs_status_open(struct inode *inode, struct file *file) return single_open(file, _sde_debugfs_status_show, inode->i_private); } +static ssize_t _sde_debugfs_hw_fence_features_mask_wr(struct file *file, + const char __user *user_buf, size_t count, loff_t *ppos) +{ + struct sde_crtc *sde_crtc; + u32 bit, enable; + char buf[10]; + + if (!file || !file->private_data) + return -EINVAL; + + if (count >= sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, user_buf, count)) { + SDE_ERROR("buffer copy failed\n"); + return -EINVAL; + } + + buf[count] = 0; /* end of string */ + sde_crtc = file->private_data; + + if (sscanf(buf, "%u %u", &bit, &enable) != 2) { + SDE_ERROR("incorrect usage: expected 2 parameters, bit and enable\n"); + return -EINVAL; + } + + if (enable) + set_bit(bit, sde_crtc->hwfence_features_mask); + else + clear_bit(bit, sde_crtc->hwfence_features_mask); + + return count; +} + +static ssize_t _sde_debugfs_hw_fence_features_mask_rd(struct file *file, + char __user *user_buff, size_t count, loff_t *ppos) +{ + struct sde_crtc *sde_crtc; + ssize_t len = 0; + char buf[256] = {'\0'}; + int i; + + if (*ppos) + return 0; + + if (!file || !file->private_data) + return -EINVAL; + + sde_crtc = file->private_data; + + for (i = HW_FENCE_OUT_FENCES_ENABLE; i < HW_FENCE_FEATURES_MAX; i++) { + len += scnprintf(buf + len, 256 - len, + "bit %d: %d\n", i, test_bit(i, sde_crtc->hwfence_features_mask)); + } + + if (count <= len) + return 0; + + if (copy_to_user(user_buff, buf, len)) + return -EFAULT; + + *ppos += len; /* increase offset */ + + return len; +} + static ssize_t _sde_crtc_misr_setup(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) { @@ -7164,6 +7421,12 @@ static int _sde_crtc_init_debugfs(struct drm_crtc *crtc) .read = seq_read, }; + static const struct file_operations debugfs_hw_fence_features_fops = { + .open = simple_open, + .read = _sde_debugfs_hw_fence_features_mask_rd, + .write = _sde_debugfs_hw_fence_features_mask_wr, + }; + if (!crtc) return -EINVAL; sde_crtc = to_sde_crtc(crtc); @@ -7192,6 +7455,13 @@ static int _sde_crtc_init_debugfs(struct drm_crtc *crtc) debugfs_create_file("fence_status", 0400, sde_crtc->debugfs_root, sde_crtc, &debugfs_fence_fops); + if (sde_kms->catalog->hw_fence_rev) { + debugfs_create_file("hwfence_features_mask", 0600, sde_crtc->debugfs_root, + &sde_crtc->base, &debugfs_hw_fence_features_fops); + debugfs_create_u32("hwfence_out_fences_skip", 0600, sde_crtc->debugfs_root, + &sde_crtc->hwfence_out_fences_skip); + } + return 0; } @@ -7643,6 +7913,11 @@ struct drm_crtc *sde_crtc_init(struct drm_device *dev, struct drm_plane *plane) drm_crtc_init_with_planes(dev, crtc, plane, NULL, crtc_funcs, NULL); drm_crtc_helper_add(crtc, &sde_crtc_helper_funcs); + if (kms->catalog->hw_fence_rev) { + set_bit(HW_FENCE_OUT_FENCES_ENABLE, sde_crtc->hwfence_features_mask); + set_bit(HW_FENCE_IN_FENCES_ENABLE, sde_crtc->hwfence_features_mask); + } + /* save user friendly CRTC name for later */ snprintf(sde_crtc->name, SDE_CRTC_NAME_SIZE, "crtc%u", crtc->base.id); @@ -7685,7 +7960,10 @@ struct drm_crtc *sde_crtc_init(struct drm_device *dev, struct drm_plane *plane) kthread_init_delayed_work(&sde_crtc->static_cache_read_work, __sde_crtc_static_cache_read_work); - SDE_DEBUG("%s: successfully initialized crtc\n", sde_crtc->name); + SDE_DEBUG("%s: successfully initialized crtc, hwfence_out:%d, hwfence_in:%d\n", + sde_crtc->name, + test_bit(HW_FENCE_OUT_FENCES_ENABLE, sde_crtc->hwfence_features_mask), + test_bit(HW_FENCE_IN_FENCES_ENABLE, sde_crtc->hwfence_features_mask)); return crtc; } diff --git a/msm/sde/sde_crtc.h b/msm/sde/sde_crtc.h index efef3a470a..655ee9a8e0 100644 --- a/msm/sde/sde_crtc.h +++ b/msm/sde/sde_crtc.h @@ -242,6 +242,22 @@ struct sde_opr_value { u32 opr_value[MAX_DSI_DISPLAYS]; }; +/** + * enum sde_crtc_hw_fence_flags - flags to enable/disable hw fence features + * @HW_FENCE_OUT_FENCES_ENABLE: enables creation of hw fences for crtc output fences + * @HW_FENCE_IN_FENCES_ENABLE: enables hw fences for input-fences that are candidates for hw wait + * (i.e. they have the dma-fence flag for dma-fences set), this allows to + * selectively enable/disable input-fences, regardless of the dma-fence flags. + * @HW_FENCE-IN_FENCES_NO_OVERRIDE: skip the sw-override of the input hw-fences signal. + * @HW_FENCE_FEATURES_MAX: max number of features. + */ +enum sde_crtc_hw_fence_flags { + HW_FENCE_OUT_FENCES_ENABLE, + HW_FENCE_IN_FENCES_ENABLE, + HW_FENCE_IN_FENCES_NO_OVERRIDE, + HW_FENCE_FEATURES_MAX, +}; + /** * struct sde_crtc - virtualized CRTC data structure * @base : Base drm crtc structure @@ -329,6 +345,10 @@ struct sde_opr_value { * @frame_data : Framedata data structure * @previous_opr_value : store previous opr values * @opr_event_notify_enabled : Flag to indicate if opr event notify is enabled or not + * @hwfence_features_mask : u32 mask to enable/disable hw fence features. See enum + * sde_crtc_hw_fence_flags for available fields. + * @hwfence_out_fences_skip: number of frames to skip before create a new hw-fence, this can be + * used to slow-down creation of output hw-fences for debugging purposes. */ struct sde_crtc { struct drm_crtc base; @@ -438,6 +458,9 @@ struct sde_crtc { struct sde_opr_value previous_opr_value; bool opr_event_notify_enabled; + + DECLARE_BITMAP(hwfence_features_mask, HW_FENCE_FEATURES_MAX); + u32 hwfence_out_fences_skip; }; enum sde_crtc_dirty_flags { @@ -499,6 +522,7 @@ struct sde_line_insertion_param { * @cp_range_payload: array storing state user_data passed via range props * @cont_splash_populated: State was populated as part of cont. splash * @param: sde line insertion parameters + * @hwfence_in_fences_set: input hw fences are configured for the commit */ struct sde_crtc_state { struct drm_crtc_state base; @@ -539,6 +563,7 @@ struct sde_crtc_state { cp_range_payload[SDE_CP_CRTC_MAX_FEATURES]; bool cont_splash_populated; struct sde_line_insertion_param line_insertion; + bool hwfence_in_fences_set; }; enum sde_crtc_irq_state { diff --git a/msm/sde/sde_encoder.c b/msm/sde/sde_encoder.c index 23e0bb5c15..9b1b40ef12 100644 --- a/msm/sde/sde_encoder.c +++ b/msm/sde/sde_encoder.c @@ -44,6 +44,7 @@ #include "sde_hw_qdss.h" #include "sde_encoder_dce.h" #include "sde_vm.h" +#include "sde_fence.h" #define SDE_DEBUG_ENC(e, fmt, ...) SDE_DEBUG("enc%d " fmt,\ (e) ? (e)->base.base.id : -1, ##__VA_ARGS__) @@ -3297,6 +3298,13 @@ static void sde_encoder_virt_disable(struct drm_encoder *drm_enc) sde_encoder_virt_reset(drm_enc); } +static void _trigger_encoder_hw_fences_override(struct sde_kms *sde_kms, struct sde_hw_ctl *ctl) +{ + /* trigger hw-fences override signal */ + if (sde_kms && sde_kms->catalog->hw_fence_rev && ctl->ops.hw_fence_trigger_sw_override) + ctl->ops.hw_fence_trigger_sw_override(ctl); +} + void sde_encoder_helper_phys_disable(struct sde_encoder_phys *phys_enc, struct sde_encoder_phys_wb *wb_enc) { @@ -3378,6 +3386,8 @@ void sde_encoder_helper_phys_disable(struct sde_encoder_phys *phys_enc, } } + _trigger_encoder_hw_fences_override(phys_enc->sde_kms, ctl); + sde_crtc_disable_cp_features(sde_enc->base.crtc); ctl->ops.get_pending_flush(ctl, &cfg); SDE_EVT32(DRMID(phys_enc->parent), cfg.pending_flush_mask); @@ -3691,6 +3701,25 @@ int sde_encoder_idle_request(struct drm_encoder *drm_enc) return 0; } +/** +* _sde_encoder_update_retire_txq - update tx queue for a retire hw fence +* phys: Pointer to physical encoder structure +* +*/ +static inline void _sde_encoder_update_retire_txq(struct sde_encoder_phys *phys) +{ + struct sde_connector *c_conn; + + c_conn = to_sde_connector(phys->connector); + if (!c_conn) { + SDE_ERROR("invalid connector"); + return; + } + + if (c_conn->hwfence_wb_retire_fences_enable) + sde_fence_update_hw_fences_txq(c_conn->retire_fence, false); +} + /** * _sde_encoder_trigger_flush - trigger flush for a physical encoder * drm_enc: Pointer to drm encoder structure @@ -3898,6 +3927,21 @@ void sde_encoder_helper_hw_reset(struct sde_encoder_phys *phys_enc) phys_enc->enable_state = SDE_ENC_ENABLED; } +void sde_encoder_helper_update_out_fence_txq(struct sde_encoder_virt *sde_enc, bool is_vid) +{ + struct sde_crtc *sde_crtc; + + if (!sde_enc || !sde_enc->crtc) { + SDE_ERROR("invalid encoder %d\n", !sde_enc); + return; + } + + sde_crtc = to_sde_crtc(sde_enc->crtc); + + SDE_EVT32(DRMID(sde_enc->crtc), is_vid); + sde_fence_update_hw_fences_txq(sde_crtc->output_fence, is_vid); +} + /** * _sde_encoder_kickoff_phys - handle physical encoder kickoff * Iterate through the physical encoders and perform consolidated flush @@ -3987,8 +4031,7 @@ static void _sde_encoder_kickoff_phys(struct sde_encoder_virt *sde_enc, pending_kickoff_cnt = sde_encoder_phys_inc_pending(phys); SDE_EVT32(pending_kickoff_cnt, - pending_flush.pending_flush_mask, - SDE_EVTLOG_FUNC_CASE2); + pending_flush.pending_flush_mask, SDE_EVTLOG_FUNC_CASE2); } } @@ -4572,6 +4615,10 @@ void sde_encoder_kickoff(struct drm_encoder *drm_enc, bool config_changed) SDE_EVT32(DRMID(drm_enc), i, SDE_EVTLOG_FUNC_CASE1); } + /* update txq for any output retire hw-fence (wb-path) */ + if (sde_enc->cur_master) + _sde_encoder_update_retire_txq(sde_enc->cur_master); + /* All phys encs are ready to go, trigger the kickoff */ _sde_encoder_kickoff_phys(sde_enc, config_changed); @@ -5907,6 +5954,33 @@ bool sde_encoder_needs_dsc_disable(struct drm_encoder *drm_enc) return TOPOLOGY_DSC_MODE(conn_state->old_topology_name); } +struct sde_hw_ctl *sde_encoder_get_hw_ctl(struct sde_connector *c_conn) +{ + struct drm_encoder *drm_enc; + struct sde_encoder_virt *sde_enc; + struct sde_encoder_phys *cur_master; + struct sde_hw_ctl *hw_ctl = NULL; + + if (!c_conn || !c_conn->hwfence_wb_retire_fences_enable) + goto exit; + + /* get encoder to find the hw_ctl for this connector */ + drm_enc = c_conn->encoder; + if (!drm_enc) + goto exit; + + sde_enc = to_sde_encoder_virt(drm_enc); + cur_master = sde_enc->phys_encs[0]; + if (!cur_master || !cur_master->hw_ctl) + goto exit; + + hw_ctl = cur_master->hw_ctl; + SDE_DEBUG("conn hw_ctl idx:%d intf_mode:%d\n", hw_ctl->idx, cur_master->intf_mode); + +exit: + return hw_ctl; +} + void sde_encoder_add_data_to_minidump_va(struct drm_encoder *drm_enc) { struct sde_encoder_virt *sde_enc; diff --git a/msm/sde/sde_encoder.h b/msm/sde/sde_encoder.h index c0660ff070..c19f888120 100644 --- a/msm/sde/sde_encoder.h +++ b/msm/sde/sde_encoder.h @@ -607,6 +607,13 @@ bool sde_encoder_needs_dsc_disable(struct drm_encoder *drm_enc); void sde_encoder_get_transfer_time(struct drm_encoder *drm_enc, u32 *transfer_time_us); +/** + * sde_encoder_helper_update_out_fence_txq - updates hw-fence tx queue + * @sde_enc: Pointer to sde encoder structure + * @is_vid: Boolean to indicate if is video-mode + */ +void sde_encoder_helper_update_out_fence_txq(struct sde_encoder_virt *sde_enc, bool is_vid); + /* * sde_encoder_get_dfps_maxfps - get dynamic FPS max frame rate of the given encoder @@ -691,6 +698,13 @@ static inline bool sde_encoder_is_widebus_enabled(struct drm_encoder *drm_enc) */ bool sde_encoder_is_line_insertion_supported(struct drm_encoder *drm_enc); +/** + * sde_encoder_get_hw_ctl - gets hw ctl from the connector + * @c_conn: sde connector + * @Return: pointer to the hw ctl from the encoder upon success, otherwise null + */ +struct sde_hw_ctl *sde_encoder_get_hw_ctl(struct sde_connector *c_conn); + void sde_encoder_add_data_to_minidump_va(struct drm_encoder *drm_enc); /** diff --git a/msm/sde/sde_encoder_phys_vid.c b/msm/sde/sde_encoder_phys_vid.c index 575691f716..4622f14e26 100644 --- a/msm/sde/sde_encoder_phys_vid.c +++ b/msm/sde/sde_encoder_phys_vid.c @@ -927,6 +927,21 @@ static int sde_encoder_phys_vid_wait_for_vblank( return _sde_encoder_phys_vid_wait_for_vblank(phys_enc, true); } +static void sde_encoder_phys_vid_update_txq(struct sde_encoder_phys *phys_enc) +{ + struct sde_encoder_virt *sde_enc; + + if (!phys_enc) + return; + + sde_enc = to_sde_encoder_virt(phys_enc->parent); + if (!sde_enc) + return; + + SDE_EVT32(DRMID(phys_enc->parent)); + sde_encoder_helper_update_out_fence_txq(sde_enc, true); +} + static int sde_encoder_phys_vid_wait_for_commit_done( struct sde_encoder_phys *phys_enc) { @@ -936,6 +951,9 @@ static int sde_encoder_phys_vid_wait_for_commit_done( if (rc) sde_encoder_helper_phys_reset(phys_enc); + /* Update TxQ for the incoming frame */ + sde_encoder_phys_vid_update_txq(phys_enc); + return rc; } diff --git a/msm/sde/sde_fence.c b/msm/sde/sde_fence.c index 3f4f1895c4..6f8faeacdc 100644 --- a/msm/sde/sde_fence.c +++ b/msm/sde/sde_fence.c @@ -16,6 +16,481 @@ #define SPEC_FENCE_FLAG_FENCE_ARRAY 0x10 #define SPEC_FENCE_FLAG_ARRAY_BIND 0x11 +/** + * struct sde_fence - release/retire fence structure + * @base: base fence structure + * @ctx: fence context + * @name: name of each fence- it is fence timeline + commit_count + * @fence_list: list to associated this fence on timeline/context + * @fd: fd attached to this fence - debugging purpose. + * @hwfence_out_ctl: hw ctl for the output fence + * @hwfence_index: hw fence index for this fence + * @txq_updated_fence: flag to indicate that a fence has been updated in txq + */ +struct sde_fence { + struct dma_fence base; + struct sde_fence_context *ctx; + char name[SDE_FENCE_NAME_SIZE]; + struct list_head fence_list; + int fd; + struct sde_hw_ctl *hwfence_out_ctl; + u64 hwfence_index; + bool txq_updated_fence; +}; + +/** + * enum sde_hw_fence_clients - sde clients for the hw-fence feature + * + * Do not modify the order of this struct and/or add more elements + * without modify/add fields in the 'hw_fence_data' structs. + */ +enum sde_hw_fence_clients { + SDE_HW_FENCE_CLIENT_CTL_0, + SDE_HW_FENCE_CLIENT_CTL_1, + SDE_HW_FENCE_CLIENT_CTL_2, + SDE_HW_FENCE_CLIENT_CTL_3, + SDE_HW_FENCE_CLIENT_CTL_4, + SDE_HW_FENCE_CLIENT_CTL_5, + SDE_HW_FENCE_CLIENT_MAX, +}; + +/** + * hw_fence_data_dpu_client - this table maps the dpu ipcc input and output signals for each display + * clients to communicate with the fence controller. + * This struct must match the order of the 'sde_hw_fence_clients' enum, + * the output signal must match with the signals that FenceCTL expects for each display client. + * This 'hw_fence_data_dpu_client' must be used for HW that does not support dpu-signal. + */ +struct sde_hw_fence_data hw_fence_data_no_dpu[SDE_HW_FENCE_CLIENT_MAX] = { + {SDE_HW_FENCE_CLIENT_CTL_0, HW_FENCE_CLIENT_ID_CTL0, NULL, {0}, 8, 14, {2, 3}, 0, 8, 8}, + {SDE_HW_FENCE_CLIENT_CTL_1, HW_FENCE_CLIENT_ID_CTL1, NULL, {0}, 8, 15, {4, 5}, 0, 8, 8}, + {SDE_HW_FENCE_CLIENT_CTL_2, HW_FENCE_CLIENT_ID_CTL2, NULL, {0}, 8, 16, {6, 7}, 0, 8, 8}, + {SDE_HW_FENCE_CLIENT_CTL_3, HW_FENCE_CLIENT_ID_CTL3, NULL, {0}, 8, 17, {8, 9}, 0, 8, 8}, + {SDE_HW_FENCE_CLIENT_CTL_4, HW_FENCE_CLIENT_ID_CTL4, NULL, {0}, 8, 18, {10, 11}, 0, 8, 8}, + {SDE_HW_FENCE_CLIENT_CTL_5, HW_FENCE_CLIENT_ID_CTL5, NULL, {0}, 8, 19, {12, 13}, 0, 8, 8} +}; + +/** + * hw_fence_data_dpu_client - this table maps the dpu ipcc input and output signals for each display + * clients to communicate with the fence controller. + * This struct must match the order of the 'sde_hw_fence_clients' enum, + * the output signal must match with the signals that FenceCTL expects for each display client. + * This 'hw_fence_data_dpu_client' must be used for HW that supports dpu-signal + */ +struct sde_hw_fence_data hw_fence_data_dpu_client[SDE_HW_FENCE_CLIENT_MAX] = { + {SDE_HW_FENCE_CLIENT_CTL_0, HW_FENCE_CLIENT_ID_CTL0, NULL, {0}, 8, 0, {0, 6}, 0, 8, 25}, + {SDE_HW_FENCE_CLIENT_CTL_1, HW_FENCE_CLIENT_ID_CTL1, NULL, {0}, 8, 1, {1, 7}, 0, 8, 25}, + {SDE_HW_FENCE_CLIENT_CTL_2, HW_FENCE_CLIENT_ID_CTL2, NULL, {0}, 8, 2, {2, 8}, 0, 8, 25}, + {SDE_HW_FENCE_CLIENT_CTL_3, HW_FENCE_CLIENT_ID_CTL3, NULL, {0}, 8, 3, {3, 9}, 0, 8, 25}, + {SDE_HW_FENCE_CLIENT_CTL_4, HW_FENCE_CLIENT_ID_CTL4, NULL, {0}, 8, 4, {4, 10}, 0, 8, 25}, + {SDE_HW_FENCE_CLIENT_CTL_5, HW_FENCE_CLIENT_ID_CTL5, NULL, {0}, 8, 5, {5, 11}, 0, 8, 25} +}; + +int sde_hw_fence_init(struct sde_hw_ctl *hw_ctl, bool use_dpu_ipcc) +{ + struct sde_hw_fence_data *sde_hw_fence_data; + struct sde_hw_fence_data *hwfence_data; + int ctl_id; + + if (!hw_ctl) + return -EINVAL; + + ctl_id = hw_ctl->idx - CTL_0; + if (ctl_id >= SDE_HW_FENCE_CLIENT_MAX || ctl_id < 0) { + SDE_ERROR("unexpected ctl_id:%d\n", ctl_id); + return -EINVAL; + } + + hwfence_data = &hw_ctl->hwfence_data; + sde_hw_fence_data = use_dpu_ipcc ? hw_fence_data_dpu_client : hw_fence_data_no_dpu; + + if (sde_hw_fence_data[ctl_id].client_id != ctl_id) { + SDE_ERROR("Unexpected client_id:%d for ctl_id:%d\n", + sde_hw_fence_data[ctl_id].client_id, ctl_id); + return -EINVAL; + } + + /* init the default fence-data for this client */ + memcpy(hwfence_data, &sde_hw_fence_data[ctl_id], sizeof(struct sde_hw_fence_data)); + + SDE_DEBUG("hwfence register ctl:%d client:%d\n", ctl_id, hwfence_data->hw_fence_client_id); + hwfence_data->hw_fence_handle = msm_hw_fence_register(hwfence_data->hw_fence_client_id, + &hwfence_data->mem_descriptor); + if (IS_ERR_OR_NULL(hwfence_data->hw_fence_handle)) { + + hwfence_data->hw_fence_handle = NULL; + + SDE_ERROR("error cannot register ctl_id:%d hw-fence client:%d\n", ctl_id, + hwfence_data->hw_fence_client_id); + return -EINVAL; + } + + SDE_DEBUG("hwfence registered ctl_id:%d hw_fence_client_id:%d handle:0x%p\n", + ctl_id, hwfence_data->hw_fence_client_id, hwfence_data->hw_fence_handle); + + return 0; +} + +void sde_hw_fence_deinit(struct sde_hw_ctl *hw_ctl) +{ + struct sde_hw_fence_data *hwfence_data; + + if (!hw_ctl) + return; + + hwfence_data = &hw_ctl->hwfence_data; + + /* client was not registered */ + if (IS_ERR_OR_NULL(hwfence_data->hw_fence_handle)) + return; + + SDE_DEBUG("hwfence deregister ctl_id:%d hw_fence_client_id:%d\n", + hw_ctl->idx - CTL_0, hwfence_data->hw_fence_client_id); + + msm_hw_fence_deregister(hwfence_data->hw_fence_handle); + + hwfence_data->hw_fence_handle = NULL; +} + +static int sde_fence_create_hw_fence(struct sde_hw_ctl *hw_ctl, struct sde_fence *sde_fence) +{ + struct sde_hw_fence_data *data; + struct msm_hw_fence_create_params params; + int ctl_id; + u64 hwfence_index; + int ret; + + if (!hw_ctl) + return -EINVAL; + + ctl_id = hw_ctl->idx - CTL_0; + data = &hw_ctl->hwfence_data; + + if (IS_ERR_OR_NULL(data->hw_fence_handle)) { + SDE_ERROR("unexpected handle for ctl_id:%d\n", ctl_id); + return -EINVAL; + } + params.fence = &sde_fence->base; + params.handle = &hwfence_index; + + /* Create the HW fence */ + ret = msm_hw_fence_create(data->hw_fence_handle, ¶ms); + if (ret) { + SDE_ERROR("failed to create hw_fence for client:%d ctx:%llu seqno:%llu\n", ctl_id, + sde_fence->base.context, sde_fence->base.seqno); + } else { + /* store ctl and index for this fence */ + sde_fence->hwfence_out_ctl = hw_ctl; + sde_fence->hwfence_index = hwfence_index; + + SDE_DEBUG("create hfence index:%llu ctl:%d ctx:%llu seqno:%llu name:%s\n", + sde_fence->hwfence_index, ctl_id, sde_fence->base.context, + sde_fence->base.seqno, sde_fence->name); + } + + return ret; +} + +static inline char *_get_client_id_name(int hw_fence_client_id) +{ + switch (hw_fence_client_id) { + case HW_FENCE_CLIENT_ID_CTX0: + return "HW_FENCE_CLIENT_ID_CTX0"; + case HW_FENCE_CLIENT_ID_CTL0: + return "HW_FENCE_CLIENT_ID_CTL0"; + case HW_FENCE_CLIENT_ID_CTL1: + return "HW_FENCE_CLIENT_ID_CTL1"; + case HW_FENCE_CLIENT_ID_CTL2: + return "HW_FENCE_CLIENT_ID_CTL2"; + case HW_FENCE_CLIENT_ID_CTL3: + return "HW_FENCE_CLIENT_ID_CTL3"; + case HW_FENCE_CLIENT_ID_CTL4: + return "HW_FENCE_CLIENT_ID_CTL4"; + case HW_FENCE_CLIENT_ID_CTL5: + return "HW_FENCE_CLIENT_ID_CTL15"; + default: + return "Unknown"; + } + + return "unknown"; +} + +int sde_fence_register_hw_fences_wait(struct sde_hw_ctl *hw_ctl, struct dma_fence **fences, + u32 num_fences) +{ + struct sde_hw_fence_data *data; + int i, ret; + int ctl_id; + + if (!hw_ctl) { + SDE_ERROR("wrong ctl\n"); + return -EINVAL; + } + + ctl_id = hw_ctl->idx - CTL_0; + data = &hw_ctl->hwfence_data; + if (IS_ERR_OR_NULL(data->hw_fence_handle)) { + SDE_ERROR("unexpected handle for ctl_id:%d\n", ctl_id); + return -EINVAL; + } + + SDE_DEBUG("register for wait fences:%d ctl_id:%d hw_fence_client:%s\n", + num_fences, ctl_id, _get_client_id_name(data->hw_fence_client_id)); + + for (i = 0; i < num_fences; i++) { + SDE_DEBUG("registering fence: ctx:%llu seqno:%llu\n", + (fences[i])->context, (fences[i])->seqno); + } + + /* register for wait */ + ret = msm_hw_fence_wait_update(data->hw_fence_handle, fences, num_fences, true); + if (ret) + SDE_ERROR("failed to register wait fences for ctl_id:%d ret:%d\n", ctl_id, ret); + SDE_EVT32_VERBOSE(ctl_id, num_fences, ret); + + return ret; +} + +static int _arm_output_hw_fence(struct sde_hw_ctl *hw_ctl) +{ + struct sde_hw_fence_data *data; + u32 ipcc_out_signal; + int ctl_id; + + if (!hw_ctl || !hw_ctl->ops.hw_fence_trigger_output_fence || + !hw_ctl->ops.hw_fence_update_output_fence) { + SDE_ERROR("missing ctl/trigger or update fence %d\n", !hw_ctl); + return -EINVAL; + } + + ctl_id = hw_ctl->idx - CTL_0; + data = &hw_ctl->hwfence_data; + if (data->ipcc_out_signal_pp_idx >= MAX_SDE_HFENCE_OUT_SIGNAL_PING_PONG) { + /* This should not have happened!, review the ping pong calculation */ + SDE_ERROR("Wrong pp_idx:%d, max:%d\n", data->ipcc_out_signal_pp_idx, + MAX_SDE_HFENCE_OUT_SIGNAL_PING_PONG); + return -EINVAL; + } + + ipcc_out_signal = data->ipcc_out_signal_pp[data->ipcc_out_signal_pp_idx]; + data->ipcc_out_signal_pp_idx = (++data->ipcc_out_signal_pp_idx % + MAX_SDE_HFENCE_OUT_SIGNAL_PING_PONG); + + SDE_DEBUG("out-fence ctl_id:%d out_signal:%d hw_fence_client:%s\n", + ctl_id, ipcc_out_signal, _get_client_id_name(data->hw_fence_client_id)); + + /* update client/signal output fence */ + hw_ctl->ops.hw_fence_update_output_fence(hw_ctl, data->ipcc_out_client, ipcc_out_signal); + SDE_EVT32_VERBOSE(ctl_id, ipcc_out_signal); + + /* arm dpu to trigger output fence signal once ready */ + hw_ctl->ops.hw_fence_trigger_output_fence(hw_ctl, HW_FENCE_TRIGGER_SEL_CTRL_DONE); + + return 0; +} + +static int _sde_fence_arm_output_hw_fence(struct sde_fence_context *ctx) +{ + struct sde_hw_ctl *hw_ctl = NULL; + struct sde_fence *fc, *next; + + spin_lock(&ctx->list_lock); + if (list_empty(&ctx->fence_list_head)) { + spin_unlock(&ctx->list_lock); + return 0; + } + + list_for_each_entry_safe(fc, next, &ctx->fence_list_head, fence_list) { + struct dma_fence *fence = &fc->base; + + /* this is not hw-fence, or already processed */ + if (!test_bit(MSM_HW_FENCE_FLAG_ENABLED_BIT, &fence->flags)) + continue; + + hw_ctl = fc->hwfence_out_ctl; + if (!hw_ctl) { + /* + * We flaged an output dma-fence as hw-fence but the hw ctl to handle + * it is not available, this should not have happened, but if it does, + * this can translate to a fence-timeout! + */ + SDE_ERROR("invalid hw ctl, this can cause a fence-timeout!\n"); + SDE_EVT32(SDE_EVTLOG_ERROR, SDE_EVTLOG_FUNC_CASE1, fence->flags, + fence->context, fence->seqno); + + spin_unlock(&ctx->list_lock); + return -EINVAL; + } + } + spin_unlock(&ctx->list_lock); + + /* arm dpu to trigger output hw-fence ipcc signal upon completion */ + if (hw_ctl) + _arm_output_hw_fence(hw_ctl); + + return 0; +} + +/* update output hw_fences txq */ +int sde_fence_update_hw_fences_txq(struct sde_fence_context *ctx, bool vid_mode) +{ + int ret = 0; + struct sde_hw_fence_data *data; + struct sde_fence *fc, *next; + struct sde_hw_ctl *hw_ctl = NULL; + int ctl_id; + bool txq_updated = false; + + spin_lock(&ctx->list_lock); + if (list_empty(&ctx->fence_list_head)) { + spin_unlock(&ctx->list_lock); + return 0; + } + + list_for_each_entry_safe(fc, next, &ctx->fence_list_head, fence_list) { + struct dma_fence *fence = &fc->base; + + /* this is not hw-fence, or already processed */ + if (!test_bit(MSM_HW_FENCE_FLAG_ENABLED_BIT, &fence->flags) || + fc->txq_updated_fence) + continue; + + hw_ctl = fc->hwfence_out_ctl; + if (!hw_ctl) { + /* We flaged an output dma-fence as hw-fence but the hw ctl to handle + * it is not available, this should not have happened, but if it does, + * this can translate to a fence-timeout! + */ + SDE_ERROR("invalid hw ctl, this can cause a fence-timeout!\n"); + SDE_EVT32(SDE_EVTLOG_FUNC_CASE1, fence->flags, fence->context, + fence->seqno, SDE_EVTLOG_ERROR); + ret = -EINVAL; + goto exit; + } + + ctl_id = hw_ctl->idx - CTL_0; + data = &hw_ctl->hwfence_data; + if (IS_ERR_OR_NULL(data->hw_fence_handle)) { + SDE_ERROR("unexpected handle for ctl_id:%d, this can fence-timeout\n", + ctl_id); + SDE_EVT32(SDE_EVTLOG_FUNC_CASE2, fence->flags, fence->context, + fence->seqno, ctl_id, SDE_EVTLOG_ERROR); + ret = -EINVAL; + goto exit; + } + + /* update hw-fence tx queue */ + ret = msm_hw_fence_update_txq(data->hw_fence_handle, fc->hwfence_index, 0, 0); + if (ret) { + SDE_ERROR("fail txq update index:%llu fctx:%llu seqno:%llu client:%d\n", + fc->hwfence_index, fence->context, fence->seqno, + data->hw_fence_client_id); + SDE_EVT32(SDE_EVTLOG_FUNC_CASE3, fence->flags, fence->context, + fence->seqno, ctl_id, SDE_EVTLOG_ERROR); + goto exit; + } + /* avoid updating txq more than once and avoid repeating the same fence twice */ + txq_updated = fc->txq_updated_fence = true; + + SDE_DEBUG("update txq fence:0x%pK ctx:%llu seqno:%llu f:0x%llx ctl:%d vid:%d\n", + fence, fence->context, fence->seqno, fence->flags, ctl_id, vid_mode); + + /* We will update TxQ one time per frame */ + if (txq_updated) + break; + } + +exit: + spin_unlock(&ctx->list_lock); + + /* arm dpu to trigger output hw-fence ipcc signal upon completion in vid-mode */ + if (txq_updated && hw_ctl) + _sde_fence_arm_output_hw_fence(ctx); + + return ret; +} + +static void _sde_hw_fence_release(struct sde_fence *f) +{ + struct sde_hw_fence_data *data; + struct sde_hw_ctl *hw_ctl = f->hwfence_out_ctl; + int ctl_id; + int ret; + + if (!hw_ctl) { + SDE_ERROR("invalid hw_ctl\n"); + return; + } + + ctl_id = hw_ctl->idx - CTL_0; + data = &hw_ctl->hwfence_data; + if (IS_ERR_OR_NULL(data->hw_fence_handle)) { + SDE_ERROR("unexpected handle for ctl_id:%d\n", ctl_id); + return; + } + + SDE_DEBUG("destroy hw fence ctl_id:%d ctx:%llu seqno:%llu name:%s\n", + ctl_id, f->base.context, f->base.seqno, f->name); + + /* Delete the HW fence */ + ret = msm_hw_fence_destroy(data->hw_fence_handle, &f->base); + if (ret) + SDE_ERROR("failed to destroy hw_fence for ctl_id:%d ctx:%llu seqno:%llu\n", ctl_id, + f->base.context, f->base.seqno); +} + +static int _reset_hw_fence_timeline(struct sde_hw_ctl *hw_ctl, u32 flags) +{ + struct sde_hw_fence_data *data; + int ret = 0; + + data = &hw_ctl->hwfence_data; + + if (!IS_ERR_OR_NULL(data->hw_fence_handle)) { + SDE_EVT32(data->hw_fence_client_id); + ret = msm_hw_fence_reset_client(data->hw_fence_handle, flags); + if (ret) { + pr_err("failed to reset client %d\n", data->hw_fence_client_id); + return -EINVAL; + } + } + + return ret; +} + +int sde_fence_update_input_hw_fence_signal(struct sde_hw_ctl *hw_ctl) +{ + struct sde_hw_fence_data *data; + u32 ipcc_signal_id; + u32 ipcc_client_id; + int ctl_id; + + /* we must support sw_override as well, so check both functions */ + if (!hw_ctl || !hw_ctl->ops.hw_fence_update_input_fence || + !hw_ctl->ops.hw_fence_trigger_sw_override) { + SDE_ERROR("missing ctl/override/update fence %d\n", !hw_ctl); + return -EINVAL; + } + + ctl_id = hw_ctl->idx - CTL_0; + data = &hw_ctl->hwfence_data; + + ipcc_signal_id = data->ipcc_in_signal; + ipcc_client_id = data->ipcc_in_client; + + SDE_DEBUG("configure input signal:%d out client:%d ctl_id:%d\n", ipcc_signal_id, + ipcc_client_id, ctl_id); + SDE_EVT32(ctl_id, ipcc_signal_id, ipcc_client_id); + + /* configure dpu hw for the client/signal pair signaling input-fence */ + hw_ctl->ops.hw_fence_update_input_fence(hw_ctl, ipcc_client_id, ipcc_signal_id); + + /* Enable hw-fence for this ctrl-path */ + hw_ctl->ops.hw_fence_ctrl(hw_ctl, true, true, 1); + + return 0; +} + void *sde_sync_get(uint64_t fd) { /* force signed compare, fdget accepts an int argument */ @@ -117,21 +592,6 @@ uint32_t sde_sync_get_name_prefix(void *fence) return prefix; } -/** - * struct sde_fence - release/retire fence structure - * @fence: base fence structure - * @name: name of each fence- it is fence timeline + commit_count - * @fence_list: list to associated this fence on timeline/context - * @fd: fd attached to this fence - debugging purpose. - */ -struct sde_fence { - struct dma_fence base; - struct sde_fence_context *ctx; - char name[SDE_FENCE_NAME_SIZE]; - struct list_head fence_list; - int fd; -}; - static void sde_fence_destroy(struct kref *kref) { struct sde_fence_context *ctx; @@ -186,6 +646,11 @@ static void sde_fence_release(struct dma_fence *fence) if (fence) { f = to_sde_fence(fence); + + /* Delete the HW fence */ + if (test_bit(MSM_HW_FENCE_FLAG_ENABLED_BIT, &fence->flags)) + _sde_hw_fence_release(f); + kref_put(&f->ctx->kref, sde_fence_destroy); kfree(f); } @@ -228,7 +693,7 @@ static struct dma_fence_ops sde_fence_ops = { * @val: Timeline value at which to signal the fence * Return: File descriptor on success, or error code on error */ -static int _sde_fence_create_fd(void *fence_ctx, uint32_t val) +static int _sde_fence_create_fd(void *fence_ctx, uint32_t val, struct sde_hw_ctl *hw_ctl) { struct sde_fence *sde_fence; struct sync_file *sync_file; @@ -270,6 +735,10 @@ static int _sde_fence_create_fd(void *fence_ctx, uint32_t val) goto exit; } + /* If ctl_id is valid, try to create a hw-fence */ + if (hw_ctl) + sde_fence_create_hw_fence(hw_ctl, sde_fence); + fd_install(fd, sync_file->file); sde_fence->fd = fd; @@ -366,7 +835,7 @@ end: } int sde_fence_create(struct sde_fence_context *ctx, uint64_t *val, - uint32_t offset) + uint32_t offset, struct sde_hw_ctl *hw_ctl) { uint32_t trigger_value; int fd, rc = -EINVAL; @@ -390,19 +859,19 @@ int sde_fence_create(struct sde_fence_context *ctx, uint64_t *val, trigger_value = ctx->commit_count + offset; spin_unlock_irqrestore(&ctx->lock, flags); - fd = _sde_fence_create_fd(ctx, trigger_value); + fd = _sde_fence_create_fd(ctx, trigger_value, hw_ctl); *val = fd; SDE_DEBUG("fd:%d trigger:%d commit:%d offset:%d\n", fd, trigger_value, ctx->commit_count, offset); - SDE_EVT32(ctx->drm_id, trigger_value, fd); + SDE_EVT32(ctx->drm_id, trigger_value, fd, hw_ctl ? hw_ctl->idx : 0); rc = (fd >= 0) ? 0 : fd; return rc; } void sde_fence_signal(struct sde_fence_context *ctx, ktime_t ts, - enum sde_fence_event fence_event) + enum sde_fence_event fence_event, struct sde_hw_ctl *hw_ctl) { unsigned long flags; @@ -413,6 +882,11 @@ void sde_fence_signal(struct sde_fence_context *ctx, ktime_t ts, spin_lock_irqsave(&ctx->lock, flags); if (fence_event == SDE_FENCE_RESET_TIMELINE) { + /* reset hw-fences without error */ + if (hw_ctl) + _reset_hw_fence_timeline(hw_ctl, MSM_HW_FENCE_RESET_WITHOUT_ERROR | + MSM_HW_FENCE_RESET_WITHOUT_DESTROY); + if ((int)(ctx->done_count - ctx->commit_count) < 0) { SDE_DEBUG( "timeline reset attempt! done count:%d commit:%d\n", diff --git a/msm/sde/sde_fence.h b/msm/sde/sde_fence.h index 10177eecbb..d1822bf9df 100644 --- a/msm/sde/sde_fence.h +++ b/msm/sde/sde_fence.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved. */ @@ -9,13 +10,18 @@ #include #include #include +#include #ifndef CHAR_BIT #define CHAR_BIT 8 /* define this if limits.h not available */ #endif +#define HW_FENCE_TRIGGER_SEL_CTRL_DONE 0x0 +#define HW_FENCE_TRIGGER_SEL_PROG_LINE_COUNT 0x1 + #define SDE_FENCE_NAME_SIZE 24 +#define MAX_SDE_HFENCE_OUT_SIGNAL_PING_PONG 2 /** * struct sde_fence_context - release/retire fence context/timeline structure * @commit_count: Number of detected commits since bootup @@ -52,6 +58,32 @@ enum sde_fence_event { SDE_FENCE_SIGNAL_ERROR }; +/** + * struct sde_hw_fence_data - contains the information of each display-client of the hw-fences + * to communicate with the fence controller. + * @client_id: client_id enum for the display driver. + * @hw_fence_client_id: client_id enum for the hw-fence driver. + * @mem_descriptor: memory descriptor with the hfi for the rx/tx queues mapping. + * @ipcc_in_client: ipcc client triggering the signal: IN_CLIENT (APPS) -> DPU + * @ipcc_in_signal: ipcc signal triggered from client to dpu: IN_SIGNAL (APPS) -> DPU + * @ipcc_out_signal_pp: output signal from dpu to fctl, ping-pongs between two signals + * @ipcc_out_signal_pp_idx: index of the output signal ping-pong + * @ipcc_out_client: destination client id (APPS for the FCTL) + * @ipcc_this_client: ipcc dpu client id (For Waipio: APPS, For Kailua: DPU HW) + */ +struct sde_hw_fence_data { + int client_id; + enum hw_fence_client_id hw_fence_client_id; + void *hw_fence_handle; + struct msm_hw_fence_mem_addr mem_descriptor; + u32 ipcc_in_client; + u32 ipcc_in_signal; + u32 ipcc_out_signal_pp[MAX_SDE_HFENCE_OUT_SIGNAL_PING_PONG]; + u32 ipcc_out_signal_pp_idx; + u32 ipcc_out_client; + u32 ipcc_this_client; +}; + #if IS_ENABLED(CONFIG_SYNC_FILE) /** * sde_sync_get - Query sync fence object from a file handle @@ -89,6 +121,7 @@ signed long sde_sync_wait(void *fence, long timeout_ms); /** * sde_sync_get_name_prefix - get integer representation of fence name prefix + * * @fence: Pointer to opaque fence structure * * Return: 32-bit integer containing first 4 characters of fence name, @@ -98,13 +131,65 @@ uint32_t sde_sync_get_name_prefix(void *fence); /** * sde_fence_init - initialize fence object + * * @drm_id: ID number of owning DRM Object * @name: Timeline name + * * Returns: fence context object on success */ struct sde_fence_context *sde_fence_init(const char *name, uint32_t drm_id); +/** + * sde_fence_hw_fence_init - initialize hw-fence clients + * + * @hw_ctl: hw ctl client to init. + * @use_ipcc: boolean to indicate if hw should use dpu ipcc signals. + * + * Returns: Zero on success, otherwise returns an error code. + */ +int sde_hw_fence_init(struct sde_hw_ctl *hw_ctl, bool use_dpu_ipcc); + +/** + * sde_fence_hw_fence_deinit - deinitialize hw-fence clients + * + * @hw_ctl: hw ctl client to init. + */ +void sde_hw_fence_deinit(struct sde_hw_ctl *hw_ctl); + +/** + * sde_fence_register_hw_fences_wait - registers dpu-client for wait on hw fence or fences + * + * @hw_ctl: hw ctl client used to register for wait. + * @fences: list of dma-fences that have hw-fence support to wait-on + * @num_fences: number of fences in the above list + * + * Returns: Zero on success, otherwise returns an error code. + */ +int sde_fence_register_hw_fences_wait(struct sde_hw_ctl *hw_ctl, struct dma_fence **fences, + u32 num_fences); + +/** + * sde_fence_update_hw_fences_txq - updates the hw-fence txq with the list of hw-fences to signal + * upon triggering the ipcc signal. + * + * @ctx: sde fence context + * @vid_mode: is video-mode update + * + * Returns: Zero on success, otherwise returns an error code. + */ +int sde_fence_update_hw_fences_txq(struct sde_fence_context *ctx, bool vid_mode); + +/** + * sde_fence_update_input_hw_fence_signal - updates input-fence ipcc signal in dpu and enables + * hw-fences for the ctl. + * + * @ctl: hw ctl to update the input-fence and enable hw-fences + * + * Returns: Zero on success, otherwise returns an error code. + */ +int sde_fence_update_input_hw_fence_signal(struct sde_hw_ctl *ctl); + /** * sde_fence_deinit - deinit fence container * @fence: Pointer fence container @@ -122,19 +207,21 @@ void sde_fence_prepare(struct sde_fence_context *fence); * @fence: Pointer fence container * @val: Pointer to output value variable, fence fd will be placed here * @offset: Fence signal commit offset, e.g., +1 to signal on next commit + * @hw_ctl: Ctl for hw fences * Returns: Zero on success */ int sde_fence_create(struct sde_fence_context *fence, uint64_t *val, - uint32_t offset); + uint32_t offset, struct sde_hw_ctl *hw_ctl); /** * sde_fence_signal - advance fence timeline to signal outstanding fences * @fence: Pointer fence container * @ts: fence timestamp * @fence_event: fence event to indicate nature of fence signal. + * @hw_ctl: ctl to signal fences for the timeline rest event */ void sde_fence_signal(struct sde_fence_context *fence, ktime_t ts, - enum sde_fence_event fence_event); + enum sde_fence_event fence_event, struct sde_hw_ctl *hw_ctl); /** * sde_fence_timeline_status - prints fence timeline status diff --git a/msm/sde/sde_hw_catalog.c b/msm/sde/sde_hw_catalog.c index 3de7cd0a69..6836e35eb9 100644 --- a/msm/sde/sde_hw_catalog.c +++ b/msm/sde/sde_hw_catalog.c @@ -178,6 +178,7 @@ *************************************************************/ enum { SDE_HW_VERSION, + SDE_HW_FENCE_VERSION, SDE_HW_PROP_MAX, }; @@ -574,6 +575,7 @@ struct sde_dt_props { *************************************************************/ static struct sde_prop_type sde_hw_prop[] = { {SDE_HW_VERSION, "qcom,sde-hw-version", false, PROP_TYPE_U32}, + {SDE_HW_FENCE_VERSION, "qcom,hw-fence-sw-version", false, PROP_TYPE_U32}, }; static struct sde_prop_type sde_prop[] = { @@ -5215,6 +5217,22 @@ end: return rc; } +static void _sde_hw_fence_caps(struct sde_mdss_cfg *sde_cfg) +{ + struct sde_ctl_cfg *ctl; + int i; + + if (!sde_cfg->hw_fence_rev) + return; + + set_bit(SDE_FEATURE_HW_FENCE_IPCC, sde_cfg->features); + + for (i = 0; i < sde_cfg->ctl_count; i++) { + ctl = sde_cfg->ctl + i; + set_bit(SDE_CTL_HW_FENCE, &ctl->features); + } +} + static int _sde_hardware_post_caps(struct sde_mdss_cfg *sde_cfg, uint32_t hw_rev) { @@ -5262,6 +5280,8 @@ static int _sde_hardware_post_caps(struct sde_mdss_cfg *sde_cfg, sde_cfg->min_display_width = MIN_DISPLAY_WIDTH; sde_cfg->max_cwb = min_t(u32, sde_cfg->wb_count, MAX_CWB_SESSIONS); + _sde_hw_fence_caps(sde_cfg); + rc = _sde_hw_dnsc_blur_filter_caps(sde_cfg); return rc; @@ -5356,6 +5376,11 @@ static int sde_hw_ver_parse_dt(struct drm_device *dev, struct device_node *np, else cfg->hw_rev = sde_kms_get_hw_version(dev); + if (prop_exists[SDE_HW_FENCE_VERSION]) + cfg->hw_fence_rev = PROP_VALUE_ACCESS(prop_value, SDE_HW_FENCE_VERSION, 0); + else + cfg->hw_fence_rev = 0; /* disable hw-fences */ + end: kfree(prop_value); return rc; diff --git a/msm/sde/sde_hw_catalog.h b/msm/sde/sde_hw_catalog.h index aac0d8a829..a9afabeb38 100644 --- a/msm/sde/sde_hw_catalog.h +++ b/msm/sde/sde_hw_catalog.h @@ -557,6 +557,7 @@ enum { * blocks * @SDE_CTL_UIDLE CTL supports uidle * @SDE_CTL_UNIFIED_DSPP_FLUSH CTL supports only one flush bit for DSPP + * @SDE_CTL_HW_FENCE CTL supports hw fencing * @SDE_CTL_MAX */ enum { @@ -566,6 +567,7 @@ enum { SDE_CTL_ACTIVE_CFG, SDE_CTL_UIDLE, SDE_CTL_UNIFIED_DSPP_FLUSH, + SDE_CTL_HW_FENCE, SDE_CTL_MAX }; @@ -721,6 +723,7 @@ enum { * @SDE_FEATURE_VBIF_CLK_SPLIT VBIF clock split supported * @SDE_FEATURE_CTL_DONE Support for CTL DONE irq * @SDE_FEATURE_SYS_CACHE_NSE Support for no-self-evict feature + * @SDE_FEATURE_HW_FENCE_IPCC HW fence supports ipcc signaling in dpu * @SDE_FEATURE_MAX: MAX features value */ enum sde_mdss_features { @@ -761,6 +764,7 @@ enum sde_mdss_features { SDE_FEATURE_VBIF_CLK_SPLIT, SDE_FEATURE_CTL_DONE, SDE_FEATURE_SYS_CACHE_NSE, + SDE_FEATURE_HW_FENCE_IPCC, SDE_FEATURE_MAX }; @@ -1752,6 +1756,7 @@ struct sde_perf_cfg { * @ts_prefill_rev prefill traffic shaper feature revision * @true_inline_rot_rev inline rotator feature revision * @dnsc_blur_rev downscale blur HW block version + * @hw_fence_rev hw fence feature revision * @mdss_count number of valid MDSS HW blocks * @mdss array of pointers to MDSS HW blocks * @mdss_hw_block_size max offset of MDSS_HW block (0 offset), used for debug @@ -1857,6 +1862,7 @@ struct sde_mdss_cfg { u32 ts_prefill_rev; u32 true_inline_rot_rev; u32 dnsc_blur_rev; + u32 hw_fence_rev; /* HW Blocks */ u32 mdss_count; diff --git a/msm/sde/sde_hw_ctl.c b/msm/sde/sde_hw_ctl.c index dab3f8f63f..b5a3317bc6 100644 --- a/msm/sde/sde_hw_ctl.c +++ b/msm/sde/sde_hw_ctl.c @@ -54,6 +54,12 @@ #define CTL_INTF_MASTER 0x134 #define CTL_UIDLE_ACTIVE 0x138 +#define CTL_HW_FENCE_CTRL 0x250 +#define CTL_FENCE_READY_SW_OVERRIDE 0x254 +#define CTL_INPUT_FENCE_ID 0x258 +#define CTL_OUTPUT_FENCE_CTRL 0x25C +#define CTL_OUTPUT_FENCE_ID 0x260 + #define CTL_MIXER_BORDER_OUT BIT(24) #define CTL_FLUSH_MASK_ROT BIT(27) #define CTL_FLUSH_MASK_CTL BIT(17) @@ -318,6 +324,47 @@ static inline bool _is_dspp_flush_pending(struct sde_hw_ctl *ctx) return false; } +static inline void sde_hw_ctl_update_input_fence(struct sde_hw_ctl *ctx, + u32 client_id, u32 signal_id) +{ + u32 val = (client_id << 16) | (0xFFFF & signal_id); + + SDE_REG_WRITE(&ctx->hw, CTL_INPUT_FENCE_ID, val); +} + +static inline void sde_hw_ctl_update_output_fence(struct sde_hw_ctl *ctx, + u32 client_id, u32 signal_id) +{ + u32 val = (client_id << 16) | (0xFFFF & signal_id); + + SDE_REG_WRITE(&ctx->hw, CTL_OUTPUT_FENCE_ID, val); +} + +static inline void sde_hw_ctl_trigger_output_fence(struct sde_hw_ctl *ctx, u32 trigger_sel) +{ + u32 val = ((trigger_sel & 0xF) << 4) | 0x1; + + SDE_REG_WRITE(&ctx->hw, CTL_OUTPUT_FENCE_CTRL, val); +} + +static inline void sde_hw_ctl_hw_fence_ctrl(struct sde_hw_ctl *ctx, bool sw_override_set, + bool sw_override_clear, u32 mode) +{ + u32 val; + + val = SDE_REG_READ(&ctx->hw, CTL_HW_FENCE_CTRL); + val |= (0x1 & mode) | (sw_override_set ? BIT(5) : 0) | (sw_override_clear ? BIT(4) : 0); + SDE_REG_WRITE(&ctx->hw, CTL_HW_FENCE_CTRL, val); +} + +static inline void sde_hw_ctl_trigger_sw_override(struct sde_hw_ctl *ctx) +{ + /* clear input fence before override */ + sde_hw_ctl_update_input_fence(ctx, 0, 0); + + SDE_REG_WRITE(&ctx->hw, CTL_FENCE_READY_SW_OVERRIDE, 0x1); +} + static inline int sde_hw_ctl_trigger_start(struct sde_hw_ctl *ctx) { if (!ctx) @@ -1366,9 +1413,17 @@ static void _setup_ctl_ops(struct sde_hw_ctl_ops *ops, sde_hw_ctl_update_bitmask_dspp_pavlut; } + if (cap & BIT(SDE_CTL_HW_FENCE)) { + ops->hw_fence_update_input_fence = sde_hw_ctl_update_input_fence; + ops->hw_fence_update_output_fence = sde_hw_ctl_update_output_fence; + ops->hw_fence_trigger_output_fence = sde_hw_ctl_trigger_output_fence; + ops->hw_fence_ctrl = sde_hw_ctl_hw_fence_ctrl; + ops->hw_fence_trigger_sw_override = sde_hw_ctl_trigger_sw_override; + } + if (cap & BIT(SDE_CTL_UIDLE)) ops->uidle_enable = sde_hw_ctl_uidle_enable; -}; +} struct sde_hw_blk_reg_map *sde_hw_ctl_init(enum sde_ctl idx, void __iomem *addr, diff --git a/msm/sde/sde_hw_ctl.h b/msm/sde/sde_hw_ctl.h index b842e80c3f..2f70354f3e 100644 --- a/msm/sde/sde_hw_ctl.h +++ b/msm/sde/sde_hw_ctl.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved. */ @@ -10,6 +11,7 @@ #include "sde_hw_util.h" #include "sde_hw_catalog.h" #include "sde_hw_sspp.h" +#include "sde_fence.h" #define INVALID_CTL_STATUS 0xfffff88e #define CTL_MAX_DSPP_COUNT (DSPP_MAX - DSPP_0) @@ -169,6 +171,41 @@ struct sde_ctl_flush_cfg { * Assumption is these functions will be called after clocks are enabled */ struct sde_hw_ctl_ops { + /** + * hw fence control + * @ctx : ctl path ctx pointer + */ + void (*hw_fence_ctrl)(struct sde_hw_ctl *ctx, bool sw_set, bool sw_clear, u32 mode); + + /** + * trigger hw fence fence-ready sw override + * @ctx : ctl path ctx pointer + */ + void (*hw_fence_trigger_sw_override)(struct sde_hw_ctl *ctx); + + /** + * configure output hw fence trigger + * @ctx : ctl path ctx pointer + * @trigger_sel : select upon which event the output trigger will happen + */ + void (*hw_fence_trigger_output_fence)(struct sde_hw_ctl *ctx, u32 trigger_sel); + + /** + * update output hw fence ipcc client_id and signal_id + * @ctx : ctl path ctx pointer + * @client_id : value to write to update the client_id + * @signal_id : value to write to update the signal_id + */ + void (*hw_fence_update_output_fence)(struct sde_hw_ctl *ctx, u32 client_id, u32 signal_id); + + /** + * update input hw fence ipcc client_id and signal_id + * @ctx : ctl path ctx pointer + * @client_id : value to write to update the client_id + * @signal_id : value to write to update the signal_id + */ + void (*hw_fence_update_input_fence)(struct sde_hw_ctl *ctx, u32 client_id, u32 signal_id); + /** * kickoff hw operation for Sw controlled interfaces * DSI cmd mode and WB interface are SW controlled @@ -499,6 +536,9 @@ struct sde_hw_ctl { const struct sde_lm_cfg *mixer_hw_caps; struct sde_ctl_flush_cfg flush; + /* hw fence */ + struct sde_hw_fence_data hwfence_data; + /* ops */ struct sde_hw_ctl_ops ops; }; diff --git a/msm/sde/sde_hw_top.c b/msm/sde/sde_hw_top.c index 38f9c598c9..b32fef0ac8 100644 --- a/msm/sde/sde_hw_top.c +++ b/msm/sde/sde_hw_top.c @@ -86,6 +86,36 @@ #define ROT_SID_ID_VAL 0x1c +/* HW Fences */ +#define MDP_CTL_HW_FENCE_CTRL 0x14000 +#define MDP_CTL_HW_FENCE_ID_START_ADDR 0x14004 +#define MDP_CTL_HW_FENCE_ID_STATUS 0x14008 +#define MDP_CTL_HW_FENCE_ID_TIMESTAMP_CTRL 0x1400c +#define MDP_CTL_HW_FENCE_INPUT_START_TIMESTAMP0 0x14010 +#define MDP_CTL_HW_FENCE_INPUT_START_TIMESTAMP1 0x14014 +#define MDP_CTL_HW_FENCE_INPUT_END_TIMESTAMP0 0x14018 +#define MDP_CTL_HW_FENCE_INPUT_END_TIMESTAMP1 0x1401c +#define MDP_CTL_HW_FENCE_QOS 0x14020 +#define MDP_CTL_HW_FENCE_IDn_ISR 0x14050 +#define MDP_CTL_HW_FENCE_IDm_ADDR 0x14054 +#define MDP_CTL_HW_FENCE_IDm_DATA 0x14058 +#define MDP_CTL_HW_FENCE_IDm_MASK 0x1405c +#define MDP_CTL_HW_FENCE_IDm_ATTR 0x14060 +#define HW_FENCE_IPCC_PROTOCOL_ID_COMPUTE_L1 0x2 +#define HW_FENCE_IPCC_PROTOCOLp_CLIENTc_SEND(ba, p, c) ((ba+0xc) + (0x40000*p) + (0x1000*c)) +#define HW_FENCE_IPCC_PROTOCOLp_CLIENTc_RECV_ID(ba, p, c) ((ba+0x10) + (0x40000*p) + (0x1000*c)) +#define HW_FENCE_IPCC_SEND_BA 0x40000c +#define HW_FENCE_IPCC_RECV_ID_BA 0x400010 +#define MDP_CTL_HW_FENCE_ID_OFFSET_n(base, n) (base + (0x14*n)) +#define MDP_CTL_HW_FENCE_ID_OFFSET_m(base, m) (base + (0x14*m)) +#define MDP_CTL_FENCE_ATTRS(devicetype, size, resp_req) \ + (((resp_req & 0x1) << 16) | ((size & 0x7) << 4) | (devicetype & 0xf)) +#define MDP_CTL_FENCE_ISR_OP_CODE(opcode, op0, op1, op2) \ + (((op2 & 0xff) << 24) | ((op1 & 0xff) << 16) | ((op0 & 0xff) << 8) | (opcode & 0xff)) + +#define HW_FENCE_DPU_INPUT_FENCE_START_N 0 +#define HW_FENCE_DPU_OUTPUT_FENCE_START_N 4 + static void sde_hw_setup_split_pipe(struct sde_hw_mdp *mdp, struct split_pipe_cfg *cfg) { @@ -565,8 +595,103 @@ static u32 sde_hw_get_autorefresh_status(struct sde_hw_mdp *mdp, u32 intf_idx) return autorefresh_status; } -static void _setup_mdp_ops(struct sde_hw_mdp_ops *ops, - unsigned long cap) +static void sde_hw_setup_hw_fences_config(struct sde_hw_mdp *mdp) +{ + u32 val, offset; + struct sde_hw_blk_reg_map c; + + if (!mdp) { + SDE_ERROR("invalid mdp, won't configure hw-fences\n"); + return; + } + + /* start from the base-address of the mdss */ + c = mdp->hw; + c.blk_off = 0x0; + + /* select ipcc protocol id for dpu */ + SDE_REG_WRITE(&c, MDP_CTL_HW_FENCE_CTRL, HW_FENCE_IPCC_PROTOCOL_ID_COMPUTE_L1); + + /* configure the start of the FENCE_IDn_ISR ops for input and output fence isr's */ + val = (HW_FENCE_DPU_OUTPUT_FENCE_START_N << 16) | (HW_FENCE_DPU_INPUT_FENCE_START_N & 0xFF); + SDE_REG_WRITE(&c, MDP_CTL_HW_FENCE_ID_START_ADDR, val); + + /* setup input fence isr */ + + /* configure the attribs for the isr read_reg op */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_ADDR, 0); + val = HW_FENCE_IPCC_PROTOCOLp_CLIENTc_RECV_ID(HW_FENCE_IPCC_RECV_ID_BA, + HW_FENCE_IPCC_PROTOCOL_ID_COMPUTE_L1, 25); + SDE_REG_WRITE(&c, offset, val); + + offset = MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_ATTR, 0); + val = MDP_CTL_FENCE_ATTRS(0x1, 0x2, 0x1); + SDE_REG_WRITE(&c, offset, val); + + offset = MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_MASK, 0); + SDE_REG_WRITE(&c, offset, 0xFFFFFFFF); + + /* configure the attribs for the write if eq data */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_DATA, 1); + SDE_REG_WRITE(&c, offset, 0x1); + + /* program input-fence isr ops */ + + /* set read_reg op */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_n(MDP_CTL_HW_FENCE_IDn_ISR, + HW_FENCE_DPU_INPUT_FENCE_START_N); + val = MDP_CTL_FENCE_ISR_OP_CODE(0x0, 0x0, 0x0, 0x0); + SDE_REG_WRITE(&c, offset, val); + + /* set write if eq op for flush ready */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_n(MDP_CTL_HW_FENCE_IDn_ISR, + (HW_FENCE_DPU_INPUT_FENCE_START_N + 1)); + val = MDP_CTL_FENCE_ISR_OP_CODE(0x7, 0x0, 0x1, 0x0); + SDE_REG_WRITE(&c, offset, val); + + /* set exit op */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_n(MDP_CTL_HW_FENCE_IDn_ISR, + (HW_FENCE_DPU_INPUT_FENCE_START_N + 2)); + val = MDP_CTL_FENCE_ISR_OP_CODE(0xf, 0x0, 0x0, 0x0); + SDE_REG_WRITE(&c, offset, val); + + /*setup output fence isr */ + + /* configure the attribs for the isr load_data op */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_ADDR, 4); + val = HW_FENCE_IPCC_PROTOCOLp_CLIENTc_SEND(HW_FENCE_IPCC_SEND_BA, + HW_FENCE_IPCC_PROTOCOL_ID_COMPUTE_L1, 25); + SDE_REG_WRITE(&c, offset, val); + + offset = MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_ATTR, 4); + val = MDP_CTL_FENCE_ATTRS(0x1, 0x2, 0x0); + SDE_REG_WRITE(&c, offset, val); + + offset = MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_MASK, 4); + SDE_REG_WRITE(&c, offset, 0xFFFFFFFF); + + /* program output-fence isr ops */ + + /* set load_data op*/ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_n(MDP_CTL_HW_FENCE_IDn_ISR, + HW_FENCE_DPU_OUTPUT_FENCE_START_N); + val = MDP_CTL_FENCE_ISR_OP_CODE(0x6, 0x0, 0x4, 0x0); + SDE_REG_WRITE(&c, offset, val); + + /* set write_reg op */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_n(MDP_CTL_HW_FENCE_IDn_ISR, + (HW_FENCE_DPU_OUTPUT_FENCE_START_N + 1)); + val = MDP_CTL_FENCE_ISR_OP_CODE(0x2, 0x4, 0x0, 0x0); + SDE_REG_WRITE(&c, offset, val); + + /* set exit op */ + offset = MDP_CTL_HW_FENCE_ID_OFFSET_n(MDP_CTL_HW_FENCE_IDn_ISR, + (HW_FENCE_DPU_OUTPUT_FENCE_START_N + 2)); + val = MDP_CTL_FENCE_ISR_OP_CODE(0xf, 0x0, 0x0, 0x0); + SDE_REG_WRITE(&c, offset, val); +} + +static void _setup_mdp_ops(struct sde_hw_mdp_ops *ops, unsigned long cap, u32 hw_fence_rev) { ops->setup_split_pipe = sde_hw_setup_split_pipe; ops->setup_pp_split = sde_hw_setup_pp_split; @@ -586,6 +711,9 @@ static void _setup_mdp_ops(struct sde_hw_mdp_ops *ops, cap & BIT(SDE_MDP_DHDR_MEMPOOL)) ops->set_hdr_plus_metadata = sde_hw_set_hdr_plus_metadata; ops->get_autorefresh_status = sde_hw_get_autorefresh_status; + + if (hw_fence_rev) + ops->setup_hw_fences = sde_hw_setup_hw_fences_config; } static const struct sde_mdp_cfg *_top_offset(enum sde_mdp mdp, @@ -637,7 +765,7 @@ struct sde_hw_mdp *sde_hw_mdptop_init(enum sde_mdp idx, */ mdp->idx = idx; mdp->caps = cfg; - _setup_mdp_ops(&mdp->ops, mdp->caps->features); + _setup_mdp_ops(&mdp->ops, mdp->caps->features, m->hw_fence_rev); sde_dbg_reg_register_dump_range(SDE_DBG_NAME, "mdss_hw", 0, m->mdss_hw_block_size, 0); @@ -652,6 +780,10 @@ struct sde_hw_mdp *sde_hw_mdptop_init(enum sde_mdp idx, sde_dbg_reg_register_dump_range(SDE_DBG_NAME, name, mdp->hw.blk_off + MDP_SSPP_TOP2, mdp->hw.blk_off + mdp->hw.length, mdp->hw.xin_id); + + /* do not use blk_off, following offsets start from mdp_phys */ + sde_dbg_reg_register_dump_range(SDE_DBG_NAME, "hw_fence", MDP_CTL_HW_FENCE_CTRL, + MDP_CTL_HW_FENCE_ID_OFFSET_m(MDP_CTL_HW_FENCE_IDm_ATTR, 5), mdp->hw.xin_id); } else { sde_dbg_reg_register_dump_range(SDE_DBG_NAME, cfg->name, mdp->hw.blk_off, mdp->hw.blk_off + mdp->hw.length, diff --git a/msm/sde/sde_hw_top.h b/msm/sde/sde_hw_top.h index c74f764949..c7464eec3c 100644 --- a/msm/sde/sde_hw_top.h +++ b/msm/sde/sde_hw_top.h @@ -202,6 +202,13 @@ struct sde_hw_mdp_ops { */ u32 (*get_autorefresh_status)(struct sde_hw_mdp *mdp, u32 intf_idx); + + /** + * setup_hw_fences - configure hw fences top registers + * @mdp: mdp top context driver + */ + void (*setup_hw_fences)(struct sde_hw_mdp *mdp); + }; struct sde_hw_mdp { diff --git a/msm/sde/sde_kms.c b/msm/sde/sde_kms.c index b4c68c48cb..8134a0421c 100644 --- a/msm/sde/sde_kms.c +++ b/msm/sde/sde_kms.c @@ -54,6 +54,7 @@ #include "sde_reg_dma.h" #include "sde_connector.h" #include "sde_vm.h" +#include "sde_fence.h" #include #include @@ -4251,6 +4252,15 @@ static void sde_kms_init_rot_sid_hw(struct sde_kms *sde_kms) sde_hw_set_rotator_sid(sde_kms->hw_sid); } +static void sde_kms_init_hw_fences(struct sde_kms *sde_kms) +{ + if (!sde_kms || !sde_kms->hw_mdp) + return; + + if (sde_kms->hw_mdp->ops.setup_hw_fences) + sde_kms->hw_mdp->ops.setup_hw_fences(sde_kms->hw_mdp); +} + static void sde_kms_init_shared_hw(struct sde_kms *sde_kms) { if (!sde_kms || !sde_kms->hw_mdp || !sde_kms->catalog) @@ -4410,10 +4420,11 @@ static void sde_kms_handle_power_event(u32 event_type, void *usr) sde_kms->first_kickoff = true; /** - * Rotator sid needs to be programmed since uefi doesn't - * configure it during continuous splash + * Rotator sid and hw fences need to be programmed since uefi doesn't + * configure them during continuous splash */ sde_kms_init_rot_sid_hw(sde_kms); + sde_kms_init_hw_fences(sde_kms); if (sde_kms->splash_data.num_splash_displays || sde_in_trusted_vm(sde_kms)) return; diff --git a/msm/sde/sde_plane.c b/msm/sde/sde_plane.c index 1dcd618f5e..c4a7750d58 100644 --- a/msm/sde/sde_plane.c +++ b/msm/sde/sde_plane.c @@ -78,55 +78,6 @@ enum sde_plane_qos { SDE_PLANE_QOS_PANIC_CTRL = BIT(2), }; -struct sde_plane { - struct drm_plane base; - - struct mutex lock; - - enum sde_sspp pipe; - uint64_t features; /* capabilities from catalog */ - uint32_t perf_features; /* perf capabilities from catalog */ - uint32_t nformats; - uint32_t formats[64]; - - struct sde_hw_pipe *pipe_hw; - struct sde_hw_pipe_cfg pipe_cfg; - struct sde_hw_sharp_cfg sharp_cfg; - struct sde_hw_pipe_qos_cfg pipe_qos_cfg; - uint32_t color_fill; - bool is_error; - bool is_rt_pipe; - enum sde_wb_usage_type wb_usage_type; - bool is_virtual; - struct list_head mplane_list; - struct sde_mdss_cfg *catalog; - bool revalidate; - bool xin_halt_forced_clk; - - struct sde_csc_cfg csc_cfg; - struct sde_csc_cfg *csc_usr_ptr; - struct sde_csc_cfg *csc_ptr; - - uint32_t cached_lut_flag; - struct sde_hw_scaler3_cfg scaler3_cfg; - struct sde_hw_pixel_ext pixel_ext; - - const struct sde_sspp_sub_blks *pipe_sblk; - - char pipe_name[SDE_NAME_SIZE]; - - struct msm_property_info property_info; - struct msm_property_data property_data[PLANE_PROP_COUNT]; - struct drm_property_blob *blob_info; - struct drm_property_blob *blob_rot_caps; - - /* debugfs related stuff */ - struct dentry *debugfs_root; - bool debugfs_default_scale; -}; - -#define to_sde_plane(x) container_of(x, struct sde_plane, base) - static int plane_prop_array[PLANE_PROP_COUNT] = {SDE_PLANE_DIRTY_ALL}; static struct sde_kms *_sde_plane_get_kms(struct drm_plane *plane) diff --git a/msm/sde/sde_plane.h b/msm/sde/sde_plane.h index 3d518246f7..7eab4726a8 100644 --- a/msm/sde/sde_plane.h +++ b/msm/sde/sde_plane.h @@ -50,6 +50,55 @@ SDE_PLANE_DIRTY_FP16_UNMULT) #define SDE_PLANE_DIRTY_ALL (0xFFFFFFFF & ~(SDE_PLANE_DIRTY_CP)) +struct sde_plane { + struct drm_plane base; + + struct mutex lock; + + enum sde_sspp pipe; + uint64_t features; /* capabilities from catalog */ + uint32_t perf_features; /* perf capabilities from catalog */ + uint32_t nformats; + uint32_t formats[64]; + + struct sde_hw_pipe *pipe_hw; + struct sde_hw_pipe_cfg pipe_cfg; + struct sde_hw_sharp_cfg sharp_cfg; + struct sde_hw_pipe_qos_cfg pipe_qos_cfg; + uint32_t color_fill; + bool is_error; + bool is_rt_pipe; + enum sde_wb_usage_type wb_usage_type; + bool is_virtual; + struct list_head mplane_list; + struct sde_mdss_cfg *catalog; + bool revalidate; + bool xin_halt_forced_clk; + + struct sde_csc_cfg csc_cfg; + struct sde_csc_cfg *csc_usr_ptr; + struct sde_csc_cfg *csc_ptr; + + uint32_t cached_lut_flag; + struct sde_hw_scaler3_cfg scaler3_cfg; + struct sde_hw_pixel_ext pixel_ext; + + const struct sde_sspp_sub_blks *pipe_sblk; + + char pipe_name[SDE_NAME_SIZE]; + + struct msm_property_info property_info; + struct msm_property_data property_data[PLANE_PROP_COUNT]; + struct drm_property_blob *blob_info; + struct drm_property_blob *blob_rot_caps; + + /* debugfs related stuff */ + struct dentry *debugfs_root; + bool debugfs_default_scale; +}; + +#define to_sde_plane(x) container_of(x, struct sde_plane, base) + /** * enum sde_layout * Describes SSPP to LM staging layout when using more than 1 pair of LMs diff --git a/msm/sde/sde_rm.c b/msm/sde/sde_rm.c index 588379c7d0..0b98f88d06 100644 --- a/msm/sde/sde_rm.c +++ b/msm/sde/sde_rm.c @@ -562,6 +562,18 @@ static void _sde_rm_hw_destroy(enum sde_hw_blk_type type, struct sde_hw_blk_reg_ } } +static void _deinit_hw_fences(struct sde_rm *rm) +{ + struct sde_rm_hw_iter iter; + + sde_rm_init_hw_iter(&iter, 0, SDE_HW_BLK_CTL); + while (_sde_rm_get_hw_locked(rm, &iter)) { + struct sde_hw_ctl *ctl = to_sde_hw_ctl(iter.blk->hw); + + sde_hw_fence_deinit(ctl); + } +} + int sde_rm_destroy(struct sde_rm *rm) { @@ -574,6 +586,8 @@ int sde_rm_destroy(struct sde_rm *rm) return -EINVAL; } + _deinit_hw_fences(rm); + list_for_each_entry_safe(rsvp_cur, rsvp_nxt, &rm->rsvps, list) { list_del(&rsvp_cur->list); kfree(rsvp_cur); @@ -693,6 +707,29 @@ static int _sde_rm_hw_blk_create( return 0; } +static int _init_hw_fences(struct sde_rm *rm, bool use_ipcc) +{ + struct sde_rm_hw_iter iter; + int ret = 0; + + sde_rm_init_hw_iter(&iter, 0, SDE_HW_BLK_CTL); + while (_sde_rm_get_hw_locked(rm, &iter)) { + struct sde_hw_ctl *ctl = to_sde_hw_ctl(iter.blk->hw); + + if (sde_hw_fence_init(ctl, use_ipcc)) { + pr_err("failed to init hw_fence idx:%d\n", ctl->idx); + ret = -EINVAL; + break; + } + SDE_DEBUG("init hw-fence for ctl %d", iter.blk->id); + } + + if (ret) + _deinit_hw_fences(rm); + + return ret; +} + static int _sde_rm_hw_blk_create_new(struct sde_rm *rm, struct sde_mdss_cfg *cat, void __iomem *mmio) @@ -778,6 +815,13 @@ static int _sde_rm_hw_blk_create_new(struct sde_rm *rm, } } + if (cat->hw_fence_rev) { + if (_init_hw_fences(rm, test_bit(SDE_FEATURE_HW_FENCE_IPCC, cat->features))) { + SDE_INFO("failed to init hw-fences, disabling hw-fences\n"); + cat->hw_fence_rev = 0; + } + } + for (i = 0; i < cat->cdm_count; i++) { rc = _sde_rm_hw_blk_create(rm, cat, mmio, SDE_HW_BLK_CDM, cat->cdm[i].id, &cat->cdm[i]);