diff --git a/asoc/codecs/swr-haptics.c b/asoc/codecs/swr-haptics.c
index 7452819a23..34dd53e44b 100644
--- a/asoc/codecs/swr-haptics.c
+++ b/asoc/codecs/swr-haptics.c
@@ -313,6 +313,7 @@ static int hap_enable_swr_dac_port(struct snd_soc_dapm_widget *w,
 		if (rc < 0) {
 			dev_err_ratelimited(swr_hap->dev, "%s: Enable hpwr_vreg failed, rc=%d\n",
 					__func__, rc);
+			swr_device_wakeup_unvote(swr_hap->swr_slave);
 			return rc;
 		}
 
@@ -327,16 +328,20 @@ static int hap_enable_swr_dac_port(struct snd_soc_dapm_widget *w,
 			swr_slvdev_datapath_control(swr_hap->swr_slave,
 					swr_hap->swr_slave->dev_num, false);
 			swr_hap_disable_hpwr_vreg(swr_hap);
+			swr_device_wakeup_unvote(swr_hap->swr_slave);
 			return rc;
 		}
+		swr_device_wakeup_unvote(swr_hap->swr_slave);
 		break;
 	case SND_SOC_DAPM_PRE_PMD:
+		swr_device_wakeup_vote(swr_hap->swr_slave);
 		/* stop SWR play */
 		val = SWR_PLAY_SRC_VAL_SWR;
 		rc = regmap_write(swr_hap->regmap, SWR_PLAY_REG, val);
 		if (rc) {
 			dev_err_ratelimited(swr_hap->dev, "%s: Enable SWR_PLAY failed, rc=%d\n",
 					__func__, rc);
+			swr_device_wakeup_unvote(swr_hap->swr_slave);
 			return rc;
 		}
 
@@ -344,6 +349,7 @@ static int hap_enable_swr_dac_port(struct snd_soc_dapm_widget *w,
 		if (rc < 0) {
 			dev_err_ratelimited(swr_hap->dev, "%s: Disable hpwr_vreg failed, rc=%d\n",
 					__func__, rc);
+			swr_device_wakeup_unvote(swr_hap->swr_slave);
 			return rc;
 		}
 		break;