diff --git a/dsp/cdsp-loader.c b/dsp/cdsp-loader.c
index d3b1dec433..e63718d27e 100644
--- a/dsp/cdsp-loader.c
+++ b/dsp/cdsp-loader.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2012-2014, 2017-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/init.h>
@@ -156,6 +156,7 @@ static void cdsp_loader_unload(struct platform_device *pdev)
 		dev_dbg(&pdev->dev, "%s: calling subsystem_put\n", __func__);
 		rproc_shutdown(priv->pil_h);
 		priv->pil_h = NULL;
+		cdsp_state = CDSP_SUBSYS_DOWN;
 	}
 }
 
@@ -229,6 +230,7 @@ static int cdsp_loader_remove(struct platform_device *pdev)
 	if (priv->pil_h) {
 		rproc_shutdown(priv->pil_h);
 		priv->pil_h = NULL;
+		cdsp_state = CDSP_SUBSYS_DOWN;
 	}
 
 	if (priv->boot_cdsp_obj) {