soc: swr-mstr-ctrl: remove unnecessary check in swr_remove_from_group

swrm->num_cfg_devs is never changed and it would cause
swr_remove_from_group doesn't work. Remove swrm->num_cfg_devs
to fix this issue.

Change-Id: Id3e39e3e95dfa32e1d81234b1669af3d2b6737bf
Signed-off-by: Meng Wang <[email protected]>

soc/swr-mstr-ctrl.c

@@ -1164,8 +1164,7 @@ static bool swrm_remove_from_group(struct swr_master *master)
 		goto end;
 
 	mutex_lock(&swrm->mlock);
-	if ((swrm->num_rx_chs > 1) &&
-	    (swrm->num_rx_chs == swrm->num_cfg_devs)) {
+	if (swrm->num_rx_chs > 1) {
 		list_for_each_entry(swr_dev, &master->devices,
 				dev_list) {
 			swr_dev->group_id = SWR_GROUP_NONE;