--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -992,8 +992,7 @@ static int ieee80211_start_ap(struct wip
 		      BSS_CHANGED_P2P_PS |
 		      BSS_CHANGED_TXPOWER |
 		      BSS_CHANGED_TWT |
-		      BSS_CHANGED_HE_OBSS_PD |
-		      BSS_CHANGED_HE_BSS_COLOR;
+		      BSS_CHANGED_HE_OBSS_PD;
 	int err;
 	int prev_beacon_int;
 
@@ -1019,6 +1018,7 @@ static int ieee80211_start_ap(struct wip
 		sdata->vif.bss_conf.frame_time_rts_th =
 			le32_get_bits(params->he_oper->he_oper_params,
 			      IEEE80211_HE_OPERATION_RTS_THRESHOLD_MASK);
+		changed |= BSS_CHANGED_HE_BSS_COLOR;
 	}
 
 	if (params->he_cap) {