diff --git a/modules/module-pw-audio-softdsp-endpoint.c b/modules/module-pw-audio-softdsp-endpoint.c
index 9dac1dec230e6f4f4d533a3671c2bb2a047c896f..94652e3d1f41c309600c1f08aee8145b38279a77 100644
--- a/modules/module-pw-audio-softdsp-endpoint.c
+++ b/modules/module-pw-audio-softdsp-endpoint.c
@@ -633,50 +633,9 @@ endpoint_factory (WpFactory * factory, GType type, GVariant * properties)
   return ep;
 }
 
-static void
-global_endpoint_notify_control_value (WpEndpoint * ep, guint control_id,
-    WpCore * core)
-{
-  WpPwAudioSoftdspEndpoint *sdspep = WP_PW_AUDIO_SOFTDSP_ENDPOINT (ep);
-  g_autoptr (GPtrArray) a = NULL;
-  int i;
-
-  /* when an endpoint becomes "selected", unselect
-   * all other endpoints of the same media class */
-  if (control_id == CONTROL_SELECTED && sdspep->selected) {
-    g_debug ("selected: %p", ep);
-    a = wp_endpoint_find (core, wp_endpoint_get_media_class (ep));
-
-    for (i = 0; i < a->len; i++) {
-      WpEndpoint *other = g_ptr_array_index (a, i);
-      if (!WP_PW_IS_AUDIO_SOFTDSP_ENDPOINT (ep)
-          || other == ep
-          || !WP_PW_AUDIO_SOFTDSP_ENDPOINT (other)->selected)
-        continue;
-
-      g_debug ("unselecting %p", other);
-      WP_PW_AUDIO_SOFTDSP_ENDPOINT (other)->selected = FALSE;
-      wp_endpoint_notify_control_value (other, CONTROL_SELECTED);
-    }
-  }
-}
-
-static void
-global_endpoint_added (WpCore *core, GQuark key, WpEndpoint *ep, gpointer data)
-{
-  if (WP_PW_IS_AUDIO_SOFTDSP_ENDPOINT (ep)) {
-    g_debug ("connecting to notify-control-value for %p", ep);
-    g_signal_connect (ep, "notify-control-value",
-        (GCallback) global_endpoint_notify_control_value, core);
-  }
-}
-
 void
 wireplumber__module_init (WpModule * module, WpCore * core, GVariant * args)
 {
   /* Register the softdsp endpoint */
   wp_factory_new (core, "pw-audio-softdsp-endpoint", endpoint_factory);
-
-  g_signal_connect (core, "global-added::endpoint",
-      (GCallback) global_endpoint_added, NULL);
 }
diff --git a/modules/module-simple-policy.c b/modules/module-simple-policy.c
index ff3f6132fbfa691a116944a14bf5786919b34292..79807bd4a490e1c166597cbcfe840ad08243bed8 100644
--- a/modules/module-simple-policy.c
+++ b/modules/module-simple-policy.c
@@ -8,9 +8,16 @@
 
 #include <wp/wp.h>
 
+enum {
+  DIRECTION_SINK = 0,
+  DIRECTION_SOURCE
+};
+
 struct _WpSimplePolicy
 {
   WpPolicy parent;
+  WpEndpoint *selected[2];
+  guint32 selected_ctl_id[2];
 };
 
 G_DECLARE_FINAL_TYPE (WpSimplePolicy, simple_policy, WP, SIMPLE_POLICY, WpPolicy)
@@ -21,6 +28,174 @@ simple_policy_init (WpSimplePolicy *self)
 {
 }
 
+static void
+endpoint_notify_control_value (WpEndpoint * ep, guint control_id,
+    WpSimplePolicy *self)
+{
+  g_autoptr (GVariant) v = NULL;
+  const gchar *media_class;
+  guint32 tmp_id;
+  gint direction;
+  WpEndpoint *old_selected;
+  guint32 old_ctl_id;
+
+  /* when an endpoint becomes "selected", unselect
+   * all other endpoints of the same media class */
+
+  /* the already "selected" endpoint cannot become even more "selected",
+   * so we skip it */
+  if (ep == self->selected[DIRECTION_SINK] ||
+      ep == self->selected[DIRECTION_SOURCE])
+    return;
+
+  /* verify that the changed control is the "selected" */
+  tmp_id = wp_endpoint_find_control (ep, WP_STREAM_ID_NONE, "selected");
+  if (control_id != tmp_id)
+    return;
+
+  /* verify it changed to TRUE */
+  v = wp_endpoint_get_control_value (ep, control_id);
+  if (g_variant_get_boolean (v) == FALSE)
+    return;
+
+  media_class = wp_endpoint_get_media_class (ep);
+  direction = strstr(media_class, "Sink") ? DIRECTION_SINK : DIRECTION_SOURCE;
+
+  g_debug ("selected %s: %p, unselecting %p",
+      (direction == DIRECTION_SINK) ? "sink" : "source",
+      ep, self->selected);
+
+  old_selected = self->selected[direction];
+  old_ctl_id = self->selected_ctl_id[direction];
+  self->selected[direction] = ep;
+  self->selected_ctl_id[direction] = control_id;
+
+  /* update the control value */
+  wp_endpoint_set_control_value (old_selected, old_ctl_id,
+      g_variant_new_boolean (FALSE));
+
+  /* notify policy watchers that things have changed */
+  wp_policy_notify_changed (WP_POLICY (self));
+}
+
+static void
+select_endpoint (WpSimplePolicy *self, gint direction, WpEndpoint *ep,
+    guint32 control_id)
+{
+  g_debug ("selecting %s %p (%s)",
+      (direction == DIRECTION_SINK) ? "sink" : "source",
+      ep, wp_endpoint_get_name (ep));
+
+  self->selected[direction] = ep;
+  self->selected_ctl_id[direction] = control_id;
+
+  /* update the control value */
+  wp_endpoint_set_control_value (ep, control_id,
+      g_variant_new_boolean (TRUE));
+
+  /* notify policy watchers that things have changed */
+  wp_policy_notify_changed (WP_POLICY (self));
+}
+
+static gboolean
+select_new_endpoint (WpSimplePolicy *self)
+{
+  g_autoptr (WpCore) core = NULL;
+  g_autoptr (GPtrArray) ptr_array = NULL;
+  const gchar *media_class = NULL;
+  WpEndpoint *other;
+  guint32 control_id;
+  gint direction, i;
+
+  if (!self->selected[DIRECTION_SINK]) {
+    direction = DIRECTION_SINK;
+    media_class = "Audio/Sink";
+  } else if (!self->selected[DIRECTION_SOURCE]) {
+    direction = DIRECTION_SOURCE;
+    media_class = "Audio/Source";
+  } else
+    return G_SOURCE_REMOVE;
+
+  core = wp_policy_get_core (WP_POLICY (self));
+
+  /* Get all the endpoints with the same media class */
+  ptr_array = wp_endpoint_find (core, media_class);
+
+  /* select the first available that has the "selected" control */
+  for (i = 0; i < (ptr_array ? ptr_array->len : 0); i++) {
+    other = g_ptr_array_index (ptr_array, i);
+
+    control_id =
+        wp_endpoint_find_control (other, WP_STREAM_ID_NONE, "selected");
+    if (control_id == WP_CONTROL_ID_NONE)
+      continue;
+
+    select_endpoint (self, direction, other, control_id);
+    break;
+  }
+
+  return G_SOURCE_REMOVE;
+}
+
+static void
+simple_policy_endpoint_added (WpPolicy *policy, WpEndpoint *ep)
+{
+  WpSimplePolicy *self = WP_SIMPLE_POLICY (policy);
+  const gchar *media_class = wp_endpoint_get_media_class (ep);
+  guint32 control_id;
+  gint direction;
+
+  /* we only care about audio device endpoints here */
+  if (!g_str_has_prefix (media_class, "Audio/"))
+    return;
+
+  /* verify it has the "selected" control available */
+  control_id = wp_endpoint_find_control (ep, WP_STREAM_ID_NONE, "selected");
+  if (control_id == WP_CONTROL_ID_NONE)
+    return;
+
+  /* connect to control value changes */
+  g_debug ("connecting to notify-control-value for %p", ep);
+  g_signal_connect_object (ep, "notify-control-value",
+      (GCallback) endpoint_notify_control_value, self, 0);
+
+  /* select this endpoint if no other is already selected */
+  direction = strstr (media_class, "Sink") ? DIRECTION_SINK : DIRECTION_SOURCE;
+
+  if (!self->selected[direction]) {
+    select_endpoint (self, direction, ep, control_id);
+  }
+}
+
+static void
+simple_policy_endpoint_removed (WpPolicy *policy, WpEndpoint *ep)
+{
+  WpSimplePolicy *self = WP_SIMPLE_POLICY (policy);
+  gint direction;
+
+  /* if the "selected" endpoint was removed, select another one */
+
+  if (ep == self->selected[DIRECTION_SINK])
+    direction = DIRECTION_SINK;
+  else if (ep == self->selected[DIRECTION_SOURCE])
+    direction = DIRECTION_SOURCE;
+  else
+    return;
+
+  self->selected[direction] = NULL;
+  self->selected_ctl_id[direction] = WP_CONTROL_ID_NONE;
+
+  /* do the rest later, to possibly let other endpoints be removed as well
+   * before we try to pick a new one, to avoid multiple notifications
+   * and to also avoid crashing when the pipewire remote disconnects
+   * (in which case all endpoints are getting removed and changing controls
+   * triggers a crash in remote-endpoint, trying to export a value change
+   * without a valid connection)
+   */
+  g_idle_add_full (G_PRIORITY_HIGH, (GSourceFunc) select_new_endpoint,
+      g_object_ref (self), g_object_unref);
+}
+
 static gboolean
 simple_policy_handle_endpoint (WpPolicy *policy, WpEndpoint *ep)
 {
@@ -120,6 +295,8 @@ simple_policy_class_init (WpSimplePolicyClass *klass)
 {
   WpPolicyClass *policy_class = (WpPolicyClass *) klass;
 
+  policy_class->endpoint_added = simple_policy_endpoint_added;
+  policy_class->endpoint_removed = simple_policy_endpoint_removed;
   policy_class->handle_endpoint = simple_policy_handle_endpoint;
   policy_class->find_endpoint = simple_policy_find_endpoint;
 }