diff --git a/compositor/compositor.c b/compositor/compositor.c
index d0cdcf24..7b1adec9 100644
--- a/compositor/compositor.c
+++ b/compositor/compositor.c
@@ -129,6 +129,8 @@ wlsc_surface_create(struct wlsc_compositor *compositor,
 	if (surface == NULL)
 		return NULL;
 
+	wl_list_init(&surface->surface.destroy_listener_list);
+
 	glGenTextures(1, &surface->texture);
 	glBindTexture(GL_TEXTURE_2D, surface->texture);
 	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
@@ -159,13 +161,17 @@ destroy_surface(struct wl_resource *resource, struct wl_client *client)
 	struct wlsc_surface *surface =
 		container_of(resource, struct wlsc_surface, surface.resource);
 	struct wlsc_compositor *compositor = surface->compositor;
-	struct wlsc_listener *l;
+	struct wl_listener *l, *next;
 
 	wl_list_remove(&surface->link);
 	glDeleteTextures(1, &surface->texture);
 
 	wl_list_for_each(l, &compositor->surface_destroy_listener_list, link)
-		l->func(l, surface);
+		l->func(l, &surface->surface);
+
+	wl_list_for_each_safe(l, next,
+			      &surface->surface.destroy_listener_list, link)
+		l->func(l, &surface->surface);
 
 	free(surface);
 
@@ -574,7 +580,7 @@ get_time(void)
 
 struct wlsc_drag {
 	struct wl_drag drag;
-	struct wlsc_listener listener;
+	struct wl_listener listener;
 };
 
 static void
@@ -596,14 +602,14 @@ destroy_drag(struct wl_resource *resource, struct wl_client *client)
 const static struct wl_drag_interface drag_interface;
 
 static void
-drag_handle_surface_destroy(struct wlsc_listener *listener,
-			    struct wlsc_surface *surface)
+drag_handle_surface_destroy(struct wl_listener *listener,
+			    struct wl_surface *surface)
 {
 	struct wlsc_drag *drag =
 		container_of(listener, struct wlsc_drag, listener);
 	uint32_t time = get_time();
 
-	if (drag->drag.pointer_focus == &surface->surface)
+	if (drag->drag.pointer_focus == surface)
 		wl_drag_set_pointer_focus(&drag->drag, NULL, time, 0, 0, 0, 0);
 }
 
@@ -984,24 +990,6 @@ const static struct wl_input_device_interface input_device_interface = {
 	input_device_attach,
 };
 
-static void
-handle_surface_destroy(struct wlsc_listener *listener,
-		       struct wlsc_surface *surface)
-{
-	struct wlsc_input_device *device =
-		container_of(listener, struct wlsc_input_device, listener);
-	uint32_t time = get_time();
-
-	if (device->input_device.keyboard_focus == &surface->surface)
-		wl_input_device_set_keyboard_focus(&device->input_device, NULL, time);
-	if (device->input_device.pointer_focus == &surface->surface)
-		wl_input_device_set_pointer_focus(&device->input_device, NULL, time,
-						    0, 0, 0, 0);
-	if (device->input_device.pointer_focus == &surface->surface ||
-	    (device->input_device.pointer_focus == &wl_grab_surface &&
-	     device->grab_surface == surface))
-		wlsc_input_device_end_grab(device, time);
-}
 
 static void
 wl_drag_set_pointer_focus(struct wl_drag *drag,
@@ -1165,6 +1153,32 @@ static const struct wl_drag_interface drag_interface = {
 	drag_cancel,
 };
 
+static void
+lose_pointer_focus(struct wl_listener *listener,
+		   struct wl_surface *surface)
+{
+	uint32_t time = get_time();
+	struct wlsc_input_device *device =
+		container_of(listener, struct wlsc_input_device,
+			     input_device.pointer_focus_listener);
+
+	wl_input_device_set_pointer_focus(&device->input_device,
+					  NULL, time, 0, 0, 0, 0);
+	wlsc_input_device_end_grab(device, time);
+}
+
+static void
+lose_keyboard_focus(struct wl_listener *listener,
+		    struct wl_surface *surface)
+{
+	uint32_t time = get_time();
+	struct wlsc_input_device *device =
+		container_of(listener, struct wlsc_input_device,
+			     input_device.keyboard_focus_listener);
+
+	wl_input_device_set_keyboard_focus(&device->input_device, NULL, time);
+}
+
 void
 wlsc_input_device_init(struct wlsc_input_device *device,
 		       struct wlsc_compositor *ec)
@@ -1183,9 +1197,11 @@ wlsc_input_device_init(struct wlsc_input_device *device,
 	device->hotspot_x = 16;
 	device->hotspot_y = 16;
 
-	device->listener.func = handle_surface_destroy;
-	wl_list_insert(ec->surface_destroy_listener_list.prev,
-		       &device->listener.link);
+	wl_list_init(&device->input_device.pointer_focus_listener.link);
+	device->input_device.pointer_focus_listener.func = lose_pointer_focus;
+	wl_list_init(&device->input_device.keyboard_focus_listener.link);
+	device->input_device.keyboard_focus_listener.func = lose_keyboard_focus;
+
 	wl_list_insert(ec->input_device_list.prev, &device->link);
 
 	wlsc_input_device_set_pointer_image(device, WLSC_POINTER_LEFT_PTR);
diff --git a/compositor/compositor.h b/compositor/compositor.h
index 6502c589..16d01a78 100644
--- a/compositor/compositor.h
+++ b/compositor/compositor.h
@@ -41,12 +41,6 @@ struct wlsc_matrix {
 
 struct wlsc_surface;
 
-struct wlsc_listener {
-	struct wl_list link;
-	void (*func)(struct wlsc_listener *listener,
-		     struct wlsc_surface *surface);
-};
-
 struct wlsc_output {
 	struct wl_object object;
 	struct wl_list link;
@@ -106,7 +100,7 @@ struct wlsc_input_device {
 	uint32_t grab_button;
 	struct wl_drag *drag;
 
-	struct wlsc_listener listener;
+	struct wl_listener listener;
 };
 
 struct wlsc_drm {
diff --git a/wayland/wayland-server.c b/wayland/wayland-server.c
index 57599faa..487bddc5 100644
--- a/wayland/wayland-server.c
+++ b/wayland/wayland-server.c
@@ -79,7 +79,14 @@ struct wl_global {
 	struct wl_list link;
 };
 
-WL_EXPORT struct wl_surface wl_grab_surface;
+WL_EXPORT struct wl_surface wl_grab_surface = {
+	{},
+	NULL,
+	{
+		&wl_grab_surface.destroy_listener_list,
+		&wl_grab_surface.destroy_listener_list
+	}
+};
 
 static int wl_debug = 0;
 
@@ -333,6 +340,11 @@ wl_input_device_set_pointer_focus(struct wl_input_device *device,
 
 	device->pointer_focus = surface;
 	device->pointer_focus_time = time;
+
+	wl_list_remove(&device->pointer_focus_listener.link);
+	if (surface)
+		wl_list_insert(surface->destroy_listener_list.prev,
+			       &device->pointer_focus_listener.link);
 }
 
 WL_EXPORT void
@@ -358,6 +370,11 @@ wl_input_device_set_keyboard_focus(struct wl_input_device *device,
 
 	device->keyboard_focus = surface;
 	device->keyboard_focus_time = time;
+
+	wl_list_remove(&device->keyboard_focus_listener.link);
+	if (surface)
+		wl_list_insert(surface->destroy_listener_list.prev,
+			       &device->keyboard_focus_listener.link);
 }
 
 static void
diff --git a/wayland/wayland-server.h b/wayland/wayland-server.h
index 8b22b42c..15a02a4f 100644
--- a/wayland/wayland-server.h
+++ b/wayland/wayland-server.h
@@ -119,9 +119,16 @@ struct wl_buffer {
 		       int32_t x, int32_t y, int32_t width, int32_t height);
 };
 
+struct wl_listener {
+	struct wl_list link;
+	void (*func)(struct wl_listener *listener,
+		     struct wl_surface *surface);
+};
+
 struct wl_surface {
 	struct wl_resource resource;
 	struct wl_client *client;
+	struct wl_list destroy_listener_list;
 };
 
 struct wl_shell {
@@ -135,6 +142,8 @@ struct wl_input_device {
 	struct wl_array keys;
 	uint32_t pointer_focus_time;
 	uint32_t keyboard_focus_time;
+	struct wl_listener pointer_focus_listener;
+	struct wl_listener keyboard_focus_listener;
 };
 
 struct wl_visual {