motif: restore `BS_NORMAL` correctly (#5135)

motif hints: restore BS_NORMAL correctly

include/window.h

@@ -94,7 +94,7 @@ void window_update_hints(i3Window *win, xcb_get_property_reply_t *prop, bool *ur
  * it is still in use by popular widget toolkits such as GTK+ and Java AWT.
  *
  */
-void window_update_motif_hints(i3Window *win, xcb_get_property_reply_t *prop, border_style_t *motif_border_style);
+bool window_update_motif_hints(i3Window *win, xcb_get_property_reply_t *prop, border_style_t *motif_border_style);
 
 /**
  * Updates the WM_CLIENT_MACHINE

release-notes/bugfixes/9-bs-normal

@@ -0,0 +1 @@
+Restore BS_NORMAL _MOTIF_WM_HINTS correctly

src/handlers.c

@@ -1187,9 +1187,9 @@ static bool handle_machine_change(Con *con, xcb_get_property_reply_t *prop) {
  */
 static bool handle_motif_hints_change(Con *con, xcb_get_property_reply_t *prop) {
     border_style_t motif_border_style;
-    window_update_motif_hints(con->window, prop, &motif_border_style);
+    bool has_mwm_hints = window_update_motif_hints(con->window, prop, &motif_border_style);
 
-    if (motif_border_style != con->border_style && motif_border_style != BS_NORMAL) {
+    if (has_mwm_hints && motif_border_style != con->border_style) {
         DLOG("Update border style of con %p to %d\n", con, motif_border_style);
         con_set_border_style(con, motif_border_style, con->current_border_width);
 

src/manage.c

@@ -214,8 +214,8 @@ void manage_window(xcb_window_t window, xcb_get_window_attributes_cookie_t cooki
     window_update_role(cwindow, xcb_get_property_reply(conn, role_cookie, NULL));
     bool urgency_hint;
     window_update_hints(cwindow, xcb_get_property_reply(conn, wm_hints_cookie, NULL), &urgency_hint);
-    border_style_t motif_border_style = BS_NORMAL;
-    window_update_motif_hints(cwindow, xcb_get_property_reply(conn, motif_wm_hints_cookie, NULL), &motif_border_style);
+    border_style_t motif_border_style;
+    bool has_mwm_hints = window_update_motif_hints(cwindow, xcb_get_property_reply(conn, motif_wm_hints_cookie, NULL), &motif_border_style);
     window_update_normal_hints(cwindow, xcb_get_property_reply(conn, wm_normal_hints_cookie, NULL), geom);
     window_update_machine(cwindow, xcb_get_property_reply(conn, wm_machine_cookie, NULL));
     xcb_get_property_reply_t *type_reply = xcb_get_property_reply(conn, wm_type_cookie, NULL);
@@ -511,7 +511,7 @@ void manage_window(xcb_window_t window, xcb_get_window_attributes_cookie_t cooki
     if (nc->geometry.width == 0)
         nc->geometry = (Rect){geom->x, geom->y, geom->width, geom->height};
 
-    if (motif_border_style != BS_NORMAL) {
+    if (has_mwm_hints) {
         DLOG("MOTIF_WM_HINTS specifies decorations (border_style = %d)\n", motif_border_style);
         if (want_floating) {
             con_set_border_style(nc, motif_border_style, config.default_floating_border_width);

src/window.c

@@ -415,12 +415,10 @@ void window_update_hints(i3Window *win, xcb_get_property_reply_t *prop, bool *ur
  * it is still in use by popular widget toolkits such as GTK+ and Java AWT.
  *
  */
-void window_update_motif_hints(i3Window *win, xcb_get_property_reply_t *prop, border_style_t *motif_border_style) {
-    /* This implementation simply mirrors Gnome's Metacity. Official
-     * documentation of this hint is nowhere to be found.
-     * For more information see:
-     * https://people.gnome.org/~tthurman/docs/metacity/xprops_8h-source.html
-     * https://stackoverflow.com/questions/13787553/detect-if-a-x11-window-has-decorations
+bool window_update_motif_hints(i3Window *win, xcb_get_property_reply_t *prop, border_style_t *motif_border_style) {
+    /* See `man VendorShell' for more info, `XmNmwmDecorations' section:
+     * https://linux.die.net/man/3/vendorshell
+     * The following constants are adapted from <Xm/MwmUtil.h>.
      */
 #define MWM_HINTS_FLAGS_FIELD 0
 #define MWM_HINTS_DECORATIONS_FIELD 2
@@ -435,7 +433,7 @@ void window_update_motif_hints(i3Window *win, xcb_get_property_reply_t *prop, bo
 
     if (prop == NULL || xcb_get_property_value_length(prop) == 0) {
         FREE(prop);
-        return;
+        return false;
     }
 
     /* The property consists of an array of 5 uint32_t's. The first value is a
@@ -461,6 +459,7 @@ void window_update_motif_hints(i3Window *win, xcb_get_property_reply_t *prop, bo
     }
 
     FREE(prop);
+    return true;
 
 #undef MWM_HINTS_FLAGS_FIELD
 #undef MWM_HINTS_DECORATIONS_FIELD