Wayland: Fix issue in maximization of a borderless window (#1099)

Also fixes scenarios mixing fullscreen and maximization:
- maximize
- set fullscreen
- unset fullscreen
- un-maximize
with and without window border.

FL/Fl_Window.H

@@ -217,7 +217,9 @@ public:
   */
   void resize(int X,int Y,int W,int H) FL_OVERRIDE;
   /** Sets whether or not the window manager border is around the window.
-  The default value is true. <I>With some X window
+  The default value is true. The macOS platform ignores requests to change the
+  border state of a fullscreen or maximized window.
+   <I>With some X window
   managers, this does not work after show() has been called.</I>
   */
   void border(int b);

src/Fl_Window.cxx

@@ -954,13 +954,14 @@ fl_uintptr_t Fl_Window::os_id() { return pWindowDriver->os_id();}
 
  This function is effective only with a show()'n, resizable, top-level window.
  Bordered and borderless windows can be used.
+ Fullscreen windows can't be used.
  \see Fl_Window::un_maximize(), Fl_Window::maximize_active()
  */
 void Fl_Window::maximize() {
-  if (!shown() || parent() || !is_resizable() || maximize_active()) return;
+  if (!shown() || parent() || !is_resizable() || maximize_active() || fullscreen_active())
+    return;
   set_flag(MAXIMIZED);
-  if (border()) pWindowDriver->maximize();
-  else pWindowDriver->Fl_Window_Driver::maximize();
+  pWindowDriver->maximize();
 }
 
 /**
@@ -968,10 +969,9 @@ void Fl_Window::maximize() {
  \see Fl_Window::maximize()
 */
 void Fl_Window::un_maximize() {
-  if (!shown() || parent() || !is_resizable() || !maximize_active()) return;
+  if (!shown() || parent() || !is_resizable() || !maximize_active() || fullscreen_active()) return;
   clear_flag(MAXIMIZED);
-  if (border()) pWindowDriver->un_maximize();
-  else pWindowDriver->Fl_Window_Driver::un_maximize();
+  pWindowDriver->un_maximize();
 }
 
 void Fl_Window::is_maximized_(bool b) {

src/Fl_Window_fullscreen.cxx

@@ -37,10 +37,12 @@ void Fl_Window::border(int b) {
 */
 void Fl_Window::fullscreen() {
   if (!is_resizable()) return;
-  no_fullscreen_x = x();
-  no_fullscreen_y = y();
-  no_fullscreen_w = w();
-  no_fullscreen_h = h();
+  if (!maximize_active()) {
+    no_fullscreen_x = x();
+    no_fullscreen_y = y();
+    no_fullscreen_w = w();
+    no_fullscreen_h = h();
+  }
   if (shown() && !(flags() & Fl_Widget::FULLSCREEN)) {
     pWindowDriver->fullscreen_on();
   } else {
@@ -54,7 +56,8 @@ void Fl_Window::fullscreen_off(int X,int Y,int W,int H) {
   } else {
     clear_flag(FULLSCREEN);
   }
-  no_fullscreen_x = no_fullscreen_y = no_fullscreen_w = no_fullscreen_h = 0;
+  if (!maximize_active())
+    no_fullscreen_x = no_fullscreen_y = no_fullscreen_w = no_fullscreen_h = 0;
 }
 
 void Fl_Window::fullscreen_off() {

src/Fl_cocoa.mm

@@ -3328,12 +3328,14 @@ void Fl_Cocoa_Window_Driver::fullscreen_on() {
 
 
 void Fl_Cocoa_Window_Driver::maximize() {
-  [fl_xid(pWindow) performZoom:nil];
+  if (border()) [fl_xid(pWindow) performZoom:nil];
+  else Fl_Window_Driver::maximize();
 }
 
 
 void Fl_Cocoa_Window_Driver::un_maximize() {
-  [fl_xid(pWindow) performZoom:nil];
+  if (border()) [fl_xid(pWindow) performZoom:nil];
+  else Fl_Window_Driver::un_maximize();
 }
 
 
@@ -3387,6 +3389,7 @@ void Fl_Cocoa_Window_Driver::fullscreen_off(int X, int Y, int W, int H) {
     [nswin setStyleMask:calc_win_style(pWindow)]; //10.6
     restore_window_title_and_icon(pWindow, icon_image);
     pWindow->resize(X, Y, W, H);
+    if (pWindow->maximize_active()) Fl_Window_Driver::maximize();
     if (has_focus) [nswin makeKeyAndOrderFront:nil];
     else [nswin orderFront:nil];
   } else

src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx

@@ -2175,10 +2175,12 @@ Fl_Wayland_Plugin *Fl_Wayland_Window_Driver::gl_plugin() {
 void Fl_Wayland_Window_Driver::maximize() {
   struct wld_window *xid = (struct wld_window *)Fl_X::flx(pWindow)->xid;
   if (xid->kind == DECORATED) libdecor_frame_set_maximized(xid->frame);
+  else Fl_Window_Driver::maximize();
 }
 
 
 void Fl_Wayland_Window_Driver::un_maximize() {
   struct wld_window *xid = (struct wld_window *)Fl_X::flx(pWindow)->xid;
   if (xid->kind == DECORATED) libdecor_frame_unset_maximized(xid->frame);
+  else Fl_Window_Driver::un_maximize();
 }

src/drivers/WinAPI/Fl_WinAPI_Window_Driver.cxx

@@ -610,6 +610,7 @@ void Fl_WinAPI_Window_Driver::fullscreen_off(int X, int Y, int W, int H) {
   scaledW += bx*2;
   scaledH += by*2+bt;
   SetWindowLong(fl_xid(pWindow), GWL_STYLE, style);
+  if (pWindow->maximize_active()) return this->maximize();
   SetWindowPos(fl_xid(pWindow), 0, scaledX, scaledY, scaledW, scaledH,
                SWP_NOACTIVATE | SWP_NOZORDER | SWP_FRAMECHANGED);
   Fl::handle(FL_FULLSCREEN, pWindow);
@@ -617,10 +618,12 @@ void Fl_WinAPI_Window_Driver::fullscreen_off(int X, int Y, int W, int H) {
 
 
 void Fl_WinAPI_Window_Driver::maximize() {
+  if (!border()) return Fl_Window_Driver::maximize();
   ShowWindow(fl_xid(pWindow), SW_SHOWMAXIMIZED);
 }
 
 void Fl_WinAPI_Window_Driver::un_maximize() {
+  if (!border()) return Fl_Window_Driver::un_maximize();
   ShowWindow(fl_xid(pWindow), SW_SHOWNORMAL);
 }
 

test/fullscreen.cxx

@@ -183,14 +183,13 @@ void border_cb(Fl_Button *b, Fl_Window *w) {
 }
 
 
-void maximize_cb(Fl_Widget *o, void *p) {
-  Fl_Window *w = (Fl_Window *)p;
+void maximize_cb(Fl_Button *b, Fl_Window *w) {
   if (w->maximize_active()) {
     w->un_maximize();
-    //((Fl_Button*)o)->set();
+    if (w->maximize_active()) b->set();
   } else {
     w->maximize();
-    //((Fl_Button*)o)->clear();
+    if (!w->maximize_active()) b->clear();
   }
 }
 
@@ -349,7 +348,7 @@ int main(int argc, char **argv) {
   y+=30;
 
   window.b3_maxi = new Fl_Toggle_Light_Button(50,y,window.w()-60,30,"Maximize");
-  window.b3_maxi->callback(maximize_cb,w);
+  window.b3_maxi->callback((Fl_Callback*)maximize_cb,w);
   y+=30;
 
   window.b4 = new Fl_Toggle_Light_Button(50,y,window.w()-60,30,"All Screens");