Improve support of child windows that may leak outside their parent window.

1) add Wayland code that prevent subwindows from leaking outside their parent.
This does not cover GL subwindows.

2) add macOS code that prevent GL subwindows from leaking outside their parent.
This fixes issue #494 for the macOS platform.

N.B.: Wayland GL subwindows are not prevented from leaking because no solution
that would not require any change in client applications was found. Code that
would cover Wayland GL subwindows but would require client applications to always
use the FL_ALPHA flag is included in this commit in commented out form.

FL/Fl_Gl_Window.H

@@ -47,6 +47,11 @@ class Fl_Gl_Window_Driver;
   to add a selection of widgets to an OpenGL window. The widgets will draw on top
   of any OpenGL rendering. The number of supported widgets will increase as the
   driver development improves. Program test/cube.cxx illustrates how to do that.
+ 
+  \note FLTK expects that when an Fl_Gl_Window is a child of a parent Fl_Window,
+  the child window lies entirely inside its parent window. If that's not the case, what
+  happens to the part of the GL subwindow which leaks outside its parent is undefined
+  and susceptible to be platform-specific.
 */
 class FL_EXPORT Fl_Gl_Window : public Fl_Window {
   friend class Fl_Gl_Window_Driver;

src/Fl_Gl_Window.cxx

@@ -372,7 +372,7 @@ void Fl_Gl_Window::draw_begin() {
   glPointSize((GLfloat)(drv->pixels_per_unit_));
   glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
   glEnable(GL_BLEND);
-  glDisable(GL_SCISSOR_TEST);
+  if (!pGlWindowDriver->need_scissor()) glDisable(GL_SCISSOR_TEST);
   // TODO: all of the settings should be saved on the GL stack
 }
 

src/Fl_Gl_Window_Driver.H

@@ -101,6 +101,9 @@ public:
   virtual Fl_Font_Descriptor** fontnum_to_fontdescriptor(int fnum);
   virtual Fl_RGB_Image* capture_gl_rectangle(int x, int y, int w, int h);
   static inline Fl_Gl_Window_Driver* driver(const Fl_Gl_Window *win) {return win->pGlWindowDriver;}
+  // true means the platform uses glScissor() to make sure GL subwindows
+  // don't leak outside their parent window
+  virtual bool need_scissor() { return false; }
 };
 
 #endif /* Fl_Gl_Window_Driver_H */

src/Fl_cocoa.mm

@@ -2930,10 +2930,22 @@ NSOpenGLContext* Fl_Cocoa_Window_Driver::create_GLcontext_for_window(NSOpenGLPix
       addr(view, @selector(setWantsBestResolutionOpenGLSurface:), Fl::use_high_res_GL() != 0);
     }
     [context setView:view];
+    if (Fl_Cocoa_Window_Driver::driver(window)->subRect()) {
+      remove_gl_context_opacity(context);
+    }
   }
   return context;
 }
 
+void Fl_Cocoa_Window_Driver::remove_gl_context_opacity(NSOpenGLContext *ctx) {
+  GLint gl_opacity;
+  [ctx getValues:&gl_opacity forParameter:NSOpenGLContextParameterSurfaceOpacity];
+  if (gl_opacity != 0) {
+    gl_opacity = 0;
+    [ctx setValues:&gl_opacity forParameter:NSOpenGLContextParameterSurfaceOpacity];
+  }
+}
+
 void Fl_Cocoa_Window_Driver::GLcontext_update(NSOpenGLContext* ctxt)
 {
   [ctxt update];
@@ -3408,6 +3420,9 @@ void Fl_Cocoa_Window_Driver::resize(int X, int Y, int W, int H) {
     }
     through_resize(0);
   }
+  
+  // make sure subwindow doesn't leak outside parent
+  if (pWindow->parent()) [fl_xid(pWindow) checkSubwindowFrame];
 }
 
 

src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H

@@ -41,6 +41,8 @@ class Fl_Cocoa_Gl_Window_Driver : public Fl_Gl_Window_Driver {
   virtual void gl_start();
   virtual char *alpha_mask_for_string(const char *str, int n, int w, int h, Fl_Fontsize fs);
   virtual Fl_RGB_Image* capture_gl_rectangle(int x, int y, int w, int h);
+  virtual bool need_scissor() { return true; }
+  void apply_scissor();
 };
 
 

src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx

@@ -69,6 +69,9 @@ GLContext Fl_Cocoa_Gl_Window_Driver::create_gl_context(Fl_Window* window, const
   context = Fl_Cocoa_Window_Driver::create_GLcontext_for_window(((Fl_Cocoa_Gl_Choice*)g)->pixelformat, (NSOpenGLContext*)shared_ctx, window);
   if (!context) return 0;
   add_context(context);
+  Fl_Cocoa_Window_Driver::GLcontext_makecurrent((NSOpenGLContext*)context);
+  glClearColor(0., 0., 0., 1.);
+  apply_scissor();
   return (context);
 }
 
@@ -185,9 +188,28 @@ void Fl_Cocoa_Gl_Window_Driver::swap_buffers() {
 char Fl_Cocoa_Gl_Window_Driver::swap_type() {return copy;}
 
 void Fl_Cocoa_Gl_Window_Driver::resize(int is_a_resize, int w, int h) {
+  if (pWindow->shown()) apply_scissor();
   Fl_Cocoa_Window_Driver::GLcontext_update((NSOpenGLContext*)pWindow->context());
 }
 
+void Fl_Cocoa_Gl_Window_Driver::apply_scissor() {
+  CGRect *extents = Fl_Cocoa_Window_Driver::driver(pWindow)->subRect();
+  if (extents) {
+    Fl_Cocoa_Window_Driver::remove_gl_context_opacity((NSOpenGLContext*)pWindow->context());
+    glDisable(GL_SCISSOR_TEST);
+    GLdouble vals[4];
+    glGetDoublev(GL_COLOR_CLEAR_VALUE, vals);
+    glClearColor(0., 0., 0., 0.);
+    glClear(GL_COLOR_BUFFER_BIT);
+    glClearColor(vals[0], vals[1], vals[2], vals[3]);
+    float s = pWindow->pixels_per_unit();
+    glScissor(s*extents->origin.x, s*extents->origin.y, s*extents->size.width, s*extents->size.height);
+//printf("apply_scissor %dx%d %dx%d\n",extents->x, extents->y, extents->width, extents->height);
+    glEnable(GL_SCISSOR_TEST);
+  }
+}
+
+
 /* Some old Apple hardware doesn't implement the GL_EXT_texture_rectangle extension.
  For it, draw_string_legacy_glut() is used to draw text. */
 

src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H

@@ -154,6 +154,7 @@ public:
   static void GLcontext_makecurrent(NSOpenGLContext*); // uses Objective-c
   static void GL_cleardrawable(void); // uses Objective-c
   static void gl_start(NSOpenGLContext*); // uses Objective-c
+  static void remove_gl_context_opacity(NSOpenGLContext*); // uses Objective-c
 
   //icons
   virtual void icons(const Fl_RGB_Image *icons[], int count);

src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.H

@@ -59,6 +59,9 @@ class Fl_Wayland_Gl_Window_Driver : public Fl_Gl_Window_Driver {
   void init();
   struct wl_egl_window *egl_window;
   EGLSurface egl_surface;
+public:
+  //virtual bool need_scissor() { return true; } // CONTROL_LEAKING_SUB_GL_WINDOWS
+  //void apply_scissor(); // CONTROL_LEAKING_SUB_GL_WINDOWS
 };
 
 #endif // HAVE_GL

src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.cxx

@@ -27,12 +27,17 @@
 #include <EGL/egl.h>
 #include <FL/gl.h>
 
-/* Implementation note about OpenGL drawing on the Wayland platform
+/* Implementation notes about OpenGL drawing on the Wayland platform
 
-After eglCreateWindowSurface() with attributes {EGL_RENDER_BUFFER, EGL_SINGLE_BUFFER, EGL_NONE},
+* After eglCreateWindowSurface() with attributes {EGL_RENDER_BUFFER, EGL_SINGLE_BUFFER, EGL_NONE},
 eglQueryContext() reports that EGL_RENDER_BUFFER equals EGL_BACK_BUFFER.
 This experiment suggests that the platform only supports double-buffer drawing.
 Consequently, FL_DOUBLE is enforced in all Fl_Gl_Window::mode_ values under Wayland.
+ 
+* Commented out code marked with CONTROL_LEAKING_SUB_GL_WINDOWS aims to prevent
+ sub GL windows from leaking out from their parent by making leaking parts fully transparent.
+ This code is commented out because it requires the FL_ALPHA flag to be on
+ which not all client applications do.
 */
 
 // Describes crap needed to create a GLContext.
@@ -120,6 +125,7 @@ char *Fl_Wayland_Gl_Window_Driver::alpha_mask_for_string(const char *str, int n,
 Fl_Gl_Choice *Fl_Wayland_Gl_Window_Driver::find(int m, const int *alistp)
 {
   m |= FL_DOUBLE;
+  //if (pWindow->parent()) m |= FL_ALPHA; // CONTROL_LEAKING_SUB_GL_WINDOWS
   Fl_Wayland_Gl_Choice *g = (Fl_Wayland_Gl_Choice*)Fl_Gl_Window_Driver::find_begin(m, alistp);
   if (g) return g;
 
@@ -177,8 +183,15 @@ GLContext Fl_Wayland_Gl_Window_Driver::create_gl_context(Fl_Window* window, cons
   static const EGLint context_attribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
   GLContext ctx = (GLContext)eglCreateContext(egl_display, ((Fl_Wayland_Gl_Choice*)g)->egl_conf, shared_ctx?(EGLContext)shared_ctx:EGL_NO_CONTEXT, context_attribs);
 //fprintf(stderr, "eglCreateContext=%p shared_ctx=%p\n", ctx, shared_ctx);
-  if (ctx)
+  if (ctx) {
     add_context(ctx);
+    /* CONTROL_LEAKING_SUB_GL_WINDOWS
+    if (egl_surface) {
+      eglMakeCurrent(egl_display, egl_surface, egl_surface, (EGLContext)ctx);
+      glClearColor(0., 0., 0., 1.); // set opaque black as starting background color
+      apply_scissor();
+    }*/
+  }
   return ctx;
 }
 
@@ -216,6 +229,24 @@ void Fl_Wayland_Gl_Window_Driver::set_gl_context(Fl_Window* w, GLContext context
   }
 }
 
+/* CONTROL_LEAKING_SUB_GL_WINDOWS
+void Fl_Wayland_Gl_Window_Driver::apply_scissor() {
+  cairo_rectangle_int_t *extents = Fl_Wayland_Window_Driver::driver(pWindow)->subRect();
+  if (extents) {
+    glDisable(GL_SCISSOR_TEST);
+    GLdouble vals[4];
+    glGetDoublev(GL_COLOR_CLEAR_VALUE, vals);
+    glClearColor(0., 0., 0., 0.);
+    glClear(GL_COLOR_BUFFER_BIT);
+    glClearColor(vals[0], vals[1], vals[2], vals[3]);
+    float s = pWindow->pixels_per_unit();
+    glScissor(s*extents->x, s*extents->y, s*extents->width, s*extents->height);
+//printf("apply_scissor %dx%d %dx%d\n",extents->x, extents->y, extents->width, extents->height);
+    glEnable(GL_SCISSOR_TEST);
+  }
+}*/
+
+
 void Fl_Wayland_Gl_Window_Driver::delete_gl_context(GLContext context) {
   if (cached_context == context) {
     cached_context = 0;
@@ -355,6 +386,11 @@ public:
 
 static Fl_Wayland_Gl_Plugin Gl_Overlay_Plugin;
 
+/* CONTROL_LEAKING_SUB_GL_WINDOWS
+static void delayed_scissor(Fl_Wayland_Gl_Window_Driver *dr) {
+  dr->apply_scissor();
+}*/
+
 static void delayed_flush(Fl_Gl_Window *win) {
   win->flush();
 }
@@ -374,6 +410,11 @@ void Fl_Wayland_Gl_Window_Driver::resize(int is_a_resize, int W, int H) {
         Fl::add_timeout(0.01, (Fl_Timeout_Handler)delayed_flush, pWindow);
     }
   }
+  /* CONTROL_LEAKING_SUB_GL_WINDOWS
+  if (Fl_Wayland_Window_Driver::driver(pWindow)->subRect()) {
+    pWindow->redraw();
+    Fl::add_timeout(0.01, (Fl_Timeout_Handler)delayed_scissor, this);
+  }*/
 }
 
 char Fl_Wayland_Gl_Window_Driver::swap_type() {

src/drivers/Wayland/Fl_Wayland_Window_Driver.H

@@ -41,6 +41,7 @@
  */
 
 typedef struct _cairo_pattern cairo_pattern_t;
+typedef struct _cairo_rectangle_int cairo_rectangle_int_t;
 class Fl_Wayland_Plugin;
 
 
@@ -56,6 +57,7 @@ private:
     Fl_Image* shape_; ///<  shape image
     cairo_pattern_t *mask_pattern_;
   } *shape_data_;
+  cairo_rectangle_int_t *subRect_;      // makes sure subwindow remains inside its parent window
   static bool in_flush; // useful for progressive window drawing
   static Fl_Wayland_Plugin *gl_plugin();
   struct wl_cursor *cursor_;
@@ -78,6 +80,9 @@ public:
   void shape_bitmap_(Fl_Image* b);
   void shape_alpha_(Fl_Image* img, int offset);
   void update_scale();
+  cairo_rectangle_int_t *subRect() { return subRect_; } // getter
+  void subRect(cairo_rectangle_int_t *r); // setter
+  void checkSubwindowFrame();
   enum kind {DECORATED, SUBWINDOW, POPUP, UNFRAMED};
   struct xdg_toplevel *xdg_toplevel();
   Fl_Wayland_Window_Driver(Fl_Window*);

src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx

@@ -48,6 +48,7 @@ extern "C" {
 }
 
 #define fl_max(a,b) ((a) > (b) ? (a) : (b))
+#define fl_min(a,b) ((a) < (b) ? (a) : (b))
 
 
 struct wld_window *Fl_Wayland_Window_Driver::wld_window = NULL;
@@ -66,6 +67,7 @@ Fl_Wayland_Window_Driver::Fl_Wayland_Window_Driver(Fl_Window *win) : Fl_Window_D
   in_handle_configure = false;
   screen_num_ = -1;
   gl_start_support_ = NULL;
+  subRect_ = NULL;
 }
 
 void Fl_Wayland_Window_Driver::delete_cursor_() {
@@ -97,6 +99,7 @@ Fl_Wayland_Window_Driver::~Fl_Wayland_Window_Driver()
     delete shape_data_;
   }
   delete_cursor_();
+  if (subRect_) delete subRect_;
   if (gl_start_support_) { // occurs only if gl_start/gl_finish was used
     gl_plugin()->destroy(gl_start_support_);
   }
@@ -354,7 +357,6 @@ void Fl_Wayland_Window_Driver::make_current() {
     Fl_Wayland_Graphics_Driver::buffer_commit(window, &surface_frame_listener);
   }
 
-  fl_graphics_driver->clip_region(0);
   Fl_Wayland_Window_Driver::wld_window = window;
   float scale = Fl::screen_scale(pWindow->screen_num()) * window->scale;
   if (!window->buffer) {
@@ -364,6 +366,14 @@ void Fl_Wayland_Window_Driver::make_current() {
                                             &window->buffer->draw_buffer_needs_commit);
   }
   ((Fl_Wayland_Graphics_Driver*)fl_graphics_driver)->set_buffer(window->buffer, scale);
+  cairo_rectangle_int_t *extents = subRect();
+  if (extents) { // make damage-to-buffer not to leak outside parent
+    Fl_Region clip_region = fl_graphics_driver->XRectangleRegion(extents->x, extents->y,
+                                                                 extents->width, extents->height);
+//printf("make_current: %dx%d %dx%d\n",extents->x, extents->y, extents->width, extents->height);
+    Fl_X::i(pWindow)->region = clip_region;
+  }
+  else fl_graphics_driver->clip_region(0);
 
 #ifdef FLTK_HAVE_CAIROEXT
   // update the cairo_t context
@@ -1060,6 +1070,7 @@ Fl_X *Fl_Wayland_Window_Driver::makeWindow()
     new_window->configured_height = pWindow->h();
     wait_for_expose_value = 0;
     pWindow->border(0);
+    checkSubwindowFrame(); // make sure subwindow doesn't leak outside parent
 
   } else { // a window without decoration
     new_window->kind = UNFRAMED;
@@ -1438,8 +1449,70 @@ void Fl_Wayland_Window_Driver::resize(int X, int Y, int W, int H) {
       }
     }
   }
+  
+  if (fl_win && fl_win->kind == SUBWINDOW && fl_win->subsurface)
+      checkSubwindowFrame(); // make sure subwindow doesn't leak outside parent
 }
 
+
+static void crect_intersect(cairo_rectangle_int_t *to, cairo_rectangle_int_t *with) {
+  int x = fl_max(to->x, with->x);
+  to->width = fl_min(to->x + to->width, with->x + with->width) - x;
+  if (to->width < 0) to->width = 0;
+  int y = fl_max(to->y, with->y);
+  to->height = fl_min(to->y + to->height, with->y + with->height) - y;
+  if (to->height < 0) to->height = 0;
+  to->x = x;
+  to->y = y;
+}
+
+
+static bool crect_equal(cairo_rectangle_int_t *to, cairo_rectangle_int_t *with) {
+  return (to->x == with->x && to->y == with->y && to->width == with->width && to->height == with->height);
+}
+
+
+void Fl_Wayland_Window_Driver::checkSubwindowFrame() {
+  if (!pWindow->parent()) return;
+  // make sure this subwindow doesn't leak out of its parent window
+  Fl_Window *from = pWindow, *parent;
+  cairo_rectangle_int_t full = {0, 0, pWindow->w(), pWindow->h()}; // full subwindow area
+  cairo_rectangle_int_t srect = full; // will become new subwindow clip
+  int fromx = 0, fromy = 0;
+  while ((parent = from->window()) != NULL) { // loop over all parent windows
+    fromx -= from->x(); // parent origin in subwindow's coordinates
+    fromy -= from->y();
+    cairo_rectangle_int_t prect = {fromx, fromy, parent->w(), parent->h()};
+    crect_intersect(&srect, &prect); // area of subwindow inside its parent
+    from = parent;
+  }
+  cairo_rectangle_int_t *r = subRect();
+  // current subwindow clip
+  cairo_rectangle_int_t current_clip = (r ? *r : full);
+  if (!crect_equal(&srect, &current_clip)) { // if new clip differs from current clip
+    if (crect_equal(&srect, &full)) r = NULL;
+    else {
+      r = &srect;
+      if (r->width == 0 || r->height == 0) {
+        r = NULL;
+      }
+    }
+    subRect(r);
+  }
+}
+
+
+void Fl_Wayland_Window_Driver::subRect(cairo_rectangle_int_t *r) {
+  if (subRect_) delete subRect_;
+  cairo_rectangle_int_t *r2 = NULL;
+  if (r) {
+    r2 = new cairo_rectangle_int_t;
+    *r2 = *r;
+  }
+  subRect_ = r2;
+}
+
+
 void Fl_Wayland_Window_Driver::reposition_menu_window(int x, int y) {
   struct wld_window * xid_menu = fl_wl_xid(pWindow);
   if (y == pWindow->y() && y >= 0) return;