Improve support of child windows that may leak outside their parent window.

1) add Wayland code that prevent subwindows from leaking outside their parent.
This does not cover GL subwindows.

2) add macOS code that prevent GL subwindows from leaking outside their parent.
This fixes issue #494 for the macOS platform.

N.B.: Wayland GL subwindows are not prevented from leaking because no solution
that would not require any change in client applications was found. Code that
would cover Wayland GL subwindows but would require client applications to always
use the FL_ALPHA flag is included in this commit in commented out form.

10 files changed, 170 insertions, 5 deletions

diff --git a/src/Fl_Gl_Window.cxx b/src/Fl_Gl_Window.cxx
index 7e9678460..2842338fe 100644
--- a/src/Fl_Gl_Window.cxx
+++ b/src/Fl_Gl_Window.cxx
@@ -372,7 +372,7 @@ void Fl_Gl_Window::draw_begin() {
   glPointSize((GLfloat)(drv->pixels_per_unit_));
   glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
   glEnable(GL_BLEND);
-  glDisable(GL_SCISSOR_TEST);
+  if (!pGlWindowDriver->need_scissor()) glDisable(GL_SCISSOR_TEST);
   // TODO: all of the settings should be saved on the GL stack
 }
 
diff --git a/src/Fl_Gl_Window_Driver.H b/src/Fl_Gl_Window_Driver.H
index e123c177a..8f3f6cf29 100644
--- a/src/Fl_Gl_Window_Driver.H
+++ b/src/Fl_Gl_Window_Driver.H
@@ -101,6 +101,9 @@ public:
   virtual Fl_Font_Descriptor** fontnum_to_fontdescriptor(int fnum);
   virtual Fl_RGB_Image* capture_gl_rectangle(int x, int y, int w, int h);
   static inline Fl_Gl_Window_Driver* driver(const Fl_Gl_Window *win) {return win->pGlWindowDriver;}
+  // true means the platform uses glScissor() to make sure GL subwindows
+  // don't leak outside their parent window
+  virtual bool need_scissor() { return false; }
 };
 
 #endif /* Fl_Gl_Window_Driver_H */
diff --git a/src/Fl_cocoa.mm b/src/Fl_cocoa.mm
index 663fd2956..f021cb750 100644
--- a/src/Fl_cocoa.mm
+++ b/src/Fl_cocoa.mm
@@ -2930,10 +2930,22 @@ NSOpenGLContext* Fl_Cocoa_Window_Driver::create_GLcontext_for_window(NSOpenGLPix
       addr(view, @selector(setWantsBestResolutionOpenGLSurface:), Fl::use_high_res_GL() != 0);
     }
     [context setView:view];
+    if (Fl_Cocoa_Window_Driver::driver(window)->subRect()) {
+      remove_gl_context_opacity(context);
+    }
   }
   return context;
 }
 
+void Fl_Cocoa_Window_Driver::remove_gl_context_opacity(NSOpenGLContext *ctx) {
+  GLint gl_opacity;
+  [ctx getValues:&gl_opacity forParameter:NSOpenGLContextParameterSurfaceOpacity];
+  if (gl_opacity != 0) {
+    gl_opacity = 0;
+    [ctx setValues:&gl_opacity forParameter:NSOpenGLContextParameterSurfaceOpacity];
+  }
+}
+
 void Fl_Cocoa_Window_Driver::GLcontext_update(NSOpenGLContext* ctxt)
 {
   [ctxt update];
@@ -3408,6 +3420,9 @@ void Fl_Cocoa_Window_Driver::resize(int X, int Y, int W, int H) {
     }
     through_resize(0);
   }
+  
+  // make sure subwindow doesn't leak outside parent
+  if (pWindow->parent()) [fl_xid(pWindow) checkSubwindowFrame];
 }
 
 
diff --git a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H
index 526cad688..9a3d06086 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H
+++ b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H
@@ -41,6 +41,8 @@ class Fl_Cocoa_Gl_Window_Driver : public Fl_Gl_Window_Driver {
   virtual void gl_start();
   virtual char *alpha_mask_for_string(const char *str, int n, int w, int h, Fl_Fontsize fs);
   virtual Fl_RGB_Image* capture_gl_rectangle(int x, int y, int w, int h);
+  virtual bool need_scissor() { return true; }
+  void apply_scissor();
 };
 
 
diff --git a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx
index d1ed8df1e..a77ff7776 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx
+++ b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx
@@ -69,6 +69,9 @@ GLContext Fl_Cocoa_Gl_Window_Driver::create_gl_context(Fl_Window* window, const
   context = Fl_Cocoa_Window_Driver::create_GLcontext_for_window(((Fl_Cocoa_Gl_Choice*)g)->pixelformat, (NSOpenGLContext*)shared_ctx, window);
   if (!context) return 0;
   add_context(context);
+  Fl_Cocoa_Window_Driver::GLcontext_makecurrent((NSOpenGLContext*)context);
+  glClearColor(0., 0., 0., 1.);
+  apply_scissor();
   return (context);
 }
 
@@ -185,9 +188,28 @@ void Fl_Cocoa_Gl_Window_Driver::swap_buffers() {
 char Fl_Cocoa_Gl_Window_Driver::swap_type() {return copy;}
 
 void Fl_Cocoa_Gl_Window_Driver::resize(int is_a_resize, int w, int h) {
+  if (pWindow->shown()) apply_scissor();
   Fl_Cocoa_Window_Driver::GLcontext_update((NSOpenGLContext*)pWindow->context());
 }
 
+void Fl_Cocoa_Gl_Window_Driver::apply_scissor() {
+  CGRect *extents = Fl_Cocoa_Window_Driver::driver(pWindow)->subRect();
+  if (extents) {
+    Fl_Cocoa_Window_Driver::remove_gl_context_opacity((NSOpenGLContext*)pWindow->context());
+    glDisable(GL_SCISSOR_TEST);
+    GLdouble vals[4];
+    glGetDoublev(GL_COLOR_CLEAR_VALUE, vals);
+    glClearColor(0., 0., 0., 0.);
+    glClear(GL_COLOR_BUFFER_BIT);
+    glClearColor(vals[0], vals[1], vals[2], vals[3]);
+    float s = pWindow->pixels_per_unit();
+    glScissor(s*extents->origin.x, s*extents->origin.y, s*extents->size.width, s*extents->size.height);
+//printf("apply_scissor %dx%d %dx%d\n",extents->x, extents->y, extents->width, extents->height);
+    glEnable(GL_SCISSOR_TEST);
+  }
+}
+
+
 /* Some old Apple hardware doesn't implement the GL_EXT_texture_rectangle extension.
  For it, draw_string_legacy_glut() is used to draw text. */
 
diff --git a/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H b/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
index ce3b15629..cbd853776 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
+++ b/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
@@ -154,6 +154,7 @@ public:
   static void GLcontext_makecurrent(NSOpenGLContext*); // uses Objective-c
   static void GL_cleardrawable(void); // uses Objective-c
   static void gl_start(NSOpenGLContext*); // uses Objective-c
+  static void remove_gl_context_opacity(NSOpenGLContext*); // uses Objective-c
 
   //icons
   virtual void icons(const Fl_RGB_Image *icons[], int count);
diff --git a/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.H b/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.H
index 7c4a1c6ea..5ecfeead2 100644
--- a/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.H
+++ b/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.H
@@ -59,6 +59,9 @@ class Fl_Wayland_Gl_Window_Driver : public Fl_Gl_Window_Driver {
   void init();
   struct wl_egl_window *egl_window;
   EGLSurface egl_surface;
+public:
+  //virtual bool need_scissor() { return true; } // CONTROL_LEAKING_SUB_GL_WINDOWS
+  //void apply_scissor(); // CONTROL_LEAKING_SUB_GL_WINDOWS
 };
 
 #endif // HAVE_GL
diff --git a/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.cxx b/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.cxx
index 89014826f..3f91d0cbd 100644
--- a/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.cxx
+++ b/src/drivers/Wayland/Fl_Wayland_Gl_Window_Driver.cxx
@@ -27,12 +27,17 @@
 #include <EGL/egl.h>
 #include <FL/gl.h>
 
-/* Implementation note about OpenGL drawing on the Wayland platform
+/* Implementation notes about OpenGL drawing on the Wayland platform
 
-After eglCreateWindowSurface() with attributes {EGL_RENDER_BUFFER, EGL_SINGLE_BUFFER, EGL_NONE},
+* After eglCreateWindowSurface() with attributes {EGL_RENDER_BUFFER, EGL_SINGLE_BUFFER, EGL_NONE},
 eglQueryContext() reports that EGL_RENDER_BUFFER equals EGL_BACK_BUFFER.
 This experiment suggests that the platform only supports double-buffer drawing.
 Consequently, FL_DOUBLE is enforced in all Fl_Gl_Window::mode_ values under Wayland.
+ 
+* Commented out code marked with CONTROL_LEAKING_SUB_GL_WINDOWS aims to prevent
+ sub GL windows from leaking out from their parent by making leaking parts fully transparent.
+ This code is commented out because it requires the FL_ALPHA flag to be on
+ which not all client applications do.
 */
 
 // Describes crap needed to create a GLContext.
@@ -120,6 +125,7 @@ char *Fl_Wayland_Gl_Window_Driver::alpha_mask_for_string(const char *str, int n,
 Fl_Gl_Choice *Fl_Wayland_Gl_Window_Driver::find(int m, const int *alistp)
 {
   m |= FL_DOUBLE;
+  //if (pWindow->parent()) m |= FL_ALPHA; // CONTROL_LEAKING_SUB_GL_WINDOWS
   Fl_Wayland_Gl_Choice *g = (Fl_Wayland_Gl_Choice*)Fl_Gl_Window_Driver::find_begin(m, alistp);
   if (g) return g;
 
@@ -177,8 +183,15 @@ GLContext Fl_Wayland_Gl_Window_Driver::create_gl_context(Fl_Window* window, cons
   static const EGLint context_attribs[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
   GLContext ctx = (GLContext)eglCreateContext(egl_display, ((Fl_Wayland_Gl_Choice*)g)->egl_conf, shared_ctx?(EGLContext)shared_ctx:EGL_NO_CONTEXT, context_attribs);
 //fprintf(stderr, "eglCreateContext=%p shared_ctx=%p\n", ctx, shared_ctx);
-  if (ctx)
+  if (ctx) {
     add_context(ctx);
+    /* CONTROL_LEAKING_SUB_GL_WINDOWS
+    if (egl_surface) {
+      eglMakeCurrent(egl_display, egl_surface, egl_surface, (EGLContext)ctx);
+      glClearColor(0., 0., 0., 1.); // set opaque black as starting background color
+      apply_scissor();
+    }*/
+  }
   return ctx;
 }
 
@@ -216,6 +229,24 @@ void Fl_Wayland_Gl_Window_Driver::set_gl_context(Fl_Window* w, GLContext context
   }
 }
 
+/* CONTROL_LEAKING_SUB_GL_WINDOWS
+void Fl_Wayland_Gl_Window_Driver::apply_scissor() {
+  cairo_rectangle_int_t *extents = Fl_Wayland_Window_Driver::driver(pWindow)->subRect();
+  if (extents) {
+    glDisable(GL_SCISSOR_TEST);
+    GLdouble vals[4];
+    glGetDoublev(GL_COLOR_CLEAR_VALUE, vals);
+    glClearColor(0., 0., 0., 0.);
+    glClear(GL_COLOR_BUFFER_BIT);
+    glClearColor(vals[0], vals[1], vals[2], vals[3]);
+    float s = pWindow->pixels_per_unit();
+    glScissor(s*extents->x, s*extents->y, s*extents->width, s*extents->height);
+//printf("apply_scissor %dx%d %dx%d\n",extents->x, extents->y, extents->width, extents->height);
+    glEnable(GL_SCISSOR_TEST);
+  }
+}*/
+
+
 void Fl_Wayland_Gl_Window_Driver::delete_gl_context(GLContext context) {
   if (cached_context == context) {
     cached_context = 0;
@@ -355,6 +386,11 @@ public:
 
 static Fl_Wayland_Gl_Plugin Gl_Overlay_Plugin;
 
+/* CONTROL_LEAKING_SUB_GL_WINDOWS
+static void delayed_scissor(Fl_Wayland_Gl_Window_Driver *dr) {
+  dr->apply_scissor();
+}*/
+
 static void delayed_flush(Fl_Gl_Window *win) {
   win->flush();
 }
@@ -374,6 +410,11 @@ void Fl_Wayland_Gl_Window_Driver::resize(int is_a_resize, int W, int H) {
         Fl::add_timeout(0.01, (Fl_Timeout_Handler)delayed_flush, pWindow);
     }
   }
+  /* CONTROL_LEAKING_SUB_GL_WINDOWS
+  if (Fl_Wayland_Window_Driver::driver(pWindow)->subRect()) {
+    pWindow->redraw();
+    Fl::add_timeout(0.01, (Fl_Timeout_Handler)delayed_scissor, this);
+  }*/
 }
 
 char Fl_Wayland_Gl_Window_Driver::swap_type() {
diff --git a/src/drivers/Wayland/Fl_Wayland_Window_Driver.H b/src/drivers/Wayland/Fl_Wayland_Window_Driver.H
index 5bd73ab04..c8b959d17 100644
--- a/src/drivers/Wayland/Fl_Wayland_Window_Driver.H
+++ b/src/drivers/Wayland/Fl_Wayland_Window_Driver.H
@@ -41,6 +41,7 @@
  */
 
 typedef struct _cairo_pattern cairo_pattern_t;
+typedef struct _cairo_rectangle_int cairo_rectangle_int_t;
 class Fl_Wayland_Plugin;
 
 
@@ -56,6 +57,7 @@ private:
     Fl_Image* shape_; ///<  shape image
     cairo_pattern_t *mask_pattern_;
   } *shape_data_;
+  cairo_rectangle_int_t *subRect_;      // makes sure subwindow remains inside its parent window
   static bool in_flush; // useful for progressive window drawing
   static Fl_Wayland_Plugin *gl_plugin();
   struct wl_cursor *cursor_;
@@ -78,6 +80,9 @@ public:
   void shape_bitmap_(Fl_Image* b);
   void shape_alpha_(Fl_Image* img, int offset);
   void update_scale();
+  cairo_rectangle_int_t *subRect() { return subRect_; } // getter
+  void subRect(cairo_rectangle_int_t *r); // setter
+  void checkSubwindowFrame();
   enum kind {DECORATED, SUBWINDOW, POPUP, UNFRAMED};
   struct xdg_toplevel *xdg_toplevel();
   Fl_Wayland_Window_Driver(Fl_Window*);
diff --git a/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx b/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx
index 6c436ea25..2692e8ae1 100644
--- a/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx
+++ b/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx
@@ -48,6 +48,7 @@ extern "C" {
 }
 
 #define fl_max(a,b) ((a) > (b) ? (a) : (b))
+#define fl_min(a,b) ((a) < (b) ? (a) : (b))
 
 
 struct wld_window *Fl_Wayland_Window_Driver::wld_window = NULL;
@@ -66,6 +67,7 @@ Fl_Wayland_Window_Driver::Fl_Wayland_Window_Driver(Fl_Window *win) : Fl_Window_D
   in_handle_configure = false;
   screen_num_ = -1;
   gl_start_support_ = NULL;
+  subRect_ = NULL;
 }
 
 void Fl_Wayland_Window_Driver::delete_cursor_() {
@@ -97,6 +99,7 @@ Fl_Wayland_Window_Driver::~Fl_Wayland_Window_Driver()
     delete shape_data_;
   }
   delete_cursor_();
+  if (subRect_) delete subRect_;
   if (gl_start_support_) { // occurs only if gl_start/gl_finish was used
     gl_plugin()->destroy(gl_start_support_);
   }
@@ -354,7 +357,6 @@ void Fl_Wayland_Window_Driver::make_current() {
     Fl_Wayland_Graphics_Driver::buffer_commit(window, &surface_frame_listener);
   }
 
-  fl_graphics_driver->clip_region(0);
   Fl_Wayland_Window_Driver::wld_window = window;
   float scale = Fl::screen_scale(pWindow->screen_num()) * window->scale;
   if (!window->buffer) {
@@ -364,6 +366,14 @@ void Fl_Wayland_Window_Driver::make_current() {
                                             &window->buffer->draw_buffer_needs_commit);
   }
   ((Fl_Wayland_Graphics_Driver*)fl_graphics_driver)->set_buffer(window->buffer, scale);
+  cairo_rectangle_int_t *extents = subRect();
+  if (extents) { // make damage-to-buffer not to leak outside parent
+    Fl_Region clip_region = fl_graphics_driver->XRectangleRegion(extents->x, extents->y,
+                                                                 extents->width, extents->height);
+//printf("make_current: %dx%d %dx%d\n",extents->x, extents->y, extents->width, extents->height);
+    Fl_X::i(pWindow)->region = clip_region;
+  }
+  else fl_graphics_driver->clip_region(0);
 
 #ifdef FLTK_HAVE_CAIROEXT
   // update the cairo_t context
@@ -1060,6 +1070,7 @@ Fl_X *Fl_Wayland_Window_Driver::makeWindow()
     new_window->configured_height = pWindow->h();
     wait_for_expose_value = 0;
     pWindow->border(0);
+    checkSubwindowFrame(); // make sure subwindow doesn't leak outside parent
 
   } else { // a window without decoration
     new_window->kind = UNFRAMED;
@@ -1438,8 +1449,70 @@ void Fl_Wayland_Window_Driver::resize(int X, int Y, int W, int H) {
       }
     }
   }
+  
+  if (fl_win && fl_win->kind == SUBWINDOW && fl_win->subsurface)
+      checkSubwindowFrame(); // make sure subwindow doesn't leak outside parent
+}
+
+
+static void crect_intersect(cairo_rectangle_int_t *to, cairo_rectangle_int_t *with) {
+  int x = fl_max(to->x, with->x);
+  to->width = fl_min(to->x + to->width, with->x + with->width) - x;
+  if (to->width < 0) to->width = 0;
+  int y = fl_max(to->y, with->y);
+  to->height = fl_min(to->y + to->height, with->y + with->height) - y;
+  if (to->height < 0) to->height = 0;
+  to->x = x;
+  to->y = y;
+}
+
+
+static bool crect_equal(cairo_rectangle_int_t *to, cairo_rectangle_int_t *with) {
+  return (to->x == with->x && to->y == with->y && to->width == with->width && to->height == with->height);
 }
 
+
+void Fl_Wayland_Window_Driver::checkSubwindowFrame() {
+  if (!pWindow->parent()) return;
+  // make sure this subwindow doesn't leak out of its parent window
+  Fl_Window *from = pWindow, *parent;
+  cairo_rectangle_int_t full = {0, 0, pWindow->w(), pWindow->h()}; // full subwindow area
+  cairo_rectangle_int_t srect = full; // will become new subwindow clip
+  int fromx = 0, fromy = 0;
+  while ((parent = from->window()) != NULL) { // loop over all parent windows
+    fromx -= from->x(); // parent origin in subwindow's coordinates
+    fromy -= from->y();
+    cairo_rectangle_int_t prect = {fromx, fromy, parent->w(), parent->h()};
+    crect_intersect(&srect, &prect); // area of subwindow inside its parent
+    from = parent;
+  }
+  cairo_rectangle_int_t *r = subRect();
+  // current subwindow clip
+  cairo_rectangle_int_t current_clip = (r ? *r : full);
+  if (!crect_equal(&srect, &current_clip)) { // if new clip differs from current clip
+    if (crect_equal(&srect, &full)) r = NULL;
+    else {
+      r = &srect;
+      if (r->width == 0 || r->height == 0) {
+        r = NULL;
+      }
+    }
+    subRect(r);
+  }
+}
+
+
+void Fl_Wayland_Window_Driver::subRect(cairo_rectangle_int_t *r) {
+  if (subRect_) delete subRect_;
+  cairo_rectangle_int_t *r2 = NULL;
+  if (r) {
+    r2 = new cairo_rectangle_int_t;
+    *r2 = *r;
+  }
+  subRect_ = r2;
+}
+
+
 void Fl_Wayland_Window_Driver::reposition_menu_window(int x, int y) {
   struct wld_window * xid_menu = fl_wl_xid(pWindow);
   if (y == pWindow->y() && y >= 0) return;