Improve support of child windows that may leak outside their parent window.

1) add Wayland code that prevent subwindows from leaking outside their parent.
This does not cover GL subwindows.

2) add macOS code that prevent GL subwindows from leaking outside their parent.
This fixes issue #494 for the macOS platform.

N.B.: Wayland GL subwindows are not prevented from leaking because no solution
that would not require any change in client applications was found. Code that
would cover Wayland GL subwindows but would require client applications to always
use the FL_ALPHA flag is included in this commit in commented out form.

3 files changed, 25 insertions, 0 deletions

diff --git a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H
index 526cad688..9a3d06086 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H
+++ b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.H
@@ -41,6 +41,8 @@ class Fl_Cocoa_Gl_Window_Driver : public Fl_Gl_Window_Driver {
   virtual void gl_start();
   virtual char *alpha_mask_for_string(const char *str, int n, int w, int h, Fl_Fontsize fs);
   virtual Fl_RGB_Image* capture_gl_rectangle(int x, int y, int w, int h);
+  virtual bool need_scissor() { return true; }
+  void apply_scissor();
 };
 
 
diff --git a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx
index d1ed8df1e..a77ff7776 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx
+++ b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.cxx
@@ -69,6 +69,9 @@ GLContext Fl_Cocoa_Gl_Window_Driver::create_gl_context(Fl_Window* window, const
   context = Fl_Cocoa_Window_Driver::create_GLcontext_for_window(((Fl_Cocoa_Gl_Choice*)g)->pixelformat, (NSOpenGLContext*)shared_ctx, window);
   if (!context) return 0;
   add_context(context);
+  Fl_Cocoa_Window_Driver::GLcontext_makecurrent((NSOpenGLContext*)context);
+  glClearColor(0., 0., 0., 1.);
+  apply_scissor();
   return (context);
 }
 
@@ -185,9 +188,28 @@ void Fl_Cocoa_Gl_Window_Driver::swap_buffers() {
 char Fl_Cocoa_Gl_Window_Driver::swap_type() {return copy;}
 
 void Fl_Cocoa_Gl_Window_Driver::resize(int is_a_resize, int w, int h) {
+  if (pWindow->shown()) apply_scissor();
   Fl_Cocoa_Window_Driver::GLcontext_update((NSOpenGLContext*)pWindow->context());
 }
 
+void Fl_Cocoa_Gl_Window_Driver::apply_scissor() {
+  CGRect *extents = Fl_Cocoa_Window_Driver::driver(pWindow)->subRect();
+  if (extents) {
+    Fl_Cocoa_Window_Driver::remove_gl_context_opacity((NSOpenGLContext*)pWindow->context());
+    glDisable(GL_SCISSOR_TEST);
+    GLdouble vals[4];
+    glGetDoublev(GL_COLOR_CLEAR_VALUE, vals);
+    glClearColor(0., 0., 0., 0.);
+    glClear(GL_COLOR_BUFFER_BIT);
+    glClearColor(vals[0], vals[1], vals[2], vals[3]);
+    float s = pWindow->pixels_per_unit();
+    glScissor(s*extents->origin.x, s*extents->origin.y, s*extents->size.width, s*extents->size.height);
+//printf("apply_scissor %dx%d %dx%d\n",extents->x, extents->y, extents->width, extents->height);
+    glEnable(GL_SCISSOR_TEST);
+  }
+}
+
+
 /* Some old Apple hardware doesn't implement the GL_EXT_texture_rectangle extension.
  For it, draw_string_legacy_glut() is used to draw text. */
 
diff --git a/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H b/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
index ce3b15629..cbd853776 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
+++ b/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
@@ -154,6 +154,7 @@ public:
   static void GLcontext_makecurrent(NSOpenGLContext*); // uses Objective-c
   static void GL_cleardrawable(void); // uses Objective-c
   static void gl_start(NSOpenGLContext*); // uses Objective-c
+  static void remove_gl_context_opacity(NSOpenGLContext*); // uses Objective-c
 
   //icons
   virtual void icons(const Fl_RGB_Image *icons[], int count);