Improve support of child windows that may leak outside their parent window.

1) add Wayland code that prevent subwindows from leaking outside their parent.
This does not cover GL subwindows.

2) add macOS code that prevent GL subwindows from leaking outside their parent.
This fixes issue #494 for the macOS platform.

N.B.: Wayland GL subwindows are not prevented from leaking because no solution
that would not require any change in client applications was found. Code that
would cover Wayland GL subwindows but would require client applications to always
use the FL_ALPHA flag is included in this commit in commented out form.

1 files changed, 15 insertions, 0 deletions

diff --git a/src/Fl_cocoa.mm b/src/Fl_cocoa.mm
index 663fd2956..f021cb750 100644
--- a/src/Fl_cocoa.mm
+++ b/src/Fl_cocoa.mm
@@ -2930,10 +2930,22 @@ NSOpenGLContext* Fl_Cocoa_Window_Driver::create_GLcontext_for_window(NSOpenGLPix
       addr(view, @selector(setWantsBestResolutionOpenGLSurface:), Fl::use_high_res_GL() != 0);
     }
     [context setView:view];
+    if (Fl_Cocoa_Window_Driver::driver(window)->subRect()) {
+      remove_gl_context_opacity(context);
+    }
   }
   return context;
 }
 
+void Fl_Cocoa_Window_Driver::remove_gl_context_opacity(NSOpenGLContext *ctx) {
+  GLint gl_opacity;
+  [ctx getValues:&gl_opacity forParameter:NSOpenGLContextParameterSurfaceOpacity];
+  if (gl_opacity != 0) {
+    gl_opacity = 0;
+    [ctx setValues:&gl_opacity forParameter:NSOpenGLContextParameterSurfaceOpacity];
+  }
+}
+
 void Fl_Cocoa_Window_Driver::GLcontext_update(NSOpenGLContext* ctxt)
 {
   [ctxt update];
@@ -3408,6 +3420,9 @@ void Fl_Cocoa_Window_Driver::resize(int X, int Y, int W, int H) {
     }
     through_resize(0);
   }
+  
+  // make sure subwindow doesn't leak outside parent
+  if (pWindow->parent()) [fl_xid(pWindow) checkSubwindowFrame];
 }