Make Fl_Cocoa_Gl_Window_Driver::capture_gl_rectangle() return a depth-4 image

This change allows to capture the rounded corners of a top-level GL window.

Also, rename Fl_Cocoa_Window_Driver::capture_decorated_window_10_5()
from capture_decorated_window_10_6() because this function uses
CGWindowListCreateImageFromArray() available from macOS 10.5.

3 files changed, 26 insertions, 28 deletions

diff --git a/src/Fl_cocoa.mm b/src/Fl_cocoa.mm
index face4e91b..468c4b0d0 100644
--- a/src/Fl_cocoa.mm
+++ b/src/Fl_cocoa.mm
@@ -4542,14 +4542,17 @@ static NSBitmapImageRep* GL_rect_to_nsbitmap(Fl_Window *win, int x, int y, int w
   if (!plugin) return nil;
   Fl_RGB_Image *img = plugin->rectangle_capture(win, x, y, w, h);
   NSBitmapImageRep* bitmap = [[NSBitmapImageRep alloc] initWithBitmapDataPlanes:NULL pixelsWide:img->w() pixelsHigh:img->h() bitsPerSample:8 samplesPerPixel:4 hasAlpha:YES isPlanar:NO colorSpaceName:NSDeviceRGBColorSpace bytesPerRow:4*img->w() bitsPerPixel:32];
-  memset([bitmap bitmapData], 0xFF, [bitmap bytesPerPlane]);
-  const uchar *from = img->array;
-  for (int r = 0; r < img->h(); r++) {
-    uchar *to = [bitmap bitmapData] + r * [bitmap bytesPerRow];
-    for (int c = 0; c < img->w(); c++) {
-      memcpy(to, from, 3);
-      from += 3;
-      to += 4;
+  if (img->d() == 4) memcpy([bitmap bitmapData], img->array, 4*img->data_w()*img->data_h());
+  else {
+    memset([bitmap bitmapData], 0xFF, [bitmap bytesPerPlane]);
+    const uchar *from = img->array;
+    for (int r = 0; r < img->h(); r++) {
+      uchar *to = [bitmap bitmapData] + r * [bitmap bytesPerRow];
+      for (int c = 0; c < img->w(); c++) {
+        memcpy(to, from, 3);
+        from += 3;
+        to += 4;
+      }
     }
   }
   delete img;
@@ -4800,8 +4803,8 @@ static CGImageRef capture_decorated_window_SCK(NSWindow *nswin) {
 #endif //MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_15_0
 
 
-CGImageRef Fl_Cocoa_Window_Driver::capture_decorated_window_10_6(NSWindow *nswin) {
-  // usable with 10.6 and above
+CGImageRef Fl_Cocoa_Window_Driver::capture_decorated_window_10_5(NSWindow *nswin) {
+  // usable with 10.5 and above
   CGImageRef img = NULL;
 #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5
 #  if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_15_0
@@ -4824,15 +4827,18 @@ CGImageRef Fl_Cocoa_Window_Driver::capture_decorated_window_10_6(NSWindow *nswin
 
 static CGImageRef capture_window_titlebar(Fl_Window *win, Fl_Cocoa_Window_Driver *cocoa_dr) {
   CGImageRef img;
+#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
   if (fl_mac_os_version >= 100600) { // verified OK from 10.6
     FLWindow *nswin = fl_xid(win);
-    CGImageRef img_full = Fl_Cocoa_Window_Driver::capture_decorated_window_10_6(nswin);
+    CGImageRef img_full = Fl_Cocoa_Window_Driver::capture_decorated_window_10_5(nswin);
     int bt =  [nswin frame].size.height - [[nswin contentView] frame].size.height;
     int s = CGImageGetWidth(img_full) / [nswin frame].size.width;
     CGRect cgr = CGRectMake(0, 0, CGImageGetWidth(img_full), bt * s);
-    img = CGImageCreateWithImageInRect(img_full, cgr);
+    img = CGImageCreateWithImageInRect(img_full, cgr); // 10.4
     CGImageRelease(img_full);
-  } else {
+  } else
+#endif
+  {
     int w = win->w(), h = win->decorated_h() - win->h();
     Fl_Graphics_Driver::default_driver().scale(1);
     img = cocoa_dr->CGImage_from_window_rect(0, -h, w, h, false);
diff --git a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.mm b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.mm
index ad309ee55..fd846f1c8 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.mm
+++ b/src/drivers/Cocoa/Fl_Cocoa_Gl_Window_Driver.mm
@@ -453,7 +453,7 @@ static uchar *convert_BGRA_to_RGB(uchar *baseAddress, int w, int h, int mByteWid
 }
 
 
-static Fl_RGB_Image *cgimage_to_rgb3(CGImageRef img) {
+static Fl_RGB_Image *cgimage_to_rgb4(CGImageRef img) {
   int w = (int)CGImageGetWidth(img);
   int h = (int)CGImageGetHeight(img);
   CGColorSpaceRef cspace = CGColorSpaceCreateDeviceRGB();
@@ -463,15 +463,7 @@ static Fl_RGB_Image *cgimage_to_rgb3(CGImageRef img) {
   CGColorSpaceRelease(cspace);
   CGContextDrawImage(auxgc, CGRectMake(0, 0, w, h), img);
   CGContextRelease(auxgc);
-  uchar *rgb3 = new uchar[3 * w * h]; // transform RGBA pixel array into RGB array
-  uchar *p = rgba, *q = rgb3, *last = rgba + 4 * w * h;
-  while ( p < last) {
-    memcpy(q, p, 3);
-    p += 4;
-    q += 3;
-  }
-  delete[] rgba;
-  Fl_RGB_Image *rgb = new Fl_RGB_Image(rgb3, w, h, 3);
+  Fl_RGB_Image *rgb = new Fl_RGB_Image(rgba, w, h, 4);
   rgb->alloc_array = 1;
   return rgb;
 }
@@ -484,16 +476,16 @@ Fl_RGB_Image* Fl_Cocoa_Gl_Window_Driver::capture_gl_rectangle(int x, int y, int
   if (factor != 1) {
     w *= factor; h *= factor; x *= factor; y *= factor;
   }
-#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4
-  if (fl_mac_os_version >= 100600) {
+#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5
+  if (fl_mac_os_version >= 100500) {
     NSWindow *nswin = (NSWindow*)fl_mac_xid(pWindow);
-    CGImageRef img_full = Fl_Cocoa_Window_Driver::capture_decorated_window_10_6(nswin);
+    CGImageRef img_full = Fl_Cocoa_Window_Driver::capture_decorated_window_10_5(nswin);
     int bt =  [nswin frame].size.height - [[nswin contentView] frame].size.height;
     bt *= (factor / Fl_Graphics_Driver::default_driver().scale());
     CGRect cgr = CGRectMake(x, y + bt, w, h); // add vertical offset to bypass titlebar
     CGImageRef cgimg = CGImageCreateWithImageInRect(img_full, cgr); // 10.4
     CGImageRelease(img_full);
-    Fl_RGB_Image *rgb = cgimage_to_rgb3(cgimg);
+    Fl_RGB_Image *rgb = cgimage_to_rgb4(cgimg);
     CGImageRelease(cgimg);
     return rgb;
   }
diff --git a/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H b/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
index c4d4e31da..4fa1a7491 100644
--- a/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
+++ b/src/drivers/Cocoa/Fl_Cocoa_Window_Driver.H
@@ -97,7 +97,7 @@ public:
   NSCursor *cursor;
   static void q_release_context(Fl_Cocoa_Window_Driver *x = 0); // free all resources associated with gc
   static void clip_to_rounded_corners(CGContextRef gc, int w, int h);
-  static CGImageRef capture_decorated_window_10_6(NSWindow *nswin);
+  static CGImageRef capture_decorated_window_10_5(NSWindow *nswin);
   void set_key_window();
   bool mapped_to_retina();      // is window mapped to retina display?
   void mapped_to_retina(bool);  // sets whether window is mapped to retina display