Fix "Huge memory leak under Wayland" (#730)

3 files changed, 52 insertions, 6 deletions

diff --git a/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.H b/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.H
index b9dbb78be..b6697df40 100644
--- a/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.H
+++ b/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.H
@@ -72,6 +72,12 @@ class Fl_Wayland_Graphics_Driver : public Fl_Cairo_Graphics_Driver {
 private:
   struct fl_wld_buffer *buffer_;
 public:
+  struct wld_shm_pool_data { // attached to wl_shm_pool objects
+    char *pool_memory; // start of mmap'ed memory encapsulated by the wl_shm_pool
+    int pool_size; // size of encapsulated memory
+    int use_count; // nber of active wl_buffer objects in this wl_shm_pool
+    bool destroyed; // true after the wl_shm_pool was destroyed
+  };
   Fl_Wayland_Graphics_Driver();
   static const uint32_t wld_format;
   void set_buffer(struct fl_wld_buffer *buffer, float scale = 0);
diff --git a/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.cxx b/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.cxx
index c8793d55d..cfe1ff494 100644
--- a/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.cxx
+++ b/src/drivers/Wayland/Fl_Wayland_Graphics_Driver.cxx
@@ -33,7 +33,6 @@ struct fl_wld_buffer *Fl_Wayland_Graphics_Driver::create_shm_buffer(int width, i
   struct fl_wld_buffer *buffer;
   int stride = cairo_format_stride_for_width(Fl_Cairo_Graphics_Driver::cairo_format, width);
   int size = stride * height;
-  static char *pool_memory = NULL;
   static int pool_size = 10000000; // gets increased if necessary
   static int chunk_offset = pool_size;
   static int fd = -1;
@@ -41,23 +40,39 @@ struct fl_wld_buffer *Fl_Wayland_Graphics_Driver::create_shm_buffer(int width, i
   if (chunk_offset + size > pool_size) {
     chunk_offset = 0;
     if (pool) {
+      struct wld_shm_pool_data *pool_data =
+        (struct wld_shm_pool_data *)wl_shm_pool_get_user_data(pool);
       wl_shm_pool_destroy(pool);
       close(fd);
+      pool_data->destroyed = true;
+      if (pool_data->use_count == 0) {
+        /*int err =*/ munmap(pool_data->pool_memory, pool_data->pool_size);
+//printf("munmap(%p)->%d\n", pool_data->pool_memory, err);
+        free(pool_data);
+      }
     }
     if (size > pool_size) pool_size = 2 * size;
     fd = os_create_anonymous_file(pool_size);
-    pool_memory = (char*)mmap(NULL, pool_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-    if (pool_memory == MAP_FAILED) {
+    struct wld_shm_pool_data *pool_data = (struct wld_shm_pool_data*)
+        calloc(1, sizeof(struct wld_shm_pool_data));
+    pool_data->pool_memory = (char*)mmap(NULL, pool_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (pool_data->pool_memory == MAP_FAILED) {
       close(fd);
       Fl::fatal("mmap failed: %s\n", strerror(errno));
     }
     Fl_Wayland_Screen_Driver *scr_driver = (Fl_Wayland_Screen_Driver*)Fl::screen_driver();
     pool = wl_shm_create_pool(scr_driver->wl_shm, fd, pool_size);
+    pool_data->pool_size = pool_size;
+    wl_shm_pool_set_user_data(pool, pool_data);
   }
   buffer = (struct fl_wld_buffer*)calloc(1, sizeof(struct fl_wld_buffer));
   buffer->stride = stride;
   buffer->wl_buffer = wl_shm_pool_create_buffer(pool, chunk_offset, width, height, stride, Fl_Wayland_Graphics_Driver::wld_format);
-  buffer->data = (void*)(pool_memory + chunk_offset);
+  struct wld_shm_pool_data *pool_data =
+    (struct wld_shm_pool_data *)wl_shm_pool_get_user_data(pool);
+  pool_data->use_count++;
+  wl_buffer_set_user_data(buffer->wl_buffer, pool_data);
+  buffer->data = (void*)(pool_data->pool_memory + chunk_offset);
   chunk_offset += size;
   buffer->data_size = size;
   buffer->width = width;
@@ -159,7 +174,16 @@ void Fl_Wayland_Graphics_Driver::buffer_release(struct wld_window *window)
 {
   if (window->buffer) {
     if (window->buffer->cb) wl_callback_destroy(window->buffer->cb);
+    struct wld_shm_pool_data *pool_data =
+      (struct wld_shm_pool_data *)wl_buffer_get_user_data(window->buffer->wl_buffer);
     wl_buffer_destroy(window->buffer->wl_buffer);
+//printf("wl_buffer_destroy(%p)\n",window->buffer->wl_buffer);
+    pool_data->use_count--;
+    if (pool_data->destroyed && pool_data->use_count == 0) {
+      /*int err =*/ munmap(pool_data->pool_memory, pool_data->pool_size);
+//printf("munmap(%p)->%d\n", pool_data->pool_memory, err);
+      free(pool_data);
+    }
     delete[] window->buffer->draw_buffer;
     window->buffer->draw_buffer = NULL;
     cairo_destroy(window->buffer->cairo_);
diff --git a/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx b/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx
index 182f2f4a0..dbd6acf32 100644
--- a/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx
+++ b/src/drivers/Wayland/Fl_Wayland_Window_Driver.cxx
@@ -70,12 +70,23 @@ Fl_Wayland_Window_Driver::Fl_Wayland_Window_Driver(Fl_Window *win) : Fl_Window_D
   subRect_ = NULL;
 }
 
+
+struct custom_cursor_data {
+  struct fl_wld_buffer *offscreen;
+  struct Fl_Wayland_Graphics_Driver::wld_shm_pool_data *buf_data;
+};
+
+
 void Fl_Wayland_Window_Driver::delete_cursor_(struct wld_window *xid, bool delete_rgb) {
   struct wld_window::custom_cursor_ *custom = xid->custom_cursor;
   if (custom) {
     struct wl_cursor *wl_cursor = custom->wl_cursor;
     struct cursor_image *new_image = (struct cursor_image*)wl_cursor->images[0];
-    struct fl_wld_buffer *offscreen = (struct fl_wld_buffer *)wl_buffer_get_user_data(new_image->buffer);
+    custom_cursor_data *buf_data =
+      (custom_cursor_data *)wl_buffer_get_user_data(new_image->buffer);
+    struct fl_wld_buffer *offscreen = buf_data->offscreen;
+    wl_buffer_set_user_data(new_image->buffer, buf_data->buf_data);
+    free(buf_data);
     struct wld_window fake_xid;
     fake_xid.buffer = offscreen;
     Fl_Wayland_Graphics_Driver::buffer_release(&fake_xid);
@@ -1554,7 +1565,12 @@ int Fl_Wayland_Window_Driver::set_cursor_4args(const Fl_RGB_Image *rgb, int hotx
   //create a Wayland buffer and have it used as an image of the new cursor
   struct fl_wld_buffer *offscreen = Fl_Wayland_Graphics_Driver::create_shm_buffer(new_image->image.width, new_image->image.height);
   new_image->buffer = offscreen->wl_buffer;
-  wl_buffer_set_user_data(new_image->buffer, offscreen);
+  custom_cursor_data *new_buf_data =
+    (custom_cursor_data*)malloc(sizeof(custom_cursor_data));
+  new_buf_data->buf_data = (struct Fl_Wayland_Graphics_Driver::wld_shm_pool_data *)
+      wl_buffer_get_user_data(new_image->buffer);
+  new_buf_data->offscreen = offscreen;
+  wl_buffer_set_user_data(new_image->buffer, new_buf_data);
   new_cursor->image_count = 1;
   new_cursor->images = (struct wl_cursor_image**)malloc(sizeof(struct wl_cursor_image*));
   new_cursor->images[0] = (struct wl_cursor_image*)new_image;