Merging the UTF8 patch, consisting of O'ksi'd s original 1.1.6 patch and additions by Ian. PLEASE BE AWARE that the patch in its current incarnation is a regression in many aspects and further work is required before we can announce Unicode support.

git-svn-id: file:///fltk/svn/fltk/branches/branch-1.3@6212 ea41ed52-d2ee-0310-a9c1-e6b18d33e121

1 files changed, 814 insertions, 0 deletions

diff --git a/src/fl_utf.c b/src/fl_utf.c
new file mode 100644
index 000000000..33b7a2cfc
--- /dev/null
+++ b/src/fl_utf.c
@@ -0,0 +1,814 @@
+/* This is the utf.c file from fltk2 adapted for use in my fltk1.1 port */
+
+/* Copyright 2006 by Bill Spitzak and others.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Please report all bugs and problems to "fltk-bugs@fltk.org".
+ */
+
+// Modified to obey rfc3629, which limits unicode to 0-0x10ffff
+
+#include <FL/fl_utf8.H>
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Set to 1 to turn bad UTF8 bytes into ISO-8859-1. If this is to zero
+   they are instead turned into the Unicode REPLACEMENT CHARACTER, of
+   value 0xfffd.
+   If this is on fl_utf8decode will correctly map most (perhaps all)
+   human-readable text that is in ISO-8859-1. This may allow you
+   to completely ignore character sets in your code because virtually
+   everything is either ISO-8859-1 or UTF-8.
+*/
+#define ERRORS_TO_ISO8859_1 1
+
+/* Set to 1 to turn bad UTF8 bytes in the 0x80-0x9f range into the
+   Unicode index for Microsoft's CP1252 character set. You should
+   also set ERRORS_TO_ISO8859_1. With this a huge amount of more
+   available text (such as all web pages) are correctly converted
+   to Unicode.
+*/
+#define ERRORS_TO_CP1252 1
+
+/* A number of Unicode code points are in fact illegal and should not
+   be produced by a UTF-8 converter. Turn this on will replace the
+   bytes in those encodings with errors. If you do this then converting
+   arbitrary 16-bit data to UTF-8 and then back is not an identity,
+   which will probably break a lot of software.
+*/
+#define STRICT_RFC3629 0
+
+#if ERRORS_TO_CP1252
+// Codes 0x80..0x9f from the Microsoft CP1252 character set, translated
+// to Unicode:
+static unsigned short cp1252[32] = {
+  0x20ac, 0x0081, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021,
+  0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008d, 0x017d, 0x008f,
+  0x0090, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014,
+  0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0x009d, 0x017e, 0x0178
+};
+#endif
+
+/*! Decode a single UTF-8 encoded character starting at \e p. The
+    resulting Unicode value (in the range 0-0x10ffff) is returned,
+    and \e len is set the the number of bytes in the UTF-8 encoding
+    (adding \e len to \e p will point at the next character).
+
+    If \a p points at an illegal UTF-8 encoding, including one that
+    would go past \e end, or where a code is uses more bytes than
+    necessary, then *(unsigned char*)p is translated as though it is
+    in the Microsoft CP1252 character set and \e len is set to 1.
+    Treating errors this way allows this to decode almost any
+    ISO-8859-1 or CP1252 text that has been mistakenly placed where
+    UTF-8 is expected, and has proven very useful.
+
+    If you want errors to be converted to error characters (as the
+    standards recommend), adding a test to see if the length is
+    unexpectedly 1 will work:
+
+\code
+    if (*p & 0x80) { // what should be a multibyte encoding
+      code = fl_utf8decode(p,end,&len);
+      if (len<2) code = 0xFFFD; // Turn errors into REPLACEMENT CHARACTER
+    } else { // handle the 1-byte utf8 encoding:
+      code = *p;
+      len = 1;
+    }
+\endcode
+
+    Direct testing for the 1-byte case (as shown above) will also
+    speed up the scanning of strings where the majority of characters
+    are ASCII.
+*/
+unsigned fl_utf8decode(const char* p, const char* end, int* len)
+{
+  unsigned char c = *(unsigned char*)p;
+  if (c < 0x80) {
+    *len = 1;
+    return c;
+#if ERRORS_TO_CP1252
+  } else if (c < 0xa0) {
+    *len = 1;
+    return cp1252[c-0x80];
+#endif
+  } else if (c < 0xc2) {
+    goto FAIL;
+  }
+  if (p+1 >= end || (p[1]&0xc0) != 0x80) goto FAIL;
+  if (c < 0xe0) {
+    *len = 2;
+    return
+      ((p[0] & 0x1f) << 6) +
+      ((p[1] & 0x3f));
+  } else if (c == 0xe0) {
+    if (((unsigned char*)p)[1] < 0xa0) goto FAIL;
+    goto UTF8_3;
+#if STRICT_RFC3629
+  } else if (c == 0xed) {
+    // RFC 3629 says surrogate chars are illegal.
+    if (((unsigned char*)p)[1] >= 0xa0) goto FAIL;
+    goto UTF8_3;
+  } else if (c == 0xef) {
+    // 0xfffe and 0xffff are also illegal characters
+    if (((unsigned char*)p)[1]==0xbf &&
+	((unsigned char*)p)[2]>=0xbe) goto FAIL;
+    goto UTF8_3;
+#endif
+  } else if (c < 0xf0) {
+  UTF8_3:
+    if (p+2 >= end || (p[2]&0xc0) != 0x80) goto FAIL;
+    *len = 3;
+    return
+      ((p[0] & 0x0f) << 12) +
+      ((p[1] & 0x3f) << 6) +
+      ((p[2] & 0x3f));
+  } else if (c == 0xf0) {
+    if (((unsigned char*)p)[1] < 0x90) goto FAIL;
+    goto UTF8_4;
+  } else if (c < 0xf4) {
+  UTF8_4:
+    if (p+3 >= end || (p[2]&0xc0) != 0x80 || (p[3]&0xc0) != 0x80) goto FAIL;
+    *len = 4;
+#if STRICT_RFC3629
+    // RFC 3629 says all codes ending in fffe or ffff are illegal:
+    if ((p[1]&0xf)==0xf &&
+	((unsigned char*)p)[2] == 0xbf &&
+	((unsigned char*)p)[3] >= 0xbe) goto FAIL;
+#endif
+    return
+      ((p[0] & 0x07) << 18) +
+      ((p[1] & 0x3f) << 12) +
+      ((p[2] & 0x3f) << 6) +
+      ((p[3] & 0x3f));
+  } else if (c == 0xf4) {
+    if (((unsigned char*)p)[1] > 0x8f) goto FAIL; // after 0x10ffff
+    goto UTF8_4;
+  } else {
+  FAIL:
+    *len = 1;
+#if ERRORS_TO_ISO8859_1
+    return c;
+#else
+    return 0xfffd; // Unicode REPLACEMENT CHARACTER
+#endif
+  }
+}
+
+/*! Move \a p forward until it points to the start of a UTF-8
+  character. If it already points at the start of one then it
+  is returned unchanged. Any UTF-8 errors are treated as though each
+  byte of the error is an individual character.
+
+  \e start is the start of the string and is used to limit the
+  backwards search for the start of a utf8 character.
+
+  \e end is the end of the string and is assummed to be a break
+  between characters. It is assummed to be greater than p.
+
+  This function is for moving a pointer that was jumped to the
+  middle of a string, such as when doing a binary search for
+  a position. You should use either this or fl_utf8back() depending
+  on which direction your algorithim can handle the pointer
+  moving. Do not use this to scan strings, use fl_utf8decode()
+  instead.
+*/
+const char* fl_utf8fwd(const char* p, const char* start, const char* end)
+{
+  const char* a;
+  int len;
+  // if we are not pointing at a continuation character, we are done:
+  if ((*p&0xc0) != 0x80) return p;
+  // search backwards for a 0xc0 starting the character:
+  for (a = p-1; ; --a) {
+    if (a < start) return p;
+    if (!(a[0]&0x80)) return p;
+    if ((a[0]&0x40)) break;
+  }
+  fl_utf8decode(a,end,&len);
+  a += len;
+  if (a > p) return a;
+  return p;
+}
+
+/*! Move \a p backward until it points to the start of a UTF-8
+  character. If it already points at the start of one then it
+  is returned unchanged. Any UTF-8 errors are treated as though each
+  byte of the error is an individual character.
+
+  \e start is the start of the string and is used to limit the
+  backwards search for the start of a UTF-8 character.
+
+  \e end is the end of the string and is assummed to be a break
+  between characters. It is assummed to be greater than p.
+
+  If you wish to decrement a UTF-8 pointer, pass p-1 to this.
+*/
+const char* fl_utf8back(const char* p, const char* start, const char* end)
+{
+  const char* a;
+  int len;
+  // if we are not pointing at a continuation character, we are done:
+  if ((*p&0xc0) != 0x80) return p;
+  // search backwards for a 0xc0 starting the character:
+  for (a = p-1; ; --a) {
+    if (a < start) return p;
+    if (!(a[0]&0x80)) return p;
+    if ((a[0]&0x40)) break;
+  }
+  fl_utf8decode(a,end,&len);
+  if (a+len > p) return a;
+  return p;
+}
+
+/*! Returns number of bytes that utf8encode() will use to encode the
+  character \a ucs. */
+int fl_utf8bytes(unsigned ucs) {
+  if (ucs < 0x000080U) {
+    return 1;
+  } else if (ucs < 0x000800U) {
+    return 2;
+  } else if (ucs < 0x010000U) {
+    return 3;
+  } else if (ucs < 0x10ffffU) {
+    return 4;
+  } else {
+    return 3; // length of the illegal character encoding
+  }
+}
+
+/*! Write the UTF-8 encoding of \e ucs into \e buf and return the
+    number of bytes written. Up to 4 bytes may be written. If you know
+    that \a ucs is less than 0x10000 then at most 3 bytes will be written.
+    If you wish to speed this up, remember that anything less than 0x80
+    is written as a single byte.
+
+    If ucs is greater than 0x10ffff this is an illegal character
+    according to RFC 3629. These are converted as though they are
+    0xFFFD (REPLACEMENT CHARACTER).
+
+    RFC 3629 also says many other values for \a ucs are illegal (in
+    the range 0xd800 to 0xdfff, or ending with 0xfffe or
+    0xffff). However I encode these as though they are legal, so that
+    utf8encode/fl_utf8decode will be the identity for all codes between 0
+    and 0x10ffff.
+*/
+int fl_utf8encode(unsigned ucs, char* buf) {
+  if (ucs < 0x000080U) {
+    buf[0] = ucs;
+    return 1;
+  } else if (ucs < 0x000800U) {
+    buf[0] = 0xc0 | (ucs >> 6);
+    buf[1] = 0x80 | (ucs & 0x3F);
+    return 2;
+  } else if (ucs < 0x010000U) {
+    buf[0] = 0xe0 | (ucs >> 12);
+    buf[1] = 0x80 | ((ucs >> 6) & 0x3F);
+    buf[2] = 0x80 | (ucs & 0x3F);
+    return 3;
+  } else if (ucs < 0x0010ffffU) {
+    buf[0] = 0xf0 | (ucs >> 18);
+    buf[1] = 0x80 | ((ucs >> 12) & 0x3F);
+    buf[2] = 0x80 | ((ucs >> 6) & 0x3F);
+    buf[3] = 0x80 | (ucs & 0x3F);
+    return 4;
+  } else {
+    // encode 0xfffd:
+    buf[0] = 0xefU;
+    buf[1] = 0xbfU;
+    buf[2] = 0xbdU;
+    return 3;
+  }
+}
+
+/*! Convert a UTF-8 sequence into an array of wchar_t. These
+    are used by some system calls, especially on Windows.
+
+    \a src points at the UTF-8, and \a srclen is the number of bytes to
+    convert.
+
+    \a dst points at an array to write, and \a dstlen is the number of
+    locations in this array. At most \a dstlen-1 words will be
+    written there, plus a 0 terminating word. Thus this function
+    will never overwrite the buffer and will always return a
+    zero-terminated string. If \a dstlen is zero then \a dst can be
+    null and no data is written, but the length is returned.
+
+    The return value is the number of words that \e would be written
+    to \a dst if it were long enough, not counting the terminating
+    zero. If the return value is greater or equal to \a dstlen it
+    indicates truncation, you can then allocate a new array of size
+    return+1 and call this again.
+
+    Errors in the UTF-8 are converted as though each byte in the
+    erroneous string is in the Microsoft CP1252 encoding. This allows
+    ISO-8859-1 text mistakenly identified as UTF-8 to be printed
+    correctly.
+
+    Notice that sizeof(wchar_t) is 2 on Windows and is 4 on Linux
+    and most other systems. Where wchar_t is 16 bits, Unicode
+    characters in the range 0x10000 to 0x10ffff are converted to
+    "surrogate pairs" which take two words each (this is called UTF-16
+    encoding). If wchar_t is 32 bits this rather nasty problem is
+    avoided.
+*/
+unsigned fl_utf8toUtf16(const char* src, unsigned srclen,
+		  unsigned short* dst, unsigned dstlen)
+{
+  const char* p = src;
+  const char* e = src+srclen;
+  unsigned count = 0;
+  if (dstlen) for (;;) {
+    if (p >= e) {dst[count] = 0; return count;}
+    if (!(*p & 0x80)) { // ascii
+      dst[count] = *p++;
+    } else {
+      int len; unsigned ucs = fl_utf8decode(p,e,&len);
+      p += len;
+      if (ucs < 0x10000) {
+	dst[count] = ucs;
+      } else {
+	// make a surrogate pair:
+	if (count+2 >= dstlen) {dst[count] = 0; count += 2; break;}
+	dst[count] = (((ucs-0x10000u)>>10)&0x3ff) | 0xd800;
+	dst[++count] = (ucs&0x3ff) | 0xdc00;
+      }
+    }
+    if (++count == dstlen) {dst[count-1] = 0; break;}
+  }
+  // we filled dst, measure the rest:
+  while (p < e) {
+    if (!(*p & 0x80)) p++;
+    else {
+      int len; unsigned ucs = fl_utf8decode(p,e,&len);
+      p += len;
+      if (ucs >= 0x10000) ++count;
+    }
+    ++count;
+  }
+  return count;
+}
+
+
+/* This function generates 32-bit wchar_t (e.g. "ucs4" as it were) except on win32 where
+   it returns Utf16 with surrogate pairs where required. */
+unsigned fl_utf8towc(const char* src, unsigned srclen,
+		  wchar_t* dst, unsigned dstlen)
+{
+#ifdef _WIN32
+	return fl_utf8toUtf16(src, srclen, (unsigned short*)dst, dstlen);
+#else
+  const char* p = src;
+  const char* e = src+srclen;
+  unsigned count = 0;
+  if (dstlen) for (;;) {
+    if (p >= e) {dst[count] = 0; return count;}
+    if (!(*p & 0x80)) { // ascii
+      dst[count] = *p++;
+    } else {
+      int len; unsigned ucs = fl_utf8decode(p,e,&len);
+      p += len;
+      dst[count] = (wchar_t)ucs;
+    }
+    if (++count == dstlen) {dst[count-1] = 0; break;}
+  }
+  // we filled dst, measure the rest:
+  while (p < e) {
+    if (!(*p & 0x80)) p++;
+    else {
+      int len; fl_utf8decode(p,e,&len);
+      p += len;
+    }
+    ++count;
+  }
+  return count;
+#endif
+}
+
+/*! Convert a UTF-8 sequence into an array of 1-byte characters.
+
+    If the UTF-8 decodes to a character greater than 0xff then it is
+    replaced with '?'.
+
+    Errors in the UTF-8 are converted as individual bytes, same as
+    fl_utf8decode() does. This allows ISO-8859-1 text mistakenly identified
+    as UTF-8 to be printed correctly (and possibly CP1512 on Windows).
+
+    \a src points at the UTF-8, and \a srclen is the number of bytes to
+    convert.
+
+    Up to \a dstlen bytes are written to \a dst, including a null
+    terminator. The return value is the number of bytes that would be
+    written, not counting the null terminator. If greater or equal to
+    \a dstlen then if you malloc a new array of size n+1 you will have
+    the space needed for the entire string. If \a dstlen is zero then
+    nothing is written and this call just measures the storage space
+    needed.
+*/
+unsigned fl_utf8toa(const char* src, unsigned srclen,
+		 char* dst, unsigned dstlen)
+{
+  const char* p = src;
+  const char* e = src+srclen;
+  unsigned count = 0;
+  if (dstlen) for (;;) {
+    unsigned char c;
+    if (p >= e) {dst[count] = 0; return count;}
+    c = *(unsigned char*)p;
+    if (c < 0xC2) { // ascii or bad code
+      dst[count] = c;
+      p++;
+    } else {
+      int len; unsigned ucs = fl_utf8decode(p,e,&len);
+      p += len;
+      if (ucs < 0x100) dst[count] = ucs;
+      else dst[count] = '?';
+    }
+    if (++count >= dstlen) {dst[count-1] = 0; break;}
+  }
+  // we filled dst, measure the rest:
+  while (p < e) {
+    if (!(*p & 0x80)) p++;
+    else {
+      int len;
+      fl_utf8decode(p,e,&len);
+      p += len;
+    }
+    ++count;
+  }
+  return count;
+}
+
+/*! Turn "wide characters" as returned by some system calls
+    (especially on Windows) into UTF-8.
+
+    Up to \a dstlen bytes are written to \a dst, including a null
+    terminator. The return value is the number of bytes that would be
+    written, not counting the null terminator. If greater or equal to
+    \a dstlen then if you malloc a new array of size n+1 you will have
+    the space needed for the entire string. If \a dstlen is zero then
+    nothing is written and this call just measures the storage space
+    needed.
+
+    \a srclen is the number of words in \a src to convert. On Windows
+    this is not necessairly the number of characters, due to there
+    possibly being "surrogate pairs" in the UTF-16 encoding used.
+    On Unix wchar_t is 32 bits and each location is a character.
+
+    On Unix if a src word is greater than 0x10ffff then this is an
+    illegal character according to RFC 3629. These are converted as
+    though they are 0xFFFD (REPLACEMENT CHARACTER). Characters in the
+    range 0xd800 to 0xdfff, or ending with 0xfffe or 0xffff are also
+    illegal according to RFC 3629. However I encode these as though
+    they are legal, so that fl_utf8towc will return the original data.
+
+    On Windows "surrogate pairs" are converted to a single character
+    and UTF-8 encoded (as 4 bytes). Mismatched halves of surrogate
+    pairs are converted as though they are individual characters.
+*/
+unsigned fl_utf8fromwc(char* dst, unsigned dstlen,
+		    const wchar_t* src, unsigned srclen) {
+  unsigned i = 0;
+  unsigned count = 0;
+  if (dstlen) for (;;) {
+    unsigned ucs;
+    if (i >= srclen) {dst[count] = 0; return count;}
+    ucs = src[i++];
+    if (ucs < 0x80U) {
+      dst[count++] = ucs;
+      if (count >= dstlen) {dst[count-1] = 0; break;}
+    } else if (ucs < 0x800U) { // 2 bytes
+      if (count+2 >= dstlen) {dst[count] = 0; count += 2; break;}
+      dst[count++] = 0xc0 | (ucs >> 6);
+      dst[count++] = 0x80 | (ucs & 0x3F);
+#ifdef _WIN32
+    } else if (ucs >= 0xd800 && ucs <= 0xdbff && i < srclen &&
+	       src[i] >= 0xdc00 && src[i] <= 0xdfff) {
+      // surrogate pair
+      unsigned ucs2 = src[i++];
+      ucs = 0x10000U + ((ucs&0x3ff)<<10) + (ucs2&0x3ff);
+      // all surrogate pairs turn into 4-byte utf8
+#else
+    } else if (ucs >= 0x10000) {
+      if (ucs > 0x10ffff) {
+	ucs = 0xfffd;
+	goto J1;
+      }
+#endif
+      if (count+4 >= dstlen) {dst[count] = 0; count += 4; break;}
+      dst[count++] = 0xf0 | (ucs >> 18);
+      dst[count++] = 0x80 | ((ucs >> 12) & 0x3F);
+      dst[count++] = 0x80 | ((ucs >> 6) & 0x3F);
+      dst[count++] = 0x80 | (ucs & 0x3F);
+    } else {
+#ifndef _WIN32
+    J1:
+#endif
+      // all others are 3 bytes:
+      if (count+3 >= dstlen) {dst[count] = 0; count += 3; break;}
+      dst[count++] = 0xe0 | (ucs >> 12);
+      dst[count++] = 0x80 | ((ucs >> 6) & 0x3F);
+      dst[count++] = 0x80 | (ucs & 0x3F);
+    }
+  }
+  // we filled dst, measure the rest:
+  while (i < srclen) {
+    unsigned ucs = src[i++];
+    if (ucs < 0x80U) {
+      count++;
+    } else if (ucs < 0x800U) { // 2 bytes
+      count += 2;
+#ifdef _WIN32
+    } else if (ucs >= 0xd800 && ucs <= 0xdbff && i < srclen-1 &&
+	       src[i+1] >= 0xdc00 && src[i+1] <= 0xdfff) {
+      // surrogate pair
+      ++i;
+#else
+    } else if (ucs >= 0x10000 && ucs <= 0x10ffff) {
+#endif
+      count += 4;
+    } else {
+      count += 3;
+    }
+  }
+  return count;
+}
+
+/*! Convert an ISO-8859-1 (ie normal c-string) byte stream to UTF-8.
+
+    It is possible this should convert Microsoft's CP1252 to UTF-8
+    instead. This would translate the codes in the range 0x80-0x9f
+    to different characters. Currently it does not do this.
+
+    Up to \a dstlen bytes are written to \a dst, including a null
+    terminator. The return value is the number of bytes that would be
+    written, not counting the null terminator. If greater or equal to
+    \a dstlen then if you malloc a new array of size n+1 you will have
+    the space needed for the entire string. If \a dstlen is zero then
+    nothing is written and this call just measures the storage space
+    needed.
+
+    \a srclen is the number of bytes in \a src to convert.
+
+    If the return value equals \a srclen then this indicates that
+    no conversion is necessary, as only ASCII characters are in the
+    string.
+*/
+unsigned fl_utf8froma(char* dst, unsigned dstlen,
+		   const char* src, unsigned srclen) {
+  const char* p = src;
+  const char* e = src+srclen;
+  unsigned count = 0;
+  if (dstlen) for (;;) {
+    unsigned char ucs;
+    if (p >= e) {dst[count] = 0; return count;}
+    ucs = *(unsigned char*)p++;
+    if (ucs < 0x80U) {
+      dst[count++] = ucs;
+      if (count >= dstlen) {dst[count-1] = 0; break;}
+    } else { // 2 bytes (note that CP1252 translate could make 3 bytes!)
+      if (count+2 >= dstlen) {dst[count] = 0; count += 2; break;}
+      dst[count++] = 0xc0 | (ucs >> 6);
+      dst[count++] = 0x80 | (ucs & 0x3F);
+    }
+  }
+  // we filled dst, measure the rest:
+  while (p < e) {
+    unsigned char ucs = *(unsigned char*)p++;
+    if (ucs < 0x80U) {
+      count++;
+    } else {
+      count += 2;
+    }
+  }
+  return count;
+}
+
+#ifdef _WIN32
+# include <windows.h>
+#endif
+
+/*! Return true if the "locale" seems to indicate that UTF-8 encoding
+    is used. If true the fl_utf8to_mb and fl_utf8from_mb don't do anything
+    useful.
+
+    <i>It is highly recommended that you change your system so this
+    does return true.</i> On Windows this is done by setting the
+    "codepage" to CP_UTF8.  On Unix this is done by setting $LC_CTYPE
+    to a string containing the letters "utf" or "UTF" in it, or by
+    deleting all $LC* and $LANG environment variables. In the future
+    it is likely that all non-Asian Unix systems will return true,
+    due to the compatability of UTF-8 with ISO-8859-1.
+*/
+int fl_utf8locale(void) {
+  static int ret = 2;
+  if (ret == 2) {
+#ifdef _WIN32
+    ret = GetACP() == CP_UTF8;
+#else
+    char* s;
+    ret = 1; // assumme UTF-8 if no locale
+    if (((s = getenv("LC_CTYPE")) && *s) ||
+	((s = getenv("LC_ALL"))   && *s) ||
+	((s = getenv("LANG"))     && *s)) {
+      ret = (strstr(s,"utf") || strstr(s,"UTF"));
+    }
+#endif
+  }
+  return ret;
+}
+
+/*! Convert the UTF-8 used by FLTK to the locale-specific encoding
+    used for filenames (and sometimes used for data in files).
+    Unfortunatley due to stupid design you will have to do this as
+    needed for filenames. This is a bug on both Unix and Windows.
+
+    Up to \a dstlen bytes are written to \a dst, including a null
+    terminator. The return value is the number of bytes that would be
+    written, not counting the null terminator. If greater or equal to
+    \a dstlen then if you malloc a new array of size n+1 you will have
+    the space needed for the entire string. If \a dstlen is zero then
+    nothing is written and this call just measures the storage space
+    needed.
+
+    If fl_utf8locale() returns true then this does not change the data.
+    It is copied and truncated as necessary to
+    the destination buffer and \a srclen is always returned.  */
+unsigned fl_utf8to_mb(const char* src, unsigned srclen,
+		  char* dst, unsigned dstlen)
+{
+  if (!fl_utf8locale()) {
+#ifdef _WIN32
+    wchar_t lbuf[1024];
+    wchar_t* buf = lbuf;
+    unsigned length = fl_utf8towc(src, srclen, buf, 1024);
+    unsigned ret;
+    if (length >= 1024) {
+      buf = (wchar_t*)(malloc((length+1)*sizeof(wchar_t)));
+      fl_utf8towc(src, srclen, buf, length+1);
+    }
+    if (dstlen) {
+      // apparently this does not null-terminate, even though msdn
+      // documentation claims it does:
+      ret =
+        WideCharToMultiByte(GetACP(), 0, buf, length, dst, dstlen, 0, 0);
+      dst[ret] = 0;
+    }
+    // if it overflows or measuring length, get the actual length:
+    if (dstlen==0 || ret >= dstlen-1)
+      ret =
+	WideCharToMultiByte(GetACP(), 0, buf, length, 0, 0, 0, 0);
+    if (buf != lbuf) free((void*)buf);
+    return ret;
+#else
+    wchar_t lbuf[1024];
+    wchar_t* buf = lbuf;
+    unsigned length = fl_utf8towc(src, srclen, buf, 1024);
+    int ret;
+    if (length >= 1024) {
+      buf = (wchar_t*)(malloc((length+1)*sizeof(wchar_t)));
+      fl_utf8towc(src, srclen, buf, length+1);
+    }
+    if (dstlen) {
+      ret = wcstombs(dst, buf, dstlen);
+      if (ret >= dstlen-1) ret = wcstombs(0,buf,0);
+    } else {
+      ret = wcstombs(0,buf,0);
+    }
+    if (buf != lbuf) free((void*)buf);
+    if (ret >= 0) return (unsigned)ret;
+    // on any errors we return the UTF-8 as raw text...
+#endif
+  }
+  // identity transform:
+  if (srclen < dstlen) {
+    memcpy(dst, src, srclen);
+    dst[srclen] = 0;
+  } else {
+    memcpy(dst, src, dstlen-1);
+    dst[dstlen-1] = 0;
+  }
+  return srclen;
+}
+
+/*! Convert a filename from the locale-specific multibyte encoding
+    used by Windows to UTF-8 as used by FLTK.
+
+    Up to \a dstlen bytes are written to \a dst, including a null
+    terminator. The return value is the number of bytes that would be
+    written, not counting the null terminator. If greater or equal to
+    \a dstlen then if you malloc a new array of size n+1 you will have
+    the space needed for the entire string. If \a dstlen is zero then
+    nothing is written and this call just measures the storage space
+    needed.
+
+    On Unix or on Windows when a UTF-8 locale is in effect, this
+    does not change the data. It is copied and truncated as necessary to
+    the destination buffer and \a srclen is always returned.
+    You may also want to check if fl_utf8test() returns non-zero, so that
+    the filesystem can store filenames in UTF-8 encoding regardless of
+    the locale.
+*/
+unsigned fl_utf8from_mb(char* dst, unsigned dstlen,
+		    const char* src, unsigned srclen)
+{
+  if (!fl_utf8locale()) {
+#ifdef _WIN32
+    wchar_t lbuf[1024];
+    wchar_t* buf = lbuf;
+    unsigned length;
+    unsigned ret;
+    length =
+      MultiByteToWideChar(GetACP(), 0, src, srclen, buf, 1024);
+    if (length >= 1024) {
+      length = MultiByteToWideChar(GetACP(), 0, src, srclen, 0, 0);
+      buf = (wchar_t*)(malloc(length*sizeof(wchar_t)));
+      MultiByteToWideChar(GetACP(), 0, src, srclen, buf, length);
+    }
+    ret = fl_utf8fromwc(dst, dstlen, buf, length);
+    if (buf != lbuf) free((void*)buf);
+    return ret;
+#else
+    wchar_t lbuf[1024];
+    wchar_t* buf = lbuf;
+    int length;
+    unsigned ret;
+    length = mbstowcs(buf, src, 1024);
+    if (length >= 1024) {
+      length = mbstowcs(0, src, 0)+1;
+      buf = (wchar_t*)(malloc(length*sizeof(unsigned short)));
+      mbstowcs(buf, src, length);
+    }
+    if (length >= 0) {
+      ret = fl_utf8fromwc(dst, dstlen, buf, length);
+      if (buf != lbuf) free((void*)buf);
+      return ret;
+    }
+    // errors in conversion return the UTF-8 unchanged
+#endif
+  }
+  // identity transform:
+  if (srclen < dstlen) {
+    memcpy(dst, src, srclen);
+    dst[srclen] = 0;
+  } else {
+    memcpy(dst, src, dstlen-1);
+    dst[dstlen-1] = 0;
+  }
+  return srclen;
+}
+
+/*! Examines the first \a srclen bytes in \a src and return a verdict
+    on whether it is UTF-8 or not.
+    - Returns 0 if there is any illegal UTF-8 sequences, using the
+      same rules as fl_utf8decode(). Note that some UCS values considered
+      illegal by RFC 3629, such as 0xffff, are considered legal by this.
+    - Returns 1 if there are only single-byte characters (ie no bytes
+      have the high bit set). This is legal UTF-8, but also indicates
+      plain ASCII. It also returns 1 if \a srclen is zero.
+    - Returns 2 if there are only characters less than 0x800.
+    - Returns 3 if there are only characters less than 0x10000.
+    - Returns 4 if there are characters in the 0x10000 to 0x10ffff range.
+
+    Because there are many illegal sequences in UTF-8, it is almost
+    impossible for a string in another encoding to be confused with
+    UTF-8. This is very useful for transitioning Unix to UTF-8
+    filenames, you can simply test each filename with this to decide
+    if it is UTF-8 or in the locale encoding. My hope is that if
+    this is done we will be able to cleanly transition to a locale-less
+    encoding.
+*/
+int fl_utf8test(const char* src, unsigned srclen) {
+  int ret = 1;
+  const char* p = src;
+  const char* e = src+srclen;
+  while (p < e) {
+    if (*p & 0x80) {
+      int len; fl_utf8decode(p,e,&len);
+      if (len < 2) return 0;
+      if (len > ret) ret = len;
+      p += len;
+    } else {
+      p++;
+    }
+  }
+  return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif