UTF8 Text Display and Editor: added tons of tests for utf8 alignment, fixed a bunch of methods that did not understand utf8. Still lots of places to visit.

git-svn-id: file:///fltk/svn/fltk/branches/branch-1.3@7800 ea41ed52-d2ee-0310-a9c1-e6b18d33e121

4 files changed, 198 insertions, 172 deletions

diff --git a/src/Fl_Text_Buffer.cxx b/src/Fl_Text_Buffer.cxx
index 565e6183b..5cbce5a4c 100644
--- a/src/Fl_Text_Buffer.cxx
+++ b/src/Fl_Text_Buffer.cxx
@@ -63,17 +63,11 @@
 
 #ifndef min
 
-/*
- Unicode safe.
- */
 static int max(int i1, int i2)
 {
   return i1 >= i2 ? i1 : i2;
 }
 
-/*
- Unicode safe.
- */
 static int min(int i1, int i2)
 {
   return i1 <= i2 ? i1 : i2;
@@ -92,7 +86,6 @@ static int undoyankcut;		// length of valid contents of buffer, even if undocut=
 
 /*
  Resize the undo buffer to match at least the requested size.
- Unicode safe.
  */
 static void undobuffersize(int n)
 {
@@ -112,7 +105,6 @@ static void undobuffersize(int n)
 
 /*
  Initialize all variables.
- Unicode safe.
  */
 Fl_Text_Buffer::Fl_Text_Buffer(int requestedSize, int preferredGapSize)
 {
@@ -141,7 +133,6 @@ Fl_Text_Buffer::Fl_Text_Buffer(int requestedSize, int preferredGapSize)
 
 /*
  Free all resources.
- Unicode safe.
  */
 Fl_Text_Buffer::~Fl_Text_Buffer()
 {
@@ -160,7 +151,6 @@ Fl_Text_Buffer::~Fl_Text_Buffer()
 /*
  This function copies verbose whatever is in front and after the gap into a
  single buffer.
- Unicode safe.
  */
 char *Fl_Text_Buffer::text() const {
   char *t = (char *) malloc(mLength + 1);
@@ -173,10 +163,11 @@ char *Fl_Text_Buffer::text() const {
 
 /*
  Set the text buffer to a new string.
- Unicode safe.
  */
 void Fl_Text_Buffer::text(const char *t)
 {
+  IS_UTF8_ALIGNED(t)
+  
   call_predelete_callbacks(0, length());
   
   /* Save information for redisplay, and get rid of the old buffer */
@@ -203,9 +194,11 @@ void Fl_Text_Buffer::text(const char *t)
 
 /*
  Creates a range of text to a new buffer and copies verbose from around the gap.
- Unicode safe.
  */
 char *Fl_Text_Buffer::text_range(int start, int end) const {
+  IS_UTF8_ALIGNED(address(start))
+  IS_UTF8_ALIGNED(address(start))
+  
   char *s = NULL;
   
   /* Make sure start and end are ok, and allocate memory for returned string.
@@ -242,11 +235,14 @@ char *Fl_Text_Buffer::text_range(int start, int end) const {
 
 /*
  Return a UCS-4 character at the given index.
- Unicode safe. Pos must be at a character boundary.
+ Pos must be at a character boundary.
  */
-unsigned int Fl_Text_Buffer::char_at(int pos) const {
+unsigned int Fl_Text_Buffer::char_at(int pos) const {  
   if (pos < 0 || pos >= mLength)
     return '\0';
+  
+  IS_UTF8_ALIGNED(address(pos))
+  
   const char *src = address(pos);
   return fl_utf8decode(src, 0, 0);
 } 
@@ -266,10 +262,13 @@ char Fl_Text_Buffer::byte_at(int pos) const {
 
 /*
  Insert some text at the given index.
- Unicode safe. Pos must be at a character boundary.
+ Pos must be at a character boundary.
 */
 void Fl_Text_Buffer::insert(int pos, const char *text)
 {
+  IS_UTF8_ALIGNED(address(pos))
+  IS_UTF8_ALIGNED(text)
+  
   /* check if there is actually any text */
   if (!text || !*text)
     return;
@@ -286,13 +285,14 @@ void Fl_Text_Buffer::insert(int pos, const char *text)
   /* insert and redisplay */
   int nInserted = insert_(pos, text);
   mCursorPosHint = pos + nInserted;
+  IS_UTF8_ALIGNED(address(mCursorPosHint))
   call_modify_callbacks(pos, 0, nInserted, 0, NULL);
 }
 
 
 /*
  Replace a range of text with new text.
- Unicode safe. Start and end must be at a character boundary.
+ Start and end must be at a character boundary.
 */
 void Fl_Text_Buffer::replace(int start, int end, const char *text)
 {
@@ -303,6 +303,10 @@ void Fl_Text_Buffer::replace(int start, int end, const char *text)
     start = 0;
   if (end > mLength)
     end = mLength;
+
+  IS_UTF8_ALIGNED(address(start))
+  IS_UTF8_ALIGNED(address(end))
+  IS_UTF8_ALIGNED(text)
   
   call_predelete_callbacks(start, end - start);
   const char *deletedText = text_range(start, end);
@@ -316,7 +320,7 @@ void Fl_Text_Buffer::replace(int start, int end, const char *text)
 
 /*
  Remove a range of text.
- Unicode safe. Start and End must be at a character boundary.
+ Start and End must be at a character boundary.
 */
 void Fl_Text_Buffer::remove(int start, int end)
 {
@@ -334,6 +338,9 @@ void Fl_Text_Buffer::remove(int start, int end)
     end = mLength;
   if (end < 0)
     end = 0;
+
+  IS_UTF8_ALIGNED(address(start))
+  IS_UTF8_ALIGNED(address(end))  
   
   if (start == end)
     return;
@@ -350,11 +357,15 @@ void Fl_Text_Buffer::remove(int start, int end)
 
 /*
  Copy a range of text from another text buffer.
- Unicode safe. FromDtart, fromEnd, and toPos must be at a character boundary.
+ FromStart, fromEnd, and toPos must be at a character boundary.
  */
 void Fl_Text_Buffer::copy(Fl_Text_Buffer * fromBuf, int fromStart,
 			  int fromEnd, int toPos)
 {
+  IS_UTF8_ALIGNED(fromBuf->address(fromStart))
+  IS_UTF8_ALIGNED(fromBuf->address(fromEnd))
+  IS_UTF8_ALIGNED(address(toPos))
+  
   int copiedLength = fromEnd - fromStart;
   
   /* Prepare the buffer to receive the new text.  If the new text fits in
@@ -389,7 +400,7 @@ void Fl_Text_Buffer::copy(Fl_Text_Buffer * fromBuf, int fromStart,
 /*
  Take the previous changes and undo them. Return the previous
  cursor position in cursorPos. Returns 1 if the undo was applied.
- Unicode safe. CursorPos will be at a character boundary.
+ CursorPos will be at a character boundary.
  */ 
 int Fl_Text_Buffer::undo(int *cursorPos)
 {
@@ -431,7 +442,6 @@ int Fl_Text_Buffer::undo(int *cursorPos)
 
 /*
  Set a flag is undo function will work.
- Unicode safe.
  */
 void Fl_Text_Buffer::canUndo(char flag)
 {
@@ -448,7 +458,6 @@ void Fl_Text_Buffer::canUndo(char flag)
  Matt: I am not entirely sure why we need to trigger callbacks because
  tabs are only a graphical hint, not changing any text at all, but I leave
  this in here for back compatibility. 
- Unicode safe.
  */
 void Fl_Text_Buffer::tab_distance(int tabDist)
 {
@@ -469,10 +478,13 @@ void Fl_Text_Buffer::tab_distance(int tabDist)
 
 /*
  Select a range of text.
- Unicode safe. Start and End must be at a character boundary.
+ Start and End must be at a character boundary.
  */
 void Fl_Text_Buffer::select(int start, int end)
 {
+  IS_UTF8_ALIGNED(address(start))
+  IS_UTF8_ALIGNED(address(end))  
+  
   Fl_Text_Selection oldSelection = mPrimary;
   
   mPrimary.set(start, end);
@@ -482,7 +494,6 @@ void Fl_Text_Buffer::select(int start, int end)
 
 /*
  Clear the primary selection.
- Unicode safe.
  */
 void Fl_Text_Buffer::unselect()
 {
@@ -495,7 +506,6 @@ void Fl_Text_Buffer::unselect()
   
 /*
  Return the primary selection range.
- Unicode safe.
  */
 int Fl_Text_Buffer::selection_position(int *start, int *end)
 {
@@ -505,7 +515,6 @@ int Fl_Text_Buffer::selection_position(int *start, int *end)
 
 /*
  Return a copy of the selected text.
- Unicode safe.
  */
 char *Fl_Text_Buffer::selection_text()
 {
@@ -515,7 +524,6 @@ char *Fl_Text_Buffer::selection_text()
 
 /*
  Remove the selected text.
- Unicode safe.
  */
 void Fl_Text_Buffer::remove_selection()
 {
@@ -525,7 +533,6 @@ void Fl_Text_Buffer::remove_selection()
 
 /*
  Replace the selected text.
- Unicode safe.
  */
 void Fl_Text_Buffer::replace_selection(const char *text)
 {
@@ -535,7 +542,7 @@ void Fl_Text_Buffer::replace_selection(const char *text)
 
 /*
  Select text.
- Unicode safe. Start and End must be at a character boundary.
+ Start and End must be at a character boundary.
  */
 void Fl_Text_Buffer::secondary_select(int start, int end)
 {
@@ -548,7 +555,6 @@ void Fl_Text_Buffer::secondary_select(int start, int end)
 
 /*
  Deselect text.
- Unicode safe.
  */
 void Fl_Text_Buffer::secondary_unselect()
 {
@@ -561,7 +567,6 @@ void Fl_Text_Buffer::secondary_unselect()
   
 /*
  Return the selected range.
- Unicode safe.
  */
 int Fl_Text_Buffer::secondary_selection_position(int *start, int *end)
 {
@@ -571,7 +576,6 @@ int Fl_Text_Buffer::secondary_selection_position(int *start, int *end)
 
 /*
  Return a copy of the text in this selection.
- Unicode safe.
  */
 char *Fl_Text_Buffer::secondary_selection_text()
 {
@@ -581,7 +585,6 @@ char *Fl_Text_Buffer::secondary_selection_text()
 
 /*
  Remove the selected text.
- Unicode safe.
  */
 void Fl_Text_Buffer::remove_secondary_selection()
 {
@@ -591,7 +594,6 @@ void Fl_Text_Buffer::remove_secondary_selection()
 
 /*
  Replace selected text.
- Unicode safe.
  */
 void Fl_Text_Buffer::replace_secondary_selection(const char *text)
 {
@@ -601,7 +603,7 @@ void Fl_Text_Buffer::replace_secondary_selection(const char *text)
 
 /*
  Highlight a range of text.
- Unicode safe. Start and End must be at a character boundary.
+ Start and End must be at a character boundary.
  */
 void Fl_Text_Buffer::highlight(int start, int end)
 {
@@ -614,7 +616,6 @@ void Fl_Text_Buffer::highlight(int start, int end)
 
 /*
  Remove text highlighting.
- Unicode safe.
  */
 void Fl_Text_Buffer::unhighlight()
 {
@@ -627,7 +628,6 @@ void Fl_Text_Buffer::unhighlight()
   
 /*
  Return position of highlight.
- Unicode safe.
  */
 int Fl_Text_Buffer::highlight_position(int *start, int *end)
 {
@@ -637,7 +637,6 @@ int Fl_Text_Buffer::highlight_position(int *start, int *end)
 
 /*
  Return a copy of highlighted text.
- Unicode safe.
  */
 char *Fl_Text_Buffer::highlight_text()
 {
@@ -647,7 +646,6 @@ char *Fl_Text_Buffer::highlight_text()
 
 /*
  Add a callback that is called whenever text is modified.
- Unicode safe.
  */
 void Fl_Text_Buffer::add_modify_callback(Fl_Text_Modify_Cb bufModifiedCB,
 					 void *cbArg)
@@ -673,7 +671,6 @@ void Fl_Text_Buffer::add_modify_callback(Fl_Text_Modify_Cb bufModifiedCB,
 
 /*
  Remove a callback.
- Unicode safe.
  */
 void Fl_Text_Buffer::remove_modify_callback(Fl_Text_Modify_Cb bufModifiedCB, 
                                             void *cbArg)
@@ -725,7 +722,6 @@ void Fl_Text_Buffer::remove_modify_callback(Fl_Text_Modify_Cb bufModifiedCB,
 
 /*
  Add a callback that is called before deleting text.
- Unicode safe.
  */
 void Fl_Text_Buffer::add_predelete_callback(Fl_Text_Predelete_Cb bufPreDeleteCB, 
                                             void *cbArg)
@@ -751,7 +747,6 @@ void Fl_Text_Buffer::add_predelete_callback(Fl_Text_Predelete_Cb bufPreDeleteCB,
 
 /*
  Remove a callback.
- Unicode safe.
  */
 void Fl_Text_Buffer::remove_predelete_callback(Fl_Text_Predelete_Cb bufPreDeleteCB, void *cbArg)
 {
@@ -803,7 +798,7 @@ void Fl_Text_Buffer::remove_predelete_callback(Fl_Text_Predelete_Cb bufPreDelete
 
 /*
  Return a copy of the line that contains a given index.
- Unicode safe. Pos must be at a character boundary.
+ Pos must be at a character boundary.
  */
 char *Fl_Text_Buffer::line_text(int pos) const {
   return text_range(line_start(pos), line_end(pos));
@@ -812,11 +807,9 @@ char *Fl_Text_Buffer::line_text(int pos) const {
 
 /*
  Find the beginning of the line.
- NOT UNICODE SAFE.
  */
 int Fl_Text_Buffer::line_start(int pos) const 
 {
-  // FIXME: this currently works for unicode, but will be very inefficent when findchar_backward is fixed.
   if (!findchar_backward(pos, '\n', &pos))
     return 0;
   return pos + 1;
@@ -825,10 +818,8 @@ int Fl_Text_Buffer::line_start(int pos) const
 
 /*
  Find the end of the line.
- NOT UNICODE SAFE.
  */
 int Fl_Text_Buffer::line_end(int pos) const {
-  // FIXME: this currently works for unicode, but will be very inefficent when findchar_forward is fixed.
   if (!findchar_forward(pos, '\n', &pos))
     pos = mLength;
   return pos;
@@ -841,12 +832,12 @@ int Fl_Text_Buffer::line_end(int pos) const {
  */
 int Fl_Text_Buffer::word_start(int pos) const {
   // FIXME: character is ucs-4
-  while (pos && (isalnum(char_at(pos)) || char_at(pos) == '_')) {
-    pos--;
+  while (pos>0 && (isalnum(char_at(pos)) || char_at(pos) == '_')) {
+    pos = prev_char(pos);
   } 
   // FIXME: character is ucs-4
   if (!(isalnum(char_at(pos)) || char_at(pos) == '_'))
-    pos++;
+    pos = next_char(pos);
   return pos;
 }
 
@@ -859,7 +850,7 @@ int Fl_Text_Buffer::word_end(int pos) const {
   // FIXME: character is ucs-4
   while (pos < length() && (isalnum(char_at(pos)) || char_at(pos) == '_'))
   {
-    pos++;
+    pos = next_char(pos);
   } return pos;
 }
 
@@ -868,17 +859,20 @@ int Fl_Text_Buffer::word_end(int pos) const {
  Matt: I am not sure why we need this function. Does it still make sense in
  the world of proportional characters?
  */
+// FIXME: this is misleading and mey be used to count bytes instead of characters!
 int Fl_Text_Buffer::count_displayed_characters(int lineStartPos,
 					       int targetPos) const
 {
+  IS_UTF8_ALIGNED(address(lineStartPos))
+  IS_UTF8_ALIGNED(address(targetPos))
+  
   // TODO: is this function still needed? If it is, put this functionality in handle_vline?
   int charCount = 0;
   
   int pos = lineStartPos;
   while (pos < targetPos) {
-    int len = fl_utf8len(*address(pos));
-    charCount += 1;
-    pos += len;
+    pos = next_char(pos);
+    charCount++;
   }
   return charCount;
 } 
@@ -888,20 +882,20 @@ int Fl_Text_Buffer::count_displayed_characters(int lineStartPos,
  Matt: I am not sure why we need this function. Does it still make sense in
  the world of proportional characters?
  */
+// FIXME: this is misleading and mey be used to count bytes instead of characters!
 // All values are number of bytes. 
 // - unicode ok?
 int Fl_Text_Buffer::skip_displayed_characters(int lineStartPos, int nChars)
 {
+  IS_UTF8_ALIGNED(address(lineStartPos))
   // FIXME: is this function still needed?
   int pos = lineStartPos;
   
-  for (int charCount = 0; charCount < nChars && pos < mLength;) {
-    const char *src = address(pos);
-    char c = *src;
+  for (int charCount = 0; charCount < nChars && pos < mLength; charCount++) {
+    unsigned int c = char_at(pos);
     if (c == '\n')
       return pos;
-    charCount++;
-    pos += fl_utf8len(c);
+    pos = next_char(pos);
   }
   return pos;
 }
@@ -909,9 +903,13 @@ int Fl_Text_Buffer::skip_displayed_characters(int lineStartPos, int nChars)
 
 /*
  Count the number of newline characters between start and end.
- Unicode safe. StartPos and endPos must be at a character boundary.
+ StartPos and endPos must be at a character boundary.
+ This function is optimized for speed by not using UTF-8 calls.
  */
 int Fl_Text_Buffer::count_lines(int startPos, int endPos) const {
+  IS_UTF8_ALIGNED(address(startPos))
+  IS_UTF8_ALIGNED(address(endPos))
+  
   int gapLen = mGapEnd - mGapStart;
   int lineCount = 0;
   
@@ -935,10 +933,13 @@ int Fl_Text_Buffer::count_lines(int startPos, int endPos) const {
 
 /*
  Skip to the first character, n lines ahead.
- Unicode safe. StartPos must be at a character boundary.
+ StartPos must be at a character boundary.
+ This function is optimized for speed by not using UTF-8 calls.
  */
 int Fl_Text_Buffer::skip_lines(int startPos, int nLines)
 {
+  IS_UTF8_ALIGNED(address(startPos))
+  
   if (nLines == 0)
     return startPos;
   
@@ -948,27 +949,35 @@ int Fl_Text_Buffer::skip_lines(int startPos, int nLines)
   while (pos < mGapStart) {
     if (mBuf[pos++] == '\n') {
       lineCount++;
-      if (lineCount == nLines)
+      if (lineCount == nLines) {
+        IS_UTF8_ALIGNED(address(pos))
 	return pos;
+      }
     }
   }
   while (pos < mLength) {
     if (mBuf[pos++ + gapLen] == '\n') {
       lineCount++;
-      if (lineCount >= nLines)
+      if (lineCount >= nLines) {
+        IS_UTF8_ALIGNED(address(pos))
 	return pos;
+      }
     }
   }
+  IS_UTF8_ALIGNED(address(pos))
   return pos;
 }
 
 
 /*
  Skip to the first character, n lines back.
- Unicode safe. StartPos must be at a character boundary.
+ StartPos must be at a character boundary.
+ This function is optimized for speed by not using UTF-8 calls.
  */
 int Fl_Text_Buffer::rewind_lines(int startPos, int nLines)
 {
+  IS_UTF8_ALIGNED(address(startPos))
+  
   int pos = startPos - 1;
   if (pos <= 0)
     return 0;
@@ -977,15 +986,19 @@ int Fl_Text_Buffer::rewind_lines(int startPos, int nLines)
   int lineCount = -1;
   while (pos >= mGapStart) {
     if (mBuf[pos + gapLen] == '\n') {
-      if (++lineCount >= nLines)
+      if (++lineCount >= nLines) {
+        IS_UTF8_ALIGNED(address(pos+1))
 	return pos + 1;
+      }
     }
     pos--;
   }
   while (pos >= 0) {
     if (mBuf[pos] == '\n') {
-      if (++lineCount >= nLines)
+      if (++lineCount >= nLines) {
+        IS_UTF8_ALIGNED(address(pos+1))
 	return pos + 1;
+      }
     }
     pos--;
   }
@@ -995,30 +1008,56 @@ int Fl_Text_Buffer::rewind_lines(int startPos, int nLines)
 
 /*
  Find a matching string in the buffer.
- NOT TESTED FOR UNICODE.
  */
 int Fl_Text_Buffer::search_forward(int startPos, const char *searchString,
 				   int *foundPos, int matchCase) const 
 {
-  // FIXME: Unicode?
+  IS_UTF8_ALIGNED(address(startPos))
+  IS_UTF8_ALIGNED(searchString)
+  
   if (!searchString)
     return 0;
   int bp;
   const char *sp;
-  while (startPos < length()) {
-    bp = startPos;
-    sp = searchString;
-    do {
-      if (!*sp) {
-        *foundPos = startPos;
-        return 1;
+  if (matchCase) {
+    while (startPos < length()) {
+      bp = startPos;
+      sp = searchString;
+      for (;;) {
+        char c = *sp;
+        // we reached the end of the "needle", so we found the string!
+        if (!c) {
+          *foundPos = startPos;
+          return 1;
+        }
+        int l = fl_utf8len(c);
+        if (memcmp(sp, address(bp), l))
+          break;
+        sp += l; bp += l;
       }
-      // FIXME: character is ucs-4
-    } while ((matchCase ? char_at(bp++) == (unsigned int)*sp++ :
-              toupper(char_at(bp++)) == toupper(*sp++))
-             && bp < length());
-    startPos++;
-  }
+      startPos = next_char(startPos);
+    }
+  } else {
+    while (startPos < length()) {
+      bp = startPos;
+      sp = searchString;
+      for (;;) {
+        // we reached the end of the "needle", so we found the string!
+        if (!*sp) {
+          *foundPos = startPos;
+          return 1;
+        }
+        int l;
+        unsigned int b = char_at(bp);
+        unsigned int s = fl_utf8decode(sp, 0, &l);
+        if (fl_tolower(b)!=fl_tolower(s))
+          break;
+        sp += l; 
+        bp = next_char(bp);
+      }
+      startPos = next_char(startPos);
+    }
+  }  
   return 0;
 }
 
@@ -1286,9 +1325,9 @@ void Fl_Text_Buffer::remove_selection_(Fl_Text_Selection * sel)
   
   if (!sel->position(&start, &end))
     return;
-    remove(start, end);
-    //undoyankcut = undocut;
-  }
+  remove(start, end);
+  //undoyankcut = undocut;
+}
 
 
 /*
@@ -1322,6 +1361,7 @@ void Fl_Text_Buffer::replace_selection_(Fl_Text_Selection * sel,
 void Fl_Text_Buffer::call_modify_callbacks(int pos, int nDeleted,
 					   int nInserted, int nRestyled,
 					   const char *deletedText) const {
+  IS_UTF8_ALIGNED(address(pos))
   for (int i = 0; i < mNModifyProcs; i++)
     (*mModifyProcs[i]) (pos, nInserted, nDeleted, nRestyled,
 			deletedText, mCbArgs[i]);
@@ -1590,6 +1630,8 @@ int Fl_Text_Buffer::prev_char_clipped(int pos) const
   if (pos<=0)
     return 0;
 
+  IS_UTF8_ALIGNED(address(pos))  
+
   char c;
   do {
     pos--;
@@ -1598,6 +1640,7 @@ int Fl_Text_Buffer::prev_char_clipped(int pos) const
     c = byte_at(pos);
   } while ( (c&0xc0) == 0x80);
   
+  IS_UTF8_ALIGNED(address(pos))  
   return pos;
 }
 
@@ -1619,10 +1662,12 @@ int Fl_Text_Buffer::prev_char(int pos) const
  */
 int Fl_Text_Buffer::next_char(int pos) const
 {
+  IS_UTF8_ALIGNED(address(pos))  
   int n = fl_utf8len(byte_at(pos));
   pos += n;
   if (pos>=mLength)
     return mLength;
+  IS_UTF8_ALIGNED(address(pos))  
   return pos;
 }
 
@@ -1633,11 +1678,21 @@ int Fl_Text_Buffer::next_char(int pos) const
  */
 int Fl_Text_Buffer::next_char_clipped(int pos) const
 {
-  int n = next_char(pos);
-  if (pos==mLength) return pos;
-  return n;
+  return next_char(pos);
 }
 
+/*
+ Align an index to the current utf8 boundary
+ */
+int Fl_Text_Buffer::utf8_align(int pos) const 
+{
+  char c = byte_at(pos);
+  while ( (c&0xc0) == 0x80) {
+    pos--;
+    c = byte_at(pos);
+  }
+  return pos;
+}
 
 //
 // End of "$Id$".
diff --git a/src/Fl_Text_Display.cxx b/src/Fl_Text_Display.cxx
index a0904c07d..f706219df 100644
--- a/src/Fl_Text_Display.cxx
+++ b/src/Fl_Text_Display.cxx
@@ -38,15 +38,6 @@
 #include <FL/Fl_Text_Display.H>
 #include <FL/Fl_Window.H>
 
-#define ASSERT_UTF8
-
-#ifdef ASSERT_UTF8
-#include <assert.h>
-#define IS_UTF8_ALIGNED(a) if (a && *a) assert(fl_utf8len(*(a))>0);
-#else
-#define IS_UTF8_ALIGNED(a)
-#endif
-
 #undef min
 #undef max
 
@@ -520,8 +511,8 @@ void Fl_Text_Display::redisplay_range(int startpos, int endpos) {
  \param endpos index after last character to draw
  */
 void Fl_Text_Display::draw_range(int startpos, int endpos) {
-  IS_UTF8_ALIGNED(buffer()->address(startpos))
-  IS_UTF8_ALIGNED(buffer()->address(endpos))
+  startpos = buffer()->utf8_align(startpos);
+  endpos = buffer()->utf8_align(endpos);
   
   int i, startLine, lastLine, startIndex, endIndex;
   
@@ -1076,18 +1067,6 @@ int Fl_Text_Display::move_up() {
   /* move the cursor */
   insert_position( newPos );
   
-  int ok = 0;
-  while (!ok) {
-    int pos = insert_position();
-    // FIXME: character is ucs-4
-    char c = buffer()->char_at( pos );
-    if (!((c & 0x80) && !(c & 0x40))) {
-      ok = 1;
-    } else {
-      insert_position( mCursorPos + 1 );
-    }
-  }
-  
   /* if a preferred column wasn't aleady established, establish it */
   mCursorPreferredCol = column;
   return 1;
@@ -1120,17 +1099,6 @@ int Fl_Text_Display::move_down() {
     newPos = min(newPos, line_end(nextLineStartPos, true));
   
   insert_position( newPos );
-  int ok = 0;
-  while (!ok) {
-    int pos = insert_position();
-    // FIXME: character is ucs-4
-    char c = buffer()->char_at( pos );
-    if (!((c & 0x80) && !(c & 0x40))) {
-      ok = 1;
-    } else {
-      insert_position( mCursorPos + 1 );
-    }
-  }
   mCursorPreferredCol = column;
   return 1;
 }
@@ -1422,18 +1390,19 @@ void Fl_Text_Display::buffer_predelete_cb(int pos, int nDeleted, void *cbArg) {
  \param nRestyled ??
  \param deletedText this is what was removed
  \param cbArg "this" pointer for static callback function
- 
- \todo Unicode?
  */
 void Fl_Text_Display::buffer_modified_cb( int pos, int nInserted, int nDeleted,
                                          int nRestyled, const char *deletedText, void *cbArg ) {
   int linesInserted, linesDeleted, startDispPos, endDispPos;
   Fl_Text_Display *textD = ( Fl_Text_Display * ) cbArg;
   Fl_Text_Buffer *buf = textD->mBuffer;
-  IS_UTF8_ALIGNED(buf->address(pos))  
   int oldFirstChar = textD->mFirstChar;
   int scrolled, origCursorPos = textD->mCursorPos;
   int wrapModStart, wrapModEnd;
+
+  IS_UTF8_ALIGNED(buf->address(pos))  
+  IS_UTF8_ALIGNED(buf->address(oldFirstChar))  
+  IS_UTF8_ALIGNED(buf->address(origCursorPos))  
   
   /* buffer modification cancels vertical cursor motion column */
   if ( nInserted != 0 || nDeleted != 0 )
@@ -1445,8 +1414,7 @@ void Fl_Text_Display::buffer_modified_cb( int pos, int nInserted, int nDeleted,
     textD->find_wrap_range(deletedText, pos, nInserted, nDeleted,
                            &wrapModStart, &wrapModEnd, &linesInserted, &linesDeleted);
   } else {
-    linesInserted = nInserted == 0 ? 0 :
-    buf->count_lines( pos, pos + nInserted );
+    linesInserted = nInserted == 0 ? 0 : buf->count_lines( pos, pos + nInserted );
     linesDeleted = nDeleted == 0 ? 0 : countlines( deletedText );
   }
   
@@ -1469,7 +1437,7 @@ void Fl_Text_Display::buffer_modified_cb( int pos, int nInserted, int nDeleted,
       (nInserted != 0 || nDeleted != 0)) {
     if (pos + nDeleted < oldFirstChar)
       textD->mAbsTopLineNum += buf->count_lines(pos, pos + nInserted) -
-      countlines(deletedText);
+                               countlines(deletedText);
     else if (pos < oldFirstChar)
       textD->reset_absolute_top_line_number();
   }    	    
@@ -1508,21 +1476,28 @@ void Fl_Text_Display::buffer_modified_cb( int pos, int nInserted, int nDeleted,
    old cursor gets erased, and erase the bits of the cursor which extend
    beyond the left and right edges of the text. */
   startDispPos = textD->mContinuousWrap ? wrapModStart : pos;
+  IS_UTF8_ALIGNED(buf->address(startDispPos))  
+  
   if ( origCursorPos == startDispPos && textD->mCursorPos != startDispPos )
-    startDispPos = min( startDispPos, origCursorPos - 1 );
+    startDispPos = min( startDispPos, buf->prev_char_clipped(origCursorPos) );
+  IS_UTF8_ALIGNED(buf->address(startDispPos))  
+  
   if ( linesInserted == linesDeleted ) {
     if ( nInserted == 0 && nDeleted == 0 )
       endDispPos = pos + nRestyled;
     else {
-      endDispPos = textD->mContinuousWrap ? wrapModEnd :
-      buf->line_end( pos + nInserted ) + 1;
+      if (textD->mContinuousWrap)
+        endDispPos = wrapModEnd;
+      else
+        endDispPos = buf->next_char(buf->line_end( pos + nInserted ));
+      
       // CET - FIXME      if ( origCursorPos >= startDispPos &&
       //                ( origCursorPos <= endDispPos || endDispPos == buf->length() ) )
     }
     
     if (linesInserted > 1) textD->draw_line_numbers(false);
   } else {
-    endDispPos = textD->mLastChar + 1;
+    endDispPos = buf->next_char(textD->mLastChar);
     // CET - FIXME   if ( origCursorPos >= pos )
     /* If more than one line is inserted/deleted, a line break may have
      been inserted or removed in between, and the line numbers may
@@ -1531,6 +1506,8 @@ void Fl_Text_Display::buffer_modified_cb( int pos, int nInserted, int nDeleted,
      results in at least two lines being redrawn). */
     textD->draw_line_numbers(false);
   }
+  IS_UTF8_ALIGNED(buf->address(startDispPos))  
+  IS_UTF8_ALIGNED(buf->address(endDispPos))  
   
   /* If there is a style buffer, check if the modification caused additional
    changes that need to be redisplayed.  (Redisplaying separately would
@@ -1538,9 +1515,11 @@ void Fl_Text_Display::buffer_modified_cb( int pos, int nInserted, int nDeleted,
    text).  Extend the redraw range to incorporate style changes */
   if ( textD->mStyleBuffer )
     textD->extend_range_for_styles( &startDispPos, &endDispPos );
+  IS_UTF8_ALIGNED(buf->address(startDispPos))  
+  IS_UTF8_ALIGNED(buf->address(endDispPos))  
   
   /* Redisplay computed range */
-  textD->redisplay_range( startDispPos, endDispPos );  // FIXME utf8
+  textD->redisplay_range( startDispPos, endDispPos );
 }
 
 
@@ -1735,8 +1714,10 @@ int Fl_Text_Display::handle_vline(
       style = position_style(lineStartPos, lineLen, -1);
       draw_string( style|BG_ONLY_MASK, text_area.x, Y, text_area.x+text_area.w, lineStr, lineLen );
     }
-    if (mode==FIND_INDEX)
+    if (mode==FIND_INDEX) {
+      IS_UTF8_ALIGNED(buffer()->address(lineStartPos))
       return lineStartPos;
+    }
     return 0;
   }
   
@@ -1756,6 +1737,7 @@ int Fl_Text_Display::handle_vline(
         // find x pos inside block
         int di = find_x(lineStr+startIndex, i-startIndex, style, rightClip-startX);
         free(lineStr);
+        IS_UTF8_ALIGNED(buffer()->address(lineStartPos+startIndex+di))
         return lineStartPos + startIndex + di;
       }
       style = charStyle;
@@ -1775,6 +1757,7 @@ int Fl_Text_Display::handle_vline(
     // find x pos inside block
     int di = find_x(lineStr+startIndex, i-startIndex, style, rightClip-startX);
     free(lineStr);
+    IS_UTF8_ALIGNED(buffer()->address(lineStartPos+startIndex+di))
     return lineStartPos + startIndex + di;
   }
   if (mode==GET_WIDTH) {
@@ -1789,6 +1772,7 @@ int Fl_Text_Display::handle_vline(
     draw_string( style|BG_ONLY_MASK, startX, Y, text_area.x+text_area.w, lineStr, lineLen );
   
   free(lineStr);
+  IS_UTF8_ALIGNED(buffer()->address(lineStartPos+lineLen))
   return lineStartPos + lineLen;
 }
 
@@ -3351,6 +3335,9 @@ int Fl_Text_Display::wrap_uses_character(int lineEndPos) const {
  \todo Unicode?
  */
 void Fl_Text_Display::extend_range_for_styles( int *startpos, int *endpos ) {
+  IS_UTF8_ALIGNED(buffer()->address(*startpos))  
+  IS_UTF8_ALIGNED(buffer()->address(*endpos))  
+  
   Fl_Text_Selection * sel = mStyleBuffer->primary_selection();
   int extended = 0;
   
@@ -3367,10 +3354,15 @@ void Fl_Text_Display::extend_range_for_styles( int *startpos, int *endpos ) {
   if ( sel->selected() ) {
     if ( sel->start() < *startpos ) {
       *startpos = sel->start();
+      // FIXME: somewhere while deleting, alignment is lost!
+      *startpos = buffer()->utf8_align(*startpos);
+      IS_UTF8_ALIGNED(buffer()->address(*startpos))  
       extended = 1;
     }
     if ( sel->end() > *endpos ) {
       *endpos = sel->end();
+      *endpos = buffer()->utf8_align(*endpos);
+      IS_UTF8_ALIGNED(buffer()->address(*endpos))  
       extended = 1;
     }
   }
@@ -3380,6 +3372,8 @@ void Fl_Text_Display::extend_range_for_styles( int *startpos, int *endpos ) {
    redraw characters exposed by possible font size changes */
   if ( extended )
     *endpos = mBuffer->line_end( *endpos ) + 1;
+  
+  IS_UTF8_ALIGNED(buffer()->address(*endpos))
 }
 
 
@@ -3631,16 +3625,6 @@ int Fl_Text_Display::handle(int event) {
       if (Fl::event_state()&FL_SHIFT) return handle(FL_DRAG);
       dragging = 1;
       int pos = xy_to_position(Fl::event_x(), Fl::event_y(), CURSOR_POS);
-      int ok = 0;
-      while (!ok) {
-        // FIXME: character is ucs-4
-        char c = buffer()->char_at( pos );
-        if (!((c & 0x80) && !(c & 0x40))) {
-          ok = 1;
-        } else {
-          pos++;
-        }
-      }
       dragType = Fl::event_clicks();
       dragPos = pos;
       if (dragType == DRAG_CHAR)
@@ -3697,16 +3681,7 @@ int Fl_Text_Display::handle(int event) {
           scroll_direction = 0;
         }
         pos = xy_to_position(X, Y, CURSOR_POS);
-        int ok = 0;
-        while (!ok) {
-          // FIXME: character is ucs-4
-          char c = buffer()->char_at( pos );
-          if (!((c & 0x80) && !(c & 0x40))) {
-            ok = 1;
-          } else {
-            pos++;
-          }
-        }
+        pos = buffer()->next_char(pos);
       }
       fl_text_drag_me(pos, this);
       return 1;
diff --git a/src/Fl_Text_Editor.cxx b/src/Fl_Text_Editor.cxx
index dc4730fe2..c6294332a 100644
--- a/src/Fl_Text_Editor.cxx
+++ b/src/Fl_Text_Editor.cxx
@@ -3,7 +3,7 @@
 //
 // Copyright 2001-2009 by Bill Spitzak and others.
 // Original code Copyright Mark Edel.  Permission to distribute under
-// the LGPL for the FLTK library granted by Mark Edel.
+// the LGPL for the FLTK library granted by Mark E.
 //
 // This library is free software; you can redistribute it and/or
 // modify it under the terms of the GNU Library General Public
@@ -138,7 +138,7 @@ static struct {
   { FL_Down,      FL_CTRL|FL_SHIFT,         Fl_Text_Editor::kf_c_s_move   },
   { FL_Page_Up,   FL_CTRL|FL_SHIFT,         Fl_Text_Editor::kf_c_s_move   },
   { FL_Page_Down, FL_CTRL|FL_SHIFT,         Fl_Text_Editor::kf_c_s_move   },
-//{ FL_Clear,	  0,                        Fl_Text_Editor::delete_to_eol },
+//{ FL_Clear,	  0,                        Fl_Text_Editor::ete_to_eol },
   { 'z',          FL_CTRL,                  Fl_Text_Editor::kf_undo	  },
   { '/',          FL_CTRL,                  Fl_Text_Editor::kf_undo	  },
   { 'x',          FL_CTRL,                  Fl_Text_Editor::kf_cut        },
@@ -253,13 +253,9 @@ int Fl_Text_Editor::kf_ignore(int, Fl_Text_Editor*) {
 /**  Does a backspace in the current buffer.*/
 int Fl_Text_Editor::kf_backspace(int, Fl_Text_Editor* e) {
   if (!e->buffer()->selected() && e->move_left()) {
-    int l = 1;
-    // FIXME: character is ucs-4
-    char c = e->buffer()->char_at(e->insert_position());
-    if (c & 0x80 && c & 0x40) {
-      l = fl_utf8len(c);
-    }
-    e->buffer()->select(e->insert_position(), e->insert_position()+l);
+    int p1 = e->insert_position();
+    int p2 = e->buffer()->next_char(p1);
+    e->buffer()->select(p1, p2);
   }
   kill_selection(e);
   e->show_insert_position();
@@ -449,13 +445,9 @@ int Fl_Text_Editor::kf_insert(int, Fl_Text_Editor* e) {
 /**  Does a delete of selected text or the current character in the current buffer.*/
 int Fl_Text_Editor::kf_delete(int, Fl_Text_Editor* e) {
   if (!e->buffer()->selected()) {
-    int l = 1;
-    // FIXME: character is ucs-4
-    char c = e->buffer()->char_at(e->insert_position());
-    if (c & 0x80 && c & 0x40) {
-      l = fl_utf8len(c);
-    }
-    e->buffer()->select(e->insert_position(), e->insert_position()+l);
+    int p1 = e->insert_position();
+    int p2 = e->buffer()->next_char(p1);
+    e->buffer()->select(p1, p2);
   }
 
   kill_selection(e);
@@ -519,7 +511,11 @@ int Fl_Text_Editor::handle_key() {
   // bytes to delete and a string to insert:
   int del = 0;
   if (Fl::compose(del)) {
-    if (del) buffer()->select(insert_position()-del, insert_position());
+    if (del) {
+      int dp = insert_position(), di = del;
+      while (di--) dp = buffer()->prev_char_clipped(dp);
+      buffer()->select(dp, insert_position());
+    }
     kill_selection(this);
     if (Fl::event_length()) {
       if (insert_mode()) insert(Fl::event_text());
diff --git a/src/fl_utf.c b/src/fl_utf.c
index 64d715261..b30ca82c4 100644
--- a/src/fl_utf.c
+++ b/src/fl_utf.c
@@ -151,7 +151,7 @@ unsigned fl_utf8decode(const char* p, const char* end, int* len)
   } else if (c < 0xc2) {
     goto FAIL;
   }
-  if (p+1 >= end || (p[1]&0xc0) != 0x80) goto FAIL;
+  if ( (end && p+1 >= end) || (p[1]&0xc0) != 0x80) goto FAIL;
   if (c < 0xe0) {
     if (len) *len = 2;
     return
@@ -173,7 +173,7 @@ unsigned fl_utf8decode(const char* p, const char* end, int* len)
 #endif
   } else if (c < 0xf0) {
   UTF8_3:
-    if (p+2 >= end || (p[2]&0xc0) != 0x80) goto FAIL;
+    if ( (end && p+2 >= end) || (p[2]&0xc0) != 0x80) goto FAIL;
     if (len) *len = 3;
     return
       ((p[0] & 0x0f) << 12) +
@@ -184,7 +184,7 @@ unsigned fl_utf8decode(const char* p, const char* end, int* len)
     goto UTF8_4;
   } else if (c < 0xf4) {
   UTF8_4:
-    if (p+3 >= end || (p[2]&0xc0) != 0x80 || (p[3]&0xc0) != 0x80) goto FAIL;
+    if ( (end && p+3 >= end) || (p[2]&0xc0) != 0x80 || (p[3]&0xc0) != 0x80) goto FAIL;
     if (len) *len = 4;
 #if STRICT_RFC3629
     /* RFC 3629 says all codes ending in fffe or ffff are illegal: */