make the parser explicitly handle utf8

rendering ascii will be significantly faster, so we should try to do
that when possible.

4 files changed, 359 insertions, 262 deletions

diff --git a/display.c b/display.c
index eba573f..a21621c 100644
--- a/display.c
+++ b/display.c
@@ -119,7 +119,7 @@ void runes_display_move_to(RunesTerm *t, int row, int col)
     runes_display_position_cursor(t, t->cr);
 }
 
-void runes_display_show_string(RunesTerm *t, char *buf, size_t len)
+void runes_display_show_string_ascii(RunesTerm *t, char *buf, size_t len)
 {
     if (len) {
         int remaining = len, space_in_row = t->cols - t->col;
@@ -157,7 +157,6 @@ void runes_display_show_string(RunesTerm *t, char *buf, size_t len)
                 runes_display_move_to(t, t->row + 1, 0);
             }
             else {
-                /* XXX broken with utf8 */
                 t->col += len;
                 runes_display_position_cursor(t, t->cr);
             }
@@ -165,6 +164,12 @@ void runes_display_show_string(RunesTerm *t, char *buf, size_t len)
     }
 }
 
+void runes_display_show_string_utf8(RunesTerm *t, char *buf, size_t len)
+{
+    /* XXX */
+    runes_display_show_string_ascii(t, buf, len);
+}
+
 void runes_display_clear_screen(RunesTerm *t)
 {
     runes_display_paint_rectangle(
diff --git a/display.h b/display.h
index 0b4538c..0cd2dbe 100644
--- a/display.h
+++ b/display.h
@@ -6,7 +6,8 @@ void runes_display_set_window_size(RunesTerm *t);
 void runes_display_focus_in(RunesTerm *t);
 void runes_display_focus_out(RunesTerm *t);
 void runes_display_move_to(RunesTerm *t, int row, int col);
-void runes_display_show_string(RunesTerm *t, char *buf, size_t len);
+void runes_display_show_string_ascii(RunesTerm *t, char *buf, size_t len);
+void runes_display_show_string_utf8(RunesTerm *t, char *buf, size_t len);
 void runes_display_clear_screen(RunesTerm *t);
 void runes_display_clear_screen_forward(RunesTerm *t);
 void runes_display_kill_line_forward(RunesTerm *t);
diff --git a/parser.c b/parser.c
index e81e368..747f5ed 100644
--- a/parser.c
+++ b/parser.c
@@ -351,8 +351,8 @@ static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner );
 	*yy_cp = '\0'; \
 	yyg->yy_c_buf_p = yy_cp;
 
-#define YY_NUM_RULES 40
-#define YY_END_OF_BUFFER 41
+#define YY_NUM_RULES 42
+#define YY_END_OF_BUFFER 43
 /* This struct is not used in this scanner,
    but its presence is necessary. */
 struct yy_trans_info
@@ -360,17 +360,22 @@ struct yy_trans_info
 	flex_int32_t yy_verify;
 	flex_int32_t yy_nxt;
 	};
-static yyconst flex_int16_t yy_accept[82] =
+static yyconst flex_int16_t yy_accept[129] =
     {   0,
-        0,    0,   41,   39,    2,    3,    4,    5,    6,    7,
-        8,   39,    1,   37,   38,   13,   14,    9,   10,   11,
-       38,   38,   12,    1,   33,   34,   34,   34,   34,   15,
-       16,   17,   18,   19,   20,   21,   22,   23,   24,   25,
-       26,   27,   35,   36,   36,   36,   36,   34,   34,   34,
-       28,   29,    0,    0,    0,    0,    0,   34,   30,    0,
-       31,    0,   32,    0,   34,   34,    0,    0,    0,    0,
-       24,   25,   34,   34,   34,    0,   34,   34,    0,   34,
-        0
+        0,    0,   43,   40,    1,    2,    3,    4,    5,    6,
+        7,   40,   32,   41,   41,   41,   41,   38,   39,   12,
+       13,    8,    9,   10,   39,   39,   11,    0,    0,    0,
+       32,    0,    0,    0,   33,    0,    0,   34,   35,   35,
+       35,   35,   14,   15,   16,   17,   18,   19,   20,   21,
+       22,   23,   24,   25,   26,    0,    0,    0,   36,   37,
+       37,   37,   37,    0,    0,    0,    0,    0,    0,   35,
+       35,   35,   27,   28,    0,    0,    0,    0,    0,    0,
+        0,    0,    0,    0,   35,    0,   29,    0,    0,    0,
+        0,   30,    0,    0,    0,    0,   31,    0,    0,    0,
+
+        0,    0,   35,   35,    0,    0,    0,    0,    0,    0,
+        0,    0,    0,    0,   23,   24,   35,    0,    0,    0,
+       35,   35,    0,   35,   35,    0,   35,    0
     } ;
 
 static yyconst flex_int32_t yy_ec[256] =
@@ -388,149 +393,200 @@ static yyconst flex_int32_t yy_ec[256] =
 
        10,   10,   34,   35,   10,   10,   10,   36,   37,   10,
        10,   10,   10,   38,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,    1,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10,   10,   10,   10,   10,   10,
-       10,   10,   10,   10,   10
+       10,   10,   10,   10,   10,   10,    1,   39,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       39,   40,   40,   40,   40,   40,   40,   40,   40,   40,
+
+       40,   40,   40,   40,   40,   40,   40,   40,   40,   40,
+       40,   40,   40,   40,   40,   40,   40,   40,   40,   40,
+       40,   40,   40,   41,   41,   41,   41,   41,   41,   41,
+       41,   41,   41,   41,   41,   41,   41,   41,   41,   42,
+       42,   42,   42,   42,   42,   42,   42,   43,   43,   43,
+       43,   43,   43,   43,   43
     } ;
 
-static yyconst flex_int32_t yy_meta[39] =
+static yyconst flex_int32_t yy_meta[44] =
     {   0,
         1,    2,    1,    1,    1,    1,    1,    1,    1,    3,
-        4,    4,    4,    4,    4,    4,    3,    5,    5,    3,
-        5,    3,    3,    3,    3,    3,    3,    3,    3,    3,
-        3,    3,    3,    3,    3,    3,    3,    3
+        4,    4,    4,    4,    4,    4,    5,    6,    6,    3,
+        6,    3,    3,    3,    3,    3,    3,    3,    3,    3,
+        3,    3,    3,    3,    5,    5,    3,    3,    7,    3,
+        3,    3,    7
     } ;
 
-static yyconst flex_int16_t yy_base[95] =
+static yyconst flex_int16_t yy_base[143] =
     {   0,
-        0,    9,  170,  338,  338,  338,  338,  338,  338,  338,
-      338,   18,    0,  338,  338,  338,  338,  338,  338,  338,
-       56,   94,  338,    0,  338,  338,   91,  129,  145,  338,
-      338,  338,  338,  338,  338,  338,  338,  338,  338,  338,
-      338,  338,  338,  338,  149,  145,  138,   95,  163,   87,
-      338,  338,  137,  126,  124,  189,    0,   99,  338,  119,
-      338,  117,  338,  113,    0,  106,  146,  217,  223,  146,
-      338,  338,  239,    0,  150,  245,  252,  165,  259,    0,
-      338,  297,  300,  305,  310,  312,  316,  320,  324,  107,
-      326,  328,  330,  332
-
+        0,   43,  315,  511,  511,  511,  511,  511,  511,  511,
+      511,   86,   13,  511,  274,  273,  272,  511,  511,  511,
+      511,  511,  511,  511,  128,  170,  511,  271,  270,  257,
+       16,  256,  255,  254,   19,  253,  252,  511,  511,    0,
+      212,  244,  511,  511,  511,  511,  511,  511,  511,  511,
+      511,  511,  511,  511,  511,  249,  248,  247,  511,  511,
+      268,  267,  266,  243,  242,  241,  231,  230,  228,    4,
+       45,  262,  511,  511,  227,  226,   24,   27,   30,  225,
+      224,  223,  288,    0,    9,  222,  511,   33,  212,  170,
+      169,  511,  165,  165,  164,  163,  511,  266,  162,  161,
+
+      158,  157,    0,   15,  163,  316,  156,  155,  154,  153,
+      151,  147,  334,   58,  511,  511,  350,   86,   39,   24,
+      368,  166,  384,  400,  170,  416,    0,  511,  454,  459,
+      465,  469,  472,  477,  482,  487,  490,  493,  496,  498,
+      501,  504
     } ;
 
-static yyconst flex_int16_t yy_def[95] =
+static yyconst flex_int16_t yy_def[143] =
     {   0,
-       82,   82,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   83,   81,   81,   81,   81,   81,   81,   81,
-       81,   84,   81,   83,   81,   81,   21,   85,   85,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   86,   29,   29,
-       81,   81,   87,   88,   89,   49,   90,   91,   81,   87,
-       81,   88,   81,   89,   56,   92,   90,   49,   56,   90,
-       81,   81,   49,   69,   93,   69,   69,   94,   69,   79,
-        0,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81
-
+      129,  129,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  130,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  131,  128,  128,  128,  128,
+      130,  128,  128,  128,  132,  128,  128,  128,  128,   25,
+      128,   41,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  133,
+       41,   41,  128,  128,  128,  128,  134,  135,  136,  128,
+      128,  128,   41,  128,  137,  128,  128,  134,  128,  128,
+      128,  128,  135,  128,  128,  128,  128,  136,  128,  128,
+
+      128,  128,   83,  138,  139,   72,  128,  128,  128,  128,
+      128,  128,   83,  140,  128,  128,   72,  128,  128,  128,
+       83,  141,   83,   83,  142,   83,  126,    0,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128
     } ;
 
-static yyconst flex_int16_t yy_nxt[377] =
+static yyconst flex_int16_t yy_nxt[555] =
     {   0,
-        4,    5,    6,    7,    8,    9,   10,   11,   12,    4,
-        5,    6,    7,    8,    9,   10,   11,   12,   14,   14,
-       14,   14,   14,   14,   14,   14,   14,   15,   15,   15,
-       15,   15,   16,   17,   15,   15,   18,   19,   15,   15,
-       15,   15,   15,   15,   15,   15,   15,   20,   15,   21,
-       22,   23,   15,   15,   15,   15,   25,   25,   25,   25,
-       25,   25,   25,   25,   25,   26,   27,   27,   27,   27,
-       27,   27,   26,   28,   28,   26,   29,   30,   31,   32,
-       33,   34,   35,   36,   37,   26,   38,   26,   26,   26,
-       39,   40,   41,   42,   43,   43,   43,   43,   43,   43,
-
-       43,   43,   43,   58,   45,   46,   47,   48,   26,   26,
-       67,   26,   57,   57,   63,   57,   57,   57,   61,   57,
-       59,   39,   40,   57,   57,   63,   57,   61,   26,   25,
-       25,   25,   25,   25,   25,   25,   25,   25,   59,   49,
-       49,   49,   49,   49,   49,   25,   25,   25,   25,   25,
-       25,   25,   25,   25,   55,   50,   50,   50,   50,   50,
-       50,   54,   70,   57,   57,   53,   57,   57,   57,   81,
-       57,   51,   52,   49,   49,   49,   49,   49,   49,   58,
-       71,   72,   57,   57,   81,   57,   81,   81,   81,   26,
-       26,   81,   81,   81,   81,   81,   81,   39,   40,   65,
-
-       65,   65,   65,   65,   65,   66,   81,   81,   81,   81,
-       81,   81,   81,   81,   34,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   41,   42,   73,   73,   73,
-       73,   73,   73,   74,   74,   74,   74,   74,   74,   75,
-       81,   81,   81,   81,   81,   81,   81,   81,   26,   73,
-       73,   73,   73,   73,   73,   77,   77,   77,   77,   77,
-       77,   78,   77,   77,   77,   77,   77,   77,   78,   80,
-       80,   80,   80,   80,   80,   78,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   26,   13,   13,   13,
-
-       13,   13,   24,   24,   24,   44,   44,   44,   44,   44,
-       26,   26,   26,   26,   26,   56,   56,   60,   60,   60,
-       60,   62,   62,   62,   62,   64,   64,   64,   64,   68,
-       68,   69,   69,   76,   76,   79,   79,    3,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81
+        4,    5,    6,    7,    8,    9,   10,   11,   12,  128,
+      105,  105,  105,  105,  105,  105,   70,   39,   39,  128,
+       39,   84,   84,  128,   84,   87,   84,   84,   92,   84,
+      128,   97,   84,   84,   87,   84,  128,   39,   14,   15,
+       16,   17,   14,    4,    5,    6,    7,    8,    9,   10,
+       11,   12,   32,   33,   34,   32,   33,   34,   32,   33,
+       34,   85,   98,   89,   90,   91,   94,   95,   96,   99,
+      100,  101,   89,   90,   91,   84,   84,   93,   84,   52,
+       53,   14,   15,   16,   17,   14,   18,   18,   18,   18,
+       18,   18,   18,   18,   18,   19,   19,   19,   19,   19,
+
+       20,   21,   19,   19,   22,   23,   19,   19,   19,   19,
+       19,   19,   19,   19,   19,   24,   19,   25,   26,   27,
+       19,   19,   19,   19,   88,   28,   29,   30,   38,   38,
+       38,   38,   38,   38,   38,   38,   38,   39,   40,   40,
+       40,   40,   40,   40,   39,   41,   41,   39,   42,   43,
+       44,   45,   46,   47,   48,   49,   50,   39,   51,   39,
+       39,   39,   52,   53,   54,   55,   92,   56,   57,   58,
+       59,   59,   59,   59,   59,   59,   59,   59,   59,  114,
+       61,   62,   63,   84,   84,  120,   84,   84,   84,   98,
+       84,  119,   93,  118,   88,   60,  112,  115,  116,  111,
+
+       98,  110,  109,   93,   94,   95,   96,  108,  107,   64,
+       65,   66,   38,   38,   38,   38,   38,   38,   38,   38,
+       38,   39,   71,   71,   71,   71,   71,   71,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       39,   39,   39,   39,   39,   39,   39,   39,   39,   39,
+       88,   56,   57,   58,   72,   72,   72,   72,   72,   72,
+       39,   19,  102,   60,   86,   39,   35,   97,   82,   19,
+       73,   74,   72,   72,   72,   72,   72,   72,   85,   81,
+       80,   60,   79,   78,   77,   76,   75,   39,   73,   74,
+       69,   35,   37,   36,   35,   68,   52,   53,  103,  103,
+
+      103,  103,  103,  103,  104,   99,  100,  101,   67,   19,
+       37,   36,   35,   47,  128,  128,  128,  128,  128,  128,
+      128,  128,   52,   53,   54,   55,  117,  117,  117,  117,
+      117,  117,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,   39,   39,  121,  121,  121,  121,  121,  121,
+      122,  128,  128,  128,  128,  128,  128,  128,  128,   39,
+      117,  117,  117,  117,  117,  117,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,   39,   39,  121,  121,
+      121,  121,  121,  121,  122,  128,  128,  128,  128,  128,
+      128,  128,  128,   39,  124,  124,  124,  124,  124,  124,
+
+      125,  128,  128,  128,  128,  128,  128,  128,  128,   39,
+      124,  124,  124,  124,  124,  124,  125,  128,  128,  128,
+      128,  128,  128,  128,  128,   39,  127,  127,  127,  127,
+      127,  127,  125,  128,  128,  128,  128,  128,  128,  128,
+      128,   39,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,   39,   13,   13,   13,   13,   13,   13,
+       13,   31,   31,   31,   31,   60,   60,   60,   60,   60,
+       60,   35,   35,   35,   35,   83,  128,   83,   88,   88,
+       88,   88,   88,   93,   93,   93,   93,   93,   98,   98,
+       98,   98,   98,  106,  128,  106,  113,  128,  113,  105,
+
+      105,  105,  128,  105,  123,  128,  123,  126,  128,  126,
+        3,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128
     } ;
 
-static yyconst flex_int16_t yy_chk[377] =
+static yyconst flex_int16_t yy_chk[555] =
     {   0,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    2,
-        2,    2,    2,    2,    2,    2,    2,    2,   12,   12,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    0,
+       84,   84,   84,   84,   84,   84,   40,   40,   40,    0,
+       40,   70,   70,    0,   70,   77,   85,   85,   78,   85,
+        0,   79,  104,  104,   88,  104,    0,   40,    1,    1,
+        1,    1,    1,    2,    2,    2,    2,    2,    2,    2,
+        2,    2,   13,   13,   13,   31,   31,   31,   35,   35,
+       35,   71,  120,   77,   77,   77,   78,   78,   78,   79,
+       79,   79,   88,   88,   88,  114,  114,  119,  114,   71,
+       71,    2,    2,    2,    2,    2,   12,   12,   12,   12,
        12,   12,   12,   12,   12,   12,   12,   12,   12,   12,
+
        12,   12,   12,   12,   12,   12,   12,   12,   12,   12,
        12,   12,   12,   12,   12,   12,   12,   12,   12,   12,
-       12,   12,   12,   12,   12,   12,   21,   21,   21,   21,
-       21,   21,   21,   21,   21,   21,   21,   21,   21,   21,
-       21,   21,   21,   21,   21,   21,   21,   21,   21,   21,
-       21,   21,   21,   21,   21,   21,   21,   21,   21,   21,
-       21,   21,   21,   21,   22,   22,   22,   22,   22,   22,
-
-       22,   22,   22,   50,   22,   22,   22,   27,   27,   27,
-       90,   27,   48,   48,   64,   48,   58,   58,   62,   58,
-       60,   50,   50,   66,   66,   55,   66,   54,   27,   28,
-       28,   28,   28,   28,   28,   28,   28,   28,   53,   28,
-       28,   28,   28,   28,   28,   29,   29,   29,   29,   29,
-       29,   29,   29,   29,   47,   29,   29,   29,   29,   29,
-       29,   46,   67,   70,   70,   45,   70,   75,   75,    3,
-       75,   29,   29,   49,   49,   49,   49,   49,   49,   49,
-       67,   67,   78,   78,    0,   78,    0,    0,    0,   49,
-       49,    0,    0,    0,    0,    0,    0,   49,   49,   56,
-
-       56,   56,   56,   56,   56,   56,    0,    0,    0,    0,
-        0,    0,    0,    0,   56,    0,    0,    0,    0,    0,
-        0,    0,    0,    0,    0,   56,   56,   68,   68,   68,
-       68,   68,   68,   69,   69,   69,   69,   69,   69,   69,
-        0,    0,    0,    0,    0,    0,    0,    0,   69,   73,
-       73,   73,   73,   73,   73,   76,   76,   76,   76,   76,
-       76,   76,   77,   77,   77,   77,   77,   77,   77,   79,
-       79,   79,   79,   79,   79,   79,    0,    0,    0,    0,
-        0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
-        0,    0,    0,    0,    0,    0,   79,   82,   82,   82,
-
-       82,   82,   83,   83,   83,   84,   84,   84,   84,   84,
-       85,   85,   85,   85,   85,   86,   86,   87,   87,   87,
-       87,   88,   88,   88,   88,   89,   89,   89,   89,   91,
-       91,   92,   92,   93,   93,   94,   94,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81,   81,   81,   81,   81,
-       81,   81,   81,   81,   81,   81
+       12,   12,   12,   12,  118,   12,   12,   12,   25,   25,
+       25,   25,   25,   25,   25,   25,   25,   25,   25,   25,
+       25,   25,   25,   25,   25,   25,   25,   25,   25,   25,
+       25,   25,   25,   25,   25,   25,   25,   25,   25,   25,
+       25,   25,   25,   25,   25,   25,   93,   25,   25,   25,
+       26,   26,   26,   26,   26,   26,   26,   26,   26,  105,
+       26,   26,   26,  122,  122,  112,  122,  125,  125,  111,
+      125,  110,  109,  108,  107,  102,  101,  105,  105,  100,
+
+       99,   96,   95,   94,   93,   93,   93,   91,   90,   26,
+       26,   26,   41,   41,   41,   41,   41,   41,   41,   41,
+       41,   41,   41,   41,   41,   41,   41,   41,   41,   41,
+       41,   41,   41,   41,   41,   41,   41,   41,   41,   41,
+       41,   41,   41,   41,   41,   41,   41,   41,   41,   41,
+       89,   41,   41,   41,   42,   42,   42,   42,   42,   42,
+       86,   82,   81,   80,   76,   75,   69,   98,   68,   67,
+       42,   42,   72,   72,   72,   72,   72,   72,   72,   66,
+       65,   64,   63,   62,   61,   58,   57,   56,   72,   72,
+       37,   36,   34,   33,   32,   30,   72,   72,   83,   83,
+
+       83,   83,   83,   83,   83,   98,   98,   98,   29,   28,
+       17,   16,   15,   83,    3,    0,    0,    0,    0,    0,
+        0,    0,   83,   83,   83,   83,  106,  106,  106,  106,
+      106,  106,    0,    0,    0,    0,    0,    0,    0,    0,
+        0,    0,  106,  106,  113,  113,  113,  113,  113,  113,
+      113,    0,    0,    0,    0,    0,    0,    0,    0,  113,
+      117,  117,  117,  117,  117,  117,    0,    0,    0,    0,
+        0,    0,    0,    0,    0,    0,  117,  117,  121,  121,
+      121,  121,  121,  121,  121,    0,    0,    0,    0,    0,
+        0,    0,    0,  121,  123,  123,  123,  123,  123,  123,
+
+      123,    0,    0,    0,    0,    0,    0,    0,    0,  123,
+      124,  124,  124,  124,  124,  124,  124,    0,    0,    0,
+        0,    0,    0,    0,    0,  124,  126,  126,  126,  126,
+      126,  126,  126,    0,    0,    0,    0,    0,    0,    0,
+        0,  126,    0,    0,    0,    0,    0,    0,    0,    0,
+        0,    0,    0,  126,  129,  129,  129,  129,  129,  129,
+      129,  130,  130,  130,  130,  131,  131,  131,  131,  131,
+      131,  132,  132,  132,  132,  133,    0,  133,  134,  134,
+      134,  134,  134,  135,  135,  135,  135,  135,  136,  136,
+      136,  136,  136,  137,    0,  137,  138,    0,  138,  139,
+
+      139,  140,    0,  140,  141,    0,  141,  142,    0,  142,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128,  128,  128,  128,  128,  128,  128,
+      128,  128,  128,  128
     } ;
 
 /* The intent behind this definition is that it'll catch
@@ -550,8 +606,7 @@ static yyconst flex_int16_t yy_chk[377] =
 
 #define YY_EXIT_FAILURE (UNUSED(yyscanner), 2)
 #define YY_NO_INPUT 1
-#line 67 "parser.l"
-static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len);
+#line 73 "parser.l"
 static void runes_parser_handle_bel(RunesTerm *t);
 static void runes_parser_handle_bs(RunesTerm *t);
 static void runes_parser_handle_tab(RunesTerm *t);
@@ -585,7 +640,9 @@ static void runes_parser_handle_decsel(RunesTerm *t, char *buf, size_t len);
 static void runes_parser_handle_osc0(RunesTerm *t, char *buf, size_t len);
 static void runes_parser_handle_osc1(RunesTerm *t, char *buf, size_t len);
 static void runes_parser_handle_osc2(RunesTerm *t, char *buf, size_t len);
-#line 589 "parser.c"
+static void runes_parser_handle_ascii(RunesTerm *t, char *text, size_t len);
+static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len);
+#line 646 "parser.c"
 
 #define INITIAL 0
 
@@ -833,10 +890,10 @@ YY_DECL
 		}
 
 	{
-#line 103 "parser.l"
+#line 110 "parser.l"
 
 
-#line 840 "parser.c"
+#line 897 "parser.c"
 
 	while ( 1 )		/* loops until end-of-file is reached */
 		{
@@ -863,13 +920,13 @@ yy_match:
 			while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 				{
 				yy_current_state = (int) yy_def[yy_current_state];
-				if ( yy_current_state >= 82 )
+				if ( yy_current_state >= 129 )
 					yy_c = yy_meta[(unsigned int) yy_c];
 				}
 			yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
 			++yy_cp;
 			}
-		while ( yy_current_state != 81 );
+		while ( yy_current_state != 128 );
 		yy_cp = yyg->yy_last_accepting_cpos;
 		yy_current_state = yyg->yy_last_accepting_state;
 
@@ -891,230 +948,242 @@ do_action:	/* This label is used only to access EOF actions. */
 
 case 1:
 YY_RULE_SETUP
-#line 105 "parser.l"
-runes_parser_handle_text(yyextra, yytext, yyleng);
-	YY_BREAK
-case 2:
-YY_RULE_SETUP
-#line 107 "parser.l"
+#line 112 "parser.l"
 runes_parser_handle_bel(yyextra);
 	YY_BREAK
-case 3:
+case 2:
 YY_RULE_SETUP
-#line 108 "parser.l"
+#line 113 "parser.l"
 runes_parser_handle_bs(yyextra);
 	YY_BREAK
-case 4:
+case 3:
 YY_RULE_SETUP
-#line 109 "parser.l"
+#line 114 "parser.l"
 runes_parser_handle_tab(yyextra);
 	YY_BREAK
+case 4:
+/* rule 4 can match eol */
+#line 116 "parser.l"
 case 5:
 /* rule 5 can match eol */
-#line 111 "parser.l"
+#line 117 "parser.l"
 case 6:
 /* rule 6 can match eol */
-#line 112 "parser.l"
-case 7:
-/* rule 7 can match eol */
 YY_RULE_SETUP
-#line 112 "parser.l"
+#line 117 "parser.l"
 runes_parser_handle_lf(yyextra);
 	YY_BREAK
-case 8:
+case 7:
 YY_RULE_SETUP
-#line 113 "parser.l"
+#line 118 "parser.l"
 runes_parser_handle_cr(yyextra);
 	YY_BREAK
-case 9:
+case 8:
 YY_RULE_SETUP
-#line 115 "parser.l"
+#line 120 "parser.l"
 runes_parser_handle_deckpam(yyextra);
 	YY_BREAK
-case 10:
+case 9:
 YY_RULE_SETUP
-#line 116 "parser.l"
+#line 121 "parser.l"
 runes_parser_handle_deckpnm(yyextra);
 	YY_BREAK
-case 11:
+case 10:
 YY_RULE_SETUP
-#line 117 "parser.l"
+#line 122 "parser.l"
 runes_parser_handle_ri(yyextra);
 	YY_BREAK
-case 12:
+case 11:
 YY_RULE_SETUP
-#line 118 "parser.l"
+#line 123 "parser.l"
 runes_parser_handle_vb(yyextra);
 	YY_BREAK
-case 13:
+case 12:
 YY_RULE_SETUP
-#line 119 "parser.l"
+#line 124 "parser.l"
 runes_parser_handle_decsc(yyextra);
 	YY_BREAK
-case 14:
+case 13:
 YY_RULE_SETUP
-#line 120 "parser.l"
+#line 125 "parser.l"
 runes_parser_handle_decrc(yyextra);
 	YY_BREAK
-case 15:
+case 14:
 YY_RULE_SETUP
-#line 122 "parser.l"
+#line 127 "parser.l"
 runes_parser_handle_cuu(yyextra, yytext, yyleng);
 	YY_BREAK
-case 16:
+case 15:
 YY_RULE_SETUP
-#line 123 "parser.l"
+#line 128 "parser.l"
 runes_parser_handle_cud(yyextra, yytext, yyleng);
 	YY_BREAK
-case 17:
+case 16:
 YY_RULE_SETUP
-#line 124 "parser.l"
+#line 129 "parser.l"
 runes_parser_handle_cuf(yyextra, yytext, yyleng);
 	YY_BREAK
-case 18:
+case 17:
 YY_RULE_SETUP
-#line 125 "parser.l"
+#line 130 "parser.l"
 runes_parser_handle_cub(yyextra, yytext, yyleng);
 	YY_BREAK
-case 19:
+case 18:
 YY_RULE_SETUP
-#line 126 "parser.l"
+#line 131 "parser.l"
 runes_parser_handle_cup(yyextra, yytext, yyleng);
 	YY_BREAK
-case 20:
+case 19:
 YY_RULE_SETUP
-#line 127 "parser.l"
+#line 132 "parser.l"
 runes_parser_handle_ed(yyextra, yytext, yyleng);
 	YY_BREAK
-case 21:
+case 20:
 YY_RULE_SETUP
-#line 128 "parser.l"
+#line 133 "parser.l"
 runes_parser_handle_el(yyextra, yytext, yyleng);
 	YY_BREAK
-case 22:
+case 21:
 YY_RULE_SETUP
-#line 129 "parser.l"
+#line 134 "parser.l"
 runes_parser_handle_il(yyextra, yytext, yyleng);
 	YY_BREAK
-case 23:
+case 22:
 YY_RULE_SETUP
-#line 130 "parser.l"
+#line 135 "parser.l"
 runes_parser_handle_dch(yyextra, yytext, yyleng);
 	YY_BREAK
-case 24:
+case 23:
 YY_RULE_SETUP
-#line 131 "parser.l"
+#line 136 "parser.l"
 runes_parser_handle_sm(yyextra, yytext, yyleng);
 	YY_BREAK
-case 25:
+case 24:
 YY_RULE_SETUP
-#line 132 "parser.l"
+#line 137 "parser.l"
 runes_parser_handle_rm(yyextra, yytext, yyleng);
 	YY_BREAK
-case 26:
+case 25:
 YY_RULE_SETUP
-#line 133 "parser.l"
+#line 138 "parser.l"
 runes_parser_handle_sgr(yyextra, yytext, yyleng);
 	YY_BREAK
-case 27:
+case 26:
 YY_RULE_SETUP
-#line 134 "parser.l"
+#line 139 "parser.l"
 runes_parser_handle_csr(yyextra, yytext, yyleng);
 	YY_BREAK
-case 28:
+case 27:
 YY_RULE_SETUP
-#line 136 "parser.l"
+#line 141 "parser.l"
 runes_parser_handle_decsed(yyextra, yytext, yyleng);
 	YY_BREAK
-case 29:
+case 28:
 YY_RULE_SETUP
-#line 137 "parser.l"
+#line 142 "parser.l"
 runes_parser_handle_decsel(yyextra, yytext, yyleng);
 	YY_BREAK
-case 30:
+case 29:
 YY_RULE_SETUP
-#line 139 "parser.l"
+#line 144 "parser.l"
 runes_parser_handle_osc0(yyextra, yytext, yyleng);
 	YY_BREAK
-case 31:
+case 30:
 YY_RULE_SETUP
-#line 140 "parser.l"
+#line 145 "parser.l"
 runes_parser_handle_osc1(yyextra, yytext, yyleng);
 	YY_BREAK
-case 32:
+case 31:
 YY_RULE_SETUP
-#line 141 "parser.l"
+#line 146 "parser.l"
 runes_parser_handle_osc2(yyextra, yytext, yyleng);
 	YY_BREAK
+case 32:
+YY_RULE_SETUP
+#line 148 "parser.l"
+runes_parser_handle_ascii(yyextra, yytext, yyleng);
+	YY_BREAK
 case 33:
-/* rule 33 can match eol */
 YY_RULE_SETUP
-#line 143 "parser.l"
+#line 149 "parser.l"
+runes_parser_handle_text(yyextra, yytext, yyleng);
+	YY_BREAK
+case 34:
+/* rule 34 can match eol */
+YY_RULE_SETUP
+#line 151 "parser.l"
 {
     fprintf(
         stderr, "unhandled CSI sequence: \\033%*s\\%hho\n",
         (int)yyleng - 2, yytext + 1, yytext[yyleng - 1]);
 }
 	YY_BREAK
-case 34:
+case 35:
 YY_RULE_SETUP
-#line 149 "parser.l"
+#line 157 "parser.l"
 {
     fprintf(
         stderr, "unhandled CSI sequence: \\033%*s\n",
         (int)yyleng - 1, yytext + 1);
 }
 	YY_BREAK
-case 35:
-/* rule 35 can match eol */
+case 36:
+/* rule 36 can match eol */
 YY_RULE_SETUP
-#line 155 "parser.l"
+#line 163 "parser.l"
 {
     fprintf(
         stderr, "unhandled OSC sequence: \\033%*s\\%hho\n",
         (int)yyleng - 2, yytext + 1, yytext[yyleng - 1]);
 }
 	YY_BREAK
-case 36:
+case 37:
 YY_RULE_SETUP
-#line 161 "parser.l"
+#line 169 "parser.l"
 {
     fprintf(
         stderr, "unhandled OSC sequence: \\033%*s\n",
         (int)yyleng - 1, yytext + 1);
 }
 	YY_BREAK
-case 37:
-/* rule 37 can match eol */
+case 38:
+/* rule 38 can match eol */
 YY_RULE_SETUP
-#line 167 "parser.l"
+#line 175 "parser.l"
 {
     fprintf(stderr, "unhandled escape sequence: \\%hho\n", yytext[1]);
-    yyless(1);
 }
 	YY_BREAK
-case 38:
+case 39:
 YY_RULE_SETUP
-#line 172 "parser.l"
+#line 179 "parser.l"
 {
-    fprintf(stderr, "unhandled escape sequence: %c\n", yytext[1]);
-    yyless(1);
+    fprintf(
+        stderr, "unhandled escape sequence: %*s\n",
+        (int)yyleng - 1, yytext + 1);
 }
 	YY_BREAK
-case 39:
-/* rule 39 can match eol */
+case 40:
+/* rule 40 can match eol */
 YY_RULE_SETUP
-#line 177 "parser.l"
+#line 185 "parser.l"
 {
     fprintf(stderr, "unhandled control character: \\%hho\n", yytext[0]);
 }
 	YY_BREAK
-case 40:
+case 41:
 YY_RULE_SETUP
-#line 181 "parser.l"
+#line 189 "parser.l"
+{
+    fprintf(stderr, "invalid utf8 byte: \\%hho\n", yytext[0]);
+}
+	YY_BREAK
+case 42:
+YY_RULE_SETUP
+#line 193 "parser.l"
 YY_FATAL_ERROR( "flex scanner jammed" );
 	YY_BREAK
-#line 1118 "parser.c"
+#line 1187 "parser.c"
 case YY_STATE_EOF(INITIAL):
 	yyterminate();
 
@@ -1410,7 +1479,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 		while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 			{
 			yy_current_state = (int) yy_def[yy_current_state];
-			if ( yy_current_state >= 82 )
+			if ( yy_current_state >= 129 )
 				yy_c = yy_meta[(unsigned int) yy_c];
 			}
 		yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -1439,11 +1508,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner)
 	while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 		{
 		yy_current_state = (int) yy_def[yy_current_state];
-		if ( yy_current_state >= 82 )
+		if ( yy_current_state >= 129 )
 			yy_c = yy_meta[(unsigned int) yy_c];
 		}
 	yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
-	yy_is_jam = (yy_current_state == 81);
+	yy_is_jam = (yy_current_state == 128);
 
 	(void)yyg;
 	return yy_is_jam ? 0 : yy_current_state;
@@ -2208,7 +2277,7 @@ static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner)
 
 #define YYTABLES_NAME "yytables"
 
-#line 181 "parser.l"
+#line 193 "parser.l"
 
 
 
@@ -2227,11 +2296,6 @@ void runes_parser_process_string(RunesTerm *t, char *buf, size_t len)
     runes_parser_yylex_destroy(scanner);
 }
 
-static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len)
-{
-    runes_display_show_string(t, text, len);
-}
-
 static void runes_parser_handle_bel(RunesTerm *t)
 {
     runes_window_backend_request_visual_bell(t);
@@ -2634,6 +2698,16 @@ static void runes_parser_handle_osc2(RunesTerm *t, char *buf, size_t len)
     runes_window_backend_set_window_title(t, buf + 4, len - 5);
 }
 
+static void runes_parser_handle_ascii(RunesTerm *t, char *text, size_t len)
+{
+    runes_display_show_string_ascii(t, text, len);
+}
+
+static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len)
+{
+    runes_display_show_string_utf8(t, text, len);
+}
+
 /* XXX these are copied from the generated file so that I can add the UNUSED
  * declarations, otherwise we get compilation errors */
 void *runes_parser_yyalloc(yy_size_t size, yyscan_t yyscanner)
diff --git a/parser.l b/parser.l
index a6586e1..19f02e7 100644
--- a/parser.l
+++ b/parser.l
@@ -13,8 +13,14 @@
 %option prefix="runes_parser_yy"
 %option extra-type="RunesTerm *"
 
-CTRL [\000-\037\177]
-TEXT [^\000-\037\177]
+CTRL    [\000-\037\177]
+ASCII   [\040-\176]
+LEAD2   [\300-\337]
+LEAD3   [\340-\357]
+LEAD4   [\360-\367]
+CONT    [\200-\277]
+UNICHAR ({LEAD2}{CONT}|{LEAD3}{CONT}{CONT}|{LEAD4}{CONT}{CONT}{CONT})
+CHAR    ({ASCII}|{UNICHAR})
 
 ST  \007
 BEL \007
@@ -59,12 +65,11 @@ CSR {CSI}{CSIPARAM24}r
 DECSED {DECCSI}{CSIPARAM1}J
 DECSEL {DECCSI}{CSIPARAM1}K
 
-OSC0 {OSC}0;{TEXT}*{ST}
-OSC1 {OSC}1;{TEXT}*{ST}
-OSC2 {OSC}2;{TEXT}*{ST}
+OSC0 {OSC}0;{CHAR}*{ST}
+OSC1 {OSC}1;{CHAR}*{ST}
+OSC2 {OSC}2;{CHAR}*{ST}
 
 %{
-static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len);
 static void runes_parser_handle_bel(RunesTerm *t);
 static void runes_parser_handle_bs(RunesTerm *t);
 static void runes_parser_handle_tab(RunesTerm *t);
@@ -98,12 +103,12 @@ static void runes_parser_handle_decsel(RunesTerm *t, char *buf, size_t len);
 static void runes_parser_handle_osc0(RunesTerm *t, char *buf, size_t len);
 static void runes_parser_handle_osc1(RunesTerm *t, char *buf, size_t len);
 static void runes_parser_handle_osc2(RunesTerm *t, char *buf, size_t len);
+static void runes_parser_handle_ascii(RunesTerm *t, char *text, size_t len);
+static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len);
 %}
 
 %%
 
-{TEXT}+ runes_parser_handle_text(yyextra, yytext, yyleng);
-
 {BEL} runes_parser_handle_bel(yyextra);
 {BS}  runes_parser_handle_bs(yyextra);
 {TAB} runes_parser_handle_tab(yyextra);
@@ -140,13 +145,16 @@ static void runes_parser_handle_osc2(RunesTerm *t, char *buf, size_t len);
 {OSC1} runes_parser_handle_osc1(yyextra, yytext, yyleng);
 {OSC2} runes_parser_handle_osc2(yyextra, yytext, yyleng);
 
+{ASCII}+ runes_parser_handle_ascii(yyextra, yytext, yyleng);
+{CHAR}+  runes_parser_handle_text(yyextra, yytext, yyleng);
+
 {CSI}[<=?]?{CSIPARAMS}{CTRL} {
     fprintf(
         stderr, "unhandled CSI sequence: \\033%*s\\%hho\n",
         (int)yyleng - 2, yytext + 1, yytext[yyleng - 1]);
 }
 
-{CSI}[<=?]?{CSIPARAMS}{TEXT} {
+{CSI}[<=?]?{CSIPARAMS}{CHAR} {
     fprintf(
         stderr, "unhandled CSI sequence: \\033%*s\n",
         (int)yyleng - 1, yytext + 1);
@@ -158,7 +166,7 @@ static void runes_parser_handle_osc2(RunesTerm *t, char *buf, size_t len);
         (int)yyleng - 2, yytext + 1, yytext[yyleng - 1]);
 }
 
-{OSC}{TEXT} {
+{OSC}{CHAR} {
     fprintf(
         stderr, "unhandled OSC sequence: \\033%*s\n",
         (int)yyleng - 1, yytext + 1);
@@ -166,18 +174,22 @@ static void runes_parser_handle_osc2(RunesTerm *t, char *buf, size_t len);
 
 {ESC}{CTRL} {
     fprintf(stderr, "unhandled escape sequence: \\%hho\n", yytext[1]);
-    yyless(1);
 }
 
-{ESC}{TEXT} {
-    fprintf(stderr, "unhandled escape sequence: %c\n", yytext[1]);
-    yyless(1);
+{ESC}{CHAR} {
+    fprintf(
+        stderr, "unhandled escape sequence: %*s\n",
+        (int)yyleng - 1, yytext + 1);
 }
 
 {CTRL} {
     fprintf(stderr, "unhandled control character: \\%hho\n", yytext[0]);
 }
 
+(?s:.) {
+    fprintf(stderr, "invalid utf8 byte: \\%hho\n", yytext[0]);
+}
+
 %%
 
 void runes_parser_process_string(RunesTerm *t, char *buf, size_t len)
@@ -195,11 +207,6 @@ void runes_parser_process_string(RunesTerm *t, char *buf, size_t len)
     yylex_destroy(scanner);
 }
 
-static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len)
-{
-    runes_display_show_string(t, text, len);
-}
-
 static void runes_parser_handle_bel(RunesTerm *t)
 {
     runes_window_backend_request_visual_bell(t);
@@ -602,6 +609,16 @@ static void runes_parser_handle_osc2(RunesTerm *t, char *buf, size_t len)
     runes_window_backend_set_window_title(t, buf + 4, len - 5);
 }
 
+static void runes_parser_handle_ascii(RunesTerm *t, char *text, size_t len)
+{
+    runes_display_show_string_ascii(t, text, len);
+}
+
+static void runes_parser_handle_text(RunesTerm *t, char *text, size_t len)
+{
+    runes_display_show_string_utf8(t, text, len);
+}
+
 /* XXX these are copied from the generated file so that I can add the UNUSED
  * declarations, otherwise we get compilation errors */
 void *runes_parser_yyalloc(yy_size_t size, yyscan_t yyscanner)