Implement UTF8 text display

This implements straightforward UTF8 to wchar_t to charcell drawing,
sufficient to display languages without nonspacing characters. Those are
all european languages, chinese, and japanese. Languages requiring
multiple characters per cell or complex layout rules are not supported
and can not be supported without a full layout engine, which I am not
qualified to write. Closes #33.

config.h.in

@@ -61,13 +61,13 @@
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
+#include <locale.h>
 #include <assert.h>
 #include <stdio.h>
 #include <time.h>
 #include "os-support.h"
 #ifdef LOCALEPATH
     #include <libintl.h>
-    #include <locale.h>
     #define _(String) gettext (String)
 #else
     #define _(String) (String)

conversions.c

@@ -18,6 +18,7 @@
 #include "main.h"
 #include "conversions.h"
 #include "md5.h"
+#include "ui-support.h"
 #include <iconv.h>
 #include <libxml/HTMLparser.h>
 #include <langinfo.h>
@@ -293,7 +294,7 @@ char* WrapText (const char* text, unsigned width)
 		break;
 	    }
 
-	    if (strlen (chunk) > width) {
+	    if (utf8_length (chunk) > width) {
 		// First copy remaining stuff in line to newtext.
 		newtext = realloc (newtext, strlen (newtext) + strlen (line) + 2);
 		strcat (newtext, line);

interface.c

@@ -73,7 +73,7 @@ static void UIDisplayItem (const struct newsitem* current_item, const struct fee
 	    free (newtext);
 
 	    int columns = COLS - 10;
-	    mvaddnstr (2, 1, converted, columns);
+	    mvaddn_utf8 (2, 1, converted, columns);
 	    if (xmlStrlen ((xmlChar *) converted) > columns)
 		mvaddstr (2, columns + 1, "...");
 
@@ -88,13 +88,13 @@ static void UIDisplayItem (const struct newsitem* current_item, const struct fee
 		free (newtext);
 
 		int columns = COLS - 10;
-		mvaddnstr (2, 1, converted, columns);
+		mvaddn_utf8 (2, 1, converted, columns);
 		if (xmlStrlen ((xmlChar *) converted) > columns)
 		    mvaddstr (2, columns + 1, "...");
 
 		free (converted);
 	    } else
-		mvaddstr (2, 1, current_feed->title);
+		mvadd_utf8 (2, 1, current_feed->title);
 	}
 
 	attroff (WA_BOLD);
@@ -122,21 +122,21 @@ static void UIDisplayItem (const struct newsitem* current_item, const struct fee
 
 	    int columns = COLS - 6;
 	    if (xmlStrlen ((xmlChar *) converted) > columns) {
-		mvaddnstr (4, 1, converted, columns);
+		mvaddn_utf8 (4, 1, converted, columns);
 		if (xmlStrlen ((xmlChar *) converted) > columns)
 		    mvaddstr (4, columns + 1, "...");
 	    } else
-		mvprintw (4, (COLS / 2) - (xmlStrlen ((xmlChar *) converted) / 2), "%s", converted);
+		mvadd_utf8 (4, (COLS / 2) - (xmlStrlen ((xmlChar *) converted) / 2), converted);
 
 	    free (converted);
 	} else
-	    mvaddstr (4, (COLS / 2) - (strlen (_("No title")) / 2), _("No title"));
+	    mvadd_utf8 (4, (COLS / 2) - (strlen (_("No title")) / 2), _("No title"));
 
 	if (current_item->data->description == NULL)
-	    mvprintw (6, 1, _("No description available."));
+	    mvadd_utf8 (6, 1, _("No description available."));
 	else {
 	    if (strlen (current_item->data->description) == 0)
-		mvprintw (6, 1, _("No description available."));
+		mvadd_utf8 (6, 1, _("No description available."));
 	    else {
 		// Only generate a new scroll list if we need to rewrap everything.
 		// Otherwise just typeaheadskip this block.
@@ -188,7 +188,7 @@ static void UIDisplayItem (const struct newsitem* current_item, const struct fee
 		// We sould now have the linked list setup'ed... hopefully.
 		unsigned ypos = 6;
 		for (; ypos <= LINES - 4u && l; ++ientry, ++ypos, l = l->next)
-		    mvaddstr (ypos, 2, l->line);
+		    mvadd_utf8 (ypos, 2, l->line);
 	    }
 	}
 
@@ -202,9 +202,11 @@ static void UIDisplayItem (const struct newsitem* current_item, const struct fee
 
 	// Print article URL.
 	if (current_item->data->link == NULL)
-	    mvaddstr (LINES - 2, 1, _("-> No link"));
-	else
-	    mvprintw (LINES - 2, 1, "-> %s", current_item->data->link);
+	    mvadd_utf8 (LINES - 2, 1, _("-> No link"));
+	else {
+	    mvaddstr (LINES - 2, 1, "-> ");
+	    add_utf8 (current_item->data->link);
+	}
 
 	// Disable color style.
 	if (!_settings.monochrome) {
@@ -391,13 +393,13 @@ static int UIDisplayFeed (struct feed* current_feed)
 	    free (newtext);
 
 	    int columns = COLS - 10;
-	    mvaddnstr (2, 1, converted, columns);
+	    mvaddn_utf8 (2, 1, converted, columns);
 	    if (xmlStrlen ((xmlChar *) converted) > columns)
 		mvaddstr (2, columns + 1, "...");
 
 	    free (converted);
 	} else
-	    mvprintw (2, 1, "%s", current_feed->title);
+	    mvadd_utf8 (2, 1, current_feed->title);
 
 	if (_settings.color.feedtitle > -1) {
 	    attroff (COLOR_PAIR (4));
@@ -447,7 +449,7 @@ static int UIDisplayFeed (struct feed* current_feed)
 		free (newtext);
 
 		int columns = COLS - 6;	// Cut max item length.
-		mvaddnstr (ypos, 1, converted, columns);
+		mvaddn_utf8 (ypos, 1, converted, columns);
 		if (xmlStrlen ((xmlChar *) converted) > columns)
 		    mvaddstr (ypos, columns + 1, "...");
 
@@ -456,9 +458,9 @@ static int UIDisplayFeed (struct feed* current_feed)
 		if (current_feed->smartfeed == 1) {
 		    char tmpstr[256];
 		    snprintf (tmpstr, sizeof (tmpstr), "(%s) %s", item->data->parent->title, _("No title"));
-		    mvaddstr (ypos, 1, tmpstr);
+		    mvadd_utf8 (ypos, 1, tmpstr);
 		} else
-		    mvaddstr (ypos, 1, _("No title"));
+		    mvadd_utf8 (ypos, 1, _("No title"));
 	    }
 
 	    ++ypos;
@@ -489,9 +491,11 @@ static int UIDisplayFeed (struct feed* current_feed)
 
 	// Print feed URL.
 	if (current_feed->link == NULL)
-	    mvaddstr (LINES - 2, 1, _("-> No link"));
-	else
-	    mvprintw (LINES - 2, 1, "-> %s", current_feed->link);
+	    mvadd_utf8 (LINES - 2, 1, _("-> No link"));
+	else {
+	    mvaddstr (LINES - 2, 1, "-> ");
+	    add_utf8 (current_feed->link);
+	}
 
 	// Disable color style.
 	if (!_settings.monochrome) {
@@ -609,7 +613,7 @@ static int UIDisplayFeed (struct feed* current_feed)
 	    else if (uiinput == _settings.keybindings.urljump2 && highlighted)
 		UISupportURLJump (highlighted->data->link);
 	    else if (uiinput == _settings.keybindings.markread) {	// Mark everything read.
-		for (struct newsitem * i = current_feed->items; i; i = i->next) {
+		for (struct newsitem* i = current_feed->items; i; i = i->next) {
 		    if (!i->data->readstatus) {
 			i->data->readstatus = true;
 			current_feed->mtime = curtime;
@@ -624,8 +628,7 @@ static int UIDisplayFeed (struct feed* current_feed)
 	    else if (uiinput == _settings.keybindings.feedinfo)
 		FeedInfo (current_feed);
 	    else if (resize_dirty || uiinput == KEY_RESIZE) {
-		endwin();
-		refresh();
+		clear();
 		resize_dirty = false;
 	    } else if (uiinput == 12)	// Redraw screen on ^L
 		clear();
@@ -893,7 +896,7 @@ void UIMainInterface (void)
 	    else
 		columns = COLS - 9 - strlen (_("new"));
 
-	    mvaddnstr (ypos, 1, cur_ptr->title, columns);
+	    mvaddn_utf8 (ypos, 1, cur_ptr->title, columns);
 	    if (xmlStrlen ((xmlChar *) cur_ptr->title) > columns)
 		mvaddstr (ypos, columns + 1, "...");
 
@@ -905,11 +908,11 @@ void UIMainInterface (void)
 		const char* localized_msg = ngettext ("%3u new", "%3u new", newcount);
 		char msgbuf[16];
 		snprintf (msgbuf, sizeof (msgbuf), localized_msg, newcount);
-		mvaddstr (ypos, COLS - 1 - strlen (localized_msg), msgbuf);
+		mvadd_utf8 (ypos, COLS - 1 - strlen (localized_msg), msgbuf);
 	    }
 
 	    if (cur_ptr->feedcategories != NULL)
-		mvaddnstr (ypos, COLS - 21 - strlen (_("new")), cur_ptr->feedcategories->name, 15);
+		mvaddn_utf8 (ypos, COLS - 21 - strlen (_("new")), cur_ptr->feedcategories->name, 15);
 
 	    ++ypos;
 
@@ -1286,8 +1289,7 @@ void UIMainInterface (void)
 		andxor = !andxor;
 		filteractivated = true;	// Reconstruct filter.
 	    } else if (resize_dirty || uiinput == KEY_RESIZE) {
-		endwin();
-		refresh();
+		clear();
 		resize_dirty = false;
 	    } else if (uiinput == 12)	// Redraw screen on ^L
 		clear();

main.c

@@ -172,7 +172,7 @@ static void CleanupStderrLog (void)
 }
 
 // Redirects stderr into a log file in /tmp
-void RedirectStderrToLog (void)
+static void RedirectStderrToLog (void)
 {
     char logfile[PATH_MAX];
     if (0 > MakeStderrLogFileName (logfile, sizeof (logfile)))
@@ -293,11 +293,11 @@ static void srandrand (void)
 int main (int argc, char* argv[])
 {
     InstallSignalHandlers();
-    srandrand();	       // Init the pRNG. See about.c for usages of rand() ;)
-#ifdef LOCALEPATH
+    srandrand();
     setlocale (LC_ALL, "");
-    bindtextdomain ("snownews", LOCALEPATH);
-    textdomain ("snownews");
+#ifdef LOCALEPATH
+    bindtextdomain (SNOWNEWS_NAME, LOCALEPATH);
+    textdomain (SNOWNEWS_NAME);
 #endif
     RedirectStderrToLog();
 

ui-support.c

@@ -39,7 +39,7 @@ void UIStatus (const char* text, int delay, int warning)
     attron (attr);
     clearLine (LINES - 1, INVERSE);
     attron (WA_REVERSE);
-    mvaddnstr (LINES - 1, 1, text, COLS - 2);
+    mvaddn_utf8 (LINES - 1, 1, text, COLS - 2);
     attroff (attr);
     refresh();
     if (delay)
@@ -234,3 +234,71 @@ void clearLine (int line, clear_line how)
     if (how == INVERSE)
 	attron (WA_REVERSE);
 }
+
+// Returns the number of bytes in the character whose first char is c
+static unsigned utf8_ibytes (char c)
+{
+    // Count the leading bits. Header bits are 1 * nBytes followed by a 0.
+    //	0 - single byte character. Take 7 bits (0xff >> 1)
+    //	1 - error, in the middle of the character. Take 6 bits (0xff >> 2)
+    //	    so you will keep reading invalid entries until you hit the next character.
+    //	>2 - multibyte character. Take remaining bits, and get the next bytes.
+    //
+    unsigned n = 0u;
+    for (uint8_t mask = 0x80; c & mask; ++n)
+	mask >>= 1;
+    return n+!n; // A sequence is always at least 1 byte.
+}
+
+static wchar_t utf8_next (const char** pt)
+{
+    const char* i = *pt;
+    unsigned n = utf8_ibytes (*i);
+    wchar_t v = *i & (0xff >> n);	// First byte contains bits after the header.
+    while (--n && *++i)			// Each subsequent byte has 6 bits.
+	v = (v << 6) | (*i & 0x3f);
+    *pt = ++i;
+    return v;
+}
+
+unsigned utf8_length (const char* s)
+{
+    unsigned l = 0;
+    while (utf8_next (&s))
+	++l;
+    return l;
+}
+
+void addn_utf8 (const char* s, unsigned n)
+{
+    attr_t attr = 0;
+    short cpair = 0;
+    attr_get (&attr, &cpair, NULL);
+
+    wchar_t wchzs[2] = { 0, 0 };
+    cchar_t ch = {};
+
+    while (n-- && (wchzs[0] = utf8_next (&s))) {
+	setcchar (&ch, wchzs, attr, cpair, NULL);
+	add_wch (&ch);
+    }
+}
+
+void add_utf8 (const char* s)
+{
+    int x = getcurx (stdscr);
+    unsigned w = getmaxx (stdscr);
+    addn_utf8 (s, w-x);
+}
+
+void mvaddn_utf8 (int y, int x, const char* s, unsigned n)
+{
+    move (y, x);
+    addn_utf8 (s, n);
+}
+
+void mvadd_utf8 (int y, int x, const char* s)
+{
+    move (y, x);
+    add_utf8 (s);
+}

ui-support.h

@@ -34,3 +34,8 @@ void SmartFeedNewitems (struct feed* smart_feed);
 bool SmartFeedExists (const char* smartfeed);
 void DrawProgressBar (int numobjects, int titlestrlen);
 void clearLine (int line, clear_line how);
+unsigned utf8_length (const char* s);
+void add_utf8 (const char* s);
+void addn_utf8 (const char* s, unsigned n);
+void mvadd_utf8 (int y, int x, const char* s);
+void mvaddn_utf8 (int y, int x, const char* s, unsigned n);