SLUDGE: use U32String to replace sludge utf8 library

8 files changed, 238 insertions, 706 deletions

diff --git a/engines/sludge/CommonCode/utf8.cpp b/engines/sludge/CommonCode/utf8.cpp
deleted file mode 100644
index 291bef4fb9..0000000000
--- a/engines/sludge/CommonCode/utf8.cpp
+++ /dev/null
@@ -1,555 +0,0 @@
-/* ScummVM - Graphic Adventure Engine
- *
- * ScummVM is the legal property of its developers, whose names
- * are too numerous to list here. Please refer to the COPYRIGHT
- * file distributed with this source distribution.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- *
- */
-/*
-  Basic UTF-8 manipulation routines
-  by Jeff Bezanson
-  placed in the public domain Fall 2005
-
-  This code is designed to provide the utilities you need to manipulate
-  UTF-8 as an internal string encoding. These functions do not perform the
-  error checking normally needed when handling UTF-8 data, so if you happen
-  to be from the Unicode Consortium you will want to flay me alive.
-  I do this because error checking can be performed at the boundaries (I/O),
-  with these routines reserved for higher performance on data known to be
-  valid.
-*/
-
-#include "common/debug.h"
-
-#include "sludge/CommonCode/utf8.h"
-
-static const uint32 offsetsFromUTF8[6] = {
-    0x00000000UL, 0x00003080UL, 0x000E2080UL,
-    0x03C82080UL, 0xFA082080UL, 0x82082080UL
-};
-
-static const char trailingBytesForUTF8[256] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
-};
-
-/* returns length of next utf-8 sequence */
-int u8_seqlen(char *s)
-{
-    return trailingBytesForUTF8[(unsigned int)(unsigned char)s[0]] + 1;
-}
-
-/* conversions without error checking
-   only works for valid UTF-8, i.e. no 5- or 6-byte sequences
-   srcsz = source size in bytes, or -1 if 0-terminated
-   sz = dest size in # of wide characters
-
-   returns # characters converted
-   dest will always be L'\0'-terminated, even if there isn't enough room
-   for all the characters.
-   if sz = srcsz+1 (i.e. 4*srcsz+4 bytes), there will always be enough space.
-*/
-int u8_toucs(uint32 *dest, int sz, char *src, int srcsz)
-{
-    uint32 ch;
-    char *src_end = src + srcsz;
-    int nb;
-    int i=0;
-
-    while (i < sz-1) {
-        nb = trailingBytesForUTF8[(unsigned char)*src];
-        if (srcsz == -1) {
-            if (*src == 0)
-                goto done_toucs;
-        }
-        else {
-            if (src + nb >= src_end)
-                goto done_toucs;
-        }
-        ch = 0;
-        switch (nb) {
-            /* these fall through deliberately */
-        case 3: ch += (unsigned char)*src++; ch <<= 6;
-        case 2: ch += (unsigned char)*src++; ch <<= 6;
-        case 1: ch += (unsigned char)*src++; ch <<= 6;
-        case 0: ch += (unsigned char)*src++;
-        }
-        ch -= offsetsFromUTF8[nb];
-        dest[i++] = ch;
-    }
- done_toucs:
-    dest[i] = 0;
-    return i;
-}
-
-/* srcsz = number of source characters, or -1 if 0-terminated
-   sz = size of dest buffer in bytes
-
-   returns # characters converted
-   dest will only be '\0'-terminated if there is enough space. this is
-   for consistency; imagine there are 2 bytes of space left, but the next
-   character requires 3 bytes. in this case we could NUL-terminate, but in
-   general we can't when there's insufficient space. therefore this function
-   only NUL-terminates if all the characters fit, and there's space for
-   the NUL as well.
-   the destination string will never be bigger than the source string.
-*/
-int u8_toutf8(char *dest, int sz, uint32 *src, int srcsz)
-{
-    uint32 ch;
-    int i = 0;
-    char *dest_end = dest + sz;
-
-    while (srcsz<0 ? src[i]!=0 : i < srcsz) {
-        ch = src[i];
-        if (ch < 0x80) {
-            if (dest >= dest_end)
-                return i;
-            *dest++ = (char)ch;
-        }
-        else if (ch < 0x800) {
-            if (dest >= dest_end-1)
-                return i;
-            *dest++ = (ch>>6) | 0xC0;
-            *dest++ = (ch & 0x3F) | 0x80;
-        }
-        else if (ch < 0x10000) {
-            if (dest >= dest_end-2)
-                return i;
-            *dest++ = (ch>>12) | 0xE0;
-            *dest++ = ((ch>>6) & 0x3F) | 0x80;
-            *dest++ = (ch & 0x3F) | 0x80;
-        }
-        else if (ch < 0x110000) {
-            if (dest >= dest_end-3)
-                return i;
-            *dest++ = (ch>>18) | 0xF0;
-            *dest++ = ((ch>>12) & 0x3F) | 0x80;
-            *dest++ = ((ch>>6) & 0x3F) | 0x80;
-            *dest++ = (ch & 0x3F) | 0x80;
-        }
-        i++;
-    }
-    if (dest < dest_end)
-        *dest = '\0';
-    return i;
-}
-
-int u8_wc_toutf8(char *dest, uint32 ch)
-{
-    if (ch < 0x80) {
-        dest[0] = (char)ch;
-        return 1;
-    }
-    if (ch < 0x800) {
-        dest[0] = (ch>>6) | 0xC0;
-        dest[1] = (ch & 0x3F) | 0x80;
-        return 2;
-    }
-    if (ch < 0x10000) {
-        dest[0] = (ch>>12) | 0xE0;
-        dest[1] = ((ch>>6) & 0x3F) | 0x80;
-        dest[2] = (ch & 0x3F) | 0x80;
-        return 3;
-    }
-    if (ch < 0x110000) {
-        dest[0] = (ch>>18) | 0xF0;
-        dest[1] = ((ch>>12) & 0x3F) | 0x80;
-        dest[2] = ((ch>>6) & 0x3F) | 0x80;
-        dest[3] = (ch & 0x3F) | 0x80;
-        return 4;
-    }
-    return 0;
-}
-
-/* charnum => byte offset */
-int u8_offset(char *str, int charnum)
-{
-    int offs=0;
-
-    while (charnum > 0 && str[offs]) {
-        (void)(isutf(str[++offs]) || isutf(str[++offs]) ||
-               isutf(str[++offs]) || ++offs);
-        charnum--;
-    }
-    return offs;
-}
-
-/* byte offset => charnum */
-int u8_charnum(char *s, int offset)
-{
-    int charnum = 0, offs=0;
-
-    while (offs < offset && s[offs]) {
-        (void)(isutf(s[++offs]) || isutf(s[++offs]) ||
-               isutf(s[++offs]) || ++offs);
-        charnum++;
-    }
-    return charnum;
-}
-
-/* number of characters */
-int u8_strlen(char *s)
-{
-    int count = 0;
-    int i = 0;
-
-    while (u8_nextchar(s, &i) != 0)
-        count++;
-
-    return count;
-}
-
-/* reads the next utf-8 sequence out of a string, updating an index */
-uint32 u8_nextchar(const char *s, int *i)
-{
-    uint32 ch = 0;
-    int sz = 0;
-
-    do {
-        ch <<= 6;
-        ch += (unsigned char)s[(*i)++];
-        sz++;
-    } while (s[*i] && !isutf(s[*i]));
-    ch -= offsetsFromUTF8[sz-1];
-
-    return ch;
-}
-
-void u8_inc(char *s, int *i)
-{
-    (void)(isutf(s[++(*i)]) || isutf(s[++(*i)]) ||
-           isutf(s[++(*i)]) || ++(*i));
-}
-
-void u8_dec(char *s, int *i)
-{
-    (void)(isutf(s[--(*i)]) || isutf(s[--(*i)]) ||
-           isutf(s[--(*i)]) || --(*i));
-}
-
-int octal_digit(char c)
-{
-    return (c >= '0' && c <= '7');
-}
-
-int hex_digit(char c)
-{
-    return ((c >= '0' && c <= '9') ||
-            (c >= 'A' && c <= 'F') ||
-            (c >= 'a' && c <= 'f'));
-}
-
-/* assumes that src points to the character after a backslash
-   returns number of input characters processed */
-int u8_read_escape_sequence(char *str, uint32 *dest)
-{
-    uint32 ch;
-    char digs[9]="\0\0\0\0\0\0\0\0";
-    int dno=0, i=1;
-
-    ch = (uint32)str[0];    /* take literal character */
-    if (str[0] == 'n')
-        ch = L'\n';
-    else if (str[0] == 't')
-        ch = L'\t';
-    else if (str[0] == 'r')
-        ch = L'\r';
-    else if (str[0] == 'b')
-        ch = L'\b';
-    else if (str[0] == 'f')
-        ch = L'\f';
-    else if (str[0] == 'v')
-        ch = L'\v';
-    else if (str[0] == 'a')
-        ch = L'\a';
-    else if (octal_digit(str[0])) {
-        i = 0;
-        do {
-            digs[dno++] = str[i++];
-        } while (octal_digit(str[i]) && dno < 3);
-        ch = strtol(digs, NULL, 8);
-    }
-    else if (str[0] == 'x') {
-        while (hex_digit(str[i]) && dno < 2) {
-            digs[dno++] = str[i++];
-        }
-        if (dno > 0)
-            ch = strtol(digs, NULL, 16);
-    }
-    else if (str[0] == 'u') {
-        while (hex_digit(str[i]) && dno < 4) {
-            digs[dno++] = str[i++];
-        }
-        if (dno > 0)
-            ch = strtol(digs, NULL, 16);
-    }
-    else if (str[0] == 'U') {
-        while (hex_digit(str[i]) && dno < 8) {
-            digs[dno++] = str[i++];
-        }
-        if (dno > 0)
-            ch = strtol(digs, NULL, 16);
-    }
-    *dest = ch;
-
-    return i;
-}
-
-/* convert a string with literal \uxxxx or \Uxxxxxxxx characters to UTF-8
-   example: u8_unescape(mybuf, 256, "hello\\u220e")
-   note the double backslash is needed if called on a C string literal */
-int u8_unescape(char *buf, int sz, char *src)
-{
-    int c=0, amt;
-    uint32 ch;
-    char temp[4];
-
-    while (*src && c < sz) {
-        if (*src == '\\') {
-            src++;
-            amt = u8_read_escape_sequence(src, &ch);
-        }
-        else {
-            ch = (uint32)*src;
-            amt = 1;
-        }
-        src += amt;
-        amt = u8_wc_toutf8(temp, ch);
-        if (amt > sz-c)
-            break;
-        memcpy(&buf[c], temp, amt);
-        c += amt;
-    }
-    if (c < sz)
-        buf[c] = '\0';
-    return c;
-}
-
-int u8_escape_wchar(char *buf, int sz, uint32 ch)
-{
-    if (ch == L'\n')
-        return snprintf(buf, sz, "\\n");
-    else if (ch == L'\t')
-        return snprintf(buf, sz, "\\t");
-    else if (ch == L'\r')
-        return snprintf(buf, sz, "\\r");
-    else if (ch == L'\b')
-        return snprintf(buf, sz, "\\b");
-    else if (ch == L'\f')
-        return snprintf(buf, sz, "\\f");
-    else if (ch == L'\v')
-        return snprintf(buf, sz, "\\v");
-    else if (ch == L'\a')
-        return snprintf(buf, sz, "\\a");
-    else if (ch == L'\\')
-        return snprintf(buf, sz, "\\\\");
-    else if (ch < 32 || ch == 0x7f)
-        return snprintf(buf, sz, "\\x%hhX", (unsigned char)ch);
-    else if (ch > 0xFFFF)
-        return snprintf(buf, sz, "\\U%.8X", (uint32)ch);
-    else if (ch >= 0x80 && ch <= 0xFFFF)
-        return snprintf(buf, sz, "\\u%.4hX", (unsigned short)ch);
-
-    return snprintf(buf, sz, "%c", (char)ch);
-}
-
-int u8_escape(char *buf, int sz, char *src, int escape_quotes)
-{
-    int c=0, i=0, amt;
-
-    while (src[i] && c < sz) {
-        if (escape_quotes && src[i] == '"') {
-            amt = snprintf(buf, sz - c, "\\\"");
-            i++;
-        }
-        else {
-            amt = u8_escape_wchar(buf, sz - c, u8_nextchar(src, &i));
-        }
-        c += amt;
-        buf += amt;
-    }
-    if (c < sz)
-        *buf = '\0';
-    return c;
-}
-
-char *u8_strchr(char *s, uint32 ch, int *charn)
-{
-    int i = 0, lasti=0;
-    uint32 c;
-
-    *charn = 0;
-    while (s[i]) {
-        c = u8_nextchar(s, &i);
-        if (c == ch) {
-            return &s[lasti];
-        }
-        lasti = i;
-        (*charn)++;
-    }
-    return NULL;
-}
-
-char *u8_memchr(char *s, uint32 ch, size_t sz, int *charn)
-{
-    int i = 0, lasti=0;
-    uint32 c;
-    int csz;
-
-    *charn = 0;
-    while (i < sz) {
-        c = csz = 0;
-        do {
-            c <<= 6;
-            c += (unsigned char)s[i++];
-            csz++;
-        } while (i < sz && !isutf(s[i]));
-        c -= offsetsFromUTF8[csz-1];
-
-        if (c == ch) {
-            return &s[lasti];
-        }
-        lasti = i;
-        (*charn)++;
-    }
-    return NULL;
-}
-
-int u8_is_locale_utf8(char *locale)
-{
-    /* this code based on libutf8 */
-    const char* cp = locale;
-
-    for (; *cp != '\0' && *cp != '@' && *cp != '+' && *cp != ','; cp++) {
-        if (*cp == '.') {
-            const char* encoding = ++cp;
-            for (; *cp != '\0' && *cp != '@' && *cp != '+' && *cp != ','; cp++)
-                ;
-            if ((cp-encoding == 5 && !strncmp(encoding, "UTF-8", 5))
-                || (cp-encoding == 4 && !strncmp(encoding, "utf8", 4)))
-                return 1; /* it's UTF-8 */
-            break;
-        }
-    }
-    return 0;
-}
-
-int u8_vprintf(char *fmt, va_list ap)
-{
-    int cnt, sz=0;
-    char *buf;
-    uint32 *wcs;
-
-    sz = 512;
-    buf = (char*)alloca(sz);
- try_print:
-    cnt = vsnprintf(buf, sz, fmt, ap);
-    if (cnt >= sz) {
-        buf = (char*)alloca(cnt - sz + 1);
-        sz = cnt + 1;
-        goto try_print;
-    }
-    wcs = (uint32*)alloca((cnt+1) * sizeof(uint32));
-    cnt = u8_toucs(wcs, cnt+1, buf, cnt);
-    debug("%ls", (wchar_t*)wcs);
-    return cnt;
-}
-
-int u8_printf(char *fmt, ...)
-{
-    int cnt;
-    va_list args;
-
-    va_start(args, fmt);
-
-    cnt = u8_vprintf(fmt, args);
-
-    va_end(args);
-    return cnt;
-}
-
-
-// This function checks a string to see if it's valid UTF-8
-// It returns true if the string is valid.
-//
-// based on the valid_utf8 routine from the PCRE library by Philip Hazel
-
-int u8_isvalid(const char *str)
-{
-    const unsigned char *p;
-    unsigned char c;
-    int ab;
-	
-    for (p = (unsigned char*)str; *p; p++) {
-        c = *p;
-        if (c < 128)
-            continue;
-        if ((c & 0xc0) != 0xc0)
-            return 0;
-        ab = trailingBytesForUTF8[c];
-		
-        p++;
-        /* Check top bits in the second byte */
-        if ((*p & 0xc0) != 0x80)
-            return 0;
-		
-        /* Check for overlong sequences for each different length */
-        switch (ab) {
-				/* Check for xx00 000x */
-			case 1:
-				if ((c & 0x3e) == 0) return 0;
-				continue;   /* We know there aren't any more bytes to check */
-				
-				/* Check for 1110 0000, xx0x xxxx */
-			case 2:
-				if (c == 0xe0 && (*p & 0x20) == 0) return 0;
-				break;
-				
-				/* Check for 1111 0000, xx00 xxxx */
-			case 3:
-				if (c == 0xf0 && (*p & 0x30) == 0) return 0;
-				break;
-				
-				/* Check for 1111 1000, xx00 0xxx */
-			case 4:
-				if (c == 0xf8 && (*p & 0x38) == 0) return 0;
-				break;
-				
-				/* Check for leading 0xfe or 0xff,
-				 and then for 1111 1100, xx00 00xx */
-			case 5:
-				if (c == 0xfe || c == 0xff ||
-					(c == 0xfc && (*p & 0x3c) == 0)) return 0;
-				break;
-        }
-		
-        /* Check for valid bytes after the 2nd, if any; all must start 10 */
-        while (--ab > 0) {
-            if ((*(++p) & 0xc0) != 0x80) return 0;
-        }
-    }
-	
-    return 1;
-}
-
diff --git a/engines/sludge/CommonCode/utf8.h b/engines/sludge/CommonCode/utf8.h
deleted file mode 100644
index c9af92e2ca..0000000000
--- a/engines/sludge/CommonCode/utf8.h
+++ /dev/null
@@ -1,96 +0,0 @@
-/* ScummVM - Graphic Adventure Engine
- *
- * ScummVM is the legal property of its developers, whose names
- * are too numerous to list here. Please refer to the COPYRIGHT
- * file distributed with this source distribution.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- *
- */
-
-#include "common/system.h"
-
-/* is c the start of a utf8 sequence? */
-#define isutf(c) (((c)&0xC0)!=0x80)
-
-/* convert UTF-8 data to wide character */
-int u8_toucs(uint32 *dest, int sz, char *src, int srcsz);
-
-/* the opposite conversion */
-int u8_toutf8(char *dest, int sz, uint32 *src, int srcsz);
-
-/* single character to UTF-8 */
-int u8_wc_toutf8(char *dest, uint32 ch);
-
-/* character number to byte offset */
-int u8_offset(char *str, int charnum);
-
-/* byte offset to character number */
-int u8_charnum(char *s, int offset);
-
-/* return next character, updating an index variable */
-uint32 u8_nextchar(const char *s, int *i);
-
-/* move to next character */
-void u8_inc(char *s, int *i);
-
-/* move to previous character */
-void u8_dec(char *s, int *i);
-
-/* returns length of next utf-8 sequence */
-int u8_seqlen(char *s);
-
-/* assuming src points to the character after a backslash, read an
-   escape sequence, storing the result in dest and returning the number of
-   input characters processed */
-int u8_read_escape_sequence(char *src, uint32 *dest);
-
-/* given a wide character, convert it to an ASCII escape sequence stored in
-   buf, where buf is "sz" bytes. returns the number of characters output. */
-int u8_escape_wchar(char *buf, int sz, uint32 ch);
-
-/* convert a string "src" containing escape sequences to UTF-8 */
-int u8_unescape(char *buf, int sz, char *src);
-
-/* convert UTF-8 "src" to ASCII with escape sequences.
-   if escape_quotes is nonzero, quote characters will be preceded by
-   backslashes as well. */
-int u8_escape(char *buf, int sz, char *src, int escape_quotes);
-
-/* utility predicates used by the above */
-int octal_digit(char c);
-int hex_digit(char c);
-
-/* return a pointer to the first occurrence of ch in s, or NULL if not
-   found. character index of found character returned in *charn. */
-char *u8_strchr(char *s, uint32 ch, int *charn);
-
-/* same as the above, but searches a buffer of a given size instead of
-   a NUL-terminated string. */
-char *u8_memchr(char *s, uint32 ch, size_t sz, int *charn);
-
-/* count the number of characters in a UTF-8 string */
-int u8_strlen(char *s);
-
-int u8_is_locale_utf8(char *locale);
-
-/* printf where the format string and arguments may be in UTF-8.
-   you can avoid this function and just use ordinary printf() if the current
-   locale is UTF-8. */
-int u8_vprintf(char *fmt, va_list ap);
-int u8_printf(char *fmt, ...);
-
-
-int u8_isvalid(const char *input);
diff --git a/engines/sludge/builtin.cpp b/engines/sludge/builtin.cpp
index 1f19fc3b14..fc92161e3c 100644
--- a/engines/sludge/builtin.cpp
+++ b/engines/sludge/builtin.cpp
@@ -50,7 +50,7 @@
 #include "sludge/thumbnail.h"
 #include "sludge/graphics.h"
 #include "sludge/sludge.h"
-#include "sludge/CommonCode/utf8.h"
+#include "sludge/utf8.h"
 
 namespace Sludge {
 
@@ -558,9 +558,12 @@ builtIn(substring) {
 	wholeString = getTextFromAnyVar(fun->stack->thisVar);
 	trimStack(fun->stack);
 
-	if (u8_strlen(wholeString) < start + length) {
-		length = u8_strlen(wholeString) - start;
-		if (u8_strlen(wholeString) < start) {
+	UTF8Converter convert(wholeString);
+	Common::U32String str32 = convert.getU32String();
+
+	if (str32.size() < start + length) {
+		length = str32.size() - start;
+		if (str32.size() < start) {
 			start = 0;
 		}
 	}
@@ -568,8 +571,8 @@ builtIn(substring) {
 		length = 0;
 	}
 
-	int startoffset = u8_offset(wholeString, start);
-	int endoffset = u8_offset(wholeString, start + length);
+	int startoffset = convert.getOriginOffset(start);
+	int endoffset = convert.getOriginOffset(start + length);
 
 	newString = new char[endoffset - startoffset + 1];
 	if (!checkNew(newString)) {
@@ -580,7 +583,7 @@ builtIn(substring) {
 	newString[endoffset - startoffset] = 0;
 
 	makeTextVar(fun->reg, newString);
-	delete newString;
+	delete []newString;
 	return BR_CONTINUE;
 }
 
diff --git a/engines/sludge/fonttext.cpp b/engines/sludge/fonttext.cpp
index c3c877c80b..a35c51bd24 100644
--- a/engines/sludge/fonttext.cpp
+++ b/engines/sludge/fonttext.cpp
@@ -28,17 +28,17 @@
 #include "sludge/newfatal.h"
 #include "sludge/moreio.h"
 #include "sludge/platform-dependent.h"
-#include "sludge/CommonCode/utf8.h"
+#include "sludge/utf8.h"
 
 namespace Sludge {
 
 spriteBank theFont;
 int fontHeight = 0, numFontColours, loadedFontNum;
-char *fontOrderString = NULL;
+Common::U32String fontOrderString;
 short fontSpace = -1;
 
 uint32 *fontTable = NULL;
-unsigned int fontTableSize = 0;
+uint fontTableSize = 0;
 
 #define fontInTable(x) ((x<fontTableSize) ? fontTable[(uint32) x] : 0)
 
@@ -50,30 +50,33 @@ bool isInFont(char *theText) {
 	if (!theText[0])
 		return 0;
 
+	Common::U32String str32 = UTF8Converter::convertUtf8ToUtf32(theText);
+
 	// We don't want to compare strings. Only single characters allowed!
-	if (u8_strlen(theText) > 1)
+	if (str32.size() > 1)
 		return false;
 
-	int i = 0;
-	uint32 c = u8_nextchar(theText, &i);
+	uint32 c = str32[0];
 
-	return u8_strchr(fontOrderString, c, &i);
+	// check if font order contains the utf8 char
+	return fontOrderString.contains(c);
 }
 
 int stringLength(char *theText) {
-	return u8_strlen(theText);
+	Common::U32String str32 = UTF8Converter::convertUtf8ToUtf32(theText);
+	return str32.size();
 }
 
 int stringWidth(char *theText) {
-	int a = 0;
-	uint32 c;
 	int xOff = 0;
 
 	if (!fontTableSize)
 		return 0;
 
-	while (theText[a]) {
-		c = u8_nextchar(theText, &a);
+	Common::U32String str32 = UTF8Converter::convertUtf8ToUtf32(theText);
+
+	for (uint i = 0; i < str32.size(); ++i) {
+		uint32 c = str32[i];
 		xOff += theFont.sprites[fontInTable(c)].surface.w + fontSpace;
 	}
 
@@ -81,51 +84,46 @@ int stringWidth(char *theText) {
 }
 
 void pasteString(char *theText, int xOff, int y, spritePalette &thePal) {
-	sprite *mySprite;
-	int a = 0;
-	uint32 c;
-
 	if (!fontTableSize)
 		return;
 
 	xOff += (int)((float)(fontSpace >> 1) / cameraZoom);
-	while (theText[a]) {
-		c = u8_nextchar(theText, &a);
-		mySprite = &theFont.sprites[fontInTable(c)];
+
+	Common::U32String str32 = UTF8Converter::convertUtf8ToUtf32(theText);
+
+	for (uint32 i = 0; i < str32.size(); ++i) {
+		uint32 c = str32[i];
+		sprite *mySprite = &theFont.sprites[fontInTable(c)];
 		fontSprite(xOff, y, *mySprite, thePal);
 		xOff += (int)((double)(mySprite->surface.w + fontSpace) / cameraZoom);
 	}
 }
 
 void pasteStringToBackdrop(char *theText, int xOff, int y, spritePalette &thePal) {
-	sprite *mySprite;
-	int a = 0;
-	uint32 c;
-
 	if (!fontTableSize)
 		return;
 
+	Common::U32String str32 = UTF8Converter::convertUtf8ToUtf32(theText);
+
 	xOff += fontSpace >> 1;
-	while (theText[a]) {
-		c = u8_nextchar(theText, &a);
-		mySprite = &theFont.sprites[fontInTable(c)];
+	for (uint32 i = 0; i < str32.size(); ++i) {
+		uint32 c = str32[i];
+		sprite *mySprite = &theFont.sprites[fontInTable(c)];
 		pasteSpriteToBackDrop(xOff, y, *mySprite, thePal);
 		xOff += mySprite->surface.w + fontSpace;
 	}
 }
 
 void burnStringToBackdrop(char *theText, int xOff, int y, spritePalette &thePal) {
-	sprite *mySprite;
-	int a = 0;
-	uint32 c;
-
 	if (!fontTableSize)
 		return;
 
+	Common::U32String str32 = UTF8Converter::convertUtf8ToUtf32(theText);
+
 	xOff += fontSpace >> 1;
-	while (theText[a]) {
-		c = u8_nextchar(theText, &a);
-		mySprite = &theFont.sprites[fontInTable(c)];
+	for (uint i = 0; i < str32.size(); ++i) {
+		uint32 c = str32[i];
+		sprite *mySprite = &theFont.sprites[fontInTable(c)];
 		burnSpriteToBackDrop(xOff, y, *mySprite, thePal);
 		xOff += mySprite->surface.w + fontSpace;
 	}
@@ -165,38 +163,35 @@ void setFontColour(spritePalette &sP, byte r, byte g, byte b) {
 }
 
 bool loadFont(int filenum, const char *charOrder, int h) {
-	int a = 0;
-	uint32 c;
-
-	delete[] fontOrderString;
-	fontOrderString = copyString(charOrder);
+	fontOrderString.clear();
+	fontOrderString = UTF8Converter::convertUtf8ToUtf32(charOrder);
 
 	forgetSpriteBank(theFont);
 
 	loadedFontNum = filenum;
 
+	// get max value among all utf8 chars
 	fontTableSize = 0;
-	while (charOrder[a]) {
-		c = u8_nextchar(charOrder, &a);
+	for (uint32 i = 0; i < fontOrderString.size(); ++i) {
+		uint32 c = fontOrderString[i];
 		if (c > fontTableSize)
 			fontTableSize = c;
 	}
 	fontTableSize++;
 
+	// create an index table from utf8 char to the index
 	delete[] fontTable;
 	fontTable = new uint32[fontTableSize];
 	if (!checkNew(fontTable))
 		return false;
 
-	for (a = 0; a < fontTableSize; a++) {
-		fontTable[a] = 0;
+	for (uint i = 0; i < fontTableSize; i++) {
+		fontTable[i] = 0;
 	}
-	a = 0;
-	int i = 0;
-	while (charOrder[a]) {
-		c = u8_nextchar(charOrder, &a);
+
+	for (uint i = 0; i < fontOrderString.size(); ++i) {
+		uint32 c = fontOrderString[i];
 		fontTable[c] = i;
-		i++;
 	}
 
 	if (!loadSpriteBank(filenum, theFont, true)) {
diff --git a/engines/sludge/fonttext.h b/engines/sludge/fonttext.h
index 92520a087c..cabb52d476 100644
--- a/engines/sludge/fonttext.h
+++ b/engines/sludge/fonttext.h
@@ -22,6 +22,8 @@
 #ifndef SLUDGE_FONTTEXT_H

 #define SLUDGE_FONTTEXT_H

 

+#include "common/ustr.h"

+

 namespace Sludge {

 

 bool loadFont(int filenum, const char *charOrder, int);

diff --git a/engines/sludge/module.mk b/engines/sludge/module.mk
index f1e4cadc42..afe1d88b8b 100644
--- a/engines/sludge/module.mk
+++ b/engines/sludge/module.mk
@@ -39,9 +39,9 @@ MODULE_OBJS := \
 	thumbnail.o \
 	timing.o \
 	transition.o \
+	utf8.o \
 	variable.o \
 	zbuffer.o \
-	CommonCode/utf8.o \
 #	shaders.o \
 #	libwebm/mkvparser.o \
 #	libwebm/mkvreader.o \
diff --git a/engines/sludge/utf8.cpp b/engines/sludge/utf8.cpp
new file mode 100644
index 0000000000..95f9e91f8a
--- /dev/null
+++ b/engines/sludge/utf8.cpp
@@ -0,0 +1,101 @@
+/* ScummVM - Graphic Adventure Engine
+ *
+ * ScummVM is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+/*
+ Basic UTF-8 manipulation routines
+ by Jeff Bezanson
+ placed in the public domain Fall 2005
+
+ This code is designed to provide the utilities you need to manipulate
+ UTF-8 as an internal string encoding. These functions do not perform the
+ error checking normally needed when handling UTF-8 data, so if you happen
+ to be from the Unicode Consortium you will want to flay me alive.
+ I do this because error checking can be performed at the boundaries (I/O),
+ with these routines reserved for higher performance on data known to be
+ valid.
+ */
+
+#include "common/debug.h"
+
+#include "sludge/utf8.h"
+
+namespace Sludge {
+
+const uint32 UTF8Converter::offsetsFromUTF8[6] = {
+		0x00000000UL, 0x00003080UL,
+		0x000E2080UL, 0x03C82080UL,
+		0xFA082080UL, 0x82082080UL };
+
+/* reads the next utf-8 sequence out of a string, updating an index */
+uint32 UTF8Converter::nextchar(const char *s, int *i) {
+	uint32 ch = 0;
+	int sz = 0;
+
+	do {
+		ch <<= 6;
+		ch += (unsigned char)s[(*i)++];
+		sz++;
+	} while (s[*i] && !isutf(s[*i]));
+	ch -= offsetsFromUTF8[sz - 1];
+
+	return ch;
+}
+
+Common::U32String UTF8Converter::convertUtf8ToUtf32(const Common::String &str) {
+	// we assume one character in a Common::String is one byte
+	// but in this case it's actually an UTF-8 string
+	// with up to 4 bytes per character. To work around this,
+	// convert it to an U32String before any further operation
+	Common::U32String u32str;
+	int i = 0;
+	while (i < (int)str.size()) {
+		uint32 chr = nextchar(str.c_str(), &i);
+		u32str += chr;
+	}
+	return u32str;
+}
+
+/* utf32 index => original byte offset */
+int UTF8Converter::getOriginOffset(int origIdx) {
+	int offs = 0;
+
+	while (origIdx > 0 && _str[offs]) {
+		// increment if it's not the start of a utf8 sequence
+		(void)(isutf(_str[++offs]) || isutf(_str[++offs]) || isutf(_str[++offs]) || ++offs);
+		origIdx--;
+	}
+	return offs;
+}
+
+/** Construct a UTF8String with original char array to convert */
+UTF8Converter::UTF8Converter(const char *str) {
+	setUTF8String(str);
+}
+
+/** set a utf8 string to convert */
+void UTF8Converter::setUTF8String(Common::String str) {
+	_str32.clear();
+	_str32 = convertUtf8ToUtf32(str);
+	_str.clear();
+	_str = str;
+}
+
+} // End of namespace Sludge
diff --git a/engines/sludge/utf8.h b/engines/sludge/utf8.h
new file mode 100644
index 0000000000..4595a6b86f
--- /dev/null
+++ b/engines/sludge/utf8.h
@@ -0,0 +1,82 @@
+/* ScummVM - Graphic Adventure Engine
+ *
+ * ScummVM is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+#ifndef SLUDGE_UTF8_H
+#define SLUDGE_UTF8_H
+
+#include "common/system.h"
+#include "common/ustr.h"
+
+namespace Sludge {
+
+class UTF8Converter {
+private:
+	static const uint32 offsetsFromUTF8[6];
+
+	/**
+	 * we assume one character in a Common::String is one byte
+	 * but in this case it's actually an UTF-8 string
+	 */
+	Common::String _str;
+
+	/**
+	 * wrap a converted U32String
+	 */
+	Common::U32String _str32;
+
+	/** A tool function for string conversion
+	 * return next character, updating an index variable
+	 */
+	static uint32 nextchar(const char *s, int *i);
+
+	/** A tool function for string conversion
+	 * is this byte the start of a utf8 sequence?
+	 */
+	static inline bool isutf(char c) { return (((c)&0xC0)!=0x80); }
+
+public:
+	/** Construct a new empty string. */
+	UTF8Converter() {};
+
+	/** Construct a UTF8String with original char array to convert */
+	UTF8Converter(const char *data);
+
+	/** U32 character index to origin char offset */
+	int getOriginOffset(int origIdx);
+
+	/**
+	 * set a char array to this UTF8String
+	 */
+	void setUTF8String(Common::String str);
+
+	/**
+	 * get converted U32String
+	 */
+	Common::U32String getU32String() { return _str32; };
+
+	/** Convert UTF8 String to UTF32 String
+	 */
+	static Common::U32String convertUtf8ToUtf32(const Common::String &str);
+};
+
+} // End of namespace Sludge
+
+#endif