COMMON & WINTERMUTE: Use non-1252 for 125X games (PR 1698)

* WINTERMUTE: Add detection for "The Driller Incident"

"The Driller Incident" is a small freeware game for Wintermute,
avaliable in English and Russian: http://questzone.ru/enzi/files/1645

* WINTERMUTE: Add detection table for "One Helluva Day" demo

"One Helluva Day" is a point-and-click adventure demo in English / Czech
/ Russian.
Free download:
https://store.steampowered.com/app/603680/One_helluva_day/

* WINTERMUTE: Support CHARSET property for TT fonts

"// we don't need this anymore" was a mistake =)

Surely, most Wintermute games are either designed for 1252 language
(DE_DEU, EN_ANY, ES_ESP, FR_FRA, IT_ITA, PT_BRA), or don't use TrueType
fonts (5ma, deadcity, grotter1, grotter2, thekite, tib), or use
CHARSET=1 with UTF strings (dirtysplit, reversion1, reversion2, twc),
which meen this conversion is not needed for those games.

However, there are some games that explicitly states CHARSET=10 (driller
(RU_RUS), oknytt (RU_RUS), onehelluvaday (UNK_LANG when playing as
Russian)) and there are some games with CHARSET=1 with non-1252 in mind
(bookofgron (RU_RUS excepts 1251), carolreed4 (RU_RUS excepts 1251),
kulivocko (CZ_CZE excepts 1250)).

This fixes text in some games: bookofgron, carolreed4, driller, kulivocko,
oknytt, onehelluvaday.

* WINTERMUTE: Break savegame compatibility

sizeof(BaseFontTT) was changed, so let's break savegame compatibility

* COMMON: Add conversion tables for win1253 and win1257

* COMMON: Add string conversion from U32String back to Common::String

convertUtf32ToUtf8 code is based on Wintermute::ConvertUTF32toUTF8
convertFromU32String use convertUtf32ToUtf8 for UTF8 or lookup through
conversion table for single-byte encodings

* WINTERMUTE: Use Common::convert functions for non-UTF charsets

* WINTERMUTE: Fix whitespaces at detection tables

* WINTERMUTE: Add TODO comments

2 files changed, 7 insertions, 4 deletions

diff --git a/engines/wintermute/base/font/base_font_truetype.cpp b/engines/wintermute/base/font/base_font_truetype.cpp
index 93084ca8a1..03d82cb940 100644
--- a/engines/wintermute/base/font/base_font_truetype.cpp
+++ b/engines/wintermute/base/font/base_font_truetype.cpp
@@ -48,6 +48,7 @@ IMPLEMENT_PERSISTENT(BaseFontTT, false)
 BaseFontTT::BaseFontTT(BaseGame *inGame) : BaseFont(inGame) {
 	_fontHeight = 12;
 	_isBold = _isItalic = _isUnderline = _isStriked = false;
+	_charset = CHARSET_ANSI;
 
 	_fontFile = nullptr;
 	_font = nullptr;
@@ -116,7 +117,7 @@ int BaseFontTT::getTextWidth(const byte *text, int maxLength) {
 	if (_gameRef->_textEncoding == TEXT_UTF8) {
 		textStr = StringUtil::utf8ToWide((const char *)text);
 	} else {
-		textStr = StringUtil::ansiToWide((const char *)text);
+		textStr = StringUtil::ansiToWide((const char *)text, _charset);
 	}
 
 	if (maxLength >= 0 && textStr.size() > (uint32)maxLength) {
@@ -137,7 +138,7 @@ int BaseFontTT::getTextHeight(const byte *text, int width) {
 	if (_gameRef->_textEncoding == TEXT_UTF8) {
 		textStr = StringUtil::utf8ToWide((const char *)text);
 	} else {
-		textStr = StringUtil::ansiToWide((const char *)text);
+		textStr = StringUtil::ansiToWide((const char *)text, _charset);
 	}
 
 
@@ -162,7 +163,7 @@ void BaseFontTT::drawText(const byte *text, int x, int y, int width, TTextAlign
 	if (_gameRef->_textEncoding == TEXT_UTF8) {
 		textStr = StringUtil::utf8ToWide((const char *)text);
 	} else {
-		textStr = StringUtil::ansiToWide((const char *)text);
+		textStr = StringUtil::ansiToWide((const char *)text, _charset);
 	}
 
 	if (maxLength >= 0 && textStr.size() > (uint32)maxLength) {
@@ -412,7 +413,7 @@ bool BaseFontTT::loadBuffer(char *buffer) {
 			break;
 
 		case TOKEN_CHARSET:
-			// we don't need this anymore
+			parser.scanStr(params, "%d", &_charset);
 			break;
 
 		case TOKEN_COLOR: {
@@ -519,6 +520,7 @@ bool BaseFontTT::persist(BasePersistenceManager *persistMgr) {
 	persistMgr->transferBool(TMEMBER(_isStriked));
 	persistMgr->transferSint32(TMEMBER(_fontHeight));
 	persistMgr->transferCharPtr(TMEMBER(_fontFile));
+	persistMgr->transferSint32(TMEMBER_INT(_charset));
 
 
 	// persist layers
diff --git a/engines/wintermute/base/font/base_font_truetype.h b/engines/wintermute/base/font/base_font_truetype.h
index c0349eccd4..d74efa067f 100644
--- a/engines/wintermute/base/font/base_font_truetype.h
+++ b/engines/wintermute/base/font/base_font_truetype.h
@@ -141,6 +141,7 @@ private:
 	bool _isStriked;
 	int32 _fontHeight;
 	char *_fontFile;
+	TTextCharset _charset;
 
 	BaseArray<BaseTTFontLayer *> _layers;
 	void clearCache();