GRAPHICS/GUI: Implement charset mapping for TTF fonts.

The charsets used by the translations now need to have a "$(name).cp" file,
which contains an charset index => unicode mapping. Otherwise
create_translations will fail.

5 files changed, 255 insertions, 10 deletions

diff --git a/devtools/create_translations/cp_parser.cpp b/devtools/create_translations/cp_parser.cpp
new file mode 100644
index 0000000000..a4202bf153
--- /dev/null
+++ b/devtools/create_translations/cp_parser.cpp
@@ -0,0 +1,104 @@
+/* ScummVM - Graphic Adventure Engine
+ *
+ * ScummVM is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * This is a utility for create the translations.dat file from all the po files.
+ * The generated files is used by ScummVM to propose translation of its GUI.
+ */
+
+#include "cp_parser.h"
+
+#include <fstream>
+#include <stdio.h>
+
+Codepage::Codepage(const std::string &name, const uint32 *mapping) : _name(name) {
+	if (!mapping) {
+		// Default to a ISO-8859-1 mapping
+		for (unsigned int i = 0; i < 256; ++i)
+			_mapping[i] = i;
+	} else {
+		for (unsigned int i = 0; i < 256; ++i)
+			_mapping[i] = *mapping++;
+	}
+}
+
+Codepage *parseCodepageMapping(const std::string &filename) {
+	size_t start = filename.find_last_of("/\\");
+	if (start == std::string::npos)
+		start = 0;
+	// Strip off the filename extension
+	const size_t pos = filename.find_last_of('.');
+	const std::string charset(filename.substr(start + 1, pos != std::string::npos ? (pos - start - 1) : std::string::npos));
+
+	std::ifstream in(filename.c_str());
+	if (!in) {
+		fprintf(stderr, "ERROR: Could not open file \"%s\"!", filename.c_str());
+		return 0;
+	}
+
+	uint32 mapping[256];
+
+	int processedMappings = 0;
+	std::string line;
+	while (processedMappings < 256) {
+		std::getline(in, line);
+		if (in.eof())
+			break;
+		if (in.fail()) {
+			fprintf(stderr, "ERROR: Reading from file \"%s\" failed!", filename.c_str());
+			return 0;
+		}
+
+		// In case the line starts with a "#" it is a comment. We also ignore
+		// empty lines.
+		if (line.empty() || line[0] == '#')
+			continue;
+
+		// Read the unicode number, we thereby ignore everything else on the line
+		int unicode, required;
+		const int parsed = sscanf(line.c_str(), "%d %d", &unicode, &required);
+		if (parsed < 1 || parsed > 2) {
+			fprintf(stderr, "ERROR: Line \"%s\" is malformed!", filename.c_str());
+			return 0;
+		}
+		// In case no required integer was given, we assume the character is
+		// required
+		if (parsed == 1)
+			required = 1;
+
+		// -1 means default mapping
+		if (unicode == -1)
+			unicode = processedMappings;
+
+		uint32 unicodeValue = unicode;
+		if (required)
+			unicodeValue |= 0x80000000;
+
+		mapping[processedMappings++] = (uint32)unicodeValue;
+	}
+
+	// Check whether all character encodings are mapped, if not error out
+	if (processedMappings != 256) {
+		fprintf(stderr, "ERROR: File \"%s\" does not contain mappings for exactly 256 characters!", filename.c_str());
+		return 0;
+	}
+
+	// Return the codepage
+	return new Codepage(charset, mapping);
+}
diff --git a/devtools/create_translations/cp_parser.h b/devtools/create_translations/cp_parser.h
new file mode 100644
index 0000000000..b748430ca0
--- /dev/null
+++ b/devtools/create_translations/cp_parser.h
@@ -0,0 +1,54 @@
+/* ScummVM - Graphic Adventure Engine
+ *
+ * ScummVM is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * This is a utility for create the translations.dat file from all the po files.
+ * The generated files is used by ScummVM to propose translation of its GUI.
+ */
+
+#ifndef CP_PARSER_H
+#define CP_PARSER_H
+
+#include "create_translations.h"
+
+#include <string>
+
+/**
+ * Codepage description.
+ *
+ * This includes a name, and the codepage -> unicode mapping.
+ */
+class Codepage {
+public:
+	Codepage(const std::string &name, const uint32 *mapping);
+
+	const std::string &getName() const { return _name; }
+
+	uint32 getMapping(unsigned char src) const { return _mapping[src]; }
+private:
+	std::string _name;
+	uint32 _mapping[256];
+};
+
+/**
+ * Parse the codepage file and create a codepage.
+ */
+Codepage *parseCodepageMapping(const std::string &filename);
+
+#endif
diff --git a/devtools/create_translations/create_translations.cpp b/devtools/create_translations/create_translations.cpp
index 9fcf3b4a31..a153632c47 100644
--- a/devtools/create_translations/create_translations.cpp
+++ b/devtools/create_translations/create_translations.cpp
@@ -25,6 +25,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <ctype.h>
+#include <vector>
 
  // HACK to allow building with the SDL backend on MinGW
 // see bug #1800764 "TOOLS: MinGW tools building broken"
@@ -34,8 +36,23 @@
 
 #include "create_translations.h"
 #include "po_parser.h"
+#include "cp_parser.h"
 
-#define TRANSLATIONS_DAT_VER 2	// 1 byte
+#define TRANSLATIONS_DAT_VER 3	// 1 byte
+
+// Portable implementation of stricmp / strcasecmp / strcmpi.
+int scumm_stricmp(const char *s1, const char *s2) {
+	uint8 l1, l2;
+	do {
+		// Don't use ++ inside tolower, in case the macro uses its
+		// arguments more than once.
+		l1 = (uint8)*s1++;
+		l1 = tolower(l1);
+		l2 = (uint8)*s2++;
+		l2 = tolower(l2);
+	} while (l1 == l2 && l1 != 0);
+	return l1 - l2;
+}
 
 // Padding buffer (filled with 0) used if we want to aligned writes
 // static uint8 padBuf[DATAALIGNMENT];
@@ -52,6 +69,13 @@ void writeUint16BE(FILE *fp, uint16 value) {
 	writeByte(fp, (uint8)(value & 0xFF));
 }
 
+void writeUint32BE(FILE *fp, uint32 value) {
+	writeByte(fp, (uint8)(value >> 24));
+	writeByte(fp, (uint8)(value >> 16));
+	writeByte(fp, (uint8)(value >>  8));
+	writeByte(fp, (uint8)(value & 0xFF));
+}
+
 int stringSize(const char *string) {
 	// Each string is preceded by its size coded on 2 bytes
 	if (string == NULL)
@@ -82,14 +106,51 @@ void writeString(FILE *fp, const char *string) {
 
 // Main
 int main(int argc, char *argv[]) {
-	// Build the translation list
+	std::vector<Codepage *> codepages;
+	// Add default codepages, we won't store them in the output later on
+	codepages.push_back(new Codepage("ascii", 0));
+	codepages.push_back(new Codepage("iso-8859-1", 0));
+
+	// Build the translation and codepage list
 	PoMessageList messageIds;
-	PoMessageEntryList **translations = new PoMessageEntryList*[argc - 1];
+	std::vector<PoMessageEntryList *> translations;
 	int numLangs = 0;
 	for (int i = 1; i < argc; ++i) {
-		translations[numLangs] = parsePoFile(argv[i], messageIds);
-		if (translations[numLangs] != NULL)
-			++numLangs;
+		// Check file extension
+		int len = strlen(argv[i]);
+		if (scumm_stricmp(argv[i] + len - 2, "po") == 0) {
+			PoMessageEntryList *po = parsePoFile(argv[i], messageIds);
+			if (po != NULL) {
+				translations.push_back(po);
+				++numLangs;
+			}
+		} else if (scumm_stricmp(argv[i] + len - 2, "cp") == 0) {
+			// Else try to parse an codepage
+			Codepage *co = parseCodepageMapping(argv[i]);
+			if (co)
+				codepages.push_back(co);
+		}
+	}
+
+	// Parse all charset mappings
+	for (int i = 0; i < numLangs; ++i) {
+		bool found = false;
+		for (size_t j = 0; j < codepages.size(); ++j) {
+			if (scumm_stricmp(codepages[j]->getName().c_str(), translations[i]->charset()) == 0) {
+				found = true;
+				break;
+			}
+		}
+
+		// In case the codepage was not found error out
+		if (!found) {
+			fprintf(stderr, "ERROR: No codepage mapping for codepage \"%s\" present!\n", translations[i]->charset());
+			for (size_t j = 0; j < translations.size(); ++j)
+				delete translations[j];
+			for (size_t j = 0; j < codepages.size(); ++j)
+				delete codepages[j];
+			return -1;
+		}
 	}
 
 	FILE *outFile;
@@ -110,6 +171,8 @@ int main(int argc, char *argv[]) {
 
 	// Write number of translations
 	writeUint16BE(outFile, numLangs);
+	// Write number of codepages, we don't save ascii and iso-8859-1
+	writeUint16BE(outFile, codepages.size() - 2);
 
 	// Write the length of each data block here.
 	// We could write it at the start of each block but that would mean that
@@ -119,9 +182,14 @@ int main(int argc, char *argv[]) {
 	// file and can then skip to the block we want.
 	// Blocks are:
 	//   1. List of languages with the language name
-	//   2. Original messages (i.e. english)
-	//   3. First translation
-	//   4. Second translation
+	//   2. List of codepages
+	//   3. Original messages (i.e. english)
+	//   4. First translation
+	//   5. Second translation
+	//   ...
+	//   n. First codepage (These don't have any data size, since they are all
+	//                      256 * 4 bytes long)
+	//   n+1. Second codepage
 	//   ...
 
 	// Write length for translation description
@@ -132,6 +200,12 @@ int main(int argc, char *argv[]) {
 	}
 	writeUint16BE(outFile, len);
 
+	// Write length for the codepage names
+	len = 0;
+	for (size_t j = 2; j < codepages.size(); ++j)
+		len += stringSize(codepages[j]->getName().c_str());
+	writeUint16BE(outFile, len);
+
 	// Write size for the original language (english) block
 	// It starts with the number of strings coded on 2 bytes followed by each
 	// string (two bytes for the number of chars and the string itself).
@@ -159,6 +233,11 @@ int main(int argc, char *argv[]) {
 		writeString(outFile, translations[lang]->languageName());
 	}
 
+	// Write list of codepages
+	for (size_t j = 2; j < codepages.size(); ++j) {
+		writeString(outFile, codepages[j]->getName().c_str());
+	}
+
 	// Write original messages
 	writeUint16BE(outFile, messageIds.size());
 	for (i = 0; i < messageIds.size(); ++i) {
@@ -176,12 +255,18 @@ int main(int argc, char *argv[]) {
 		}
 	}
 
+	// Write codepages
+	for (size_t j = 2; j < codepages.size(); ++j) {
+		const Codepage *cp = codepages[j];
+		for (i = 0; i < 256; ++i)
+			writeUint32BE(outFile, cp->getMapping(i));
+	}
+
 	fclose(outFile);
 
 	// Clean the memory
 	for (i = 0; i < numLangs; ++i)
 		delete translations[i];
-	delete[] translations;
 
 	return 0;
 }
diff --git a/devtools/create_translations/create_translations.h b/devtools/create_translations/create_translations.h
index 0ece8102f0..9ccbd39b2b 100644
--- a/devtools/create_translations/create_translations.h
+++ b/devtools/create_translations/create_translations.h
@@ -25,6 +25,7 @@
 
 typedef unsigned char   uint8;
 typedef unsigned short uint16;
+typedef unsigned int   uint32;
 typedef signed short int16;
 
 #endif /* CREATE_TRANSLATIONS_H */
diff --git a/devtools/create_translations/module.mk b/devtools/create_translations/module.mk
index 430cf91976..9a2b302aef 100644
--- a/devtools/create_translations/module.mk
+++ b/devtools/create_translations/module.mk
@@ -1,6 +1,7 @@
 MODULE := devtools/create_translations
 
 MODULE_OBJS := \
+	cp_parser.o \
 	po_parser.o \
 	create_translations.o