SUPERNOVA: Add parsing of po file in create_supernova

This means the tool is now complete and can add translations to the
supernova engine data file.

4 files changed, 318 insertions, 15 deletions

diff --git a/devtools/create_supernova/create_supernova.cpp b/devtools/create_supernova/create_supernova.cpp
index 0d4ebe2fe7..bf04004b1a 100644
--- a/devtools/create_supernova/create_supernova.cpp
+++ b/devtools/create_supernova/create_supernova.cpp
@@ -1,6 +1,7 @@
 #include "create_supernova.h"
 #include "gametext.h"
 #include "file.h"
+#include "po_parser.h"
 
 // HACK to allow building with the SDL backend on MinGW
 // see bug #1800764 "TOOLS: MinGW tools building broken"
@@ -105,10 +106,10 @@ void writeGermanStrings(File& outputFile) {
 }
 
 void writeStrings(File& outputFile, const char* language) {
-	File poFile;
 	char fileName[16];
 	sprintf(fileName, "strings-%s.po", language);
-	if (!poFile.open(fileName, kFileReadMode)) {
+	PoMessageList* poList = parsePoFile(fileName);
+	if (!poList) {
 		printf("Cannot find strings file for language '%s'.\n", language);
 		return;
 	}
@@ -130,19 +131,21 @@ void writeStrings(File& outputFile, const char* language) {
 	uint32 blockSize = 0;
 	outputFile.writeLong(blockSize);
 
-
-
-	// TODO: read all the strings
-	// build german - translated map
-	// Then iterate on gameText and for each (german) string look for the corresponding translated string
-	// and write it (or write "" if not found).
-	// Doing so also count the number of byte written.
-	// Then finally go back to block size pos and write the block size.
-	// And go back to end of file.
-
-
-
-	poFile.close();
+	// Write all the strings.
+	// If a string is not translated we use the German one.
+	const char **s = &gameText[0];
+	while (*s) {
+		const char* translation = poList->findTranslation(*s);
+		if (translation) {
+			outputFile.writeString(translation);
+			blockSize += strlen(translation) + 1;
+		} else {
+			outputFile.writeString(*s);
+			blockSize += strlen(*s) + 1;
+		}
+		++s;
+	}
+	delete poList;
 
 	// Now write the block size and then go back to the end of the file.
 	outputFile.seek(blockSizePos, SEEK_SET);
diff --git a/devtools/create_supernova/module.mk b/devtools/create_supernova/module.mk
index 9591233168..806cccadaa 100644
--- a/devtools/create_supernova/module.mk
+++ b/devtools/create_supernova/module.mk
@@ -2,6 +2,7 @@ MODULE := devtools/create_supernova
 
 MODULE_OBJS := \
         file.o \
+        po_parser.o \
         create_supernova.o
 
 # Set the name of the executable
diff --git a/devtools/create_supernova/po_parser.cpp b/devtools/create_supernova/po_parser.cpp
new file mode 100644
index 0000000000..f4a9b96388
--- /dev/null
+++ b/devtools/create_supernova/po_parser.cpp
@@ -0,0 +1,221 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "po_parser.h"
+
+PoMessageList::PoMessageList() : _list(NULL), _size(0), _allocated(0) {
+}
+
+PoMessageList::~PoMessageList() {
+	for (int i = 0; i < _size; ++i)
+		delete _list[i];
+	delete[] _list;
+}
+
+int PoMessageList::compareString(const char* left, const char* right) {
+	if (left == NULL && right == NULL)
+		return 0;
+	if (left == NULL)
+		return -1;
+	if (right == NULL)
+		return 1;
+	return strcmp(left, right);
+}
+
+int PoMessageList::compareMessage(const char *msgLeft, const char *contextLeft, const char *msgRight, const char *contextRight) {
+	int compare = compareString(msgLeft, msgRight);
+	if (compare != 0)
+		return compare;
+	return compareString(contextLeft, contextRight);
+}
+
+void PoMessageList::insert(const char *translation, const char *msg, const char *context) {
+	if (msg == NULL || *msg == '\0' || translation == NULL || *translation == '\0')
+		return;
+
+	// binary-search for the insertion index
+	int leftIndex = 0;
+	int rightIndex = _size - 1;
+	while (rightIndex >= leftIndex) {
+		int midIndex = (leftIndex + rightIndex) / 2;
+		int compareResult = compareMessage(msg, context, _list[midIndex]->msgid, _list[midIndex]->msgctxt);
+		if (compareResult == 0)
+			return; // The message is already in this list
+		else if (compareResult < 0)
+			rightIndex = midIndex - 1;
+		else
+			leftIndex = midIndex + 1;
+	}
+	// We now have rightIndex = leftIndex - 1 and we need to insert the new message
+	// between the two (i.a. at leftIndex).
+	if (_size + 1 > _allocated) {
+		_allocated += 100;
+		PoMessage **newList = new PoMessage*[_allocated];
+		for (int i = 0; i < leftIndex; ++i)
+			newList[i] = _list[i];
+		for (int i = leftIndex; i < _size; ++i)
+			newList[i + 1] = _list[i];
+		delete[] _list;
+		_list = newList;
+	} else {
+		for (int i = _size - 1; i >= leftIndex; --i)
+			_list[i + 1] = _list[i];
+	}
+	_list[leftIndex] = new PoMessage(translation, msg, context);
+	++_size;
+}
+
+const char *PoMessageList::findTranslation(const char *msg, const char *context) {
+	if (msg == NULL || *msg == '\0')
+		NULL;
+
+	// binary-search for the message
+	int leftIndex = 0;
+	int rightIndex = _size - 1;
+	while (rightIndex >= leftIndex) {
+		int midIndex = (leftIndex + rightIndex) / 2;
+		int compareResult = compareMessage(msg, context, _list[midIndex]->msgid, _list[midIndex]->msgctxt);
+		if (compareResult == 0)
+			return _list[midIndex]->msgstr;
+		else if (compareResult < 0)
+			rightIndex = midIndex - 1;
+		else
+			leftIndex = midIndex + 1;
+	}
+	return NULL;
+}
+
+PoMessageList *parsePoFile(const char *file) {
+	FILE *inFile = fopen(file, "r");
+	if (!inFile)
+		return NULL;
+
+	char msgidBuf[1024], msgctxtBuf[1024], msgstrBuf[1024];
+	char line[1024], *currentBuf = msgstrBuf;
+
+	PoMessageList *list = new PoMessageList();
+
+	// Initialize the message attributes.
+	bool fuzzy = false;
+	bool fuzzy_next = false;
+
+	// Parse the file line by line.
+	// The msgstr is always the last line of an entry (i.e. msgid and msgctxt always
+	// precede the corresponding msgstr).
+	msgidBuf[0] = msgstrBuf[0] = msgctxtBuf[0] = '\0';
+	while (!feof(inFile) && fgets(line, 1024, inFile)) {
+		if (line[0] == '#' && line[1] == ',') {
+			// Handle message attributes.
+			if (strstr(line, "fuzzy")) {
+				fuzzy_next = true;
+				continue;
+			}
+		}
+		// Skip empty and comment line
+		if (*line == '\n' || *line == '#')
+			continue;
+		if (strncmp(line, "msgid", 5) == 0) {
+			if (currentBuf == msgstrBuf) {
+				// add previous entry
+				if (*msgstrBuf != '\0' && !fuzzy)
+					list->insert(msgstrBuf, msgidBuf, msgctxtBuf);
+				msgidBuf[0] = msgstrBuf[0] = msgctxtBuf[0] = '\0';
+
+				// Reset the attribute flags.
+				fuzzy = fuzzy_next;
+				fuzzy_next = false;
+			}
+			strcpy(msgidBuf, stripLine(line));
+			currentBuf = msgidBuf;
+		} else if (strncmp(line, "msgctxt", 7) == 0) {
+			if (currentBuf == msgstrBuf) {
+				// add previous entry
+				if (*msgstrBuf != '\0' && !fuzzy)
+					list->insert(msgstrBuf, msgidBuf, msgctxtBuf);
+				msgidBuf[0] = msgstrBuf[0] = msgctxtBuf[0] = '\0';
+
+				// Reset the attribute flags
+				fuzzy = fuzzy_next;
+				fuzzy_next = false;
+			}
+			strcpy(msgctxtBuf, stripLine(line));
+			currentBuf = msgctxtBuf;
+		} else if (strncmp(line, "msgstr", 6) == 0) {
+			strcpy(msgstrBuf, stripLine(line));
+			currentBuf = msgstrBuf;
+		} else {
+			// concatenate the string at the end of the current buffer
+			if (currentBuf)
+				strcat(currentBuf, stripLine(line));
+		}
+	}
+	if (currentBuf == msgstrBuf) {
+		// add last entry
+		if (*msgstrBuf != '\0' && !fuzzy)
+			list->insert(msgstrBuf, msgidBuf, msgctxtBuf);
+	}
+
+	fclose(inFile);
+	return list;
+}
+
+char *stripLine(char *const line) {
+	// This function modifies line in place and return it.
+	// Keep only the text between the first two unprotected quotes.
+	// It also look for literal special characters (e.g. preceded by '\n', '\\', '\"', '\'', '\t')
+	// and replace them by the special character so that strcmp() can match them at run time.
+	// Look for the first quote
+	char const *src = line;
+	while (*src != '\0' && *src++ != '"') {}
+	// shift characters until we reach the end of the string or an unprotected quote
+	char *dst = line;
+	while (*src != '\0' && *src != '"') {
+		char c = *src++;
+		if (c == '\\') {
+			switch (c = *src++) {
+				case  'n': c = '\n'; break;
+				case  't': c = '\t'; break;
+				case '\"': c = '\"'; break;
+				case '\'': c = '\''; break;
+				case '\\': c = '\\'; break;
+				default:
+					// Just skip
+					fprintf(stderr, "Unsupported special character \"\\%c\" in string. Please contact ScummVM developers.\n", c);
+					continue;
+			}
+		}
+		*dst++ = c;
+	}
+	*dst = '\0';
+	return line;
+}
+
+char *parseLine(const char *line, const char *field) {
+	// This function allocate and return a new char*.
+	// It will return a NULL pointer if the field is not found.
+	// It is used to parse the header of the po files to find the language name
+	// and the charset.
+	const char *str = strstr(line, field);
+	if (str == NULL)
+		return NULL;
+	str += strlen(field);
+	// Skip spaces
+	while (*str != '\0' && isspace(*str)) {
+		++str;
+	}
+	// Find string length (stop at the first '\n')
+	int len = 0;
+	while (str[len] != '\0' && str[len] != '\n') {
+		++len;
+	}
+	if (len == 0)
+		return NULL;
+	// Create result string
+	char *result = new char[len + 1];
+	strncpy(result, str, len);
+	result[len] = '\0';
+	return result;
+}
+
diff --git a/devtools/create_supernova/po_parser.h b/devtools/create_supernova/po_parser.h
new file mode 100644
index 0000000000..7c358e32f6
--- /dev/null
+++ b/devtools/create_supernova/po_parser.h
@@ -0,0 +1,78 @@
+/* ScummVM - Graphic Adventure Engine
+ *
+ * ScummVM is the legal property of its developers, whose names
+ * are too numerous to list here. Please refer to the COPYRIGHT
+ * file distributed with this source distribution.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * This is a utility for generating a data file for the supernova engine.
+ * It contains strings extracted from the original executable as well
+ * as translations and is required for the engine to work properly.
+ */
+
+#ifndef PO_PARSER_H
+#define PO_PARSER_H
+
+struct PoMessage {
+	char *msgstr;
+	char *msgid;
+	char *msgctxt;
+
+	PoMessage(const char *translation, const char *message, const char *context = NULL) :
+	msgstr(NULL), msgid(NULL), msgctxt(NULL)
+	{
+		if (translation != NULL && *translation != '\0') {
+			msgstr = new char[1 + strlen(translation)];
+			strcpy(msgstr, translation);
+		}
+		if (message != NULL && *message != '\0') {
+			msgid = new char[1 + strlen(message)];
+			strcpy(msgid, message);
+		}
+		if (context != NULL && *context != '\0') {
+			msgctxt = new char[1 + strlen(context)];
+			strcpy(msgctxt, context);
+		}
+	}
+	~PoMessage() {
+		delete[] msgstr;
+		delete[] msgid;
+		delete[] msgctxt;
+	}
+};
+
+class PoMessageList {
+public:
+	PoMessageList();
+	~PoMessageList();
+
+	void insert(const char *translation, const char *msg, const char *context = NULL);
+	const char *findTranslation(const char *msg, const char *context = NULL);
+
+private:
+	int compareString(const char *left, const char *right);
+	int compareMessage(const char *msgLeft, const char *contextLeft, const char *msgRight, const char *contextRight);
+
+	PoMessage **_list;
+	int _size;
+	int _allocated;
+};
+
+PoMessageList *parsePoFile(const char *file);
+char *stripLine(char *);
+char *parseLine(const char *line, const char *field);
+
+#endif /* PO_PARSER_H */