SCI32: Add SaveFileRewriteStream for read/write access to files

At least Phantasmagoria and PQ:SWAT require this.

This adds a ReadStream/WriteStream to support this. It replaces
the previous VirtualIndexFile which required reimplementation of
most read/write functions.

5 files changed, 158 insertions, 268 deletions

diff --git a/engines/sci/engine/file.cpp b/engines/sci/engine/file.cpp
index 0b1001bfda..156f6f51f7 100644
--- a/engines/sci/engine/file.cpp
+++ b/engines/sci/engine/file.cpp
@@ -22,6 +22,7 @@
 
 #include "common/savefile.h"
 #include "common/stream.h"
+#include "common/memstream.h"
 
 #include "sci/sci.h"
 #include "sci/engine/file.h"
@@ -32,6 +33,112 @@
 
 namespace Sci {
 
+#ifdef ENABLE_SCI32
+/**
+ * A MemoryWriteStreamDynamic with additional read functionality.
+ * The read and write functions share a single stream position.
+ */
+class MemoryDynamicRWStream : public Common::MemoryWriteStreamDynamic, public Common::SeekableReadStream {
+protected:
+	bool _eos;
+public:
+	MemoryDynamicRWStream(DisposeAfterUse::Flag disposeMemory = DisposeAfterUse::NO) : MemoryWriteStreamDynamic(disposeMemory), _eos(false) { }
+
+	uint32 read(void *dataPtr, uint32 dataSize);
+
+	bool eos() const { return _eos; }
+	int32 pos() const { return _pos; }
+	int32 size() const { return _size; }
+	void clearErr() { _eos = false; Common::MemoryWriteStreamDynamic::clearErr(); }
+	bool seek(int32 offs, int whence = SEEK_SET) { return Common::MemoryWriteStreamDynamic::seek(offs, whence); }
+
+};
+
+uint32 MemoryDynamicRWStream::read(void *dataPtr, uint32 dataSize)
+{
+	// Read at most as many bytes as are still available...
+	if (dataSize > _size - _pos) {
+		dataSize = _size - _pos;
+		_eos = true;
+	}
+	memcpy(dataPtr, _ptr, dataSize);
+
+	_ptr += dataSize;
+	_pos += dataSize;
+
+	return dataSize;
+}
+
+/**
+ * A MemoryDynamicRWStream intended to re-write a file.
+ * It reads the contents of `inFile` in the constructor, and writes back
+ * the changes to `fileName` in the destructor (and when calling commit() ).
+ */
+class SaveFileRewriteStream : public MemoryDynamicRWStream {
+public:
+	SaveFileRewriteStream(Common::String fileName,
+	                      Common::SeekableReadStream *inFile,
+	                      bool truncate, bool compress);
+	virtual ~SaveFileRewriteStream();
+
+	virtual uint32 write(const void *dataPtr, uint32 dataSize) { _changed = true; return MemoryDynamicRWStream::write(dataPtr, dataSize); }
+
+	void commit(); //< Save back to disk
+
+protected:
+	Common::String _fileName;
+	bool _compress;
+	bool _changed;
+};
+
+SaveFileRewriteStream::SaveFileRewriteStream(Common::String fileName,
+                                             Common::SeekableReadStream *inFile,
+                                             bool truncate,
+                                             bool compress)
+: MemoryDynamicRWStream(DisposeAfterUse::YES),
+  _fileName(fileName), _compress(compress)
+{
+	if (!truncate && inFile) {
+		unsigned int s = inFile->size();
+		ensureCapacity(s);
+		inFile->read(_data, s);
+		_changed = false;
+	} else {
+		_changed = true;
+	}
+}
+
+SaveFileRewriteStream::~SaveFileRewriteStream() {
+	commit();
+}
+
+void SaveFileRewriteStream::commit() {
+	// Write contents of buffer back to file
+
+	if (_changed) {
+		Common::WriteStream *outFile = g_sci->getSaveFileManager()->openForSaving(_fileName, _compress);
+		outFile->write(_data, _size);
+		delete outFile;
+		_changed = false;
+	}
+}
+
+#endif
+
+uint findFreeFileHandle(EngineState *s) {
+	// Find a free file handle
+	uint handle = 1; // Ignore _fileHandles[0]
+	while ((handle < s->_fileHandles.size()) && s->_fileHandles[handle].isOpen())
+		handle++;
+
+	if (handle == s->_fileHandles.size()) {
+		// Hit size limit => Allocate more space
+		s->_fileHandles.resize(s->_fileHandles.size() + 1);
+	}
+
+	return handle;
+}
+
 /*
  * Note on how file I/O is implemented: In ScummVM, one can not create/write
  * arbitrary data files, simply because many of our target platforms do not
@@ -91,6 +198,27 @@ reg_t file_open(EngineState *s, const Common::String &filename, int mode, bool u
 		break;
 	}
 
+#ifdef ENABLE_SCI32
+	if (mode != _K_FILE_MODE_OPEN_OR_FAIL && (
+	    (g_sci->getGameId() == GID_PHANTASMAGORIA && filename == "phantsg.dir") ||
+	    (g_sci->getGameId() == GID_PQSWAT && filename == "swat.dat"))) {
+		debugC(kDebugLevelFile, "  -> file_open opening %s for rewriting", wrappedName.c_str());
+
+		inFile = saveFileMan->openForLoading(wrappedName);
+		// If no matching savestate exists: fall back to reading from a regular
+		// file
+		if (!inFile)
+			inFile = SearchMan.createReadStreamForMember(englishName);
+
+		SaveFileRewriteStream *stream;
+		stream = new SaveFileRewriteStream(wrappedName, inFile, mode == _K_FILE_MODE_CREATE, isCompressed);
+
+		delete inFile;
+
+		inFile = stream;
+		outFile = stream;
+	} else
+#endif
 	if (mode == _K_FILE_MODE_OPEN_OR_FAIL) {
 		// Try to open file, abort if not possible
 		inFile = saveFileMan->openForLoading(wrappedName);
@@ -126,15 +254,7 @@ reg_t file_open(EngineState *s, const Common::String &filename, int mode, bool u
 		return SIGNAL_REG;
 	}
 
-	// Find a free file handle
-	uint handle = 1; // Ignore _fileHandles[0]
-	while ((handle < s->_fileHandles.size()) && s->_fileHandles[handle].isOpen())
-		handle++;
-
-	if (handle == s->_fileHandles.size()) {
-		// Hit size limit => Allocate more space
-		s->_fileHandles.resize(s->_fileHandles.size() + 1);
-	}
+	uint handle = findFreeFileHandle(s);
 
 	s->_fileHandles[handle]._in = inFile;
 	s->_fileHandles[handle]._out = outFile;
@@ -252,8 +372,12 @@ FileHandle::~FileHandle() {
 }
 
 void FileHandle::close() {
-	delete _in;
-	delete _out;
+	// NB: It is possible _in and _out are both non-null, but
+	// then they point to the same object.
+	if (_in)
+		delete _in;
+	else
+		delete _out;
 	_in = 0;
 	_out = 0;
 	_name.clear();
@@ -365,119 +489,4 @@ reg_t DirSeeker::nextFile(SegManager *segMan) {
 	return _outbuffer;
 }
 
-
-#ifdef ENABLE_SCI32
-
-VirtualIndexFile::VirtualIndexFile(Common::String fileName) : _fileName(fileName), _changed(false) {
-	Common::SeekableReadStream *inFile = g_sci->getSaveFileManager()->openForLoading(fileName);
-
-	_bufferSize = inFile->size();
-	_buffer = new char[_bufferSize];
-	inFile->read(_buffer, _bufferSize);
-	_ptr = _buffer;
-	delete inFile;
-}
-
-VirtualIndexFile::VirtualIndexFile(uint32 initialSize) : _changed(false) {
-	_bufferSize = initialSize;
-	_buffer = new char[_bufferSize];
-	_ptr = _buffer;
-}
-
-VirtualIndexFile::~VirtualIndexFile() {
-	close();
-
-	_bufferSize = 0;
-	delete[] _buffer;
-	_buffer = 0;
-}
-
-uint32 VirtualIndexFile::read(char *buffer, uint32 size) {
-	uint32 curPos = _ptr - _buffer;
-	uint32 finalSize = MIN<uint32>(size, _bufferSize - curPos);
-	char *localPtr = buffer;
-
-	for (uint32 i = 0; i < finalSize; i++)
-		*localPtr++ = *_ptr++;
-
-	return finalSize;
-}
-
-uint32 VirtualIndexFile::write(const char *buffer, uint32 size) {
-	_changed = true;
-	uint32 curPos = _ptr - _buffer;
-
-	// Check if the buffer needs to be resized
-	if (curPos + size >= _bufferSize) {
-		_bufferSize = curPos + size + 1;
-		char *tmp = _buffer;
-		_buffer = new char[_bufferSize];
-		_ptr = _buffer + curPos;
-		memcpy(_buffer, tmp, _bufferSize);
-		delete[] tmp;
-	}
-
-	for (uint32 i = 0; i < size; i++)
-		*_ptr++ = *buffer++;
-
-	return size;
-}
-
-uint32 VirtualIndexFile::readLine(char *buffer, uint32 size) {
-	uint32 startPos = _ptr - _buffer;
-	uint32 bytesRead = 0;
-	char *localPtr = buffer;
-
-	// This is not a full-blown implementation of readLine, but it
-	// suffices for Phantasmagoria
-	while (startPos + bytesRead < size) {
-		bytesRead++;
-
-		if (*_ptr == 0 || *_ptr == 0x0A) {
-			_ptr++;
-			*localPtr = 0;
-			return bytesRead;
-		} else {
-			*localPtr++ = *_ptr++;
-		}
-	}
-
-	return bytesRead;
-}
-
-bool VirtualIndexFile::seek(int32 offset, int whence) {
-	uint32 startPos = _ptr - _buffer;
-	assert(offset >= 0);
-
-	switch (whence) {
-	case SEEK_CUR:
-		assert(startPos + offset < _bufferSize);
-		_ptr += offset;
-		break;
-	case SEEK_SET:
-		assert(offset < (int32)_bufferSize);
-		_ptr = _buffer + offset;
-		break;
-	case SEEK_END:
-		assert((int32)_bufferSize - offset >= 0);
-		_ptr = _buffer + (_bufferSize - offset);
-		break;
-	}
-
-	return true;
-}
-
-void VirtualIndexFile::close() {
-	if (_changed && !_fileName.empty()) {
-		Common::WriteStream *outFile = g_sci->getSaveFileManager()->openForSaving(_fileName);
-		outFile->write(_buffer, _bufferSize);
-		delete outFile;
-	}
-
-	// Maintain the buffer, and seek to the beginning of it
-	_ptr = _buffer;
-}
-
-#endif
-
 } // End of namespace Sci
diff --git a/engines/sci/engine/file.h b/engines/sci/engine/file.h
index 54627d5228..982d7b7823 100644
--- a/engines/sci/engine/file.h
+++ b/engines/sci/engine/file.h
@@ -43,7 +43,6 @@ enum {
 
 #define VIRTUALFILE_HANDLE_START 32000
 #define VIRTUALFILE_HANDLE_SCI32SAVE 32100
-#define PHANTASMAGORIA_SAVEGAME_INDEX "phantsg.dir"
 #define VIRTUALFILE_HANDLE_SCIAUDIO 32300
 #define VIRTUALFILE_HANDLE_END 32300
 
@@ -93,50 +92,7 @@ private:
 	void addAsVirtualFiles(Common::String title, Common::String fileMask);
 };
 
-
-#ifdef ENABLE_SCI32
-
-/**
- * An implementation of a virtual file that supports basic read and write
- * operations simultaneously.
- *
- * This class has been initially implemented for Phantasmagoria, which has its
- * own custom save/load code. The load code keeps checking for the existence
- * of the save index file and keeps closing and reopening it for each save
- * slot. This is notoriously slow and clumsy, and introduces noticeable delays,
- * especially for non-desktop systems. Also, its game scripts request to open
- * the index file for reading and writing with the same parameters
- * (SaveManager::setCurrentSave and SaveManager::getCurrentSave). Moreover,
- * the game scripts reopen the index file for writing in order to update it
- * and seek within it. We do not support seeking in writeable streams, and the
- * fact that our saved games are ZIP files makes this operation even more
- * expensive. Finally, the savegame index file is supposed to be expanded when
- * a new save slot is added.
- * For the aforementioned reasons, this class has been implemented, which offers
- * the basic functionality needed by the game scripts in Phantasmagoria.
- */
-class VirtualIndexFile {
-public:
-	VirtualIndexFile(Common::String fileName);
-	VirtualIndexFile(uint32 initialSize);
-	~VirtualIndexFile();
-
-	uint32 read(char *buffer, uint32 size);
-	uint32 readLine(char *buffer, uint32 size);
-	uint32 write(const char *buffer, uint32 size);
-	bool seek(int32 offset, int whence);
-	void close();
-
-private:
-	char *_buffer;
-	uint32 _bufferSize;
-	char *_ptr;
-
-	Common::String _fileName;
-	bool _changed;
-};
-
-#endif
+uint findFreeFileHandle(EngineState *s);
 
 } // End of namespace Sci
 
diff --git a/engines/sci/engine/kfile.cpp b/engines/sci/engine/kfile.cpp
index d604bb85d0..4508a481a0 100644
--- a/engines/sci/engine/kfile.cpp
+++ b/engines/sci/engine/kfile.cpp
@@ -29,6 +29,7 @@
 #include "common/savefile.h"
 #include "common/system.h"
 #include "common/translation.h"
+#include "common/memstream.h"
 
 #include "gui/saveload.h"
 
@@ -263,19 +264,6 @@ reg_t kFileIOOpen(EngineState *s, int argc, reg_t *argv) {
 	}
 
 #ifdef ENABLE_SCI32
-	if (name == PHANTASMAGORIA_SAVEGAME_INDEX) {
-		if (s->_virtualIndexFile) {
-			return make_reg(0, VIRTUALFILE_HANDLE_SCI32SAVE);
-		} else {
-			Common::String englishName = g_sci->getSciLanguageString(name, K_LANG_ENGLISH);
-			Common::String wrappedName = g_sci->wrapFilename(englishName);
-			if (!g_sci->getSaveFileManager()->listSavefiles(wrappedName).empty()) {
-				s->_virtualIndexFile = new VirtualIndexFile(wrappedName);
-				return make_reg(0, VIRTUALFILE_HANDLE_SCI32SAVE);
-			}
-		}
-	}
-
 	// Shivers is trying to store savegame descriptions and current spots in
 	// separate .SG files, which are hardcoded in the scripts.
 	// Essentially, there is a normal save file, created by the executable
@@ -313,18 +301,18 @@ reg_t kFileIOOpen(EngineState *s, int argc, reg_t *argv) {
 			listSavegames(saves);
 			int savegameNr = findSavegame(saves, slotNumber - SAVEGAMEID_OFFICIALRANGE_START);
 
-			if (!s->_virtualIndexFile) {
-				// Make the virtual file buffer big enough to avoid having it grow dynamically.
-				// 50 bytes should be more than enough.
-				s->_virtualIndexFile = new VirtualIndexFile(50);
-			}
+			int size = strlen(saves[savegameNr].name) + 2;
+			char *buf = (char *)malloc(size);
+			strcpy(buf, saves[savegameNr].name);
+			buf[size - 1] = 0; // Spot description (empty)
 
-			s->_virtualIndexFile->seek(0, SEEK_SET);
-			s->_virtualIndexFile->write(saves[savegameNr].name, strlen(saves[savegameNr].name));
-			s->_virtualIndexFile->write("\0", 1);
-			s->_virtualIndexFile->write("\0", 1);	// Spot description (empty)
-			s->_virtualIndexFile->seek(0, SEEK_SET);
-			return make_reg(0, VIRTUALFILE_HANDLE_SCI32SAVE);
+			uint handle = findFreeFileHandle(s);
+
+			s->_fileHandles[handle]._in = new Common::MemoryReadStream((byte *)buf, size, DisposeAfterUse::YES);
+			s->_fileHandles[handle]._out = nullptr;
+			s->_fileHandles[handle]._name = "";
+
+			return make_reg(0, handle);
 		}
 	}
 #endif
@@ -349,13 +337,6 @@ reg_t kFileIOClose(EngineState *s, int argc, reg_t *argv) {
 
 	uint16 handle = argv[0].toUint16();
 
-#ifdef ENABLE_SCI32
-	if (handle == VIRTUALFILE_HANDLE_SCI32SAVE) {
-		s->_virtualIndexFile->close();
-		return SIGNAL_REG;
-	}
-#endif
-
 	if (handle >= VIRTUALFILE_HANDLE_START) {
 		// it's a virtual handle? ignore it
 		return SIGNAL_REG;
@@ -381,17 +362,9 @@ reg_t kFileIOReadRaw(EngineState *s, int argc, reg_t *argv) {
 	char *buf = new char[size];
 	debugC(kDebugLevelFile, "kFileIO(readRaw): %d, %d", handle, size);
 
-#ifdef ENABLE_SCI32
-	if (handle == VIRTUALFILE_HANDLE_SCI32SAVE) {
-		bytesRead = s->_virtualIndexFile->read(buf, size);
-	} else {
-#endif
-		FileHandle *f = getFileFromHandle(s, handle);
-		if (f)
-			bytesRead = f->_in->read(buf, size);
-#ifdef ENABLE_SCI32
-	}
-#endif
+	FileHandle *f = getFileFromHandle(s, handle);
+	if (f)
+		bytesRead = f->_in->read(buf, size);
 
 	// TODO: What happens if less bytes are read than what has
 	// been requested? (i.e. if bytesRead is non-zero, but still
@@ -411,20 +384,11 @@ reg_t kFileIOWriteRaw(EngineState *s, int argc, reg_t *argv) {
 	s->_segMan->memcpy((byte *)buf, argv[1], size);
 	debugC(kDebugLevelFile, "kFileIO(writeRaw): %d, %d", handle, size);
 
-#ifdef ENABLE_SCI32
-	if (handle == VIRTUALFILE_HANDLE_SCI32SAVE) {
-		s->_virtualIndexFile->write(buf, size);
+	FileHandle *f = getFileFromHandle(s, handle);
+	if (f) {
+		f->_out->write(buf, size);
 		success = true;
-	} else {
-#endif
-		FileHandle *f = getFileFromHandle(s, handle);
-		if (f) {
-			f->_out->write(buf, size);
-			success = true;
-		}
-#ifdef ENABLE_SCI32
 	}
-#endif
 
 	delete[] buf;
 	if (success)
@@ -463,13 +427,6 @@ reg_t kFileIOUnlink(EngineState *s, int argc, reg_t *argv) {
 			const Common::String wrappedName = g_sci->wrapFilename(name);
 			result = saveFileMan->removeSavefile(wrappedName);
 		}
-
-#ifdef ENABLE_SCI32
-		if (name == PHANTASMAGORIA_SAVEGAME_INDEX) {
-			delete s->_virtualIndexFile;
-			s->_virtualIndexFile = 0;
-		}
-#endif
 	} else {
 		const Common::String wrappedName = g_sci->wrapFilename(name);
 		result = saveFileMan->removeSavefile(wrappedName);
@@ -488,12 +445,7 @@ reg_t kFileIOReadString(EngineState *s, int argc, reg_t *argv) {
 	debugC(kDebugLevelFile, "kFileIO(readString): %d, %d", handle, maxsize);
 	uint32 bytesRead;
 
-#ifdef ENABLE_SCI32
-	if (handle == VIRTUALFILE_HANDLE_SCI32SAVE)
-		bytesRead = s->_virtualIndexFile->readLine(buf, maxsize);
-	else
-#endif
-		bytesRead = fgets_wrapper(s, buf, maxsize, handle);
+	bytesRead = fgets_wrapper(s, buf, maxsize, handle);
 
 	s->_segMan->memcpy(argv[0], (const byte*)buf, maxsize);
 	delete[] buf;
@@ -520,13 +472,6 @@ reg_t kFileIOWriteString(EngineState *s, int argc, reg_t *argv) {
 		return NULL_REG;
 	}
 
-#ifdef ENABLE_SCI32
-	if (handle == VIRTUALFILE_HANDLE_SCI32SAVE) {
-		s->_virtualIndexFile->write(str.c_str(), str.size());
-		return NULL_REG;
-	}
-#endif
-
 	FileHandle *f = getFileFromHandle(s, handle);
 
 	if (f) {
@@ -547,11 +492,6 @@ reg_t kFileIOSeek(EngineState *s, int argc, reg_t *argv) {
 	uint16 whence = argv[2].toUint16();
 	debugC(kDebugLevelFile, "kFileIO(seek): %d, %d, %d", handle, offset, whence);
 
-#ifdef ENABLE_SCI32
-	if (handle == VIRTUALFILE_HANDLE_SCI32SAVE)
-		return make_reg(0, s->_virtualIndexFile->seek(offset, whence));
-#endif
-
 	FileHandle *f = getFileFromHandle(s, handle);
 
 	if (f && f->_in) {
@@ -591,14 +531,6 @@ reg_t kFileIOFindNext(EngineState *s, int argc, reg_t *argv) {
 reg_t kFileIOExists(EngineState *s, int argc, reg_t *argv) {
 	Common::String name = s->_segMan->getString(argv[0]);
 
-#ifdef ENABLE_SCI32
-	// Cache the file existence result for the Phantasmagoria
-	// save index file, as the game scripts keep checking for
-	// its existence.
-	if (name == PHANTASMAGORIA_SAVEGAME_INDEX && s->_virtualIndexFile)
-		return TRUE_REG;
-#endif
-
 	bool exists = false;
 
 	if (g_sci->getGameId() == GID_PEPPER) {
@@ -611,6 +543,9 @@ reg_t kFileIOExists(EngineState *s, int argc, reg_t *argv) {
 			return NULL_REG;
 	}
 
+	// TODO: It may apparently be worth caching the existence of
+	// phantsg.dir, and possibly even keeping it open persistently
+
 	// Check for regular file
 	exists = Common::File::exists(name);
 
diff --git a/engines/sci/engine/state.cpp b/engines/sci/engine/state.cpp
index fda78317b5..2c85907628 100644
--- a/engines/sci/engine/state.cpp
+++ b/engines/sci/engine/state.cpp
@@ -70,9 +70,6 @@ static const uint16 s_halfWidthSJISMap[256] = {
 
 EngineState::EngineState(SegManager *segMan)
 : _segMan(segMan),
-#ifdef ENABLE_SCI32
-	_virtualIndexFile(0),
-#endif
 	_dirseeker() {
 
 	reset(false);
@@ -80,9 +77,6 @@ EngineState::EngineState(SegManager *segMan)
 
 EngineState::~EngineState() {
 	delete _msgState;
-#ifdef ENABLE_SCI32
-	delete _virtualIndexFile;
-#endif
 }
 
 void EngineState::reset(bool isRestoring) {
diff --git a/engines/sci/engine/state.h b/engines/sci/engine/state.h
index cf9a753f5c..baca4a503b 100644
--- a/engines/sci/engine/state.h
+++ b/engines/sci/engine/state.h
@@ -131,10 +131,6 @@ public:
 	int16 _lastSaveVirtualId; // last virtual id fed to kSaveGame, if no kGetSaveFiles was called inbetween
 	int16 _lastSaveNewId;    // last newly created filename-id by kSaveGame
 
-#ifdef ENABLE_SCI32
-	VirtualIndexFile *_virtualIndexFile;
-#endif
-
 	// see detection.cpp / SciEngine::loadGameState()
 	bool _delayedRestoreGame;  // boolean, that triggers delayed restore (triggered by ScummVM menu)
 	int _delayedRestoreGameId; // the saved game id, that it supposed to get restored (triggered by ScummVM menu)