OPENGL: Use premultiplied alpha for color-keyed cursors

This fixes colour fringing on keyed cursors when using filtering.

Fixes Trac#10594.

4 files changed, 101 insertions, 43 deletions

diff --git a/backends/graphics/opengl/framebuffer.cpp b/backends/graphics/opengl/framebuffer.cpp
index 7191aab8bc..671c448d26 100644
--- a/backends/graphics/opengl/framebuffer.cpp
+++ b/backends/graphics/opengl/framebuffer.cpp
@@ -28,7 +28,7 @@ namespace OpenGL {
 
 Framebuffer::Framebuffer()
     : _viewport(), _projectionMatrix(), _isActive(false), _clearColor(),
-      _blendState(false), _scissorTestState(false), _scissorBox() {
+      _blendState(kBlendModeDisabled), _scissorTestState(false), _scissorBox() {
 }
 
 void Framebuffer::activate() {
@@ -62,8 +62,8 @@ void Framebuffer::setClearColor(GLfloat r, GLfloat g, GLfloat b, GLfloat a) {
 	}
 }
 
-void Framebuffer::enableBlend(bool enable) {
-	_blendState = enable;
+void Framebuffer::enableBlend(BlendMode mode) {
+	_blendState = mode;
 
 	// Directly apply changes when we are active.
 	if (isActive()) {
@@ -105,10 +105,18 @@ void Framebuffer::applyClearColor() {
 }
 
 void Framebuffer::applyBlendState() {
-	if (_blendState) {
-		GL_CALL(glEnable(GL_BLEND));
-	} else {
-		GL_CALL(glDisable(GL_BLEND));
+	switch (_blendState) {
+		case kBlendModeDisabled:
+			GL_CALL(glDisable(GL_BLEND));
+			break;
+		case kBlendModeTraditionalTransparency:
+			GL_CALL(glEnable(GL_BLEND));
+			GL_CALL(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA));
+			break;
+		case kBlendModePremultipliedTransparency:
+			GL_CALL(glEnable(GL_BLEND));
+			GL_CALL(glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA));
+			break;
 	}
 }
 
diff --git a/backends/graphics/opengl/framebuffer.h b/backends/graphics/opengl/framebuffer.h
index c44c98ddc4..2f0245faa7 100644
--- a/backends/graphics/opengl/framebuffer.h
+++ b/backends/graphics/opengl/framebuffer.h
@@ -37,6 +37,29 @@ public:
 	virtual ~Framebuffer() {};
 
 public:
+	enum BlendMode {
+		/**
+		 * Newly drawn pixels overwrite the existing contents of the framebuffer
+		 * without mixing with them
+		 */
+		kBlendModeDisabled,
+
+		/**
+		 * Newly drawn pixels mix with the framebuffer based on their alpha value
+		 * for transparency.
+		 */
+		kBlendModeTraditionalTransparency,
+
+		/**
+		 * Newly drawn pixels mix with the framebuffer based on their alpha value
+		 * for transparency.
+		 *
+		 * Requires the image data being drawn to have its color values pre-multipled
+		 * with the alpha value.
+		 */
+		kBlendModePremultipliedTransparency
+	};
+
 	/**
 	 * Set the clear color of the framebuffer.
 	 */
@@ -45,7 +68,7 @@ public:
 	/**
 	 * Enable/disable GL_BLEND.
 	 */
-	void enableBlend(bool enable);
+	void enableBlend(BlendMode mode);
 
 	/**
 	 * Enable/disable GL_SCISSOR_TEST.
@@ -102,7 +125,7 @@ private:
 	GLfloat _clearColor[4];
 	void applyClearColor();
 
-	bool _blendState;
+	BlendMode _blendState;
 	void applyBlendState();
 
 	bool _scissorTestState;
diff --git a/backends/graphics/opengl/opengl-graphics.cpp b/backends/graphics/opengl/opengl-graphics.cpp
index 7d29a81f12..cfddc93e2f 100644
--- a/backends/graphics/opengl/opengl-graphics.cpp
+++ b/backends/graphics/opengl/opengl-graphics.cpp
@@ -420,16 +420,22 @@ void OpenGLGraphicsManager::updateScreen() {
 
 	const GLfloat shakeOffset = _gameScreenShakeOffset * (GLfloat)_gameDrawRect.height() / _gameScreen->getHeight();
 
+	// Alpha blending is disabled when drawing the screen
+	_backBuffer.enableBlend(Framebuffer::kBlendModeDisabled);
+
 	// First step: Draw the (virtual) game screen.
 	g_context.getActivePipeline()->drawTexture(_gameScreen->getGLTexture(), _gameDrawRect.left, _gameDrawRect.top + shakeOffset, _gameDrawRect.width(), _gameDrawRect.height());
 
 	// Second step: Draw the overlay if visible.
 	if (_overlayVisible) {
+		_backBuffer.enableBlend(Framebuffer::kBlendModeTraditionalTransparency);
 		g_context.getActivePipeline()->drawTexture(_overlay->getGLTexture(), 0, 0, _overlayDrawRect.width(), _overlayDrawRect.height());
 	}
 
 	// Third step: Draw the cursor if visible.
 	if (_cursorVisible && _cursor) {
+		_backBuffer.enableBlend(Framebuffer::kBlendModePremultipliedTransparency);
+
 		// Adjust game screen shake position, but only when the overlay is not
 		// visible.
 		const GLfloat cursorOffset = _overlayVisible ? 0 : shakeOffset;
@@ -446,6 +452,10 @@ void OpenGLGraphicsManager::updateScreen() {
 
 #ifdef USE_OSD
 	// Fourth step: Draw the OSD.
+	if (_osdMessageSurface || _osdIconSurface) {
+		_backBuffer.enableBlend(Framebuffer::kBlendModeTraditionalTransparency);
+	}
+
 	if (_osdMessageSurface) {
 		// Update alpha value.
 		const int diff = g_system->getMillis(false) - _osdMessageFadeStartTime;
@@ -549,20 +559,35 @@ void OpenGLGraphicsManager::grabOverlay(void *buf, int pitch) const {
 }
 
 namespace {
-template<typename DstPixel, typename SrcPixel>
-void applyColorKey(DstPixel *dst, const SrcPixel *src, uint w, uint h, uint dstPitch, uint srcPitch, SrcPixel keyColor, DstPixel alphaMask) {
-	const uint srcAdd = srcPitch - w * sizeof(SrcPixel);
-	const uint dstAdd = dstPitch - w * sizeof(DstPixel);
-
-	while (h-- > 0) {
-		for (uint x = w; x > 0; --x, ++dst, ++src) {
-			if (*src == keyColor) {
-				*dst &= ~alphaMask;
+template<typename SrcColor, typename DstColor>
+void multiplyColorWithAlpha(const byte *src, byte *dst, const uint w, const uint h,
+                            const Graphics::PixelFormat &srcFmt, const Graphics::PixelFormat &dstFmt,
+                            const uint srcPitch, const uint dstPitch, const SrcColor keyColor) {
+	for (uint y = 0; y < h; ++y) {
+		for (uint x = 0; x < w; ++x) {
+			const uint32 color = *(const SrcColor *)src;
+
+			if (color == keyColor) {
+				*(DstColor *)dst = 0;
+			} else {
+				byte a, r, g, b;
+				srcFmt.colorToARGB(color, a, r, g, b);
+
+				if (a != 0xFF) {
+					r = (int) r * a / 255;
+					g = (int) g * a / 255;
+					b = (int) b * a / 255;
+				}
+
+				*(DstColor *)dst = dstFmt.ARGBToColor(a, r, g, b);
 			}
+
+			src += sizeof(SrcColor);
+			dst += sizeof(DstColor);
 		}
 
-		dst = (DstPixel *)((byte *)dst + dstAdd);
-		src = (const SrcPixel *)((const byte *)src + srcAdd);
+		src += srcPitch - w * srcFmt.bytesPerPixel;
+		dst += dstPitch - w * dstFmt.bytesPerPixel;
 	}
 }
 } // End of anonymous namespace
@@ -629,27 +654,26 @@ void OpenGLGraphicsManager::setMouseCursor(const void *buf, uint w, uint h, int
 
 		// Copy the cursor data to the actual texture surface. This will make
 		// sure that the data is also converted to the expected format.
-		Graphics::crossBlit((byte *)dst->getPixels(), (const byte *)buf, dst->pitch, srcPitch,
-		                    w, h, dst->format, inputFormat);
 
-		// We apply the color key by setting the alpha bits of the pixels to
-		// fully transparent.
-		const uint32 aMask = (0xFF >> dst->format.aLoss) << dst->format.aShift;
+		// Also multiply the color values with the alpha channel.
+		// The pre-multiplication allows using a blend mode that prevents
+		// color fringes due to filtering.
+
 		if (dst->format.bytesPerPixel == 2) {
 			if (inputFormat.bytesPerPixel == 2) {
-				applyColorKey<uint16, uint16>((uint16 *)dst->getPixels(), (const uint16 *)buf, w, h,
-				                              dst->pitch, srcPitch, keycolor, aMask);
+				multiplyColorWithAlpha<uint16, uint16>((const byte *) buf, (byte *) dst->getPixels(), w, h,
+				                                       inputFormat, dst->format, srcPitch, dst->pitch, keycolor);
 			} else if (inputFormat.bytesPerPixel == 4) {
-				applyColorKey<uint16, uint32>((uint16 *)dst->getPixels(), (const uint32 *)buf, w, h,
-				                              dst->pitch, srcPitch, keycolor, aMask);
+				multiplyColorWithAlpha<uint32, uint16>((const byte *) buf, (byte *) dst->getPixels(), w, h,
+				                                       inputFormat, dst->format, srcPitch, dst->pitch, keycolor);
 			}
 		} else {
 			if (inputFormat.bytesPerPixel == 2) {
-				applyColorKey<uint32, uint16>((uint32 *)dst->getPixels(), (const uint16 *)buf, w, h,
-				                              dst->pitch, srcPitch, keycolor, aMask);
+				multiplyColorWithAlpha<uint16, uint32>((const byte *) buf, (byte *) dst->getPixels(), w, h,
+				                                       inputFormat, dst->format, srcPitch, dst->pitch, keycolor);
 			} else if (inputFormat.bytesPerPixel == 4) {
-				applyColorKey<uint32, uint32>((uint32 *)dst->getPixels(), (const uint32 *)buf, w, h,
-				                              dst->pitch, srcPitch, keycolor, aMask);
+				multiplyColorWithAlpha<uint32, uint32>((const byte *) buf, (byte *) dst->getPixels(), w, h,
+				                                       inputFormat, dst->format, srcPitch, dst->pitch, keycolor);
 			}
 		}
 
@@ -881,14 +905,10 @@ void OpenGLGraphicsManager::notifyContextCreate(const Graphics::PixelFormat &def
 
 	g_context.getActivePipeline()->setColor(1.0f, 1.0f, 1.0f, 1.0f);
 
-	GL_CALL(glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA));
-
 	// Setup backbuffer state.
 
 	// Default to black as clear color.
 	_backBuffer.setClearColor(0.0f, 0.0f, 0.0f, 0.0f);
-	// Setup alpha blend (for overlay and cursor).
-	_backBuffer.enableBlend(true);
 
 	g_context.getActivePipeline()->setFramebuffer(&_backBuffer);
 
diff --git a/backends/graphics/opengl/texture.cpp b/backends/graphics/opengl/texture.cpp
index 33598b5488..2a078534ab 100644
--- a/backends/graphics/opengl/texture.cpp
+++ b/backends/graphics/opengl/texture.cpp
@@ -344,16 +344,16 @@ Graphics::PixelFormat TextureCLUT8::getFormat() const {
 }
 
 void TextureCLUT8::setColorKey(uint colorKey) {
-	// We remove all alpha bits from the palette entry of the color key.
-	// This makes sure its properly handled as color key.
-	const uint32 aMask = (0xFF >> _format.aLoss) << _format.aShift;
-
+	// The key color is set to black so the color value is pre-multiplied with the alpha value
+	// to avoid color fringes due to filtering.
+	// Erasing the color data is not a problem as the palette is always fully re-initialized
+	// before setting the key color.
 	if (_format.bytesPerPixel == 2) {
 		uint16 *palette = (uint16 *)_palette + colorKey;
-		*palette &= ~aMask;
+		*palette = 0;
 	} else if (_format.bytesPerPixel == 4) {
 		uint32 *palette = (uint32 *)_palette + colorKey;
-		*palette &= ~aMask;
+		*palette = 0;
 	} else {
 		warning("TextureCLUT8::setColorKey: Unsupported pixel depth %d", _format.bytesPerPixel);
 	}
@@ -581,6 +581,13 @@ Graphics::PixelFormat TextureCLUT8GPU::getFormat() const {
 }
 
 void TextureCLUT8GPU::setColorKey(uint colorKey) {
+	// The key color is set to black so the color value is pre-multiplied with the alpha value
+	// to avoid color fringes due to filtering.
+	// Erasing the color data is not a problem as the palette is always fully re-initialized
+	// before setting the key color.
+	_palette[colorKey * 4    ] = 0x00;
+	_palette[colorKey * 4 + 1] = 0x00;
+	_palette[colorKey * 4 + 2] = 0x00;
 	_palette[colorKey * 4 + 3] = 0x00;
 
 	_paletteDirty = true;