EGLX: Optional hacks to support Chrome/ANGLE

Chrome/ANGLE attempts to reproduce the visual matching algorithm from
GTK v3.15.2 through 4.3.1 (refer to
398e941), but it does so by sending raw
GLX requests to and receiving raw GLX replies from the X server.  Thus,
there is nothing for VirtualGL to interpose, and Chrome picks "system"
and "RGBA" visuals based solely on OpenGL rendering attributes assigned
to the 2D X server visuals by the 2D X server's GLX implementation, not
on OpenGL rendering attributes assigned to the visuals by VirtualGL.  As
a result, the RGBA visual that Chrome picks is not associated with any
EGLConfig that VirtualGL returns.  To work around this, two hacks can
now be enabled by setting an undocumented environment variable
(VGL_CHROMEHACK) to 1:

- glxvisual::buildVisAttribTable() scores the 2D X server visuals in the
  same way that Chrome does, preferring an alpha-enabled visual without
  a depth buffer, a stencil buffer, or multisampling.
- The EGL/X11 interposer chooses the highest-scoring visual and assigns
  its visual ID to the value of the EGL_NATIVE_VISUAL_ID attribute for
  any EGLConfigs that have an 8-bit alpha channel.

VGL_CHROMEHACK automatically enables VGL_PROBEGLX.

This should allow VirtualGL to work with Chrome v112+ if Chrome is
passed '--disable-seccomp-filter-sandbox --use-angle=gl-egl'.

Refer to #229

common/rr.h

@@ -1,6 +1,6 @@
 /* Copyright (C)2004 Landmark Graphics Corporation
  * Copyright (C)2005-2007 Sun Microsystems, Inc.
- * Copyright (C)2009-2013, 2015-2022 D. R. Commander
+ * Copyright (C)2009-2013, 2015-2022, 2024 D. R. Commander
  *
  * This library is free software and may be redistributed and/or modified under
  * the terms of the wxWindows Library License, Version 3.1 or (at your option)
@@ -209,6 +209,7 @@ typedef struct _FakerConfig
   char ocllib[MAXSTR];
   char amdgpuHack;
   char exitfunction[MAXSTR];
+  char chromeHack;
 } FakerConfig;
 
 #if !defined(__SUNPRO_CC) && !defined(__SUNPRO_C)

server/faker-egl.cpp

@@ -109,7 +109,7 @@ static XVisualInfo *getVisualFromConfig(faker::EGLXDisplay *eglxdpy,
 {
 	if(!eglxdpy || !config) return NULL;
 
-	EGLint redSize, greenSize, blueSize;
+	EGLint redSize, greenSize, blueSize, alphaSize;
 	int depth = 24;
 
 	if(_eglGetConfigAttrib(eglxdpy->edpy, config, EGL_RED_SIZE, &redSize)
@@ -118,6 +118,15 @@ static XVisualInfo *getVisualFromConfig(faker::EGLXDisplay *eglxdpy,
 		&& redSize == 10 && greenSize == 10 && blueSize == 10)
 		depth = 30;
 
+	if(fconfig.chromeHack
+		&& _eglGetConfigAttrib(eglxdpy->edpy, config, EGL_ALPHA_SIZE, &alphaSize)
+		&& alphaSize == 8)
+	{
+		XVisualInfo *v = glxvisual::getHighestScoringVisual(eglxdpy->x11dpy,
+			eglxdpy->screen);
+		if(v) return v;
+	}
+
 	return getVisual(eglxdpy->x11dpy, eglxdpy->screen, depth, TrueColor);
 }
 

server/fakerconfig.cpp

@@ -1,4 +1,4 @@
-// Copyright (C)2009-2023 D. R. Commander
+// Copyright (C)2009-2024 D. R. Commander
 //
 // This library is free software and may be redistributed and/or modified under
 // the terms of the wxWindows Library License, Version 3.1 or (at your option)
@@ -280,6 +280,7 @@ void fconfig_reloadenv(void)
 	FETCHENV_BOOL("VGL_ALLOWINDIRECT", allowindirect);
 	FETCHENV_BOOL("VGL_AMDGPUHACK", amdgpuHack);
 	FETCHENV_BOOL("VGL_AUTOTEST", autotest);
+	FETCHENV_BOOL("VGL_CHROMEHACK", chromeHack);
 	FETCHENV_STR("VGL_CLIENT", client);
 	if((env = getenv("VGL_SUBSAMP")) != NULL && strlen(env) > 0)
 	{
@@ -481,6 +482,8 @@ void fconfig_reloadenv(void)
 		if(fconfig.subsamp < 0) fconfig.subsamp = 1;
 	}
 
+	if(fconfig.chromeHack) fconfig.probeglx = 1;
+
 	fconfig_envset = true;
 }
 
@@ -620,6 +623,7 @@ void fconfig_print(FakerConfig &fc)
 {
 	PRCONF_INT(allowindirect);
 	PRCONF_INT(amdgpuHack);
+	PRCONF_INT(chromeHack);
 	PRCONF_STR(client);
 	PRCONF_INT(compress);
 	PRCONF_STR(config);

server/glxvisual.cpp

@@ -1,6 +1,6 @@
 // Copyright (C)2004 Landmark Graphics Corporation
 // Copyright (C)2005 Sun Microsystems, Inc.
-// Copyright (C)2009-2016, 2019-2023 D. R. Commander
+// Copyright (C)2009-2016, 2019-2024 D. R. Commander
 //
 // This library is free software and may be redistributed and/or modified under
 // the terms of the wxWindows Library License, Version 3.1 or (at your option)
@@ -33,6 +33,7 @@ typedef struct
 	VGLFBConfig config;
 	int depth, c_class, bpc, nVisuals;
 	int isStereo, isDB, isGL;
+	int score;
 	GLXAttrib glx;
 } VisAttrib;
 
@@ -248,12 +249,31 @@ static bool buildVisAttribTable(Display *dpy, int screen)
 			va[i].c_class = visuals[i].c_class;
 			va[i].bpc = visuals[i].bits_per_rgb;
 			va[i].nVisuals = nVisuals;
+			va[i].score = -1;
 
 			if(clientGLX)
 			{
 				_glXGetConfig(dpy, &visuals[i], GLX_DOUBLEBUFFER, &va[i].isDB);
 				_glXGetConfig(dpy, &visuals[i], GLX_USE_GL, &va[i].isGL);
 				_glXGetConfig(dpy, &visuals[i], GLX_STEREO, &va[i].isStereo);
+				// Simulate Chrome's ARGB visual selection mechanism, which
+				// unfortunately uses raw GLX requests/replies that VirtualGL cannot
+				// interpose.
+				if(fconfig.chromeHack && va[i].depth == 32 && va[i].bpc == 8
+					&& va[i].isDB && !va[i].isStereo)
+				{
+					va[i].score = 0;
+					int alphaSize = -1, depthSize = -1, stencilSize = -1,
+						sampleBuffers = -1;
+					_glXGetConfig(dpy, &visuals[i], GLX_ALPHA_SIZE, &alphaSize);
+					if(alphaSize > 0) va[i].score++;
+					_glXGetConfig(dpy, &visuals[i], GLX_DEPTH_SIZE, &depthSize);
+					_glXGetConfig(dpy, &visuals[i], GLX_STENCIL_SIZE, &stencilSize);
+					if(depthSize == 0 && stencilSize == 0) va[i].score++;
+					_glXGetConfig(dpy, &visuals[i], GLX_SAMPLE_BUFFERS_ARB,
+						&sampleBuffers);
+					if(sampleBuffers == 0) va[i].score++;
+				}
 			}
 			va[i].glx.alphaSize = va[i].glx.depthSize = va[i].glx.stencilSize =
 				va[i].glx.samples = -1;
@@ -1067,4 +1087,28 @@ VGLFBConfig getDefaultFBConfig(Display *dpy, int screen, VisualID vid)
 	return 0;
 }
 
+
+XVisualInfo *getHighestScoringVisual(Display *dpy, int screen)
+{
+	if(!dpy) return NULL;
+
+	int highestScore = -1;
+	VisualID bestVisualID =
+		matchVisual2D(dpy, screen, 32, TrueColor, 8, 0, true);
+
+	GET_VA_TABLE();
+
+	for(int i = 0; i < vaEntries; i++)
+	{
+		if(va[i].score > highestScore)
+		{
+			highestScore = va[i].score;
+			bestVisualID = va[i].visualID;
+		}
+	}
+
+	if(bestVisualID) return visualFromID(dpy, screen, bestVisualID);
+	else return NULL;
+}
+
 }  // namespace

server/glxvisual.h

@@ -1,6 +1,6 @@
 // Copyright (C)2004 Landmark Graphics Corporation
 // Copyright (C)2005 Sun Microsystems, Inc.
-// Copyright (C)2014, 2019-2021 D. R. Commander
+// Copyright (C)2014, 2019-2021, 2024 D. R. Commander
 //
 // This library is free software and may be redistributed and/or modified under
 // the terms of the wxWindows Library License, Version 3.1 or (at your option)
@@ -77,6 +77,8 @@ namespace glxvisual
 	// This function returns the default FB config attached to a given visual ID
 	// in the visual attribute table.
 	VGLFBConfig getDefaultFBConfig(Display *dpy, int screen, VisualID vid);
+
+	XVisualInfo *getHighestScoringVisual(Display *dpy, int screen);
 }