From 176db99cc522322f8e48b15e330d116388f917d9 Mon Sep 17 00:00:00 2001
From: Simon Zeni <simon@bl4ckb0ne.ca>
Date: Sat, 30 Oct 2021 22:31:16 -0400
Subject: [PATCH] render/vulkan: get only available validation layers

---
 render/vulkan/vulkan.c | 97 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 89 insertions(+), 8 deletions(-)

diff --git a/render/vulkan/vulkan.c b/render/vulkan/vulkan.c
index 4932ec4da4..2b18a436b0 100644
--- a/render/vulkan/vulkan.c
+++ b/render/vulkan/vulkan.c
@@ -33,6 +33,81 @@ static const char *find_extensions(const VkExtensionProperties *avail,
 	return NULL;
 }
 
+static char **get_validation_layers(uint32_t *layer_count) {
+	// TODO request layer via env var
+	static const char *layers[] = {
+		"VK_LAYER_KHRONOS_validation",
+		// "VK_LAYER_RENDERDOC_Capture",
+		// "VK_LAYER_live_introspection",
+	};
+
+	static const size_t layers_len = sizeof(layers) / sizeof(layers[0]);
+
+	VkLayerProperties *layer_props = NULL;
+	char **found_layers = NULL;
+	uint32_t found_layers_count = 0;
+
+	uint32_t count;
+	vkEnumerateInstanceLayerProperties(&count, NULL);
+
+	if (count == 0) {
+		wlr_log(WLR_DEBUG, "No validation layers found");
+		goto layers_err;
+	}
+	wlr_log(WLR_DEBUG, "Found %"PRIu32" validation layers", count);
+
+	layer_props = calloc((size_t)count, sizeof(VkLayerProperties));
+	if (layer_props == NULL) {
+		wlr_log(WLR_ERROR, "Failed to allocate %"PRIu32" VkLayerProperties",
+			count);
+		goto layers_err;
+	}
+
+	found_layers = calloc((size_t)count, sizeof(char*));
+	if (found_layers == NULL) {
+		wlr_log(WLR_ERROR, "Failed to allocate validation layers");
+		goto layers_err;
+	}
+
+	vkEnumerateInstanceLayerProperties(&count, layer_props);
+	for (size_t i = 0; i < (size_t)count; ++i) {
+		wlr_log(WLR_DEBUG, "Vulkan instance validation layer %s v%"PRIu32,
+			layer_props[i].layerName, layer_props[i].implementationVersion);
+		for (size_t j = 0; j < layers_len; ++j) {
+			if (strcmp(layer_props[i].layerName, layers[j]) == 0) {
+				found_layers[found_layers_count] = calloc(
+					VK_MAX_EXTENSION_NAME_SIZE, sizeof(char));
+				if (found_layers[found_layers_count] == NULL) {
+					wlr_log(WLR_ERROR, "Failed to allocate validation layer");
+					goto layers_err;
+				}
+
+				strcpy(found_layers[found_layers_count], layers[j]);
+				found_layers_count++;
+				break;
+			}
+		}
+	}
+
+	free(layer_props);
+
+	*layer_count = found_layers_count;
+	return found_layers;
+
+layers_err:
+	free(layer_props);
+
+	if (found_layers) {
+		for (uint32_t i = 0; i < found_layers_count; ++i) {
+			free(found_layers[i]);
+		}
+	}
+
+	free(found_layers);
+	*layer_count = 0;
+	return NULL;
+}
+
 static VkBool32 debug_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
 		VkDebugUtilsMessageTypeFlagsEXT type,
 		const VkDebugUtilsMessengerCallbackDataEXT *debug_data,
@@ -163,13 +238,13 @@ struct wlr_vk_instance *vulkan_instance_create(size_t ext_count,
 	application_info.engineVersion = WLR_VERSION_NUM;
 	application_info.apiVersion = VK_API_VERSION_1_1;
 
-	const char *layers[] = {
-		"VK_LAYER_KHRONOS_validation",
-		// "VK_LAYER_RENDERDOC_Capture",
-		// "VK_LAYER_live_introspection",
-	};
-
-	unsigned layer_count = debug * (sizeof(layers) / sizeof(layers[0]));
+	uint32_t layer_count = 0;
+	char **layers = get_validation_layers(&layer_count);
+	wlr_log(WLR_DEBUG, "Using %"PRIu32" instance validation layers",
+		layer_count);
+	for (uint32_t i = 0; i < layer_count; ++i) {
+		wlr_log(WLR_DEBUG, "%s", layers[i]);
+	}
 
 	VkInstanceCreateInfo instance_info = {0};
 	instance_info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
@@ -177,7 +252,7 @@ struct wlr_vk_instance *vulkan_instance_create(size_t ext_count,
 	instance_info.enabledExtensionCount = ini->extension_count;
 	instance_info.ppEnabledExtensionNames = ini->extensions;
 	instance_info.enabledLayerCount = layer_count;
-	instance_info.ppEnabledLayerNames = layers;
+	instance_info.ppEnabledLayerNames = (const char *const *)layers;
 
 	VkDebugUtilsMessageSeverityFlagsEXT severity =
 		// VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
@@ -204,6 +279,12 @@ struct wlr_vk_instance *vulkan_instance_create(size_t ext_count,
 	}
 
 	res = vkCreateInstance(&instance_info, NULL, &ini->instance);
+
+	for (size_t i = 0; i < layer_count; ++i) {
+		free(layers[i]);
+	}
+	free(layers);
+
 	if (res != VK_SUCCESS) {
 		wlr_vk_error("Could not create instance", res);
 		goto error;