From 8e8f27a22aa3fe8a0470dd19ebbd87b1b9d72ed0 Mon Sep 17 00:00:00 2001
From: outfoxxed <outfoxxed@outfoxxed.me>
Date: Mon, 23 Feb 2026 23:03:48 -0800
Subject: [PATCH] wayland/screencopy: enable vulkan dmabuf support on session
 locks

Also reformat dmabuf
---
 src/wayland/buffer/dmabuf.cpp | 27 ++++++++++-----------------
 src/wayland/session_lock.cpp  | 10 ++++++++++
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/src/wayland/buffer/dmabuf.cpp b/src/wayland/buffer/dmabuf.cpp
index 7d17884..ed9dbeb 100644
--- a/src/wayland/buffer/dmabuf.cpp
+++ b/src/wayland/buffer/dmabuf.cpp
@@ -28,6 +28,7 @@
 #include <qscopedpointer.h>
 #include <qsgrendererinterface.h>
 #include <qsgtexture_platform.h>
+#include <qtypes.h>
 #include <qvulkanfunctions.h>
 #include <qvulkaninstance.h>
 #include <qwayland-linux-dmabuf-v1.h>
@@ -35,7 +36,6 @@
 #include <sys/mman.h>
 #include <sys/types.h>
 #include <unistd.h>
-#include <qtypes.h>
 #include <vulkan/vulkan_core.h>
 #include <wayland-client-protocol.h>
 #include <wayland-linux-dmabuf-v1-client-protocol.h>
@@ -80,10 +80,8 @@ bool drmFormatHasAlpha(uint32_t drmFormat) {
 	case DRM_FORMAT_ABGR8888:
 	case DRM_FORMAT_ARGB2101010:
 	case DRM_FORMAT_ABGR2101010:
-	case DRM_FORMAT_ABGR16161616F:
-		return true;
-	default:
-		return false;
+	case DRM_FORMAT_ABGR16161616F: return true;
+	default: return false;
 	}
 }
 
@@ -818,7 +816,8 @@ WlBufferQSGTexture* WlDmaBuffer::createQsgTextureVulkan(QQuickWindow* window) co
 
 	// dup() is required because vkAllocateMemory with VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT
 	// takes ownership of the fd on succcess. Without dup, WlDmaBuffer would double-close.
-	const int dupFd = dup(this->planes[0].fd); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
+	const int dupFd =
+	    dup(this->planes[0].fd); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
 	if (dupFd < 0) {
 		qCWarning(logDmabuf) << "Failed to dup() fd for DMA-BUF import";
 		goto cleanup_fail; // NOLINT
@@ -909,12 +908,12 @@ WlBufferQSGTexture* WlDmaBuffer::createQsgTextureVulkan(QQuickWindow* window) co
 			// find the graphics queue family index for the ownrship transfer.
 			uint32_t graphicsQueueFamily = 0;
 			uint32_t queueFamilyCount = 0;
-			instFuncs->vkGetPhysicalDeviceQueueFamilyProperties(
-			    physDevice, &queueFamilyCount, nullptr
-			);
+			instFuncs->vkGetPhysicalDeviceQueueFamilyProperties(physDevice, &queueFamilyCount, nullptr);
 			std::vector<VkQueueFamilyProperties> queueFamilies(queueFamilyCount);
 			instFuncs->vkGetPhysicalDeviceQueueFamilyProperties(
-			    physDevice, &queueFamilyCount, queueFamilies.data()
+			    physDevice,
+			    &queueFamilyCount,
+			    queueFamilies.data()
 			);
 			for (uint32_t i = 0; i < queueFamilyCount; ++i) {
 				if (queueFamilies[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) {
@@ -989,13 +988,7 @@ WlBufferQSGTexture* WlDmaBuffer::createQsgTextureVulkan(QQuickWindow* window) co
 			}
 		}
 
-		auto* tex = new WlDmaBufferVulkanQSGTexture(
-		    devFuncs,
-		    device,
-		    image,
-		    memory,
-		    qsgTexture
-		);
+		auto* tex = new WlDmaBufferVulkanQSGTexture(devFuncs, device, image, memory, qsgTexture);
 		qCDebug(logDmabuf) << "Created WlDmaBufferVulkanQSGTexture" << tex << "from" << this;
 		return tex;
 	}
diff --git a/src/wayland/session_lock.cpp b/src/wayland/session_lock.cpp
index d5a3e53..2ebe3fd 100644
--- a/src/wayland/session_lock.cpp
+++ b/src/wayland/session_lock.cpp
@@ -9,6 +9,7 @@
 #include <qqmlcomponent.h>
 #include <qqmlengine.h>
 #include <qqmllist.h>
+#include <qquickgraphicsconfiguration.h>
 #include <qquickitem.h>
 #include <qquickwindow.h>
 #include <qscreen.h>
@@ -216,6 +217,15 @@ void WlSessionLockSurface::onReload(QObject* oldInstance) {
 
 	if (this->window == nullptr) {
 		this->window = new QQuickWindow();
+
+		// needed for vulkan dmabuf import, qt ignores these if not applicable
+		auto graphicsConfig = this->window->graphicsConfiguration();
+		graphicsConfig.setDeviceExtensions({
+		    "VK_KHR_external_memory_fd",
+		    "VK_EXT_external_memory_dma_buf",
+		    "VK_EXT_image_drm_format_modifier",
+		});
+		this->window->setGraphicsConfiguration(graphicsConfig);
 	}
 
 	this->mContentItem->setParentItem(this->window->contentItem());