path: root/dolphin-emu-wayland-support.patch

commit e249a7690535a32ea33e700c52c324d9d2300b30
Author: Artemis Tosini <me@artem.ist>
Date:   Mon Apr 6 19:47:13 2020 +0000

    DolphinQt: Add support for Vulkan on Wayland
    
    Currently, Linux users of DolphinQt can only use the X11 windowing
    system. This commit adds support for running on top of Wayland, which
    some compositors require in order to access full resolution on HiDPI
    displays (such as sway). This uses the Vulkan ICD to get a Vulkan
    surface, though does not support using wayland-egl to get an EGL
    surface or software rendering to a native Wayland surface.
    
    This commit does not set Wayland as the default backend when it is availaible.
    Users who want to use it must set -DENABLE_WAYLAND=ON when compiling Dolphin
    and run Dolphin with QT_QPA_PLATFORM=wayland.

diff --git a/CMake/FindWayland.cmake b/CMake/FindWayland.cmake
new file mode 100644
index 0000000000..f93218b873
--- /dev/null
+++ b/CMake/FindWayland.cmake
@@ -0,0 +1,66 @@
+# Try to find Wayland on a Unix system
+#
+# This will define:
+#
+#   WAYLAND_FOUND       - True if Wayland is found
+#   WAYLAND_LIBRARIES   - Link these to use Wayland
+#   WAYLAND_INCLUDE_DIR - Include directory for Wayland
+#   WAYLAND_DEFINITIONS - Compiler flags for using Wayland
+#
+# In addition the following more fine grained variables will be defined:
+#
+#   WAYLAND_CLIENT_FOUND  WAYLAND_CLIENT_INCLUDE_DIR  WAYLAND_CLIENT_LIBRARIES
+#   WAYLAND_SERVER_FOUND  WAYLAND_SERVER_INCLUDE_DIR  WAYLAND_SERVER_LIBRARIES
+#   WAYLAND_EGL_FOUND     WAYLAND_EGL_INCLUDE_DIR     WAYLAND_EGL_LIBRARIES
+#
+# Copyright (c) 2013 Martin Gräßlin <mgraesslin@kde.org>
+#
+# Redistribution and use is allowed according to the terms of the BSD license.
+# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+
+IF (NOT WIN32)
+  IF (WAYLAND_INCLUDE_DIR AND WAYLAND_LIBRARIES)
+    # In the cache already
+    SET(WAYLAND_FIND_QUIETLY TRUE)
+  ENDIF ()
+
+  # Use pkg-config to get the directories and then use these values
+  # in the FIND_PATH() and FIND_LIBRARY() calls
+  FIND_PACKAGE(PkgConfig)
+  PKG_CHECK_MODULES(PKG_WAYLAND QUIET wayland-client wayland-server wayland-egl wayland-cursor)
+
+  SET(WAYLAND_DEFINITIONS ${PKG_WAYLAND_CFLAGS})
+
+  FIND_PATH(WAYLAND_CLIENT_INCLUDE_DIR  NAMES wayland-client.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_SERVER_INCLUDE_DIR  NAMES wayland-server.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_EGL_INCLUDE_DIR     NAMES wayland-egl.h    HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_CURSOR_INCLUDE_DIR  NAMES wayland-cursor.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+
+  FIND_LIBRARY(WAYLAND_CLIENT_LIBRARIES NAMES wayland-client   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_SERVER_LIBRARIES NAMES wayland-server   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_EGL_LIBRARIES    NAMES wayland-egl      HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_CURSOR_LIBRARIES NAMES wayland-cursor   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+
+  set(WAYLAND_INCLUDE_DIR ${WAYLAND_CLIENT_INCLUDE_DIR} ${WAYLAND_SERVER_INCLUDE_DIR} ${WAYLAND_EGL_INCLUDE_DIR} ${WAYLAND_CURSOR_INCLUDE_DIR})
+
+  set(WAYLAND_LIBRARIES ${WAYLAND_CLIENT_LIBRARIES} ${WAYLAND_SERVER_LIBRARIES} ${WAYLAND_EGL_LIBRARIES} ${WAYLAND_CURSOR_LIBRARIES})
+
+  list(REMOVE_DUPLICATES WAYLAND_INCLUDE_DIR)
+
+  include(FindPackageHandleStandardArgs)
+
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_CLIENT  DEFAULT_MSG  WAYLAND_CLIENT_LIBRARIES  WAYLAND_CLIENT_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_SERVER  DEFAULT_MSG  WAYLAND_SERVER_LIBRARIES  WAYLAND_SERVER_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_EGL     DEFAULT_MSG  WAYLAND_EGL_LIBRARIES     WAYLAND_EGL_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_CURSOR  DEFAULT_MSG  WAYLAND_CURSOR_LIBRARIES  WAYLAND_CURSOR_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND         DEFAULT_MSG  WAYLAND_LIBRARIES         WAYLAND_INCLUDE_DIR)
+
+  MARK_AS_ADVANCED(
+        WAYLAND_INCLUDE_DIR         WAYLAND_LIBRARIES
+        WAYLAND_CLIENT_INCLUDE_DIR  WAYLAND_CLIENT_LIBRARIES
+        WAYLAND_SERVER_INCLUDE_DIR  WAYLAND_SERVER_LIBRARIES
+        WAYLAND_EGL_INCLUDE_DIR     WAYLAND_EGL_LIBRARIES
+        WAYLAND_CURSOR_INCLUDE_DIR  WAYLAND_CURSOR_LIBRARIES
+  )
+
+ENDIF ()
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 04bcdde4c2..a67ea09589 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -43,6 +43,7 @@ set(DOLPHIN_DEFAULT_UPDATE_TRACK "" CACHE STRING "Name of the default update tra
 
 if(UNIX AND NOT APPLE AND NOT ANDROID)
   option(ENABLE_X11 "Enables X11 Support" ON)
+	option(ENABLE_WAYLAND "Enables Wayland Support" OFF)
 endif()
 if(NOT WIN32 AND NOT APPLE AND NOT HAIKU)
   option(ENABLE_EGL "Enables EGL OpenGL Interface" ON)
@@ -549,6 +550,17 @@ if(ENABLE_X11)
   endif()
 endif()
 
+if(ENABLE_WAYLAND)
+	find_package(Wayland)
+	if(WAYLAND_FOUND)
+		add_definitions(-DHAVE_WAYLAND=1)
+		message(STATUS "Wayland support enabled")
+  else()
+		message(WARNING "Wayland support enabled but not found. This build will not support Wayland.")
+  endif()
+endif()
+
+
 if(ENABLE_EGL)
   find_package(EGL)
   if(EGL_FOUND)
diff --git a/Source/Core/Common/WindowSystemInfo.h b/Source/Core/Common/WindowSystemInfo.h
index 8936ad1a02..604e963ab5 100644
--- a/Source/Core/Common/WindowSystemInfo.h
+++ b/Source/Core/Common/WindowSystemInfo.h
@@ -19,9 +19,9 @@ struct WindowSystemInfo
 {
   WindowSystemInfo() = default;
   WindowSystemInfo(WindowSystemType type_, void* display_connection_, void* render_window_,
-                   void* render_surface_)
+                   void* render_surface_, int width_, int height_)
       : type(type_), display_connection(display_connection_), render_window(render_window_),
-        render_surface(render_surface_)
+        render_surface(render_surface_), width(width_), height(height_)
   {
   }
 
@@ -41,6 +41,12 @@ struct WindowSystemInfo
   // during video backend startup the surface pointer may change (MoltenVK).
   void* render_surface = nullptr;
 
+  // Width and height of the render surface. This is necessary on Wayland as
+  // vkGetPhysicalDeviceSurfaceCapabilitiesKHR does not return the size of
+  // the VkSurfaceKHR created with vkCreateWaylandSurfaceKHR
+  int width = -1;
+  int height = -1;
+
   // Scale of the render surface. For hidpi systems, this will be >1.
   float render_surface_scale = 1.0f;
 };
diff --git a/Source/Core/DolphinQt/Host.cpp b/Source/Core/DolphinQt/Host.cpp
index 5ccd10189c..0ef8318cc4 100644
--- a/Source/Core/DolphinQt/Host.cpp
+++ b/Source/Core/DolphinQt/Host.cpp
@@ -189,7 +189,7 @@ void Host::SetRenderFullscreen(bool fullscreen)
 void Host::ResizeSurface(int new_width, int new_height)
 {
   if (g_renderer)
-    g_renderer->ResizeSurface();
+    g_renderer->ResizeSurface(new_width, new_height);
 }
 
 std::vector<std::string> Host_GetPreferredLocales()
diff --git a/Source/Core/DolphinQt/MainWindow.cpp b/Source/Core/DolphinQt/MainWindow.cpp
index 52a2a0290c..47d46cb719 100644
--- a/Source/Core/DolphinQt/MainWindow.cpp
+++ b/Source/Core/DolphinQt/MainWindow.cpp
@@ -183,7 +183,12 @@ static WindowSystemInfo GetWindowSystemInfo(QWindow* window)
   QPlatformNativeInterface* pni = QGuiApplication::platformNativeInterface();
   wsi.display_connection = pni->nativeResourceForWindow("display", window);
   if (wsi.type == WindowSystemType::Wayland)
+  {
     wsi.render_window = window ? pni->nativeResourceForWindow("surface", window) : nullptr;
+    QSize size = window->size();
+    wsi.width = size.width();
+    wsi.height = size.height();
+  }
   else
     wsi.render_window = window ? reinterpret_cast<void*>(window->winId()) : nullptr;
   wsi.render_surface = wsi.render_window;
@@ -1254,8 +1259,12 @@ void MainWindow::ShowGraphicsWindow()
           static_cast<Display*>(QGuiApplication::platformNativeInterface()->nativeResourceForWindow(
               "display", windowHandle())),
           winId());
+      m_graphics_window = new GraphicsWindow(m_xrr_config.get(), this);
+    }
+    else
+    {
+      m_graphics_window = new GraphicsWindow(nullptr, this);
     }
-    m_graphics_window = new GraphicsWindow(m_xrr_config.get(), this);
 #else
     m_graphics_window = new GraphicsWindow(nullptr, this);
 #endif
diff --git a/Source/Core/VideoBackends/OGL/CMakeLists.txt b/Source/Core/VideoBackends/OGL/CMakeLists.txt
index 9a9d9caf2d..f4f6f46d35 100644
--- a/Source/Core/VideoBackends/OGL/CMakeLists.txt
+++ b/Source/Core/VideoBackends/OGL/CMakeLists.txt
@@ -32,6 +32,7 @@ PUBLIC
 
 PRIVATE
   ${X11_LIBRARIES}
+	${WAYLAND_LIBRARIES}
 )
 
 if(MSVC)
diff --git a/Source/Core/VideoBackends/Vulkan/VKRenderer.cpp b/Source/Core/VideoBackends/Vulkan/VKRenderer.cpp
index 23dba4613b..9e48e15cf5 100644
--- a/Source/Core/VideoBackends/Vulkan/VKRenderer.cpp
+++ b/Source/Core/VideoBackends/Vulkan/VKRenderer.cpp
@@ -412,6 +412,7 @@ void Renderer::CheckForSurfaceResize()
   g_command_buffer_mgr->CheckLastPresentFail();
 
   // Resize the swap chain.
+  m_swap_chain->UpdateSize(m_new_width, m_new_height);
   m_swap_chain->RecreateSwapChain();
   OnSwapChainResized();
 }
diff --git a/Source/Core/VideoBackends/Vulkan/VKSwapChain.cpp b/Source/Core/VideoBackends/Vulkan/VKSwapChain.cpp
index a7f5bb929d..9854144e78 100644
--- a/Source/Core/VideoBackends/Vulkan/VKSwapChain.cpp
+++ b/Source/Core/VideoBackends/Vulkan/VKSwapChain.cpp
@@ -21,6 +21,10 @@
 #include <X11/Xlib.h>
 #endif
 
+#if defined(VK_USE_PLATFORM_WAYLAND_KHR)
+#include <wayland-client.h>
+#endif
+
 namespace Vulkan
 {
 SwapChain::SwapChain(const WindowSystemInfo& wsi, VkSurfaceKHR surface, bool vsync)
@@ -84,6 +88,29 @@ VkSurfaceKHR SwapChain::CreateVulkanSurface(VkInstance instance, const WindowSys
   }
 #endif
 
+#if defined(VK_USE_PLATFORM_WAYLAND_KHR)
+  if (wsi.type == WindowSystemType::Wayland)
+  {
+    VkWaylandSurfaceCreateInfoKHR surface_create_info = {
+        VK_STRUCTURE_TYPE_WAYLAND_SURFACE_CREATE_INFO_KHR,  // VkStructureType                sType
+        nullptr,                                            // const void*                    pNext
+        0,                                                  // VkWaylandSurfaceCreateFlagsKHR flags
+        static_cast<struct wl_display*>(wsi.display_connection),  // struct wl_display* display
+        static_cast<struct wl_surface*>(wsi.render_surface)       // struct wl_surface* surface
+    };
+
+    VkSurfaceKHR surface;
+    VkResult res = vkCreateWaylandSurfaceKHR(instance, &surface_create_info, nullptr, &surface);
+    if (res != VK_SUCCESS)
+    {
+      LOG_VULKAN_ERROR(res, "vkCreateWaylandSurfaceKHR failed: ");
+      return VK_NULL_HANDLE;
+    }
+
+    return surface;
+  }
+#endif
+
 #if defined(VK_USE_PLATFORM_ANDROID_KHR)
   if (wsi.type == WindowSystemType::Android)
   {
@@ -263,7 +290,12 @@ bool SwapChain::CreateSwapChain()
   // Determine the dimensions of the swap chain. Values of -1 indicate the size we specify here
   // determines window size?
   VkExtent2D size = surface_capabilities.currentExtent;
-  if (size.width == UINT32_MAX)
+  if (size.width == UINT32_MAX && m_wsi.type == WindowSystemType::Wayland)
+  {
+    size.width = m_wsi.width;
+    size.height = m_wsi.height;
+  }
+  else if (size.width == UINT32_MAX)
   {
     size.width = std::max(g_renderer->GetBackbufferWidth(), 1);
     size.height = std::max(g_renderer->GetBackbufferHeight(), 1);
diff --git a/Source/Core/VideoBackends/Vulkan/VKSwapChain.h b/Source/Core/VideoBackends/Vulkan/VKSwapChain.h
index 5e67217f2d..16b4a209d4 100644
--- a/Source/Core/VideoBackends/Vulkan/VKSwapChain.h
+++ b/Source/Core/VideoBackends/Vulkan/VKSwapChain.h
@@ -52,6 +52,12 @@ public:
   }
   VkResult AcquireNextImage();
 
+  void UpdateSize(int width, int height)
+  {
+    m_wsi.width = width;
+    m_wsi.height = height;
+  }
+
   bool RecreateSurface(void* native_handle);
   bool ResizeSwapChain();
   bool RecreateSwapChain();
diff --git a/Source/Core/VideoBackends/Vulkan/VulkanContext.cpp b/Source/Core/VideoBackends/Vulkan/VulkanContext.cpp
index cad326c83a..681551b086 100644
--- a/Source/Core/VideoBackends/Vulkan/VulkanContext.cpp
+++ b/Source/Core/VideoBackends/Vulkan/VulkanContext.cpp
@@ -204,6 +204,13 @@ bool VulkanContext::SelectInstanceExtensions(std::vector<const char*>* extension
     return false;
   }
 #endif
+#if defined(VK_USE_PLATFORM_WAYLAND_KHR)
+  if (wstype == WindowSystemType::Wayland &&
+      !AddExtension(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME, true))
+  {
+    return false;
+  }
+#endif
 #if defined(VK_USE_PLATFORM_ANDROID_KHR)
   if (wstype == WindowSystemType::Android &&
       !AddExtension(VK_KHR_ANDROID_SURFACE_EXTENSION_NAME, true))
diff --git a/Source/Core/VideoBackends/Vulkan/VulkanEntryPoints.inl b/Source/Core/VideoBackends/Vulkan/VulkanEntryPoints.inl
index 8c6db283c9..8035f66ddd 100644
--- a/Source/Core/VideoBackends/Vulkan/VulkanEntryPoints.inl
+++ b/Source/Core/VideoBackends/Vulkan/VulkanEntryPoints.inl
@@ -48,6 +48,11 @@ VULKAN_INSTANCE_ENTRY_POINT(vkCreateXlibSurfaceKHR, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceXlibPresentationSupportKHR, false)
 #endif
 
+#if defined(VK_USE_PLATFORM_WAYLAND_KHR)
+VULKAN_INSTANCE_ENTRY_POINT(vkCreateWaylandSurfaceKHR, false)
+VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceWaylandPresentationSupportKHR, false)
+#endif
+
 #if defined(VK_USE_PLATFORM_ANDROID_KHR)
 VULKAN_INSTANCE_ENTRY_POINT(vkCreateAndroidSurfaceKHR, false)
 #endif
diff --git a/Source/Core/VideoBackends/Vulkan/VulkanLoader.h b/Source/Core/VideoBackends/Vulkan/VulkanLoader.h
index c20ea698d3..7652fdf1c9 100644
--- a/Source/Core/VideoBackends/Vulkan/VulkanLoader.h
+++ b/Source/Core/VideoBackends/Vulkan/VulkanLoader.h
@@ -13,6 +13,10 @@
 #define VK_USE_PLATFORM_XLIB_KHR
 #endif
 
+#if defined(HAVE_WAYLAND)
+#define VK_USE_PLATFORM_WAYLAND_KHR
+#endif
+
 #if defined(ANDROID)
 #define VK_USE_PLATFORM_ANDROID_KHR
 #endif
diff --git a/Source/Core/VideoCommon/RenderBase.cpp b/Source/Core/VideoCommon/RenderBase.cpp
index e51307fa15..b6c1066417 100644
--- a/Source/Core/VideoCommon/RenderBase.cpp
+++ b/Source/Core/VideoCommon/RenderBase.cpp
@@ -732,9 +732,11 @@ void Renderer::ChangeSurface(void* new_surface_handle)
   m_surface_changed.Set();
 }
 
-void Renderer::ResizeSurface()
+void Renderer::ResizeSurface(int width, int height)
 {
   std::lock_guard<std::mutex> lock(m_swap_mutex);
+  m_new_width = width;
+  m_new_height = height;
   m_surface_resized.Set();
 }
 
diff --git a/Source/Core/VideoCommon/RenderBase.h b/Source/Core/VideoCommon/RenderBase.h
index 8824e6ff77..7daf0ef322 100644
--- a/Source/Core/VideoCommon/RenderBase.h
+++ b/Source/Core/VideoCommon/RenderBase.h
@@ -246,7 +246,8 @@ public:
   // Final surface changing
   // This is called when the surface is resized (WX) or the window changes (Android).
   void ChangeSurface(void* new_surface_handle);
-  void ResizeSurface();
+  void ResizeSurface(int width, int height);
+  void ResizeSurface() { ResizeSurface(-1, -1); }
   bool UseVertexDepthRange() const;
   void DoState(PointerWrap& p);
 
@@ -351,6 +352,10 @@ protected:
   std::mutex m_imgui_mutex;
   u64 m_imgui_last_frame_time;
 
+  // Set when we receive a window resize event. Useful on Wayland
+  int m_new_width = -1;
+  int m_new_height = -1;
+
 private:
   std::tuple<int, int> CalculateOutputDimensions(int width, int height) const;