MXVK Vulkan Framework 0.35.0
C++20 Vulkan rendering framework for practical 2D and 3D application development with SDL3.
Loading...
Searching...
No Matches
mxvk_cv.cpp
Go to the documentation of this file.
1/**
2 * @file mxvk_cv.cpp
3 * @brief Implementation of mxvk::VK_Capture (Vulkan + OpenCV variant).
4 */
5#include "mxvk/mxvk_cv.hpp"
6#include "mxvk/mxvk_abstract_model.hpp"
7#include "mxvk/mxvk_opencv_compat.hpp"
8#include <cstdlib>
9#include <filesystem>
10#include <iostream>
11
12namespace mxvk {
13
14 bool VK_Capture::open(const std::string &filename) {
15 const bool ok = cap.open(filename);
16 if (ok)
17 std::cout << std::format("mxvk_cv: Opened file: {}\n", filename);
18 else
19 std::cout << std::format("mxvk_cv: Failed to open file: {}\n", filename);
20 return ok;
21 }
22
23 bool VK_Capture::open(int id, int mode) {
24#ifdef __linux__
25 if (mode == 0)
26 mode = cv::CAP_V4L2;
27#endif
28 if (cap.open(id, mode)) {
29 cap.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G'));
30 std::cout << std::format("mxvk_cv: Opened device: {}\n", id);
31 return true;
32 }
33 std::cout << std::format("mxvk_cv: Failed to open device: {}\n", id);
34 return false;
35 }
36
37 void VK_Capture::close() {
38 cap.release();
39 std::cout << "mxvk_cv: Capture closed\n";
40 }
41
42 void VK_Capture::resetSprite() { sprite.reset(); }
43
44 bool VK_Capture::createImage(VkDevice device, VkPhysicalDevice physDev, VkQueue gQueue, VkCommandPool cmdPool, size_t width, size_t height, const std::string &vert, const std::string &frag) {
45 sprite = std::make_unique<VK_Sprite>(device, physDev, gQueue, cmdPool);
46 sprite->createEmptySprite(static_cast<int>(width), static_cast<int>(height), vert, frag);
47 sprite->enableExtendedUBO();
48 sprite->rebuildPipeline();
49 std::cout << std::format("mxvk_cv: Sprite created: {}x{}\n", width, height);
50 return true;
51 }
52
53 bool VK_Capture::reload(size_t width, size_t height, const std::string &vert, const std::string &frag) {
54 if (sprite) {
55 sprite->createEmptySprite(static_cast<int>(width), static_cast<int>(height), vert, frag);
56 sprite->enableExtendedUBO();
57 std::cout << std::format("mxvk_cv: Shader reloaded: vert={} frag={}\n", std::filesystem::path(vert).filename().string(), std::filesystem::path(frag).filename().string());
58 return true;
59 }
60 std::cout << "mxvk_cv: Reload failed: no sprite\n";
61 return false;
62 }
63
64 bool VK_Capture::read(cv::Mat &frame) { return cap.read(frame); }
65
66 bool VK_Capture::grab() { return cap.grab(); }
67
68 bool VK_Capture::readRgba(cv::Mat &rgba, bool flipY) {
69#ifdef MXVK_CUDA
70 cv::Mat cpuFallbackFrame;
71
72 try {
73 if (initializeCuda()) {
74 if (cudaMappedInput) {
75 if (!cap.read(mappedFrame) || mappedFrame.empty()) {
76 return false;
77 }
78 cpuFallbackFrame = host_mem_mat_header(mappedFrame);
79 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
80 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
81 } else {
82 if (!cap.read(frame) || frame.empty()) {
83 return false;
84 }
85 cpuFallbackFrame = frame;
86 gpuFrame.upload(frame, cuda_stream);
87 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
88 }
89
90 if (flipY) {
91 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
92 } else {
93 gpuVulkanRgba = gpuRgba;
94 }
95
96 if (!cudaPipelineLogged) {
97 std::cout << std::format("mxvk_cv: CUDA RGBA path active: capture -> GpuMat -> cv::cuda::cvtColor(BGR to RGBA) -> {} -> download\n", flipY ? "cv::cuda::flip(Y)" : "no Y flip");
98 cudaPipelineLogged = true;
99 }
100
101 pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
102 gpuVulkanRgba.download(pinnedRgba, cuda_stream);
103 cuda_stream.waitForCompletion();
104 rgba = host_mem_mat_header(pinnedRgba);
105 return true;
106 }
107 } catch (const cv::Exception &e) {
108 std::cout << std::format("mxvk_cv: CUDA RGBA path failed; falling back to CPU path: {}\n", e.what());
109 cudaAvailable = false;
110 if (cpuFallbackFrame.empty()) {
111 return false;
112 }
113 cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
114 if (flipY) {
115 cv::flip(rgba, rgba, 0);
116 }
117 return true;
118 }
119#endif
120
121 if (!cap.read(frame) || frame.empty()) {
122 return false;
123 }
124 cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
125 if (flipY) {
126 cv::flip(rgba, rgba, 0);
127 }
128 return true;
129 }
130
131#ifdef MXVK_CUDA
132 bool VK_Capture::initializeCuda() {
133 if (cudaChecked) {
134 return cudaAvailable;
135 }
136
137 cudaChecked = true;
138 std::cout << "mxvk_cv: CUDA init: MXVK_CUDA compile definition is enabled\n";
139 try {
140 const int deviceCount = cv::cuda::getCudaEnabledDeviceCount();
141 std::cout << std::format("mxvk_cv: CUDA init: OpenCV reports {} CUDA-enabled device(s)\n", deviceCount);
142 if (deviceCount <= 0) {
143 std::cout << "mxvk_cv: CUDA path unavailable: no CUDA-enabled OpenCV device\n";
144 return false;
145 }
146
147 cv::cuda::setDevice(0);
148 const cv::cuda::DeviceInfo deviceInfo(0);
149 cudaMappedInput = deviceInfo.canMapHostMemory();
150 if (const char *flipEnv = std::getenv("MXVK_CUDA_FLIP_Y")) {
151 cudaFlipYForVulkan = std::string(flipEnv) != "0";
152 }
153 cudaAvailable = true;
154 std::cout << std::format("mxvk_cv: CUDA init: selected device 0: {}\n", deviceInfo.name());
155 std::cout << std::format("mxvk_cv: CUDA init: compute capability {}.{}\n", deviceInfo.majorVersion(), deviceInfo.minorVersion());
156 std::cout << std::format("mxvk_cv: CUDA init: unified addressing={}, host memory mapping={}\n", deviceInfo.unifiedAddressing() ? "true" : "false", cudaMappedInput ? "true" : "false");
157 std::cout << std::format("mxvk_cv: CUDA init: capture input mode={}\n", cudaMappedInput ? "mapped HostMem::SHARED GpuMat header" : "cv::Mat upload to GpuMat");
158 std::cout << std::format("mxvk_cv: CUDA init: Vulkan upload Y flip={} (default on for CUDA external-memory textures; override with MXVK_CUDA_FLIP_Y=0 or 1)\n", cudaFlipYForVulkan ? "enabled" : "disabled");
159 std::cout << std::format("mxvk_cv: CUDA path enabled on {} ({})\n", deviceInfo.name(), cudaMappedInput ? "mapped zero-copy input" : "pinned async input upload");
160 std::cout << "mxvk_cv: CUDA pipeline: capture -> GpuMat -> cv::cuda::cvtColor(BGR to RGBA) -> direct Vulkan texture when available\n";
161 } catch (const cv::Exception &e) {
162 std::cout << std::format("mxvk_cv: CUDA path unavailable: {}\n", e.what());
163 cudaAvailable = false;
164 }
165
166 return cudaAvailable;
167 }
168
169 bool VK_Capture::readCuda() {
170 cv::Mat cpuFallbackFrame;
171
172 try {
173 if (cudaMappedInput) {
174 if (!cap.read(mappedFrame)) {
175 return false;
176 }
177 if (mappedFrame.empty()) {
178 return false;
179 }
180
181 cpuFallbackFrame = host_mem_mat_header(mappedFrame);
182 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
183 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
184 } else {
185 if (!cap.read(frame)) {
186 return false;
187 }
188 if (frame.empty()) {
189 return false;
190 }
191
192 cpuFallbackFrame = frame;
193 gpuFrame.upload(frame, cuda_stream);
194 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
195 }
196
197 if (cudaFlipYForVulkan) {
198 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
199 } else {
200 gpuVulkanRgba = gpuRgba;
201 }
202
203 if (sprite && sprite->updateTextureCuda(gpuVulkanRgba, cuda_stream)) {
204 if (!cudaPipelineLogged) {
205 std::cout << std::format("mxvk_cv: CUDA interop active: RGBA GpuMat -> {} -> Vulkan external-memory image\n", cudaFlipYForVulkan ? "cv::cuda::flip(Y)" : "no Y flip");
206 cudaPipelineLogged = true;
207 }
208 return true;
209 }
210
211 if (!cudaPipelineLogged) {
212 std::cout << "mxvk_cv: CUDA interop unavailable for this sprite; using pinned CUDA download -> Vulkan staging upload\n";
213 cudaPipelineLogged = true;
214 }
215 pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
216 gpuVulkanRgba.download(pinnedRgba, cuda_stream);
217 cuda_stream.waitForCompletion();
218
219 pinnedRgbaMat = host_mem_mat_header(pinnedRgba);
220 sprite->updateTexture(pinnedRgbaMat.ptr(), pinnedRgbaMat.cols, pinnedRgbaMat.rows, static_cast<int>(pinnedRgbaMat.step));
221 return true;
222 } catch (const cv::Exception &e) {
223 std::cout << std::format("mxvk_cv: CUDA frame path failed; falling back to CPU path: {}\n", e.what());
224 cudaAvailable = false;
225 }
226
227 if (cpuFallbackFrame.empty()) {
228 return false;
229 }
230
231 cv::Mat rgba;
232 cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
233 sprite->updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
234 return true;
235 }
236
237 bool VK_Capture::readGpuRgba(cv::cuda::GpuMat &rgba, bool flipY) {
238 if (!initializeCuda()) {
239 return false;
240 }
241
242 try {
243 if (cudaMappedInput) {
244 if (!cap.read(mappedFrame) || mappedFrame.empty()) {
245 return false;
246 }
247 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
248 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
249 } else {
250 if (!cap.read(frame) || frame.empty()) {
251 return false;
252 }
253 gpuFrame.upload(frame, cuda_stream);
254 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
255 }
256
257 if (flipY) {
258 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
259 } else {
260 gpuVulkanRgba = gpuRgba;
261 }
262
263 if (!cudaPipelineLogged) {
264 std::cout << std::format("mxvk_cv: CUDA GpuMat path active: capture -> GpuMat -> cv::cuda::cvtColor(BGR to RGBA) -> {} -> resident GpuMat for caller\n", flipY ? "cv::cuda::flip(Y)" : "no Y flip");
265 cudaPipelineLogged = true;
266 }
267
268 rgba = gpuVulkanRgba;
269 return true;
270 } catch (const cv::Exception &e) {
271 std::cout << std::format("mxvk_cv: CUDA GpuMat path failed: {}\n", e.what());
272 cudaAvailable = false;
273 return false;
274 }
275 }
276#endif
277
278 bool VK_Capture::readToModelTexture(VKAbstractModel &model, bool flipY) {
279#ifdef MXVK_CUDA
280 if (initializeCuda()) {
281 cv::Mat cpuFallbackFrame;
282
283 try {
284 if (cudaMappedInput) {
285 if (!cap.read(mappedFrame) || mappedFrame.empty()) {
286 return false;
287 }
288 cpuFallbackFrame = host_mem_mat_header(mappedFrame);
289 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
290 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
291 } else {
292 if (!cap.read(frame) || frame.empty()) {
293 return false;
294 }
295 cpuFallbackFrame = frame;
296 gpuFrame.upload(frame, cuda_stream);
297 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
298 }
299
300 if (flipY) {
301 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
302 } else {
303 gpuVulkanRgba = gpuRgba;
304 }
305
306 if (model.updatePrimaryTextureCuda(gpuVulkanRgba, cuda_stream)) {
307 if (!cudaPipelineLogged) {
308 std::cout << std::format("mxvk_cv: CUDA interop active for model texture: RGBA GpuMat -> {} -> Vulkan external-memory image array (no download)\n", flipY ? "cv::cuda::flip(Y)" : "no Y flip");
309 cudaPipelineLogged = true;
310 }
311 return true;
312 }
313
314 if (!cudaPipelineLogged) {
315 std::cout << "mxvk_cv: CUDA interop unavailable for model texture; using pinned CUDA download -> Vulkan staging upload\n";
316 cudaPipelineLogged = true;
317 }
318 pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
319 gpuVulkanRgba.download(pinnedRgba, cuda_stream);
320 cuda_stream.waitForCompletion();
321
322 pinnedRgbaMat = host_mem_mat_header(pinnedRgba);
323 return model.updatePrimaryTexture(pinnedRgbaMat.ptr(), pinnedRgbaMat.cols, pinnedRgbaMat.rows, static_cast<int>(pinnedRgbaMat.step));
324 } catch (const cv::Exception &e) {
325 std::cout << std::format("mxvk_cv: CUDA model-texture path failed; falling back to CPU path: {}\n", e.what());
326 cudaAvailable = false;
327 if (cpuFallbackFrame.empty()) {
328 return false;
329 }
330 cv::Mat rgba;
331 cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
332 if (flipY) {
333 cv::flip(rgba, rgba, 0);
334 }
335 return model.updatePrimaryTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
336 }
337 }
338#endif
339 cv::Mat rgba;
340 if (!readRgba(rgba, flipY)) {
341 return false;
342 }
343 return model.updatePrimaryTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
344 }
345
346 bool VK_Capture::readToSprite(VK_Sprite &targetSprite) {
347#ifdef MXVK_CUDA
348 if (initializeCuda()) {
349 return readToSprite(targetSprite, cudaFlipYForVulkan);
350 }
351#endif
352 return readToSprite(targetSprite, false);
353 }
354
355 bool VK_Capture::readToSprite(VK_Sprite &targetSprite, bool flipY) {
356#ifdef MXVK_CUDA
357 if (initializeCuda()) {
358 cv::Mat cpuFallbackFrame;
359
360 try {
361 if (cudaMappedInput) {
362 if (!cap.read(mappedFrame) || mappedFrame.empty()) {
363 return false;
364 }
365 cpuFallbackFrame = host_mem_mat_header(mappedFrame);
366 const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
367 cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
368 } else {
369 if (!cap.read(frame) || frame.empty()) {
370 return false;
371 }
372 cpuFallbackFrame = frame;
373 gpuFrame.upload(frame, cuda_stream);
374 cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
375 }
376
377 if (flipY) {
378 cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
379 } else {
380 gpuVulkanRgba = gpuRgba;
381 }
382
383 if (targetSprite.updateTextureCuda(gpuVulkanRgba, cuda_stream)) {
384 if (!cudaPipelineLogged) {
385 std::cout << std::format("mxvk_cv: CUDA interop active for external sprite: RGBA GpuMat -> {} -> Vulkan external-memory image\n", flipY ? "cv::cuda::flip(Y)" : "no Y flip");
386 cudaPipelineLogged = true;
387 }
388 return true;
389 }
390
391 if (!cudaPipelineLogged) {
392 std::cout << "mxvk_cv: CUDA interop unavailable for external sprite; using pinned CUDA download -> Vulkan staging upload\n";
393 cudaPipelineLogged = true;
394 }
395 pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
396 gpuVulkanRgba.download(pinnedRgba, cuda_stream);
397 cuda_stream.waitForCompletion();
398
399 pinnedRgbaMat = host_mem_mat_header(pinnedRgba);
400 targetSprite.updateTexture(pinnedRgbaMat.ptr(), pinnedRgbaMat.cols, pinnedRgbaMat.rows, static_cast<int>(pinnedRgbaMat.step));
401 return true;
402 } catch (const cv::Exception &e) {
403 std::cout << std::format("mxvk_cv: CUDA external-sprite path failed; falling back to CPU path: {}\n", e.what());
404 cudaAvailable = false;
405 if (cpuFallbackFrame.empty()) {
406 return false;
407 }
408 cv::Mat rgba;
409 cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
410 if (flipY) {
411 cv::flip(rgba, rgba, 0);
412 }
413 targetSprite.updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
414 return true;
415 }
416 }
417#endif
418 if (!cap.read(frame) || frame.empty()) {
419 return false;
420 }
421 cv::Mat rgba;
422 cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
423 if (flipY) {
424 cv::flip(rgba, rgba, 0);
425 }
426 targetSprite.updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
427 return true;
428 }
429
430 bool VK_Capture::read() {
431#ifdef MXVK_CUDA
432 if (initializeCuda()) {
433 return readCuda();
434 }
435#endif
436 if (!cap.read(frame)) {
437 return false;
438 }
439 cv::Mat rgba;
440 cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
441 sprite->updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
442 return true;
443 }
444
445 void VK_Capture::draw(int x, int y, int width, int height) {
446 if (sprite)
447 sprite->drawSpriteRect(x, y, width, height);
448 }
449
450 void VK_Capture::draw(int x, int y) {
451 if (sprite)
452 sprite->drawSprite(x, y);
453 }
454
455 void VK_Capture::set(unsigned int option, double value) { cap.set(option, value); }
456
457 double VK_Capture::get(unsigned int option) { return cap.get(option); }
458
459} // namespace mxvk
mxvk::VKAbstractModel
Convenience wrapper that owns mesh, textures, descriptors, and pipeline state.
Definition mxvk_abstract_model.hpp:71
mxvk::VK_Capture::readToModelTexture
bool readToModelTexture(VKAbstractModel &model, bool flipY=false)
Definition mxvk_cv.cpp:278
mxvk::VK_Capture::open
bool open(const std::string &filename)
Open a video file.
Definition mxvk_cv.cpp:14
mxvk::VK_Capture::reload
bool reload(size_t width, size_t height, const std::string &vert, const std::string &frag)
Replace the vertex/fragment shaders.
Definition mxvk_cv.cpp:53
mxvk::VK_Capture::resetSprite
void resetSprite()
Destroy the current sprite and its Vulkan resources.
Definition mxvk_cv.cpp:42
mxvk::VK_Capture::grab
bool grab()
Decode and discard the next capture frame without conversion.
Definition mxvk_cv.cpp:66
mxvk::VK_Capture::close
void close()
Close the capture device.
Definition mxvk_cv.cpp:37
mxvk::VK_Capture::get
double get(unsigned int option)
Query a VideoCapture property.
Definition mxvk_cv.cpp:457
mxvk::VK_Capture::readToSprite
bool readToSprite(VK_Sprite &targetSprite)
Definition mxvk_cv.cpp:346
mxvk::VK_Capture::readRgba
bool readRgba(cv::Mat &rgba, bool flipY=false)
Definition mxvk_cv.cpp:68
mxvk::VK_Capture::createImage
bool createImage(VkDevice device, VkPhysicalDevice physDev, VkQueue gQueue, VkCommandPool cmdPool, size_t width, size_t height, const std::string &vert, const std::string &frag)
Allocate the VKSprite and upload the initial texture.
Definition mxvk_cv.cpp:44
mxvk::VK_Capture::draw
void draw(int x, int y, int width, int height)
Draw the current frame at a specified position and size.
Definition mxvk_cv.cpp:445
mxvk::VK_Capture::read
bool read()
Capture and upload the next video frame.
Definition mxvk_cv.cpp:430
mxvk::VK_Capture::set
void set(unsigned int option, double value)
Set a VideoCapture property.
Definition mxvk_cv.cpp:455
mxvk::VK_Sprite
Definition mxvk_sprite.hpp:56
mxvk::VK_Sprite::updateTexture
void updateTexture(SDL_Surface *surface)
Replace the sprite texture from an SDL_Surface.
Definition mxvk_sprite.cpp:1740
mxvk_abstract_model.hpp
High-level model wrapper integrated with MXVK dynamic rendering.
mxvk_cv.hpp
OpenCV video-capture integration for the Vulkan backend.
mxvk_opencv_compat.hpp
Small compatibility wrappers around OpenCV CUDA APIs.
model
Definition model.py:1
mxvk
Utilities for loading and saving PNG images.
Definition mxvk.hpp:31