mxvk::VK_Capture Class Reference

Vulkan OpenCV video capture source. More...

#include <mxvk/include/mxvk/mxvk_cv.hpp>

Public Member Functions
void	close ()
	Close the capture device.
bool	createImage (VkDevice device, VkPhysicalDevice physDev, VkQueue gQueue, VkCommandPool cmdPool, size_t width, size_t height, const std::string &vert, const std::string &frag)
	Allocate the VKSprite and upload the initial texture.
void	draw (int x, int y)
	Draw using the sprite's native dimensions.
void	draw (int x, int y, int width, int height)
	Draw the current frame at a specified position and size.
double	get (unsigned int option)
	Query a VideoCapture property.
VK_Sprite *	getSprite ()
	Access the backing sprite.
bool	grab ()
	Decode and discard the next capture frame without conversion.
bool	is_open () const
	Check whether the capture device is open.
bool	open (const std::string &filename)
	Open a video file.
bool	open (int id, int mode=0)
	Open a camera device.
VK_Capture &	operator= (const VK_Capture &)=delete
VK_Capture &	operator= (VK_Capture &&)=delete
bool	read ()
	Capture and upload the next video frame.
bool	read (cv::Mat &frame)
	Capture the next frame into a cv::Mat.
bool	readRgba (cv::Mat &rgba, bool flipY=false)
bool	readToModelTexture (VKAbstractModel &model, bool flipY=false)
bool	readToSprite (VK_Sprite &targetSprite)
bool	readToSprite (VK_Sprite &targetSprite, bool flipY)
bool	reload (size_t width, size_t height, const std::string &vert, const std::string &frag)
	Replace the vertex/fragment shaders.
void	resetSprite ()
	Destroy the current sprite and its Vulkan resources.
void	set (unsigned int option, double value)
	Set a VideoCapture property.
	VK_Capture ()=default
	Default constructor.
	VK_Capture (const VK_Capture &)=delete
	VK_Capture (VK_Capture &&)=delete
	~VK_Capture ()=default
	Destructor.

Detailed Description

Vulkan OpenCV video capture source.

Opens a video file or camera, decodes frames, and uploads them to a VKSprite texture for Vulkan-accelerated rendering inside a VKWindow.

Definition at line 30 of file mxvk_cv.hpp.

Constructor & Destructor Documentation

VK_Capture() [1/3]

mxvk::VK_Capture::VK_Capture ( )

default

Default constructor.

~VK_Capture()

mxvk::VK_Capture::~VK_Capture ( )

default

Destructor.

VK_Capture() [2/3]

mxvk::VK_Capture::VK_Capture ( const VK_Capture & )

delete

VK_Capture() [3/3]

mxvk::VK_Capture::VK_Capture ( VK_Capture && )

delete

Member Function Documentation

close()

void mxvk::VK_Capture::close

(

)

Close the capture device.

Definition at line 37 of file mxvk_cv.cpp.

                           {
        cap.release();
        std::cout << "mxvk_cv: Capture closed\n";
    }

createImage()

bool mxvk::VK_Capture::createImage	(	VkDevice	device,
		VkPhysicalDevice	physDev,
		VkQueue	gQueue,
		VkCommandPool	cmdPool,
		size_t	width,
		size_t	height,
		const std::string &	vert,
		const std::string &	frag )

Allocate the VKSprite and upload the initial texture.

Parameters

device	Logical Vulkan device.
physDev	Physical device.
gQueue	Graphics queue.
cmdPool	Command pool.
width	Display width.
height	Display height.
vert	Vertex shader path.
frag	Fragment shader path.

Returns

true on success.

Definition at line 46 of file mxvk_cv.cpp.

                                                                               {
        sprite = std::make_unique<VK_Sprite>(device, physDev, gQueue, cmdPool);
        sprite->createEmptySprite(static_cast<int>(width), static_cast<int>(height), vert, frag);
        sprite->enableExtendedUBO();
        sprite->rebuildPipeline();
        std::cout << std::format("mxvk_cv: Sprite created: {}x{}\n", width, height);
        return true;
    }

draw() [1/2]

void mxvk::VK_Capture::draw	(	int	x,
		int	y )

Draw using the sprite's native dimensions.

Parameters

x	Destination X.
y	Destination Y.

Definition at line 480 of file mxvk_cv.cpp.

                                      {
        if (sprite)
            sprite->drawSprite(x, y);
    }

draw() [2/2]

void mxvk::VK_Capture::draw	(	int	x,
		int	y,
		int	width,
		int	height )

Draw the current frame at a specified position and size.

Parameters

x	Width destination X.
y	Width destination Y.
width	Display width.
height	Display height.

Definition at line 475 of file mxvk_cv.cpp.

                                                             {
        if (sprite)
            sprite->drawSpriteRect(x, y, width, height);
    }

get()

double mxvk::VK_Capture::get

(

unsigned int

option

)

Query a VideoCapture property.

Parameters

option

cv::VideoCaptureProperties constant.

Returns

Current value.

Definition at line 489 of file mxvk_cv.cpp.

                                              {
        return cap.get(option);
    }

getSprite()

VK_Sprite * mxvk::VK_Capture::getSprite ( )

inline

Access the backing sprite.

Returns

Pointer to the VKSprite (may be null before createImage).

Definition at line 93 of file mxvk_cv.hpp.

{ return sprite.get(); }

grab()

bool mxvk::VK_Capture::grab

(

)

Decode and discard the next capture frame without conversion.

Returns

true when the backend advanced to another frame.

Definition at line 74 of file mxvk_cv.cpp.

                          {
        return cap.grab();
    }

is_open()

bool mxvk::VK_Capture::is_open ( ) const

inline

Check whether the capture device is open.

Returns

true if the VideoCapture is opened.

Definition at line 71 of file mxvk_cv.hpp.

{ return cap.isOpened(); }

open() [1/2]

bool mxvk::VK_Capture::open

(

const std::string &

filename

)

Open a video file.

Parameters

filename

Path to the video file.

Returns

true on success.

Definition at line 14 of file mxvk_cv.cpp.

                                                   {
        const bool ok = cap.open(filename);
        if (ok)
            std::cout << std::format("mxvk_cv: Opened file: {}\n", filename);
        else
            std::cout << std::format("mxvk_cv: Failed to open file: {}\n", filename);
        return ok;
    }

open() [2/2]

bool mxvk::VK_Capture::open	(	int	id,
		int	mode = 0 )

Open a camera device.

Parameters

id	Camera index.
mode	Backend hint (0 = auto).

Returns

true on success.

Definition at line 23 of file mxvk_cv.cpp.

                                          {
#ifdef __linux__
        if (mode == 0)
            mode = cv::CAP_V4L2;
#endif
        if (cap.open(id, mode)) {
            cap.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G'));
            std::cout << std::format("mxvk_cv: Opened device: {}\n", id);
            return true;
        }
        std::cout << std::format("mxvk_cv: Failed to open device: {}\n", id);
        return false;
    }

operator=() [1/2]

VK_Capture & mxvk::VK_Capture::operator= ( const VK_Capture & )

delete

operator=() [2/2]

VK_Capture & mxvk::VK_Capture::operator= ( VK_Capture && )

delete

read() [1/2]

bool mxvk::VK_Capture::read

(

)

Capture and upload the next video frame.

Returns

true if a frame was available.

Definition at line 460 of file mxvk_cv.cpp.

                          {
#ifdef MXVK_CUDA
        if (initializeCuda()) {
            return readCuda();
        }
#endif
        if (!cap.read(frame)) {
            return false;
        }
        cv::Mat rgba;
        cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
        sprite->updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
        return true;
    }

read() [2/2]

bool mxvk::VK_Capture::read

(

cv::Mat &

frame

)

Capture the next frame into a cv::Mat.

Parameters

frame

Output matrix.

Returns

true if a frame was available.

Definition at line 70 of file mxvk_cv.cpp.

                                      {
        return cap.read(frame);
    }

readRgba()

bool mxvk::VK_Capture::readRgba	(	cv::Mat &	rgba,
		bool	flipY = false )

Definition at line 78 of file mxvk_cv.cpp.

                                                     {
#ifdef MXVK_CUDA
        cv::Mat cpuFallbackFrame;
 
        try {
            if (initializeCuda()) {
                if (cudaMappedInput) {
                    if (!cap.read(mappedFrame) || mappedFrame.empty()) {
                        return false;
                    }
                    cpuFallbackFrame = host_mem_mat_header(mappedFrame);
                    const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
                    cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
                } else {
                    if (!cap.read(frame) || frame.empty()) {
                        return false;
                    }
                    cpuFallbackFrame = frame;
                    gpuFrame.upload(frame, cuda_stream);
                    cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
                }
 
                if (flipY) {
                    cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
                } else {
                    gpuVulkanRgba = gpuRgba;
                }
 
                if (!cudaPipelineLogged) {
                    std::cout << std::format(
                        "mxvk_cv: CUDA RGBA path active: capture -> GpuMat -> cv::cuda::cvtColor(BGR to RGBA) -> {} -> download\n",
                        flipY ? "cv::cuda::flip(Y)" : "no Y flip");
                    cudaPipelineLogged = true;
                }
 
                pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
                gpuVulkanRgba.download(pinnedRgba, cuda_stream);
                cuda_stream.waitForCompletion();
                rgba = host_mem_mat_header(pinnedRgba);
                return true;
            }
        } catch (const cv::Exception &e) {
            std::cout << std::format("mxvk_cv: CUDA RGBA path failed; falling back to CPU path: {}\n", e.what());
            cudaAvailable = false;
            if (cpuFallbackFrame.empty()) {
                return false;
            }
            cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
            if (flipY) {
                cv::flip(rgba, rgba, 0);
            }
            return true;
        }
#endif
 
        if (!cap.read(frame) || frame.empty()) {
            return false;
        }
        cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
        if (flipY) {
            cv::flip(rgba, rgba, 0);
        }
        return true;
    }

readToModelTexture()

bool mxvk::VK_Capture::readToModelTexture	(	VKAbstractModel &	model,
		bool	flipY = false )

Definition at line 302 of file mxvk_cv.cpp.

                                                                          {
#ifdef MXVK_CUDA
        if (initializeCuda()) {
            cv::Mat cpuFallbackFrame;
 
            try {
                if (cudaMappedInput) {
                    if (!cap.read(mappedFrame) || mappedFrame.empty()) {
                        return false;
                    }
                    cpuFallbackFrame = host_mem_mat_header(mappedFrame);
                    const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
                    cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
                } else {
                    if (!cap.read(frame) || frame.empty()) {
                        return false;
                    }
                    cpuFallbackFrame = frame;
                    gpuFrame.upload(frame, cuda_stream);
                    cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
                }
 
                if (flipY) {
                    cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
                } else {
                    gpuVulkanRgba = gpuRgba;
                }
 
                if (model.updatePrimaryTextureCuda(gpuVulkanRgba, cuda_stream)) {
                    if (!cudaPipelineLogged) {
                        std::cout << std::format(
                            "mxvk_cv: CUDA interop active for model texture: RGBA GpuMat -> {} -> Vulkan external-memory image array (no download)\n",
                            flipY ? "cv::cuda::flip(Y)" : "no Y flip");
                        cudaPipelineLogged = true;
                    }
                    return true;
                }
 
                if (!cudaPipelineLogged) {
                    std::cout << "mxvk_cv: CUDA interop unavailable for model texture; using pinned CUDA download -> Vulkan staging upload\n";
                    cudaPipelineLogged = true;
                }
                pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
                gpuVulkanRgba.download(pinnedRgba, cuda_stream);
                cuda_stream.waitForCompletion();
 
                pinnedRgbaMat = host_mem_mat_header(pinnedRgba);
                return model.updatePrimaryTexture(pinnedRgbaMat.ptr(), pinnedRgbaMat.cols, pinnedRgbaMat.rows,
                                                  static_cast<int>(pinnedRgbaMat.step));
            } catch (const cv::Exception &e) {
                std::cout << std::format("mxvk_cv: CUDA model-texture path failed; falling back to CPU path: {}\n", e.what());
                cudaAvailable = false;
                if (cpuFallbackFrame.empty()) {
                    return false;
                }
                cv::Mat rgba;
                cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
                if (flipY) {
                    cv::flip(rgba, rgba, 0);
                }
                return model.updatePrimaryTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
            }
        }
#endif
        cv::Mat rgba;
        if (!readRgba(rgba, flipY)) {
            return false;
        }
        return model.updatePrimaryTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
    }

readToSprite() [1/2]

bool mxvk::VK_Capture::readToSprite

(

VK_Sprite &

targetSprite

)

Definition at line 373 of file mxvk_cv.cpp.

                                                         {
#ifdef MXVK_CUDA
        if (initializeCuda()) {
            return readToSprite(targetSprite, cudaFlipYForVulkan);
        }
#endif
        return readToSprite(targetSprite, false);
    }

readToSprite() [2/2]

bool mxvk::VK_Capture::readToSprite	(	VK_Sprite &	targetSprite,
		bool	flipY )

Definition at line 382 of file mxvk_cv.cpp.

                                                                     {
#ifdef MXVK_CUDA
        if (initializeCuda()) {
            cv::Mat cpuFallbackFrame;
 
            try {
                if (cudaMappedInput) {
                    if (!cap.read(mappedFrame) || mappedFrame.empty()) {
                        return false;
                    }
                    cpuFallbackFrame = host_mem_mat_header(mappedFrame);
                    const cv::cuda::GpuMat mappedInput = host_mem_gpu_header(mappedFrame);
                    cv::cuda::cvtColor(mappedInput, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
                } else {
                    if (!cap.read(frame) || frame.empty()) {
                        return false;
                    }
                    cpuFallbackFrame = frame;
                    gpuFrame.upload(frame, cuda_stream);
                    cv::cuda::cvtColor(gpuFrame, gpuRgba, cv::COLOR_BGR2RGBA, 0, cuda_stream);
                }
 
                if (flipY) {
                    cv::cuda::flip(gpuRgba, gpuVulkanRgba, 0, cuda_stream);
                } else {
                    gpuVulkanRgba = gpuRgba;
                }
 
                if (targetSprite.updateTextureCuda(gpuVulkanRgba, cuda_stream)) {
                    if (!cudaPipelineLogged) {
                        std::cout << std::format(
                            "mxvk_cv: CUDA interop active for external sprite: RGBA GpuMat -> {} -> Vulkan external-memory image\n",
                            flipY ? "cv::cuda::flip(Y)" : "no Y flip");
                        cudaPipelineLogged = true;
                    }
                    return true;
                }
 
                if (!cudaPipelineLogged) {
                    std::cout << "mxvk_cv: CUDA interop unavailable for external sprite; using pinned CUDA download -> Vulkan staging upload\n";
                    cudaPipelineLogged = true;
                }
                pinnedRgba.create(gpuVulkanRgba.size(), gpuVulkanRgba.type());
                gpuVulkanRgba.download(pinnedRgba, cuda_stream);
                cuda_stream.waitForCompletion();
 
                pinnedRgbaMat = host_mem_mat_header(pinnedRgba);
                targetSprite.updateTexture(pinnedRgbaMat.ptr(), pinnedRgbaMat.cols, pinnedRgbaMat.rows,
                                           static_cast<int>(pinnedRgbaMat.step));
                return true;
            } catch (const cv::Exception &e) {
                std::cout << std::format("mxvk_cv: CUDA external-sprite path failed; falling back to CPU path: {}\n", e.what());
                cudaAvailable = false;
                if (cpuFallbackFrame.empty()) {
                    return false;
                }
                cv::Mat rgba;
                cv::cvtColor(cpuFallbackFrame, rgba, cv::COLOR_BGR2RGBA);
                if (flipY) {
                    cv::flip(rgba, rgba, 0);
                }
                targetSprite.updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
                return true;
            }
        }
#endif
        if (!cap.read(frame) || frame.empty()) {
            return false;
        }
        cv::Mat rgba;
        cv::cvtColor(frame, rgba, cv::COLOR_BGR2RGBA);
        if (flipY) {
            cv::flip(rgba, rgba, 0);
        }
        targetSprite.updateTexture(rgba.ptr(), rgba.cols, rgba.rows, static_cast<int>(rgba.step));
        return true;
    }

reload()

bool mxvk::VK_Capture::reload	(	size_t	width,
		size_t	height,
		const std::string &	vert,
		const std::string &	frag )

Replace the vertex/fragment shaders.

Parameters

width	Display width.
height	Display height.
vert	New vertex shader path.
frag	New fragment shader path.

Returns

true on success.

Definition at line 57 of file mxvk_cv.cpp.

                                                                                                     {
        if (sprite) {
            sprite->createEmptySprite(static_cast<int>(width), static_cast<int>(height), vert, frag);
            sprite->enableExtendedUBO();
            std::cout << std::format("mxvk_cv: Shader reloaded: vert={} frag={}\n",
                                     std::filesystem::path(vert).filename().string(),
                                     std::filesystem::path(frag).filename().string());
            return true;
        }
        std::cout << "mxvk_cv: Reload failed: no sprite\n";
        return false;
    }

resetSprite()

void mxvk::VK_Capture::resetSprite

(

)

Destroy the current sprite and its Vulkan resources.

Call this before swapchain/command-pool teardown to avoid destroying sprite staging command buffers against an invalid command pool.

Definition at line 42 of file mxvk_cv.cpp.

                                 {
        sprite.reset();
    }

set()

void mxvk::VK_Capture::set	(	unsigned int	option,
		double	value )

Set a VideoCapture property.

Parameters

option	cv::VideoCaptureProperties constant.
value	Desired value.

Definition at line 485 of file mxvk_cv.cpp.

                                                          {
        cap.set(option, value);
    }

---

The documentation for this class was generated from the following files:

• mxvk/include/mxvk/mxvk_cv.hpp
• mxvk/src/mxvk_cv.cpp