MXVK Vulkan Framework 0.33.1
C++20 Vulkan rendering framework for practical 2D and 3D application development with SDL3.
Loading...
Searching...
No Matches
main.cpp
Go to the documentation of this file.
1#include "mxvk/argz.hpp"
2#include "mxvk/mxvk.hpp"
3#include "mxvk/mxvk_cv.hpp"
6#if defined(MXVK_WITH_FFMPEG_CAPTURE)
8#endif
10#if defined(MXWRITE_ENABLED)
11#include "mxwrite.hpp"
12#endif
13#include <algorithm>
14#include <array>
15#include <cmath>
16#include <cstdint>
17#include <cstdlib>
18#include <cstring>
19#include <ctime>
20#include <fstream>
21#include <iostream>
22#include <opencv2/opencv.hpp>
23#include <string>
24#include <string_view>
25#include <thread>
26#include <vector>
27#ifdef MXVK_CUDA
28#include <cuda_runtime_api.h>
29#include <opencv2/core/cuda.hpp>
30#include <opencv2/cudaimgproc.hpp>
31#include <opencv2/cudawarping.hpp>
32#include <unistd.h>
33#endif
34
35#ifndef compute_shader_ASSET_DIR
36#define compute_shader_ASSET_DIR "."
37#endif
38
39static constexpr int HISTORY_SIZE = 8;
40static constexpr const char *MODE_SHADER_NAME = "acidcam_filters.spv";
41static constexpr std::array<std::string_view, 50> ACIDCAM_FILTER_MODE_NAMES = {
42 "Block Pixelate",
43 "Block Mirror X",
44 "Block Mirror Y",
45 "Combine Pixels",
46 "History XOR",
47 "Temporal Blend",
48 "Scanline Warp",
49 "RGB Split",
50 "Horizontal Mirror",
51 "Vertical Mirror",
52 "Kaleidoscope",
53 "Dynamic Kaleidoscope",
54 "Negate",
55 "Posterize",
56 "Threshold",
57 "Gamma Darken",
58 "Brightness Contrast",
59 "Sepia",
60 "Solarize",
61 "Hue Rotate",
62 "Saturate",
63 "Desaturate",
64 "Box Blur",
65 "Sharpen",
66 "Emboss",
67 "Sobel",
68 "Edge Detect",
69 "Dilate",
70 "Erode",
71 "Posterize Scale",
72 "Wave",
73 "Ripple",
74 "Twirl",
75 "Zoom Pulse",
76 "Crosshatch",
77 "Noise Grain",
78 "Strobe Bars",
79 "Scanline XOR",
80 "Block Shuffle",
81 "Diagonal Slice",
82 "Frame Blend",
83 "Trail Blend",
84 "History Median",
85 "Row Blend",
86 "Column Blend",
87 "Color Cycle",
88 "Gradient Ramp",
89 "Flash Invert",
90 "XOR Grid",
91 "Mirror Trail",
92};
93
94struct ComputePC {
95 int32_t mode;
96 int32_t historyCount;
97 int32_t historyIdx;
98 int32_t square_size;
99 int32_t history_dir;
100 float alpha;
101 int32_t do_swap;
102 int32_t do_invert;
103};
104
106 public:
107 explicit ComputeWindow(const Arguments &args)
108 : mxvk::VK_Window("-[ VK Compute CV ]-", args.width, args.height, args.fullscreen, MXVK_VALIDATION, args.enable_vsync),
109 assetRoot(args.path.empty() ? std::string(compute_shader_ASSET_DIR) : args.path),
110 inputFilename(args.filename),
111 usingFile(!inputFilename.empty()),
112 fastMode(args.fast),
113 explicitResolution(args.resolutionSpecified),
114 fullscreenMode(args.fullscreen),
115 outputFilename(args.output),
116 outputCrf(args.crf),
117 encodePreset(args.encodePreset),
118 encodeTune(args.encodeTune),
119 encodeCodec(args.encodeCodec),
120 encodeRealtime(args.encodeRealtime),
121 mxwriteBlockWhenFull(args.mxwriteBlockWhenFull),
122 repeat(args.repeat),
123 cameraIndex(args.camera_index),
124 requestedShaderIndex(args.shader_index),
125 initialShaderMode(
126 args.index > 0
127 ? std::clamp(args.index - 1, 0, static_cast<int>(ACIDCAM_FILTER_MODE_NAMES.size()) - 1)
128 : 0) {
129 recordWidth = args.width;
130 recordHeight = args.height;
131 shaderMode = initialShaderMode;
132 initComputeResources();
133 }
134
135 ~ComputeWindow() override {
136 capture.close();
137#if defined(MXVK_WITH_FFMPEG_CAPTURE)
138 ffCapture.close();
139#endif
140 destroyComputeResources();
141 }
142
143 void proc() override {
144 bool frameUploaded = false;
145 if (usingFile && !fastMode) {
146 throttleVideoPlayback();
147 }
148
149#if defined(MXVK_WITH_FFMPEG_CAPTURE)
150 if (usingFfCapture) {
151 frameUploaded = readFfFrameToCompute();
152 if (!frameUploaded && usingFile) {
153 if (repeat) {
154 restartFfCapture();
155 frameUploaded = readFfFrameToCompute();
156 } else {
157 std::cout << "compute_shader: video file reached EOF, shutting down\n";
158 exit();
159 return;
160 }
161 }
162 } else
163#endif
164 {
165 cv::Mat frame;
166#ifdef MXVK_CUDA
167 cv::cuda::GpuMat gpuFrame;
168 if (capture.readGpuRgba(gpuFrame) && !gpuFrame.empty()) {
169 frameUploaded = uploadGpuFrameToCompute(gpuFrame, capture.cudaStream());
170 if (frameUploaded && !captureUploadPathLogged) {
171 std::cout << "compute_shader: CUDA interop capture path active: capture -> GpuMat -> optional CUDA resize -> cudaMemcpy2DToArrayAsync -> Vulkan compute storage image (no download)\n";
172 captureUploadPathLogged = true;
173 }
174 }
175#endif
176 if (!frameUploaded && capture.readRgba(frame) && !frame.empty()) {
177 if (!captureUploadPathLogged) {
178#ifdef MXVK_CUDA
179 std::cout << "compute_shader: CUDA interop unavailable; fallback path active: CUDA/CPU RGBA -> optional CPU resize -> Vulkan staging upload\n";
180#else
181 std::cout << "compute_shader: CPU capture path active: readRgba converts to RGBA, optional CPU resize, then uploads through Vulkan staging\n";
182#endif
183 captureUploadPathLogged = true;
184 }
185 uploadCpuFrameToCompute(frame);
186 frameUploaded = true;
187 } else if (!frameUploaded && usingFile) {
188 if (repeat) {
189 capture.close();
190 if (capture.open(inputFilename)) {
191 configureVideoPlaybackRate();
192 cv::Mat restartedFrame;
193 if (capture.readRgba(restartedFrame) && !restartedFrame.empty()) {
194 if (!captureUploadPathLogged) {
195#ifdef MXVK_CUDA
196 std::cout << "compute_shader: CUDA interop unavailable; fallback path active: CUDA/CPU RGBA -> optional CPU resize -> Vulkan staging upload\n";
197#else
198 std::cout << "compute_shader: CPU capture path active: readRgba converts to RGBA, optional CPU resize, then uploads through Vulkan staging\n";
199#endif
200 captureUploadPathLogged = true;
201 }
202 uploadCpuFrameToCompute(restartedFrame);
203 frameUploaded = true;
204 }
205 }
206 } else {
207 std::cout << "compute_shader: video file reached EOF, shutting down\n";
208 exit();
209 return;
210 }
211 }
212 }
213
214 if (frameUploaded) {
215 tickAnimState();
216 runComputeFrame();
217 recordProcessedFrame();
218 }
219 updateFpsOverlay(frameUploaded);
220 }
221
222 void onSwapchainRecreated() override {
223 rebuildDisplayPipeline();
224 }
225
226 void onRecordCustomRendering(VkCommandBuffer cmd, [[maybe_unused]] uint32_t imageIndex) override {
227 renderComputeOutput(cmd);
228 }
229
230 void event(SDL_Event &e) override {
231 if (e.type == SDL_EVENT_QUIT ||
232 (e.type == SDL_EVENT_KEY_DOWN && e.key.key == SDLK_ESCAPE)) {
233 exit();
234 return;
235 }
236
237 if (e.type == SDL_EVENT_KEY_DOWN && !spvFiles.empty()) {
238 if (e.key.key == SDLK_UP) {
239 currentSpvIndex =
240 (currentSpvIndex - 1 + static_cast<int>(spvFiles.size())) % static_cast<int>(spvFiles.size());
241 std::cout << "Current index: " << spvFiles[currentSpvIndex] << "\n";
242 reloadPipeline();
243 } else if (e.key.key == SDLK_DOWN) {
244 currentSpvIndex = (currentSpvIndex + 1) % static_cast<int>(spvFiles.size());
245 std::cout << "Current index: " << spvFiles[currentSpvIndex] << "\n";
246 reloadPipeline();
247 } else if (spvFiles[currentSpvIndex] == MODE_SHADER_NAME &&
248 (e.key.key == SDLK_LEFT || e.key.key == SDLK_RIGHT)) {
249 const int delta = (e.key.key == SDLK_LEFT) ? -1 : 1;
250 shaderMode = (shaderMode + delta + 50) % 50;
251 std::cout << "Mode shader mode: " << shaderMode << "\n";
252 }
253 }
254 }
255
256 private:
257 struct ComputeImage {
258 VkImage image = VK_NULL_HANDLE;
259 VkDeviceMemory memory = VK_NULL_HANDLE;
260 VkImageView view = VK_NULL_HANDLE;
261#ifdef MXVK_CUDA
262 VkDeviceSize cudaExportMemorySize = 0;
263 cudaExternalMemory_t cudaExternalMemory = nullptr;
264 cudaMipmappedArray_t cudaMipmappedArray = nullptr;
265 cudaArray_t cudaArray = nullptr;
266 bool cudaInteropEnabled = false;
267 bool cudaInteropUnavailableLogged = false;
268 bool cudaUploadLogged = false;
269 bool cudaBarrierLogged = false;
270#endif
271 };
272
273 std::string assetRoot;
274 mxvk::VK_Capture capture{};
275#if defined(MXVK_WITH_FFMPEG_CAPTURE)
276 mxvk::VK_FF_Capture ffCapture{};
277 bool usingFfCapture = false;
278 std::vector<uint8_t> ffFrameRgba{};
279 int ffFramePitch = 0;
280#ifdef MXVK_CUDA
281 cv::cuda::Stream ffCudaStream{};
282#endif
283#endif
284 std::string inputFilename;
285 bool usingFile = false;
286 bool fastMode = false;
287 bool explicitResolution = false;
288 bool fullscreenMode = false;
289 int recordWidth = 1920;
290 int recordHeight = 1080;
291 int sourceWidth = 1920;
292 int sourceHeight = 1080;
293 std::string outputFilename;
294 std::string outputCrf;
295 std::string encodePreset;
296 std::string encodeTune;
297 std::string encodeCodec;
298 bool encodeRealtime = false;
299 bool mxwriteBlockWhenFull = false;
300 bool repeat = false;
301 mxvk::Font fpsFont{};
302 int cameraIndex = 0;
303 int texWidth = 1920;
304 int texHeight = 1080;
305
306 std::array<ComputeImage, 2> workImg{};
307 std::array<ComputeImage, HISTORY_SIZE> histImg{};
308 ComputeImage outImg{};
309
310 VkSampler computeSampler = VK_NULL_HANDLE;
311
312 VkBuffer stagingBuf = VK_NULL_HANDLE;
313 VkDeviceMemory stagingMem = VK_NULL_HANDLE;
314 VkBuffer readbackBuf = VK_NULL_HANDLE;
315 VkDeviceMemory readbackMem = VK_NULL_HANDLE;
316
317 VkDescriptorSetLayout compDSLayout = VK_NULL_HANDLE;
318 VkPipelineLayout compPipeLayout = VK_NULL_HANDLE;
319 VkPipeline compPipeline = VK_NULL_HANDLE;
320 VkDescriptorPool compDSPool = VK_NULL_HANDLE;
321
322 std::array<VkDescriptorSet, 2> blurDS{};
323 std::array<VkDescriptorSet, 2> blendDS{};
324
325 VkDescriptorSetLayout displayDSLayout = VK_NULL_HANDLE;
326 VkDescriptorPool displayDSPool = VK_NULL_HANDLE;
327 VkDescriptorSet displayDS = VK_NULL_HANDLE;
328 VkPipelineLayout displayPipeLayout = VK_NULL_HANDLE;
329 VkPipeline displayPipeline = VK_NULL_HANDLE;
330 VkBuffer displayVertexBuffer = VK_NULL_HANDLE;
331 VkDeviceMemory displayVertexMemory = VK_NULL_HANDLE;
332 VkBuffer displayIndexBuffer = VK_NULL_HANDLE;
333 VkDeviceMemory displayIndexMemory = VK_NULL_HANDLE;
334
335 int historyIndex = 0;
336 int historyCount = 0;
337 int currentSquare = 4;
338 int squareDir = 1;
339 int currentHistIdx = 0;
340 int currentDir = 1;
341 int requestedShaderIndex = 0;
342 int shaderMode = 0;
343 int initialShaderMode = 0;
344 float alpha = 1.0f;
345 bool captureUploadPathLogged = false;
346 double videoFps = 0.0;
347 double sourceFps = 0.0;
348 std::chrono::duration<double> videoFrameInterval{0.0};
349 std::chrono::steady_clock::time_point nextVideoFrameDeadline{std::chrono::steady_clock::now()};
350 double currentFps = 0.0;
351 uint32_t fpsFrameCount = 0;
352 std::chrono::steady_clock::time_point fpsSampleTime{std::chrono::steady_clock::now()};
353 std::chrono::steady_clock::time_point playbackStartTime{std::chrono::steady_clock::now()};
354 std::string fpsText = "FPS: --";
355 uint64_t processedVideoFrames = 0;
356 bool recordingEnabled = false;
357 [[maybe_unused]] bool recordingWarningLogged = false;
358 bool processedRecordPathLogged = false;
359 std::vector<uint8_t> recordScratch{};
360#ifdef MXVK_CUDA
361 cv::cuda::Stream processedRecordStream{};
362 cv::cuda::GpuMat processedRecordGpuFrame{};
363 cv::cuda::GpuMat computeInputGpuFrame{};
364#endif
365#if defined(MXWRITE_ENABLED)
366 Writer videoWriter{};
367 bool videoWriterOpen = false;
368#endif
369
370 std::vector<std::string> spvFiles{};
371 int currentSpvIndex = 0;
372
373 void loadSPV() {
374 std::ifstream file(assetRoot + "/data/index.txt");
375 if (!file.is_open()) {
376 throw mxvk::Exception("Cannot open: " + assetRoot + "/data/index.txt");
377 }
378
379 std::string line;
380 while (std::getline(file, line)) {
381 if (!line.empty()) {
382 spvFiles.push_back(line);
383 }
384 }
385
386 if (spvFiles.empty()) {
387 throw mxvk::Exception("index.txt contains no entries");
388 }
389
390 currentSpvIndex =
391 std::clamp(requestedShaderIndex, 0, static_cast<int>(spvFiles.size()) - 1);
392 }
393
394 [[nodiscard]] double configureCameraFps() {
395 static constexpr std::array<double, 3> fpsChoices = {60.0, 30.0, 24.0};
396
397 for (const double requestedFps : fpsChoices) {
398 capture.set(cv::CAP_PROP_FPS, requestedFps);
399 const double reportedFps = capture.get(cv::CAP_PROP_FPS);
400 if (reportedFps > 0.0 && reportedFps + 0.5 >= requestedFps) {
401 return reportedFps;
402 }
403 }
404
405 capture.set(cv::CAP_PROP_FPS, fpsChoices.back());
406 const double reportedFps = capture.get(cv::CAP_PROP_FPS);
407 return (reportedFps > 0.0) ? reportedFps : fpsChoices.back();
408 }
409
410 void resetVideoPlaybackClock() {
411 nextVideoFrameDeadline = std::chrono::steady_clock::now();
412 }
413
414 void configureVideoPlaybackRate() {
415 double reportedFps = 0.0;
416#if defined(MXVK_WITH_FFMPEG_CAPTURE)
417 if (usingFfCapture) {
418 reportedFps = ffCapture.fps();
419 } else
420#endif
421 {
422 reportedFps = capture.get(cv::CAP_PROP_FPS);
423 }
424 videoFps = (reportedFps > 0.0) ? reportedFps : 30.0;
425 sourceFps = videoFps;
426 if (videoFps <= 0.0) {
427 videoFps = 30.0;
428 sourceFps = videoFps;
429 }
430 videoFrameInterval = std::chrono::duration<double>(1.0 / videoFps);
431 resetVideoPlaybackClock();
432 std::cout << "compute_shader: video file FPS " << videoFps
433 << " fps, fast=" << (fastMode ? "true" : "false") << "\n";
434 }
435
436 bool openVideoSource() {
437#if defined(MXVK_WITH_FFMPEG_CAPTURE)
438 usingFfCapture = false;
439 if (ffCapture.open(inputFilename)) {
440 usingFfCapture = true;
441 std::cout << "compute_shader: FFmpeg capture path active for file input"
442 << (ffCapture.using_hardware_decode() ? " (CUDA decode)\n" : " (software decode)\n");
443 return true;
444 }
445 std::cout << "compute_shader: FFmpeg capture failed; falling back to VK_Capture/OpenCV file input\n";
446#endif
447 return capture.open(inputFilename);
448 }
449
450 void uploadCpuFrameToCompute(const cv::Mat &frame) {
451 if (frame.empty()) {
452 return;
453 }
454 if (frame.cols == texWidth && frame.rows == texHeight) {
455 uploadToImage(frame.ptr(), static_cast<int>(frame.step), workImg[0]);
456 return;
457 }
458
459 cv::Mat resizedFrame;
460 cv::resize(frame, resizedFrame, cv::Size(texWidth, texHeight), 0.0, 0.0, cv::INTER_LINEAR);
461 uploadToImage(resizedFrame.ptr(), static_cast<int>(resizedFrame.step), workImg[0]);
462 }
463
464#ifdef MXVK_CUDA
465 bool uploadGpuFrameToCompute(const cv::cuda::GpuMat &gpuFrame, cv::cuda::Stream &stream) {
466 if (gpuFrame.empty()) {
467 return false;
468 }
469 if (gpuFrame.cols == texWidth && gpuFrame.rows == texHeight) {
470 return uploadGpuToImage(gpuFrame, stream, workImg[0]);
471 }
472
473 cv::cuda::resize(gpuFrame, computeInputGpuFrame, cv::Size(texWidth, texHeight), 0.0, 0.0, cv::INTER_LINEAR, stream);
474 stream.waitForCompletion();
475 return uploadGpuToImage(computeInputGpuFrame, stream, workImg[0]);
476 }
477#endif
478
479#if defined(MXVK_WITH_FFMPEG_CAPTURE)
480 void restartFfCapture() {
481 ffCapture.close();
482 if (ffCapture.open(inputFilename)) {
483 usingFfCapture = true;
484 configureVideoPlaybackRate();
485 }
486 }
487
488 bool readFfFrameToCompute() {
489#ifdef MXVK_CUDA
490 if (ffCapture.using_hardware_decode()) {
491 cv::cuda::GpuMat gpuFrame;
492 if (ffCapture.readGpuRgba(gpuFrame, ffCudaStream) && !gpuFrame.empty()) {
493 const bool uploadedWithInterop = uploadGpuFrameToCompute(gpuFrame, ffCudaStream);
494 if (uploadedWithInterop) {
495 if (!captureUploadPathLogged) {
496 std::cout << "compute_shader: FFmpeg CUDA path active: NVDEC/CUDA decode -> CUDA NV12/RGBA conversion -> optional CUDA resize -> cudaMemcpy2DToArrayAsync -> Vulkan compute storage image (no CPU download)\n";
497 captureUploadPathLogged = true;
498 }
499 return true;
500 }
501
502 cv::Mat cpuFrame;
503 gpuFrame.download(cpuFrame, ffCudaStream);
504 ffCudaStream.waitForCompletion();
505 if (!cpuFrame.empty()) {
506 if (!captureUploadPathLogged) {
507 std::cout << "compute_shader: FFmpeg CUDA decode active, Vulkan CUDA interop unavailable; downloading RGBA for optional CPU resize and staging upload\n";
508 captureUploadPathLogged = true;
509 }
510 uploadCpuFrameToCompute(cpuFrame);
511 return true;
512 }
513 }
514 }
515#endif
516 int frameWidth = 0;
517 int frameHeight = 0;
518 if (!ffCapture.readRgba(ffFrameRgba, frameWidth, frameHeight, ffFramePitch) || ffFrameRgba.empty()) {
519 return false;
520 }
521 if (frameWidth <= 0 || frameHeight <= 0) {
522 return false;
523 }
524 if (!captureUploadPathLogged) {
525 std::cout << "compute_shader: FFmpeg capture path active: decoded RGBA -> optional CPU resize -> Vulkan staging upload\n";
526 captureUploadPathLogged = true;
527 }
528 cv::Mat frame(frameHeight, frameWidth, CV_8UC4, ffFrameRgba.data(), static_cast<size_t>(ffFramePitch));
529 uploadCpuFrameToCompute(frame);
530 return true;
531 }
532#endif
533
534 void configureRecordingDefaults() {
535 if (outputFilename.empty()) {
536 outputFilename = assetRoot + "/compute_shader_output.mp4";
537 }
538 if (outputCrf.empty()) {
539 outputCrf = "24";
540 }
541 }
542
543 [[nodiscard]] int overlayFontSizeForCanvas() const {
544 const int canvasMinDim = std::min(texWidth, texHeight);
545 return std::clamp(canvasMinDim / 30, 8, 36);
546 }
547
548 void maybeResizeWindowToSource() {
549 if (usingFile && !explicitResolution && !fullscreenMode && getSDLWindow() != nullptr) {
550 SDL_SetWindowSize(getSDLWindow(), texWidth, texHeight);
551 std::cout << "compute_shader: window resized to source frame size " << texWidth << "x" << texHeight
552 << " (pass -r/--resolution to override)\n";
553 }
554 }
555
556#if defined(MXWRITE_ENABLED)
557 [[nodiscard]] int parseCrf() const {
558 try {
559 size_t parsedChars = 0;
560 const int value = std::stoi(outputCrf, &parsedChars);
561 if (parsedChars == outputCrf.size() && value >= 0 && value <= 51) {
562 return value;
563 }
564 } catch (const std::exception &) {
565 }
566 throw mxvk::Exception("compute_shader: invalid CRF '" + outputCrf + "'; expected integer 0..51");
567 }
568
569 void openVideoWriter() {
570 if (videoWriterOpen) {
571 return;
572 }
573 if (sourceFps <= 0.0) {
574 sourceFps = usingFile ? videoFps : 30.0;
575 }
576 if (sourceFps <= 0.0) {
577 sourceFps = 30.0;
578 }
579 EncodeOptions encodeOptions{};
580 encodeOptions.crf = parseCrf();
581 if (!encodePreset.empty()) {
582 encodeOptions.preset = encodePreset;
583 }
584 if (!encodeTune.empty()) {
585 encodeOptions.tune = encodeTune;
586 }
587 if (!encodeCodec.empty()) {
588 encodeOptions.codec = encodeCodec;
589 }
590 encodeOptions.realtime = encodeRealtime;
591 encodeOptions.block_when_full = mxwriteBlockWhenFull;
592
593 if (!videoWriter.open(outputFilename, recordWidth, recordHeight, static_cast<float>(sourceFps), encodeOptions)) {
594 throw mxvk::Exception("compute_shader: failed to open MXWrite output file '" + outputFilename + "'");
595 }
596 videoWriterOpen = true;
597 std::cout << "compute_shader: recording to " << outputFilename << " at " << sourceFps
598 << " fps with crf " << encodeOptions.crf
599 << ", codec=" << encodeOptions.codec
600 << ", preset=" << encodeOptions.preset
601 << ", tune=" << (encodeOptions.tune.empty() ? "none" : encodeOptions.tune)
602 << ", realtime=" << (encodeOptions.realtime ? "true" : "false")
603 << ", block_when_full=" << (mxwriteBlockWhenFull ? "true" : "false") << "\n";
604 }
605
606 void recordFrame(const cv::Mat &frame) {
607 if (videoWriterOpen && !frame.empty()) {
608 recordFrame(frame.ptr(), frame.cols, frame.rows, static_cast<int>(frame.step));
609 }
610 }
611
612 void recordFrame(const uint8_t *data, int width, int height, int pitch) {
613 if (!videoWriterOpen || data == nullptr || width != recordWidth || height != recordHeight) {
614 return;
615 }
616
617 const int tightPitch = recordWidth * 4;
618 if (pitch == tightPitch) {
619 videoWriter.write(const_cast<uint8_t *>(data));
620 return;
621 }
622
623 if (pitch < tightPitch) {
624 return;
625 }
626
627 const int recordTightPitch = recordWidth * 4;
628 recordScratch.resize(static_cast<size_t>(recordTightPitch) * static_cast<size_t>(recordHeight));
629 for (int row = 0; row < recordHeight; ++row) {
630 std::memcpy(recordScratch.data() + static_cast<size_t>(row) * static_cast<size_t>(tightPitch),
631 data + static_cast<size_t>(row) * static_cast<size_t>(pitch),
632 static_cast<size_t>(recordTightPitch));
633 }
634 videoWriter.write(recordScratch.data());
635 }
636
637#ifdef MXVK_CUDA
638 void recordGpuFrame(cv::cuda::GpuMat &gpuFrame) {
639 if (!videoWriterOpen || gpuFrame.empty()) {
640 return;
641 }
642#if defined(MXWRITE_HAS_CUDA_COPY)
643 if (videoWriter.is_hardware_encode() &&
644 videoWriter.write_cuda_rgba(gpuFrame.ptr(), static_cast<int>(gpuFrame.step))) {
645 return;
646 }
647#endif
648 cv::Mat cpuFrame;
649 gpuFrame.download(cpuFrame);
650 recordFrame(cpuFrame);
651 }
652#endif
653#else
654 void openVideoWriter() {
655 if (!recordingEnabled) {
656 return;
657 }
658 if (!recordingWarningLogged) {
659 std::cout << "compute_shader: MXWrite is unavailable; video recording disabled\n";
660 recordingWarningLogged = true;
661 }
662 recordingEnabled = false;
663 }
664
665 void recordFrame(const cv::Mat &) {}
666 void recordFrame(const uint8_t *, int, int, int) {}
667#ifdef MXVK_CUDA
668 void recordGpuFrame(cv::cuda::GpuMat &) {}
669#endif
670#endif
671
672#ifdef MXVK_CUDA
673 bool recordProcessedFrameCuda() {
674 if (!recordingEnabled || !ensureCudaInterop(outImg)) {
675 return false;
676 }
677
678 const VkCommandBuffer cmd = beginSingleTimeCommands();
679 transitionImageLayout(
680 cmd,
681 outImg.image,
682 VK_IMAGE_LAYOUT_GENERAL,
683 VK_IMAGE_LAYOUT_GENERAL,
684 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
685 VK_ACCESS_2_MEMORY_WRITE_BIT,
686 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
687 VK_ACCESS_2_MEMORY_READ_BIT);
688 endSingleTimeCommands(cmd);
689
690 processedRecordGpuFrame.create(texHeight, texWidth, CV_8UC4);
691 cudaStream_t cudaStream = mxvk::cuda_stream_handle(processedRecordStream);
692 cudaError_t cudaResult = cudaMemcpy2DFromArrayAsync(
693 processedRecordGpuFrame.ptr(),
694 processedRecordGpuFrame.step,
695 outImg.cudaArray,
696 0,
697 0,
698 static_cast<size_t>(texWidth) * 4U,
699 static_cast<size_t>(texHeight),
700 cudaMemcpyDeviceToDevice,
701 cudaStream);
702 if (cudaResult != cudaSuccess) {
703 std::cout << "compute_shader: CUDA processed-frame readback failed: " << cudaGetErrorString(cudaResult) << "\n";
704 return false;
705 }
706
707 cudaResult = cudaStreamSynchronize(cudaStream);
708 if (cudaResult != cudaSuccess) {
709 std::cout << "compute_shader: CUDA processed-frame readback sync failed: " << cudaGetErrorString(cudaResult) << "\n";
710 return false;
711 }
712
713 if (!processedRecordPathLogged) {
714 std::cout << "compute_shader: processed recording path active: Vulkan compute output image -> CUDA array -> RGBA GpuMat -> MXWrite"
715#if defined(MXWRITE_HAS_CUDA_COPY)
716 << " CUDA ingestion when hardware encode is active"
717#else
718 << " CPU fallback when MXWrite CUDA ingestion is unavailable"
719#endif
720 << "\n";
721 processedRecordPathLogged = true;
722 }
723 recordGpuFrame(processedRecordGpuFrame);
724 return true;
725 }
726#endif
727
728 void recordProcessedFrame() {
729 if (!recordingEnabled || readbackBuf == VK_NULL_HANDLE || readbackMem == VK_NULL_HANDLE) {
730 return;
731 }
732
733#ifdef MXVK_CUDA
734 if (recordProcessedFrameCuda()) {
735 return;
736 }
737#endif
738
739 const int tightPitch = texWidth * 4;
740 const VkDeviceSize bytes = static_cast<VkDeviceSize>(tightPitch) * static_cast<VkDeviceSize>(texHeight);
741 const VkCommandBuffer cmd = beginSingleTimeCommands();
742
743 transitionImageLayout(
744 cmd,
745 outImg.image,
746 VK_IMAGE_LAYOUT_GENERAL,
747 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
748 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
749 VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT,
750 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
751 VK_ACCESS_2_TRANSFER_READ_BIT);
752
753 VkBufferImageCopy2 region{};
754 region.sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2;
755 region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
756 region.imageExtent = {static_cast<uint32_t>(texWidth), static_cast<uint32_t>(texHeight), 1};
757
758 VkCopyImageToBufferInfo2 copyInfo{};
759 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_BUFFER_INFO_2;
760 copyInfo.srcImage = outImg.image;
761 copyInfo.srcImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
762 copyInfo.dstBuffer = readbackBuf;
763 copyInfo.regionCount = 1;
764 copyInfo.pRegions = &region;
765 vkCmdCopyImageToBuffer2(cmd, &copyInfo);
766
767 transitionImageLayout(
768 cmd,
769 outImg.image,
770 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
771 VK_IMAGE_LAYOUT_GENERAL,
772 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
773 VK_ACCESS_2_TRANSFER_READ_BIT,
774 VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT,
775 VK_ACCESS_2_SHADER_SAMPLED_READ_BIT);
776
777 endSingleTimeCommands(cmd);
778
779 void *mapped = nullptr;
780 VK_CHECK_RESULT(vkMapMemory(device, readbackMem, 0, bytes, 0, &mapped));
781 cv::Mat sourceFrame(texHeight, texWidth, CV_8UC4, mapped, tightPitch);
782 if (!processedRecordPathLogged) {
783 std::cout << "compute_shader: processed recording path active: Vulkan compute output image -> readback buffer -> MXWrite\n";
784 processedRecordPathLogged = true;
785 }
786 recordFrame(sourceFrame);
787 vkUnmapMemory(device, readbackMem);
788 }
789
790 void throttleVideoPlayback() {
791 if (!usingFile || fastMode || videoFps <= 0.0) {
792 return;
793 }
794
795 const auto now = std::chrono::steady_clock::now();
796 if (nextVideoFrameDeadline > now) {
797 std::this_thread::sleep_until(nextVideoFrameDeadline);
798 }
799 nextVideoFrameDeadline += std::chrono::duration_cast<std::chrono::steady_clock::duration>(videoFrameInterval);
800 }
801
802 void initComputeResources() {
803 try {
804 if (device == VK_NULL_HANDLE) {
805 throw mxvk::Exception("Compute resources require an initialized Vulkan device");
806 }
807 if (swapchain == VK_NULL_HANDLE || command_pool == VK_NULL_HANDLE) {
808 createDevice();
809 }
810 setFont(assetRoot + "/data/font.ttf", 20);
811
812 if (usingFile) {
813 if (!openVideoSource()) {
814 throw mxvk::Exception("Failed to open video file " + inputFilename);
815 }
816 configureVideoPlaybackRate();
817 } else if (!capture.open(cameraIndex)) {
818 throw mxvk::Exception("Failed to open camera " + std::to_string(cameraIndex));
819 }
820
821 if (!usingFile) {
822 capture.set(cv::CAP_PROP_FRAME_WIDTH, recordWidth);
823 capture.set(cv::CAP_PROP_FRAME_HEIGHT,recordHeight);
824 const double selectedFps = configureCameraFps();
825 sourceFps = selectedFps;
826 std::cout << "compute_shader: requested camera FPS fallback order 60 -> 30 -> 24; selected "
827 << selectedFps << " fps\n";
828 }
829
830#if defined(MXVK_WITH_FFMPEG_CAPTURE)
831 if (usingFfCapture) {
832 sourceWidth = ffCapture.width();
833 sourceHeight = ffCapture.height();
834 if (sourceWidth <= 0 || sourceHeight <= 0) {
835 throw mxvk::Exception("Failed to query FFmpeg video dimensions");
836 }
837 } else
838#endif
839 {
840 cv::Mat frame;
841 if (!capture.read(frame) || frame.empty()) {
842 throw mxvk::Exception(usingFile ? "Failed to read initial video frame" : "Failed to read initial camera frame");
843 }
844
845 sourceWidth = frame.cols;
846 sourceHeight = frame.rows;
847 }
848
849 if (explicitResolution) {
850 texWidth = recordWidth;
851 texHeight = recordHeight;
852 std::cout << "compute_shader: compute canvas set from explicit resolution " << texWidth << "x" << texHeight
853 << "; source frames are " << sourceWidth << "x" << sourceHeight << "\n";
854 } else {
855 texWidth = sourceWidth;
856 texHeight = sourceHeight;
857 recordWidth = texWidth;
858 recordHeight = texHeight;
859 }
860
861 configureRecordingDefaults();
862 recordingEnabled = true;
863 openVideoWriter();
864 fpsFont.reset(assetRoot + "/data/font.ttf", overlayFontSizeForCanvas());
865 playbackStartTime = std::chrono::steady_clock::now();
866 maybeResizeWindowToSource();
867
868 const VkDeviceSize imgBytes = static_cast<VkDeviceSize>(texWidth) * texHeight * 4;
869
870 createBuffer(
871 imgBytes,
872 VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
873 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
874 stagingBuf,
875 stagingMem);
876 createBuffer(
877 imgBytes,
878 VK_BUFFER_USAGE_TRANSFER_DST_BIT,
879 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
880 readbackBuf,
881 readbackMem);
882
883 {
884 const VkCommandBuffer cmd = beginSingleTimeCommands();
885#ifdef MXVK_CUDA
886 allocCImg(workImg[0], cmd, true);
887#else
888 allocCImg(workImg[0], cmd);
889#endif
890 allocCImg(workImg[1], cmd);
891 for (ComputeImage &img : histImg) {
892 allocCImg(img, cmd);
893 }
894#ifdef MXVK_CUDA
895 allocCImg(outImg, cmd, true);
896#else
897 allocCImg(outImg, cmd);
898#endif
899 endSingleTimeCommands(cmd);
900 }
901
902 VkSamplerCreateInfo samplerInfo{};
903 samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
904 samplerInfo.magFilter = VK_FILTER_LINEAR;
905 samplerInfo.minFilter = VK_FILTER_LINEAR;
906 samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
907 samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
908 samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
909 samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
910 samplerInfo.maxAnisotropy = 1.0f;
911 VK_CHECK_RESULT(vkCreateSampler(device, &samplerInfo, nullptr, &computeSampler));
912
913 loadSPV();
914 buildDescriptorSetLayout();
915 buildComputePipeline();
916 buildDescriptorSets();
917 createDisplayResources();
918 } catch (...) {
919 // Constructor failure bypasses ~ComputeWindow; cleanup partial Vulkan state here.
920 capture.close();
921#if defined(MXVK_WITH_FFMPEG_CAPTURE)
922 ffCapture.close();
923#endif
924 destroyComputeResources();
925 throw;
926 }
927 }
928
929 void updateFpsOverlay(bool frameUploaded) {
930 if (!active || !frameUploaded) {
931 return;
932 }
933
934 try {
936 } catch (const std::exception &ex) {
937 std::cerr << "compute_shader: failed to clear stale text overlay queue: " << ex.what() << "\n";
938 }
939
940 if (frameUploaded) {
941 ++fpsFrameCount;
942 ++processedVideoFrames;
943 }
944
945 const auto now = std::chrono::steady_clock::now();
946 const double elapsed = std::chrono::duration<double>(now - fpsSampleTime).count();
947 if (elapsed >= 0.25) {
948 currentFps = static_cast<double>(fpsFrameCount) / elapsed;
949 fpsFrameCount = 0;
950 fpsSampleTime = now;
951 fpsText = std::format("FPS: {:.1f}", currentFps);
952 }
953
954 const auto recElapsed = now - playbackStartTime;
955 const uint64_t recTotalSeconds =
956 static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::seconds>(recElapsed).count());
957 const uint64_t recHours = recTotalSeconds / 3600U;
958 const uint64_t recMinutes = (recTotalSeconds / 60U) % 60U;
959 const uint64_t recSeconds = recTotalSeconds % 60U;
960 const std::string recText = std::format("Rec: {:02}:{:02}:{:02}", recHours, recMinutes, recSeconds);
961
962 const double effectiveSourceFps = (sourceFps > 0.0) ? sourceFps : videoFps;
963 const uint64_t totalTenths = (effectiveSourceFps > 0.0)
964 ? static_cast<uint64_t>(std::llround((static_cast<double>(processedVideoFrames) / effectiveSourceFps) * 10.0))
965 : 0U;
966 const uint64_t hours = totalTenths / 36000U;
967 const uint64_t minutes = (totalTenths / 600U) % 60U;
968 const uint64_t seconds = (totalTenths / 10U) % 60U;
969 const uint64_t tenths = totalTenths % 10U;
970 const std::string timeText = std::format("Output: {:02}:{:02}:{:02}.{:01}", hours, minutes, seconds, tenths);
971 const std::string overlayText = std::format(
972 "Source FPS: {:.1f} Current FPS: {:.1f} | {} | {}",
973 sourceFps,
974 currentFps,
975 recText,
976 timeText);
977
978 printText(overlayText, 18, 18, SDL_Color{255, 240, 0, 255}, fpsFont);
979 if (!spvFiles.empty()) {
980 if (spvFiles[currentSpvIndex] == MODE_SHADER_NAME) {
981 const int modeIndex = std::clamp(shaderMode, 0, static_cast<int>(ACIDCAM_FILTER_MODE_NAMES.size()) - 1);
982 const std::string modeText = std::format(
983 "Mode: {} {}/{}",
984 ACIDCAM_FILTER_MODE_NAMES[modeIndex],
985 modeIndex + 1,
986 ACIDCAM_FILTER_MODE_NAMES.size());
987 printText(modeText, 15, 68, SDL_Color{255, 105, 180, 255});
988 } else {
989 const std::string spvText = std::format("{}: {}", currentSpvIndex, spvFiles[currentSpvIndex]);
990 printText(spvText, 15, 68, SDL_Color{80, 160, 255, 255});
991 }
992 }
993 }
994
995 [[nodiscard]] uint32_t findMemoryType(uint32_t typeFilter, VkMemoryPropertyFlags properties) const {
996 VkPhysicalDeviceMemoryProperties memProperties{};
997 vkGetPhysicalDeviceMemoryProperties(physical_device, &memProperties);
998
999 for (uint32_t index = 0; index < memProperties.memoryTypeCount; ++index) {
1000 const bool typeMatches = (typeFilter & (1U << index)) != 0U;
1001 const bool propertyMatches =
1002 (memProperties.memoryTypes[index].propertyFlags & properties) == properties;
1003 if (typeMatches && propertyMatches) {
1004 return index;
1005 }
1006 }
1007
1008 throw mxvk::Exception("Failed to find suitable memory type");
1009 }
1010
1011 void createBuffer(VkDeviceSize size,
1012 VkBufferUsageFlags usage,
1013 VkMemoryPropertyFlags properties,
1014 VkBuffer &buffer,
1015 VkDeviceMemory &bufferMemory) {
1016 VkBuffer newBuffer = VK_NULL_HANDLE;
1017 VkDeviceMemory newMemory = VK_NULL_HANDLE;
1018
1019 VkBufferCreateInfo bufferInfo{};
1020 bufferInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
1021 bufferInfo.size = size;
1022 bufferInfo.usage = usage;
1023 bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
1024
1025 try {
1026 VK_CHECK_RESULT(vkCreateBuffer(device, &bufferInfo, nullptr, &newBuffer));
1027
1028 VkMemoryRequirements memRequirements{};
1029 vkGetBufferMemoryRequirements(device, newBuffer, &memRequirements);
1030
1031 VkMemoryAllocateInfo allocInfo{};
1032 allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
1033 allocInfo.allocationSize = memRequirements.size;
1034 allocInfo.memoryTypeIndex = findMemoryType(memRequirements.memoryTypeBits, properties);
1035 VK_CHECK_RESULT(vkAllocateMemory(device, &allocInfo, nullptr, &newMemory));
1036 VK_CHECK_RESULT(vkBindBufferMemory(device, newBuffer, newMemory, 0));
1037 } catch (...) {
1038 if (newBuffer != VK_NULL_HANDLE) {
1039 vkDestroyBuffer(device, newBuffer, nullptr);
1040 }
1041 if (newMemory != VK_NULL_HANDLE) {
1042 vkFreeMemory(device, newMemory, nullptr);
1043 }
1044 throw;
1045 }
1046
1047 if (buffer != VK_NULL_HANDLE) {
1048 vkDestroyBuffer(device, buffer, nullptr);
1049 }
1050 if (bufferMemory != VK_NULL_HANDLE) {
1051 vkFreeMemory(device, bufferMemory, nullptr);
1052 }
1053 buffer = newBuffer;
1054 bufferMemory = newMemory;
1055 }
1056
1057 [[nodiscard]] VkCommandBuffer beginSingleTimeCommands() const {
1058 VkCommandBufferAllocateInfo allocInfo{};
1059 allocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
1060 allocInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
1061 allocInfo.commandPool = command_pool;
1062 allocInfo.commandBufferCount = 1;
1063
1064 VkCommandBuffer commandBuffer = VK_NULL_HANDLE;
1065 VK_CHECK_RESULT(vkAllocateCommandBuffers(device, &allocInfo, &commandBuffer));
1066
1067 VkCommandBufferBeginInfo beginInfo{};
1068 beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
1069 beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
1070 VK_CHECK_RESULT(vkBeginCommandBuffer(commandBuffer, &beginInfo));
1071
1072 return commandBuffer;
1073 }
1074
1075 void endSingleTimeCommands(VkCommandBuffer commandBuffer) const {
1076 VK_CHECK_RESULT(vkEndCommandBuffer(commandBuffer));
1077
1078 VkCommandBufferSubmitInfo commandBufferInfo{};
1079 commandBufferInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO;
1080 commandBufferInfo.commandBuffer = commandBuffer;
1081
1082 VkSubmitInfo2 submitInfo{};
1083 submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2;
1084 submitInfo.commandBufferInfoCount = 1;
1085 submitInfo.pCommandBufferInfos = &commandBufferInfo;
1086
1087 VK_CHECK_RESULT(vkQueueSubmit2(graphics_queue, 1, &submitInfo, VK_NULL_HANDLE));
1088 VK_CHECK_RESULT(vkQueueWaitIdle(graphics_queue));
1089 vkFreeCommandBuffers(device, command_pool, 1, &commandBuffer);
1090 }
1091
1092 void createImage(uint32_t width,
1093 uint32_t height,
1094 VkFormat format,
1095 VkImageTiling tiling,
1096 VkImageUsageFlags usage,
1097 VkMemoryPropertyFlags properties,
1098 VkImage &image,
1099 VkDeviceMemory &imageMemory) {
1100 VkImage newImage = VK_NULL_HANDLE;
1101 VkDeviceMemory newMemory = VK_NULL_HANDLE;
1102
1103 VkImageCreateInfo imageInfo{};
1104 imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
1105 imageInfo.imageType = VK_IMAGE_TYPE_2D;
1106 imageInfo.extent.width = width;
1107 imageInfo.extent.height = height;
1108 imageInfo.extent.depth = 1;
1109 imageInfo.mipLevels = 1;
1110 imageInfo.arrayLayers = 1;
1111 imageInfo.format = format;
1112 imageInfo.tiling = tiling;
1113 imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
1114 imageInfo.usage = usage;
1115 imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
1116 imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
1117
1118 try {
1119 VK_CHECK_RESULT(vkCreateImage(device, &imageInfo, nullptr, &newImage));
1120
1121 VkMemoryRequirements memRequirements{};
1122 vkGetImageMemoryRequirements(device, newImage, &memRequirements);
1123
1124 VkMemoryAllocateInfo allocInfo{};
1125 allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
1126 allocInfo.allocationSize = memRequirements.size;
1127 allocInfo.memoryTypeIndex = findMemoryType(memRequirements.memoryTypeBits, properties);
1128 VK_CHECK_RESULT(vkAllocateMemory(device, &allocInfo, nullptr, &newMemory));
1129 VK_CHECK_RESULT(vkBindImageMemory(device, newImage, newMemory, 0));
1130 } catch (...) {
1131 if (newImage != VK_NULL_HANDLE) {
1132 vkDestroyImage(device, newImage, nullptr);
1133 }
1134 if (newMemory != VK_NULL_HANDLE) {
1135 vkFreeMemory(device, newMemory, nullptr);
1136 }
1137 throw;
1138 }
1139
1140 if (image != VK_NULL_HANDLE) {
1141 vkDestroyImage(device, image, nullptr);
1142 }
1143 if (imageMemory != VK_NULL_HANDLE) {
1144 vkFreeMemory(device, imageMemory, nullptr);
1145 }
1146 image = newImage;
1147 imageMemory = newMemory;
1148 }
1149
1150#ifdef MXVK_CUDA
1151 void createCudaExportableImage(ComputeImage &img) {
1152 std::cout << "compute_shader: CUDA interop init: requesting exportable compute input image "
1153 << texWidth << "x" << texHeight << " RGBA8 optimal-tiled OPAQUE_FD\n";
1154
1155 VkExternalMemoryImageCreateInfo externalImageInfo{};
1156 externalImageInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO;
1157 externalImageInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
1158
1159 VkImageCreateInfo imageInfo{};
1160 imageInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
1161 imageInfo.pNext = &externalImageInfo;
1162 imageInfo.imageType = VK_IMAGE_TYPE_2D;
1163 imageInfo.extent.width = static_cast<uint32_t>(texWidth);
1164 imageInfo.extent.height = static_cast<uint32_t>(texHeight);
1165 imageInfo.extent.depth = 1;
1166 imageInfo.mipLevels = 1;
1167 imageInfo.arrayLayers = 1;
1168 imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM;
1169 imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
1170 imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
1171 imageInfo.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
1172 VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
1173 imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
1174 imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
1175
1176 VK_CHECK_RESULT(vkCreateImage(device, &imageInfo, nullptr, &img.image));
1177
1178 VkMemoryRequirements memRequirements{};
1179 vkGetImageMemoryRequirements(device, img.image, &memRequirements);
1180
1181 VkExportMemoryAllocateInfo exportMemoryInfo{};
1182 exportMemoryInfo.sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
1183 exportMemoryInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
1184
1185 VkMemoryAllocateInfo allocInfo{};
1186 allocInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
1187 allocInfo.pNext = &exportMemoryInfo;
1188 allocInfo.allocationSize = memRequirements.size;
1189
1190 try {
1191 allocInfo.memoryTypeIndex = findMemoryType(memRequirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
1192 VK_CHECK_RESULT(vkAllocateMemory(device, &allocInfo, nullptr, &img.memory));
1193 VK_CHECK_RESULT(vkBindImageMemory(device, img.image, img.memory, 0));
1194 img.cudaExportMemorySize = memRequirements.size;
1195 img.cudaInteropUnavailableLogged = false;
1196 std::cout << "compute_shader: CUDA interop init: exportable compute input image allocated (memorySize="
1197 << static_cast<unsigned long long>(memRequirements.size)
1198 << " bytes, memoryType=" << allocInfo.memoryTypeIndex
1199 << "); optimal image memory will be imported as cudaArray\n";
1200 } catch (...) {
1201 if (img.image != VK_NULL_HANDLE) {
1202 vkDestroyImage(device, img.image, nullptr);
1203 img.image = VK_NULL_HANDLE;
1204 }
1205 if (img.memory != VK_NULL_HANDLE) {
1206 vkFreeMemory(device, img.memory, nullptr);
1207 img.memory = VK_NULL_HANDLE;
1208 }
1209 img.cudaExportMemorySize = 0;
1210 throw;
1211 }
1212 }
1213
1214 void destroyCudaInterop(ComputeImage &img) {
1215 if (img.cudaInteropEnabled || img.cudaExternalMemory != nullptr || img.cudaMipmappedArray != nullptr) {
1216 std::cout << "compute_shader: CUDA interop: destroying imported compute input image resources\n";
1217 }
1218 if (img.cudaMipmappedArray != nullptr) {
1219 cudaFreeMipmappedArray(img.cudaMipmappedArray);
1220 img.cudaMipmappedArray = nullptr;
1221 img.cudaArray = nullptr;
1222 }
1223 if (img.cudaExternalMemory != nullptr) {
1224 cudaDestroyExternalMemory(img.cudaExternalMemory);
1225 img.cudaExternalMemory = nullptr;
1226 }
1227 img.cudaInteropEnabled = false;
1228 img.cudaExportMemorySize = 0;
1229 img.cudaUploadLogged = false;
1230 img.cudaBarrierLogged = false;
1231 }
1232
1233 bool ensureCudaInterop(ComputeImage &img) {
1234 if (img.cudaInteropEnabled) {
1235 return true;
1236 }
1237 if (img.memory == VK_NULL_HANDLE || img.cudaExportMemorySize == 0) {
1238 if (!img.cudaInteropUnavailableLogged) {
1239 std::cout << "compute_shader: CUDA interop init: compute input image is not exportable\n";
1240 img.cudaInteropUnavailableLogged = true;
1241 }
1242 return false;
1243 }
1244 if (vkGetMemoryFdKHR == nullptr) {
1245 if (!img.cudaInteropUnavailableLogged) {
1246 std::cout << "compute_shader: CUDA interop init: vkGetMemoryFdKHR was not loaded\n";
1247 img.cudaInteropUnavailableLogged = true;
1248 }
1249 return false;
1250 }
1251
1252 VkMemoryGetFdInfoKHR fdInfo{};
1253 fdInfo.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR;
1254 fdInfo.memory = img.memory;
1255 fdInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
1256
1257 int memoryFd = -1;
1258 const VkResult fdResult = vkGetMemoryFdKHR(device, &fdInfo, &memoryFd);
1259 if (fdResult != VK_SUCCESS) {
1260 if (!img.cudaInteropUnavailableLogged) {
1261 std::cout << "compute_shader: CUDA interop init: vkGetMemoryFdKHR failed (" << static_cast<int>(fdResult) << ")\n";
1262 img.cudaInteropUnavailableLogged = true;
1263 }
1264 return false;
1265 }
1266 std::cout << "compute_shader: CUDA interop init: exported compute input image memory fd=" << memoryFd << "\n";
1267
1268 cudaExternalMemoryHandleDesc externalMemoryDesc{};
1269 externalMemoryDesc.type = cudaExternalMemoryHandleTypeOpaqueFd;
1270 externalMemoryDesc.handle.fd = memoryFd;
1271 externalMemoryDesc.size = img.cudaExportMemorySize;
1272
1273 cudaError_t cudaResult = cudaImportExternalMemory(&img.cudaExternalMemory, &externalMemoryDesc);
1274 if (cudaResult != cudaSuccess) {
1275 close(memoryFd);
1276 if (!img.cudaInteropUnavailableLogged) {
1277 std::cout << "compute_shader: CUDA interop init: cudaImportExternalMemory failed: "
1278 << cudaGetErrorString(cudaResult) << "\n";
1279 img.cudaInteropUnavailableLogged = true;
1280 }
1281 img.cudaExternalMemory = nullptr;
1282 return false;
1283 }
1284 std::cout << "compute_shader: CUDA interop init: imported compute input image external memory into CUDA ("
1285 << static_cast<unsigned long long>(img.cudaExportMemorySize) << " bytes)\n";
1286
1287 cudaExternalMemoryMipmappedArrayDesc arrayDesc{};
1288 arrayDesc.offset = 0;
1289 arrayDesc.formatDesc = cudaCreateChannelDesc<uchar4>();
1290 arrayDesc.extent = make_cudaExtent(static_cast<size_t>(texWidth), static_cast<size_t>(texHeight), 0);
1291 arrayDesc.flags = cudaArrayColorAttachment;
1292 arrayDesc.numLevels = 1;
1293
1294 cudaResult = cudaExternalMemoryGetMappedMipmappedArray(&img.cudaMipmappedArray, img.cudaExternalMemory, &arrayDesc);
1295 if (cudaResult != cudaSuccess) {
1296 if (!img.cudaInteropUnavailableLogged) {
1297 std::cout << "compute_shader: CUDA interop init: cudaExternalMemoryGetMappedMipmappedArray failed: "
1298 << cudaGetErrorString(cudaResult) << "\n";
1299 img.cudaInteropUnavailableLogged = true;
1300 }
1301 destroyCudaInterop(img);
1302 return false;
1303 }
1304 std::cout << "compute_shader: CUDA interop init: mapped compute input CUDA mipmapped array "
1305 << texWidth << "x" << texHeight << " uchar4\n";
1306
1307 cudaResult = cudaGetMipmappedArrayLevel(&img.cudaArray, img.cudaMipmappedArray, 0);
1308 if (cudaResult != cudaSuccess) {
1309 if (!img.cudaInteropUnavailableLogged) {
1310 std::cout << "compute_shader: CUDA interop init: cudaGetMipmappedArrayLevel failed: "
1311 << cudaGetErrorString(cudaResult) << "\n";
1312 img.cudaInteropUnavailableLogged = true;
1313 }
1314 destroyCudaInterop(img);
1315 return false;
1316 }
1317
1318 img.cudaInteropEnabled = true;
1319 std::cout << "compute_shader: CUDA interop init: direct CUDA-to-compute-input upload is ready\n";
1320 return true;
1321 }
1322#endif
1323
1324 [[nodiscard]] VkImageView createImageView(VkImage image,
1325 VkFormat format,
1326 VkImageAspectFlags aspectFlags) const {
1327 VkImageViewCreateInfo viewInfo{};
1328 viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
1329 viewInfo.image = image;
1330 viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D;
1331 viewInfo.format = format;
1332 viewInfo.subresourceRange.aspectMask = aspectFlags;
1333 viewInfo.subresourceRange.baseMipLevel = 0;
1334 viewInfo.subresourceRange.levelCount = 1;
1335 viewInfo.subresourceRange.baseArrayLayer = 0;
1336 viewInfo.subresourceRange.layerCount = 1;
1337
1338 VkImageView imageView = VK_NULL_HANDLE;
1339 VK_CHECK_RESULT(vkCreateImageView(device, &viewInfo, nullptr, &imageView));
1340 return imageView;
1341 }
1342
1343 void allocCImg(ComputeImage &img, VkCommandBuffer cmd, [[maybe_unused]] bool cudaExportable = false) {
1344#ifdef MXVK_CUDA
1345 if (cudaExportable) {
1346 try {
1347 createCudaExportableImage(img);
1348 } catch (const std::exception &ex) {
1349 std::cout << "compute_shader: CUDA exportable input image unavailable: " << ex.what()
1350 << "; using Vulkan staging fallback\n";
1351 createImage(
1352 static_cast<uint32_t>(texWidth),
1353 static_cast<uint32_t>(texHeight),
1354 VK_FORMAT_R8G8B8A8_UNORM,
1355 VK_IMAGE_TILING_OPTIMAL,
1356 VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
1357 VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1358 VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
1359 img.image,
1360 img.memory);
1361 }
1362 } else {
1363 createImage(
1364 static_cast<uint32_t>(texWidth),
1365 static_cast<uint32_t>(texHeight),
1366 VK_FORMAT_R8G8B8A8_UNORM,
1367 VK_IMAGE_TILING_OPTIMAL,
1368 VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
1369 VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1370 VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
1371 img.image,
1372 img.memory);
1373 }
1374#else
1375 createImage(
1376 static_cast<uint32_t>(texWidth),
1377 static_cast<uint32_t>(texHeight),
1378 VK_FORMAT_R8G8B8A8_UNORM,
1379 VK_IMAGE_TILING_OPTIMAL,
1380 VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
1381 VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
1382 VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
1383 img.image,
1384 img.memory);
1385#endif
1386 img.view = createImageView(img.image, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_ASPECT_COLOR_BIT);
1387
1388 transitionImageLayout(
1389 cmd,
1390 img.image,
1391 VK_IMAGE_LAYOUT_UNDEFINED,
1392 VK_IMAGE_LAYOUT_GENERAL,
1393 VK_PIPELINE_STAGE_2_NONE,
1394 VK_ACCESS_2_NONE,
1395 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
1396 VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1397 }
1398
1399 void reloadPipeline() {
1400 vkDeviceWaitIdle(device);
1401 if (compPipeline != VK_NULL_HANDLE) {
1402 vkDestroyPipeline(device, compPipeline, nullptr);
1403 compPipeline = VK_NULL_HANDLE;
1404 }
1405 if (compPipeLayout != VK_NULL_HANDLE) {
1406 vkDestroyPipelineLayout(device, compPipeLayout, nullptr);
1407 compPipeLayout = VK_NULL_HANDLE;
1408 }
1409 shaderMode = initialShaderMode;
1410 buildComputePipeline();
1411 }
1412
1413 void buildDescriptorSetLayout() {
1414 std::array<VkDescriptorSetLayoutBinding, 3> bindings{};
1415
1416 bindings[0].binding = 0;
1417 bindings[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
1418 bindings[0].descriptorCount = 1;
1419 bindings[0].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1420
1421 bindings[1].binding = 1;
1422 bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1423 bindings[1].descriptorCount = 1;
1424 bindings[1].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1425
1426 bindings[2].binding = 2;
1427 bindings[2].descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1428 bindings[2].descriptorCount = HISTORY_SIZE;
1429 bindings[2].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1430
1431 VkDescriptorSetLayoutCreateInfo createInfo{};
1432 createInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
1433 createInfo.bindingCount = static_cast<uint32_t>(bindings.size());
1434 createInfo.pBindings = bindings.data();
1435 VK_CHECK_RESULT(vkCreateDescriptorSetLayout(device, &createInfo, nullptr, &compDSLayout));
1436 }
1437
1438 void buildComputePipeline() {
1439 const std::string spvPath = assetRoot + "/data/" + spvFiles[currentSpvIndex];
1440 const std::vector<char> spv = mxvk::load_spv(spvPath);
1441 VkShaderModule module = mxvk::create_shader_module(device, spv);
1442
1443 VkPushConstantRange pushConstantRange{};
1444 pushConstantRange.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
1445 pushConstantRange.offset = 0;
1446 pushConstantRange.size = sizeof(ComputePC);
1447
1448 VkPipelineLayoutCreateInfo pipelineLayoutInfo{};
1449 pipelineLayoutInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;
1450 pipelineLayoutInfo.setLayoutCount = 1;
1451 pipelineLayoutInfo.pSetLayouts = &compDSLayout;
1452 pipelineLayoutInfo.pushConstantRangeCount = 1;
1453 pipelineLayoutInfo.pPushConstantRanges = &pushConstantRange;
1454 VK_CHECK_RESULT(vkCreatePipelineLayout(device, &pipelineLayoutInfo, nullptr, &compPipeLayout));
1455
1456 VkComputePipelineCreateInfo pipelineInfo{};
1457 pipelineInfo.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO;
1458 pipelineInfo.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
1459 pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
1460 pipelineInfo.stage.module = module;
1461 pipelineInfo.stage.pName = "main";
1462 pipelineInfo.layout = compPipeLayout;
1463 VK_CHECK_RESULT(vkCreateComputePipelines(device, VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &compPipeline));
1464
1465 vkDestroyShaderModule(device, module, nullptr);
1466 }
1467
1468 void buildDescriptorSets() {
1469 std::array<VkDescriptorPoolSize, 2> poolSizes{};
1470 poolSizes[0] = {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 4};
1471 poolSizes[1] = {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 4 * (1 + HISTORY_SIZE)};
1472
1473 VkDescriptorPoolCreateInfo poolInfo{};
1474 poolInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
1475 poolInfo.maxSets = 4;
1476 poolInfo.poolSizeCount = static_cast<uint32_t>(poolSizes.size());
1477 poolInfo.pPoolSizes = poolSizes.data();
1478 VK_CHECK_RESULT(vkCreateDescriptorPool(device, &poolInfo, nullptr, &compDSPool));
1479
1480 std::array<VkDescriptorSetLayout, 4> layouts{};
1481 layouts.fill(compDSLayout);
1482
1483 VkDescriptorSetAllocateInfo allocInfo{};
1484 allocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
1485 allocInfo.descriptorPool = compDSPool;
1486 allocInfo.descriptorSetCount = static_cast<uint32_t>(layouts.size());
1487 allocInfo.pSetLayouts = layouts.data();
1488
1489 std::array<VkDescriptorSet, 4> raw{};
1490 VK_CHECK_RESULT(vkAllocateDescriptorSets(device, &allocInfo, raw.data()));
1491 blurDS[0] = raw[0];
1492 blurDS[1] = raw[1];
1493 blendDS[0] = raw[2];
1494 blendDS[1] = raw[3];
1495
1496 writeBlurDS(blurDS[0], workImg[0].view, workImg[1].view);
1497 writeBlurDS(blurDS[1], workImg[1].view, workImg[0].view);
1498 writeBlendDS(blendDS[0], workImg[0].view);
1499 writeBlendDS(blendDS[1], workImg[1].view);
1500 }
1501
1502 void writeBlurDS(VkDescriptorSet descriptorSet, VkImageView destView, VkImageView srcView) {
1503 VkDescriptorImageInfo destInfo{VK_NULL_HANDLE, destView, VK_IMAGE_LAYOUT_GENERAL};
1504 VkDescriptorImageInfo srcInfo{computeSampler, srcView, VK_IMAGE_LAYOUT_GENERAL};
1505 std::vector<VkDescriptorImageInfo> historyInfos(
1506 HISTORY_SIZE,
1507 VkDescriptorImageInfo{computeSampler, srcView, VK_IMAGE_LAYOUT_GENERAL});
1508
1509 std::array<VkWriteDescriptorSet, 3> writes{};
1510 writes[0] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, descriptorSet, 0, 0, 1,
1511 VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &destInfo, nullptr, nullptr};
1512 writes[1] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, descriptorSet, 1, 0, 1,
1513 VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, &srcInfo, nullptr, nullptr};
1514 writes[2] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, descriptorSet, 2, 0, HISTORY_SIZE,
1515 VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, historyInfos.data(), nullptr, nullptr};
1516 vkUpdateDescriptorSets(device, static_cast<uint32_t>(writes.size()), writes.data(), 0, nullptr);
1517 }
1518
1519 [[nodiscard]] std::vector<char> readDisplayShader(const std::string &name) const {
1520 const std::array<std::string, 3> candidates = {
1521 assetRoot + "/data/" + name,
1522 assetRoot + "/" + name,
1523 std::string("data/") + name,
1524 };
1525
1526 for (const std::string &path : candidates) {
1527 std::ifstream file(path, std::ios::binary);
1528 if (file.is_open()) {
1529 return mxvk::load_spv(path);
1530 }
1531 }
1532
1533 throw mxvk::Exception("Cannot open compute display shader: " + name);
1534 }
1535
1536 void createDisplayBuffers() {
1537 if (displayVertexBuffer != VK_NULL_HANDLE && displayIndexBuffer != VK_NULL_HANDLE) {
1538 return;
1539 }
1540
1541 const std::array<float, 16> vertices = {
1542 0.0f,
1543 0.0f,
1544 0.0f,
1545 0.0f,
1546 1.0f,
1547 0.0f,
1548 1.0f,
1549 0.0f,
1550 1.0f,
1551 1.0f,
1552 1.0f,
1553 1.0f,
1554 0.0f,
1555 1.0f,
1556 0.0f,
1557 1.0f,
1558 };
1559 const std::array<uint16_t, 6> indices = {0, 1, 2, 0, 2, 3};
1560
1561 createBuffer(sizeof(float) * vertices.size(),
1562 VK_BUFFER_USAGE_VERTEX_BUFFER_BIT,
1563 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
1564 displayVertexBuffer,
1565 displayVertexMemory);
1566 createBuffer(sizeof(uint16_t) * indices.size(),
1567 VK_BUFFER_USAGE_INDEX_BUFFER_BIT,
1568 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
1569 displayIndexBuffer,
1570 displayIndexMemory);
1571
1572 void *mapped = nullptr;
1573 VK_CHECK_RESULT(vkMapMemory(device, displayVertexMemory, 0, sizeof(float) * vertices.size(), 0, &mapped));
1574 std::memcpy(mapped, vertices.data(), sizeof(float) * vertices.size());
1575 vkUnmapMemory(device, displayVertexMemory);
1576
1577 VK_CHECK_RESULT(vkMapMemory(device, displayIndexMemory, 0, sizeof(uint16_t) * indices.size(), 0, &mapped));
1578 std::memcpy(mapped, indices.data(), sizeof(uint16_t) * indices.size());
1579 vkUnmapMemory(device, displayIndexMemory);
1580 }
1581
1582 void createDisplayDescriptorSet() {
1583 VkDescriptorSetLayoutBinding binding{};
1584 binding.binding = 0;
1585 binding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1586 binding.descriptorCount = 1;
1587 binding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT;
1588
1589 VkDescriptorSetLayoutCreateInfo layoutInfo{};
1590 layoutInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
1591 layoutInfo.bindingCount = 1;
1592 layoutInfo.pBindings = &binding;
1593 VK_CHECK_RESULT(vkCreateDescriptorSetLayout(device, &layoutInfo, nullptr, &displayDSLayout));
1594
1595 VkDescriptorPoolSize poolSize{};
1596 poolSize.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1597 poolSize.descriptorCount = 1;
1598
1599 VkDescriptorPoolCreateInfo poolInfo{};
1600 poolInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO;
1601 poolInfo.poolSizeCount = 1;
1602 poolInfo.pPoolSizes = &poolSize;
1603 poolInfo.maxSets = 1;
1604 VK_CHECK_RESULT(vkCreateDescriptorPool(device, &poolInfo, nullptr, &displayDSPool));
1605
1606 VkDescriptorSetAllocateInfo allocInfo{};
1607 allocInfo.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO;
1608 allocInfo.descriptorPool = displayDSPool;
1609 allocInfo.descriptorSetCount = 1;
1610 allocInfo.pSetLayouts = &displayDSLayout;
1611 VK_CHECK_RESULT(vkAllocateDescriptorSets(device, &allocInfo, &displayDS));
1612
1613 VkDescriptorImageInfo imageInfo{};
1614 imageInfo.sampler = computeSampler;
1615 imageInfo.imageView = outImg.view;
1616 imageInfo.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
1617
1618 VkWriteDescriptorSet write{};
1619 write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
1620 write.dstSet = displayDS;
1621 write.dstBinding = 0;
1622 write.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;
1623 write.descriptorCount = 1;
1624 write.pImageInfo = &imageInfo;
1625 vkUpdateDescriptorSets(device, 1, &write, 0, nullptr);
1626 }
1627
1628 void rebuildDisplayPipeline() {
1629 if (displayPipeline != VK_NULL_HANDLE) {
1630 vkDestroyPipeline(device, displayPipeline, nullptr);
1631 displayPipeline = VK_NULL_HANDLE;
1632 }
1633 if (displayPipeLayout != VK_NULL_HANDLE) {
1634 vkDestroyPipelineLayout(device, displayPipeLayout, nullptr);
1635 displayPipeLayout = VK_NULL_HANDLE;
1636 }
1637 if (displayDSLayout == VK_NULL_HANDLE || swapchain_format == VK_FORMAT_UNDEFINED) {
1638 return;
1639 }
1640
1641 const VkShaderModule vertModule = mxvk::create_shader_module(device, readDisplayShader("sprite.vert.spv"));
1642 const VkShaderModule fragModule = mxvk::create_shader_module(device, readDisplayShader("sprite.frag.spv"));
1643
1644 VkPipelineShaderStageCreateInfo vertStage{};
1645 vertStage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
1646 vertStage.stage = VK_SHADER_STAGE_VERTEX_BIT;
1647 vertStage.module = vertModule;
1648 vertStage.pName = "main";
1649
1650 VkPipelineShaderStageCreateInfo fragStage{};
1651 fragStage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
1652 fragStage.stage = VK_SHADER_STAGE_FRAGMENT_BIT;
1653 fragStage.module = fragModule;
1654 fragStage.pName = "main";
1655
1656 const std::array<VkPipelineShaderStageCreateInfo, 2> stages = {vertStage, fragStage};
1657
1658 VkVertexInputBindingDescription binding{};
1659 binding.binding = 0;
1660 binding.stride = sizeof(float) * 4;
1661 binding.inputRate = VK_VERTEX_INPUT_RATE_VERTEX;
1662
1663 std::array<VkVertexInputAttributeDescription, 2> attrs{};
1664 attrs[0].binding = 0;
1665 attrs[0].location = 0;
1666 attrs[0].format = VK_FORMAT_R32G32_SFLOAT;
1667 attrs[0].offset = 0;
1668 attrs[1].binding = 0;
1669 attrs[1].location = 1;
1670 attrs[1].format = VK_FORMAT_R32G32_SFLOAT;
1671 attrs[1].offset = sizeof(float) * 2;
1672
1673 VkPipelineVertexInputStateCreateInfo vertexInput{};
1674 vertexInput.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO;
1675 vertexInput.vertexBindingDescriptionCount = 1;
1676 vertexInput.pVertexBindingDescriptions = &binding;
1677 vertexInput.vertexAttributeDescriptionCount = static_cast<uint32_t>(attrs.size());
1678 vertexInput.pVertexAttributeDescriptions = attrs.data();
1679
1680 VkPipelineInputAssemblyStateCreateInfo inputAssembly{};
1681 inputAssembly.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO;
1682 inputAssembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
1683
1684 const std::array<VkDynamicState, 2> dynamicStates = {VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR};
1685 VkPipelineDynamicStateCreateInfo dynamicInfo{};
1686 dynamicInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO;
1687 dynamicInfo.dynamicStateCount = static_cast<uint32_t>(dynamicStates.size());
1688 dynamicInfo.pDynamicStates = dynamicStates.data();
1689
1690 VkPipelineViewportStateCreateInfo viewportState{};
1691 viewportState.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO;
1692 viewportState.viewportCount = 1;
1693 viewportState.scissorCount = 1;
1694
1695 VkPipelineRasterizationStateCreateInfo rasterizer{};
1696 rasterizer.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO;
1697 rasterizer.polygonMode = VK_POLYGON_MODE_FILL;
1698 rasterizer.cullMode = VK_CULL_MODE_NONE;
1699 rasterizer.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
1700 rasterizer.lineWidth = 1.0f;
1701
1702 VkPipelineMultisampleStateCreateInfo multisample{};
1703 multisample.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO;
1704 multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;
1705
1706 VkPipelineDepthStencilStateCreateInfo depthStencil{};
1707 depthStencil.sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO;
1708 depthStencil.depthTestEnable = VK_FALSE;
1709 depthStencil.depthWriteEnable = VK_FALSE;
1710
1711 VkPipelineColorBlendAttachmentState blendAttachment{};
1712 blendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
1713 VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
1714 blendAttachment.blendEnable = VK_FALSE;
1715
1716 VkPipelineColorBlendStateCreateInfo colorBlend{};
1717 colorBlend.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO;
1718 colorBlend.attachmentCount = 1;
1719 colorBlend.pAttachments = &blendAttachment;
1720
1721 VkPushConstantRange pushRange{};
1722 pushRange.stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT;
1723 pushRange.size = sizeof(float) * 12;
1724
1725 VkPipelineLayoutCreateInfo layoutInfo{};
1726 layoutInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;
1727 layoutInfo.setLayoutCount = 1;
1728 layoutInfo.pSetLayouts = &displayDSLayout;
1729 layoutInfo.pushConstantRangeCount = 1;
1730 layoutInfo.pPushConstantRanges = &pushRange;
1731 VK_CHECK_RESULT(vkCreatePipelineLayout(device, &layoutInfo, nullptr, &displayPipeLayout));
1732
1733 VkPipelineRenderingCreateInfo renderingInfo{};
1734 renderingInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO;
1735 renderingInfo.colorAttachmentCount = 1;
1736 renderingInfo.pColorAttachmentFormats = &swapchain_format;
1737 if (depth_format != VK_FORMAT_UNDEFINED) {
1738 renderingInfo.depthAttachmentFormat = depth_format;
1739 }
1740
1741 VkGraphicsPipelineCreateInfo pipelineInfo{};
1742 pipelineInfo.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO;
1743 pipelineInfo.pNext = &renderingInfo;
1744 pipelineInfo.stageCount = static_cast<uint32_t>(stages.size());
1745 pipelineInfo.pStages = stages.data();
1746 pipelineInfo.pVertexInputState = &vertexInput;
1747 pipelineInfo.pInputAssemblyState = &inputAssembly;
1748 pipelineInfo.pViewportState = &viewportState;
1749 pipelineInfo.pRasterizationState = &rasterizer;
1750 pipelineInfo.pMultisampleState = &multisample;
1751 pipelineInfo.pDepthStencilState = &depthStencil;
1752 pipelineInfo.pColorBlendState = &colorBlend;
1753 pipelineInfo.pDynamicState = &dynamicInfo;
1754 pipelineInfo.layout = displayPipeLayout;
1755 pipelineInfo.renderPass = VK_NULL_HANDLE;
1756 VK_CHECK_RESULT(vkCreateGraphicsPipelines(device, VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &displayPipeline));
1757
1758 vkDestroyShaderModule(device, fragModule, nullptr);
1759 vkDestroyShaderModule(device, vertModule, nullptr);
1760 }
1761
1762 void createDisplayResources() {
1763 createDisplayBuffers();
1764 createDisplayDescriptorSet();
1765 rebuildDisplayPipeline();
1766 std::cout << "compute_shader: display path active: Vulkan compute output image -> fullscreen sampled draw (no readback, no sprite upload)\n";
1767 }
1768
1769 void writeBlendDS(VkDescriptorSet descriptorSet, VkImageView srcView) {
1770 VkDescriptorImageInfo destInfo{VK_NULL_HANDLE, outImg.view, VK_IMAGE_LAYOUT_GENERAL};
1771 VkDescriptorImageInfo srcInfo{computeSampler, srcView, VK_IMAGE_LAYOUT_GENERAL};
1772
1773 std::vector<VkDescriptorImageInfo> historyInfos(HISTORY_SIZE);
1774 for (int index = 0; index < HISTORY_SIZE; ++index) {
1775 historyInfos[index] = {computeSampler, histImg[index].view, VK_IMAGE_LAYOUT_GENERAL};
1776 }
1777
1778 std::array<VkWriteDescriptorSet, 3> writes{};
1779 writes[0] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, descriptorSet, 0, 0, 1,
1780 VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &destInfo, nullptr, nullptr};
1781 writes[1] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, descriptorSet, 1, 0, 1,
1782 VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, &srcInfo, nullptr, nullptr};
1783 writes[2] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, descriptorSet, 2, 0, HISTORY_SIZE,
1784 VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, historyInfos.data(), nullptr, nullptr};
1785 vkUpdateDescriptorSets(device, static_cast<uint32_t>(writes.size()), writes.data(), 0, nullptr);
1786 }
1787
1788 void uploadToImage(const void *data, int srcPitch, ComputeImage &img) {
1789 const VkDeviceSize bytes = static_cast<VkDeviceSize>(texWidth) * texHeight * 4;
1790 const int tightPitch = texWidth * 4;
1791 if (data == nullptr || srcPitch < tightPitch) {
1792 return;
1793 }
1794
1795 void *mapped = nullptr;
1796 VK_CHECK_RESULT(vkMapMemory(device, stagingMem, 0, bytes, 0, &mapped));
1797 if (srcPitch == tightPitch) {
1798 std::memcpy(mapped, data, static_cast<size_t>(bytes));
1799 } else {
1800 const auto *src = static_cast<const uint8_t *>(data);
1801 auto *dst = static_cast<uint8_t *>(mapped);
1802 for (int row = 0; row < texHeight; ++row) {
1803 std::memcpy(dst + static_cast<size_t>(row) * tightPitch,
1804 src + static_cast<size_t>(row) * srcPitch,
1805 static_cast<size_t>(tightPitch));
1806 }
1807 }
1808 vkUnmapMemory(device, stagingMem);
1809
1810 const VkCommandBuffer cmd = beginSingleTimeCommands();
1811
1812 transitionImageLayout(
1813 cmd,
1814 img.image,
1815 VK_IMAGE_LAYOUT_GENERAL,
1816 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
1817 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
1818 VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT,
1819 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
1820 VK_ACCESS_2_TRANSFER_WRITE_BIT);
1821
1822 VkBufferImageCopy2 region{};
1823 region.sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2;
1824 region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
1825 region.imageExtent = {static_cast<uint32_t>(texWidth), static_cast<uint32_t>(texHeight), 1};
1826
1827 VkCopyBufferToImageInfo2 copyInfo{};
1828 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_BUFFER_TO_IMAGE_INFO_2;
1829 copyInfo.srcBuffer = stagingBuf;
1830 copyInfo.dstImage = img.image;
1831 copyInfo.dstImageLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
1832 copyInfo.regionCount = 1;
1833 copyInfo.pRegions = &region;
1834 vkCmdCopyBufferToImage2(cmd, &copyInfo);
1835
1836 transitionImageLayout(
1837 cmd,
1838 img.image,
1839 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
1840 VK_IMAGE_LAYOUT_GENERAL,
1841 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
1842 VK_ACCESS_2_TRANSFER_WRITE_BIT,
1843 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
1844 VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1845
1846 endSingleTimeCommands(cmd);
1847 }
1848
1849#ifdef MXVK_CUDA
1850 bool uploadGpuToImage(const cv::cuda::GpuMat &rgba, cv::cuda::Stream &stream, ComputeImage &img) {
1851 if (rgba.empty() || rgba.type() != CV_8UC4 || rgba.cols != texWidth || rgba.rows != texHeight) {
1852 return false;
1853 }
1854 if (!ensureCudaInterop(img)) {
1855 return false;
1856 }
1857
1858 cudaStream_t cudaStream = mxvk::cuda_stream_handle(stream);
1859 if (!img.cudaUploadLogged) {
1860 std::cout << "compute_shader: CUDA interop upload: copying "
1861 << rgba.cols << "x" << rgba.rows
1862 << " RGBA GpuMat to Vulkan compute storage image via cudaArray (source pitch="
1863 << static_cast<unsigned long long>(rgba.step)
1864 << " bytes, copy row bytes="
1865 << static_cast<unsigned long long>(static_cast<size_t>(rgba.cols) * 4U)
1866 << ")\n";
1867 img.cudaUploadLogged = true;
1868 }
1869
1870 cudaError_t cudaResult = cudaMemcpy2DToArrayAsync(
1871 img.cudaArray, 0, 0, rgba.ptr(), rgba.step,
1872 static_cast<size_t>(rgba.cols) * 4U, static_cast<size_t>(rgba.rows),
1873 cudaMemcpyDeviceToDevice, cudaStream);
1874 if (cudaResult != cudaSuccess) {
1875 std::cout << "compute_shader: CUDA interop compute input copy failed: "
1876 << cudaGetErrorString(cudaResult) << "\n";
1877 return false;
1878 }
1879
1880 cudaResult = cudaStreamSynchronize(cudaStream);
1881 if (cudaResult != cudaSuccess) {
1882 std::cout << "compute_shader: CUDA interop compute input sync failed: "
1883 << cudaGetErrorString(cudaResult) << "\n";
1884 return false;
1885 }
1886
1887 const VkCommandBuffer cmd = beginSingleTimeCommands();
1888 transitionImageLayout(
1889 cmd,
1890 img.image,
1891 VK_IMAGE_LAYOUT_GENERAL,
1892 VK_IMAGE_LAYOUT_GENERAL,
1893 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
1894 VK_ACCESS_2_MEMORY_WRITE_BIT,
1895 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
1896 VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1897 endSingleTimeCommands(cmd);
1898
1899 if (!img.cudaBarrierLogged) {
1900 std::cout << "compute_shader: CUDA interop sync: CUDA stream synchronized; Vulkan records GENERAL -> GENERAL memory barrier before compute sampling\n";
1901 img.cudaBarrierLogged = true;
1902 }
1903 return true;
1904 }
1905#endif
1906
1907 void dispatchOne(VkCommandBuffer cmd, VkDescriptorSet descriptorSet, int mode) {
1908 ComputePC pc{};
1909 pc.mode = (!spvFiles.empty() && spvFiles[currentSpvIndex] == MODE_SHADER_NAME) ? shaderMode : mode;
1910 pc.historyCount = historyCount;
1911 pc.historyIdx = currentHistIdx;
1912 pc.square_size = currentSquare;
1913 pc.history_dir = currentDir;
1914 pc.alpha = alpha;
1915 pc.do_invert = 0;
1916 pc.do_swap = 0;
1917
1918 vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, compPipeline);
1919 vkCmdBindDescriptorSets(
1920 cmd,
1921 VK_PIPELINE_BIND_POINT_COMPUTE,
1922 compPipeLayout,
1923 0,
1924 1,
1925 &descriptorSet,
1926 0,
1927 nullptr);
1928 vkCmdPushConstants(cmd, compPipeLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
1929
1930 const uint32_t groupX = (static_cast<uint32_t>(texWidth) + 15) / 16;
1931 const uint32_t groupY = (static_cast<uint32_t>(texHeight) + 15) / 16;
1932 vkCmdDispatch(cmd, groupX, groupY, 1);
1933 }
1934
1935 void renderComputeOutput(VkCommandBuffer cmd) {
1936 if (displayPipeline == VK_NULL_HANDLE || displayPipeLayout == VK_NULL_HANDLE ||
1937 displayDS == VK_NULL_HANDLE || displayVertexBuffer == VK_NULL_HANDLE ||
1938 displayIndexBuffer == VK_NULL_HANDLE) {
1939 return;
1940 }
1941
1942 vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, displayPipeline);
1943 vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, displayPipeLayout,
1944 0, 1, &displayDS, 0, nullptr);
1945
1946 const VkDeviceSize offset = 0;
1947 vkCmdBindVertexBuffers(cmd, 0, 1, &displayVertexBuffer, &offset);
1948 vkCmdBindIndexBuffer(cmd, displayIndexBuffer, 0, VK_INDEX_TYPE_UINT16);
1949
1950 struct DisplayPC {
1951 float screenWidth;
1952 float screenHeight;
1953 float spritePosX;
1954 float spritePosY;
1955 float spriteSizeW;
1956 float spriteSizeH;
1957 float effectsOn;
1958 float padding2;
1959 float params[4];
1960 } pc{
1961 static_cast<float>(swapchain_extent.width),
1962 static_cast<float>(swapchain_extent.height),
1963 0.0f,
1964 0.0f,
1965 static_cast<float>(swapchain_extent.width),
1966 static_cast<float>(swapchain_extent.height),
1967 0.0f,
1968 0.0f,
1969 {0.0f, 0.0f, 0.0f, 0.0f},
1970 };
1971
1972 vkCmdPushConstants(cmd, displayPipeLayout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT,
1973 0, sizeof(DisplayPC), &pc);
1974 vkCmdDrawIndexed(cmd, 6, 1, 0, 0, 0);
1975 }
1976
1977 void computeBarrier(VkCommandBuffer cmd, VkImage img) const {
1978 transitionImageLayout(
1979 cmd,
1980 img,
1981 VK_IMAGE_LAYOUT_GENERAL,
1982 VK_IMAGE_LAYOUT_GENERAL,
1983 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
1984 VK_ACCESS_2_SHADER_WRITE_BIT,
1985 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
1986 VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
1987 }
1988
1989 void runComputeFrame() {
1990 const VkCommandBuffer cmd = beginSingleTimeCommands();
1991 int srcIdx = 0;
1992 int dstIdx = 1;
1993 const bool isModeShader = !spvFiles.empty() && spvFiles[currentSpvIndex] == MODE_SHADER_NAME;
1994
1995 if (isModeShader) {
1996 std::array<VkImageMemoryBarrier2, 2> barriers{};
1997 barriers[0] = makeImageBarrier(
1998 workImg[srcIdx].image,
1999 VK_IMAGE_LAYOUT_GENERAL,
2000 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
2001 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2002 VK_ACCESS_2_SHADER_WRITE_BIT,
2003 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2004 VK_ACCESS_2_TRANSFER_READ_BIT);
2005
2006 barriers[1] = makeImageBarrier(
2007 histImg[historyIndex].image,
2008 VK_IMAGE_LAYOUT_GENERAL,
2009 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
2010 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2011 VK_ACCESS_2_SHADER_READ_BIT,
2012 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2013 VK_ACCESS_2_TRANSFER_WRITE_BIT);
2014
2015 VkDependencyInfo dependencyInfo{};
2016 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
2017 dependencyInfo.imageMemoryBarrierCount = static_cast<uint32_t>(barriers.size());
2018 dependencyInfo.pImageMemoryBarriers = barriers.data();
2019 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
2020
2021 VkImageCopy2 copy{};
2022 copy.sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2;
2023 copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
2024 copy.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
2025 copy.extent = {static_cast<uint32_t>(texWidth), static_cast<uint32_t>(texHeight), 1};
2026
2027 VkCopyImageInfo2 copyInfo{};
2028 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_INFO_2;
2029 copyInfo.srcImage = workImg[srcIdx].image;
2030 copyInfo.srcImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
2031 copyInfo.dstImage = histImg[historyIndex].image;
2032 copyInfo.dstImageLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2033 copyInfo.regionCount = 1;
2034 copyInfo.pRegions = &copy;
2035 vkCmdCopyImage2(cmd, &copyInfo);
2036
2037 barriers[0] = makeImageBarrier(
2038 workImg[srcIdx].image,
2039 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
2040 VK_IMAGE_LAYOUT_GENERAL,
2041 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2042 VK_ACCESS_2_TRANSFER_READ_BIT,
2043 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2044 VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
2045
2046 barriers[1] = makeImageBarrier(
2047 histImg[historyIndex].image,
2048 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
2049 VK_IMAGE_LAYOUT_GENERAL,
2050 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2051 VK_ACCESS_2_TRANSFER_WRITE_BIT,
2052 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2053 VK_ACCESS_2_SHADER_READ_BIT);
2054
2055 dependencyInfo = {};
2056 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
2057 dependencyInfo.imageMemoryBarrierCount = static_cast<uint32_t>(barriers.size());
2058 dependencyInfo.pImageMemoryBarriers = barriers.data();
2059 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
2060
2061 if (historyCount < HISTORY_SIZE) {
2062 ++historyCount;
2063 }
2064 historyIndex = (historyIndex + 1) % HISTORY_SIZE;
2065 dispatchOne(cmd, blendDS[srcIdx], shaderMode);
2066 } else {
2067 const int passes = 3 + (std::rand() % 7);
2068 for (int pass = 0; pass < passes; ++pass) {
2069 dispatchOne(cmd, blurDS[dstIdx], 0);
2070 computeBarrier(cmd, workImg[dstIdx].image);
2071 std::swap(srcIdx, dstIdx);
2072 }
2073
2074 std::array<VkImageMemoryBarrier2, 2> barriers{};
2075 barriers[0] = makeImageBarrier(
2076 workImg[srcIdx].image,
2077 VK_IMAGE_LAYOUT_GENERAL,
2078 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
2079 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2080 VK_ACCESS_2_SHADER_WRITE_BIT,
2081 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2082 VK_ACCESS_2_TRANSFER_READ_BIT);
2083
2084 barriers[1] = makeImageBarrier(
2085 histImg[historyIndex].image,
2086 VK_IMAGE_LAYOUT_GENERAL,
2087 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
2088 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2089 VK_ACCESS_2_SHADER_READ_BIT,
2090 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2091 VK_ACCESS_2_TRANSFER_WRITE_BIT);
2092
2093 VkDependencyInfo dependencyInfo{};
2094 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
2095 dependencyInfo.imageMemoryBarrierCount = static_cast<uint32_t>(barriers.size());
2096 dependencyInfo.pImageMemoryBarriers = barriers.data();
2097 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
2098
2099 VkImageCopy2 copy{};
2100 copy.sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2;
2101 copy.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
2102 copy.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
2103 copy.extent = {static_cast<uint32_t>(texWidth), static_cast<uint32_t>(texHeight), 1};
2104
2105 VkCopyImageInfo2 copyInfo{};
2106 copyInfo.sType = VK_STRUCTURE_TYPE_COPY_IMAGE_INFO_2;
2107 copyInfo.srcImage = workImg[srcIdx].image;
2108 copyInfo.srcImageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
2109 copyInfo.dstImage = histImg[historyIndex].image;
2110 copyInfo.dstImageLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2111 copyInfo.regionCount = 1;
2112 copyInfo.pRegions = &copy;
2113 vkCmdCopyImage2(cmd, &copyInfo);
2114
2115 barriers[0] = makeImageBarrier(
2116 workImg[srcIdx].image,
2117 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
2118 VK_IMAGE_LAYOUT_GENERAL,
2119 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2120 VK_ACCESS_2_TRANSFER_READ_BIT,
2121 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2122 VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT);
2123
2124 barriers[1] = makeImageBarrier(
2125 histImg[historyIndex].image,
2126 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
2127 VK_IMAGE_LAYOUT_GENERAL,
2128 VK_PIPELINE_STAGE_2_TRANSFER_BIT,
2129 VK_ACCESS_2_TRANSFER_WRITE_BIT,
2130 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2131 VK_ACCESS_2_SHADER_READ_BIT);
2132
2133 dependencyInfo = {};
2134 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
2135 dependencyInfo.imageMemoryBarrierCount = static_cast<uint32_t>(barriers.size());
2136 dependencyInfo.pImageMemoryBarriers = barriers.data();
2137 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
2138
2139 if (historyCount < HISTORY_SIZE) {
2140 ++historyCount;
2141 }
2142 historyIndex = (historyIndex + 1) % HISTORY_SIZE;
2143
2144 const bool isMetalMedian =
2145 !spvFiles.empty() && spvFiles[currentSpvIndex].find("metalmedianblend") != std::string::npos;
2146 dispatchOne(cmd, blendDS[srcIdx], isMetalMedian ? 2 : 1);
2147 }
2148
2149 transitionImageLayout(
2150 cmd,
2151 outImg.image,
2152 VK_IMAGE_LAYOUT_GENERAL,
2153 VK_IMAGE_LAYOUT_GENERAL,
2154 VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
2155 VK_ACCESS_2_SHADER_WRITE_BIT,
2156 VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT,
2157 VK_ACCESS_2_SHADER_SAMPLED_READ_BIT);
2158
2159 endSingleTimeCommands(cmd);
2160 }
2161
2162 void tickAnimState() {
2163 if (currentDir == 1) {
2164 if (++currentHistIdx >= HISTORY_SIZE - 1) {
2165 currentHistIdx = HISTORY_SIZE - 1;
2166 currentDir = -1;
2167 }
2168 } else if (--currentHistIdx <= 0) {
2169 currentHistIdx = 0;
2170 currentDir = 1;
2171 }
2172
2173 if (squareDir == 1) {
2174 currentSquare += 2;
2175 if (currentSquare >= 64) {
2176 currentSquare = 64;
2177 squareDir = 0;
2178 }
2179 } else {
2180 currentSquare -= 2;
2181 if (currentSquare <= 2) {
2182 currentSquare = 2;
2183 squareDir = 1;
2184 }
2185 }
2186
2187 static int alphaDir = 1;
2188 if (alphaDir == 1) {
2189 alpha += 0.005f;
2190 if (alpha >= (255.0f / 32.0f)) {
2191 alpha = 255.0f / 32.0f;
2192 alphaDir = -1;
2193 }
2194 } else {
2195 alpha -= 0.005f;
2196 if (alpha <= 1.0f) {
2197 alpha = 1.0f;
2198 alphaDir = 1;
2199 }
2200 }
2201 }
2202
2203 static VkImageMemoryBarrier2 makeImageBarrier(VkImage img,
2204 VkImageLayout oldLayout,
2205 VkImageLayout newLayout,
2206 VkPipelineStageFlags2 srcStage,
2207 VkAccessFlags2 srcAccess,
2208 VkPipelineStageFlags2 dstStage,
2209 VkAccessFlags2 dstAccess) {
2210 VkImageMemoryBarrier2 barrier{};
2211 barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2;
2212 barrier.srcStageMask = srcStage;
2213 barrier.srcAccessMask = srcAccess;
2214 barrier.dstStageMask = dstStage;
2215 barrier.dstAccessMask = dstAccess;
2216 barrier.oldLayout = oldLayout;
2217 barrier.newLayout = newLayout;
2218 barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
2219 barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
2220 barrier.image = img;
2221 barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
2222 return barrier;
2223 }
2224
2225 static void transitionImageLayout(VkCommandBuffer cmd,
2226 VkImage img,
2227 VkImageLayout oldLayout,
2228 VkImageLayout newLayout,
2229 VkPipelineStageFlags2 srcStage,
2230 VkAccessFlags2 srcAccess,
2231 VkPipelineStageFlags2 dstStage,
2232 VkAccessFlags2 dstAccess) {
2233 const VkImageMemoryBarrier2 barrier = makeImageBarrier(
2234 img,
2235 oldLayout,
2236 newLayout,
2237 srcStage,
2238 srcAccess,
2239 dstStage,
2240 dstAccess);
2241
2242 VkDependencyInfo dependencyInfo{};
2243 dependencyInfo.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO;
2244 dependencyInfo.imageMemoryBarrierCount = 1;
2245 dependencyInfo.pImageMemoryBarriers = &barrier;
2246 vkCmdPipelineBarrier2(cmd, &dependencyInfo);
2247 }
2248
2249 void destroyDisplayResources() {
2250 if (displayPipeline != VK_NULL_HANDLE) {
2251 vkDestroyPipeline(device, displayPipeline, nullptr);
2252 displayPipeline = VK_NULL_HANDLE;
2253 }
2254 if (displayPipeLayout != VK_NULL_HANDLE) {
2255 vkDestroyPipelineLayout(device, displayPipeLayout, nullptr);
2256 displayPipeLayout = VK_NULL_HANDLE;
2257 }
2258 if (displayDSPool != VK_NULL_HANDLE) {
2259 vkDestroyDescriptorPool(device, displayDSPool, nullptr);
2260 displayDSPool = VK_NULL_HANDLE;
2261 displayDS = VK_NULL_HANDLE;
2262 }
2263 if (displayDSLayout != VK_NULL_HANDLE) {
2264 vkDestroyDescriptorSetLayout(device, displayDSLayout, nullptr);
2265 displayDSLayout = VK_NULL_HANDLE;
2266 }
2267 if (displayVertexBuffer != VK_NULL_HANDLE) {
2268 vkDestroyBuffer(device, displayVertexBuffer, nullptr);
2269 displayVertexBuffer = VK_NULL_HANDLE;
2270 }
2271 if (displayVertexMemory != VK_NULL_HANDLE) {
2272 vkFreeMemory(device, displayVertexMemory, nullptr);
2273 displayVertexMemory = VK_NULL_HANDLE;
2274 }
2275 if (displayIndexBuffer != VK_NULL_HANDLE) {
2276 vkDestroyBuffer(device, displayIndexBuffer, nullptr);
2277 displayIndexBuffer = VK_NULL_HANDLE;
2278 }
2279 if (displayIndexMemory != VK_NULL_HANDLE) {
2280 vkFreeMemory(device, displayIndexMemory, nullptr);
2281 displayIndexMemory = VK_NULL_HANDLE;
2282 }
2283 }
2284
2285 void destroyComputeResources() {
2286 if (device == VK_NULL_HANDLE) {
2287 return;
2288 }
2289
2290 vkDeviceWaitIdle(device);
2291 destroyDisplayResources();
2292
2293 auto destroyImage = [&](ComputeImage &img) {
2294#ifdef MXVK_CUDA
2295 destroyCudaInterop(img);
2296#endif
2297 if (img.view != VK_NULL_HANDLE) {
2298 vkDestroyImageView(device, img.view, nullptr);
2299 }
2300 if (img.image != VK_NULL_HANDLE) {
2301 vkDestroyImage(device, img.image, nullptr);
2302 }
2303 if (img.memory != VK_NULL_HANDLE) {
2304 vkFreeMemory(device, img.memory, nullptr);
2305 }
2306 img = {};
2307 };
2308
2309 destroyImage(workImg[0]);
2310 destroyImage(workImg[1]);
2311 for (ComputeImage &img : histImg) {
2312 destroyImage(img);
2313 }
2314 destroyImage(outImg);
2315
2316 if (computeSampler != VK_NULL_HANDLE) {
2317 vkDestroySampler(device, computeSampler, nullptr);
2318 computeSampler = VK_NULL_HANDLE;
2319 }
2320 if (compDSPool != VK_NULL_HANDLE) {
2321 vkDestroyDescriptorPool(device, compDSPool, nullptr);
2322 compDSPool = VK_NULL_HANDLE;
2323 }
2324 if (compPipeline != VK_NULL_HANDLE) {
2325 vkDestroyPipeline(device, compPipeline, nullptr);
2326 compPipeline = VK_NULL_HANDLE;
2327 }
2328 if (compPipeLayout != VK_NULL_HANDLE) {
2329 vkDestroyPipelineLayout(device, compPipeLayout, nullptr);
2330 compPipeLayout = VK_NULL_HANDLE;
2331 }
2332 if (compDSLayout != VK_NULL_HANDLE) {
2333 vkDestroyDescriptorSetLayout(device, compDSLayout, nullptr);
2334 compDSLayout = VK_NULL_HANDLE;
2335 }
2336 if (stagingBuf != VK_NULL_HANDLE) {
2337 vkDestroyBuffer(device, stagingBuf, nullptr);
2338 stagingBuf = VK_NULL_HANDLE;
2339 }
2340 if (stagingMem != VK_NULL_HANDLE) {
2341 vkFreeMemory(device, stagingMem, nullptr);
2342 stagingMem = VK_NULL_HANDLE;
2343 }
2344 if (readbackBuf != VK_NULL_HANDLE) {
2345 vkDestroyBuffer(device, readbackBuf, nullptr);
2346 readbackBuf = VK_NULL_HANDLE;
2347 }
2348 if (readbackMem != VK_NULL_HANDLE) {
2349 vkFreeMemory(device, readbackMem, nullptr);
2350 readbackMem = VK_NULL_HANDLE;
2351 }
2352 }
2353};
2354
2355int main(int argc, char **argv) {
2356 std::srand(static_cast<unsigned>(std::time(nullptr)));
2357
2358 try {
2359 Arguments args = proc_args(argc, argv);
2360 if (args.path == ".") {
2362 }
2363
2364 ComputeWindow window(args);
2365 window.loop();
2366 } catch (const mxvk::Exception &e) {
2367 std::cerr << "mxvk: Exception: " << e.text() << "\n";
2368 return EXIT_FAILURE;
2369 } catch (const ArgException<std::string> &e) {
2370 std::cerr << std::format("mxvk: Argument Exception: {}\n", e.text());
2371 return EXIT_FAILURE;
2372 }
2373
2374 return EXIT_SUCCESS;
2375}
Lightweight, header-only, template command-line argument parser.
Arguments proc_args(int &argc, char **argv)
Parse standard libmx2 command-line options from main()'s argv.
Definition argz.hpp:872
Exception thrown by Argz::proc() on unrecognised or malformed options.
Definition argz.hpp:178
~ComputeWindow() override
Definition main.cpp:135
void onRecordCustomRendering(VkCommandBuffer cmd, uint32_t imageIndex) override
Optional hook for derived classes to record extra draw commands.
Definition main.cpp:226
void event(SDL_Event &e) override
Handle one SDL event.
Definition main.cpp:230
void proc() override
Execute one processing/update step.
Definition main.cpp:143
ComputeWindow(const Arguments &args)
Definition main.cpp:107
void onSwapchainRecreated() override
Called after swapchain and render resources are recreated.
Definition main.cpp:222
bool is_hardware_encode() const
True when FFmpeg identifies the active encoder as hardware or hybrid.
Definition mxwrite.hpp:244
bool write_cuda_rgba(void *cuda_rgba_buffer, int src_stride, bool bottom_up=false)
Queue a CUDA RGBA frame for encoding.
Definition mxwrite.cpp:2072
bool open(const std::string &filename, int width, int height, float fps, const char *crf)
Open an output file using the legacy CRF string interface.
Definition mxwrite.cpp:1039
void write(void *rgba_buffer)
Queue a host RGBA frame for immediate-mode encoding.
Definition mxwrite.cpp:1859
std::string text() const
void close()
Close the active file and release decoder resources.
double fps() const
Source frame rate, falling back to 30 fps when unknown.
int width() const
Source width in pixels.
int height() const
Source height in pixels.
bool open(const std::string &filename)
Open a video file.
bool readRgba(std::vector< uint8_t > &rgba, int &width, int &height, int &pitch, bool flipY=false)
Decode the next frame as tightly packed RGBA8.
bool using_hardware_decode() const
True when CUDA hardware decode is active.
Main Vulkan window wrapper for MXVK.
Definition mxvk.hpp:38
VkSwapchainKHR swapchain
Definition mxvk.hpp:603
void loop()
Run the main event/render loop.
Definition mxvk.cpp:627
VkDevice device
Definition mxvk.hpp:596
VkFormat swapchain_format
Definition mxvk.hpp:604
VkExtent2D swapchain_extent
Definition mxvk.hpp:606
VkFormat depth_format
Definition mxvk.hpp:605
void createDevice()
Create final device resources.
Definition mxvk.cpp:2086
void clearTextQueue()
Clear all queued text draw calls for the current frame.
Definition mxvk.cpp:4096
SDL_Window * getSDLWindow() const noexcept
Get the underlying SDL window handle.
Definition mxvk.hpp:189
void exit()
Request loop termination.
Definition mxvk.cpp:1429
VkCommandPool command_pool
Definition mxvk.hpp:616
VK_Window()=default
Construct an empty window object.
VkPhysicalDevice physical_device
Definition mxvk.hpp:595
void setFont(const std::string &fontPath, int fontSize=24)
Set the active text-render font.
Definition mxvk.cpp:4002
void printText(const std::string &text, int x, int y, const SDL_Color &col)
Queue a text string for rendering during the current frame.
Definition mxvk.cpp:4051
VkQueue graphics_queue
Definition mxvk.hpp:599
int main(int argc, char **argv)
Definition main.cpp:173
#define compute_shader_ASSET_DIR
Definition main.cpp:36
#define MXVK_VALIDATION
Definition mxvk.hpp:28
OpenCV video-capture integration for the Vulkan backend.
FFmpeg video-file capture with optional CUDA hardware decoding.
Small compatibility wrappers around OpenCV CUDA APIs.
#define VK_CHECK_RESULT(f)
FFmpeg-based video writer used by MXWrite.
Utilities for loading and saving PNG images.
Definition mxvk.hpp:31
VkShaderModule create_shader_module(VkDevice device, const std::vector< char > &spv_bytes)
Create a shader module from SPIR-V bytecode.
std::vector< char > load_spv(const std::string &path)
Load a SPIR-V file from disk.
Plain data structure returned by proc_args() with all common libmx2 CLI options.
Definition argz.hpp:730
int height
Viewport height in pixels (default: 720).
Definition argz.hpp:733
std::string path
Asset root; proc_args() defaults it to the executable directory.
Definition argz.hpp:735
int width
Viewport width in pixels (default: 1280).
Definition argz.hpp:732
int32_t mode
Definition main.cpp:95
int32_t do_swap
Definition main.cpp:101
int32_t historyIdx
Definition main.cpp:97
int32_t history_dir
Definition main.cpp:99
int32_t historyCount
Definition main.cpp:96
float alpha
Definition main.cpp:100
int32_t square_size
Definition main.cpp:98
int32_t do_invert
Definition main.cpp:102
std::string codec
Encoder selection policy or exact FFmpeg encoder name.
Definition mxwrite.hpp:96
std::string tune
Optional tuning mode.
Definition mxwrite.hpp:94
bool realtime
Enable low-latency settings.
Definition mxwrite.hpp:98
int crf
Constant Rate Factor.
Definition mxwrite.hpp:95
bool block_when_full
Pace producers to encoder throughput instead of dropping frames.
Definition mxwrite.hpp:99
std::string preset
Encoder preset name.
Definition mxwrite.hpp:93