fix(linux): fall back to the software encoder under a vendor preference

FTHRcapture_linux/CMakeLists.txt

@@ -534,6 +534,26 @@ if(BUILD_TESTING)
     add_test(NAME linux_encoder_timestamps
         COMMAND fthr_linux_encoder_timestamp_test)
 
+    add_executable(fthr_linux_encoder_candidates_test
+        tests/linux_encoder_candidates_test.cpp
+        src/encoder.cpp)
+    target_include_directories(fthr_linux_encoder_candidates_test PRIVATE src)
+    target_include_directories(fthr_linux_encoder_candidates_test SYSTEM PRIVATE
+        ${FFMPEG_INCLUDE_DIRS})
+    target_link_libraries(fthr_linux_encoder_candidates_test PRIVATE
+        $<$<BOOL:${FTHR_FFMPEG_BUNDLED}>:${FTHR_FFMPEG_LINK_LIBRARIES}>
+        $<$<NOT:$<BOOL:${FTHR_FFMPEG_BUNDLED}>>:PkgConfig::FFMPEG>
+        pthread ${CMAKE_DL_LIBS})
+    target_compile_options(fthr_linux_encoder_candidates_test PRIVATE
+        -Wall -Wextra -Wpedantic -UNDEBUG)
+    target_link_options(fthr_linux_encoder_candidates_test PRIVATE
+        "-Wl,--disable-new-dtags")
+    set_target_properties(fthr_linux_encoder_candidates_test PROPERTIES
+        BUILD_WITH_INSTALL_RPATH TRUE
+        INSTALL_RPATH "$ORIGIN/../third_party/ffmpeg/lib")
+    add_test(NAME linux_encoder_candidates
+        COMMAND fthr_linux_encoder_candidates_test)
+
     add_executable(fthr_linux_encoder_resize_test
         tests/linux_encoder_resize_test.cpp
         src/encoder.cpp)

FTHRcapture_linux/src/encoder.cpp

@@ -74,7 +74,7 @@ bool ApplyPreset(AVCodecContext* ctx, const char* codec_name, int p) {
         av_opt_set(ctx->priv_data, "preset", kKvz[p - 1], 0);
     } else if (strstr(codec_name, "libx26")) {
         // Retained for distro FFmpeg builds that still ship x264/x265. FTHR
-        // never selects these itself (see BuildCodecList) — this branch only
+        // never selects these itself (see EncoderCandidates) — this branch only
         // matters if a downstream packager re-adds them deliberately.
         static const char* kX26x[] = {
             "ultrafast","superfast","veryfast","fast","medium","slow","veryslow"};
@@ -180,15 +180,18 @@ bool Encoder::TryOpen(const char* codec_name, const EncoderConfig& cfg) {
 
 // Build priority list for Open()
 
-static void BuildCodecList(
-    CodecPref codec_pref,
-    EncoderPref encoder_pref,
-    std::vector<const char*>& out) {
+std::vector<const char*> EncoderCandidates(CodecPref codec_pref,
+                                           EncoderPref encoder_pref) {
+    std::vector<const char*> out;
     auto codec_allowed = [&](CodecPref codec) {
         return codec_pref == CodecPref::Auto || codec_pref == codec;
     };
+    // Software stays allowed under a hardware preference: a stale setting, a
+    // missing vendor runtime or a driver update must not leave the user with
+    // no replay buffer at all.
     auto backend_allowed = [&](EncoderPref backend) {
-        return encoder_pref == EncoderPref::Auto || encoder_pref == backend;
+        return encoder_pref == EncoderPref::Auto || encoder_pref == backend ||
+               backend == EncoderPref::Software;
     };
     auto add_backend = [&](EncoderPref backend) {
         if (!backend_allowed(backend)) return;
@@ -227,13 +230,14 @@ static void BuildCodecList(
     add_backend(EncoderPref::Amd);
     add_backend(EncoderPref::Intel);
     add_backend(EncoderPref::Software);
+    return out;
 }
 
 // Open — try codecs in priority order
 
 bool Encoder::Open(const EncoderConfig& cfg, std::string& codec_used_out) {
-    std::vector<const char*> candidates;
-    BuildCodecList(cfg.codec_pref, cfg.encoder_pref, candidates);
+    const std::vector<const char*> candidates =
+        EncoderCandidates(cfg.codec_pref, cfg.encoder_pref);
 
     for (const char* name : candidates) {
         std::cout << "[Encoder] Trying codec: " << name << std::endl;

FTHRcapture_linux/src/encoder.h

@@ -100,4 +100,10 @@ class Encoder {
 // before avcodec_open2. Returns false if codec_name is unrecognised.
 bool ApplyPreset(AVCodecContext* ctx, const char* codec_name, int preset_1_7);
 
+// Encoders Open tries, in order. A hardware preference narrows the hardware
+// candidates to that vendor; the LGPL software encoders always follow, so a
+// missing driver costs speed rather than the replay buffer.
+std::vector<const char*> EncoderCandidates(CodecPref codec_pref,
+                                           EncoderPref encoder_pref);
+
 } // namespace fthr

FTHRcapture_linux/tests/linux_encoder_candidates_test.cpp

@@ -0,0 +1,59 @@
+#include "encoder.h"
+
+#include <cassert>
+#include <string>
+#include <vector>
+
+namespace {
+
+std::vector<std::string> Names(fthr::CodecPref codec, fthr::EncoderPref pref) {
+    std::vector<std::string> names;
+    for (const char* name : fthr::EncoderCandidates(codec, pref)) {
+        names.emplace_back(name);
+    }
+    return names;
+}
+
+bool Contains(const std::vector<std::string>& names, const std::string& name) {
+    for (const auto& candidate : names) {
+        if (candidate == name) return true;
+    }
+    return false;
+}
+
+}  // namespace
+
+int main() {
+    using fthr::CodecPref;
+    using fthr::EncoderPref;
+
+    // Auto is backend-first: NVENC, VA-API/AMF, QSV, then software.
+    const auto automatic = Names(CodecPref::H264, EncoderPref::Auto);
+    assert((automatic == std::vector<std::string>{
+        "h264_nvenc", "h264_vaapi", "h264_amf", "h264_qsv", "libopenh264"}));
+
+    // A hardware preference narrows the hardware candidates to that vendor
+    // but keeps the software encoder last, so a missing driver does not end
+    // capture.
+    assert((Names(CodecPref::H264, EncoderPref::Nvidia) ==
+            std::vector<std::string>{"h264_nvenc", "libopenh264"}));
+    assert((Names(CodecPref::H264, EncoderPref::Amd) ==
+            std::vector<std::string>{"h264_vaapi", "h264_amf", "libopenh264"}));
+    assert((Names(CodecPref::H264, EncoderPref::Intel) ==
+            std::vector<std::string>{"h264_qsv", "libopenh264"}));
+
+    // The codec preference still holds for the software fallback.
+    assert((Names(CodecPref::HEVC, EncoderPref::Nvidia) ==
+            std::vector<std::string>{"hevc_nvenc", "libkvazaar"}));
+    const auto av1_amd = Names(CodecPref::AV1, EncoderPref::Amd);
+    assert(av1_amd.front() == "av1_vaapi");
+    assert(av1_amd.back() == "librav1e");
+    assert(Contains(av1_amd, "libsvtav1"));
+    assert(!Contains(av1_amd, "av1_nvenc"));
+
+    // Software never includes a hardware encoder.
+    for (const auto& name : Names(CodecPref::Auto, EncoderPref::Software)) {
+        assert(name.rfind("lib", 0) == 0);
+    }
+    return 0;
+}

RELEASE_NOTES.md

@@ -1,3 +1,10 @@
+# Unreleased (Linux)
+
+- Choosing NVIDIA, AMD or Intel as the encoder no longer ends capture when
+  that vendor's encoder cannot open (for example NVENC without the CUDA
+  libraries, or Intel without a oneVPL runtime); the software encoder for the
+  chosen codec is used instead, as it already was for Auto.
+
 # FTHR Clips 1.1.3-alpha
 
 Linux hardening release before the UI overhaul. Windows artifacts are unchanged apart from shared test and packaging fixes.

docs/engine-startup.md

@@ -37,6 +37,10 @@ Windows also validates output dimensions and crop geometry.
 An encoder preference doesn't guarantee availability. Windows public-alpha
 startup requires a supported hardware backend on the capture adapter. It
 doesn't silently switch to raw replay or another adapter.
+On Linux a hardware preference (1–3) limits the hardware encoders to that
+vendor, and the LGPL software encoders for the chosen codec are still tried
+afterwards; preference 0 tries every hardware backend first. The encoder
+actually opened is reported back to the UI.
 
 Microphone capture is native on Windows and handled by Python on Linux.
 Completed clips pass through UI finalization to produce the chosen audio layout.

docs/linux.md

@@ -188,6 +188,9 @@ ls -l /dev/dri/renderD*
 ```
 
 If VA-API initialization fails, the application falls back to software encoding.
+This also applies when an encoder vendor is chosen in Settings: NVIDIA, AMD or
+Intel limits which hardware encoders are tried, and the software encoder is
+still used if none of them opens.
 That fallback keeps recording available but does not qualify hardware encoding.
 The current Linux qualification does not claim NVIDIA NVENC or Intel hardware
 encoding.