From 6c497b8a80314912599e32673f2d9294ebc54af7 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Sat, 8 Aug 2026 15:11:19 +0100 Subject: [PATCH] Updated to llama.cpp `815a2a5915f22ce6a760c676389c5dfe8535c08f` (b10221) --- LLama.KernelMemory/BuilderExtensions.cs | 3 +- .../LLamaSharpTextEmbeddingGenerator.cs | 6 +- LLama.KernelMemory/LlamaSharpTextGenerator.cs | 6 +- LLama.Unittest/NativeAbiTests.cs | 69 --------- LLama.Web/Common/ModelOptions.cs | 15 +- LLama/Abstractions/IModelParams.cs | 25 ++-- LLama/Common/ModelParams.cs | 15 +- LLama/Extensions/IContextParamsExtensions.cs | 3 + LLama/Extensions/IModelParamsExtensions.cs | 17 ++- LLama/LLamaSharp.csproj | 2 +- LLama/Native/LLamaContextParams.cs | 11 ++ LLama/Native/LLamaFtype.cs | 5 + LLama/Native/LLamaModelParams.cs | 50 +++---- LLama/Native/LLamaVocabNative.cs | 9 ++ LLama/Native/Load/LLamaLoadMode.cs | 52 +++++++ LLama/Native/MtmdContextParams.cs | 20 ++- LLama/Native/NativeApi.Mtmd.cs | 131 ++++++++++++++++-- LLama/Native/NativeApi.Quantize.cs | 8 ++ LLama/Native/SafeLLamaContextHandle.cs | 18 --- LLama/Native/SafeLlamaModelHandle.cs | 26 ++++ LLama/Native/SafeMtmdModelHandle.cs | 5 +- 21 files changed, 316 insertions(+), 180 deletions(-) create mode 100644 LLama/Native/Load/LLamaLoadMode.cs diff --git a/LLama.KernelMemory/BuilderExtensions.cs b/LLama.KernelMemory/BuilderExtensions.cs index 6ab04a8bc..0749a61e9 100644 --- a/LLama.KernelMemory/BuilderExtensions.cs +++ b/LLama.KernelMemory/BuilderExtensions.cs @@ -77,8 +77,7 @@ public static IKernelMemoryBuilder WithLLamaSharpDefaults(this IKernelMemoryBuil SplitMode = config.SplitMode, BatchSize = 512, UBatchSize = 512, - FlashAttention = true, - UseMemorymap = true + FlashAttention = true }; if (weights == null) diff --git a/LLama.KernelMemory/LLamaSharpTextEmbeddingGenerator.cs b/LLama.KernelMemory/LLamaSharpTextEmbeddingGenerator.cs index 0635015df..b8b77eb2d 100644 --- a/LLama.KernelMemory/LLamaSharpTextEmbeddingGenerator.cs +++ b/LLama.KernelMemory/LLamaSharpTextEmbeddingGenerator.cs @@ -37,11 +37,10 @@ public LLamaSharpTextEmbeddingGenerator(LLamaSharpConfig config) ContextSize = config?.ContextSize ?? 2048, GpuLayerCount = config?.GpuLayerCount ?? 20, MainGpu = config?.MainGpu ?? 0, - SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer, + SplitMode = config?.SplitMode ?? GPUSplitMode.Layer, BatchSize = 512, UBatchSize = 512, FlashAttention = true, - UseMemorymap = true, PoolingType = LLamaPoolingType.Mean, }; @@ -65,11 +64,10 @@ public LLamaSharpTextEmbeddingGenerator(LLamaSharpConfig config, LLamaWeights we ContextSize = config?.ContextSize ?? 2048, GpuLayerCount = config?.GpuLayerCount ?? 20, MainGpu = config?.MainGpu ?? 0, - SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer, + SplitMode = config?.SplitMode ?? GPUSplitMode.Layer, BatchSize = 512, UBatchSize = 512, FlashAttention = true, - UseMemorymap = true, PoolingType = LLamaPoolingType.Mean, }; _weights = weights; diff --git a/LLama.KernelMemory/LlamaSharpTextGenerator.cs b/LLama.KernelMemory/LlamaSharpTextGenerator.cs index 5c965b266..18ca1fda6 100644 --- a/LLama.KernelMemory/LlamaSharpTextGenerator.cs +++ b/LLama.KernelMemory/LlamaSharpTextGenerator.cs @@ -38,8 +38,7 @@ public LlamaSharpTextGenerator(LLamaSharpConfig config) SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer, BatchSize = 512, UBatchSize = 512, - FlashAttention = true, - UseMemorymap = true + FlashAttention = true }; _weights = LLamaWeights.LoadFromFile(@params); _executor = new StatelessExecutor(_weights, @params); @@ -66,8 +65,7 @@ public LlamaSharpTextGenerator(LLamaWeights weights, LLamaSharpConfig config, St SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer, BatchSize = 512, UBatchSize = 512, - FlashAttention = true, - UseMemorymap = true + FlashAttention = true }; _executor = executor ?? new StatelessExecutor(_weights, @params); _defaultInferenceParams = inferenceParams; diff --git a/LLama.Unittest/NativeAbiTests.cs b/LLama.Unittest/NativeAbiTests.cs index b5c1667c0..f2fe58e3d 100644 --- a/LLama.Unittest/NativeAbiTests.cs +++ b/LLama.Unittest/NativeAbiTests.cs @@ -1,5 +1,4 @@ using LLama.Native; -using System.Net.Mime; using System.Runtime.InteropServices; namespace LLama.Unittest @@ -33,74 +32,6 @@ public void TokenDataArrayLayoutMatchesNative() Assert.Equal(expectedSize, Marshal.SizeOf()); } - [Fact] - public void ContextParamsSizeMatchesNative() - { - var pointerSize = IntPtr.Size; - var fields = new List<(int size, int align)> - { - (sizeof(uint), 4), // n_ctx - (sizeof(uint), 4), // n_batch - (sizeof(uint), 4), // n_ubatch - (sizeof(uint), 4), // n_seq_max - (sizeof(uint), 4), // n_rs_seq - (sizeof(int), 4), // n_threads - (sizeof(int), 4), // n_threads_batch - (sizeof(LLamaContextType), 4), // ctx_type - (sizeof(int), 4), // rope_scaling_type - (sizeof(int), 4), // pooling_type - (sizeof(int), 4), // attention_type - (sizeof(int), 4), // flash_attn_type - (sizeof(float), 4), // rope_freq_base - (sizeof(float), 4), // rope_freq_scale - (sizeof(float), 4), // yarn_ext_factor - (sizeof(float), 4), // yarn_attn_factor - (sizeof(float), 4), // yarn_beta_fast - (sizeof(float), 4), // yarn_beta_slow - (sizeof(uint), 4), // yarn_orig_ctx - (sizeof(float), 4), // defrag_thold - (pointerSize, pointerSize), // cb_eval - (pointerSize, pointerSize), // cb_eval_user_data - (sizeof(int), 4), // type_k - (sizeof(int), 4), // type_v - (pointerSize, pointerSize), // abort_callback - (pointerSize, pointerSize), // abort_callback_user_data - (sizeof(sbyte), 1), // embeddings - (sizeof(sbyte), 1), // offload_kqv - (sizeof(sbyte), 1), // no_perf - (sizeof(sbyte), 1), // op_offload - (sizeof(sbyte), 1), // swa_full - (sizeof(sbyte), 1), // kv_unified - (pointerSize, pointerSize), // samplers - (pointerSize, pointerSize), // n_samplers - }; - - var expectedSize = ComputeSize(fields); - Assert.Equal(expectedSize, Marshal.SizeOf()); - } - - [Fact] - public void ModelParamsBoolBlockMatchesNative() - { - var pointerSize = IntPtr.Size; - - // Get the field immediately before the first boolean field - var kvOffset = Marshal.OffsetOf(nameof(LLamaModelParams.kv_overrides)).ToInt32(); - - // Get the first boolean field - var vocabOffset = Marshal.OffsetOf("_vocab_only").ToInt32(); - - // Check first boolean field is one ptr-size after the other - Assert.Equal(kvOffset + pointerSize, vocabOffset); - Assert.Equal(vocabOffset + 1, Marshal.OffsetOf("_use_mmap").ToInt32()); - Assert.Equal(vocabOffset + 2, Marshal.OffsetOf("_use_direct_io").ToInt32()); - Assert.Equal(vocabOffset + 3, Marshal.OffsetOf("_use_mlock").ToInt32()); - Assert.Equal(vocabOffset + 4, Marshal.OffsetOf("_check_tensors").ToInt32()); - Assert.Equal(vocabOffset + 5, Marshal.OffsetOf("_use_extra_bufts").ToInt32()); - Assert.Equal(vocabOffset + 6, Marshal.OffsetOf("_no_host").ToInt32()); - Assert.Equal(vocabOffset + 7, Marshal.OffsetOf("_no_alloc").ToInt32()); - } - [Fact] public void SamplerInterfaceSizeMatchesNative() { diff --git a/LLama.Web/Common/ModelOptions.cs b/LLama.Web/Common/ModelOptions.cs index a8d4fb4ee..5f0bb64d2 100644 --- a/LLama.Web/Common/ModelOptions.cs +++ b/LLama.Web/Common/ModelOptions.cs @@ -43,15 +43,6 @@ public class ModelOptions /// public bool Embeddings { get; set; } - /// - public bool UseMemorymap { get; set; } = true; - - /// - public bool UseDirectIO { get; } - - /// - public bool UseMemoryLock { get; set; } = false; - /// public string ModelPath { get; set; } @@ -76,6 +67,12 @@ public class ModelOptions /// public List MetadataOverrides { get; } = new(); + /// + public LLamaLoadMode LoadMode { get; set; } + + /// + public bool LoadMTP { get; set; } + /// public float? RopeFrequencyBase { get; set; } diff --git a/LLama/Abstractions/IModelParams.cs b/LLama/Abstractions/IModelParams.cs index 49af09001..6cce4372c 100644 --- a/LLama/Abstractions/IModelParams.cs +++ b/LLama/Abstractions/IModelParams.cs @@ -49,21 +49,6 @@ public interface IModelParams /// int GpuLayerCount { get; } - /// - /// Use mmap for faster loads (use_mmap) - /// - bool UseMemorymap { get; } - - /// - /// Use direct io, takes precedence over use_mmap when supported - /// - bool UseDirectIO { get; } - - /// - /// Use mlock to keep model in memory (use_mlock) - /// - bool UseMemoryLock { get; } - /// /// Model path (model) /// @@ -88,6 +73,16 @@ public interface IModelParams /// Override specific metadata items in the model /// List MetadataOverrides { get; } + + /// + /// How the load this model + /// + LLamaLoadMode LoadMode { get; } + + /// + /// Whether to load MTP layers + /// + bool LoadMTP { get; } } /// diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs index 4579dbaf8..55e7c4300 100644 --- a/LLama/Common/ModelParams.cs +++ b/LLama/Common/ModelParams.cs @@ -37,15 +37,6 @@ public record ModelParams /// public uint RecurrentRollbackSnapshots { get; set; } = 0; - /// - public bool UseMemorymap { get; set; } = true; - - /// - public bool UseDirectIO { get; set; } - - /// - public bool UseMemoryLock { get; set; } - /// public string ModelPath { get; set; } @@ -73,6 +64,12 @@ public record ModelParams /// public List MetadataOverrides { get; set; } = new(); + /// + public LLamaLoadMode LoadMode { get; set; } + + /// + public bool LoadMTP { get; set; } + /// public float? RopeFrequencyBase { get; set; } diff --git a/LLama/Extensions/IContextParamsExtensions.cs b/LLama/Extensions/IContextParamsExtensions.cs index be76b3516..c13bc9a59 100644 --- a/LLama/Extensions/IContextParamsExtensions.cs +++ b/LLama/Extensions/IContextParamsExtensions.cs @@ -28,6 +28,7 @@ public static void ToLlamaContextParams(this IContextParams @params, out LLamaCo result.n_ubatch = @params.UBatchSize; result.n_seq_max = @params.SeqMax; result.n_rs_seq = @params.RecurrentRollbackSnapshots; + result.n_outputs_max = 0; // 0 = n_batch result.embeddings = @params.Embeddings; result.rope_freq_base = @params.RopeFrequencyBase ?? 0; result.rope_freq_scale = @params.RopeFrequencyScale ?? 0; @@ -70,6 +71,8 @@ public static void ToLlamaContextParams(this IContextParams @params, out LLamaCo result.op_offload = @params.OpOffload.Value; if (@params.KVUnified.HasValue) result.kv_unified = @params.KVUnified.Value; + + result.ctx_other = IntPtr.Zero; } private static int Threads(int? value) diff --git a/LLama/Extensions/IModelParamsExtensions.cs b/LLama/Extensions/IModelParamsExtensions.cs index 9f41f3562..5ed152c7e 100644 --- a/LLama/Extensions/IModelParamsExtensions.cs +++ b/LLama/Extensions/IModelParamsExtensions.cs @@ -22,10 +22,14 @@ public static class IModelParamsExtensions /// public static IDisposable ToLlamaModelParams(this IModelParams @params, out LLamaModelParams result) { - if (@params.UseMemoryLock && !NativeApi.llama_supports_mlock()) - throw new NotSupportedException("'UseMemoryLock' is not supported (llama_supports_mlock() == false)"); - if (@params.UseMemorymap && !NativeApi.llama_supports_mmap()) - throw new NotSupportedException("'UseMemorymap' is not supported (llama_supports_mmap() == false)"); + var supportsMmap = NativeApi.llama_supports_mmap(); + var supportMlock = NativeApi.llama_supports_mlock(); + if (@params.LoadMode == LLamaLoadMode.MemoryLock && !supportMlock) + throw new NotSupportedException("'LLamaLoadMode.MemoryLock' is not supported (llama_supports_mlock() == false)"); + if (@params.LoadMode == LLamaLoadMode.MemoryMap && !supportsMmap) + throw new NotSupportedException("'LLamaLoadMode.MemoryLock' is not supported (llama_supports_mmap() == false)"); + if (@params.LoadMode == LLamaLoadMode.MemoryMapAndLock && (!supportsMmap || !supportMlock)) + throw new NotSupportedException($"'LLamaLoadMode.MemoryLock' is not supported (llama_supports_mmap() == {supportsMmap}, llama_supports_mlock() == {supportMlock}"); var disposer = new GroupDisposable(); @@ -36,11 +40,10 @@ public static IDisposable ToLlamaModelParams(this IModelParams @params, out LLam if (@params.SplitMode.HasValue) result.split_mode = @params.SplitMode.Value; - result.use_mlock = @params.UseMemoryLock; - result.use_mmap = @params.UseMemorymap; - result.use_direct_io = @params.UseDirectIO; + result.load_mode = @params.LoadMode; result.vocab_only = @params.VocabOnly; result.check_tensors = @params.CheckTensors; + result.load_mtp = @params.LoadMTP; unsafe { diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 353c209b4..bd13b1c9b 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -59,7 +59,7 @@ - c0c7e147e7efa6c58587 + 815a2a5915f22c diff --git a/LLama/Native/LLamaContextParams.cs b/LLama/Native/LLamaContextParams.cs index 198ee834f..b9f6d9d97 100644 --- a/LLama/Native/LLamaContextParams.cs +++ b/LLama/Native/LLamaContextParams.cs @@ -45,6 +45,11 @@ public struct LLamaContextParams /// public uint n_rs_seq; + /// + /// max outputs in a ubatch (0 = n_batch) + /// + public uint n_outputs_max; + /// /// number of threads to use for generation /// @@ -229,6 +234,12 @@ public bool kv_unified /// public nuint n_samplers; + /// + /// a source/target/parent context. + /// can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts + /// + public IntPtr ctx_other; + /// /// Get the default LLamaContextParams /// diff --git a/LLama/Native/LLamaFtype.cs b/LLama/Native/LLamaFtype.cs index 78a682130..08688f5ac 100644 --- a/LLama/Native/LLamaFtype.cs +++ b/LLama/Native/LLamaFtype.cs @@ -217,6 +217,11 @@ public enum LLamaFtype /// LLAMA_FTYPE_MOSTLY_Q1_0 = 40, + /// + /// Except 1d tensors + /// + LLAMA_FTYPE_MOSTLY_Q2_0 = 41, + /// /// File type was not specified /// diff --git a/LLama/Native/LLamaModelParams.cs b/LLama/Native/LLamaModelParams.cs index 2bdcad7a8..ce6ab617a 100644 --- a/LLama/Native/LLamaModelParams.cs +++ b/LLama/Native/LLamaModelParams.cs @@ -28,7 +28,12 @@ public unsafe struct LLamaModelParams /// how to split the model across multiple GPUs /// public GPUSplitMode split_mode; - + + /// + /// How to load the model + /// + public LLamaLoadMode load_mode; + /// /// the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE /// @@ -71,36 +76,6 @@ public bool vocab_only } private sbyte _vocab_only; - /// - /// use mmap if possible - /// - public bool use_mmap - { - readonly get => Convert.ToBoolean(_use_mmap); - set => _use_mmap = Convert.ToSByte(value); - } - private sbyte _use_mmap; - - /// - /// use direct io, takes precedence over use_mmap when supported - /// - public bool use_direct_io - { - readonly get => Convert.ToBoolean(_use_direct_io); - set => _use_direct_io = Convert.ToSByte(value); - } - private sbyte _use_direct_io; - - /// - /// force system to keep model in RAM - /// - public bool use_mlock - { - readonly get => Convert.ToBoolean(_use_mlock); - set => _use_mlock = Convert.ToSByte(value); - } - private sbyte _use_mlock; - /// /// validate model tensor data /// @@ -139,7 +114,18 @@ public bool no_alloc readonly get => Convert.ToBoolean(_no_alloc); set => _no_alloc = Convert.ToSByte(value); } - private sbyte _no_alloc; + private sbyte _no_alloc; + + /// + /// whether to load MTP layers + /// + public bool load_mtp + { + readonly get => Convert.ToBoolean(_load_mtp); + set => _load_mtp = Convert.ToSByte(value); + } + private sbyte _load_mtp; + /// /// Create a LLamaModelParams with default values /// diff --git a/LLama/Native/LLamaVocabNative.cs b/LLama/Native/LLamaVocabNative.cs index 05347aa4e..03a9b8cef 100644 --- a/LLama/Native/LLamaVocabNative.cs +++ b/LLama/Native/LLamaVocabNative.cs @@ -131,4 +131,13 @@ internal struct LLamaVocabNative [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] [return: MarshalAs(UnmanagedType.U1)] public static extern unsafe bool llama_vocab_get_add_sep(LLamaVocabNative* vocab); + + /// + /// model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) + /// + /// + /// + /// + [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern unsafe LLamaToken* llama_vocab_get_suppress_tokens(LLamaVocabNative* vocab, int* n_suppress_tokens); } \ No newline at end of file diff --git a/LLama/Native/Load/LLamaLoadMode.cs b/LLama/Native/Load/LLamaLoadMode.cs new file mode 100644 index 000000000..c16f6126f --- /dev/null +++ b/LLama/Native/Load/LLamaLoadMode.cs @@ -0,0 +1,52 @@ +namespace LLama.Native; + +/// +/// +/// +/// llama_load_mode +public enum LLamaLoadMode +{ + /// + /// no special loading mode + /// + None = 0, + + /// + /// memory map the model + /// + MemoryMap = 1, + + /// + /// force system to keep model in RAM rather than swapping or compressing + /// + MemoryLock = 2, + + /// + /// mmap + force system to keep model in RAM rather than swapping or compressing + /// + MemoryMapAndLock = 3, + + /// + /// Use direct I/O if available + /// + DirectIO = 4, +} + +public static partial class NativeApi +{ + /// + /// Get the canonical name of a particular load mode + /// + /// + /// + [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern string llama_load_mode_name(LLamaLoadMode load_mode); + + /// + /// Parse a load mode from a string + /// + /// + /// + [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern LLamaLoadMode llama_load_mode_from_str(string str); +} \ No newline at end of file diff --git a/LLama/Native/MtmdContextParams.cs b/LLama/Native/MtmdContextParams.cs index 003fdc1fc..b3dc7f399 100644 --- a/LLama/Native/MtmdContextParams.cs +++ b/LLama/Native/MtmdContextParams.cs @@ -53,6 +53,13 @@ public class MtmdContextParams /// public int ImageMaxTokens { get; set; } + /// + /// maximum number of output tokens in a batch + /// (note: this is not a hard-limit, the first image will always be added even if it exceeds this limit) + /// (default: 1024) + /// + public int BatchMaxTokens; + /// /// Create a managed copy of the native defaults returned by . /// @@ -69,7 +76,8 @@ public static MtmdContextParams Default() FlashAttentionType = native.flash_attn_type, Warmup = native.warmup, ImageMinTokens = native.image_min_tokens, - ImageMaxTokens = native.image_max_tokens + ImageMaxTokens = native.image_max_tokens, + BatchMaxTokens = native.batch_max_tokens, }; } @@ -145,6 +153,16 @@ public IntPtr Pointer } } + public nuint Length + { + get + { + if (_buffer is null || !_handle.IsAllocated) + return 0; + return (nuint)_buffer.Length; + } + } + public void Dispose() { if (_buffer is not null && _handle.IsAllocated) diff --git a/LLama/Native/NativeApi.Mtmd.cs b/LLama/Native/NativeApi.Mtmd.cs index d35a3fb1a..685200bee 100644 --- a/LLama/Native/NativeApi.Mtmd.cs +++ b/LLama/Native/NativeApi.Mtmd.cs @@ -8,10 +8,11 @@ namespace LLama.Native; /// public static partial class NativeApi { - + /// /// Native context parameters returned by . /// + /// mtmd_context_params [StructLayout(LayoutKind.Sequential)] internal struct mtmd_context_params { @@ -27,6 +28,19 @@ internal struct mtmd_context_params private IntPtr /* ggml_backend_sched_eval_callback */ cb_eval; private IntPtr cb_eval_user_data; + + /// + /// maximum number of output tokens in a batch + /// (note: this is not a hard-limit, the first image will always be added even if it exceeds this limit) + /// (default: 1024) + /// + public int batch_max_tokens; + + // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. + // If the provided progress_callback returns true, model loading continues. + // If it returns false, model loading is immediately aborted. + private IntPtr progress_callback; + private IntPtr progress_callback_user_data; } [DllImport(mtmdLibraryName, EntryPoint = "mtmd_default_marker", CallingConvention = CallingConvention.Cdecl)] @@ -38,6 +52,14 @@ internal struct mtmd_context_params public static string? MtmdDefaultMarker() => mtmd_default_marker().PtrToString(); + /// + /// get the current marker string + /// + /// + /// + [DllImport(mtmdLibraryName, EntryPoint = "mtmd_context_params_default", CallingConvention = CallingConvention.Cdecl)] + public static extern string mtmd_get_marker(SafeMtmdModelHandle ctx); + [DllImport(mtmdLibraryName, EntryPoint = "mtmd_context_params_default", CallingConvention = CallingConvention.Cdecl)] internal static extern mtmd_context_params mtmd_context_params_default(); @@ -227,6 +249,7 @@ internal struct mtmd_decoder_pos internal unsafe struct mtmd_input_text_native { public byte* text; + public nuint text_len; [MarshalAs(UnmanagedType.I1)] public bool add_special; [MarshalAs(UnmanagedType.I1)] public bool parse_special; } @@ -245,6 +268,7 @@ public MtmdInputTextScope(string text, bool addSpecial, bool parseSpecial) Value = new mtmd_input_text_native { text = (byte*)_text.Pointer, + text_len = _text.Length, add_special = addSpecial, parse_special = parseSpecial }; @@ -281,9 +305,14 @@ internal static unsafe int mtmd_tokenize(SafeMtmdModelHandle ctx, IntPtr output, return mtmd_tokenize_native(ctx, output, &scope.Value, bitmaps, n_bitmaps); } - [DllImport(mtmdLibraryName, EntryPoint = "mtmd_encode", CallingConvention = CallingConvention.Cdecl)] - internal static extern int mtmd_encode(IntPtr ctx, IntPtr image_tokens); - + /// + /// text chunk will be ignored silently, only media chunk will be encoded + /// returns 0 on success + /// returns 1 on generic error + /// + /// + /// + /// [DllImport(mtmdLibraryName, EntryPoint = "mtmd_encode_chunk", CallingConvention = CallingConvention.Cdecl)] internal static extern int mtmd_encode_chunk(IntPtr ctx, IntPtr chunk); @@ -292,20 +321,31 @@ internal static unsafe int mtmd_tokenize(SafeMtmdModelHandle ctx, IntPtr output, // helper ------------------------------------------------------------ + internal struct mtmd_helper_bitmap_wrapper + { + public IntPtr /* mtmd_bitmap* */ bitmap; + public IntPtr /* mtmd_helper_video* */ video_ctx; + }; + [DllImport(mtmdLibraryName, EntryPoint = "mtmd_test_create_input_chunks", CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr mtmd_test_create_input_chunks(); [DllImport(mtmdLibraryName, EntryPoint = "mtmd_helper_bitmap_init_from_file", CallingConvention = CallingConvention.Cdecl)] - private static extern unsafe IntPtr mtmd_helper_bitmap_init_from_file_native(SafeMtmdModelHandle ctx, byte* fname); + private static extern unsafe mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file_native(SafeMtmdModelHandle ctx, byte* fname, bool placeholder); internal static unsafe IntPtr mtmd_helper_bitmap_init_from_file(SafeMtmdModelHandle ctx, string fname) { using var pinned = PinnedUtf8String.Create(fname) ?? throw new ArgumentNullException(nameof(fname)); - return mtmd_helper_bitmap_init_from_file_native(ctx, (byte*)pinned.Pointer); + return mtmd_helper_bitmap_init_from_file_native(ctx, (byte*)pinned.Pointer, false).bitmap; } [DllImport(mtmdLibraryName, EntryPoint = "mtmd_helper_bitmap_init_from_buf", CallingConvention = CallingConvention.Cdecl)] - internal static extern unsafe IntPtr mtmd_helper_bitmap_init_from_buf(SafeMtmdModelHandle ctx, byte* buf, nuint len); + internal static extern unsafe mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf_native(SafeMtmdModelHandle ctx, byte* buf, nuint len, bool placeholder); + + internal static unsafe IntPtr mtmd_helper_bitmap_init_from_buf(SafeMtmdModelHandle ctx, byte* buf, nuint len) + { + return mtmd_helper_bitmap_init_from_buf_native(ctx, buf, len, false).bitmap; + } [DllImport(mtmdLibraryName, EntryPoint = "mtmd_helper_get_n_tokens", CallingConvention = CallingConvention.Cdecl)] internal static extern UIntPtr mtmd_helper_get_n_tokens(SafeMtmdInputChunks chunks); @@ -353,7 +393,10 @@ internal static extern int mtmd_helper_decode_image_chunk( int n_past, int seq_id, int n_batch, - ref int new_n_past); + ref int new_n_past, + IntPtr /* mtmd_helper_post_decode_callback */ callback, + IntPtr user_data + ); /* * // EXPERIMENTAL API to get mmproj's capabilities without initializing the full context @@ -364,4 +407,76 @@ struct mtmd_caps { }; MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); */ + + /* + * // batch encoding API + // chunks are not owned by the batch, they will not be freed by mtmd_batch_free() + // batch is valid for a given context, cannot be shared across contexts + MTMD_API mtmd_batch * mtmd_batch_init(mtmd_context * ctx); + MTMD_API void mtmd_batch_free(mtmd_batch * batch); + + // only media chunks are allowed, text chunks will be rejected + // returns 0 on success + // returns 1 on generic error + // returns 2 if the batch is too large (chunk won't be added) + // returns 3 if it cannot be batched with the existing chunks in the batch + MTMD_API int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk); + + // returns 0 on success + // returns 1 on generic error + MTMD_API int32_t mtmd_batch_encode(mtmd_batch * batch); + MTMD_API float * mtmd_batch_get_output_embd(mtmd_batch * batch, const mtmd_input_chunk * chunk); + */ + + /* + * // + // video input helpers (requires ffmpeg/ffprobe installed on the system) + // the notion of video only exists at the helper level, it is not visible to the core mtmd library + // + // NOTE: this implementation is model-agnostic, it can be used with any vision-capable model + // however, it may not be accurate for some specific models + // (this is expected for now, to keep the implementation simple) + // + + struct mtmd_helper_video_info { + uint32_t width; + uint32_t height; + float fps; // effective fps (fps_target if set, else original video fps) + int32_t n_frames; // estimated total frames at effective fps (-1 if unknown) + }; + + struct mtmd_helper_video_init_params { + float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f + const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH + int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms + // TODO @ngxson : allow "placeholder" bitmap output for counting tokens + }; + + MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void); + + // returns NULL on failure (ffprobe not found, file unreadable, etc.) + MTMD_API mtmd_helper_video * mtmd_helper_video_init( + struct mtmd_context * mctx, + const char * path, + struct mtmd_helper_video_init_params params); + + // Same as mtmd_helper_video_init(), but reads from an in-memory buffer. + // The buffer is copied internally; the caller does not need to keep it alive. + // Note: pipe input is not seekable, so seeking will use output-side seeking + // (ffmpeg decodes and discards frames up to the target position). + MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( + struct mtmd_context * mctx, + const unsigned char * buf, size_t len, + struct mtmd_helper_video_init_params params); + MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx); + MTMD_API struct mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx); + + // Read the next item from the video stream; exactly one of out_bitmap or out_text is set per call. + // *out_bitmap - heap-allocated; caller must free with mtmd_bitmap_free() + // *out_text - heap-allocated (always via strdup/malloc); caller must free with free() + // returns 0 on success, -1 on EOF, -2 on error + MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, + mtmd_bitmap ** out_bitmap, + char ** out_text); + */ } diff --git a/LLama/Native/NativeApi.Quantize.cs b/LLama/Native/NativeApi.Quantize.cs index 0e4225330..aa6dac4ea 100644 --- a/LLama/Native/NativeApi.Quantize.cs +++ b/LLama/Native/NativeApi.Quantize.cs @@ -11,5 +11,13 @@ public static partial class NativeApi /// Returns 0 on success [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] public static extern uint llama_model_quantize(string fname_inp, string fname_out, ref LLamaModelQuantizeParams param); + + /// + /// Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" + /// + /// + /// + [DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)] + public static extern string llama_ftype_name(LLamaFtype ftype); } } diff --git a/LLama/Native/SafeLLamaContextHandle.cs b/LLama/Native/SafeLLamaContextHandle.cs index f805addbc..0eb8ca9f1 100644 --- a/LLama/Native/SafeLLamaContextHandle.cs +++ b/LLama/Native/SafeLLamaContextHandle.cs @@ -424,15 +424,6 @@ static SafeLLamaContextHandle() [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr llama_get_memory(SafeLLamaContextHandle ctx); - /// - /// Set whether the model is in warmup mode or not - /// If true, all model tensors are activated during llama_decode() to load and cache their weights. - /// - /// - /// - [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] - private static extern void llama_set_warmup(SafeLLamaContextHandle ctx, [MarshalAs(UnmanagedType.U1)] bool warmup); - /// /// Set whether to use causal attention or not. If set to true, the model will only attend to the past tokens /// @@ -465,15 +456,6 @@ static SafeLLamaContextHandle() #endregion #region Setters - /// - /// Set whether the model is in warmup mode or not - /// If true, all model tensors are activated during to load and cache their weights. - /// - public void SetWarmup(bool value) - { - llama_set_warmup(this, value); - } - /// /// Set whether to use causal attention or not. If set to true, the model will only attend to the past tokens /// diff --git a/LLama/Native/SafeLlamaModelHandle.cs b/LLama/Native/SafeLlamaModelHandle.cs index ae4a7a333..c27d4a507 100644 --- a/LLama/Native/SafeLlamaModelHandle.cs +++ b/LLama/Native/SafeLlamaModelHandle.cs @@ -50,6 +50,11 @@ public sealed class SafeLlamaModelHandle /// public int LayerCount => llama_model_n_layer(this); + /// + /// Get the number of MTP layers in this model + /// + public int LayerCountMtp => llama_model_n_layer_nextn(this); + /// /// Get the number of heads in this model /// @@ -106,6 +111,11 @@ public string Description } } + /// + /// Get the file quantisation type of this model + /// + public LLamaFtype FileType => llama_model_ftype(this); + /// /// Get the number of metadata key/value pairs /// @@ -338,6 +348,14 @@ private static int llama_model_meta_val_str(SafeLlamaModelHandle model, string k [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] private static extern int llama_model_n_layer(SafeLlamaModelHandle model); + /// + /// Get the number of MTP layers in this model + /// + /// + /// + [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] + private static extern int llama_model_n_layer_nextn(SafeLlamaModelHandle model); + /// /// Get the number of heads in this model /// @@ -372,6 +390,14 @@ private static int llama_model_meta_val_str(SafeLlamaModelHandle model, string k [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] private static extern unsafe int llama_model_desc(SafeLlamaModelHandle model, byte* buf, nint bufSize); + /// + /// Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 + /// + /// + /// The size of the model + [DllImport(NativeApi.libraryName, CallingConvention = CallingConvention.Cdecl)] + private static extern LLamaFtype llama_model_ftype(SafeLlamaModelHandle model); + /// /// Get the size of the model in bytes /// diff --git a/LLama/Native/SafeMtmdModelHandle.cs b/LLama/Native/SafeMtmdModelHandle.cs index 24ccc8ee2..c5b541bcf 100644 --- a/LLama/Native/SafeMtmdModelHandle.cs +++ b/LLama/Native/SafeMtmdModelHandle.cs @@ -301,7 +301,10 @@ public int DecodeImageChunk(IntPtr chunkPtr, SafeLLamaContextHandle llamaContext nPast, seqId, nBatch, - ref newNPast); + ref newNPast, + IntPtr.Zero, + IntPtr.Zero + ); if (result == 0) nPast = newNPast;