Skip to content

Updated to llama.cpp 815a2a5915f22ce6a760c676389c5dfe8535c08f (b10221) #1414

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
martindevans wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Updated to llama.cpp 815a2a5915f22ce6a760c676389c5dfe8535c08f (b10221) #1414

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions LLama.KernelMemory/BuilderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,7 @@ public static IKernelMemoryBuilder WithLLamaSharpDefaults(this IKernelMemoryBuil
SplitMode = config.SplitMode,
BatchSize = 512,
UBatchSize = 512,
FlashAttention = true,
UseMemorymap = true
FlashAttention = true
};

if (weights == null)
Expand Down
6 changes: 2 additions & 4 deletions LLama.KernelMemory/LLamaSharpTextEmbeddingGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,10 @@ public LLamaSharpTextEmbeddingGenerator(LLamaSharpConfig config)
ContextSize = config?.ContextSize ?? 2048,
GpuLayerCount = config?.GpuLayerCount ?? 20,
MainGpu = config?.MainGpu ?? 0,
SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer,
SplitMode = config?.SplitMode ?? GPUSplitMode.Layer,
BatchSize = 512,
UBatchSize = 512,
FlashAttention = true,
UseMemorymap = true,
PoolingType = LLamaPoolingType.Mean,
};

Expand All @@ -65,11 +64,10 @@ public LLamaSharpTextEmbeddingGenerator(LLamaSharpConfig config, LLamaWeights we
ContextSize = config?.ContextSize ?? 2048,
GpuLayerCount = config?.GpuLayerCount ?? 20,
MainGpu = config?.MainGpu ?? 0,
SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer,
SplitMode = config?.SplitMode ?? GPUSplitMode.Layer,
BatchSize = 512,
UBatchSize = 512,
FlashAttention = true,
UseMemorymap = true,
PoolingType = LLamaPoolingType.Mean,
};
_weights = weights;
Expand Down
6 changes: 2 additions & 4 deletions LLama.KernelMemory/LlamaSharpTextGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ public LlamaSharpTextGenerator(LLamaSharpConfig config)
SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer,
BatchSize = 512,
UBatchSize = 512,
FlashAttention = true,
UseMemorymap = true
FlashAttention = true
};
_weights = LLamaWeights.LoadFromFile(@params);
_executor = new StatelessExecutor(_weights, @params);
Expand All @@ -66,8 +65,7 @@ public LlamaSharpTextGenerator(LLamaWeights weights, LLamaSharpConfig config, St
SplitMode = config?.SplitMode ?? LLama.Native.GPUSplitMode.Layer,
BatchSize = 512,
UBatchSize = 512,
FlashAttention = true,
UseMemorymap = true
FlashAttention = true
};
_executor = executor ?? new StatelessExecutor(_weights, @params);
_defaultInferenceParams = inferenceParams;
Expand Down
69 changes: 0 additions & 69 deletions LLama.Unittest/NativeAbiTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using LLama.Native;
using System.Net.Mime;
using System.Runtime.InteropServices;

namespace LLama.Unittest
Expand Down Expand Up @@ -33,74 +32,6 @@ public void TokenDataArrayLayoutMatchesNative()
Assert.Equal(expectedSize, Marshal.SizeOf<LLamaTokenDataArrayNative>());
}

[Fact]
public void ContextParamsSizeMatchesNative()
{
var pointerSize = IntPtr.Size;
var fields = new List<(int size, int align)>
{
(sizeof(uint), 4), // n_ctx
(sizeof(uint), 4), // n_batch
(sizeof(uint), 4), // n_ubatch
(sizeof(uint), 4), // n_seq_max
(sizeof(uint), 4), // n_rs_seq
(sizeof(int), 4), // n_threads
(sizeof(int), 4), // n_threads_batch
(sizeof(LLamaContextType), 4), // ctx_type
(sizeof(int), 4), // rope_scaling_type
(sizeof(int), 4), // pooling_type
(sizeof(int), 4), // attention_type
(sizeof(int), 4), // flash_attn_type
(sizeof(float), 4), // rope_freq_base
(sizeof(float), 4), // rope_freq_scale
(sizeof(float), 4), // yarn_ext_factor
(sizeof(float), 4), // yarn_attn_factor
(sizeof(float), 4), // yarn_beta_fast
(sizeof(float), 4), // yarn_beta_slow
(sizeof(uint), 4), // yarn_orig_ctx
(sizeof(float), 4), // defrag_thold
(pointerSize, pointerSize), // cb_eval
(pointerSize, pointerSize), // cb_eval_user_data
(sizeof(int), 4), // type_k
(sizeof(int), 4), // type_v
(pointerSize, pointerSize), // abort_callback
(pointerSize, pointerSize), // abort_callback_user_data
(sizeof(sbyte), 1), // embeddings
(sizeof(sbyte), 1), // offload_kqv
(sizeof(sbyte), 1), // no_perf
(sizeof(sbyte), 1), // op_offload
(sizeof(sbyte), 1), // swa_full
(sizeof(sbyte), 1), // kv_unified
(pointerSize, pointerSize), // samplers
(pointerSize, pointerSize), // n_samplers
};

var expectedSize = ComputeSize(fields);
Assert.Equal(expectedSize, Marshal.SizeOf<LLamaContextParams>());
}

[Fact]
public void ModelParamsBoolBlockMatchesNative()
{
var pointerSize = IntPtr.Size;

// Get the field immediately before the first boolean field
var kvOffset = Marshal.OffsetOf<LLamaModelParams>(nameof(LLamaModelParams.kv_overrides)).ToInt32();

// Get the first boolean field
var vocabOffset = Marshal.OffsetOf<LLamaModelParams>("_vocab_only").ToInt32();

// Check first boolean field is one ptr-size after the other
Assert.Equal(kvOffset + pointerSize, vocabOffset);
Assert.Equal(vocabOffset + 1, Marshal.OffsetOf<LLamaModelParams>("_use_mmap").ToInt32());
Assert.Equal(vocabOffset + 2, Marshal.OffsetOf<LLamaModelParams>("_use_direct_io").ToInt32());
Assert.Equal(vocabOffset + 3, Marshal.OffsetOf<LLamaModelParams>("_use_mlock").ToInt32());
Assert.Equal(vocabOffset + 4, Marshal.OffsetOf<LLamaModelParams>("_check_tensors").ToInt32());
Assert.Equal(vocabOffset + 5, Marshal.OffsetOf<LLamaModelParams>("_use_extra_bufts").ToInt32());
Assert.Equal(vocabOffset + 6, Marshal.OffsetOf<LLamaModelParams>("_no_host").ToInt32());
Assert.Equal(vocabOffset + 7, Marshal.OffsetOf<LLamaModelParams>("_no_alloc").ToInt32());
}

[Fact]
public void SamplerInterfaceSizeMatchesNative()
{
Expand Down
15 changes: 6 additions & 9 deletions LLama.Web/Common/ModelOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,6 @@ public class ModelOptions
/// <inheritdoc />
public bool Embeddings { get; set; }

/// <inheritdoc />
public bool UseMemorymap { get; set; } = true;

/// <inheritdoc />
public bool UseDirectIO { get; }

/// <inheritdoc />
public bool UseMemoryLock { get; set; } = false;

/// <inheritdoc />
public string ModelPath { get; set; }

Expand All @@ -76,6 +67,12 @@ public class ModelOptions
/// <inheritdoc />
public List<MetadataOverride> MetadataOverrides { get; } = new();

/// <inheritdoc />
public LLamaLoadMode LoadMode { get; set; }

/// <inheritdoc />
public bool LoadMTP { get; set; }

/// <inheritdoc />
public float? RopeFrequencyBase { get; set; }

Expand Down
25 changes: 10 additions & 15 deletions LLama/Abstractions/IModelParams.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,21 +49,6 @@ public interface IModelParams
/// </summary>
int GpuLayerCount { get; }

/// <summary>
/// Use mmap for faster loads (use_mmap)
/// </summary>
bool UseMemorymap { get; }

/// <summary>
/// Use direct io, takes precedence over use_mmap when supported
/// </summary>
bool UseDirectIO { get; }

/// <summary>
/// Use mlock to keep model in memory (use_mlock)
/// </summary>
bool UseMemoryLock { get; }

/// <summary>
/// Model path (model)
/// </summary>
Expand All @@ -88,6 +73,16 @@ public interface IModelParams
/// Override specific metadata items in the model
/// </summary>
List<MetadataOverride> MetadataOverrides { get; }

/// <summary>
/// How the load this model
/// </summary>
LLamaLoadMode LoadMode { get; }

/// <summary>
/// Whether to load MTP layers
/// </summary>
bool LoadMTP { get; }
}

/// <summary>
Expand Down
15 changes: 6 additions & 9 deletions LLama/Common/ModelParams.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,6 @@ public record ModelParams
/// <inheritdoc />
public uint RecurrentRollbackSnapshots { get; set; } = 0;

/// <inheritdoc />
public bool UseMemorymap { get; set; } = true;

/// <inheritdoc />
public bool UseDirectIO { get; set; }

/// <inheritdoc />
public bool UseMemoryLock { get; set; }

/// <inheritdoc />
public string ModelPath { get; set; }

Expand Down Expand Up @@ -73,6 +64,12 @@ public record ModelParams
/// <inheritdoc />
public List<MetadataOverride> MetadataOverrides { get; set; } = new();

/// <inheritdoc />
public LLamaLoadMode LoadMode { get; set; }

/// <inheritdoc />
public bool LoadMTP { get; set; }

/// <inheritdoc />
public float? RopeFrequencyBase { get; set; }

Expand Down
3 changes: 3 additions & 0 deletions LLama/Extensions/IContextParamsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public static void ToLlamaContextParams(this IContextParams @params, out LLamaCo
result.n_ubatch = @params.UBatchSize;
result.n_seq_max = @params.SeqMax;
result.n_rs_seq = @params.RecurrentRollbackSnapshots;
result.n_outputs_max = 0; // 0 = n_batch
result.embeddings = @params.Embeddings;
result.rope_freq_base = @params.RopeFrequencyBase ?? 0;
result.rope_freq_scale = @params.RopeFrequencyScale ?? 0;
Expand Down Expand Up @@ -70,6 +71,8 @@ public static void ToLlamaContextParams(this IContextParams @params, out LLamaCo
result.op_offload = @params.OpOffload.Value;
if (@params.KVUnified.HasValue)
result.kv_unified = @params.KVUnified.Value;

result.ctx_other = IntPtr.Zero;
}

private static int Threads(int? value)
Expand Down
17 changes: 10 additions & 7 deletions LLama/Extensions/IModelParamsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,14 @@
/// <exception cref="ArgumentException"></exception>
public static IDisposable ToLlamaModelParams(this IModelParams @params, out LLamaModelParams result)
{
if (@params.UseMemoryLock && !NativeApi.llama_supports_mlock())
throw new NotSupportedException("'UseMemoryLock' is not supported (llama_supports_mlock() == false)");
if (@params.UseMemorymap && !NativeApi.llama_supports_mmap())
throw new NotSupportedException("'UseMemorymap' is not supported (llama_supports_mmap() == false)");
var supportsMmap = NativeApi.llama_supports_mmap();
var supportMlock = NativeApi.llama_supports_mlock();
if (@params.LoadMode == LLamaLoadMode.MemoryLock && !supportMlock)
throw new NotSupportedException("'LLamaLoadMode.MemoryLock' is not supported (llama_supports_mlock() == false)");
if (@params.LoadMode == LLamaLoadMode.MemoryMap && !supportsMmap)
throw new NotSupportedException("'LLamaLoadMode.MemoryLock' is not supported (llama_supports_mmap() == false)");
if (@params.LoadMode == LLamaLoadMode.MemoryMapAndLock && (!supportsMmap || !supportMlock))
throw new NotSupportedException($"'LLamaLoadMode.MemoryLock' is not supported (llama_supports_mmap() == {supportsMmap}, llama_supports_mlock() == {supportMlock}");

var disposer = new GroupDisposable();

Expand All @@ -36,11 +40,10 @@
if (@params.SplitMode.HasValue)
result.split_mode = @params.SplitMode.Value;

result.use_mlock = @params.UseMemoryLock;
result.use_mmap = @params.UseMemorymap;
result.use_direct_io = @params.UseDirectIO;
result.load_mode = @params.LoadMode;
result.vocab_only = @params.VocabOnly;
result.check_tensors = @params.CheckTensors;
result.load_mtp = @params.LoadMTP;

unsafe
{
Expand Down Expand Up @@ -120,7 +123,7 @@
if (string.IsNullOrEmpty(name))
continue;

result[name] = buft;

Check warning on line 126 in LLama/Extensions/IModelParamsExtensions.cs

View workflow job for this annotation

GitHub Actions / Windows x64 CPU 

Possible null reference argument for parameter 'key' in 'IntPtr Dictionary<string, IntPtr>.this[string key]'.

Check warning on line 126 in LLama/Extensions/IModelParamsExtensions.cs

View workflow job for this annotation

GitHub Actions / Linux x64 CPU 

Possible null reference argument for parameter 'key' in 'IntPtr Dictionary<string, IntPtr>.this[string key]'.

Check warning on line 126 in LLama/Extensions/IModelParamsExtensions.cs

View workflow job for this annotation

GitHub Actions / macOS ARM64 Metal 

Possible null reference argument for parameter 'key' in 'IntPtr Dictionary<string, IntPtr>.this[string key]'.

Check warning on line 126 in LLama/Extensions/IModelParamsExtensions.cs

View workflow job for this annotation

GitHub Actions / Linux ARM64 CPU 

Possible null reference argument for parameter 'key' in 'IntPtr Dictionary<string, IntPtr>.this[string key]'.
}

return result;
Expand Down
2 changes: 1 addition & 1 deletion LLama/LLamaSharp.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
</ItemGroup>

<PropertyGroup>
<BinaryReleaseId>c0c7e147e7efa6c58587</BinaryReleaseId>
<BinaryReleaseId>815a2a5915f22c</BinaryReleaseId>
</PropertyGroup>

<PropertyGroup>
Expand Down
11 changes: 11 additions & 0 deletions LLama/Native/LLamaContextParams.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ public struct LLamaContextParams
/// </summary>
public uint n_rs_seq;

/// <summary>
/// max outputs in a ubatch (0 = n_batch)
/// </summary>
public uint n_outputs_max;

/// <summary>
/// number of threads to use for generation
/// </summary>
Expand Down Expand Up @@ -229,6 +234,12 @@ public bool kv_unified
/// </summary>
public nuint n_samplers;

/// <summary>
/// a source/target/parent context.
/// can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts
/// </summary>
public IntPtr ctx_other;

/// <summary>
/// Get the default LLamaContextParams
/// </summary>
Expand Down
5 changes: 5 additions & 0 deletions LLama/Native/LLamaFtype.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,11 @@ public enum LLamaFtype
/// </summary>
LLAMA_FTYPE_MOSTLY_Q1_0 = 40,

/// <summary>
/// Except 1d tensors
/// </summary>
LLAMA_FTYPE_MOSTLY_Q2_0 = 41,

/// <summary>
/// File type was not specified
/// </summary>
Expand Down
Loading
Loading