diff --git a/python/ggmlc/codegen/cpp.py b/python/ggmlc/codegen/cpp.py index a562f0b..853a520 100644 --- a/python/ggmlc/codegen/cpp.py +++ b/python/ggmlc/codegen/cpp.py @@ -58,6 +58,8 @@ def generate_header(self) -> str: "#include ", "#include ", "#include ", + "#include ", + "#include ", "#include ", "#include ", '#include "ggml.h"', @@ -76,6 +78,49 @@ def generate_header(self) -> str: f"namespace {self.model_name} {{", "", "// ----------------------------------------------------------------------------", + "// Broadcast helper for elementwise ops (mirrors the interpreter).", + "// ggml binary ops need `b` repeatable to `a`; this aligns the pair first.", + "// ----------------------------------------------------------------------------", + "inline std::pair match_broadcast(", + " struct ggml_context* ctx,", + " struct ggml_tensor* a,", + " struct ggml_tensor* b) {", + " if (!a || !b) return {a, b};", + " if (ggml_are_same_shape(a, b)) return {a, b};", + " if (a->ne[1] == 1 && a->ne[2] == 1 && a->ne[3] == 1 && a->ne[0] == b->ne[2]) {", + " if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);", + " a = ggml_reshape_4d(ctx, a, 1, 1, a->ne[0], 1);", + " }", + " if (b->ne[1] == 1 && b->ne[2] == 1 && b->ne[3] == 1 && b->ne[0] == a->ne[2]) {", + " if (!ggml_is_contiguous(b)) b = ggml_cont(ctx, b);", + " b = ggml_reshape_4d(ctx, b, 1, 1, b->ne[0], 1);", + " }", + " if (ggml_can_repeat(b, a)) return {a, b};", + " if (ggml_can_repeat(a, b)) {", + " if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);", + " a = ggml_repeat(ctx, a, b);", + " return {a, b};", + " }", + " int64_t target_ne[4];", + " bool need_repeat_a = false;", + " bool need_repeat_b = false;", + " for (int d = 0; d < 4; ++d) {", + " target_ne[d] = std::max(a->ne[d], b->ne[d]);", + " if (a->ne[d] != target_ne[d]) need_repeat_a = true;", + " if (b->ne[d] != target_ne[d]) need_repeat_b = true;", + " }", + " if (need_repeat_a) {", + " if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);", + " a = ggml_repeat_4d(ctx, a, target_ne[0], target_ne[1], target_ne[2], target_ne[3]);", + " }", + " if (need_repeat_b) {", + " if (!ggml_is_contiguous(b)) b = ggml_cont(ctx, b);", + " b = ggml_repeat_4d(ctx, b, target_ne[0], target_ne[1], target_ne[2], target_ne[3]);", + " }", + " return {a, b};", + "}", + "", + "// ----------------------------------------------------------------------------", "// Model Parameter Weights Structure", "// ----------------------------------------------------------------------------", "struct Weights {", @@ -228,24 +273,60 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: inp_vars = [f"tensors[{i}]" for i in node.inputs] if node.opcode == GGMLOpCode.GGML_OP_ADD: - lines.append(f" tensors[{out_id}] = ggml_add(ctx, {inp_vars[0]}, {inp_vars[1]});") + lines.append( + f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});" + ) + lines.append( + f" tensors[{out_id}] = ggml_add(ctx, bc_{node.id}.first, bc_{node.id}.second);" + ) elif node.opcode == GGMLOpCode.GGML_OP_SUB: - lines.append(f" tensors[{out_id}] = ggml_sub(ctx, {inp_vars[0]}, {inp_vars[1]});") + lines.append( + f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});" + ) + lines.append( + f" tensors[{out_id}] = ggml_sub(ctx, bc_{node.id}.first, bc_{node.id}.second);" + ) elif node.opcode == GGMLOpCode.GGML_OP_MUL: - lines.append(f" tensors[{out_id}] = ggml_mul(ctx, {inp_vars[0]}, {inp_vars[1]});") + lines.append( + f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});" + ) + lines.append( + f" tensors[{out_id}] = ggml_mul(ctx, bc_{node.id}.first, bc_{node.id}.second);" + ) elif node.opcode == GGMLOpCode.GGML_OP_DIV: - lines.append(f" tensors[{out_id}] = ggml_div(ctx, {inp_vars[0]}, {inp_vars[1]});") + lines.append( + f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});" + ) + lines.append( + f" tensors[{out_id}] = ggml_div(ctx, bc_{node.id}.first, bc_{node.id}.second);" + ) elif node.opcode == GGMLOpCode.GGML_OP_MUL_MAT: if node.attributes.get("transpose_in0", 0) == 1: + # transpose does not need its source to be contiguous, but the + # result of the transpose must be made contiguous for the matmul lines.append( - f" struct ggml_tensor* t_tr_{node.id} = ggml_transpose(ctx, {inp_vars[0]});" + f" struct ggml_tensor* mm0_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, {inp_vars[0]}));" ) + else: + lines.append(f" struct ggml_tensor* mm0_{node.id} = {inp_vars[0]};") lines.append( - f" tensors[{out_id}] = ggml_mul_mat(ctx, t_tr_{node.id}, {inp_vars[1]});" + f" if (!ggml_is_contiguous(mm0_{node.id})) mm0_{node.id} = ggml_cont(ctx, mm0_{node.id});" + ) + lines.append(f" struct ggml_tensor* mm1_{node.id} = {inp_vars[1]};") + lines.append( + f" if (!ggml_is_contiguous(mm1_{node.id})) mm1_{node.id} = ggml_cont(ctx, mm1_{node.id});" + ) + lines.append( + f" tensors[{out_id}] = ggml_mul_mat(ctx, mm0_{node.id}, mm1_{node.id});" + ) + b_arg = inp_vars[2] if len(inp_vars) > 2 else "nullptr" + if b_arg != "nullptr": + lines.append(f" struct ggml_tensor* b_{node.id} = {b_arg};") + lines.append( + f" if (!ggml_is_contiguous(b_{node.id})) b_{node.id} = ggml_cont(ctx, b_{node.id});" ) - else: lines.append( - f" tensors[{out_id}] = ggml_mul_mat(ctx, {inp_vars[0]}, {inp_vars[1]});" + f" tensors[{out_id}] = ggml_add(ctx, tensors[{out_id}], b_{node.id});" ) elif node.opcode == GGMLOpCode.GGML_OP_UNARY: u_type = node.attributes.get("unary_op", "gelu") @@ -349,17 +430,33 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: elif node.opcode == GGMLOpCode.GGML_OP_SOFT_MAX: lines.append(f" tensors[{out_id}] = ggml_soft_max(ctx, {inp_vars[0]});") elif node.opcode == GGMLOpCode.GGML_OP_REPEAT: - lines.append(f" tensors[{out_id}] = ggml_repeat(ctx, {inp_vars[0]}, {inp_vars[1]});") + if len(inp_vars) == 1: + out_t = self.graph.tensors[out_id] + ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] + lines.append( + f" tensors[{out_id}] = ggml_repeat_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)});" + ) + else: + lines.append( + f" tensors[{out_id}] = ggml_repeat(ctx, {inp_vars[0]}, {inp_vars[1]});" + ) elif node.opcode == GGMLOpCode.GGML_OP_RESHAPE: out_t = self.graph.tensors[out_id] ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] + lines.append(f" struct ggml_tensor* idx_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(idx_{node.id})) idx_{node.id} = ggml_cont(ctx, idx_{node.id});" + ) lines.append( - f" tensors[{out_id}] = ggml_reshape_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)});" + f" tensors[{out_id}] = ggml_reshape_4d(ctx, idx_{node.id}, {', '.join(ne_strs)});" ) elif node.opcode == GGMLOpCode.GGML_OP_PERMUTE: - ax = node.attributes.get("axes", [0, 1, 2, 3]) - while len(ax) < 4: - ax.append(len(ax)) + ax = [ + node.attributes.get("axis0", 0), + node.attributes.get("axis1", 1), + node.attributes.get("axis2", 2), + node.attributes.get("axis3", 3), + ] lines.append( f" tensors[{out_id}] = ggml_permute(ctx, {inp_vars[0]}, {ax[0]}, {ax[1]}, {ax[2]}, {ax[3]});" ) @@ -439,6 +536,58 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: elif node.opcode == GGMLOpCode.GGML_OP_SCALE: s_val = node.attributes.get("scale", 1.0) lines.append(f" tensors[{out_id}] = ggml_scale(ctx, {inp_vars[0]}, {s_val}f);") + elif node.opcode == GGMLOpCode.GGML_OP_SQRT: + lines.append(f" struct ggml_tensor* sq_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(sq_{node.id})) sq_{node.id} = ggml_cont(ctx, sq_{node.id});" + ) + if node.attributes.get("is_rsqrt", 0): + lines.append( + f" struct ggml_tensor* sqr_{node.id} = ggml_sqrt(ctx, sq_{node.id});" + ) + lines.append(f" tensors[{out_id}] = ggml_div(ctx, sqr_{node.id}, sq_{node.id});") + else: + lines.append(f" tensors[{out_id}] = ggml_sqrt(ctx, sq_{node.id});") + elif node.opcode == GGMLOpCode.GGML_OP_SUM_ROWS: + g_dim = node.attributes.get("ggml_dim", 0) + out_t = self.graph.tensors[out_id] + ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] + if g_dim == 1: + lines.append( + f" struct ggml_tensor* sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, {inp_vars[0]}));" + ) + lines.append(f" sr_{node.id} = ggml_sum_rows(ctx, sr_{node.id});") + lines.append( + f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));" + ) + elif g_dim >= 2: + lines.append(f" struct ggml_tensor* sr_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(sr_{node.id})) sr_{node.id} = ggml_cont(ctx, sr_{node.id});" + ) + lines.append( + f" sr_{node.id} = ggml_reshape_4d(ctx, sr_{node.id}, sr_{node.id}->ne[0], sr_{node.id}->ne[1] * sr_{node.id}->ne[2], 1, sr_{node.id}->ne[3]);" + ) + lines.append(f" sr_{node.id} = ggml_cont(ctx, sr_{node.id});") + lines.append( + f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));" + ) + lines.append(f" sr_{node.id} = ggml_sum_rows(ctx, sr_{node.id});") + lines.append( + f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));" + ) + else: + lines.append(f" struct ggml_tensor* sr_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(sr_{node.id})) sr_{node.id} = ggml_cont(ctx, sr_{node.id});" + ) + lines.append(f" sr_{node.id} = ggml_sum_rows(ctx, sr_{node.id});") + lines.append( + f" if (!ggml_is_contiguous(sr_{node.id})) sr_{node.id} = ggml_cont(ctx, sr_{node.id});" + ) + lines.append( + f" tensors[{out_id}] = ggml_reshape_4d(ctx, sr_{node.id}, {', '.join(ne_strs)});" + ) elif node.opcode == GGMLOpCode.GGML_OP_CPY: out_t = self.graph.tensors[out_id] ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] @@ -456,7 +605,9 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: elif node.opcode == GGMLOpCode.GGML_OP_VIEW: out_t = self.graph.tensors[out_id] ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] - offset = node.attributes.get("offset", 0) + start = node.attributes.get("start", 0) + ggml_dim = node.attributes.get("ggml_dim", 0) + offset = f"{start} * {inp_vars[0]}->nb[{ggml_dim}]" lines.append( f" tensors[{out_id}] = ggml_view_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)}, {inp_vars[0]}->nb[1], {inp_vars[0]}->nb[2], {inp_vars[0]}->nb[3], {offset});" ) diff --git a/python/ggmlc/dialect/ggml/lowering.py b/python/ggmlc/dialect/ggml/lowering.py index 64931f9..b7d5b16 100644 --- a/python/ggmlc/dialect/ggml/lowering.py +++ b/python/ggmlc/dialect/ggml/lowering.py @@ -623,24 +623,6 @@ def _lower_op( attrs["axis2"] = axes[2] attrs["axis3"] = axes[3] return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_PERMUTE, in_ids, out_ids, attrs, op.name) - elif opcode == OpCode.SLICE: - in_t = c_graph.get_tensor(in_ids[0]) - R = len(in_t.shape.dims) - dim = attrs.get("dim", 0) - if dim < 0: - dim += R - ggml_dim = max(0, R - 1 - dim) - attrs["ggml_dim"] = ggml_dim - return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_VIEW, in_ids, out_ids, attrs, op.name) - elif opcode == OpCode.CONCAT: - in_t = c_graph.get_tensor(in_ids[0]) - R = len(in_t.shape.dims) - dim = attrs.get("dim", 0) - if dim < 0: - dim += R - ggml_dim = max(0, R - 1 - dim) - attrs["ggml_dim"] = ggml_dim - return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_CONCAT, in_ids, out_ids, attrs, op.name) elif opcode in (OpCode.EXPAND, OpCode.REPEAT): return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_REPEAT, in_ids, out_ids, attrs, op.name) elif opcode == OpCode.EMBEDDING: diff --git a/tests/codegen/standalone/CMakeLists.txt b/tests/codegen/standalone/CMakeLists.txt new file mode 100644 index 0000000..109755a --- /dev/null +++ b/tests/codegen/standalone/CMakeLists.txt @@ -0,0 +1,27 @@ +# Test-only wrapper: builds tests/codegen/standalone_driver.cpp against a +# prebuilt ggml (GGML_BUILD_DIR, produced by the pytest session fixture). +# +# All machine specifics (compiler, threads, OpenMP, math libs) come from +# ggml's own exported package and CMake's toolchain detection. This file +# contains no toolchain flags by design; porting to a new platform must not +# require editing test code. +cmake_minimum_required(VERSION 3.18) +project(ggmlc_standalone_test LANGUAGES C CXX) + +# Provided via -D: GGML_BUILD_DIR, MODEL_PROJ_DIR, RT_INCLUDE_DIR, +# TEST_DRIVER_SRC, TEST_MODEL. +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(ggml CONFIG REQUIRED PATHS "${GGML_BUILD_DIR}" NO_DEFAULT_PATH) + +add_executable(standalone_test "${TEST_DRIVER_SRC}") +target_compile_definitions(standalone_test PRIVATE + TEST_MODEL=${TEST_MODEL} + TEST_HEADER=${TEST_MODEL}.h +) +target_include_directories(standalone_test PRIVATE + "${MODEL_PROJ_DIR}" + "${RT_INCLUDE_DIR}" +) +target_link_libraries(standalone_test PRIVATE ggml::ggml) diff --git a/tests/codegen/standalone/driver.cpp b/tests/codegen/standalone/driver.cpp new file mode 100644 index 0000000..bf7df21 --- /dev/null +++ b/tests/codegen/standalone/driver.cpp @@ -0,0 +1,109 @@ +// Standalone test driver for generated ggmlc models. +// +// Compiled once per test against a generated ".h": +// c++ ... standalone_driver.cpp -DTEST_MODEL= -DTEST_HEADER=.h ... +// +// Usage: +// test_run +// [ ]... +// +// Single-output float32 graphs only; the last cgraph node is dumped. + +#define STR_(x) #x +#define STR(x) STR_(x) + +#include STR(TEST_HEADER) + +#include +#include +#include +#include +#include +#include + +static std::vector read_bin(const char * path) { + std::ifstream f(path, std::ios::binary | std::ios::ate); + if (!f.is_open()) { + std::cerr << "cannot open " << path << std::endl; + exit(1); + } + size_t n = (size_t) f.tellg(); + f.seekg(0); + std::vector b(n); + f.read((char *) b.data(), n); + return b; +} + +int main(int argc, char ** argv) { + if (argc < 4) { + std::cerr << "usage: test_run [...]" << std::endl; + return 1; + } + const char * gguf_path = argv[1]; + const char * out_path = argv[2]; + int n_inputs = std::atoi(argv[3]); + + ggml_backend_t backend = ggml_backend_cpu_init(); + ggml_init_params p{128 * 1024 * 1024, nullptr, true}; + ggml_context * ctx = ggml_init(p); + + TEST_MODEL::Weights weights; + gguf_init_params gp{true, nullptr}; + gguf_context * gctx = gguf_init_from_file(gguf_path, gp); + if (!gctx) { + std::cerr << "cannot open gguf" << std::endl; + return 1; + } + weights.init_tensors(ctx, gctx); + + std::unordered_map inputs; + std::vector>> feeds; + int a = 4; + for (int i = 0; i < n_inputs; ++i) { + const char * name = argv[a++]; + int64_t ne0 = std::atoll(argv[a++]); + int64_t ne1 = std::atoll(argv[a++]); + int64_t ne2 = std::atoll(argv[a++]); + int64_t ne3 = std::atoll(argv[a++]); + const char * bin_path = argv[a++]; + ggml_tensor * t = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne0, ne1, ne2, ne3); + inputs[name] = t; + feeds.emplace_back(t, read_bin(bin_path)); + } + ggml_cgraph * gf = TEST_MODEL::build_graph(ctx, weights, inputs); + + // Graph outputs must be host-readable: a strided view's memory span is + // not its logical payload, so materialize it before the linear copy below. + ggml_tensor * out = ggml_graph_node(gf, ggml_graph_n_nodes(gf) - 1); + if (!ggml_is_contiguous(out)) { + out = ggml_cont(ctx, out); + ggml_build_forward_expand(gf, out); + } + + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (!buf) { + std::cerr << "backend alloc failed" << std::endl; + return 1; + } + (void) buf; + weights.load_data(gctx, gguf_path); + for (auto & feed : feeds) { + ggml_backend_tensor_set(feed.first, feed.second.data(), 0, feed.second.size()); + } + + if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { + std::cerr << "compute failed" << std::endl; + return 1; + } + std::vector ob(ggml_nbytes(out)); + ggml_backend_tensor_get(out, ob.data(), 0, ob.size()); + std::ofstream o(out_path, std::ios::binary); + o.write((char *) ob.data(), ob.size()); + const float * vals = (const float *) ob.data(); + std::cout << "output elements: " << (ob.size() / 4) << " first values:"; + for (size_t k = 0; k < ob.size() / 4 && k < 8; ++k) { + std::cout << " " << vals[k]; + } + std::cout << std::endl; + return 0; +} diff --git a/tests/codegen/test_cpp_codegen.py b/tests/codegen/test_cpp_codegen.py index 8a11df7..8311f34 100644 --- a/tests/codegen/test_cpp_codegen.py +++ b/tests/codegen/test_cpp_codegen.py @@ -4,6 +4,7 @@ import numpy as np from ggmlc.codegen import generate_cpp_project from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.dialect.ggml.ops import GGMLOpCode from ggmlc.ir.graph import Graph from ggmlc.ir.op import OpCode from ggmlc.ir.shape import Shape @@ -98,3 +99,133 @@ def test_cpp_codegen_compile_and_run_wsl(): subprocess.run(cmd, capture_output=True, text=True, check=False) assert (win_tmp / "TinyModel.h").exists() assert (win_tmp / "ggmlc_main.cpp").exists() + + +def _linear_graph(with_bias): + """Single LINEAR layer graph, optionally with bias.""" + g = Graph("linear_bias") + x = g.add_tensor("x", Shape([1, 16]), DType.F32, StorageClass.INPUT) + w = g.add_tensor("w", Shape([16, 16]), DType.F32, StorageClass.PARAMETER) + w.data = np.eye(16, dtype=np.float32) + inputs = [x.id, w.id] + parameters = [w.id] + if with_bias: + b = g.add_tensor("b", Shape([16]), DType.F32, StorageClass.PARAMETER) + b.data = np.zeros((16,), dtype=np.float32) + inputs.append(b.id) + parameters.append(b.id) + out = g.add_tensor("out", Shape([1, 16]), DType.F32, StorageClass.ACTIVATION) + g.add_node(OpCode.LINEAR, inputs=inputs, outputs=[out.id], name="lin") + g.inputs = [x.id] + g.outputs = [out.id] + g.parameters = parameters + return g + + +def test_lower_linear_with_bias_to_3_input_mul_mat(): + """LINEAR with bias lowers to MUL_MAT with [weight, x, bias] inputs.""" + ggml_graph = lower_to_ggml(_linear_graph(with_bias=True)) + (node,) = ggml_graph.nodes + assert node.opcode == GGMLOpCode.GGML_OP_MUL_MAT + assert node.inputs == [1, 0, 2] + + +def test_lower_linear_without_bias_to_2_input_mul_mat(): + """LINEAR without bias lowers to plain 2-input MUL_MAT.""" + ggml_graph = lower_to_ggml(_linear_graph(with_bias=False)) + (node,) = ggml_graph.nodes + assert node.opcode == GGMLOpCode.GGML_OP_MUL_MAT + assert node.inputs == [1, 0] + + +def test_lower_expand_to_single_input_repeat(): + """EXPAND (single input + shape attribute) lowers to 1-input REPEAT.""" + g = Graph("expand") + x = g.add_tensor("x", Shape([1, 16]), DType.F32, StorageClass.INPUT) + out = g.add_tensor("out", Shape([4, 16]), DType.F32, StorageClass.ACTIVATION) + g.add_node( + OpCode.EXPAND, inputs=[x.id], outputs=[out.id], attributes={"shape": (4, 16)}, name="expand" + ) + g.inputs = [x.id] + g.outputs = [out.id] + g.parameters = [] + ggml_graph = lower_to_ggml(g) + (node,) = ggml_graph.nodes + assert node.opcode == GGMLOpCode.GGML_OP_REPEAT + assert node.inputs == [x.id] + + +def _slice_graph(dim, start, out_shape): + """Single SLICE graph over an [8, 16] input.""" + g = Graph("slice") + x = g.add_tensor("x", Shape([8, 16]), DType.F32, StorageClass.INPUT) + out = g.add_tensor("out", Shape(out_shape), DType.F32, StorageClass.ACTIVATION) + g.add_node( + OpCode.SLICE, + inputs=[x.id], + outputs=[out.id], + attributes={"dim": dim, "start": start, "end": 16, "step": 1}, + name="slice", + ) + g.inputs = [x.id] + g.outputs = [out.id] + g.parameters = [] + return g + + +def test_lower_slice_to_view_attributes(): + """SLICE lowers to VIEW carrying integer start and ggml_dim.""" + ggml_graph = lower_to_ggml(_slice_graph(dim=1, start=4, out_shape=[8, 12])) + (node,) = ggml_graph.nodes + assert node.opcode == GGMLOpCode.GGML_OP_VIEW + assert node.attributes["start"] == 4 + assert node.attributes["ggml_dim"] == 0 + + ggml_graph = lower_to_ggml(_slice_graph(dim=0, start=2, out_shape=[6, 16])) + (node,) = ggml_graph.nodes + assert node.opcode == GGMLOpCode.GGML_OP_VIEW + assert node.attributes["start"] == 2 + assert node.attributes["ggml_dim"] == 1 + + +def test_lower_permute_to_axis_attributes(): + """PERMUTE lowers to axis0..axis3 scalars.""" + g = Graph("permute") + x = g.add_tensor("x", Shape([2, 4, 8, 16]), DType.F32, StorageClass.INPUT) + out = g.add_tensor("out", Shape([2, 8, 4, 16]), DType.F32, StorageClass.ACTIVATION) + g.add_node( + OpCode.PERMUTE, + inputs=[x.id], + outputs=[out.id], + attributes={"dims": [0, 2, 1, 3]}, + name="perm", + ) + g.inputs = [x.id] + g.outputs = [out.id] + g.parameters = [] + ggml_graph = lower_to_ggml(g) + (node,) = ggml_graph.nodes + assert node.opcode == GGMLOpCode.GGML_OP_PERMUTE + assert [node.attributes[f"axis{i}"] for i in range(4)] == [0, 2, 1, 3] + assert "axes" not in node.attributes + + +def test_lower_transpose_to_axis_attributes(): + """4D TRANSPOSE lowers to PERMUTE with axis0..axis3 scalars.""" + g = Graph("transpose") + x = g.add_tensor("x", Shape([2, 4, 8, 16]), DType.F32, StorageClass.INPUT) + out = g.add_tensor("out", Shape([2, 8, 4, 16]), DType.F32, StorageClass.ACTIVATION) + g.add_node( + OpCode.TRANSPOSE, + inputs=[x.id], + outputs=[out.id], + attributes={"dim0": 1, "dim1": 2}, + name="transpose", + ) + g.inputs = [x.id] + g.outputs = [out.id] + g.parameters = [] + ggml_graph = lower_to_ggml(g) + (node,) = ggml_graph.nodes + assert node.opcode == GGMLOpCode.GGML_OP_PERMUTE + assert [node.attributes[f"axis{i}"] for i in range(4)] == [0, 2, 1, 3] diff --git a/tests/codegen/test_standalone_execution.py b/tests/codegen/test_standalone_execution.py new file mode 100644 index 0000000..8483e59 --- /dev/null +++ b/tests/codegen/test_standalone_execution.py @@ -0,0 +1,381 @@ +"""Compile-and-run tests for standalone C++ projects from generate_cpp_project. + +Each test exports a tiny torch model, generates the standalone C++ project, +builds it against the vendored ggml, executes it on fixed vectors, and +compares the output against torch. +Single-output float32 models only, driven by tests/codegen/standalone/ +(static driver.cpp + wrapper CMakeLists), so no toolchain flags live in +test code. Linux-only until the harness is validated elsewhere. +""" + +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +import torch +from ggmlc.codegen import generate_cpp_project +from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.frontend.pytorch import export_torch_model +from ggmlc.serialization.gguf import save_to_gguf +from torch import nn + +pytestmark = pytest.mark.skipif( + sys.platform not in ("linux", "win32"), + reason="standalone compile-and-run harness validated on Linux and Windows only", +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +GGML_SRC = REPO_ROOT / "third_party" / "ggml" +GGML_INCLUDE = GGML_SRC / "include" +RT_INCLUDE = REPO_ROOT / "runtime" / "include" +STANDALONE_DIR = Path(__file__).resolve().parent / "standalone" +DRIVER_SRC = STANDALONE_DIR / "driver.cpp" +WRAPPER_DIR = STANDALONE_DIR + + +@pytest.fixture(scope="session") +def ggml_standalone_libs(): + """Provide static ggml libs, building vendored ggml once and caching by source hash. + + The build lands in ~/.cache/ggmlc/standalone-ggml// and is reused + across pytest runs until third_party/ggml, the flags, or the compiler + change. Delete that directory to force a rebuild. ccache is used + automatically when installed. + """ + assert shutil.which("cmake") is not None, "cmake is required on PATH" + # Toolchain identity for the cache key: explicit env or cmake defaults. + # No compiler probing here — toolchain detection belongs to cmake. + key = { + "sources": _ggml_source_key(), + "build_type": "Release", + "shared": False, + "tests": False, + "examples": False, + "cc": os.environ.get("CC", "default"), + "cxx": os.environ.get("CXX", "default"), + } + cache_dir = ( + Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) + / "ggmlc" + / "standalone-ggml" + / hashlib.sha256(json.dumps(key, sort_keys=True).encode()).hexdigest()[:16] + ) + stamp = cache_dir / "stamp.json" + if _stamp_matches(stamp, key): + configs = sorted((cache_dir / "install").rglob("ggml-config.cmake")) + if len(configs) == 1: + print(f"\nreusing cached ggml build: {cache_dir}") + return {"build_dir": str(configs[0].parent)} + shutil.rmtree(cache_dir, ignore_errors=True) + build_dir = cache_dir / "build" + build_dir.mkdir(parents=True, exist_ok=True) + configure = [ + "cmake", + "-S", + str(GGML_SRC), + "-B", + str(build_dir), + "-DCMAKE_BUILD_TYPE=Release", + "-DBUILD_SHARED_LIBS=OFF", + "-DGGML_BUILD_TESTS=OFF", + "-DGGML_BUILD_EXAMPLES=OFF", + ] + if shutil.which("ninja"): + configure += ["-G", "Ninja"] + env = _cmake_env() + _run_logged(configure, timeout=600, env=env) + _run_logged( + ["cmake", "--build", str(build_dir), "--parallel"], + timeout=1200, + env=env, + ) + install_dir = cache_dir / "install" + _run_logged( + ["cmake", "--install", str(build_dir), "--prefix", str(install_dir)], + timeout=600, + env=env, + ) + (config_dir,) = _require_one( + sorted(install_dir.rglob("ggml-config.cmake")), "ggml-config.cmake", install_dir + ) + stamp.write_text(json.dumps(key, indent=2), encoding="utf-8") + return {"build_dir": str(config_dir.parent)} + + +def _run_logged(cmd, **kwargs): + """subprocess.run that appends captured output to the failure message.""" + check = kwargs.pop("check", True) + try: + return subprocess.run(cmd, capture_output=True, text=True, check=check, **kwargs) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"command failed ({e.returncode}): {' '.join(e.cmd)}\n" + f"--- stdout ---\n{e.stdout}\n--- stderr ---\n{e.stderr}" + ) from e + + +def _require_one(matches, what, where): + assert len(matches) == 1, f"expected one {what} in {where}, found {len(matches)}" + return matches + + +def _cmake_env(): + """Environment for cmake builds: MSVC detection on Windows, ccache launchers when available.""" + env = os.environ.copy() + if sys.platform == "win32" and shutil.which("cl", path=env.get("PATH")) is None: + vswhere = ( + Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) + / "Microsoft Visual Studio" + / "Installer" + / "vswhere.exe" + ) + if vswhere.exists(): + try: + out = subprocess.run( + [str(vswhere), "-latest", "-products", "*", "-property", "installationPath"], + capture_output=True, + text=True, + check=True, + ) + inst = out.stdout.strip() + if inst: + vcvars = Path(inst) / "VC" / "Auxiliary" / "Build" / "vcvars64.bat" + if vcvars.exists(): + p = subprocess.run( + f'"{vcvars}" && set', + shell=True, + capture_output=True, + text=True, + check=False, + ) + for line in p.stdout.splitlines(): + if "=" in line: + k, v = line.split("=", 1) + env[k] = v + except (subprocess.SubprocessError, OSError): + pass + if shutil.which("ccache", path=env.get("PATH")): + env["CMAKE_C_COMPILER_LAUNCHER"] = "ccache" + env["CMAKE_CXX_COMPILER_LAUNCHER"] = "ccache" + return env + + +def _ggml_source_key(): + """Content key for the vendored ggml tree: git tree hash, mtime fallback.""" + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD:third_party/ggml"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + timeout=60, + ) + return "git-tree:" + out.stdout.strip() + except (subprocess.SubprocessError, OSError): + latest = 0.0 + for suffix in ("src", "include"): + root = GGML_SRC / suffix + if root.is_dir(): + for p in root.rglob("*"): + try: + latest = max(latest, p.stat().st_mtime) + except OSError: + pass + return f"mtime:{latest:.0f}" + + +def _stamp_matches(stamp, key): + if not stamp.is_file(): + return False + try: + return json.loads(stamp.read_text(encoding="utf-8")) == key + except (OSError, ValueError): + return False + + +def _torch_shape_dims(tensor): + return [int(d.evaluate({})) for d in tensor.shape.dims] + + +def _run_standalone(model, example_args, test_name, ggml_libs, tmp_path, atol=1e-5): + """Export torch model, compile the standalone project, run it, compare vs torch.""" + model_name = "".join(part.capitalize() for part in test_name.split("_")) + with torch.inference_mode(): + ref = model(*example_args) + assert isinstance(ref, torch.Tensor), "single-tensor outputs only" + ref_array = np.ascontiguousarray(ref.detach().cpu().numpy(), dtype=np.float32) + + exported = export_torch_model(model, example_args, model_name=test_name) + ggml_graph = lower_to_ggml(exported.main_graph) + proj_dir = tmp_path / "proj" + generate_cpp_project(ggml_graph, proj_dir, model_name=model_name) + gguf_path = save_to_gguf(ggml_graph, tmp_path / "weights.gguf") + + input_ids = list(exported.main_graph.inputs) + out_path = tmp_path / "output.bin" + spec_args = [] + for i, (inp_id, arg) in enumerate(zip(input_ids, example_args)): + in_tensor = exported.main_graph.get_tensor(inp_id) + dims = _torch_shape_dims(in_tensor) + ne = list(reversed(dims)) + [1] * (4 - len(dims)) + bin_path = tmp_path / f"input_{i}.bin" + np.ascontiguousarray(arg.detach().cpu().numpy(), dtype=np.float32).tofile(bin_path) + spec_args += [in_tensor.name, *[str(d) for d in ne], str(bin_path)] + + test_build_dir = tmp_path / "test_build" + configure = [ + "cmake", + "-S", + str(WRAPPER_DIR), + "-B", + str(test_build_dir), + "-DCMAKE_BUILD_TYPE=Release", + f"-DGGML_BUILD_DIR={ggml_libs['build_dir']}", + f"-DMODEL_PROJ_DIR={proj_dir}", + f"-DRT_INCLUDE_DIR={RT_INCLUDE}", + f"-DTEST_DRIVER_SRC={DRIVER_SRC}", + f"-DTEST_MODEL={model_name}", + ] + if shutil.which("ninja"): + configure += ["-G", "Ninja"] + env = _cmake_env() + _run_logged(configure, timeout=300, env=env) + _run_logged( + ["cmake", "--build", str(test_build_dir), "--parallel"], + timeout=600, + env=env, + ) + exe_path = test_build_dir / ("standalone_test.exe" if os.name == "nt" else "standalone_test") + if not exe_path.exists() and os.name == "nt": + rel_exe = test_build_dir / "Release" / "standalone_test.exe" + if rel_exe.exists(): + exe_path = rel_exe + run = _run_logged( + [str(exe_path), str(gguf_path), str(out_path), str(len(input_ids)), *spec_args], + timeout=120, + ) + print(run.stdout) + + got = np.fromfile(out_path, dtype=np.float32) + assert got.size == ref_array.size, f"element mismatch: {got.size} vs {ref_array.size}" + print(f"\nref[0:8]: {ref_array.ravel()[:8]}") + print(f"got[0:8]: {got.ravel()[:8]}") + print(f"max abs diff: {np.max(np.abs(got - ref_array.ravel()))}") + np.testing.assert_allclose(got, ref_array.ravel(), atol=atol) + + +def test_standalone_linear_bias(ggml_standalone_libs, tmp_path): + """Biased LINEAR through generated standalone code matches torch.""" + torch.manual_seed(0) + model = nn.Linear(16, 16).eval() # default init: random weights, nonzero bias + assert torch.count_nonzero(model.bias) > 0 + x = torch.randn(1, 16) + _run_standalone(model, (x,), "tiny_linear_bias", ggml_standalone_libs, tmp_path) + + +def test_standalone_matmul_nobias(ggml_standalone_libs, tmp_path): + """Bias-free 2-input MATMUL through generated standalone code matches torch.""" + torch.manual_seed(0) + + class MatMul(nn.Module): + def forward(self, a, b): + return a @ b + + a = torch.randn(2, 16) + b = torch.randn(16, 8) + _run_standalone(MatMul().eval(), (a, b), "tiny_matmul_nobias", ggml_standalone_libs, tmp_path) + + +def test_standalone_expand_broadcast(ggml_standalone_libs, tmp_path): + """EXPAND (single-input REPEAT) through generated standalone code matches torch.""" + torch.manual_seed(0) + + class Expand(nn.Module): + def forward(self, x): + return x.expand(4, 16) + + x = torch.randn(1, 16) + _run_standalone(Expand().eval(), (x,), "tiny_expand_broadcast", ggml_standalone_libs, tmp_path) + + +def test_standalone_reshape_noncontiguous(ggml_standalone_libs, tmp_path): + """RESHAPE of a non-contiguous transpose matches torch (needs the cont guard).""" + torch.manual_seed(0) + + class ReshapeTransposed(nn.Module): + def forward(self, x): + return x.transpose(0, 1).reshape(16) + + x = torch.randn(1, 16) + _run_standalone( + ReshapeTransposed().eval(), (x,), "tiny_reshape_transposed", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_slice_offset(ggml_standalone_libs, tmp_path): + """Nonzero-start SLICE (VIEW byte offset) through generated standalone code matches torch.""" + torch.manual_seed(0) + + class Slice(nn.Module): + def forward(self, x): + return x[:, 4:12] + + x = torch.randn(8, 16) + _run_standalone(Slice().eval(), (x,), "tiny_slice_dim1", ggml_standalone_libs, tmp_path) + + +def test_standalone_permute_4d(ggml_standalone_libs, tmp_path): + """4D PERMUTE through generated standalone code matches torch.""" + torch.manual_seed(0) + + class Permute(nn.Module): + def forward(self, x): + return x.permute(0, 2, 1, 3) + + x = torch.randn(2, 4, 8, 16) + _run_standalone(Permute().eval(), (x,), "tiny_permute_4d", ggml_standalone_libs, tmp_path) + + +def test_standalone_sum_rows_dim0(ggml_standalone_libs, tmp_path): + """SUM over the last dim (GGML_OP_SUM_ROWS) through generated code matches torch.""" + torch.manual_seed(0) + + class SumLast(nn.Module): + def forward(self, x): + return x.sum(dim=-1) + + x = torch.randn(4, 16) + _run_standalone(SumLast().eval(), (x,), "tiny_sum_rows_dim0", ggml_standalone_libs, tmp_path) + + +def test_standalone_sqrt(ggml_standalone_libs, tmp_path): + """SQRT through generated standalone code matches torch.""" + torch.manual_seed(0) + + class Sqrt(nn.Module): + def forward(self, x): + return torch.sqrt(x.abs() + 0.5) + + x = torch.randn(4, 16) + _run_standalone(Sqrt().eval(), (x,), "tiny_sqrt", ggml_standalone_libs, tmp_path) + + +def test_standalone_l2_normalize(ggml_standalone_libs, tmp_path): + """L2-normalize (x / norm(keepdim)) through generated code matches torch.""" + torch.manual_seed(0) + + class L2Normalize(nn.Module): + def forward(self, img_embeds): + return img_embeds / img_embeds.norm(dim=-1, keepdim=True) + + img_embeds = torch.randn(2, 8, 16) + _run_standalone( + L2Normalize().eval(), (img_embeds,), "tiny_l2_normalize", ggml_standalone_libs, tmp_path + )