Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 165 additions & 14 deletions python/ggmlc/codegen/cpp.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ def generate_header(self) -> str:
"#include <unordered_map>",
"#include <memory>",
"#include <cmath>",
"#include <algorithm>",
"#include <utility>",
"#include <fstream>",
"#include <iostream>",
'#include "ggml.h"',
Expand All @@ -76,6 +78,49 @@ def generate_header(self) -> str:
f"namespace {self.model_name} {{",
"",
"// ----------------------------------------------------------------------------",
"// Broadcast helper for elementwise ops (mirrors the interpreter).",
"// ggml binary ops need `b` repeatable to `a`; this aligns the pair first.",
"// ----------------------------------------------------------------------------",
"inline std::pair<struct ggml_tensor*, struct ggml_tensor*> match_broadcast(",
" struct ggml_context* ctx,",
" struct ggml_tensor* a,",
" struct ggml_tensor* b) {",
" if (!a || !b) return {a, b};",
" if (ggml_are_same_shape(a, b)) return {a, b};",
" if (a->ne[1] == 1 && a->ne[2] == 1 && a->ne[3] == 1 && a->ne[0] == b->ne[2]) {",
" if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);",
" a = ggml_reshape_4d(ctx, a, 1, 1, a->ne[0], 1);",
" }",
" if (b->ne[1] == 1 && b->ne[2] == 1 && b->ne[3] == 1 && b->ne[0] == a->ne[2]) {",
" if (!ggml_is_contiguous(b)) b = ggml_cont(ctx, b);",
" b = ggml_reshape_4d(ctx, b, 1, 1, b->ne[0], 1);",
" }",
" if (ggml_can_repeat(b, a)) return {a, b};",
" if (ggml_can_repeat(a, b)) {",
" if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);",
" a = ggml_repeat(ctx, a, b);",
" return {a, b};",
" }",
" int64_t target_ne[4];",
" bool need_repeat_a = false;",
" bool need_repeat_b = false;",
" for (int d = 0; d < 4; ++d) {",
" target_ne[d] = std::max(a->ne[d], b->ne[d]);",
" if (a->ne[d] != target_ne[d]) need_repeat_a = true;",
" if (b->ne[d] != target_ne[d]) need_repeat_b = true;",
" }",
" if (need_repeat_a) {",
" if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);",
" a = ggml_repeat_4d(ctx, a, target_ne[0], target_ne[1], target_ne[2], target_ne[3]);",
" }",
" if (need_repeat_b) {",
" if (!ggml_is_contiguous(b)) b = ggml_cont(ctx, b);",
" b = ggml_repeat_4d(ctx, b, target_ne[0], target_ne[1], target_ne[2], target_ne[3]);",
" }",
" return {a, b};",
"}",
"",
"// ----------------------------------------------------------------------------",
"// Model Parameter Weights Structure",
"// ----------------------------------------------------------------------------",
"struct Weights {",
Expand Down Expand Up @@ -228,24 +273,60 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]:
inp_vars = [f"tensors[{i}]" for i in node.inputs]

if node.opcode == GGMLOpCode.GGML_OP_ADD:
lines.append(f" tensors[{out_id}] = ggml_add(ctx, {inp_vars[0]}, {inp_vars[1]});")
lines.append(
f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});"
)
lines.append(
f" tensors[{out_id}] = ggml_add(ctx, bc_{node.id}.first, bc_{node.id}.second);"
)
elif node.opcode == GGMLOpCode.GGML_OP_SUB:
lines.append(f" tensors[{out_id}] = ggml_sub(ctx, {inp_vars[0]}, {inp_vars[1]});")
lines.append(
f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});"
)
lines.append(
f" tensors[{out_id}] = ggml_sub(ctx, bc_{node.id}.first, bc_{node.id}.second);"
)
elif node.opcode == GGMLOpCode.GGML_OP_MUL:
lines.append(f" tensors[{out_id}] = ggml_mul(ctx, {inp_vars[0]}, {inp_vars[1]});")
lines.append(
f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});"
)
lines.append(
f" tensors[{out_id}] = ggml_mul(ctx, bc_{node.id}.first, bc_{node.id}.second);"
)
elif node.opcode == GGMLOpCode.GGML_OP_DIV:
lines.append(f" tensors[{out_id}] = ggml_div(ctx, {inp_vars[0]}, {inp_vars[1]});")
lines.append(
f" auto bc_{node.id} = match_broadcast(ctx, {inp_vars[0]}, {inp_vars[1]});"
)
lines.append(
f" tensors[{out_id}] = ggml_div(ctx, bc_{node.id}.first, bc_{node.id}.second);"
)
elif node.opcode == GGMLOpCode.GGML_OP_MUL_MAT:
if node.attributes.get("transpose_in0", 0) == 1:
# transpose does not need its source to be contiguous, but the
# result of the transpose must be made contiguous for the matmul
lines.append(
f" struct ggml_tensor* t_tr_{node.id} = ggml_transpose(ctx, {inp_vars[0]});"
f" struct ggml_tensor* mm0_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, {inp_vars[0]}));"
)
else:
lines.append(f" struct ggml_tensor* mm0_{node.id} = {inp_vars[0]};")
lines.append(
f" tensors[{out_id}] = ggml_mul_mat(ctx, t_tr_{node.id}, {inp_vars[1]});"
f" if (!ggml_is_contiguous(mm0_{node.id})) mm0_{node.id} = ggml_cont(ctx, mm0_{node.id});"
)
lines.append(f" struct ggml_tensor* mm1_{node.id} = {inp_vars[1]};")
lines.append(
f" if (!ggml_is_contiguous(mm1_{node.id})) mm1_{node.id} = ggml_cont(ctx, mm1_{node.id});"
)
lines.append(
f" tensors[{out_id}] = ggml_mul_mat(ctx, mm0_{node.id}, mm1_{node.id});"
)
b_arg = inp_vars[2] if len(inp_vars) > 2 else "nullptr"
if b_arg != "nullptr":
lines.append(f" struct ggml_tensor* b_{node.id} = {b_arg};")
lines.append(
f" if (!ggml_is_contiguous(b_{node.id})) b_{node.id} = ggml_cont(ctx, b_{node.id});"
)
else:
lines.append(
f" tensors[{out_id}] = ggml_mul_mat(ctx, {inp_vars[0]}, {inp_vars[1]});"
f" tensors[{out_id}] = ggml_add(ctx, tensors[{out_id}], b_{node.id});"
)
elif node.opcode == GGMLOpCode.GGML_OP_UNARY:
u_type = node.attributes.get("unary_op", "gelu")
Expand Down Expand Up @@ -349,17 +430,33 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]:
elif node.opcode == GGMLOpCode.GGML_OP_SOFT_MAX:
lines.append(f" tensors[{out_id}] = ggml_soft_max(ctx, {inp_vars[0]});")
elif node.opcode == GGMLOpCode.GGML_OP_REPEAT:
lines.append(f" tensors[{out_id}] = ggml_repeat(ctx, {inp_vars[0]}, {inp_vars[1]});")
if len(inp_vars) == 1:
out_t = self.graph.tensors[out_id]
ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne]
lines.append(
f" tensors[{out_id}] = ggml_repeat_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)});"
)
else:
lines.append(
f" tensors[{out_id}] = ggml_repeat(ctx, {inp_vars[0]}, {inp_vars[1]});"
)
elif node.opcode == GGMLOpCode.GGML_OP_RESHAPE:
out_t = self.graph.tensors[out_id]
ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne]
lines.append(f" struct ggml_tensor* idx_{node.id} = {inp_vars[0]};")
lines.append(
f" if (!ggml_is_contiguous(idx_{node.id})) idx_{node.id} = ggml_cont(ctx, idx_{node.id});"
)
lines.append(
f" tensors[{out_id}] = ggml_reshape_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)});"
f" tensors[{out_id}] = ggml_reshape_4d(ctx, idx_{node.id}, {', '.join(ne_strs)});"
)
elif node.opcode == GGMLOpCode.GGML_OP_PERMUTE:
ax = node.attributes.get("axes", [0, 1, 2, 3])
while len(ax) < 4:
ax.append(len(ax))
ax = [
node.attributes.get("axis0", 0),
node.attributes.get("axis1", 1),
node.attributes.get("axis2", 2),
node.attributes.get("axis3", 3),
]
lines.append(
f" tensors[{out_id}] = ggml_permute(ctx, {inp_vars[0]}, {ax[0]}, {ax[1]}, {ax[2]}, {ax[3]});"
)
Expand Down Expand Up @@ -439,6 +536,58 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]:
elif node.opcode == GGMLOpCode.GGML_OP_SCALE:
s_val = node.attributes.get("scale", 1.0)
lines.append(f" tensors[{out_id}] = ggml_scale(ctx, {inp_vars[0]}, {s_val}f);")
elif node.opcode == GGMLOpCode.GGML_OP_SQRT:
lines.append(f" struct ggml_tensor* sq_{node.id} = {inp_vars[0]};")
lines.append(
f" if (!ggml_is_contiguous(sq_{node.id})) sq_{node.id} = ggml_cont(ctx, sq_{node.id});"
)
if node.attributes.get("is_rsqrt", 0):
lines.append(
f" struct ggml_tensor* sqr_{node.id} = ggml_sqrt(ctx, sq_{node.id});"
)
lines.append(f" tensors[{out_id}] = ggml_div(ctx, sqr_{node.id}, sq_{node.id});")
else:
lines.append(f" tensors[{out_id}] = ggml_sqrt(ctx, sq_{node.id});")
elif node.opcode == GGMLOpCode.GGML_OP_SUM_ROWS:
g_dim = node.attributes.get("ggml_dim", 0)
out_t = self.graph.tensors[out_id]
ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne]
if g_dim == 1:
lines.append(
f" struct ggml_tensor* sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, {inp_vars[0]}));"
)
lines.append(f" sr_{node.id} = ggml_sum_rows(ctx, sr_{node.id});")
lines.append(
f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));"
)
elif g_dim >= 2:
lines.append(f" struct ggml_tensor* sr_{node.id} = {inp_vars[0]};")
lines.append(
f" if (!ggml_is_contiguous(sr_{node.id})) sr_{node.id} = ggml_cont(ctx, sr_{node.id});"
)
lines.append(
f" sr_{node.id} = ggml_reshape_4d(ctx, sr_{node.id}, sr_{node.id}->ne[0], sr_{node.id}->ne[1] * sr_{node.id}->ne[2], 1, sr_{node.id}->ne[3]);"
)
lines.append(f" sr_{node.id} = ggml_cont(ctx, sr_{node.id});")
lines.append(
f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));"
)
lines.append(f" sr_{node.id} = ggml_sum_rows(ctx, sr_{node.id});")
lines.append(
f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));"
)
else:
lines.append(f" struct ggml_tensor* sr_{node.id} = {inp_vars[0]};")
lines.append(
f" if (!ggml_is_contiguous(sr_{node.id})) sr_{node.id} = ggml_cont(ctx, sr_{node.id});"
)
lines.append(f" sr_{node.id} = ggml_sum_rows(ctx, sr_{node.id});")
lines.append(
f" if (!ggml_is_contiguous(sr_{node.id})) sr_{node.id} = ggml_cont(ctx, sr_{node.id});"
)
lines.append(
f" tensors[{out_id}] = ggml_reshape_4d(ctx, sr_{node.id}, {', '.join(ne_strs)});"
)
elif node.opcode == GGMLOpCode.GGML_OP_CPY:
out_t = self.graph.tensors[out_id]
ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne]
Expand All @@ -456,7 +605,9 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]:
elif node.opcode == GGMLOpCode.GGML_OP_VIEW:
out_t = self.graph.tensors[out_id]
ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne]
offset = node.attributes.get("offset", 0)
start = node.attributes.get("start", 0)
ggml_dim = node.attributes.get("ggml_dim", 0)
offset = f"{start} * {inp_vars[0]}->nb[{ggml_dim}]"
lines.append(
f" tensors[{out_id}] = ggml_view_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)}, {inp_vars[0]}->nb[1], {inp_vars[0]}->nb[2], {inp_vars[0]}->nb[3], {offset});"
)
Expand Down
18 changes: 0 additions & 18 deletions python/ggmlc/dialect/ggml/lowering.py
Original file line number Diff line number Diff line change
Expand Up @@ -623,24 +623,6 @@ def _lower_op(
attrs["axis2"] = axes[2]
attrs["axis3"] = axes[3]
return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_PERMUTE, in_ids, out_ids, attrs, op.name)
elif opcode == OpCode.SLICE:
in_t = c_graph.get_tensor(in_ids[0])
R = len(in_t.shape.dims)
dim = attrs.get("dim", 0)
if dim < 0:
dim += R
ggml_dim = max(0, R - 1 - dim)
attrs["ggml_dim"] = ggml_dim
return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_VIEW, in_ids, out_ids, attrs, op.name)
elif opcode == OpCode.CONCAT:
in_t = c_graph.get_tensor(in_ids[0])
R = len(in_t.shape.dims)
dim = attrs.get("dim", 0)
if dim < 0:
dim += R
ggml_dim = max(0, R - 1 - dim)
attrs["ggml_dim"] = ggml_dim
return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_CONCAT, in_ids, out_ids, attrs, op.name)
elif opcode in (OpCode.EXPAND, OpCode.REPEAT):
return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_REPEAT, in_ids, out_ids, attrs, op.name)
elif opcode == OpCode.EMBEDDING:
Expand Down
27 changes: 27 additions & 0 deletions tests/codegen/standalone/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Test-only wrapper: builds tests/codegen/standalone_driver.cpp against a
# prebuilt ggml (GGML_BUILD_DIR, produced by the pytest session fixture).
#
# All machine specifics (compiler, threads, OpenMP, math libs) come from
# ggml's own exported package and CMake's toolchain detection. This file
# contains no toolchain flags by design; porting to a new platform must not
# require editing test code.
cmake_minimum_required(VERSION 3.18)
project(ggmlc_standalone_test LANGUAGES C CXX)

# Provided via -D: GGML_BUILD_DIR, MODEL_PROJ_DIR, RT_INCLUDE_DIR,
# TEST_DRIVER_SRC, TEST_MODEL.
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(ggml CONFIG REQUIRED PATHS "${GGML_BUILD_DIR}" NO_DEFAULT_PATH)

add_executable(standalone_test "${TEST_DRIVER_SRC}")
target_compile_definitions(standalone_test PRIVATE
TEST_MODEL=${TEST_MODEL}
TEST_HEADER=${TEST_MODEL}.h
)
target_include_directories(standalone_test PRIVATE
"${MODEL_PROJ_DIR}"
"${RT_INCLUDE_DIR}"
)
target_link_libraries(standalone_test PRIVATE ggml::ggml)
109 changes: 109 additions & 0 deletions tests/codegen/standalone/driver.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Standalone test driver for generated ggmlc models.
//
// Compiled once per test against a generated "<Model>.h":
// c++ ... standalone_driver.cpp -DTEST_MODEL=<Model> -DTEST_HEADER=<Model>.h ...
//
// Usage:
// test_run <weights.gguf> <output.bin> <n_inputs>
// [<name> <ne0> <ne1> <ne2> <ne3> <in.bin>]...
//
// Single-output float32 graphs only; the last cgraph node is dumped.

#define STR_(x) #x
#define STR(x) STR_(x)

#include STR(TEST_HEADER)

#include <cstdint>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <unordered_map>
#include <vector>

static std::vector<uint8_t> read_bin(const char * path) {
std::ifstream f(path, std::ios::binary | std::ios::ate);
if (!f.is_open()) {
std::cerr << "cannot open " << path << std::endl;
exit(1);
}
size_t n = (size_t) f.tellg();
f.seekg(0);
std::vector<uint8_t> b(n);
f.read((char *) b.data(), n);
return b;
}

int main(int argc, char ** argv) {
if (argc < 4) {
std::cerr << "usage: test_run <weights.gguf> <output.bin> <n_inputs> [...]" << std::endl;
return 1;
}
const char * gguf_path = argv[1];
const char * out_path = argv[2];
int n_inputs = std::atoi(argv[3]);

ggml_backend_t backend = ggml_backend_cpu_init();
ggml_init_params p{128 * 1024 * 1024, nullptr, true};
ggml_context * ctx = ggml_init(p);

TEST_MODEL::Weights weights;
gguf_init_params gp{true, nullptr};
gguf_context * gctx = gguf_init_from_file(gguf_path, gp);
if (!gctx) {
std::cerr << "cannot open gguf" << std::endl;
return 1;
}
weights.init_tensors(ctx, gctx);

std::unordered_map<std::string, ggml_tensor *> inputs;
std::vector<std::pair<ggml_tensor *, std::vector<uint8_t>>> feeds;
int a = 4;
for (int i = 0; i < n_inputs; ++i) {
const char * name = argv[a++];
int64_t ne0 = std::atoll(argv[a++]);
int64_t ne1 = std::atoll(argv[a++]);
int64_t ne2 = std::atoll(argv[a++]);
int64_t ne3 = std::atoll(argv[a++]);
const char * bin_path = argv[a++];
ggml_tensor * t = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne0, ne1, ne2, ne3);
inputs[name] = t;
feeds.emplace_back(t, read_bin(bin_path));
}
ggml_cgraph * gf = TEST_MODEL::build_graph(ctx, weights, inputs);

// Graph outputs must be host-readable: a strided view's memory span is
// not its logical payload, so materialize it before the linear copy below.
ggml_tensor * out = ggml_graph_node(gf, ggml_graph_n_nodes(gf) - 1);
if (!ggml_is_contiguous(out)) {
out = ggml_cont(ctx, out);
ggml_build_forward_expand(gf, out);
}

ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
if (!buf) {
std::cerr << "backend alloc failed" << std::endl;
return 1;
}
(void) buf;
weights.load_data(gctx, gguf_path);
for (auto & feed : feeds) {
ggml_backend_tensor_set(feed.first, feed.second.data(), 0, feed.second.size());
}

if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) {
std::cerr << "compute failed" << std::endl;
return 1;
}
std::vector<uint8_t> ob(ggml_nbytes(out));
ggml_backend_tensor_get(out, ob.data(), 0, ob.size());
std::ofstream o(out_path, std::ios::binary);
o.write((char *) ob.data(), ob.size());
const float * vals = (const float *) ob.data();
std::cout << "output elements: " << (ob.size() / 4) << " first values:";
for (size_t k = 0; k < ob.size() / 4 && k < 8; ++k) {
std::cout << " " << vals[k];
}
std::cout << std::endl;
return 0;
}
Loading
Loading