Dev/cuda

.gitignore

@@ -1,4 +1,5 @@
 .vscode/*
+.venv/*
 build/
 bin/
 __pycache__/*
@@ -28,3 +29,6 @@ wheelhouse/
 dist/
 *.egg-info/
 _skbuild/
+__pycache__/
+*.pyc
+logs/

CMakeLists.txt

@@ -144,6 +144,10 @@ endif()
 if(DEEPITY_ENABLE_CUDA)
     find_package(CUDAToolkit QUIET)
     if(CUDAToolkit_FOUND)
+        enable_language(CUDA)
+        set(CMAKE_CUDA_STANDARD 17)
+        set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+        set(CMAKE_CUDA_ARCHITECTURES 86)
         message(STATUS "CUDA support enabled.")
     else()
         message(STATUS "CUDA toolkit not found. Building CPU-only.")
@@ -199,10 +203,6 @@ else()
     set(DEEPITY_BLAS_RESOLVED FALSE)
 
     if(DEEPITY_USE_MKL)
-        # Modern, recommended integration: Intel's own MKLConfig.cmake,
-        # present if oneAPI/MKL is properly installed and sourced (e.g.
-        # via `source /opt/intel/oneapi/setvars.sh`). Exposes a single
-        # target (MKL::MKL) with both include dirs and link libraries.
         find_package(MKL CONFIG QUIET)
         if(MKL_FOUND)
             message(STATUS "Using Intel MKL (found via MKLConfig.cmake).")
@@ -273,8 +273,11 @@ add_library(Deepity
     src/GaussSeidelPCNetwork.cpp
     src/SimpleConvPCNetwork.cpp
     src/DirectKPPCNetwork.cpp
-    src/ModelIO.cpp
     src/StreamAlignedBatcher.cpp
+    src/backend/Backend.cpp
+    src/backend/CPUBackend.cpp
+    src/backend/CUDABackend.cu
+    src/backend/Tensor.cpp
 )
 
 if(DEEPITY_USE_MKL AND DEEPITY_BLAS_RESOLVED)
@@ -304,6 +307,29 @@ endif()
 
 target_compile_definitions(Deepity PUBLIC SLEEF_STATIC_LIBS)
 
+# --- Optiona: Use Google Benchmark -----------------------------------
+
+include(FetchContent)
+FetchContent_Declare(
+    benchmark
+    GIT_REPOSITORY https://github.com/google/benchmark.git
+    GIT_TAG v1.9.1
+)
+set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
+FetchContent_MakeAvailable(benchmark)
+
+# --- Tests -----------------------------------------------------------
+
+option(DEEPITY_BUILD_TESTS "Build test/verification executables" ON)
+
+add_executable(ActivationBenchmark tests/tActivations.cpp)
+target_link_libraries(ActivationBenchmark PRIVATE Deepity benchmark::benchmark)
+set_target_properties(ActivationBenchmark PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin
+)
+
+add_test(NAME ActivationBenchmark COMMAND ActivationBenchmark)
+
 # --- Compiler flags -------------------------------------------------
 
 if(MSVC)

CMakePresets.json

@@ -1,5 +1,5 @@
 {
-    "version": 6,
+    "version": 3,
     "configurePresets": [
         {
             "name": "windows-clang",
@@ -35,6 +35,25 @@
                 "CMAKE_CXX_COMPILER": "/usr/bin/clang++",
                 "CMAKE_BUILD_TYPE": "Debug"
             }
+        },
+        {
+            "name": "gpu",
+            "displayName": "Linux GPU (CUDA + Python 3.12)",
+            "description": "Cloud GPU instance configuration -- Python 3.12 pinned explicitly (avoids Ubuntu 22.04's default python3 resolving to the system's 3.10, which has no nanobind installed and no ABI match with anything built here). CMAKE_CUDA_ARCHITECTURES set for Ampere (compute_86, matches RTX 3090/A4000/etc) -- change to 89 for Ada Lovelace (RTX 4060 Ti/4070/4090), 80 for A100, 90 for H100.",
+            "generator": "Ninja",
+            "binaryDir": "${sourceDir}/build/Release",
+            "condition": {
+                "type": "notEquals",
+                "lhs": "${hostSystemName}",
+                "rhs": "Windows"
+            },
+            "cacheVariables": {
+                "CMAKE_BUILD_TYPE": "Release",
+                "Python_EXECUTABLE": "${sourceDir}/.venv/bin/python3.12",
+                "DEEPITY_ENABLE_CUDA": "ON",
+                "CMAKE_CUDA_ARCHITECTURES": "86",
+                "CMAKE_CUDA_STANDARD": "17"
+            }
         }
     ]
 }

bindings/pybinding.cpp

@@ -52,8 +52,12 @@ namespace
             return Deep::tanh;
         if (act == "sigmoid")
             return Deep::sigmoid;
+        if (act == "esigmoid")
+            return Deep::e_sigmoid;
         if (act == "relu")
             return Deep::relu;
+        if (act == "gelu")
+            return Deep::gelu;
         if (act == "linear")
             return Deep::linear;
         return Deep::relu;
@@ -65,8 +69,12 @@ namespace
             return Deep::dTanh;
         if (act == "dsigmoid")
             return Deep::dSigmoid;
+        if (act == "d_esigmoid")
+            return Deep::d_eSigmoid;
         if (act == "drelu")
             return Deep::dRelu;
+        if (act == "dgelu")
+            return Deep::dGelu;
         if (act == "dLinear")
             return Deep::dLinear;
         return Deep::dRelu;
@@ -82,12 +90,18 @@ namespace
             return Deep::ActivationType::RELU;
         if (act == "drelu")
             return Deep::ActivationType::dRELU;
+        if (act == "gelu")
+            return Deep::ActivationType::GELU;
+        if (act == "dgelu")
+            return Deep::ActivationType::dGELU;
         if (act == "sigmoid")
             return Deep::ActivationType::SIGMOID;
         if (act == "dsigmoid")
             return Deep::ActivationType::dSIGMOID;
         if (act == "esigmoid")
             return Deep::ActivationType::eSIGMOID;
+        if (act == "d_esigmoid")
+            return Deep::ActivationType::d_eSIGMOID;
         if (act == "dlinear")
             return Deep::ActivationType::dLINEAR;
         return Deep::ActivationType::LINEAR;
@@ -136,7 +150,7 @@ namespace
     template <typename LayerT>
     void BindCommonPCLayer(nb::class_<LayerT, Deep::Layer> &cls, const char *className)
     {
-        cls.def("calculate_state", &LayerT::CalculateState)
+        cls.def("calculate_state", static_cast<float (LayerT::*)() noexcept>(&LayerT::CalculateState))
             .def("update_state", &LayerT::UpdateState)
             .def("update_weights", &LayerT::UpdateWeights)
             .def("flush", &LayerT::Flush)
@@ -420,30 +434,39 @@ void bind_networks(nb::module_ &m)
 
     auto simpleNetCls = nb::class_<Deep::SimplePCNetwork>(m, "SimplePCNetwork", "Predictive Coding Network built from SimplePCLayers.");
     BindCommonPCNetwork<Deep::SimplePCNetwork>(simpleNetCls, "SimplePCNetwork");
+    simpleNetCls.def("__init__", [](Deep::SimplePCNetwork *self, int batch_size, const std::string &device)
+                     {
+    Deep::DeviceType dt = (device == "cuda" || device == "gpu")
+        ? Deep::DeviceType::DEVICE_GPU
+        : Deep::DeviceType::DEVICE_CPU;
+    new (self) Deep::SimplePCNetwork(batch_size, dt); }, nb::arg("batch_size"), nb::arg("device") = "cpu", "Construct a network with a fixed batch size and device (\"cpu\" or \"cuda\"/\"gpu\").");
     simpleNetCls.def("add_layer", [](Deep::SimplePCNetwork &self, int size, int next_size, float lr, float ir, float lmbda, const std::string &activation, const std::string &activation_deriv)
                      { self.AddLayer(size, next_size, lr, ir, lmbda, resolveActEnum(activation), resolveActEnum(activation_deriv)); }, nb::arg("size"), nb::arg("next_size"), nb::arg("lr") = 1e-6f, nb::arg("ir") = 0.1f, nb::arg("lmbda") = 1e-2f, nb::arg("activation") = "relu", nb::arg("activation_deriv") = "drelu", "Add a layer to the network.")
         .def("set_optimizer", [](Deep::SimplePCNetwork &self, const std::string &opt)
              {
-            if (opt == "ADAM") self.SetOptimizer(Deep::OptimizerType::ADAM);
-            else if (opt == "ADAMW") self.SetOptimizer(Deep::OptimizerType::ADAMW);
-            else self.SetOptimizer(Deep::OptimizerType::SGD); }, nb::arg("optimizer"), "Sets the optimizer: ADAM, ADAMW, or SGD.")
-
+        if (opt == "ADAM") self.SetOptimizer(Deep::OptimizerType::ADAM);
+        else if (opt == "ADAMW") self.SetOptimizer(Deep::OptimizerType::ADAMW);
+        else self.SetOptimizer(Deep::OptimizerType::SGD); }, nb::arg("optimizer"), "Sets the optimizer: ADAM, ADAMW, or SGD.")
         .def("project_forward", &Deep::SimplePCNetwork::ProjectForward, "Seeds hidden layers from a genuine forward pass through current "
                                                                         "weights, instead of zero-init. Call AFTER clamp_input(), BEFORE "
                                                                         "the settling loop.")
-
-        .def("train_step_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, FloatArray y, int steps)
+        .def("train_step_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, FloatArray y, int steps, bool computeEnergy)
              {
-    std::vector<float> xvec(x.data(), x.data() + x.size());
-    std::vector<float> yvec(y.data(), y.data() + y.size());
-    return self.TrainStepWithProjection(xvec, yvec, steps); }, nb::arg("x"), nb::arg("y"), nb::arg("steps"))
+std::vector<float> xvec(x.data(), x.data() + x.size());
+std::vector<float> yvec(y.data(), y.data() + y.size());
+return self.TrainStepWithProjection(xvec, yvec, steps, computeEnergy); }, nb::arg("x"), nb::arg("y"), nb::arg("steps"), nb::arg("computeEnergy") = true)
         .def("predict_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, int steps)
              {
-    std::vector<float> xvec(x.data(), x.data() + x.size());
-
-    std::vector<float> out_beliefs = self.PredictWithProjection(xvec, steps);
+std::vector<float> xvec(x.data(), x.data() + x.size());
+std::vector<float> out_beliefs = self.PredictWithProjection(xvec, steps);
+return CopyToNewArray(out_beliefs.data(), {out_beliefs.size()}); }, nb::arg("x"), nb::arg("steps"), "Runs forward-projection init and settling loop entirely in C++, returning terminal beliefs.")
+        .def("randomize_weights", [](Deep::SimplePCNetwork &self, const std::string &distribution)
+             {
+    std::random_device rd;
+    std::mt19937 rng(rd());
+    self.RandomizeWeights(rng, distribution.c_str()); }, nb::arg("distribution"), "Initialize every layer's weights using a distribution string, "
+                                           "e.g. \"normal(0, 1)\" or \"uniform(-0.3, 0.3)\".");
 
-    return CopyToNewArray(out_beliefs.data(), {out_beliefs.size()}); }, nb::arg("x"), nb::arg("steps"), "Runs forward-projection init and settling loop entirely in C++, returning terminal beliefs.");
     nb::class_<Deep::GaussSeidelPCNetwork>(m, "GaussSeidelPCNetwork", "Predictive Coding Network with Gauss-Seidel settling dynamics.")
         .def(nb::init<int>(), nb::arg("batch_size"))
         .def("add_layer", [](Deep::GaussSeidelPCNetwork &self, int size, int next_size, float lr, float ir, float lmbda, const std::string &activation, const std::string &activation_deriv)
@@ -490,7 +513,12 @@ void bind_networks(nb::module_ &m)
            return layers[index].get(); }, nb::rv_policy::reference_internal);
 
     nb::class_<Deep::DirectKPPCNetwork>(m, "DirectKPPCNetwork", "Predictive Coding Network with Direct Kolen-Pollack feedback alignment.")
-        .def(nb::init<int>(), nb::arg("batch_size"))
+        .def("__init__", [](Deep::DirectKPPCNetwork *self, int batch_size, const std::string &device)
+             {
+    Deep::DeviceType dt = (device == "cuda" || device == "gpu")
+        ? Deep::DeviceType::DEVICE_GPU
+        : Deep::DeviceType::DEVICE_CPU;
+    new (self) Deep::DirectKPPCNetwork(batch_size, dt); }, nb::arg("batch_size"), nb::arg("device") = "cpu")
         .def("add_layer", [](Deep::DirectKPPCNetwork &self, size_t size, size_t next_size, size_t terminal_size, float lr, float ir, float fl, float lmbda, const std::string &activation, const std::string &activation_deriv)
              { self.AddLayer(size, next_size, terminal_size, lr, ir, fl, lmbda, resolveActEnum(activation), resolveActEnum(activation_deriv)); }, nb::arg("size"), nb::arg("next_size"), nb::arg("terminal_size"), nb::arg("lr") = 1e-6f, nb::arg("ir") = 0.1f, nb::arg("fl") = 1e-4f, nb::arg("lmbda") = 1e-2f, nb::arg("activation") = "relu", nb::arg("activation_deriv") = "drelu")
         .def("compile", &Deep::DirectKPPCNetwork::Compile)
@@ -700,9 +728,7 @@ void bind_utilities(nb::module_ &m)
           { Deep::dSigmoid(x.data(), x.size()); });
 }
 
-// ============================================================================
 // Main Module Entry
-// ============================================================================
 NB_MODULE(pydeepity, m)
 {
     m.doc() = "Deepity: A high-performance Predictive Coding library.";

deepity_build/cli.py

@@ -79,7 +79,7 @@ def _merge_pgo_profiles(config):
 
     print(f"--- PGO: profile ready: {output} ---")
 
-def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]:
     parser = argparse.ArgumentParser(
         description="Deepity Cross-Platform Build & Test Runner"
     )
@@ -190,7 +190,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
         ),
     )
 
-    args = parser.parse_args(argv)
+    args, unknown_args = parser.parse_known_args(argv)
 
     if args.list_profiles:
         for profile in ARCH_PROFILES.values():
@@ -206,14 +206,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
         args.arch_profile = DEFAULT_ARCH_PROFILE
 
     if args.cuda is None:
-        # Distributed builds default to CPU-only, since a CUDA-linked binary
-        # isn't portable either. Everything else keeps the old default (ON).
         args.cuda = args.arch_profile != "distributed"
 
-    return args
+    return args, unknown_args
 
 
-def build_config_from_args(args: argparse.Namespace) -> BuildConfig:
+def build_config_from_args(args: argparse.Namespace, extra_args: list[str]) -> BuildConfig:
     return BuildConfig(
         build_type=args.build_type,
         jobs=args.jobs,
@@ -226,6 +224,7 @@ def build_config_from_args(args: argparse.Namespace) -> BuildConfig:
         clean=args.clean,
         verbose=args.verbose,
         pgo=args.pgo,
+        extra_cmake_args=tuple(extra_args)
     )
 
 
@@ -349,8 +348,8 @@ def _run_pgo_workload(config: BuildConfig) -> None:
 
 
 def main(argv: list[str] | None = None) -> None:
-    args = parse_args(argv)
-    config = build_config_from_args(args)
+    args, extra_args = parse_args(argv)
+    config = build_config_from_args(args, extra_args)
 
     if config.clean and config.build_dir.exists():
         shutil.rmtree(config.build_dir, onexc=_rmtree_onexc)

deepity_build/cmake_runner.py

@@ -40,6 +40,9 @@ def configure_command(config: BuildConfig, ninja: str | None, pgo_phase: str | N
     if profile.msvc_flags:
         cmd.append(f"-DDEEPITY_MSVC_ARCH_FLAGS={profile.msvc_flags}")
 
+    if config.extra_cmake_args:
+        cmd.extend(config.extra_cmake_args)
+
     if sys.platform == "win32" and ninja:
         # CC being unset doesn't mean clang isn't in play -- CMake can
         # auto-detect and pick it up on its own (as this project's own

deepity_build/config.py

@@ -75,6 +75,7 @@ class BuildConfig:
     clean: bool
     verbose: bool
     pgo: bool = False
+    extra_cmake_args: tuple[str,...] = ()
     build_root: Path = Path("build")
 
     @property