Feat/sde

error.txt

@@ -0,0 +1,44 @@
+-- Configuring done (0.3s)
+-- Generating done (0.0s)
+-- Build files have been written to: /home/wslarch/Documents/Coding/DEV/soft/soft-cuda/build
+
+real	0m0.356s
+user	0m0.157s
+sys	0m0.181s
+[  5%] Built target gtest
+[ 10%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/base/debug.cpp.o
+[ 10%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/bias_add.cpp.o
+[ 13%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/mean.cpp.o
+[ 16%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/mse.cpp.o
+[ 18%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/mul.cpp.o
+[ 21%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/relu.cpp.o
+[ 24%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/add.cpp.o
+[ 29%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/backprop/backprop_b.cpp.o
+[ 29%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/scalar.cpp.o
+[ 32%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/square.cpp.o
+[ 35%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/transpose.cpp.o
+[ 37%] Building CXX object CMakeFiles/soft_lib.dir/src/backend_cpu/math/sub.cpp.o
+[ 40%] Building CUDA object CMakeFiles/soft_lib.dir/src/backend_gpu/math/matmul.cu.o
+[ 43%] Building CUDA object CMakeFiles/soft_lib.dir/src/backend_gpu/math/add.cu.o
+[ 48%] Built target gtest_main
+[ 54%] Built target gmock
+[ 56%] Building CXX object CMakeFiles/soft_lib.dir/src/core/JSON/json_utils.cpp.o
+[ 62%] Built target gmock_main
+[ 64%] Building CXX object CMakeFiles/soft_lib.dir/src/core/graph/DAGbuild.cpp.o
+[ 67%] Building CUDA object CMakeFiles/soft_lib.dir/src/core/graph/assignBackend.cu.o
+[ 70%] Building CXX object CMakeFiles/soft_lib.dir/src/core/graph/train.cpp.o
+[ 72%] Building CUDA object CMakeFiles/soft_lib.dir/src/core/pool/pool.cu.o
+[ 75%] Building CXX object CMakeFiles/soft_lib.dir/src/core/tensor/tensor.cpp.o
+/home/wslarch/Documents/Coding/DEV/soft/soft-cuda/src/core/tensor/tensor.cpp: In function ‘bool tensor_evaluate(tensor_pool_t*, tensor_t*, float*, float*, float*)’:
+/home/wslarch/Documents/Coding/DEV/soft/soft-cuda/src/core/tensor/tensor.cpp:176:19: error: ‘tensor_op_square’ was not declared in this scope; did you mean ‘tensor_square’?
+  176 |         success = tensor_op_square(pool, t);
+      |                   ^~~~~~~~~~~~~~~~
+      |                   tensor_square
+make[2]: *** [CMakeFiles/soft_lib.dir/build.make:363: CMakeFiles/soft_lib.dir/src/core/tensor/tensor.cpp.o] Error 1
+make[2]: *** Waiting for unfinished jobs....
+make[1]: *** [CMakeFiles/Makefile2:181: CMakeFiles/soft_lib.dir/all] Error 2
+make: *** [Makefile:146: all] Error 2
+
+real	0m11.154s
+user	1m11.334s
+sys	0m16.624s

include/soft-cuda/tensor/api.h

@@ -84,7 +84,7 @@ uint32_t tensor_id(tensor_t *t);
  */
 // DONE
 tensor_t *tensor_create(tensor_pool_t *pool, tensor_dtype_t dtype, uint32_t num_dims,
-                        uint32_t *dims, void *elems);
+                        uint32_t *dims, void *elems, bool grad_status = true);
 
 /*
  * Establish a new memory arena
@@ -186,6 +186,7 @@ void tensor_print_data(tensor_t *t);
 // DONE
 tensor_t *tensor_mul(tensor_pool_t *pool, tensor_t *x, tensor_t *y);
 
+tensor_t *tensor_square(tensor_pool_t *pool, tensor_t *x);
 /* The naive version of matrix multiplication
  * @param pool       Pointer to the tensor pool
  * @param x            Pointer to the tensor which will be mulptiplied
@@ -221,6 +222,16 @@ tensor_t *tensor_add(tensor_pool_t *pool, tensor_t *x, tensor_t *y);
 // DONE
 tensor_t *tensor_add_bias(tensor_pool_t *pool, const tensor_t *xw, const tensor_t *bias);
 
+/*
+ * Do matrix subtraction
+ * @param out          Pointer to the tensor where result will be stored
+ * @param x            Pointer to the tensor which will be subtracted from 
+ * @param y            Pointer to the tensor which will be subtracted
+ *
+ * @return             Returns a tensor object with operation set.
+ * */
+// DONE
+tensor_t *tensor_sub(tensor_pool_t *pool, tensor_t *a, tensor_t *b);
 /////////////////////////////////////////////////////////////
 /// DEPRECEATED tensor_mul operation handles it automatically
 // /*
@@ -242,23 +253,25 @@ tensor_t *tensor_add_bias(tensor_pool_t *pool, const tensor_t *xw, const tensor_
 // DONE
 tensor_t *tensor_relu(tensor_pool_t *pool, tensor_t *a);
 
+// DONE
+tensor_t *tensor_mean(tensor_pool_t *pool, tensor_t *a);
 // Compares result
 // return how correct we were b/w 0-1
 tensor_t *tensor_mse_loss(tensor_pool_t *pool, tensor_t *predictions, tensor_t *target);
 
-// Fills an existing tensor with normally distributed random numbers.
+// // Fills an existing tensor with normally distributed random numbers.
 bool tensor_fill_random_normal(tensor_t *t, float mean, float std_dev);
-
-// Fused operation combining Softmax and Cross-Entropy for stability
-tensor_t *tensor_cross_entropy_loss(tensor_pool_t *pool, const tensor_t *predictions,
-                                    const tensor_t *targets);
+//
+// // Fused operation combining Softmax and Cross-Entropy for stability
+// tensor_t *tensor_cross_entropy_loss(tensor_pool_t *pool, const tensor_t *predictions,
+//                                     const tensor_t *targets);
 
 // ***********************************************************************************
 // TODO: HAVE TO UPDATED FUNC SIG SPEC 
 // Evalutes the operation(Forward) with depth=1
 // @return             boolean flag for status
 // DONE
-bool tensor_evaluate( tensor_pool_t *pool,tensor_t *t,  float *d_a, float *d_b, float *d_res);
+bool tensor_evaluate( tensor_pool_t *pool,tensor_t *t,  float *d_a = nullptr, float *d_b = nullptr, float *d_res= nullptr);
 // DONE
 bool tensor_evaluate_GPU( tensor_pool_t *pool,tensor_t *t,  float *d_a, float *d_b, float *d_res);
 
@@ -273,9 +286,7 @@ bool tensor_evaluate_GPU( tensor_pool_t *pool,tensor_t *t,  float *d_a, float *d
 bool tensor_backward(tensor_pool_t *pool, tensor_t *t);
 
 // The Optimizer
-void tensor_sgd_template(tensor_pool_t *static_weights_pool, double learning_rate);
-
-// API UNSTABLE
+void tensor_sgd(std::vector<execution_node_t *> &nodes, float learning_rate);
 /////////////////////////////////////////////////////////////
 // GRAPH OPERATIONS
 
@@ -321,6 +332,7 @@ bool verifyIfDAG(tensor_pool_t *pool, tensor_t *t, std::vector<execution_node_t
 // DONE
 void assignBackendGraph(tensor_pool_t *pool,std::vector<execution_node_t *> &nodes);
 
+void assignGradMemory(tensor_pool_t *pool_grad_cpu, tensor_pool_t *pool_grad_gpu, std::vector<execution_node_t *> &nodes);
 /* @params  Take execution_node_t which you wanna know
  * @returns the postion of the execution_node_t after verifyIfDAG
  * */
@@ -340,14 +352,15 @@ void printExecutionNode(execution_node_t *et);
  * */
 bool tensor_graph_forward_evaluate(tensor_pool_t *pool_cpu, tensor_pool_t *pool_gpu, std::vector<execution_node_t *> &nodes);
 
-/*
- * Evaluate the whole graph backward operation.
- * @params            graph struct for ops sequence
- * @return            boolean status flag
- * */
-bool tensor_graph_backward_evaluate(tensor_graph_t *g);
-
 //////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////
+// BACKWARD PASS FUNCTIONS
+
+void gradInitializer(std::vector<execution_node_t *> &nodes);
+
+bool tensor_graph_backward(std::vector<execution_node_t *> &nodes);
+
+void assignGradMemory(tensor_pool_t *pool_grad_cpu, tensor_pool_t *pool_grad_gpu, std::vector<execution_node_t *> &nodes);
 
 #ifdef __cplusplus
 }

main.cpp

@@ -2,75 +2,123 @@
 #include "soft-cuda/tensor/debug_api.h"
 #include <iostream>
 #include <vector>
+#include <cassert>
 
 using namespace std;
 
 int main() {
-    // Create pools
-    tensor_pool_t *pool = tensor_pool_create(1024 * 1024);
-    tensor_pool_t *pool_2 = tensor_pool_create(1024 * 1024);
-    tensor_pool_t *pool_gpu = tensor_pool_create(1024 * 1024, true);
+    tensor_pool_t *pool          = tensor_pool_create(1024 * 1024);
+    tensor_pool_t *pool_meta     = tensor_pool_create(1024 * 1024);
+    tensor_pool_t *pool_grad_cpu = tensor_pool_create(1024 * 1024);
+
+    tensor_pool_t *pool_gpu      = tensor_pool_create(1024 * 1024, true);
+    tensor_pool_t *pool_grad_gpu = tensor_pool_create(1024 * 1024, true);
+
     assert(pool != NULL);
     assert(pool_gpu != NULL);
 
     cout << "=========================================== \n";
-    cout << "=============TESTING DAG VERIFICATION============== \n";
+    cout << "========= XOR IMPLEMENTATION (4-NEURON) ==== \n";
+
+    float val_X[8]{0,0, 0,1, 1,0, 1,1};
+    float val_Y[4]{0, 1, 1, 0};
 
-    float val_a[900]{};
-    float val_b[900]{};
-    float val_c[900]{};
-    float val_d[900]{};
+    float val_W1[8]{}; 
+    float val_W2[4]{}; 
+    float val_b1[4]{}; 
+    float val_b2[1]{}; 
+
+    uint32_t dims_X[]  = {4, 2};
+    uint32_t dims_Y[]  = {4, 1};
+    uint32_t dims_W1[] = {2, 4}; 
+    uint32_t dims_b1[] = {1, 4};
+    uint32_t dims_W2[] = {4, 1};
+    uint32_t dims_b2[] = {1, 1};
+
+    tensor_t *X  = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_X,  val_X, false);
+    tensor_t *Y  = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_Y,  val_Y, false);
 
-    // TACTICAL FIX: Make 'e' the exact same size to avoid a GPU segfault 
-    // before we implement a dedicated GPU broadcasting kernel!
-    float val_e[900]{};
-
-    uint32_t dims_a[] = {30, 30};
-    uint32_t dims_b[] = {30, 30};
-    uint32_t dims_c[] = {30, 30};
-    uint32_t dims_d[] = {30, 30};
-    uint32_t dims_e[] = {30, 30}; // Matched dimensions!
-
-    tensor_t *a = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_a, val_a);
-    tensor_t *b = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_b, val_b);
-    tensor_t *c = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_c, val_c);
-    tensor_t *d = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_d, val_d);
-    tensor_t *e = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_e, val_e);
-    tensor_fill_random_normal(a, 10.1, 5.7);
-    tensor_fill_random_normal(b, 10.1, 5.7);
-    tensor_fill_random_normal(c, 10.1, 5.7);
-    tensor_fill_random_normal(d, 10.1, 5.7);
-    tensor_fill_random_normal(e, 10.1, 5.7);
-    tensor_t *f = tensor_mul(pool, a, b);
-    tensor_t *g = tensor_mul(pool, b, c);
-    tensor_t *h = tensor_mul(pool, e, f);
-    tensor_t *i = tensor_mul(pool, h, a);
-    tensor_t *j = tensor_mul(pool, i, g);
+    tensor_t *W1 = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_W1, val_W1, true);
+    tensor_t *W2 = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_W2, val_W2, true);
+    tensor_t *b1 = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_b1, val_b1, true);
+    tensor_t *b2 = tensor_create(pool, tensor_dtype_t::FLOAT32_T, 2, dims_b2, val_b2, true);
+
+    tensor_fill_random_normal(W1, 0.5, 0.2);
+    tensor_fill_random_normal(W2, 0.5, 0.2);
+    tensor_fill_random_normal(b1, 0.0, 0.1);
+    tensor_fill_random_normal(b2, 0.0, 0.1);
 
     cout << "=========================================== \n";
-    cout << "==============LAZY EVAL DONE=============== \n";
+    cout << "============== DEFINING OPS =============== \n";
+
+    tensor_t *H_pred_bef = tensor_mul_naive(pool, X, W1);
+    tensor_t *H_pred     = tensor_add(pool, H_pred_bef, b1);
+    tensor_t *H          = tensor_relu(pool, H_pred);
+    tensor_t *Y_pred_bef = tensor_mul_naive(pool, H, W2);
+    tensor_t *Y_pred     = tensor_add(pool, Y_pred_bef, b2);
 
-    std::vector<execution_node_t*> seq;
-    bool oki = verifyIfDAG(pool_2, j, seq);
+    tensor_t *L_sub      = tensor_sub(pool, Y_pred, Y);
+    tensor_t *L_squ      = tensor_square(pool, L_sub);
+    tensor_t *mse        = tensor_mean(pool, L_squ);
 
-    // Ensure assignDevice is returning device_type::GPU under the hood!
+    cout << "=========================================== \n";
+    cout << "============== LAZY EVAL DONE ============= \n";
+
+    cout << "=========================================== \n";
+    cout << "=========== PREPARING THE KITCHEN ========= \n";
+    std::vector<execution_node_t*> seq;
+    bool oki = verifyIfDAG(pool_meta, mse, seq);
     assignBackendGraph(pool_gpu, seq);
-    tensor_graph_forward_evaluate(pool, pool_gpu, seq);
-    
+    assignGradMemory(pool_grad_cpu, pool_grad_gpu, seq);
+
     if (oki) {
         cout << "=========================================== \n";
-        cout << "========= VRAM TRACE ============= \n";
+        cout << "============= STARTING TRAINING =========== \n";
 
-            execution_node_t *node = seq.back();
+        for (int i = 0; i <= 10000; i++) {
+            tensor_graph_forward_evaluate(pool, pool_gpu, seq);
+            gradInitializer(seq);
+            tensor_graph_backward(seq);
+
+            if (i % 1000 == 0) {
+                std::cout << "EPOCH " << i << "\n";
+                execution_node_t *mse_node = seq.back();
+                printExecutionNode(mse_node);
+            }
 
-                // Pull it back to the CPU
-                execution_node_to_host(node);
-                printExecutionNode(node);
-                cout << "\n";
+            tensor_sgd(seq, 0.05); 
+        }
+
+        tensor_graph_forward_evaluate(pool, pool_gpu, seq);
+
+        float* inputs      = (float*)tensor_get_data(X);
+        float* targets     = (float*)tensor_get_data(Y);
+        float* predictions = (float*)tensor_get_data(Y_pred);
+
+        cout << "=========================================== \n";
+        cout << "============= XOR(4 NEURONS RESULT) ============= \n";
+        cout << "X1\tX2\t|\tTarget\t|\tPredicted\n";
+        cout << "---------------------------------------------------\n";
+
+        for (int i = 0; i < 4; i++) {
+            float x1     = inputs[i * 2 + 0];
+            float x2     = inputs[i * 2 + 1];
+            float y_true = targets[i];
+            float y_pred = predictions[i];
+
+            cout << x1 << "\t" << x2 << "\t|\t" << y_true << "\t|\t" << y_pred << "\n";
+        }
+        cout << "=========================================== \n";
+
+    } else {
+        cout << "WARNING: DAG Verification failed. Aborting expedition.\n";
     }
 
     tensor_pool_destroy(pool);
-    tensor_pool_destroy(pool_2);
+    tensor_pool_destroy(pool_meta);
+    tensor_pool_destroy(pool_grad_cpu);
+    tensor_pool_destroy(pool_grad_gpu);
     tensor_pool_destroy(pool_gpu);
+
     return 0;
 }

make.sh

@@ -1,2 +1,2 @@
-cmake -B build -DCMAKE_BUILD_TYPE=Debug
-cmake --build build
+time cmake -B build -DCMAKE_BUILD_TYPE=Debug
+time cmake --build build -j16

notes/Backward/MemoryModel.md

@@ -0,0 +1,53 @@
+## Memory model for backward pass(Unstable)
+
+The problem we have is that when we do hybrid forward pass, it causes some data to be on GPU and some to be on CPU, this I thought
+would be a problem but that is not the case.
+
+### Observation
+> Since data flow is only one directional and due to our contagious logic
+
+```
+            assignPlaceOnDeviceMemory(pool, (int32_t)(node->pos), nodes);
+            if (node->t->a == NULL)
+                goto trp;
+            if (node->t->a->device == device_type::CPU) {
+                int32_t a_idx = getTheExecutionNodeIndex(node, 0);
+                assert(a_idx != -1);
+                nodes[((size_t)a_idx)]->to_device_needed = true;
+                assignPlaceOnDeviceMemory(pool, a_idx, nodes);
+            }
+        trp:
+            if (node->t->b == NULL)
+                continue;
+            if (node->t->b->device == device_type::CPU) {
+                int32_t b_idx = getTheExecutionNodeIndex(node, 1);
+                assert(b_idx != -1);
+                nodes[((size_t)b_idx)]->to_device_needed = true;
+                assignPlaceOnDeviceMemory(pool, b_idx, nodes);
+            }
+
+
+```
+We can conclude that all the data we need for grad computation is available on each device. and since we copy the parents we can 
+confidently say that we don't need to retransfer tensors inbetween eliminating the thrashing.
+
+* So here comes the part where what we will do in
+`void assignBackendGraph(tensor_pool_t *pool,std::vector<execution_node_t *> &nodes);` 
+is we will allocate the grad space for tensors on GPU as `pool_gpu_grad` and on CPU as `pool_cpu_grad` then we assign it to the home tensor(refering to the current tensor)
+then during backward pass data is assigned to each pool and is written as it is.
+since we don't want transfer overhead during computation path we will construct another data transfering fucnction which will get grad data from gpu to host.
+and then we do optimization and then free resources.
+
+Also the device_ptr_gpu is stored in execution_node_t;
+
+===================================================================================================
+
+actually i think we can't do grad placement in the AssignBackendGraph function which we are implementing the contagious logic we will need a 4th pass i think where so we can do this maneuvour
+
+P1->GPU, P2->GPU, C1->GPU
+
+but now if P1 and P2 was bought to GPU and it's copy exist on cpu we can jump to it's DRAM storage and i think since it's address we will know due to well how we can detect it you may ask it would be if it have address on execution node and also on tensor instance right then we switch to CPU allocation right.
+
+Now for this to work we will need to walk reverse in graph so one more reason for 4th seperate pass plus this also helps that we will not be touching much of things.
+
+Now if we are already doing 4th pass it would be better to instead have a seperate explict function for this so func params also don't get bloated and confusing.