From 6e283def6dbe82aa756a18b0dc4a67018a6308cd Mon Sep 17 00:00:00 2001 From: danmcleran Date: Sun, 30 Aug 2026 22:44:12 -0600 Subject: [PATCH] test: make the seeded RNG independent of the C++ standard library The nightly MSan job links an instrumented libc++, and under it unit_test/nn failed two tolerance checks that pass under libstdc++. Not a defect: several suites initialize network weights from a seeded RNG and then assert on what training converges to, and std::uniform_real_distribution does not specify the mapping from engine output to value. libstdc++ and libc++ return different doubles from identical engine state, so the weights differ and trained results move. std::default_random_engine is likewise an implementation-chosen typedef. unit_test/include/portable_test_random.hpp freezes the sequence the tests were written against, in portable code: minstd_rand0 (x = 16807x mod 2^31-1) plus the generate_canonical mapping libstdc++ applies over it, which for this engine consumes exactly two draws. Verified bit-identical to libstdc++ over 20,000 draws -- 0 mismatches, max difference 0 -- so every tolerance in these suites keeps its current meaning and not one needed changing. std::mt19937, whose output the standard does specify, was the first choice and was rejected on evidence. It is statistically indistinguishable (mean ~0, mean|x| ~0.5 over 2000 draws) but produces a different sequence, and two tests do not survive that: lstm_weight_serialization at 0.605 against a 0.02 bound, and rmsprop_fixedpoint_xor at average error 9 against a bound of 4. Loosening those to accommodate a new generator would weaken two real gates to settle a question neither is asking about. test_case_portable_uniform_real_matches_frozen_sequence locks the first eight draws as an exact assertion. Every training test here is calibrated against those numbers, so a future edit to the generator would otherwise silently re-tune all of them; now it fails loudly. Verified to have teeth by perturbing an expected value and confirming the failure. Applied to unit_test/nn, unit_test/kan and unit_test/qlearn. All three pass under both g++ and clang++. qlearn's std::uniform_int_distribution use for maze state selection is left alone and documented: that mapping is implementation-defined too, but the suite passes under the instrumented libc++ today, and changing a passing suite's draw sequence risks its assertions for no observed benefit. --- unit_test/include/portable_test_random.hpp | 125 + unit_test/kan/Makefile | 2 +- unit_test/kan/kan_unit_test.cpp | 15 +- unit_test/nn/Makefile | 2 +- unit_test/nn/nn_unit_test.cpp | 17011 ++++++++++--------- unit_test/qlearn/Makefile | 2 +- unit_test/qlearn/qlearn_unit_test.cpp | 1959 ++- 7 files changed, 9633 insertions(+), 9483 deletions(-) create mode 100644 unit_test/include/portable_test_random.hpp diff --git a/unit_test/include/portable_test_random.hpp b/unit_test/include/portable_test_random.hpp new file mode 100644 index 00000000..26f5f1ea --- /dev/null +++ b/unit_test/include/portable_test_random.hpp @@ -0,0 +1,125 @@ +/** +* Copyright (c) 2026 Dan McLeran +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#ifndef TINYMIND_PORTABLE_TEST_RANDOM_HPP +#define TINYMIND_PORTABLE_TEST_RANDOM_HPP + +#include + +// Test-only. Not part of the shipped library. +// +// Why this exists +// --------------- +// Several suites initialize network weights from a seeded RNG and then assert +// on what training converges to. That is only reproducible if the sequence is +// reproducible -- and std::uniform_real_distribution is NOT. The standard +// specifies the distribution, not the mapping from engine output to value, so +// libstdc++ and libc++ return different doubles from identical engine state. +// std::default_random_engine is likewise a typedef the implementation picks. +// +// The consequence was real: the nightly MSan job, which links an instrumented +// libc++, saw unit_test/nn fail two tolerance checks that pass under libstdc++ +// -- not a defect, just a different draw sequence. +// +// Why it reproduces libstdc++ rather than picking something cleaner +// ---------------------------------------------------------------- +// The obvious fix -- switch to std::mt19937, whose output IS specified, and do +// the scaling by hand -- was tried first and rejected on evidence. It is a +// perfectly good generator: statistically indistinguishable from the current +// one (mean ~0, mean|x| ~0.5 over 2000 draws). But it produces a different +// sequence, and two tests do not survive that: +// +// test_case_lstm_weight_serialization 0.605 against a 0.02 bound +// test_case_rmsprop_fixedpoint_xor average error 9 against a bound of 4 +// +// Those tolerances are calibrated against the draws the tests have always seen. +// Loosening them to accommodate a new generator would weaken two real gates to +// settle a question neither is asking about. +// +// Making them robust to any initialization is the deeper fix and a much larger +// change: it means deciding what each of those tolerances should assert, which +// risks masking genuine regressions if done in bulk. Freezing the sequence the +// tests were written against gets the portability without touching a single +// tolerance, and leaves that larger cleanup as separate work. +// +// What is frozen +// -------------- +// libstdc++'s std::default_random_engine is minstd_rand0: x = 16807x mod +// (2^31 - 1). Its uniform_real_distribution goes through +// generate_canonical, which for this engine consumes exactly two +// draws (b = 53 bits, log2(r) = 30, so k = ceil(53/30) = 2) and forms +// (d1 + d2*r) / r^2 before scaling into [low, high). +// +// Verified bit-identical to libstdc++'s output over 20,000 draws: 0 mismatches, +// max difference 0. test_case_portable_uniform_real_matches_frozen_sequence in +// nn_unit_test.cpp locks the first draws in as an assertion, so a future edit +// cannot quietly change the sequence and silently re-tune every training test +// that depends on it. +class PortableUniformReal +{ +public: + PortableUniformReal(const double low, const double high, const uint32_t s) + : mState(s != 0u ? s : 1u), mLow(low), mHigh(high) + { + } + + double operator()() + { + // r = engine.max() - engine.min() + 1, i.e. (2^31 - 2) - 1 + 1. + const double r = 2147483646.0; + const double d1 = static_cast(next() - 1u); + const double d2 = static_cast(next() - 1u); + const double sum = d1 + (d2 * r); + const double canonical = sum / (r * r); + + return (canonical * (mHigh - mLow)) + mLow; + } + + void seed(const uint32_t s) + { + mState = (s != 0u) ? s : 1u; + } + +private: + uint32_t next() + { + mState = static_cast((16807ULL * static_cast(mState)) % 2147483647ULL); + + return mState; + } + + uint32_t mState; + double mLow; + double mHigh; +}; + +// Not covered here +// ---------------- +// unit_test/qlearn still uses std::default_random_engine with +// std::uniform_int_distribution to pick maze states and actions. That mapping +// is implementation-defined too, so the sequence differs across libraries in +// principle. It is left alone deliberately: the suite passes under the +// instrumented libc++ in the nightly MSan job, so there is no observed problem +// to fix, and changing the draw sequence of a passing suite risks breaking its +// assertions for no benefit. Worth revisiting only if it actually diverges. + +#endif // TINYMIND_PORTABLE_TEST_RANDOM_HPP diff --git a/unit_test/kan/Makefile b/unit_test/kan/Makefile index c23959b7..e9b5d7f9 100644 --- a/unit_test/kan/Makefile +++ b/unit_test/kan/Makefile @@ -4,7 +4,7 @@ CC=g++ WARN=-Wall -Wextra -Werror -Wpedantic SOURCES=kan_unit_test.cpp ../../cpp/lookupTables.cpp DEFINES=-DTINYMIND_ENABLE_OSTREAMS=1 -DTINYMIND_ENABLE_FLOAT=1 -DTINYMIND_ENABLE_STD=1 -DTINYMIND_ENABLE_HOSTED_RAND=1 -DTINYMIND_USE_SIGMOID_8_8=1 -INCLUDES=-I ../../include -I../../cpp -I../../cpp/include -I../../include -I${BOOST_HOME} +INCLUDES=-I../include -I ../../include -I../../cpp -I../../cpp/include -I../../include -I${BOOST_HOME} OUT=./output/kan_unit_test kan_unit_test: $(MKDIR) diff --git a/unit_test/kan/kan_unit_test.cpp b/unit_test/kan/kan_unit_test.cpp index d648c534..45f7c896 100644 --- a/unit_test/kan/kan_unit_test.cpp +++ b/unit_test/kan/kan_unit_test.cpp @@ -39,6 +39,7 @@ TINYMIND_DISABLE_WARNING_POP #include #include +#include "portable_test_random.hpp" #include "qformat.hpp" #include "bspline.hpp" #include "kan.hpp" @@ -384,9 +385,10 @@ struct DoubleRandomNumberGenerator { static double generateRandomWeight() { - static std::default_random_engine generator(RANDOM_SEED); - static std::uniform_real_distribution distribution(-0.5, 0.5); - return distribution(generator); + // Implementation-independent sequence -- see + // unit_test/include/portable_test_random.hpp. + static PortableUniformReal generator(-0.5, 0.5, RANDOM_SEED); + return generator(); } }; @@ -575,9 +577,10 @@ struct SinusoidRandomNumberGenerator { static double generateRandomWeight() { - static std::default_random_engine generator(RANDOM_SEED); - static std::uniform_real_distribution distribution(-0.1, 0.1); - return distribution(generator); + // Implementation-independent sequence -- see + // unit_test/include/portable_test_random.hpp. + static PortableUniformReal generator(-0.1, 0.1, RANDOM_SEED); + return generator(); } }; diff --git a/unit_test/nn/Makefile b/unit_test/nn/Makefile index 6099c578..5892f010 100644 --- a/unit_test/nn/Makefile +++ b/unit_test/nn/Makefile @@ -6,7 +6,7 @@ CC=g++ WARN=-Wall -Wextra -Werror -Wpedantic SOURCES=nn_unit_test.cpp ../../cpp/lookupTables.cpp DEFINES=-DTINYMIND_ENABLE_OSTREAMS=1 -DTINYMIND_ENABLE_FLOAT=1 -DTINYMIND_ENABLE_STD=1 -DTINYMIND_ENABLE_HOSTED_IO=1 -DTINYMIND_ENABLE_HOSTED_RAND=1 -DTINYMIND_USE_SIGMOID_8_8=1 -DTINYMIND_USE_SIGMOID_16_16=1 -DTINYMIND_USE_LOG_16_16=1 -DTINYMIND_USE_TANH_8_8=1 -DTINYMIND_USE_TANH_8_24=1 -DTINYMIND_USE_TANH_16_16=1 -DTINYMIND_USE_EXP_16_16=1 -DTINYMIND_USE_EXP_8_8=1 -INCLUDES=-I ../../include -I../../cpp -I../../cpp/include -I../../include -I${BOOST_HOME} +INCLUDES=-I../include -I ../../include -I../../cpp -I../../cpp/include -I../../include -I${BOOST_HOME} OUT=./output/nn_unit_test nn_unit_test: $(MKDIR) diff --git a/unit_test/nn/nn_unit_test.cpp b/unit_test/nn/nn_unit_test.cpp index 0eb60c58..299194fc 100644 --- a/unit_test/nn/nn_unit_test.cpp +++ b/unit_test/nn/nn_unit_test.cpp @@ -1,8493 +1,8518 @@ -/** -* Copyright (c) 2020 Intel Corporation -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -// nn_unit_test.cpp : Defines the entry point for the neural network template unit tests. - -#include "compiler.h" - -#define BOOST_TEST_MODULE nn_unit_test -TINYMIND_DISABLE_WARNING_PUSH -TINYMIND_DISABLE_WARNING_GCC_ONLY("-Wdangling-reference") -#include -TINYMIND_DISABLE_WARNING_POP - -#include -#include - -#include "qformat.hpp" -#include "neuralnet.hpp" -#include "activationFunctions.hpp" -#include "fixedPointTransferFunctions.hpp" -#include "gradientClipping.hpp" -#include "weightDecay.hpp" -#include "learningRateSchedule.hpp" -#include "adam.hpp" -#include "earlyStopping.hpp" -#include "teacherForcing.hpp" -#include "dual.hpp" -#include "pinn.hpp" -#include "truncatedBPTT.hpp" -#include "conv1d.hpp" -#include "pool1d.hpp" -#include "upsample1d.hpp" -#include "conv2d.hpp" -#include "depthwiseconv2d.hpp" -#include "pointwiseconv2d.hpp" -#include "pool2d.hpp" -#include "bench/platform.hpp" -#include "bench/report.hpp" -#include "dropout.hpp" -#include "batchnorm.hpp" -#include "rmsprop.hpp" -#include "binarylayer.hpp" -#include "ternarylayer.hpp" -#include "selfattention1d.hpp" -#include "anfis.hpp" -#include "moe.hpp" -#include "fft1d.hpp" -#include "networkStats.hpp" -#include "random.hpp" -#include "nnproperties.hpp" -#include "xavier.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace tinymind { - template<> - struct TanhActivationPolicy - { - static double activationFunction(const double& value) - { - return std::tanh(value); - } - - static double activationFunctionDerivative(const double& value) - { - //Approximation for 1st derivative of tanh - return (static_cast(1.0) - (value * value)); - } - }; -} - -namespace tinymind { - template<> - struct SigmoidActivationPolicy - { - static double activationFunction(const double& value) - { - return (1.0 / (1.0 + std::exp(-value))); - } - - static double activationFunctionDerivative(const double& value) - { - return (value * (1.0 - value)); - } - }; -} - -namespace tinymind { - template<> - struct ZeroToleranceCalculator - { - static bool isWithinZeroTolerance(const double& value) - { - static const double zeroTolerance(0.004); - static const double negativeTolerance = (static_cast(-1.0) * zeroTolerance); - - return ((0 == value) || ((value < zeroTolerance) && (value > negativeTolerance))); - } - }; -} - -namespace tinymind { - template<> - struct Constants - { - static float one() - { - return 1.0f; - } - - static float negativeOne() - { - return -1.0f; - } - - static float zero() - { - return 0.0f; - } - }; - - template<> - struct Constants - { - static double one() - { - return 1.0; - } - - static double negativeOne() - { - return -1.0; - } - - static double zero() - { - return 0.0; - } - }; -} - -using namespace std; - -#define TRAINING_ITERATIONS 2000 -#define NUM_SAMPLES_AVG_ERROR 20 -#define STOP_ON_AVG_ERROR 0 -#define USE_WEIGHTS_INPUT_FILE 0 // the weights input file uses the initial values from a successful training run -#define RANDOM_SEED 7U - -template -struct ValueHelper -{ - typedef typename ValueType::FullWidthValueType FullWidthValueType; - - static FullWidthValueType getErrorLimit() - { - static const FullWidthValueType ERROR_LIMIT = (1 << (ValueType::NumberOfFractionalBits - 6)); - - return ERROR_LIMIT; - } -}; - -template<> -struct ValueHelper -{ - static double getErrorLimit() - { - return 0.1; - } -}; - -template -struct UniformRealRandomNumberGenerator -{ - typedef tinymind::ValueConverter WeightConverterPolicy; - - static ValueType generateRandomWeight() - { - const double temp = distribution(generator); - const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); - - return weight; - } - - static void seed(unsigned int s) - { - generator.seed(s); - } -private: - static std::default_random_engine generator; - static std::uniform_real_distribution distribution; -}; - -template -std::default_random_engine UniformRealRandomNumberGenerator::generator(RANDOM_SEED); - -template -std::uniform_real_distribution UniformRealRandomNumberGenerator::distribution(-1.0, 1.0); - -template -struct XavierUniformRandomNumberGenerator -{ - typedef tinymind::XavierWeightInitializer XavierWeightInitializerType; - typedef tinymind::ValueConverter WeightConverterPolicy; - - static ValueType generateRandomWeight() - { - static XavierWeightInitializerType xavierWeightInitializer; - const double temp = xavierWeightInitializer.generateUniformWeight(); - const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); - - return weight; - } -}; - -template -struct XavierNormalRandomNumberGenerator -{ - typedef tinymind::XavierWeightInitializer XavierWeightInitializerType; - typedef tinymind::ValueConverter WeightConverterPolicy; - - static ValueType generateRandomWeight() - { - static XavierWeightInitializerType xavierWeightInitializer; - const double temp = xavierWeightInitializer.generateNormalWeight(); - const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); - - return weight; - } -}; - -template -struct XavierUniformHeterogeneousRandomNumberGenerator -{ - typedef tinymind::XavierWeightInitializerForLayers XavierWeightInitializerType; - typedef tinymind::ValueConverter WeightConverterPolicy; - - static ValueType generateRandomWeight() - { - static XavierWeightInitializerType xavierWeightInitializer; - const double temp = xavierWeightInitializer.generateUniformWeight(); - const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); - - return weight; - } -}; - -template< - typename ValueType, - template class TransferFunctionRandomNumberGeneratorPolicy, - template class TransferFunctionHiddenNeuronActivationPolicy, - template class TransferFunctionOutputNeuronActivationPolicy, - template class TransferFunctionGatedNeuronActivationPolicy = tinymind::NullActivationPolicy, - template class TransferFunctionZeroTolerancePolicy = tinymind::ZeroToleranceCalculator, - unsigned NumberOfOutputNeurons = 1> -struct FloatingPointTransferFunctions -{ - typedef ValueType TransferFunctionsValueType; - typedef TransferFunctionRandomNumberGeneratorPolicy RandomNumberGeneratorPolicy; - typedef TransferFunctionHiddenNeuronActivationPolicy HiddenNeuronActivationPolicy; - typedef TransferFunctionOutputNeuronActivationPolicy OutputNeuronActivationPolicy; - typedef TransferFunctionGatedNeuronActivationPolicy GatedNeuronActivationPolicy; - typedef TransferFunctionZeroTolerancePolicy ZeroToleranceCalculatorPolicy; - - static const unsigned NumberOfTransferFunctionsOutputNeurons = NumberOfOutputNeurons; - - static ValueType calculateError(ValueType const* const targetValues, ValueType const* const outputValues) - { - ValueType error(0); - - //calculate overall error RMS - for (uint32_t neuron = 0; neuron < NumberOfOutputNeurons; neuron++) - { - const ValueType delta = (targetValues[neuron] - outputValues[neuron]); - error += (delta * delta); - } - - if (NumberOfOutputNeurons > 1) - { - error /= NumberOfOutputNeurons; - } - - return error; - } - - static ValueType calculateOutputGradient(const ValueType& targetValue, const ValueType& outputValue) - { - const ValueType delta = targetValue - outputValue; - - return (delta * OutputNeuronActivationPolicy::activationFunctionDerivative(outputValue)); - } - - static ValueType gateActivationFunction(const ValueType& value) - { - return GatedNeuronActivationPolicy::activationFunction(value); - } - - static ValueType generateRandomWeight() - { - return RandomNumberGeneratorPolicy::generateRandomWeight(); - } - - static ValueType hiddenNeuronActivationFunction(const ValueType& value) - { - return HiddenNeuronActivationPolicy::activationFunction(value); - } - - static ValueType hiddenNeuronActivationFunctionDerivative(const ValueType& value) - { - return HiddenNeuronActivationPolicy::activationFunctionDerivative(value); - } - - static ValueType outputNeuronActivationFunction(const ValueType& value) - { - return OutputNeuronActivationPolicy::activationFunction(value); - } - - static ValueType outputNeuronActivationFunctionDerivative(const ValueType& value) - { - return OutputNeuronActivationPolicy::activationFunctionDerivative(value); - } - - static ValueType initialAccelerationRate() - { - ValueType rate(0.1); - - return rate; - } - - static ValueType initialBiasOutputValue() - { - const ValueType value(1.0); - - return value; - } - - static ValueType initialDeltaWeight() - { - const ValueType delta(0); - - return delta; - } - - static ValueType initialGradientValue() - { - const ValueType gradient(0); - - return gradient; - } - - static ValueType initialLearningRate() - { - ValueType rate(0.15); - - return rate; - } - - static ValueType initialMomentumRate() - { - ValueType rate(0.5); - - return rate; - } - - static ValueType initialOutputValue() - { - const ValueType value(0); - - return value; - } - - static bool isWithinZeroTolerance(const ValueType& value) - { - return ZeroToleranceCalculatorPolicy::isWithinZeroTolerance(value); - } - - static ValueType negate(const ValueType& value) - { - return (-1.0 * value); - } - - static ValueType neuronActivationFunction(const ValueType& value) - { - return HiddenNeuronActivationPolicy::activationFunction(value); - } - - static ValueType neuronActivationFunctionDerivative(const ValueType& value) - { - return HiddenNeuronActivationPolicy::activationFunctionDerivative(value); - } - - static ValueType noOpDeltaWeight() - { - const ValueType value(1.0); - - return value; - } - - static ValueType noOpWeight() - { - const ValueType value(1.0); - - return value; - } -}; - -template -static void generateXorValues(T* values, T* output) -{ - const uint32_t x = (rand() & 1); - const uint32_t y = (rand() & 1); - const uint32_t z = x ^ y; - - *values = static_cast(x); - ++values; - *values = static_cast(y); - *output = z; -} - -template -static void generateFixedPointXorValues(T* values, T* output) -{ - const uint32_t x = (rand() & 1); - const uint32_t y = (rand() & 1); - const uint32_t z = x ^ y; - - // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and - // left-shifting a negative value is undefined behavior before C++20. - const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; - *values = static_cast(x * scale); - ++values; - *values = static_cast(y * scale); - *output = (z * scale); -} - -template -static void generateAndValues(T* values, T* output) -{ - const uint32_t x = (rand() & 1); - const uint32_t y = (rand() & 1); - const uint32_t z = x & y; - - *values = static_cast(x); - ++values; - *values = static_cast(y); - *output = z; -} - -template -static void generateFixedPointAndValues(T* values, T* output) -{ - const uint32_t x = (rand() & 1); - const uint32_t y = (rand() & 1); - const uint32_t z = x & y; - - // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and - // left-shifting a negative value is undefined behavior before C++20. - const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; - *values = static_cast(x * scale); - ++values; - *values = static_cast(y * scale); - *output = (z * scale); -} - -template -static void generateOrValues(T* values, T* output) -{ - const uint32_t x = (rand() & 1); - const uint32_t y = (rand() & 1); - const uint32_t z = x | y; - - *values = static_cast(x); - ++values; - *values = static_cast(y); - *output = z; -} - -template -static void generateFixedPointOrValues(T* values, T* output) -{ - const uint32_t x = (rand() & 1); - const uint32_t y = (rand() & 1); - const uint32_t z = x | y; - - // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and - // left-shifting a negative value is undefined behavior before C++20. - const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; - *values = static_cast(x * scale); - ++values; - *values = static_cast(y * scale); - *output = (z * scale); -} - -template -static void generateFixedPointNorValues(T* values, T* output) -{ - const uint32_t x = (rand() & 1); - const uint32_t y = (rand() & 1); - const uint32_t z = !(x | y); - - // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and - // left-shifting a negative value is undefined behavior before C++20. - const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; - *values = static_cast(x * scale); - ++values; - *values = static_cast(y * scale); - *output = (z * scale); -} - -template -static void generateRecurrentValues(T* values, T* output) -{ - static int32_t x = -1; - static int32_t y = 0; - const int32_t z = x + y; - - *values = static_cast(x); - ++values; - *values = static_cast(y); - *output = static_cast(z); - - if (++x > 1) - x = -1; - - if (++y > 1) - y = -1; -} - -template -static void generateFixedPointRecurrentValues(T* values, T* output) -{ - static int32_t x = -1; - static int32_t y = 0; - const int32_t z = x + y; - - // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and - // left-shifting a negative value is undefined behavior before C++20. - const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; - *values = static_cast(x * scale); - ++values; - *values = static_cast(y * scale); - *output = (z * scale); - - if (++x > 1) - x = -1; - - if (++y > 1) - y = -1; -} - -template -static void testFloatingPointNN( NeuralNetworkType& neuralNetwork, - void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), - char const* const path, - const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; - static const ValueType ERROR_LIMIT = 0.1; - ofstream results(path); - ofstream weightsOutputFile; - std::string weightsOutputPath(path); - std::deque errors; - ValueType error; - - ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; - double learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; - - weightsOutputPath.replace(weightsOutputPath.find("."), std::string::npos, "_weights.txt"); - - tinymind::NetworkPropertiesFileManager::writeHeader(results); - - for (int i = 0; i < numberOfTrainingIterations; ++i) - { - pValuesFn(values, output); - - neuralNetwork.feedForward(&values[0]); - error = neuralNetwork.calculateError(&output[0]); - if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - neuralNetwork.trainNetwork(&output[0]); - } - neuralNetwork.getLearnedValues(&learnedValues[0]); - - errors.push_front(error); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - -#if STOP_ON_AVG_ERROR - const double totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - if (averageError <= ERROR_LIMIT) - { - break; - } -#endif // STOP_ON_AVG_ERROR - } - - tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); - results << error << std::endl; - } - - weightsOutputFile.open(weightsOutputPath.c_str()); - - tinymind::NetworkPropertiesFileManager::storeNetworkWeights(neuralNetwork, weightsOutputFile); - - weightsOutputFile.close(); - - const double totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -template -static void testFloatingPointNN_Xor(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - testFloatingPointNN(neuralNetwork, generateXorValues, path, numberOfTrainingIterations); -} - -template -static void testFloatingPointNN_And(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - testFloatingPointNN(neuralNetwork, generateAndValues, path, numberOfTrainingIterations); -} - -template -static void testFloatingPointNN_Or(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - testFloatingPointNN(neuralNetwork, generateOrValues, path, numberOfTrainingIterations); -} - -template -static void testFixedPointNeuralNetwork( NeuralNetworkType& neuralNetwork, - void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), - char const* const path, - const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ -#if USE_WEIGHTS_INPUT_FILE == 1 - typedef tinymind::NetworkPropertiesFileManager NetworkPropertiesFileManagerType; -#endif // USE_WEIGHTS_INPUT_FILE - typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - typedef ValueHelper ValueHelperType; - static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); - ofstream results(path); - ofstream weightsOutputFile; - std::string initialWeightsInputPath(path); - std::string weightsOutputPath(path); - std::string binaryWeightsOutputPath(path); - ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - - initialWeightsInputPath.replace(weightsOutputPath.find("."), std::string::npos, "_initial_weights.txt"); - weightsOutputPath.replace(weightsOutputPath.find("."), std::string::npos, "_weights.txt"); - binaryWeightsOutputPath.replace(binaryWeightsOutputPath.find(".txt"), std::string::npos, ".bin"); - - tinymind::NetworkPropertiesFileManager::writeHeader(results); - -#if USE_WEIGHTS_INPUT_FILE - initialWeightsInputPath.insert(0, "../input/"); - ifstream weightsInputFile(initialWeightsInputPath); - if (weightsInputFile.is_open()) - { - NetworkPropertiesFileManagerType::template loadNetworkWeights(neuralNetwork, weightsInputFile); - } - else - { - cout << "Did not find initial weights input file: " << initialWeightsInputPath << endl; - } -#endif - - for (int i = 0; i < numberOfTrainingIterations; ++i) - { - pValuesFn(values, output); - - neuralNetwork.feedForward(&values[0]); - error = neuralNetwork.calculateError(&output[0]); - if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - neuralNetwork.trainNetwork(&output[0]); - } - neuralNetwork.getLearnedValues(&learnedValues[0]); - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - -#if STOP_ON_AVG_ERROR - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - if (averageError <= ERROR_LIMIT) - { - break; - } -#endif // STOP_ON_AVG_ERROR - } - - tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); - results << error << std::endl; - } - - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - - weightsOutputFile.open(weightsOutputPath.c_str()); - - tinymind::NetworkPropertiesFileManager::storeNetworkWeights(neuralNetwork, weightsOutputFile); - - weightsOutputFile.close(); - - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -template -static void testFixedPointNeuralNetwork_Xor(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointXorValues, path, numberOfTrainingIterations); -} - -template -static void testFixedPointNeuralNetwork_And(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointAndValues, path, numberOfTrainingIterations); -} - -template -static void testFixedPointNeuralNetwork_Or(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointOrValues, path, numberOfTrainingIterations); -} - -template -static void testFixedPointNeuralNetwork_Nor(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointNorValues, path, numberOfTrainingIterations); -} - -template -static void testFixedPointNeuralNetwork_No_Train( NeuralNetworkType& neuralNetwork, - void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), - char const* const path, - char const* const weightsInputPath) -{ - typedef tinymind::NetworkPropertiesFileManager NetworkPropertiesFileManagerType; - typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - typedef ValueHelper ValueHelperType; - static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); - ofstream results(path); - ifstream weightsInputFile(weightsInputPath); - ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - - NetworkPropertiesFileManagerType::template loadNetworkWeights(neuralNetwork, weightsInputFile); - tinymind::NetworkPropertiesFileManager::writeHeader(results); - - for (int i = 0; i < TRAINING_ITERATIONS; ++i) - { - pValuesFn(values, output); - - neuralNetwork.feedForward(&values[0]); - error = neuralNetwork.calculateError(&output[0]); - neuralNetwork.getLearnedValues(&learnedValues[0]); - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - -#if STOP_ON_AVG_ERROR - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - if (averageError <= ERROR_LIMIT) - { - break; - } -#endif // STOP_ON_AVG_ERROR - } - - tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); - results << error << std::endl; - } - - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -template -static void testFixedPointNeuralNetwork_Xor_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) -{ - testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointXorValues, path, weightsInputPath); -} - -template -static void testFixedPointNeuralNetwork_And_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) -{ - testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointAndValues, path, weightsInputPath); -} - -template -static void testFixedPointNeuralNetwork_Or_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) -{ - testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointOrValues, path, weightsInputPath); -} - -template -static void testFixedPointNeuralNetwork_Nor_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) -{ - testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointNorValues, path, weightsInputPath); -} - -template -static void testFixedPointNeuralNetwork_No_Train_Float_Weights( NeuralNetworkType& neuralNetwork, - void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), - char const* const path, - char const* const weightsInputPath) -{ - typedef tinymind::NetworkPropertiesFileManager NetworkPropertiesFileManagerType; - typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - typedef ValueHelper ValueHelperType; - static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); - ofstream results(path); - ifstream weightsInputFile(weightsInputPath); - ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - - NetworkPropertiesFileManagerType::template loadNetworkWeights(neuralNetwork, weightsInputFile); - tinymind::NetworkPropertiesFileManager::writeHeader(results); - - for (int i = 0; i < TRAINING_ITERATIONS; ++i) - { - pValuesFn(values, output); - - neuralNetwork.feedForward(&values[0]); - error = neuralNetwork.calculateError(&output[0]); - neuralNetwork.getLearnedValues(&learnedValues[0]); - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - -#if STOP_ON_AVG_ERROR - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - if (averageError <= ERROR_LIMIT) - { - break; - } -#endif // STOP_ON_AVG_ERROR - } - - tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); - results << error << std::endl; - } - - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -template -static void testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) -{ - testFixedPointNeuralNetwork_No_Train_Float_Weights(neuralNetwork, generateFixedPointXorValues, path, weightsInputPath); -} - -template -static void testFixedPointNeuralNetwork_And_No_Train_Float_Weights(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) -{ - testFixedPointNeuralNetwork_No_Train_Float_Weights(neuralNetwork, generateFixedPointAndValues, path, weightsInputPath); -} - -template -static void testFixedPointNeuralNetwork_Or_No_Train_Float_Weights(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) -{ - testFixedPointNeuralNetwork_No_Train_Float_Weights(neuralNetwork, generateFixedPointOrValues, path, weightsInputPath); -} - -template -static void testNeuralNetwork_Recurrent(NeuralNetworkType& neuralNetwork, char const* const path) -{ - typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - typedef ValueHelper ValueHelperType; - static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); - ofstream results(path); - ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - - tinymind::NetworkPropertiesFileManager::writeHeader(results); - - for (int i = 0; i < TRAINING_ITERATIONS; ++i) - { - generateFixedPointRecurrentValues(values, output); - - neuralNetwork.feedForward(&values[0]); - error = neuralNetwork.calculateError(&output[0]); - if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - neuralNetwork.trainNetwork(&output[0]); - } - neuralNetwork.getLearnedValues(&learnedValues[0]); - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - -#if STOP_ON_AVG_ERROR - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - if (averageError <= ERROR_LIMIT) - { - break; - } -#endif // STOP_ON_AVG_ERROR - } - - tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); - results << error << std::endl; - } - - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -template -static void testFloatingPointNeuralNetwork_Recurrent(NeuralNetworkType& neuralNetwork, char const* const path, - const int numberOfTrainingIterations = TRAINING_ITERATIONS) -{ - typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; - typedef double FullWidthValueType; - typedef ValueHelper ValueHelperType; - static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); - ofstream results(path); - ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - - tinymind::NetworkPropertiesFileManager::writeHeader(results); - - for (int i = 0; i < numberOfTrainingIterations; ++i) - { - generateRecurrentValues(values, output); - - neuralNetwork.feedForward(&values[0]); - error = neuralNetwork.calculateError(&output[0]); - if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - neuralNetwork.trainNetwork(&output[0]); - } - neuralNetwork.getLearnedValues(&learnedValues[0]); - - errors.push_front(error); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - -#if STOP_ON_AVG_ERROR - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - if (averageError <= ERROR_LIMIT) - { - break; - } -#endif // STOP_ON_AVG_ERROR - } - - tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); - results << error << std::endl; - } - - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -BOOST_AUTO_TEST_SUITE(test_suite_nn) - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_xavier_uniform) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - XavierUniformRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_xor_xavier_uniform.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_xavier_normal) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - XavierNormalRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_xor_xavier_normal.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_xavier_heterogeneous) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::HiddenLayers<6, 4> HiddenLayersDesc; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - XavierUniformHeterogeneousRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - HiddenLayersDesc, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointHeterogeneousNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_xor_xavier_heterogeneous.txt"; - FixedPointHeterogeneousNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_nn_copy) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_xor.txt"; - char const* const pathCopy = "output/nn_fixed_xor_copy.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - FixedPointMultiLayerPerceptronNetworkType nnCopy; - - testFixedPointNeuralNetwork_Xor(nn, path); - nnCopy.setWeights(nn); - testFixedPointNeuralNetwork_Xor(nnCopy, pathCopy); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_And(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_or.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Or(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_nor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_nor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Nor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - static const bool TRAINABLE = false; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_no_train_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor_No_Train(nn, path, "output/nn_fixed_xor_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - static const bool TRAINABLE = false; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_no_train_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_And_No_Train(nn, path, "output/nn_fixed_and_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - static const bool TRAINABLE = false; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_no_train_or.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Or_No_Train(nn, path, "output/nn_fixed_or_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_nor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - static const bool TRAINABLE = false; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_no_train_nor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Nor_No_Train(nn, path, "output/nn_fixed_nor_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_5_hidden_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_5_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_5_hidden_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_5_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_And(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_5_hidden_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_5_or.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Or(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_16_16_nn_5_hidden_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 16; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_16_16_5_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_16_16_nn_5_hidden_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 16; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_16_16_5_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_And(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_16_16_nn_5_hidden_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 16; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_16_16_5_or.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Or(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_8_24_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_8_24_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_And(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_8_24_or.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Or(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_2_8_24_nn_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = true; - static const size_t BATCH_SIZE = 2; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE, - BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_batch_2_8_24_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - const ValueType learningRate(nn.getLearningRate() / 4); - const ValueType accelerationRate(nn.getAccelerationRate() / 4); - const ValueType momentumRate(nn.getMomentumRate() / 4); - - nn.setLearningRate(learningRate); - nn.setAccelerationRate(accelerationRate); - nn.setMomentumRate(momentumRate); - - testFixedPointNeuralNetwork_Xor(nn, path, 10000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_2_8_24_nn_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = true; - static const size_t BATCH_SIZE = 2; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE, - BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_batch_2_8_24_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - const ValueType learningRate(nn.getLearningRate() / 4); - const ValueType accelerationRate(nn.getAccelerationRate() / 4); - const ValueType momentumRate(nn.getMomentumRate() / 4); - - nn.setLearningRate(learningRate); - nn.setAccelerationRate(accelerationRate); - nn.setMomentumRate(momentumRate); - - testFixedPointNeuralNetwork_And(nn, path, 10000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_2_8_24_nn_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = true; - static const size_t BATCH_SIZE = 2; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE, - BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_batch_2_8_24_or.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - const ValueType learningRate(nn.getLearningRate() / 4); - const ValueType accelerationRate(nn.getAccelerationRate() / 4); - const ValueType momentumRate(nn.getMomentumRate() / 4); - - nn.setLearningRate(learningRate); - nn.setAccelerationRate(accelerationRate); - nn.setMomentumRate(momentumRate); - - testFixedPointNeuralNetwork_Or(nn, path, 10000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_4_8_24_nn_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = true; - static const size_t BATCH_SIZE = 4; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE, - BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_batch_4_8_24_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - const ValueType learningRate(nn.getLearningRate() / 4); - const ValueType accelerationRate(nn.getAccelerationRate() / 4); - const ValueType momentumRate(nn.getMomentumRate() / 4); - - nn.setLearningRate(learningRate); - nn.setAccelerationRate(accelerationRate); - nn.setMomentumRate(momentumRate); - - testFixedPointNeuralNetwork_Xor(nn, path, 10000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_4_8_24_nn_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = true; - static const size_t BATCH_SIZE = 4; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE, - BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_batch_4_8_24_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - const ValueType learningRate(nn.getLearningRate() / 4); - const ValueType accelerationRate(nn.getAccelerationRate() / 4); - const ValueType momentumRate(nn.getMomentumRate() / 4); - - nn.setLearningRate(learningRate); - nn.setAccelerationRate(accelerationRate); - nn.setMomentumRate(momentumRate); - - testFixedPointNeuralNetwork_And(nn, path, 10000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_4_8_24_nn_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = true; - static const size_t BATCH_SIZE = 4; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE, - BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_batch_4_8_24_or.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - const ValueType learningRate(nn.getLearningRate() / 4); - const ValueType accelerationRate(nn.getAccelerationRate() / 4); - const ValueType momentumRate(nn.getMomentumRate() / 4); - - nn.setLearningRate(learningRate); - nn.setAccelerationRate(accelerationRate); - nn.setMomentumRate(momentumRate); - - testFixedPointNeuralNetwork_Or(nn, path, 10000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_elman_nn) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::ElmanNetwork< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointElmanNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_elman.txt"; - FixedPointElmanNetworkType nn; - - testNeuralNetwork_Recurrent(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_floatingpoint_elman_nn) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::ElmanNetwork< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointElmanNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_elman.txt"; - FloatingPointElmanNetworkType nn; - - testFloatingPointNeuralNetwork_Recurrent(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_xor.txt"; - FloatingPointMultiLayerPerceptronNetworkType nn; - - testFloatingPointNN_Xor(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_and.txt"; - FloatingPointMultiLayerPerceptronNetworkType nn; - - testFloatingPointNN_And(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_or) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_or.txt"; - FloatingPointMultiLayerPerceptronNetworkType nn; - - testFloatingPointNN_Or(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_float_weights_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = false; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_no_train_float_weights_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_xor_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_float_weights_and) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = false; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_no_train_float_weights_and.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_And_No_Train_Float_Weights(nn, path, "output/nn_float_and_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_relu_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_relu_xor.txt"; - FloatingPointMultiLayerPerceptronNetworkType nn; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - testFloatingPointNN_Xor(nn, path, 200000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_relu_xor_no_train) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = false; - static const size_t NUMBER_OF_FIXED_BITS = 16; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - tinymind::NullRandomNumberPolicy, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_relu_xor_no_train.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_relu_xor_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_relu_xor_no_train) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = false; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - tinymind::NullRandomNumberPolicy, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_8_24_relu_xor_no_train.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_relu_xor_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_8_nn_relu_xor_no_train) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const bool TRAINABLE = false; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - tinymind::NullRandomNumberPolicy, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_8_8_relu_xor_no_train.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_relu_xor_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_floatingpoint_2_hidden_nn_relu_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_2_hidden_relu_xor.txt"; - FloatingPointMultiLayerPerceptronNetworkType nn; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - testFloatingPointNN_Xor(nn, path, 100000); -} - -BOOST_AUTO_TEST_CASE(test_case_floatingpoint_2_hidden_nn_relu_xor_copy) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_2_hidden_relu_xor.txt"; - char const* const pathCopy = "output/nn_float_2_hidden_relu_xor_copy.txt"; - FloatingPointMultiLayerPerceptronNetworkType nn; - FloatingPointMultiLayerPerceptronNetworkType nnCopy; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - testFloatingPointNN_Xor(nn, path, 100000); - nnCopy.setWeights(nn); - testFloatingPointNN_Xor(nnCopy, pathCopy, 100000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_2_hidden_nn_relu_xor_no_train) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 16; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - tinymind::NullRandomNumberPolicy, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_2_hidden_relu_xor_no_train.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_2_hidden_relu_xor_weights.txt"); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_sigmoid_xor) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::SigmoidActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_sigmoid_xor.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path, 75000); -} - -BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_4_hidden_layers) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_HIDDEN_LAYERS = 4; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_xor_4_hidden_layers.txt"; - FixedPointMultiLayerPerceptronNetworkType nn; - - testFixedPointNeuralNetwork_Xor(nn, path, TRAINING_ITERATIONS * 100); -} - -BOOST_AUTO_TEST_SUITE_END() - -typedef tinymind::QValue<8, 8, true, tinymind::TruncatePolicy, tinymind::MinMaxSaturatePolicy> SignedQ8_8SatPolicyType; -typedef tinymind::QValue<8, 8, false, tinymind::TruncatePolicy, tinymind::MinMaxSaturatePolicy> UnsignedQ8_8SatPolicyType; - -BOOST_AUTO_TEST_SUITE(SoftMaxLinearClassificationTests) - -// Test case: 2x2 grayscale image with 2 classes -BOOST_AUTO_TEST_CASE(test_softmax_2x2_image_2_classes) -{ - // 2x2 grayscale image (4 input features) - const size_t inputSize = 4; // 2x2 pixels - const size_t numClasses = 2; // Binary classification - - // Example: grayscale values normalized to Q8.8 format - // Image pixels: [100, 150, 200, 50] -> normalized to [-1, 1] range - SignedQ8_8SatPolicyType inputImage[inputSize]; - inputImage[0] = SignedQ8_8SatPolicyType(-1, 0); // Dark pixel (-1.0) - inputImage[1] = SignedQ8_8SatPolicyType(0, 0); // Medium pixel (0.0) - inputImage[2] = SignedQ8_8SatPolicyType(1, 0); // Bright pixel (1.0) - inputImage[3] = SignedQ8_8SatPolicyType(-1, 128); // Dark-ish pixel (-0.5) - - // Linear layer weights: [numClasses x inputSize] - // Class 0 weights favor dark pixels, Class 1 favors bright pixels - SignedQ8_8SatPolicyType weights[numClasses][inputSize] = { - // Class 0: favors dark pixels - {SignedQ8_8SatPolicyType(1, 0), SignedQ8_8SatPolicyType(0, 128), - SignedQ8_8SatPolicyType(-1, 0), SignedQ8_8SatPolicyType(1, 128)}, - // Class 1: favors bright pixels - {SignedQ8_8SatPolicyType(-1, 0), SignedQ8_8SatPolicyType(0, 128), - SignedQ8_8SatPolicyType(2, 0), SignedQ8_8SatPolicyType(-1, 128)} - }; - - // Bias terms for each class - SignedQ8_8SatPolicyType bias[numClasses] = { - SignedQ8_8SatPolicyType(0, 128), // 0.5 bias for class 0 - SignedQ8_8SatPolicyType(-1, 128) // -0.5 bias for class 1 - }; - - // Linear transformation: logits = weights * input + bias - SignedQ8_8SatPolicyType logits[numClasses]; - for (size_t c = 0; c < numClasses; ++c) - { - logits[c] = bias[c]; - for (size_t i = 0; i < inputSize; ++i) - { - logits[c] += weights[c][i] * inputImage[i]; - } - } - - // Apply SoftMax - SignedQ8_8SatPolicyType probabilities[numClasses]; - tinymind::SoftmaxActivationPolicy::activationFunction( - logits, - probabilities, - numClasses - ); - - // Verify: Sum of probabilities should be close to 1.0 - SignedQ8_8SatPolicyType sum(0, 0); - for (size_t c = 0; c < numClasses; ++c) - { - sum += probabilities[c]; - } - - // Expected sum: 1.0 in Q8.8 = 0x0100 = 256 - SignedQ8_8SatPolicyType expectedSum(1, 0); - - // Allow tolerance due to Q-format rounding and exp approximation - // Tolerance: ±3 LSB = ±0.01171875 in Q8.8 - const int tolerance = 3; - int sumValue = sum.getValue(); - int expectedValue = expectedSum.getValue(); - - // Test to make sure sum of probabilities should be ~1.0 - BOOST_TEST(sumValue >= (expectedValue - tolerance)); - BOOST_TEST(sumValue <= (expectedValue + tolerance)); - - // Verify each probability is in [0, 1] range - for (size_t c = 0; c < numClasses; ++c) - { - BOOST_TEST(probabilities[c].getValue() >= 0); - BOOST_TEST(probabilities[c].getValue() <= SignedQ8_8SatPolicyType(1, 0).getValue()); - } -} - -// ========================================================================= -// Tests for the new NeuralNetwork class with heterogeneous hidden layers -// ========================================================================= - -BOOST_AUTO_TEST_CASE(test_case_new_nn_uniform_hidden_layers_xor) -{ - // NeuralNetwork with uniform hidden layers should work like MultilayerPerceptron - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - tinymind::HiddenLayers<5>, - 1, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - NNType nn; - ValueType error; - - nn.setLearningRate(0.1); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - ValueType values[2]; - ValueType output[1]; - - for (int i = 0; i < 10000; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - if (!TransferFunctionsType::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - } - - // Verify the network has learned XOR - std::deque errors; - - for (int i = 0; i < 100; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - errors.push_back(error); - } - - double totalError = 0.0; - for (size_t i = 0; i < errors.size(); ++i) - { - totalError += std::abs(errors[i]); - } - double averageError = totalError / static_cast(errors.size()); - BOOST_TEST(averageError <= 0.1); -} - -BOOST_AUTO_TEST_CASE(test_case_new_nn_2_uniform_hidden_layers_xor) -{ - // NeuralNetwork with 2 uniform hidden layers - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - tinymind::HiddenLayers<5, 5>, - 1, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - NNType nn; - ValueType error; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - ValueType values[2]; - ValueType output[1]; - - for (int i = 0; i < 100000; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - if (!TransferFunctionsType::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - } - - // Verify convergence - std::deque errors; - for (int i = 0; i < 100; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - errors.push_back(error); - } - - double totalError = 0.0; - for (size_t i = 0; i < errors.size(); ++i) - { - totalError += std::abs(errors[i]); - } - double averageError = totalError / static_cast(errors.size()); - BOOST_TEST(averageError <= 0.1); -} - -BOOST_AUTO_TEST_CASE(test_case_new_nn_heterogeneous_hidden_layers_xor) -{ - // NeuralNetwork with heterogeneous hidden layers: 8 neurons, then 4 neurons - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - tinymind::HiddenLayers<8, 4>, - 1, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - NNType nn; - ValueType error; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - ValueType values[2]; - ValueType output[1]; - - for (int i = 0; i < 100000; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - if (!TransferFunctionsType::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - } - - // Verify convergence - std::deque errors; - for (int i = 0; i < 100; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - errors.push_back(error); - } - - double totalError = 0.0; - for (size_t i = 0; i < errors.size(); ++i) - { - totalError += std::abs(errors[i]); - } - double averageError = totalError / static_cast(errors.size()); - BOOST_TEST(averageError <= 0.1); -} - -BOOST_AUTO_TEST_CASE(test_case_new_nn_3_heterogeneous_hidden_layers_xor) -{ - // NeuralNetwork with 3 heterogeneous hidden layers: 10, 5, 3 - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - tinymind::HiddenLayers<10, 5, 3>, - 1, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - NNType nn; - ValueType error; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - ValueType values[2]; - ValueType output[1]; - - for (int i = 0; i < 100000; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - if (!TransferFunctionsType::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - } - - // Verify convergence - std::deque errors; - for (int i = 0; i < 100; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - errors.push_back(error); - } - - double totalError = 0.0; - for (size_t i = 0; i < errors.size(); ++i) - { - totalError += std::abs(errors[i]); - } - double averageError = totalError / static_cast(errors.size()); - BOOST_TEST(averageError <= 0.1); -} - -BOOST_AUTO_TEST_CASE(test_case_new_nn_uniform_alias_xor) -{ - // Verify UniformHiddenLayersAlias works correctly - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - typename tinymind::UniformHiddenLayersAlias<2, 5>::type, - 1, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - NNType nn; - ValueType error; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - ValueType values[2]; - ValueType output[1]; - - for (int i = 0; i < 100000; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - if (!TransferFunctionsType::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - } - - // Verify convergence - std::deque errors; - for (int i = 0; i < 100; ++i) - { - generateXorValues(values, output); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - errors.push_back(error); - } - - double totalError = 0.0; - for (size_t i = 0; i < errors.size(); ++i) - { - totalError += std::abs(errors[i]); - } - double averageError = totalError / static_cast(errors.size()); - BOOST_TEST(averageError <= 0.1); -} - -BOOST_AUTO_TEST_CASE(test_case_new_nn_not_trainable) -{ - // Verify non-trainable NeuralNetwork compiles and runs feedForward - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - tinymind::HiddenLayers<4, 3>, - 1, - TransferFunctionsType, - false> NNType; - NNType nn; - - ValueType values[2] = {1.0, 0.0}; - nn.feedForward(&values[0]); - - // Just verify it runs without crashing - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_new_nn_heterogeneous_2_hidden_layers_weight_copy) -{ - // NeuralNetwork with heterogeneous hidden layers (8, 4): train, copy weights, verify copy - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - tinymind::HiddenLayers<8, 4>, - 1, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_hetero_2_hidden_xor.txt"; - char const* const pathCopy = "output/nn_float_hetero_2_hidden_xor_copy.txt"; - NNType nn; - NNType nnCopy; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - testFloatingPointNN_Xor(nn, path, 100000); - nnCopy.setWeights(nn); - testFloatingPointNN_Xor(nnCopy, pathCopy, 100000); -} - -BOOST_AUTO_TEST_CASE(test_case_new_nn_heterogeneous_3_hidden_layers_weight_copy) -{ - // NeuralNetwork with 3 heterogeneous hidden layers (10, 5, 3): train, copy weights, verify copy - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::ReluActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::NeuralNetwork< ValueType, - 2, - tinymind::HiddenLayers<10, 5, 3>, - 1, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_hetero_3_hidden_xor.txt"; - char const* const pathCopy = "output/nn_float_hetero_3_hidden_xor_copy.txt"; - NNType nn; - NNType nnCopy; - - nn.setLearningRate(0.005); - nn.setAccelerationRate(0.03); - nn.setMomentumRate(0.16); - - testFloatingPointNN_Xor(nn, path, 100000); - nnCopy.setWeights(nn); - testFloatingPointNN_Xor(nnCopy, pathCopy, 100000); -} - -// ========================================================================= -// Tests for RecurrentNeuralNetwork and ElmanNeuralNetwork -// ========================================================================= - -BOOST_AUTO_TEST_CASE(test_case_elman_neural_network_floating_point) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::ElmanNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointElmanNeuralNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_elman_neural_network.txt"; - FloatingPointElmanNeuralNetworkType nn; - - testFloatingPointNeuralNetwork_Recurrent(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_elman_neural_network_fixed_point) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t NUMBER_OF_FIXED_BITS = 8; - static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; - typedef tinymind::QValue ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::ElmanNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointElmanNeuralNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_fixed_elman_neural_network.txt"; - FixedPointElmanNeuralNetworkType nn; - - testNeuralNetwork_Recurrent(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_recurrent_neural_network_floating_point) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::RecurrentNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers<3>, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> RecurrentNNType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_recurrent_neural_network.txt"; - RecurrentNNType nn; - - testFloatingPointNeuralNetwork_Recurrent(nn, path); -} - -BOOST_AUTO_TEST_CASE(test_case_recurrent_neural_network_heterogeneous_layers) -{ - // RecurrentNeuralNetwork with heterogeneous hidden layers - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::RecurrentNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers<4, 3>, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> RecurrentNNType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_recurrent_nn_hetero.txt"; - RecurrentNNType nn; - - testFloatingPointNeuralNetwork_Recurrent(nn, path); -} - -// ========================================================================= -// Tests for LstmNeuralNetwork -// ========================================================================= - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_floating_point) -{ - // LSTM networks require more training iterations than simple RNNs due to - // the additional gate parameters (input, forget, output gates) that must - // all converge together. - static const int LSTM_TRAINING_ITERATIONS = 5000; - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointLstmNeuralNetworkType; - srand(RANDOM_SEED); - char const* const path = "output/nn_float_lstm_neural_network.txt"; - FloatingPointLstmNeuralNetworkType nn; - - testFloatingPointNeuralNetwork_Recurrent(nn, path, LSTM_TRAINING_ITERATIONS); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_fixed_point) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointLstmNeuralNetworkType; - srand(RANDOM_SEED); - FixedPointLstmNeuralNetworkType nn; - - ValueType values[FixedPointLstmNeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[FixedPointLstmNeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[FixedPointLstmNeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType error; - ValueType firstError; - bool firstErrorCaptured = false; - - for (int i = 0; i < TRAINING_ITERATIONS; ++i) - { - generateFixedPointRecurrentValues(values, output); - - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - - if (!firstErrorCaptured) - { - firstError = error; - firstErrorCaptured = true; - } - - if (!FixedPointLstmNeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - nn.getLearnedValues(&learnedValues[0]); - } - - // Verify feedforward produces valid output - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - BOOST_TEST(learnedValues[0].getValue() != 0); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_multi_layer) -{ - // Test multi-layer LSTM: two LSTM hidden layers (8 and 4 neurons). - // Both inner and last hidden layers should be LSTM layers with their - // own recurrent connections and gate weights. - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers<8, 4>, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> MultiLayerLstmType; - srand(RANDOM_SEED); - MultiLayerLstmType nn; - - ValueType values[MultiLayerLstmType::NumberOfInputLayerNeurons]; - ValueType output[MultiLayerLstmType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[MultiLayerLstmType::NumberOfOutputLayerNeurons]; - - // Verify feedforward works with multi-layer LSTM - values[0] = 1.0; - values[1] = 0.0; - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - BOOST_TEST(!std::isnan(learnedValues[0])); - BOOST_TEST(!std::isinf(learnedValues[0])); - - // Verify output changes with different input - (void)learnedValues[0]; // first output verified above - values[0] = 0.0; - values[1] = 1.0; - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - BOOST_TEST(!std::isnan(learnedValues[0])); - BOOST_TEST(!std::isinf(learnedValues[0])); - - // Verify successive calls produce different output (LSTM state accumulates) - const double secondOutput = learnedValues[0]; - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - BOOST_TEST(learnedValues[0] != secondOutput); - - // Verify training doesn't crash - output[0] = 1.0; - nn.feedForward(&values[0]); - const ValueType error = nn.calculateError(&output[0]); - nn.trainNetwork(&output[0]); - BOOST_TEST(!std::isnan(error)); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_feedforward_produces_output) -{ - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> LstmNNType; - srand(RANDOM_SEED); - LstmNNType nn; - - ValueType values[LstmNNType::NumberOfInputLayerNeurons]; - ValueType output[LstmNNType::NumberOfOutputLayerNeurons]; - - values[0] = 1.0; - values[1] = 0.0; - - nn.feedForward(&values[0]); - nn.getLearnedValues(&output[0]); - - // After feedForward with initialized weights, the output should be a valid number - BOOST_TEST(!std::isnan(output[0])); - BOOST_TEST(!std::isinf(output[0])); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_sequential_inputs) -{ - // Test that LSTM maintains state across sequential feedForward calls - static const size_t NUMBER_OF_INPUTS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> LstmNNType; - srand(RANDOM_SEED); - LstmNNType nn; - - ValueType values[LstmNNType::NumberOfInputLayerNeurons]; - ValueType output1[LstmNNType::NumberOfOutputLayerNeurons]; - ValueType output2[LstmNNType::NumberOfOutputLayerNeurons]; - - // First feedForward - values[0] = 1.0; - nn.feedForward(&values[0]); - nn.getLearnedValues(&output1[0]); - - // Second feedForward with same input - output should differ due to recurrent state - nn.feedForward(&values[0]); - nn.getLearnedValues(&output2[0]); - - BOOST_TEST(!std::isnan(output1[0])); - BOOST_TEST(!std::isnan(output2[0])); - // Due to LSTM cell state, the same input should produce different outputs - BOOST_TEST(output1[0] != output2[0]); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_char_sequence_prediction) -{ - // Train a floating-point LSTM to predict the next value in a repeating - // sequence of 3 values: 0.2, 0.5, 0.8, 0.2, 0.5, 0.8, ... - // Input is (previous, current), target is the next value. - // The 3 training pairs are: - // (0.8, 0.2) -> 0.5 - // (0.2, 0.5) -> 0.8 - // (0.5, 0.8) -> 0.2 - // After training, verify the average error over the last samples is - // below a threshold, confirming the network has learned the pattern. - static const size_t SEQUENCE_LENGTH = 3; - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 8; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> LstmSeqNNType; - srand(RANDOM_SEED); - LstmSeqNNType nn; - - // Repeating sequence values - const ValueType sequence[SEQUENCE_LENGTH] = {0.2, 0.5, 0.8}; - - ValueType values[LstmSeqNNType::NumberOfInputLayerNeurons]; - ValueType target[LstmSeqNNType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[LstmSeqNNType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - static const int SEQ_TRAINING_ITERATIONS = 20000; - size_t seqIndex = 0; - - for (int i = 0; i < SEQ_TRAINING_ITERATIONS; ++i) - { - const size_t prevIndex = (seqIndex + SEQUENCE_LENGTH - 1) % SEQUENCE_LENGTH; - const size_t nextIndex = (seqIndex + 1) % SEQUENCE_LENGTH; - - values[0] = sequence[prevIndex]; - values[1] = sequence[seqIndex]; - target[0] = sequence[nextIndex]; - - nn.feedForward(&values[0]); - error = nn.calculateError(&target[0]); - - if (!LstmSeqNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&target[0]); - } - nn.getLearnedValues(&learnedValues[0]); - - errors.push_front(error); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - - seqIndex = nextIndex; - } - - // Verify: the average error over the last NUM_SAMPLES_AVG_ERROR samples - // should be below the error limit, confirming the LSTM learned the - // repeating sequence pattern. - const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); - const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - static const double SEQ_ERROR_LIMIT = 0.15; - - BOOST_TEST(averageError <= SEQ_ERROR_LIMIT); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_fixed_point_sequence_prediction) -{ - // Train a Q16.16 fixed-point LSTM to predict the next value in a - // repeating 2-value sequence: 0.3, 0.7, 0.3, 0.7, ... - // Input is (previous, current) and the target is the next value. - // The 2 training pairs are: - // (0.7, 0.3) -> 0.7 - // (0.3, 0.7) -> 0.3 - // After training, verify the average error over the last samples is - // below a threshold, confirming the network learned to predict the - // next number in the repeating sequence. - static const size_t SEQUENCE_LENGTH = 2; - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointLstmSeqNNType; - srand(RANDOM_SEED); - FixedPointLstmSeqNNType nn; - - // Repeating sequence: 0.3, 0.7 in Q16.16 fixed-point - typedef tinymind::ValueConverter ConverterType; - const ValueType sequence[SEQUENCE_LENGTH] = { - ConverterType::convertToDestinationType(0.3), - ConverterType::convertToDestinationType(0.7) - }; - ValueType values[FixedPointLstmSeqNNType::NumberOfInputLayerNeurons]; - ValueType target[FixedPointLstmSeqNNType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[FixedPointLstmSeqNNType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - static const int FP_SEQ_TRAINING_ITERATIONS = 20000; - size_t seqIndex = 0; - - for (int i = 0; i < FP_SEQ_TRAINING_ITERATIONS; ++i) - { - const size_t prevIndex = (seqIndex + SEQUENCE_LENGTH - 1) % SEQUENCE_LENGTH; - const size_t nextIndex = (seqIndex + 1) % SEQUENCE_LENGTH; - - values[0] = sequence[prevIndex]; - values[1] = sequence[seqIndex]; - target[0] = sequence[nextIndex]; - - nn.feedForward(&values[0]); - error = nn.calculateError(&target[0]); - - if (!FixedPointLstmSeqNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&target[0]); - } - nn.getLearnedValues(&learnedValues[0]); - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - - seqIndex = nextIndex; - } - - // Verify: the average error over the last NUM_SAMPLES_AVG_ERROR samples - // should be below the error limit, confirming the LSTM learned to - // predict the next number in the repeating sequence. - static const FullWidthValueType FP_SEQ_ERROR_LIMIT = (1 << (ValueType::NumberOfFractionalBits - 4)); - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - - BOOST_TEST(averageError <= FP_SEQ_ERROR_LIMIT); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_float_sinusoid_prediction) -{ - // Train a floating-point LSTM on a sampled sinusoid using sequential - // input processing: one value per timestep, with LSTM hidden state - // carrying temporal context between steps. - // - // Training: feed sin[i], target sin[i+1], for each step in the sequence. - // LSTM state is reset at the start of each epoch so the network learns - // the dynamics from a clean state each time. - // - // Prediction: prime the LSTM with the training sequence, then - // auto-regressively predict PREDICTION_LENGTH values. - static const size_t NUMBER_OF_INPUTS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 16; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t SEQUENCE_LENGTH = 10; - static const size_t PREDICTION_LENGTH = 20; - static const int SIN_TRAINING_ITERATIONS = 100000; - static const double SIN_TRAINING_ERROR_LIMIT = 0.15; - static const double SIN_PREDICTION_TOLERANCE = 0.90; - - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> LstmSinNNType; - srand(RANDOM_SEED); - UniformRealRandomNumberGenerator::seed(RANDOM_SEED); - LstmSinNNType nn; - - // Generate sinusoid samples scaled to [0, 1] - // sin(x) ranges [-1, 1], so (sin(x) + 1) / 2 maps to [0, 1] - const double step = 2.0 * M_PI / static_cast(SEQUENCE_LENGTH); - double sinSamples[SEQUENCE_LENGTH + PREDICTION_LENGTH]; - for (size_t i = 0; i < SEQUENCE_LENGTH + PREDICTION_LENGTH; ++i) - { - sinSamples[i] = (sin(static_cast(i) * step) + 1.0) / 2.0; - } - - ValueType values[LstmSinNNType::NumberOfInputLayerNeurons]; - ValueType target[LstmSinNNType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[LstmSinNNType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - - // Train: feed one value per timestep, LSTM state carries context. - // Reset state at the start of each epoch for clean temporal learning. - for (int epoch = 0; epoch < SIN_TRAINING_ITERATIONS; ++epoch) - { - nn.resetState(); - - for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) - { - values[0] = sinSamples[i]; - target[0] = sinSamples[i + 1]; - - nn.feedForward(&values[0]); - error = nn.calculateError(&target[0]); - - if (!LstmSinNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&target[0]); - } - - errors.push_front(error); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - } - } - - // Verify training converged - const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); - const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - BOOST_TEST(averageError <= SIN_TRAINING_ERROR_LIMIT); - - // Prime the LSTM by feeding the training sequence (no training) - nn.resetState(); - for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) - { - values[0] = sinSamples[i]; - nn.feedForward(&values[0]); - } - - // Auto-regressively predict: feed last known value, get prediction, - // feed prediction as next input - double curr = sinSamples[SEQUENCE_LENGTH - 1]; - - ofstream predictionOutput("output/nn_float_lstm_sinusoid_prediction.txt"); - predictionOutput << "Step,Actual,Predicted\n"; - for (size_t i = 0; i < SEQUENCE_LENGTH; ++i) - { - predictionOutput << i << "," << sinSamples[i] << ",\n"; - } - - for (size_t p = 0; p < PREDICTION_LENGTH; ++p) - { - values[0] = curr; - - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - - const double predicted = learnedValues[0]; - const double expected = sinSamples[SEQUENCE_LENGTH + p]; - const double predictionError = fabs(predicted - expected); - - predictionOutput << (SEQUENCE_LENGTH + p) << "," << expected << "," << predicted << "\n"; - - BOOST_TEST(predictionError <= SIN_PREDICTION_TOLERANCE); - - // Feed prediction as next input - curr = predicted; - } -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_fixed_point_sinusoid_prediction) -{ - // Train a Q16.16 fixed-point LSTM on a sampled sinusoid using sequential - // input processing: one value per timestep with LSTM state carrying - // temporal context. State is reset at the start of each epoch. - static const size_t NUMBER_OF_INPUTS = 1; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 16; - static const size_t NUMBER_OF_OUTPUTS = 1; - static const size_t SEQUENCE_LENGTH = 10; - static const size_t PREDICTION_LENGTH = 20; - static const int FP_SIN_TRAINING_ITERATIONS = 50000; - - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - typedef tinymind::ValueConverter ConverterType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointLstmSinNNType; - srand(RANDOM_SEED); - UniformRealRandomNumberGenerator::seed(RANDOM_SEED); - FixedPointLstmSinNNType nn; - - // Generate sinusoid samples scaled to [0, 1] and convert to fixed-point - const double step = 2.0 * M_PI / static_cast(SEQUENCE_LENGTH); - ValueType sinSamples[SEQUENCE_LENGTH + PREDICTION_LENGTH]; - double sinSamplesDouble[SEQUENCE_LENGTH + PREDICTION_LENGTH]; - for (size_t i = 0; i < SEQUENCE_LENGTH + PREDICTION_LENGTH; ++i) - { - sinSamplesDouble[i] = (sin(static_cast(i) * step) + 1.0) / 2.0; - sinSamples[i] = ConverterType::convertToDestinationType(sinSamplesDouble[i]); - } - - ValueType values[FixedPointLstmSinNNType::NumberOfInputLayerNeurons]; - ValueType target[FixedPointLstmSinNNType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[FixedPointLstmSinNNType::NumberOfOutputLayerNeurons]; - std::deque errors; - ValueType error; - - // Train: feed one value per timestep, reset state each epoch - for (int epoch = 0; epoch < FP_SIN_TRAINING_ITERATIONS; ++epoch) - { - nn.resetState(); - - for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) - { - values[0] = sinSamples[i]; - target[0] = sinSamples[i + 1]; - - nn.feedForward(&values[0]); - error = nn.calculateError(&target[0]); - - if (!FixedPointLstmSinNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&target[0]); - } - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - } - } - - // Verify training converged - // Error limit: 1/4 of full fractional range (generous for fixed-point sinusoid) - static const FullWidthValueType FP_SIN_ERROR_LIMIT = (1 << (ValueType::NumberOfFractionalBits - 2)); - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - BOOST_TEST(averageError <= FP_SIN_ERROR_LIMIT); - - // Prime the LSTM with the training sequence - nn.resetState(); - for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) - { - values[0] = sinSamples[i]; - nn.feedForward(&values[0]); - } - - // Auto-regressively predict - ValueType curr = sinSamples[SEQUENCE_LENGTH - 1]; - // Prediction tolerance in fixed-point: ~0.90 = 0.90 * 65536 ≈ 58982 - static const FullWidthValueType FP_SIN_PREDICTION_TOLERANCE = static_cast(0.90 * (1 << ValueType::NumberOfFractionalBits)); - - ofstream predictionOutput("output/nn_fixed_lstm_sinusoid_prediction.txt"); - predictionOutput << "Step,Actual,Predicted\n"; - for (size_t i = 0; i < SEQUENCE_LENGTH; ++i) - { - predictionOutput << i << "," << sinSamplesDouble[i] << ",\n"; - } - - for (size_t p = 0; p < PREDICTION_LENGTH; ++p) - { - values[0] = curr; - - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - - const FullWidthValueType predicted = learnedValues[0].getValue(); - const FullWidthValueType expected = sinSamples[SEQUENCE_LENGTH + p].getValue(); - const FullWidthValueType predictionError = (predicted > expected) ? (predicted - expected) : (expected - predicted); - - predictionOutput << (SEQUENCE_LENGTH + p) << "," << sinSamplesDouble[SEQUENCE_LENGTH + p] << "," << learnedValues[0] << "\n"; - - BOOST_TEST(predictionError <= FP_SIN_PREDICTION_TOLERANCE); - - // Feed prediction as next input - curr = learnedValues[0]; - } -} - -BOOST_AUTO_TEST_SUITE_END() - -// ========================================================================= -// Test suite for Tier 1 features: gradient clipping, weight decay, -// learning rate scheduling, and GRU networks. -// ========================================================================= -BOOST_AUTO_TEST_SUITE(test_suite_tier1_features) - -BOOST_AUTO_TEST_CASE(test_case_gradient_clipping_policy_fixed_point) -{ - // Verify GradientClipByValue clips Q8.8 gradients correctly - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::GradientClipByValue ClipPolicy; - - // Value within range [-1, 1] should pass through - ValueType small(0, 128); // 0.5 - ValueType clipped = ClipPolicy::clip(small); - BOOST_TEST(clipped.getValue() == small.getValue()); - - // Null policy should pass through unchanged - ValueType large(2, 0); // 2.0 - ValueType unchanged = tinymind::NullGradientClippingPolicy::clip(large); - BOOST_TEST(unchanged.getValue() == large.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_weight_decay_policy_fixed_point) -{ - // Verify L2WeightDecay reduces weight magnitude for Q16.16 - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - - ValueType weight(2, 0); // 2.0 - ValueType lr(0, (1 << 14)); // ~0.25 - // Use lambda = (0, 256) = 256/65536 ≈ 0.004 for Q16.16 - ValueType result = tinymind::L2WeightDecay::applyDecay(weight, lr); - - // Weight should be reduced (decay pulls toward zero) - BOOST_TEST(result.getValue() < weight.getValue()); - - // Null policy should leave weight unchanged - ValueType unchanged = tinymind::NullWeightDecayPolicy::applyDecay(weight, lr); - BOOST_TEST(unchanged.getValue() == weight.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_learning_rate_schedule_fixed_point) -{ - // Verify FixedLearningRatePolicy never changes the rate - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::FixedLearningRatePolicy schedule; - - ValueType rate(0, (1 << 6)); // ~0.25 - schedule.initialize(rate); - - for (int i = 0; i < 100; ++i) - { - rate = schedule.step(rate); - } - BOOST_TEST(rate.getValue() == ValueType(0, (1 << 6)).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_fixed_point) -{ - // Verify StepDecaySchedule reduces rate after interval for Q8.8 - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - // Decay factor: 0, 230 => 230/256 ≈ 0.898, interval 3 steps - tinymind::StepDecaySchedule schedule; - - ValueType rate(0, 128); // 0.5 - schedule.initialize(rate); - - // Steps 1-2: unchanged - ValueType r1 = schedule.step(rate); - BOOST_TEST(r1.getValue() == rate.getValue()); - ValueType r2 = schedule.step(rate); - BOOST_TEST(r2.getValue() == rate.getValue()); - - // Step 3: decays - ValueType r3 = schedule.step(rate); - BOOST_TEST(r3.getValue() < rate.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_multi_cycle) -{ - // Realistic usage: rate = schedule.step(rate). Across multiple intervals - // the rate must strictly decrease each cycle and stay flat in between. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::StepDecaySchedule schedule; - - ValueType rate(0, 128); // 0.5 - schedule.initialize(rate); - - ValueType prev = rate; - for (int cycle = 0; cycle < 3; ++cycle) - { - // Two flat steps: rate unchanged. - rate = schedule.step(rate); - BOOST_TEST(rate.getValue() == prev.getValue()); - rate = schedule.step(rate); - BOOST_TEST(rate.getValue() == prev.getValue()); - // Third step decays. - rate = schedule.step(rate); - BOOST_TEST(rate.getValue() < prev.getValue()); - prev = rate; - } -} - -BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_initialize_resets_counter) -{ - // initialize() must reset the internal step counter; decay should fire - // exactly StepInterval steps after the reset, not sooner. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::StepDecaySchedule schedule; - - ValueType rate(0, 128); // 0.5 - - // Burn 2 steps worth of counter so the next step would have decayed. - schedule.initialize(rate); - (void)schedule.step(rate); - (void)schedule.step(rate); - - // Reset; next two steps must NOT decay even though only one more would have triggered before. - schedule.initialize(rate); - BOOST_TEST(schedule.step(rate).getValue() == rate.getValue()); - BOOST_TEST(schedule.step(rate).getValue() == rate.getValue()); - // Third step after reset triggers decay. - BOOST_TEST(schedule.step(rate).getValue() < rate.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_interval_one) -{ - // StepInterval = 1 means decay every single step (no flat plateau). - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::StepDecaySchedule schedule; - - ValueType rate(0, 128); // 0.5 - schedule.initialize(rate); - - ValueType prev = rate; - for (int i = 0; i < 5; ++i) - { - ValueType next = schedule.step(prev); - BOOST_TEST(next.getValue() < prev.getValue()); - prev = next; - } -} - -BOOST_AUTO_TEST_CASE(test_case_fixed_point_gradient_clipping_xor) -{ - // Verify fixed-point XOR training with gradient clipping enabled via FixedPointTransferFunctions - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - NUMBER_OF_OUTPUTS, - tinymind::DefaultNetworkInitializer, - tinymind::MeanSquaredErrorCalculator, - tinymind::ZeroToleranceCalculator, - tinymind::GradientClipByValue> TransferFunctionsType; - - typedef tinymind::MultilayerPerceptron< ValueType, - NUMBER_OF_INPUTS, - 1, 3, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> NNType; - srand(RANDOM_SEED); - NNType nn; - - ValueType values[2]; - ValueType output[1]; - ValueType error; - - for (int i = 0; i < TRAINING_ITERATIONS; ++i) - { - generateFixedPointXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - nn.trainNetwork(&output[0]); - } - - // Network should have learned XOR to some degree - BOOST_TEST(error.getValue() < ValueHelper::getErrorLimit()); -} - -BOOST_AUTO_TEST_CASE(test_case_gru_neural_network_floating_point) -{ - // Verify GRU network can be instantiated, trained, and produce valid output - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::GruNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FloatingPointGruNeuralNetworkType; - srand(RANDOM_SEED); - FloatingPointGruNeuralNetworkType nn; - - ValueType values[2]; - ValueType output[1]; - ValueType learnedValues[1]; - ValueType error; - - for (int i = 0; i < TRAINING_ITERATIONS; ++i) - { - generateXorValues(&values[0], &output[0]); - - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - - nn.trainNetwork(&output[0]); - nn.getLearnedValues(&learnedValues[0]); - } - - // Verify feedforward produces valid, finite output - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - BOOST_TEST(!std::isnan(learnedValues[0])); - BOOST_TEST(!std::isinf(learnedValues[0])); - BOOST_TEST(!std::isnan(error)); - BOOST_TEST(!std::isinf(error)); -} - -BOOST_AUTO_TEST_CASE(test_case_gru_neural_network_fixed_point) -{ - // Verify GRU network works with fixed-point arithmetic - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::GruNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> FixedPointGruNeuralNetworkType; - srand(RANDOM_SEED); - FixedPointGruNeuralNetworkType nn; - - ValueType values[FixedPointGruNeuralNetworkType::NumberOfInputLayerNeurons]; - ValueType output[FixedPointGruNeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType learnedValues[FixedPointGruNeuralNetworkType::NumberOfOutputLayerNeurons]; - ValueType error; - ValueType firstError; - bool firstErrorCaptured = false; - - for (int i = 0; i < TRAINING_ITERATIONS; ++i) - { - generateFixedPointRecurrentValues(values, output); - - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - - if (!firstErrorCaptured) - { - firstError = error; - firstErrorCaptured = true; - } - - if (!FixedPointGruNeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - nn.getLearnedValues(&learnedValues[0]); - } - - // Verify feedforward produces valid output - nn.feedForward(&values[0]); - nn.getLearnedValues(&learnedValues[0]); - BOOST_TEST(learnedValues[0].getValue() != 0); -} - -BOOST_AUTO_TEST_CASE(test_case_gru_reset_state) -{ - // Verify GRU resetState clears accumulated state so a second - // reset+feedforward pair produces the same output as the first. - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::GruNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers<4>, - NUMBER_OF_OUTPUTS, - TransferFunctionsType> GruType; - - srand(RANDOM_SEED); - GruType nn; - - ValueType values[2] = {0.5, 0.3}; - ValueType output1[1]; - ValueType output2[1]; - - // Reset, then feedforward to get baseline - nn.resetState(); - nn.feedForward(&values[0]); - nn.getLearnedValues(&output1[0]); - - // Feed forward multiple times to accumulate state - nn.feedForward(&values[0]); - nn.feedForward(&values[0]); - nn.feedForward(&values[0]); - - // Reset and feedforward again — should match baseline - nn.resetState(); - nn.feedForward(&values[0]); - nn.getLearnedValues(&output2[0]); - - BOOST_TEST(fabs(output1[0] - output2[0]) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_gru_non_trainable) -{ - // Verify GRU network can be instantiated as non-trainable (inference only) - static const size_t NUMBER_OF_INPUTS = 2; - static const size_t NUMBER_OF_OUTPUTS = 1; - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::GruNeuralNetwork< ValueType, - NUMBER_OF_INPUTS, - tinymind::HiddenLayers<4>, - NUMBER_OF_OUTPUTS, - TransferFunctionsType, - false> NonTrainableGruType; - - NonTrainableGruType nn; - - ValueType values[2] = {0.5, 0.3}; - ValueType output[1]; - - // Should be able to feed forward without crash - nn.feedForward(&values[0]); - nn.getLearnedValues(&output[0]); - - // Output should be finite - BOOST_TEST(!std::isnan(output[0])); - BOOST_TEST(!std::isinf(output[0])); -} - -BOOST_AUTO_TEST_CASE(test_case_early_stopping) -{ - // Verify EarlyStopping correctly detects convergence - tinymind::EarlyStopping stopper; - - // Decreasing error: should not stop - BOOST_TEST(!stopper.shouldStop(1.0)); - BOOST_TEST(!stopper.shouldStop(0.5)); - BOOST_TEST(!stopper.shouldStop(0.3)); - - // Stagnant error: should stop after patience exhausted - BOOST_TEST(!stopper.shouldStop(0.4)); - BOOST_TEST(!stopper.shouldStop(0.4)); - BOOST_TEST(!stopper.shouldStop(0.4)); - BOOST_TEST(!stopper.shouldStop(0.4)); - BOOST_TEST(stopper.shouldStop(0.4)); // 5th non-improving step - - // Best error should be the minimum seen - BOOST_TEST(fabs(stopper.getBestError() - 0.3) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_early_stopping_reset) -{ - // Verify EarlyStopping reset works - tinymind::EarlyStopping stopper; - - stopper.shouldStop(1.0); - stopper.shouldStop(2.0); - stopper.shouldStop(2.0); - stopper.shouldStop(2.0); - BOOST_TEST(stopper.shouldStop(2.0)); // should stop - - stopper.reset(); - BOOST_TEST(!stopper.shouldStop(5.0)); // should NOT stop after reset -} - -BOOST_AUTO_TEST_CASE(test_case_early_stopping_with_training) -{ - // Verify early stopping works in a training loop - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TransferFunctionsType; - typedef tinymind::MultilayerPerceptron NNType; - srand(RANDOM_SEED); - NNType nn; - - tinymind::EarlyStopping stopper; - ValueType values[2], output[1], error; - int iterationsUsed = 0; - - for (int i = 0; i < 10000; ++i) - { - generateXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - if (stopper.shouldStop(error)) break; - nn.trainNetwork(&output[0]); - ++iterationsUsed; - } - - // Should have stopped before 10000 iterations - BOOST_TEST(iterationsUsed < 10000); - BOOST_TEST(!std::isnan(error)); -} - -struct AdamTransferFunctions -{ - typedef double TransferFunctionsValueType; - typedef UniformRealRandomNumberGenerator RandomNumberGeneratorPolicy; - typedef tinymind::TanhActivationPolicy HiddenNeuronActivationPolicy; - typedef tinymind::TanhActivationPolicy OutputNeuronActivationPolicy; - typedef tinymind::ZeroToleranceCalculator ZeroToleranceCalculatorPolicy; - typedef tinymind::AdamOptimizerFloat OptimizerPolicyType; - - static const unsigned NumberOfTransferFunctionsOutputNeurons = 1; - - static double calculateError(double const* const targetValues, double const* const outputValues) - { - const double delta = (targetValues[0] - outputValues[0]); - return (delta * delta); - } - - static double calculateOutputGradient(const double& targetValue, const double& outputValue) - { - return (targetValue - outputValue) * OutputNeuronActivationPolicy::activationFunctionDerivative(outputValue); - } - - static double generateRandomWeight() { return RandomNumberGeneratorPolicy::generateRandomWeight(); } - static double hiddenNeuronActivationFunction(const double& value) { return HiddenNeuronActivationPolicy::activationFunction(value); } - static double hiddenNeuronActivationFunctionDerivative(const double& value) { return HiddenNeuronActivationPolicy::activationFunctionDerivative(value); } - static double outputNeuronActivationFunction(const double& value) { return OutputNeuronActivationPolicy::activationFunction(value); } - static double outputNeuronActivationFunctionDerivative(const double& value) { return OutputNeuronActivationPolicy::activationFunctionDerivative(value); } - static double initialAccelerationRate() { return 0.0; } - static double initialBiasOutputValue() { return 1.0; } - static double initialDeltaWeight() { return 0.0; } - static double initialGradientValue() { return 0.0; } - static double initialLearningRate() { return 0.01; } - static double initialMomentumRate() { return 0.0; } - static double initialOutputValue() { return 0.0; } - static bool isWithinZeroTolerance(const double& value) { return (fabs(value) < 0.004); } - static double negate(const double& value) { return -value; } - static double noOpDeltaWeight() { return 1.0; } - static double noOpWeight() { return 1.0; } -}; - -BOOST_AUTO_TEST_CASE(test_case_adam_optimizer_xor_training) -{ - // Verify Adam optimizer converges on XOR - typedef double ValueType; - typedef tinymind::MultilayerPerceptron AdamNNType; - srand(RANDOM_SEED); - AdamNNType nn; - - static const double ERROR_LIMIT = ValueHelper::getErrorLimit(); - ValueType values[2], output[1], error; - std::deque errors; - - static const int ADAM_TRAINING_ITERATIONS = 10000; - - for (int i = 0; i < ADAM_TRAINING_ITERATIONS; ++i) - { - generateXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - - if (!AdamTransferFunctions::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - - errors.push_front(error); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - } - - const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); - const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -BOOST_AUTO_TEST_CASE(test_case_adam_fixedpoint_xor_training) -{ - // Verify Adam optimizer converges on XOR (fixed-point Q16.16) - // Q16.16 provides sufficient precision for Adam's moment calculations - // and bias correction, which lose accuracy rapidly in Q8.8. - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - 1, - tinymind::DefaultNetworkInitializer, - tinymind::MeanSquaredErrorCalculator, - tinymind::ZeroToleranceCalculator, - tinymind::GradientClipByValue, - tinymind::NullWeightDecayPolicy, - tinymind::FixedLearningRatePolicy, - tinymind::AdamOptimizer> TransferFunctionsType; - - typedef tinymind::MultilayerPerceptron NNType; - - srand(RANDOM_SEED); - NNType nn; - - // Adam needs a lower learning rate than the default SGD rate. - // QValue(0, 655) ≈ 655/65536 ≈ 0.01 - nn.setLearningRate(ValueType(0, 655)); - - static const FullWidthValueType ERROR_LIMIT = ValueHelper::getErrorLimit(); - ValueType values[2], output[1], error; - std::deque errors; - - static const int ADAM_FP_ITERATIONS = 10000; - - for (int i = 0; i < ADAM_FP_ITERATIONS; ++i) - { - generateFixedPointXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - - if (!NNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - } - - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -BOOST_AUTO_TEST_CASE(test_case_lstm_weight_serialization) -{ - // Verify LSTM weights can be saved and loaded - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::LstmNeuralNetwork, 1, - TransferFunctionsType> LstmType; - srand(RANDOM_SEED); - LstmType nn; - - // Train briefly to get non-trivial weights - ValueType values[2], output[1]; - for (int i = 0; i < 100; ++i) - { - generateXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - nn.trainNetwork(&output[0]); - } - - // Save weights - { - std::ofstream outFile("output/lstm_weights.txt"); - tinymind::RecurrentNetworkPropertiesFileManager::storeNetworkWeights(nn, outFile); - } - - // Create a new network and load weights - LstmType nn2; - - { - std::ifstream inFile("output/lstm_weights.txt"); - tinymind::RecurrentNetworkPropertiesFileManager::template loadNetworkWeights(nn2, inFile); - } - - // Reset state so recurrent layer doesn't affect comparison - nn.resetState(); - nn2.resetState(); - - // Both networks should produce the same output for the same input - values[0] = 0.5; values[1] = 0.3; - nn.feedForward(&values[0]); - nn2.feedForward(&values[0]); - - ValueType out1[1], out2[1]; - nn.getLearnedValues(&out1[0]); - nn2.getLearnedValues(&out2[0]); - - // Tolerance accounts for text serialization precision loss - BOOST_TEST(fabs(out1[0] - out2[0]) < 0.02); -} - -BOOST_AUTO_TEST_CASE(test_case_gru_weight_serialization) -{ - // Verify GRU weights can be saved and loaded - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TransferFunctionsType; - typedef tinymind::GruNeuralNetwork, 1, - TransferFunctionsType> GruType; - srand(RANDOM_SEED); - GruType nn; - - // Train briefly - ValueType values[2], output[1]; - for (int i = 0; i < 100; ++i) - { - generateXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - nn.trainNetwork(&output[0]); - } - - // Save weights - { - std::ofstream outFile("output/gru_weights.txt"); - tinymind::RecurrentNetworkPropertiesFileManager::storeNetworkWeights(nn, outFile); - } - - // Load weights into new network - GruType nn2; - - { - std::ifstream inFile("output/gru_weights.txt"); - tinymind::RecurrentNetworkPropertiesFileManager::template loadNetworkWeights(nn2, inFile); - } - - // Both should produce same output - values[0] = 0.5; values[1] = 0.3; - nn.resetState(); - nn2.resetState(); - nn.feedForward(&values[0]); - nn2.feedForward(&values[0]); - - ValueType out1[1], out2[1]; - nn.getLearnedValues(&out1[0]); - nn2.getLearnedValues(&out2[0]); - - BOOST_TEST(fabs(out1[0] - out2[0]) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_elu_activation_fixed_point) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::EluActivationPolicy EluPolicy; - - // Positive values: f(x) = x - ValueType pos(0, 128); // 0.5 - ValueType result = EluPolicy::activationFunction(pos); - BOOST_TEST(result.getValue() == pos.getValue()); - - // Derivative for positive: f'(x) = 1 - ValueType deriv = EluPolicy::activationFunctionDerivative(pos); - BOOST_TEST(deriv.getValue() == tinymind::Constants::one().getValue()); - - // Negative values exercise the exp-table branch: f(x) = exp(x) - 1 < 0, - // and the derivative path f'(x) = x + 1, which is in (0, 1) for x in (-1, 0). - ValueType neg(-1, 128); // -0.5 - ValueType negResult = EluPolicy::activationFunction(neg); - BOOST_TEST(negResult < tinymind::Constants::zero()); - BOOST_TEST(negResult > ValueType(-1, 0)); - - ValueType negDeriv = EluPolicy::activationFunctionDerivative(neg); - BOOST_TEST(negDeriv > tinymind::Constants::zero()); - BOOST_TEST(negDeriv < tinymind::Constants::one()); -} - -BOOST_AUTO_TEST_CASE(test_case_gelu_activation_fixed_point) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::GeluActivationPolicy GeluPolicy; - - // GELU(0) should be ~0 (x * sigmoid(0) = 0 * 0.5 = 0) - ValueType zero(0); - ValueType result = GeluPolicy::activationFunction(zero); - BOOST_TEST(result.getValue() == 0); - - // GELU for positive input should be positive - ValueType pos(1, 0); // 1.0 - ValueType posResult = GeluPolicy::activationFunction(pos); - BOOST_TEST(posResult.getValue() > 0); -} - -BOOST_AUTO_TEST_CASE(test_case_network_stats) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TF; - typedef tinymind::NeuralNetwork, 1, TF> NNType; - typedef tinymind::NetworkStats Stats; - - // Use static_assert for compile-time verification (no ODR-use issues) - static_assert(Stats::NumberOfInputs == 2, "Wrong input count"); - static_assert(Stats::NumberOfHiddenNeurons == 5, "Wrong hidden count"); - static_assert(Stats::NumberOfOutputs == 1, "Wrong output count"); - static_assert(Stats::NumberOfHiddenLayers == 1, "Wrong layer count"); - static_assert(Stats::ValueSizeBytes == sizeof(ValueType), "Wrong value size"); - static_assert(Stats::InstanceSizeBytes == sizeof(NNType), "Wrong instance size"); - static_assert(Stats::InstanceSizeBytes > 0, "Instance size must be positive"); - BOOST_TEST(true); // test counts toward suite -} - -BOOST_AUTO_TEST_CASE(test_case_hidden_layer_accessor_out_of_range) -{ - // With a single hidden layer, InnerHiddenLayerChainType is EmptyLayerChain. - // An out-of-range hidden-layer index falls past the last-hidden-layer fast - // path into the ChainHiddenLayerAccessor terminal methods: - // getters return a default-constructed value, setters are no-ops. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> TF; - typedef tinymind::NeuralNetwork, 1, TF> NNType; - - NNType nn; - const size_t outOfRange = 1; // only valid hidden-layer index is 0 - - BOOST_TEST(nn.getHiddenLayerBiasNeuronWeightForConnection(outOfRange, 0) == - ValueType()); - BOOST_TEST(nn.getHiddenLayerWeightForNeuronAndConnection(outOfRange, 0, 0) == - ValueType()); - - // Setters on the terminal chain are no-ops; they must not touch the real - // layer-0 weights. Capture a valid weight, run the no-op setters, re-read. - const ValueType before = - nn.getHiddenLayerWeightForNeuronAndConnection(0, 0, 0); - nn.setHiddenLayerBiasNeuronWeightForConnection(outOfRange, 0, ValueType(1, 0)); - nn.setHiddenLayerWeightForNeuronAndConnection(outOfRange, 0, 0, ValueType(1, 0)); - BOOST_TEST(nn.getHiddenLayerWeightForNeuronAndConnection(0, 0, 0) == before); -} - -BOOST_AUTO_TEST_CASE(test_case_forward_as_dual_differentiable) -{ - // tinymind::pinn::forwardAs re-evaluates the STOCK NeuralNetwork in an - // arbitrary scalar type without touching its own forward path: with double - // it must reproduce getLearnedValues exactly, and with Dual it - // yields the network's input derivative (du/dx), matching finite difference. - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, tinymind::TanhActivationPolicy> TF; - typedef tinymind::MultilayerPerceptron NN; - - srand(RANDOM_SEED); - NN nn; - - double inputs[2] = { 0.3, -0.7 }; - - nn.feedForward(inputs); - double nativeOut[1]; - nn.getLearnedValues(nativeOut); - - double myOut[1]; - tinymind::pinn::forwardAs(nn, inputs, myOut); - BOOST_TEST(myOut[0] == nativeOut[0], boost::test_tools::tolerance(1e-12)); - - typedef tinymind::Dual D1; - D1 din[2] = { D1(0.3, 1.0), D1(-0.7, 0.0) }; // seed d/dx0 - D1 dout[1]; - tinymind::pinn::forwardAs(nn, din, dout); - - const double h = 1e-6; - double a[2] = { 0.3 + h, -0.7 }, b[2] = { 0.3 - h, -0.7 }; - double fa[1], fb[1]; - nn.feedForward(a); nn.getLearnedValues(fa); - nn.feedForward(b); nn.getLearnedValues(fb); - const double fd = (fa[0] - fb[0]) / (2.0 * h); - BOOST_TEST(dout[0].deriv == fd, boost::test_tools::tolerance(1e-5)); -} - -BOOST_AUTO_TEST_CASE(test_case_forward_as_multi_hidden_layer) -{ - // forwardAs handles multiple uniform-width hidden layers: a 2->4->4->1 MLP. - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, tinymind::TanhActivationPolicy> TF; - typedef tinymind::MultilayerPerceptron NN; - - srand(RANDOM_SEED); - NN nn; - - double inputs[2] = { 0.4, -0.2 }; - nn.feedForward(inputs); - double nativeOut[1]; - nn.getLearnedValues(nativeOut); - - double myOut[1]; - tinymind::pinn::forwardAs(nn, inputs, myOut); - BOOST_TEST(myOut[0] == nativeOut[0], boost::test_tools::tolerance(1e-12)); -} - -BOOST_AUTO_TEST_CASE(test_case_forward_as_linear_output) -{ - // forwardAs with a LINEAR output activation -- the canonical PINN-field - // configuration (tanh hidden + linear readout). The output activation policy - // must match the network's own, so forwardAs<...,LinearActivation> reproduces - // getLearnedValues and yields the correct input derivative. - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, tinymind::LinearActivationPolicy> TF; - typedef tinymind::MultilayerPerceptron NN; - - srand(RANDOM_SEED); - NN nn; - - double inputs[2] = { 0.25, -0.6 }; - nn.feedForward(inputs); - double nativeOut[1]; - nn.getLearnedValues(nativeOut); - - double myOut[1]; - tinymind::pinn::forwardAs(nn, inputs, myOut); - BOOST_TEST(myOut[0] == nativeOut[0], boost::test_tools::tolerance(1e-12)); - - typedef tinymind::Dual D1; - D1 din[2] = { D1(0.25, 1.0), D1(-0.6, 0.0) }; // seed d/dx0 - D1 dout[1]; - tinymind::pinn::forwardAs(nn, din, dout); - - const double h = 1e-6; - double a[2] = { 0.25 + h, -0.6 }, b[2] = { 0.25 - h, -0.6 }, fa[1], fb[1]; - nn.feedForward(a); nn.getLearnedValues(fa); - nn.feedForward(b); nn.getLearnedValues(fb); - const double fd = (fa[0] - fb[0]) / (2.0 * h); - BOOST_TEST(dout[0].deriv == fd, boost::test_tools::tolerance(1e-5)); -} - -BOOST_AUTO_TEST_CASE(test_case_teacher_forcing) -{ - tinymind::ScheduledSampling sampler(100); - - // At step 0, teacher forcing ratio should be 1.0 (always ground truth) - BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 1.0) < 0.01); - - // Advance halfway - for (int i = 0; i < 50; ++i) sampler.step(); - BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 0.5) < 0.01); - - // Advance to end - for (int i = 0; i < 50; ++i) sampler.step(); - BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 0.0) < 0.01); - - // After decay, selectInput should return prediction - double result = sampler.selectInput(1.0, 0.5); - BOOST_TEST(fabs(result - 0.5) < 0.001); - - // Reset should restore ratio to 1.0 - sampler.reset(); - BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 1.0) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_teacher_forcing_rand_branch) -{ - // Exercises the TINYMIND_ENABLE_HOSTED_RAND path of selectInput, which the - // post-decay test above skips (it only hits the early-out at step >= total). - tinymind::ScheduledSampling sampler(100); - - // Step 0: threshold == 1000, so rand() % 1000 is always below it -> - // selectInput always returns ground truth. - for (int i = 0; i < 50; ++i) - { - BOOST_TEST(fabs(sampler.selectInput(1.0, 0.5) - 1.0) < 1e-9); - } - - // Halfway through decay (threshold ~500): both outcomes must occur across - // many rolls. rand() is deterministically seeded, so this is reproducible. - for (int i = 0; i < 50; ++i) sampler.step(); - bool saw_ground_truth = false; - bool saw_prediction = false; - for (int i = 0; i < 500; ++i) - { - const double r = sampler.selectInput(1.0, 0.5); - if (fabs(r - 1.0) < 1e-9) saw_ground_truth = true; - if (fabs(r - 0.5) < 1e-9) saw_prediction = true; - } - BOOST_TEST(saw_ground_truth); - BOOST_TEST(saw_prediction); -} - -BOOST_AUTO_TEST_CASE(test_case_qformat_minmax_saturate_policy) -{ - // Direct coverage of MinMaxSaturatePolicy saturation arms, which the - // default QValue overflow policy does not route through. - typedef tinymind::MinMaxSaturatePolicy Policy; - const int lo = -100; - const int hi = 100; - - // Addition: positive overflow, negative underflow, and the zero no-op. - BOOST_TEST(Policy::saturate(90, 20, lo, hi, tinymind::AdditionOp) == hi); - BOOST_TEST(Policy::saturate(-90, -20, lo, hi, tinymind::AdditionOp) == lo); - BOOST_TEST(Policy::saturate(7, 0, lo, hi, tinymind::AdditionOp) == 7); - - // Subtraction: positive-target underflow, negative-target overflow, zero. - BOOST_TEST(Policy::saturate(-90, 20, lo, hi, tinymind::SubtractionOp) == lo); - BOOST_TEST(Policy::saturate(90, -20, lo, hi, tinymind::SubtractionOp) == hi); - BOOST_TEST(Policy::saturate(7, 0, lo, hi, tinymind::SubtractionOp) == 7); - - // Multiplication: overflow and underflow saturation. - BOOST_TEST(Policy::saturate(50, 50, lo, hi, tinymind::MultiplicationOp) == hi); - BOOST_TEST(Policy::saturate(50, -50, lo, hi, tinymind::MultiplicationOp) == lo); - - // Division: by-zero (sign-dependent), plus overflow / underflow. - BOOST_TEST(Policy::saturate(5, 0, lo, hi, tinymind::DivisionOp) == hi); - BOOST_TEST(Policy::saturate(-5, 0, lo, hi, tinymind::DivisionOp) == lo); - BOOST_TEST(Policy::saturate(1000, 2, lo, hi, tinymind::DivisionOp) == hi); - BOOST_TEST(Policy::saturate(-1000, 2, lo, hi, tinymind::DivisionOp) == lo); -} - -BOOST_AUTO_TEST_CASE(test_case_conv1d_forward) -{ - // Conv1D with 5-point input, kernel=3, stride=1, 1 filter - tinymind::Conv1D conv; - - // Set known kernel weights: [1, 0, -1] with bias=0 - conv.setFilterWeight(0, 0, 1.0); - conv.setFilterWeight(0, 1, 0.0); - conv.setFilterWeight(0, 2, -1.0); - conv.setFilterWeight(0, 3, 0.0); // bias - - double input[5] = {1.0, 2.0, 3.0, 4.0, 5.0}; - double output[3]; // (5 - 3) / 1 + 1 = 3 - - conv.forward(input, output); - - // kernel [1, 0, -1]: - // pos 0: 1*1 + 0*2 + (-1)*3 = -2 - // pos 1: 1*2 + 0*3 + (-1)*4 = -2 - // pos 2: 1*3 + 0*4 + (-1)*5 = -2 - BOOST_TEST(fabs(output[0] - (-2.0)) < 0.001); - BOOST_TEST(fabs(output[1] - (-2.0)) < 0.001); - BOOST_TEST(fabs(output[2] - (-2.0)) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_conv1d_multi_filter) -{ - // Conv1D with 4-point input, kernel=2, stride=1, 2 filters - tinymind::Conv1D conv; - - // Filter 0: [1, 1], bias=0 (sum filter) - conv.setFilterWeight(0, 0, 1.0); - conv.setFilterWeight(0, 1, 1.0); - conv.setFilterWeight(0, 2, 0.0); - - // Filter 1: [1, -1], bias=0 (difference filter) - conv.setFilterWeight(1, 0, 1.0); - conv.setFilterWeight(1, 1, -1.0); - conv.setFilterWeight(1, 2, 0.0); - - double input[4] = {1.0, 3.0, 2.0, 4.0}; - double output[6]; // 2 filters * 3 positions - - conv.forward(input, output); - - // Filter 0 (sum): [4, 5, 6] - BOOST_TEST(fabs(output[0] - 4.0) < 0.001); - BOOST_TEST(fabs(output[1] - 5.0) < 0.001); - BOOST_TEST(fabs(output[2] - 6.0) < 0.001); - - // Filter 1 (diff): [-2, 1, -2] - BOOST_TEST(fabs(output[3] - (-2.0)) < 0.001); - BOOST_TEST(fabs(output[4] - 1.0) < 0.001); - BOOST_TEST(fabs(output[5] - (-2.0)) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_conv1d_static_sizes) -{ - typedef tinymind::Conv1D ConvType; - - // OutputLength = (100 - 5) / 2 + 1 = 48 - static_assert(ConvType::OutputLength == 48, "Wrong output length"); - static_assert(ConvType::OutputSize == 384, "Wrong output size"); // 8 * 48 - static_assert(ConvType::TotalWeights == 48, "Wrong total weights"); // 8 * (5 + 1) - BOOST_TEST(true); // test counts toward suite -} - -BOOST_AUTO_TEST_CASE(test_case_conv1d_fixed_point) -{ - // Verify Conv1D works with Q8.8 fixed-point - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::Conv1D conv; - - // Set kernel [1, 0, -1], bias=0 - conv.setFilterWeight(0, 0, ValueType(1, 0)); - conv.setFilterWeight(0, 1, ValueType(0)); - conv.setFilterWeight(0, 2, ValueType(-1, 0)); - conv.setFilterWeight(0, 3, ValueType(0)); // bias - - ValueType input[5]; - input[0] = ValueType(1, 0); - input[1] = ValueType(2, 0); - input[2] = ValueType(3, 0); - input[3] = ValueType(4, 0); - input[4] = ValueType(5, 0); - - ValueType output[3]; - conv.forward(input, output); - - // kernel [1, 0, -1]: each output should be -2 - ValueType expected(-2, 0); - BOOST_TEST(output[0].getValue() == expected.getValue()); - BOOST_TEST(output[1].getValue() == expected.getValue()); - BOOST_TEST(output[2].getValue() == expected.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_conv1d_gradient_sanity) -{ - // y = sum(w * x) + b. With outputDelta = 1 the kernel weight gradients - // accumulate the input values at each stride position, and the bias - // gradient counts the number of output positions. - tinymind::Conv1D conv; - for (size_t i = 0; i < conv.TotalWeights; ++i) conv.setWeight(i, 0.0); - - double input[5] = {1.0, 2.0, 3.0, 4.0, 5.0}; - double output[3]; - conv.forward(input, output); - double outputDeltas[3] = {1.0, 1.0, 1.0}; - conv.computeGradients(outputDeltas, input); - - // Output positions slide kernel over input. Kernel-weight gradient at k - // accumulates input[k + pos*stride] for pos in [0, OutputLength). - // For stride=1, kernelSize=3, inputLength=5: outputLength=3. - // grad[k=0] = input[0] + input[1] + input[2] = 1+2+3 = 6 - // grad[k=1] = input[1] + input[2] + input[3] = 2+3+4 = 9 - // grad[k=2] = input[2] + input[3] + input[4] = 3+4+5 = 12 - BOOST_TEST(std::fabs(conv.getGradient(0) - 6.0) < 1e-9); - BOOST_TEST(std::fabs(conv.getGradient(1) - 9.0) < 1e-9); - BOOST_TEST(std::fabs(conv.getGradient(2) - 12.0) < 1e-9); - // Bias gradient = sum of deltas across output positions = 3. - BOOST_TEST(std::fabs(conv.getGradient(3) - 3.0) < 1e-9); - - // updateWeights with lr=0.1 should move weight 0 by 0.1 * 6.0 = 0.6. - conv.updateWeights(0.1); - BOOST_TEST(std::fabs(conv.getFilterWeight(0, 0) - 0.6) < 1e-9); - BOOST_TEST(std::fabs(conv.getFilterWeight(0, 3) - 0.3) < 1e-9); // bias -} - -BOOST_AUTO_TEST_CASE(test_case_conv1d_gradient_fixed_point) -{ - // Same gradient invariant in Q8.8. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::Conv1D conv; - for (size_t i = 0; i < conv.TotalWeights; ++i) conv.setWeight(i, ValueType(0)); - - ValueType input[5] = {ValueType(1, 0), ValueType(2, 0), ValueType(3, 0), - ValueType(4, 0), ValueType(5, 0)}; - ValueType output[3]; - conv.forward(input, output); - ValueType outputDeltas[3] = {ValueType(1, 0), ValueType(1, 0), ValueType(1, 0)}; - conv.computeGradients(outputDeltas, input); - - BOOST_TEST(conv.getGradient(0).getValue() == ValueType( 6, 0).getValue()); - BOOST_TEST(conv.getGradient(1).getValue() == ValueType( 9, 0).getValue()); - BOOST_TEST(conv.getGradient(2).getValue() == ValueType(12, 0).getValue()); - BOOST_TEST(conv.getGradient(3).getValue() == ValueType( 3, 0).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_truncated_bptt) -{ - // Test that TruncatedBPTT accumulates steps before training - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - tinymind::SigmoidActivationPolicy> TF; - typedef tinymind::LstmNeuralNetwork, 1, TF> LstmType; - - LstmType nn; - tinymind::TruncatedBPTT trainer; - - ValueType input[1] = {0.5}; - ValueType target[1] = {0.8}; - - // Steps 1-2: accumulate, no training yet - trainer.step(nn, input, target); - BOOST_TEST(trainer.getStepCount() == 1u); - trainer.step(nn, input, target); - BOOST_TEST(trainer.getStepCount() == 2u); - - // Step 3: window full, trains and resets - trainer.step(nn, input, target); - BOOST_TEST(trainer.getStepCount() == 0u); - - // Flush with partial window - trainer.step(nn, input, target); - trainer.flush(nn); - BOOST_TEST(trainer.getStepCount() == 0u); -} - -// ============================================================ -// MaxPool1D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_maxpool1d_forward) -{ - // MaxPool1D: 6-element input, pool size 2, stride 2, 1 channel - tinymind::MaxPool1D pool; - - double input[6] = {1.0, 3.0, 2.0, 5.0, 4.0, 6.0}; - double output[3]; // (6 - 2) / 2 + 1 = 3 - - pool.forward(input, output); - - // Window [1,3] -> 3, [2,5] -> 5, [4,6] -> 6 - BOOST_TEST(fabs(output[0] - 3.0) < 0.001); - BOOST_TEST(fabs(output[1] - 5.0) < 0.001); - BOOST_TEST(fabs(output[2] - 6.0) < 0.001); - - // Verify argmax indices - BOOST_TEST(pool.getArgMaxIndex(0) == 1u); - BOOST_TEST(pool.getArgMaxIndex(1) == 3u); - BOOST_TEST(pool.getArgMaxIndex(2) == 5u); -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool1d_backward) -{ - tinymind::MaxPool1D pool; - - double input[6] = {1.0, 3.0, 2.0, 5.0, 4.0, 6.0}; - double output[3]; - pool.forward(input, output); - - // Backprop: gradients should route to argmax positions - double outputDeltas[3] = {0.1, 0.2, 0.3}; - double inputDeltas[6]; - pool.backward(outputDeltas, inputDeltas); - - BOOST_TEST(fabs(inputDeltas[0]) < 0.001); // not max - BOOST_TEST(fabs(inputDeltas[1] - 0.1) < 0.001); // max of window 0 - BOOST_TEST(fabs(inputDeltas[2]) < 0.001); // not max - BOOST_TEST(fabs(inputDeltas[3] - 0.2) < 0.001); // max of window 1 - BOOST_TEST(fabs(inputDeltas[4]) < 0.001); // not max - BOOST_TEST(fabs(inputDeltas[5] - 0.3) < 0.001); // max of window 2 -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool1d_backward_fixed_point) -{ - // Same argmax routing in Q8.8: only the position holding each max - // should receive the upstream delta; others stay zero. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::MaxPool1D pool; - - ValueType input[6] = {ValueType(1, 0), ValueType(3, 0), ValueType(2, 0), - ValueType(5, 0), ValueType(4, 0), ValueType(6, 0)}; - ValueType output[3]; - pool.forward(input, output); - - ValueType outputDeltas[3] = {ValueType(1, 0), ValueType(2, 0), ValueType(3, 0)}; - ValueType inputDeltas[6]; - pool.backward(outputDeltas, inputDeltas); - - BOOST_TEST(inputDeltas[0].getValue() == 0); - BOOST_TEST(inputDeltas[1].getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(inputDeltas[2].getValue() == 0); - BOOST_TEST(inputDeltas[3].getValue() == ValueType(2, 0).getValue()); - BOOST_TEST(inputDeltas[4].getValue() == 0); - BOOST_TEST(inputDeltas[5].getValue() == ValueType(3, 0).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool1d_multichannel) -{ - // 2 channels, 4 elements each, pool size 2, stride 2 - tinymind::MaxPool1D pool; - - // Channel-major: [ch0: 1,4,2,3, ch1: 5,2,6,1] - double input[8] = {1.0, 4.0, 2.0, 3.0, 5.0, 2.0, 6.0, 1.0}; - double output[4]; // 2 channels * 2 outputs - - pool.forward(input, output); - - // Ch0: [1,4]->4, [2,3]->3 - BOOST_TEST(fabs(output[0] - 4.0) < 0.001); - BOOST_TEST(fabs(output[1] - 3.0) < 0.001); - // Ch1: [5,2]->5, [6,1]->6 - BOOST_TEST(fabs(output[2] - 5.0) < 0.001); - BOOST_TEST(fabs(output[3] - 6.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool1d_stride1) -{ - // Overlapping windows: pool size 3, stride 1 - tinymind::MaxPool1D pool; - - double input[5] = {2.0, 1.0, 4.0, 3.0, 5.0}; - double output[3]; // (5 - 3) / 1 + 1 = 3 - - pool.forward(input, output); - - // [2,1,4]->4, [1,4,3]->4, [4,3,5]->5 - BOOST_TEST(fabs(output[0] - 4.0) < 0.001); - BOOST_TEST(fabs(output[1] - 4.0) < 0.001); - BOOST_TEST(fabs(output[2] - 5.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool1d_static_sizes) -{ - typedef tinymind::MaxPool1D PoolType; - - // OutputLength = (100 - 4) / 4 + 1 = 25 - static_assert(PoolType::OutputLength == 25, "Wrong output length"); - static_assert(PoolType::OutputSize == 75, "Wrong output size"); // 3 * 25 - static_assert(PoolType::InputSize == 300, "Wrong input size"); // 3 * 100 - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool1d_fixed_point) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::MaxPool1D pool; - - ValueType input[4]; - input[0] = ValueType(1, 0); - input[1] = ValueType(3, 0); - input[2] = ValueType(2, 0); - input[3] = ValueType(5, 0); - - ValueType output[2]; - pool.forward(input, output); - - BOOST_TEST(output[0].getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(output[1].getValue() == ValueType(5, 0).getValue()); -} - -// ============================================================ -// AvgPool1D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_avgpool1d_forward) -{ - tinymind::AvgPool1D pool; - - double input[6] = {1.0, 3.0, 2.0, 6.0, 4.0, 8.0}; - double output[3]; - - pool.forward(input, output); - - // [1,3]->2, [2,6]->4, [4,8]->6 - BOOST_TEST(fabs(output[0] - 2.0) < 0.001); - BOOST_TEST(fabs(output[1] - 4.0) < 0.001); - BOOST_TEST(fabs(output[2] - 6.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool1d_backward) -{ - tinymind::AvgPool1D pool; - - double outputDeltas[3] = {0.2, 0.4, 0.6}; - double inputDeltas[6]; - pool.backward(outputDeltas, inputDeltas); - - // Each gradient splits evenly across pool window - BOOST_TEST(fabs(inputDeltas[0] - 0.1) < 0.001); - BOOST_TEST(fabs(inputDeltas[1] - 0.1) < 0.001); - BOOST_TEST(fabs(inputDeltas[2] - 0.2) < 0.001); - BOOST_TEST(fabs(inputDeltas[3] - 0.2) < 0.001); - BOOST_TEST(fabs(inputDeltas[4] - 0.3) < 0.001); - BOOST_TEST(fabs(inputDeltas[5] - 0.3) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool1d_multichannel) -{ - tinymind::AvgPool1D pool; - - double input[8] = {2.0, 4.0, 6.0, 8.0, 1.0, 3.0, 5.0, 7.0}; - double output[4]; - - pool.forward(input, output); - - // Ch0: [2,4]->3, [6,8]->7 - BOOST_TEST(fabs(output[0] - 3.0) < 0.001); - BOOST_TEST(fabs(output[1] - 7.0) < 0.001); - // Ch1: [1,3]->2, [5,7]->6 - BOOST_TEST(fabs(output[2] - 2.0) < 0.001); - BOOST_TEST(fabs(output[3] - 6.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool1d_pool3) -{ - tinymind::AvgPool1D pool; - - double input[6] = {3.0, 6.0, 9.0, 12.0, 15.0, 18.0}; - double output[2]; - - pool.forward(input, output); - - // [3,6,9]->6, [12,15,18]->15 - BOOST_TEST(fabs(output[0] - 6.0) < 0.001); - BOOST_TEST(fabs(output[1] - 15.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool1d_fixed_point) -{ - // Pin down the fixed-point divisor: average of [2.0, 4.0] in Q8.8 must be - // exactly 3.0, not raw=PoolSize. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::AvgPool1D pool; - - ValueType input[4] = {ValueType(2, 0), ValueType(4, 0), ValueType(6, 0), ValueType(8, 0)}; - ValueType output[2]; - pool.forward(input, output); - - // (2+4)/2 = 3.0; (6+8)/2 = 7.0. - BOOST_TEST(output[0].getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(output[1].getValue() == ValueType(7, 0).getValue()); -} - -// ============================================================ -// UpsampleNearest1D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_forward) -{ - // Repeat each element by factor 2: dual of a stride-2 pool. - tinymind::UpsampleNearest1D up; - - double input[4] = {3.0, 7.0, 2.0, 9.0}; - double output[8]; // 4 * 2 - - up.forward(input, output); - - const double expected[8] = {3.0, 3.0, 7.0, 7.0, 2.0, 2.0, 9.0, 9.0}; - for (size_t i = 0; i < 8; ++i) - { - BOOST_TEST(fabs(output[i] - expected[i]) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_backward) -{ - // Backward accumulates the ScaleFactor output grads onto each input. - tinymind::UpsampleNearest1D up; - - double outputDeltas[9] = {0.1, 0.2, 0.3, 1.0, 1.0, 1.0, 0.5, 0.0, -0.5}; - double inputDeltas[3]; - up.backward(outputDeltas, inputDeltas); - - BOOST_TEST(fabs(inputDeltas[0] - 0.6) < 0.001); // 0.1+0.2+0.3 - BOOST_TEST(fabs(inputDeltas[1] - 3.0) < 0.001); // 1+1+1 - BOOST_TEST(fabs(inputDeltas[2] - 0.0) < 0.001); // 0.5+0-0.5 -} - -BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_multichannel) -{ - // 2 channels, 3 elements each, factor 2, channel-major. - tinymind::UpsampleNearest1D up; - - double input[6] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; - double output[12]; - up.forward(input, output); - - const double expected[12] = {1.0, 1.0, 2.0, 2.0, 3.0, 3.0, - 4.0, 4.0, 5.0, 5.0, 6.0, 6.0}; - for (size_t i = 0; i < 12; ++i) - { - BOOST_TEST(fabs(output[i] - expected[i]) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_static_sizes) -{ - typedef tinymind::UpsampleNearest1D UpType; - - static_assert(UpType::OutputLength == 200, "Wrong output length"); // 50 * 4 - static_assert(UpType::OutputSize == 600, "Wrong output size"); // 3 * 200 - static_assert(UpType::InputSize == 150, "Wrong input size"); // 3 * 50 - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_fixed_point) -{ - // No arithmetic: exact bit copies in Q8.8. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::UpsampleNearest1D up; - - ValueType input[2] = {ValueType(3, 0), ValueType(5, 0)}; - ValueType output[6]; - up.forward(input, output); - - BOOST_TEST(output[0].getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(output[1].getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(output[2].getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(output[3].getValue() == ValueType(5, 0).getValue()); - BOOST_TEST(output[4].getValue() == ValueType(5, 0).getValue()); - BOOST_TEST(output[5].getValue() == ValueType(5, 0).getValue()); -} - -// ============================================================ -// UpsampleLinear1D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_forward) -{ - // Factor 2: input nodes land on even outputs; odds are the midpoints. - tinymind::UpsampleLinear1D up; - - double input[4] = {0.0, 4.0, 2.0, 10.0}; - double output[8]; - up.forward(input, output); - - // node, mid, node, mid, node, mid, node, tail-clamped last value - const double expected[8] = {0.0, 2.0, 4.0, 3.0, 2.0, 6.0, 10.0, 10.0}; - for (size_t i = 0; i < 8; ++i) - { - BOOST_TEST(fabs(output[i] - expected[i]) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_nodes_preserved) -{ - // Every input element must reappear exactly at index k * ScaleFactor. - tinymind::UpsampleLinear1D up; - - double input[5] = {1.0, 2.0, 4.0, 8.0, 16.0}; - double output[15]; - up.forward(input, output); - - for (size_t k = 0; k < 5; ++k) - { - BOOST_TEST(fabs(output[k * 3] - input[k]) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_backward) -{ - // Backward is the transpose: a unit grad on a midpoint splits by weight. - tinymind::UpsampleLinear1D up; - - // outputs: [node0, mid, node1, tail]; mid weight = 0.5 to each node. - double outputDeltas[4] = {0.0, 1.0, 0.0, 0.0}; - double inputDeltas[2]; - up.backward(outputDeltas, inputDeltas); - - BOOST_TEST(fabs(inputDeltas[0] - 0.5) < 0.001); - BOOST_TEST(fabs(inputDeltas[1] - 0.5) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_fixed_point) -{ - // Midpoint of [2.0, 6.0] in Q8.8 must be exactly 4.0 (weight 0.5 built via - // the (FixedPart, FractionalPart) constructor, not raw bits). - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::UpsampleLinear1D up; - - ValueType input[2] = {ValueType(2, 0), ValueType(6, 0)}; - ValueType output[4]; - up.forward(input, output); - - BOOST_TEST(output[0].getValue() == ValueType(2, 0).getValue()); // node - BOOST_TEST(output[1].getValue() == ValueType(4, 0).getValue()); // midpoint - BOOST_TEST(output[2].getValue() == ValueType(6, 0).getValue()); // node - BOOST_TEST(output[3].getValue() == ValueType(6, 0).getValue()); // tail clamp -} - -// ============================================================ -// ANFIS membership function tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_anfis_triangular_membership_function) -{ - typedef tinymind::TriangularMembershipFunction MfType; - - static_assert(MfType::NumberOfParameters == 3, "Triangle takes {a, b, c}"); - - const double parameters[3] = {0.0, 1.0, 2.0}; - - BOOST_TEST(fabs(MfType::evaluate(parameters, -1.0) - 0.0) < 0.001); // left of support - BOOST_TEST(fabs(MfType::evaluate(parameters, 0.0) - 0.0) < 0.001); // left foot - BOOST_TEST(fabs(MfType::evaluate(parameters, 0.25) - 0.25) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, 1.0) - 1.0) < 0.001); // peak - BOOST_TEST(fabs(MfType::evaluate(parameters, 1.5) - 0.5) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, 2.0) - 0.0) < 0.001); // right foot - BOOST_TEST(fabs(MfType::evaluate(parameters, 9.0) - 0.0) < 0.001); // right of support -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_triangular_shoulders) -{ - // a == b is a left shoulder, b == c a right shoulder. Neither may divide - // by zero: the degenerate side is unreachable from inside the support. - typedef tinymind::TriangularMembershipFunction MfType; - - const double leftShoulder[3] = {0.0, 0.0, 2.0}; - BOOST_TEST(fabs(MfType::evaluate(leftShoulder, 1.0) - 0.5) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(leftShoulder, 0.0) - 0.0) < 0.001); - - const double rightShoulder[3] = {0.0, 2.0, 2.0}; - BOOST_TEST(fabs(MfType::evaluate(rightShoulder, 1.0) - 0.5) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(rightShoulder, 2.0) - 0.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_trapezoidal_membership_function) -{ - typedef tinymind::TrapezoidalMembershipFunction MfType; - - static_assert(MfType::NumberOfParameters == 4, "Trapezoid takes {a, b, c, d}"); - - const double parameters[4] = {0.0, 1.0, 3.0, 4.0}; - - BOOST_TEST(fabs(MfType::evaluate(parameters, 0.0) - 0.0) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, 0.5) - 0.5) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, 1.0) - 1.0) < 0.001); // plateau start - BOOST_TEST(fabs(MfType::evaluate(parameters, 2.0) - 1.0) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, 3.0) - 1.0) < 0.001); // plateau end - BOOST_TEST(fabs(MfType::evaluate(parameters, 3.5) - 0.5) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, 4.0) - 0.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_generalized_bell_membership_function) -{ - // mu = 1 / (1 + (((x - c) / a)^2)^BellExponent) - typedef tinymind::GeneralizedBellMembershipFunction Bell1Type; - typedef tinymind::GeneralizedBellMembershipFunction Bell2Type; - - static_assert(Bell1Type::NumberOfParameters == 2, "Bell takes {a, c}"); - - const double parameters[2] = {1.0, 0.0}; // width 1, centered at 0 - - BOOST_TEST(fabs(Bell1Type::evaluate(parameters, 0.0) - 1.0) < 0.001); - BOOST_TEST(fabs(Bell1Type::evaluate(parameters, 1.0) - 0.5) < 0.001); // the half-width point - BOOST_TEST(fabs(Bell2Type::evaluate(parameters, 1.0) - 0.5) < 0.001); // b does not move it - BOOST_TEST(fabs(Bell2Type::evaluate(parameters, 0.5) - (1.0 / 1.0625)) < 0.001); - - // t >= 64 is reported as 0 so a narrow Q format cannot overflow. The clamp - // sits both before the exponent loop (t itself is already over) and inside - // it (t is under, but t^BellExponent is not: t = 9 -> t^2 = 81). - BOOST_TEST(fabs(Bell1Type::evaluate(parameters, 100.0) - 0.0) < 0.001); - BOOST_TEST(fabs(Bell2Type::evaluate(parameters, 3.0) - 0.0) < 0.001); - - // a == 0 degenerates to an impulse rather than dividing by zero. - const double impulse[2] = {0.0, 3.0}; - BOOST_TEST(fabs(Bell1Type::evaluate(impulse, 3.0) - 1.0) < 0.001); - BOOST_TEST(fabs(Bell1Type::evaluate(impulse, 3.5) - 0.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_gaussian_membership_function) -{ - // Parameters are {c, sigma, 1/sigma}; the reciprocal is precomputed so the - // inference path carries no divide. - typedef tinymind::GaussianMembershipFunction MfType; - - static_assert(MfType::NumberOfParameters == 3, "Gaussian takes {c, sigma, 1/sigma}"); - - const double parameters[3] = {0.0, 1.0, 1.0}; - - BOOST_TEST(fabs(MfType::evaluate(parameters, 0.0) - 1.0) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, 1.0) - exp(-0.5)) < 0.001); - BOOST_TEST(fabs(MfType::evaluate(parameters, -1.0) - exp(-0.5)) < 0.001); // symmetric - BOOST_TEST(fabs(MfType::evaluate(parameters, 4.0) - 0.0) < 0.001); // hard cut at 4 sigma -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_gaussian_membership_function_fixed_point) -{ - // The Q-format Gaussian rides the integer exp lookup table, so it needs no - // FPU. It must track the float form closely. - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::GaussianMembershipFunction MfType; - - const ValueType parameters[3] = {ValueType(0, 0), ValueType(1, 0), ValueType(1, 0)}; - - const ValueType mu = MfType::evaluate(parameters, ValueType(1, 0)); - const double muAsDouble = static_cast(mu.getValue()) / 65536.0; - BOOST_TEST(fabs(muAsDouble - exp(-0.5)) < 0.01); - - const ValueType far = MfType::evaluate(parameters, ValueType(5, 0)); - BOOST_TEST(far.getValue() == 0); -} - -// ============================================================ -// ANFIS rule table tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_anfis_full_grid_rule_table) -{ - typedef tinymind::AnfisFullGridRuleTable<2, 3> GridType; - - static_assert(GridType::NumberOfRules == 9, "3^2 == 9"); - static_assert(GridType::RuleTableSize == 18, "9 rules * 2 inputs"); - - uint8_t ruleTable[GridType::RuleTableSize]; - GridType::generate(ruleTable); - - // Odometer order: the last input varies fastest. - const uint8_t expected[18] = {0, 0, 0, 1, 0, 2, - 1, 0, 1, 1, 1, 2, - 2, 0, 2, 1, 2, 2}; - for (size_t i = 0; i < GridType::RuleTableSize; ++i) - { - BOOST_TEST(ruleTable[i] == expected[i]); - } -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_full_grid_rule_count_explodes) -{ - // The reason the rule base is an explicit table and not an implicit grid. - static_assert(tinymind::AnfisFullGridRuleTable<4, 3>::NumberOfRules == 81, "3^4"); - static_assert(tinymind::AnfisFullGridRuleTable<8, 3>::NumberOfRules == 6561, "3^8"); - BOOST_TEST(true); -} - -// ============================================================ -// ANFIS inference tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_anfis_zeroth_order_inference) -{ - // One input, two triangles crossing at 0.5, constant consequents 10 and 20. - // Both rules fire at 0.5 -> the output is the midpoint. - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::Anfis AnfisType; - - static_assert(AnfisType::NumberOfPremiseParameters == 6, "2 MFs * 3 parameters"); - static_assert(AnfisType::NumberOfConsequentParametersPerRule == 1, "Zeroth order is a constant"); - static_assert(AnfisType::NumberOfConsequentParameters == 2, "2 rules * 1 output * 1 parameter"); - - const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, - 0.0, 1.0, 2.0}; - const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; - const double consequent[AnfisType::NumberOfConsequentParameters] = {10.0, 20.0}; - - AnfisType anfis(premise, ruleTable, consequent); - - double input[1] = {0.5}; - double output[1]; - anfis.forward(input, output); - - BOOST_TEST(fabs(output[0] - 15.0) < 0.001); - BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(0) - 0.5) < 0.001); - BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(1) - 0.5) < 0.001); - - // Sitting on a peak makes that rule the only one that fires. - input[0] = 0.0; - anfis.forward(input, output); - BOOST_TEST(fabs(output[0] - 10.0) < 0.001); - BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(0) - 1.0) < 0.001); - BOOST_TEST(anfis.getDominantRule() == 0); - - input[0] = 1.0; - anfis.forward(input, output); - BOOST_TEST(fabs(output[0] - 20.0) < 0.001); - BOOST_TEST(anfis.getDominantRule() == 1); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_first_order_inference) -{ - // Two inputs, two triangles each, the full 2x2 grid, linear consequents. - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::AnfisFullGridRuleTable<2, 2> GridType; - typedef tinymind::Anfis AnfisType; - - static_assert(AnfisType::NumberOfConsequentParametersPerRule == 3, "2 coefficients + 1 constant"); - - uint8_t ruleTable[GridType::RuleTableSize]; - GridType::generate(ruleTable); - - const double premise[AnfisType::NumberOfPremiseParameters] = { - -1.0, 0.0, 1.0, 0.0, 1.0, 2.0, // input 0 - -1.0, 0.0, 1.0, 0.0, 1.0, 2.0}; // input 1 - - // f_r = p0 * x0 + p1 * x1 + q - const double consequent[AnfisType::NumberOfConsequentParameters] = { - 1.0, 0.0, 0.0, // rule 0: x0 - 0.0, 1.0, 0.0, // rule 1: x1 - 0.0, 0.0, 5.0, // rule 2: 5 - 1.0, 1.0, 1.0}; // rule 3: x0 + x1 + 1 - - AnfisType anfis(premise, ruleTable, consequent); - - double input[2] = {0.5, 0.5}; - double output[1]; - anfis.forward(input, output); - - // Every grade is 0.5, so every firing strength is 0.25 and the total is 1. - // f = [0.5, 0.5, 5.0, 2.0] -> y = 0.25 * 8.0 = 2.0 - BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 1.0) < 0.001); - BOOST_TEST(fabs(output[0] - 2.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_multiple_outputs_share_the_premise) -{ - // Two outputs off one premise layer: the membership grades and firing - // strengths are computed once and reused. - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::Anfis AnfisType; - - static_assert(AnfisType::NumberOfConsequentParameters == 4, "2 rules * 2 outputs * 1 parameter"); - - const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, - 0.0, 1.0, 2.0}; - const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; - // [rule][output] - const double consequent[AnfisType::NumberOfConsequentParameters] = {10.0, -1.0, - 20.0, 1.0}; - - AnfisType anfis(premise, ruleTable, consequent); - - double input[1] = {0.5}; - double output[2]; - anfis.forward(input, output); - - BOOST_TEST(fabs(output[0] - 15.0) < 0.001); - BOOST_TEST(fabs(output[1] - 0.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_dont_care_antecedent) -{ - // A pruned rule ignores an input entirely: the antecedent contributes a - // grade of 1 no matter how far outside every membership function it sits. - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::Anfis AnfisType; - - const double premise[AnfisType::NumberOfPremiseParameters] = { - -1.0, 0.0, 1.0, 0.0, 1.0, 2.0, - -1.0, 0.0, 1.0, 0.0, 1.0, 2.0}; - const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, AnfisType::DontCareIndex}; - const double consequent[AnfisType::NumberOfConsequentParameters] = {7.0}; - - AnfisType anfis(premise, ruleTable, consequent); - - double input[2] = {0.5, 99.0}; // input 1 is outside every membership function - double output[1]; - anfis.forward(input, output); - - BOOST_TEST(fabs(anfis.getFiringStrength(0) - 0.5) < 0.001); // grade of input 0 alone - BOOST_TEST(fabs(output[0] - 7.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_no_rule_fires) -{ - // Outside the support of every membership function nothing fires. The - // output must be zeroed without attempting a divide by zero. - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::Anfis AnfisType; - - const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0}; - const uint8_t ruleTable[AnfisType::RuleTableSize] = {0}; - const double consequent[AnfisType::NumberOfConsequentParameters] = {42.0}; - - AnfisType anfis(premise, ruleTable, consequent); - - double input[1] = {5.0}; - double output[1] = {123.0}; - anfis.forward(input, output); - - BOOST_TEST(fabs(output[0] - 0.0) < 0.001); - BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 0.0) < 0.001); - BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(0) - 0.0) < 0.001); - - // Back inside the support the same system produces its consequent again: - // the dead path must not be sticky. - input[0] = 0.0; - anfis.forward(input, output); - BOOST_TEST(fabs(output[0] - 42.0) < 0.001); - BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 1.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_membership_grades_are_readable) -{ - // The premise layer is readable too, not just the rule layer. - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::Anfis AnfisType; - - const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, - 0.0, 1.0, 2.0}; - const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; - const double consequent[AnfisType::NumberOfConsequentParameters] = {10.0, 20.0}; - - AnfisType anfis(premise, ruleTable, consequent); - - double input[1] = {0.25}; - double output[1]; - anfis.forward(input, output); - - BOOST_TEST(fabs(anfis.getMembershipGrade(0, 0) - 0.75) < 0.001); - BOOST_TEST(fabs(anfis.getMembershipGrade(0, 1) - 0.25) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_partition_of_unity) -{ - // Two triangles that sum to 1 across the crossover region make a - // zeroth-order ANFIS interpolate linearly between its two consequents. - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::Anfis AnfisType; - - const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, - 0.0, 1.0, 2.0}; - const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; - const double consequent[AnfisType::NumberOfConsequentParameters] = {0.0, 10.0}; - - AnfisType anfis(premise, ruleTable, consequent); - - for (size_t step = 1; step < 10; ++step) - { - const double x = static_cast(step) / 10.0; - double input[1] = {x}; - double output[1]; - anfis.forward(input, output); - - BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 1.0) < 0.001); - BOOST_TEST(fabs(output[0] - (10.0 * x)) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_anfis_fixed_point_inference) -{ - // Q16.16 is the recommended deployment format: the fused - // sum(w * f) / sum(w) defuzzification needs headroom for the weighted sum. - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::TriangularMembershipFunction MfType; - typedef tinymind::Anfis AnfisType; - - const ValueType premise[AnfisType::NumberOfPremiseParameters] = { - ValueType(-1, 0), ValueType(0, 0), ValueType(1, 0), - ValueType(0, 0), ValueType(1, 0), ValueType(2, 0)}; - const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; - const ValueType consequent[AnfisType::NumberOfConsequentParameters] = {ValueType(10, 0), - ValueType(20, 0)}; - - AnfisType anfis(premise, ruleTable, consequent); - - ValueType input[1] = {ValueType(0, 1u << 15)}; // 0.5 - ValueType output[1]; - anfis.forward(input, output); - - BOOST_TEST(output[0].getValue() == ValueType(15, 0).getValue()); - - input[0] = ValueType(1, 0); - anfis.forward(input, output); - BOOST_TEST(output[0].getValue() == ValueType(20, 0).getValue()); -} - -// ============================================================ -// Dropout tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_dropout_inference_passthrough) -{ - // In inference mode, dropout should be identity - tinymind::Dropout dropout; - dropout.setTraining(false); - - double input[4] = {1.0, 2.0, 3.0, 4.0}; - double output[4]; - - dropout.forward(input, output); - - BOOST_TEST(fabs(output[0] - 1.0) < 0.001); - BOOST_TEST(fabs(output[1] - 2.0) < 0.001); - BOOST_TEST(fabs(output[2] - 3.0) < 0.001); - BOOST_TEST(fabs(output[3] - 4.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_training_zeros_some) -{ - // In training mode, some outputs should be zero - srand(RANDOM_SEED); - tinymind::Dropout dropout; - dropout.setTraining(true); - - double input[100]; - double output[100]; - for (size_t i = 0; i < 100; ++i) - { - input[i] = 1.0; - } - - dropout.forward(input, output); - - size_t zeroCount = 0; - size_t nonZeroCount = 0; - for (size_t i = 0; i < 100; ++i) - { - if (fabs(output[i]) < 0.001) - { - ++zeroCount; - } - else - { - ++nonZeroCount; - } - } - - // With 50% dropout on 100 elements, expect roughly 50 zeros - // Allow wide tolerance for randomness - BOOST_TEST(zeroCount > 20u); - BOOST_TEST(nonZeroCount > 20u); -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_inverted_scaling) -{ - // Verify inverted dropout scaling: survivors should be scaled by 1/(1-p) - // With 50% dropout, scale = 2.0 - srand(RANDOM_SEED); - tinymind::Dropout dropout; - dropout.setTraining(true); - - double input[10]; - double output[10]; - for (size_t i = 0; i < 10; ++i) - { - input[i] = 1.0; - } - - dropout.forward(input, output); - - for (size_t i = 0; i < 10; ++i) - { - if (dropout.getMask(i)) - { - // Kept: should be scaled by 2.0 (1/(1-0.5)) - BOOST_TEST(fabs(output[i] - 2.0) < 0.001); - } - else - { - // Dropped: should be zero - BOOST_TEST(fabs(output[i]) < 0.001); - } - } -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_backward_mask) -{ - srand(RANDOM_SEED); - tinymind::Dropout dropout; - dropout.setTraining(true); - - double input[5] = {1.0, 1.0, 1.0, 1.0, 1.0}; - double output[5]; - dropout.forward(input, output); // generates mask - - double outputDeltas[5] = {0.5, 0.5, 0.5, 0.5, 0.5}; - double inputDeltas[5]; - dropout.backward(outputDeltas, inputDeltas); - - for (size_t i = 0; i < 5; ++i) - { - if (dropout.getMask(i)) - { - // Gradient scaled by 1/(1-p) = 2.0 - BOOST_TEST(fabs(inputDeltas[i] - 1.0) < 0.001); - } - else - { - BOOST_TEST(fabs(inputDeltas[i]) < 0.001); - } - } -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_zero_percent) -{ - // 0% dropout should pass everything through even in training mode - tinymind::Dropout dropout; - dropout.setTraining(true); - - double input[4] = {1.0, 2.0, 3.0, 4.0}; - double output[4]; - - dropout.forward(input, output); - - BOOST_TEST(fabs(output[0] - 1.0) < 0.001); - BOOST_TEST(fabs(output[1] - 2.0) < 0.001); - BOOST_TEST(fabs(output[2] - 3.0) < 0.001); - BOOST_TEST(fabs(output[3] - 4.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_mode_toggle) -{ - tinymind::Dropout dropout; - - BOOST_TEST(dropout.isTraining() == true); // default is training - dropout.setTraining(false); - BOOST_TEST(dropout.isTraining() == false); - dropout.setTraining(true); - BOOST_TEST(dropout.isTraining() == true); -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_fixed_point_inference_passthrough) -{ - // In inference mode the forward pass is a verbatim copy, even with - // fixed-point types. No scaling, no mask sampling. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::Dropout dropout; - dropout.setTraining(false); - - ValueType input[4] = {ValueType(1, 0), ValueType(2, 0), ValueType(-1, 0), ValueType(0, 128)}; - ValueType output[4]; - dropout.forward(input, output); - - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(output[i].getValue() == input[i].getValue()); - } -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_fixed_point_training_scaling) -{ - // Q8.8 with 50% dropout: survivors must scale by exactly 2.0 = raw 512. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - srand(RANDOM_SEED); - tinymind::Dropout dropout; - dropout.setTraining(true); - - ValueType input[8]; - for (size_t i = 0; i < 8; ++i) - { - input[i] = ValueType(1, 0); // 1.0 - } - ValueType output[8]; - dropout.forward(input, output); - - for (size_t i = 0; i < 8; ++i) - { - if (dropout.getMask(i)) - { - BOOST_TEST(output[i].getValue() == ValueType(2, 0).getValue()); - } - else - { - BOOST_TEST(output[i].getValue() == 0); - } - } -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_high_rate_boundary) -{ - // 99% dropout is the largest legal rate (the layer static_asserts < 100). - // Scale = 100. Use Q16.16 since 100 doesn't fit in Q8.8's signed fixed range. - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - srand(RANDOM_SEED); - tinymind::Dropout dropout; - dropout.setTraining(true); - - ValueType input[16]; - for (size_t i = 0; i < 16; ++i) - { - input[i] = ValueType(1, 0); - } - ValueType output[16]; - dropout.forward(input, output); - - const ValueType expectedKept(100, 0); - bool sawDropped = false; - for (size_t i = 0; i < 16; ++i) - { - if (dropout.getMask(i)) - { - // 1 LSB tolerance for Q16.16 division rounding in scale construction. - const auto delta = output[i].getValue() - expectedKept.getValue(); - BOOST_TEST(std::abs(delta) <= 1); - } - else - { - BOOST_TEST(output[i].getValue() == 0); - sawDropped = true; - } - } - // With 99% drop on 16 elements the chance of zero drops is ~1.5e-32. - BOOST_TEST(sawDropped); -} - -BOOST_AUTO_TEST_CASE(test_case_dropout_seed_determinism) -{ - // Identical srand seeds must produce identical masks across separate - // dropout instances — needed for reproducible training runs. - tinymind::Dropout a; - tinymind::Dropout b; - double input[32]; - for (size_t i = 0; i < 32; ++i) input[i] = 1.0; - double outA[32], outB[32]; - - srand(RANDOM_SEED); - a.forward(input, outA); - srand(RANDOM_SEED); - b.forward(input, outB); - - for (size_t i = 0; i < 32; ++i) - { - BOOST_TEST(a.getMask(i) == b.getMask(i)); - BOOST_TEST(outA[i] == outB[i]); - } -} - -// ============================================================ -// RMSprop optimizer tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_rmsprop_float_xor) -{ - // Verify RMSprop optimizer converges on XOR (floating-point) - typedef double ValueType; - typedef FloatingPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy> BaseTF; - - struct RmsPropTF : public BaseTF - { - typedef tinymind::RmsPropOptimizerFloat OptimizerPolicyType; - }; - - typedef tinymind::MultilayerPerceptron NNType; - - srand(RANDOM_SEED); - NNType nn; - - static const double ERROR_LIMIT = ValueHelper::getErrorLimit(); - ValueType values[2], output[1], error; - std::deque errors; - - for (int i = 0; i < 10000; ++i) - { - generateXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - nn.trainNetwork(&output[0]); - - errors.push_front(error); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - } - - const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); - const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -BOOST_AUTO_TEST_CASE(test_case_rmsprop_fixedpoint_xor) -{ - // Verify RMSprop optimizer converges on XOR (fixed-point) - // Adaptive optimizers need gradient clipping with fixed-point to - // prevent overflow in the moment accumulation - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - typedef typename ValueType::FullWidthValueType FullWidthValueType; - - typedef tinymind::FixedPointTransferFunctions< - ValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - 1, - tinymind::DefaultNetworkInitializer, - tinymind::MeanSquaredErrorCalculator, - tinymind::ZeroToleranceCalculator, - tinymind::GradientClipByValue, - tinymind::NullWeightDecayPolicy, - tinymind::FixedLearningRatePolicy, - tinymind::RmsPropOptimizer> TransferFunctionsType; - - typedef tinymind::MultilayerPerceptron NNType; - - srand(RANDOM_SEED); - NNType nn; - - static const FullWidthValueType ERROR_LIMIT = ValueHelper::getErrorLimit(); - ValueType values[2], output[1], error; - std::deque errors; - - static const int RMSPROP_FP_ITERATIONS = TRAINING_ITERATIONS; - - for (int i = 0; i < RMSPROP_FP_ITERATIONS; ++i) - { - generateFixedPointXorValues(&values[0], &output[0]); - nn.feedForward(&values[0]); - error = nn.calculateError(&output[0]); - - if (!NNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) - { - nn.trainNetwork(&output[0]); - } - - errors.push_front(error.getValue()); - if (errors.size() > NUM_SAMPLES_AVG_ERROR) - { - errors.pop_back(); - } - } - - const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); - const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); - BOOST_TEST(averageError <= ERROR_LIMIT); -} - -// ============================================================ -// Conv1D + Pool1D integration test -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_conv1d_maxpool1d_pipeline) -{ - // Conv1D(8 input, kernel=3) -> MaxPool1D(6 input, pool=2) - tinymind::Conv1D conv; - typedef tinymind::MaxPool1D PoolType; - PoolType pool; - - // Set filter 0: [1, 1, 1], bias=0 (moving sum) - conv.setFilterWeight(0, 0, 1.0); - conv.setFilterWeight(0, 1, 1.0); - conv.setFilterWeight(0, 2, 1.0); - conv.setFilterWeight(0, 3, 0.0); - - // Set filter 1: [1, 0, -1], bias=0 (edge detect) - conv.setFilterWeight(1, 0, 1.0); - conv.setFilterWeight(1, 1, 0.0); - conv.setFilterWeight(1, 2, -1.0); - conv.setFilterWeight(1, 3, 0.0); - - double input[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}; - double convOutput[12]; // 2 filters * 6 positions - double poolOutput[6]; // 2 channels * 3 positions - - conv.forward(input, convOutput); - - // Filter 0 (sum): [3, 6, 9, 12, 15, 18] - BOOST_TEST(fabs(convOutput[0] - 3.0) < 0.001); - BOOST_TEST(fabs(convOutput[5] - 18.0) < 0.001); - - pool.forward(convOutput, poolOutput); - - // Pool ch0: [3,6]->6, [9,12]->12, [15,18]->18 - BOOST_TEST(fabs(poolOutput[0] - 6.0) < 0.001); - BOOST_TEST(fabs(poolOutput[1] - 12.0) < 0.001); - BOOST_TEST(fabs(poolOutput[2] - 18.0) < 0.001); -} - -// ============================================================ -// BatchNorm1D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_normalizes_output) -{ - // BatchNorm should normalize input to approximately zero mean - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - double input[4] = {2.0, 4.0, 6.0, 8.0}; - double output[4]; - - bn.forward(input, output); - - // Mean of normalized output should be ~0 - double mean = 0.0; - for (size_t i = 0; i < 4; ++i) - { - mean += output[i]; - } - mean /= 4.0; - BOOST_TEST(fabs(mean) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_unit_variance) -{ - // With default gamma=1, beta=0, output should have unit variance - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - double input[4] = {1.0, 3.0, 5.0, 7.0}; - double output[4]; - - bn.forward(input, output); - - // Compute variance of output - double mean = 0.0; - for (size_t i = 0; i < 4; ++i) - { - mean += output[i]; - } - mean /= 4.0; - - double variance = 0.0; - for (size_t i = 0; i < 4; ++i) - { - const double diff = output[i] - mean; - variance += diff * diff; - } - variance /= 4.0; - - // Variance should be ~1.0 (normalized) - BOOST_TEST(fabs(variance - 1.0) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_gamma_beta) -{ - // Custom gamma and beta should scale and shift the normalized output - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - // Set gamma=2, beta=3 for all elements - for (size_t i = 0; i < 4; ++i) - { - bn.setGamma(i, 2.0); - bn.setBeta(i, 3.0); - } - - double input[4] = {1.0, 3.0, 5.0, 7.0}; - double output[4]; - - bn.forward(input, output); - - // Output mean should be ~3 (beta) since normalized mean is 0 - double mean = 0.0; - for (size_t i = 0; i < 4; ++i) - { - mean += output[i]; - } - mean /= 4.0; - BOOST_TEST(fabs(mean - 3.0) < 0.01); - - // Output variance should be ~4 (gamma^2 * 1.0) - double variance = 0.0; - for (size_t i = 0; i < 4; ++i) - { - const double diff = output[i] - mean; - variance += diff * diff; - } - variance /= 4.0; - BOOST_TEST(fabs(variance - 4.0) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_inference_uses_running_stats) -{ - // After training, inference should use running mean/variance - tinymind::BatchNorm1D bn; // momentum=100% so running stats match batch stats immediately - - double input[4] = {2.0, 4.0, 6.0, 8.0}; - double trainOutput[4]; - double inferOutput[4]; - - // Training forward pass to populate running stats - bn.setTraining(true); - bn.forward(input, trainOutput); - - // Switch to inference - bn.setTraining(false); - bn.forward(input, inferOutput); - - // With 100% momentum, running stats equal batch stats, so outputs should match - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(fabs(trainOutput[i] - inferOutput[i]) < 0.01); - } -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_constant_input) -{ - // Constant input should produce zero-centered output (all zeros with default gamma=1, beta=0) - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - double input[4] = {5.0, 5.0, 5.0, 5.0}; - double output[4]; - - bn.forward(input, output); - - // All inputs equal -> variance ~0 -> normalized is ~0 -> output is ~beta=0 - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(fabs(output[i]) < 0.1); - } -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_backward_gradients) -{ - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - double input[4] = {1.0, 2.0, 3.0, 4.0}; - double output[4]; - bn.forward(input, output); - - // Backward with uniform gradients - double outputDeltas[4] = {1.0, 1.0, 1.0, 1.0}; - double inputDeltas[4]; - bn.backward(outputDeltas, inputDeltas); - - // Beta gradient should be sum of outputDeltas = 4.0 total, 1.0 per element - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(fabs(bn.getBetaGradient(i) - 1.0) < 0.001); - } - - // Gamma gradient = outputDelta * normalized, and normalized values - // should have zero mean, so gamma gradients should sum to ~0 - double gammaGradSum = 0.0; - for (size_t i = 0; i < 4; ++i) - { - gammaGradSum += bn.getGammaGradient(i); - } - BOOST_TEST(fabs(gammaGradSum) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_mode_toggle) -{ - tinymind::BatchNorm1D bn; - - BOOST_TEST(bn.isTraining() == true); // default is training - bn.setTraining(false); - BOOST_TEST(bn.isTraining() == false); - bn.setTraining(true); - BOOST_TEST(bn.isTraining() == true); -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_update_parameters) -{ - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - double input[4] = {1.0, 2.0, 3.0, 4.0}; - double output[4]; - bn.forward(input, output); - - double outputDeltas[4] = {1.0, 1.0, 1.0, 1.0}; - double inputDeltas[4]; - bn.backward(outputDeltas, inputDeltas); - - // Update with a learning rate - bn.updateParameters(0.01); - - // Gamma should have changed (gradient was non-zero for at least some elements) - // Beta should have changed (gradient was 1.0 for all elements) - bool betaChanged = false; - for (size_t i = 0; i < 4; ++i) - { - if (fabs(bn.getBeta(i)) > 0.001) - { - betaChanged = true; - } - } - BOOST_TEST(betaChanged); - - // Gradients should be zeroed after update - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(fabs(bn.getGammaGradient(i)) < 0.001); - BOOST_TEST(fabs(bn.getBetaGradient(i)) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_square_root_qvalue_perfect_squares) -{ - // Integer Newton's method must produce exact results on perfect squares. - typedef tinymind::QValue<8, 8, true> Q88; - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(0, 0)).getValue() - == 0); - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(1, 0)).getValue() - == Q88(1, 0).getValue()); - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(4, 0)).getValue() - == Q88(2, 0).getValue()); - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(9, 0)).getValue() - == Q88(3, 0).getValue()); - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(16, 0)).getValue() - == Q88(4, 0).getValue()); - - typedef tinymind::QValue<16, 16, true> Q1616; - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q1616(100, 0)).getValue() - == Q1616(10, 0).getValue()); - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q1616(10000, 0)).getValue() - == Q1616(100, 0).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_square_root_qvalue_non_squares) -{ - // Compare integer-sqrt result against the float reference within - // one raw-LSB tolerance (truncation rounding). - typedef tinymind::QValue<16, 16, true> ValueType; - const double inputs[] = { 0.25, 0.5, 2.0, 3.0, 0.01, 0.1, 1.5, 0.75 }; - for (size_t i = 0; i < sizeof(inputs) / sizeof(inputs[0]); ++i) - { - const ValueType v = tinymind::ValueConverter::convertToDestinationType(inputs[i]); - const ValueType result = tinymind::SquareRootApproximation::sqrt(v); - const double resultD = static_cast(result.getValue()) - / static_cast(1ULL << ValueType::NumberOfFractionalBits); - const double expected = std::sqrt(inputs[i]); - // Up to ~3 Q-LSBs of error: input quantization (truncation) compounds - // with the integer-sqrt truncation. Use 4 LSBs as a safe tolerance. - BOOST_TEST(std::fabs(resultD - expected) < 4.0 / (1ULL << ValueType::NumberOfFractionalBits)); - } -} - -BOOST_AUTO_TEST_CASE(test_case_square_root_qvalue_clamps_negative) -{ - // Negative inputs must clamp to zero, not invoke std::sqrt(NaN). - typedef tinymind::QValue<8, 8, true> Q88; - Q88 negative; - negative = static_cast(-256); // -1.0 - BOOST_TEST(tinymind::SquareRootApproximation::sqrt(negative).getValue() == 0); -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_qvalue_hyperparam_helpers) -{ - // BatchNorm with QValue must derive epsilon/momentum/sizeValue without - // going through ValueConverter. This catches a bug we - // had where gating the float specializations in nnproperties.hpp left - // batchnorm with a silently-zero epsilon (then SIGFPE on sqrt(0+0)). - typedef tinymind::QValue<16, 16, true> ValueType; - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - ValueType input[4]; - input[0] = ValueType(0, 0); - input[1] = ValueType(0, 0); - input[2] = ValueType(0, 0); - input[3] = ValueType(0, 0); - - ValueType output[4]; - // All-zero input -> mean=0, variance=0. Without a non-zero epsilon - // we'd hit a divide-by-zero in invStd. The helper must produce - // epsilon ~ 0.01 for EpsilonPercent=1. - bn.forward(input, output); - - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(output[i].getValue() == 0); - } -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_fixed_point) -{ - // Verify BatchNorm compiles and runs with Q16.16 fixed-point - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - tinymind::BatchNorm1D bn; - bn.setTraining(true); - - ValueType input[4]; - input[0] = ValueType(1, 0); - input[1] = ValueType(3, 0); - input[2] = ValueType(5, 0); - input[3] = ValueType(7, 0); - - ValueType output[4]; - bn.forward(input, output); - - // Normalized output should have mean ~0 - // Sum the raw values - they should roughly cancel out - typename ValueType::FullWidthValueType sum = 0; - for (size_t i = 0; i < 4; ++i) - { - sum += output[i].getValue(); - } - // Allow tolerance for fixed-point rounding - const typename ValueType::FullWidthValueType tolerance = 1 << (ValueType::NumberOfFractionalBits - 2); - BOOST_TEST(std::abs(sum) <= tolerance); -} - -BOOST_AUTO_TEST_CASE(test_case_batchnorm_conv1d_pipeline) -{ - // Conv1D -> BatchNorm -> output pipeline - tinymind::Conv1D conv; - tinymind::BatchNorm1D bn; // Conv output is (8-3)/1+1 = 6 - - // Set kernel to [1, 1, 1], bias=0 (moving sum) - conv.setFilterWeight(0, 0, 1.0); - conv.setFilterWeight(0, 1, 1.0); - conv.setFilterWeight(0, 2, 1.0); - conv.setFilterWeight(0, 3, 0.0); - - double input[8] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}; - double convOutput[6]; - double bnOutput[6]; - - conv.forward(input, convOutput); - // Conv output: [6, 9, 12, 15, 18, 21] - - bn.setTraining(true); - bn.forward(convOutput, bnOutput); - - // BatchNorm output should have zero mean - double mean = 0.0; - for (size_t i = 0; i < 6; ++i) - { - mean += bnOutput[i]; - } - mean /= 6.0; - BOOST_TEST(fabs(mean) < 0.01); -} - -// ============================================================ -// BinaryDense tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_forward_basic) -{ - // 4 inputs, 2 outputs - tinymind::BinaryDense layer; - - // Set latent weights: output 0 = [+1, +1, -1, -1], output 1 = [+1, -1, +1, -1] - layer.setLatentWeight(0, 0, 0.5); - layer.setLatentWeight(0, 1, 0.3); - layer.setLatentWeight(0, 2, -0.7); - layer.setLatentWeight(0, 3, -0.2); - - layer.setLatentWeight(1, 0, 0.9); - layer.setLatentWeight(1, 1, -0.4); - layer.setLatentWeight(1, 2, 0.6); - layer.setLatentWeight(1, 3, -0.1); - - layer.setBias(0, 0.0); - layer.setBias(1, 0.0); - - layer.binarizeWeights(); - - // Verify binarization: positive latent -> +1, negative latent -> -1 - BOOST_TEST(layer.getBinaryWeight(0, 0) == 1.0); - BOOST_TEST(layer.getBinaryWeight(0, 1) == 1.0); - BOOST_TEST(layer.getBinaryWeight(0, 2) == -1.0); - BOOST_TEST(layer.getBinaryWeight(0, 3) == -1.0); - - BOOST_TEST(layer.getBinaryWeight(1, 0) == 1.0); - BOOST_TEST(layer.getBinaryWeight(1, 1) == -1.0); - BOOST_TEST(layer.getBinaryWeight(1, 2) == 1.0); - BOOST_TEST(layer.getBinaryWeight(1, 3) == -1.0); - - // Input: [1.0, -1.0, 1.0, -1.0] => sign = [+1, -1, +1, -1] - double input[4] = {1.0, -1.0, 1.0, -1.0}; - double output[2]; - layer.forward(input, output); - - // output[0]: sign(input) dot weights[0] = (+1)(+1) + (-1)(+1) + (+1)(-1) + (-1)(-1) - // = 1 - 1 - 1 + 1 = 0 - BOOST_TEST(fabs(output[0] - 0.0) < 0.01); - - // output[1]: sign(input) dot weights[1] = (+1)(+1) + (-1)(-1) + (+1)(+1) + (-1)(-1) - // = 1 + 1 + 1 + 1 = 4 - BOOST_TEST(fabs(output[1] - 4.0) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_forward_with_bias) -{ - tinymind::BinaryDense layer; - - // All positive latent weights => binary weights all +1 - layer.setLatentWeight(0, 0, 0.5); - layer.setLatentWeight(0, 1, 0.5); - layer.setLatentWeight(0, 2, 0.5); - layer.setBias(0, 2.0); - layer.binarizeWeights(); - - // Input: [1.0, 1.0, 1.0] => sign = [+1, +1, +1] - // dot product = 3, plus bias 2 = 5 - double input[3] = {1.0, 1.0, 1.0}; - double output[1]; - layer.forward(input, output); - - BOOST_TEST(fabs(output[0] - 5.0) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_static_sizes) -{ - typedef tinymind::BinaryDense BinType; - - // 64 * 16 = 1024 binary weights - static_assert(BinType::TotalBinaryWeights == 1024, "Wrong total binary weights"); - // 1024 / 32 = 32 packed words - static_assert(BinType::PackedWeightWords == 32, "Wrong packed weight words"); - // 64 / 32 = 2 packed input words - static_assert(BinType::PackedInputWords == 2, "Wrong packed input words"); - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_packed_storage) -{ - // Verify that packed weights use minimal storage - typedef tinymind::BinaryDense BinType; - - // 33 bits needs 2 words (ceil(33/32) = 2) - static_assert(BinType::PackedWeightWords == 2, "Wrong packed words for 33 bits"); - static_assert(BinType::PackedInputWords == 2, "Wrong packed input words for 33 bits"); - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_backward_ste) -{ - tinymind::BinaryDense layer; - - // Set latent weights within [-1, 1] - layer.setLatentWeight(0, 0, 0.5); - layer.setLatentWeight(0, 1, -0.3); - layer.setBias(0, 0.0); - layer.binarizeWeights(); - - double input[2] = {1.0, -1.0}; - double output[1]; - layer.forward(input, output); - - double outputDeltas[1] = {1.0}; - double inputDeltas[2]; - layer.backward(outputDeltas, input, inputDeltas); - - // Learning rate = -0.1 (negative because gradient descent) - layer.updateWeights(-0.1); - - // Latent weights should have been updated - // w0: 0.5 + (-0.1) * (1.0 * sign(1.0)) = 0.5 - 0.1 = 0.4 - // w1: -0.3 + (-0.1) * (1.0 * sign(-1.0)) = -0.3 + 0.1 = -0.2 - BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - 0.4) < 0.01); - BOOST_TEST(fabs(layer.getLatentWeight(0, 1) - (-0.2)) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_updateweights_sign_flip) -{ - // updateWeights() must call binarizeWeights() so that a latent weight - // crossing zero immediately flips the packed weight sign on the next - // forward pass. - tinymind::BinaryDense layer; - layer.setLatentWeight(0, 0, 0.05); // small positive - layer.setBias(0, 0.0); - layer.binarizeWeights(); - BOOST_TEST(layer.getBinaryWeight(0, 0) == 1.0); - - // Run backward to populate gradient. With input=+1 and outputDelta=+1, - // mLatentGradients[0] = +1.0 (within STE clip range). - double input[1] = {1.0}; - double output[1]; - layer.forward(input, output); - double outputDeltas[1] = {1.0}; - layer.backward(outputDeltas, input, nullptr); - - // lr = -0.1 pushes latent to 0.05 - 0.1 = -0.05; re-binarize to -1. - layer.updateWeights(-0.1); - BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - (-0.05)) < 1e-9); - BOOST_TEST(layer.getBinaryWeight(0, 0) == -1.0); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_ste_clipping) -{ - tinymind::BinaryDense layer; - - // Set one latent weight outside [-1, 1] — STE should zero its gradient - layer.setLatentWeight(0, 0, 1.5); // outside [-1, 1] - layer.setLatentWeight(0, 1, 0.5); // inside [-1, 1] - layer.setBias(0, 0.0); - layer.binarizeWeights(); - - double input[2] = {1.0, 1.0}; - double output[1]; - layer.forward(input, output); - - double outputDeltas[1] = {1.0}; - layer.backward(outputDeltas, input, nullptr); - layer.updateWeights(-0.1); - - // w0 at 1.5 is outside [-1,1], gradient clipped to 0 => stays at 1.5 - BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - 1.5) < 0.01); - // w1 at 0.5 is inside [-1,1], gradient applies => 0.5 + (-0.1)(1.0*1.0) = 0.4 - BOOST_TEST(fabs(layer.getLatentWeight(0, 1) - 0.4) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_fixed_point) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::BinaryDense layer; - - // Set latent weights using ValueConverter - typedef tinymind::ValueConverter Conv; - layer.setLatentWeight(0, 0, Conv::convertToDestinationType(0.5)); - layer.setLatentWeight(0, 1, Conv::convertToDestinationType(-0.5)); - layer.setLatentWeight(0, 2, Conv::convertToDestinationType(0.5)); - layer.setLatentWeight(0, 3, Conv::convertToDestinationType(-0.5)); - layer.setBias(0, Conv::convertToDestinationType(0.0)); - layer.binarizeWeights(); - - // Binary weights should be [+1, -1, +1, -1] - ValueType posOne = Conv::convertToDestinationType(1.0); - ValueType negOne = Conv::convertToDestinationType(-1.0); - BOOST_TEST(layer.getBinaryWeight(0, 0).getValue() == posOne.getValue()); - BOOST_TEST(layer.getBinaryWeight(0, 1).getValue() == negOne.getValue()); - BOOST_TEST(layer.getBinaryWeight(0, 2).getValue() == posOne.getValue()); - BOOST_TEST(layer.getBinaryWeight(0, 3).getValue() == negOne.getValue()); - - // Input: all +1 => sign = [+1, +1, +1, +1] - // dot = (+1)(+1) + (+1)(-1) + (+1)(+1) + (+1)(-1) = 0 - ValueType input[4]; - input[0] = Conv::convertToDestinationType(1.0); - input[1] = Conv::convertToDestinationType(1.0); - input[2] = Conv::convertToDestinationType(1.0); - input[3] = Conv::convertToDestinationType(1.0); - - ValueType output[1]; - layer.forward(input, output); - - ValueType expected = Conv::convertToDestinationType(0.0); - BOOST_TEST(output[0].getValue() == expected.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_binary_dense_input_gradient_propagation) -{ - tinymind::BinaryDense layer; - - layer.setLatentWeight(0, 0, 0.5); - layer.setLatentWeight(0, 1, -0.5); - layer.setLatentWeight(0, 2, 0.5); - - layer.setLatentWeight(1, 0, -0.5); - layer.setLatentWeight(1, 1, 0.5); - layer.setLatentWeight(1, 2, -0.5); - - layer.setBias(0, 0.0); - layer.setBias(1, 0.0); - layer.binarizeWeights(); - - double input[3] = {1.0, 1.0, 1.0}; - double output[2]; - layer.forward(input, output); - - // output deltas both = 1.0 - double outputDeltas[2] = {1.0, 1.0}; - double inputDeltas[3]; - layer.backward(outputDeltas, input, inputDeltas); - - // inputDeltas[0] = delta0 * sign(w[0][0]) + delta1 * sign(w[1][0]) - // = 1.0 * (+1) + 1.0 * (-1) = 0 - BOOST_TEST(fabs(inputDeltas[0] - 0.0) < 0.01); - // inputDeltas[1] = 1.0 * (-1) + 1.0 * (+1) = 0 - BOOST_TEST(fabs(inputDeltas[1] - 0.0) < 0.01); - // inputDeltas[2] = 1.0 * (+1) + 1.0 * (-1) = 0 - BOOST_TEST(fabs(inputDeltas[2] - 0.0) < 0.01); -} - -// ============================================================ -// TernaryDense tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_forward_basic) -{ - // 4 inputs, 2 outputs, threshold 50% - tinymind::TernaryDense layer; - - // Set latent weights with clear magnitudes - // Large values => +1 or -1, small values near 0 => 0 - layer.setLatentWeight(0, 0, 0.9); // should be +1 - layer.setLatentWeight(0, 1, 0.01); // should be 0 (below threshold) - layer.setLatentWeight(0, 2, -0.8); // should be -1 - layer.setLatentWeight(0, 3, 0.02); // should be 0 (below threshold) - - layer.setLatentWeight(1, 0, -0.7); // should be -1 - layer.setLatentWeight(1, 1, 0.85); // should be +1 - layer.setLatentWeight(1, 2, 0.03); // should be 0 (below threshold) - layer.setLatentWeight(1, 3, -0.9); // should be -1 - - layer.setBias(0, 0.0); - layer.setBias(1, 0.0); - layer.ternarizeWeights(); - - // Verify ternarization - // Mean |w| = (0.9+0.01+0.8+0.02+0.7+0.85+0.03+0.9)/8 = 4.21/8 = 0.52625 - // threshold = 0.5 * 0.52625 = 0.263125 - // w[0][0]=0.9 > 0.263 => +1 - // w[0][1]=0.01 < 0.263 => 0 - // w[0][2]=-0.8, |0.8| > 0.263 => -1 - // w[0][3]=0.02 < 0.263 => 0 - BOOST_TEST(layer.getTernaryWeight(0, 0) == tinymind::detail::TERNARY_POS); - BOOST_TEST(layer.getTernaryWeight(0, 1) == tinymind::detail::TERNARY_ZERO); - BOOST_TEST(layer.getTernaryWeight(0, 2) == tinymind::detail::TERNARY_NEG); - BOOST_TEST(layer.getTernaryWeight(0, 3) == tinymind::detail::TERNARY_ZERO); - - BOOST_TEST(layer.getTernaryWeight(1, 0) == tinymind::detail::TERNARY_NEG); - BOOST_TEST(layer.getTernaryWeight(1, 1) == tinymind::detail::TERNARY_POS); - BOOST_TEST(layer.getTernaryWeight(1, 2) == tinymind::detail::TERNARY_ZERO); - BOOST_TEST(layer.getTernaryWeight(1, 3) == tinymind::detail::TERNARY_NEG); - - // getTernaryWeightValue maps the packed code to +1 / 0 / -1; the - // dead-zone weights exercise the zero() return. - BOOST_TEST(layer.getTernaryWeightValue(0, 0) == 1.0); - BOOST_TEST(layer.getTernaryWeightValue(0, 1) == 0.0); - BOOST_TEST(layer.getTernaryWeightValue(0, 2) == -1.0); - BOOST_TEST(layer.getTernaryWeightValue(0, 3) == 0.0); - - // Input: [2.0, 3.0, 4.0, 5.0] - double input[4] = {2.0, 3.0, 4.0, 5.0}; - double output[2]; - layer.forward(input, output); - - // output[0] = (+1)*2 + 0*3 + (-1)*4 + 0*5 = 2 - 4 = -2 - BOOST_TEST(fabs(output[0] - (-2.0)) < 0.01); - - // output[1] = (-1)*2 + (+1)*3 + 0*4 + (-1)*5 = -2 + 3 - 5 = -4 - BOOST_TEST(fabs(output[1] - (-4.0)) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_forward_with_bias) -{ - tinymind::TernaryDense layer; - - // Both large positive => both +1 - layer.setLatentWeight(0, 0, 0.9); - layer.setLatentWeight(0, 1, 0.8); - layer.setBias(0, 1.5); - layer.ternarizeWeights(); - - double input[2] = {3.0, 4.0}; - double output[1]; - layer.forward(input, output); - - // output = (+1)*3 + (+1)*4 + 1.5 = 8.5 - BOOST_TEST(fabs(output[0] - 8.5) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_static_sizes) -{ - typedef tinymind::TernaryDense TernType; - - static_assert(TernType::TotalTernaryWeights == 1024, "Wrong total ternary weights"); - // 1024 ternary values, 16 per word = 64 words - static_assert(TernType::PackedWeightWords == 64, "Wrong packed weight words"); - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_all_zero_threshold) -{ - // With very high threshold, most weights should become 0 - tinymind::TernaryDense layer; - - layer.setLatentWeight(0, 0, 0.5); - layer.setLatentWeight(0, 1, -0.5); - layer.setLatentWeight(0, 2, 0.5); - layer.setLatentWeight(0, 3, -0.5); - layer.setBias(0, 0.0); - layer.ternarizeWeights(); - - // Mean |w| = 0.5, threshold = 0.99 * 0.5 = 0.495 - // All |w| = 0.5 > 0.495, so all should still be non-zero - // (need truly small weights relative to mean for zeroing) - double input[4] = {1.0, 1.0, 1.0, 1.0}; - double output[1]; - layer.forward(input, output); - - // With very uniform weights, threshold 99% of mean still passes - // output = (+1)*1 + (-1)*1 + (+1)*1 + (-1)*1 = 0 - BOOST_TEST(fabs(output[0]) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_backward_ste) -{ - tinymind::TernaryDense layer; - - layer.setLatentWeight(0, 0, 0.8); - layer.setLatentWeight(0, 1, -0.6); - layer.setBias(0, 0.0); - layer.ternarizeWeights(); - - double input[2] = {2.0, 3.0}; - double output[1]; - layer.forward(input, output); - - double outputDeltas[1] = {1.0}; - double inputDeltas[2]; - layer.backward(outputDeltas, input, inputDeltas); - layer.updateWeights(-0.1); - - // Gradient for w0: delta * input[0] = 1.0 * 2.0 = 2.0 - // w0: 0.8 + (-0.1)(2.0) = 0.6 - BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - 0.6) < 0.01); - - // Gradient for w1: delta * input[1] = 1.0 * 3.0 = 3.0 - // w1: -0.6 + (-0.1)(3.0) = -0.9 - BOOST_TEST(fabs(layer.getLatentWeight(0, 1) - (-0.9)) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_updateweights_sign_flip) -{ - // Verify updateWeights re-ternarizes after a latent weight crosses zero. - // With a single weight the threshold collapses to 0.5*|w|, so any nonzero - // latent maps to its sign. Initial latent +0.5 -> +1; gradient pushes it - // negative, packed value should flip to -1 immediately. - tinymind::TernaryDense layer; - layer.setLatentWeight(0, 0, 0.5); - layer.setBias(0, 0.0); - layer.ternarizeWeights(); - BOOST_TEST(layer.getTernaryWeightValue(0, 0) == 1.0); - - double input[1] = {2.0}; - double output[1]; - layer.forward(input, output); - double outputDeltas[1] = {1.0}; - layer.backward(outputDeltas, input, nullptr); - // Gradient = delta * input = 2.0; lr = -0.4 -> latent becomes 0.5 - 0.8 = -0.3. - layer.updateWeights(-0.4); - BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - (-0.3)) < 1e-6); - BOOST_TEST(layer.getTernaryWeightValue(0, 0) == -1.0); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_input_gradient_propagation) -{ - tinymind::TernaryDense layer; - - // All large => all non-zero ternary - layer.setLatentWeight(0, 0, 0.9); // +1 - layer.setLatentWeight(0, 1, -0.8); // -1 - layer.setLatentWeight(0, 2, 0.7); // +1 - layer.setBias(0, 0.0); - layer.ternarizeWeights(); - - double input[3] = {1.0, 2.0, 3.0}; - double output[1]; - layer.forward(input, output); - - double outputDeltas[1] = {2.0}; - double inputDeltas[3]; - layer.backward(outputDeltas, input, inputDeltas); - - // inputDeltas[0] = delta * ternary(w[0][0]) = 2.0 * (+1) = 2.0 - BOOST_TEST(fabs(inputDeltas[0] - 2.0) < 0.01); - // inputDeltas[1] = delta * ternary(w[0][1]) = 2.0 * (-1) = -2.0 - BOOST_TEST(fabs(inputDeltas[1] - (-2.0)) < 0.01); - // inputDeltas[2] = delta * ternary(w[0][2]) = 2.0 * (+1) = 2.0 - BOOST_TEST(fabs(inputDeltas[2] - 2.0) < 0.01); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_fixed_point) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - typedef tinymind::ValueConverter Conv; - tinymind::TernaryDense layer; - - layer.setLatentWeight(0, 0, Conv::convertToDestinationType(0.9)); - layer.setLatentWeight(0, 1, Conv::convertToDestinationType(-0.8)); - layer.setLatentWeight(0, 2, Conv::convertToDestinationType(0.7)); - layer.setLatentWeight(0, 3, Conv::convertToDestinationType(-0.6)); - layer.setBias(0, Conv::convertToDestinationType(0.0)); - layer.ternarizeWeights(); - - // All weights are large magnitude => all non-zero ternary - BOOST_TEST(layer.getTernaryWeight(0, 0) == tinymind::detail::TERNARY_POS); - BOOST_TEST(layer.getTernaryWeight(0, 1) == tinymind::detail::TERNARY_NEG); - BOOST_TEST(layer.getTernaryWeight(0, 2) == tinymind::detail::TERNARY_POS); - BOOST_TEST(layer.getTernaryWeight(0, 3) == tinymind::detail::TERNARY_NEG); - - // Input: [2, 1, 3, 2] - // output = (+1)*2 + (-1)*1 + (+1)*3 + (-1)*2 = 2 - 1 + 3 - 2 = 2 - ValueType input[4]; - input[0] = Conv::convertToDestinationType(2.0); - input[1] = Conv::convertToDestinationType(1.0); - input[2] = Conv::convertToDestinationType(3.0); - input[3] = Conv::convertToDestinationType(2.0); - - ValueType output[1]; - layer.forward(input, output); - - ValueType expected = Conv::convertToDestinationType(2.0); - BOOST_TEST(output[0].getValue() == expected.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_ternary_dense_get_value_accessors) -{ - tinymind::TernaryDense layer; - - layer.setLatentWeight(0, 0, 0.9); - layer.setLatentWeight(0, 1, -0.8); - layer.setBias(0, 0.0); - layer.ternarizeWeights(); - - BOOST_TEST(fabs(layer.getTernaryWeightValue(0, 0) - 1.0) < 0.01); - BOOST_TEST(fabs(layer.getTernaryWeightValue(0, 1) - (-1.0)) < 0.01); -} - -// ============================================================ -// SelfAttention1D tests (floating-point) -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_static_sizes) -{ - typedef tinymind::SelfAttention1D AttnType; - - static_assert(AttnType::InputSize == 128, "Wrong input size"); // 16 * 8 - static_assert(AttnType::OutputSize == 64, "Wrong output size"); // 16 * 4 - static_assert(AttnType::WeightsPerProjection == 32, "Wrong weights per proj"); // 8 * 4 - static_assert(AttnType::TotalWeights == 96, "Wrong total weights"); // 3 * 32 - static_assert(AttnType::BiasesPerProjection == 4, "Wrong biases per proj"); - static_assert(AttnType::TotalBiases == 12, "Wrong total biases"); // 3 * 4 - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_identity_projection) -{ - // With identity-like projections and no ReLU clipping, - // verify output is non-zero and has expected dimensions. - // N=4, D=2, P=2 - tinymind::SelfAttention1D attn; - - // Set W_q = W_k = W_v = identity matrix - for (size_t proj = 0; proj < 3; ++proj) - { - for (size_t r = 0; r < 2; ++r) - { - for (size_t c = 0; c < 2; ++c) - { - attn.setProjectionWeight(proj, r, c, (r == c) ? 1.0 : 0.0); - } - } - } - - // Input: 4 time steps x 2 features, all positive (so ReLU passes through) - double input[8] = { - 1.0, 2.0, // t0 - 3.0, 4.0, // t1 - 5.0, 6.0, // t2 - 7.0, 8.0 // t3 - }; - double output[8]; // 4 x 2 - - attn.forward(input, output); - - // With identity projections on positive input: - // Q' = K' = V = X (ReLU is pass-through for positive values) - // KV = X^T * X (2x2 gram matrix) - // Out = X * KV = X * (X^T * X) - // Verify output is non-zero - bool anyNonZero = false; - for (size_t i = 0; i < 8; ++i) - { - if (fabs(output[i]) > 0.001) - { - anyNonZero = true; - break; - } - } - BOOST_TEST(anyNonZero); - - // Manually compute expected output: - // X^T * X = [[1+9+25+49, 2+12+30+56], [2+12+30+56, 4+16+36+64]] - // = [[84, 100], [100, 120]] - // Out[0] = X[0] * KV = [1,2] * [[84,100],[100,120]] = [284, 340] - BOOST_TEST(fabs(output[0] - 284.0) < 0.001); - BOOST_TEST(fabs(output[1] - 340.0) < 0.001); - - // Out[1] = [3,4] * [[84,100],[100,120]] = [652, 780] - BOOST_TEST(fabs(output[2] - 652.0) < 0.001); - BOOST_TEST(fabs(output[3] - 780.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_relu_clipping) -{ - // Verify ReLU zeroes out negative Q and K projections - // N=2, D=2, P=2 - tinymind::SelfAttention1D attn; - - // W_q = [[-1, 0], [0, -1]] (negates input -> ReLU clips to zero) - attn.setProjectionWeight(0, 0, 0, -1.0); - attn.setProjectionWeight(0, 0, 1, 0.0); - attn.setProjectionWeight(0, 1, 0, 0.0); - attn.setProjectionWeight(0, 1, 1, -1.0); - - // W_k = identity - attn.setProjectionWeight(1, 0, 0, 1.0); - attn.setProjectionWeight(1, 0, 1, 0.0); - attn.setProjectionWeight(1, 1, 0, 0.0); - attn.setProjectionWeight(1, 1, 1, 1.0); - - // W_v = identity - attn.setProjectionWeight(2, 0, 0, 1.0); - attn.setProjectionWeight(2, 0, 1, 0.0); - attn.setProjectionWeight(2, 1, 0, 0.0); - attn.setProjectionWeight(2, 1, 1, 1.0); - - double input[4] = {1.0, 2.0, 3.0, 4.0}; - double output[4]; - - attn.forward(input, output); - - // Q = ReLU([[-1,0],[0,-1]] * input) = ReLU(negative) = 0 - // Output = Q' * KV = 0 for all - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(fabs(output[i]) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_bias) -{ - // Verify biases are applied to projections - // N=2, D=2, P=2, zero weights, positive Q bias to produce non-zero output - tinymind::SelfAttention1D attn; - - // All weights zero (default) - // Set Q bias to positive values so Q' = ReLU(0 + bias) = bias - attn.setProjectionBias(0, 0, 1.0); - attn.setProjectionBias(0, 1, 1.0); - - // Set K bias so K' has positive values - attn.setProjectionBias(1, 0, 1.0); - attn.setProjectionBias(1, 1, 1.0); - - // Set V bias - attn.setProjectionBias(2, 0, 2.0); - attn.setProjectionBias(2, 1, 3.0); - - double input[4] = {0.0, 0.0, 0.0, 0.0}; - double output[4]; - - attn.forward(input, output); - - // Q' = [1,1] for each time step (from bias, ReLU pass-through) - // K' = [1,1] for each time step - // V = [2,3] for each time step - // KV = K'^T * V = [[1,1],[1,1]]^T (2x2) * [[2,3],[2,3]] (2x2) - // K'^T is (2x2): col0=[1,1], col1=[1,1] - // KV[0][0] = K'[0,0]*V[0,0] + K'[1,0]*V[1,0] = 1*2 + 1*2 = 4 - // KV[0][1] = K'[0,0]*V[0,1] + K'[1,0]*V[1,1] = 1*3 + 1*3 = 6 - // KV[1][0] = K'[0,1]*V[0,0] + K'[1,1]*V[1,0] = 1*2 + 1*2 = 4 - // KV[1][1] = K'[0,1]*V[0,1] + K'[1,1]*V[1,1] = 1*3 + 1*3 = 6 - // Out = Q' * KV: - // Out[0] = [1,1] * [[4,6],[4,6]] = [8, 12] - BOOST_TEST(fabs(output[0] - 8.0) < 0.001); - BOOST_TEST(fabs(output[1] - 12.0) < 0.001); - BOOST_TEST(fabs(output[2] - 8.0) < 0.001); - BOOST_TEST(fabs(output[3] - 12.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_weight_accessors) -{ - tinymind::SelfAttention1D attn; - - // Set and verify projection weights - attn.setProjectionWeight(0, 1, 0, 3.14); - BOOST_TEST(fabs(attn.getProjectionWeight(0, 1, 0) - 3.14) < 0.001); - - attn.setProjectionWeight(2, 2, 1, -1.5); - BOOST_TEST(fabs(attn.getProjectionWeight(2, 2, 1) - (-1.5)) < 0.001); - - // Verify flat accessor matches structured accessor - // proj=0, row=1, col=0 -> flat index = 0*6 + 1*2 + 0 = 2 - BOOST_TEST(fabs(attn.getWeight(2) - 3.14) < 0.001); - - // proj=2, row=2, col=1 -> flat index = 2*6 + 2*2 + 1 = 17 - BOOST_TEST(fabs(attn.getWeight(17) - (-1.5)) < 0.001); - - // Bias accessors - attn.setProjectionBias(1, 0, 0.5); - BOOST_TEST(fabs(attn.getProjectionBias(1, 0) - 0.5) < 0.001); - BOOST_TEST(fabs(attn.getBias(2) - 0.5) < 0.001); // flat index = 1*2 + 0 = 2 -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_gradients_nonzero) -{ - // After forward + computeGradients, weight gradients should be non-zero - tinymind::SelfAttention1D attn; - - // Identity projections - for (size_t proj = 0; proj < 3; ++proj) - { - attn.setProjectionWeight(proj, 0, 0, 1.0); - attn.setProjectionWeight(proj, 0, 1, 0.0); - attn.setProjectionWeight(proj, 1, 0, 0.0); - attn.setProjectionWeight(proj, 1, 1, 1.0); - } - - double input[8] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}; - double output[8]; - attn.forward(input, output); - - // Non-zero output deltas - double deltas[8] = {0.1, -0.1, 0.2, -0.2, 0.3, -0.3, 0.4, -0.4}; - attn.computeGradients(deltas); - - // At least some gradients should be non-zero - bool anyGradNonZero = false; - for (size_t i = 0; i < tinymind::SelfAttention1D::TotalWeights; ++i) - { - if (fabs(attn.getGradient(i)) > 1e-10) - { - anyGradNonZero = true; - break; - } - } - BOOST_TEST(anyGradNonZero); - - // Bias gradients should also be non-zero - bool anyBiasGradNonZero = false; - for (size_t i = 0; i < tinymind::SelfAttention1D::TotalBiases; ++i) - { - if (fabs(attn.getBiasGradient(i)) > 1e-10) - { - anyBiasGradNonZero = true; - break; - } - } - BOOST_TEST(anyBiasGradNonZero); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_weight_update) -{ - // Verify that updateWeights modifies weights in the correct direction - tinymind::SelfAttention1D attn; - - for (size_t proj = 0; proj < 3; ++proj) - { - attn.setProjectionWeight(proj, 0, 0, 1.0); - attn.setProjectionWeight(proj, 0, 1, 0.0); - attn.setProjectionWeight(proj, 1, 0, 0.0); - attn.setProjectionWeight(proj, 1, 1, 1.0); - } - - double input[4] = {1.0, 2.0, 3.0, 4.0}; - double output[4]; - attn.forward(input, output); - - double deltas[4] = {0.1, 0.2, 0.3, 0.4}; - attn.computeGradients(deltas); - - // Record weights before update - double wBefore = attn.getProjectionWeight(0, 0, 0); - double bBefore = attn.getProjectionBias(0, 0); - - // Negative learning rate for gradient descent - attn.updateWeights(-0.01); - - double wAfter = attn.getProjectionWeight(0, 0, 0); - double bAfter = attn.getProjectionBias(0, 0); - - // Weights should have changed - BOOST_TEST(fabs(wAfter - wBefore) > 1e-10); - BOOST_TEST(fabs(bAfter - bBefore) > 1e-10); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_conv1d_pipeline) -{ - // Conv1D -> SelfAttention1D pipeline - // Conv1D: 8 input, kernel=3, stride=1, 2 filters -> output 6*2=12 values - // Reshape as: 6 time steps x 2 features - // SelfAttention1D: N=6, D=2, P=2 - tinymind::Conv1D conv; - tinymind::SelfAttention1D attn; - - // Set conv filter 0: [1, 0, 0], bias=0 - conv.setFilterWeight(0, 0, 1.0); - conv.setFilterWeight(0, 1, 0.0); - conv.setFilterWeight(0, 2, 0.0); - conv.setFilterWeight(0, 3, 0.0); - - // Set conv filter 1: [0, 0, 1], bias=0 - conv.setFilterWeight(1, 0, 0.0); - conv.setFilterWeight(1, 1, 0.0); - conv.setFilterWeight(1, 2, 1.0); - conv.setFilterWeight(1, 3, 0.0); - - double convInput[8] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}; - double convOutput[12]; // 2 filters * 6 positions - - conv.forward(convInput, convOutput); - - // Reshape conv output (filter-major) to attention input (time-step-major) - double attnInput[12]; - for (size_t t = 0; t < 6; ++t) - { - attnInput[t * 2 + 0] = convOutput[t]; // filter 0 - attnInput[t * 2 + 1] = convOutput[6 + t]; // filter 1 - } - - // Identity projections for attention - for (size_t proj = 0; proj < 3; ++proj) - { - attn.setProjectionWeight(proj, 0, 0, 1.0); - attn.setProjectionWeight(proj, 0, 1, 0.0); - attn.setProjectionWeight(proj, 1, 0, 0.0); - attn.setProjectionWeight(proj, 1, 1, 1.0); - } - - double attnOutput[12]; // 6 * 2 - attn.forward(attnInput, attnOutput); - - // Verify output is non-zero and plausible - bool anyNonZero = false; - for (size_t i = 0; i < 12; ++i) - { - if (fabs(attnOutput[i]) > 0.001) - { - anyNonZero = true; - break; - } - } - BOOST_TEST(anyNonZero); -} - -BOOST_AUTO_TEST_CASE(test_case_self_attention_gradients_relu_derivative) -{ - // A freshly-constructed layer has zero weights, so the ReLU-kernel - // projections Q' and K' are all zero. computeGradients runs reluDerivative - // over them, exercising the activation <= 0 branch that zeroes the - // gradient. The resulting weight gradients must therefore all be zero. - tinymind::SelfAttention1D attn; - - double input[12]; - for (size_t i = 0; i < 12; ++i) input[i] = static_cast(i) - 6.0; - double output[8]; - attn.forward(input, output); - - double outputDeltas[8]; - for (size_t i = 0; i < 8; ++i) outputDeltas[i] = 1.0; - attn.computeGradients(outputDeltas); - - for (size_t i = 0; i < attn.TotalWeights; ++i) - { - BOOST_TEST(fabs(attn.getGradient(i)) < 1e-12); - } -} - -// ============================================================ -// SelfAttention1D tests (fixed-point) -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_forward) -{ - // Verify SelfAttention1D compiles and runs with Q8.8 fixed-point - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::SelfAttention1D attn; - - // Identity projections - for (size_t proj = 0; proj < 3; ++proj) - { - attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); - attn.setProjectionWeight(proj, 0, 1, ValueType(0)); - attn.setProjectionWeight(proj, 1, 0, ValueType(0)); - attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); - } - - ValueType input[4]; - input[0] = ValueType(1, 0); - input[1] = ValueType(2, 0); - input[2] = ValueType(1, 0); - input[3] = ValueType(1, 0); - - ValueType output[4]; - attn.forward(input, output); - - // Output should be non-zero (positive input through identity + ReLU) - bool anyNonZero = false; - for (size_t i = 0; i < 4; ++i) - { - if (output[i].getValue() != 0) - { - anyNonZero = true; - break; - } - } - BOOST_TEST(anyNonZero); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_relu) -{ - // Verify ReLU works correctly with Q8.8 - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::SelfAttention1D attn; - - // W_q negates (ReLU should clip to zero) - attn.setProjectionWeight(0, 0, 0, ValueType(-1, 0)); - attn.setProjectionWeight(0, 0, 1, ValueType(0)); - attn.setProjectionWeight(0, 1, 0, ValueType(0)); - attn.setProjectionWeight(0, 1, 1, ValueType(-1, 0)); - - // W_k, W_v = identity - for (size_t proj = 1; proj < 3; ++proj) - { - attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); - attn.setProjectionWeight(proj, 0, 1, ValueType(0)); - attn.setProjectionWeight(proj, 1, 0, ValueType(0)); - attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); - } - - ValueType input[4]; - input[0] = ValueType(1, 0); - input[1] = ValueType(2, 0); - input[2] = ValueType(3, 0); - input[3] = ValueType(4, 0); - - ValueType output[4]; - attn.forward(input, output); - - // Q' = ReLU(negative) = 0, so output = 0 - ValueType zero(0); - for (size_t i = 0; i < 4; ++i) - { - BOOST_TEST(output[i].getValue() == zero.getValue()); - } -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_q16_16) -{ - // Verify with Q16.16 (the recommended mid-range format) - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - tinymind::SelfAttention1D attn; - - // Identity projections - for (size_t proj = 0; proj < 3; ++proj) - { - attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); - attn.setProjectionWeight(proj, 0, 1, ValueType(0)); - attn.setProjectionWeight(proj, 1, 0, ValueType(0)); - attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); - } - - ValueType input[8]; - input[0] = ValueType(1, 0); - input[1] = ValueType(2, 0); - input[2] = ValueType(3, 0); - input[3] = ValueType(4, 0); - input[4] = ValueType(5, 0); - input[5] = ValueType(6, 0); - input[6] = ValueType(7, 0); - input[7] = ValueType(8, 0); - - ValueType output[8]; - attn.forward(input, output); - - // With identity projections on positive input, output = X * (X^T * X) - // Same math as the floating-point test: - // X^T * X = [[84, 100], [100, 120]] - // Out[0] = [1,2] * [[84,100],[100,120]] = [284, 340] - // Verify first output element (allow fixed-point rounding tolerance) - const typename ValueType::FullWidthValueType tolerance = 1 << (ValueType::NumberOfFractionalBits - 1); - ValueType expected284(284, 0); - BOOST_TEST(std::abs(output[0].getValue() - expected284.getValue()) <= tolerance); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_gradients) -{ - // Verify gradients compile and produce non-zero values with Q16.16 - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - tinymind::SelfAttention1D attn; - - // Identity projections - for (size_t proj = 0; proj < 3; ++proj) - { - attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); - attn.setProjectionWeight(proj, 0, 1, ValueType(0)); - attn.setProjectionWeight(proj, 1, 0, ValueType(0)); - attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); - } - - ValueType input[4]; - input[0] = ValueType(1, 0); - input[1] = ValueType(2, 0); - input[2] = ValueType(1, 0); - input[3] = ValueType(1, 0); - - ValueType output[4]; - attn.forward(input, output); - - // Compute gradients with non-zero deltas - ValueType deltas[4]; - deltas[0] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 1)); // 0.5 - deltas[1] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 1)); - deltas[2] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 2)); // 0.25 - deltas[3] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 2)); - - attn.computeGradients(deltas); - - // At least some gradients should be non-zero - bool anyNonZero = false; - for (size_t i = 0; i < tinymind::SelfAttention1D::TotalWeights; ++i) - { - if (attn.getGradient(i).getValue() != 0) - { - anyNonZero = true; - break; - } - } - BOOST_TEST(anyNonZero); -} - -BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_bias) -{ - // Verify biases work with Q16.16 - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - tinymind::SelfAttention1D attn; - - // All weights zero, set biases - attn.setProjectionBias(0, 0, ValueType(1, 0)); - attn.setProjectionBias(0, 1, ValueType(1, 0)); - attn.setProjectionBias(1, 0, ValueType(1, 0)); - attn.setProjectionBias(1, 1, ValueType(1, 0)); - attn.setProjectionBias(2, 0, ValueType(2, 0)); - attn.setProjectionBias(2, 1, ValueType(3, 0)); - - ValueType input[4]; - input[0] = ValueType(0); - input[1] = ValueType(0); - input[2] = ValueType(0); - input[3] = ValueType(0); - - ValueType output[4]; - attn.forward(input, output); - - // Same math as floating-point bias test: - // Q' = K' = [1,1] per step, V = [2,3] per step - // KV = [[4,6],[4,6]], Out = [[8,12],[8,12]] - const typename ValueType::FullWidthValueType tolerance = 1 << (ValueType::NumberOfFractionalBits - 1); - ValueType expected8(8, 0); - ValueType expected12(12, 0); - BOOST_TEST(std::abs(output[0].getValue() - expected8.getValue()) <= tolerance); - BOOST_TEST(std::abs(output[1].getValue() - expected12.getValue()) <= tolerance); - BOOST_TEST(std::abs(output[2].getValue() - expected8.getValue()) <= tolerance); - BOOST_TEST(std::abs(output[3].getValue() - expected12.getValue()) <= tolerance); -} - -// ============================================================================ -// MixtureOfExperts tests (phase 1: float, top-1 argmax routing) -// ============================================================================ - -BOOST_AUTO_TEST_CASE(test_case_moe_static_sizes) -{ - typedef tinymind::MixtureOfExperts MoEType; - - static_assert(MoEType::InputSize == 8, "Wrong input size"); - static_assert(MoEType::OutputSize == 4, "Wrong output size"); - static_assert(MoEType::NumberOfExperts == 3, "Wrong expert count"); - - // Default expert is LinearExpert with matching dims - static_assert(MoEType::Expert::InputSize == 8, "Wrong expert input size"); - static_assert(MoEType::Expert::OutputSize == 4, "Wrong expert output size"); - static_assert(MoEType::Expert::TotalWeights == 32, "Wrong expert weights"); // 8 * 4 - // Router maps InputDim -> NumExperts - static_assert(MoEType::Router::OutputSize == 3, "Wrong router output size"); - static_assert(MoEType::Router::TotalWeights == 24, "Wrong router weights"); // 8 * 3 - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_linear_expert_forward) -{ - // output[o] = bias[o] + sum_i W(o,i) * input[i] - tinymind::LinearExpert expert; - - // W = [[1, 2, 3], [4, 5, 6]], bias = [10, 20] - expert.setWeightAt(0, 0, 1.0); expert.setWeightAt(0, 1, 2.0); expert.setWeightAt(0, 2, 3.0); - expert.setWeightAt(1, 0, 4.0); expert.setWeightAt(1, 1, 5.0); expert.setWeightAt(1, 2, 6.0); - expert.setBias(0, 10.0); - expert.setBias(1, 20.0); - - double input[3] = {1.0, 1.0, 1.0}; - double output[2]; - expert.forward(input, output); - - // out[0] = 10 + (1+2+3) = 16 ; out[1] = 20 + (4+5+6) = 35 - BOOST_TEST(fabs(output[0] - 16.0) < 1e-9); - BOOST_TEST(fabs(output[1] - 35.0) < 1e-9); - - // Flat accessor agrees with structured: W(1,2) at index 1*3 + 2 = 5 - BOOST_TEST(fabs(expert.getWeight(5) - 6.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_routes_to_argmax_expert) -{ - // Router picks the expert with the highest logit; that expert's output - // is what forward() returns. - tinymind::MixtureOfExperts moe; - - // Router: drive logit of expert 1 highest for this input. - // logit[e] = sum_i W(e,i) * input[i] (biases default 0) - moe.router().setWeightAt(0, 0, 0.0); moe.router().setWeightAt(0, 1, 0.0); // expert 0 -> 0 - moe.router().setWeightAt(1, 0, 1.0); moe.router().setWeightAt(1, 1, 1.0); // expert 1 -> 2 - moe.router().setWeightAt(2, 0, 0.5); moe.router().setWeightAt(2, 1, 0.0); // expert 2 -> 0.5 - - // Give each expert a distinct constant output via bias so we can identify it. - moe.expert(0).setBias(0, 100.0); - moe.expert(1).setBias(0, 200.0); - moe.expert(2).setBias(0, 300.0); - - double input[2] = {1.0, 1.0}; - double output[1]; - moe.forward(input, output); - - // logits = [0, 2, 0.5] -> argmax = expert 1 -> output 200 - BOOST_TEST(moe.getLastSelectedExpert() == 1u); - BOOST_TEST(fabs(output[0] - 200.0) < 1e-9); - - // route() agrees and reports the logits - double logits[3]; - const size_t sel = moe.route(input, logits); - BOOST_TEST(sel == 1u); - BOOST_TEST(fabs(logits[1] - 2.0) < 1e-9); - BOOST_TEST(fabs(logits[2] - 0.5) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_selection_changes_with_input) -{ - // Different inputs route to different experts. - tinymind::MixtureOfExperts moe; - - // expert 0 wins when input[0] dominates; expert 1 wins when input[1] does. - moe.router().setWeightAt(0, 0, 1.0); moe.router().setWeightAt(0, 1, 0.0); - moe.router().setWeightAt(1, 0, 0.0); moe.router().setWeightAt(1, 1, 1.0); - - moe.expert(0).setBias(0, -1.0); - moe.expert(1).setBias(0, 1.0); - - double output[1]; - - double inA[2] = {5.0, 1.0}; - moe.forward(inA, output); - BOOST_TEST(moe.getLastSelectedExpert() == 0u); - BOOST_TEST(fabs(output[0] - (-1.0)) < 1e-9); - - double inB[2] = {1.0, 5.0}; - moe.forward(inB, output); - BOOST_TEST(moe.getLastSelectedExpert() == 1u); - BOOST_TEST(fabs(output[0] - 1.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_argmax_ties_lowest_index) -{ - // Equal logits resolve to the lowest expert index. - tinymind::MixtureOfExperts moe; - - // All router weights zero -> all logits 0 (tie) - double input[1] = {1.0}; - double output[1]; - moe.expert(0).setBias(0, 7.0); - moe.forward(input, output); - - BOOST_TEST(moe.getLastSelectedExpert() == 0u); - BOOST_TEST(fabs(output[0] - 7.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_fixed_point) -{ - // Top-1 routing works with Q-format value types. - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> Q16; - tinymind::MixtureOfExperts moe; - - moe.router().setWeightAt(0, 0, Q16(0, 0)); moe.router().setWeightAt(0, 1, Q16(0, 0)); - moe.router().setWeightAt(1, 0, Q16(1, 0)); moe.router().setWeightAt(1, 1, Q16(0, 0)); - - moe.expert(0).setBias(0, Q16(3, 0)); - moe.expert(1).setBias(0, Q16(9, 0)); - - Q16 input[2] = {Q16(1, 0), Q16(0, 0)}; - Q16 output[1]; - moe.forward(input, output); - - // logits = [0, 1] -> expert 1 -> output 9 - BOOST_TEST(moe.getLastSelectedExpert() == 1u); - BOOST_TEST(output[0].getValue() == Q16(9, 0).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_initialize_and_run) -{ - // Random init then a forward pass: selection is in range, output finite. - tinymind::MixtureOfExperts moe; - moe.initializeWeights>(); - - double input[4] = {0.5, -0.5, 0.25, -0.25}; - double output[2]; - moe.forward(input, output); - - BOOST_TEST(moe.getLastSelectedExpert() < 3u); - BOOST_TEST(std::isfinite(output[0])); - BOOST_TEST(std::isfinite(output[1])); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_custom_expert_type) -{ - // Plug a SelfAttention-like expert? Use LinearExpert explicitly as the - // template-template argument to confirm the pluggable path compiles and - // routes identically to the default. - tinymind::MixtureOfExperts moe; - - moe.router().setWeightAt(0, 0, 0.0); moe.router().setWeightAt(0, 1, 0.0); moe.router().setWeightAt(0, 2, 0.0); - moe.router().setWeightAt(1, 0, 1.0); moe.router().setWeightAt(1, 1, 1.0); moe.router().setWeightAt(1, 2, 1.0); - - // expert 1 = identity - for (size_t r = 0; r < 3; ++r) - { - moe.expert(1).setWeightAt(r, r, 1.0); - } - - double input[3] = {2.0, 3.0, 4.0}; - double output[3]; - moe.forward(input, output); - - // logits = [0, 9] -> expert 1 (identity) -> output == input - BOOST_TEST(moe.getLastSelectedExpert() == 1u); - BOOST_TEST(fabs(output[0] - 2.0) < 1e-9); - BOOST_TEST(fabs(output[1] - 3.0) < 1e-9); - BOOST_TEST(fabs(output[2] - 4.0) < 1e-9); -} - -// ============================================================================ -// TopKMixtureOfExperts tests (phase 5: float, softmax-gated blend) -// ============================================================================ - -BOOST_AUTO_TEST_CASE(test_case_moe_topk_static_sizes) -{ - typedef tinymind::TopKMixtureOfExperts MoEType; - static_assert(MoEType::InputSize == 6, "Wrong input size"); - static_assert(MoEType::OutputSize == 3, "Wrong output size"); - static_assert(MoEType::NumberOfExperts == 4, "Wrong expert count"); - static_assert(MoEType::TopK == 2, "Wrong K"); - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_topk_k1_equals_top1) -{ - // K=1 collapses to top-1: gate weight 1.0, output == argmax expert. - tinymind::TopKMixtureOfExperts moe; - - moe.router().setWeightAt(0, 0, 0.0); moe.router().setWeightAt(0, 1, 0.0); - moe.router().setWeightAt(1, 0, 1.0); moe.router().setWeightAt(1, 1, 1.0); - moe.router().setWeightAt(2, 0, 0.5); moe.router().setWeightAt(2, 1, 0.0); - - moe.expert(0).setBias(0, 100.0); - moe.expert(1).setBias(0, 200.0); - moe.expert(2).setBias(0, 300.0); - - double input[2] = {1.0, 1.0}; - double output[1]; - moe.forward(input, output); - - // logits {0,2,0.5} -> expert 1, gate 1.0 - BOOST_TEST(moe.getSelectedExpert(0) == 1u); - BOOST_TEST(fabs(moe.getGate(0) - 1.0) < 1e-12); - BOOST_TEST(fabs(output[0] - 200.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_topk_dense_average) -{ - // Dense MoE (K==NumExperts), tied logits -> equal 0.5/0.5 gates -> - // output is the average of the two expert outputs. - tinymind::TopKMixtureOfExperts moe; - - // Router weights zero -> logits {0,0} tie. - moe.expert(0).setBias(0, 2.0); - moe.expert(1).setBias(0, 4.0); - - double input[1] = {1.0}; - double output[1]; - moe.forward(input, output); - - BOOST_TEST(fabs(moe.getGate(0) - 0.5) < 1e-9); - BOOST_TEST(fabs(moe.getGate(1) - 0.5) < 1e-9); - BOOST_TEST(fabs(output[0] - 3.0) < 1e-9); // 0.5*2 + 0.5*4 -} - -BOOST_AUTO_TEST_CASE(test_case_moe_topk_top2_of_3_blend) -{ - // Top-2 of 3: the two highest logits are softmax-blended, the lowest - // expert is dropped entirely. - tinymind::TopKMixtureOfExperts moe; - - // Logits = weight * input(=1): expert0=0, expert1=2, expert2=1. - moe.router().setWeightAt(0, 0, 0.0); - moe.router().setWeightAt(1, 0, 2.0); - moe.router().setWeightAt(2, 0, 1.0); - - moe.expert(0).setBias(0, -999.0); // must be excluded - moe.expert(1).setBias(0, 10.0); - moe.expert(2).setBias(0, 20.0); - - double input[1] = {1.0}; - double output[1]; - moe.forward(input, output); - - // Selected: expert 1 (logit 2) then expert 2 (logit 1). - BOOST_TEST(moe.getSelectedExpert(0) == 1u); - BOOST_TEST(moe.getSelectedExpert(1) == 2u); - - // softmax over {2,1}: w1 = e^2/(e^2+e^1), w2 = e^1/(e^2+e^1). - const double e2 = exp(2.0), e1 = exp(1.0); - const double w1 = e2 / (e2 + e1), w2 = e1 / (e2 + e1); - const double expected = w1 * 10.0 + w2 * 20.0; - BOOST_TEST(fabs(output[0] - expected) < 1e-9); - // Excluded expert 0 never contributes. - BOOST_TEST(output[0] > 0.0); -} - -BOOST_AUTO_TEST_CASE(test_case_moe_topk_gates_sum_to_one) -{ - // Random init: the K gates always form a convex combination. - tinymind::TopKMixtureOfExperts moe; - moe.initializeWeights>(); - - double input[4] = {0.3, -0.7, 0.5, 0.1}; - double output[2]; - moe.forward(input, output); - - double gsum = 0.0; - for (size_t j = 0; j < 3; ++j) - { - BOOST_TEST(moe.getGate(j) >= 0.0); - gsum += moe.getGate(j); - } - BOOST_TEST(fabs(gsum - 1.0) < 1e-9); -} - -// ============================================================================ -// FFT1D tests -// ============================================================================ - -BOOST_AUTO_TEST_CASE(test_case_fft1d_static_sizes) -{ - typedef tinymind::FFT1D FFTType; - - static_assert(FFTType::Length == 8, "Wrong FFT length"); - static_assert(FFTType::HalfLength == 4, "Wrong half length"); - static_assert(FFTType::NumStages == 3, "Wrong number of stages"); - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_float_dc_signal) -{ - // A constant (DC) signal should produce energy only in bin 0 - const size_t N = 8; - tinymind::FFT1D fft; - - // Compute twiddle factors - double cosTable[N / 2]; - double sinTable[N / 2]; - for (size_t k = 0; k < N / 2; ++k) - { - cosTable[k] = std::cos(-2.0 * M_PI * k / N); - sinTable[k] = std::sin(-2.0 * M_PI * k / N); - } - fft.setTwiddleFactors(cosTable, sinTable); - - double real[N] = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; - double imag[N] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - - fft.forward(real, imag); - - // Scaled FFT divides by N, so DC bin = sum/N = 8/8 = 1.0 - BOOST_TEST(fabs(real[0] - 1.0) < 0.001); - BOOST_TEST(fabs(imag[0]) < 0.001); - - // All other bins should be zero - for (size_t i = 1; i < N; ++i) - { - BOOST_TEST(fabs(real[i]) < 0.001); - BOOST_TEST(fabs(imag[i]) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_float_impulse) -{ - // Impulse at index 0: FFT should produce constant magnitude across all bins - const size_t N = 8; - tinymind::FFT1D fft; - - double cosTable[N / 2]; - double sinTable[N / 2]; - for (size_t k = 0; k < N / 2; ++k) - { - cosTable[k] = std::cos(-2.0 * M_PI * k / N); - sinTable[k] = std::sin(-2.0 * M_PI * k / N); - } - fft.setTwiddleFactors(cosTable, sinTable); - - double real[N] = {1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - double imag[N] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - - fft.forward(real, imag); - - // Scaled by 1/N, so each bin should have magnitude 1/N = 0.125 - double magSq[N]; - tinymind::FFT1D::magnitudeSquared(real, imag, magSq); - - const double expectedMagSq = (1.0 / N) * (1.0 / N); // 0.015625 - for (size_t i = 0; i < N; ++i) - { - BOOST_TEST(fabs(magSq[i] - expectedMagSq) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_float_nyquist) -{ - // Alternating +1/-1 signal: energy at Nyquist frequency (bin N/2) - const size_t N = 8; - tinymind::FFT1D fft; - - double cosTable[N / 2]; - double sinTable[N / 2]; - for (size_t k = 0; k < N / 2; ++k) - { - cosTable[k] = std::cos(-2.0 * M_PI * k / N); - sinTable[k] = std::sin(-2.0 * M_PI * k / N); - } - fft.setTwiddleFactors(cosTable, sinTable); - - double real[N] = {1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0}; - double imag[N] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - - fft.forward(real, imag); - - // Scaled FFT: Nyquist bin (N/2=4) should have magnitude 1.0 - BOOST_TEST(fabs(real[N / 2] - 1.0) < 0.001); - BOOST_TEST(fabs(imag[N / 2]) < 0.001); - - // All other bins should be zero - for (size_t i = 0; i < N; ++i) - { - if (i != N / 2) - { - BOOST_TEST(fabs(real[i]) < 0.001); - BOOST_TEST(fabs(imag[i]) < 0.001); - } - } -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_float_roundtrip) -{ - // Forward then inverse should recover the original signal - const size_t N = 8; - tinymind::FFT1D fft; - - double cosTable[N / 2]; - double sinTable[N / 2]; - for (size_t k = 0; k < N / 2; ++k) - { - cosTable[k] = std::cos(-2.0 * M_PI * k / N); - sinTable[k] = std::sin(-2.0 * M_PI * k / N); - } - fft.setTwiddleFactors(cosTable, sinTable); - - double original[N] = {0.1, 0.5, -0.3, 0.8, -0.2, 0.6, 0.0, -0.4}; - double real[N]; - double imag[N]; - for (size_t i = 0; i < N; ++i) - { - real[i] = original[i]; - imag[i] = 0.0; - } - - fft.forward(real, imag); - fft.inverse(real, imag); - - // Should recover original (within floating-point tolerance) - for (size_t i = 0; i < N; ++i) - { - BOOST_TEST(fabs(real[i] - original[i]) < 0.001); - BOOST_TEST(fabs(imag[i]) < 0.001); - } -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_float_single_frequency) -{ - // Pure cosine at bin 1 frequency - const size_t N = 16; - tinymind::FFT1D fft; - - double cosTable[N / 2]; - double sinTable[N / 2]; - for (size_t k = 0; k < N / 2; ++k) - { - cosTable[k] = std::cos(-2.0 * M_PI * k / N); - sinTable[k] = std::sin(-2.0 * M_PI * k / N); - } - fft.setTwiddleFactors(cosTable, sinTable); - - double real[N]; - double imag[N]; - for (size_t i = 0; i < N; ++i) - { - real[i] = std::cos(2.0 * M_PI * i / N); // frequency bin 1 - imag[i] = 0.0; - } - - fft.forward(real, imag); - - // Scaled FFT: bins 1 and N-1 should have magnitude 0.5 - // (cosine splits into positive and negative frequency) - BOOST_TEST(fabs(real[1] - 0.5) < 0.001); - BOOST_TEST(fabs(imag[1]) < 0.001); - BOOST_TEST(fabs(real[N - 1] - 0.5) < 0.001); - BOOST_TEST(fabs(imag[N - 1]) < 0.001); - - // All other bins should be near zero - for (size_t i = 0; i < N; ++i) - { - if (i != 1 && i != N - 1) - { - BOOST_TEST(fabs(real[i]) < 0.001); - BOOST_TEST(fabs(imag[i]) < 0.001); - } - } -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_float_magnitude_squared) -{ - // Verify magnitudeSquared computes real^2 + imag^2 - const size_t N = 4; - double real[N] = {3.0, 0.0, -1.0, 0.0}; - double imag[N] = {4.0, 0.0, 2.0, 0.0}; - double magSq[N]; - - tinymind::FFT1D::magnitudeSquared(real, imag, magSq); - - BOOST_TEST(fabs(magSq[0] - 25.0) < 0.001); // 9 + 16 - BOOST_TEST(fabs(magSq[1] - 0.0) < 0.001); - BOOST_TEST(fabs(magSq[2] - 5.0) < 0.001); // 1 + 4 - BOOST_TEST(fabs(magSq[3] - 0.0) < 0.001); -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_fixed_point_dc_signal) -{ - // DC signal test with Q8.8 fixed-point - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - const size_t N = 8; - tinymind::FFT1D fft; - - // Pre-computed twiddle factors for N=8: cos(-2*pi*k/8) and sin(-2*pi*k/8) - // k=0: cos(0)=1, sin(0)=0 - // k=1: cos(-pi/4)=0.707, sin(-pi/4)=-0.707 - // k=2: cos(-pi/2)=0, sin(-pi/2)=-1 - // k=3: cos(-3pi/4)=-0.707, sin(-3pi/4)=-0.707 - ValueType cosTable[N / 2]; - ValueType sinTable[N / 2]; - cosTable[0] = ValueType(1, 0); - cosTable[1] = ValueType(0, 181); // ~0.707 - cosTable[2] = ValueType(0, 0); - cosTable[3] = ValueType(-1, 75); // ~-0.707 - - sinTable[0] = ValueType(0, 0); - sinTable[1] = ValueType(-1, 75); // ~-0.707 - sinTable[2] = ValueType(-1, 0); - sinTable[3] = ValueType(-1, 75); // ~-0.707 - - fft.setTwiddleFactors(cosTable, sinTable); - - ValueType real[N]; - ValueType imag[N]; - for (size_t i = 0; i < N; ++i) - { - real[i] = ValueType(1, 0); - imag[i] = ValueType(0); - } - - fft.forward(real, imag); - - // DC bin should have the dominant energy - // With scaled FFT, DC = 1.0 (sum/N = 8/8) - ValueType expected(1, 0); - const typename ValueType::FullWidthValueType tolerance = 3 << (ValueType::NumberOfFractionalBits - 3); - - BOOST_TEST(std::abs(real[0].getValue() - expected.getValue()) <= tolerance); - BOOST_TEST(std::abs(imag[0].getValue()) <= tolerance); - - // Other bins should be near zero - for (size_t i = 1; i < N; ++i) - { - BOOST_TEST(std::abs(real[i].getValue()) <= tolerance); - BOOST_TEST(std::abs(imag[i].getValue()) <= tolerance); - } -} - -BOOST_AUTO_TEST_CASE(test_case_fft1d_fixed_point_roundtrip) -{ - // Forward then inverse should approximately recover the signal - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - const size_t N = 8; - tinymind::FFT1D fft; - - ValueType cosTable[N / 2]; - ValueType sinTable[N / 2]; - cosTable[0] = ValueType(1, 0); - cosTable[1] = ValueType(0, 181); - cosTable[2] = ValueType(0, 0); - cosTable[3] = ValueType(-1, 75); - - sinTable[0] = ValueType(0, 0); - sinTable[1] = ValueType(-1, 75); - sinTable[2] = ValueType(-1, 0); - sinTable[3] = ValueType(-1, 75); - - fft.setTwiddleFactors(cosTable, sinTable); - - ValueType original[N]; - original[0] = ValueType(0, 128); // 0.5 - original[1] = ValueType(1, 0); - original[2] = ValueType(-1, 0); - original[3] = ValueType(0, 64); // 0.25 - original[4] = ValueType(0, 0); - original[5] = ValueType(0, 192); // 0.75 - original[6] = ValueType(-1, 128); // -0.5 - original[7] = ValueType(0, 32); // 0.125 - - ValueType real[N]; - ValueType imag[N]; - for (size_t i = 0; i < N; ++i) - { - real[i] = original[i]; - imag[i] = ValueType(0); - } - - fft.forward(real, imag); - fft.inverse(real, imag); - - // With Q8.8 and 3 stages, expect some rounding error - // Tolerance: ~4 LSBs of the fractional part - const typename ValueType::FullWidthValueType tolerance = 4; - for (size_t i = 0; i < N; ++i) - { - BOOST_TEST(std::abs(real[i].getValue() - original[i].getValue()) <= tolerance); - } -} - -// ============================================================ -// Conv2D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_conv2d_static_sizes) -{ - typedef tinymind::Conv2D ConvType; - static_assert(ConvType::OutputHeight == 30, "Wrong output height"); - static_assert(ConvType::OutputWidth == 30, "Wrong output width"); - static_assert(ConvType::OutputSize == 30 * 30 * 16, "Wrong output size"); - // 16 filters * (3*3*3 + 1) = 16 * 28 = 448 - static_assert(ConvType::TotalWeights == 448, "Wrong total weights"); - BOOST_TEST(true); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_forward_identity) -{ - // 3x3 input, 1 channel, 3x3 kernel, 1 filter = 1x1 output - // Kernel is all zeros except center = 1; output should equal input center. - tinymind::Conv2D conv; - - conv.setFilterWeight(0, 1, 1, 0, 1.0); // center tap - conv.setFilterBias(0, 0.0); - - double input[9] = { - 1, 2, 3, - 4, 5, 6, - 7, 8, 9 - }; - double output[1]; - conv.forward(input, output); - - BOOST_TEST(std::fabs(output[0] - 5.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_forward_sum_kernel) -{ - // 4x4 input, 1 channel, 2x2 kernel (all ones), stride 2 -> 2x2 output - tinymind::Conv2D conv; - for (size_t kh = 0; kh < 2; ++kh) - { - for (size_t kw = 0; kw < 2; ++kw) - { - conv.setFilterWeight(0, kh, kw, 0, 1.0); - } - } - conv.setFilterBias(0, 0.0); - - double input[16] = { - 1, 2, 3, 4, - 5, 6, 7, 8, - 9, 10, 11, 12, - 13, 14, 15, 16 - }; - double output[4]; - conv.forward(input, output); - - // Top-left 2x2 = 1+2+5+6 = 14; top-right = 3+4+7+8 = 22 - // Bot-left = 9+10+13+14 = 46; bot-right = 11+12+15+16 = 54 - BOOST_TEST(std::fabs(output[0] - 14.0) < 1e-9); - BOOST_TEST(std::fabs(output[1] - 22.0) < 1e-9); - BOOST_TEST(std::fabs(output[2] - 46.0) < 1e-9); - BOOST_TEST(std::fabs(output[3] - 54.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_multi_channel_and_filter) -{ - // 2x2 input, 2 channels, 2x2 kernel, 2 filters -> 1x1x2 output - tinymind::Conv2D conv; - - // Filter 0: sum over all channels and positions - for (size_t kh = 0; kh < 2; ++kh) - for (size_t kw = 0; kw < 2; ++kw) - for (size_t ci = 0; ci < 2; ++ci) - conv.setFilterWeight(0, kh, kw, ci, 1.0); - conv.setFilterBias(0, 0.0); - - // Filter 1: channel 0 positive, channel 1 negative - for (size_t kh = 0; kh < 2; ++kh) - { - for (size_t kw = 0; kw < 2; ++kw) - { - conv.setFilterWeight(1, kh, kw, 0, 1.0); - conv.setFilterWeight(1, kh, kw, 1, -1.0); - } - } - conv.setFilterBias(1, 0.0); - - // NHWC input, each pixel has [ch0, ch1] - double input[8] = { - 1, 10, 2, 20, // row 0 - 3, 30, 4, 40 // row 1 - }; - double output[2]; - conv.forward(input, output); - - // Filter 0 = sum all = (1+2+3+4) + (10+20+30+40) = 10 + 100 = 110 - BOOST_TEST(std::fabs(output[0] - 110.0) < 1e-9); - // Filter 1 = (1+2+3+4) - (10+20+30+40) = 10 - 100 = -90 - BOOST_TEST(std::fabs(output[1] - (-90.0)) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_gradient_sanity) -{ - // Single-filter, 3x3 input, 3x3 kernel -> scalar output. - // For output y = sum(w * x) + b, dy/dw = x, dy/db = 1. - // With delta=1 the weight gradient equals input and bias grad = 1. - tinymind::Conv2D conv; - for (size_t i = 0; i < conv.TotalWeights; ++i) - { - conv.setWeight(i, 0.1); - } - - double input[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; - double delta[1] = {1.0}; - conv.computeGradients(delta, input); - - for (size_t kh = 0; kh < 3; ++kh) - { - for (size_t kw = 0; kw < 3; ++kw) - { - const size_t idx = kh * 3 + kw; - const double g = conv.getGradient(0 * conv.WeightsPerFilter + idx); - BOOST_TEST(std::fabs(g - input[idx]) < 1e-9); - } - } - // Bias gradient - const double biasGrad = conv.getGradient(conv.WeightsPerFilter - 1); - BOOST_TEST(std::fabs(biasGrad - 1.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_fixed_point) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::Conv2D conv; - - conv.setFilterWeight(0, 1, 1, 0, ValueType(1, 0)); - conv.setFilterBias(0, ValueType(0)); - - ValueType input[9]; - for (size_t i = 0; i < 9; ++i) - { - input[i] = ValueType(static_cast(i + 1), 0); - } - ValueType output[1]; - conv.forward(input, output); - - ValueType expected(5, 0); - BOOST_TEST(output[0].getValue() == expected.getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_gradient_fixed_point) -{ - // Q8.8 gradient sanity: dy/dw = x for each kernel position; dy/db = 1. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::Conv2D conv; - for (size_t i = 0; i < conv.TotalWeights; ++i) conv.setWeight(i, ValueType(0)); - - ValueType input[9]; - for (size_t i = 0; i < 9; ++i) - { - input[i] = ValueType(static_cast(i + 1), 0); - } - ValueType delta[1] = {ValueType(1, 0)}; - conv.computeGradients(delta, input); - - for (size_t kh = 0; kh < 3; ++kh) - { - for (size_t kw = 0; kw < 3; ++kw) - { - const size_t idx = kh * 3 + kw; - BOOST_TEST(conv.getGradient(idx).getValue() == input[idx].getValue()); - } - } - BOOST_TEST(conv.getGradient(conv.WeightsPerFilter - 1).getValue() == ValueType(1, 0).getValue()); -} - -// ============================================================ -// DepthwiseConv2D + PointwiseConv2D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_independence) -{ - // Two channels, kernel picks channel 0 only. - tinymind::DepthwiseConv2D dw; - - // Channel 0: ones; Channel 1: ones. Biases 0. - for (size_t c = 0; c < 2; ++c) - { - for (size_t kh = 0; kh < 2; ++kh) - { - for (size_t kw = 0; kw < 2; ++kw) - { - dw.setChannelWeight(c, kh, kw, 1.0); - } - } - dw.setChannelBias(c, 0.0); - } - - double input[8] = { - 1, 10, 2, 20, - 3, 30, 4, 40 - }; - double output[2]; // OutputH = OutputW = 1, Channels = 2 - dw.forward(input, output); - - BOOST_TEST(std::fabs(output[0] - 10.0) < 1e-9); // ch0 sum - BOOST_TEST(std::fabs(output[1] - 100.0) < 1e-9); // ch1 sum — no mixing -} - -BOOST_AUTO_TEST_CASE(test_case_pointwiseconv2d_channel_mix) -{ - // 1x1 spatial, 3 in-channels, 2 filters. - tinymind::PointwiseConv2D pw; - - // Filter 0: [1, 2, 3] dot input - pw.setFilterWeight(0, 0, 1.0); - pw.setFilterWeight(0, 1, 2.0); - pw.setFilterWeight(0, 2, 3.0); - pw.setFilterBias(0, 0.0); - - // Filter 1: mean with bias - pw.setFilterWeight(1, 0, 1.0); - pw.setFilterWeight(1, 1, 1.0); - pw.setFilterWeight(1, 2, 1.0); - pw.setFilterBias(1, 0.5); - - double input[3] = {4.0, 5.0, 6.0}; - double output[2]; - pw.forward(input, output); - - BOOST_TEST(std::fabs(output[0] - (1*4 + 2*5 + 3*6)) < 1e-9); // 32 - BOOST_TEST(std::fabs(output[1] - (4 + 5 + 6 + 0.5)) < 1e-9); // 15.5 -} - -BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_gradient_sanity) -{ - // For y = sum(w * x) + b within a channel, dy/dw = x and dy/db = 1. - // With output delta = 1.0 we expect mGradients to equal the input - // patch values in each channel and 1.0 for each bias. - tinymind::DepthwiseConv2D dw; - for (size_t c = 0; c < 2; ++c) - { - for (size_t kh = 0; kh < 2; ++kh) - for (size_t kw = 0; kw < 2; ++kw) - dw.setChannelWeight(c, kh, kw, 0.0); - dw.setChannelBias(c, 0.0); - } - double input[8] = { - 1, 10, 2, 20, - 3, 30, 4, 40 - }; - double output[2]; - dw.forward(input, output); - double outputDeltas[2] = {1.0, 1.0}; - dw.computeGradients(outputDeltas, input); - - // Channel 0 weight gradients should equal channel-0 inputs (1,2,3,4). - BOOST_TEST(std::fabs(dw.getChannelWeight(0, 0, 0)) < 1e-9); // unchanged before update - BOOST_TEST(std::fabs(dw.getGradient(0) - 1.0) < 1e-9); // ch0 (kh=0,kw=0) - BOOST_TEST(std::fabs(dw.getGradient(1) - 2.0) < 1e-9); - BOOST_TEST(std::fabs(dw.getGradient(2) - 3.0) < 1e-9); - BOOST_TEST(std::fabs(dw.getGradient(3) - 4.0) < 1e-9); - BOOST_TEST(std::fabs(dw.getGradient(4) - 1.0) < 1e-9); // bias ch0 - // Channel 1 weight gradients should equal channel-1 inputs (10,20,30,40). - BOOST_TEST(std::fabs(dw.getGradient(5) - 10.0) < 1e-9); - BOOST_TEST(std::fabs(dw.getGradient(6) - 20.0) < 1e-9); - BOOST_TEST(std::fabs(dw.getGradient(7) - 30.0) < 1e-9); - BOOST_TEST(std::fabs(dw.getGradient(8) - 40.0) < 1e-9); - BOOST_TEST(std::fabs(dw.getGradient(9) - 1.0) < 1e-9); // bias ch1 - - // updateWeights moves weights by lr * grad. With lr = 0.1 and starting - // weights at zero, the new ch0 (kh=0,kw=0) weight should be 0.1 * 1.0 = 0.1. - dw.updateWeights(0.1); - BOOST_TEST(std::fabs(dw.getChannelWeight(0, 0, 0) - 0.1) < 1e-9); - BOOST_TEST(std::fabs(dw.getChannelBias(1) - 0.1) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_pointwiseconv2d_gradient_sanity) -{ - // 1x1 spatial, 3 in-channels, 2 filters. Linear: dy_f/dw_{f,ci} = x_ci. - tinymind::PointwiseConv2D pw; - for (size_t f = 0; f < 2; ++f) - { - for (size_t ci = 0; ci < 3; ++ci) pw.setFilterWeight(f, ci, 0.0); - pw.setFilterBias(f, 0.0); - } - double input[3] = {4.0, 5.0, 6.0}; - double output[2]; - pw.forward(input, output); - - // Output deltas: filter 0 -> 1.0, filter 1 -> 2.0. - double outputDeltas[2] = {1.0, 2.0}; - pw.computeGradients(outputDeltas, input); - - // Filter 0: gradients = delta * input = (4,5,6); bias grad = 1. - BOOST_TEST(std::fabs(pw.getGradient(0) - 4.0) < 1e-9); - BOOST_TEST(std::fabs(pw.getGradient(1) - 5.0) < 1e-9); - BOOST_TEST(std::fabs(pw.getGradient(2) - 6.0) < 1e-9); - BOOST_TEST(std::fabs(pw.getGradient(3) - 1.0) < 1e-9); - // Filter 1: gradients = 2 * input = (8,10,12); bias grad = 2. - BOOST_TEST(std::fabs(pw.getGradient(4) - 8.0) < 1e-9); - BOOST_TEST(std::fabs(pw.getGradient(5) - 10.0) < 1e-9); - BOOST_TEST(std::fabs(pw.getGradient(6) - 12.0) < 1e-9); - BOOST_TEST(std::fabs(pw.getGradient(7) - 2.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_gradient_fixed_point) -{ - // Same gradient invariant in Q8.8: dy/dw equals the input patch. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::DepthwiseConv2D dw; - for (size_t kh = 0; kh < 2; ++kh) - for (size_t kw = 0; kw < 2; ++kw) - dw.setChannelWeight(0, kh, kw, ValueType(0)); - dw.setChannelBias(0, ValueType(0)); - - ValueType input[4] = {ValueType(1, 0), ValueType(2, 0), ValueType(3, 0), ValueType(4, 0)}; - ValueType output[1]; - dw.forward(input, output); - ValueType outputDeltas[1] = {ValueType(1, 0)}; - dw.computeGradients(outputDeltas, input); - - BOOST_TEST(dw.getGradient(0).getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(dw.getGradient(1).getValue() == ValueType(2, 0).getValue()); - BOOST_TEST(dw.getGradient(2).getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(dw.getGradient(3).getValue() == ValueType(4, 0).getValue()); - BOOST_TEST(dw.getGradient(4).getValue() == ValueType(1, 0).getValue()); // bias -} - -BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_updateweights_fixed_point) -{ - // After computeGradients, updateWeights must compute weights[i] += lr*grad[i] - // through Q-format multiply-add. Use Q16.16 for headroom. - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - tinymind::DepthwiseConv2D dw; - for (size_t kh = 0; kh < 2; ++kh) - for (size_t kw = 0; kw < 2; ++kw) - dw.setChannelWeight(0, kh, kw, ValueType(0)); - dw.setChannelBias(0, ValueType(0)); - - ValueType input[4] = {ValueType(2, 0), ValueType(4, 0), ValueType(6, 0), ValueType(8, 0)}; - ValueType output[1]; - dw.forward(input, output); - ValueType outputDeltas[1] = {ValueType(1, 0)}; - dw.computeGradients(outputDeltas, input); - - // lr = 0.5, gradients = (2, 4, 6, 8); expected new weights = (1, 2, 3, 4); bias = 0.5. - const ValueType lr(0, 1u << 15); // 0.5 in Q16.16 - dw.updateWeights(lr); - - BOOST_TEST(dw.getChannelWeight(0, 0, 0).getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(dw.getChannelWeight(0, 0, 1).getValue() == ValueType(2, 0).getValue()); - BOOST_TEST(dw.getChannelWeight(0, 1, 0).getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(dw.getChannelWeight(0, 1, 1).getValue() == ValueType(4, 0).getValue()); - BOOST_TEST(dw.getChannelBias(0).getValue() == ValueType(0, 1u << 15).getValue()); // 0.5 -} - -BOOST_AUTO_TEST_CASE(test_case_pointwiseconv2d_updateweights_fixed_point) -{ - typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; - tinymind::PointwiseConv2D pw; - for (size_t ci = 0; ci < 3; ++ci) pw.setFilterWeight(0, ci, ValueType(0)); - pw.setFilterBias(0, ValueType(0)); - - ValueType input[3] = {ValueType(2, 0), ValueType(4, 0), ValueType(6, 0)}; - ValueType output[1]; - pw.forward(input, output); - ValueType outputDeltas[1] = {ValueType(1, 0)}; - pw.computeGradients(outputDeltas, input); - - const ValueType lr(0, 1u << 15); // 0.5 - pw.updateWeights(lr); - // Filter 0 weight gradients = input = (2, 4, 6); * 0.5 = (1, 2, 3). - BOOST_TEST(pw.getFilterWeight(0, 0).getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(pw.getFilterWeight(0, 1).getValue() == ValueType(2, 0).getValue()); - BOOST_TEST(pw.getFilterWeight(0, 2).getValue() == ValueType(3, 0).getValue()); - BOOST_TEST(pw.getFilterBias(0).getValue() == ValueType(0, 1u << 15).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_separable_pipeline) -{ - // Verify a depthwise-then-pointwise pipeline produces the same - // result as manual per-channel + mix math. - tinymind::DepthwiseConv2D dw; - tinymind::PointwiseConv2D pw; - - for (size_t c = 0; c < 2; ++c) - { - for (size_t kh = 0; kh < 3; ++kh) - { - for (size_t kw = 0; kw < 3; ++kw) - { - dw.setChannelWeight(c, kh, kw, 1.0); - } - } - dw.setChannelBias(c, 0.0); - } - pw.setFilterWeight(0, 0, 0.5); - pw.setFilterWeight(0, 1, 2.0); - pw.setFilterBias(0, 0.0); - - double input[18]; - for (size_t i = 0; i < 18; ++i) - { - input[i] = (i % 2 == 0) ? 1.0 : 2.0; - } - - double dwOut[2]; - dw.forward(input, dwOut); - // dwOut[0] = sum of 9 ch0 values = 9.0; dwOut[1] = 18.0 - BOOST_TEST(std::fabs(dwOut[0] - 9.0) < 1e-9); - BOOST_TEST(std::fabs(dwOut[1] - 18.0) < 1e-9); - - double pwOut[1]; - pw.forward(dwOut, pwOut); - // 0.5 * 9 + 2.0 * 18 = 4.5 + 36 = 40.5 - BOOST_TEST(std::fabs(pwOut[0] - 40.5) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_asymmetric_kernel) -{ - // 3x1 kernel over a 4x3 input (vertical sum). Output shape (4-3+1) x (3-1+1) = 2x3. - tinymind::Conv2D conv; - for (size_t kh = 0; kh < 3; ++kh) - { - conv.setFilterWeight(0, kh, 0, 0, 1.0); - } - conv.setFilterBias(0, 0.0); - - double input[12] = { - 1, 2, 3, - 4, 5, 6, - 7, 8, 9, - 10, 11, 12 - }; - double output[6]; - conv.forward(input, output); - - // Row 0 of output: column sums of input rows 0..2 = (1+4+7, 2+5+8, 3+6+9) = (12, 15, 18) - // Row 1 of output: column sums of input rows 1..3 = (4+7+10, 5+8+11, 6+9+12) = (21, 24, 27) - BOOST_TEST(std::fabs(output[0] - 12.0) < 1e-9); - BOOST_TEST(std::fabs(output[1] - 15.0) < 1e-9); - BOOST_TEST(std::fabs(output[2] - 18.0) < 1e-9); - BOOST_TEST(std::fabs(output[3] - 21.0) < 1e-9); - BOOST_TEST(std::fabs(output[4] - 24.0) < 1e-9); - BOOST_TEST(std::fabs(output[5] - 27.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_conv2d_stride_three) -{ - // 7x7 input, 2x2 kernel of ones, stride 3x3. Output: floor((7-2)/3)+1 = 2 each axis. - tinymind::Conv2D conv; - static_assert(decltype(conv)::OutputHeight == 2, "Wrong output height for stride 3"); - static_assert(decltype(conv)::OutputWidth == 2, "Wrong output width for stride 3"); - for (size_t kh = 0; kh < 2; ++kh) - { - for (size_t kw = 0; kw < 2; ++kw) - { - conv.setFilterWeight(0, kh, kw, 0, 1.0); - } - } - conv.setFilterBias(0, 0.0); - - double input[49]; - for (size_t i = 0; i < 49; ++i) - { - input[i] = static_cast(i + 1); - } - double output[4]; - conv.forward(input, output); - - // Window (row, col) sums a 2x2 patch starting at (3*row, 3*col) of a 1-indexed - // sequence laid out row-major. Patch at (0,0): rows {0,1}, cols {0,1} -> - // values {1,2,8,9} sum to 20. Patch at (0,1): cols {3,4} -> {4,5,11,12} = 32. - // Patch at (1,0): rows {3,4} cols {0,1} -> {22,23,29,30} = 104. - // Patch at (1,1): rows {3,4} cols {3,4} -> {25,26,32,33} = 116. - BOOST_TEST(std::fabs(output[0] - 20.0) < 1e-9); - BOOST_TEST(std::fabs(output[1] - 32.0) < 1e-9); - BOOST_TEST(std::fabs(output[2] - 104.0) < 1e-9); - BOOST_TEST(std::fabs(output[3] - 116.0) < 1e-9); -} - -// ============================================================ -// Pool2D tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_maxpool2d_forward) -{ - tinymind::MaxPool2D pool; - double input[16] = { - 1, 3, 2, 4, - 5, 7, 6, 8, - 9, 11, 10, 12, - 13, 15, 14, 16 - }; - double output[4]; - pool.forward(input, output); - - // Each 2x2 block max: 7, 8, 15, 16 - BOOST_TEST(std::fabs(output[0] - 7.0) < 1e-9); - BOOST_TEST(std::fabs(output[1] - 8.0) < 1e-9); - BOOST_TEST(std::fabs(output[2] - 15.0) < 1e-9); - BOOST_TEST(std::fabs(output[3] - 16.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool2d_backward) -{ - tinymind::MaxPool2D pool; - double input[16] = { - 1, 3, 2, 4, - 5, 7, 6, 8, - 9, 11, 10, 12, - 13, 15, 14, 16 - }; - double output[4]; - pool.forward(input, output); - - double outputDeltas[4] = {1.0, 1.0, 1.0, 1.0}; - double inputDeltas[16]; - pool.backward(outputDeltas, inputDeltas); - - // Only argmax positions get gradient; sum should equal 4. - double sum = 0.0; - for (size_t i = 0; i < 16; ++i) sum += inputDeltas[i]; - BOOST_TEST(std::fabs(sum - 4.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool2d_forward) -{ - tinymind::AvgPool2D pool; - double input[16] = { - 1, 1, 2, 2, - 1, 1, 2, 2, - 3, 3, 4, 4, - 3, 3, 4, 4 - }; - double output[4]; - pool.forward(input, output); - - BOOST_TEST(std::fabs(output[0] - 1.0) < 1e-9); - BOOST_TEST(std::fabs(output[1] - 2.0) < 1e-9); - BOOST_TEST(std::fabs(output[2] - 3.0) < 1e-9); - BOOST_TEST(std::fabs(output[3] - 4.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_forward) -{ - // 3x3x2 input -> 2 outputs - tinymind::GlobalAvgPool2D gap; - - double input[18]; - for (size_t i = 0; i < 9; ++i) - { - input[i * 2 + 0] = 1.0; // channel 0 all ones - input[i * 2 + 1] = 3.0; // channel 1 all threes - } - - double output[2]; - gap.forward(input, output); - - BOOST_TEST(std::fabs(output[0] - 1.0) < 1e-9); - BOOST_TEST(std::fabs(output[1] - 3.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool2d_fixed_point) -{ - // 4x4 input, 2x2 window, stride 2 -> 2x2 output. Divisor must be 4.0 - // (a Q-format value), not raw=4. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::AvgPool2D pool; - - ValueType input[16]; - // Row-major NHWC, channel 0. Use values that average to whole numbers. - const int values[16] = { - 2, 2, 4, 4, - 2, 2, 4, 4, - 6, 6, 8, 8, - 6, 6, 8, 8 - }; - for (size_t i = 0; i < 16; ++i) input[i] = ValueType(values[i], 0); - - ValueType output[4]; - pool.forward(input, output); - - BOOST_TEST(output[0].getValue() == ValueType(2, 0).getValue()); - BOOST_TEST(output[1].getValue() == ValueType(4, 0).getValue()); - BOOST_TEST(output[2].getValue() == ValueType(6, 0).getValue()); - BOOST_TEST(output[3].getValue() == ValueType(8, 0).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_fixed_point) -{ - // 3x3x2 input, all ones / threes per channel; GAP must produce exactly 1.0 / 3.0. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::GlobalAvgPool2D gap; - - ValueType input[18]; - for (size_t i = 0; i < 9; ++i) - { - input[i * 2 + 0] = ValueType(1, 0); - input[i * 2 + 1] = ValueType(3, 0); - } - ValueType output[2]; - gap.forward(input, output); - - BOOST_TEST(output[0].getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(output[1].getValue() == ValueType(3, 0).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool2d_backward) -{ - // 4x4 single-channel, 2x2 window stride 2. Each output cell distributes - // its delta uniformly across the 4 input cells in its window: grad = d/4. - tinymind::AvgPool2D pool; - double outputDeltas[4] = {4.0, 8.0, 12.0, 16.0}; - double inputDeltas[16]; - pool.backward(outputDeltas, inputDeltas); - - // Top-left window (output 0): grad = 1.0 across 4 cells. - BOOST_TEST(std::fabs(inputDeltas[0] - 1.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[1] - 1.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[4] - 1.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[5] - 1.0) < 1e-9); - // Top-right window (output 1): grad = 2.0 - BOOST_TEST(std::fabs(inputDeltas[2] - 2.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[3] - 2.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[6] - 2.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[7] - 2.0) < 1e-9); - // Bot-right window (output 3): grad = 4.0 - BOOST_TEST(std::fabs(inputDeltas[10] - 4.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[15] - 4.0) < 1e-9); -} - -BOOST_AUTO_TEST_CASE(test_case_avgpool2d_backward_fixed_point) -{ - // Same input pattern as above in Q8.8: confirms divisor() is correct on - // the backward path, not just forward. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::AvgPool2D pool; - ValueType outputDeltas[4] = {ValueType(4, 0), ValueType(8, 0), ValueType(12, 0), ValueType(16, 0)}; - ValueType inputDeltas[16]; - pool.backward(outputDeltas, inputDeltas); - - BOOST_TEST(inputDeltas[0].getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(inputDeltas[5].getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(inputDeltas[3].getValue() == ValueType(2, 0).getValue()); - BOOST_TEST(inputDeltas[10].getValue() == ValueType(4, 0).getValue()); - BOOST_TEST(inputDeltas[15].getValue() == ValueType(4, 0).getValue()); -} - -BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_backward) -{ - // GAP backward distributes each output delta uniformly across its - // spatial extent (H*W positions per channel). - tinymind::GlobalAvgPool2D gap; - double outputDeltas[2] = {9.0, 18.0}; // chosen so per-cell grad is 1.0 and 2.0 - double inputDeltas[18]; - gap.backward(outputDeltas, inputDeltas); - - for (size_t i = 0; i < 9; ++i) - { - BOOST_TEST(std::fabs(inputDeltas[i * 2 + 0] - 1.0) < 1e-9); - BOOST_TEST(std::fabs(inputDeltas[i * 2 + 1] - 2.0) < 1e-9); - } -} - -BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_backward_fixed_point) -{ - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::GlobalAvgPool2D gap; - ValueType outputDeltas[2] = {ValueType(9, 0), ValueType(18, 0)}; - ValueType inputDeltas[18]; - gap.backward(outputDeltas, inputDeltas); - - for (size_t i = 0; i < 9; ++i) - { - BOOST_TEST(inputDeltas[i * 2 + 0].getValue() == ValueType(1, 0).getValue()); - BOOST_TEST(inputDeltas[i * 2 + 1].getValue() == ValueType(2, 0).getValue()); - } -} - -BOOST_AUTO_TEST_CASE(test_case_maxpool2d_backward_fixed_point) -{ - // Argmax-routed gradients: only the position holding each max receives - // the upstream delta. Other positions stay zero. - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; - tinymind::MaxPool2D pool; - ValueType input[16]; - const int values[16] = { - 1, 3, 2, 4, - 5, 7, 6, 8, - 9, 11, 10, 12, - 13, 15, 14, 16 - }; - for (size_t i = 0; i < 16; ++i) input[i] = ValueType(values[i], 0); - - ValueType output[4]; - pool.forward(input, output); // populates argmax indices - - ValueType outputDeltas[4] = {ValueType(1, 0), ValueType(1, 0), ValueType(1, 0), ValueType(1, 0)}; - ValueType inputDeltas[16]; - pool.backward(outputDeltas, inputDeltas); - - typename ValueType::FullWidthValueType sum = 0; - size_t hits = 0; - for (size_t i = 0; i < 16; ++i) - { - sum += inputDeltas[i].getValue(); - if (inputDeltas[i].getValue() != 0) ++hits; - } - // Exactly four argmax positions, each receiving raw=256 (= 1.0). - BOOST_TEST(hits == 4u); - BOOST_TEST(sum == 4 * ValueType(1, 0).getValue()); -} - -// ============================================================ -// Benchmark harness tests -// ============================================================ - -BOOST_AUTO_TEST_CASE(test_case_bench_stack_watermark) -{ - uint8_t buffer[256]; - tinymind::bench::paintStack(buffer, sizeof(buffer)); - - // Untouched: high water == 0 - BOOST_TEST(tinymind::bench::stackHighWater(buffer, sizeof(buffer)) == 0u); - - // Stack grows down: simulate a 40-byte frame by zeroing the *highest* - // 40 bytes of the buffer. stackHighWater should report 40. - for (size_t i = sizeof(buffer) - 40; i < sizeof(buffer); ++i) - { - buffer[i] = 0x00; - } - // The canary pattern is intact up to byte (256 - 40 - 1). The first - // non-A5 byte is at offset 216, so used = 256 - 216 = 40. - BOOST_TEST(tinymind::bench::stackHighWater(buffer, sizeof(buffer)) == 40u); - - // Touching the bottom as well should push the watermark to full size. - buffer[0] = 0x00; - BOOST_TEST(tinymind::bench::stackHighWater(buffer, sizeof(buffer)) == sizeof(buffer)); -} - -BOOST_AUTO_TEST_CASE(test_case_bench_cycle_counter_monotonic) -{ - tinymind::bench::enableCycleCounter(); - const tinymind::bench::Cycles a = tinymind::bench::readCycleCounter(); - // Do some work the compiler cannot elide. - volatile uint64_t acc = 0; - for (uint64_t i = 0; i < 10000; ++i) acc += i; - const tinymind::bench::Cycles b = tinymind::bench::readCycleCounter(); - - // On host, a/b are ns-since-first-call; require b >= a modulo wrap. - // Cast to signed to tolerate legitimate wrap. - BOOST_TEST(static_cast(b - a) >= 0); - BOOST_TEST(acc > 0u); // keep the loop live -} - -// Exercise the EmptyLayerChain terminal accessors (recursion base case of the -// heterogeneous layer chain) across every value type / policy combination, the -// inverted-Dropout training-mode path, and two QValue operators that the -// runtime suites had not otherwise reached -- closing the per-instantiation -// function-coverage gaps that line coverage cannot reflect. -BOOST_AUTO_TEST_CASE(test_case_function_coverage_terminal_accessors_and_ops) -{ - typedef tinymind::ChainHiddenLayerAccessor A; - tinymind::EmptyLayerChain ec; - - // No-op terminal accessors: instantiate + call all four for each type. - auto hit = [&](auto sample) { - typedef decltype(sample) V; - (void)A::template getBiasNeuronWeightForConnection(ec, 0, 0); - (void)A::template getWeightForNeuronAndConnection(ec, 0, 0, 0); - A::template setBiasNeuronWeightForConnection(ec, 0, 0, V()); - A::template setWeightForNeuronAndConnection(ec, 0, 0, 0, V()); - }; - hit(double()); - hit(tinymind::QValue<8, 8, true, tinymind::TruncatePolicy, tinymind::WrapPolicy>()); - hit(tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy, tinymind::WrapPolicy>()); - hit(tinymind::QValue<16, 16, true, tinymind::TruncatePolicy, tinymind::WrapPolicy>()); - hit(tinymind::QValue<8, 24, true, tinymind::RoundUpPolicy, tinymind::WrapPolicy>()); - hit(tinymind::QValue<8, 24, true, tinymind::TruncatePolicy, tinymind::WrapPolicy>()); - BOOST_TEST(true); // accessors are no-ops; reaching here means they ran - - // Inverted dropout training path: forward() in training mode runs - // generateMask() + scale() + fromInteger() for each instantiation. - { - tinymind::Dropout dd; // ctor leaves training=true - double din[4] = { 1.0, 1.0, 1.0, 1.0 }; - double dout[4] = { 0.0, 0.0, 0.0, 0.0 }; - dd.forward(din, dout); - - typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy, tinymind::WrapPolicy> Qv; - tinymind::Dropout dq; - Qv qin[4] = { Qv(1, 0), Qv(1, 0), Qv(1, 0), Qv(1, 0) }; - Qv qout[4] = { Qv(0, 0), Qv(0, 0), Qv(0, 0), Qv(0, 0) }; - dq.forward(qin, qout); - BOOST_TEST(true); - } - - // QValue<8.8 MinMaxSaturate> assignment-from-short and QValue<16.16> ostream. - { - tinymind::QValue<8, 8, true, tinymind::TruncatePolicy, - tinymind::MinMaxSaturatePolicy> qsat; - qsat = static_cast(3); - BOOST_TEST(qsat.getValue() != 0); - - std::ostringstream oss; - oss << tinymind::QValue<16, 16, true, tinymind::TruncatePolicy, - tinymind::WrapPolicy>(2, 0); - BOOST_TEST(!oss.str().empty()); - } -} - -BOOST_AUTO_TEST_SUITE_END() +/** +* Copyright (c) 2020 Intel Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +// nn_unit_test.cpp : Defines the entry point for the neural network template unit tests. + +#include "compiler.h" + +#define BOOST_TEST_MODULE nn_unit_test +TINYMIND_DISABLE_WARNING_PUSH +TINYMIND_DISABLE_WARNING_GCC_ONLY("-Wdangling-reference") +#include +TINYMIND_DISABLE_WARNING_POP + +#include +#include + +#include "portable_test_random.hpp" +#include "qformat.hpp" +#include "neuralnet.hpp" +#include "activationFunctions.hpp" +#include "fixedPointTransferFunctions.hpp" +#include "gradientClipping.hpp" +#include "weightDecay.hpp" +#include "learningRateSchedule.hpp" +#include "adam.hpp" +#include "earlyStopping.hpp" +#include "teacherForcing.hpp" +#include "dual.hpp" +#include "pinn.hpp" +#include "truncatedBPTT.hpp" +#include "conv1d.hpp" +#include "pool1d.hpp" +#include "upsample1d.hpp" +#include "conv2d.hpp" +#include "depthwiseconv2d.hpp" +#include "pointwiseconv2d.hpp" +#include "pool2d.hpp" +#include "bench/platform.hpp" +#include "bench/report.hpp" +#include "dropout.hpp" +#include "batchnorm.hpp" +#include "rmsprop.hpp" +#include "binarylayer.hpp" +#include "ternarylayer.hpp" +#include "selfattention1d.hpp" +#include "anfis.hpp" +#include "moe.hpp" +#include "fft1d.hpp" +#include "networkStats.hpp" +#include "random.hpp" +#include "nnproperties.hpp" +#include "xavier.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace tinymind { + template<> + struct TanhActivationPolicy + { + static double activationFunction(const double& value) + { + return std::tanh(value); + } + + static double activationFunctionDerivative(const double& value) + { + //Approximation for 1st derivative of tanh + return (static_cast(1.0) - (value * value)); + } + }; +} + +namespace tinymind { + template<> + struct SigmoidActivationPolicy + { + static double activationFunction(const double& value) + { + return (1.0 / (1.0 + std::exp(-value))); + } + + static double activationFunctionDerivative(const double& value) + { + return (value * (1.0 - value)); + } + }; +} + +namespace tinymind { + template<> + struct ZeroToleranceCalculator + { + static bool isWithinZeroTolerance(const double& value) + { + static const double zeroTolerance(0.004); + static const double negativeTolerance = (static_cast(-1.0) * zeroTolerance); + + return ((0 == value) || ((value < zeroTolerance) && (value > negativeTolerance))); + } + }; +} + +namespace tinymind { + template<> + struct Constants + { + static float one() + { + return 1.0f; + } + + static float negativeOne() + { + return -1.0f; + } + + static float zero() + { + return 0.0f; + } + }; + + template<> + struct Constants + { + static double one() + { + return 1.0; + } + + static double negativeOne() + { + return -1.0; + } + + static double zero() + { + return 0.0; + } + }; +} + +using namespace std; + +#define TRAINING_ITERATIONS 2000 +#define NUM_SAMPLES_AVG_ERROR 20 +#define STOP_ON_AVG_ERROR 0 +#define USE_WEIGHTS_INPUT_FILE 0 // the weights input file uses the initial values from a successful training run +#ifndef RANDOM_SEED +#define RANDOM_SEED 7U +#endif + +template +struct ValueHelper +{ + typedef typename ValueType::FullWidthValueType FullWidthValueType; + + static FullWidthValueType getErrorLimit() + { + static const FullWidthValueType ERROR_LIMIT = (1 << (ValueType::NumberOfFractionalBits - 6)); + + return ERROR_LIMIT; + } +}; + +template<> +struct ValueHelper +{ + static double getErrorLimit() + { + return 0.1; + } +}; + +template +struct UniformRealRandomNumberGenerator +{ + typedef tinymind::ValueConverter WeightConverterPolicy; + + static ValueType generateRandomWeight() + { + const double temp = generator(); + const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); + + return weight; + } + + static void seed(unsigned int s) + { + generator.seed(static_cast(s)); + } +private: + static PortableUniformReal generator; +}; + +template +PortableUniformReal UniformRealRandomNumberGenerator::generator(-1.0, 1.0, RANDOM_SEED); + +template +struct XavierUniformRandomNumberGenerator +{ + typedef tinymind::XavierWeightInitializer XavierWeightInitializerType; + typedef tinymind::ValueConverter WeightConverterPolicy; + + static ValueType generateRandomWeight() + { + static XavierWeightInitializerType xavierWeightInitializer; + const double temp = xavierWeightInitializer.generateUniformWeight(); + const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); + + return weight; + } +}; + +template +struct XavierNormalRandomNumberGenerator +{ + typedef tinymind::XavierWeightInitializer XavierWeightInitializerType; + typedef tinymind::ValueConverter WeightConverterPolicy; + + static ValueType generateRandomWeight() + { + static XavierWeightInitializerType xavierWeightInitializer; + const double temp = xavierWeightInitializer.generateNormalWeight(); + const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); + + return weight; + } +}; + +template +struct XavierUniformHeterogeneousRandomNumberGenerator +{ + typedef tinymind::XavierWeightInitializerForLayers XavierWeightInitializerType; + typedef tinymind::ValueConverter WeightConverterPolicy; + + static ValueType generateRandomWeight() + { + static XavierWeightInitializerType xavierWeightInitializer; + const double temp = xavierWeightInitializer.generateUniformWeight(); + const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); + + return weight; + } +}; + +template< + typename ValueType, + template class TransferFunctionRandomNumberGeneratorPolicy, + template class TransferFunctionHiddenNeuronActivationPolicy, + template class TransferFunctionOutputNeuronActivationPolicy, + template class TransferFunctionGatedNeuronActivationPolicy = tinymind::NullActivationPolicy, + template class TransferFunctionZeroTolerancePolicy = tinymind::ZeroToleranceCalculator, + unsigned NumberOfOutputNeurons = 1> +struct FloatingPointTransferFunctions +{ + typedef ValueType TransferFunctionsValueType; + typedef TransferFunctionRandomNumberGeneratorPolicy RandomNumberGeneratorPolicy; + typedef TransferFunctionHiddenNeuronActivationPolicy HiddenNeuronActivationPolicy; + typedef TransferFunctionOutputNeuronActivationPolicy OutputNeuronActivationPolicy; + typedef TransferFunctionGatedNeuronActivationPolicy GatedNeuronActivationPolicy; + typedef TransferFunctionZeroTolerancePolicy ZeroToleranceCalculatorPolicy; + + static const unsigned NumberOfTransferFunctionsOutputNeurons = NumberOfOutputNeurons; + + static ValueType calculateError(ValueType const* const targetValues, ValueType const* const outputValues) + { + ValueType error(0); + + //calculate overall error RMS + for (uint32_t neuron = 0; neuron < NumberOfOutputNeurons; neuron++) + { + const ValueType delta = (targetValues[neuron] - outputValues[neuron]); + error += (delta * delta); + } + + if (NumberOfOutputNeurons > 1) + { + error /= NumberOfOutputNeurons; + } + + return error; + } + + static ValueType calculateOutputGradient(const ValueType& targetValue, const ValueType& outputValue) + { + const ValueType delta = targetValue - outputValue; + + return (delta * OutputNeuronActivationPolicy::activationFunctionDerivative(outputValue)); + } + + static ValueType gateActivationFunction(const ValueType& value) + { + return GatedNeuronActivationPolicy::activationFunction(value); + } + + static ValueType generateRandomWeight() + { + return RandomNumberGeneratorPolicy::generateRandomWeight(); + } + + static ValueType hiddenNeuronActivationFunction(const ValueType& value) + { + return HiddenNeuronActivationPolicy::activationFunction(value); + } + + static ValueType hiddenNeuronActivationFunctionDerivative(const ValueType& value) + { + return HiddenNeuronActivationPolicy::activationFunctionDerivative(value); + } + + static ValueType outputNeuronActivationFunction(const ValueType& value) + { + return OutputNeuronActivationPolicy::activationFunction(value); + } + + static ValueType outputNeuronActivationFunctionDerivative(const ValueType& value) + { + return OutputNeuronActivationPolicy::activationFunctionDerivative(value); + } + + static ValueType initialAccelerationRate() + { + ValueType rate(0.1); + + return rate; + } + + static ValueType initialBiasOutputValue() + { + const ValueType value(1.0); + + return value; + } + + static ValueType initialDeltaWeight() + { + const ValueType delta(0); + + return delta; + } + + static ValueType initialGradientValue() + { + const ValueType gradient(0); + + return gradient; + } + + static ValueType initialLearningRate() + { + ValueType rate(0.15); + + return rate; + } + + static ValueType initialMomentumRate() + { + ValueType rate(0.5); + + return rate; + } + + static ValueType initialOutputValue() + { + const ValueType value(0); + + return value; + } + + static bool isWithinZeroTolerance(const ValueType& value) + { + return ZeroToleranceCalculatorPolicy::isWithinZeroTolerance(value); + } + + static ValueType negate(const ValueType& value) + { + return (-1.0 * value); + } + + static ValueType neuronActivationFunction(const ValueType& value) + { + return HiddenNeuronActivationPolicy::activationFunction(value); + } + + static ValueType neuronActivationFunctionDerivative(const ValueType& value) + { + return HiddenNeuronActivationPolicy::activationFunctionDerivative(value); + } + + static ValueType noOpDeltaWeight() + { + const ValueType value(1.0); + + return value; + } + + static ValueType noOpWeight() + { + const ValueType value(1.0); + + return value; + } +}; + +template +static void generateXorValues(T* values, T* output) +{ + const uint32_t x = (rand() & 1); + const uint32_t y = (rand() & 1); + const uint32_t z = x ^ y; + + *values = static_cast(x); + ++values; + *values = static_cast(y); + *output = z; +} + +template +static void generateFixedPointXorValues(T* values, T* output) +{ + const uint32_t x = (rand() & 1); + const uint32_t y = (rand() & 1); + const uint32_t z = x ^ y; + + // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and + // left-shifting a negative value is undefined behavior before C++20. + const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; + *values = static_cast(x * scale); + ++values; + *values = static_cast(y * scale); + *output = (z * scale); +} + +template +static void generateAndValues(T* values, T* output) +{ + const uint32_t x = (rand() & 1); + const uint32_t y = (rand() & 1); + const uint32_t z = x & y; + + *values = static_cast(x); + ++values; + *values = static_cast(y); + *output = z; +} + +template +static void generateFixedPointAndValues(T* values, T* output) +{ + const uint32_t x = (rand() & 1); + const uint32_t y = (rand() & 1); + const uint32_t z = x & y; + + // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and + // left-shifting a negative value is undefined behavior before C++20. + const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; + *values = static_cast(x * scale); + ++values; + *values = static_cast(y * scale); + *output = (z * scale); +} + +template +static void generateOrValues(T* values, T* output) +{ + const uint32_t x = (rand() & 1); + const uint32_t y = (rand() & 1); + const uint32_t z = x | y; + + *values = static_cast(x); + ++values; + *values = static_cast(y); + *output = z; +} + +template +static void generateFixedPointOrValues(T* values, T* output) +{ + const uint32_t x = (rand() & 1); + const uint32_t y = (rand() & 1); + const uint32_t z = x | y; + + // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and + // left-shifting a negative value is undefined behavior before C++20. + const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; + *values = static_cast(x * scale); + ++values; + *values = static_cast(y * scale); + *output = (z * scale); +} + +template +static void generateFixedPointNorValues(T* values, T* output) +{ + const uint32_t x = (rand() & 1); + const uint32_t y = (rand() & 1); + const uint32_t z = !(x | y); + + // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and + // left-shifting a negative value is undefined behavior before C++20. + const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; + *values = static_cast(x * scale); + ++values; + *values = static_cast(y * scale); + *output = (z * scale); +} + +template +static void generateRecurrentValues(T* values, T* output) +{ + static int32_t x = -1; + static int32_t y = 0; + const int32_t z = x + y; + + *values = static_cast(x); + ++values; + *values = static_cast(y); + *output = static_cast(z); + + if (++x > 1) + x = -1; + + if (++y > 1) + y = -1; +} + +template +static void generateFixedPointRecurrentValues(T* values, T* output) +{ + static int32_t x = -1; + static int32_t y = 0; + const int32_t z = x + y; + + // Scale by 2^frac via multiply, not left shift: x/y/z may be negative and + // left-shifting a negative value is undefined behavior before C++20. + const int32_t scale = static_cast(1) << T::NumberOfFractionalBits; + *values = static_cast(x * scale); + ++values; + *values = static_cast(y * scale); + *output = (z * scale); + + if (++x > 1) + x = -1; + + if (++y > 1) + y = -1; +} + +template +static void testFloatingPointNN( NeuralNetworkType& neuralNetwork, + void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), + char const* const path, + const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; + static const ValueType ERROR_LIMIT = 0.1; + ofstream results(path); + ofstream weightsOutputFile; + std::string weightsOutputPath(path); + std::deque errors; + ValueType error; + + ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; + double learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; + + weightsOutputPath.replace(weightsOutputPath.find("."), std::string::npos, "_weights.txt"); + + tinymind::NetworkPropertiesFileManager::writeHeader(results); + + for (int i = 0; i < numberOfTrainingIterations; ++i) + { + pValuesFn(values, output); + + neuralNetwork.feedForward(&values[0]); + error = neuralNetwork.calculateError(&output[0]); + if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + neuralNetwork.trainNetwork(&output[0]); + } + neuralNetwork.getLearnedValues(&learnedValues[0]); + + errors.push_front(error); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + +#if STOP_ON_AVG_ERROR + const double totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + if (averageError <= ERROR_LIMIT) + { + break; + } +#endif // STOP_ON_AVG_ERROR + } + + tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); + results << error << std::endl; + } + + weightsOutputFile.open(weightsOutputPath.c_str()); + + tinymind::NetworkPropertiesFileManager::storeNetworkWeights(neuralNetwork, weightsOutputFile); + + weightsOutputFile.close(); + + const double totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +template +static void testFloatingPointNN_Xor(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + testFloatingPointNN(neuralNetwork, generateXorValues, path, numberOfTrainingIterations); +} + +template +static void testFloatingPointNN_And(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + testFloatingPointNN(neuralNetwork, generateAndValues, path, numberOfTrainingIterations); +} + +template +static void testFloatingPointNN_Or(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + testFloatingPointNN(neuralNetwork, generateOrValues, path, numberOfTrainingIterations); +} + +template +static void testFixedPointNeuralNetwork( NeuralNetworkType& neuralNetwork, + void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), + char const* const path, + const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ +#if USE_WEIGHTS_INPUT_FILE == 1 + typedef tinymind::NetworkPropertiesFileManager NetworkPropertiesFileManagerType; +#endif // USE_WEIGHTS_INPUT_FILE + typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + typedef ValueHelper ValueHelperType; + static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); + ofstream results(path); + ofstream weightsOutputFile; + std::string initialWeightsInputPath(path); + std::string weightsOutputPath(path); + std::string binaryWeightsOutputPath(path); + ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + + initialWeightsInputPath.replace(weightsOutputPath.find("."), std::string::npos, "_initial_weights.txt"); + weightsOutputPath.replace(weightsOutputPath.find("."), std::string::npos, "_weights.txt"); + binaryWeightsOutputPath.replace(binaryWeightsOutputPath.find(".txt"), std::string::npos, ".bin"); + + tinymind::NetworkPropertiesFileManager::writeHeader(results); + +#if USE_WEIGHTS_INPUT_FILE + initialWeightsInputPath.insert(0, "../input/"); + ifstream weightsInputFile(initialWeightsInputPath); + if (weightsInputFile.is_open()) + { + NetworkPropertiesFileManagerType::template loadNetworkWeights(neuralNetwork, weightsInputFile); + } + else + { + cout << "Did not find initial weights input file: " << initialWeightsInputPath << endl; + } +#endif + + for (int i = 0; i < numberOfTrainingIterations; ++i) + { + pValuesFn(values, output); + + neuralNetwork.feedForward(&values[0]); + error = neuralNetwork.calculateError(&output[0]); + if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + neuralNetwork.trainNetwork(&output[0]); + } + neuralNetwork.getLearnedValues(&learnedValues[0]); + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + +#if STOP_ON_AVG_ERROR + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + if (averageError <= ERROR_LIMIT) + { + break; + } +#endif // STOP_ON_AVG_ERROR + } + + tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); + results << error << std::endl; + } + + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + + weightsOutputFile.open(weightsOutputPath.c_str()); + + tinymind::NetworkPropertiesFileManager::storeNetworkWeights(neuralNetwork, weightsOutputFile); + + weightsOutputFile.close(); + + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +template +static void testFixedPointNeuralNetwork_Xor(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointXorValues, path, numberOfTrainingIterations); +} + +template +static void testFixedPointNeuralNetwork_And(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointAndValues, path, numberOfTrainingIterations); +} + +template +static void testFixedPointNeuralNetwork_Or(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointOrValues, path, numberOfTrainingIterations); +} + +template +static void testFixedPointNeuralNetwork_Nor(NeuralNetworkType& neuralNetwork, char const* const path, const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + testFixedPointNeuralNetwork(neuralNetwork, generateFixedPointNorValues, path, numberOfTrainingIterations); +} + +template +static void testFixedPointNeuralNetwork_No_Train( NeuralNetworkType& neuralNetwork, + void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), + char const* const path, + char const* const weightsInputPath) +{ + typedef tinymind::NetworkPropertiesFileManager NetworkPropertiesFileManagerType; + typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + typedef ValueHelper ValueHelperType; + static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); + ofstream results(path); + ifstream weightsInputFile(weightsInputPath); + ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + + NetworkPropertiesFileManagerType::template loadNetworkWeights(neuralNetwork, weightsInputFile); + tinymind::NetworkPropertiesFileManager::writeHeader(results); + + for (int i = 0; i < TRAINING_ITERATIONS; ++i) + { + pValuesFn(values, output); + + neuralNetwork.feedForward(&values[0]); + error = neuralNetwork.calculateError(&output[0]); + neuralNetwork.getLearnedValues(&learnedValues[0]); + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + +#if STOP_ON_AVG_ERROR + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + if (averageError <= ERROR_LIMIT) + { + break; + } +#endif // STOP_ON_AVG_ERROR + } + + tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); + results << error << std::endl; + } + + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +template +static void testFixedPointNeuralNetwork_Xor_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) +{ + testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointXorValues, path, weightsInputPath); +} + +template +static void testFixedPointNeuralNetwork_And_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) +{ + testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointAndValues, path, weightsInputPath); +} + +template +static void testFixedPointNeuralNetwork_Or_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) +{ + testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointOrValues, path, weightsInputPath); +} + +template +static void testFixedPointNeuralNetwork_Nor_No_Train(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) +{ + testFixedPointNeuralNetwork_No_Train(neuralNetwork, generateFixedPointNorValues, path, weightsInputPath); +} + +template +static void testFixedPointNeuralNetwork_No_Train_Float_Weights( NeuralNetworkType& neuralNetwork, + void(*pValuesFn)(typename NeuralNetworkType::NeuralNetworkValueType*, typename NeuralNetworkType::NeuralNetworkValueType*), + char const* const path, + char const* const weightsInputPath) +{ + typedef tinymind::NetworkPropertiesFileManager NetworkPropertiesFileManagerType; + typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + typedef ValueHelper ValueHelperType; + static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); + ofstream results(path); + ifstream weightsInputFile(weightsInputPath); + ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + + NetworkPropertiesFileManagerType::template loadNetworkWeights(neuralNetwork, weightsInputFile); + tinymind::NetworkPropertiesFileManager::writeHeader(results); + + for (int i = 0; i < TRAINING_ITERATIONS; ++i) + { + pValuesFn(values, output); + + neuralNetwork.feedForward(&values[0]); + error = neuralNetwork.calculateError(&output[0]); + neuralNetwork.getLearnedValues(&learnedValues[0]); + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + +#if STOP_ON_AVG_ERROR + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + if (averageError <= ERROR_LIMIT) + { + break; + } +#endif // STOP_ON_AVG_ERROR + } + + tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); + results << error << std::endl; + } + + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +template +static void testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) +{ + testFixedPointNeuralNetwork_No_Train_Float_Weights(neuralNetwork, generateFixedPointXorValues, path, weightsInputPath); +} + +template +static void testFixedPointNeuralNetwork_And_No_Train_Float_Weights(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) +{ + testFixedPointNeuralNetwork_No_Train_Float_Weights(neuralNetwork, generateFixedPointAndValues, path, weightsInputPath); +} + +template +static void testFixedPointNeuralNetwork_Or_No_Train_Float_Weights(NeuralNetworkType& neuralNetwork, char const* const path, char const* const weightsInputPath) +{ + testFixedPointNeuralNetwork_No_Train_Float_Weights(neuralNetwork, generateFixedPointOrValues, path, weightsInputPath); +} + +template +static void testNeuralNetwork_Recurrent(NeuralNetworkType& neuralNetwork, char const* const path) +{ + typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + typedef ValueHelper ValueHelperType; + static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); + ofstream results(path); + ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + + tinymind::NetworkPropertiesFileManager::writeHeader(results); + + for (int i = 0; i < TRAINING_ITERATIONS; ++i) + { + generateFixedPointRecurrentValues(values, output); + + neuralNetwork.feedForward(&values[0]); + error = neuralNetwork.calculateError(&output[0]); + if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + neuralNetwork.trainNetwork(&output[0]); + } + neuralNetwork.getLearnedValues(&learnedValues[0]); + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + +#if STOP_ON_AVG_ERROR + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + if (averageError <= ERROR_LIMIT) + { + break; + } +#endif // STOP_ON_AVG_ERROR + } + + tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); + results << error << std::endl; + } + + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +template +static void testFloatingPointNeuralNetwork_Recurrent(NeuralNetworkType& neuralNetwork, char const* const path, + const int numberOfTrainingIterations = TRAINING_ITERATIONS) +{ + typedef typename NeuralNetworkType::NeuralNetworkValueType ValueType; + typedef double FullWidthValueType; + typedef ValueHelper ValueHelperType; + static const FullWidthValueType ERROR_LIMIT = ValueHelperType::getErrorLimit(); + ofstream results(path); + ValueType values[NeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[NeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[NeuralNetworkType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + + tinymind::NetworkPropertiesFileManager::writeHeader(results); + + for (int i = 0; i < numberOfTrainingIterations; ++i) + { + generateRecurrentValues(values, output); + + neuralNetwork.feedForward(&values[0]); + error = neuralNetwork.calculateError(&output[0]); + if (!NeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + neuralNetwork.trainNetwork(&output[0]); + } + neuralNetwork.getLearnedValues(&learnedValues[0]); + + errors.push_front(error); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + +#if STOP_ON_AVG_ERROR + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + if (averageError <= ERROR_LIMIT) + { + break; + } +#endif // STOP_ON_AVG_ERROR + } + + tinymind::NetworkPropertiesFileManager::storeNetworkProperties(neuralNetwork, results, &output[0], &learnedValues[0]); + results << error << std::endl; + } + + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +BOOST_AUTO_TEST_SUITE(test_suite_nn) + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_xavier_uniform) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + XavierUniformRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_xor_xavier_uniform.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_xavier_normal) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + XavierNormalRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_xor_xavier_normal.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_xavier_heterogeneous) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::HiddenLayers<6, 4> HiddenLayersDesc; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + XavierUniformHeterogeneousRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + HiddenLayersDesc, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointHeterogeneousNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_xor_xavier_heterogeneous.txt"; + FixedPointHeterogeneousNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_nn_copy) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_xor.txt"; + char const* const pathCopy = "output/nn_fixed_xor_copy.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + FixedPointMultiLayerPerceptronNetworkType nnCopy; + + testFixedPointNeuralNetwork_Xor(nn, path); + nnCopy.setWeights(nn); + testFixedPointNeuralNetwork_Xor(nnCopy, pathCopy); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_And(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_or.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Or(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_nor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_nor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Nor(nn, path); +} + +// Locks the frozen draw sequence. Every training test in this file is +// calibrated against these exact numbers, so a change here silently re-tunes +// all of them -- this makes that change fail loudly instead. The expected +// values are libstdc++'s std::uniform_real_distribution(-1, 1) over +// std::default_random_engine seeded with RANDOM_SEED, captured when the +// generator was made implementation-independent. +BOOST_AUTO_TEST_CASE(test_case_portable_uniform_real_matches_frozen_sequence) +{ + static const double expected[8] = { + 0.84152903393058454, + -0.57889815353756857, + 0.065428608837350577, + 0.5041060362322427, + 0.085700543449050981, + 0.27182920924978693, + -0.51599045256081322, + 0.41580270676876796 + }; + PortableUniformReal generator(-1.0, 1.0, RANDOM_SEED); + + for (size_t i = 0; i < 8; ++i) + { + BOOST_TEST(generator() == expected[i]); + } +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + static const bool TRAINABLE = false; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_no_train_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor_No_Train(nn, path, "output/nn_fixed_xor_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + static const bool TRAINABLE = false; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_no_train_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_And_No_Train(nn, path, "output/nn_fixed_and_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + static const bool TRAINABLE = false; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_no_train_or.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Or_No_Train(nn, path, "output/nn_fixed_or_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_nor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + static const bool TRAINABLE = false; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_no_train_nor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Nor_No_Train(nn, path, "output/nn_fixed_nor_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_5_hidden_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_5_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_5_hidden_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_5_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_And(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_5_hidden_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_5_or.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Or(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_16_16_nn_5_hidden_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 16; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_16_16_5_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_16_16_nn_5_hidden_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 16; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_16_16_5_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_And(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_16_16_nn_5_hidden_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 16; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_16_16_5_or.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Or(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_8_24_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_8_24_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_And(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_8_24_or.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Or(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_2_8_24_nn_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = true; + static const size_t BATCH_SIZE = 2; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE, + BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_batch_2_8_24_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + const ValueType learningRate(nn.getLearningRate() / 4); + const ValueType accelerationRate(nn.getAccelerationRate() / 4); + const ValueType momentumRate(nn.getMomentumRate() / 4); + + nn.setLearningRate(learningRate); + nn.setAccelerationRate(accelerationRate); + nn.setMomentumRate(momentumRate); + + testFixedPointNeuralNetwork_Xor(nn, path, 10000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_2_8_24_nn_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = true; + static const size_t BATCH_SIZE = 2; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE, + BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_batch_2_8_24_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + const ValueType learningRate(nn.getLearningRate() / 4); + const ValueType accelerationRate(nn.getAccelerationRate() / 4); + const ValueType momentumRate(nn.getMomentumRate() / 4); + + nn.setLearningRate(learningRate); + nn.setAccelerationRate(accelerationRate); + nn.setMomentumRate(momentumRate); + + testFixedPointNeuralNetwork_And(nn, path, 10000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_2_8_24_nn_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = true; + static const size_t BATCH_SIZE = 2; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE, + BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_batch_2_8_24_or.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + const ValueType learningRate(nn.getLearningRate() / 4); + const ValueType accelerationRate(nn.getAccelerationRate() / 4); + const ValueType momentumRate(nn.getMomentumRate() / 4); + + nn.setLearningRate(learningRate); + nn.setAccelerationRate(accelerationRate); + nn.setMomentumRate(momentumRate); + + testFixedPointNeuralNetwork_Or(nn, path, 10000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_4_8_24_nn_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = true; + static const size_t BATCH_SIZE = 4; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE, + BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_batch_4_8_24_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + const ValueType learningRate(nn.getLearningRate() / 4); + const ValueType accelerationRate(nn.getAccelerationRate() / 4); + const ValueType momentumRate(nn.getMomentumRate() / 4); + + nn.setLearningRate(learningRate); + nn.setAccelerationRate(accelerationRate); + nn.setMomentumRate(momentumRate); + + testFixedPointNeuralNetwork_Xor(nn, path, 10000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_4_8_24_nn_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = true; + static const size_t BATCH_SIZE = 4; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE, + BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_batch_4_8_24_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + const ValueType learningRate(nn.getLearningRate() / 4); + const ValueType accelerationRate(nn.getAccelerationRate() / 4); + const ValueType momentumRate(nn.getMomentumRate() / 4); + + nn.setLearningRate(learningRate); + nn.setAccelerationRate(accelerationRate); + nn.setMomentumRate(momentumRate); + + testFixedPointNeuralNetwork_And(nn, path, 10000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_batch_4_8_24_nn_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = true; + static const size_t BATCH_SIZE = 4; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE, + BATCH_SIZE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_batch_4_8_24_or.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + const ValueType learningRate(nn.getLearningRate() / 4); + const ValueType accelerationRate(nn.getAccelerationRate() / 4); + const ValueType momentumRate(nn.getMomentumRate() / 4); + + nn.setLearningRate(learningRate); + nn.setAccelerationRate(accelerationRate); + nn.setMomentumRate(momentumRate); + + testFixedPointNeuralNetwork_Or(nn, path, 10000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_elman_nn) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::ElmanNetwork< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointElmanNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_elman.txt"; + FixedPointElmanNetworkType nn; + + testNeuralNetwork_Recurrent(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_floatingpoint_elman_nn) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::ElmanNetwork< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointElmanNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_elman.txt"; + FloatingPointElmanNetworkType nn; + + testFloatingPointNeuralNetwork_Recurrent(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_xor.txt"; + FloatingPointMultiLayerPerceptronNetworkType nn; + + testFloatingPointNN_Xor(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_and.txt"; + FloatingPointMultiLayerPerceptronNetworkType nn; + + testFloatingPointNN_And(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_or) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_or.txt"; + FloatingPointMultiLayerPerceptronNetworkType nn; + + testFloatingPointNN_Or(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_float_weights_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = false; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_no_train_float_weights_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_xor_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_no_train_float_weights_and) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = false; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_no_train_float_weights_and.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_And_No_Train_Float_Weights(nn, path, "output/nn_float_and_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_floatingpoint_nn_relu_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_relu_xor.txt"; + FloatingPointMultiLayerPerceptronNetworkType nn; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + testFloatingPointNN_Xor(nn, path, 200000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_relu_xor_no_train) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = false; + static const size_t NUMBER_OF_FIXED_BITS = 16; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + tinymind::NullRandomNumberPolicy, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_relu_xor_no_train.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_relu_xor_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_24_nn_relu_xor_no_train) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = false; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 24; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + tinymind::NullRandomNumberPolicy, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_8_24_relu_xor_no_train.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_relu_xor_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_8_8_nn_relu_xor_no_train) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const bool TRAINABLE = false; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + tinymind::NullRandomNumberPolicy, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + TRAINABLE> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_8_8_relu_xor_no_train.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_relu_xor_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_floatingpoint_2_hidden_nn_relu_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_2_hidden_relu_xor.txt"; + FloatingPointMultiLayerPerceptronNetworkType nn; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + testFloatingPointNN_Xor(nn, path, 100000); +} + +BOOST_AUTO_TEST_CASE(test_case_floatingpoint_2_hidden_nn_relu_xor_copy) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_2_hidden_relu_xor.txt"; + char const* const pathCopy = "output/nn_float_2_hidden_relu_xor_copy.txt"; + FloatingPointMultiLayerPerceptronNetworkType nn; + FloatingPointMultiLayerPerceptronNetworkType nnCopy; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + testFloatingPointNN_Xor(nn, path, 100000); + nnCopy.setWeights(nn); + testFloatingPointNN_Xor(nnCopy, pathCopy, 100000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_2_hidden_nn_relu_xor_no_train) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 5; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 16; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 16; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + tinymind::NullRandomNumberPolicy, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_2_hidden_relu_xor_no_train.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor_No_Train_Float_Weights(nn, path, "output/nn_float_2_hidden_relu_xor_weights.txt"); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_sigmoid_xor) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::SigmoidActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_sigmoid_xor.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path, 75000); +} + +BOOST_AUTO_TEST_CASE(test_case_fixedpoint_nn_xor_4_hidden_layers) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_HIDDEN_LAYERS = 4; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointMultiLayerPerceptronNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_xor_4_hidden_layers.txt"; + FixedPointMultiLayerPerceptronNetworkType nn; + + testFixedPointNeuralNetwork_Xor(nn, path, TRAINING_ITERATIONS * 100); +} + +BOOST_AUTO_TEST_SUITE_END() + +typedef tinymind::QValue<8, 8, true, tinymind::TruncatePolicy, tinymind::MinMaxSaturatePolicy> SignedQ8_8SatPolicyType; +typedef tinymind::QValue<8, 8, false, tinymind::TruncatePolicy, tinymind::MinMaxSaturatePolicy> UnsignedQ8_8SatPolicyType; + +BOOST_AUTO_TEST_SUITE(SoftMaxLinearClassificationTests) + +// Test case: 2x2 grayscale image with 2 classes +BOOST_AUTO_TEST_CASE(test_softmax_2x2_image_2_classes) +{ + // 2x2 grayscale image (4 input features) + const size_t inputSize = 4; // 2x2 pixels + const size_t numClasses = 2; // Binary classification + + // Example: grayscale values normalized to Q8.8 format + // Image pixels: [100, 150, 200, 50] -> normalized to [-1, 1] range + SignedQ8_8SatPolicyType inputImage[inputSize]; + inputImage[0] = SignedQ8_8SatPolicyType(-1, 0); // Dark pixel (-1.0) + inputImage[1] = SignedQ8_8SatPolicyType(0, 0); // Medium pixel (0.0) + inputImage[2] = SignedQ8_8SatPolicyType(1, 0); // Bright pixel (1.0) + inputImage[3] = SignedQ8_8SatPolicyType(-1, 128); // Dark-ish pixel (-0.5) + + // Linear layer weights: [numClasses x inputSize] + // Class 0 weights favor dark pixels, Class 1 favors bright pixels + SignedQ8_8SatPolicyType weights[numClasses][inputSize] = { + // Class 0: favors dark pixels + {SignedQ8_8SatPolicyType(1, 0), SignedQ8_8SatPolicyType(0, 128), + SignedQ8_8SatPolicyType(-1, 0), SignedQ8_8SatPolicyType(1, 128)}, + // Class 1: favors bright pixels + {SignedQ8_8SatPolicyType(-1, 0), SignedQ8_8SatPolicyType(0, 128), + SignedQ8_8SatPolicyType(2, 0), SignedQ8_8SatPolicyType(-1, 128)} + }; + + // Bias terms for each class + SignedQ8_8SatPolicyType bias[numClasses] = { + SignedQ8_8SatPolicyType(0, 128), // 0.5 bias for class 0 + SignedQ8_8SatPolicyType(-1, 128) // -0.5 bias for class 1 + }; + + // Linear transformation: logits = weights * input + bias + SignedQ8_8SatPolicyType logits[numClasses]; + for (size_t c = 0; c < numClasses; ++c) + { + logits[c] = bias[c]; + for (size_t i = 0; i < inputSize; ++i) + { + logits[c] += weights[c][i] * inputImage[i]; + } + } + + // Apply SoftMax + SignedQ8_8SatPolicyType probabilities[numClasses]; + tinymind::SoftmaxActivationPolicy::activationFunction( + logits, + probabilities, + numClasses + ); + + // Verify: Sum of probabilities should be close to 1.0 + SignedQ8_8SatPolicyType sum(0, 0); + for (size_t c = 0; c < numClasses; ++c) + { + sum += probabilities[c]; + } + + // Expected sum: 1.0 in Q8.8 = 0x0100 = 256 + SignedQ8_8SatPolicyType expectedSum(1, 0); + + // Allow tolerance due to Q-format rounding and exp approximation + // Tolerance: ±3 LSB = ±0.01171875 in Q8.8 + const int tolerance = 3; + int sumValue = sum.getValue(); + int expectedValue = expectedSum.getValue(); + + // Test to make sure sum of probabilities should be ~1.0 + BOOST_TEST(sumValue >= (expectedValue - tolerance)); + BOOST_TEST(sumValue <= (expectedValue + tolerance)); + + // Verify each probability is in [0, 1] range + for (size_t c = 0; c < numClasses; ++c) + { + BOOST_TEST(probabilities[c].getValue() >= 0); + BOOST_TEST(probabilities[c].getValue() <= SignedQ8_8SatPolicyType(1, 0).getValue()); + } +} + +// ========================================================================= +// Tests for the new NeuralNetwork class with heterogeneous hidden layers +// ========================================================================= + +BOOST_AUTO_TEST_CASE(test_case_new_nn_uniform_hidden_layers_xor) +{ + // NeuralNetwork with uniform hidden layers should work like MultilayerPerceptron + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + tinymind::HiddenLayers<5>, + 1, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + NNType nn; + ValueType error; + + nn.setLearningRate(0.1); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + ValueType values[2]; + ValueType output[1]; + + for (int i = 0; i < 10000; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + if (!TransferFunctionsType::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + } + + // Verify the network has learned XOR + std::deque errors; + + for (int i = 0; i < 100; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + errors.push_back(error); + } + + double totalError = 0.0; + for (size_t i = 0; i < errors.size(); ++i) + { + totalError += std::abs(errors[i]); + } + double averageError = totalError / static_cast(errors.size()); + BOOST_TEST(averageError <= 0.1); +} + +BOOST_AUTO_TEST_CASE(test_case_new_nn_2_uniform_hidden_layers_xor) +{ + // NeuralNetwork with 2 uniform hidden layers + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + tinymind::HiddenLayers<5, 5>, + 1, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + NNType nn; + ValueType error; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + ValueType values[2]; + ValueType output[1]; + + for (int i = 0; i < 100000; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + if (!TransferFunctionsType::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + } + + // Verify convergence + std::deque errors; + for (int i = 0; i < 100; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + errors.push_back(error); + } + + double totalError = 0.0; + for (size_t i = 0; i < errors.size(); ++i) + { + totalError += std::abs(errors[i]); + } + double averageError = totalError / static_cast(errors.size()); + BOOST_TEST(averageError <= 0.1); +} + +BOOST_AUTO_TEST_CASE(test_case_new_nn_heterogeneous_hidden_layers_xor) +{ + // NeuralNetwork with heterogeneous hidden layers: 8 neurons, then 4 neurons + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + tinymind::HiddenLayers<8, 4>, + 1, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + NNType nn; + ValueType error; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + ValueType values[2]; + ValueType output[1]; + + for (int i = 0; i < 100000; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + if (!TransferFunctionsType::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + } + + // Verify convergence + std::deque errors; + for (int i = 0; i < 100; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + errors.push_back(error); + } + + double totalError = 0.0; + for (size_t i = 0; i < errors.size(); ++i) + { + totalError += std::abs(errors[i]); + } + double averageError = totalError / static_cast(errors.size()); + BOOST_TEST(averageError <= 0.1); +} + +BOOST_AUTO_TEST_CASE(test_case_new_nn_3_heterogeneous_hidden_layers_xor) +{ + // NeuralNetwork with 3 heterogeneous hidden layers: 10, 5, 3 + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + tinymind::HiddenLayers<10, 5, 3>, + 1, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + NNType nn; + ValueType error; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + ValueType values[2]; + ValueType output[1]; + + for (int i = 0; i < 100000; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + if (!TransferFunctionsType::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + } + + // Verify convergence + std::deque errors; + for (int i = 0; i < 100; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + errors.push_back(error); + } + + double totalError = 0.0; + for (size_t i = 0; i < errors.size(); ++i) + { + totalError += std::abs(errors[i]); + } + double averageError = totalError / static_cast(errors.size()); + BOOST_TEST(averageError <= 0.1); +} + +BOOST_AUTO_TEST_CASE(test_case_new_nn_uniform_alias_xor) +{ + // Verify UniformHiddenLayersAlias works correctly + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + typename tinymind::UniformHiddenLayersAlias<2, 5>::type, + 1, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + NNType nn; + ValueType error; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + ValueType values[2]; + ValueType output[1]; + + for (int i = 0; i < 100000; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + if (!TransferFunctionsType::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + } + + // Verify convergence + std::deque errors; + for (int i = 0; i < 100; ++i) + { + generateXorValues(values, output); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + errors.push_back(error); + } + + double totalError = 0.0; + for (size_t i = 0; i < errors.size(); ++i) + { + totalError += std::abs(errors[i]); + } + double averageError = totalError / static_cast(errors.size()); + BOOST_TEST(averageError <= 0.1); +} + +BOOST_AUTO_TEST_CASE(test_case_new_nn_not_trainable) +{ + // Verify non-trainable NeuralNetwork compiles and runs feedForward + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + tinymind::HiddenLayers<4, 3>, + 1, + TransferFunctionsType, + false> NNType; + NNType nn; + + ValueType values[2] = {1.0, 0.0}; + nn.feedForward(&values[0]); + + // Just verify it runs without crashing + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_new_nn_heterogeneous_2_hidden_layers_weight_copy) +{ + // NeuralNetwork with heterogeneous hidden layers (8, 4): train, copy weights, verify copy + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + tinymind::HiddenLayers<8, 4>, + 1, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_hetero_2_hidden_xor.txt"; + char const* const pathCopy = "output/nn_float_hetero_2_hidden_xor_copy.txt"; + NNType nn; + NNType nnCopy; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + testFloatingPointNN_Xor(nn, path, 100000); + nnCopy.setWeights(nn); + testFloatingPointNN_Xor(nnCopy, pathCopy, 100000); +} + +BOOST_AUTO_TEST_CASE(test_case_new_nn_heterogeneous_3_hidden_layers_weight_copy) +{ + // NeuralNetwork with 3 heterogeneous hidden layers (10, 5, 3): train, copy weights, verify copy + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::ReluActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::NeuralNetwork< ValueType, + 2, + tinymind::HiddenLayers<10, 5, 3>, + 1, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_hetero_3_hidden_xor.txt"; + char const* const pathCopy = "output/nn_float_hetero_3_hidden_xor_copy.txt"; + NNType nn; + NNType nnCopy; + + nn.setLearningRate(0.005); + nn.setAccelerationRate(0.03); + nn.setMomentumRate(0.16); + + testFloatingPointNN_Xor(nn, path, 100000); + nnCopy.setWeights(nn); + testFloatingPointNN_Xor(nnCopy, pathCopy, 100000); +} + +// ========================================================================= +// Tests for RecurrentNeuralNetwork and ElmanNeuralNetwork +// ========================================================================= + +BOOST_AUTO_TEST_CASE(test_case_elman_neural_network_floating_point) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::ElmanNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointElmanNeuralNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_elman_neural_network.txt"; + FloatingPointElmanNeuralNetworkType nn; + + testFloatingPointNeuralNetwork_Recurrent(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_elman_neural_network_fixed_point) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t NUMBER_OF_FIXED_BITS = 8; + static const size_t NUMBER_OF_FRACTIONAL_BITS = 8; + typedef tinymind::QValue ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::ElmanNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + NUMBER_OF_NEURONS_PER_HIDDEN_LAYER, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointElmanNeuralNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_fixed_elman_neural_network.txt"; + FixedPointElmanNeuralNetworkType nn; + + testNeuralNetwork_Recurrent(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_recurrent_neural_network_floating_point) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::RecurrentNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers<3>, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> RecurrentNNType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_recurrent_neural_network.txt"; + RecurrentNNType nn; + + testFloatingPointNeuralNetwork_Recurrent(nn, path); +} + +BOOST_AUTO_TEST_CASE(test_case_recurrent_neural_network_heterogeneous_layers) +{ + // RecurrentNeuralNetwork with heterogeneous hidden layers + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::RecurrentNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers<4, 3>, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> RecurrentNNType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_recurrent_nn_hetero.txt"; + RecurrentNNType nn; + + testFloatingPointNeuralNetwork_Recurrent(nn, path); +} + +// ========================================================================= +// Tests for LstmNeuralNetwork +// ========================================================================= + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_floating_point) +{ + // LSTM networks require more training iterations than simple RNNs due to + // the additional gate parameters (input, forget, output gates) that must + // all converge together. + static const int LSTM_TRAINING_ITERATIONS = 5000; + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointLstmNeuralNetworkType; + srand(RANDOM_SEED); + char const* const path = "output/nn_float_lstm_neural_network.txt"; + FloatingPointLstmNeuralNetworkType nn; + + testFloatingPointNeuralNetwork_Recurrent(nn, path, LSTM_TRAINING_ITERATIONS); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_fixed_point) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointLstmNeuralNetworkType; + srand(RANDOM_SEED); + FixedPointLstmNeuralNetworkType nn; + + ValueType values[FixedPointLstmNeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[FixedPointLstmNeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[FixedPointLstmNeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType error; + ValueType firstError; + bool firstErrorCaptured = false; + + for (int i = 0; i < TRAINING_ITERATIONS; ++i) + { + generateFixedPointRecurrentValues(values, output); + + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + + if (!firstErrorCaptured) + { + firstError = error; + firstErrorCaptured = true; + } + + if (!FixedPointLstmNeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + nn.getLearnedValues(&learnedValues[0]); + } + + // Verify feedforward produces valid output + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + BOOST_TEST(learnedValues[0].getValue() != 0); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_multi_layer) +{ + // Test multi-layer LSTM: two LSTM hidden layers (8 and 4 neurons). + // Both inner and last hidden layers should be LSTM layers with their + // own recurrent connections and gate weights. + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers<8, 4>, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> MultiLayerLstmType; + srand(RANDOM_SEED); + MultiLayerLstmType nn; + + ValueType values[MultiLayerLstmType::NumberOfInputLayerNeurons]; + ValueType output[MultiLayerLstmType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[MultiLayerLstmType::NumberOfOutputLayerNeurons]; + + // Verify feedforward works with multi-layer LSTM + values[0] = 1.0; + values[1] = 0.0; + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + BOOST_TEST(!std::isnan(learnedValues[0])); + BOOST_TEST(!std::isinf(learnedValues[0])); + + // Verify output changes with different input + (void)learnedValues[0]; // first output verified above + values[0] = 0.0; + values[1] = 1.0; + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + BOOST_TEST(!std::isnan(learnedValues[0])); + BOOST_TEST(!std::isinf(learnedValues[0])); + + // Verify successive calls produce different output (LSTM state accumulates) + const double secondOutput = learnedValues[0]; + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + BOOST_TEST(learnedValues[0] != secondOutput); + + // Verify training doesn't crash + output[0] = 1.0; + nn.feedForward(&values[0]); + const ValueType error = nn.calculateError(&output[0]); + nn.trainNetwork(&output[0]); + BOOST_TEST(!std::isnan(error)); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_feedforward_produces_output) +{ + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> LstmNNType; + srand(RANDOM_SEED); + LstmNNType nn; + + ValueType values[LstmNNType::NumberOfInputLayerNeurons]; + ValueType output[LstmNNType::NumberOfOutputLayerNeurons]; + + values[0] = 1.0; + values[1] = 0.0; + + nn.feedForward(&values[0]); + nn.getLearnedValues(&output[0]); + + // After feedForward with initialized weights, the output should be a valid number + BOOST_TEST(!std::isnan(output[0])); + BOOST_TEST(!std::isinf(output[0])); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_sequential_inputs) +{ + // Test that LSTM maintains state across sequential feedForward calls + static const size_t NUMBER_OF_INPUTS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 3; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> LstmNNType; + srand(RANDOM_SEED); + LstmNNType nn; + + ValueType values[LstmNNType::NumberOfInputLayerNeurons]; + ValueType output1[LstmNNType::NumberOfOutputLayerNeurons]; + ValueType output2[LstmNNType::NumberOfOutputLayerNeurons]; + + // First feedForward + values[0] = 1.0; + nn.feedForward(&values[0]); + nn.getLearnedValues(&output1[0]); + + // Second feedForward with same input - output should differ due to recurrent state + nn.feedForward(&values[0]); + nn.getLearnedValues(&output2[0]); + + BOOST_TEST(!std::isnan(output1[0])); + BOOST_TEST(!std::isnan(output2[0])); + // Due to LSTM cell state, the same input should produce different outputs + BOOST_TEST(output1[0] != output2[0]); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_char_sequence_prediction) +{ + // Train a floating-point LSTM to predict the next value in a repeating + // sequence of 3 values: 0.2, 0.5, 0.8, 0.2, 0.5, 0.8, ... + // Input is (previous, current), target is the next value. + // The 3 training pairs are: + // (0.8, 0.2) -> 0.5 + // (0.2, 0.5) -> 0.8 + // (0.5, 0.8) -> 0.2 + // After training, verify the average error over the last samples is + // below a threshold, confirming the network has learned the pattern. + static const size_t SEQUENCE_LENGTH = 3; + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 8; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> LstmSeqNNType; + srand(RANDOM_SEED); + LstmSeqNNType nn; + + // Repeating sequence values + const ValueType sequence[SEQUENCE_LENGTH] = {0.2, 0.5, 0.8}; + + ValueType values[LstmSeqNNType::NumberOfInputLayerNeurons]; + ValueType target[LstmSeqNNType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[LstmSeqNNType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + static const int SEQ_TRAINING_ITERATIONS = 20000; + size_t seqIndex = 0; + + for (int i = 0; i < SEQ_TRAINING_ITERATIONS; ++i) + { + const size_t prevIndex = (seqIndex + SEQUENCE_LENGTH - 1) % SEQUENCE_LENGTH; + const size_t nextIndex = (seqIndex + 1) % SEQUENCE_LENGTH; + + values[0] = sequence[prevIndex]; + values[1] = sequence[seqIndex]; + target[0] = sequence[nextIndex]; + + nn.feedForward(&values[0]); + error = nn.calculateError(&target[0]); + + if (!LstmSeqNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&target[0]); + } + nn.getLearnedValues(&learnedValues[0]); + + errors.push_front(error); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + + seqIndex = nextIndex; + } + + // Verify: the average error over the last NUM_SAMPLES_AVG_ERROR samples + // should be below the error limit, confirming the LSTM learned the + // repeating sequence pattern. + const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); + const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + static const double SEQ_ERROR_LIMIT = 0.15; + + BOOST_TEST(averageError <= SEQ_ERROR_LIMIT); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_fixed_point_sequence_prediction) +{ + // Train a Q16.16 fixed-point LSTM to predict the next value in a + // repeating 2-value sequence: 0.3, 0.7, 0.3, 0.7, ... + // Input is (previous, current) and the target is the next value. + // The 2 training pairs are: + // (0.7, 0.3) -> 0.7 + // (0.3, 0.7) -> 0.3 + // After training, verify the average error over the last samples is + // below a threshold, confirming the network learned to predict the + // next number in the repeating sequence. + static const size_t SEQUENCE_LENGTH = 2; + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointLstmSeqNNType; + srand(RANDOM_SEED); + FixedPointLstmSeqNNType nn; + + // Repeating sequence: 0.3, 0.7 in Q16.16 fixed-point + typedef tinymind::ValueConverter ConverterType; + const ValueType sequence[SEQUENCE_LENGTH] = { + ConverterType::convertToDestinationType(0.3), + ConverterType::convertToDestinationType(0.7) + }; + ValueType values[FixedPointLstmSeqNNType::NumberOfInputLayerNeurons]; + ValueType target[FixedPointLstmSeqNNType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[FixedPointLstmSeqNNType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + static const int FP_SEQ_TRAINING_ITERATIONS = 20000; + size_t seqIndex = 0; + + for (int i = 0; i < FP_SEQ_TRAINING_ITERATIONS; ++i) + { + const size_t prevIndex = (seqIndex + SEQUENCE_LENGTH - 1) % SEQUENCE_LENGTH; + const size_t nextIndex = (seqIndex + 1) % SEQUENCE_LENGTH; + + values[0] = sequence[prevIndex]; + values[1] = sequence[seqIndex]; + target[0] = sequence[nextIndex]; + + nn.feedForward(&values[0]); + error = nn.calculateError(&target[0]); + + if (!FixedPointLstmSeqNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&target[0]); + } + nn.getLearnedValues(&learnedValues[0]); + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + + seqIndex = nextIndex; + } + + // Verify: the average error over the last NUM_SAMPLES_AVG_ERROR samples + // should be below the error limit, confirming the LSTM learned to + // predict the next number in the repeating sequence. + static const FullWidthValueType FP_SEQ_ERROR_LIMIT = (1 << (ValueType::NumberOfFractionalBits - 4)); + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + + BOOST_TEST(averageError <= FP_SEQ_ERROR_LIMIT); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_float_sinusoid_prediction) +{ + // Train a floating-point LSTM on a sampled sinusoid using sequential + // input processing: one value per timestep, with LSTM hidden state + // carrying temporal context between steps. + // + // Training: feed sin[i], target sin[i+1], for each step in the sequence. + // LSTM state is reset at the start of each epoch so the network learns + // the dynamics from a clean state each time. + // + // Prediction: prime the LSTM with the training sequence, then + // auto-regressively predict PREDICTION_LENGTH values. + static const size_t NUMBER_OF_INPUTS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 16; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t SEQUENCE_LENGTH = 10; + static const size_t PREDICTION_LENGTH = 20; + static const int SIN_TRAINING_ITERATIONS = 100000; + static const double SIN_TRAINING_ERROR_LIMIT = 0.15; + static const double SIN_PREDICTION_TOLERANCE = 0.90; + + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> LstmSinNNType; + srand(RANDOM_SEED); + UniformRealRandomNumberGenerator::seed(RANDOM_SEED); + LstmSinNNType nn; + + // Generate sinusoid samples scaled to [0, 1] + // sin(x) ranges [-1, 1], so (sin(x) + 1) / 2 maps to [0, 1] + const double step = 2.0 * M_PI / static_cast(SEQUENCE_LENGTH); + double sinSamples[SEQUENCE_LENGTH + PREDICTION_LENGTH]; + for (size_t i = 0; i < SEQUENCE_LENGTH + PREDICTION_LENGTH; ++i) + { + sinSamples[i] = (sin(static_cast(i) * step) + 1.0) / 2.0; + } + + ValueType values[LstmSinNNType::NumberOfInputLayerNeurons]; + ValueType target[LstmSinNNType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[LstmSinNNType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + + // Train: feed one value per timestep, LSTM state carries context. + // Reset state at the start of each epoch for clean temporal learning. + for (int epoch = 0; epoch < SIN_TRAINING_ITERATIONS; ++epoch) + { + nn.resetState(); + + for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) + { + values[0] = sinSamples[i]; + target[0] = sinSamples[i + 1]; + + nn.feedForward(&values[0]); + error = nn.calculateError(&target[0]); + + if (!LstmSinNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&target[0]); + } + + errors.push_front(error); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + } + } + + // Verify training converged + const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); + const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + BOOST_TEST(averageError <= SIN_TRAINING_ERROR_LIMIT); + + // Prime the LSTM by feeding the training sequence (no training) + nn.resetState(); + for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) + { + values[0] = sinSamples[i]; + nn.feedForward(&values[0]); + } + + // Auto-regressively predict: feed last known value, get prediction, + // feed prediction as next input + double curr = sinSamples[SEQUENCE_LENGTH - 1]; + + ofstream predictionOutput("output/nn_float_lstm_sinusoid_prediction.txt"); + predictionOutput << "Step,Actual,Predicted\n"; + for (size_t i = 0; i < SEQUENCE_LENGTH; ++i) + { + predictionOutput << i << "," << sinSamples[i] << ",\n"; + } + + for (size_t p = 0; p < PREDICTION_LENGTH; ++p) + { + values[0] = curr; + + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + + const double predicted = learnedValues[0]; + const double expected = sinSamples[SEQUENCE_LENGTH + p]; + const double predictionError = fabs(predicted - expected); + + predictionOutput << (SEQUENCE_LENGTH + p) << "," << expected << "," << predicted << "\n"; + + BOOST_TEST(predictionError <= SIN_PREDICTION_TOLERANCE); + + // Feed prediction as next input + curr = predicted; + } +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_neural_network_fixed_point_sinusoid_prediction) +{ + // Train a Q16.16 fixed-point LSTM on a sampled sinusoid using sequential + // input processing: one value per timestep with LSTM state carrying + // temporal context. State is reset at the start of each epoch. + static const size_t NUMBER_OF_INPUTS = 1; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 16; + static const size_t NUMBER_OF_OUTPUTS = 1; + static const size_t SEQUENCE_LENGTH = 10; + static const size_t PREDICTION_LENGTH = 20; + static const int FP_SIN_TRAINING_ITERATIONS = 50000; + + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + typedef tinymind::ValueConverter ConverterType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointLstmSinNNType; + srand(RANDOM_SEED); + UniformRealRandomNumberGenerator::seed(RANDOM_SEED); + FixedPointLstmSinNNType nn; + + // Generate sinusoid samples scaled to [0, 1] and convert to fixed-point + const double step = 2.0 * M_PI / static_cast(SEQUENCE_LENGTH); + ValueType sinSamples[SEQUENCE_LENGTH + PREDICTION_LENGTH]; + double sinSamplesDouble[SEQUENCE_LENGTH + PREDICTION_LENGTH]; + for (size_t i = 0; i < SEQUENCE_LENGTH + PREDICTION_LENGTH; ++i) + { + sinSamplesDouble[i] = (sin(static_cast(i) * step) + 1.0) / 2.0; + sinSamples[i] = ConverterType::convertToDestinationType(sinSamplesDouble[i]); + } + + ValueType values[FixedPointLstmSinNNType::NumberOfInputLayerNeurons]; + ValueType target[FixedPointLstmSinNNType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[FixedPointLstmSinNNType::NumberOfOutputLayerNeurons]; + std::deque errors; + ValueType error; + + // Train: feed one value per timestep, reset state each epoch + for (int epoch = 0; epoch < FP_SIN_TRAINING_ITERATIONS; ++epoch) + { + nn.resetState(); + + for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) + { + values[0] = sinSamples[i]; + target[0] = sinSamples[i + 1]; + + nn.feedForward(&values[0]); + error = nn.calculateError(&target[0]); + + if (!FixedPointLstmSinNNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&target[0]); + } + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + } + } + + // Verify training converged + // Error limit: 1/4 of full fractional range (generous for fixed-point sinusoid) + static const FullWidthValueType FP_SIN_ERROR_LIMIT = (1 << (ValueType::NumberOfFractionalBits - 2)); + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + BOOST_TEST(averageError <= FP_SIN_ERROR_LIMIT); + + // Prime the LSTM with the training sequence + nn.resetState(); + for (size_t i = 0; i < SEQUENCE_LENGTH - 1; ++i) + { + values[0] = sinSamples[i]; + nn.feedForward(&values[0]); + } + + // Auto-regressively predict + ValueType curr = sinSamples[SEQUENCE_LENGTH - 1]; + // Prediction tolerance in fixed-point: ~0.90 = 0.90 * 65536 ≈ 58982 + static const FullWidthValueType FP_SIN_PREDICTION_TOLERANCE = static_cast(0.90 * (1 << ValueType::NumberOfFractionalBits)); + + ofstream predictionOutput("output/nn_fixed_lstm_sinusoid_prediction.txt"); + predictionOutput << "Step,Actual,Predicted\n"; + for (size_t i = 0; i < SEQUENCE_LENGTH; ++i) + { + predictionOutput << i << "," << sinSamplesDouble[i] << ",\n"; + } + + for (size_t p = 0; p < PREDICTION_LENGTH; ++p) + { + values[0] = curr; + + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + + const FullWidthValueType predicted = learnedValues[0].getValue(); + const FullWidthValueType expected = sinSamples[SEQUENCE_LENGTH + p].getValue(); + const FullWidthValueType predictionError = (predicted > expected) ? (predicted - expected) : (expected - predicted); + + predictionOutput << (SEQUENCE_LENGTH + p) << "," << sinSamplesDouble[SEQUENCE_LENGTH + p] << "," << learnedValues[0] << "\n"; + + BOOST_TEST(predictionError <= FP_SIN_PREDICTION_TOLERANCE); + + // Feed prediction as next input + curr = learnedValues[0]; + } +} + +BOOST_AUTO_TEST_SUITE_END() + +// ========================================================================= +// Test suite for Tier 1 features: gradient clipping, weight decay, +// learning rate scheduling, and GRU networks. +// ========================================================================= +BOOST_AUTO_TEST_SUITE(test_suite_tier1_features) + +BOOST_AUTO_TEST_CASE(test_case_gradient_clipping_policy_fixed_point) +{ + // Verify GradientClipByValue clips Q8.8 gradients correctly + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::GradientClipByValue ClipPolicy; + + // Value within range [-1, 1] should pass through + ValueType small(0, 128); // 0.5 + ValueType clipped = ClipPolicy::clip(small); + BOOST_TEST(clipped.getValue() == small.getValue()); + + // Null policy should pass through unchanged + ValueType large(2, 0); // 2.0 + ValueType unchanged = tinymind::NullGradientClippingPolicy::clip(large); + BOOST_TEST(unchanged.getValue() == large.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_weight_decay_policy_fixed_point) +{ + // Verify L2WeightDecay reduces weight magnitude for Q16.16 + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + + ValueType weight(2, 0); // 2.0 + ValueType lr(0, (1 << 14)); // ~0.25 + // Use lambda = (0, 256) = 256/65536 ≈ 0.004 for Q16.16 + ValueType result = tinymind::L2WeightDecay::applyDecay(weight, lr); + + // Weight should be reduced (decay pulls toward zero) + BOOST_TEST(result.getValue() < weight.getValue()); + + // Null policy should leave weight unchanged + ValueType unchanged = tinymind::NullWeightDecayPolicy::applyDecay(weight, lr); + BOOST_TEST(unchanged.getValue() == weight.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_learning_rate_schedule_fixed_point) +{ + // Verify FixedLearningRatePolicy never changes the rate + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::FixedLearningRatePolicy schedule; + + ValueType rate(0, (1 << 6)); // ~0.25 + schedule.initialize(rate); + + for (int i = 0; i < 100; ++i) + { + rate = schedule.step(rate); + } + BOOST_TEST(rate.getValue() == ValueType(0, (1 << 6)).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_fixed_point) +{ + // Verify StepDecaySchedule reduces rate after interval for Q8.8 + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + // Decay factor: 0, 230 => 230/256 ≈ 0.898, interval 3 steps + tinymind::StepDecaySchedule schedule; + + ValueType rate(0, 128); // 0.5 + schedule.initialize(rate); + + // Steps 1-2: unchanged + ValueType r1 = schedule.step(rate); + BOOST_TEST(r1.getValue() == rate.getValue()); + ValueType r2 = schedule.step(rate); + BOOST_TEST(r2.getValue() == rate.getValue()); + + // Step 3: decays + ValueType r3 = schedule.step(rate); + BOOST_TEST(r3.getValue() < rate.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_multi_cycle) +{ + // Realistic usage: rate = schedule.step(rate). Across multiple intervals + // the rate must strictly decrease each cycle and stay flat in between. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::StepDecaySchedule schedule; + + ValueType rate(0, 128); // 0.5 + schedule.initialize(rate); + + ValueType prev = rate; + for (int cycle = 0; cycle < 3; ++cycle) + { + // Two flat steps: rate unchanged. + rate = schedule.step(rate); + BOOST_TEST(rate.getValue() == prev.getValue()); + rate = schedule.step(rate); + BOOST_TEST(rate.getValue() == prev.getValue()); + // Third step decays. + rate = schedule.step(rate); + BOOST_TEST(rate.getValue() < prev.getValue()); + prev = rate; + } +} + +BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_initialize_resets_counter) +{ + // initialize() must reset the internal step counter; decay should fire + // exactly StepInterval steps after the reset, not sooner. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::StepDecaySchedule schedule; + + ValueType rate(0, 128); // 0.5 + + // Burn 2 steps worth of counter so the next step would have decayed. + schedule.initialize(rate); + (void)schedule.step(rate); + (void)schedule.step(rate); + + // Reset; next two steps must NOT decay even though only one more would have triggered before. + schedule.initialize(rate); + BOOST_TEST(schedule.step(rate).getValue() == rate.getValue()); + BOOST_TEST(schedule.step(rate).getValue() == rate.getValue()); + // Third step after reset triggers decay. + BOOST_TEST(schedule.step(rate).getValue() < rate.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_step_decay_schedule_interval_one) +{ + // StepInterval = 1 means decay every single step (no flat plateau). + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::StepDecaySchedule schedule; + + ValueType rate(0, 128); // 0.5 + schedule.initialize(rate); + + ValueType prev = rate; + for (int i = 0; i < 5; ++i) + { + ValueType next = schedule.step(prev); + BOOST_TEST(next.getValue() < prev.getValue()); + prev = next; + } +} + +BOOST_AUTO_TEST_CASE(test_case_fixed_point_gradient_clipping_xor) +{ + // Verify fixed-point XOR training with gradient clipping enabled via FixedPointTransferFunctions + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + NUMBER_OF_OUTPUTS, + tinymind::DefaultNetworkInitializer, + tinymind::MeanSquaredErrorCalculator, + tinymind::ZeroToleranceCalculator, + tinymind::GradientClipByValue> TransferFunctionsType; + + typedef tinymind::MultilayerPerceptron< ValueType, + NUMBER_OF_INPUTS, + 1, 3, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> NNType; + srand(RANDOM_SEED); + NNType nn; + + ValueType values[2]; + ValueType output[1]; + ValueType error; + + for (int i = 0; i < TRAINING_ITERATIONS; ++i) + { + generateFixedPointXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + nn.trainNetwork(&output[0]); + } + + // Network should have learned XOR to some degree + BOOST_TEST(error.getValue() < ValueHelper::getErrorLimit()); +} + +BOOST_AUTO_TEST_CASE(test_case_gru_neural_network_floating_point) +{ + // Verify GRU network can be instantiated, trained, and produce valid output + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::GruNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FloatingPointGruNeuralNetworkType; + srand(RANDOM_SEED); + FloatingPointGruNeuralNetworkType nn; + + ValueType values[2]; + ValueType output[1]; + ValueType learnedValues[1]; + ValueType error; + + for (int i = 0; i < TRAINING_ITERATIONS; ++i) + { + generateXorValues(&values[0], &output[0]); + + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + + nn.trainNetwork(&output[0]); + nn.getLearnedValues(&learnedValues[0]); + } + + // Verify feedforward produces valid, finite output + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + BOOST_TEST(!std::isnan(learnedValues[0])); + BOOST_TEST(!std::isinf(learnedValues[0])); + BOOST_TEST(!std::isnan(error)); + BOOST_TEST(!std::isinf(error)); +} + +BOOST_AUTO_TEST_CASE(test_case_gru_neural_network_fixed_point) +{ + // Verify GRU network works with fixed-point arithmetic + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_NEURONS_PER_HIDDEN_LAYER = 4; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::GruNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> FixedPointGruNeuralNetworkType; + srand(RANDOM_SEED); + FixedPointGruNeuralNetworkType nn; + + ValueType values[FixedPointGruNeuralNetworkType::NumberOfInputLayerNeurons]; + ValueType output[FixedPointGruNeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType learnedValues[FixedPointGruNeuralNetworkType::NumberOfOutputLayerNeurons]; + ValueType error; + ValueType firstError; + bool firstErrorCaptured = false; + + for (int i = 0; i < TRAINING_ITERATIONS; ++i) + { + generateFixedPointRecurrentValues(values, output); + + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + + if (!firstErrorCaptured) + { + firstError = error; + firstErrorCaptured = true; + } + + if (!FixedPointGruNeuralNetworkType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + nn.getLearnedValues(&learnedValues[0]); + } + + // Verify feedforward produces valid output + nn.feedForward(&values[0]); + nn.getLearnedValues(&learnedValues[0]); + BOOST_TEST(learnedValues[0].getValue() != 0); +} + +BOOST_AUTO_TEST_CASE(test_case_gru_reset_state) +{ + // Verify GRU resetState clears accumulated state so a second + // reset+feedforward pair produces the same output as the first. + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::GruNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers<4>, + NUMBER_OF_OUTPUTS, + TransferFunctionsType> GruType; + + srand(RANDOM_SEED); + GruType nn; + + ValueType values[2] = {0.5, 0.3}; + ValueType output1[1]; + ValueType output2[1]; + + // Reset, then feedforward to get baseline + nn.resetState(); + nn.feedForward(&values[0]); + nn.getLearnedValues(&output1[0]); + + // Feed forward multiple times to accumulate state + nn.feedForward(&values[0]); + nn.feedForward(&values[0]); + nn.feedForward(&values[0]); + + // Reset and feedforward again — should match baseline + nn.resetState(); + nn.feedForward(&values[0]); + nn.getLearnedValues(&output2[0]); + + BOOST_TEST(fabs(output1[0] - output2[0]) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_gru_non_trainable) +{ + // Verify GRU network can be instantiated as non-trainable (inference only) + static const size_t NUMBER_OF_INPUTS = 2; + static const size_t NUMBER_OF_OUTPUTS = 1; + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::GruNeuralNetwork< ValueType, + NUMBER_OF_INPUTS, + tinymind::HiddenLayers<4>, + NUMBER_OF_OUTPUTS, + TransferFunctionsType, + false> NonTrainableGruType; + + NonTrainableGruType nn; + + ValueType values[2] = {0.5, 0.3}; + ValueType output[1]; + + // Should be able to feed forward without crash + nn.feedForward(&values[0]); + nn.getLearnedValues(&output[0]); + + // Output should be finite + BOOST_TEST(!std::isnan(output[0])); + BOOST_TEST(!std::isinf(output[0])); +} + +BOOST_AUTO_TEST_CASE(test_case_early_stopping) +{ + // Verify EarlyStopping correctly detects convergence + tinymind::EarlyStopping stopper; + + // Decreasing error: should not stop + BOOST_TEST(!stopper.shouldStop(1.0)); + BOOST_TEST(!stopper.shouldStop(0.5)); + BOOST_TEST(!stopper.shouldStop(0.3)); + + // Stagnant error: should stop after patience exhausted + BOOST_TEST(!stopper.shouldStop(0.4)); + BOOST_TEST(!stopper.shouldStop(0.4)); + BOOST_TEST(!stopper.shouldStop(0.4)); + BOOST_TEST(!stopper.shouldStop(0.4)); + BOOST_TEST(stopper.shouldStop(0.4)); // 5th non-improving step + + // Best error should be the minimum seen + BOOST_TEST(fabs(stopper.getBestError() - 0.3) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_early_stopping_reset) +{ + // Verify EarlyStopping reset works + tinymind::EarlyStopping stopper; + + stopper.shouldStop(1.0); + stopper.shouldStop(2.0); + stopper.shouldStop(2.0); + stopper.shouldStop(2.0); + BOOST_TEST(stopper.shouldStop(2.0)); // should stop + + stopper.reset(); + BOOST_TEST(!stopper.shouldStop(5.0)); // should NOT stop after reset +} + +BOOST_AUTO_TEST_CASE(test_case_early_stopping_with_training) +{ + // Verify early stopping works in a training loop + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TransferFunctionsType; + typedef tinymind::MultilayerPerceptron NNType; + srand(RANDOM_SEED); + NNType nn; + + tinymind::EarlyStopping stopper; + ValueType values[2], output[1], error; + int iterationsUsed = 0; + + for (int i = 0; i < 10000; ++i) + { + generateXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + if (stopper.shouldStop(error)) break; + nn.trainNetwork(&output[0]); + ++iterationsUsed; + } + + // Should have stopped before 10000 iterations + BOOST_TEST(iterationsUsed < 10000); + BOOST_TEST(!std::isnan(error)); +} + +struct AdamTransferFunctions +{ + typedef double TransferFunctionsValueType; + typedef UniformRealRandomNumberGenerator RandomNumberGeneratorPolicy; + typedef tinymind::TanhActivationPolicy HiddenNeuronActivationPolicy; + typedef tinymind::TanhActivationPolicy OutputNeuronActivationPolicy; + typedef tinymind::ZeroToleranceCalculator ZeroToleranceCalculatorPolicy; + typedef tinymind::AdamOptimizerFloat OptimizerPolicyType; + + static const unsigned NumberOfTransferFunctionsOutputNeurons = 1; + + static double calculateError(double const* const targetValues, double const* const outputValues) + { + const double delta = (targetValues[0] - outputValues[0]); + return (delta * delta); + } + + static double calculateOutputGradient(const double& targetValue, const double& outputValue) + { + return (targetValue - outputValue) * OutputNeuronActivationPolicy::activationFunctionDerivative(outputValue); + } + + static double generateRandomWeight() { return RandomNumberGeneratorPolicy::generateRandomWeight(); } + static double hiddenNeuronActivationFunction(const double& value) { return HiddenNeuronActivationPolicy::activationFunction(value); } + static double hiddenNeuronActivationFunctionDerivative(const double& value) { return HiddenNeuronActivationPolicy::activationFunctionDerivative(value); } + static double outputNeuronActivationFunction(const double& value) { return OutputNeuronActivationPolicy::activationFunction(value); } + static double outputNeuronActivationFunctionDerivative(const double& value) { return OutputNeuronActivationPolicy::activationFunctionDerivative(value); } + static double initialAccelerationRate() { return 0.0; } + static double initialBiasOutputValue() { return 1.0; } + static double initialDeltaWeight() { return 0.0; } + static double initialGradientValue() { return 0.0; } + static double initialLearningRate() { return 0.01; } + static double initialMomentumRate() { return 0.0; } + static double initialOutputValue() { return 0.0; } + static bool isWithinZeroTolerance(const double& value) { return (fabs(value) < 0.004); } + static double negate(const double& value) { return -value; } + static double noOpDeltaWeight() { return 1.0; } + static double noOpWeight() { return 1.0; } +}; + +BOOST_AUTO_TEST_CASE(test_case_adam_optimizer_xor_training) +{ + // Verify Adam optimizer converges on XOR + typedef double ValueType; + typedef tinymind::MultilayerPerceptron AdamNNType; + srand(RANDOM_SEED); + AdamNNType nn; + + static const double ERROR_LIMIT = ValueHelper::getErrorLimit(); + ValueType values[2], output[1], error; + std::deque errors; + + static const int ADAM_TRAINING_ITERATIONS = 10000; + + for (int i = 0; i < ADAM_TRAINING_ITERATIONS; ++i) + { + generateXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + + if (!AdamTransferFunctions::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + + errors.push_front(error); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + } + + const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); + const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +BOOST_AUTO_TEST_CASE(test_case_adam_fixedpoint_xor_training) +{ + // Verify Adam optimizer converges on XOR (fixed-point Q16.16) + // Q16.16 provides sufficient precision for Adam's moment calculations + // and bias correction, which lose accuracy rapidly in Q8.8. + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + 1, + tinymind::DefaultNetworkInitializer, + tinymind::MeanSquaredErrorCalculator, + tinymind::ZeroToleranceCalculator, + tinymind::GradientClipByValue, + tinymind::NullWeightDecayPolicy, + tinymind::FixedLearningRatePolicy, + tinymind::AdamOptimizer> TransferFunctionsType; + + typedef tinymind::MultilayerPerceptron NNType; + + srand(RANDOM_SEED); + NNType nn; + + // Adam needs a lower learning rate than the default SGD rate. + // QValue(0, 655) ≈ 655/65536 ≈ 0.01 + nn.setLearningRate(ValueType(0, 655)); + + static const FullWidthValueType ERROR_LIMIT = ValueHelper::getErrorLimit(); + ValueType values[2], output[1], error; + std::deque errors; + + static const int ADAM_FP_ITERATIONS = 10000; + + for (int i = 0; i < ADAM_FP_ITERATIONS; ++i) + { + generateFixedPointXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + + if (!NNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + } + + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +BOOST_AUTO_TEST_CASE(test_case_lstm_weight_serialization) +{ + // Verify LSTM weights can be saved and loaded + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::LstmNeuralNetwork, 1, + TransferFunctionsType> LstmType; + srand(RANDOM_SEED); + LstmType nn; + + // Train briefly to get non-trivial weights + ValueType values[2], output[1]; + for (int i = 0; i < 100; ++i) + { + generateXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + nn.trainNetwork(&output[0]); + } + + // Save weights + { + std::ofstream outFile("output/lstm_weights.txt"); + tinymind::RecurrentNetworkPropertiesFileManager::storeNetworkWeights(nn, outFile); + } + + // Create a new network and load weights + LstmType nn2; + + { + std::ifstream inFile("output/lstm_weights.txt"); + tinymind::RecurrentNetworkPropertiesFileManager::template loadNetworkWeights(nn2, inFile); + } + + // Reset state so recurrent layer doesn't affect comparison + nn.resetState(); + nn2.resetState(); + + // Both networks should produce the same output for the same input + values[0] = 0.5; values[1] = 0.3; + nn.feedForward(&values[0]); + nn2.feedForward(&values[0]); + + ValueType out1[1], out2[1]; + nn.getLearnedValues(&out1[0]); + nn2.getLearnedValues(&out2[0]); + + // Tolerance accounts for text serialization precision loss + BOOST_TEST(fabs(out1[0] - out2[0]) < 0.02); +} + +BOOST_AUTO_TEST_CASE(test_case_gru_weight_serialization) +{ + // Verify GRU weights can be saved and loaded + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TransferFunctionsType; + typedef tinymind::GruNeuralNetwork, 1, + TransferFunctionsType> GruType; + srand(RANDOM_SEED); + GruType nn; + + // Train briefly + ValueType values[2], output[1]; + for (int i = 0; i < 100; ++i) + { + generateXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + nn.trainNetwork(&output[0]); + } + + // Save weights + { + std::ofstream outFile("output/gru_weights.txt"); + tinymind::RecurrentNetworkPropertiesFileManager::storeNetworkWeights(nn, outFile); + } + + // Load weights into new network + GruType nn2; + + { + std::ifstream inFile("output/gru_weights.txt"); + tinymind::RecurrentNetworkPropertiesFileManager::template loadNetworkWeights(nn2, inFile); + } + + // Both should produce same output + values[0] = 0.5; values[1] = 0.3; + nn.resetState(); + nn2.resetState(); + nn.feedForward(&values[0]); + nn2.feedForward(&values[0]); + + ValueType out1[1], out2[1]; + nn.getLearnedValues(&out1[0]); + nn2.getLearnedValues(&out2[0]); + + BOOST_TEST(fabs(out1[0] - out2[0]) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_elu_activation_fixed_point) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::EluActivationPolicy EluPolicy; + + // Positive values: f(x) = x + ValueType pos(0, 128); // 0.5 + ValueType result = EluPolicy::activationFunction(pos); + BOOST_TEST(result.getValue() == pos.getValue()); + + // Derivative for positive: f'(x) = 1 + ValueType deriv = EluPolicy::activationFunctionDerivative(pos); + BOOST_TEST(deriv.getValue() == tinymind::Constants::one().getValue()); + + // Negative values exercise the exp-table branch: f(x) = exp(x) - 1 < 0, + // and the derivative path f'(x) = x + 1, which is in (0, 1) for x in (-1, 0). + ValueType neg(-1, 128); // -0.5 + ValueType negResult = EluPolicy::activationFunction(neg); + BOOST_TEST(negResult < tinymind::Constants::zero()); + BOOST_TEST(negResult > ValueType(-1, 0)); + + ValueType negDeriv = EluPolicy::activationFunctionDerivative(neg); + BOOST_TEST(negDeriv > tinymind::Constants::zero()); + BOOST_TEST(negDeriv < tinymind::Constants::one()); +} + +BOOST_AUTO_TEST_CASE(test_case_gelu_activation_fixed_point) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::GeluActivationPolicy GeluPolicy; + + // GELU(0) should be ~0 (x * sigmoid(0) = 0 * 0.5 = 0) + ValueType zero(0); + ValueType result = GeluPolicy::activationFunction(zero); + BOOST_TEST(result.getValue() == 0); + + // GELU for positive input should be positive + ValueType pos(1, 0); // 1.0 + ValueType posResult = GeluPolicy::activationFunction(pos); + BOOST_TEST(posResult.getValue() > 0); +} + +BOOST_AUTO_TEST_CASE(test_case_network_stats) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TF; + typedef tinymind::NeuralNetwork, 1, TF> NNType; + typedef tinymind::NetworkStats Stats; + + // Use static_assert for compile-time verification (no ODR-use issues) + static_assert(Stats::NumberOfInputs == 2, "Wrong input count"); + static_assert(Stats::NumberOfHiddenNeurons == 5, "Wrong hidden count"); + static_assert(Stats::NumberOfOutputs == 1, "Wrong output count"); + static_assert(Stats::NumberOfHiddenLayers == 1, "Wrong layer count"); + static_assert(Stats::ValueSizeBytes == sizeof(ValueType), "Wrong value size"); + static_assert(Stats::InstanceSizeBytes == sizeof(NNType), "Wrong instance size"); + static_assert(Stats::InstanceSizeBytes > 0, "Instance size must be positive"); + BOOST_TEST(true); // test counts toward suite +} + +BOOST_AUTO_TEST_CASE(test_case_hidden_layer_accessor_out_of_range) +{ + // With a single hidden layer, InnerHiddenLayerChainType is EmptyLayerChain. + // An out-of-range hidden-layer index falls past the last-hidden-layer fast + // path into the ChainHiddenLayerAccessor terminal methods: + // getters return a default-constructed value, setters are no-ops. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> TF; + typedef tinymind::NeuralNetwork, 1, TF> NNType; + + NNType nn; + const size_t outOfRange = 1; // only valid hidden-layer index is 0 + + BOOST_TEST(nn.getHiddenLayerBiasNeuronWeightForConnection(outOfRange, 0) == + ValueType()); + BOOST_TEST(nn.getHiddenLayerWeightForNeuronAndConnection(outOfRange, 0, 0) == + ValueType()); + + // Setters on the terminal chain are no-ops; they must not touch the real + // layer-0 weights. Capture a valid weight, run the no-op setters, re-read. + const ValueType before = + nn.getHiddenLayerWeightForNeuronAndConnection(0, 0, 0); + nn.setHiddenLayerBiasNeuronWeightForConnection(outOfRange, 0, ValueType(1, 0)); + nn.setHiddenLayerWeightForNeuronAndConnection(outOfRange, 0, 0, ValueType(1, 0)); + BOOST_TEST(nn.getHiddenLayerWeightForNeuronAndConnection(0, 0, 0) == before); +} + +BOOST_AUTO_TEST_CASE(test_case_forward_as_dual_differentiable) +{ + // tinymind::pinn::forwardAs re-evaluates the STOCK NeuralNetwork in an + // arbitrary scalar type without touching its own forward path: with double + // it must reproduce getLearnedValues exactly, and with Dual it + // yields the network's input derivative (du/dx), matching finite difference. + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, tinymind::TanhActivationPolicy> TF; + typedef tinymind::MultilayerPerceptron NN; + + srand(RANDOM_SEED); + NN nn; + + double inputs[2] = { 0.3, -0.7 }; + + nn.feedForward(inputs); + double nativeOut[1]; + nn.getLearnedValues(nativeOut); + + double myOut[1]; + tinymind::pinn::forwardAs(nn, inputs, myOut); + BOOST_TEST(myOut[0] == nativeOut[0], boost::test_tools::tolerance(1e-12)); + + typedef tinymind::Dual D1; + D1 din[2] = { D1(0.3, 1.0), D1(-0.7, 0.0) }; // seed d/dx0 + D1 dout[1]; + tinymind::pinn::forwardAs(nn, din, dout); + + const double h = 1e-6; + double a[2] = { 0.3 + h, -0.7 }, b[2] = { 0.3 - h, -0.7 }; + double fa[1], fb[1]; + nn.feedForward(a); nn.getLearnedValues(fa); + nn.feedForward(b); nn.getLearnedValues(fb); + const double fd = (fa[0] - fb[0]) / (2.0 * h); + BOOST_TEST(dout[0].deriv == fd, boost::test_tools::tolerance(1e-5)); +} + +BOOST_AUTO_TEST_CASE(test_case_forward_as_multi_hidden_layer) +{ + // forwardAs handles multiple uniform-width hidden layers: a 2->4->4->1 MLP. + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, tinymind::TanhActivationPolicy> TF; + typedef tinymind::MultilayerPerceptron NN; + + srand(RANDOM_SEED); + NN nn; + + double inputs[2] = { 0.4, -0.2 }; + nn.feedForward(inputs); + double nativeOut[1]; + nn.getLearnedValues(nativeOut); + + double myOut[1]; + tinymind::pinn::forwardAs(nn, inputs, myOut); + BOOST_TEST(myOut[0] == nativeOut[0], boost::test_tools::tolerance(1e-12)); +} + +BOOST_AUTO_TEST_CASE(test_case_forward_as_linear_output) +{ + // forwardAs with a LINEAR output activation -- the canonical PINN-field + // configuration (tanh hidden + linear readout). The output activation policy + // must match the network's own, so forwardAs<...,LinearActivation> reproduces + // getLearnedValues and yields the correct input derivative. + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, tinymind::LinearActivationPolicy> TF; + typedef tinymind::MultilayerPerceptron NN; + + srand(RANDOM_SEED); + NN nn; + + double inputs[2] = { 0.25, -0.6 }; + nn.feedForward(inputs); + double nativeOut[1]; + nn.getLearnedValues(nativeOut); + + double myOut[1]; + tinymind::pinn::forwardAs(nn, inputs, myOut); + BOOST_TEST(myOut[0] == nativeOut[0], boost::test_tools::tolerance(1e-12)); + + typedef tinymind::Dual D1; + D1 din[2] = { D1(0.25, 1.0), D1(-0.6, 0.0) }; // seed d/dx0 + D1 dout[1]; + tinymind::pinn::forwardAs(nn, din, dout); + + const double h = 1e-6; + double a[2] = { 0.25 + h, -0.6 }, b[2] = { 0.25 - h, -0.6 }, fa[1], fb[1]; + nn.feedForward(a); nn.getLearnedValues(fa); + nn.feedForward(b); nn.getLearnedValues(fb); + const double fd = (fa[0] - fb[0]) / (2.0 * h); + BOOST_TEST(dout[0].deriv == fd, boost::test_tools::tolerance(1e-5)); +} + +BOOST_AUTO_TEST_CASE(test_case_teacher_forcing) +{ + tinymind::ScheduledSampling sampler(100); + + // At step 0, teacher forcing ratio should be 1.0 (always ground truth) + BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 1.0) < 0.01); + + // Advance halfway + for (int i = 0; i < 50; ++i) sampler.step(); + BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 0.5) < 0.01); + + // Advance to end + for (int i = 0; i < 50; ++i) sampler.step(); + BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 0.0) < 0.01); + + // After decay, selectInput should return prediction + double result = sampler.selectInput(1.0, 0.5); + BOOST_TEST(fabs(result - 0.5) < 0.001); + + // Reset should restore ratio to 1.0 + sampler.reset(); + BOOST_TEST(fabs(sampler.getTeacherForcingRatio() - 1.0) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_teacher_forcing_rand_branch) +{ + // Exercises the TINYMIND_ENABLE_HOSTED_RAND path of selectInput, which the + // post-decay test above skips (it only hits the early-out at step >= total). + tinymind::ScheduledSampling sampler(100); + + // Step 0: threshold == 1000, so rand() % 1000 is always below it -> + // selectInput always returns ground truth. + for (int i = 0; i < 50; ++i) + { + BOOST_TEST(fabs(sampler.selectInput(1.0, 0.5) - 1.0) < 1e-9); + } + + // Halfway through decay (threshold ~500): both outcomes must occur across + // many rolls. rand() is deterministically seeded, so this is reproducible. + for (int i = 0; i < 50; ++i) sampler.step(); + bool saw_ground_truth = false; + bool saw_prediction = false; + for (int i = 0; i < 500; ++i) + { + const double r = sampler.selectInput(1.0, 0.5); + if (fabs(r - 1.0) < 1e-9) saw_ground_truth = true; + if (fabs(r - 0.5) < 1e-9) saw_prediction = true; + } + BOOST_TEST(saw_ground_truth); + BOOST_TEST(saw_prediction); +} + +BOOST_AUTO_TEST_CASE(test_case_qformat_minmax_saturate_policy) +{ + // Direct coverage of MinMaxSaturatePolicy saturation arms, which the + // default QValue overflow policy does not route through. + typedef tinymind::MinMaxSaturatePolicy Policy; + const int lo = -100; + const int hi = 100; + + // Addition: positive overflow, negative underflow, and the zero no-op. + BOOST_TEST(Policy::saturate(90, 20, lo, hi, tinymind::AdditionOp) == hi); + BOOST_TEST(Policy::saturate(-90, -20, lo, hi, tinymind::AdditionOp) == lo); + BOOST_TEST(Policy::saturate(7, 0, lo, hi, tinymind::AdditionOp) == 7); + + // Subtraction: positive-target underflow, negative-target overflow, zero. + BOOST_TEST(Policy::saturate(-90, 20, lo, hi, tinymind::SubtractionOp) == lo); + BOOST_TEST(Policy::saturate(90, -20, lo, hi, tinymind::SubtractionOp) == hi); + BOOST_TEST(Policy::saturate(7, 0, lo, hi, tinymind::SubtractionOp) == 7); + + // Multiplication: overflow and underflow saturation. + BOOST_TEST(Policy::saturate(50, 50, lo, hi, tinymind::MultiplicationOp) == hi); + BOOST_TEST(Policy::saturate(50, -50, lo, hi, tinymind::MultiplicationOp) == lo); + + // Division: by-zero (sign-dependent), plus overflow / underflow. + BOOST_TEST(Policy::saturate(5, 0, lo, hi, tinymind::DivisionOp) == hi); + BOOST_TEST(Policy::saturate(-5, 0, lo, hi, tinymind::DivisionOp) == lo); + BOOST_TEST(Policy::saturate(1000, 2, lo, hi, tinymind::DivisionOp) == hi); + BOOST_TEST(Policy::saturate(-1000, 2, lo, hi, tinymind::DivisionOp) == lo); +} + +BOOST_AUTO_TEST_CASE(test_case_conv1d_forward) +{ + // Conv1D with 5-point input, kernel=3, stride=1, 1 filter + tinymind::Conv1D conv; + + // Set known kernel weights: [1, 0, -1] with bias=0 + conv.setFilterWeight(0, 0, 1.0); + conv.setFilterWeight(0, 1, 0.0); + conv.setFilterWeight(0, 2, -1.0); + conv.setFilterWeight(0, 3, 0.0); // bias + + double input[5] = {1.0, 2.0, 3.0, 4.0, 5.0}; + double output[3]; // (5 - 3) / 1 + 1 = 3 + + conv.forward(input, output); + + // kernel [1, 0, -1]: + // pos 0: 1*1 + 0*2 + (-1)*3 = -2 + // pos 1: 1*2 + 0*3 + (-1)*4 = -2 + // pos 2: 1*3 + 0*4 + (-1)*5 = -2 + BOOST_TEST(fabs(output[0] - (-2.0)) < 0.001); + BOOST_TEST(fabs(output[1] - (-2.0)) < 0.001); + BOOST_TEST(fabs(output[2] - (-2.0)) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_conv1d_multi_filter) +{ + // Conv1D with 4-point input, kernel=2, stride=1, 2 filters + tinymind::Conv1D conv; + + // Filter 0: [1, 1], bias=0 (sum filter) + conv.setFilterWeight(0, 0, 1.0); + conv.setFilterWeight(0, 1, 1.0); + conv.setFilterWeight(0, 2, 0.0); + + // Filter 1: [1, -1], bias=0 (difference filter) + conv.setFilterWeight(1, 0, 1.0); + conv.setFilterWeight(1, 1, -1.0); + conv.setFilterWeight(1, 2, 0.0); + + double input[4] = {1.0, 3.0, 2.0, 4.0}; + double output[6]; // 2 filters * 3 positions + + conv.forward(input, output); + + // Filter 0 (sum): [4, 5, 6] + BOOST_TEST(fabs(output[0] - 4.0) < 0.001); + BOOST_TEST(fabs(output[1] - 5.0) < 0.001); + BOOST_TEST(fabs(output[2] - 6.0) < 0.001); + + // Filter 1 (diff): [-2, 1, -2] + BOOST_TEST(fabs(output[3] - (-2.0)) < 0.001); + BOOST_TEST(fabs(output[4] - 1.0) < 0.001); + BOOST_TEST(fabs(output[5] - (-2.0)) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_conv1d_static_sizes) +{ + typedef tinymind::Conv1D ConvType; + + // OutputLength = (100 - 5) / 2 + 1 = 48 + static_assert(ConvType::OutputLength == 48, "Wrong output length"); + static_assert(ConvType::OutputSize == 384, "Wrong output size"); // 8 * 48 + static_assert(ConvType::TotalWeights == 48, "Wrong total weights"); // 8 * (5 + 1) + BOOST_TEST(true); // test counts toward suite +} + +BOOST_AUTO_TEST_CASE(test_case_conv1d_fixed_point) +{ + // Verify Conv1D works with Q8.8 fixed-point + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::Conv1D conv; + + // Set kernel [1, 0, -1], bias=0 + conv.setFilterWeight(0, 0, ValueType(1, 0)); + conv.setFilterWeight(0, 1, ValueType(0)); + conv.setFilterWeight(0, 2, ValueType(-1, 0)); + conv.setFilterWeight(0, 3, ValueType(0)); // bias + + ValueType input[5]; + input[0] = ValueType(1, 0); + input[1] = ValueType(2, 0); + input[2] = ValueType(3, 0); + input[3] = ValueType(4, 0); + input[4] = ValueType(5, 0); + + ValueType output[3]; + conv.forward(input, output); + + // kernel [1, 0, -1]: each output should be -2 + ValueType expected(-2, 0); + BOOST_TEST(output[0].getValue() == expected.getValue()); + BOOST_TEST(output[1].getValue() == expected.getValue()); + BOOST_TEST(output[2].getValue() == expected.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_conv1d_gradient_sanity) +{ + // y = sum(w * x) + b. With outputDelta = 1 the kernel weight gradients + // accumulate the input values at each stride position, and the bias + // gradient counts the number of output positions. + tinymind::Conv1D conv; + for (size_t i = 0; i < conv.TotalWeights; ++i) conv.setWeight(i, 0.0); + + double input[5] = {1.0, 2.0, 3.0, 4.0, 5.0}; + double output[3]; + conv.forward(input, output); + double outputDeltas[3] = {1.0, 1.0, 1.0}; + conv.computeGradients(outputDeltas, input); + + // Output positions slide kernel over input. Kernel-weight gradient at k + // accumulates input[k + pos*stride] for pos in [0, OutputLength). + // For stride=1, kernelSize=3, inputLength=5: outputLength=3. + // grad[k=0] = input[0] + input[1] + input[2] = 1+2+3 = 6 + // grad[k=1] = input[1] + input[2] + input[3] = 2+3+4 = 9 + // grad[k=2] = input[2] + input[3] + input[4] = 3+4+5 = 12 + BOOST_TEST(std::fabs(conv.getGradient(0) - 6.0) < 1e-9); + BOOST_TEST(std::fabs(conv.getGradient(1) - 9.0) < 1e-9); + BOOST_TEST(std::fabs(conv.getGradient(2) - 12.0) < 1e-9); + // Bias gradient = sum of deltas across output positions = 3. + BOOST_TEST(std::fabs(conv.getGradient(3) - 3.0) < 1e-9); + + // updateWeights with lr=0.1 should move weight 0 by 0.1 * 6.0 = 0.6. + conv.updateWeights(0.1); + BOOST_TEST(std::fabs(conv.getFilterWeight(0, 0) - 0.6) < 1e-9); + BOOST_TEST(std::fabs(conv.getFilterWeight(0, 3) - 0.3) < 1e-9); // bias +} + +BOOST_AUTO_TEST_CASE(test_case_conv1d_gradient_fixed_point) +{ + // Same gradient invariant in Q8.8. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::Conv1D conv; + for (size_t i = 0; i < conv.TotalWeights; ++i) conv.setWeight(i, ValueType(0)); + + ValueType input[5] = {ValueType(1, 0), ValueType(2, 0), ValueType(3, 0), + ValueType(4, 0), ValueType(5, 0)}; + ValueType output[3]; + conv.forward(input, output); + ValueType outputDeltas[3] = {ValueType(1, 0), ValueType(1, 0), ValueType(1, 0)}; + conv.computeGradients(outputDeltas, input); + + BOOST_TEST(conv.getGradient(0).getValue() == ValueType( 6, 0).getValue()); + BOOST_TEST(conv.getGradient(1).getValue() == ValueType( 9, 0).getValue()); + BOOST_TEST(conv.getGradient(2).getValue() == ValueType(12, 0).getValue()); + BOOST_TEST(conv.getGradient(3).getValue() == ValueType( 3, 0).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_truncated_bptt) +{ + // Test that TruncatedBPTT accumulates steps before training + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + tinymind::SigmoidActivationPolicy> TF; + typedef tinymind::LstmNeuralNetwork, 1, TF> LstmType; + + LstmType nn; + tinymind::TruncatedBPTT trainer; + + ValueType input[1] = {0.5}; + ValueType target[1] = {0.8}; + + // Steps 1-2: accumulate, no training yet + trainer.step(nn, input, target); + BOOST_TEST(trainer.getStepCount() == 1u); + trainer.step(nn, input, target); + BOOST_TEST(trainer.getStepCount() == 2u); + + // Step 3: window full, trains and resets + trainer.step(nn, input, target); + BOOST_TEST(trainer.getStepCount() == 0u); + + // Flush with partial window + trainer.step(nn, input, target); + trainer.flush(nn); + BOOST_TEST(trainer.getStepCount() == 0u); +} + +// ============================================================ +// MaxPool1D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_maxpool1d_forward) +{ + // MaxPool1D: 6-element input, pool size 2, stride 2, 1 channel + tinymind::MaxPool1D pool; + + double input[6] = {1.0, 3.0, 2.0, 5.0, 4.0, 6.0}; + double output[3]; // (6 - 2) / 2 + 1 = 3 + + pool.forward(input, output); + + // Window [1,3] -> 3, [2,5] -> 5, [4,6] -> 6 + BOOST_TEST(fabs(output[0] - 3.0) < 0.001); + BOOST_TEST(fabs(output[1] - 5.0) < 0.001); + BOOST_TEST(fabs(output[2] - 6.0) < 0.001); + + // Verify argmax indices + BOOST_TEST(pool.getArgMaxIndex(0) == 1u); + BOOST_TEST(pool.getArgMaxIndex(1) == 3u); + BOOST_TEST(pool.getArgMaxIndex(2) == 5u); +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool1d_backward) +{ + tinymind::MaxPool1D pool; + + double input[6] = {1.0, 3.0, 2.0, 5.0, 4.0, 6.0}; + double output[3]; + pool.forward(input, output); + + // Backprop: gradients should route to argmax positions + double outputDeltas[3] = {0.1, 0.2, 0.3}; + double inputDeltas[6]; + pool.backward(outputDeltas, inputDeltas); + + BOOST_TEST(fabs(inputDeltas[0]) < 0.001); // not max + BOOST_TEST(fabs(inputDeltas[1] - 0.1) < 0.001); // max of window 0 + BOOST_TEST(fabs(inputDeltas[2]) < 0.001); // not max + BOOST_TEST(fabs(inputDeltas[3] - 0.2) < 0.001); // max of window 1 + BOOST_TEST(fabs(inputDeltas[4]) < 0.001); // not max + BOOST_TEST(fabs(inputDeltas[5] - 0.3) < 0.001); // max of window 2 +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool1d_backward_fixed_point) +{ + // Same argmax routing in Q8.8: only the position holding each max + // should receive the upstream delta; others stay zero. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::MaxPool1D pool; + + ValueType input[6] = {ValueType(1, 0), ValueType(3, 0), ValueType(2, 0), + ValueType(5, 0), ValueType(4, 0), ValueType(6, 0)}; + ValueType output[3]; + pool.forward(input, output); + + ValueType outputDeltas[3] = {ValueType(1, 0), ValueType(2, 0), ValueType(3, 0)}; + ValueType inputDeltas[6]; + pool.backward(outputDeltas, inputDeltas); + + BOOST_TEST(inputDeltas[0].getValue() == 0); + BOOST_TEST(inputDeltas[1].getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(inputDeltas[2].getValue() == 0); + BOOST_TEST(inputDeltas[3].getValue() == ValueType(2, 0).getValue()); + BOOST_TEST(inputDeltas[4].getValue() == 0); + BOOST_TEST(inputDeltas[5].getValue() == ValueType(3, 0).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool1d_multichannel) +{ + // 2 channels, 4 elements each, pool size 2, stride 2 + tinymind::MaxPool1D pool; + + // Channel-major: [ch0: 1,4,2,3, ch1: 5,2,6,1] + double input[8] = {1.0, 4.0, 2.0, 3.0, 5.0, 2.0, 6.0, 1.0}; + double output[4]; // 2 channels * 2 outputs + + pool.forward(input, output); + + // Ch0: [1,4]->4, [2,3]->3 + BOOST_TEST(fabs(output[0] - 4.0) < 0.001); + BOOST_TEST(fabs(output[1] - 3.0) < 0.001); + // Ch1: [5,2]->5, [6,1]->6 + BOOST_TEST(fabs(output[2] - 5.0) < 0.001); + BOOST_TEST(fabs(output[3] - 6.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool1d_stride1) +{ + // Overlapping windows: pool size 3, stride 1 + tinymind::MaxPool1D pool; + + double input[5] = {2.0, 1.0, 4.0, 3.0, 5.0}; + double output[3]; // (5 - 3) / 1 + 1 = 3 + + pool.forward(input, output); + + // [2,1,4]->4, [1,4,3]->4, [4,3,5]->5 + BOOST_TEST(fabs(output[0] - 4.0) < 0.001); + BOOST_TEST(fabs(output[1] - 4.0) < 0.001); + BOOST_TEST(fabs(output[2] - 5.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool1d_static_sizes) +{ + typedef tinymind::MaxPool1D PoolType; + + // OutputLength = (100 - 4) / 4 + 1 = 25 + static_assert(PoolType::OutputLength == 25, "Wrong output length"); + static_assert(PoolType::OutputSize == 75, "Wrong output size"); // 3 * 25 + static_assert(PoolType::InputSize == 300, "Wrong input size"); // 3 * 100 + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool1d_fixed_point) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::MaxPool1D pool; + + ValueType input[4]; + input[0] = ValueType(1, 0); + input[1] = ValueType(3, 0); + input[2] = ValueType(2, 0); + input[3] = ValueType(5, 0); + + ValueType output[2]; + pool.forward(input, output); + + BOOST_TEST(output[0].getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(output[1].getValue() == ValueType(5, 0).getValue()); +} + +// ============================================================ +// AvgPool1D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_avgpool1d_forward) +{ + tinymind::AvgPool1D pool; + + double input[6] = {1.0, 3.0, 2.0, 6.0, 4.0, 8.0}; + double output[3]; + + pool.forward(input, output); + + // [1,3]->2, [2,6]->4, [4,8]->6 + BOOST_TEST(fabs(output[0] - 2.0) < 0.001); + BOOST_TEST(fabs(output[1] - 4.0) < 0.001); + BOOST_TEST(fabs(output[2] - 6.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool1d_backward) +{ + tinymind::AvgPool1D pool; + + double outputDeltas[3] = {0.2, 0.4, 0.6}; + double inputDeltas[6]; + pool.backward(outputDeltas, inputDeltas); + + // Each gradient splits evenly across pool window + BOOST_TEST(fabs(inputDeltas[0] - 0.1) < 0.001); + BOOST_TEST(fabs(inputDeltas[1] - 0.1) < 0.001); + BOOST_TEST(fabs(inputDeltas[2] - 0.2) < 0.001); + BOOST_TEST(fabs(inputDeltas[3] - 0.2) < 0.001); + BOOST_TEST(fabs(inputDeltas[4] - 0.3) < 0.001); + BOOST_TEST(fabs(inputDeltas[5] - 0.3) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool1d_multichannel) +{ + tinymind::AvgPool1D pool; + + double input[8] = {2.0, 4.0, 6.0, 8.0, 1.0, 3.0, 5.0, 7.0}; + double output[4]; + + pool.forward(input, output); + + // Ch0: [2,4]->3, [6,8]->7 + BOOST_TEST(fabs(output[0] - 3.0) < 0.001); + BOOST_TEST(fabs(output[1] - 7.0) < 0.001); + // Ch1: [1,3]->2, [5,7]->6 + BOOST_TEST(fabs(output[2] - 2.0) < 0.001); + BOOST_TEST(fabs(output[3] - 6.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool1d_pool3) +{ + tinymind::AvgPool1D pool; + + double input[6] = {3.0, 6.0, 9.0, 12.0, 15.0, 18.0}; + double output[2]; + + pool.forward(input, output); + + // [3,6,9]->6, [12,15,18]->15 + BOOST_TEST(fabs(output[0] - 6.0) < 0.001); + BOOST_TEST(fabs(output[1] - 15.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool1d_fixed_point) +{ + // Pin down the fixed-point divisor: average of [2.0, 4.0] in Q8.8 must be + // exactly 3.0, not raw=PoolSize. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::AvgPool1D pool; + + ValueType input[4] = {ValueType(2, 0), ValueType(4, 0), ValueType(6, 0), ValueType(8, 0)}; + ValueType output[2]; + pool.forward(input, output); + + // (2+4)/2 = 3.0; (6+8)/2 = 7.0. + BOOST_TEST(output[0].getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(output[1].getValue() == ValueType(7, 0).getValue()); +} + +// ============================================================ +// UpsampleNearest1D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_forward) +{ + // Repeat each element by factor 2: dual of a stride-2 pool. + tinymind::UpsampleNearest1D up; + + double input[4] = {3.0, 7.0, 2.0, 9.0}; + double output[8]; // 4 * 2 + + up.forward(input, output); + + const double expected[8] = {3.0, 3.0, 7.0, 7.0, 2.0, 2.0, 9.0, 9.0}; + for (size_t i = 0; i < 8; ++i) + { + BOOST_TEST(fabs(output[i] - expected[i]) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_backward) +{ + // Backward accumulates the ScaleFactor output grads onto each input. + tinymind::UpsampleNearest1D up; + + double outputDeltas[9] = {0.1, 0.2, 0.3, 1.0, 1.0, 1.0, 0.5, 0.0, -0.5}; + double inputDeltas[3]; + up.backward(outputDeltas, inputDeltas); + + BOOST_TEST(fabs(inputDeltas[0] - 0.6) < 0.001); // 0.1+0.2+0.3 + BOOST_TEST(fabs(inputDeltas[1] - 3.0) < 0.001); // 1+1+1 + BOOST_TEST(fabs(inputDeltas[2] - 0.0) < 0.001); // 0.5+0-0.5 +} + +BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_multichannel) +{ + // 2 channels, 3 elements each, factor 2, channel-major. + tinymind::UpsampleNearest1D up; + + double input[6] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}; + double output[12]; + up.forward(input, output); + + const double expected[12] = {1.0, 1.0, 2.0, 2.0, 3.0, 3.0, + 4.0, 4.0, 5.0, 5.0, 6.0, 6.0}; + for (size_t i = 0; i < 12; ++i) + { + BOOST_TEST(fabs(output[i] - expected[i]) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_static_sizes) +{ + typedef tinymind::UpsampleNearest1D UpType; + + static_assert(UpType::OutputLength == 200, "Wrong output length"); // 50 * 4 + static_assert(UpType::OutputSize == 600, "Wrong output size"); // 3 * 200 + static_assert(UpType::InputSize == 150, "Wrong input size"); // 3 * 50 + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_upsample_nearest1d_fixed_point) +{ + // No arithmetic: exact bit copies in Q8.8. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::UpsampleNearest1D up; + + ValueType input[2] = {ValueType(3, 0), ValueType(5, 0)}; + ValueType output[6]; + up.forward(input, output); + + BOOST_TEST(output[0].getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(output[1].getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(output[2].getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(output[3].getValue() == ValueType(5, 0).getValue()); + BOOST_TEST(output[4].getValue() == ValueType(5, 0).getValue()); + BOOST_TEST(output[5].getValue() == ValueType(5, 0).getValue()); +} + +// ============================================================ +// UpsampleLinear1D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_forward) +{ + // Factor 2: input nodes land on even outputs; odds are the midpoints. + tinymind::UpsampleLinear1D up; + + double input[4] = {0.0, 4.0, 2.0, 10.0}; + double output[8]; + up.forward(input, output); + + // node, mid, node, mid, node, mid, node, tail-clamped last value + const double expected[8] = {0.0, 2.0, 4.0, 3.0, 2.0, 6.0, 10.0, 10.0}; + for (size_t i = 0; i < 8; ++i) + { + BOOST_TEST(fabs(output[i] - expected[i]) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_nodes_preserved) +{ + // Every input element must reappear exactly at index k * ScaleFactor. + tinymind::UpsampleLinear1D up; + + double input[5] = {1.0, 2.0, 4.0, 8.0, 16.0}; + double output[15]; + up.forward(input, output); + + for (size_t k = 0; k < 5; ++k) + { + BOOST_TEST(fabs(output[k * 3] - input[k]) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_backward) +{ + // Backward is the transpose: a unit grad on a midpoint splits by weight. + tinymind::UpsampleLinear1D up; + + // outputs: [node0, mid, node1, tail]; mid weight = 0.5 to each node. + double outputDeltas[4] = {0.0, 1.0, 0.0, 0.0}; + double inputDeltas[2]; + up.backward(outputDeltas, inputDeltas); + + BOOST_TEST(fabs(inputDeltas[0] - 0.5) < 0.001); + BOOST_TEST(fabs(inputDeltas[1] - 0.5) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_upsample_linear1d_fixed_point) +{ + // Midpoint of [2.0, 6.0] in Q8.8 must be exactly 4.0 (weight 0.5 built via + // the (FixedPart, FractionalPart) constructor, not raw bits). + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::UpsampleLinear1D up; + + ValueType input[2] = {ValueType(2, 0), ValueType(6, 0)}; + ValueType output[4]; + up.forward(input, output); + + BOOST_TEST(output[0].getValue() == ValueType(2, 0).getValue()); // node + BOOST_TEST(output[1].getValue() == ValueType(4, 0).getValue()); // midpoint + BOOST_TEST(output[2].getValue() == ValueType(6, 0).getValue()); // node + BOOST_TEST(output[3].getValue() == ValueType(6, 0).getValue()); // tail clamp +} + +// ============================================================ +// ANFIS membership function tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_anfis_triangular_membership_function) +{ + typedef tinymind::TriangularMembershipFunction MfType; + + static_assert(MfType::NumberOfParameters == 3, "Triangle takes {a, b, c}"); + + const double parameters[3] = {0.0, 1.0, 2.0}; + + BOOST_TEST(fabs(MfType::evaluate(parameters, -1.0) - 0.0) < 0.001); // left of support + BOOST_TEST(fabs(MfType::evaluate(parameters, 0.0) - 0.0) < 0.001); // left foot + BOOST_TEST(fabs(MfType::evaluate(parameters, 0.25) - 0.25) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, 1.0) - 1.0) < 0.001); // peak + BOOST_TEST(fabs(MfType::evaluate(parameters, 1.5) - 0.5) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, 2.0) - 0.0) < 0.001); // right foot + BOOST_TEST(fabs(MfType::evaluate(parameters, 9.0) - 0.0) < 0.001); // right of support +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_triangular_shoulders) +{ + // a == b is a left shoulder, b == c a right shoulder. Neither may divide + // by zero: the degenerate side is unreachable from inside the support. + typedef tinymind::TriangularMembershipFunction MfType; + + const double leftShoulder[3] = {0.0, 0.0, 2.0}; + BOOST_TEST(fabs(MfType::evaluate(leftShoulder, 1.0) - 0.5) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(leftShoulder, 0.0) - 0.0) < 0.001); + + const double rightShoulder[3] = {0.0, 2.0, 2.0}; + BOOST_TEST(fabs(MfType::evaluate(rightShoulder, 1.0) - 0.5) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(rightShoulder, 2.0) - 0.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_trapezoidal_membership_function) +{ + typedef tinymind::TrapezoidalMembershipFunction MfType; + + static_assert(MfType::NumberOfParameters == 4, "Trapezoid takes {a, b, c, d}"); + + const double parameters[4] = {0.0, 1.0, 3.0, 4.0}; + + BOOST_TEST(fabs(MfType::evaluate(parameters, 0.0) - 0.0) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, 0.5) - 0.5) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, 1.0) - 1.0) < 0.001); // plateau start + BOOST_TEST(fabs(MfType::evaluate(parameters, 2.0) - 1.0) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, 3.0) - 1.0) < 0.001); // plateau end + BOOST_TEST(fabs(MfType::evaluate(parameters, 3.5) - 0.5) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, 4.0) - 0.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_generalized_bell_membership_function) +{ + // mu = 1 / (1 + (((x - c) / a)^2)^BellExponent) + typedef tinymind::GeneralizedBellMembershipFunction Bell1Type; + typedef tinymind::GeneralizedBellMembershipFunction Bell2Type; + + static_assert(Bell1Type::NumberOfParameters == 2, "Bell takes {a, c}"); + + const double parameters[2] = {1.0, 0.0}; // width 1, centered at 0 + + BOOST_TEST(fabs(Bell1Type::evaluate(parameters, 0.0) - 1.0) < 0.001); + BOOST_TEST(fabs(Bell1Type::evaluate(parameters, 1.0) - 0.5) < 0.001); // the half-width point + BOOST_TEST(fabs(Bell2Type::evaluate(parameters, 1.0) - 0.5) < 0.001); // b does not move it + BOOST_TEST(fabs(Bell2Type::evaluate(parameters, 0.5) - (1.0 / 1.0625)) < 0.001); + + // t >= 64 is reported as 0 so a narrow Q format cannot overflow. The clamp + // sits both before the exponent loop (t itself is already over) and inside + // it (t is under, but t^BellExponent is not: t = 9 -> t^2 = 81). + BOOST_TEST(fabs(Bell1Type::evaluate(parameters, 100.0) - 0.0) < 0.001); + BOOST_TEST(fabs(Bell2Type::evaluate(parameters, 3.0) - 0.0) < 0.001); + + // a == 0 degenerates to an impulse rather than dividing by zero. + const double impulse[2] = {0.0, 3.0}; + BOOST_TEST(fabs(Bell1Type::evaluate(impulse, 3.0) - 1.0) < 0.001); + BOOST_TEST(fabs(Bell1Type::evaluate(impulse, 3.5) - 0.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_gaussian_membership_function) +{ + // Parameters are {c, sigma, 1/sigma}; the reciprocal is precomputed so the + // inference path carries no divide. + typedef tinymind::GaussianMembershipFunction MfType; + + static_assert(MfType::NumberOfParameters == 3, "Gaussian takes {c, sigma, 1/sigma}"); + + const double parameters[3] = {0.0, 1.0, 1.0}; + + BOOST_TEST(fabs(MfType::evaluate(parameters, 0.0) - 1.0) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, 1.0) - exp(-0.5)) < 0.001); + BOOST_TEST(fabs(MfType::evaluate(parameters, -1.0) - exp(-0.5)) < 0.001); // symmetric + BOOST_TEST(fabs(MfType::evaluate(parameters, 4.0) - 0.0) < 0.001); // hard cut at 4 sigma +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_gaussian_membership_function_fixed_point) +{ + // The Q-format Gaussian rides the integer exp lookup table, so it needs no + // FPU. It must track the float form closely. + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::GaussianMembershipFunction MfType; + + const ValueType parameters[3] = {ValueType(0, 0), ValueType(1, 0), ValueType(1, 0)}; + + const ValueType mu = MfType::evaluate(parameters, ValueType(1, 0)); + const double muAsDouble = static_cast(mu.getValue()) / 65536.0; + BOOST_TEST(fabs(muAsDouble - exp(-0.5)) < 0.01); + + const ValueType far = MfType::evaluate(parameters, ValueType(5, 0)); + BOOST_TEST(far.getValue() == 0); +} + +// ============================================================ +// ANFIS rule table tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_anfis_full_grid_rule_table) +{ + typedef tinymind::AnfisFullGridRuleTable<2, 3> GridType; + + static_assert(GridType::NumberOfRules == 9, "3^2 == 9"); + static_assert(GridType::RuleTableSize == 18, "9 rules * 2 inputs"); + + uint8_t ruleTable[GridType::RuleTableSize]; + GridType::generate(ruleTable); + + // Odometer order: the last input varies fastest. + const uint8_t expected[18] = {0, 0, 0, 1, 0, 2, + 1, 0, 1, 1, 1, 2, + 2, 0, 2, 1, 2, 2}; + for (size_t i = 0; i < GridType::RuleTableSize; ++i) + { + BOOST_TEST(ruleTable[i] == expected[i]); + } +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_full_grid_rule_count_explodes) +{ + // The reason the rule base is an explicit table and not an implicit grid. + static_assert(tinymind::AnfisFullGridRuleTable<4, 3>::NumberOfRules == 81, "3^4"); + static_assert(tinymind::AnfisFullGridRuleTable<8, 3>::NumberOfRules == 6561, "3^8"); + BOOST_TEST(true); +} + +// ============================================================ +// ANFIS inference tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_anfis_zeroth_order_inference) +{ + // One input, two triangles crossing at 0.5, constant consequents 10 and 20. + // Both rules fire at 0.5 -> the output is the midpoint. + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::Anfis AnfisType; + + static_assert(AnfisType::NumberOfPremiseParameters == 6, "2 MFs * 3 parameters"); + static_assert(AnfisType::NumberOfConsequentParametersPerRule == 1, "Zeroth order is a constant"); + static_assert(AnfisType::NumberOfConsequentParameters == 2, "2 rules * 1 output * 1 parameter"); + + const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, + 0.0, 1.0, 2.0}; + const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; + const double consequent[AnfisType::NumberOfConsequentParameters] = {10.0, 20.0}; + + AnfisType anfis(premise, ruleTable, consequent); + + double input[1] = {0.5}; + double output[1]; + anfis.forward(input, output); + + BOOST_TEST(fabs(output[0] - 15.0) < 0.001); + BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(0) - 0.5) < 0.001); + BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(1) - 0.5) < 0.001); + + // Sitting on a peak makes that rule the only one that fires. + input[0] = 0.0; + anfis.forward(input, output); + BOOST_TEST(fabs(output[0] - 10.0) < 0.001); + BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(0) - 1.0) < 0.001); + BOOST_TEST(anfis.getDominantRule() == 0); + + input[0] = 1.0; + anfis.forward(input, output); + BOOST_TEST(fabs(output[0] - 20.0) < 0.001); + BOOST_TEST(anfis.getDominantRule() == 1); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_first_order_inference) +{ + // Two inputs, two triangles each, the full 2x2 grid, linear consequents. + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::AnfisFullGridRuleTable<2, 2> GridType; + typedef tinymind::Anfis AnfisType; + + static_assert(AnfisType::NumberOfConsequentParametersPerRule == 3, "2 coefficients + 1 constant"); + + uint8_t ruleTable[GridType::RuleTableSize]; + GridType::generate(ruleTable); + + const double premise[AnfisType::NumberOfPremiseParameters] = { + -1.0, 0.0, 1.0, 0.0, 1.0, 2.0, // input 0 + -1.0, 0.0, 1.0, 0.0, 1.0, 2.0}; // input 1 + + // f_r = p0 * x0 + p1 * x1 + q + const double consequent[AnfisType::NumberOfConsequentParameters] = { + 1.0, 0.0, 0.0, // rule 0: x0 + 0.0, 1.0, 0.0, // rule 1: x1 + 0.0, 0.0, 5.0, // rule 2: 5 + 1.0, 1.0, 1.0}; // rule 3: x0 + x1 + 1 + + AnfisType anfis(premise, ruleTable, consequent); + + double input[2] = {0.5, 0.5}; + double output[1]; + anfis.forward(input, output); + + // Every grade is 0.5, so every firing strength is 0.25 and the total is 1. + // f = [0.5, 0.5, 5.0, 2.0] -> y = 0.25 * 8.0 = 2.0 + BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 1.0) < 0.001); + BOOST_TEST(fabs(output[0] - 2.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_multiple_outputs_share_the_premise) +{ + // Two outputs off one premise layer: the membership grades and firing + // strengths are computed once and reused. + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::Anfis AnfisType; + + static_assert(AnfisType::NumberOfConsequentParameters == 4, "2 rules * 2 outputs * 1 parameter"); + + const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, + 0.0, 1.0, 2.0}; + const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; + // [rule][output] + const double consequent[AnfisType::NumberOfConsequentParameters] = {10.0, -1.0, + 20.0, 1.0}; + + AnfisType anfis(premise, ruleTable, consequent); + + double input[1] = {0.5}; + double output[2]; + anfis.forward(input, output); + + BOOST_TEST(fabs(output[0] - 15.0) < 0.001); + BOOST_TEST(fabs(output[1] - 0.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_dont_care_antecedent) +{ + // A pruned rule ignores an input entirely: the antecedent contributes a + // grade of 1 no matter how far outside every membership function it sits. + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::Anfis AnfisType; + + const double premise[AnfisType::NumberOfPremiseParameters] = { + -1.0, 0.0, 1.0, 0.0, 1.0, 2.0, + -1.0, 0.0, 1.0, 0.0, 1.0, 2.0}; + const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, AnfisType::DontCareIndex}; + const double consequent[AnfisType::NumberOfConsequentParameters] = {7.0}; + + AnfisType anfis(premise, ruleTable, consequent); + + double input[2] = {0.5, 99.0}; // input 1 is outside every membership function + double output[1]; + anfis.forward(input, output); + + BOOST_TEST(fabs(anfis.getFiringStrength(0) - 0.5) < 0.001); // grade of input 0 alone + BOOST_TEST(fabs(output[0] - 7.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_no_rule_fires) +{ + // Outside the support of every membership function nothing fires. The + // output must be zeroed without attempting a divide by zero. + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::Anfis AnfisType; + + const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0}; + const uint8_t ruleTable[AnfisType::RuleTableSize] = {0}; + const double consequent[AnfisType::NumberOfConsequentParameters] = {42.0}; + + AnfisType anfis(premise, ruleTable, consequent); + + double input[1] = {5.0}; + double output[1] = {123.0}; + anfis.forward(input, output); + + BOOST_TEST(fabs(output[0] - 0.0) < 0.001); + BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 0.0) < 0.001); + BOOST_TEST(fabs(anfis.getNormalizedFiringStrength(0) - 0.0) < 0.001); + + // Back inside the support the same system produces its consequent again: + // the dead path must not be sticky. + input[0] = 0.0; + anfis.forward(input, output); + BOOST_TEST(fabs(output[0] - 42.0) < 0.001); + BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 1.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_membership_grades_are_readable) +{ + // The premise layer is readable too, not just the rule layer. + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::Anfis AnfisType; + + const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, + 0.0, 1.0, 2.0}; + const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; + const double consequent[AnfisType::NumberOfConsequentParameters] = {10.0, 20.0}; + + AnfisType anfis(premise, ruleTable, consequent); + + double input[1] = {0.25}; + double output[1]; + anfis.forward(input, output); + + BOOST_TEST(fabs(anfis.getMembershipGrade(0, 0) - 0.75) < 0.001); + BOOST_TEST(fabs(anfis.getMembershipGrade(0, 1) - 0.25) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_partition_of_unity) +{ + // Two triangles that sum to 1 across the crossover region make a + // zeroth-order ANFIS interpolate linearly between its two consequents. + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::Anfis AnfisType; + + const double premise[AnfisType::NumberOfPremiseParameters] = {-1.0, 0.0, 1.0, + 0.0, 1.0, 2.0}; + const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; + const double consequent[AnfisType::NumberOfConsequentParameters] = {0.0, 10.0}; + + AnfisType anfis(premise, ruleTable, consequent); + + for (size_t step = 1; step < 10; ++step) + { + const double x = static_cast(step) / 10.0; + double input[1] = {x}; + double output[1]; + anfis.forward(input, output); + + BOOST_TEST(fabs(anfis.getTotalFiringStrength() - 1.0) < 0.001); + BOOST_TEST(fabs(output[0] - (10.0 * x)) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_anfis_fixed_point_inference) +{ + // Q16.16 is the recommended deployment format: the fused + // sum(w * f) / sum(w) defuzzification needs headroom for the weighted sum. + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::TriangularMembershipFunction MfType; + typedef tinymind::Anfis AnfisType; + + const ValueType premise[AnfisType::NumberOfPremiseParameters] = { + ValueType(-1, 0), ValueType(0, 0), ValueType(1, 0), + ValueType(0, 0), ValueType(1, 0), ValueType(2, 0)}; + const uint8_t ruleTable[AnfisType::RuleTableSize] = {0, 1}; + const ValueType consequent[AnfisType::NumberOfConsequentParameters] = {ValueType(10, 0), + ValueType(20, 0)}; + + AnfisType anfis(premise, ruleTable, consequent); + + ValueType input[1] = {ValueType(0, 1u << 15)}; // 0.5 + ValueType output[1]; + anfis.forward(input, output); + + BOOST_TEST(output[0].getValue() == ValueType(15, 0).getValue()); + + input[0] = ValueType(1, 0); + anfis.forward(input, output); + BOOST_TEST(output[0].getValue() == ValueType(20, 0).getValue()); +} + +// ============================================================ +// Dropout tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_dropout_inference_passthrough) +{ + // In inference mode, dropout should be identity + tinymind::Dropout dropout; + dropout.setTraining(false); + + double input[4] = {1.0, 2.0, 3.0, 4.0}; + double output[4]; + + dropout.forward(input, output); + + BOOST_TEST(fabs(output[0] - 1.0) < 0.001); + BOOST_TEST(fabs(output[1] - 2.0) < 0.001); + BOOST_TEST(fabs(output[2] - 3.0) < 0.001); + BOOST_TEST(fabs(output[3] - 4.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_training_zeros_some) +{ + // In training mode, some outputs should be zero + srand(RANDOM_SEED); + tinymind::Dropout dropout; + dropout.setTraining(true); + + double input[100]; + double output[100]; + for (size_t i = 0; i < 100; ++i) + { + input[i] = 1.0; + } + + dropout.forward(input, output); + + size_t zeroCount = 0; + size_t nonZeroCount = 0; + for (size_t i = 0; i < 100; ++i) + { + if (fabs(output[i]) < 0.001) + { + ++zeroCount; + } + else + { + ++nonZeroCount; + } + } + + // With 50% dropout on 100 elements, expect roughly 50 zeros + // Allow wide tolerance for randomness + BOOST_TEST(zeroCount > 20u); + BOOST_TEST(nonZeroCount > 20u); +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_inverted_scaling) +{ + // Verify inverted dropout scaling: survivors should be scaled by 1/(1-p) + // With 50% dropout, scale = 2.0 + srand(RANDOM_SEED); + tinymind::Dropout dropout; + dropout.setTraining(true); + + double input[10]; + double output[10]; + for (size_t i = 0; i < 10; ++i) + { + input[i] = 1.0; + } + + dropout.forward(input, output); + + for (size_t i = 0; i < 10; ++i) + { + if (dropout.getMask(i)) + { + // Kept: should be scaled by 2.0 (1/(1-0.5)) + BOOST_TEST(fabs(output[i] - 2.0) < 0.001); + } + else + { + // Dropped: should be zero + BOOST_TEST(fabs(output[i]) < 0.001); + } + } +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_backward_mask) +{ + srand(RANDOM_SEED); + tinymind::Dropout dropout; + dropout.setTraining(true); + + double input[5] = {1.0, 1.0, 1.0, 1.0, 1.0}; + double output[5]; + dropout.forward(input, output); // generates mask + + double outputDeltas[5] = {0.5, 0.5, 0.5, 0.5, 0.5}; + double inputDeltas[5]; + dropout.backward(outputDeltas, inputDeltas); + + for (size_t i = 0; i < 5; ++i) + { + if (dropout.getMask(i)) + { + // Gradient scaled by 1/(1-p) = 2.0 + BOOST_TEST(fabs(inputDeltas[i] - 1.0) < 0.001); + } + else + { + BOOST_TEST(fabs(inputDeltas[i]) < 0.001); + } + } +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_zero_percent) +{ + // 0% dropout should pass everything through even in training mode + tinymind::Dropout dropout; + dropout.setTraining(true); + + double input[4] = {1.0, 2.0, 3.0, 4.0}; + double output[4]; + + dropout.forward(input, output); + + BOOST_TEST(fabs(output[0] - 1.0) < 0.001); + BOOST_TEST(fabs(output[1] - 2.0) < 0.001); + BOOST_TEST(fabs(output[2] - 3.0) < 0.001); + BOOST_TEST(fabs(output[3] - 4.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_mode_toggle) +{ + tinymind::Dropout dropout; + + BOOST_TEST(dropout.isTraining() == true); // default is training + dropout.setTraining(false); + BOOST_TEST(dropout.isTraining() == false); + dropout.setTraining(true); + BOOST_TEST(dropout.isTraining() == true); +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_fixed_point_inference_passthrough) +{ + // In inference mode the forward pass is a verbatim copy, even with + // fixed-point types. No scaling, no mask sampling. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::Dropout dropout; + dropout.setTraining(false); + + ValueType input[4] = {ValueType(1, 0), ValueType(2, 0), ValueType(-1, 0), ValueType(0, 128)}; + ValueType output[4]; + dropout.forward(input, output); + + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(output[i].getValue() == input[i].getValue()); + } +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_fixed_point_training_scaling) +{ + // Q8.8 with 50% dropout: survivors must scale by exactly 2.0 = raw 512. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + srand(RANDOM_SEED); + tinymind::Dropout dropout; + dropout.setTraining(true); + + ValueType input[8]; + for (size_t i = 0; i < 8; ++i) + { + input[i] = ValueType(1, 0); // 1.0 + } + ValueType output[8]; + dropout.forward(input, output); + + for (size_t i = 0; i < 8; ++i) + { + if (dropout.getMask(i)) + { + BOOST_TEST(output[i].getValue() == ValueType(2, 0).getValue()); + } + else + { + BOOST_TEST(output[i].getValue() == 0); + } + } +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_high_rate_boundary) +{ + // 99% dropout is the largest legal rate (the layer static_asserts < 100). + // Scale = 100. Use Q16.16 since 100 doesn't fit in Q8.8's signed fixed range. + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + srand(RANDOM_SEED); + tinymind::Dropout dropout; + dropout.setTraining(true); + + ValueType input[16]; + for (size_t i = 0; i < 16; ++i) + { + input[i] = ValueType(1, 0); + } + ValueType output[16]; + dropout.forward(input, output); + + const ValueType expectedKept(100, 0); + bool sawDropped = false; + for (size_t i = 0; i < 16; ++i) + { + if (dropout.getMask(i)) + { + // 1 LSB tolerance for Q16.16 division rounding in scale construction. + const auto delta = output[i].getValue() - expectedKept.getValue(); + BOOST_TEST(std::abs(delta) <= 1); + } + else + { + BOOST_TEST(output[i].getValue() == 0); + sawDropped = true; + } + } + // With 99% drop on 16 elements the chance of zero drops is ~1.5e-32. + BOOST_TEST(sawDropped); +} + +BOOST_AUTO_TEST_CASE(test_case_dropout_seed_determinism) +{ + // Identical srand seeds must produce identical masks across separate + // dropout instances — needed for reproducible training runs. + tinymind::Dropout a; + tinymind::Dropout b; + double input[32]; + for (size_t i = 0; i < 32; ++i) input[i] = 1.0; + double outA[32], outB[32]; + + srand(RANDOM_SEED); + a.forward(input, outA); + srand(RANDOM_SEED); + b.forward(input, outB); + + for (size_t i = 0; i < 32; ++i) + { + BOOST_TEST(a.getMask(i) == b.getMask(i)); + BOOST_TEST(outA[i] == outB[i]); + } +} + +// ============================================================ +// RMSprop optimizer tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_rmsprop_float_xor) +{ + // Verify RMSprop optimizer converges on XOR (floating-point) + typedef double ValueType; + typedef FloatingPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy> BaseTF; + + struct RmsPropTF : public BaseTF + { + typedef tinymind::RmsPropOptimizerFloat OptimizerPolicyType; + }; + + typedef tinymind::MultilayerPerceptron NNType; + + srand(RANDOM_SEED); + NNType nn; + + static const double ERROR_LIMIT = ValueHelper::getErrorLimit(); + ValueType values[2], output[1], error; + std::deque errors; + + for (int i = 0; i < 10000; ++i) + { + generateXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + nn.trainNetwork(&output[0]); + + errors.push_front(error); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + } + + const double totalError = std::accumulate(errors.begin(), errors.end(), 0.0); + const double averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +BOOST_AUTO_TEST_CASE(test_case_rmsprop_fixedpoint_xor) +{ + // Verify RMSprop optimizer converges on XOR (fixed-point) + // Adaptive optimizers need gradient clipping with fixed-point to + // prevent overflow in the moment accumulation + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + typedef typename ValueType::FullWidthValueType FullWidthValueType; + + typedef tinymind::FixedPointTransferFunctions< + ValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + 1, + tinymind::DefaultNetworkInitializer, + tinymind::MeanSquaredErrorCalculator, + tinymind::ZeroToleranceCalculator, + tinymind::GradientClipByValue, + tinymind::NullWeightDecayPolicy, + tinymind::FixedLearningRatePolicy, + tinymind::RmsPropOptimizer> TransferFunctionsType; + + typedef tinymind::MultilayerPerceptron NNType; + + srand(RANDOM_SEED); + NNType nn; + + static const FullWidthValueType ERROR_LIMIT = ValueHelper::getErrorLimit(); + ValueType values[2], output[1], error; + std::deque errors; + + static const int RMSPROP_FP_ITERATIONS = TRAINING_ITERATIONS; + + for (int i = 0; i < RMSPROP_FP_ITERATIONS; ++i) + { + generateFixedPointXorValues(&values[0], &output[0]); + nn.feedForward(&values[0]); + error = nn.calculateError(&output[0]); + + if (!NNType::NeuralNetworkTransferFunctionsPolicy::isWithinZeroTolerance(error)) + { + nn.trainNetwork(&output[0]); + } + + errors.push_front(error.getValue()); + if (errors.size() > NUM_SAMPLES_AVG_ERROR) + { + errors.pop_back(); + } + } + + const FullWidthValueType totalError = std::accumulate(errors.begin(), errors.end(), static_cast(0)); + const FullWidthValueType averageError = (totalError / static_cast(NUM_SAMPLES_AVG_ERROR)); + BOOST_TEST(averageError <= ERROR_LIMIT); +} + +// ============================================================ +// Conv1D + Pool1D integration test +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_conv1d_maxpool1d_pipeline) +{ + // Conv1D(8 input, kernel=3) -> MaxPool1D(6 input, pool=2) + tinymind::Conv1D conv; + typedef tinymind::MaxPool1D PoolType; + PoolType pool; + + // Set filter 0: [1, 1, 1], bias=0 (moving sum) + conv.setFilterWeight(0, 0, 1.0); + conv.setFilterWeight(0, 1, 1.0); + conv.setFilterWeight(0, 2, 1.0); + conv.setFilterWeight(0, 3, 0.0); + + // Set filter 1: [1, 0, -1], bias=0 (edge detect) + conv.setFilterWeight(1, 0, 1.0); + conv.setFilterWeight(1, 1, 0.0); + conv.setFilterWeight(1, 2, -1.0); + conv.setFilterWeight(1, 3, 0.0); + + double input[8] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0}; + double convOutput[12]; // 2 filters * 6 positions + double poolOutput[6]; // 2 channels * 3 positions + + conv.forward(input, convOutput); + + // Filter 0 (sum): [3, 6, 9, 12, 15, 18] + BOOST_TEST(fabs(convOutput[0] - 3.0) < 0.001); + BOOST_TEST(fabs(convOutput[5] - 18.0) < 0.001); + + pool.forward(convOutput, poolOutput); + + // Pool ch0: [3,6]->6, [9,12]->12, [15,18]->18 + BOOST_TEST(fabs(poolOutput[0] - 6.0) < 0.001); + BOOST_TEST(fabs(poolOutput[1] - 12.0) < 0.001); + BOOST_TEST(fabs(poolOutput[2] - 18.0) < 0.001); +} + +// ============================================================ +// BatchNorm1D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_normalizes_output) +{ + // BatchNorm should normalize input to approximately zero mean + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + double input[4] = {2.0, 4.0, 6.0, 8.0}; + double output[4]; + + bn.forward(input, output); + + // Mean of normalized output should be ~0 + double mean = 0.0; + for (size_t i = 0; i < 4; ++i) + { + mean += output[i]; + } + mean /= 4.0; + BOOST_TEST(fabs(mean) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_unit_variance) +{ + // With default gamma=1, beta=0, output should have unit variance + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + double input[4] = {1.0, 3.0, 5.0, 7.0}; + double output[4]; + + bn.forward(input, output); + + // Compute variance of output + double mean = 0.0; + for (size_t i = 0; i < 4; ++i) + { + mean += output[i]; + } + mean /= 4.0; + + double variance = 0.0; + for (size_t i = 0; i < 4; ++i) + { + const double diff = output[i] - mean; + variance += diff * diff; + } + variance /= 4.0; + + // Variance should be ~1.0 (normalized) + BOOST_TEST(fabs(variance - 1.0) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_gamma_beta) +{ + // Custom gamma and beta should scale and shift the normalized output + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + // Set gamma=2, beta=3 for all elements + for (size_t i = 0; i < 4; ++i) + { + bn.setGamma(i, 2.0); + bn.setBeta(i, 3.0); + } + + double input[4] = {1.0, 3.0, 5.0, 7.0}; + double output[4]; + + bn.forward(input, output); + + // Output mean should be ~3 (beta) since normalized mean is 0 + double mean = 0.0; + for (size_t i = 0; i < 4; ++i) + { + mean += output[i]; + } + mean /= 4.0; + BOOST_TEST(fabs(mean - 3.0) < 0.01); + + // Output variance should be ~4 (gamma^2 * 1.0) + double variance = 0.0; + for (size_t i = 0; i < 4; ++i) + { + const double diff = output[i] - mean; + variance += diff * diff; + } + variance /= 4.0; + BOOST_TEST(fabs(variance - 4.0) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_inference_uses_running_stats) +{ + // After training, inference should use running mean/variance + tinymind::BatchNorm1D bn; // momentum=100% so running stats match batch stats immediately + + double input[4] = {2.0, 4.0, 6.0, 8.0}; + double trainOutput[4]; + double inferOutput[4]; + + // Training forward pass to populate running stats + bn.setTraining(true); + bn.forward(input, trainOutput); + + // Switch to inference + bn.setTraining(false); + bn.forward(input, inferOutput); + + // With 100% momentum, running stats equal batch stats, so outputs should match + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(fabs(trainOutput[i] - inferOutput[i]) < 0.01); + } +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_constant_input) +{ + // Constant input should produce zero-centered output (all zeros with default gamma=1, beta=0) + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + double input[4] = {5.0, 5.0, 5.0, 5.0}; + double output[4]; + + bn.forward(input, output); + + // All inputs equal -> variance ~0 -> normalized is ~0 -> output is ~beta=0 + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(fabs(output[i]) < 0.1); + } +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_backward_gradients) +{ + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + double input[4] = {1.0, 2.0, 3.0, 4.0}; + double output[4]; + bn.forward(input, output); + + // Backward with uniform gradients + double outputDeltas[4] = {1.0, 1.0, 1.0, 1.0}; + double inputDeltas[4]; + bn.backward(outputDeltas, inputDeltas); + + // Beta gradient should be sum of outputDeltas = 4.0 total, 1.0 per element + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(fabs(bn.getBetaGradient(i) - 1.0) < 0.001); + } + + // Gamma gradient = outputDelta * normalized, and normalized values + // should have zero mean, so gamma gradients should sum to ~0 + double gammaGradSum = 0.0; + for (size_t i = 0; i < 4; ++i) + { + gammaGradSum += bn.getGammaGradient(i); + } + BOOST_TEST(fabs(gammaGradSum) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_mode_toggle) +{ + tinymind::BatchNorm1D bn; + + BOOST_TEST(bn.isTraining() == true); // default is training + bn.setTraining(false); + BOOST_TEST(bn.isTraining() == false); + bn.setTraining(true); + BOOST_TEST(bn.isTraining() == true); +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_update_parameters) +{ + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + double input[4] = {1.0, 2.0, 3.0, 4.0}; + double output[4]; + bn.forward(input, output); + + double outputDeltas[4] = {1.0, 1.0, 1.0, 1.0}; + double inputDeltas[4]; + bn.backward(outputDeltas, inputDeltas); + + // Update with a learning rate + bn.updateParameters(0.01); + + // Gamma should have changed (gradient was non-zero for at least some elements) + // Beta should have changed (gradient was 1.0 for all elements) + bool betaChanged = false; + for (size_t i = 0; i < 4; ++i) + { + if (fabs(bn.getBeta(i)) > 0.001) + { + betaChanged = true; + } + } + BOOST_TEST(betaChanged); + + // Gradients should be zeroed after update + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(fabs(bn.getGammaGradient(i)) < 0.001); + BOOST_TEST(fabs(bn.getBetaGradient(i)) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_square_root_qvalue_perfect_squares) +{ + // Integer Newton's method must produce exact results on perfect squares. + typedef tinymind::QValue<8, 8, true> Q88; + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(0, 0)).getValue() + == 0); + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(1, 0)).getValue() + == Q88(1, 0).getValue()); + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(4, 0)).getValue() + == Q88(2, 0).getValue()); + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(9, 0)).getValue() + == Q88(3, 0).getValue()); + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q88(16, 0)).getValue() + == Q88(4, 0).getValue()); + + typedef tinymind::QValue<16, 16, true> Q1616; + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q1616(100, 0)).getValue() + == Q1616(10, 0).getValue()); + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(Q1616(10000, 0)).getValue() + == Q1616(100, 0).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_square_root_qvalue_non_squares) +{ + // Compare integer-sqrt result against the float reference within + // one raw-LSB tolerance (truncation rounding). + typedef tinymind::QValue<16, 16, true> ValueType; + const double inputs[] = { 0.25, 0.5, 2.0, 3.0, 0.01, 0.1, 1.5, 0.75 }; + for (size_t i = 0; i < sizeof(inputs) / sizeof(inputs[0]); ++i) + { + const ValueType v = tinymind::ValueConverter::convertToDestinationType(inputs[i]); + const ValueType result = tinymind::SquareRootApproximation::sqrt(v); + const double resultD = static_cast(result.getValue()) + / static_cast(1ULL << ValueType::NumberOfFractionalBits); + const double expected = std::sqrt(inputs[i]); + // Up to ~3 Q-LSBs of error: input quantization (truncation) compounds + // with the integer-sqrt truncation. Use 4 LSBs as a safe tolerance. + BOOST_TEST(std::fabs(resultD - expected) < 4.0 / (1ULL << ValueType::NumberOfFractionalBits)); + } +} + +BOOST_AUTO_TEST_CASE(test_case_square_root_qvalue_clamps_negative) +{ + // Negative inputs must clamp to zero, not invoke std::sqrt(NaN). + typedef tinymind::QValue<8, 8, true> Q88; + Q88 negative; + negative = static_cast(-256); // -1.0 + BOOST_TEST(tinymind::SquareRootApproximation::sqrt(negative).getValue() == 0); +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_qvalue_hyperparam_helpers) +{ + // BatchNorm with QValue must derive epsilon/momentum/sizeValue without + // going through ValueConverter. This catches a bug we + // had where gating the float specializations in nnproperties.hpp left + // batchnorm with a silently-zero epsilon (then SIGFPE on sqrt(0+0)). + typedef tinymind::QValue<16, 16, true> ValueType; + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + ValueType input[4]; + input[0] = ValueType(0, 0); + input[1] = ValueType(0, 0); + input[2] = ValueType(0, 0); + input[3] = ValueType(0, 0); + + ValueType output[4]; + // All-zero input -> mean=0, variance=0. Without a non-zero epsilon + // we'd hit a divide-by-zero in invStd. The helper must produce + // epsilon ~ 0.01 for EpsilonPercent=1. + bn.forward(input, output); + + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(output[i].getValue() == 0); + } +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_fixed_point) +{ + // Verify BatchNorm compiles and runs with Q16.16 fixed-point + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + tinymind::BatchNorm1D bn; + bn.setTraining(true); + + ValueType input[4]; + input[0] = ValueType(1, 0); + input[1] = ValueType(3, 0); + input[2] = ValueType(5, 0); + input[3] = ValueType(7, 0); + + ValueType output[4]; + bn.forward(input, output); + + // Normalized output should have mean ~0 + // Sum the raw values - they should roughly cancel out + typename ValueType::FullWidthValueType sum = 0; + for (size_t i = 0; i < 4; ++i) + { + sum += output[i].getValue(); + } + // Allow tolerance for fixed-point rounding + const typename ValueType::FullWidthValueType tolerance = 1 << (ValueType::NumberOfFractionalBits - 2); + BOOST_TEST(std::abs(sum) <= tolerance); +} + +BOOST_AUTO_TEST_CASE(test_case_batchnorm_conv1d_pipeline) +{ + // Conv1D -> BatchNorm -> output pipeline + tinymind::Conv1D conv; + tinymind::BatchNorm1D bn; // Conv output is (8-3)/1+1 = 6 + + // Set kernel to [1, 1, 1], bias=0 (moving sum) + conv.setFilterWeight(0, 0, 1.0); + conv.setFilterWeight(0, 1, 1.0); + conv.setFilterWeight(0, 2, 1.0); + conv.setFilterWeight(0, 3, 0.0); + + double input[8] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}; + double convOutput[6]; + double bnOutput[6]; + + conv.forward(input, convOutput); + // Conv output: [6, 9, 12, 15, 18, 21] + + bn.setTraining(true); + bn.forward(convOutput, bnOutput); + + // BatchNorm output should have zero mean + double mean = 0.0; + for (size_t i = 0; i < 6; ++i) + { + mean += bnOutput[i]; + } + mean /= 6.0; + BOOST_TEST(fabs(mean) < 0.01); +} + +// ============================================================ +// BinaryDense tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_forward_basic) +{ + // 4 inputs, 2 outputs + tinymind::BinaryDense layer; + + // Set latent weights: output 0 = [+1, +1, -1, -1], output 1 = [+1, -1, +1, -1] + layer.setLatentWeight(0, 0, 0.5); + layer.setLatentWeight(0, 1, 0.3); + layer.setLatentWeight(0, 2, -0.7); + layer.setLatentWeight(0, 3, -0.2); + + layer.setLatentWeight(1, 0, 0.9); + layer.setLatentWeight(1, 1, -0.4); + layer.setLatentWeight(1, 2, 0.6); + layer.setLatentWeight(1, 3, -0.1); + + layer.setBias(0, 0.0); + layer.setBias(1, 0.0); + + layer.binarizeWeights(); + + // Verify binarization: positive latent -> +1, negative latent -> -1 + BOOST_TEST(layer.getBinaryWeight(0, 0) == 1.0); + BOOST_TEST(layer.getBinaryWeight(0, 1) == 1.0); + BOOST_TEST(layer.getBinaryWeight(0, 2) == -1.0); + BOOST_TEST(layer.getBinaryWeight(0, 3) == -1.0); + + BOOST_TEST(layer.getBinaryWeight(1, 0) == 1.0); + BOOST_TEST(layer.getBinaryWeight(1, 1) == -1.0); + BOOST_TEST(layer.getBinaryWeight(1, 2) == 1.0); + BOOST_TEST(layer.getBinaryWeight(1, 3) == -1.0); + + // Input: [1.0, -1.0, 1.0, -1.0] => sign = [+1, -1, +1, -1] + double input[4] = {1.0, -1.0, 1.0, -1.0}; + double output[2]; + layer.forward(input, output); + + // output[0]: sign(input) dot weights[0] = (+1)(+1) + (-1)(+1) + (+1)(-1) + (-1)(-1) + // = 1 - 1 - 1 + 1 = 0 + BOOST_TEST(fabs(output[0] - 0.0) < 0.01); + + // output[1]: sign(input) dot weights[1] = (+1)(+1) + (-1)(-1) + (+1)(+1) + (-1)(-1) + // = 1 + 1 + 1 + 1 = 4 + BOOST_TEST(fabs(output[1] - 4.0) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_forward_with_bias) +{ + tinymind::BinaryDense layer; + + // All positive latent weights => binary weights all +1 + layer.setLatentWeight(0, 0, 0.5); + layer.setLatentWeight(0, 1, 0.5); + layer.setLatentWeight(0, 2, 0.5); + layer.setBias(0, 2.0); + layer.binarizeWeights(); + + // Input: [1.0, 1.0, 1.0] => sign = [+1, +1, +1] + // dot product = 3, plus bias 2 = 5 + double input[3] = {1.0, 1.0, 1.0}; + double output[1]; + layer.forward(input, output); + + BOOST_TEST(fabs(output[0] - 5.0) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_static_sizes) +{ + typedef tinymind::BinaryDense BinType; + + // 64 * 16 = 1024 binary weights + static_assert(BinType::TotalBinaryWeights == 1024, "Wrong total binary weights"); + // 1024 / 32 = 32 packed words + static_assert(BinType::PackedWeightWords == 32, "Wrong packed weight words"); + // 64 / 32 = 2 packed input words + static_assert(BinType::PackedInputWords == 2, "Wrong packed input words"); + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_packed_storage) +{ + // Verify that packed weights use minimal storage + typedef tinymind::BinaryDense BinType; + + // 33 bits needs 2 words (ceil(33/32) = 2) + static_assert(BinType::PackedWeightWords == 2, "Wrong packed words for 33 bits"); + static_assert(BinType::PackedInputWords == 2, "Wrong packed input words for 33 bits"); + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_backward_ste) +{ + tinymind::BinaryDense layer; + + // Set latent weights within [-1, 1] + layer.setLatentWeight(0, 0, 0.5); + layer.setLatentWeight(0, 1, -0.3); + layer.setBias(0, 0.0); + layer.binarizeWeights(); + + double input[2] = {1.0, -1.0}; + double output[1]; + layer.forward(input, output); + + double outputDeltas[1] = {1.0}; + double inputDeltas[2]; + layer.backward(outputDeltas, input, inputDeltas); + + // Learning rate = -0.1 (negative because gradient descent) + layer.updateWeights(-0.1); + + // Latent weights should have been updated + // w0: 0.5 + (-0.1) * (1.0 * sign(1.0)) = 0.5 - 0.1 = 0.4 + // w1: -0.3 + (-0.1) * (1.0 * sign(-1.0)) = -0.3 + 0.1 = -0.2 + BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - 0.4) < 0.01); + BOOST_TEST(fabs(layer.getLatentWeight(0, 1) - (-0.2)) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_updateweights_sign_flip) +{ + // updateWeights() must call binarizeWeights() so that a latent weight + // crossing zero immediately flips the packed weight sign on the next + // forward pass. + tinymind::BinaryDense layer; + layer.setLatentWeight(0, 0, 0.05); // small positive + layer.setBias(0, 0.0); + layer.binarizeWeights(); + BOOST_TEST(layer.getBinaryWeight(0, 0) == 1.0); + + // Run backward to populate gradient. With input=+1 and outputDelta=+1, + // mLatentGradients[0] = +1.0 (within STE clip range). + double input[1] = {1.0}; + double output[1]; + layer.forward(input, output); + double outputDeltas[1] = {1.0}; + layer.backward(outputDeltas, input, nullptr); + + // lr = -0.1 pushes latent to 0.05 - 0.1 = -0.05; re-binarize to -1. + layer.updateWeights(-0.1); + BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - (-0.05)) < 1e-9); + BOOST_TEST(layer.getBinaryWeight(0, 0) == -1.0); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_ste_clipping) +{ + tinymind::BinaryDense layer; + + // Set one latent weight outside [-1, 1] — STE should zero its gradient + layer.setLatentWeight(0, 0, 1.5); // outside [-1, 1] + layer.setLatentWeight(0, 1, 0.5); // inside [-1, 1] + layer.setBias(0, 0.0); + layer.binarizeWeights(); + + double input[2] = {1.0, 1.0}; + double output[1]; + layer.forward(input, output); + + double outputDeltas[1] = {1.0}; + layer.backward(outputDeltas, input, nullptr); + layer.updateWeights(-0.1); + + // w0 at 1.5 is outside [-1,1], gradient clipped to 0 => stays at 1.5 + BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - 1.5) < 0.01); + // w1 at 0.5 is inside [-1,1], gradient applies => 0.5 + (-0.1)(1.0*1.0) = 0.4 + BOOST_TEST(fabs(layer.getLatentWeight(0, 1) - 0.4) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_fixed_point) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::BinaryDense layer; + + // Set latent weights using ValueConverter + typedef tinymind::ValueConverter Conv; + layer.setLatentWeight(0, 0, Conv::convertToDestinationType(0.5)); + layer.setLatentWeight(0, 1, Conv::convertToDestinationType(-0.5)); + layer.setLatentWeight(0, 2, Conv::convertToDestinationType(0.5)); + layer.setLatentWeight(0, 3, Conv::convertToDestinationType(-0.5)); + layer.setBias(0, Conv::convertToDestinationType(0.0)); + layer.binarizeWeights(); + + // Binary weights should be [+1, -1, +1, -1] + ValueType posOne = Conv::convertToDestinationType(1.0); + ValueType negOne = Conv::convertToDestinationType(-1.0); + BOOST_TEST(layer.getBinaryWeight(0, 0).getValue() == posOne.getValue()); + BOOST_TEST(layer.getBinaryWeight(0, 1).getValue() == negOne.getValue()); + BOOST_TEST(layer.getBinaryWeight(0, 2).getValue() == posOne.getValue()); + BOOST_TEST(layer.getBinaryWeight(0, 3).getValue() == negOne.getValue()); + + // Input: all +1 => sign = [+1, +1, +1, +1] + // dot = (+1)(+1) + (+1)(-1) + (+1)(+1) + (+1)(-1) = 0 + ValueType input[4]; + input[0] = Conv::convertToDestinationType(1.0); + input[1] = Conv::convertToDestinationType(1.0); + input[2] = Conv::convertToDestinationType(1.0); + input[3] = Conv::convertToDestinationType(1.0); + + ValueType output[1]; + layer.forward(input, output); + + ValueType expected = Conv::convertToDestinationType(0.0); + BOOST_TEST(output[0].getValue() == expected.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_binary_dense_input_gradient_propagation) +{ + tinymind::BinaryDense layer; + + layer.setLatentWeight(0, 0, 0.5); + layer.setLatentWeight(0, 1, -0.5); + layer.setLatentWeight(0, 2, 0.5); + + layer.setLatentWeight(1, 0, -0.5); + layer.setLatentWeight(1, 1, 0.5); + layer.setLatentWeight(1, 2, -0.5); + + layer.setBias(0, 0.0); + layer.setBias(1, 0.0); + layer.binarizeWeights(); + + double input[3] = {1.0, 1.0, 1.0}; + double output[2]; + layer.forward(input, output); + + // output deltas both = 1.0 + double outputDeltas[2] = {1.0, 1.0}; + double inputDeltas[3]; + layer.backward(outputDeltas, input, inputDeltas); + + // inputDeltas[0] = delta0 * sign(w[0][0]) + delta1 * sign(w[1][0]) + // = 1.0 * (+1) + 1.0 * (-1) = 0 + BOOST_TEST(fabs(inputDeltas[0] - 0.0) < 0.01); + // inputDeltas[1] = 1.0 * (-1) + 1.0 * (+1) = 0 + BOOST_TEST(fabs(inputDeltas[1] - 0.0) < 0.01); + // inputDeltas[2] = 1.0 * (+1) + 1.0 * (-1) = 0 + BOOST_TEST(fabs(inputDeltas[2] - 0.0) < 0.01); +} + +// ============================================================ +// TernaryDense tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_forward_basic) +{ + // 4 inputs, 2 outputs, threshold 50% + tinymind::TernaryDense layer; + + // Set latent weights with clear magnitudes + // Large values => +1 or -1, small values near 0 => 0 + layer.setLatentWeight(0, 0, 0.9); // should be +1 + layer.setLatentWeight(0, 1, 0.01); // should be 0 (below threshold) + layer.setLatentWeight(0, 2, -0.8); // should be -1 + layer.setLatentWeight(0, 3, 0.02); // should be 0 (below threshold) + + layer.setLatentWeight(1, 0, -0.7); // should be -1 + layer.setLatentWeight(1, 1, 0.85); // should be +1 + layer.setLatentWeight(1, 2, 0.03); // should be 0 (below threshold) + layer.setLatentWeight(1, 3, -0.9); // should be -1 + + layer.setBias(0, 0.0); + layer.setBias(1, 0.0); + layer.ternarizeWeights(); + + // Verify ternarization + // Mean |w| = (0.9+0.01+0.8+0.02+0.7+0.85+0.03+0.9)/8 = 4.21/8 = 0.52625 + // threshold = 0.5 * 0.52625 = 0.263125 + // w[0][0]=0.9 > 0.263 => +1 + // w[0][1]=0.01 < 0.263 => 0 + // w[0][2]=-0.8, |0.8| > 0.263 => -1 + // w[0][3]=0.02 < 0.263 => 0 + BOOST_TEST(layer.getTernaryWeight(0, 0) == tinymind::detail::TERNARY_POS); + BOOST_TEST(layer.getTernaryWeight(0, 1) == tinymind::detail::TERNARY_ZERO); + BOOST_TEST(layer.getTernaryWeight(0, 2) == tinymind::detail::TERNARY_NEG); + BOOST_TEST(layer.getTernaryWeight(0, 3) == tinymind::detail::TERNARY_ZERO); + + BOOST_TEST(layer.getTernaryWeight(1, 0) == tinymind::detail::TERNARY_NEG); + BOOST_TEST(layer.getTernaryWeight(1, 1) == tinymind::detail::TERNARY_POS); + BOOST_TEST(layer.getTernaryWeight(1, 2) == tinymind::detail::TERNARY_ZERO); + BOOST_TEST(layer.getTernaryWeight(1, 3) == tinymind::detail::TERNARY_NEG); + + // getTernaryWeightValue maps the packed code to +1 / 0 / -1; the + // dead-zone weights exercise the zero() return. + BOOST_TEST(layer.getTernaryWeightValue(0, 0) == 1.0); + BOOST_TEST(layer.getTernaryWeightValue(0, 1) == 0.0); + BOOST_TEST(layer.getTernaryWeightValue(0, 2) == -1.0); + BOOST_TEST(layer.getTernaryWeightValue(0, 3) == 0.0); + + // Input: [2.0, 3.0, 4.0, 5.0] + double input[4] = {2.0, 3.0, 4.0, 5.0}; + double output[2]; + layer.forward(input, output); + + // output[0] = (+1)*2 + 0*3 + (-1)*4 + 0*5 = 2 - 4 = -2 + BOOST_TEST(fabs(output[0] - (-2.0)) < 0.01); + + // output[1] = (-1)*2 + (+1)*3 + 0*4 + (-1)*5 = -2 + 3 - 5 = -4 + BOOST_TEST(fabs(output[1] - (-4.0)) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_forward_with_bias) +{ + tinymind::TernaryDense layer; + + // Both large positive => both +1 + layer.setLatentWeight(0, 0, 0.9); + layer.setLatentWeight(0, 1, 0.8); + layer.setBias(0, 1.5); + layer.ternarizeWeights(); + + double input[2] = {3.0, 4.0}; + double output[1]; + layer.forward(input, output); + + // output = (+1)*3 + (+1)*4 + 1.5 = 8.5 + BOOST_TEST(fabs(output[0] - 8.5) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_static_sizes) +{ + typedef tinymind::TernaryDense TernType; + + static_assert(TernType::TotalTernaryWeights == 1024, "Wrong total ternary weights"); + // 1024 ternary values, 16 per word = 64 words + static_assert(TernType::PackedWeightWords == 64, "Wrong packed weight words"); + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_all_zero_threshold) +{ + // With very high threshold, most weights should become 0 + tinymind::TernaryDense layer; + + layer.setLatentWeight(0, 0, 0.5); + layer.setLatentWeight(0, 1, -0.5); + layer.setLatentWeight(0, 2, 0.5); + layer.setLatentWeight(0, 3, -0.5); + layer.setBias(0, 0.0); + layer.ternarizeWeights(); + + // Mean |w| = 0.5, threshold = 0.99 * 0.5 = 0.495 + // All |w| = 0.5 > 0.495, so all should still be non-zero + // (need truly small weights relative to mean for zeroing) + double input[4] = {1.0, 1.0, 1.0, 1.0}; + double output[1]; + layer.forward(input, output); + + // With very uniform weights, threshold 99% of mean still passes + // output = (+1)*1 + (-1)*1 + (+1)*1 + (-1)*1 = 0 + BOOST_TEST(fabs(output[0]) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_backward_ste) +{ + tinymind::TernaryDense layer; + + layer.setLatentWeight(0, 0, 0.8); + layer.setLatentWeight(0, 1, -0.6); + layer.setBias(0, 0.0); + layer.ternarizeWeights(); + + double input[2] = {2.0, 3.0}; + double output[1]; + layer.forward(input, output); + + double outputDeltas[1] = {1.0}; + double inputDeltas[2]; + layer.backward(outputDeltas, input, inputDeltas); + layer.updateWeights(-0.1); + + // Gradient for w0: delta * input[0] = 1.0 * 2.0 = 2.0 + // w0: 0.8 + (-0.1)(2.0) = 0.6 + BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - 0.6) < 0.01); + + // Gradient for w1: delta * input[1] = 1.0 * 3.0 = 3.0 + // w1: -0.6 + (-0.1)(3.0) = -0.9 + BOOST_TEST(fabs(layer.getLatentWeight(0, 1) - (-0.9)) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_updateweights_sign_flip) +{ + // Verify updateWeights re-ternarizes after a latent weight crosses zero. + // With a single weight the threshold collapses to 0.5*|w|, so any nonzero + // latent maps to its sign. Initial latent +0.5 -> +1; gradient pushes it + // negative, packed value should flip to -1 immediately. + tinymind::TernaryDense layer; + layer.setLatentWeight(0, 0, 0.5); + layer.setBias(0, 0.0); + layer.ternarizeWeights(); + BOOST_TEST(layer.getTernaryWeightValue(0, 0) == 1.0); + + double input[1] = {2.0}; + double output[1]; + layer.forward(input, output); + double outputDeltas[1] = {1.0}; + layer.backward(outputDeltas, input, nullptr); + // Gradient = delta * input = 2.0; lr = -0.4 -> latent becomes 0.5 - 0.8 = -0.3. + layer.updateWeights(-0.4); + BOOST_TEST(fabs(layer.getLatentWeight(0, 0) - (-0.3)) < 1e-6); + BOOST_TEST(layer.getTernaryWeightValue(0, 0) == -1.0); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_input_gradient_propagation) +{ + tinymind::TernaryDense layer; + + // All large => all non-zero ternary + layer.setLatentWeight(0, 0, 0.9); // +1 + layer.setLatentWeight(0, 1, -0.8); // -1 + layer.setLatentWeight(0, 2, 0.7); // +1 + layer.setBias(0, 0.0); + layer.ternarizeWeights(); + + double input[3] = {1.0, 2.0, 3.0}; + double output[1]; + layer.forward(input, output); + + double outputDeltas[1] = {2.0}; + double inputDeltas[3]; + layer.backward(outputDeltas, input, inputDeltas); + + // inputDeltas[0] = delta * ternary(w[0][0]) = 2.0 * (+1) = 2.0 + BOOST_TEST(fabs(inputDeltas[0] - 2.0) < 0.01); + // inputDeltas[1] = delta * ternary(w[0][1]) = 2.0 * (-1) = -2.0 + BOOST_TEST(fabs(inputDeltas[1] - (-2.0)) < 0.01); + // inputDeltas[2] = delta * ternary(w[0][2]) = 2.0 * (+1) = 2.0 + BOOST_TEST(fabs(inputDeltas[2] - 2.0) < 0.01); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_fixed_point) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + typedef tinymind::ValueConverter Conv; + tinymind::TernaryDense layer; + + layer.setLatentWeight(0, 0, Conv::convertToDestinationType(0.9)); + layer.setLatentWeight(0, 1, Conv::convertToDestinationType(-0.8)); + layer.setLatentWeight(0, 2, Conv::convertToDestinationType(0.7)); + layer.setLatentWeight(0, 3, Conv::convertToDestinationType(-0.6)); + layer.setBias(0, Conv::convertToDestinationType(0.0)); + layer.ternarizeWeights(); + + // All weights are large magnitude => all non-zero ternary + BOOST_TEST(layer.getTernaryWeight(0, 0) == tinymind::detail::TERNARY_POS); + BOOST_TEST(layer.getTernaryWeight(0, 1) == tinymind::detail::TERNARY_NEG); + BOOST_TEST(layer.getTernaryWeight(0, 2) == tinymind::detail::TERNARY_POS); + BOOST_TEST(layer.getTernaryWeight(0, 3) == tinymind::detail::TERNARY_NEG); + + // Input: [2, 1, 3, 2] + // output = (+1)*2 + (-1)*1 + (+1)*3 + (-1)*2 = 2 - 1 + 3 - 2 = 2 + ValueType input[4]; + input[0] = Conv::convertToDestinationType(2.0); + input[1] = Conv::convertToDestinationType(1.0); + input[2] = Conv::convertToDestinationType(3.0); + input[3] = Conv::convertToDestinationType(2.0); + + ValueType output[1]; + layer.forward(input, output); + + ValueType expected = Conv::convertToDestinationType(2.0); + BOOST_TEST(output[0].getValue() == expected.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_ternary_dense_get_value_accessors) +{ + tinymind::TernaryDense layer; + + layer.setLatentWeight(0, 0, 0.9); + layer.setLatentWeight(0, 1, -0.8); + layer.setBias(0, 0.0); + layer.ternarizeWeights(); + + BOOST_TEST(fabs(layer.getTernaryWeightValue(0, 0) - 1.0) < 0.01); + BOOST_TEST(fabs(layer.getTernaryWeightValue(0, 1) - (-1.0)) < 0.01); +} + +// ============================================================ +// SelfAttention1D tests (floating-point) +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_static_sizes) +{ + typedef tinymind::SelfAttention1D AttnType; + + static_assert(AttnType::InputSize == 128, "Wrong input size"); // 16 * 8 + static_assert(AttnType::OutputSize == 64, "Wrong output size"); // 16 * 4 + static_assert(AttnType::WeightsPerProjection == 32, "Wrong weights per proj"); // 8 * 4 + static_assert(AttnType::TotalWeights == 96, "Wrong total weights"); // 3 * 32 + static_assert(AttnType::BiasesPerProjection == 4, "Wrong biases per proj"); + static_assert(AttnType::TotalBiases == 12, "Wrong total biases"); // 3 * 4 + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_identity_projection) +{ + // With identity-like projections and no ReLU clipping, + // verify output is non-zero and has expected dimensions. + // N=4, D=2, P=2 + tinymind::SelfAttention1D attn; + + // Set W_q = W_k = W_v = identity matrix + for (size_t proj = 0; proj < 3; ++proj) + { + for (size_t r = 0; r < 2; ++r) + { + for (size_t c = 0; c < 2; ++c) + { + attn.setProjectionWeight(proj, r, c, (r == c) ? 1.0 : 0.0); + } + } + } + + // Input: 4 time steps x 2 features, all positive (so ReLU passes through) + double input[8] = { + 1.0, 2.0, // t0 + 3.0, 4.0, // t1 + 5.0, 6.0, // t2 + 7.0, 8.0 // t3 + }; + double output[8]; // 4 x 2 + + attn.forward(input, output); + + // With identity projections on positive input: + // Q' = K' = V = X (ReLU is pass-through for positive values) + // KV = X^T * X (2x2 gram matrix) + // Out = X * KV = X * (X^T * X) + // Verify output is non-zero + bool anyNonZero = false; + for (size_t i = 0; i < 8; ++i) + { + if (fabs(output[i]) > 0.001) + { + anyNonZero = true; + break; + } + } + BOOST_TEST(anyNonZero); + + // Manually compute expected output: + // X^T * X = [[1+9+25+49, 2+12+30+56], [2+12+30+56, 4+16+36+64]] + // = [[84, 100], [100, 120]] + // Out[0] = X[0] * KV = [1,2] * [[84,100],[100,120]] = [284, 340] + BOOST_TEST(fabs(output[0] - 284.0) < 0.001); + BOOST_TEST(fabs(output[1] - 340.0) < 0.001); + + // Out[1] = [3,4] * [[84,100],[100,120]] = [652, 780] + BOOST_TEST(fabs(output[2] - 652.0) < 0.001); + BOOST_TEST(fabs(output[3] - 780.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_relu_clipping) +{ + // Verify ReLU zeroes out negative Q and K projections + // N=2, D=2, P=2 + tinymind::SelfAttention1D attn; + + // W_q = [[-1, 0], [0, -1]] (negates input -> ReLU clips to zero) + attn.setProjectionWeight(0, 0, 0, -1.0); + attn.setProjectionWeight(0, 0, 1, 0.0); + attn.setProjectionWeight(0, 1, 0, 0.0); + attn.setProjectionWeight(0, 1, 1, -1.0); + + // W_k = identity + attn.setProjectionWeight(1, 0, 0, 1.0); + attn.setProjectionWeight(1, 0, 1, 0.0); + attn.setProjectionWeight(1, 1, 0, 0.0); + attn.setProjectionWeight(1, 1, 1, 1.0); + + // W_v = identity + attn.setProjectionWeight(2, 0, 0, 1.0); + attn.setProjectionWeight(2, 0, 1, 0.0); + attn.setProjectionWeight(2, 1, 0, 0.0); + attn.setProjectionWeight(2, 1, 1, 1.0); + + double input[4] = {1.0, 2.0, 3.0, 4.0}; + double output[4]; + + attn.forward(input, output); + + // Q = ReLU([[-1,0],[0,-1]] * input) = ReLU(negative) = 0 + // Output = Q' * KV = 0 for all + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(fabs(output[i]) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_bias) +{ + // Verify biases are applied to projections + // N=2, D=2, P=2, zero weights, positive Q bias to produce non-zero output + tinymind::SelfAttention1D attn; + + // All weights zero (default) + // Set Q bias to positive values so Q' = ReLU(0 + bias) = bias + attn.setProjectionBias(0, 0, 1.0); + attn.setProjectionBias(0, 1, 1.0); + + // Set K bias so K' has positive values + attn.setProjectionBias(1, 0, 1.0); + attn.setProjectionBias(1, 1, 1.0); + + // Set V bias + attn.setProjectionBias(2, 0, 2.0); + attn.setProjectionBias(2, 1, 3.0); + + double input[4] = {0.0, 0.0, 0.0, 0.0}; + double output[4]; + + attn.forward(input, output); + + // Q' = [1,1] for each time step (from bias, ReLU pass-through) + // K' = [1,1] for each time step + // V = [2,3] for each time step + // KV = K'^T * V = [[1,1],[1,1]]^T (2x2) * [[2,3],[2,3]] (2x2) + // K'^T is (2x2): col0=[1,1], col1=[1,1] + // KV[0][0] = K'[0,0]*V[0,0] + K'[1,0]*V[1,0] = 1*2 + 1*2 = 4 + // KV[0][1] = K'[0,0]*V[0,1] + K'[1,0]*V[1,1] = 1*3 + 1*3 = 6 + // KV[1][0] = K'[0,1]*V[0,0] + K'[1,1]*V[1,0] = 1*2 + 1*2 = 4 + // KV[1][1] = K'[0,1]*V[0,1] + K'[1,1]*V[1,1] = 1*3 + 1*3 = 6 + // Out = Q' * KV: + // Out[0] = [1,1] * [[4,6],[4,6]] = [8, 12] + BOOST_TEST(fabs(output[0] - 8.0) < 0.001); + BOOST_TEST(fabs(output[1] - 12.0) < 0.001); + BOOST_TEST(fabs(output[2] - 8.0) < 0.001); + BOOST_TEST(fabs(output[3] - 12.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_weight_accessors) +{ + tinymind::SelfAttention1D attn; + + // Set and verify projection weights + attn.setProjectionWeight(0, 1, 0, 3.14); + BOOST_TEST(fabs(attn.getProjectionWeight(0, 1, 0) - 3.14) < 0.001); + + attn.setProjectionWeight(2, 2, 1, -1.5); + BOOST_TEST(fabs(attn.getProjectionWeight(2, 2, 1) - (-1.5)) < 0.001); + + // Verify flat accessor matches structured accessor + // proj=0, row=1, col=0 -> flat index = 0*6 + 1*2 + 0 = 2 + BOOST_TEST(fabs(attn.getWeight(2) - 3.14) < 0.001); + + // proj=2, row=2, col=1 -> flat index = 2*6 + 2*2 + 1 = 17 + BOOST_TEST(fabs(attn.getWeight(17) - (-1.5)) < 0.001); + + // Bias accessors + attn.setProjectionBias(1, 0, 0.5); + BOOST_TEST(fabs(attn.getProjectionBias(1, 0) - 0.5) < 0.001); + BOOST_TEST(fabs(attn.getBias(2) - 0.5) < 0.001); // flat index = 1*2 + 0 = 2 +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_gradients_nonzero) +{ + // After forward + computeGradients, weight gradients should be non-zero + tinymind::SelfAttention1D attn; + + // Identity projections + for (size_t proj = 0; proj < 3; ++proj) + { + attn.setProjectionWeight(proj, 0, 0, 1.0); + attn.setProjectionWeight(proj, 0, 1, 0.0); + attn.setProjectionWeight(proj, 1, 0, 0.0); + attn.setProjectionWeight(proj, 1, 1, 1.0); + } + + double input[8] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}; + double output[8]; + attn.forward(input, output); + + // Non-zero output deltas + double deltas[8] = {0.1, -0.1, 0.2, -0.2, 0.3, -0.3, 0.4, -0.4}; + attn.computeGradients(deltas); + + // At least some gradients should be non-zero + bool anyGradNonZero = false; + for (size_t i = 0; i < tinymind::SelfAttention1D::TotalWeights; ++i) + { + if (fabs(attn.getGradient(i)) > 1e-10) + { + anyGradNonZero = true; + break; + } + } + BOOST_TEST(anyGradNonZero); + + // Bias gradients should also be non-zero + bool anyBiasGradNonZero = false; + for (size_t i = 0; i < tinymind::SelfAttention1D::TotalBiases; ++i) + { + if (fabs(attn.getBiasGradient(i)) > 1e-10) + { + anyBiasGradNonZero = true; + break; + } + } + BOOST_TEST(anyBiasGradNonZero); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_weight_update) +{ + // Verify that updateWeights modifies weights in the correct direction + tinymind::SelfAttention1D attn; + + for (size_t proj = 0; proj < 3; ++proj) + { + attn.setProjectionWeight(proj, 0, 0, 1.0); + attn.setProjectionWeight(proj, 0, 1, 0.0); + attn.setProjectionWeight(proj, 1, 0, 0.0); + attn.setProjectionWeight(proj, 1, 1, 1.0); + } + + double input[4] = {1.0, 2.0, 3.0, 4.0}; + double output[4]; + attn.forward(input, output); + + double deltas[4] = {0.1, 0.2, 0.3, 0.4}; + attn.computeGradients(deltas); + + // Record weights before update + double wBefore = attn.getProjectionWeight(0, 0, 0); + double bBefore = attn.getProjectionBias(0, 0); + + // Negative learning rate for gradient descent + attn.updateWeights(-0.01); + + double wAfter = attn.getProjectionWeight(0, 0, 0); + double bAfter = attn.getProjectionBias(0, 0); + + // Weights should have changed + BOOST_TEST(fabs(wAfter - wBefore) > 1e-10); + BOOST_TEST(fabs(bAfter - bBefore) > 1e-10); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_conv1d_pipeline) +{ + // Conv1D -> SelfAttention1D pipeline + // Conv1D: 8 input, kernel=3, stride=1, 2 filters -> output 6*2=12 values + // Reshape as: 6 time steps x 2 features + // SelfAttention1D: N=6, D=2, P=2 + tinymind::Conv1D conv; + tinymind::SelfAttention1D attn; + + // Set conv filter 0: [1, 0, 0], bias=0 + conv.setFilterWeight(0, 0, 1.0); + conv.setFilterWeight(0, 1, 0.0); + conv.setFilterWeight(0, 2, 0.0); + conv.setFilterWeight(0, 3, 0.0); + + // Set conv filter 1: [0, 0, 1], bias=0 + conv.setFilterWeight(1, 0, 0.0); + conv.setFilterWeight(1, 1, 0.0); + conv.setFilterWeight(1, 2, 1.0); + conv.setFilterWeight(1, 3, 0.0); + + double convInput[8] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}; + double convOutput[12]; // 2 filters * 6 positions + + conv.forward(convInput, convOutput); + + // Reshape conv output (filter-major) to attention input (time-step-major) + double attnInput[12]; + for (size_t t = 0; t < 6; ++t) + { + attnInput[t * 2 + 0] = convOutput[t]; // filter 0 + attnInput[t * 2 + 1] = convOutput[6 + t]; // filter 1 + } + + // Identity projections for attention + for (size_t proj = 0; proj < 3; ++proj) + { + attn.setProjectionWeight(proj, 0, 0, 1.0); + attn.setProjectionWeight(proj, 0, 1, 0.0); + attn.setProjectionWeight(proj, 1, 0, 0.0); + attn.setProjectionWeight(proj, 1, 1, 1.0); + } + + double attnOutput[12]; // 6 * 2 + attn.forward(attnInput, attnOutput); + + // Verify output is non-zero and plausible + bool anyNonZero = false; + for (size_t i = 0; i < 12; ++i) + { + if (fabs(attnOutput[i]) > 0.001) + { + anyNonZero = true; + break; + } + } + BOOST_TEST(anyNonZero); +} + +BOOST_AUTO_TEST_CASE(test_case_self_attention_gradients_relu_derivative) +{ + // A freshly-constructed layer has zero weights, so the ReLU-kernel + // projections Q' and K' are all zero. computeGradients runs reluDerivative + // over them, exercising the activation <= 0 branch that zeroes the + // gradient. The resulting weight gradients must therefore all be zero. + tinymind::SelfAttention1D attn; + + double input[12]; + for (size_t i = 0; i < 12; ++i) input[i] = static_cast(i) - 6.0; + double output[8]; + attn.forward(input, output); + + double outputDeltas[8]; + for (size_t i = 0; i < 8; ++i) outputDeltas[i] = 1.0; + attn.computeGradients(outputDeltas); + + for (size_t i = 0; i < attn.TotalWeights; ++i) + { + BOOST_TEST(fabs(attn.getGradient(i)) < 1e-12); + } +} + +// ============================================================ +// SelfAttention1D tests (fixed-point) +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_forward) +{ + // Verify SelfAttention1D compiles and runs with Q8.8 fixed-point + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::SelfAttention1D attn; + + // Identity projections + for (size_t proj = 0; proj < 3; ++proj) + { + attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); + attn.setProjectionWeight(proj, 0, 1, ValueType(0)); + attn.setProjectionWeight(proj, 1, 0, ValueType(0)); + attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); + } + + ValueType input[4]; + input[0] = ValueType(1, 0); + input[1] = ValueType(2, 0); + input[2] = ValueType(1, 0); + input[3] = ValueType(1, 0); + + ValueType output[4]; + attn.forward(input, output); + + // Output should be non-zero (positive input through identity + ReLU) + bool anyNonZero = false; + for (size_t i = 0; i < 4; ++i) + { + if (output[i].getValue() != 0) + { + anyNonZero = true; + break; + } + } + BOOST_TEST(anyNonZero); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_relu) +{ + // Verify ReLU works correctly with Q8.8 + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::SelfAttention1D attn; + + // W_q negates (ReLU should clip to zero) + attn.setProjectionWeight(0, 0, 0, ValueType(-1, 0)); + attn.setProjectionWeight(0, 0, 1, ValueType(0)); + attn.setProjectionWeight(0, 1, 0, ValueType(0)); + attn.setProjectionWeight(0, 1, 1, ValueType(-1, 0)); + + // W_k, W_v = identity + for (size_t proj = 1; proj < 3; ++proj) + { + attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); + attn.setProjectionWeight(proj, 0, 1, ValueType(0)); + attn.setProjectionWeight(proj, 1, 0, ValueType(0)); + attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); + } + + ValueType input[4]; + input[0] = ValueType(1, 0); + input[1] = ValueType(2, 0); + input[2] = ValueType(3, 0); + input[3] = ValueType(4, 0); + + ValueType output[4]; + attn.forward(input, output); + + // Q' = ReLU(negative) = 0, so output = 0 + ValueType zero(0); + for (size_t i = 0; i < 4; ++i) + { + BOOST_TEST(output[i].getValue() == zero.getValue()); + } +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_q16_16) +{ + // Verify with Q16.16 (the recommended mid-range format) + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + tinymind::SelfAttention1D attn; + + // Identity projections + for (size_t proj = 0; proj < 3; ++proj) + { + attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); + attn.setProjectionWeight(proj, 0, 1, ValueType(0)); + attn.setProjectionWeight(proj, 1, 0, ValueType(0)); + attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); + } + + ValueType input[8]; + input[0] = ValueType(1, 0); + input[1] = ValueType(2, 0); + input[2] = ValueType(3, 0); + input[3] = ValueType(4, 0); + input[4] = ValueType(5, 0); + input[5] = ValueType(6, 0); + input[6] = ValueType(7, 0); + input[7] = ValueType(8, 0); + + ValueType output[8]; + attn.forward(input, output); + + // With identity projections on positive input, output = X * (X^T * X) + // Same math as the floating-point test: + // X^T * X = [[84, 100], [100, 120]] + // Out[0] = [1,2] * [[84,100],[100,120]] = [284, 340] + // Verify first output element (allow fixed-point rounding tolerance) + const typename ValueType::FullWidthValueType tolerance = 1 << (ValueType::NumberOfFractionalBits - 1); + ValueType expected284(284, 0); + BOOST_TEST(std::abs(output[0].getValue() - expected284.getValue()) <= tolerance); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_gradients) +{ + // Verify gradients compile and produce non-zero values with Q16.16 + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + tinymind::SelfAttention1D attn; + + // Identity projections + for (size_t proj = 0; proj < 3; ++proj) + { + attn.setProjectionWeight(proj, 0, 0, ValueType(1, 0)); + attn.setProjectionWeight(proj, 0, 1, ValueType(0)); + attn.setProjectionWeight(proj, 1, 0, ValueType(0)); + attn.setProjectionWeight(proj, 1, 1, ValueType(1, 0)); + } + + ValueType input[4]; + input[0] = ValueType(1, 0); + input[1] = ValueType(2, 0); + input[2] = ValueType(1, 0); + input[3] = ValueType(1, 0); + + ValueType output[4]; + attn.forward(input, output); + + // Compute gradients with non-zero deltas + ValueType deltas[4]; + deltas[0] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 1)); // 0.5 + deltas[1] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 1)); + deltas[2] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 2)); // 0.25 + deltas[3] = ValueType(0, 1 << (ValueType::NumberOfFractionalBits - 2)); + + attn.computeGradients(deltas); + + // At least some gradients should be non-zero + bool anyNonZero = false; + for (size_t i = 0; i < tinymind::SelfAttention1D::TotalWeights; ++i) + { + if (attn.getGradient(i).getValue() != 0) + { + anyNonZero = true; + break; + } + } + BOOST_TEST(anyNonZero); +} + +BOOST_AUTO_TEST_CASE(test_case_selfattention1d_fixed_point_bias) +{ + // Verify biases work with Q16.16 + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + tinymind::SelfAttention1D attn; + + // All weights zero, set biases + attn.setProjectionBias(0, 0, ValueType(1, 0)); + attn.setProjectionBias(0, 1, ValueType(1, 0)); + attn.setProjectionBias(1, 0, ValueType(1, 0)); + attn.setProjectionBias(1, 1, ValueType(1, 0)); + attn.setProjectionBias(2, 0, ValueType(2, 0)); + attn.setProjectionBias(2, 1, ValueType(3, 0)); + + ValueType input[4]; + input[0] = ValueType(0); + input[1] = ValueType(0); + input[2] = ValueType(0); + input[3] = ValueType(0); + + ValueType output[4]; + attn.forward(input, output); + + // Same math as floating-point bias test: + // Q' = K' = [1,1] per step, V = [2,3] per step + // KV = [[4,6],[4,6]], Out = [[8,12],[8,12]] + const typename ValueType::FullWidthValueType tolerance = 1 << (ValueType::NumberOfFractionalBits - 1); + ValueType expected8(8, 0); + ValueType expected12(12, 0); + BOOST_TEST(std::abs(output[0].getValue() - expected8.getValue()) <= tolerance); + BOOST_TEST(std::abs(output[1].getValue() - expected12.getValue()) <= tolerance); + BOOST_TEST(std::abs(output[2].getValue() - expected8.getValue()) <= tolerance); + BOOST_TEST(std::abs(output[3].getValue() - expected12.getValue()) <= tolerance); +} + +// ============================================================================ +// MixtureOfExperts tests (phase 1: float, top-1 argmax routing) +// ============================================================================ + +BOOST_AUTO_TEST_CASE(test_case_moe_static_sizes) +{ + typedef tinymind::MixtureOfExperts MoEType; + + static_assert(MoEType::InputSize == 8, "Wrong input size"); + static_assert(MoEType::OutputSize == 4, "Wrong output size"); + static_assert(MoEType::NumberOfExperts == 3, "Wrong expert count"); + + // Default expert is LinearExpert with matching dims + static_assert(MoEType::Expert::InputSize == 8, "Wrong expert input size"); + static_assert(MoEType::Expert::OutputSize == 4, "Wrong expert output size"); + static_assert(MoEType::Expert::TotalWeights == 32, "Wrong expert weights"); // 8 * 4 + // Router maps InputDim -> NumExperts + static_assert(MoEType::Router::OutputSize == 3, "Wrong router output size"); + static_assert(MoEType::Router::TotalWeights == 24, "Wrong router weights"); // 8 * 3 + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_linear_expert_forward) +{ + // output[o] = bias[o] + sum_i W(o,i) * input[i] + tinymind::LinearExpert expert; + + // W = [[1, 2, 3], [4, 5, 6]], bias = [10, 20] + expert.setWeightAt(0, 0, 1.0); expert.setWeightAt(0, 1, 2.0); expert.setWeightAt(0, 2, 3.0); + expert.setWeightAt(1, 0, 4.0); expert.setWeightAt(1, 1, 5.0); expert.setWeightAt(1, 2, 6.0); + expert.setBias(0, 10.0); + expert.setBias(1, 20.0); + + double input[3] = {1.0, 1.0, 1.0}; + double output[2]; + expert.forward(input, output); + + // out[0] = 10 + (1+2+3) = 16 ; out[1] = 20 + (4+5+6) = 35 + BOOST_TEST(fabs(output[0] - 16.0) < 1e-9); + BOOST_TEST(fabs(output[1] - 35.0) < 1e-9); + + // Flat accessor agrees with structured: W(1,2) at index 1*3 + 2 = 5 + BOOST_TEST(fabs(expert.getWeight(5) - 6.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_routes_to_argmax_expert) +{ + // Router picks the expert with the highest logit; that expert's output + // is what forward() returns. + tinymind::MixtureOfExperts moe; + + // Router: drive logit of expert 1 highest for this input. + // logit[e] = sum_i W(e,i) * input[i] (biases default 0) + moe.router().setWeightAt(0, 0, 0.0); moe.router().setWeightAt(0, 1, 0.0); // expert 0 -> 0 + moe.router().setWeightAt(1, 0, 1.0); moe.router().setWeightAt(1, 1, 1.0); // expert 1 -> 2 + moe.router().setWeightAt(2, 0, 0.5); moe.router().setWeightAt(2, 1, 0.0); // expert 2 -> 0.5 + + // Give each expert a distinct constant output via bias so we can identify it. + moe.expert(0).setBias(0, 100.0); + moe.expert(1).setBias(0, 200.0); + moe.expert(2).setBias(0, 300.0); + + double input[2] = {1.0, 1.0}; + double output[1]; + moe.forward(input, output); + + // logits = [0, 2, 0.5] -> argmax = expert 1 -> output 200 + BOOST_TEST(moe.getLastSelectedExpert() == 1u); + BOOST_TEST(fabs(output[0] - 200.0) < 1e-9); + + // route() agrees and reports the logits + double logits[3]; + const size_t sel = moe.route(input, logits); + BOOST_TEST(sel == 1u); + BOOST_TEST(fabs(logits[1] - 2.0) < 1e-9); + BOOST_TEST(fabs(logits[2] - 0.5) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_selection_changes_with_input) +{ + // Different inputs route to different experts. + tinymind::MixtureOfExperts moe; + + // expert 0 wins when input[0] dominates; expert 1 wins when input[1] does. + moe.router().setWeightAt(0, 0, 1.0); moe.router().setWeightAt(0, 1, 0.0); + moe.router().setWeightAt(1, 0, 0.0); moe.router().setWeightAt(1, 1, 1.0); + + moe.expert(0).setBias(0, -1.0); + moe.expert(1).setBias(0, 1.0); + + double output[1]; + + double inA[2] = {5.0, 1.0}; + moe.forward(inA, output); + BOOST_TEST(moe.getLastSelectedExpert() == 0u); + BOOST_TEST(fabs(output[0] - (-1.0)) < 1e-9); + + double inB[2] = {1.0, 5.0}; + moe.forward(inB, output); + BOOST_TEST(moe.getLastSelectedExpert() == 1u); + BOOST_TEST(fabs(output[0] - 1.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_argmax_ties_lowest_index) +{ + // Equal logits resolve to the lowest expert index. + tinymind::MixtureOfExperts moe; + + // All router weights zero -> all logits 0 (tie) + double input[1] = {1.0}; + double output[1]; + moe.expert(0).setBias(0, 7.0); + moe.forward(input, output); + + BOOST_TEST(moe.getLastSelectedExpert() == 0u); + BOOST_TEST(fabs(output[0] - 7.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_fixed_point) +{ + // Top-1 routing works with Q-format value types. + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> Q16; + tinymind::MixtureOfExperts moe; + + moe.router().setWeightAt(0, 0, Q16(0, 0)); moe.router().setWeightAt(0, 1, Q16(0, 0)); + moe.router().setWeightAt(1, 0, Q16(1, 0)); moe.router().setWeightAt(1, 1, Q16(0, 0)); + + moe.expert(0).setBias(0, Q16(3, 0)); + moe.expert(1).setBias(0, Q16(9, 0)); + + Q16 input[2] = {Q16(1, 0), Q16(0, 0)}; + Q16 output[1]; + moe.forward(input, output); + + // logits = [0, 1] -> expert 1 -> output 9 + BOOST_TEST(moe.getLastSelectedExpert() == 1u); + BOOST_TEST(output[0].getValue() == Q16(9, 0).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_initialize_and_run) +{ + // Random init then a forward pass: selection is in range, output finite. + tinymind::MixtureOfExperts moe; + moe.initializeWeights>(); + + double input[4] = {0.5, -0.5, 0.25, -0.25}; + double output[2]; + moe.forward(input, output); + + BOOST_TEST(moe.getLastSelectedExpert() < 3u); + BOOST_TEST(std::isfinite(output[0])); + BOOST_TEST(std::isfinite(output[1])); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_custom_expert_type) +{ + // Plug a SelfAttention-like expert? Use LinearExpert explicitly as the + // template-template argument to confirm the pluggable path compiles and + // routes identically to the default. + tinymind::MixtureOfExperts moe; + + moe.router().setWeightAt(0, 0, 0.0); moe.router().setWeightAt(0, 1, 0.0); moe.router().setWeightAt(0, 2, 0.0); + moe.router().setWeightAt(1, 0, 1.0); moe.router().setWeightAt(1, 1, 1.0); moe.router().setWeightAt(1, 2, 1.0); + + // expert 1 = identity + for (size_t r = 0; r < 3; ++r) + { + moe.expert(1).setWeightAt(r, r, 1.0); + } + + double input[3] = {2.0, 3.0, 4.0}; + double output[3]; + moe.forward(input, output); + + // logits = [0, 9] -> expert 1 (identity) -> output == input + BOOST_TEST(moe.getLastSelectedExpert() == 1u); + BOOST_TEST(fabs(output[0] - 2.0) < 1e-9); + BOOST_TEST(fabs(output[1] - 3.0) < 1e-9); + BOOST_TEST(fabs(output[2] - 4.0) < 1e-9); +} + +// ============================================================================ +// TopKMixtureOfExperts tests (phase 5: float, softmax-gated blend) +// ============================================================================ + +BOOST_AUTO_TEST_CASE(test_case_moe_topk_static_sizes) +{ + typedef tinymind::TopKMixtureOfExperts MoEType; + static_assert(MoEType::InputSize == 6, "Wrong input size"); + static_assert(MoEType::OutputSize == 3, "Wrong output size"); + static_assert(MoEType::NumberOfExperts == 4, "Wrong expert count"); + static_assert(MoEType::TopK == 2, "Wrong K"); + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_topk_k1_equals_top1) +{ + // K=1 collapses to top-1: gate weight 1.0, output == argmax expert. + tinymind::TopKMixtureOfExperts moe; + + moe.router().setWeightAt(0, 0, 0.0); moe.router().setWeightAt(0, 1, 0.0); + moe.router().setWeightAt(1, 0, 1.0); moe.router().setWeightAt(1, 1, 1.0); + moe.router().setWeightAt(2, 0, 0.5); moe.router().setWeightAt(2, 1, 0.0); + + moe.expert(0).setBias(0, 100.0); + moe.expert(1).setBias(0, 200.0); + moe.expert(2).setBias(0, 300.0); + + double input[2] = {1.0, 1.0}; + double output[1]; + moe.forward(input, output); + + // logits {0,2,0.5} -> expert 1, gate 1.0 + BOOST_TEST(moe.getSelectedExpert(0) == 1u); + BOOST_TEST(fabs(moe.getGate(0) - 1.0) < 1e-12); + BOOST_TEST(fabs(output[0] - 200.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_topk_dense_average) +{ + // Dense MoE (K==NumExperts), tied logits -> equal 0.5/0.5 gates -> + // output is the average of the two expert outputs. + tinymind::TopKMixtureOfExperts moe; + + // Router weights zero -> logits {0,0} tie. + moe.expert(0).setBias(0, 2.0); + moe.expert(1).setBias(0, 4.0); + + double input[1] = {1.0}; + double output[1]; + moe.forward(input, output); + + BOOST_TEST(fabs(moe.getGate(0) - 0.5) < 1e-9); + BOOST_TEST(fabs(moe.getGate(1) - 0.5) < 1e-9); + BOOST_TEST(fabs(output[0] - 3.0) < 1e-9); // 0.5*2 + 0.5*4 +} + +BOOST_AUTO_TEST_CASE(test_case_moe_topk_top2_of_3_blend) +{ + // Top-2 of 3: the two highest logits are softmax-blended, the lowest + // expert is dropped entirely. + tinymind::TopKMixtureOfExperts moe; + + // Logits = weight * input(=1): expert0=0, expert1=2, expert2=1. + moe.router().setWeightAt(0, 0, 0.0); + moe.router().setWeightAt(1, 0, 2.0); + moe.router().setWeightAt(2, 0, 1.0); + + moe.expert(0).setBias(0, -999.0); // must be excluded + moe.expert(1).setBias(0, 10.0); + moe.expert(2).setBias(0, 20.0); + + double input[1] = {1.0}; + double output[1]; + moe.forward(input, output); + + // Selected: expert 1 (logit 2) then expert 2 (logit 1). + BOOST_TEST(moe.getSelectedExpert(0) == 1u); + BOOST_TEST(moe.getSelectedExpert(1) == 2u); + + // softmax over {2,1}: w1 = e^2/(e^2+e^1), w2 = e^1/(e^2+e^1). + const double e2 = exp(2.0), e1 = exp(1.0); + const double w1 = e2 / (e2 + e1), w2 = e1 / (e2 + e1); + const double expected = w1 * 10.0 + w2 * 20.0; + BOOST_TEST(fabs(output[0] - expected) < 1e-9); + // Excluded expert 0 never contributes. + BOOST_TEST(output[0] > 0.0); +} + +BOOST_AUTO_TEST_CASE(test_case_moe_topk_gates_sum_to_one) +{ + // Random init: the K gates always form a convex combination. + tinymind::TopKMixtureOfExperts moe; + moe.initializeWeights>(); + + double input[4] = {0.3, -0.7, 0.5, 0.1}; + double output[2]; + moe.forward(input, output); + + double gsum = 0.0; + for (size_t j = 0; j < 3; ++j) + { + BOOST_TEST(moe.getGate(j) >= 0.0); + gsum += moe.getGate(j); + } + BOOST_TEST(fabs(gsum - 1.0) < 1e-9); +} + +// ============================================================================ +// FFT1D tests +// ============================================================================ + +BOOST_AUTO_TEST_CASE(test_case_fft1d_static_sizes) +{ + typedef tinymind::FFT1D FFTType; + + static_assert(FFTType::Length == 8, "Wrong FFT length"); + static_assert(FFTType::HalfLength == 4, "Wrong half length"); + static_assert(FFTType::NumStages == 3, "Wrong number of stages"); + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_float_dc_signal) +{ + // A constant (DC) signal should produce energy only in bin 0 + const size_t N = 8; + tinymind::FFT1D fft; + + // Compute twiddle factors + double cosTable[N / 2]; + double sinTable[N / 2]; + for (size_t k = 0; k < N / 2; ++k) + { + cosTable[k] = std::cos(-2.0 * M_PI * k / N); + sinTable[k] = std::sin(-2.0 * M_PI * k / N); + } + fft.setTwiddleFactors(cosTable, sinTable); + + double real[N] = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; + double imag[N] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + + fft.forward(real, imag); + + // Scaled FFT divides by N, so DC bin = sum/N = 8/8 = 1.0 + BOOST_TEST(fabs(real[0] - 1.0) < 0.001); + BOOST_TEST(fabs(imag[0]) < 0.001); + + // All other bins should be zero + for (size_t i = 1; i < N; ++i) + { + BOOST_TEST(fabs(real[i]) < 0.001); + BOOST_TEST(fabs(imag[i]) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_float_impulse) +{ + // Impulse at index 0: FFT should produce constant magnitude across all bins + const size_t N = 8; + tinymind::FFT1D fft; + + double cosTable[N / 2]; + double sinTable[N / 2]; + for (size_t k = 0; k < N / 2; ++k) + { + cosTable[k] = std::cos(-2.0 * M_PI * k / N); + sinTable[k] = std::sin(-2.0 * M_PI * k / N); + } + fft.setTwiddleFactors(cosTable, sinTable); + + double real[N] = {1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + double imag[N] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + + fft.forward(real, imag); + + // Scaled by 1/N, so each bin should have magnitude 1/N = 0.125 + double magSq[N]; + tinymind::FFT1D::magnitudeSquared(real, imag, magSq); + + const double expectedMagSq = (1.0 / N) * (1.0 / N); // 0.015625 + for (size_t i = 0; i < N; ++i) + { + BOOST_TEST(fabs(magSq[i] - expectedMagSq) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_float_nyquist) +{ + // Alternating +1/-1 signal: energy at Nyquist frequency (bin N/2) + const size_t N = 8; + tinymind::FFT1D fft; + + double cosTable[N / 2]; + double sinTable[N / 2]; + for (size_t k = 0; k < N / 2; ++k) + { + cosTable[k] = std::cos(-2.0 * M_PI * k / N); + sinTable[k] = std::sin(-2.0 * M_PI * k / N); + } + fft.setTwiddleFactors(cosTable, sinTable); + + double real[N] = {1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0}; + double imag[N] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + + fft.forward(real, imag); + + // Scaled FFT: Nyquist bin (N/2=4) should have magnitude 1.0 + BOOST_TEST(fabs(real[N / 2] - 1.0) < 0.001); + BOOST_TEST(fabs(imag[N / 2]) < 0.001); + + // All other bins should be zero + for (size_t i = 0; i < N; ++i) + { + if (i != N / 2) + { + BOOST_TEST(fabs(real[i]) < 0.001); + BOOST_TEST(fabs(imag[i]) < 0.001); + } + } +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_float_roundtrip) +{ + // Forward then inverse should recover the original signal + const size_t N = 8; + tinymind::FFT1D fft; + + double cosTable[N / 2]; + double sinTable[N / 2]; + for (size_t k = 0; k < N / 2; ++k) + { + cosTable[k] = std::cos(-2.0 * M_PI * k / N); + sinTable[k] = std::sin(-2.0 * M_PI * k / N); + } + fft.setTwiddleFactors(cosTable, sinTable); + + double original[N] = {0.1, 0.5, -0.3, 0.8, -0.2, 0.6, 0.0, -0.4}; + double real[N]; + double imag[N]; + for (size_t i = 0; i < N; ++i) + { + real[i] = original[i]; + imag[i] = 0.0; + } + + fft.forward(real, imag); + fft.inverse(real, imag); + + // Should recover original (within floating-point tolerance) + for (size_t i = 0; i < N; ++i) + { + BOOST_TEST(fabs(real[i] - original[i]) < 0.001); + BOOST_TEST(fabs(imag[i]) < 0.001); + } +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_float_single_frequency) +{ + // Pure cosine at bin 1 frequency + const size_t N = 16; + tinymind::FFT1D fft; + + double cosTable[N / 2]; + double sinTable[N / 2]; + for (size_t k = 0; k < N / 2; ++k) + { + cosTable[k] = std::cos(-2.0 * M_PI * k / N); + sinTable[k] = std::sin(-2.0 * M_PI * k / N); + } + fft.setTwiddleFactors(cosTable, sinTable); + + double real[N]; + double imag[N]; + for (size_t i = 0; i < N; ++i) + { + real[i] = std::cos(2.0 * M_PI * i / N); // frequency bin 1 + imag[i] = 0.0; + } + + fft.forward(real, imag); + + // Scaled FFT: bins 1 and N-1 should have magnitude 0.5 + // (cosine splits into positive and negative frequency) + BOOST_TEST(fabs(real[1] - 0.5) < 0.001); + BOOST_TEST(fabs(imag[1]) < 0.001); + BOOST_TEST(fabs(real[N - 1] - 0.5) < 0.001); + BOOST_TEST(fabs(imag[N - 1]) < 0.001); + + // All other bins should be near zero + for (size_t i = 0; i < N; ++i) + { + if (i != 1 && i != N - 1) + { + BOOST_TEST(fabs(real[i]) < 0.001); + BOOST_TEST(fabs(imag[i]) < 0.001); + } + } +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_float_magnitude_squared) +{ + // Verify magnitudeSquared computes real^2 + imag^2 + const size_t N = 4; + double real[N] = {3.0, 0.0, -1.0, 0.0}; + double imag[N] = {4.0, 0.0, 2.0, 0.0}; + double magSq[N]; + + tinymind::FFT1D::magnitudeSquared(real, imag, magSq); + + BOOST_TEST(fabs(magSq[0] - 25.0) < 0.001); // 9 + 16 + BOOST_TEST(fabs(magSq[1] - 0.0) < 0.001); + BOOST_TEST(fabs(magSq[2] - 5.0) < 0.001); // 1 + 4 + BOOST_TEST(fabs(magSq[3] - 0.0) < 0.001); +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_fixed_point_dc_signal) +{ + // DC signal test with Q8.8 fixed-point + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + const size_t N = 8; + tinymind::FFT1D fft; + + // Pre-computed twiddle factors for N=8: cos(-2*pi*k/8) and sin(-2*pi*k/8) + // k=0: cos(0)=1, sin(0)=0 + // k=1: cos(-pi/4)=0.707, sin(-pi/4)=-0.707 + // k=2: cos(-pi/2)=0, sin(-pi/2)=-1 + // k=3: cos(-3pi/4)=-0.707, sin(-3pi/4)=-0.707 + ValueType cosTable[N / 2]; + ValueType sinTable[N / 2]; + cosTable[0] = ValueType(1, 0); + cosTable[1] = ValueType(0, 181); // ~0.707 + cosTable[2] = ValueType(0, 0); + cosTable[3] = ValueType(-1, 75); // ~-0.707 + + sinTable[0] = ValueType(0, 0); + sinTable[1] = ValueType(-1, 75); // ~-0.707 + sinTable[2] = ValueType(-1, 0); + sinTable[3] = ValueType(-1, 75); // ~-0.707 + + fft.setTwiddleFactors(cosTable, sinTable); + + ValueType real[N]; + ValueType imag[N]; + for (size_t i = 0; i < N; ++i) + { + real[i] = ValueType(1, 0); + imag[i] = ValueType(0); + } + + fft.forward(real, imag); + + // DC bin should have the dominant energy + // With scaled FFT, DC = 1.0 (sum/N = 8/8) + ValueType expected(1, 0); + const typename ValueType::FullWidthValueType tolerance = 3 << (ValueType::NumberOfFractionalBits - 3); + + BOOST_TEST(std::abs(real[0].getValue() - expected.getValue()) <= tolerance); + BOOST_TEST(std::abs(imag[0].getValue()) <= tolerance); + + // Other bins should be near zero + for (size_t i = 1; i < N; ++i) + { + BOOST_TEST(std::abs(real[i].getValue()) <= tolerance); + BOOST_TEST(std::abs(imag[i].getValue()) <= tolerance); + } +} + +BOOST_AUTO_TEST_CASE(test_case_fft1d_fixed_point_roundtrip) +{ + // Forward then inverse should approximately recover the signal + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + const size_t N = 8; + tinymind::FFT1D fft; + + ValueType cosTable[N / 2]; + ValueType sinTable[N / 2]; + cosTable[0] = ValueType(1, 0); + cosTable[1] = ValueType(0, 181); + cosTable[2] = ValueType(0, 0); + cosTable[3] = ValueType(-1, 75); + + sinTable[0] = ValueType(0, 0); + sinTable[1] = ValueType(-1, 75); + sinTable[2] = ValueType(-1, 0); + sinTable[3] = ValueType(-1, 75); + + fft.setTwiddleFactors(cosTable, sinTable); + + ValueType original[N]; + original[0] = ValueType(0, 128); // 0.5 + original[1] = ValueType(1, 0); + original[2] = ValueType(-1, 0); + original[3] = ValueType(0, 64); // 0.25 + original[4] = ValueType(0, 0); + original[5] = ValueType(0, 192); // 0.75 + original[6] = ValueType(-1, 128); // -0.5 + original[7] = ValueType(0, 32); // 0.125 + + ValueType real[N]; + ValueType imag[N]; + for (size_t i = 0; i < N; ++i) + { + real[i] = original[i]; + imag[i] = ValueType(0); + } + + fft.forward(real, imag); + fft.inverse(real, imag); + + // With Q8.8 and 3 stages, expect some rounding error + // Tolerance: ~4 LSBs of the fractional part + const typename ValueType::FullWidthValueType tolerance = 4; + for (size_t i = 0; i < N; ++i) + { + BOOST_TEST(std::abs(real[i].getValue() - original[i].getValue()) <= tolerance); + } +} + +// ============================================================ +// Conv2D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_conv2d_static_sizes) +{ + typedef tinymind::Conv2D ConvType; + static_assert(ConvType::OutputHeight == 30, "Wrong output height"); + static_assert(ConvType::OutputWidth == 30, "Wrong output width"); + static_assert(ConvType::OutputSize == 30 * 30 * 16, "Wrong output size"); + // 16 filters * (3*3*3 + 1) = 16 * 28 = 448 + static_assert(ConvType::TotalWeights == 448, "Wrong total weights"); + BOOST_TEST(true); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_forward_identity) +{ + // 3x3 input, 1 channel, 3x3 kernel, 1 filter = 1x1 output + // Kernel is all zeros except center = 1; output should equal input center. + tinymind::Conv2D conv; + + conv.setFilterWeight(0, 1, 1, 0, 1.0); // center tap + conv.setFilterBias(0, 0.0); + + double input[9] = { + 1, 2, 3, + 4, 5, 6, + 7, 8, 9 + }; + double output[1]; + conv.forward(input, output); + + BOOST_TEST(std::fabs(output[0] - 5.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_forward_sum_kernel) +{ + // 4x4 input, 1 channel, 2x2 kernel (all ones), stride 2 -> 2x2 output + tinymind::Conv2D conv; + for (size_t kh = 0; kh < 2; ++kh) + { + for (size_t kw = 0; kw < 2; ++kw) + { + conv.setFilterWeight(0, kh, kw, 0, 1.0); + } + } + conv.setFilterBias(0, 0.0); + + double input[16] = { + 1, 2, 3, 4, + 5, 6, 7, 8, + 9, 10, 11, 12, + 13, 14, 15, 16 + }; + double output[4]; + conv.forward(input, output); + + // Top-left 2x2 = 1+2+5+6 = 14; top-right = 3+4+7+8 = 22 + // Bot-left = 9+10+13+14 = 46; bot-right = 11+12+15+16 = 54 + BOOST_TEST(std::fabs(output[0] - 14.0) < 1e-9); + BOOST_TEST(std::fabs(output[1] - 22.0) < 1e-9); + BOOST_TEST(std::fabs(output[2] - 46.0) < 1e-9); + BOOST_TEST(std::fabs(output[3] - 54.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_multi_channel_and_filter) +{ + // 2x2 input, 2 channels, 2x2 kernel, 2 filters -> 1x1x2 output + tinymind::Conv2D conv; + + // Filter 0: sum over all channels and positions + for (size_t kh = 0; kh < 2; ++kh) + for (size_t kw = 0; kw < 2; ++kw) + for (size_t ci = 0; ci < 2; ++ci) + conv.setFilterWeight(0, kh, kw, ci, 1.0); + conv.setFilterBias(0, 0.0); + + // Filter 1: channel 0 positive, channel 1 negative + for (size_t kh = 0; kh < 2; ++kh) + { + for (size_t kw = 0; kw < 2; ++kw) + { + conv.setFilterWeight(1, kh, kw, 0, 1.0); + conv.setFilterWeight(1, kh, kw, 1, -1.0); + } + } + conv.setFilterBias(1, 0.0); + + // NHWC input, each pixel has [ch0, ch1] + double input[8] = { + 1, 10, 2, 20, // row 0 + 3, 30, 4, 40 // row 1 + }; + double output[2]; + conv.forward(input, output); + + // Filter 0 = sum all = (1+2+3+4) + (10+20+30+40) = 10 + 100 = 110 + BOOST_TEST(std::fabs(output[0] - 110.0) < 1e-9); + // Filter 1 = (1+2+3+4) - (10+20+30+40) = 10 - 100 = -90 + BOOST_TEST(std::fabs(output[1] - (-90.0)) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_gradient_sanity) +{ + // Single-filter, 3x3 input, 3x3 kernel -> scalar output. + // For output y = sum(w * x) + b, dy/dw = x, dy/db = 1. + // With delta=1 the weight gradient equals input and bias grad = 1. + tinymind::Conv2D conv; + for (size_t i = 0; i < conv.TotalWeights; ++i) + { + conv.setWeight(i, 0.1); + } + + double input[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + double delta[1] = {1.0}; + conv.computeGradients(delta, input); + + for (size_t kh = 0; kh < 3; ++kh) + { + for (size_t kw = 0; kw < 3; ++kw) + { + const size_t idx = kh * 3 + kw; + const double g = conv.getGradient(0 * conv.WeightsPerFilter + idx); + BOOST_TEST(std::fabs(g - input[idx]) < 1e-9); + } + } + // Bias gradient + const double biasGrad = conv.getGradient(conv.WeightsPerFilter - 1); + BOOST_TEST(std::fabs(biasGrad - 1.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_fixed_point) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::Conv2D conv; + + conv.setFilterWeight(0, 1, 1, 0, ValueType(1, 0)); + conv.setFilterBias(0, ValueType(0)); + + ValueType input[9]; + for (size_t i = 0; i < 9; ++i) + { + input[i] = ValueType(static_cast(i + 1), 0); + } + ValueType output[1]; + conv.forward(input, output); + + ValueType expected(5, 0); + BOOST_TEST(output[0].getValue() == expected.getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_gradient_fixed_point) +{ + // Q8.8 gradient sanity: dy/dw = x for each kernel position; dy/db = 1. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::Conv2D conv; + for (size_t i = 0; i < conv.TotalWeights; ++i) conv.setWeight(i, ValueType(0)); + + ValueType input[9]; + for (size_t i = 0; i < 9; ++i) + { + input[i] = ValueType(static_cast(i + 1), 0); + } + ValueType delta[1] = {ValueType(1, 0)}; + conv.computeGradients(delta, input); + + for (size_t kh = 0; kh < 3; ++kh) + { + for (size_t kw = 0; kw < 3; ++kw) + { + const size_t idx = kh * 3 + kw; + BOOST_TEST(conv.getGradient(idx).getValue() == input[idx].getValue()); + } + } + BOOST_TEST(conv.getGradient(conv.WeightsPerFilter - 1).getValue() == ValueType(1, 0).getValue()); +} + +// ============================================================ +// DepthwiseConv2D + PointwiseConv2D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_independence) +{ + // Two channels, kernel picks channel 0 only. + tinymind::DepthwiseConv2D dw; + + // Channel 0: ones; Channel 1: ones. Biases 0. + for (size_t c = 0; c < 2; ++c) + { + for (size_t kh = 0; kh < 2; ++kh) + { + for (size_t kw = 0; kw < 2; ++kw) + { + dw.setChannelWeight(c, kh, kw, 1.0); + } + } + dw.setChannelBias(c, 0.0); + } + + double input[8] = { + 1, 10, 2, 20, + 3, 30, 4, 40 + }; + double output[2]; // OutputH = OutputW = 1, Channels = 2 + dw.forward(input, output); + + BOOST_TEST(std::fabs(output[0] - 10.0) < 1e-9); // ch0 sum + BOOST_TEST(std::fabs(output[1] - 100.0) < 1e-9); // ch1 sum — no mixing +} + +BOOST_AUTO_TEST_CASE(test_case_pointwiseconv2d_channel_mix) +{ + // 1x1 spatial, 3 in-channels, 2 filters. + tinymind::PointwiseConv2D pw; + + // Filter 0: [1, 2, 3] dot input + pw.setFilterWeight(0, 0, 1.0); + pw.setFilterWeight(0, 1, 2.0); + pw.setFilterWeight(0, 2, 3.0); + pw.setFilterBias(0, 0.0); + + // Filter 1: mean with bias + pw.setFilterWeight(1, 0, 1.0); + pw.setFilterWeight(1, 1, 1.0); + pw.setFilterWeight(1, 2, 1.0); + pw.setFilterBias(1, 0.5); + + double input[3] = {4.0, 5.0, 6.0}; + double output[2]; + pw.forward(input, output); + + BOOST_TEST(std::fabs(output[0] - (1*4 + 2*5 + 3*6)) < 1e-9); // 32 + BOOST_TEST(std::fabs(output[1] - (4 + 5 + 6 + 0.5)) < 1e-9); // 15.5 +} + +BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_gradient_sanity) +{ + // For y = sum(w * x) + b within a channel, dy/dw = x and dy/db = 1. + // With output delta = 1.0 we expect mGradients to equal the input + // patch values in each channel and 1.0 for each bias. + tinymind::DepthwiseConv2D dw; + for (size_t c = 0; c < 2; ++c) + { + for (size_t kh = 0; kh < 2; ++kh) + for (size_t kw = 0; kw < 2; ++kw) + dw.setChannelWeight(c, kh, kw, 0.0); + dw.setChannelBias(c, 0.0); + } + double input[8] = { + 1, 10, 2, 20, + 3, 30, 4, 40 + }; + double output[2]; + dw.forward(input, output); + double outputDeltas[2] = {1.0, 1.0}; + dw.computeGradients(outputDeltas, input); + + // Channel 0 weight gradients should equal channel-0 inputs (1,2,3,4). + BOOST_TEST(std::fabs(dw.getChannelWeight(0, 0, 0)) < 1e-9); // unchanged before update + BOOST_TEST(std::fabs(dw.getGradient(0) - 1.0) < 1e-9); // ch0 (kh=0,kw=0) + BOOST_TEST(std::fabs(dw.getGradient(1) - 2.0) < 1e-9); + BOOST_TEST(std::fabs(dw.getGradient(2) - 3.0) < 1e-9); + BOOST_TEST(std::fabs(dw.getGradient(3) - 4.0) < 1e-9); + BOOST_TEST(std::fabs(dw.getGradient(4) - 1.0) < 1e-9); // bias ch0 + // Channel 1 weight gradients should equal channel-1 inputs (10,20,30,40). + BOOST_TEST(std::fabs(dw.getGradient(5) - 10.0) < 1e-9); + BOOST_TEST(std::fabs(dw.getGradient(6) - 20.0) < 1e-9); + BOOST_TEST(std::fabs(dw.getGradient(7) - 30.0) < 1e-9); + BOOST_TEST(std::fabs(dw.getGradient(8) - 40.0) < 1e-9); + BOOST_TEST(std::fabs(dw.getGradient(9) - 1.0) < 1e-9); // bias ch1 + + // updateWeights moves weights by lr * grad. With lr = 0.1 and starting + // weights at zero, the new ch0 (kh=0,kw=0) weight should be 0.1 * 1.0 = 0.1. + dw.updateWeights(0.1); + BOOST_TEST(std::fabs(dw.getChannelWeight(0, 0, 0) - 0.1) < 1e-9); + BOOST_TEST(std::fabs(dw.getChannelBias(1) - 0.1) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_pointwiseconv2d_gradient_sanity) +{ + // 1x1 spatial, 3 in-channels, 2 filters. Linear: dy_f/dw_{f,ci} = x_ci. + tinymind::PointwiseConv2D pw; + for (size_t f = 0; f < 2; ++f) + { + for (size_t ci = 0; ci < 3; ++ci) pw.setFilterWeight(f, ci, 0.0); + pw.setFilterBias(f, 0.0); + } + double input[3] = {4.0, 5.0, 6.0}; + double output[2]; + pw.forward(input, output); + + // Output deltas: filter 0 -> 1.0, filter 1 -> 2.0. + double outputDeltas[2] = {1.0, 2.0}; + pw.computeGradients(outputDeltas, input); + + // Filter 0: gradients = delta * input = (4,5,6); bias grad = 1. + BOOST_TEST(std::fabs(pw.getGradient(0) - 4.0) < 1e-9); + BOOST_TEST(std::fabs(pw.getGradient(1) - 5.0) < 1e-9); + BOOST_TEST(std::fabs(pw.getGradient(2) - 6.0) < 1e-9); + BOOST_TEST(std::fabs(pw.getGradient(3) - 1.0) < 1e-9); + // Filter 1: gradients = 2 * input = (8,10,12); bias grad = 2. + BOOST_TEST(std::fabs(pw.getGradient(4) - 8.0) < 1e-9); + BOOST_TEST(std::fabs(pw.getGradient(5) - 10.0) < 1e-9); + BOOST_TEST(std::fabs(pw.getGradient(6) - 12.0) < 1e-9); + BOOST_TEST(std::fabs(pw.getGradient(7) - 2.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_gradient_fixed_point) +{ + // Same gradient invariant in Q8.8: dy/dw equals the input patch. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::DepthwiseConv2D dw; + for (size_t kh = 0; kh < 2; ++kh) + for (size_t kw = 0; kw < 2; ++kw) + dw.setChannelWeight(0, kh, kw, ValueType(0)); + dw.setChannelBias(0, ValueType(0)); + + ValueType input[4] = {ValueType(1, 0), ValueType(2, 0), ValueType(3, 0), ValueType(4, 0)}; + ValueType output[1]; + dw.forward(input, output); + ValueType outputDeltas[1] = {ValueType(1, 0)}; + dw.computeGradients(outputDeltas, input); + + BOOST_TEST(dw.getGradient(0).getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(dw.getGradient(1).getValue() == ValueType(2, 0).getValue()); + BOOST_TEST(dw.getGradient(2).getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(dw.getGradient(3).getValue() == ValueType(4, 0).getValue()); + BOOST_TEST(dw.getGradient(4).getValue() == ValueType(1, 0).getValue()); // bias +} + +BOOST_AUTO_TEST_CASE(test_case_depthwiseconv2d_updateweights_fixed_point) +{ + // After computeGradients, updateWeights must compute weights[i] += lr*grad[i] + // through Q-format multiply-add. Use Q16.16 for headroom. + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + tinymind::DepthwiseConv2D dw; + for (size_t kh = 0; kh < 2; ++kh) + for (size_t kw = 0; kw < 2; ++kw) + dw.setChannelWeight(0, kh, kw, ValueType(0)); + dw.setChannelBias(0, ValueType(0)); + + ValueType input[4] = {ValueType(2, 0), ValueType(4, 0), ValueType(6, 0), ValueType(8, 0)}; + ValueType output[1]; + dw.forward(input, output); + ValueType outputDeltas[1] = {ValueType(1, 0)}; + dw.computeGradients(outputDeltas, input); + + // lr = 0.5, gradients = (2, 4, 6, 8); expected new weights = (1, 2, 3, 4); bias = 0.5. + const ValueType lr(0, 1u << 15); // 0.5 in Q16.16 + dw.updateWeights(lr); + + BOOST_TEST(dw.getChannelWeight(0, 0, 0).getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(dw.getChannelWeight(0, 0, 1).getValue() == ValueType(2, 0).getValue()); + BOOST_TEST(dw.getChannelWeight(0, 1, 0).getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(dw.getChannelWeight(0, 1, 1).getValue() == ValueType(4, 0).getValue()); + BOOST_TEST(dw.getChannelBias(0).getValue() == ValueType(0, 1u << 15).getValue()); // 0.5 +} + +BOOST_AUTO_TEST_CASE(test_case_pointwiseconv2d_updateweights_fixed_point) +{ + typedef tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy> ValueType; + tinymind::PointwiseConv2D pw; + for (size_t ci = 0; ci < 3; ++ci) pw.setFilterWeight(0, ci, ValueType(0)); + pw.setFilterBias(0, ValueType(0)); + + ValueType input[3] = {ValueType(2, 0), ValueType(4, 0), ValueType(6, 0)}; + ValueType output[1]; + pw.forward(input, output); + ValueType outputDeltas[1] = {ValueType(1, 0)}; + pw.computeGradients(outputDeltas, input); + + const ValueType lr(0, 1u << 15); // 0.5 + pw.updateWeights(lr); + // Filter 0 weight gradients = input = (2, 4, 6); * 0.5 = (1, 2, 3). + BOOST_TEST(pw.getFilterWeight(0, 0).getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(pw.getFilterWeight(0, 1).getValue() == ValueType(2, 0).getValue()); + BOOST_TEST(pw.getFilterWeight(0, 2).getValue() == ValueType(3, 0).getValue()); + BOOST_TEST(pw.getFilterBias(0).getValue() == ValueType(0, 1u << 15).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_separable_pipeline) +{ + // Verify a depthwise-then-pointwise pipeline produces the same + // result as manual per-channel + mix math. + tinymind::DepthwiseConv2D dw; + tinymind::PointwiseConv2D pw; + + for (size_t c = 0; c < 2; ++c) + { + for (size_t kh = 0; kh < 3; ++kh) + { + for (size_t kw = 0; kw < 3; ++kw) + { + dw.setChannelWeight(c, kh, kw, 1.0); + } + } + dw.setChannelBias(c, 0.0); + } + pw.setFilterWeight(0, 0, 0.5); + pw.setFilterWeight(0, 1, 2.0); + pw.setFilterBias(0, 0.0); + + double input[18]; + for (size_t i = 0; i < 18; ++i) + { + input[i] = (i % 2 == 0) ? 1.0 : 2.0; + } + + double dwOut[2]; + dw.forward(input, dwOut); + // dwOut[0] = sum of 9 ch0 values = 9.0; dwOut[1] = 18.0 + BOOST_TEST(std::fabs(dwOut[0] - 9.0) < 1e-9); + BOOST_TEST(std::fabs(dwOut[1] - 18.0) < 1e-9); + + double pwOut[1]; + pw.forward(dwOut, pwOut); + // 0.5 * 9 + 2.0 * 18 = 4.5 + 36 = 40.5 + BOOST_TEST(std::fabs(pwOut[0] - 40.5) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_asymmetric_kernel) +{ + // 3x1 kernel over a 4x3 input (vertical sum). Output shape (4-3+1) x (3-1+1) = 2x3. + tinymind::Conv2D conv; + for (size_t kh = 0; kh < 3; ++kh) + { + conv.setFilterWeight(0, kh, 0, 0, 1.0); + } + conv.setFilterBias(0, 0.0); + + double input[12] = { + 1, 2, 3, + 4, 5, 6, + 7, 8, 9, + 10, 11, 12 + }; + double output[6]; + conv.forward(input, output); + + // Row 0 of output: column sums of input rows 0..2 = (1+4+7, 2+5+8, 3+6+9) = (12, 15, 18) + // Row 1 of output: column sums of input rows 1..3 = (4+7+10, 5+8+11, 6+9+12) = (21, 24, 27) + BOOST_TEST(std::fabs(output[0] - 12.0) < 1e-9); + BOOST_TEST(std::fabs(output[1] - 15.0) < 1e-9); + BOOST_TEST(std::fabs(output[2] - 18.0) < 1e-9); + BOOST_TEST(std::fabs(output[3] - 21.0) < 1e-9); + BOOST_TEST(std::fabs(output[4] - 24.0) < 1e-9); + BOOST_TEST(std::fabs(output[5] - 27.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_conv2d_stride_three) +{ + // 7x7 input, 2x2 kernel of ones, stride 3x3. Output: floor((7-2)/3)+1 = 2 each axis. + tinymind::Conv2D conv; + static_assert(decltype(conv)::OutputHeight == 2, "Wrong output height for stride 3"); + static_assert(decltype(conv)::OutputWidth == 2, "Wrong output width for stride 3"); + for (size_t kh = 0; kh < 2; ++kh) + { + for (size_t kw = 0; kw < 2; ++kw) + { + conv.setFilterWeight(0, kh, kw, 0, 1.0); + } + } + conv.setFilterBias(0, 0.0); + + double input[49]; + for (size_t i = 0; i < 49; ++i) + { + input[i] = static_cast(i + 1); + } + double output[4]; + conv.forward(input, output); + + // Window (row, col) sums a 2x2 patch starting at (3*row, 3*col) of a 1-indexed + // sequence laid out row-major. Patch at (0,0): rows {0,1}, cols {0,1} -> + // values {1,2,8,9} sum to 20. Patch at (0,1): cols {3,4} -> {4,5,11,12} = 32. + // Patch at (1,0): rows {3,4} cols {0,1} -> {22,23,29,30} = 104. + // Patch at (1,1): rows {3,4} cols {3,4} -> {25,26,32,33} = 116. + BOOST_TEST(std::fabs(output[0] - 20.0) < 1e-9); + BOOST_TEST(std::fabs(output[1] - 32.0) < 1e-9); + BOOST_TEST(std::fabs(output[2] - 104.0) < 1e-9); + BOOST_TEST(std::fabs(output[3] - 116.0) < 1e-9); +} + +// ============================================================ +// Pool2D tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_maxpool2d_forward) +{ + tinymind::MaxPool2D pool; + double input[16] = { + 1, 3, 2, 4, + 5, 7, 6, 8, + 9, 11, 10, 12, + 13, 15, 14, 16 + }; + double output[4]; + pool.forward(input, output); + + // Each 2x2 block max: 7, 8, 15, 16 + BOOST_TEST(std::fabs(output[0] - 7.0) < 1e-9); + BOOST_TEST(std::fabs(output[1] - 8.0) < 1e-9); + BOOST_TEST(std::fabs(output[2] - 15.0) < 1e-9); + BOOST_TEST(std::fabs(output[3] - 16.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool2d_backward) +{ + tinymind::MaxPool2D pool; + double input[16] = { + 1, 3, 2, 4, + 5, 7, 6, 8, + 9, 11, 10, 12, + 13, 15, 14, 16 + }; + double output[4]; + pool.forward(input, output); + + double outputDeltas[4] = {1.0, 1.0, 1.0, 1.0}; + double inputDeltas[16]; + pool.backward(outputDeltas, inputDeltas); + + // Only argmax positions get gradient; sum should equal 4. + double sum = 0.0; + for (size_t i = 0; i < 16; ++i) sum += inputDeltas[i]; + BOOST_TEST(std::fabs(sum - 4.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool2d_forward) +{ + tinymind::AvgPool2D pool; + double input[16] = { + 1, 1, 2, 2, + 1, 1, 2, 2, + 3, 3, 4, 4, + 3, 3, 4, 4 + }; + double output[4]; + pool.forward(input, output); + + BOOST_TEST(std::fabs(output[0] - 1.0) < 1e-9); + BOOST_TEST(std::fabs(output[1] - 2.0) < 1e-9); + BOOST_TEST(std::fabs(output[2] - 3.0) < 1e-9); + BOOST_TEST(std::fabs(output[3] - 4.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_forward) +{ + // 3x3x2 input -> 2 outputs + tinymind::GlobalAvgPool2D gap; + + double input[18]; + for (size_t i = 0; i < 9; ++i) + { + input[i * 2 + 0] = 1.0; // channel 0 all ones + input[i * 2 + 1] = 3.0; // channel 1 all threes + } + + double output[2]; + gap.forward(input, output); + + BOOST_TEST(std::fabs(output[0] - 1.0) < 1e-9); + BOOST_TEST(std::fabs(output[1] - 3.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool2d_fixed_point) +{ + // 4x4 input, 2x2 window, stride 2 -> 2x2 output. Divisor must be 4.0 + // (a Q-format value), not raw=4. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::AvgPool2D pool; + + ValueType input[16]; + // Row-major NHWC, channel 0. Use values that average to whole numbers. + const int values[16] = { + 2, 2, 4, 4, + 2, 2, 4, 4, + 6, 6, 8, 8, + 6, 6, 8, 8 + }; + for (size_t i = 0; i < 16; ++i) input[i] = ValueType(values[i], 0); + + ValueType output[4]; + pool.forward(input, output); + + BOOST_TEST(output[0].getValue() == ValueType(2, 0).getValue()); + BOOST_TEST(output[1].getValue() == ValueType(4, 0).getValue()); + BOOST_TEST(output[2].getValue() == ValueType(6, 0).getValue()); + BOOST_TEST(output[3].getValue() == ValueType(8, 0).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_fixed_point) +{ + // 3x3x2 input, all ones / threes per channel; GAP must produce exactly 1.0 / 3.0. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::GlobalAvgPool2D gap; + + ValueType input[18]; + for (size_t i = 0; i < 9; ++i) + { + input[i * 2 + 0] = ValueType(1, 0); + input[i * 2 + 1] = ValueType(3, 0); + } + ValueType output[2]; + gap.forward(input, output); + + BOOST_TEST(output[0].getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(output[1].getValue() == ValueType(3, 0).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool2d_backward) +{ + // 4x4 single-channel, 2x2 window stride 2. Each output cell distributes + // its delta uniformly across the 4 input cells in its window: grad = d/4. + tinymind::AvgPool2D pool; + double outputDeltas[4] = {4.0, 8.0, 12.0, 16.0}; + double inputDeltas[16]; + pool.backward(outputDeltas, inputDeltas); + + // Top-left window (output 0): grad = 1.0 across 4 cells. + BOOST_TEST(std::fabs(inputDeltas[0] - 1.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[1] - 1.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[4] - 1.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[5] - 1.0) < 1e-9); + // Top-right window (output 1): grad = 2.0 + BOOST_TEST(std::fabs(inputDeltas[2] - 2.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[3] - 2.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[6] - 2.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[7] - 2.0) < 1e-9); + // Bot-right window (output 3): grad = 4.0 + BOOST_TEST(std::fabs(inputDeltas[10] - 4.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[15] - 4.0) < 1e-9); +} + +BOOST_AUTO_TEST_CASE(test_case_avgpool2d_backward_fixed_point) +{ + // Same input pattern as above in Q8.8: confirms divisor() is correct on + // the backward path, not just forward. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::AvgPool2D pool; + ValueType outputDeltas[4] = {ValueType(4, 0), ValueType(8, 0), ValueType(12, 0), ValueType(16, 0)}; + ValueType inputDeltas[16]; + pool.backward(outputDeltas, inputDeltas); + + BOOST_TEST(inputDeltas[0].getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(inputDeltas[5].getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(inputDeltas[3].getValue() == ValueType(2, 0).getValue()); + BOOST_TEST(inputDeltas[10].getValue() == ValueType(4, 0).getValue()); + BOOST_TEST(inputDeltas[15].getValue() == ValueType(4, 0).getValue()); +} + +BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_backward) +{ + // GAP backward distributes each output delta uniformly across its + // spatial extent (H*W positions per channel). + tinymind::GlobalAvgPool2D gap; + double outputDeltas[2] = {9.0, 18.0}; // chosen so per-cell grad is 1.0 and 2.0 + double inputDeltas[18]; + gap.backward(outputDeltas, inputDeltas); + + for (size_t i = 0; i < 9; ++i) + { + BOOST_TEST(std::fabs(inputDeltas[i * 2 + 0] - 1.0) < 1e-9); + BOOST_TEST(std::fabs(inputDeltas[i * 2 + 1] - 2.0) < 1e-9); + } +} + +BOOST_AUTO_TEST_CASE(test_case_globalavgpool2d_backward_fixed_point) +{ + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::GlobalAvgPool2D gap; + ValueType outputDeltas[2] = {ValueType(9, 0), ValueType(18, 0)}; + ValueType inputDeltas[18]; + gap.backward(outputDeltas, inputDeltas); + + for (size_t i = 0; i < 9; ++i) + { + BOOST_TEST(inputDeltas[i * 2 + 0].getValue() == ValueType(1, 0).getValue()); + BOOST_TEST(inputDeltas[i * 2 + 1].getValue() == ValueType(2, 0).getValue()); + } +} + +BOOST_AUTO_TEST_CASE(test_case_maxpool2d_backward_fixed_point) +{ + // Argmax-routed gradients: only the position holding each max receives + // the upstream delta. Other positions stay zero. + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy> ValueType; + tinymind::MaxPool2D pool; + ValueType input[16]; + const int values[16] = { + 1, 3, 2, 4, + 5, 7, 6, 8, + 9, 11, 10, 12, + 13, 15, 14, 16 + }; + for (size_t i = 0; i < 16; ++i) input[i] = ValueType(values[i], 0); + + ValueType output[4]; + pool.forward(input, output); // populates argmax indices + + ValueType outputDeltas[4] = {ValueType(1, 0), ValueType(1, 0), ValueType(1, 0), ValueType(1, 0)}; + ValueType inputDeltas[16]; + pool.backward(outputDeltas, inputDeltas); + + typename ValueType::FullWidthValueType sum = 0; + size_t hits = 0; + for (size_t i = 0; i < 16; ++i) + { + sum += inputDeltas[i].getValue(); + if (inputDeltas[i].getValue() != 0) ++hits; + } + // Exactly four argmax positions, each receiving raw=256 (= 1.0). + BOOST_TEST(hits == 4u); + BOOST_TEST(sum == 4 * ValueType(1, 0).getValue()); +} + +// ============================================================ +// Benchmark harness tests +// ============================================================ + +BOOST_AUTO_TEST_CASE(test_case_bench_stack_watermark) +{ + uint8_t buffer[256]; + tinymind::bench::paintStack(buffer, sizeof(buffer)); + + // Untouched: high water == 0 + BOOST_TEST(tinymind::bench::stackHighWater(buffer, sizeof(buffer)) == 0u); + + // Stack grows down: simulate a 40-byte frame by zeroing the *highest* + // 40 bytes of the buffer. stackHighWater should report 40. + for (size_t i = sizeof(buffer) - 40; i < sizeof(buffer); ++i) + { + buffer[i] = 0x00; + } + // The canary pattern is intact up to byte (256 - 40 - 1). The first + // non-A5 byte is at offset 216, so used = 256 - 216 = 40. + BOOST_TEST(tinymind::bench::stackHighWater(buffer, sizeof(buffer)) == 40u); + + // Touching the bottom as well should push the watermark to full size. + buffer[0] = 0x00; + BOOST_TEST(tinymind::bench::stackHighWater(buffer, sizeof(buffer)) == sizeof(buffer)); +} + +BOOST_AUTO_TEST_CASE(test_case_bench_cycle_counter_monotonic) +{ + tinymind::bench::enableCycleCounter(); + const tinymind::bench::Cycles a = tinymind::bench::readCycleCounter(); + // Do some work the compiler cannot elide. + volatile uint64_t acc = 0; + for (uint64_t i = 0; i < 10000; ++i) acc += i; + const tinymind::bench::Cycles b = tinymind::bench::readCycleCounter(); + + // On host, a/b are ns-since-first-call; require b >= a modulo wrap. + // Cast to signed to tolerate legitimate wrap. + BOOST_TEST(static_cast(b - a) >= 0); + BOOST_TEST(acc > 0u); // keep the loop live +} + +// Exercise the EmptyLayerChain terminal accessors (recursion base case of the +// heterogeneous layer chain) across every value type / policy combination, the +// inverted-Dropout training-mode path, and two QValue operators that the +// runtime suites had not otherwise reached -- closing the per-instantiation +// function-coverage gaps that line coverage cannot reflect. +BOOST_AUTO_TEST_CASE(test_case_function_coverage_terminal_accessors_and_ops) +{ + typedef tinymind::ChainHiddenLayerAccessor A; + tinymind::EmptyLayerChain ec; + + // No-op terminal accessors: instantiate + call all four for each type. + auto hit = [&](auto sample) { + typedef decltype(sample) V; + (void)A::template getBiasNeuronWeightForConnection(ec, 0, 0); + (void)A::template getWeightForNeuronAndConnection(ec, 0, 0, 0); + A::template setBiasNeuronWeightForConnection(ec, 0, 0, V()); + A::template setWeightForNeuronAndConnection(ec, 0, 0, 0, V()); + }; + hit(double()); + hit(tinymind::QValue<8, 8, true, tinymind::TruncatePolicy, tinymind::WrapPolicy>()); + hit(tinymind::QValue<16, 16, true, tinymind::RoundUpPolicy, tinymind::WrapPolicy>()); + hit(tinymind::QValue<16, 16, true, tinymind::TruncatePolicy, tinymind::WrapPolicy>()); + hit(tinymind::QValue<8, 24, true, tinymind::RoundUpPolicy, tinymind::WrapPolicy>()); + hit(tinymind::QValue<8, 24, true, tinymind::TruncatePolicy, tinymind::WrapPolicy>()); + BOOST_TEST(true); // accessors are no-ops; reaching here means they ran + + // Inverted dropout training path: forward() in training mode runs + // generateMask() + scale() + fromInteger() for each instantiation. + { + tinymind::Dropout dd; // ctor leaves training=true + double din[4] = { 1.0, 1.0, 1.0, 1.0 }; + double dout[4] = { 0.0, 0.0, 0.0, 0.0 }; + dd.forward(din, dout); + + typedef tinymind::QValue<8, 8, true, tinymind::RoundUpPolicy, tinymind::WrapPolicy> Qv; + tinymind::Dropout dq; + Qv qin[4] = { Qv(1, 0), Qv(1, 0), Qv(1, 0), Qv(1, 0) }; + Qv qout[4] = { Qv(0, 0), Qv(0, 0), Qv(0, 0), Qv(0, 0) }; + dq.forward(qin, qout); + BOOST_TEST(true); + } + + // QValue<8.8 MinMaxSaturate> assignment-from-short and QValue<16.16> ostream. + { + tinymind::QValue<8, 8, true, tinymind::TruncatePolicy, + tinymind::MinMaxSaturatePolicy> qsat; + qsat = static_cast(3); + BOOST_TEST(qsat.getValue() != 0); + + std::ostringstream oss; + oss << tinymind::QValue<16, 16, true, tinymind::TruncatePolicy, + tinymind::WrapPolicy>(2, 0); + BOOST_TEST(!oss.str().empty()); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/unit_test/qlearn/Makefile b/unit_test/qlearn/Makefile index f99ea6e9..a13424b6 100644 --- a/unit_test/qlearn/Makefile +++ b/unit_test/qlearn/Makefile @@ -6,7 +6,7 @@ CC=g++ WARN=-Wall -Wextra -Werror -Wpedantic SOURCES=qlearn_unit_test.cpp ../../cpp/lookupTables.cpp DEFINES=-DTINYMIND_ENABLE_OSTREAMS=1 -DTINYMIND_ENABLE_FLOAT=1 -DTINYMIND_ENABLE_STD=1 -DTINYMIND_ENABLE_HOSTED_RAND=1 -INCLUDES=-I../../cpp -I../../cpp/include -I../../include -DTINYMIND_USE_TANH_16_16=1 -I${BOOST_HOME} +INCLUDES=-I../include -I../../cpp -I../../cpp/include -I../../include -DTINYMIND_USE_TANH_16_16=1 -I${BOOST_HOME} OUT=./output/qlearn_unit_test qlearn_unit_test: diff --git a/unit_test/qlearn/qlearn_unit_test.cpp b/unit_test/qlearn/qlearn_unit_test.cpp index f7aa348c..6f9f6632 100644 --- a/unit_test/qlearn/qlearn_unit_test.cpp +++ b/unit_test/qlearn/qlearn_unit_test.cpp @@ -1,981 +1,978 @@ -/** -* Copyright (c) 2020 Intel Corporation -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -/* -Q-Learning unit test. Learn the best path out of a simple maze. - -5 == Outside the maze -________________________________________________ -| | | -| | | -| 0 | 1 / 5 -| | | -|____________/ ________|__/ __________________|_______________________ -| | | | -| | / | -| 4 | 3 | 2 | -| / | | -|__/ __________________|_______________________|_______________________| - 5 - -The paths out of the maze: - -0->4->5 -0->4->3->1->5 -1->5 -1->3->4->5 -2->3->1->5 -2->3->4->5 -3->1->5 -3->4->5 -4->5 -4->3->1->5 -*/ - -#include "compiler.h" - -#define BOOST_TEST_MODULE test module name -TINYMIND_DISABLE_WARNING_PUSH -TINYMIND_DISABLE_WARNING_GCC_ONLY("-Wdangling-reference") -#include -TINYMIND_DISABLE_WARNING_POP - -#include -#include -#include -#include -#include -#include -#include - -#include "qformat.hpp" -#include "qlearn.hpp" -#include "fixedPointTransferFunctions.hpp" -#include "activationFunctions.hpp" -#include "nnproperties.hpp" - -#define NUMBER_OF_STATES 6 -#define NUMBER_OF_ACTIONS 6 -#define RANDOM_SEED 7U - -// Upper bound on the steps any one episode may take. The maze has 6 states, so -// a policy that is making progress reaches the goal in far fewer; this exists -// so that a policy which is *not* making progress fails the test instead of -// looping forever. -static const size_t MAX_STEPS_PER_EPISODE = 1000; - -typedef uint8_t state_t; -typedef uint8_t action_t; - -template -struct UniformRealRandomNumberGenerator -{ - typedef tinymind::ValueConverter WeightConverterPolicy; - - static ValueType generateRandomWeight() - { - const double temp = distribution()(generator()); - const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); - - return weight; - } - -private: - // Function-local statics, not namespace-scope ones. These are consumed by - // the constructors of the static QLearner objects below, and dynamic - // initialization of a static data member of a class template is *unordered* - // with respect to other non-local variables ([basic.start.dynamic]), so a - // namespace-scope definition may or may not have run by the time those - // constructors call generateRandomWeight(). It genuinely differed by - // compiler: g++ used the still-zero-initialized distribution, handing every - // weight 0.0, while clang used a properly constructed [-1, 1). Reading an - // object before its lifetime begins is UB either way. Function-local - // statics are initialized on first use, so both compilers now see a real - // distribution and identical weights. - static std::default_random_engine& generator() - { - static std::default_random_engine instance(RANDOM_SEED); - - return instance; - } - - static std::uniform_real_distribution& distribution() - { - static std::uniform_real_distribution instance(-1.0, 1.0); - - return instance; - } -}; - -template -struct MazeEnvironmentRandomNumberGeneratorPolicy -{ - // mRandomActionDecisionPoint must be initialized here: getRandomActionDecisionPoint() - // is reachable before any setRandomActionDecisionPoint() call, and reading an - // uninitialized member is UB that neither ASan nor UBSan diagnoses. - MazeEnvironmentRandomNumberGeneratorPolicy() : mRandomEngine(RANDOM_SEED), mRandomActionDecisionPoint(0) - { - } - - size_t getRandomActionDecisionPoint() const - { - return this->mRandomActionDecisionPoint; - } - - T randInt(const T min, const T max) - { - std::uniform_int_distribution distribution(min, max); - const T value = static_cast(distribution(this->mRandomEngine)); - - return value; - } - - void setRandomActionDecisionPoint(const size_t randomActionDecisionPoint) - { - if(randomActionDecisionPoint <= 100) - { - this->mRandomActionDecisionPoint = randomActionDecisionPoint; - } - } -private: - std::default_random_engine mRandomEngine; - size_t mRandomActionDecisionPoint; -}; - -template< typename StateType, - typename ActionType, - typename ValueType, - size_t NumberOfStates, - size_t NumberOfActions, - typename RewardPolicyType, - template class QLearningPolicy = tinymind::DefaultLearningPolicy - > -struct MazeEnvironment : public tinymind::QLearningEnvironment -{ - typedef tinymind::QLearningEnvironment ParentType; - static const size_t EnvironmentNumberOfStates = ParentType::EnvironmentNumberOfStates; - static const size_t EnvironmentNumberOfActions = ParentType::EnvironmentNumberOfActions; - static const size_t EnvironmentInvalidState = ParentType::EnvironmentInvalidState; - static const size_t EnvironmentInvalidAction = ParentType::EnvironmentInvalidAction; - static const ValueType EnvironmentNoRewardValue; - static const ValueType EnvironmentInvalidActionValue; - - MazeEnvironment(const ValueType& learningRate, const ValueType& discountFactor, const size_t randomActionDecisionPoint) : - ParentType(learningRate, discountFactor, randomActionDecisionPoint), mGoalState(EnvironmentInvalidState) - { - } - - StateType getGoalState() const - { - return this->mGoalState; - } - - size_t getNextStateForStateActionPair(const action_t action) const - { - return action; - } - - ValueType getRewardForStateAndAction(const StateType state, const ActionType action) const - { - return this->mRewardPolicy.getRewardForStateAndAction(state, action); - } - - size_t getValidActionsForState(const state_t state, action_t* pActions) const - { - size_t numberOfValidActions = 0; - - for(action_t action = 0;action < NumberOfActions;++action) - { - if(EnvironmentInvalidActionValue != this->getRewardForStateAndAction(state, action)) - { - *pActions = action; - ++pActions; - ++numberOfValidActions; - } - } - - return numberOfValidActions; - } - - bool isGoalState(const StateType state) const - { - return (state == this->mGoalState); - } - - void setGoalState(const StateType goalState) - { - if (goalState < EnvironmentNumberOfStates) - { - this->mGoalState = goalState; - } - } - - void setRewardForStateAndAction(const StateType state, const ActionType action, const ValueType& reward) - { - this->mRewardPolicy.setRewardForStateAndAction(state, action, reward); - } - - void takeAction(const typename ParentType::experience_t& experience) - { - this->mExperience = experience; - } - - typename ParentType::experience_t mExperience; - StateType mGoalState; -private: - RewardPolicyType mRewardPolicy; -}; - -template class QLearningPolicy> -const ValueType MazeEnvironment::EnvironmentNoRewardValue = ValueType(0); - -template class QLearningPolicy> -const ValueType MazeEnvironment::EnvironmentInvalidActionValue = ValueType(-1, 0); - -template< typename StateType, - typename ActionType, - typename ValueType, - size_t NumberOfStates, - size_t NumberOfActions, - typename RewardPolicyType, - template class QLearningPolicy = tinymind::DefaultLearningPolicy - > -struct DQNMazeEnvironment : public tinymind::QLearningEnvironment -{ - typedef tinymind::QLearningEnvironment ParentType; - static const size_t EnvironmentNumberOfStates = ParentType::EnvironmentNumberOfStates; - static const size_t EnvironmentNumberOfActions = ParentType::EnvironmentNumberOfActions; - static const size_t EnvironmentInvalidState = ParentType::EnvironmentInvalidState; - static const size_t EnvironmentInvalidAction = ParentType::EnvironmentInvalidAction; - static const ValueType EnvironmentNoRewardValue; - static const ValueType EnvironmentInvalidActionValue; - - DQNMazeEnvironment(const ValueType& learningRate, const ValueType& discountFactor, const size_t randomActionDecisionPoint) : - ParentType(learningRate, discountFactor, randomActionDecisionPoint), mGoalState(EnvironmentInvalidState) - { - } - - StateType getGoalState() const - { - return this->mGoalState; - } - - static void getInputValues(const StateType state, ValueType *pInputs) - { - static const ValueType MAX_STATE = ValueType(NUMBER_OF_STATES,0); - ValueType input = (ValueType(state, 0) / MAX_STATE); - - *pInputs = input; - } - - size_t getNextStateForStateActionPair(const action_t action) const - { - return action; - } - - ValueType getRewardForStateAndAction(const StateType state, const ActionType action) const - { - return this->mRewardPolicy.getRewardForStateAndAction(state, action); - } - - size_t getValidActionsForState(const state_t state, action_t* pActions) const - { - size_t numberOfValidActions = 0; - - for(action_t action = 0;action < NumberOfActions;++action) - { - if(EnvironmentInvalidActionValue != this->getRewardForStateAndAction(state, action)) - { - *pActions = action; - ++pActions; - ++numberOfValidActions; - } - } - - return numberOfValidActions; - } - - bool isGoalState(const StateType state) const - { - return (state == this->mGoalState); - } - - void setGoalState(const StateType goalState) - { - if (goalState < EnvironmentNumberOfStates) - { - this->mGoalState = goalState; - } - } - - void setRewardForStateAndAction(const StateType state, const ActionType action, const ValueType& reward) - { - this->mRewardPolicy.setRewardForStateAndAction(state, action, reward); - } - - void takeAction(const typename ParentType::experience_t& experience) - { - this->mExperience = experience; - } - - typename ParentType::experience_t mExperience; - StateType mGoalState; -private: - RewardPolicyType mRewardPolicy; -}; - -template class QLearningPolicy> -const ValueType DQNMazeEnvironment::EnvironmentNoRewardValue = ValueType(0); - -template class QLearningPolicy> -const ValueType DQNMazeEnvironment::EnvironmentInvalidActionValue = ValueType(-1, 0); - -typedef tinymind::QValue<16,16,true> QValueType; -typedef tinymind::QTableRewardPolicy QTableRewardPolicyType; -typedef MazeEnvironment MazeEnvironmentType; -typedef tinymind::QLearner QLearnerType; - -typedef tinymind::NullRewardPolicy NullRewardPolicyType; -typedef MazeEnvironment UntrainedMazeEnvironmentType; -typedef tinymind::QLearner UntrainedQLearnerType; - -#define NUMBER_OF_INPUT_LAYER_NEURONS 1 -#define NUMBER_OF_HIDDEN_LAYERS 1 -#define NUMBER_OF_HIDDEN_LAYER_NEURONS (NUMBER_OF_ACTIONS + 2) -#define NUMBER_OF_OUTPUT_LAYER_NEURONS NUMBER_OF_ACTIONS -#define NUMBER_OF_ITERATIONS_FOR_TARGET_NN_UPDATE 10 - -typedef DQNMazeEnvironment DQNMazeEnvironmentType; -typedef tinymind::FixedPointTransferFunctions< QValueType, - UniformRealRandomNumberGenerator, - tinymind::TanhActivationPolicy, - tinymind::TanhActivationPolicy, - NUMBER_OF_OUTPUT_LAYER_NEURONS> TransferFunctionsType; -typedef tinymind::MultilayerPerceptron< QValueType, - NUMBER_OF_INPUT_LAYER_NEURONS, - NUMBER_OF_HIDDEN_LAYERS, - NUMBER_OF_HIDDEN_LAYER_NEURONS, - NUMBER_OF_OUTPUT_LAYER_NEURONS, - TransferFunctionsType> NeuralNetworkType; -typedef tinymind::QValueNeuralNetworkPolicy QValuePolicyType; -typedef tinymind::QLearner DQNQLearnerType; - -static const QValueType learningRate = (QValueType(1,0) / QValueType(5,0)); -static const QValueType discountFactor = (QValueType(8,0) / QValueType(10,0)); -static QLearnerType qLearner(learningRate, discountFactor, 100); -static DQNQLearnerType dqnQLearner(learningRate, discountFactor, 100); -static const QValueType reward(100,0); -static UntrainedQLearnerType untrainedQLearner(QValueType(0,0), QValueType(0,0), 0); - -template -static void initializeRewardMatrix(QLearnerType& qLearner) -{ - state_t state; - - state = 0; - for(action_t action = 0;action < 4;++action) - { - qLearner.getEnvironment().setRewardForStateAndAction(state, action, MazeEnvironmentType::EnvironmentInvalidActionValue); - } - qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 5, MazeEnvironmentType::EnvironmentInvalidActionValue); - - state = 1; - for(action_t action = 0;action < 3;++action) - { - qLearner.getEnvironment().setRewardForStateAndAction(state, action, MazeEnvironmentType::EnvironmentInvalidActionValue); - } - qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 5, reward); - - state = 2; - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - qLearner.getEnvironment().setRewardForStateAndAction(state, action, MazeEnvironmentType::EnvironmentInvalidActionValue); - } - qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentNoRewardValue); - - state = 3; - qLearner.getEnvironment().setRewardForStateAndAction(state, 0, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 1, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 2, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 5, MazeEnvironmentType::EnvironmentInvalidActionValue); - - state = 4; - qLearner.getEnvironment().setRewardForStateAndAction(state, 0, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 1, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 2, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 5, reward); - - state = 5; - qLearner.getEnvironment().setRewardForStateAndAction(state, 0, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 1, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 2, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentInvalidActionValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentNoRewardValue); - qLearner.getEnvironment().setRewardForStateAndAction(state, 5, reward); -} - -BOOST_AUTO_TEST_SUITE(test_suite_qlearn) - -BOOST_AUTO_TEST_CASE(test_qlearn_init) -{ - for(state_t state = 0;state < NUMBER_OF_STATES;++state) - { - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - BOOST_TEST(static_cast(0) == qLearner.getQValue(state, action)); - BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getRewardForStateAndAction(state, action)); - } - } -} - -BOOST_AUTO_TEST_CASE(test_qlearn_setreward) -{ - initializeRewardMatrix(qLearner); - - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 4)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 3)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 3)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 1)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 2)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 4)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 0)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 3)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 1)); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 4)); - - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 0)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 1)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 2)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 5)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 0)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 1)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 2)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 4)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 0)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 1)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 2)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 4)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 5)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 0)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 3)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 5)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 1)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 2)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 4)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 0)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 2)); - BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 3)); - - BOOST_TEST(reward == qLearner.getEnvironment().getRewardForStateAndAction(1, 5)); - BOOST_TEST(reward == qLearner.getEnvironment().getRewardForStateAndAction(4, 5)); - BOOST_TEST(reward == qLearner.getEnvironment().getRewardForStateAndAction(5, 5)); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_text_next_state) -{ - BOOST_TEST(static_cast(4) == qLearner.getEnvironment().getNextStateForStateActionPair(4)); - BOOST_TEST(static_cast(3) == qLearner.getEnvironment().getNextStateForStateActionPair(3)); - BOOST_TEST(static_cast(5) == qLearner.getEnvironment().getNextStateForStateActionPair(5)); - BOOST_TEST(static_cast(3) == qLearner.getEnvironment().getNextStateForStateActionPair(3)); - BOOST_TEST(static_cast(1) == qLearner.getEnvironment().getNextStateForStateActionPair(1)); - BOOST_TEST(static_cast(2) == qLearner.getEnvironment().getNextStateForStateActionPair(2)); - BOOST_TEST(static_cast(4) == qLearner.getEnvironment().getNextStateForStateActionPair(4)); - BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getNextStateForStateActionPair(0)); - BOOST_TEST(static_cast(3) == qLearner.getEnvironment().getNextStateForStateActionPair(3)); - BOOST_TEST(static_cast(5) == qLearner.getEnvironment().getNextStateForStateActionPair(5)); - BOOST_TEST(static_cast(1) == qLearner.getEnvironment().getNextStateForStateActionPair(1)); - BOOST_TEST(static_cast(4) == qLearner.getEnvironment().getNextStateForStateActionPair(4)); - BOOST_TEST(static_cast(5) == qLearner.getEnvironment().getNextStateForStateActionPair(5)); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_change_learning_rate) -{ - static const QValueType newLearningRate = (QValueType(1,0) / QValueType(100,0)); - BOOST_TEST(learningRate != newLearningRate); - BOOST_TEST(learningRate == qLearner.getEnvironment().getLearningRate()); - qLearner.getEnvironment().setLearningRate(newLearningRate); - BOOST_TEST(newLearningRate == qLearner.getEnvironment().getLearningRate()); - qLearner.getEnvironment().setLearningRate(learningRate); - BOOST_TEST(learningRate == qLearner.getEnvironment().getLearningRate()); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_change_discount_factor) -{ - static const QValueType delta = (QValueType(1,0) / QValueType(100,0)); - static const QValueType discountFactor = qLearner.getEnvironment().getDiscountFactor(); - static const QValueType newDiscountFactor = (discountFactor + delta); - - BOOST_TEST(discountFactor != newDiscountFactor); - qLearner.getEnvironment().setDiscountFactor(newDiscountFactor); - BOOST_TEST(newDiscountFactor == qLearner.getEnvironment().getDiscountFactor()); - qLearner.getEnvironment().setDiscountFactor(discountFactor); - BOOST_TEST(discountFactor == qLearner.getEnvironment().getDiscountFactor()); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_choose_random_action) -{ - const size_t oldDecisionPoint = qLearner.getEnvironment().getRandomActionDecisionPoint(); - std::vector choices; - std::size_t t; - - qLearner.getEnvironment().setRandomActionDecisionPoint(50); - for(unsigned i = 0;i < 1000;++i) - { - choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); - } - - t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); - - BOOST_TEST(((t >= 400) && (t <= 600))); - - qLearner.getEnvironment().setRandomActionDecisionPoint(90); - choices.clear(); - for(unsigned i = 0;i < 1000;++i) - { - choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); - } - - t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); - - BOOST_TEST(((t >= 800) && (t <= 1000))); - - qLearner.getEnvironment().setRandomActionDecisionPoint(0); - choices.clear(); - for(unsigned i = 0;i < 1000;++i) - { - choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); - } - - t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); - - BOOST_TEST(0U == t); - - qLearner.getEnvironment().setRandomActionDecisionPoint(100); - choices.clear(); - for(unsigned i = 0;i < 1000;++i) - { - choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); - } - - t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); - - BOOST_TEST(1000U == t); - - qLearner.getEnvironment().setRandomActionDecisionPoint(oldDecisionPoint); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_goal_state) -{ - BOOST_TEST(static_cast(-1) == qLearner.getEnvironment().getGoalState()); - - qLearner.getEnvironment().setGoalState(0); - BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getGoalState()); - - qLearner.getEnvironment().setGoalState(-1); - BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getGoalState()); - - qLearner.getEnvironment().setGoalState(QLearnerType::NumberOfStates); - BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getGoalState()); - - qLearner.getEnvironment().setGoalState(NUMBER_OF_STATES - 1); - BOOST_TEST(static_cast(NUMBER_OF_STATES - 1) == qLearner.getEnvironment().getGoalState()); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_argmax_best_action) -{ - typedef tinymind::QValueTablePolicy QValuePolicyType; - QValuePolicyType qValuePolicy; - QValueType qValue(0, 0); - state_t state = 1; - action_t actions[NUMBER_OF_ACTIONS] = { 0, 1, 2, 3, 4, 5 }; - action_t bestAction; - - for (action_t action = 0; action < NUMBER_OF_ACTIONS; ++action) - { - qValuePolicy.setQValue(state, action, qValue); - ++qValue; - } - - bestAction = tinymind::ArgMaxPolicy::selectBestActionForState(state, &actions[0], NUMBER_OF_ACTIONS, qValuePolicy); - BOOST_TEST(static_cast(5) == bestAction); - - qValue = 0; - for (action_t action = NUMBER_OF_ACTIONS; action > 0; --action) - { - qValuePolicy.setQValue(state, action - 1, qValue); - ++qValue; - } - - bestAction = tinymind::ArgMaxPolicy::selectBestActionForState(state, &actions[0], NUMBER_OF_ACTIONS, qValuePolicy); - BOOST_TEST(static_cast(0) == bestAction); - - qValuePolicy.setQValue(state, 1, QValueType(100, 0)); - bestAction = tinymind::ArgMaxPolicy::selectBestActionForState(state, &actions[0], NUMBER_OF_ACTIONS - 2, qValuePolicy); - BOOST_TEST(static_cast(1) == bestAction); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_take_action) -{ - typename MazeEnvironmentType::ParentType::experience_t experience; - state_t state = 1; - action_t action = 5; - - initializeRewardMatrix(qLearner); - - experience.state = state; - experience.action = action; - qLearner.getEnvironment().takeAction(experience); - experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); - experience.newState = static_cast(experience.action); - qLearner.updateFromExperience(experience); - - BOOST_TEST(static_cast(1) == experience.state); - BOOST_TEST(static_cast(5) == experience.action); - BOOST_TEST(reward == experience.reward); - BOOST_TEST(static_cast(5) == experience.newState); - - state = 0; - action = 4; - - experience.state = state; - experience.action = action; - qLearner.getEnvironment().takeAction(experience); - experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); - experience.newState = static_cast(experience.action); - qLearner.updateFromExperience(experience); - - BOOST_TEST(static_cast(0) == experience.state); - BOOST_TEST(static_cast(4) == experience.action); - BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == experience.reward); - BOOST_TEST(static_cast(4) == experience.newState); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_iterate) -{ - typedef typename QValueType::FullWidthValueType FullWidthValueType; - static char const* const qTableFilePath = "qtable.txt"; - static char const* const qTableBinFilePath = "qtable.bin"; - std::default_random_engine engine(RANDOM_SEED);; - std::uniform_int_distribution stateDistribution(0, NUMBER_OF_STATES - 1); - size_t decisionPoint = 100; - std::ofstream qTableFile(qTableFilePath); - std::ofstream qTableBinFile(qTableBinFilePath, std::ios::out | std::ios::binary); - typename MazeEnvironmentType::ParentType::experience_t experience; - state_t state; - action_t action; - FullWidthValueType value; - - for (unsigned i = 0; i < 500; ++i) - { - qLearner.getEnvironment().setRandomActionDecisionPoint(decisionPoint); - - qLearner.startNewEpisode(); - state = static_cast(stateDistribution(engine)); - qLearner.setState(state); - qLearner.getEnvironment().setGoalState(5); - action = qLearner.takeAction(state); - experience.state = state; - experience.action = action; - experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); - experience.newState = static_cast(experience.action); - qLearner.updateFromExperience(experience); - - // Bounded: tabular Q-learning on this maze does reach the goal, so the - // cap is a guard rather than an expected exit. Hitting it is a failure, - // not a hang -- see the assertion after the loop. - size_t steps = 0; - while (qLearner.getState() != 5 && steps < MAX_STEPS_PER_EPISODE) - { - ++steps; - action = qLearner.takeAction(qLearner.getState()); - experience.state = qLearner.getState(); - experience.action = action; - experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); - experience.newState = static_cast(experience.action); - qLearner.updateFromExperience(experience); - } - - BOOST_TEST(steps < MAX_STEPS_PER_EPISODE); - - if (decisionPoint > 0) - { - decisionPoint -= 1; - } - } - - // store q table in text form - for(state_t state = 0;state < NUMBER_OF_STATES;++state) - { - qTableFile << state << ","; - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - qTableFile << qLearner.getQValue(state, action).getValue(); - if (action != (NUMBER_OF_ACTIONS - 1)) - { - qTableFile << ","; - } - } - - qTableFile << std::endl; - } - - qTableFile.close(); - - // store q table in binary form - for(state_t state = 0;state < NUMBER_OF_STATES;++state) - { - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - value = qLearner.getQValue(state, action).getValue(); - qTableBinFile.write(reinterpret_cast(&value), sizeof(FullWidthValueType)); - } - } - - qTableBinFile.close(); -} - -BOOST_AUTO_TEST_CASE(test_dqn_qlearn_iterate) -{ - std::default_random_engine engine(RANDOM_SEED);; - std::uniform_int_distribution stateDistribution(0, NUMBER_OF_STATES - 1); - size_t decisionPoint = 100; - typename DQNMazeEnvironmentType::ParentType::experience_t experience; - state_t state; - action_t action; - - initializeRewardMatrix(dqnQLearner); - - dqnQLearner.getEnvironment().setGoalState(5); - - for (unsigned i = 0; i < 500; ++i) - { - dqnQLearner.getEnvironment().setRandomActionDecisionPoint(decisionPoint); - - dqnQLearner.startNewEpisode(); - state = static_cast(stateDistribution(engine)); - dqnQLearner.setState(state); - action = dqnQLearner.takeAction(state); - experience.state = state; - experience.action = action; - experience.reward = dqnQLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); - experience.newState = static_cast(experience.action); - dqnQLearner.updateFromExperience(experience); - - // Bounded episode. This case asserts nothing about the policy -- it - // exercises the DQN iterate/update machinery -- so an episode that does - // not reach the goal is simply cut short rather than spinning forever. - // The greedy policy is under no obligation to reach the goal, and with - // an unbounded loop it does not: the loop only ever terminated because - // the weights were accidentally all zero (see the static-initialization - // note on UniformRealRandomNumberGenerator). - for (size_t step = 0; - step < MAX_STEPS_PER_EPISODE && - dqnQLearner.getState() != dqnQLearner.getEnvironment().getGoalState(); - ++step) - { - action = dqnQLearner.takeAction(dqnQLearner.getState()); - experience.state = dqnQLearner.getState(); - experience.action = action; - experience.reward = dqnQLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); - experience.newState = static_cast(experience.action); - dqnQLearner.updateFromExperience(experience); - } - - if (decisionPoint > 0) - { - decisionPoint -= 1; - } - } -} - -BOOST_AUTO_TEST_CASE(test_untrained_qlearner_reward) -{ - // Reward table should not exist - for(state_t state = 0;state < NUMBER_OF_STATES;++state) - { - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - BOOST_TEST(static_cast(0) == untrainedQLearner.getEnvironment().getRewardForStateAndAction(state, action)); - } - } - // Attempt to set reward - for(state_t state = 0;state < NUMBER_OF_STATES;++state) - { - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - untrainedQLearner.getEnvironment().setRewardForStateAndAction(state, action, reward); - } - } - - // Setting reward should have no effect - for(state_t state = 0;state < NUMBER_OF_STATES;++state) - { - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - BOOST_TEST(static_cast(0) == untrainedQLearner.getEnvironment().getRewardForStateAndAction(state, action)); - } - } -} - -//Load the Q table from the trained Q learner and use it. -BOOST_AUTO_TEST_CASE(test_untrained_qlearner_iterate) -{ - static const state_t goalState = 5; - std::default_random_engine engine(RANDOM_SEED);; - std::uniform_int_distribution stateDistribution(0, NUMBER_OF_STATES - 1); - size_t iterations = 0; - typename UntrainedMazeEnvironmentType::ParentType::experience_t experience; - state_t state; - action_t action; - - for(state = 0;state < NUMBER_OF_STATES;++state) - { - for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) - { - untrainedQLearner.setQValue(state, action, qLearner.getQValue(state, action)); - } - } - - do - { - untrainedQLearner.startNewEpisode(); - state = static_cast(stateDistribution(engine)); - untrainedQLearner.setState(state); - untrainedQLearner.getEnvironment().setGoalState(goalState); - // Bounded: this learner is seeded with the trained Q table, so it does - // reach the goal. The cap turns a future regression into a failure - // rather than a hang. - size_t steps = 0; - do - { - ++steps; - action = untrainedQLearner.takeAction(untrainedQLearner.getState()); - experience.state = state; - experience.action = action; - experience.newState = static_cast(experience.action); - untrainedQLearner.setState(experience.newState); - } while (untrainedQLearner.getState() != goalState && steps < MAX_STEPS_PER_EPISODE); - - BOOST_TEST(steps < MAX_STEPS_PER_EPISODE); - - ++iterations; - BOOST_TEST(goalState == untrainedQLearner.getState()); - } while(iterations < 1000); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_invalid_state_action_queries) -{ - // Out-of-range state/action exercises the invalid-query guards in both - // QTableRewardPolicy::getRewardForStateAndAction and - // QValueTablePolicy::getQValue, which all valid-path tests skip. - QLearnerType ql(learningRate, discountFactor, 100); - - const state_t badState = static_cast(NUMBER_OF_STATES + 10); - const action_t badAction = static_cast(NUMBER_OF_ACTIONS + 10); - - BOOST_TEST(static_cast(0) == - ql.getEnvironment().getRewardForStateAndAction(badState, badAction)); - BOOST_TEST(static_cast(0) == ql.getQValue(badState, badAction)); - - // numberOfValidActions == 0 hits the case-0 arm of - // chooseRandomActionFromValidActions. - action_t dummy[1] = {0}; - BOOST_TEST(static_cast(MazeEnvironmentType::EnvironmentInvalidAction) == - ql.getEnvironment().chooseRandomActionFromValidActions(dummy, 0)); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_argmax_exactly_two_actions) -{ - // decisionPoint 0 -> shouldChooseRandomAction() is always false, so - // chooseAction routes through ArgMaxPolicy::selectBestActionForState. - // A state with exactly two valid actions hits its case-2 arm; making the - // first action's Q-value larger exercises the qValue0 > qValue1 branch. - QLearnerType ql(learningRate, discountFactor, 0); - - const state_t state = 0; - for (action_t a = 0; a < NUMBER_OF_ACTIONS; ++a) - { - ql.getEnvironment().setRewardForStateAndAction( - state, a, MazeEnvironmentType::EnvironmentInvalidActionValue); - } - // Actions 1 and 2 are the only valid ones. - ql.getEnvironment().setRewardForStateAndAction( - state, 1, MazeEnvironmentType::EnvironmentNoRewardValue); - ql.getEnvironment().setRewardForStateAndAction( - state, 2, MazeEnvironmentType::EnvironmentNoRewardValue); - - ql.setQValue(state, 1, QValueType(9, 0)); - ql.setQValue(state, 2, QValueType(1, 0)); - - BOOST_TEST(static_cast(1) == ql.takeAction(state)); -} - -BOOST_AUTO_TEST_CASE(test_qlearn_future_value_with_no_valid_actions) -{ - // A newState with no valid actions makes chooseAction return InvalidAction, - // exercising calculateFutureQValue's InvalidAction arm (future value 0) as - // well as the case-0 arms of selectBestActionForState and - // chooseRandomActionFromValidActions. - QLearnerType ql(learningRate, discountFactor, 0); - - const state_t deadEnd = 4; - for (action_t a = 0; a < NUMBER_OF_ACTIONS; ++a) - { - ql.getEnvironment().setRewardForStateAndAction( - deadEnd, a, MazeEnvironmentType::EnvironmentInvalidActionValue); - } - - QLearnerType::experience_t experience; - experience.state = 0; - experience.action = 1; - experience.newState = deadEnd; - experience.reward = reward; - - // Should complete without dereferencing an invalid action; the future - // value contribution collapses to zero. - ql.updateFromExperience(experience); - BOOST_TEST(deadEnd == ql.getState()); -} - -// Directly exercise QLearningEnvironment::chooseRandomActionFromValidActions on -// the NullLearningPolicy environment instantiation (otherwise reached only via -// the epsilon-greedy training path), closing its function-coverage gap. -BOOST_AUTO_TEST_CASE(test_choose_random_action_from_valid_actions_untrained) -{ - action_t validActions[3] = { 1, 3, 5 }; - const action_t chosen = - untrainedQLearner.getEnvironment().chooseRandomActionFromValidActions(validActions, 3); - const bool isValid = (chosen == 1) || (chosen == 3) || (chosen == 5); - BOOST_TEST(isValid); -} - -BOOST_AUTO_TEST_SUITE_END() - +/** +* Copyright (c) 2020 Intel Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +/* +Q-Learning unit test. Learn the best path out of a simple maze. + +5 == Outside the maze +________________________________________________ +| | | +| | | +| 0 | 1 / 5 +| | | +|____________/ ________|__/ __________________|_______________________ +| | | | +| | / | +| 4 | 3 | 2 | +| / | | +|__/ __________________|_______________________|_______________________| + 5 + +The paths out of the maze: + +0->4->5 +0->4->3->1->5 +1->5 +1->3->4->5 +2->3->1->5 +2->3->4->5 +3->1->5 +3->4->5 +4->5 +4->3->1->5 +*/ + +#include "compiler.h" + +#define BOOST_TEST_MODULE test module name +TINYMIND_DISABLE_WARNING_PUSH +TINYMIND_DISABLE_WARNING_GCC_ONLY("-Wdangling-reference") +#include +TINYMIND_DISABLE_WARNING_POP + +#include +#include +#include +#include +#include +#include +#include + +#include "portable_test_random.hpp" +#include "qformat.hpp" +#include "qlearn.hpp" +#include "fixedPointTransferFunctions.hpp" +#include "activationFunctions.hpp" +#include "nnproperties.hpp" + +#define NUMBER_OF_STATES 6 +#define NUMBER_OF_ACTIONS 6 +#define RANDOM_SEED 7U + +// Upper bound on the steps any one episode may take. The maze has 6 states, so +// a policy that is making progress reaches the goal in far fewer; this exists +// so that a policy which is *not* making progress fails the test instead of +// looping forever. +static const size_t MAX_STEPS_PER_EPISODE = 1000; + +typedef uint8_t state_t; +typedef uint8_t action_t; + +template +struct UniformRealRandomNumberGenerator +{ + typedef tinymind::ValueConverter WeightConverterPolicy; + + static ValueType generateRandomWeight() + { + const double temp = generator()(); + const ValueType weight = WeightConverterPolicy::convertToDestinationType(temp); + + return weight; + } + +private: + // Function-local statics, not namespace-scope ones. These are consumed by + // the constructors of the static QLearner objects below, and dynamic + // initialization of a static data member of a class template is *unordered* + // with respect to other non-local variables ([basic.start.dynamic]), so a + // namespace-scope definition may or may not have run by the time those + // constructors call generateRandomWeight(). It genuinely differed by + // compiler: g++ used the still-zero-initialized distribution, handing every + // weight 0.0, while clang used a properly constructed [-1, 1). Reading an + // object before its lifetime begins is UB either way. Function-local + // statics are initialized on first use, so both compilers now see a real + // distribution and identical weights. + // Also implementation-independent: std::uniform_real_distribution's mapping + // is not specified, so the draw sequence differed between libstdc++ and + // libc++. See unit_test/include/portable_test_random.hpp. + static PortableUniformReal& generator() + { + static PortableUniformReal instance(-1.0, 1.0, RANDOM_SEED); + + return instance; + } +}; + +template +struct MazeEnvironmentRandomNumberGeneratorPolicy +{ + // mRandomActionDecisionPoint must be initialized here: getRandomActionDecisionPoint() + // is reachable before any setRandomActionDecisionPoint() call, and reading an + // uninitialized member is UB that neither ASan nor UBSan diagnoses. + MazeEnvironmentRandomNumberGeneratorPolicy() : mRandomEngine(RANDOM_SEED), mRandomActionDecisionPoint(0) + { + } + + size_t getRandomActionDecisionPoint() const + { + return this->mRandomActionDecisionPoint; + } + + T randInt(const T min, const T max) + { + std::uniform_int_distribution distribution(min, max); + const T value = static_cast(distribution(this->mRandomEngine)); + + return value; + } + + void setRandomActionDecisionPoint(const size_t randomActionDecisionPoint) + { + if(randomActionDecisionPoint <= 100) + { + this->mRandomActionDecisionPoint = randomActionDecisionPoint; + } + } +private: + std::default_random_engine mRandomEngine; + size_t mRandomActionDecisionPoint; +}; + +template< typename StateType, + typename ActionType, + typename ValueType, + size_t NumberOfStates, + size_t NumberOfActions, + typename RewardPolicyType, + template class QLearningPolicy = tinymind::DefaultLearningPolicy + > +struct MazeEnvironment : public tinymind::QLearningEnvironment +{ + typedef tinymind::QLearningEnvironment ParentType; + static const size_t EnvironmentNumberOfStates = ParentType::EnvironmentNumberOfStates; + static const size_t EnvironmentNumberOfActions = ParentType::EnvironmentNumberOfActions; + static const size_t EnvironmentInvalidState = ParentType::EnvironmentInvalidState; + static const size_t EnvironmentInvalidAction = ParentType::EnvironmentInvalidAction; + static const ValueType EnvironmentNoRewardValue; + static const ValueType EnvironmentInvalidActionValue; + + MazeEnvironment(const ValueType& learningRate, const ValueType& discountFactor, const size_t randomActionDecisionPoint) : + ParentType(learningRate, discountFactor, randomActionDecisionPoint), mGoalState(EnvironmentInvalidState) + { + } + + StateType getGoalState() const + { + return this->mGoalState; + } + + size_t getNextStateForStateActionPair(const action_t action) const + { + return action; + } + + ValueType getRewardForStateAndAction(const StateType state, const ActionType action) const + { + return this->mRewardPolicy.getRewardForStateAndAction(state, action); + } + + size_t getValidActionsForState(const state_t state, action_t* pActions) const + { + size_t numberOfValidActions = 0; + + for(action_t action = 0;action < NumberOfActions;++action) + { + if(EnvironmentInvalidActionValue != this->getRewardForStateAndAction(state, action)) + { + *pActions = action; + ++pActions; + ++numberOfValidActions; + } + } + + return numberOfValidActions; + } + + bool isGoalState(const StateType state) const + { + return (state == this->mGoalState); + } + + void setGoalState(const StateType goalState) + { + if (goalState < EnvironmentNumberOfStates) + { + this->mGoalState = goalState; + } + } + + void setRewardForStateAndAction(const StateType state, const ActionType action, const ValueType& reward) + { + this->mRewardPolicy.setRewardForStateAndAction(state, action, reward); + } + + void takeAction(const typename ParentType::experience_t& experience) + { + this->mExperience = experience; + } + + typename ParentType::experience_t mExperience; + StateType mGoalState; +private: + RewardPolicyType mRewardPolicy; +}; + +template class QLearningPolicy> +const ValueType MazeEnvironment::EnvironmentNoRewardValue = ValueType(0); + +template class QLearningPolicy> +const ValueType MazeEnvironment::EnvironmentInvalidActionValue = ValueType(-1, 0); + +template< typename StateType, + typename ActionType, + typename ValueType, + size_t NumberOfStates, + size_t NumberOfActions, + typename RewardPolicyType, + template class QLearningPolicy = tinymind::DefaultLearningPolicy + > +struct DQNMazeEnvironment : public tinymind::QLearningEnvironment +{ + typedef tinymind::QLearningEnvironment ParentType; + static const size_t EnvironmentNumberOfStates = ParentType::EnvironmentNumberOfStates; + static const size_t EnvironmentNumberOfActions = ParentType::EnvironmentNumberOfActions; + static const size_t EnvironmentInvalidState = ParentType::EnvironmentInvalidState; + static const size_t EnvironmentInvalidAction = ParentType::EnvironmentInvalidAction; + static const ValueType EnvironmentNoRewardValue; + static const ValueType EnvironmentInvalidActionValue; + + DQNMazeEnvironment(const ValueType& learningRate, const ValueType& discountFactor, const size_t randomActionDecisionPoint) : + ParentType(learningRate, discountFactor, randomActionDecisionPoint), mGoalState(EnvironmentInvalidState) + { + } + + StateType getGoalState() const + { + return this->mGoalState; + } + + static void getInputValues(const StateType state, ValueType *pInputs) + { + static const ValueType MAX_STATE = ValueType(NUMBER_OF_STATES,0); + ValueType input = (ValueType(state, 0) / MAX_STATE); + + *pInputs = input; + } + + size_t getNextStateForStateActionPair(const action_t action) const + { + return action; + } + + ValueType getRewardForStateAndAction(const StateType state, const ActionType action) const + { + return this->mRewardPolicy.getRewardForStateAndAction(state, action); + } + + size_t getValidActionsForState(const state_t state, action_t* pActions) const + { + size_t numberOfValidActions = 0; + + for(action_t action = 0;action < NumberOfActions;++action) + { + if(EnvironmentInvalidActionValue != this->getRewardForStateAndAction(state, action)) + { + *pActions = action; + ++pActions; + ++numberOfValidActions; + } + } + + return numberOfValidActions; + } + + bool isGoalState(const StateType state) const + { + return (state == this->mGoalState); + } + + void setGoalState(const StateType goalState) + { + if (goalState < EnvironmentNumberOfStates) + { + this->mGoalState = goalState; + } + } + + void setRewardForStateAndAction(const StateType state, const ActionType action, const ValueType& reward) + { + this->mRewardPolicy.setRewardForStateAndAction(state, action, reward); + } + + void takeAction(const typename ParentType::experience_t& experience) + { + this->mExperience = experience; + } + + typename ParentType::experience_t mExperience; + StateType mGoalState; +private: + RewardPolicyType mRewardPolicy; +}; + +template class QLearningPolicy> +const ValueType DQNMazeEnvironment::EnvironmentNoRewardValue = ValueType(0); + +template class QLearningPolicy> +const ValueType DQNMazeEnvironment::EnvironmentInvalidActionValue = ValueType(-1, 0); + +typedef tinymind::QValue<16,16,true> QValueType; +typedef tinymind::QTableRewardPolicy QTableRewardPolicyType; +typedef MazeEnvironment MazeEnvironmentType; +typedef tinymind::QLearner QLearnerType; + +typedef tinymind::NullRewardPolicy NullRewardPolicyType; +typedef MazeEnvironment UntrainedMazeEnvironmentType; +typedef tinymind::QLearner UntrainedQLearnerType; + +#define NUMBER_OF_INPUT_LAYER_NEURONS 1 +#define NUMBER_OF_HIDDEN_LAYERS 1 +#define NUMBER_OF_HIDDEN_LAYER_NEURONS (NUMBER_OF_ACTIONS + 2) +#define NUMBER_OF_OUTPUT_LAYER_NEURONS NUMBER_OF_ACTIONS +#define NUMBER_OF_ITERATIONS_FOR_TARGET_NN_UPDATE 10 + +typedef DQNMazeEnvironment DQNMazeEnvironmentType; +typedef tinymind::FixedPointTransferFunctions< QValueType, + UniformRealRandomNumberGenerator, + tinymind::TanhActivationPolicy, + tinymind::TanhActivationPolicy, + NUMBER_OF_OUTPUT_LAYER_NEURONS> TransferFunctionsType; +typedef tinymind::MultilayerPerceptron< QValueType, + NUMBER_OF_INPUT_LAYER_NEURONS, + NUMBER_OF_HIDDEN_LAYERS, + NUMBER_OF_HIDDEN_LAYER_NEURONS, + NUMBER_OF_OUTPUT_LAYER_NEURONS, + TransferFunctionsType> NeuralNetworkType; +typedef tinymind::QValueNeuralNetworkPolicy QValuePolicyType; +typedef tinymind::QLearner DQNQLearnerType; + +static const QValueType learningRate = (QValueType(1,0) / QValueType(5,0)); +static const QValueType discountFactor = (QValueType(8,0) / QValueType(10,0)); +static QLearnerType qLearner(learningRate, discountFactor, 100); +static DQNQLearnerType dqnQLearner(learningRate, discountFactor, 100); +static const QValueType reward(100,0); +static UntrainedQLearnerType untrainedQLearner(QValueType(0,0), QValueType(0,0), 0); + +template +static void initializeRewardMatrix(QLearnerType& qLearner) +{ + state_t state; + + state = 0; + for(action_t action = 0;action < 4;++action) + { + qLearner.getEnvironment().setRewardForStateAndAction(state, action, MazeEnvironmentType::EnvironmentInvalidActionValue); + } + qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 5, MazeEnvironmentType::EnvironmentInvalidActionValue); + + state = 1; + for(action_t action = 0;action < 3;++action) + { + qLearner.getEnvironment().setRewardForStateAndAction(state, action, MazeEnvironmentType::EnvironmentInvalidActionValue); + } + qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 5, reward); + + state = 2; + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + qLearner.getEnvironment().setRewardForStateAndAction(state, action, MazeEnvironmentType::EnvironmentInvalidActionValue); + } + qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentNoRewardValue); + + state = 3; + qLearner.getEnvironment().setRewardForStateAndAction(state, 0, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 1, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 2, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 5, MazeEnvironmentType::EnvironmentInvalidActionValue); + + state = 4; + qLearner.getEnvironment().setRewardForStateAndAction(state, 0, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 1, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 2, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 5, reward); + + state = 5; + qLearner.getEnvironment().setRewardForStateAndAction(state, 0, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 1, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 2, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 3, MazeEnvironmentType::EnvironmentInvalidActionValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 4, MazeEnvironmentType::EnvironmentNoRewardValue); + qLearner.getEnvironment().setRewardForStateAndAction(state, 5, reward); +} + +BOOST_AUTO_TEST_SUITE(test_suite_qlearn) + +BOOST_AUTO_TEST_CASE(test_qlearn_init) +{ + for(state_t state = 0;state < NUMBER_OF_STATES;++state) + { + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + BOOST_TEST(static_cast(0) == qLearner.getQValue(state, action)); + BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getRewardForStateAndAction(state, action)); + } + } +} + +BOOST_AUTO_TEST_CASE(test_qlearn_setreward) +{ + initializeRewardMatrix(qLearner); + + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 4)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 3)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 3)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 1)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 2)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 4)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 0)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 3)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 1)); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 4)); + + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 0)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 1)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 2)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(0, 5)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 0)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 1)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 2)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(1, 4)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 0)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 1)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 2)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 4)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(2, 5)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 0)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 3)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(3, 5)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 1)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 2)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(4, 4)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 0)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 2)); + BOOST_TEST(MazeEnvironmentType::EnvironmentInvalidActionValue == qLearner.getEnvironment().getRewardForStateAndAction(5, 3)); + + BOOST_TEST(reward == qLearner.getEnvironment().getRewardForStateAndAction(1, 5)); + BOOST_TEST(reward == qLearner.getEnvironment().getRewardForStateAndAction(4, 5)); + BOOST_TEST(reward == qLearner.getEnvironment().getRewardForStateAndAction(5, 5)); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_text_next_state) +{ + BOOST_TEST(static_cast(4) == qLearner.getEnvironment().getNextStateForStateActionPair(4)); + BOOST_TEST(static_cast(3) == qLearner.getEnvironment().getNextStateForStateActionPair(3)); + BOOST_TEST(static_cast(5) == qLearner.getEnvironment().getNextStateForStateActionPair(5)); + BOOST_TEST(static_cast(3) == qLearner.getEnvironment().getNextStateForStateActionPair(3)); + BOOST_TEST(static_cast(1) == qLearner.getEnvironment().getNextStateForStateActionPair(1)); + BOOST_TEST(static_cast(2) == qLearner.getEnvironment().getNextStateForStateActionPair(2)); + BOOST_TEST(static_cast(4) == qLearner.getEnvironment().getNextStateForStateActionPair(4)); + BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getNextStateForStateActionPair(0)); + BOOST_TEST(static_cast(3) == qLearner.getEnvironment().getNextStateForStateActionPair(3)); + BOOST_TEST(static_cast(5) == qLearner.getEnvironment().getNextStateForStateActionPair(5)); + BOOST_TEST(static_cast(1) == qLearner.getEnvironment().getNextStateForStateActionPair(1)); + BOOST_TEST(static_cast(4) == qLearner.getEnvironment().getNextStateForStateActionPair(4)); + BOOST_TEST(static_cast(5) == qLearner.getEnvironment().getNextStateForStateActionPair(5)); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_change_learning_rate) +{ + static const QValueType newLearningRate = (QValueType(1,0) / QValueType(100,0)); + BOOST_TEST(learningRate != newLearningRate); + BOOST_TEST(learningRate == qLearner.getEnvironment().getLearningRate()); + qLearner.getEnvironment().setLearningRate(newLearningRate); + BOOST_TEST(newLearningRate == qLearner.getEnvironment().getLearningRate()); + qLearner.getEnvironment().setLearningRate(learningRate); + BOOST_TEST(learningRate == qLearner.getEnvironment().getLearningRate()); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_change_discount_factor) +{ + static const QValueType delta = (QValueType(1,0) / QValueType(100,0)); + static const QValueType discountFactor = qLearner.getEnvironment().getDiscountFactor(); + static const QValueType newDiscountFactor = (discountFactor + delta); + + BOOST_TEST(discountFactor != newDiscountFactor); + qLearner.getEnvironment().setDiscountFactor(newDiscountFactor); + BOOST_TEST(newDiscountFactor == qLearner.getEnvironment().getDiscountFactor()); + qLearner.getEnvironment().setDiscountFactor(discountFactor); + BOOST_TEST(discountFactor == qLearner.getEnvironment().getDiscountFactor()); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_choose_random_action) +{ + const size_t oldDecisionPoint = qLearner.getEnvironment().getRandomActionDecisionPoint(); + std::vector choices; + std::size_t t; + + qLearner.getEnvironment().setRandomActionDecisionPoint(50); + for(unsigned i = 0;i < 1000;++i) + { + choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); + } + + t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); + + BOOST_TEST(((t >= 400) && (t <= 600))); + + qLearner.getEnvironment().setRandomActionDecisionPoint(90); + choices.clear(); + for(unsigned i = 0;i < 1000;++i) + { + choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); + } + + t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); + + BOOST_TEST(((t >= 800) && (t <= 1000))); + + qLearner.getEnvironment().setRandomActionDecisionPoint(0); + choices.clear(); + for(unsigned i = 0;i < 1000;++i) + { + choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); + } + + t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); + + BOOST_TEST(0U == t); + + qLearner.getEnvironment().setRandomActionDecisionPoint(100); + choices.clear(); + for(unsigned i = 0;i < 1000;++i) + { + choices.push_back(qLearner.getEnvironment().shouldChooseRandomAction()); + } + + t = std::count_if(choices.begin(), choices.end(), [](const bool value){return value;}); + + BOOST_TEST(1000U == t); + + qLearner.getEnvironment().setRandomActionDecisionPoint(oldDecisionPoint); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_goal_state) +{ + BOOST_TEST(static_cast(-1) == qLearner.getEnvironment().getGoalState()); + + qLearner.getEnvironment().setGoalState(0); + BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getGoalState()); + + qLearner.getEnvironment().setGoalState(-1); + BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getGoalState()); + + qLearner.getEnvironment().setGoalState(QLearnerType::NumberOfStates); + BOOST_TEST(static_cast(0) == qLearner.getEnvironment().getGoalState()); + + qLearner.getEnvironment().setGoalState(NUMBER_OF_STATES - 1); + BOOST_TEST(static_cast(NUMBER_OF_STATES - 1) == qLearner.getEnvironment().getGoalState()); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_argmax_best_action) +{ + typedef tinymind::QValueTablePolicy QValuePolicyType; + QValuePolicyType qValuePolicy; + QValueType qValue(0, 0); + state_t state = 1; + action_t actions[NUMBER_OF_ACTIONS] = { 0, 1, 2, 3, 4, 5 }; + action_t bestAction; + + for (action_t action = 0; action < NUMBER_OF_ACTIONS; ++action) + { + qValuePolicy.setQValue(state, action, qValue); + ++qValue; + } + + bestAction = tinymind::ArgMaxPolicy::selectBestActionForState(state, &actions[0], NUMBER_OF_ACTIONS, qValuePolicy); + BOOST_TEST(static_cast(5) == bestAction); + + qValue = 0; + for (action_t action = NUMBER_OF_ACTIONS; action > 0; --action) + { + qValuePolicy.setQValue(state, action - 1, qValue); + ++qValue; + } + + bestAction = tinymind::ArgMaxPolicy::selectBestActionForState(state, &actions[0], NUMBER_OF_ACTIONS, qValuePolicy); + BOOST_TEST(static_cast(0) == bestAction); + + qValuePolicy.setQValue(state, 1, QValueType(100, 0)); + bestAction = tinymind::ArgMaxPolicy::selectBestActionForState(state, &actions[0], NUMBER_OF_ACTIONS - 2, qValuePolicy); + BOOST_TEST(static_cast(1) == bestAction); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_take_action) +{ + typename MazeEnvironmentType::ParentType::experience_t experience; + state_t state = 1; + action_t action = 5; + + initializeRewardMatrix(qLearner); + + experience.state = state; + experience.action = action; + qLearner.getEnvironment().takeAction(experience); + experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); + experience.newState = static_cast(experience.action); + qLearner.updateFromExperience(experience); + + BOOST_TEST(static_cast(1) == experience.state); + BOOST_TEST(static_cast(5) == experience.action); + BOOST_TEST(reward == experience.reward); + BOOST_TEST(static_cast(5) == experience.newState); + + state = 0; + action = 4; + + experience.state = state; + experience.action = action; + qLearner.getEnvironment().takeAction(experience); + experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); + experience.newState = static_cast(experience.action); + qLearner.updateFromExperience(experience); + + BOOST_TEST(static_cast(0) == experience.state); + BOOST_TEST(static_cast(4) == experience.action); + BOOST_TEST(MazeEnvironmentType::EnvironmentNoRewardValue == experience.reward); + BOOST_TEST(static_cast(4) == experience.newState); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_iterate) +{ + typedef typename QValueType::FullWidthValueType FullWidthValueType; + static char const* const qTableFilePath = "qtable.txt"; + static char const* const qTableBinFilePath = "qtable.bin"; + std::default_random_engine engine(RANDOM_SEED);; + std::uniform_int_distribution stateDistribution(0, NUMBER_OF_STATES - 1); + size_t decisionPoint = 100; + std::ofstream qTableFile(qTableFilePath); + std::ofstream qTableBinFile(qTableBinFilePath, std::ios::out | std::ios::binary); + typename MazeEnvironmentType::ParentType::experience_t experience; + state_t state; + action_t action; + FullWidthValueType value; + + for (unsigned i = 0; i < 500; ++i) + { + qLearner.getEnvironment().setRandomActionDecisionPoint(decisionPoint); + + qLearner.startNewEpisode(); + state = static_cast(stateDistribution(engine)); + qLearner.setState(state); + qLearner.getEnvironment().setGoalState(5); + action = qLearner.takeAction(state); + experience.state = state; + experience.action = action; + experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); + experience.newState = static_cast(experience.action); + qLearner.updateFromExperience(experience); + + // Bounded: tabular Q-learning on this maze does reach the goal, so the + // cap is a guard rather than an expected exit. Hitting it is a failure, + // not a hang -- see the assertion after the loop. + size_t steps = 0; + while (qLearner.getState() != 5 && steps < MAX_STEPS_PER_EPISODE) + { + ++steps; + action = qLearner.takeAction(qLearner.getState()); + experience.state = qLearner.getState(); + experience.action = action; + experience.reward = qLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); + experience.newState = static_cast(experience.action); + qLearner.updateFromExperience(experience); + } + + BOOST_TEST(steps < MAX_STEPS_PER_EPISODE); + + if (decisionPoint > 0) + { + decisionPoint -= 1; + } + } + + // store q table in text form + for(state_t state = 0;state < NUMBER_OF_STATES;++state) + { + qTableFile << state << ","; + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + qTableFile << qLearner.getQValue(state, action).getValue(); + if (action != (NUMBER_OF_ACTIONS - 1)) + { + qTableFile << ","; + } + } + + qTableFile << std::endl; + } + + qTableFile.close(); + + // store q table in binary form + for(state_t state = 0;state < NUMBER_OF_STATES;++state) + { + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + value = qLearner.getQValue(state, action).getValue(); + qTableBinFile.write(reinterpret_cast(&value), sizeof(FullWidthValueType)); + } + } + + qTableBinFile.close(); +} + +BOOST_AUTO_TEST_CASE(test_dqn_qlearn_iterate) +{ + std::default_random_engine engine(RANDOM_SEED);; + std::uniform_int_distribution stateDistribution(0, NUMBER_OF_STATES - 1); + size_t decisionPoint = 100; + typename DQNMazeEnvironmentType::ParentType::experience_t experience; + state_t state; + action_t action; + + initializeRewardMatrix(dqnQLearner); + + dqnQLearner.getEnvironment().setGoalState(5); + + for (unsigned i = 0; i < 500; ++i) + { + dqnQLearner.getEnvironment().setRandomActionDecisionPoint(decisionPoint); + + dqnQLearner.startNewEpisode(); + state = static_cast(stateDistribution(engine)); + dqnQLearner.setState(state); + action = dqnQLearner.takeAction(state); + experience.state = state; + experience.action = action; + experience.reward = dqnQLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); + experience.newState = static_cast(experience.action); + dqnQLearner.updateFromExperience(experience); + + // Bounded episode. This case asserts nothing about the policy -- it + // exercises the DQN iterate/update machinery -- so an episode that does + // not reach the goal is simply cut short rather than spinning forever. + // The greedy policy is under no obligation to reach the goal, and with + // an unbounded loop it does not: the loop only ever terminated because + // the weights were accidentally all zero (see the static-initialization + // note on UniformRealRandomNumberGenerator). + for (size_t step = 0; + step < MAX_STEPS_PER_EPISODE && + dqnQLearner.getState() != dqnQLearner.getEnvironment().getGoalState(); + ++step) + { + action = dqnQLearner.takeAction(dqnQLearner.getState()); + experience.state = dqnQLearner.getState(); + experience.action = action; + experience.reward = dqnQLearner.getEnvironment().getRewardForStateAndAction(experience.state, experience.action); + experience.newState = static_cast(experience.action); + dqnQLearner.updateFromExperience(experience); + } + + if (decisionPoint > 0) + { + decisionPoint -= 1; + } + } +} + +BOOST_AUTO_TEST_CASE(test_untrained_qlearner_reward) +{ + // Reward table should not exist + for(state_t state = 0;state < NUMBER_OF_STATES;++state) + { + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + BOOST_TEST(static_cast(0) == untrainedQLearner.getEnvironment().getRewardForStateAndAction(state, action)); + } + } + // Attempt to set reward + for(state_t state = 0;state < NUMBER_OF_STATES;++state) + { + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + untrainedQLearner.getEnvironment().setRewardForStateAndAction(state, action, reward); + } + } + + // Setting reward should have no effect + for(state_t state = 0;state < NUMBER_OF_STATES;++state) + { + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + BOOST_TEST(static_cast(0) == untrainedQLearner.getEnvironment().getRewardForStateAndAction(state, action)); + } + } +} + +//Load the Q table from the trained Q learner and use it. +BOOST_AUTO_TEST_CASE(test_untrained_qlearner_iterate) +{ + static const state_t goalState = 5; + std::default_random_engine engine(RANDOM_SEED);; + std::uniform_int_distribution stateDistribution(0, NUMBER_OF_STATES - 1); + size_t iterations = 0; + typename UntrainedMazeEnvironmentType::ParentType::experience_t experience; + state_t state; + action_t action; + + for(state = 0;state < NUMBER_OF_STATES;++state) + { + for(action_t action = 0;action < NUMBER_OF_ACTIONS;++action) + { + untrainedQLearner.setQValue(state, action, qLearner.getQValue(state, action)); + } + } + + do + { + untrainedQLearner.startNewEpisode(); + state = static_cast(stateDistribution(engine)); + untrainedQLearner.setState(state); + untrainedQLearner.getEnvironment().setGoalState(goalState); + // Bounded: this learner is seeded with the trained Q table, so it does + // reach the goal. The cap turns a future regression into a failure + // rather than a hang. + size_t steps = 0; + do + { + ++steps; + action = untrainedQLearner.takeAction(untrainedQLearner.getState()); + experience.state = state; + experience.action = action; + experience.newState = static_cast(experience.action); + untrainedQLearner.setState(experience.newState); + } while (untrainedQLearner.getState() != goalState && steps < MAX_STEPS_PER_EPISODE); + + BOOST_TEST(steps < MAX_STEPS_PER_EPISODE); + + ++iterations; + BOOST_TEST(goalState == untrainedQLearner.getState()); + } while(iterations < 1000); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_invalid_state_action_queries) +{ + // Out-of-range state/action exercises the invalid-query guards in both + // QTableRewardPolicy::getRewardForStateAndAction and + // QValueTablePolicy::getQValue, which all valid-path tests skip. + QLearnerType ql(learningRate, discountFactor, 100); + + const state_t badState = static_cast(NUMBER_OF_STATES + 10); + const action_t badAction = static_cast(NUMBER_OF_ACTIONS + 10); + + BOOST_TEST(static_cast(0) == + ql.getEnvironment().getRewardForStateAndAction(badState, badAction)); + BOOST_TEST(static_cast(0) == ql.getQValue(badState, badAction)); + + // numberOfValidActions == 0 hits the case-0 arm of + // chooseRandomActionFromValidActions. + action_t dummy[1] = {0}; + BOOST_TEST(static_cast(MazeEnvironmentType::EnvironmentInvalidAction) == + ql.getEnvironment().chooseRandomActionFromValidActions(dummy, 0)); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_argmax_exactly_two_actions) +{ + // decisionPoint 0 -> shouldChooseRandomAction() is always false, so + // chooseAction routes through ArgMaxPolicy::selectBestActionForState. + // A state with exactly two valid actions hits its case-2 arm; making the + // first action's Q-value larger exercises the qValue0 > qValue1 branch. + QLearnerType ql(learningRate, discountFactor, 0); + + const state_t state = 0; + for (action_t a = 0; a < NUMBER_OF_ACTIONS; ++a) + { + ql.getEnvironment().setRewardForStateAndAction( + state, a, MazeEnvironmentType::EnvironmentInvalidActionValue); + } + // Actions 1 and 2 are the only valid ones. + ql.getEnvironment().setRewardForStateAndAction( + state, 1, MazeEnvironmentType::EnvironmentNoRewardValue); + ql.getEnvironment().setRewardForStateAndAction( + state, 2, MazeEnvironmentType::EnvironmentNoRewardValue); + + ql.setQValue(state, 1, QValueType(9, 0)); + ql.setQValue(state, 2, QValueType(1, 0)); + + BOOST_TEST(static_cast(1) == ql.takeAction(state)); +} + +BOOST_AUTO_TEST_CASE(test_qlearn_future_value_with_no_valid_actions) +{ + // A newState with no valid actions makes chooseAction return InvalidAction, + // exercising calculateFutureQValue's InvalidAction arm (future value 0) as + // well as the case-0 arms of selectBestActionForState and + // chooseRandomActionFromValidActions. + QLearnerType ql(learningRate, discountFactor, 0); + + const state_t deadEnd = 4; + for (action_t a = 0; a < NUMBER_OF_ACTIONS; ++a) + { + ql.getEnvironment().setRewardForStateAndAction( + deadEnd, a, MazeEnvironmentType::EnvironmentInvalidActionValue); + } + + QLearnerType::experience_t experience; + experience.state = 0; + experience.action = 1; + experience.newState = deadEnd; + experience.reward = reward; + + // Should complete without dereferencing an invalid action; the future + // value contribution collapses to zero. + ql.updateFromExperience(experience); + BOOST_TEST(deadEnd == ql.getState()); +} + +// Directly exercise QLearningEnvironment::chooseRandomActionFromValidActions on +// the NullLearningPolicy environment instantiation (otherwise reached only via +// the epsilon-greedy training path), closing its function-coverage gap. +BOOST_AUTO_TEST_CASE(test_choose_random_action_from_valid_actions_untrained) +{ + action_t validActions[3] = { 1, 3, 5 }; + const action_t chosen = + untrainedQLearner.getEnvironment().chooseRandomActionFromValidActions(validActions, 3); + const bool isValid = (chosen == 1) || (chosen == 3) || (chosen == 5); + BOOST_TEST(isValid); +} + +BOOST_AUTO_TEST_SUITE_END() +