Implement a Native Renderer for Compiled Language Bindings

.github/workflows/main.yml

@@ -39,6 +39,17 @@ jobs:
           cmake --build . --config Release -j 8
           popd
 
+      - name: Test
+        shell: bash
+        run: |
+          set -x
+          set -e
+
+          # Run the native tests
+          pushd ./build/native/
+          ctest -C Release --output-on-failure
+          popd
+
   wasm:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -1,12 +1,14 @@
 build
 dist
 node_modules
+.worktrees
 
 .merlin
 .cljs_node_repl
 .cpcache
 .cache
 .DS_Store
+.idea
 
 *.swp
 

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "runtime/elem/third-party/signalsmith-stretch"]
 	path = runtime/elem/third-party/signalsmith-stretch
 	url = https://github.com/Signalsmith-Audio/signalsmith-stretch.git
+[submodule "tests/third-party/googletest"]
+	path = tests/third-party/googletest
+	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -1,14 +1,23 @@
 cmake_minimum_required(VERSION 3.15)
 project(elementary VERSION 0.11.5)
 
-
 option(ONLY_BUILD_WASM "Only build the wasm subdirectory" OFF)
 set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
 
+if(MSVC)
+  # MSVC's default (non-conformant) preprocessor mishandles __VA_ARGS__
+  # expansion in nested macros, which breaks the ELEM_ARG_COUNT/DEFINE_PROPS_STRUCT
+  # macros in runtime/elem/lib/Props.h. The conformant preprocessor fixes this.
+  add_compile_options(/Zc:preprocessor)
+endif()
+
 if(${ONLY_BUILD_WASM})
   add_subdirectory(runtime)
   add_subdirectory(wasm)
 else()
+  enable_testing()
   add_subdirectory(runtime)
   add_subdirectory(cli)
+  add_subdirectory(cli-native)
+  add_subdirectory(tests)
 endif()

README.md

@@ -17,6 +17,8 @@
 * **Dynamic:** Most audio processing frameworks and tools facilitate building static processes. But what happens as your audio requirements change throughout the user journey? Elementary is designed to facilitate and adapt to the dynamic nature of modern audio applications.
 * **Portable:** By decoupling the JavaScript API from the underlying audio engine (the "what" from the "how"), Elementary enables writing portable applications. Whether the underlying engine is running in the browser, an audio plugin, or an embedded device, your JavaScript layer remains the same.
 
+Elementary also has community supported bindings for languages other than JavaScript in [Other Languages](https://www.elementary.audio/docs/other_languages/community-supported-language-bindings).
+
 ## Getting Started
 
 Every Elementary application starts with the [@elemaudio/core](https://www.elementary.audio/docs/packages/core) package ([npm](https://www.npmjs.com/package/@elemaudio/core), [source](./js/packages/core)), which provides the
@@ -27,6 +29,7 @@ If you're new to the project, we recommend studying the following workflows to g
 
 * Use the [@elemaudio/web-renderer](https://www.elementary.audio/docs/packages/web-renderer) package ([npm](https://www.npmjs.com/package/@elemaudio/web-renderer), [source](./js/packages/web-renderer)) with your favorite frontend UI library to make an audio web application
 * Use the [@elemaudio/offline-renderer](https://www.elementary.audio/docs/packages/offline-renderer) package ([npm](https://www.npmjs.com/package/@elemaudio/offline-renderer), [source](./js/packages/offline-renderer)) with Node.js for static file processing
+* Use the [elementary-swift](https://github.com/parkernilson/elementary-swift) package to develop an iOS or macOS application
 
 Once you're ready to dive in, we suggest starting with one of these ideas:
 

cli-native/CMakeLists.txt

@@ -0,0 +1,33 @@
+cmake_minimum_required(VERSION 3.15)
+project(cli-native VERSION 0.11.0)
+
+add_library(elemcli_native_graph STATIC
+  graphs/HelloSine.cpp
+  graphs/GraphRegistry.cpp)
+
+target_include_directories(elemcli_native_graph PUBLIC
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/miniaudio)
+
+target_compile_features(elemcli_native_graph PUBLIC
+  cxx_std_17)
+
+target_link_libraries(elemcli_native_graph PUBLIC
+  elem::runtime)
+
+if(MSVC)
+  target_compile_options(elemcli_native_graph PRIVATE /W4)
+else()
+  target_compile_options(elemcli_native_graph PRIVATE -Wall -Wextra)
+endif()
+
+add_executable(elemcli-native RealtimeMain.cpp)
+
+target_link_libraries(elemcli-native PRIVATE elemcli_native_graph)
+
+if(UNIX AND NOT APPLE)
+  find_package(Threads REQUIRED)
+  target_link_libraries(elemcli-native PRIVATE
+    Threads::Threads
+    ${CMAKE_DL_LIBS})
+endif()

cli-native/README.md

@@ -0,0 +1,30 @@
+# cli-native
+
+This version of the cli uses the native renderer instead of the js based renderer to render the Audio Graph.
+
+The cli tool here is not meant to be a fully featured, end-user product, but rather
+a concise demonstration of using Elementary's native renderer with the native engine.
+
+## Building
+
+```bash
+cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug
+cmake --build build --target elemcli-native
+```
+
+## Running
+
+- `./build/elemcli-native --list` lists the available graphs.
+- `./build/elemcli-native --graph <name>` plays the named graph to the
+  default audio device. Press Enter to exit.
+- `./build/elemcli-native --help` prints usage information.
+
+## Adding a graph
+
+1. Add `graphs/MyGraph.h`/`.cpp` exposing a `buildMyGraph()` function that
+   returns a `std::vector<std::shared_ptr<elem::SymbolicGraphNode>>` (see
+   `graphs/HelloSine.h`/`.cpp` for an example).
+2. Register it by adding an entry to the vector in
+   `graphs/GraphRegistry.cpp`.
+3. Add `graphs/MyGraph.cpp` to the `elemcli_native_graph` sources in
+   `CMakeLists.txt`.

cli-native/RealtimeMain.cpp

@@ -0,0 +1,176 @@
+#include <algorithm>
+#include <array>
+#include <iostream>
+#include <memory>
+#include <vector>
+
+#include "graphs/GraphRegistry.h"
+#include "elem/Renderer.h"
+#include "elem/Runtime.h"
+
+#define MINIAUDIO_IMPLEMENTATION
+#include "miniaudio.h"
+
+
+// A simple struct to proxy between the audio device and the Elementary engine
+struct DeviceProxy {
+    DeviceProxy(double sampleRate, size_t blockSize)
+        : scratchData(2 * blockSize), runtime(std::make_shared<elem::Runtime<float>>(sampleRate, blockSize))
+    {}
+
+    void process(float* outputData, size_t numChannels, size_t numFrames)
+    {
+        // We might hit this the first time around, but after that should be fine
+        if (scratchData.size() < (numChannels * numFrames))
+            scratchData.resize(numChannels * numFrames);
+
+        auto* deinterleaved = scratchData.data();
+        std::array<float*, 2> ptrs {deinterleaved, deinterleaved + numFrames};
+
+        runtime->process(
+            nullptr,
+            0,
+            ptrs.data(),
+            numChannels,
+            numFrames,
+            nullptr
+        );
+
+        for (size_t i = 0; i < numChannels; ++i)
+        {
+            for (size_t j = 0; j < numFrames; ++j)
+            {
+                outputData[i + numChannels * j] = deinterleaved[i * numFrames + j];
+            }
+        }
+    }
+
+    std::vector<float> scratchData;
+    std::shared_ptr<elem::Runtime<float>> runtime;
+};
+
+void audioCallback(ma_device* pDevice, void* pOutput, const void* /* pInput */, ma_uint32 frameCount)
+{
+    auto* proxy = static_cast<DeviceProxy*>(pDevice->pUserData);
+
+    const auto numChannels = static_cast<size_t>(pDevice->playback.channels);
+    const auto numFrames = static_cast<size_t>(frameCount);
+
+    proxy->process(static_cast<float*>(pOutput), numChannels, numFrames);
+}
+
+void printUsage(const char* programName)
+{
+    std::cout << "Usage: " << programName << " --graph <name>" << std::endl;
+    std::cout << std::endl;
+    std::cout << "Renders an Elementary audio graph to the default audio device." << std::endl;
+    std::cout << std::endl;
+    std::cout << "Options:" << std::endl;
+    std::cout << "  --graph <name>   Render the named graph" << std::endl;
+    std::cout << "  --list           List the available graphs" << std::endl;
+    std::cout << "  --help, -h       Show this usage information" << std::endl;
+    std::cout << std::endl;
+    std::cout << "Example:" << std::endl;
+    std::cout << "  " << programName << " --graph hello-sine" << std::endl;
+}
+
+void printGraphList()
+{
+    for (auto const& graphInfo : elem::lib::getGraphRegistry()) {
+        std::cout << graphInfo.name << " — " << graphInfo.description << std::endl;
+    }
+}
+
+int main(int argc, char** argv)
+{
+    std::string graphName;
+    std::vector<std::string> args(argv + 1, argv + argc);
+
+    for (size_t i = 0; i < args.size(); ++i) {
+        if (args[i] == "--help" || args[i] == "-h") {
+            printUsage(argv[0]);
+            return 0;
+        } else if (args[i] == "--list") {
+            printGraphList();
+            return 0;
+        } else if (args[i] == "--graph") {
+            if (i + 1 >= args.size()) {
+                std::cerr << "Error: --graph requires a graph name" << std::endl;
+                printUsage(argv[0]);
+                return 1;
+            }
+
+            graphName = args[++i];
+        }
+    }
+
+    if (graphName.empty()) {
+        printUsage(argv[0]);
+        return 1;
+    }
+
+    auto const& registry = elem::lib::getGraphRegistry();
+    auto it = std::find_if(registry.begin(), registry.end(), [&graphName](elem::lib::GraphInfo const& graphInfo) {
+        return graphInfo.name == graphName;
+    });
+
+    if (it == registry.end()) {
+        std::cerr << "Error: unknown graph \"" << graphName << "\". Available graphs:" << std::endl;
+        printGraphList();
+        return 1;
+    }
+
+    ma_result result;
+
+    ma_device_config deviceConfig;
+    ma_device device;
+
+    // XXX: I don't see a way to ask miniaudio for a specific block size. Let's just allocate
+    // here for 1024 and resize in the first callback if we need to.
+    std::unique_ptr<DeviceProxy> proxy = std::make_unique<DeviceProxy>(44100.0, 1024);
+
+    deviceConfig = ma_device_config_init(ma_device_type_playback);
+
+    deviceConfig.playback.pDeviceID = nullptr;
+    deviceConfig.playback.format    = ma_format_f32;
+    deviceConfig.playback.channels  = 2;
+    deviceConfig.sampleRate         = 44100;
+    deviceConfig.dataCallback       = audioCallback;
+    deviceConfig.pUserData          = proxy.get();
+
+    result = ma_device_init(nullptr, &deviceConfig, &device);
+
+    if (result != MA_SUCCESS) {
+        std::cout << "Failed to start the audio device! Exiting..." << std::endl;
+        return 1;
+    }
+
+    // Build and render the graph before audio starts
+    elem::Renderer<float> renderer(proxy->runtime);
+    auto stats = renderer.renderGraph(it->build());
+
+    std::cout << "Render result: " << stats.result << std::endl;
+    std::cout << "Nodes added: " << stats.nodesAdded << std::endl;
+    std::cout << "Edges added: " << stats.edgesAdded << std::endl;
+    std::cout << "Props written: " << stats.propsWritten << std::endl;
+
+    if (stats.result != elem::ReturnCode::Ok()) {
+        std::cerr << "Failed to render graph: " << elem::ReturnCode::describe(stats.result) << std::endl;
+        ma_device_uninit(&device);
+        return 1;
+    }
+
+    result = ma_device_start(&device);
+
+    if (result != MA_SUCCESS) {
+        std::cout << "Failed to start the audio device! Exiting..." << std::endl;
+        ma_device_uninit(&device);
+        return 1;
+    }
+
+    std::cout << "Press Enter to exit..." << std::endl;
+    getchar();
+
+    ma_device_uninit(&device);
+    return 0;
+}

cli-native/graphs/GraphRegistry.cpp

@@ -0,0 +1,13 @@
+#include "GraphRegistry.h"
+
+#include "HelloSine.h"
+
+namespace elem::lib {
+    std::vector<GraphInfo> const& getGraphRegistry() {
+        static const std::vector<GraphInfo> registry = {
+            {"hello-sine", "A single 440Hz sine tone at 0.3 gain", buildHelloSineGraph},
+        };
+
+        return registry;
+    }
+}

cli-native/graphs/GraphRegistry.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "elem/NodeRepr.h"
+
+namespace elem::lib {
+    struct GraphInfo {
+        std::string name;
+        std::string description;
+        std::function<std::vector<std::shared_ptr<elem::NodeRepr>>()> build;
+    };
+
+    std::vector<GraphInfo> const& getGraphRegistry();
+}

cli-native/graphs/HelloSine.cpp

@@ -0,0 +1,13 @@
+#include "HelloSine.h"
+
+#include "elem/lib/Math.h"
+#include "elem/lib/Oscillators.h"
+
+namespace elem::lib {
+    std::vector<std::shared_ptr<elem::NodeRepr>> buildHelloSineGraph() {
+        return {
+            mul({0.3, cycle(440.0)}),
+            mul({0.3, cycle(440.0)}),
+        };
+    }
+}

cli-native/graphs/HelloSine.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include "elem/NodeRepr.h"
+
+namespace elem::lib {
+    std::vector<std::shared_ptr<elem::NodeRepr>> buildHelloSineGraph();
+}