topology: cmake: build topologies in parallel, add -s to force sequential

alsatplg custom commands were marked USES_TERMINAL. With the Ninja
generator this places them in the single-job "console" pool, which
serializes every topology build regardless of the -j value passed to
ninja. Building all 579 topologies therefore never used more than one
core, even on a many-core machine.

Drop USES_TERMINAL by default so Ninja can run alsatplg invocations in
parallel, controlled by a new TPLG_SEQUENTIAL_BUILD CMake option
(default OFF).

Add a -s flag to build-tools.sh that sets TPLG_SEQUENTIAL_BUILD=ON, to
restore the old, easier to debug one-at-a-time behavior when needed.
It updates the CMake cache in place, so it also works on an existing
incremental build tree without deleting it.

Results
Sequential (before this patch or with -T -s with this patch):
./scripts/build-tools.sh -T -s  19.30s user 2.34s system 99% cpu 21.817 total

Parallel:
rm -rf tools/build_tools; time ./scripts/build-tools.sh -T
./scripts/build-tools.sh -T  33.79s user 3.52s system 986% cpu 3.783 total

The build time changes from ~22s to ~4s

Signed-off-by: Peter Ujfalusi <peter.ujfalusi@linux.intel.com>

Author: ujfalusi

scripts/build-tools.sh

@@ -16,7 +16,7 @@ Attention: the list of selected shortcuts below is _not_ exhaustive. To
 build _everything_ don't select any particular target; this will build
 CMake's default target "ALL".
 
-usage: $0 [-c|-f|-h|-l|-p|-t|-T|-X|-Y|-A]
+usage: $0 [-c|-f|-h|-l|-p|-s|-t|-T|-X|-Y|-A]
        -h Display help
 
        -c Rebuild ctl/
@@ -25,6 +25,8 @@ usage: $0 [-c|-f|-h|-l|-p|-t|-T|-X|-Y|-A]
        -T Rebuild topology/ (not topology/development/! Use ALL)
        -X Rebuild topology1 only
        -Y Rebuild topology2 only
+       -s Force sequential (one at a time) topology builds, for debugging.
+          Normally topologies build in parallel with -j "$NO_PROCESSORS".
        -t Rebuild test/topology/ (or tools/test/topology/tplg-build.sh directly)
        -A Clone and rebuild local ALSA lib and utils.
 
@@ -41,10 +43,19 @@ reconfigure_build()
         mkdir -p "$BUILD_TOOLS_DIR"
 
         ( cd "$BUILD_TOOLS_DIR"
-          cmake -GNinja -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" "${SOF_REPO}/tools"
+          cmake -GNinja -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \
+                -DTPLG_SEQUENTIAL_BUILD="$TPLG_SEQUENTIAL_BUILD" "${SOF_REPO}/tools"
         )
 }
 
+# Update the TPLG_SEQUENTIAL_BUILD CMake cache entry without wiping an
+# existing, already configured build tree.
+update_sequential_build_option()
+{
+        cmake -S "${SOF_REPO}/tools" -B "$BUILD_TOOLS_DIR" \
+              -DTPLG_SEQUENTIAL_BUILD="$TPLG_SEQUENTIAL_BUILD"
+}
+
 make_tool()
 {
         # if no argument provided, all the tools will be built. Empty tool is
@@ -95,12 +106,13 @@ main()
 {
         local DO_BUILD_ctl DO_BUILD_logger DO_BUILD_probes \
                 DO_BUILD_tests DO_BUILD_topologies1 DO_BUILD_topologies2 SCRIPT_DIR SOF_REPO \
-                CMAKE_ONLY BUILD_ALL
+                CMAKE_ONLY BUILD_ALL TPLG_SEQUENTIAL_BUILD
         SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
         SOF_REPO=$(dirname "$SCRIPT_DIR")
         : "${BUILD_TOOLS_DIR:=$SOF_REPO/tools/build_tools}"
         : "${NO_PROCESSORS:=$(nproc)}"
         BUILD_ALL=false
+        TPLG_SEQUENTIAL_BUILD=OFF
 
         if [ $# -eq 0 ]; then
                 BUILD_ALL=true
@@ -120,11 +132,12 @@ main()
 
         # eval is a sometimes necessary evil
         # shellcheck disable=SC2034
-        while getopts "cfhlptTCXYA" OPTION; do
+        while getopts "cfhlpstTCXYA" OPTION; do
                 case "$OPTION" in
                 c) DO_BUILD_ctl=true ;;
                 l) DO_BUILD_logger=true ;;
                 p) DO_BUILD_probes=true ;;
+                s) TPLG_SEQUENTIAL_BUILD=ON ;;
                 t) DO_BUILD_tests=true ;;
                 T) DO_BUILD_topologies1=true ; DO_BUILD_topologies2=true ;;
                 X) DO_BUILD_topologies1=true ;;
@@ -153,6 +166,11 @@ main()
             reconfigure_build
         }
 
+        # -s must apply even to an already configured, incremental build tree.
+        if [ "$TPLG_SEQUENTIAL_BUILD" = ON ]; then
+                update_sequential_build_option
+        fi
+
         if "$BUILD_ALL"; then
                 # default CMake targets
                 make_tool # trust set -e

tools/topology/CMakeLists.txt

@@ -2,6 +2,16 @@ set(SOF_TOPOLOGY_BINARY_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
 set(SOF_ALSA_TOOLS_DIR "${SOF_ROOT_SOURCE_DIRECTORY}/../tools/bin")
 set(ALSATPLG_CMD "${SOF_ALSA_TOOLS_DIR}/alsatplg")
 
+# With the Ninja generator, USES_TERMINAL forces the single-job "console"
+# pool, serializing every topology build regardless of -j. Enable this
+# option to force that serialized, easier to debug behavior back on.
+option(TPLG_SEQUENTIAL_BUILD "Build topologies one at a time (for debugging)" OFF)
+if(TPLG_SEQUENTIAL_BUILD)
+	set(TPLG_USES_TERMINAL USES_TERMINAL)
+else()
+	set(TPLG_USES_TERMINAL "")
+endif()
+
 function(alsatplg_version  OUT_STATUS  OUT_VERSION)
 	execute_process(COMMAND ${ALSATPLG_CMD} --version
 	  RESULT_VARIABLE status
@@ -69,7 +79,7 @@ macro(add_alsatplg_command)
 		# the -o(utput) file.
 		# See bug https://github.com/alsa-project/alsa-utils/issues/126
 		COMMAND ${ALSATPLG_CMD} \$\${VERBOSE:+-v 1} -c ${ARGV0} -o ${ARGV1}
-		USES_TERMINAL
+		${TPLG_USES_TERMINAL}
 	)
 endmacro()
 
@@ -97,7 +107,7 @@ macro(add_alsatplg2_command conf_header conf_target input_name output_name inclu
 		# -p to pre-process Topology2.0 conf file
 		COMMAND ALSA_CONFIG_DIR=${CMAKE_SOURCE_DIR}/topology/topology2 ${ALSATPLG_CMD} \$\${VERBOSE:+-v 1}
 		        -I ${include_path} -D "'${defines}'" -p -c ${output_name}.conf -o ${output_name}.tplg
-		USES_TERMINAL
+		${TPLG_USES_TERMINAL}
 		)
 endmacro()