diff --git a/config/cortexa725/bli_cntx_init_cortexa725.c b/config/cortexa725/bli_cntx_init_cortexa725.c new file mode 100644 index 000000000..bfe84e4f2 --- /dev/null +++ b/config/cortexa725/bli_cntx_init_cortexa725.c @@ -0,0 +1,150 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_cntx_init_cortexa725( cntx_t* cntx ) +{ + blksz_t blkszs[ BLIS_NUM_BLKSZS ]; + + // Set default kernel blocksizes and functions. + bli_cntx_init_cortexa725_ref( cntx ); + + // ------------------------------------------------------------------------- + + // Update the context with optimized native gemm micro-kernels. + bli_cntx_set_ukrs + ( + cntx, + + // level-3 native + BLIS_GEMM_UKR, BLIS_FLOAT, bli_sgemm_armv8a_asm_8x12, + BLIS_GEMM_UKR, BLIS_DOUBLE, bli_dgemm_armv8a_asm_6x8, + + // packm (optimized NEON pack kernels matching MR_s=8 / MR_d=6) + BLIS_PACKM_KER, BLIS_FLOAT, bli_spackm_armv8a_int_8x12, + BLIS_PACKM_KER, BLIS_DOUBLE, bli_dpackm_armv8a_int_6x8, + + // gemmsup (skinny/unpacked) -- double only; per source-storage combo. + // rv = row-vector (axpy) kernels; rd = row-dot kernels. Suffix m/n picks + // the loop-partition variant. Matches the armv8a 6x8 SUP register tile. + BLIS_GEMMSUP_RRR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8m, + BLIS_GEMMSUP_RRC_UKR, BLIS_DOUBLE, bli_dgemmsup_rd_armv8a_asm_6x8m, + BLIS_GEMMSUP_RCR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8m, + BLIS_GEMMSUP_RCC_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8n, + BLIS_GEMMSUP_CRR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8m, + BLIS_GEMMSUP_CRC_UKR, BLIS_DOUBLE, bli_dgemmsup_rd_armv8a_asm_6x8n, + BLIS_GEMMSUP_CCR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8n, + BLIS_GEMMSUP_CCC_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8n, + + BLIS_VA_END + ); + + // Update the context with storage preferences. + bli_cntx_set_ukr_prefs + ( + cntx, + + // level-3 native + BLIS_GEMM_UKR_ROW_PREF, BLIS_FLOAT, FALSE, + BLIS_GEMM_UKR_ROW_PREF, BLIS_DOUBLE, FALSE, + + // gemmsup + BLIS_GEMMSUP_RRR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_RRC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_RCR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_RCC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CRR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CRC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CCR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CCC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + + BLIS_VA_END + ); + + // Initialize level-3 blocksize objects with architecture-specific values. + // s d c z + bli_blksz_init_easy( &blkszs[ BLIS_MR ], 8, 6, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NR ], 12, 8, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 96, 96, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 512, 384, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 3072, 3072, -1, -1 ); + + // SUP thresholds: use the skinny/unpacked path when a dimension is at or + // below the threshold (tuned below). s uses the native path (no s SUP kernel). + // s d c z + bli_blksz_init_easy( &blkszs[ BLIS_MT ], -1, 256, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NT ], -1, 256, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_KT ], -1, 256, -1, -1 ); + + // Level-3 SUP register/cache blocksizes (double). MR_SUP=6, NR_SUP=8 match + // the 6x8 SUP micro-kernels; the second row is the packing multiple. + // s d c z + bli_blksz_init ( &blkszs[ BLIS_MR_SUP ], -1, 6, -1, -1, + -1, 9, -1, -1 ); + bli_blksz_init ( &blkszs[ BLIS_NR_SUP ], -1, 8, -1, -1, + -1, 13, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_MC_SUP ], -1, 240, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC_SUP ], -1, 1024, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC_SUP ], -1, 3072, -1, -1 ); + + // Update the context with the current architecture's register and cache + // blocksizes (and multiples) for native execution. + bli_cntx_set_blkszs + ( + cntx, + + // level-3 native + BLIS_NC, &blkszs[ BLIS_NC ], BLIS_NR, + BLIS_KC, &blkszs[ BLIS_KC ], BLIS_KR, + BLIS_MC, &blkszs[ BLIS_MC ], BLIS_MR, + BLIS_NR, &blkszs[ BLIS_NR ], BLIS_NR, + BLIS_MR, &blkszs[ BLIS_MR ], BLIS_MR, + + // SUP thresholds + BLIS_MT, &blkszs[ BLIS_MT ], BLIS_MT, + BLIS_NT, &blkszs[ BLIS_NT ], BLIS_NT, + BLIS_KT, &blkszs[ BLIS_KT ], BLIS_KT, + + // level-3 SUP + BLIS_NC_SUP, &blkszs[ BLIS_NC_SUP ], BLIS_NR_SUP, + BLIS_KC_SUP, &blkszs[ BLIS_KC_SUP ], BLIS_KR_SUP, + BLIS_MC_SUP, &blkszs[ BLIS_MC_SUP ], BLIS_MR_SUP, + BLIS_NR_SUP, &blkszs[ BLIS_NR_SUP ], BLIS_NR_SUP, + BLIS_MR_SUP, &blkszs[ BLIS_MR_SUP ], BLIS_MR_SUP, + + BLIS_VA_END + ); +} + diff --git a/config/cortexa725/bli_family_cortexa725.h b/config/cortexa725/bli_family_cortexa725.h new file mode 100644 index 000000000..4a60ed2f2 --- /dev/null +++ b/config/cortexa725/bli_family_cortexa725.h @@ -0,0 +1,76 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +//#ifndef BLIS_FAMILY_H +//#define BLIS_FAMILY_H + + +// -- MEMORY ALLOCATION -------------------------------------------------------- + +#define BLIS_SIMD_ALIGN_SIZE 16 + + +#if 0 +// -- LEVEL-3 MICRO-KERNEL CONSTANTS ------------------------------------------- + +#define BLIS_SGEMM_UKERNEL bli_sgemm_opt_8x12 +#define BLIS_DEFAULT_MR_S 8 +#define BLIS_DEFAULT_NR_S 12 +#define BLIS_DEFAULT_MC_S 120 //1536 //336 //416 // 1280 //160 // 160 // 160 //2048 //336 +#define BLIS_DEFAULT_KC_S 640 //1536 //336 //704 //1280 //672 //528 // 856 //2048 //528 +#define BLIS_DEFAULT_NC_S 3072 + +#define BLIS_DGEMM_UKERNEL bli_dgemm_opt_6x8 +#define BLIS_DEFAULT_MR_D 6 +#define BLIS_DEFAULT_NR_D 8 +#define BLIS_DEFAULT_MC_D 120 //1536 //160 //80 //176 +#define BLIS_DEFAULT_KC_D 240 //1536 //304 //336 //368 +#define BLIS_DEFAULT_NC_D 3072 + +#define BLIS_DEFAULT_MR_C 8 +#define BLIS_DEFAULT_NR_C 4 +#define BLIS_DEFAULT_MC_C 64 +#define BLIS_DEFAULT_KC_C 128 +#define BLIS_DEFAULT_NC_C 4096 + +#define BLIS_DEFAULT_MR_Z 8 +#define BLIS_DEFAULT_NR_Z 4 +#define BLIS_DEFAULT_MC_Z 64 +#define BLIS_DEFAULT_KC_Z 128 +#define BLIS_DEFAULT_NC_Z 4096 +#endif + + +//#endif + diff --git a/config/cortexa725/bli_kernel_defs_cortexa725.h b/config/cortexa725/bli_kernel_defs_cortexa725.h new file mode 100644 index 000000000..60292099c --- /dev/null +++ b/config/cortexa725/bli_kernel_defs_cortexa725.h @@ -0,0 +1,48 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2022, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +//#ifndef BLIS_KERNEL_DEFS_H +//#define BLIS_KERNEL_DEFS_H + + +// -- REGISTER BLOCK SIZES (FOR REFERENCE KERNELS) ---------------------------- + +#define BLIS_MR_s 8 +#define BLIS_MR_d 6 + +#define BLIS_NR_s 12 +#define BLIS_NR_d 8 + +//#endif + diff --git a/config/cortexa725/make_defs.mk b/config/cortexa725/make_defs.mk new file mode 100644 index 000000000..28ce3852e --- /dev/null +++ b/config/cortexa725/make_defs.mk @@ -0,0 +1,93 @@ +# +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name(s) of the copyright holder(s) nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + + +# Declare the name of the current configuration and add it to the +# running list of configurations included by common.mk. +THIS_CONFIG := cortexa725 +#CONFIGS_INCL += $(THIS_CONFIG) + +# +# --- Determine the C compiler and related flags --- +# + +# NOTE: The build system will append these variables with various +# general-purpose/configuration-agnostic flags in common.mk. You +# may specify additional flags here as needed. +CPPROCFLAGS := -D_GNU_SOURCE +CMISCFLAGS := +CPICFLAGS := -fPIC +CWARNFLAGS := + +ifneq ($(DEBUG_TYPE),off) +CDBGFLAGS := -g +endif + +ifeq ($(DEBUG_TYPE),noopt) +COPTFLAGS := -O0 +else +COPTFLAGS := -O3 -mcpu=cortex-a725 +endif + +# Flags specific to optimized kernels. +# A725 = 10x Cortex-X925 + 10x Cortex-A725 (Armv9.2-A, NEON 128b, SVE2 VL=128b). +# GCC 16 recognizes -mcpu=cortex-a725 (== -mcpu=native here). NEON asm micro-kernels are +# used because SVE offers no width advantage at VL=128b. +CKOPTFLAGS := $(COPTFLAGS) -O3 -ftree-vectorize +ifeq ($(CC_VENDOR),gcc) +CKVECFLAGS := -mcpu=cortex-a725 +else +ifeq ($(CC_VENDOR),clang) +CKVECFLAGS := -mcpu=cortex-a725 +else +$(error gcc or clang is required for this configuration.) +endif +endif + +# Flags specific to reference kernels. +CROPTFLAGS := $(CKOPTFLAGS) +ifeq ($(CC_VENDOR),gcc) +CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast +else +ifeq ($(CC_VENDOR),clang) +CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast +else +CRVECFLAGS := $(CKVECFLAGS) +endif +endif + +# Store all of the variables here to new variables containing the +# configuration name. +$(eval $(call store-make-defs,$(THIS_CONFIG))) + diff --git a/config/cortexx925/bli_cntx_init_cortexx925.c b/config/cortexx925/bli_cntx_init_cortexx925.c new file mode 100644 index 000000000..551ea5ad8 --- /dev/null +++ b/config/cortexx925/bli_cntx_init_cortexx925.c @@ -0,0 +1,150 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_cntx_init_cortexx925( cntx_t* cntx ) +{ + blksz_t blkszs[ BLIS_NUM_BLKSZS ]; + + // Set default kernel blocksizes and functions. + bli_cntx_init_cortexx925_ref( cntx ); + + // ------------------------------------------------------------------------- + + // Update the context with optimized native gemm micro-kernels. + bli_cntx_set_ukrs + ( + cntx, + + // level-3 native + BLIS_GEMM_UKR, BLIS_FLOAT, bli_sgemm_armv8a_asm_8x12, + BLIS_GEMM_UKR, BLIS_DOUBLE, bli_dgemm_armv8a_asm_6x8, + + // packm (optimized NEON pack kernels matching MR_s=8 / MR_d=6) + BLIS_PACKM_KER, BLIS_FLOAT, bli_spackm_armv8a_int_8x12, + BLIS_PACKM_KER, BLIS_DOUBLE, bli_dpackm_armv8a_int_6x8, + + // gemmsup (skinny/unpacked) -- double only; per source-storage combo. + // rv = row-vector (axpy) kernels; rd = row-dot kernels. Suffix m/n picks + // the loop-partition variant. Matches the armv8a 6x8 SUP register tile. + BLIS_GEMMSUP_RRR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8m, + BLIS_GEMMSUP_RRC_UKR, BLIS_DOUBLE, bli_dgemmsup_rd_armv8a_asm_6x8m, + BLIS_GEMMSUP_RCR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8m, + BLIS_GEMMSUP_RCC_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8n, + BLIS_GEMMSUP_CRR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8m, + BLIS_GEMMSUP_CRC_UKR, BLIS_DOUBLE, bli_dgemmsup_rd_armv8a_asm_6x8n, + BLIS_GEMMSUP_CCR_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8n, + BLIS_GEMMSUP_CCC_UKR, BLIS_DOUBLE, bli_dgemmsup_rv_armv8a_asm_6x8n, + + BLIS_VA_END + ); + + // Update the context with storage preferences. + bli_cntx_set_ukr_prefs + ( + cntx, + + // level-3 native + BLIS_GEMM_UKR_ROW_PREF, BLIS_FLOAT, FALSE, + BLIS_GEMM_UKR_ROW_PREF, BLIS_DOUBLE, FALSE, + + // gemmsup + BLIS_GEMMSUP_RRR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_RRC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_RCR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_RCC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CRR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CRC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CCR_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + BLIS_GEMMSUP_CCC_UKR_ROW_PREF, BLIS_DOUBLE, TRUE, + + BLIS_VA_END + ); + + // Initialize level-3 blocksize objects with architecture-specific values. + // s d c z + bli_blksz_init_easy( &blkszs[ BLIS_MR ], 8, 6, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NR ], 12, 8, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 120, 336, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 640, 336, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 3072, 4080, -1, -1 ); + + // SUP thresholds: use the skinny/unpacked path when a dimension is at or + // below the threshold (tuned below). s uses the native path (no s SUP kernel). + // s d c z + bli_blksz_init_easy( &blkszs[ BLIS_MT ], -1, 256, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NT ], -1, 256, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_KT ], -1, 256, -1, -1 ); + + // Level-3 SUP register/cache blocksizes (double). MR_SUP=6, NR_SUP=8 match + // the 6x8 SUP micro-kernels; the second row is the packing multiple. + // s d c z + bli_blksz_init ( &blkszs[ BLIS_MR_SUP ], -1, 6, -1, -1, + -1, 9, -1, -1 ); + bli_blksz_init ( &blkszs[ BLIS_NR_SUP ], -1, 8, -1, -1, + -1, 13, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_MC_SUP ], -1, 240, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC_SUP ], -1, 1024, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC_SUP ], -1, 3072, -1, -1 ); + + // Update the context with the current architecture's register and cache + // blocksizes (and multiples) for native execution. + bli_cntx_set_blkszs + ( + cntx, + + // level-3 native + BLIS_NC, &blkszs[ BLIS_NC ], BLIS_NR, + BLIS_KC, &blkszs[ BLIS_KC ], BLIS_KR, + BLIS_MC, &blkszs[ BLIS_MC ], BLIS_MR, + BLIS_NR, &blkszs[ BLIS_NR ], BLIS_NR, + BLIS_MR, &blkszs[ BLIS_MR ], BLIS_MR, + + // SUP thresholds + BLIS_MT, &blkszs[ BLIS_MT ], BLIS_MT, + BLIS_NT, &blkszs[ BLIS_NT ], BLIS_NT, + BLIS_KT, &blkszs[ BLIS_KT ], BLIS_KT, + + // level-3 SUP + BLIS_NC_SUP, &blkszs[ BLIS_NC_SUP ], BLIS_NR_SUP, + BLIS_KC_SUP, &blkszs[ BLIS_KC_SUP ], BLIS_KR_SUP, + BLIS_MC_SUP, &blkszs[ BLIS_MC_SUP ], BLIS_MR_SUP, + BLIS_NR_SUP, &blkszs[ BLIS_NR_SUP ], BLIS_NR_SUP, + BLIS_MR_SUP, &blkszs[ BLIS_MR_SUP ], BLIS_MR_SUP, + + BLIS_VA_END + ); +} + diff --git a/config/cortexx925/bli_family_cortexx925.h b/config/cortexx925/bli_family_cortexx925.h new file mode 100644 index 000000000..4a60ed2f2 --- /dev/null +++ b/config/cortexx925/bli_family_cortexx925.h @@ -0,0 +1,76 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +//#ifndef BLIS_FAMILY_H +//#define BLIS_FAMILY_H + + +// -- MEMORY ALLOCATION -------------------------------------------------------- + +#define BLIS_SIMD_ALIGN_SIZE 16 + + +#if 0 +// -- LEVEL-3 MICRO-KERNEL CONSTANTS ------------------------------------------- + +#define BLIS_SGEMM_UKERNEL bli_sgemm_opt_8x12 +#define BLIS_DEFAULT_MR_S 8 +#define BLIS_DEFAULT_NR_S 12 +#define BLIS_DEFAULT_MC_S 120 //1536 //336 //416 // 1280 //160 // 160 // 160 //2048 //336 +#define BLIS_DEFAULT_KC_S 640 //1536 //336 //704 //1280 //672 //528 // 856 //2048 //528 +#define BLIS_DEFAULT_NC_S 3072 + +#define BLIS_DGEMM_UKERNEL bli_dgemm_opt_6x8 +#define BLIS_DEFAULT_MR_D 6 +#define BLIS_DEFAULT_NR_D 8 +#define BLIS_DEFAULT_MC_D 120 //1536 //160 //80 //176 +#define BLIS_DEFAULT_KC_D 240 //1536 //304 //336 //368 +#define BLIS_DEFAULT_NC_D 3072 + +#define BLIS_DEFAULT_MR_C 8 +#define BLIS_DEFAULT_NR_C 4 +#define BLIS_DEFAULT_MC_C 64 +#define BLIS_DEFAULT_KC_C 128 +#define BLIS_DEFAULT_NC_C 4096 + +#define BLIS_DEFAULT_MR_Z 8 +#define BLIS_DEFAULT_NR_Z 4 +#define BLIS_DEFAULT_MC_Z 64 +#define BLIS_DEFAULT_KC_Z 128 +#define BLIS_DEFAULT_NC_Z 4096 +#endif + + +//#endif + diff --git a/config/cortexx925/bli_kernel_defs_cortexx925.h b/config/cortexx925/bli_kernel_defs_cortexx925.h new file mode 100644 index 000000000..60292099c --- /dev/null +++ b/config/cortexx925/bli_kernel_defs_cortexx925.h @@ -0,0 +1,48 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2022, The University of Texas at Austin + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +//#ifndef BLIS_KERNEL_DEFS_H +//#define BLIS_KERNEL_DEFS_H + + +// -- REGISTER BLOCK SIZES (FOR REFERENCE KERNELS) ---------------------------- + +#define BLIS_MR_s 8 +#define BLIS_MR_d 6 + +#define BLIS_NR_s 12 +#define BLIS_NR_d 8 + +//#endif + diff --git a/config/cortexx925/make_defs.mk b/config/cortexx925/make_defs.mk new file mode 100644 index 000000000..c5267ac05 --- /dev/null +++ b/config/cortexx925/make_defs.mk @@ -0,0 +1,93 @@ +# +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name(s) of the copyright holder(s) nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + + +# Declare the name of the current configuration and add it to the +# running list of configurations included by common.mk. +THIS_CONFIG := cortexx925 +#CONFIGS_INCL += $(THIS_CONFIG) + +# +# --- Determine the C compiler and related flags --- +# + +# NOTE: The build system will append these variables with various +# general-purpose/configuration-agnostic flags in common.mk. You +# may specify additional flags here as needed. +CPPROCFLAGS := -D_GNU_SOURCE +CMISCFLAGS := +CPICFLAGS := -fPIC +CWARNFLAGS := + +ifneq ($(DEBUG_TYPE),off) +CDBGFLAGS := -g +endif + +ifeq ($(DEBUG_TYPE),noopt) +COPTFLAGS := -O0 +else +COPTFLAGS := -O3 -mcpu=cortex-x925 +endif + +# Flags specific to optimized kernels. +# GB10 = 10x Cortex-X925 + 10x Cortex-A725 (Armv9.2-A, NEON 128b, SVE2 VL=128b). +# GCC 16 recognizes -mcpu=cortex-x925 (== -mcpu=native here). NEON asm micro-kernels are +# used because SVE offers no width advantage at VL=128b. +CKOPTFLAGS := $(COPTFLAGS) -O3 -ftree-vectorize +ifeq ($(CC_VENDOR),gcc) +CKVECFLAGS := -mcpu=cortex-x925 +else +ifeq ($(CC_VENDOR),clang) +CKVECFLAGS := -mcpu=cortex-x925 +else +$(error gcc or clang is required for this configuration.) +endif +endif + +# Flags specific to reference kernels. +CROPTFLAGS := $(CKOPTFLAGS) +ifeq ($(CC_VENDOR),gcc) +CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast +else +ifeq ($(CC_VENDOR),clang) +CRVECFLAGS := $(CKVECFLAGS) -funsafe-math-optimizations -ffp-contract=fast +else +CRVECFLAGS := $(CKVECFLAGS) +endif +endif + +# Store all of the variables here to new variables containing the +# configuration name. +$(eval $(call store-make-defs,$(THIS_CONFIG))) + diff --git a/config_registry b/config_registry index 815439348..bac739315 100644 --- a/config_registry +++ b/config_registry @@ -12,7 +12,7 @@ x86_64: intel64 amd64 amd64_legacy intel64: skx knl haswell sandybridge penryn generic amd64_legacy: excavator steamroller piledriver bulldozer generic amd64: zen3 zen2 zen generic -arm64: armsve firestorm thunderx2 cortexa57 cortexa53 generic +arm64: cortexx925 armsve firestorm thunderx2 cortexa57 cortexa53 generic arm32: cortexa15 cortexa9 generic power: power10 power9 generic @@ -36,6 +36,12 @@ bulldozer: bulldozer armsve: armsve/armsve a64fx: a64fx/armsve +# NVIDIA GB10 Cortex-X925 (6 pipes x 128b); also used for GB10's A725 cores. +cortexx925: cortexx925/armv8a +# Cortex-A725 (2 pipes x 128b) standalone config; not auto-selected on GB10, +# where the cortexx925 code path is used for both core types. Build explicitly +# (./configure cortexa725) for non-GB10 A725 hardware. +cortexa725: cortexa725/armv8a # ARM Neon64 (4 pipes x 128b) architectures. altramax: altramax/armv8a altra: altra/armv8a diff --git a/configure b/configure index 8bb730945..07ab9134b 100755 --- a/configure +++ b/configure @@ -1805,6 +1805,9 @@ check_compiler() # # armsve: clang11+, gcc10+ # + # cortexx925: clang19+, gcc14+ + # cortexa725: clang19+, gcc14+ + # # generic: any # # Note: These compiler requirements were originally modeled after similar @@ -1860,6 +1863,11 @@ check_compiler() if [[ ${cc_major} -lt 10 ]]; then blacklistcc_add "armsve" fi + if [[ ${cc_major} -lt 14 ]]; then + # -mcpu=cortex-x925 and -mcpu=cortex-a725 require gcc 14 or newer. + blacklistcc_add "cortexx925" + blacklistcc_add "cortexa725" + fi fi # icc @@ -1900,6 +1908,9 @@ check_compiler() blacklistcc_add "knl" blacklistcc_add "skx" fi + # Apple clang does not yet target the Armv9.2 Cortex-X925/A725. + blacklistcc_add "cortexx925" + blacklistcc_add "cortexa725" else if [[ ${cc_major} -lt 3 ]]; then echoerr_unsupportedcc @@ -1925,6 +1936,11 @@ check_compiler() if [[ ${cc_major} -lt 11 ]]; then blacklistcc_add "armsve" fi + if [[ ${cc_major} -lt 19 ]]; then + # cortex-x925 and cortex-a725 require clang 19 or newer. + blacklistcc_add "cortexx925" + blacklistcc_add "cortexa725" + fi if [[ ${cc_major} -ge 19 ]]; then blacklistcc_add "knl" fi diff --git a/frame/base/bli_arch.c b/frame/base/bli_arch.c index 776bb698b..235912574 100644 --- a/frame/base/bli_arch.c +++ b/frame/base/bli_arch.c @@ -259,6 +259,12 @@ arch_t bli_arch_query_id_impl( void ) id = BLIS_ARCH_A64FX; #endif + #ifdef BLIS_FAMILY_CORTEXX925 + id = BLIS_ARCH_CORTEXX925; + #endif + #ifdef BLIS_FAMILY_CORTEXA725 + id = BLIS_ARCH_CORTEXA725; + #endif #ifdef BLIS_FAMILY_ALTRAMAX id = BLIS_ARCH_ALTRAMAX; #endif @@ -366,6 +372,8 @@ static const char* config_name[ BLIS_NUM_ARCHS ] = "armsve", "a64fx", + "cortexx925", + "cortexa725", "altramax", "altra", "firestorm", diff --git a/frame/base/bli_cpuid.c b/frame/base/bli_cpuid.c index 550b8ca69..396b508b0 100644 --- a/frame/base/bli_cpuid.c +++ b/frame/base/bli_cpuid.c @@ -1154,6 +1154,9 @@ static uint32_t get_coretype #define ARM_CPU_PART_CORTEX_V1 0xd40 #define ARM_CPU_PART_CORTEX_N2 0xd49 #define ARM_CPU_PART_CORTEX_R82 0xd15 + // NVIDIA GB10 (DGX Spark) clusters: +#define ARM_CPU_PART_CORTEX_X925 0xd85 +#define ARM_CPU_PART_CORTEX_A725 0xd87 // // APM_CPU_PART_POTENZA 0x000 // @@ -1201,6 +1204,22 @@ static uint32_t get_coretype case ARM_CPU_IMP_ARM: // ARM switch (part) { +#ifdef BLIS_CONFIG_CORTEXX925 + // NVIDIA GB10: use the cortexx925 config for BOTH the X925 + // performance cores and the A725 efficiency cores -- a single + // code path across the heterogeneous chip. cortexx925 runs + // within ~2% of a dedicated A725 config on the A725 cores across + // all kernels, so affinity-free heterogeneous execution wins + // over per-core config selection. + case ARM_CPU_PART_CORTEX_X925: + case ARM_CPU_PART_CORTEX_A725: + return BLIS_ARCH_CORTEXX925; +#elif defined(BLIS_CONFIG_CORTEXA725) + // Standalone Cortex-A725 build (non-GB10 A725 hardware): only + // reached when cortexx925 is not part of the build. + case ARM_CPU_PART_CORTEX_A725: + return BLIS_ARCH_CORTEXA725; +#endif #ifdef BLIS_CONFIG_CORTEXA57 case ARM_CPU_PART_CORTEX_A57: return BLIS_ARCH_CORTEXA57; diff --git a/frame/include/bli_gentconf_macro_defs.h b/frame/include/bli_gentconf_macro_defs.h index f6f3af20e..599071af0 100644 --- a/frame/include/bli_gentconf_macro_defs.h +++ b/frame/include/bli_gentconf_macro_defs.h @@ -128,6 +128,18 @@ #define INSERT_GENTCONF_A64FX #endif +// -- NVIDIA GB10 (Cortex-X925 6 pipes / Cortex-A725 2 pipes x 128-bit) -- +#ifdef BLIS_CONFIG_CORTEXX925 +#define INSERT_GENTCONF_CORTEXX925 GENTCONF( CORTEXX925, cortexx925 ) +#else +#define INSERT_GENTCONF_CORTEXX925 +#endif +#ifdef BLIS_CONFIG_CORTEXA725 +#define INSERT_GENTCONF_CORTEXA725 GENTCONF( CORTEXA725, cortexa725 ) +#else +#define INSERT_GENTCONF_CORTEXA725 +#endif + // -- ARM-NEON (4 pipes x 128-bit vectors) -- #ifdef BLIS_CONFIG_ALTRA #define INSERT_GENTCONF_ALTRA GENTCONF( ALTRA, altra ) @@ -264,6 +276,8 @@ INSERT_GENTCONF_BULLDOZER \ INSERT_GENTCONF_ARMSVE \ INSERT_GENTCONF_A64FX \ \ +INSERT_GENTCONF_CORTEXX925 \ +INSERT_GENTCONF_CORTEXA725 \ INSERT_GENTCONF_ALTRAMAX \ INSERT_GENTCONF_ALTRA \ INSERT_GENTCONF_FIRESTORM \ diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 758f9eb30..bf47f9245 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -980,6 +980,9 @@ typedef enum arch_e BLIS_ARCH_ARMSVE, BLIS_ARCH_A64FX, + // NVIDIA GB10 (Cortex-X925 6 pipes / Cortex-A725 2 pipes x 128-bit) + BLIS_ARCH_CORTEXX925, + BLIS_ARCH_CORTEXA725, // ARM-NEON (4 pipes x 128-bit vectors) BLIS_ARCH_ALTRAMAX, BLIS_ARCH_ALTRA,