diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 423c310a9827a..e0cf872174ee6 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -366,6 +366,21 @@ if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_ endif() include(detect_onnxruntime_target_platform.cmake) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + if(NOT WIN32 OR NOT onnxruntime_target_platform STREQUAL "x64") + message(FATAL_ERROR "Building with clang-cl is supported only for Windows x64 targets " + "(target platform: '${onnxruntime_target_platform}').") + endif() + # The Visual Studio ClangCL toolset defaults to lld-link. Link with MSVC link.exe instead, which binary analysis + # tooling such as BinSkim supports. + if(NOT DEFINED CMAKE_LINKER_TYPE) + if(CMAKE_VERSION VERSION_LESS 3.29) + message(FATAL_ERROR "Building with clang-cl requires CMake 3.29 or newer to select the MSVC linker.") + endif() + set(CMAKE_LINKER_TYPE MSVC) + endif() +endif() include(onnxruntime_msvc_runtime.cmake) # OneCore builds link the Windows umbrella libraries (onecoreuap.lib, onecoreuap_apiset.lib) instead of CMake's default diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 19cacb56b8aed..23ce0ac82467e 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -190,6 +190,86 @@ endif() set(ONNXRUNTIME_MLAS_LIBS onnxruntime_mlas) +# +# x86-64 C++ kernel sources, grouped by the instruction set they are compiled for. MLAS selects these kernels at +# runtime via CPUID, so a group's ISA is enabled only for its own translation units. The Windows and non-MSVC x86-64 +# paths both select sources from these groups and apply their compiler's flags to them; assembly sources and +# compiler-specific exceptions stay with each path. Add new x86-64 C++ kernels to the matching group. +# +set(mlas_x64_cxx_srcs_sse2 + ${MLAS_SRC_DIR}/qgemm_kernel_sse.cpp +) +# Built without ISA flags. The scalar 2-bit packers in sqnbitgemm_kernel_avx512_2bit.cpp are also used by the AVX2 +# dispatch tables, so enabling AVX-512 would let the compiler auto-vectorize them into instructions AVX2-only CPUs +# lack. If the AVX-512 W2 pack becomes a hot spot, move an optimized variant into a separate AVX-512-only file. +set(mlas_x64_cxx_srcs_baseline + ${MLAS_SRC_DIR}/qgemm_kernel_avx2.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.cpp +) +set(mlas_x64_cxx_srcs_avx + ${MLAS_SRC_DIR}/intrinsics/avx/min_max_elements.cpp +) +set(mlas_x64_cxx_srcs_avx2 + ${MLAS_SRC_DIR}/intrinsics/avx2/qladd_avx2.cpp + ${MLAS_SRC_DIR}/intrinsics/avx2/qdwconv_avx2.cpp + ${MLAS_SRC_DIR}/intrinsics/avx2/saturation_check_avx2.cpp + ${MLAS_SRC_DIR}/intrinsics/avx2/q2_dq_avx2.cpp + ${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp + ${MLAS_SRC_DIR}/qkv_quant_kernel_avx2.cpp + ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.cpp +) +# AVX-VNNI kernels live in their own translation unit so the AVX2 kernels above cannot be auto-vectorized into +# AVX-VNNI instructions. +set(mlas_x64_cxx_srcs_avx2vnni + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2vnni.cpp +) +set(mlas_x64_cxx_srcs_avx512f + ${MLAS_SRC_DIR}/intrinsics/avx512/gelu_avx512f.cpp + ${MLAS_SRC_DIR}/intrinsics/avx512/silu_avx512f.cpp + ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp + ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_nchw_depthwise_multiplier_greater_than_1_avx512f.cpp + ${MLAS_SRC_DIR}/linear_attention_kernel_avx512f.cpp + ${MLAS_SRC_DIR}/intrinsics/avx512/reorder_avx512f.cpp + ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_depthwise_avx512f.cpp +) +set(mlas_x64_cxx_srcs_avx512core + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512.cpp +) +set(mlas_x64_cxx_srcs_avx512vnni + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni.cpp + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni_2bit.cpp + ${MLAS_SRC_DIR}/qkv_quant_kernel_avx512vnni.cpp +) +set(mlas_x64_cxx_srcs_q4gemm_avx512) +if (NOT onnxruntime_ORT_MINIMAL_BUILD) + set(mlas_x64_cxx_srcs_q4gemm_avx512 ${MLAS_SRC_DIR}/q4gemm_avx512.cpp) +endif() +set(mlas_x64_cxx_srcs_amx + ${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp +) +set(mlas_x64_cxx_hdrs + ${MLAS_SRC_DIR}/qkv_quant_common.h + ${MLAS_SRC_DIR}/qkv_quant_kernel.h + ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.h + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.h + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen32.h + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen64.h + ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen128.h + ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.h +) + +# GNU-style target-feature flags for the groups above, used by GCC/Clang and by clang-cl. +set(mlas_x64_flags_avx "-mavx") +set(mlas_x64_flags_avx2 "-mavx2 -mfma -mf16c") +set(mlas_x64_flags_avx2vnni "${mlas_x64_flags_avx2} -mavxvnni") +set(mlas_x64_flags_avx512f "-mavx512f") +set(mlas_x64_flags_avx512core "-mfma -mavx512vnni -mavx512bw -mavx512dq -mavx512vl") +set(mlas_x64_flags_avx512vnni "-mfma -mf16c -mavx512vnni -mavx512bw -mavx512dq -mavx512vl -mavx512f") +set(mlas_x64_flags_q4gemm_avx512 "-mfma -mavx512vnni -mavx512bw -mavx512dq -mavx512vl -mavx512f") +set(mlas_x64_flags_amx "-mavx2 -mavx512bw -mavx512dq -mavx512vl -mavx512f") + #TODO: set MASM flags properly function(setup_mlas_source_for_windows) @@ -342,59 +422,20 @@ function(setup_mlas_source_for_windows) ) elseif(onnxruntime_target_platform STREQUAL "x64") - file(GLOB_RECURSE mlas_platform_srcs_avx CONFIGURE_DEPENDS - "${MLAS_SRC_DIR}/intrinsics/avx/*.cpp" - ) - set_source_files_properties(${mlas_platform_srcs_avx} PROPERTIES COMPILE_FLAGS "/arch:AVX") - - file(GLOB_RECURSE mlas_platform_srcs_avx2 CONFIGURE_DEPENDS - "${MLAS_SRC_DIR}/intrinsics/avx2/*.cpp" - ) - set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "/arch:AVX2") - - set_source_files_properties(${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX2") - - set(mlas_platform_srcs_avx512 - ${MLAS_SRC_DIR}/intrinsics/avx512/gelu_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/silu_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_nchw_depthwise_multiplier_greater_than_1_avx512f.cpp - ${MLAS_SRC_DIR}/linear_attention_kernel_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/reorder_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_depthwise_avx512f.cpp - ) - - set_source_files_properties(${mlas_platform_srcs_avx512} PROPERTIES COMPILE_FLAGS "/arch:AVX512") - target_sources(onnxruntime_mlas PRIVATE ${MLAS_SRC_DIR}/dgemm.cpp - ${mlas_platform_srcs_avx} - ${mlas_platform_srcs_avx2} - ${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp - ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.h - ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp - ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp - ${MLAS_SRC_DIR}/qkv_quant_kernel.h - ${MLAS_SRC_DIR}/qkv_quant_common.h - ${MLAS_SRC_DIR}/qkv_quant_kernel_avx2.cpp - ${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp - ${MLAS_SRC_DIR}/qgemm_kernel_avx2.cpp - ${MLAS_SRC_DIR}/qgemm_kernel_sse.cpp + ${mlas_x64_cxx_srcs_sse2} ${MLAS_SRC_DIR}/qgemm_kernel_sse41.cpp - ${mlas_platform_srcs_avx512} - ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.h - ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2vnni.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.h - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen64.h - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen128.h - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen32.h - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni_2bit.cpp - ${MLAS_SRC_DIR}/qkv_quant_kernel_avx512vnni.cpp + ${mlas_x64_cxx_srcs_baseline} + ${mlas_x64_cxx_srcs_avx} + ${mlas_x64_cxx_srcs_avx2} + ${mlas_x64_cxx_srcs_avx2vnni} + ${mlas_x64_cxx_srcs_avx512f} + ${mlas_x64_cxx_srcs_avx512core} + ${mlas_x64_cxx_srcs_avx512vnni} + ${mlas_x64_cxx_srcs_q4gemm_avx512} + ${mlas_x64_cxx_srcs_amx} + ${mlas_x64_cxx_hdrs} ${MLAS_SRC_DIR}/amd64/QgemmU8S8KernelAmx.asm ${MLAS_SRC_DIR}/amd64/QgemmU8S8KernelAvx2.asm ${MLAS_SRC_DIR}/amd64/QgemmU8U8KernelAvx2.asm @@ -433,6 +474,41 @@ function(setup_mlas_source_for_windows) ${MLAS_SRC_DIR}/amd64/ErfKernelFma3.asm ) + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + # clang-cl, unlike cl.exe, only emits an intrinsic when its translation unit enables the matching target + # feature, so give each group the same flags as the non-MSVC x86-64 path. The exceptions are below. + set_source_files_properties(${mlas_x64_cxx_srcs_avx} PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx}") + set_source_files_properties(${mlas_x64_cxx_srcs_avx2} PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx2}") + set_source_files_properties(${mlas_x64_cxx_srcs_avx2vnni} PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx2vnni}") + set_source_files_properties(${mlas_x64_cxx_srcs_avx512f} PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx512f}") + set_source_files_properties(${mlas_x64_cxx_srcs_avx512vnni} PROPERTIES + COMPILE_FLAGS "${mlas_x64_flags_avx512vnni}") + if (mlas_x64_cxx_srcs_q4gemm_avx512) + set_source_files_properties(${mlas_x64_cxx_srcs_q4gemm_avx512} PROPERTIES + COMPILE_FLAGS "${mlas_x64_flags_q4gemm_avx512}") + endif() + + # sqnbitgemm_kernel_avx512.cpp backs MlasSQNBitGemmDispatchAvx512, which is selected on AVX-512 CPUs without + # VNNI, so it does not enable VNNI. + set_source_files_properties(${mlas_x64_cxx_srcs_avx512core} PROPERTIES + COMPILE_FLAGS "-mfma -mavx512f -mavx512bw -mavx512dq -mavx512vl") + # On Windows the AMX kernel uses the AMX tile intrinsics rather than inline assembly. + set_source_files_properties(${mlas_x64_cxx_srcs_amx} PROPERTIES + COMPILE_FLAGS "${mlas_x64_flags_amx} -mfma -mamx-tile -mamx-int8") + # qgemm_kernel_sse41.cpp is only built on Windows and uses SSSE3/SSE4.1 intrinsics. + set_source_files_properties(${MLAS_SRC_DIR}/qgemm_kernel_sse41.cpp PROPERTIES COMPILE_FLAGS "-msse4.1") + # Keep the 2-bit fp-zero-point dequant kernel bit-identical to the scalar kernel, as on the non-MSVC path. + set_property(SOURCE ${MLAS_SRC_DIR}/intrinsics/avx2/q2_dq_avx2.cpp + APPEND PROPERTY COMPILE_OPTIONS "/clang:-ffp-contract=off") + else() + # cl.exe does not gate intrinsics on the target ISA; only these kernels are built with /arch. + set(mlas_msvc_arch_avx2_srcs ${mlas_x64_cxx_srcs_avx2}) + list(FILTER mlas_msvc_arch_avx2_srcs INCLUDE REGEX "/intrinsics/avx2/|/layernorm_kernel_avx2\\.cpp$") + set_source_files_properties(${mlas_x64_cxx_srcs_avx} PROPERTIES COMPILE_FLAGS "/arch:AVX") + set_source_files_properties(${mlas_msvc_arch_avx2_srcs} PROPERTIES COMPILE_FLAGS "/arch:AVX2") + set_source_files_properties(${mlas_x64_cxx_srcs_avx512f} PROPERTIES COMPILE_FLAGS "/arch:AVX512") + endif() + if(onnxruntime_ENABLE_CONVSYMKERNELAVX2_SAT_CHECKER) set_source_files_properties(${MLAS_SRC_DIR}/amd64/ConvSymKernelAvx2.asm PROPERTIES COMPILE_FLAGS "-DENABLE_CONVSYMKERNELAVX2_SAT_CHECKER") endif() @@ -443,11 +519,6 @@ function(setup_mlas_source_for_windows) ) endif() - if (NOT onnxruntime_ORT_MINIMAL_BUILD) - target_sources(onnxruntime_mlas PRIVATE - ${MLAS_SRC_DIR}/q4gemm_avx512.cpp - ) - endif() else() set_source_files_properties(${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX2") target_sources(onnxruntime_mlas PRIVATE @@ -977,7 +1048,7 @@ else() # instruction set extension and explicitly set the compiler flag as appropriate. set(mlas_platform_srcs_sse2 - ${MLAS_SRC_DIR}/qgemm_kernel_sse.cpp + ${mlas_x64_cxx_srcs_sse2} ${MLAS_SRC_DIR}/x86_64/DgemmKernelSse2.S ${MLAS_SRC_DIR}/x86_64/SgemmKernelSse2.S ${MLAS_SRC_DIR}/x86_64/SgemmTransposePackB16x4Sse2.S @@ -1001,9 +1072,9 @@ else() ${MLAS_SRC_DIR}/x86_64/SconvKernelAvx.S ${MLAS_SRC_DIR}/x86_64/SpoolKernelAvx.S ${MLAS_SRC_DIR}/x86_64/SoftmaxKernelAvx.S - ${MLAS_SRC_DIR}/intrinsics/avx/min_max_elements.cpp + ${mlas_x64_cxx_srcs_avx} ) - set_source_files_properties(${mlas_platform_srcs_avx} PROPERTIES COMPILE_FLAGS "-mavx") + set_source_files_properties(${mlas_platform_srcs_avx} PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx}") set(mlas_platform_srcs_avx2 ${MLAS_SRC_DIR}/x86_64/QgemmU8S8KernelAvx2.S @@ -1019,20 +1090,7 @@ else() ${MLAS_SRC_DIR}/x86_64/LogisticKernelFma3.S ${MLAS_SRC_DIR}/x86_64/TanhKernelFma3.S ${MLAS_SRC_DIR}/x86_64/ErfKernelFma3.S - ${MLAS_SRC_DIR}/intrinsics/avx2/qladd_avx2.cpp - ${MLAS_SRC_DIR}/intrinsics/avx2/qdwconv_avx2.cpp - ${MLAS_SRC_DIR}/intrinsics/avx2/saturation_check_avx2.cpp - ${MLAS_SRC_DIR}/intrinsics/avx2/q2_dq_avx2.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.h - ${MLAS_SRC_DIR}/sqnbitgemm_lut_kernel_avx2.cpp - ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.h - ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp - ${MLAS_SRC_DIR}/rotary_embedding_kernel_avx2.cpp - ${MLAS_SRC_DIR}/qkv_quant_kernel.h - ${MLAS_SRC_DIR}/qkv_quant_common.h - ${MLAS_SRC_DIR}/qkv_quant_kernel_avx2.cpp - ${MLAS_SRC_DIR}/layernorm_kernel_avx2.cpp + ${mlas_x64_cxx_srcs_avx2} ) if(CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL 13.1 AND NOT(APPLE)) set(mlas_platform_srcs_avx2 @@ -1043,7 +1101,7 @@ else() include(CheckCXXSourceCompiles) set(OLD_CMAKE_REQUIRED_FLAGS_AVXVNNI "${CMAKE_REQUIRED_FLAGS}") - set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS_AVXVNNI} -mavx2 -mfma -mf16c -mavxvnni") + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS_AVXVNNI} ${mlas_x64_flags_avx2vnni}") check_cxx_source_compiles(" #include int main() { @@ -1061,7 +1119,7 @@ else() # sqnbitgemm_kernel_avx2.cpp is now in this set and compiled without # -mavxvnni, preventing the auto-vectorizer from emitting AVX-VNNI # instructions in the pure-AVX2 kernels. - set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "-mavx2 -mfma -mf16c") + set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx2}") # sqnbitgemm_kernel_avx2vnni.cpp is always compiled so that # MlasSQNBitGemmDispatchAvx2vnni (referenced unconditionally in @@ -1070,22 +1128,22 @@ else() # and the VNNI intrinsic paths compile out via the #if guards, leaving # only the AVX2 fallback kernels in the dispatch table. set(mlas_platform_srcs_avx2vnni - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2vnni.cpp + ${mlas_x64_cxx_srcs_avx2vnni} ) if(MLAS_COMPILER_SUPPORTS_AVXVNNI) message(STATUS "AVX-VNNI supported: compiling sqnbitgemm_kernel_avx2vnni.cpp with -mavx2 -mfma -mf16c -mavxvnni") # AVX-VNNI kernels live in a dedicated TU compiled with -mavxvnni. # Keeping them separate prevents the auto-vectorizer from emitting # VNNI instructions in the pure-AVX2 kernels in sqnbitgemm_kernel_avx2.cpp. - set_source_files_properties(${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2vnni.cpp - PROPERTIES COMPILE_FLAGS "-mavx2 -mfma -mf16c -mavxvnni") + set_source_files_properties(${mlas_platform_srcs_avx2vnni} + PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx2vnni}") else() message(STATUS "AVX-VNNI not supported by compiler/assembler: compiling sqnbitgemm_kernel_avx2vnni.cpp with -mavx2 -mfma -mf16c only (VNNI paths compiled out)") # Still compile the TU so MlasSQNBitGemmDispatchAvx2vnni is defined. # __AVXVNNI__ is absent, so all VNNI intrinsic blocks are #if-excluded # and the dispatch table falls back to pure-AVX2 kernels. - set_source_files_properties(${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx2vnni.cpp - PROPERTIES COMPILE_FLAGS "-mavx2 -mfma -mf16c") + set_source_files_properties(${mlas_platform_srcs_avx2vnni} + PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx2}") endif() # The 2-bit fp-zero-point dequant kernel relies on separate multiply @@ -1100,55 +1158,34 @@ else() ${MLAS_SRC_DIR}/x86_64/SoftmaxKernelAvx512F.S ${MLAS_SRC_DIR}/x86_64/SpoolKernelAvx512F.S ${MLAS_SRC_DIR}/x86_64/TransKernelAvx512F.S - ${MLAS_SRC_DIR}/intrinsics/avx512/gelu_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/silu_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_nchw_depthwise_multiplier_greater_than_1_avx512f.cpp - ${MLAS_SRC_DIR}/linear_attention_kernel_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/reorder_avx512f.cpp - ${MLAS_SRC_DIR}/intrinsics/avx512/sconv_depthwise_avx512f.cpp + ${mlas_x64_cxx_srcs_avx512f} ) - set_source_files_properties(${mlas_platform_srcs_avx512f} PROPERTIES COMPILE_FLAGS "-mavx512f") + set_source_files_properties(${mlas_platform_srcs_avx512f} PROPERTIES COMPILE_FLAGS "${mlas_x64_flags_avx512f}") set(mlas_platform_srcs_avx512core ${MLAS_SRC_DIR}/x86_64/QgemvU8S8KernelAvx512Core.S ${MLAS_SRC_DIR}/x86_64/QgemvU8S8KernelAvx512Vnni.S ${MLAS_SRC_DIR}/x86_64/QgemmU8X8KernelAvx512Core.S ${MLAS_SRC_DIR}/x86_64/ConvSymKernelAvx512Core.S - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.h - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen64.h - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen128.h - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit_blklen32.h + ${mlas_x64_cxx_srcs_avx512core} ) - set_source_files_properties(${mlas_platform_srcs_avx512core} PROPERTIES COMPILE_FLAGS "-mfma -mavx512vnni -mavx512bw -mavx512dq -mavx512vl") - - # NOTE: this TU is intrinsic-free scalar helpers reached from both the - # AVX2 and AVX-512 W2 dispatch tables at model load. Flags must not - # enable any ISA the AVX2-only host lacks, or the compiler may - # autovectorize the scalar loops into EVEX instructions and SIGILL. - # If the AVX-512 W2 pack ever becomes a perf hot spot, split this - # file: keep the scalar pack in a flag-less TU and put an optimized - # variant in a separate one only used by the AVX-512 tables. - set_source_files_properties(${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512_2bit.cpp PROPERTIES - COMPILE_FLAGS "") + set_source_files_properties(${mlas_platform_srcs_avx512core} PROPERTIES + COMPILE_FLAGS "${mlas_x64_flags_avx512core}") set(mlas_platform_srcs_avx512vnni - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni.cpp - ${MLAS_SRC_DIR}/sqnbitgemm_kernel_avx512vnni_2bit.cpp - ${MLAS_SRC_DIR}/qkv_quant_kernel_avx512vnni.cpp + ${mlas_x64_cxx_srcs_avx512vnni} ) set_source_files_properties( ${mlas_platform_srcs_avx512vnni} PROPERTIES - COMPILE_FLAGS "-mfma -mf16c -mavx512vnni -mavx512bw -mavx512dq -mavx512vl -mavx512f") + COMPILE_FLAGS "${mlas_x64_flags_avx512vnni}") set(mlas_platform_srcs ${MLAS_SRC_DIR}/activate_fp16.cpp ${MLAS_SRC_DIR}/dwconv.cpp ${MLAS_SRC_DIR}/dgemm.cpp ${MLAS_SRC_DIR}/pooling_fp16.cpp - ${MLAS_SRC_DIR}/qgemm_kernel_avx2.cpp + ${mlas_x64_cxx_srcs_baseline} + ${mlas_x64_cxx_hdrs} ${mlas_platform_srcs_sse2} ${mlas_platform_srcs_avx} ${mlas_platform_srcs_avx2} @@ -1158,26 +1195,28 @@ else() ${mlas_platform_srcs_avx512vnni} ) - if (NOT onnxruntime_ORT_MINIMAL_BUILD) + if (mlas_x64_cxx_srcs_q4gemm_avx512) set(mlas_platform_srcs ${mlas_platform_srcs} - ${MLAS_SRC_DIR}/q4gemm_avx512.cpp + ${mlas_x64_cxx_srcs_q4gemm_avx512} ) - set_source_files_properties(${MLAS_SRC_DIR}/q4gemm_avx512.cpp PROPERTIES COMPILE_FLAGS "-mfma -mavx512vnni -mavx512bw -mavx512dq -mavx512vl -mavx512f") + set_source_files_properties(${mlas_x64_cxx_srcs_q4gemm_avx512} PROPERTIES + COMPILE_FLAGS "${mlas_x64_flags_q4gemm_avx512}") endif() if(NOT APPLE) set(mlas_platform_srcs ${mlas_platform_srcs} ${MLAS_SRC_DIR}/x86_64/QgemmU8S8KernelAmxCommon.S - ${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp + ${mlas_x64_cxx_srcs_amx} ${MLAS_SRC_DIR}/x86_64/QgemmU8S8KernelAmx.S ) - set_source_files_properties(${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp PROPERTIES COMPILE_FLAGS "-mavx2 -mavx512bw -mavx512dq -mavx512vl -mavx512f") - set_source_files_properties(${MLAS_SRC_DIR}/x86_64/QgemmU8S8KernelAmx.S PROPERTIES COMPILE_FLAGS "-mavx2 -mavx512bw -mavx512dq -mavx512vl -mavx512f") + set_source_files_properties(${mlas_x64_cxx_srcs_amx} ${MLAS_SRC_DIR}/x86_64/QgemmU8S8KernelAmx.S PROPERTIES + COMPILE_FLAGS "${mlas_x64_flags_amx}") endif() if(onnxruntime_ENABLE_CONVSYMKERNELAVX2_SAT_CHECKER) - set_source_files_properties(${MLAS_SRC_DIR}/x86_64/ConvSymKernelAvx2.S PROPERTIES COMPILE_FLAGS "-mavx2 -mfma -mf16c -DENABLE_CONVSYMKERNELAVX2_SAT_CHECKER") + set_source_files_properties(${MLAS_SRC_DIR}/x86_64/ConvSymKernelAvx2.S PROPERTIES + COMPILE_FLAGS "${mlas_x64_flags_avx2} -DENABLE_CONVSYMKERNELAVX2_SAT_CHECKER") endif() if(ONNXRUNTIME_MLAS_MULTI_ARCH) diff --git a/cmake/onnxruntime_msvc_runtime.cmake b/cmake/onnxruntime_msvc_runtime.cmake index bcd0fbccc5f9c..7049bec1bce69 100644 --- a/cmake/onnxruntime_msvc_runtime.cmake +++ b/cmake/onnxruntime_msvc_runtime.cmake @@ -25,13 +25,50 @@ function(onnxruntime_msvc_spectre_enabled flags result_var) set(${result_var} ${spectre_enabled} PARENT_SCOPE) endfunction() +# Returns the MSVC tools root (VC/Tools/MSVC/) that provides the MSVC runtime libraries. +function(onnxruntime_get_msvc_tools_root result_var) + if (NOT (CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")) + # CMAKE_C_COMPILER is /bin/Host//cl.exe. + cmake_path(SET msvc_tools_root NORMALIZE "${CMAKE_C_COMPILER}/../../../..") + set(${result_var} "${msvc_tools_root}" PARENT_SCOPE) + return() + endif() + + # clang-cl lives under VC/Tools/Llvm, so the runtime cannot be found relative to it. The VS ClangCL toolset links + # the MSVC runtime of the MSVC tools version MSBuild selects: the generator toolset's version= field if given, + # otherwise the instance's default. + if (NOT CMAKE_GENERATOR_INSTANCE) + message(FATAL_ERROR "Linking the OneCore MSVC runtime with clang-cl requires a Visual Studio generator.") + endif() + set(msvc_tools_dir "${CMAKE_GENERATOR_INSTANCE}/VC/Tools/MSVC") + if (CMAKE_VS_PLATFORM_TOOLSET_VERSION) + file(GLOB msvc_tools_versions LIST_DIRECTORIES true RELATIVE "${msvc_tools_dir}" + "${msvc_tools_dir}/${CMAKE_VS_PLATFORM_TOOLSET_VERSION}*") + list(SORT msvc_tools_versions COMPARE NATURAL ORDER DESCENDING) + if (msvc_tools_versions) + list(GET msvc_tools_versions 0 msvc_tools_version) + endif() + else() + set(msvc_tools_version_file "${CMAKE_GENERATOR_INSTANCE}/VC/Auxiliary/Build/Microsoft.VCToolsVersion.default.txt") + if (EXISTS "${msvc_tools_version_file}") + file(STRINGS "${msvc_tools_version_file}" msvc_tools_version LIMIT_COUNT 1) + string(STRIP "${msvc_tools_version}" msvc_tools_version) + endif() + endif() + if (NOT msvc_tools_version OR NOT IS_DIRECTORY "${msvc_tools_dir}/${msvc_tools_version}") + message(FATAL_ERROR "Unable to locate the MSVC tools for clang-cl under ${msvc_tools_dir} " + "(requested version: '${CMAKE_VS_PLATFORM_TOOLSET_VERSION}').") + endif() + set(${result_var} "${msvc_tools_dir}/${msvc_tools_version}" PARENT_SCOPE) +endfunction() + function(onnxruntime_configure_msvc_onecore_runtime) # GDK builds are skipped because gdk_toolchain.cmake configures the runtime, WINAPI_FAMILY and related flags. if (onnxruntime_BUILD_FOR_ONECORE AND NOT GDK_PLATFORM AND NOT CMAKE_CROSSCOMPILING) # OneCore builds must link the OneCore flavor of the MSVC runtime. Its lib directory is prepended to the # linker search path below so it takes precedence over the desktop runtime libraries found via LIB. - # CMAKE_C_COMPILER is /bin/Host//cl.exe. - cmake_path(SET msvc_lib_root NORMALIZE "${CMAKE_C_COMPILER}/../../../../lib") + onnxruntime_get_msvc_tools_root(msvc_tools_root) + set(msvc_lib_root "${msvc_tools_root}/lib") set(msvc_onecore_platform "${onnxruntime_target_platform}") if (msvc_onecore_platform STREQUAL "ARM64EC") # MSVC ships no ARM64EC runtime directory; ARM64EC uses the ARM64 hybrid CRT libraries. @@ -41,7 +78,9 @@ function(onnxruntime_configure_msvc_onecore_runtime) onnxruntime_msvc_spectre_enabled("${CMAKE_C_FLAGS}" msvc_c_spectre_enabled) onnxruntime_msvc_spectre_enabled("${CMAKE_CXX_FLAGS}" msvc_cxx_spectre_enabled) - if (msvc_c_spectre_enabled OR msvc_cxx_spectre_enabled) + # onnxruntime_USE_MSVC_SPECTRE_RUNTIME requests the Spectre-mitigated runtime for compilers such as clang-cl + # that do not implement /Qspectre. + if (msvc_c_spectre_enabled OR msvc_cxx_spectre_enabled OR onnxruntime_USE_MSVC_SPECTRE_RUNTIME) set(msvc_onecore_lib_dir "${msvc_lib_root}/spectre/onecore/${msvc_onecore_platform}") # Only the release libraries are required: debug CRT libraries are not Spectre-mitigated by design, and # BinSkim BA2024 skips modules that link them. diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index c60c6186ffe44..1375d8ee36898 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -2231,7 +2231,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) file(GLOB onnxruntime_perf_test_src CONFIGURE_DEPENDS ${onnxruntime_perf_test_src_patterns} ) - onnxruntime_add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/platform/path_lib.cc) + onnxruntime_add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src}) # ABSL_FLAGS_STRIP_NAMES is set to 1 by default to disable flag registration when building for Android, iPhone, and "embedded devices". # See the issue: https://github.com/abseil/abseil-cpp/issues/1875 diff --git a/include/onnxruntime/core/common/float16.h b/include/onnxruntime/core/common/float16.h index e9700be59fd3f..1712b6c588456 100644 --- a/include/onnxruntime/core/common/float16.h +++ b/include/onnxruntime/core/common/float16.h @@ -79,8 +79,11 @@ struct MLFloat16 : onnxruntime_float16::Float16Impl { operator float() const noexcept { return ToFloat(); } - using Base::operator==; - using Base::operator!=; + // Compare on the derived type (as BFloat16 does). With the inherited Float16Impl-typed operator==, C++20's + // reversed candidates make MLFloat16 == MLFloat16 ambiguous, which clang reports as + // -Wambiguous-reversed-operator. operator< is not rewritten in C++20, so the inherited one is unambiguous. + bool operator==(const MLFloat16& rhs) const noexcept { return Base::operator==(rhs); } + bool operator!=(const MLFloat16& rhs) const noexcept { return Base::operator!=(rhs); } using Base::operator<; }; diff --git a/onnxruntime/contrib_ops/cpu/quantization/gguf_vec_dot_cpu.h b/onnxruntime/contrib_ops/cpu/quantization/gguf_vec_dot_cpu.h index f19a057ed70a4..b2436dbce5279 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/gguf_vec_dot_cpu.h +++ b/onnxruntime/contrib_ops/cpu/quantization/gguf_vec_dot_cpu.h @@ -121,7 +121,8 @@ inline void QuantizeBlockQ8Scalar(const float* x, GgufQ8Block* blk) noexcept { blk->bsum = bsum; } -#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) +// clang-cl is excluded: __builtin_cpu_supports needs compiler-rt's __cpu_model, which clang-cl does not link. +#if (defined(__x86_64__) || defined(_M_X64)) && (defined(__GNUC__) || defined(__clang__)) && !defined(_MSC_VER) #define ORT_GGUF_CPU_HAS_AVX2_TARGET 1 #else #define ORT_GGUF_CPU_HAS_AVX2_TARGET 0 diff --git a/onnxruntime/core/common/spin_pause.cc b/onnxruntime/core/common/spin_pause.cc index 0f62420530176..1bf082f53e38a 100644 --- a/onnxruntime/core/common/spin_pause.cc +++ b/onnxruntime/core/common/spin_pause.cc @@ -26,6 +26,26 @@ namespace onnxruntime { namespace concurrency { +// TPAUSE is compiled in on x64 Windows and on other x64 targets built with WAITPKG; it is still used only when +// CPUIDInfo::HasTPAUSE() reports support at runtime. +#if (defined(_M_AMD64) || defined(__x86_64__)) && \ + !defined(_M_ARM64EC) && \ + !defined(__ANDROID__) && \ + !defined(__APPLE__) && \ + (defined(_WIN32) || defined(__WAITPKG__)) +#define ORT_USE_TPAUSE + +// clang (including clang-cl) refuses to emit _tpause from a function not compiled with the 'waitpkg' target +// feature, so isolate it in a non-inlined helper carrying that feature. MSVC does not feature-gate intrinsics. +#if defined(__clang__) +__attribute__((target("waitpkg"), noinline)) +#endif +static void +OrtTPause(uint64_t deadline) { + _tpause(0x0, deadline); +} +#endif + // Intrinsic to use in spin-loops void SpinPause() { #if (defined(_M_AMD64) || defined(__x86_64__)) && \ @@ -35,9 +55,9 @@ void SpinPause() { static const bool has_tpause = CPUIDInfo::GetCPUIDInfo().HasTPAUSE(); if (has_tpause) { -#if (defined(_WIN32) && !defined(__clang__)) || defined(__WAITPKG__) +#if defined(ORT_USE_TPAUSE) static constexpr uint64_t tpause_spin_delay_cycles = 1000; - _tpause(0x0, __rdtsc() + tpause_spin_delay_cycles); + OrtTPause(__rdtsc() + tpause_spin_delay_cycles); #else _mm_pause(); #endif diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/DirectMLHelpers/AbstractOperatorDesc.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/DirectMLHelpers/AbstractOperatorDesc.h index 6d7a691315064..bb5aa18714686 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/DirectMLHelpers/AbstractOperatorDesc.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/External/DirectMLHelpers/AbstractOperatorDesc.h @@ -3,6 +3,9 @@ #pragma once +#include +#include + class OperatorField; struct AbstractOperatorDesc @@ -10,62 +13,101 @@ struct AbstractOperatorDesc const DML_OPERATOR_SCHEMA* schema = nullptr; std::vector fields; - AbstractOperatorDesc() = default; - AbstractOperatorDesc(const DML_OPERATOR_SCHEMA* schema, std::vector&& fields) - : schema(schema) - , fields(std::move(fields)) - {} + // NOTE (clang-cl / two-phase name lookup): + // OperatorField and AbstractOperatorDesc are mutually recursive - OperatorField holds a + // std::optional (see GeneratedSchemaTypes.h) - so OperatorField is + // necessarily an incomplete type at this point. Any member function that instantiates + // std::vector operations requiring a complete element type - the special + // member functions (which may need to destroy the vector) and the GetTensors iteration - + // must therefore be declared here and defined out of line below, after OperatorField has + // been completed. MSVC's cl.exe defers these instantiations and tolerates inline + // definitions, but conforming two-phase compilers such as clang-cl instantiate them at + // definition time and fail with "incomplete type 'OperatorField'". + AbstractOperatorDesc(); + AbstractOperatorDesc(const DML_OPERATOR_SCHEMA* schema, std::vector&& fields); + AbstractOperatorDesc(const AbstractOperatorDesc&); + AbstractOperatorDesc(AbstractOperatorDesc&&) noexcept; + AbstractOperatorDesc& operator=(const AbstractOperatorDesc&); + AbstractOperatorDesc& operator=(AbstractOperatorDesc&&) noexcept; + ~AbstractOperatorDesc(); - std::vector GetInputTensors() - { - return GetTensors(); - } + std::vector GetInputTensors(); + std::vector GetInputTensors() const; + std::vector GetOutputTensors(); + std::vector GetOutputTensors() const; - std::vector GetInputTensors() const - { - return GetTensors(); - } +private: + template + std::vector GetTensors() const; +}; - std::vector GetOutputTensors() - { - return GetTensors(); - } +// Complete OperatorField before defining the members that require it. GeneratedSchemaTypes.h is +// guarded by #pragma once, so its subsequent include from precomp.h is a no-op and the +// established include order is preserved. +#include "GeneratedSchemaTypes.h" - std::vector GetOutputTensors() const - { - return GetTensors(); - } +inline AbstractOperatorDesc::AbstractOperatorDesc() = default; -private: - template - std::vector GetTensors() const +inline AbstractOperatorDesc::AbstractOperatorDesc(const DML_OPERATOR_SCHEMA* schema, std::vector&& fields) + : schema(schema) + , fields(std::move(fields)) +{ +} + +inline AbstractOperatorDesc::AbstractOperatorDesc(const AbstractOperatorDesc&) = default; +inline AbstractOperatorDesc::AbstractOperatorDesc(AbstractOperatorDesc&&) noexcept = default; +inline AbstractOperatorDesc& AbstractOperatorDesc::operator=(const AbstractOperatorDesc&) = default; +inline AbstractOperatorDesc& AbstractOperatorDesc::operator=(AbstractOperatorDesc&&) noexcept = default; +inline AbstractOperatorDesc::~AbstractOperatorDesc() = default; + +inline std::vector AbstractOperatorDesc::GetInputTensors() +{ + return GetTensors(); +} + +inline std::vector AbstractOperatorDesc::GetInputTensors() const +{ + return GetTensors(); +} + +inline std::vector AbstractOperatorDesc::GetOutputTensors() +{ + return GetTensors(); +} + +inline std::vector AbstractOperatorDesc::GetOutputTensors() const +{ + return GetTensors(); +} + +template +std::vector AbstractOperatorDesc::GetTensors() const +{ + std::vector tensors; + for (auto& field : fields) { - std::vector tensors; - for (auto& field : fields) + const DML_SCHEMA_FIELD* fieldSchema = field.GetSchema(); + if (fieldSchema->Kind != Kind) { - const DML_SCHEMA_FIELD* fieldSchema = field.GetSchema(); - if (fieldSchema->Kind != Kind) - { - continue; - } + continue; + } - if (fieldSchema->Type == DML_SCHEMA_FIELD_TYPE_TENSOR_DESC) - { - auto& tensor = field.AsTensorDesc(); - tensors.push_back(tensor ? const_cast(&*tensor) : nullptr); - } - else if (fieldSchema->Type == DML_SCHEMA_FIELD_TYPE_TENSOR_DESC_ARRAY) + if (fieldSchema->Type == DML_SCHEMA_FIELD_TYPE_TENSOR_DESC) + { + auto& tensor = field.AsTensorDesc(); + tensors.push_back(tensor ? const_cast(&*tensor) : nullptr); + } + else if (fieldSchema->Type == DML_SCHEMA_FIELD_TYPE_TENSOR_DESC_ARRAY) + { + auto& tensorArray = field.AsTensorDescArray(); + if (tensorArray) { - auto& tensorArray = field.AsTensorDescArray(); - if (tensorArray) + for (auto& tensor : *tensorArray) { - for (auto& tensor : *tensorArray) - { - tensors.push_back(const_cast(&tensor)); - } + tensors.push_back(const_cast(&tensor)); } } } - return tensors; } -}; + return tensors; +} diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp index 4275f7930b5c4..5c58619205dc9 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp @@ -3148,7 +3148,7 @@ namespace Windows::AI::MachineLearning::Adapter #define CASE_PROTO(X, Y, Z) \ case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##X: { \ - size_t elementCount = initializer.##Z(); \ + size_t elementCount = initializer.Z(); \ tensorByteSize = elementCount * sizeof(Y); \ unpackedTensor.reset(new std::byte[tensorByteSize]); \ ORT_THROW_HR_IF(E_FAIL, !onnxruntime::utils::UnpackTensor( \ diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlDFT.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlDFT.h index 25210c146a6b6..3286c5154e1bb 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlDFT.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlDFT.h @@ -920,7 +920,7 @@ class GpuDFTOperator : public WRL::Base return dims; } - template + template void Dispatch( std::array& resources, TConstants& constants, diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlGridSample.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlGridSample.h index 6d7a089103c9b..a14e26784fdb9 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlGridSample.h +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlGridSample.h @@ -645,7 +645,7 @@ class DmlGridSampleOperator : public WRL::Base return dims; } - template + template void Dispatch( std::array& resources, TConstants& constants, diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index 98a6a0302e6f6..c17bb23c386a2 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -4093,13 +4093,7 @@ TEST(MathOpTest, Mean_8) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); // TensorRT: Input batch size is inconsistent } -#ifdef _LIBCPP_VERSION -#define MATH_NO_EXCEPT -#else -#define MATH_NO_EXCEPT noexcept -#endif - -template +template void TrigFloatTest(OpTester& test, std::initializer_list input, float abs_error = -1.0f) { std::vector dims{static_cast(input.size())}; @@ -4117,7 +4111,7 @@ void TrigFloatTest(OpTester& test, std::initializer_list input, float abs test.Run(); } -template +template void TrigDoubleTest(OpTester& test, std::initializer_list input, const std::unordered_set excluded_provider_types = {}) { std::vector dims{static_cast(input.size())}; @@ -4131,7 +4125,7 @@ void TrigDoubleTest(OpTester& test, std::initializer_list input, test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_provider_types); } -template +template void TrigFloat16Test(OpTester& test, std::initializer_list input) { std::vector dims{static_cast(input.size())}; diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 2d90026a9ec83..b21c75044024c 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -394,6 +394,21 @@ def _get_vctools_install_dir(args): return None +def get_visual_studio_toolset(args, host_arch): + """Return the CMake -T value for the Visual Studio generator.""" + # ClangCL selects the Visual Studio LLVM/clang-cl platform toolset; clang-cl uses the MSVC runtime and + # libraries of the selected MSVC tools version. + toolset = "ClangCL," if args.use_clang_cl else "" + toolset += "host=" + host_arch + if args.msvc_toolset: + toolset += ",version=" + args.msvc_toolset + if args.use_cuda and args.cuda_version: + toolset += ",cuda=" + args.cuda_version + elif args.use_cuda and args.cuda_home: + toolset += ",cuda=" + args.cuda_home + return toolset + + def get_msvc_spectre_lib_dir(args): """Return the directory that holds the MSVC Spectre-mitigated CRT/STL static libraries for the target architecture, or None if it cannot be located. @@ -1269,6 +1284,7 @@ def generate_build_tree( cxxflags = None ldflags = None nvcc_flags = [] + use_msvc_spectre_runtime = False if is_windows() and not args.android and not args.build_wasm: njobs = number_of_parallel_jobs(args) if args.use_cuda: @@ -1296,7 +1312,12 @@ def generate_build_tree( ldflags = ["/profile", "/DYNAMICBASE"] # Address Sanitizer libs do not have a Qspectre version. So they two cannot be both enabled. if not args.enable_address_sanitizer: - cflags += ["/Qspectre"] + # clang-cl does not implement /Qspectre, so only the prebuilt MSVC runtime libraries below + # are Spectre-mitigated in clang-cl builds. + if args.use_clang_cl: + use_msvc_spectre_runtime = True + else: + cflags += ["/Qspectre"] # /Qspectre only mitigates ONNX Runtime's own object files. The prebuilt MSVC # CRT/STL static libraries (libcmt.lib, libcpmt.lib, libvcruntime.lib) that are # linked into the binaries also have to be the Spectre-mitigated variants, @@ -1404,6 +1425,9 @@ def generate_build_tree( "-DCMAKE_C_FLAGS={}".format(" ".join(cflags)), "-DCMAKE_CXX_FLAGS={}".format(" ".join(cxxflags)), ] + if is_windows(): + # Always set the value so a reconfigure does not keep a value cached by an earlier configuration. + temp_cmake_args += [f"-Donnxruntime_USE_MSVC_SPECTRE_RUNTIME={'ON' if use_msvc_spectre_runtime else 'OFF'}"] if nvcc_flags is not None and len(nvcc_flags) != 0: temp_cmake_args += ["-DCMAKE_CUDA_FLAGS_INIT={}".format(" ".join(nvcc_flags))] if ldflags is not None and len(ldflags) != 0: @@ -2733,6 +2757,9 @@ def main(): cmake_extra_args = ["-A", "ARM64EC"] if args.buildasx: cmake_extra_args += ["-D", "BUILD_AS_ARM64X=ARM64EC"] + if args.use_clang_cl: + # Forward the toolset so configuration rejects the unsupported target instead of using cl.exe. + cmake_extra_args += ["-T", "ClangCL"] cmake_extra_args += ["-G", args.cmake_generator] # Cannot test on host build machine for cross-compiled # builds (Override any user-defined behavior for test if any) @@ -2754,14 +2781,7 @@ def main(): host_arch = "ARM64" else: raise BuildError("unknown python arch") - if args.msvc_toolset: - toolset = "host=" + host_arch + ",version=" + args.msvc_toolset - else: - toolset = "host=" + host_arch - if args.use_cuda and args.cuda_version: - toolset += ",cuda=" + args.cuda_version - elif args.use_cuda and args.cuda_home: - toolset += ",cuda=" + args.cuda_home + toolset = get_visual_studio_toolset(args, host_arch) if args.windows_sdk_version: target_arch += ",version=" + args.windows_sdk_version cmake_extra_args = ["-A", target_arch, "-T", toolset, "-G", args.cmake_generator] diff --git a/tools/ci_build/build_args.py b/tools/ci_build/build_args.py index 481afc0920989..1056992490a4c 100644 --- a/tools/ci_build/build_args.py +++ b/tools/ci_build/build_args.py @@ -441,6 +441,13 @@ def add_windows_specific_args(parser: argparse.ArgumentParser) -> None: """Adds arguments specific to Windows builds or Windows cross-compilation.""" # Build tools / config parser.add_argument("--msvc_toolset", help="MSVC toolset version (e.g., 14.11). Must be >=14.40") + parser.add_argument( + "--use_clang_cl", + action="store_true", + help="Build with the LLVM clang-cl compiler instead of MSVC cl.exe. " + "Requires the Visual Studio generator and the 'C++ Clang Compiler for Windows' component. " + "Selects the 'ClangCL' platform toolset. clang-cl is MSVC-ABI compatible.", + ) parser.add_argument("--windows_sdk_version", help="Windows SDK version (e.g., 10.0.19041.0).") parser.add_argument("--enable_msvc_static_runtime", action="store_true", help="Statically link MSVC runtimes.") parser.add_argument("--caller_framework", type=str, help="Name of the framework calling ONNX Runtime.") @@ -1095,6 +1102,12 @@ def convert_arg_line_to_args(self, arg_line: str) -> list[str]: if is_windows(): if getattr(args, "use_winml", False) and not getattr(args, "enable_wcos", False): parser.error("--use_winml requires --enable_wcos to be specified.") + # --use_clang_cl only selects the Visual Studio 'ClangCL' platform toolset; CMake validates the target. + if args.use_clang_cl: + if args.build_wasm or args.android: + parser.error("--use_clang_cl is only supported for native Windows builds.") + if not (args.cmake_generator or "").startswith("Visual Studio"): + parser.error("--use_clang_cl requires a Visual Studio CMake generator.") if hasattr(args, "msvc_toolset") and args.msvc_toolset: try: # Extract major.minor version parts (e.g., "14.36") diff --git a/tools/ci_build/test_build_args.py b/tools/ci_build/test_build_args.py index 9b3870dfb99d6..92e6dbfd2fdbc 100644 --- a/tools/ci_build/test_build_args.py +++ b/tools/ci_build/test_build_args.py @@ -117,6 +117,31 @@ def test_android_enables_telemetry_by_default(self): args = self._parse("--android", platform_name="linux") self.assertTrue(args.use_telemetry) + def _parse_clang_cl(self, *arguments: str, generator: str = "Visual Studio 17 2022"): + return self._parse( + "--use_clang_cl", "--cmake_generator", generator, *arguments, platform_name="windows", machine="AMD64" + ) + + def test_use_clang_cl_selects_clangcl_toolset(self): + args = self._parse_clang_cl("--msvc_toolset", "14.44") + self.assertEqual(build.get_visual_studio_toolset(args, "x64"), "ClangCL,host=x64,version=14.44") + args = self._parse("--cmake_generator", "Visual Studio 17 2022", platform_name="windows", machine="AMD64") + self.assertEqual(build.get_visual_studio_toolset(args, "x64"), "host=x64") + + def test_use_clang_cl_rejects_unsupported_builds(self): + cases = ( + ({"generator": "Ninja"}, ()), + ({}, ("--build_wasm",)), + ({}, ("--android",)), + ) + for kwargs, arguments in cases: + with ( + self.subTest(kwargs=kwargs, arguments=arguments), + mock.patch("sys.stderr"), + self.assertRaises(SystemExit), + ): + self._parse_clang_cl(*arguments, **kwargs) + def test_compiler_cache_defaults_to_ccache(self): args = self._parse("--use_cache", platform_name="linux") self.assertEqual(args.cache_tool, "ccache") diff --git a/tools/ci_build/test_onecore_spectre.py b/tools/ci_build/test_onecore_spectre.py index c202f05a7e4f4..1e75b0ecd3966 100644 --- a/tools/ci_build/test_onecore_spectre.py +++ b/tools/ci_build/test_onecore_spectre.py @@ -18,6 +18,8 @@ "libvcruntimed.lib", ) _RUNTIME_LIBRARIES = _REQUIRED_RUNTIME_LIBRARIES + _DEBUG_RUNTIME_LIBRARIES +_DEFAULT_MSVC_VERSION = "14.44.35207" +_OTHER_MSVC_VERSION = "14.39.33519" class OneCoreSpectreTest(unittest.TestCase): @@ -32,6 +34,9 @@ def _configure( cross_compiling: bool = False, missing_library: str | None = None, create_onecore_dir: bool = True, + clang_cl: bool = False, + toolset_version: str = "", + spectre_runtime: bool = False, ) -> tuple[subprocess.CompletedProcess[str], Path, Path, str]: repo_root = Path(__file__).resolve().parents[2] temporary_directory = tempfile.TemporaryDirectory(prefix="ort-onecore-spectre-") @@ -39,8 +44,21 @@ def _configure( root = Path(temporary_directory.name) source_dir = root / "source" build_dir = root / "build" - toolset_dir = root / "toolset" - compiler = toolset_dir / "bin" / "Hostx64" / "x64" / "cl.exe" + vs_instance = root / "vs" + if clang_cl: + # The VS LLVM layout: clang-cl is not under the MSVC tools root that provides the runtime. + msvc_tools_dir = vs_instance / "VC" / "Tools" / "MSVC" + selected_version = _OTHER_MSVC_VERSION if toolset_version else _DEFAULT_MSVC_VERSION + toolset_dir = msvc_tools_dir / selected_version + compiler = vs_instance / "VC" / "Tools" / "Llvm" / "x64" / "bin" / "clang-cl.exe" + unselected_version = _DEFAULT_MSVC_VERSION if toolset_version else _OTHER_MSVC_VERSION + (msvc_tools_dir / unselected_version).mkdir(parents=True) + version_file = vs_instance / "VC" / "Auxiliary" / "Build" / "Microsoft.VCToolsVersion.default.txt" + version_file.parent.mkdir(parents=True) + version_file.write_text(f"{_DEFAULT_MSVC_VERSION}\n", encoding="utf-8") + else: + toolset_dir = root / "toolset" + compiler = toolset_dir / "bin" / "Hostx64" / "x64" / "cl.exe" generic_spectre_dir = toolset_dir / "lib" / "spectre" / platform.lower() selected_platform = "ARM64" if platform == "ARM64EC" else platform spectre_onecore_dir = toolset_dir / "lib" / "spectre" / "onecore" / selected_platform @@ -66,9 +84,14 @@ def _configure( set(GDK_PLATFORM "${{TEST_GDK}}") set(CMAKE_CROSSCOMPILING "${{TEST_CROSS_COMPILING}}") set(CMAKE_C_COMPILER "${{TEST_COMPILER}}") +set(CMAKE_C_COMPILER_ID "${{TEST_COMPILER_ID}}") +set(CMAKE_C_COMPILER_FRONTEND_VARIANT "MSVC") +set(CMAKE_GENERATOR_INSTANCE "${{TEST_GENERATOR_INSTANCE}}") +set(CMAKE_VS_PLATFORM_TOOLSET_VERSION "${{TEST_TOOLSET_VERSION}}") set(CMAKE_C_FLAGS "${{TEST_C_FLAGS}}") set(CMAKE_CXX_FLAGS "${{TEST_CXX_FLAGS}}") set(onnxruntime_BUILD_FOR_ONECORE "${{TEST_ONECORE}}") +set(onnxruntime_USE_MSVC_SPECTRE_RUNTIME "${{TEST_SPECTRE_RUNTIME}}") set(onnxruntime_target_platform "${{TEST_PLATFORM}}") link_directories("${{TEST_GENERIC_SPECTRE_DIR}}") include("{cmake_module.as_posix()}") @@ -93,10 +116,14 @@ def _configure( f"-DTEST_GDK={'ON' if gdk else 'OFF'}", f"-DTEST_CROSS_COMPILING={'ON' if cross_compiling else 'OFF'}", f"-DTEST_COMPILER={compiler}", + f"-DTEST_COMPILER_ID={'Clang' if clang_cl else 'MSVC'}", + f"-DTEST_GENERATOR_INSTANCE={vs_instance if clang_cl else ''}", + f"-DTEST_TOOLSET_VERSION={toolset_version}", + f"-DTEST_SPECTRE_RUNTIME={'ON' if spectre_runtime else 'OFF'}", f"-DTEST_GENERIC_SPECTRE_DIR={generic_spectre_dir}", f"-DTEST_RESULT_FILE={result_file}", ] - result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=30) + result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=120) return result, result_file, toolset_dir, result.stdout + result.stderr def _assert_selected(self, expected_relative_path: str, **kwargs) -> None: @@ -171,6 +198,20 @@ def test_gdk_skips_override(self): def test_cross_compiling_skips_override(self): self._assert_override_skipped(cross_compiling=True) + def test_clang_cl_uses_default_msvc_tools(self): + self._assert_selected("lib/onecore/x64", clang_cl=True) + + def test_clang_cl_spectre_runtime_request_selects_spectre_libraries(self): + self._assert_selected("lib/spectre/onecore/x64", clang_cl=True, spectre_runtime=True) + + def test_clang_cl_uses_requested_msvc_toolset_version(self): + self._assert_selected("lib/onecore/x64", clang_cl=True, toolset_version="14.39") + + def test_clang_cl_unknown_msvc_toolset_version_fails(self): + result, _, _, output = self._configure(clang_cl=True, toolset_version="14.10") + self.assertNotEqual(result.returncode, 0, output) + self.assertIn("Unable to locate the MSVC tools for clang-cl", output) + def _detect_onecore(self, standard_libraries: str, win32: bool = True) -> str: repo_root = Path(__file__).resolve().parents[2] cmake_module = repo_root / "cmake" / "onnxruntime_msvc_runtime.cmake" @@ -191,7 +232,7 @@ def _detect_onecore(self, standard_libraries: str, win32: bool = True) -> str: capture_output=True, check=False, text=True, - timeout=30, + timeout=120, ) output = result.stdout + result.stderr self.assertEqual(result.returncode, 0, output) @@ -211,6 +252,11 @@ def test_detect_gdk_build(self): def test_detect_non_windows_build(self): self.assertEqual(self._detect_onecore("", win32=False), "OFF") + def test_spectre_runtime_request_requires_spectre_libraries(self): + result, _, _, output = self._configure(spectre_runtime=True, missing_library="libcmt.lib") + self.assertNotEqual(result.returncode, 0, output) + self.assertIn("Required Spectre-mitigated OneCore runtime library is missing:", output) + def test_missing_onecore_dir_fails(self): result, _, _, output = self._configure(create_onecore_dir=False) self.assertNotEqual(result.returncode, 0, output)