LCOV - code coverage report
Current view: top level - core - cpu_features.c (source / functions) Coverage Total Hit
Test: coverage.info Lines: 78.9 % 38 30
Test Date: 2026-10-05 04:47:18 Functions: 66.7 % 6 4

            Line data    Source code
       1              : /**
       2              :  * CPU Feature Detection Implementation
       3              :  *
       4              :  * Runtime detection of CPU SIMD capabilities using platform-specific methods:
       5              :  * - x86/x64 (MSVC): __cpuid/__cpuidex intrinsics + XGETBV for OS support
       6              :  * - x86/x64 (GCC/Clang): __get_cpuid_count + xgetbv for OS support
       7              :  * - ARM64: NEON is always available
       8              :  *
       9              :  * IMPORTANT: For AVX2, we must verify both:
      10              :  * 1. CPU support (CPUID leaf 7, EBX bit 5)
      11              :  * 2. OS support (OSXSAVE enabled + XCR0 bits 1,2 set for AVX state saving)
      12              :  *
      13              :  * Without OS support verification, AVX instructions will cause illegal
      14              :  * instruction exceptions on systems where the OS hasn't enabled AVX.
      15              :  *
      16              :  * Thread Safety:
      17              :  * The cache uses atomic operations with proper memory ordering to ensure
      18              :  * thread-safe initialization. The detection logic is idempotent (always
      19              :  * produces the same result), so multiple threads racing to initialize is
      20              :  * safe - they will all compute and store the same value.
      21              :  *
      22              :  * We use:
      23              :  * - C11 stdatomic.h when available (GCC 4.9+, Clang 3.1+, MSVC 2022+)
      24              :  * - GCC __atomic builtins as fallback (GCC 4.7+)
      25              :  * - MSVC Interlocked functions as final fallback
      26              :  */
      27              : 
      28              : #include "cfd/core/cpu_features.h"
      29              : 
      30              : /* ============================================================================
      31              :  * Atomic Operations Abstraction
      32              :  *
      33              :  * Provides atomic load/store with proper memory ordering across platforms.
      34              :  * Priority: C11 atomics > GCC atomics > MSVC Interlocked > volatile fallback
      35              :  * ============================================================================ */
      36              : 
      37              : /* Check for C11 atomics support */
      38              : #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
      39              :     /* C11 atomics available */
      40              :     #include <stdatomic.h>
      41              :     #define CFD_HAS_C11_ATOMICS 1
      42              : #elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7))
      43              :     /* GCC 4.7+ atomic builtins */
      44              :     #define CFD_HAS_GCC_ATOMICS 1
      45              : #elif defined(_MSC_VER)
      46              :     /* MSVC Interlocked functions */
      47              :     #include <intrin.h>
      48              :     #define CFD_HAS_MSVC_ATOMICS 1
      49              : #endif
      50              : 
      51              : /* Atomic cache type and operations */
      52              : #if defined(CFD_HAS_C11_ATOMICS)
      53              :     static _Atomic int g_simd_arch_cache = -1;
      54              : 
      55        34502 :     static inline int atomic_cache_load(void) {
      56        34502 :         return atomic_load_explicit(&g_simd_arch_cache, memory_order_acquire);
      57              :     }
      58              : 
      59           28 :     static inline void atomic_cache_store(int value) {
      60           28 :         atomic_store_explicit(&g_simd_arch_cache, value, memory_order_release);
      61              :     }
      62              : 
      63              : #elif defined(CFD_HAS_GCC_ATOMICS)
      64              :     static int g_simd_arch_cache = -1;
      65              : 
      66              :     static inline int atomic_cache_load(void) {
      67              :         return __atomic_load_n(&g_simd_arch_cache, __ATOMIC_ACQUIRE);
      68              :     }
      69              : 
      70              :     static inline void atomic_cache_store(int value) {
      71              :         __atomic_store_n(&g_simd_arch_cache, value, __ATOMIC_RELEASE);
      72              :     }
      73              : 
      74              : #elif defined(CFD_HAS_MSVC_ATOMICS)
      75              :     static volatile long g_simd_arch_cache = -1;
      76              : 
      77              :     static inline int atomic_cache_load(void) {
      78              :         /* _InterlockedCompareExchange provides full barrier on x86/x64
      79              :          * This is the intrinsic form (note the underscore prefix) */
      80              :         return (int)_InterlockedCompareExchange(&g_simd_arch_cache, -1, -1);
      81              :     }
      82              : 
      83              :     static inline void atomic_cache_store(int value) {
      84              :         _InterlockedExchange(&g_simd_arch_cache, (long)value);
      85              :     }
      86              : 
      87              : #else
      88              :     /* Fallback: volatile with no guarantees beyond single-threaded correctness.
      89              :      * This should rarely be hit on modern compilers. */
      90              :     #warning "No atomic primitives available - cache may not be thread-safe"
      91              :     static volatile int g_simd_arch_cache = -1;
      92              : 
      93              :     static inline int atomic_cache_load(void) {
      94              :         return g_simd_arch_cache;
      95              :     }
      96              : 
      97              :     static inline void atomic_cache_store(int value) {
      98              :         g_simd_arch_cache = value;
      99              :     }
     100              : #endif
     101              : 
     102              : /* ============================================================================
     103              :  * Platform-Specific CPU Detection Headers
     104              :  * ============================================================================ */
     105              : 
     106              : #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
     107              : #include <intrin.h>
     108              : #define CFD_RUNTIME_X86_MSVC 1
     109              : #elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
     110              : #include <cpuid.h>
     111              : #define CFD_RUNTIME_X86_GCC 1
     112              : 
     113              : /**
     114              :  * Inline xgetbv for GCC/Clang.
     115              :  *
     116              :  * We use inline assembly instead of _xgetbv() intrinsic because the intrinsic
     117              :  * requires -mxsave compiler flag, but we want this file to compile without
     118              :  * requiring any special CPU feature flags (since it's doing runtime detection).
     119              :  *
     120              :  * IMPORTANT: This function must ONLY be called after verifying OSXSAVE support
     121              :  * via CPUID leaf 1, ECX bit 27. The OSXSAVE bit indicates that:
     122              :  * 1. The CPU supports the XGETBV instruction
     123              :  * 2. The OS has enabled XSAVE/XRSTOR for context switching
     124              :  *
     125              :  * Calling xgetbv without OSXSAVE support will cause an undefined opcode (#UD)
     126              :  * exception. The detect_simd_arch_impl() function enforces this requirement
     127              :  * by only calling cfd_xgetbv() inside an if(osxsave) block.
     128              :  *
     129              :  * Register usage (identical on 32-bit and 64-bit x86):
     130              :  * - Input: ECX = XCR number (0 for XCR0)
     131              :  * - Output: EDX:EAX = XCR value (64-bit)
     132              :  * - xgetbv does NOT modify XMM/YMM registers - no SIMD clobbers needed
     133              :  *
     134              :  * The "memory" clobber ensures the compiler doesn't reorder this with
     135              :  * memory operations, though xgetbv itself doesn't access memory.
     136              :  */
     137           28 : static inline unsigned long long cfd_xgetbv(unsigned int xcr) {
     138           28 :     unsigned int eax, edx;
     139           56 :     __asm__ __volatile__(
     140              :         "xgetbv"
     141              :         : "=a"(eax), "=d"(edx)
     142              :         : "c"(xcr)
     143              :         : "memory"
     144              :     );
     145           28 :     return ((unsigned long long)edx << 32) | eax;
     146              : }
     147              : #endif
     148              : 
     149              : /* ============================================================================
     150              :  * Runtime Detection Implementation
     151              :  * ============================================================================ */
     152              : 
     153              : /**
     154              :  * Perform the actual SIMD detection.
     155              :  * This is a pure function - always returns the same result for a given CPU.
     156              :  */
     157           28 : static cfd_simd_arch_t detect_simd_arch_impl(void) {
     158              : #if defined(CFD_RUNTIME_X86_MSVC)
     159              :     /* MSVC on x86/x64: Use __cpuid intrinsic */
     160              :     int cpuInfo[4] = {0};
     161              :     __cpuid(cpuInfo, 0);
     162              :     int nIds = cpuInfo[0];
     163              : 
     164              :     if (nIds >= 7) {
     165              :         /* First check CPUID leaf 1 for OSXSAVE support (ECX bit 27) */
     166              :         __cpuid(cpuInfo, 1);
     167              :         int osxsave = (cpuInfo[2] & (1 << 27)) != 0;
     168              :         /* FMA3 (ECX bit 12) is a separate capability from AVX2, and the AVX2
     169              :          * build relies on it: kernels call _mm256_fmadd_* and the library is
     170              :          * compiled with FMA enabled. */
     171              :         int fma = (cpuInfo[2] & (1 << 12)) != 0;
     172              : 
     173              :         if (osxsave && fma) {
     174              :             /* OSXSAVE is enabled, now check XCR0 for AVX state support */
     175              :             /* XCR0 bits: bit 1 = SSE state, bit 2 = AVX state */
     176              :             /* Both must be set for AVX to work */
     177              :             unsigned long long xcr0 = _xgetbv(0);
     178              :             int avx_os_support = ((xcr0 & 0x6) == 0x6);
     179              : 
     180              :             if (avx_os_support) {
     181              :                 /* Now check CPU support for AVX2 */
     182              :                 __cpuidex(cpuInfo, 7, 0);
     183              :                 /* Check AVX2 bit (EBX bit 5) */
     184              :                 if (cpuInfo[1] & (1 << 5)) {
     185              :                     return CFD_SIMD_AVX2;
     186              :                 }
     187              :             }
     188              :         }
     189              :     }
     190              :     return CFD_SIMD_NONE;
     191              : 
     192              : #elif defined(CFD_RUNTIME_X86_GCC)
     193              :     /* GCC/Clang on x86/x64: Use __get_cpuid_count */
     194           28 :     unsigned int eax, ebx, ecx, edx;
     195              : 
     196              :     /* First check CPUID leaf 1 for OSXSAVE support (ECX bit 27) */
     197           56 :     if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
     198           28 :         int osxsave = (ecx & (1 << 27)) != 0;
     199              :         /* FMA3 (ECX bit 12): see the MSVC branch above. */
     200           28 :         int fma = (ecx & (1 << 12)) != 0;
     201              : 
     202           28 :         if (osxsave && fma) {
     203              :             /* OSXSAVE is enabled, now check XCR0 for AVX state support */
     204              :             /* XCR0 bits: bit 1 = SSE state, bit 2 = AVX state */
     205              :             /* Both must be set for AVX to work */
     206           28 :             unsigned long long xcr0 = cfd_xgetbv(0);
     207           28 :             int avx_os_support = ((xcr0 & 0x6) == 0x6);
     208              : 
     209           28 :             if (avx_os_support) {
     210              :                 /* Now check CPU support for AVX2 */
     211           28 :                 if (__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) {
     212              :                     /* Check AVX2 bit (EBX bit 5) */
     213           28 :                     if (ebx & (1 << 5)) {
     214              :                         return CFD_SIMD_AVX2;
     215              :                     }
     216              :                 }
     217              :             }
     218              :         }
     219              :     }
     220              :     return CFD_SIMD_NONE;
     221              : 
     222              : #elif defined(__aarch64__) || defined(_M_ARM64)
     223              :     /* ARM64: NEON is always available on AArch64 */
     224              :     return CFD_SIMD_NEON;
     225              : 
     226              : #elif defined(__ARM_NEON) || defined(__ARM_NEON__)
     227              :     /* ARMv7 with NEON: Assume available if compiled with NEON support */
     228              :     return CFD_SIMD_NEON;
     229              : 
     230              : #else
     231              :     return CFD_SIMD_NONE;
     232              : #endif
     233              : }
     234              : 
     235        34502 : cfd_simd_arch_t cfd_detect_simd_arch(void) {
     236              :     /* Read cached result with acquire semantics */
     237        34502 :     int cached = atomic_cache_load();
     238              : 
     239        34502 :     if (cached >= 0) {
     240              :         return (cfd_simd_arch_t)cached;
     241              :     }
     242              : 
     243              :     /* Perform detection - result will be the same regardless of which thread
     244              :      * computes it, so racing here is safe (just wastes some CPU cycles). */
     245           28 :     cfd_simd_arch_t detected = detect_simd_arch_impl();
     246              : 
     247              :     /* Store result with release semantics to ensure visibility to other threads */
     248           28 :     atomic_cache_store((int)detected);
     249              : 
     250           28 :     return detected;
     251              : }
     252              : 
     253              : /* ============================================================================
     254              :  * Convenience Functions
     255              :  * ============================================================================ */
     256              : 
     257            0 : bool cfd_has_avx2(void) {
     258            0 :     return cfd_detect_simd_arch() == CFD_SIMD_AVX2;
     259              : }
     260              : 
     261            0 : bool cfd_has_neon(void) {
     262            0 :     return cfd_detect_simd_arch() == CFD_SIMD_NEON;
     263              : }
     264              : 
     265            1 : bool cfd_has_simd(void) {
     266            1 :     cfd_simd_arch_t arch = cfd_detect_simd_arch();
     267            1 :     return arch == CFD_SIMD_AVX2 || arch == CFD_SIMD_NEON;
     268              : }
     269              : 
     270           38 : const char* cfd_get_simd_name(void) {
     271           38 :     switch (cfd_detect_simd_arch()) {
     272              :         case CFD_SIMD_AVX2:
     273              :             return "avx2";
     274            0 :         case CFD_SIMD_NEON:
     275            0 :             return "neon";
     276            0 :         default:
     277            0 :             return "none";
     278              :     }
     279              : }
        

Generated by: LCOV version 2.0-1