Line data Source code
1 : /**
2 : * CPU Feature Detection Implementation
3 : *
4 : * Runtime detection of CPU SIMD capabilities using platform-specific methods:
5 : * - x86/x64 (MSVC): __cpuid/__cpuidex intrinsics + XGETBV for OS support
6 : * - x86/x64 (GCC/Clang): __get_cpuid_count + xgetbv for OS support
7 : * - ARM64: NEON is always available
8 : *
9 : * IMPORTANT: For AVX2, we must verify both:
10 : * 1. CPU support (CPUID leaf 7, EBX bit 5)
11 : * 2. OS support (OSXSAVE enabled + XCR0 bits 1,2 set for AVX state saving)
12 : *
13 : * Without OS support verification, AVX instructions will cause illegal
14 : * instruction exceptions on systems where the OS hasn't enabled AVX.
15 : *
16 : * Thread Safety:
17 : * The cache uses atomic operations with proper memory ordering to ensure
18 : * thread-safe initialization. The detection logic is idempotent (always
19 : * produces the same result), so multiple threads racing to initialize is
20 : * safe - they will all compute and store the same value.
21 : *
22 : * We use:
23 : * - C11 stdatomic.h when available (GCC 4.9+, Clang 3.1+, MSVC 2022+)
24 : * - GCC __atomic builtins as fallback (GCC 4.7+)
25 : * - MSVC Interlocked functions as final fallback
26 : */
27 :
28 : #include "cfd/core/cpu_features.h"
29 :
30 : /* ============================================================================
31 : * Atomic Operations Abstraction
32 : *
33 : * Provides atomic load/store with proper memory ordering across platforms.
34 : * Priority: C11 atomics > GCC atomics > MSVC Interlocked > volatile fallback
35 : * ============================================================================ */
36 :
37 : /* Check for C11 atomics support */
38 : #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
39 : /* C11 atomics available */
40 : #include <stdatomic.h>
41 : #define CFD_HAS_C11_ATOMICS 1
42 : #elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 7))
43 : /* GCC 4.7+ atomic builtins */
44 : #define CFD_HAS_GCC_ATOMICS 1
45 : #elif defined(_MSC_VER)
46 : /* MSVC Interlocked functions */
47 : #include <intrin.h>
48 : #define CFD_HAS_MSVC_ATOMICS 1
49 : #endif
50 :
51 : /* Atomic cache type and operations */
52 : #if defined(CFD_HAS_C11_ATOMICS)
53 : static _Atomic int g_simd_arch_cache = -1;
54 :
55 34502 : static inline int atomic_cache_load(void) {
56 34502 : return atomic_load_explicit(&g_simd_arch_cache, memory_order_acquire);
57 : }
58 :
59 28 : static inline void atomic_cache_store(int value) {
60 28 : atomic_store_explicit(&g_simd_arch_cache, value, memory_order_release);
61 : }
62 :
63 : #elif defined(CFD_HAS_GCC_ATOMICS)
64 : static int g_simd_arch_cache = -1;
65 :
66 : static inline int atomic_cache_load(void) {
67 : return __atomic_load_n(&g_simd_arch_cache, __ATOMIC_ACQUIRE);
68 : }
69 :
70 : static inline void atomic_cache_store(int value) {
71 : __atomic_store_n(&g_simd_arch_cache, value, __ATOMIC_RELEASE);
72 : }
73 :
74 : #elif defined(CFD_HAS_MSVC_ATOMICS)
75 : static volatile long g_simd_arch_cache = -1;
76 :
77 : static inline int atomic_cache_load(void) {
78 : /* _InterlockedCompareExchange provides full barrier on x86/x64
79 : * This is the intrinsic form (note the underscore prefix) */
80 : return (int)_InterlockedCompareExchange(&g_simd_arch_cache, -1, -1);
81 : }
82 :
83 : static inline void atomic_cache_store(int value) {
84 : _InterlockedExchange(&g_simd_arch_cache, (long)value);
85 : }
86 :
87 : #else
88 : /* Fallback: volatile with no guarantees beyond single-threaded correctness.
89 : * This should rarely be hit on modern compilers. */
90 : #warning "No atomic primitives available - cache may not be thread-safe"
91 : static volatile int g_simd_arch_cache = -1;
92 :
93 : static inline int atomic_cache_load(void) {
94 : return g_simd_arch_cache;
95 : }
96 :
97 : static inline void atomic_cache_store(int value) {
98 : g_simd_arch_cache = value;
99 : }
100 : #endif
101 :
102 : /* ============================================================================
103 : * Platform-Specific CPU Detection Headers
104 : * ============================================================================ */
105 :
106 : #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
107 : #include <intrin.h>
108 : #define CFD_RUNTIME_X86_MSVC 1
109 : #elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__))
110 : #include <cpuid.h>
111 : #define CFD_RUNTIME_X86_GCC 1
112 :
113 : /**
114 : * Inline xgetbv for GCC/Clang.
115 : *
116 : * We use inline assembly instead of _xgetbv() intrinsic because the intrinsic
117 : * requires -mxsave compiler flag, but we want this file to compile without
118 : * requiring any special CPU feature flags (since it's doing runtime detection).
119 : *
120 : * IMPORTANT: This function must ONLY be called after verifying OSXSAVE support
121 : * via CPUID leaf 1, ECX bit 27. The OSXSAVE bit indicates that:
122 : * 1. The CPU supports the XGETBV instruction
123 : * 2. The OS has enabled XSAVE/XRSTOR for context switching
124 : *
125 : * Calling xgetbv without OSXSAVE support will cause an undefined opcode (#UD)
126 : * exception. The detect_simd_arch_impl() function enforces this requirement
127 : * by only calling cfd_xgetbv() inside an if(osxsave) block.
128 : *
129 : * Register usage (identical on 32-bit and 64-bit x86):
130 : * - Input: ECX = XCR number (0 for XCR0)
131 : * - Output: EDX:EAX = XCR value (64-bit)
132 : * - xgetbv does NOT modify XMM/YMM registers - no SIMD clobbers needed
133 : *
134 : * The "memory" clobber ensures the compiler doesn't reorder this with
135 : * memory operations, though xgetbv itself doesn't access memory.
136 : */
137 28 : static inline unsigned long long cfd_xgetbv(unsigned int xcr) {
138 28 : unsigned int eax, edx;
139 56 : __asm__ __volatile__(
140 : "xgetbv"
141 : : "=a"(eax), "=d"(edx)
142 : : "c"(xcr)
143 : : "memory"
144 : );
145 28 : return ((unsigned long long)edx << 32) | eax;
146 : }
147 : #endif
148 :
149 : /* ============================================================================
150 : * Runtime Detection Implementation
151 : * ============================================================================ */
152 :
153 : /**
154 : * Perform the actual SIMD detection.
155 : * This is a pure function - always returns the same result for a given CPU.
156 : */
157 28 : static cfd_simd_arch_t detect_simd_arch_impl(void) {
158 : #if defined(CFD_RUNTIME_X86_MSVC)
159 : /* MSVC on x86/x64: Use __cpuid intrinsic */
160 : int cpuInfo[4] = {0};
161 : __cpuid(cpuInfo, 0);
162 : int nIds = cpuInfo[0];
163 :
164 : if (nIds >= 7) {
165 : /* First check CPUID leaf 1 for OSXSAVE support (ECX bit 27) */
166 : __cpuid(cpuInfo, 1);
167 : int osxsave = (cpuInfo[2] & (1 << 27)) != 0;
168 : /* FMA3 (ECX bit 12) is a separate capability from AVX2, and the AVX2
169 : * build relies on it: kernels call _mm256_fmadd_* and the library is
170 : * compiled with FMA enabled. */
171 : int fma = (cpuInfo[2] & (1 << 12)) != 0;
172 :
173 : if (osxsave && fma) {
174 : /* OSXSAVE is enabled, now check XCR0 for AVX state support */
175 : /* XCR0 bits: bit 1 = SSE state, bit 2 = AVX state */
176 : /* Both must be set for AVX to work */
177 : unsigned long long xcr0 = _xgetbv(0);
178 : int avx_os_support = ((xcr0 & 0x6) == 0x6);
179 :
180 : if (avx_os_support) {
181 : /* Now check CPU support for AVX2 */
182 : __cpuidex(cpuInfo, 7, 0);
183 : /* Check AVX2 bit (EBX bit 5) */
184 : if (cpuInfo[1] & (1 << 5)) {
185 : return CFD_SIMD_AVX2;
186 : }
187 : }
188 : }
189 : }
190 : return CFD_SIMD_NONE;
191 :
192 : #elif defined(CFD_RUNTIME_X86_GCC)
193 : /* GCC/Clang on x86/x64: Use __get_cpuid_count */
194 28 : unsigned int eax, ebx, ecx, edx;
195 :
196 : /* First check CPUID leaf 1 for OSXSAVE support (ECX bit 27) */
197 56 : if (__get_cpuid(1, &eax, &ebx, &ecx, &edx)) {
198 28 : int osxsave = (ecx & (1 << 27)) != 0;
199 : /* FMA3 (ECX bit 12): see the MSVC branch above. */
200 28 : int fma = (ecx & (1 << 12)) != 0;
201 :
202 28 : if (osxsave && fma) {
203 : /* OSXSAVE is enabled, now check XCR0 for AVX state support */
204 : /* XCR0 bits: bit 1 = SSE state, bit 2 = AVX state */
205 : /* Both must be set for AVX to work */
206 28 : unsigned long long xcr0 = cfd_xgetbv(0);
207 28 : int avx_os_support = ((xcr0 & 0x6) == 0x6);
208 :
209 28 : if (avx_os_support) {
210 : /* Now check CPU support for AVX2 */
211 28 : if (__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) {
212 : /* Check AVX2 bit (EBX bit 5) */
213 28 : if (ebx & (1 << 5)) {
214 : return CFD_SIMD_AVX2;
215 : }
216 : }
217 : }
218 : }
219 : }
220 : return CFD_SIMD_NONE;
221 :
222 : #elif defined(__aarch64__) || defined(_M_ARM64)
223 : /* ARM64: NEON is always available on AArch64 */
224 : return CFD_SIMD_NEON;
225 :
226 : #elif defined(__ARM_NEON) || defined(__ARM_NEON__)
227 : /* ARMv7 with NEON: Assume available if compiled with NEON support */
228 : return CFD_SIMD_NEON;
229 :
230 : #else
231 : return CFD_SIMD_NONE;
232 : #endif
233 : }
234 :
235 34502 : cfd_simd_arch_t cfd_detect_simd_arch(void) {
236 : /* Read cached result with acquire semantics */
237 34502 : int cached = atomic_cache_load();
238 :
239 34502 : if (cached >= 0) {
240 : return (cfd_simd_arch_t)cached;
241 : }
242 :
243 : /* Perform detection - result will be the same regardless of which thread
244 : * computes it, so racing here is safe (just wastes some CPU cycles). */
245 28 : cfd_simd_arch_t detected = detect_simd_arch_impl();
246 :
247 : /* Store result with release semantics to ensure visibility to other threads */
248 28 : atomic_cache_store((int)detected);
249 :
250 28 : return detected;
251 : }
252 :
253 : /* ============================================================================
254 : * Convenience Functions
255 : * ============================================================================ */
256 :
257 0 : bool cfd_has_avx2(void) {
258 0 : return cfd_detect_simd_arch() == CFD_SIMD_AVX2;
259 : }
260 :
261 0 : bool cfd_has_neon(void) {
262 0 : return cfd_detect_simd_arch() == CFD_SIMD_NEON;
263 : }
264 :
265 1 : bool cfd_has_simd(void) {
266 1 : cfd_simd_arch_t arch = cfd_detect_simd_arch();
267 1 : return arch == CFD_SIMD_AVX2 || arch == CFD_SIMD_NEON;
268 : }
269 :
270 38 : const char* cfd_get_simd_name(void) {
271 38 : switch (cfd_detect_simd_arch()) {
272 : case CFD_SIMD_AVX2:
273 : return "avx2";
274 0 : case CFD_SIMD_NEON:
275 0 : return "neon";
276 0 : default:
277 0 : return "none";
278 : }
279 : }
|