Line data Source code
1 : /**
2 : * Optimized Projection Method NSSolver (Chorin's Method) with SIMD + OpenMP
3 : *
4 : * This implementation combines SIMD vectorization (AVX2) with OpenMP
5 : * parallelization for maximum performance on multi-core CPUs.
6 : *
7 : * - Predictor step: OpenMP parallelized (scalar inner loops)
8 : * - Corrector step: OpenMP parallelized with AVX2 SIMD inner loops
9 : * - Poisson solver: Uses SIMD Poisson solver for pressure computation
10 : */
11 :
12 : #include "cfd/boundary/boundary_conditions.h"
13 : #include "cfd/core/cfd_status.h"
14 : #include "cfd/core/grid.h"
15 : #include "cfd/core/indexing.h"
16 : #include "cfd/core/logging.h"
17 : #include "cfd/core/memory.h"
18 : #include "cfd/solvers/navier_stokes_solver.h"
19 : #include "cfd/solvers/poisson_solver.h"
20 : #include "../../linear/multigrid_internal.h"
21 : #include "cfd/solvers/energy_solver.h"
22 : #include "cfd/solvers/turbulence_solver.h"
23 : #include "../../energy/energy_solver_internal.h"
24 : #include "../../turbulence/turbulence_solver_internal.h"
25 :
26 : #include "../boundary_copy_utils.h"
27 : #include "../ns_convection_internal.h"
28 : #include "../ns_pressure_internal.h"
29 : #include "../ns_simd_backend_internal.h"
30 :
31 : #include <math.h>
32 : #include <stdio.h>
33 : #include <string.h>
34 :
35 : #ifndef M_PI
36 : #define M_PI 3.14159265358979323846
37 : #endif
38 :
39 : #ifdef _OPENMP
40 : #include <omp.h>
41 : #endif
42 :
43 : /* AVX2 detection
44 : * CFD_HAS_AVX2 is set by CMake when -DCFD_ENABLE_AVX2=ON.
45 : * This works consistently across all compilers (GCC, Clang, MSVC).
46 : */
47 : #if defined(CFD_HAS_AVX2)
48 : #include <immintrin.h>
49 : #define USE_AVX 1
50 : #else
51 : #define USE_AVX 0
52 : #endif
53 :
54 : // Physical limits
55 : #define MAX_VELOCITY 100.0
56 :
57 : typedef struct {
58 : double* u_star;
59 : double* v_star;
60 : double* w_star;
61 : double* p_new;
62 : double* rhs;
63 : double* u_new; /* used as p_temp for Poisson solver */
64 : double* T_ws; /* Reusable scratch for the energy step (avoids per-step alloc) */
65 : double* turb_ws; /* Reusable scratch for the turbulence step; always allocated */
66 : size_t nx;
67 : size_t ny;
68 : size_t nz;
69 : size_t stride_z;
70 : size_t k_start;
71 : size_t k_end;
72 : double inv_2dz;
73 : double inv_dz2;
74 : int initialized;
75 : int iter_count;
76 : poisson_solver_t* pressure; /**< Owned across the solver's life; see ns_pressure_internal.h */
77 : } projection_simd_context;
78 :
79 : /**
80 : * Take the z-spacing terms from this grid, rejecting a non-uniform one.
81 : *
82 : * Shared by init and step because they are grid facts, not init-time facts.
83 : * The step reads dz straight from the grid but used to take inv_2dz and
84 : * inv_dz2 from whatever init cached, so a grid that kept its shape and changed
85 : * its spacing made the two disagree inside a single step -- while
86 : * ns_pressure_ensure had already rebuilt the pressure solve for the new
87 : * spacing. Prediction, divergence and projection would then each use a
88 : * different z operator.
89 : */
90 0 : static cfd_status_t projection_simd_take_z_spacing(projection_simd_context* ctx,
91 : const grid* g) {
92 0 : if (g->nz > 1 && g->dz) {
93 0 : for (size_t kk = 1; kk < g->nz - 1; kk++) {
94 0 : if (fabs(g->dz[kk] - g->dz[0]) > 1e-14) {
95 0 : cfd_set_error(CFD_ERROR_INVALID,
96 : "the AVX2 projection uses a constant dz; this grid's z-spacing varies");
97 0 : return CFD_ERROR_INVALID;
98 : }
99 : }
100 0 : ctx->inv_2dz = 1.0 / (2.0 * g->dz[0]);
101 0 : ctx->inv_dz2 = 1.0 / (g->dz[0] * g->dz[0]);
102 : } else {
103 0 : ctx->inv_2dz = 0.0;
104 0 : ctx->inv_dz2 = 0.0;
105 : }
106 : return CFD_SUCCESS;
107 : }
108 :
109 : // Public API
110 : cfd_status_t projection_simd_init(struct NSSolver* solver, const grid* grid,
111 : const ns_solver_params_t* params);
112 : void projection_simd_destroy(struct NSSolver* solver);
113 : cfd_status_t projection_simd_step(struct NSSolver* solver, flow_field* field, const grid* grid,
114 : const ns_solver_params_t* params, ns_solver_stats_t* stats);
115 :
116 22 : cfd_status_t projection_simd_init(struct NSSolver* solver, const grid* grid,
117 : const ns_solver_params_t* params) {
118 22 : if (!solver || !grid) {
119 : return CFD_ERROR_INVALID;
120 : }
121 22 : if (grid->nx < 3 || grid->ny < 3 || (grid->nz > 1 && grid->nz < 3)) {
122 : return CFD_ERROR_INVALID;
123 : }
124 :
125 22 : cfd_status_t scheme_status = ns_check_convection_scheme(params, 1);
126 22 : if (scheme_status != CFD_SUCCESS) {
127 : return scheme_status;
128 : }
129 :
130 21 : cfd_status_t turb_status = ns_check_turbulence_model(params, 1);
131 21 : if (turb_status != CFD_SUCCESS) {
132 : return turb_status;
133 : }
134 :
135 : /* Same guard as the other SIMD solvers, so one build configuration gives one
136 : * error rather than three. */
137 21 : cfd_status_t simd_status = ns_check_simd_backend();
138 21 : if (simd_status != CFD_SUCCESS) {
139 : return simd_status;
140 : }
141 :
142 0 : cfd_status_t pressure_bc_status = ns_check_pressure_bc(params, 1);
143 0 : if (pressure_bc_status != CFD_SUCCESS) {
144 : return pressure_bc_status;
145 : }
146 :
147 : /* 0, not 1: this projection honours per-face walls but not the multigrid
148 : * pressure modes -- multigrid has no SIMD backend, and routing them to the
149 : * scalar one would be the silent cross-backend fallback the library forbids. */
150 0 : cfd_status_t pressure_solver_status = ns_check_pressure_solver(params, 0);
151 0 : if (pressure_solver_status != CFD_SUCCESS) {
152 : return pressure_solver_status;
153 : }
154 0 : projection_simd_context* ctx =
155 0 : (projection_simd_context*)cfd_calloc(1, sizeof(projection_simd_context));
156 0 : if (!ctx) {
157 : return CFD_ERROR_NOMEM;
158 : }
159 :
160 : /* Build the pressure solver here rather than probing for one and throwing it
161 : * away: the probe answered the same question and discarded the answer. An
162 : * unavailable SIMD backend, or a grid a mode cannot take, now fails at init
163 : * where the caller can still pick something else. */
164 : {
165 0 : poisson_solver_config_t cfg;
166 0 : cfd_status_t cfg_status = ns_pressure_config(params, POISSON_BACKEND_SIMD, &cfg);
167 0 : if (cfg_status != CFD_SUCCESS) {
168 0 : cfd_free(ctx);
169 0 : return cfg_status;
170 : }
171 0 : cfd_status_t pressure_status =
172 0 : ns_pressure_ensure(&ctx->pressure, &cfg, grid);
173 0 : if (pressure_status != CFD_SUCCESS) {
174 0 : cfd_free(ctx);
175 0 : return pressure_status;
176 : }
177 : }
178 :
179 0 : ctx->nx = grid->nx;
180 0 : ctx->ny = grid->ny;
181 0 : ctx->nz = grid->nz;
182 0 : size_t size = ctx->nx * ctx->ny * grid->nz * sizeof(double);
183 :
184 0 : if (projection_simd_take_z_spacing(ctx, grid) != CFD_SUCCESS) {
185 0 : ns_pressure_release(&ctx->pressure);
186 0 : cfd_free(ctx);
187 0 : return CFD_ERROR_INVALID;
188 : }
189 :
190 0 : size_t plane = ctx->nx * ctx->ny;
191 0 : ctx->stride_z = (grid->nz > 1) ? plane : 0;
192 0 : ctx->k_start = (grid->nz > 1) ? 1 : 0;
193 0 : ctx->k_end = (grid->nz > 1) ? (grid->nz - 1) : 1;
194 :
195 0 : size_t n_total = ctx->nx * ctx->ny * ctx->nz;
196 0 : ctx->u_star = (double*)cfd_aligned_malloc(size);
197 0 : ctx->v_star = (double*)cfd_aligned_malloc(size);
198 0 : ctx->w_star = (double*)cfd_aligned_malloc(size);
199 0 : ctx->p_new = (double*)cfd_aligned_malloc(size);
200 0 : ctx->rhs = (double*)cfd_aligned_malloc(size);
201 0 : ctx->u_new = (double*)cfd_aligned_malloc(size);
202 0 : ctx->T_ws = (double*)cfd_aligned_malloc(size);
203 0 : ctx->turb_ws = (double*)cfd_calloc(TURB_WORKSPACE_SIZE(n_total), sizeof(double));
204 :
205 0 : if (!ctx->u_star || !ctx->v_star || !ctx->w_star || !ctx->p_new ||
206 0 : !ctx->rhs || !ctx->u_new || !ctx->T_ws || !ctx->turb_ws) {
207 0 : if (ctx->u_star) { cfd_aligned_free(ctx->u_star); }
208 0 : if (ctx->v_star) { cfd_aligned_free(ctx->v_star); }
209 0 : if (ctx->w_star) { cfd_aligned_free(ctx->w_star); }
210 0 : if (ctx->p_new) { cfd_aligned_free(ctx->p_new); }
211 0 : if (ctx->rhs) { cfd_aligned_free(ctx->rhs); }
212 0 : if (ctx->u_new) { cfd_aligned_free(ctx->u_new); }
213 0 : if (ctx->T_ws) { cfd_aligned_free(ctx->T_ws); }
214 0 : if (ctx->turb_ws) { cfd_free(ctx->turb_ws); }
215 : /* solver->context is still NULL, so projection_simd_destroy will never
216 : * run for this ctx and the pressure solver -- a full CG workspace of
217 : * grid-sized vectors -- has to be released here. */
218 0 : ns_pressure_release(&ctx->pressure);
219 0 : cfd_free(ctx);
220 0 : return CFD_ERROR_NOMEM;
221 : }
222 :
223 0 : ctx->initialized = 1;
224 0 : solver->context = ctx;
225 0 : return CFD_SUCCESS;
226 : }
227 :
228 30 : void projection_simd_destroy(struct NSSolver* solver) {
229 30 : if (solver && solver->context) {
230 0 : projection_simd_context* ctx = (projection_simd_context*)solver->context;
231 0 : ns_pressure_release(&ctx->pressure);
232 0 : if (ctx->initialized) {
233 0 : cfd_aligned_free(ctx->u_star);
234 0 : cfd_aligned_free(ctx->v_star);
235 0 : cfd_aligned_free(ctx->w_star);
236 0 : cfd_aligned_free(ctx->p_new);
237 0 : cfd_aligned_free(ctx->rhs);
238 0 : cfd_aligned_free(ctx->u_new);
239 0 : cfd_aligned_free(ctx->T_ws);
240 0 : cfd_free(ctx->turb_ws);
241 : }
242 0 : cfd_free(ctx);
243 0 : solver->context = NULL;
244 : }
245 30 : }
246 :
247 1 : cfd_status_t projection_simd_step(struct NSSolver* solver, flow_field* field, const grid* grid,
248 : const ns_solver_params_t* params, ns_solver_stats_t* stats) {
249 1 : if (!solver || !solver->context || !field || !grid || !params) {
250 : return CFD_ERROR_INVALID;
251 : }
252 0 : if (field->nx < 3 || field->ny < 3 || (field->nz > 1 && field->nz < 3)) {
253 : return CFD_ERROR_INVALID;
254 : }
255 :
256 0 : projection_simd_context* ctx = (projection_simd_context*)solver->context;
257 :
258 : // Verify context matches current grid
259 0 : if (ctx->nx != field->nx || ctx->ny != field->ny || ctx->nz != field->nz) {
260 : return CFD_ERROR_INVALID;
261 : }
262 :
263 : /* Re-derive the pressure configuration from the params of THIS step, as the
264 : * scalar and OpenMP projections do. pressure_bc and pressure_solver are read
265 : * from params on every step -- the compatibility projection below reads
266 : * params->pressure_bc directly -- so a solver frozen at init would let the
267 : * two disagree: the mean subtraction would be skipped for a prescribed face
268 : * while the solver still carried the singular zero-gradient operator, and
269 : * the solve would then refuse the un-subtracted RHS. ns_pressure_ensure
270 : * rebuilds only when the configuration or the grid actually changed. */
271 : {
272 0 : poisson_solver_config_t cfg;
273 0 : cfd_status_t cfg_status = ns_pressure_config(params, POISSON_BACKEND_SIMD, &cfg);
274 0 : if (cfg_status != CFD_SUCCESS) {
275 0 : return cfg_status;
276 : }
277 0 : cfd_status_t pressure_status = ns_pressure_ensure(&ctx->pressure, &cfg, grid);
278 0 : if (pressure_status != CFD_SUCCESS) {
279 : return pressure_status;
280 : }
281 : /* p_new, rhs and u_new were allocated at init from the grid this context
282 : * was built for, not from the grid being stepped. */
283 0 : cfd_status_t shape_status =
284 0 : ns_pressure_check_shape(ctx->pressure, ctx->nx, ctx->ny, ctx->nz);
285 0 : if (shape_status != CFD_SUCCESS) {
286 : return shape_status;
287 : }
288 : /* Same reason, for the other half of the grid: the shape is checked
289 : * above, and the spacing is taken again here so the cached z terms
290 : * cannot describe an older grid than the pressure solve does. */
291 0 : cfd_status_t spacing_status = projection_simd_take_z_spacing(ctx, grid);
292 0 : if (spacing_status != CFD_SUCCESS) {
293 : return spacing_status;
294 : }
295 : }
296 :
297 0 : size_t nx = field->nx;
298 0 : size_t ny = field->ny;
299 0 : size_t size = nx * ny * ctx->nz;
300 :
301 0 : double dx = grid->dx[0];
302 0 : double dy = grid->dy[0];
303 0 : double dz = (ctx->nz > 1 && grid->dz) ? grid->dz[0] : 0.0;
304 0 : double dt = params->dt;
305 0 : double nu = params->mu; // Viscosity (treated as kinematic for ρ=1)
306 :
307 0 : const int turb_on = (params->turb_model != TURB_MODEL_NONE);
308 0 : const int upwind = (params->convection_scheme == NS_CONVECTION_SCHEME_UPWIND);
309 0 : const double inv_dz = 2.0 * ctx->inv_2dz;
310 :
311 0 : double* u_star = ctx->u_star;
312 0 : double* v_star = ctx->v_star;
313 0 : double* w_star = ctx->w_star;
314 0 : double* p_new = ctx->p_new;
315 0 : double* rhs = ctx->rhs;
316 :
317 : // Copy current field values to work buffers (includes boundaries)
318 0 : memcpy(u_star, field->u, size * sizeof(double));
319 0 : memcpy(v_star, field->v, size * sizeof(double));
320 0 : memcpy(w_star, field->w, size * sizeof(double));
321 0 : memcpy(p_new, field->p, size * sizeof(double));
322 0 : memset(rhs, 0, size * sizeof(double));
323 :
324 : // ============================================================
325 : // STEP 1: Predictor - Compute intermediate velocity u*
326 : // (OpenMP parallelized outer loop, scalar inner loop)
327 : // ============================================================
328 0 : int ny_int = (int)ny;
329 0 : int nx_int = (int)nx;
330 0 : int jj;
331 0 : (void)nx_int; /* suppress unused variable warning */
332 :
333 0 : for (size_t k = ctx->k_start; k < ctx->k_end; k++) {
334 0 : size_t k_off = k * ctx->stride_z;
335 : #ifdef _OPENMP
336 0 : #pragma omp parallel for schedule(static)
337 : #endif
338 : for (jj = 1; jj < ny_int - 1; jj++) {
339 : size_t j = (size_t)jj;
340 : for (size_t i = 1; i < nx - 1; i++) {
341 : size_t idx = k_off + IDX_2D(i, j, nx);
342 :
343 : double u = field->u[idx];
344 : double v = field->v[idx];
345 : double w = field->w[idx];
346 :
347 : // Convective terms: -u·∇u (central differences)
348 : double du_dx = (field->u[idx + 1] - field->u[idx - 1]) / (2.0 * dx);
349 : double du_dy = (field->u[idx + nx] - field->u[idx - nx]) / (2.0 * dy);
350 : double du_dz = (field->u[idx + ctx->stride_z] - field->u[idx - ctx->stride_z]) *
351 : ctx->inv_2dz;
352 :
353 : double dv_dx = (field->v[idx + 1] - field->v[idx - 1]) / (2.0 * dx);
354 : double dv_dy = (field->v[idx + nx] - field->v[idx - nx]) / (2.0 * dy);
355 : double dv_dz = (field->v[idx + ctx->stride_z] - field->v[idx - ctx->stride_z]) *
356 : ctx->inv_2dz;
357 :
358 : double dw_dx = (field->w[idx + 1] - field->w[idx - 1]) / (2.0 * dx);
359 : double dw_dy = (field->w[idx + nx] - field->w[idx - nx]) / (2.0 * dy);
360 : double dw_dz = (field->w[idx + ctx->stride_z] - field->w[idx - ctx->stride_z]) *
361 : ctx->inv_2dz;
362 :
363 : // Convective derivatives: central, or first-order upwind
364 : ns_conv_derivs_t cd = {du_dx, du_dy, du_dz, dv_dx, dv_dy, dv_dz,
365 : dw_dx, dw_dy, dw_dz};
366 : if (upwind) {
367 : ns_upwind_conv_derivs(field->u, field->v, field->w, idx,
368 : idx - 1, idx + 1, idx - nx, idx + nx,
369 : idx - ctx->stride_z, idx + ctx->stride_z,
370 : dx, dy, inv_dz, &cd);
371 : }
372 :
373 : double conv_u = (u * cd.du_dx) + (v * cd.du_dy) + (w * cd.du_dz);
374 : double conv_v = (u * cd.dv_dx) + (v * cd.dv_dy) + (w * cd.dv_dz);
375 : double conv_w = (u * cd.dw_dx) + (v * cd.dw_dy) + (w * cd.dw_dz);
376 :
377 : // Viscous terms: ν∇²u
378 : double d2u_dx2 = (field->u[idx + 1] - 2.0 * u + field->u[idx - 1]) / (dx * dx);
379 : double d2u_dy2 = (field->u[idx + nx] - 2.0 * u + field->u[idx - nx]) / (dy * dy);
380 : double d2u_dz2 = (field->u[idx + ctx->stride_z] - 2.0 * u +
381 : field->u[idx - ctx->stride_z]) * ctx->inv_dz2;
382 :
383 : double d2v_dx2 = (field->v[idx + 1] - 2.0 * v + field->v[idx - 1]) / (dx * dx);
384 : double d2v_dy2 = (field->v[idx + nx] - 2.0 * v + field->v[idx - nx]) / (dy * dy);
385 : double d2v_dz2 = (field->v[idx + ctx->stride_z] - 2.0 * v +
386 : field->v[idx - ctx->stride_z]) * ctx->inv_dz2;
387 :
388 : double d2w_dx2 = (field->w[idx + 1] - 2.0 * w + field->w[idx - 1]) / (dx * dx);
389 : double d2w_dy2 = (field->w[idx + nx] - 2.0 * w + field->w[idx - nx]) / (dy * dy);
390 : double d2w_dz2 = (field->w[idx + ctx->stride_z] - 2.0 * w +
391 : field->w[idx - ctx->stride_z]) * ctx->inv_dz2;
392 :
393 : double visc_u, visc_v, visc_w;
394 : if (!turb_on) {
395 : visc_u = nu * (d2u_dx2 + d2u_dy2 + d2u_dz2);
396 : visc_v = nu * (d2v_dx2 + d2v_dy2 + d2v_dz2);
397 : visc_w = nu * (d2w_dx2 + d2w_dy2 + d2w_dz2);
398 : } else {
399 : const double* nu_t = field->nu_t;
400 : double dx2 = dx * dx;
401 : double dy2 = dy * dy;
402 : double nu_xp = nu + 0.5 * (nu_t[idx] + nu_t[idx + 1]);
403 : double nu_xm = nu + 0.5 * (nu_t[idx] + nu_t[idx - 1]);
404 : double nu_yp = nu + 0.5 * (nu_t[idx] + nu_t[idx + nx]);
405 : double nu_ym = nu + 0.5 * (nu_t[idx] + nu_t[idx - nx]);
406 : visc_u = (nu_xp * (field->u[idx + 1] - u) -
407 : nu_xm * (u - field->u[idx - 1])) / dx2 +
408 : (nu_yp * (field->u[idx + nx] - u) -
409 : nu_ym * (u - field->u[idx - nx])) / dy2 +
410 : nu * d2u_dz2;
411 : visc_v = (nu_xp * (field->v[idx + 1] - v) -
412 : nu_xm * (v - field->v[idx - 1])) / dx2 +
413 : (nu_yp * (field->v[idx + nx] - v) -
414 : nu_ym * (v - field->v[idx - nx])) / dy2 +
415 : nu * d2v_dz2;
416 : visc_w = (nu_xp * (field->w[idx + 1] - w) -
417 : nu_xm * (w - field->w[idx - 1])) / dx2 +
418 : (nu_yp * (field->w[idx + nx] - w) -
419 : nu_ym * (w - field->w[idx - nx])) / dy2 +
420 : nu * d2w_dz2;
421 : }
422 :
423 : // Source terms
424 : double source_u = 0.0;
425 : double source_v = 0.0;
426 : double source_w = 0.0;
427 : double x_coord = grid->x[i];
428 : double y_coord = grid->y[j];
429 : double z_coord = (ctx->nz > 1 && grid->z) ? grid->z[k] : 0.0;
430 : compute_source_terms(x_coord, y_coord, z_coord, ctx->iter_count, dt, params,
431 : &source_u, &source_v, &source_w);
432 :
433 : // Boussinesq buoyancy source (no-op when beta == 0)
434 : energy_compute_buoyancy(field->T[idx], params,
435 : &source_u, &source_v, &source_w);
436 :
437 : // Intermediate velocity (without pressure gradient)
438 : u_star[idx] = u + (dt * (-conv_u + visc_u + source_u));
439 : v_star[idx] = v + (dt * (-conv_v + visc_v + source_v));
440 : w_star[idx] = w + (dt * (-conv_w + visc_w + source_w));
441 :
442 : // Limit velocities
443 : u_star[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, u_star[idx]));
444 : v_star[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, v_star[idx]));
445 : w_star[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, w_star[idx]));
446 : }
447 : }
448 : }
449 :
450 : // Copy boundary values from field to star arrays
451 0 : copy_boundary_velocities_3d(u_star, v_star, w_star, field->u, field->v, field->w,
452 : nx, ny, ctx->nz);
453 :
454 : // ============================================================
455 : // STEP 2: Solve Poisson equation for pressure
456 : // ∇²p = (ρ/dt) * ∇·u*
457 : // ============================================================
458 :
459 0 : double rho = field->rho[0];
460 0 : if (rho < 1e-10) {
461 0 : rho = 1.0;
462 : }
463 :
464 : // Compute RHS: divergence of intermediate velocity
465 0 : for (size_t k = ctx->k_start; k < ctx->k_end; k++) {
466 0 : size_t k_off = k * ctx->stride_z;
467 : #ifdef _OPENMP
468 0 : #pragma omp parallel for schedule(static)
469 : #endif
470 : for (jj = 1; jj < ny_int - 1; jj++) {
471 : size_t j = (size_t)jj;
472 : for (size_t i = 1; i < nx - 1; i++) {
473 : size_t idx = k_off + IDX_2D(i, j, nx);
474 :
475 : double du_star_dx = (u_star[idx + 1] - u_star[idx - 1]) / (2.0 * dx);
476 : double dv_star_dy = (v_star[idx + nx] - v_star[idx - nx]) / (2.0 * dy);
477 : double dw_star_dz = (w_star[idx + ctx->stride_z] -
478 : w_star[idx - ctx->stride_z]) * ctx->inv_2dz;
479 :
480 : double divergence = du_star_dx + dv_star_dy + dw_star_dz;
481 : rhs[idx] = (rho / dt) * divergence;
482 : }
483 : }
484 : }
485 :
486 : /* Neumann compatibility projection. The pressure Poisson system has
487 : * zero-gradient walls on every preset here, so it is singular with the
488 : * constants as its nullspace: the RHS must have zero interior mean or the
489 : * residual stalls on the component that lies in it. Discretely, div(u*)
490 : * only nearly integrates to zero, so the remainder is removed here rather
491 : * than assumed away. With a face prescribed the operator is nonsingular and
492 : * shifting the RHS would change the answer instead of making it exist. */
493 0 : if (poisson_walls_are_singular(¶ms->pressure_bc, ctx->nz)) {
494 0 : mg_subtract_interior_mean(rhs, nx, ny, ctx->nz);
495 : }
496 :
497 : // Solve with the solver this projection owns; ctx->u_new is the temp buffer.
498 : // The status passes through rather than flattening to MAX_ITER: divergence and
499 : // an unsolvable right-hand side are different problems with different fixes.
500 0 : poisson_solver_stats_t pstats = poisson_solver_stats_default();
501 0 : cfd_status_t poisson_status =
502 0 : poisson_solver_solve(ctx->pressure, p_new, ctx->u_new, rhs, &pstats);
503 :
504 0 : if (poisson_status != CFD_SUCCESS) {
505 : return poisson_status;
506 : }
507 :
508 : // ============================================================
509 : // STEP 3: Corrector - Project velocity to be divergence-free
510 : // u^(n+1) = u* - (dt/ρ) * ∇p
511 : // (OpenMP parallelized with SIMD inner loop)
512 : // ============================================================
513 :
514 0 : double dt_over_rho = dt / rho;
515 0 : double inv_2dx = 1.0 / (2.0 * dx);
516 0 : double inv_2dy = 1.0 / (2.0 * dy);
517 :
518 : #if USE_AVX
519 : __m256d dt_rho_vec = _mm256_set1_pd(dt_over_rho);
520 : __m256d inv_2dx_vec = _mm256_set1_pd(inv_2dx);
521 : __m256d inv_2dy_vec = _mm256_set1_pd(inv_2dy);
522 : __m256d inv_2dz_vec = _mm256_set1_pd(ctx->inv_2dz);
523 : __m256d max_vel_vec = _mm256_set1_pd(MAX_VELOCITY);
524 : __m256d neg_max_vel_vec = _mm256_set1_pd(-MAX_VELOCITY);
525 : #endif
526 :
527 0 : for (size_t k = ctx->k_start; k < ctx->k_end; k++) {
528 0 : size_t k_off = k * ctx->stride_z;
529 : #ifdef _OPENMP
530 0 : #pragma omp parallel for schedule(static)
531 : #endif
532 : for (jj = 1; jj < ny_int - 1; jj++) {
533 : size_t j = (size_t)jj;
534 : size_t i = 1;
535 :
536 : #if USE_AVX
537 : // SIMD loop - process 4 cells at once
538 : for (; i + 4 <= nx - 1; i += 4) {
539 : size_t idx = k_off + IDX_2D(i, j, nx);
540 :
541 : // Load pressure neighbors for gradient computation
542 : __m256d p_xp = _mm256_loadu_pd(&p_new[idx + 1]);
543 : __m256d p_xm = _mm256_loadu_pd(&p_new[idx - 1]);
544 : __m256d p_yp = _mm256_loadu_pd(&p_new[idx + nx]);
545 : __m256d p_ym = _mm256_loadu_pd(&p_new[idx - nx]);
546 : __m256d p_zp = _mm256_loadu_pd(&p_new[idx + ctx->stride_z]);
547 : __m256d p_zm = _mm256_loadu_pd(&p_new[idx - ctx->stride_z]);
548 :
549 : // Compute pressure gradients
550 : __m256d dp_dx = _mm256_mul_pd(_mm256_sub_pd(p_xp, p_xm), inv_2dx_vec);
551 : __m256d dp_dy = _mm256_mul_pd(_mm256_sub_pd(p_yp, p_ym), inv_2dy_vec);
552 : __m256d dp_dz = _mm256_mul_pd(_mm256_sub_pd(p_zp, p_zm), inv_2dz_vec);
553 :
554 : // Load intermediate velocities
555 : __m256d u_s = _mm256_loadu_pd(&u_star[idx]);
556 : __m256d v_s = _mm256_loadu_pd(&v_star[idx]);
557 : __m256d w_s = _mm256_loadu_pd(&w_star[idx]);
558 :
559 : // Corrector: u = u* - (dt/rho) * dp/dx
560 : __m256d u_new = _mm256_sub_pd(u_s, _mm256_mul_pd(dt_rho_vec, dp_dx));
561 : __m256d v_new = _mm256_sub_pd(v_s, _mm256_mul_pd(dt_rho_vec, dp_dy));
562 : __m256d w_new = _mm256_sub_pd(w_s, _mm256_mul_pd(dt_rho_vec, dp_dz));
563 :
564 : // Clamp velocities to [-MAX_VELOCITY, MAX_VELOCITY]
565 : u_new = _mm256_max_pd(neg_max_vel_vec, _mm256_min_pd(max_vel_vec, u_new));
566 : v_new = _mm256_max_pd(neg_max_vel_vec, _mm256_min_pd(max_vel_vec, v_new));
567 : w_new = _mm256_max_pd(neg_max_vel_vec, _mm256_min_pd(max_vel_vec, w_new));
568 :
569 : // Store results
570 : _mm256_storeu_pd(&field->u[idx], u_new);
571 : _mm256_storeu_pd(&field->v[idx], v_new);
572 : _mm256_storeu_pd(&field->w[idx], w_new);
573 : }
574 : #endif
575 :
576 : // Scalar remainder
577 : for (; i < nx - 1; i++) {
578 : size_t idx = k_off + IDX_2D(i, j, nx);
579 :
580 : double dp_dx = (p_new[idx + 1] - p_new[idx - 1]) * inv_2dx;
581 : double dp_dy = (p_new[idx + nx] - p_new[idx - nx]) * inv_2dy;
582 : double dp_dz = (p_new[idx + ctx->stride_z] - p_new[idx - ctx->stride_z]) *
583 : ctx->inv_2dz;
584 :
585 : field->u[idx] = u_star[idx] - (dt_over_rho * dp_dx);
586 : field->v[idx] = v_star[idx] - (dt_over_rho * dp_dy);
587 : field->w[idx] = w_star[idx] - (dt_over_rho * dp_dz);
588 :
589 : // Limit velocities
590 : field->u[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, field->u[idx]));
591 : field->v[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, field->v[idx]));
592 : field->w[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, field->w[idx]));
593 : }
594 : }
595 : }
596 :
597 : // Update pressure field
598 0 : memcpy(field->p, p_new, size * sizeof(double));
599 :
600 : // Energy equation: advance temperature after velocity correction
601 : {
602 0 : cfd_status_t energy_status = energy_step_explicit_avx2_with_workspace(
603 0 : field, grid, params, dt, ctx->iter_count * dt, ctx->T_ws, size);
604 0 : if (energy_status != CFD_SUCCESS) {
605 : return energy_status;
606 : }
607 : }
608 :
609 : // Apply configured thermal BCs to temperature field
610 : {
611 0 : cfd_status_t bc_status = energy_apply_thermal_bcs(field, params);
612 0 : if (bc_status != CFD_SUCCESS) {
613 : return bc_status;
614 : }
615 : }
616 :
617 : // Copy boundary velocity values from star arrays (which have caller's BCs)
618 0 : copy_boundary_velocities_3d(field->u, field->v, field->w, u_star, v_star, w_star,
619 : nx, ny, ctx->nz);
620 :
621 : // Turbulence transport: advance k-eps/SA with the corrected velocity,
622 : // then apply turbulence BCs (including wall functions).
623 : {
624 0 : cfd_status_t turb_status = turbulence_step_explicit_avx2_with_workspace(
625 0 : field, grid, params, dt, ctx->iter_count * dt, ctx->turb_ws,
626 : turb_on ? TURB_WORKSPACE_SIZE(size) : 0);
627 0 : if (turb_status == CFD_SUCCESS) {
628 0 : turb_status = turbulence_apply_bcs(field, grid, params);
629 : }
630 0 : if (turb_status != CFD_SUCCESS) {
631 0 : return turb_status;
632 : }
633 : }
634 :
635 : // Check for NaN
636 0 : for (size_t n = 0; n < size; n++) {
637 0 : if (!isfinite(field->u[n]) || !isfinite(field->v[n]) ||
638 0 : !isfinite(field->w[n]) || !isfinite(field->p[n])) {
639 : return CFD_ERROR_DIVERGED;
640 : }
641 : }
642 :
643 0 : ctx->iter_count++;
644 :
645 0 : if (stats) {
646 0 : stats->iterations = 1;
647 : }
648 :
649 : return CFD_SUCCESS;
650 : }
|