LCOV - code coverage report
Current view: top level - solvers/navier_stokes/avx2 - solver_projection_avx2.c (source / functions) Coverage Total Hit
Test: coverage.info Lines: 8.2 % 171 14
Test Date: 2026-10-05 04:47:18 Functions: 75.0 % 4 3

            Line data    Source code
       1              : /**
       2              :  * Optimized Projection Method NSSolver (Chorin's Method) with SIMD + OpenMP
       3              :  *
       4              :  * This implementation combines SIMD vectorization (AVX2) with OpenMP
       5              :  * parallelization for maximum performance on multi-core CPUs.
       6              :  *
       7              :  * - Predictor step: OpenMP parallelized (scalar inner loops)
       8              :  * - Corrector step: OpenMP parallelized with AVX2 SIMD inner loops
       9              :  * - Poisson solver: Uses SIMD Poisson solver for pressure computation
      10              :  */
      11              : 
      12              : #include "cfd/boundary/boundary_conditions.h"
      13              : #include "cfd/core/cfd_status.h"
      14              : #include "cfd/core/grid.h"
      15              : #include "cfd/core/indexing.h"
      16              : #include "cfd/core/logging.h"
      17              : #include "cfd/core/memory.h"
      18              : #include "cfd/solvers/navier_stokes_solver.h"
      19              : #include "cfd/solvers/poisson_solver.h"
      20              : #include "../../linear/multigrid_internal.h"
      21              : #include "cfd/solvers/energy_solver.h"
      22              : #include "cfd/solvers/turbulence_solver.h"
      23              : #include "../../energy/energy_solver_internal.h"
      24              : #include "../../turbulence/turbulence_solver_internal.h"
      25              : 
      26              : #include "../boundary_copy_utils.h"
      27              : #include "../ns_convection_internal.h"
      28              : #include "../ns_pressure_internal.h"
      29              : #include "../ns_simd_backend_internal.h"
      30              : 
      31              : #include <math.h>
      32              : #include <stdio.h>
      33              : #include <string.h>
      34              : 
      35              : #ifndef M_PI
      36              : #define M_PI 3.14159265358979323846
      37              : #endif
      38              : 
      39              : #ifdef _OPENMP
      40              : #include <omp.h>
      41              : #endif
      42              : 
      43              : /* AVX2 detection
      44              :  * CFD_HAS_AVX2 is set by CMake when -DCFD_ENABLE_AVX2=ON.
      45              :  * This works consistently across all compilers (GCC, Clang, MSVC).
      46              :  */
      47              : #if defined(CFD_HAS_AVX2)
      48              : #include <immintrin.h>
      49              : #define USE_AVX 1
      50              : #else
      51              : #define USE_AVX 0
      52              : #endif
      53              : 
      54              : // Physical limits
      55              : #define MAX_VELOCITY 100.0
      56              : 
      57              : typedef struct {
      58              :     double* u_star;
      59              :     double* v_star;
      60              :     double* w_star;
      61              :     double* p_new;
      62              :     double* rhs;
      63              :     double* u_new;  /* used as p_temp for Poisson solver */
      64              :     double* T_ws;    /* Reusable scratch for the energy step (avoids per-step alloc) */
      65              :     double* turb_ws; /* Reusable scratch for the turbulence step; always allocated */
      66              :     size_t nx;
      67              :     size_t ny;
      68              :     size_t nz;
      69              :     size_t stride_z;
      70              :     size_t k_start;
      71              :     size_t k_end;
      72              :     double inv_2dz;
      73              :     double inv_dz2;
      74              :     int initialized;
      75              :     int iter_count;
      76              :     poisson_solver_t* pressure; /**< Owned across the solver's life; see ns_pressure_internal.h */
      77              : } projection_simd_context;
      78              : 
      79              : /**
      80              :  * Take the z-spacing terms from this grid, rejecting a non-uniform one.
      81              :  *
      82              :  * Shared by init and step because they are grid facts, not init-time facts.
      83              :  * The step reads dz straight from the grid but used to take inv_2dz and
      84              :  * inv_dz2 from whatever init cached, so a grid that kept its shape and changed
      85              :  * its spacing made the two disagree inside a single step -- while
      86              :  * ns_pressure_ensure had already rebuilt the pressure solve for the new
      87              :  * spacing. Prediction, divergence and projection would then each use a
      88              :  * different z operator.
      89              :  */
      90            0 : static cfd_status_t projection_simd_take_z_spacing(projection_simd_context* ctx,
      91              :                                                    const grid* g) {
      92            0 :     if (g->nz > 1 && g->dz) {
      93            0 :         for (size_t kk = 1; kk < g->nz - 1; kk++) {
      94            0 :             if (fabs(g->dz[kk] - g->dz[0]) > 1e-14) {
      95            0 :                 cfd_set_error(CFD_ERROR_INVALID,
      96              :                     "the AVX2 projection uses a constant dz; this grid's z-spacing varies");
      97            0 :                 return CFD_ERROR_INVALID;
      98              :             }
      99              :         }
     100            0 :         ctx->inv_2dz = 1.0 / (2.0 * g->dz[0]);
     101            0 :         ctx->inv_dz2 = 1.0 / (g->dz[0] * g->dz[0]);
     102              :     } else {
     103            0 :         ctx->inv_2dz = 0.0;
     104            0 :         ctx->inv_dz2 = 0.0;
     105              :     }
     106              :     return CFD_SUCCESS;
     107              : }
     108              : 
     109              : // Public API
     110              : cfd_status_t projection_simd_init(struct NSSolver* solver, const grid* grid,
     111              :                                   const ns_solver_params_t* params);
     112              : void projection_simd_destroy(struct NSSolver* solver);
     113              : cfd_status_t projection_simd_step(struct NSSolver* solver, flow_field* field, const grid* grid,
     114              :                                   const ns_solver_params_t* params, ns_solver_stats_t* stats);
     115              : 
     116           22 : cfd_status_t projection_simd_init(struct NSSolver* solver, const grid* grid,
     117              :                                   const ns_solver_params_t* params) {
     118           22 :     if (!solver || !grid) {
     119              :         return CFD_ERROR_INVALID;
     120              :     }
     121           22 :     if (grid->nx < 3 || grid->ny < 3 || (grid->nz > 1 && grid->nz < 3)) {
     122              :         return CFD_ERROR_INVALID;
     123              :     }
     124              : 
     125           22 :     cfd_status_t scheme_status = ns_check_convection_scheme(params, 1);
     126           22 :     if (scheme_status != CFD_SUCCESS) {
     127              :         return scheme_status;
     128              :     }
     129              : 
     130           21 :     cfd_status_t turb_status = ns_check_turbulence_model(params, 1);
     131           21 :     if (turb_status != CFD_SUCCESS) {
     132              :         return turb_status;
     133              :     }
     134              : 
     135              :     /* Same guard as the other SIMD solvers, so one build configuration gives one
     136              :      * error rather than three. */
     137           21 :     cfd_status_t simd_status = ns_check_simd_backend();
     138           21 :     if (simd_status != CFD_SUCCESS) {
     139              :         return simd_status;
     140              :     }
     141              : 
     142            0 :     cfd_status_t pressure_bc_status = ns_check_pressure_bc(params, 1);
     143            0 :     if (pressure_bc_status != CFD_SUCCESS) {
     144              :         return pressure_bc_status;
     145              :     }
     146              : 
     147              :     /* 0, not 1: this projection honours per-face walls but not the multigrid
     148              :      * pressure modes -- multigrid has no SIMD backend, and routing them to the
     149              :      * scalar one would be the silent cross-backend fallback the library forbids. */
     150            0 :     cfd_status_t pressure_solver_status = ns_check_pressure_solver(params, 0);
     151            0 :     if (pressure_solver_status != CFD_SUCCESS) {
     152              :         return pressure_solver_status;
     153              :     }
     154            0 :     projection_simd_context* ctx =
     155            0 :         (projection_simd_context*)cfd_calloc(1, sizeof(projection_simd_context));
     156            0 :     if (!ctx) {
     157              :         return CFD_ERROR_NOMEM;
     158              :     }
     159              : 
     160              :     /* Build the pressure solver here rather than probing for one and throwing it
     161              :      * away: the probe answered the same question and discarded the answer. An
     162              :      * unavailable SIMD backend, or a grid a mode cannot take, now fails at init
     163              :      * where the caller can still pick something else. */
     164              :     {
     165            0 :         poisson_solver_config_t cfg;
     166            0 :         cfd_status_t cfg_status = ns_pressure_config(params, POISSON_BACKEND_SIMD, &cfg);
     167            0 :         if (cfg_status != CFD_SUCCESS) {
     168            0 :             cfd_free(ctx);
     169            0 :             return cfg_status;
     170              :         }
     171            0 :         cfd_status_t pressure_status =
     172            0 :             ns_pressure_ensure(&ctx->pressure, &cfg, grid);
     173            0 :         if (pressure_status != CFD_SUCCESS) {
     174            0 :             cfd_free(ctx);
     175            0 :             return pressure_status;
     176              :         }
     177              :     }
     178              : 
     179            0 :     ctx->nx = grid->nx;
     180            0 :     ctx->ny = grid->ny;
     181            0 :     ctx->nz = grid->nz;
     182            0 :     size_t size = ctx->nx * ctx->ny * grid->nz * sizeof(double);
     183              : 
     184            0 :     if (projection_simd_take_z_spacing(ctx, grid) != CFD_SUCCESS) {
     185            0 :         ns_pressure_release(&ctx->pressure);
     186            0 :         cfd_free(ctx);
     187            0 :         return CFD_ERROR_INVALID;
     188              :     }
     189              : 
     190            0 :     size_t plane = ctx->nx * ctx->ny;
     191            0 :     ctx->stride_z = (grid->nz > 1) ? plane : 0;
     192            0 :     ctx->k_start  = (grid->nz > 1) ? 1 : 0;
     193            0 :     ctx->k_end    = (grid->nz > 1) ? (grid->nz - 1) : 1;
     194              : 
     195            0 :     size_t n_total = ctx->nx * ctx->ny * ctx->nz;
     196            0 :     ctx->u_star  = (double*)cfd_aligned_malloc(size);
     197            0 :     ctx->v_star  = (double*)cfd_aligned_malloc(size);
     198            0 :     ctx->w_star  = (double*)cfd_aligned_malloc(size);
     199            0 :     ctx->p_new   = (double*)cfd_aligned_malloc(size);
     200            0 :     ctx->rhs     = (double*)cfd_aligned_malloc(size);
     201            0 :     ctx->u_new   = (double*)cfd_aligned_malloc(size);
     202            0 :     ctx->T_ws    = (double*)cfd_aligned_malloc(size);
     203            0 :     ctx->turb_ws = (double*)cfd_calloc(TURB_WORKSPACE_SIZE(n_total), sizeof(double));
     204              : 
     205            0 :     if (!ctx->u_star || !ctx->v_star || !ctx->w_star || !ctx->p_new ||
     206            0 :         !ctx->rhs || !ctx->u_new || !ctx->T_ws || !ctx->turb_ws) {
     207            0 :         if (ctx->u_star)   { cfd_aligned_free(ctx->u_star); }
     208            0 :         if (ctx->v_star)   { cfd_aligned_free(ctx->v_star); }
     209            0 :         if (ctx->w_star)   { cfd_aligned_free(ctx->w_star); }
     210            0 :         if (ctx->p_new)    { cfd_aligned_free(ctx->p_new); }
     211            0 :         if (ctx->rhs)      { cfd_aligned_free(ctx->rhs); }
     212            0 :         if (ctx->u_new)    { cfd_aligned_free(ctx->u_new); }
     213            0 :         if (ctx->T_ws)     { cfd_aligned_free(ctx->T_ws); }
     214            0 :         if (ctx->turb_ws)  { cfd_free(ctx->turb_ws); }
     215              :         /* solver->context is still NULL, so projection_simd_destroy will never
     216              :          * run for this ctx and the pressure solver -- a full CG workspace of
     217              :          * grid-sized vectors -- has to be released here. */
     218            0 :         ns_pressure_release(&ctx->pressure);
     219            0 :         cfd_free(ctx);
     220            0 :         return CFD_ERROR_NOMEM;
     221              :     }
     222              : 
     223            0 :     ctx->initialized = 1;
     224            0 :     solver->context = ctx;
     225            0 :     return CFD_SUCCESS;
     226              : }
     227              : 
     228           30 : void projection_simd_destroy(struct NSSolver* solver) {
     229           30 :     if (solver && solver->context) {
     230            0 :         projection_simd_context* ctx = (projection_simd_context*)solver->context;
     231            0 :         ns_pressure_release(&ctx->pressure);
     232            0 :         if (ctx->initialized) {
     233            0 :             cfd_aligned_free(ctx->u_star);
     234            0 :             cfd_aligned_free(ctx->v_star);
     235            0 :             cfd_aligned_free(ctx->w_star);
     236            0 :             cfd_aligned_free(ctx->p_new);
     237            0 :             cfd_aligned_free(ctx->rhs);
     238            0 :             cfd_aligned_free(ctx->u_new);
     239            0 :             cfd_aligned_free(ctx->T_ws);
     240            0 :             cfd_free(ctx->turb_ws);
     241              :         }
     242            0 :         cfd_free(ctx);
     243            0 :         solver->context = NULL;
     244              :     }
     245           30 : }
     246              : 
     247            1 : cfd_status_t projection_simd_step(struct NSSolver* solver, flow_field* field, const grid* grid,
     248              :                                   const ns_solver_params_t* params, ns_solver_stats_t* stats) {
     249            1 :     if (!solver || !solver->context || !field || !grid || !params) {
     250              :         return CFD_ERROR_INVALID;
     251              :     }
     252            0 :     if (field->nx < 3 || field->ny < 3 || (field->nz > 1 && field->nz < 3)) {
     253              :         return CFD_ERROR_INVALID;
     254              :     }
     255              : 
     256            0 :     projection_simd_context* ctx = (projection_simd_context*)solver->context;
     257              : 
     258              :     // Verify context matches current grid
     259            0 :     if (ctx->nx != field->nx || ctx->ny != field->ny || ctx->nz != field->nz) {
     260              :         return CFD_ERROR_INVALID;
     261              :     }
     262              : 
     263              :     /* Re-derive the pressure configuration from the params of THIS step, as the
     264              :      * scalar and OpenMP projections do. pressure_bc and pressure_solver are read
     265              :      * from params on every step -- the compatibility projection below reads
     266              :      * params->pressure_bc directly -- so a solver frozen at init would let the
     267              :      * two disagree: the mean subtraction would be skipped for a prescribed face
     268              :      * while the solver still carried the singular zero-gradient operator, and
     269              :      * the solve would then refuse the un-subtracted RHS. ns_pressure_ensure
     270              :      * rebuilds only when the configuration or the grid actually changed. */
     271              :     {
     272            0 :         poisson_solver_config_t cfg;
     273            0 :         cfd_status_t cfg_status = ns_pressure_config(params, POISSON_BACKEND_SIMD, &cfg);
     274            0 :         if (cfg_status != CFD_SUCCESS) {
     275            0 :             return cfg_status;
     276              :         }
     277            0 :         cfd_status_t pressure_status = ns_pressure_ensure(&ctx->pressure, &cfg, grid);
     278            0 :         if (pressure_status != CFD_SUCCESS) {
     279              :             return pressure_status;
     280              :         }
     281              :         /* p_new, rhs and u_new were allocated at init from the grid this context
     282              :          * was built for, not from the grid being stepped. */
     283            0 :         cfd_status_t shape_status =
     284            0 :             ns_pressure_check_shape(ctx->pressure, ctx->nx, ctx->ny, ctx->nz);
     285            0 :         if (shape_status != CFD_SUCCESS) {
     286              :             return shape_status;
     287              :         }
     288              :         /* Same reason, for the other half of the grid: the shape is checked
     289              :          * above, and the spacing is taken again here so the cached z terms
     290              :          * cannot describe an older grid than the pressure solve does. */
     291            0 :         cfd_status_t spacing_status = projection_simd_take_z_spacing(ctx, grid);
     292            0 :         if (spacing_status != CFD_SUCCESS) {
     293              :             return spacing_status;
     294              :         }
     295              :     }
     296              : 
     297            0 :     size_t nx = field->nx;
     298            0 :     size_t ny = field->ny;
     299            0 :     size_t size = nx * ny * ctx->nz;
     300              : 
     301            0 :     double dx = grid->dx[0];
     302            0 :     double dy = grid->dy[0];
     303            0 :     double dz = (ctx->nz > 1 && grid->dz) ? grid->dz[0] : 0.0;
     304            0 :     double dt = params->dt;
     305            0 :     double nu = params->mu;  // Viscosity (treated as kinematic for ρ=1)
     306              : 
     307            0 :     const int turb_on = (params->turb_model != TURB_MODEL_NONE);
     308            0 :     const int upwind = (params->convection_scheme == NS_CONVECTION_SCHEME_UPWIND);
     309            0 :     const double inv_dz = 2.0 * ctx->inv_2dz;
     310              : 
     311            0 :     double* u_star = ctx->u_star;
     312            0 :     double* v_star = ctx->v_star;
     313            0 :     double* w_star = ctx->w_star;
     314            0 :     double* p_new = ctx->p_new;
     315            0 :     double* rhs = ctx->rhs;
     316              : 
     317              :     // Copy current field values to work buffers (includes boundaries)
     318            0 :     memcpy(u_star, field->u, size * sizeof(double));
     319            0 :     memcpy(v_star, field->v, size * sizeof(double));
     320            0 :     memcpy(w_star, field->w, size * sizeof(double));
     321            0 :     memcpy(p_new, field->p, size * sizeof(double));
     322            0 :     memset(rhs, 0, size * sizeof(double));
     323              : 
     324              :     // ============================================================
     325              :     // STEP 1: Predictor - Compute intermediate velocity u*
     326              :     // (OpenMP parallelized outer loop, scalar inner loop)
     327              :     // ============================================================
     328            0 :     int ny_int = (int)ny;
     329            0 :     int nx_int = (int)nx;
     330            0 :     int jj;
     331            0 :     (void)nx_int;  /* suppress unused variable warning */
     332              : 
     333            0 :     for (size_t k = ctx->k_start; k < ctx->k_end; k++) {
     334            0 :         size_t k_off = k * ctx->stride_z;
     335              : #ifdef _OPENMP
     336            0 :         #pragma omp parallel for schedule(static)
     337              : #endif
     338              :         for (jj = 1; jj < ny_int - 1; jj++) {
     339              :             size_t j = (size_t)jj;
     340              :             for (size_t i = 1; i < nx - 1; i++) {
     341              :                 size_t idx = k_off + IDX_2D(i, j, nx);
     342              : 
     343              :                 double u = field->u[idx];
     344              :                 double v = field->v[idx];
     345              :                 double w = field->w[idx];
     346              : 
     347              :                 // Convective terms: -u·∇u (central differences)
     348              :                 double du_dx = (field->u[idx + 1] - field->u[idx - 1]) / (2.0 * dx);
     349              :                 double du_dy = (field->u[idx + nx] - field->u[idx - nx]) / (2.0 * dy);
     350              :                 double du_dz = (field->u[idx + ctx->stride_z] - field->u[idx - ctx->stride_z]) *
     351              :                                ctx->inv_2dz;
     352              : 
     353              :                 double dv_dx = (field->v[idx + 1] - field->v[idx - 1]) / (2.0 * dx);
     354              :                 double dv_dy = (field->v[idx + nx] - field->v[idx - nx]) / (2.0 * dy);
     355              :                 double dv_dz = (field->v[idx + ctx->stride_z] - field->v[idx - ctx->stride_z]) *
     356              :                                ctx->inv_2dz;
     357              : 
     358              :                 double dw_dx = (field->w[idx + 1] - field->w[idx - 1]) / (2.0 * dx);
     359              :                 double dw_dy = (field->w[idx + nx] - field->w[idx - nx]) / (2.0 * dy);
     360              :                 double dw_dz = (field->w[idx + ctx->stride_z] - field->w[idx - ctx->stride_z]) *
     361              :                                ctx->inv_2dz;
     362              : 
     363              :                 // Convective derivatives: central, or first-order upwind
     364              :                 ns_conv_derivs_t cd = {du_dx, du_dy, du_dz, dv_dx, dv_dy, dv_dz,
     365              :                                        dw_dx, dw_dy, dw_dz};
     366              :                 if (upwind) {
     367              :                     ns_upwind_conv_derivs(field->u, field->v, field->w, idx,
     368              :                                           idx - 1, idx + 1, idx - nx, idx + nx,
     369              :                                           idx - ctx->stride_z, idx + ctx->stride_z,
     370              :                                           dx, dy, inv_dz, &cd);
     371              :                 }
     372              : 
     373              :                 double conv_u = (u * cd.du_dx) + (v * cd.du_dy) + (w * cd.du_dz);
     374              :                 double conv_v = (u * cd.dv_dx) + (v * cd.dv_dy) + (w * cd.dv_dz);
     375              :                 double conv_w = (u * cd.dw_dx) + (v * cd.dw_dy) + (w * cd.dw_dz);
     376              : 
     377              :                 // Viscous terms: ν∇²u
     378              :                 double d2u_dx2 = (field->u[idx + 1] - 2.0 * u + field->u[idx - 1]) / (dx * dx);
     379              :                 double d2u_dy2 = (field->u[idx + nx] - 2.0 * u + field->u[idx - nx]) / (dy * dy);
     380              :                 double d2u_dz2 = (field->u[idx + ctx->stride_z] - 2.0 * u +
     381              :                                   field->u[idx - ctx->stride_z]) * ctx->inv_dz2;
     382              : 
     383              :                 double d2v_dx2 = (field->v[idx + 1] - 2.0 * v + field->v[idx - 1]) / (dx * dx);
     384              :                 double d2v_dy2 = (field->v[idx + nx] - 2.0 * v + field->v[idx - nx]) / (dy * dy);
     385              :                 double d2v_dz2 = (field->v[idx + ctx->stride_z] - 2.0 * v +
     386              :                                   field->v[idx - ctx->stride_z]) * ctx->inv_dz2;
     387              : 
     388              :                 double d2w_dx2 = (field->w[idx + 1] - 2.0 * w + field->w[idx - 1]) / (dx * dx);
     389              :                 double d2w_dy2 = (field->w[idx + nx] - 2.0 * w + field->w[idx - nx]) / (dy * dy);
     390              :                 double d2w_dz2 = (field->w[idx + ctx->stride_z] - 2.0 * w +
     391              :                                   field->w[idx - ctx->stride_z]) * ctx->inv_dz2;
     392              : 
     393              :                 double visc_u, visc_v, visc_w;
     394              :                 if (!turb_on) {
     395              :                     visc_u = nu * (d2u_dx2 + d2u_dy2 + d2u_dz2);
     396              :                     visc_v = nu * (d2v_dx2 + d2v_dy2 + d2v_dz2);
     397              :                     visc_w = nu * (d2w_dx2 + d2w_dy2 + d2w_dz2);
     398              :                 } else {
     399              :                     const double* nu_t = field->nu_t;
     400              :                     double dx2 = dx * dx;
     401              :                     double dy2 = dy * dy;
     402              :                     double nu_xp = nu + 0.5 * (nu_t[idx] + nu_t[idx + 1]);
     403              :                     double nu_xm = nu + 0.5 * (nu_t[idx] + nu_t[idx - 1]);
     404              :                     double nu_yp = nu + 0.5 * (nu_t[idx] + nu_t[idx + nx]);
     405              :                     double nu_ym = nu + 0.5 * (nu_t[idx] + nu_t[idx - nx]);
     406              :                     visc_u = (nu_xp * (field->u[idx + 1]  - u) -
     407              :                               nu_xm * (u - field->u[idx - 1])) / dx2 +
     408              :                              (nu_yp * (field->u[idx + nx] - u) -
     409              :                               nu_ym * (u - field->u[idx - nx])) / dy2 +
     410              :                              nu * d2u_dz2;
     411              :                     visc_v = (nu_xp * (field->v[idx + 1]  - v) -
     412              :                               nu_xm * (v - field->v[idx - 1])) / dx2 +
     413              :                              (nu_yp * (field->v[idx + nx] - v) -
     414              :                               nu_ym * (v - field->v[idx - nx])) / dy2 +
     415              :                              nu * d2v_dz2;
     416              :                     visc_w = (nu_xp * (field->w[idx + 1]  - w) -
     417              :                               nu_xm * (w - field->w[idx - 1])) / dx2 +
     418              :                              (nu_yp * (field->w[idx + nx] - w) -
     419              :                               nu_ym * (w - field->w[idx - nx])) / dy2 +
     420              :                              nu * d2w_dz2;
     421              :                 }
     422              : 
     423              :                 // Source terms
     424              :                 double source_u = 0.0;
     425              :                 double source_v = 0.0;
     426              :                 double source_w = 0.0;
     427              :                 double x_coord = grid->x[i];
     428              :                 double y_coord = grid->y[j];
     429              :                 double z_coord = (ctx->nz > 1 && grid->z) ? grid->z[k] : 0.0;
     430              :                 compute_source_terms(x_coord, y_coord, z_coord, ctx->iter_count, dt, params,
     431              :                                      &source_u, &source_v, &source_w);
     432              : 
     433              :                 // Boussinesq buoyancy source (no-op when beta == 0)
     434              :                 energy_compute_buoyancy(field->T[idx], params,
     435              :                                         &source_u, &source_v, &source_w);
     436              : 
     437              :                 // Intermediate velocity (without pressure gradient)
     438              :                 u_star[idx] = u + (dt * (-conv_u + visc_u + source_u));
     439              :                 v_star[idx] = v + (dt * (-conv_v + visc_v + source_v));
     440              :                 w_star[idx] = w + (dt * (-conv_w + visc_w + source_w));
     441              : 
     442              :                 // Limit velocities
     443              :                 u_star[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, u_star[idx]));
     444              :                 v_star[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, v_star[idx]));
     445              :                 w_star[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, w_star[idx]));
     446              :             }
     447              :         }
     448              :     }
     449              : 
     450              :     // Copy boundary values from field to star arrays
     451            0 :     copy_boundary_velocities_3d(u_star, v_star, w_star, field->u, field->v, field->w,
     452              :                                 nx, ny, ctx->nz);
     453              : 
     454              :     // ============================================================
     455              :     // STEP 2: Solve Poisson equation for pressure
     456              :     // ∇²p = (ρ/dt) * ∇·u*
     457              :     // ============================================================
     458              : 
     459            0 :     double rho = field->rho[0];
     460            0 :     if (rho < 1e-10) {
     461            0 :         rho = 1.0;
     462              :     }
     463              : 
     464              :     // Compute RHS: divergence of intermediate velocity
     465            0 :     for (size_t k = ctx->k_start; k < ctx->k_end; k++) {
     466            0 :         size_t k_off = k * ctx->stride_z;
     467              : #ifdef _OPENMP
     468            0 :         #pragma omp parallel for schedule(static)
     469              : #endif
     470              :         for (jj = 1; jj < ny_int - 1; jj++) {
     471              :             size_t j = (size_t)jj;
     472              :             for (size_t i = 1; i < nx - 1; i++) {
     473              :                 size_t idx = k_off + IDX_2D(i, j, nx);
     474              : 
     475              :                 double du_star_dx = (u_star[idx + 1] - u_star[idx - 1]) / (2.0 * dx);
     476              :                 double dv_star_dy = (v_star[idx + nx] - v_star[idx - nx]) / (2.0 * dy);
     477              :                 double dw_star_dz = (w_star[idx + ctx->stride_z] -
     478              :                                      w_star[idx - ctx->stride_z]) * ctx->inv_2dz;
     479              : 
     480              :                 double divergence = du_star_dx + dv_star_dy + dw_star_dz;
     481              :                 rhs[idx] = (rho / dt) * divergence;
     482              :             }
     483              :         }
     484              :     }
     485              : 
     486              :     /* Neumann compatibility projection. The pressure Poisson system has
     487              :      * zero-gradient walls on every preset here, so it is singular with the
     488              :      * constants as its nullspace: the RHS must have zero interior mean or the
     489              :      * residual stalls on the component that lies in it. Discretely, div(u*)
     490              :      * only nearly integrates to zero, so the remainder is removed here rather
     491              :      * than assumed away. With a face prescribed the operator is nonsingular and
     492              :      * shifting the RHS would change the answer instead of making it exist. */
     493            0 :     if (poisson_walls_are_singular(&params->pressure_bc, ctx->nz)) {
     494            0 :         mg_subtract_interior_mean(rhs, nx, ny, ctx->nz);
     495              :     }
     496              : 
     497              :     // Solve with the solver this projection owns; ctx->u_new is the temp buffer.
     498              :     // The status passes through rather than flattening to MAX_ITER: divergence and
     499              :     // an unsolvable right-hand side are different problems with different fixes.
     500            0 :     poisson_solver_stats_t pstats = poisson_solver_stats_default();
     501            0 :     cfd_status_t poisson_status =
     502            0 :         poisson_solver_solve(ctx->pressure, p_new, ctx->u_new, rhs, &pstats);
     503              : 
     504            0 :     if (poisson_status != CFD_SUCCESS) {
     505              :         return poisson_status;
     506              :     }
     507              : 
     508              :     // ============================================================
     509              :     // STEP 3: Corrector - Project velocity to be divergence-free
     510              :     // u^(n+1) = u* - (dt/ρ) * ∇p
     511              :     // (OpenMP parallelized with SIMD inner loop)
     512              :     // ============================================================
     513              : 
     514            0 :     double dt_over_rho = dt / rho;
     515            0 :     double inv_2dx = 1.0 / (2.0 * dx);
     516            0 :     double inv_2dy = 1.0 / (2.0 * dy);
     517              : 
     518              : #if USE_AVX
     519              :     __m256d dt_rho_vec      = _mm256_set1_pd(dt_over_rho);
     520              :     __m256d inv_2dx_vec     = _mm256_set1_pd(inv_2dx);
     521              :     __m256d inv_2dy_vec     = _mm256_set1_pd(inv_2dy);
     522              :     __m256d inv_2dz_vec     = _mm256_set1_pd(ctx->inv_2dz);
     523              :     __m256d max_vel_vec     = _mm256_set1_pd(MAX_VELOCITY);
     524              :     __m256d neg_max_vel_vec = _mm256_set1_pd(-MAX_VELOCITY);
     525              : #endif
     526              : 
     527            0 :     for (size_t k = ctx->k_start; k < ctx->k_end; k++) {
     528            0 :         size_t k_off = k * ctx->stride_z;
     529              : #ifdef _OPENMP
     530            0 :         #pragma omp parallel for schedule(static)
     531              : #endif
     532              :         for (jj = 1; jj < ny_int - 1; jj++) {
     533              :             size_t j = (size_t)jj;
     534              :             size_t i = 1;
     535              : 
     536              : #if USE_AVX
     537              :             // SIMD loop - process 4 cells at once
     538              :             for (; i + 4 <= nx - 1; i += 4) {
     539              :                 size_t idx = k_off + IDX_2D(i, j, nx);
     540              : 
     541              :                 // Load pressure neighbors for gradient computation
     542              :                 __m256d p_xp = _mm256_loadu_pd(&p_new[idx + 1]);
     543              :                 __m256d p_xm = _mm256_loadu_pd(&p_new[idx - 1]);
     544              :                 __m256d p_yp = _mm256_loadu_pd(&p_new[idx + nx]);
     545              :                 __m256d p_ym = _mm256_loadu_pd(&p_new[idx - nx]);
     546              :                 __m256d p_zp = _mm256_loadu_pd(&p_new[idx + ctx->stride_z]);
     547              :                 __m256d p_zm = _mm256_loadu_pd(&p_new[idx - ctx->stride_z]);
     548              : 
     549              :                 // Compute pressure gradients
     550              :                 __m256d dp_dx = _mm256_mul_pd(_mm256_sub_pd(p_xp, p_xm), inv_2dx_vec);
     551              :                 __m256d dp_dy = _mm256_mul_pd(_mm256_sub_pd(p_yp, p_ym), inv_2dy_vec);
     552              :                 __m256d dp_dz = _mm256_mul_pd(_mm256_sub_pd(p_zp, p_zm), inv_2dz_vec);
     553              : 
     554              :                 // Load intermediate velocities
     555              :                 __m256d u_s = _mm256_loadu_pd(&u_star[idx]);
     556              :                 __m256d v_s = _mm256_loadu_pd(&v_star[idx]);
     557              :                 __m256d w_s = _mm256_loadu_pd(&w_star[idx]);
     558              : 
     559              :                 // Corrector: u = u* - (dt/rho) * dp/dx
     560              :                 __m256d u_new = _mm256_sub_pd(u_s, _mm256_mul_pd(dt_rho_vec, dp_dx));
     561              :                 __m256d v_new = _mm256_sub_pd(v_s, _mm256_mul_pd(dt_rho_vec, dp_dy));
     562              :                 __m256d w_new = _mm256_sub_pd(w_s, _mm256_mul_pd(dt_rho_vec, dp_dz));
     563              : 
     564              :                 // Clamp velocities to [-MAX_VELOCITY, MAX_VELOCITY]
     565              :                 u_new = _mm256_max_pd(neg_max_vel_vec, _mm256_min_pd(max_vel_vec, u_new));
     566              :                 v_new = _mm256_max_pd(neg_max_vel_vec, _mm256_min_pd(max_vel_vec, v_new));
     567              :                 w_new = _mm256_max_pd(neg_max_vel_vec, _mm256_min_pd(max_vel_vec, w_new));
     568              : 
     569              :                 // Store results
     570              :                 _mm256_storeu_pd(&field->u[idx], u_new);
     571              :                 _mm256_storeu_pd(&field->v[idx], v_new);
     572              :                 _mm256_storeu_pd(&field->w[idx], w_new);
     573              :             }
     574              : #endif
     575              : 
     576              :             // Scalar remainder
     577              :             for (; i < nx - 1; i++) {
     578              :                 size_t idx = k_off + IDX_2D(i, j, nx);
     579              : 
     580              :                 double dp_dx = (p_new[idx + 1] - p_new[idx - 1]) * inv_2dx;
     581              :                 double dp_dy = (p_new[idx + nx] - p_new[idx - nx]) * inv_2dy;
     582              :                 double dp_dz = (p_new[idx + ctx->stride_z] - p_new[idx - ctx->stride_z]) *
     583              :                                ctx->inv_2dz;
     584              : 
     585              :                 field->u[idx] = u_star[idx] - (dt_over_rho * dp_dx);
     586              :                 field->v[idx] = v_star[idx] - (dt_over_rho * dp_dy);
     587              :                 field->w[idx] = w_star[idx] - (dt_over_rho * dp_dz);
     588              : 
     589              :                 // Limit velocities
     590              :                 field->u[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, field->u[idx]));
     591              :                 field->v[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, field->v[idx]));
     592              :                 field->w[idx] = fmax(-MAX_VELOCITY, fmin(MAX_VELOCITY, field->w[idx]));
     593              :             }
     594              :         }
     595              :     }
     596              : 
     597              :     // Update pressure field
     598            0 :     memcpy(field->p, p_new, size * sizeof(double));
     599              : 
     600              :     // Energy equation: advance temperature after velocity correction
     601              :     {
     602            0 :         cfd_status_t energy_status = energy_step_explicit_avx2_with_workspace(
     603            0 :             field, grid, params, dt, ctx->iter_count * dt, ctx->T_ws, size);
     604            0 :         if (energy_status != CFD_SUCCESS) {
     605              :             return energy_status;
     606              :         }
     607              :     }
     608              : 
     609              :     // Apply configured thermal BCs to temperature field
     610              :     {
     611            0 :         cfd_status_t bc_status = energy_apply_thermal_bcs(field, params);
     612            0 :         if (bc_status != CFD_SUCCESS) {
     613              :             return bc_status;
     614              :         }
     615              :     }
     616              : 
     617              :     // Copy boundary velocity values from star arrays (which have caller's BCs)
     618            0 :     copy_boundary_velocities_3d(field->u, field->v, field->w, u_star, v_star, w_star,
     619              :                                 nx, ny, ctx->nz);
     620              : 
     621              :     // Turbulence transport: advance k-eps/SA with the corrected velocity,
     622              :     // then apply turbulence BCs (including wall functions).
     623              :     {
     624            0 :         cfd_status_t turb_status = turbulence_step_explicit_avx2_with_workspace(
     625            0 :             field, grid, params, dt, ctx->iter_count * dt, ctx->turb_ws,
     626              :             turb_on ? TURB_WORKSPACE_SIZE(size) : 0);
     627            0 :         if (turb_status == CFD_SUCCESS) {
     628            0 :             turb_status = turbulence_apply_bcs(field, grid, params);
     629              :         }
     630            0 :         if (turb_status != CFD_SUCCESS) {
     631            0 :             return turb_status;
     632              :         }
     633              :     }
     634              : 
     635              :     // Check for NaN
     636            0 :     for (size_t n = 0; n < size; n++) {
     637            0 :         if (!isfinite(field->u[n]) || !isfinite(field->v[n]) ||
     638            0 :             !isfinite(field->w[n]) || !isfinite(field->p[n])) {
     639              :             return CFD_ERROR_DIVERGED;
     640              :         }
     641              :     }
     642              : 
     643            0 :     ctx->iter_count++;
     644              : 
     645            0 :     if (stats) {
     646            0 :         stats->iterations = 1;
     647              :     }
     648              : 
     649              :     return CFD_SUCCESS;
     650              : }
        

Generated by: LCOV version 2.0-1