LCOV - code coverage report
Current view: top level - solvers/linear - linear_solver.c (source / functions) Coverage Total Hit
Test: coverage.info Lines: 95.6 % 569 544
Test Date: 2026-10-05 04:47:18 Functions: 100.0 % 34 34

            Line data    Source code
       1              : /**
       2              :  * @file linear_solver.c
       3              :  * @brief Core linear solver implementation
       4              :  *
       5              :  * Implements:
       6              :  * - Default parameter functions
       7              :  * - Backend selection
       8              :  * - Solver lifecycle (create, init, destroy)
       9              :  * - Common solve loop
      10              :  * - Configuration validation
      11              :  * - The poisson_solve() convenience entry point
      12              :  */
      13              : 
      14              : #include "cfd/solvers/poisson_solver.h"
      15              : #include "linear_solver_internal.h"
      16              : #include "multigrid_internal.h"  /* mg_subtract_interior_mean, MG_DEFAULT_*_SMOOTH */
      17              : 
      18              : #include "cfd/boundary/boundary_conditions.h"
      19              : #include "cfd/core/indexing.h"
      20              : #include "cfd/core/logging.h"
      21              : #include "cfd/core/memory.h"
      22              : 
      23              : #include <math.h>
      24              : #include <stdio.h>
      25              : #include <string.h>
      26              : 
      27              : #ifdef _WIN32
      28              :     #define WIN32_LEAN_AND_MEAN
      29              :     #include <windows.h>
      30              : #else
      31              :     #include <sys/time.h>
      32              : #endif
      33              : 
      34              : /* ============================================================================
      35              :  * DEFAULT PARAMETERS
      36              :  * ============================================================================ */
      37              : 
      38        18628 : poisson_walls_t poisson_walls_default(void) {
      39        18628 :     poisson_walls_t walls;
      40        18628 :     memset(&walls, 0, sizeof walls);  /* POISSON_WALL_ZERO_GRADIENT is 0 */
      41        18628 :     return walls;
      42              : }
      43              : 
      44           29 : poisson_walls_t poisson_walls_uniform(poisson_wall_t type, double value) {
      45           29 :     poisson_walls_t walls = poisson_walls_default();
      46           29 :     walls.left = walls.right = walls.bottom = walls.top = walls.front = walls.back = type;
      47           29 :     if (type == POISSON_WALL_DIRICHLET) {
      48           27 :         walls.values.left = walls.values.right = value;
      49           27 :         walls.values.bottom = walls.values.top = value;
      50           27 :         walls.values.front = walls.values.back = value;
      51              :     }
      52           29 :     return walls;
      53              : }
      54              : 
      55      2360929 : bool poisson_walls_are_default(const poisson_walls_t* w)
      56              : {
      57      2360929 :     if (!w) {
      58              :         return true;  /* No walls configured is the default operator. */
      59              :     }
      60      2360928 :     return w->left == POISSON_WALL_ZERO_GRADIENT
      61      2360928 :         && w->right == POISSON_WALL_ZERO_GRADIENT
      62              :         && w->bottom == POISSON_WALL_ZERO_GRADIENT
      63      2314096 :         && w->top == POISSON_WALL_ZERO_GRADIENT
      64              :         && w->front == POISSON_WALL_ZERO_GRADIENT
      65      4721861 :         && w->back == POISSON_WALL_ZERO_GRADIENT;
      66              : }
      67              : 
      68              : /**
      69              :  * A sweep count as the multigrid hierarchy will actually read it.
      70              :  *
      71              :  * 0 means "the default" in this field, and the template resolves it that way
      72              :  * (linear_solver_multigrid_template.h). Comparing the raw values would call
      73              :  * pre_smooth = 0, post_smooth = 2 asymmetric and refuse it, when the cycle it
      74              :  * builds is the symmetric V(2,2) the rule is there to require.
      75              :  */
      76           90 : static int mg_smooth_effective(int requested, int fallback) {
      77           90 :     return requested > 0 ? requested : fallback;
      78              : }
      79              : 
      80              : /* One face. Why it is worth asking at all is on poisson_walls_are_legal() in
      81              :  * the public header; keeping the argument in one place is the point of having
      82              :  * exported the predicate. */
      83              : static int wall_type_is_legal(poisson_wall_t type) {
      84              :     return type == POISSON_WALL_ZERO_GRADIENT || type == POISSON_WALL_DIRICHLET;
      85              : }
      86              : 
      87         2374 : bool poisson_walls_are_legal(const poisson_walls_t* w)
      88              : {
      89         2374 :     if (!w) {
      90              :         return true;  /* No walls configured is the default operator. */
      91              :     }
      92         2371 :     return wall_type_is_legal(w->left) && wall_type_is_legal(w->right)
      93         2369 :         && wall_type_is_legal(w->bottom) && wall_type_is_legal(w->top)
      94         4750 :         && wall_type_is_legal(w->front) && wall_type_is_legal(w->back);
      95              : }
      96              : 
      97              : /** Whether this method's halo routines honour per-face walls (Krylov only). */
      98        17484 : static int method_honours_walls(poisson_solver_method_t method) {
      99        17484 :     return method == POISSON_METHOD_CG
     100              :         || method == POISSON_METHOD_BICGSTAB
     101        17484 :         || method == POISSON_METHOD_GMRES;
     102              : }
     103              : 
     104              : /* ============================================================================
     105              :  * CONFIGURATION VALIDATION
     106              :  *
     107              :  * One place decides what a given (method, backend) can honour, so a parameter it
     108              :  * cannot is refused at init rather than quietly ignored. The alternative -- a
     109              :  * check inside each of the twenty-odd solver inits -- is what let BiCGSTAB ignore
     110              :  * params.krylov.preconditioner on every backend, and three GPU solvers ignore a
     111              :  * caller's apply_bc, for as long as they have existed.
     112              :  * ============================================================================ */
     113              : 
     114              : /** Parameter groups, for the ownership table below. */
     115              : enum {
     116              :     PARAM_GROUP_SOR       = 1u << 0,
     117              :     PARAM_GROUP_KRYLOV    = 1u << 1,
     118              :     PARAM_GROUP_MULTIGRID = 1u << 2
     119              : };
     120              : 
     121              : /**
     122              :  * The groups this method reads. Anything else it is handed, it ignores -- which
     123              :  * is exactly what must be refused.
     124              :  */
     125         1813 : static unsigned method_owned_groups(const poisson_solver_t* solver) {
     126         1813 :     switch (solver->method) {
     127              :         case POISSON_METHOD_JACOBI:
     128              :             return 0;
     129          732 :         case POISSON_METHOD_SOR:
     130              :         case POISSON_METHOD_GAUSS_SEIDEL:
     131              :         case POISSON_METHOD_REDBLACK_SOR:
     132          732 :             return PARAM_GROUP_SOR;
     133          544 :         case POISSON_METHOD_CG:
     134              :             /* CG reads the multigrid group only when it is preconditioned by
     135              :              * one, where those parameters shape the inner V-cycle. */
     136          544 :             return PARAM_GROUP_KRYLOV
     137          544 :                  | (solver->params.krylov.preconditioner == POISSON_PRECOND_MULTIGRID
     138              :                         ? PARAM_GROUP_MULTIGRID : 0u);
     139              :         case POISSON_METHOD_BICGSTAB:
     140              :         case POISSON_METHOD_GMRES:
     141              :             return PARAM_GROUP_KRYLOV;
     142          372 :         case POISSON_METHOD_MULTIGRID:
     143          372 :             return PARAM_GROUP_MULTIGRID;
     144              :         default:
     145              :             return 0;
     146              :     }
     147              : }
     148              : 
     149              : /*
     150              :  * Whether a group is entirely at its defaults, i.e. untouched by the caller.
     151              :  *
     152              :  * Field by field, not a memcmp or a byte scan over the struct. Padding bytes
     153              :  * are not required to be zero in a caller's `poisson_solver_params_t p = {0};`
     154              :  * -- C11 6.7.9 leaves them unspecified -- so a byte scan can read garbage and
     155              :  * report a group the caller never touched as set. It also silently depends on
     156              :  * these three structs having no padding, which is true today and is not
     157              :  * something a future field can be trusted to preserve.
     158              :  */
     159         1813 : static int sor_untouched(const poisson_sor_params_t* g) {
     160         1813 :     return g->omega == 0.0;
     161              : }
     162              : 
     163         1811 : static int krylov_untouched(const poisson_krylov_params_t* g) {
     164         1729 :     return g->preconditioner == POISSON_PRECOND_NONE && g->restart == 0;
     165              : }
     166              : 
     167         1810 : static int multigrid_untouched(const poisson_multigrid_params_t* g) {
     168         1692 :     return g->cycle == 0 && g->smoother == 0 && g->bc == 0
     169         1548 :         && g->pre_smooth == 0 && g->post_smooth == 0
     170         3350 :         && g->coarse_max_iter == 0 && g->max_levels == 0;
     171              : }
     172              : 
     173         5434 : static cfd_status_t reject_unowned_group(const poisson_solver_t* solver,
     174              :                                          unsigned owned, unsigned group,
     175              :                                          int untouched,
     176              :                                          const char* name) {
     177         5434 :     if ((owned & group) || untouched) {
     178              :         return CFD_SUCCESS;
     179              :     }
     180            4 :     char msg[192];
     181            8 :     snprintf(msg, sizeof(msg),
     182              :              "params.%s is set, but the %s method does not read it; "
     183              :              "it would have no effect on this solve",
     184            4 :              name, solver->name ? solver->name : "selected");
     185            4 :     cfd_set_error(CFD_ERROR_INVALID, msg);
     186            4 :     return CFD_ERROR_INVALID;
     187              : }
     188              : 
     189         1846 : cfd_status_t poisson_solver_check_config(const poisson_solver_t* solver) {
     190         1846 :     const poisson_solver_params_t* p = &solver->params;
     191              : 
     192              :     /* ---- walls -------------------------------------------------------- */
     193              :     /* Range first: an out-of-range face is neither zero-gradient nor prescribed
     194              :      * to the code that reads it, so nothing below would agree about what the
     195              :      * operator is. */
     196         1846 :     if (!poisson_walls_are_legal(&p->walls)) {
     197            7 :         cfd_set_error(CFD_ERROR_INVALID,
     198              :             "every params.walls face must be POISSON_WALL_ZERO_GRADIENT or "
     199              :             "POISSON_WALL_DIRICHLET");
     200            7 :         return CFD_ERROR_INVALID;
     201              :     }
     202         1839 :     if (!poisson_walls_are_default(&p->walls)) {
     203              :         /* Support first, then the hook conflict: an unsupported method should say
     204              :          * so rather than report a caller error. */
     205           78 :         if (!method_honours_walls(solver->method)) {
     206            5 :             cfd_set_error(CFD_ERROR_UNSUPPORTED,
     207              :                 "per-face walls are implemented for the CG, BiCGSTAB and GMRES methods only; "
     208              :                 "the stationary and multigrid solvers apply whole-domain walls inside their sweeps");
     209            5 :             return CFD_ERROR_UNSUPPORTED;
     210              :         }
     211           73 :         if (solver->backend == POISSON_BACKEND_GPU) {
     212            0 :             cfd_set_error(CFD_ERROR_UNSUPPORTED,
     213              :                 "per-face walls are not implemented on the GPU backend, which applies the "
     214              :                 "zero-gradient walls on the device");
     215            0 :             return CFD_ERROR_UNSUPPORTED;
     216              :         }
     217           73 :         if (solver->apply_bc) {
     218            1 :             cfd_set_error(CFD_ERROR_INVALID,
     219              :                 "params.walls and a custom apply_bc both prescribe wall values; use one or the other");
     220            1 :             return CFD_ERROR_INVALID;
     221              :         }
     222           72 :         if (!isfinite(p->walls.values.left) || !isfinite(p->walls.values.right)
     223           72 :             || !isfinite(p->walls.values.bottom) || !isfinite(p->walls.values.top)
     224           72 :             || !isfinite(p->walls.values.front) || !isfinite(p->walls.values.back)) {
     225            0 :             cfd_set_error(CFD_ERROR_INVALID, "params.walls.values must be finite");
     226            0 :             return CFD_ERROR_INVALID;
     227              :         }
     228              :         /* A 2D grid has no z-faces, so poisson_apply_walls skips .front/.back and
     229              :          * poisson_walls_are_singular ignores them. Prescribing one is refused
     230              :          * only when it was the caller's only prescribed face: they meant to pin
     231              :          * the level, nothing pinned it, and the operator is still singular -- so
     232              :          * the solve either refuses their rhs or drifts, with the cause two steps
     233              :          * away. Alongside a prescribed x or y face the z-faces are simply inert,
     234              :          * which is what poisson_walls_uniform() produces on a 2D grid and is not
     235              :          * worth refusing. */
     236           72 :         if (solver->nz <= 1
     237           66 :             && (p->walls.front != POISSON_WALL_ZERO_GRADIENT
     238           66 :                 || p->walls.back != POISSON_WALL_ZERO_GRADIENT)
     239           26 :             && poisson_walls_are_singular(&p->walls, solver->nz)) {
     240            2 :             cfd_set_error(CFD_ERROR_INVALID,
     241              :                 "params.walls.front and .back describe z-faces, which a 2D grid "
     242              :                 "(nz == 1) does not have, so nothing here pins the pressure level; "
     243              :                 "prescribe an x or y face instead");
     244            2 :             return CFD_ERROR_INVALID;
     245              :         }
     246              :     }
     247              : 
     248              :     /* ---- the Helmholtz shift ------------------------------------------ */
     249         1831 :     if (!isfinite(p->helmholtz_shift) || p->helmholtz_shift < 0.0) {
     250            4 :         cfd_set_error(CFD_ERROR_INVALID,
     251              :             "params.helmholtz_shift must be finite and >= 0 (a negative shift is the "
     252              :             "indefinite Helmholtz operator, which is not SPD)");
     253            4 :         return CFD_ERROR_INVALID;
     254              :     }
     255         1827 :     if (p->helmholtz_shift != 0.0
     256           66 :         && !poisson_solver_shift_supported(solver->method, solver->backend, p)) {
     257           13 :         cfd_set_error(CFD_ERROR_UNSUPPORTED,
     258              :             "params.helmholtz_shift is implemented by the scalar and OpenMP CG solvers "
     259              :             "only, and not with the multigrid preconditioner");
     260           13 :         return CFD_ERROR_UNSUPPORTED;
     261              :     }
     262              : 
     263              :     /* ---- a caller's apply_bc ------------------------------------------ */
     264              :     /* solver->internal_apply_bc is a solver's own walls and is not a caller's
     265              :      * business; this is only about a function the caller installed. */
     266         1814 :     if (solver->apply_bc) {
     267          143 :         if (solver->backend == POISSON_BACKEND_GPU) {
     268            0 :             cfd_set_error(CFD_ERROR_UNSUPPORTED,
     269              :                 "the GPU solvers apply their walls on the device and never call apply_bc");
     270            0 :             return CFD_ERROR_UNSUPPORTED;
     271              :         }
     272          143 :         if (solver->method == POISSON_METHOD_MULTIGRID) {
     273            1 :             cfd_set_error(CFD_ERROR_UNSUPPORTED,
     274              :                 "multigrid applies its walls inside the cycle and never calls apply_bc; "
     275              :                 "use params.multigrid.bc to choose between zero-gradient and fixed walls");
     276            1 :             return CFD_ERROR_UNSUPPORTED;
     277              :         }
     278              :     }
     279              : 
     280              :     /* ---- parameter groups the method does not read --------------------- */
     281              :     {
     282         1813 :         unsigned owned = method_owned_groups(solver);
     283         1813 :         cfd_status_t status;
     284              : 
     285         1813 :         status = reject_unowned_group(solver, owned, PARAM_GROUP_SOR,
     286              :                                       sor_untouched(&p->sor), "sor");
     287         1813 :         if (status != CFD_SUCCESS) {
     288              :             return status;
     289              :         }
     290         3622 :         status = reject_unowned_group(solver, owned, PARAM_GROUP_KRYLOV,
     291              :                                       krylov_untouched(&p->krylov), "krylov");
     292         1811 :         if (status != CFD_SUCCESS) {
     293              :             return status;
     294              :         }
     295         1810 :         status = reject_unowned_group(solver, owned, PARAM_GROUP_MULTIGRID,
     296              :                                       multigrid_untouched(&p->multigrid), "multigrid");
     297         1810 :         if (status != CFD_SUCCESS) {
     298              :             return status;
     299              :         }
     300              :     }
     301              : 
     302              :     /* ---- exceptions within a group the method does own ----------------- */
     303              : 
     304              :     /* Range before capability. Every check below tests for a specific value --
     305              :      * "is it MULTIGRID", "is it not NONE" -- so an out-of-range preconditioner
     306              :      * satisfies all of them on CG and GMRES, and their backends then find it is
     307              :      * not POISSON_PRECOND_JACOBI either and run unpreconditioned. The caller
     308              :      * asked for something and got the default, which is the failure this
     309              :      * function exists to make impossible. */
     310         1809 :     if (p->krylov.preconditioner != POISSON_PRECOND_NONE
     311              :         && p->krylov.preconditioner != POISSON_PRECOND_JACOBI
     312              :         && p->krylov.preconditioner != POISSON_PRECOND_MULTIGRID) {
     313            2 :         cfd_set_error(CFD_ERROR_INVALID,
     314              :             "params.krylov.preconditioner must be POISSON_PRECOND_NONE, "
     315              :             "POISSON_PRECOND_JACOBI or POISSON_PRECOND_MULTIGRID");
     316            2 :         return CFD_ERROR_INVALID;
     317              :     }
     318              : 
     319              :     /* BiCGSTAB reads params.krylov, but has no preconditioner implementation on
     320              :      * any backend. It ignored one silently for as long as it has existed. */
     321         1807 :     if (solver->method == POISSON_METHOD_BICGSTAB
     322           35 :         && p->krylov.preconditioner != POISSON_PRECOND_NONE) {
     323            3 :         cfd_set_error(CFD_ERROR_UNSUPPORTED,
     324              :             "BiCGSTAB has no preconditioner implementation on any backend");
     325            3 :         return CFD_ERROR_UNSUPPORTED;
     326              :     }
     327              : 
     328              :     /* No GPU solver implements a preconditioner -- there is no M^-1 apply
     329              :      * anywhere under linear/gpu/ -- so one configured here would be accepted and
     330              :      * then never used, which is what BiCGSTAB did on every backend for years. */
     331         1804 :     if (p->krylov.preconditioner != POISSON_PRECOND_NONE
     332           76 :         && solver->backend == POISSON_BACKEND_GPU) {
     333            0 :         cfd_set_error(CFD_ERROR_UNSUPPORTED,
     334              :             "the GPU Krylov solvers have no preconditioner implementation; "
     335              :             "run preconditioned on a CPU backend, or unpreconditioned here");
     336            0 :         return CFD_ERROR_UNSUPPORTED;
     337              :     }
     338              : 
     339              :     /* The multigrid preconditioner is implemented by the scalar and OpenMP CG
     340              :      * solvers only. Every other Krylov backend would quietly run unpreconditioned. */
     341         1804 :     if (p->krylov.preconditioner == POISSON_PRECOND_MULTIGRID
     342           51 :         && !(solver->method == POISSON_METHOD_CG
     343           48 :              && (solver->backend == POISSON_BACKEND_SCALAR
     344           48 :                  || solver->backend == POISSON_BACKEND_OMP))) {
     345            3 :         cfd_set_error(CFD_ERROR_UNSUPPORTED,
     346              :             "POISSON_PRECOND_MULTIGRID is implemented by the scalar and OpenMP CG solvers only");
     347            3 :         return CFD_ERROR_UNSUPPORTED;
     348              :     }
     349              : 
     350              :     /* restart is the GMRES(m) basis size and means nothing to the others. */
     351         1801 :     if (p->krylov.restart != 0 && solver->method != POISSON_METHOD_GMRES) {
     352            2 :         cfd_set_error(CFD_ERROR_INVALID,
     353              :             "params.krylov.restart is the GMRES restart length; no other method reads it");
     354            2 :         return CFD_ERROR_INVALID;
     355              :     }
     356              : 
     357              :     /* Deliberately not rejected here: POISSON_METHOD_GAUSS_SEIDEL overriding
     358              :      * params.sor.omega to 1. That is what the method IS, it is documented on the
     359              :      * enum member, and test_gauss_seidel_is_sor_at_omega_one pins it. Refusing an
     360              :      * omega there would turn documented behaviour into an error, which is not the
     361              :      * class of problem this validator exists for.
     362              :      */
     363              : 
     364              :     /* The inner V-cycle of a preconditioner must stay symmetric and nonsingular,
     365              :      * or CG is no longer minimising over a Krylov space. These four are what
     366              :      * guarantee it, so they are not the caller's to change here; the rest of the
     367              :      * group is forwarded to the hierarchy by poisson_solver_create_mg_precond. */
     368         1799 :     if (p->krylov.preconditioner == POISSON_PRECOND_MULTIGRID
     369           48 :         && (p->multigrid.cycle != 0 || p->multigrid.smoother != 0 || p->multigrid.bc != 0
     370           45 :             || mg_smooth_effective(p->multigrid.pre_smooth, MG_DEFAULT_PRE_SMOOTH)
     371           83 :                    != mg_smooth_effective(p->multigrid.post_smooth, MG_DEFAULT_POST_SMOOTH))) {
     372            5 :         cfd_set_error(CFD_ERROR_INVALID,
     373              :             "a multigrid preconditioner fixes its cycle, smoother, boundary mode and an "
     374              :             "equal pre/post sweep count, which is what keeps it symmetric for CG; "
     375              :             "params.multigrid.{pre_smooth, post_smooth, coarse_max_iter, max_levels} "
     376              :             "are yours to set");
     377            5 :         return CFD_ERROR_INVALID;
     378              :     }
     379              : 
     380              :     /* iter % check_interval, with check_interval == 0, is undefined behaviour in
     381              :      * the common loop and in the CG and BiCGSTAB loops. */
     382         1794 :     if (p->check_interval <= 0) {
     383            2 :         cfd_set_error(CFD_ERROR_INVALID, "params.check_interval must be at least 1");
     384            2 :         return CFD_ERROR_INVALID;
     385              :     }
     386              : 
     387              :     return CFD_SUCCESS;
     388              : }
     389              : 
     390        27642 : bool poisson_walls_are_singular(const poisson_walls_t* walls, size_t nz) {
     391        27642 :     if (!walls) {
     392              :         return true;  /* no walls given means the defaults, which are singular */
     393              :     }
     394              :     /* The z-faces do not exist on a 2D grid, so whatever they say is irrelevant. */
     395        27641 :     if (nz > 1 && (walls->front != POISSON_WALL_ZERO_GRADIENT
     396          186 :                    || walls->back != POISSON_WALL_ZERO_GRADIENT)) {
     397              :         return false;
     398              :     }
     399        27640 :     return walls->left == POISSON_WALL_ZERO_GRADIENT
     400        27640 :         && walls->right == POISSON_WALL_ZERO_GRADIENT
     401              :         && walls->bottom == POISSON_WALL_ZERO_GRADIENT
     402        27680 :         && walls->top == POISSON_WALL_ZERO_GRADIENT;
     403              : }
     404              : 
     405        17761 : poisson_solver_params_t poisson_solver_params_default(void) {
     406              :     /* Zeroed first: the assignments below cover every named field, but a
     407              :      * projection solver compares whole params structs with memcmp to decide
     408              :      * whether its pressure solver needs rebuilding (ns_pressure_ensure), so
     409              :      * padding has to be deterministic too. Without this, that comparison could
     410              :      * differ on padding alone and rebuild the solver on every step. */
     411        17761 :     poisson_solver_params_t params;
     412        17761 :     memset(&params, 0, sizeof params);
     413        17761 :     params.tolerance = 1e-6;
     414        17761 :     params.absolute_tolerance = 1e-10;
     415        17761 :     params.max_iterations = 5000;  /* Increased from 1000 for CG on fine grids */
     416        17761 :     params.sor.omega = 0.0;  /* Auto-compute optimal omega for grid dimensions */
     417        17761 :     params.check_interval = 1;
     418        17761 :     params.verbose = false;
     419        17761 :     params.krylov.preconditioner = POISSON_PRECOND_NONE;
     420        17761 :     params.krylov.restart = 0;  /* 0 = auto (GMRES_DEFAULT_RESTART); ignored by non-GMRES methods */
     421        17761 :     params.multigrid.cycle = MG_CYCLE_V;
     422        17761 :     params.multigrid.smoother = MG_SMOOTHER_REDBLACK_GS;
     423        17761 :     params.multigrid.bc = MG_BC_NEUMANN;
     424        17761 :     params.multigrid.pre_smooth = 0;      /* 0 = default (2) */
     425        17761 :     params.multigrid.post_smooth = 0;     /* 0 = default (2) */
     426        17761 :     params.multigrid.coarse_max_iter = 0; /* 0 = default (50) */
     427        17761 :     params.multigrid.max_levels = 0;      /* 0 = auto */
     428        17761 :     params.walls = poisson_walls_default();
     429        17761 :     params.helmholtz_shift = 0.0;  /* 0 = pure Poisson */
     430        17761 :     return params;
     431              : }
     432              : 
     433        14485 : poisson_solver_config_t poisson_solver_config_preset(poisson_preset_t preset) {
     434        14485 :     poisson_solver_config_t cfg;
     435        14485 :     cfg.method = POISSON_METHOD_CG;
     436              :     /* AUTO unless the preset needs a backend that AUTO would not choose; see
     437              :      * the two multigrid cases below. */
     438        14485 :     cfg.backend = POISSON_BACKEND_AUTO;
     439        14485 :     cfg.params = poisson_solver_params_default();
     440              : 
     441        14485 :     switch (preset) {
     442          810 :         case POISSON_PRESET_ACCURATE:
     443          810 :             cfg.params.tolerance = 1e-10;
     444          810 :             cfg.params.absolute_tolerance = 1e-14;
     445          810 :             cfg.params.max_iterations = 20000;
     446          810 :             break;
     447              : 
     448            1 :         case POISSON_PRESET_NONSYMMETRIC:
     449            1 :             cfg.method = POISSON_METHOD_BICGSTAB;
     450            1 :             break;
     451              : 
     452            2 :         case POISSON_PRESET_SMOOTHER:
     453              :             /* The stationary option, and the one operator that tolerates an
     454              :              * incompatible rhs. It keeps the ordinary tolerances so an ordinary
     455              :              * solve reports success; a caller who wants a fixed sweep count
     456              :              * instead sets tolerance = 0 and max_iterations themselves, and then
     457              :              * CFD_ERROR_MAX_ITER is their own explicit choice rather than
     458              :              * something this preset returns on every call. */
     459            2 :             cfg.method = POISSON_METHOD_REDBLACK_SOR;
     460            2 :             break;
     461              : 
     462           68 :         case POISSON_PRESET_MULTIGRID:
     463           68 :             cfg.method = POISSON_METHOD_MULTIGRID;
     464           68 :             cfg.backend = POISSON_BACKEND_SCALAR;
     465           68 :             break;
     466              : 
     467           56 :         case POISSON_PRESET_MULTIGRID_PCG:
     468              :             /* The preconditioner is an ordinary parameter, so nothing downstream
     469              :              * needs a special case for it -- but the backend does need naming.
     470              :              * AUTO resolves to SIMD wherever AVX2 or NEON is present, and neither
     471              :              * multigrid nor the multigrid preconditioner has a SIMD backend, so
     472              :              * this preset would otherwise be refused at init on the very machines
     473              :              * it is most likely to run on. Scalar is the one backend AUTO could
     474              :              * have reached that implements it; POISSON_BACKEND_OMP is a single
     475              :              * assignment away for a caller who wants the threaded hierarchy. */
     476           56 :             cfg.params.krylov.preconditioner = POISSON_PRECOND_MULTIGRID;
     477           56 :             cfg.backend = POISSON_BACKEND_SCALAR;
     478           56 :             break;
     479              : 
     480              :         case POISSON_PRESET_DEFAULT:
     481              :         default:
     482              :             break;
     483              :     }
     484              : 
     485        14485 :     return cfg;
     486              : }
     487              : 
     488        17544 : poisson_solver_stats_t poisson_solver_stats_default(void) {
     489        17544 :     poisson_solver_stats_t stats;
     490        17544 :     stats.status = POISSON_ERROR;
     491        17544 :     stats.iterations = 0;
     492        17544 :     stats.initial_residual = 0.0;
     493        17544 :     stats.final_residual = 0.0;
     494        17544 :     stats.elapsed_time_ms = 0.0;
     495        17544 :     return stats;
     496              : }
     497              : 
     498           23 : const char* poisson_solver_status_string(poisson_solver_status_t status) {
     499           23 :     switch (status) {
     500              :         case POISSON_CONVERGED:        return "converged";
     501            2 :         case POISSON_MAX_ITER:         return "max iterations";
     502            3 :         case POISSON_DIVERGED:         return "diverged";
     503            4 :         case POISSON_STAGNATED:        return "stagnated";
     504            6 :         case POISSON_INCOMPATIBLE_RHS: return "incompatible rhs";
     505            6 :         case POISSON_ERROR:            return "error";
     506              :     }
     507            1 :     return "unknown";
     508              : }
     509              : 
     510              : /* ============================================================================
     511              :  * TIMING
     512              :  * ============================================================================ */
     513              : 
     514        68154 : double poisson_solver_get_time_ms(void) {
     515              : #ifdef _WIN32
     516              :     LARGE_INTEGER freq, counter;
     517              :     QueryPerformanceFrequency(&freq);
     518              :     QueryPerformanceCounter(&counter);
     519              :     return (double)counter.QuadPart * 1000.0 / (double)freq.QuadPart;
     520              : #else
     521        68154 :     struct timeval tv;
     522        68154 :     gettimeofday(&tv, NULL);
     523        68154 :     return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
     524              : #endif
     525              : }
     526              : 
     527              : /* ============================================================================
     528              :  * BACKEND SELECTION
     529              :  * ============================================================================ */
     530              : 
     531              : static poisson_solver_backend_t g_default_backend = POISSON_BACKEND_AUTO;
     532              : 
     533            2 : poisson_solver_backend_t poisson_solver_get_backend(void) {
     534            2 :     return g_default_backend;
     535              : }
     536              : 
     537            2 : const char* poisson_solver_get_backend_name(void) {
     538            2 :     switch (g_default_backend) {
     539              :         case POISSON_BACKEND_SCALAR:   return "scalar";
     540            0 :         case POISSON_BACKEND_OMP:      return "omp";
     541            0 :         case POISSON_BACKEND_SIMD: return "simd";
     542            0 :         case POISSON_BACKEND_GPU:      return "gpu";
     543            1 :         case POISSON_BACKEND_AUTO:
     544            1 :         default:                       return "auto";
     545              :     }
     546              : }
     547              : 
     548            2 : bool poisson_solver_set_backend(poisson_solver_backend_t backend) {
     549            2 :     if (!poisson_solver_backend_available(backend)) {
     550              :         return false;
     551              :     }
     552            2 :     g_default_backend = backend;
     553            2 :     return true;
     554              : }
     555              : 
     556           99 : bool poisson_solver_backend_available(poisson_solver_backend_t backend) {
     557           99 :     switch (backend) {
     558              :         case POISSON_BACKEND_AUTO:
     559              :         case POISSON_BACKEND_SCALAR:
     560              :             return true;
     561              : 
     562              :         case POISSON_BACKEND_OMP:
     563              : #ifdef CFD_ENABLE_OPENMP
     564              :             return true;
     565              : #else
     566              :             return false;
     567              : #endif
     568              : 
     569           23 :         case POISSON_BACKEND_SIMD:
     570           23 :             return poisson_solver_simd_backend_available();
     571              : 
     572              :         case POISSON_BACKEND_GPU:
     573              : #ifdef CFD_HAS_CUDA
     574              :             return true;
     575              : #else
     576              :             return false;
     577              : #endif
     578              : 
     579              :         default:
     580              :             return false;
     581              :     }
     582              : }
     583              : 
     584              : /* ============================================================================
     585              :  * SOLVER LIFECYCLE
     586              :  * ============================================================================ */
     587              : 
     588              : /**
     589              :  * Auto-select best available backend
     590              :  *
     591              :  * Priority: SIMD (runtime detection) > Scalar
     592              :  */
     593            8 : static poisson_solver_backend_t select_best_backend(void) {
     594              :     /* Prefer SIMD with runtime detection */
     595            8 :     if (poisson_solver_simd_backend_available()) {
     596            0 :         return POISSON_BACKEND_SIMD;
     597              :     }
     598              :     return POISSON_BACKEND_SCALAR;
     599              : }
     600              : 
     601              : /* Factories must set a specific last-status on NULL (see cfd_solver_create),
     602              :  * so callers can distinguish an unavailable backend from allocation failure. */
     603           58 : static poisson_solver_t* backend_unavailable(const char* method_name) {
     604           58 :     char msg[128];
     605           58 :     snprintf(msg, sizeof(msg), "Requested backend not available for %s", method_name);
     606           58 :     cfd_set_error(CFD_ERROR_UNSUPPORTED, msg);
     607           58 :     return NULL;
     608              : }
     609              : 
     610         1923 : poisson_solver_t* poisson_solver_create(
     611              :     poisson_solver_method_t method,
     612              :     poisson_solver_backend_t backend)
     613              : {
     614              :     /* Auto-select backend if requested (keep the original request: methods
     615              :      * without a SIMD backend resolve AUTO differently) */
     616         1923 :     poisson_solver_backend_t requested = backend;
     617         1923 :     if (backend == POISSON_BACKEND_AUTO) {
     618            8 :         backend = select_best_backend();
     619              :     }
     620              : 
     621              :     /* Create appropriate solver - no silent fallbacks */
     622         1923 :     switch (method) {
     623           58 :         case POISSON_METHOD_JACOBI:
     624           58 :             switch (backend) {
     625            3 :                 case POISSON_BACKEND_SIMD:
     626            3 :                     return create_jacobi_simd_solver();
     627              : #ifdef CFD_ENABLE_OPENMP
     628            9 :                 case POISSON_BACKEND_OMP:
     629            9 :                     return create_jacobi_omp_solver();
     630              : #endif
     631              : #ifdef CFD_HAS_CUDA
     632              :                 case POISSON_BACKEND_GPU:
     633              :                     return create_jacobi_gpu_solver();
     634              : #endif
     635           39 :                 case POISSON_BACKEND_SCALAR:
     636           39 :                     return create_jacobi_scalar_solver();
     637            7 :                 default:
     638            7 :                     return backend_unavailable("Jacobi");
     639              :             }
     640              : 
     641           65 :         case POISSON_METHOD_SOR:
     642              :         case POISSON_METHOD_GAUSS_SEIDEL: {
     643              :             /* Gauss-Seidel is SOR at omega = 1: the same solvers, marked with the
     644              :              * requested method so that init resolves omega to 1 */
     645           65 :             poisson_solver_t* sor;
     646           65 :             switch (backend) {
     647            7 :                 case POISSON_BACKEND_SIMD:
     648            7 :                     sor = create_sor_simd_solver();
     649            7 :                     break;
     650              : #ifdef CFD_HAS_CUDA
     651              :                 case POISSON_BACKEND_GPU:
     652              :                     sor = create_sor_gpu_solver();
     653              :                     break;
     654              : #endif
     655           39 :                 case POISSON_BACKEND_SCALAR:
     656           39 :                     sor = create_sor_scalar_solver();
     657           39 :                     break;
     658           19 :                 default:
     659           32 :                     return backend_unavailable(method == POISSON_METHOD_GAUSS_SEIDEL ? "Gauss-Seidel" : "SOR");
     660              :             }
     661           46 :             if (sor) {
     662           39 :                 sor->method = method;
     663              :             }
     664              :             return sor;
     665              :         }
     666              : 
     667          713 :         case POISSON_METHOD_REDBLACK_SOR:
     668          713 :             switch (backend) {
     669            3 :                 case POISSON_BACKEND_SIMD:
     670            3 :                     return create_redblack_simd_solver();
     671              : #ifdef CFD_ENABLE_OPENMP
     672           10 :                 case POISSON_BACKEND_OMP:
     673           10 :                     return create_redblack_omp_solver();
     674              : #endif
     675              : #ifdef CFD_HAS_CUDA
     676              :                 case POISSON_BACKEND_GPU:
     677              :                     return create_redblack_gpu_solver();
     678              : #endif
     679          695 :                 case POISSON_BACKEND_SCALAR:
     680          695 :                     return create_redblack_scalar_solver();
     681            5 :                 default:
     682            5 :                     return backend_unavailable("Red-Black SOR");
     683              :             }
     684              : 
     685          582 :         case POISSON_METHOD_CG:
     686          582 :             switch (backend) {
     687           10 :                 case POISSON_BACKEND_SIMD:
     688           10 :                     return create_cg_simd_solver();
     689              : #ifdef CFD_ENABLE_OPENMP
     690           77 :                 case POISSON_BACKEND_OMP:
     691           77 :                     return create_cg_omp_solver();
     692              : #endif
     693              : #ifdef CFD_HAS_CUDA
     694              :                 case POISSON_BACKEND_GPU:
     695              :                     return create_cg_gpu_solver();
     696              : #endif
     697          488 :                 case POISSON_BACKEND_SCALAR:
     698          488 :                     return create_cg_scalar_solver();
     699            7 :                 default:
     700            7 :                     return backend_unavailable("CG");
     701              :             }
     702              : 
     703           53 :         case POISSON_METHOD_BICGSTAB:
     704           53 :             switch (backend) {
     705            6 :                 case POISSON_BACKEND_SIMD:
     706            6 :                     return create_bicgstab_simd_solver();
     707              : #ifdef CFD_ENABLE_OPENMP
     708           13 :                 case POISSON_BACKEND_OMP:
     709           13 :                     return create_bicgstab_omp_solver();
     710              : #endif
     711              : #ifdef CFD_HAS_CUDA
     712              :                 case POISSON_BACKEND_GPU:
     713              :                     return create_bicgstab_gpu_solver();
     714              : #endif
     715           26 :                 case POISSON_BACKEND_SCALAR:
     716           26 :                     return create_bicgstab_scalar_solver();
     717            8 :                 default:
     718            8 :                     return backend_unavailable("BiCGSTAB");
     719              :             }
     720              : 
     721          103 :         case POISSON_METHOD_GMRES:
     722          103 :             switch (backend) {
     723            6 :                 case POISSON_BACKEND_SIMD:
     724            6 :                     return create_gmres_simd_solver();
     725              : #ifdef CFD_ENABLE_OPENMP
     726           37 :                 case POISSON_BACKEND_OMP:
     727           37 :                     return create_gmres_omp_solver();
     728              : #endif
     729           56 :                 case POISSON_BACKEND_SCALAR:
     730           56 :                     return create_gmres_scalar_solver();
     731            4 :                 default:
     732            4 :                     return backend_unavailable("GMRES");
     733              :             }
     734              : 
     735          348 :         case POISSON_METHOD_MULTIGRID:
     736              :             /* No SIMD/GPU multigrid. AUTO means "best available", which
     737              :              * select_best_backend() resolves to SIMD — a backend multigrid
     738              :              * lacks — so AUTO keeps resolving to the scalar reference. OMP is
     739              :              * opt-in, as for every other method; explicit SIMD/GPU requests
     740              :              * return NULL (no silent fallbacks). */
     741          348 :             if (requested == POISSON_BACKEND_AUTO) {
     742            1 :                 return create_multigrid_scalar_solver();
     743              :             }
     744          347 :             switch (backend) {
     745              : #ifdef CFD_ENABLE_OPENMP
     746          153 :                 case POISSON_BACKEND_OMP:
     747          153 :                     return create_multigrid_omp_solver();
     748              : #endif
     749          186 :                 case POISSON_BACKEND_SCALAR:
     750          186 :                     return create_multigrid_scalar_solver();
     751            8 :                 default:
     752            8 :                     return backend_unavailable("multigrid");
     753              :             }
     754              : 
     755            1 :         default:
     756            1 :             cfd_set_error(CFD_ERROR_INVALID, "Unknown Poisson solver method");
     757            1 :             return NULL;
     758              :     }
     759              : }
     760              : 
     761         1850 : cfd_status_t poisson_solver_init(
     762              :     poisson_solver_t* solver,
     763              :     size_t nx, size_t ny, size_t nz,
     764              :     double dx, double dy, double dz,
     765              :     const poisson_solver_params_t* params)
     766              : {
     767         1850 :     if (!solver) {
     768              :         return CFD_ERROR_INVALID;
     769              :     }
     770              : 
     771              :     /* Require at least one interior cell in each active dimension.
     772              :      * For 2D (nz <= 1): nx, ny >= 3.  For 3D (nz > 1): nx, ny, nz >= 3.
     773              :      * Rejects degenerate grids (e.g. nz==2) where k_start==k_end. */
     774         1849 :     if (nx < 3 || ny < 3 || (nz > 1 && nz < 3)) {
     775              :         return CFD_ERROR_INVALID;
     776              :     }
     777              : 
     778         1846 :     solver->nx = nx;
     779         1846 :     solver->ny = ny;
     780         1846 :     solver->nz = nz;
     781         1846 :     solver->dx = dx;
     782         1846 :     solver->dy = dy;
     783         1846 :     solver->dz = dz;
     784              : 
     785         1846 :     if (params) {
     786         1791 :         solver->params = *params;
     787              :     } else {
     788           55 :         solver->params = poisson_solver_params_default();
     789              :     }
     790              : 
     791              :     {
     792         1846 :         cfd_status_t wall_status = poisson_solver_check_config(solver);
     793         1846 :         if (wall_status != CFD_SUCCESS) {
     794              :             return wall_status;
     795              :         }
     796              :     }
     797              : 
     798              :     /* Adjust max_iterations for Jacobi (needs more iterations) */
     799         1792 :     if (solver->method == POISSON_METHOD_JACOBI && params == NULL) {
     800            9 :         solver->params.max_iterations = 2000;
     801              :     }
     802              : 
     803              :     /* Call solver-specific init if provided */
     804         1792 :     if (solver->init) {
     805         1792 :         return solver->init(solver, nx, ny, nz, dx, dy, dz, &solver->params);
     806              :     }
     807              : 
     808              :     return CFD_SUCCESS;
     809              : }
     810              : 
     811         2365 : void poisson_solver_destroy(poisson_solver_t* solver) {
     812         2365 :     if (!solver) {
     813              :         return;
     814              :     }
     815              : 
     816         1872 :     if (solver->destroy) {
     817         1872 :         solver->destroy(solver);
     818              :     }
     819              : 
     820         1872 :     cfd_free(solver);
     821              : }
     822              : 
     823              : /* ============================================================================
     824              :  * SOLVER OPERATIONS
     825              :  * ============================================================================ */
     826              : 
     827       188003 : double poisson_solver_compute_residual(
     828              :     poisson_solver_t* solver,
     829              :     const double* x,
     830              :     const double* rhs)
     831              : {
     832       188003 :     if (!solver || !x || !rhs) {
     833              :         return -1.0;
     834              :     }
     835              : 
     836       188000 :     size_t nx = solver->nx;
     837       188000 :     size_t ny = solver->ny;
     838       188000 :     double dx2 = solver->dx * solver->dx;
     839       188000 :     double dy2 = solver->dy * solver->dy;
     840       188000 :     double inv_dz2 = poisson_solver_compute_inv_dz2(solver->dz);
     841       188000 :     double sigma = solver->params.helmholtz_shift;
     842              : 
     843       188000 :     size_t stride_z, k_start, k_end;
     844       188000 :     poisson_solver_compute_3d_bounds(solver->nz, nx, ny,
     845              :                                      &stride_z, &k_start, &k_end);
     846              : 
     847       188000 :     double max_residual = 0.0;
     848              : 
     849       646969 :     for (size_t k = k_start; k < k_end; k++) {
     850     13332251 :         for (size_t j = 1; j < ny - 1; j++) {
     851    471459369 :             for (size_t i = 1; i < nx - 1; i++) {
     852    458586087 :                 size_t idx = k * stride_z + IDX_2D(i, j, nx);
     853              : 
     854              :                 /* Compute Laplacian: d^2x/dx^2 + d^2x/dy^2 + d^2x/dz^2 */
     855    458586087 :                 double laplacian =
     856    458586087 :                     (x[idx + 1] - 2.0 * x[idx] + x[idx - 1]) / dx2
     857    458586087 :                   + (x[idx + nx] - 2.0 * x[idx] + x[idx - nx]) / dy2
     858    458586087 :                   + (x[idx + stride_z] + x[idx - stride_z]
     859    458586087 :                      - 2.0 * x[idx]) * inv_dz2;
     860              : 
     861              :                 /* Operator is nabla^2 - sigma*I; sigma = 0 leaves this exact. */
     862    458586087 :                 double diff = laplacian - rhs[idx];
     863    458586087 :                 if (sigma != 0.0) {
     864            0 :                     diff -= sigma * x[idx];
     865              :                 }
     866    458586087 :                 double residual = fabs(diff);
     867              :                 /* A NaN compares false against everything, so without the
     868              :                  * isnan() a diverged field would read as a zero residual */
     869    458586087 :                 if (residual > max_residual || isnan(residual)) {
     870      8662173 :                     max_residual = residual;
     871              :                 }
     872              :             }
     873              :         }
     874              :     }
     875              : 
     876              :     return max_residual;
     877              : }
     878              : 
     879              : /**
     880              :  * Write one face's halo: copy the adjacent interior out for a zero-gradient face,
     881              :  * or write a constant for a Dirichlet one.
     882              :  *
     883              :  * `stride` steps inward from the face, `n_outer`/`n_inner` walk the face itself.
     884              :  */
     885       200310 : static void write_face(double* x, size_t base, ptrdiff_t stride,
     886              :                        size_t n_outer, size_t outer_stride,
     887              :                        size_t n_inner, size_t inner_stride,
     888              :                        poisson_wall_t type, double value)
     889              : {
     890       613932 :     for (size_t a = 0; a < n_outer; a++) {
     891      9076716 :         for (size_t b = 0; b < n_inner; b++) {
     892      8663094 :             size_t idx = base + (a * outer_stride) + (b * inner_stride);
     893      8663094 :             x[idx] = (type == POISSON_WALL_DIRICHLET)
     894              :                    ? value
     895      8663094 :                    : x[(size_t)((ptrdiff_t)idx + stride)];
     896              :         }
     897              :     }
     898              : }
     899              : 
     900              : /**
     901              :  * Apply per-face walls to x's halo.
     902              :  *
     903              :  * homogeneous != 0 writes 0 on every Dirichlet face: that is the homogeneous part
     904              :  * of the condition, which is what a Krylov search direction must carry. The
     905              :  * zero-gradient extension is already its own homogeneous form, so it is identical
     906              :  * either way.
     907              :  *
     908              :  * Faces run left, right, bottom, top, back, front so the order is deterministic;
     909              :  * a 5/7-point stencil never reads a corner halo cell, so which face owns a shared
     910              :  * corner does not change any answer.
     911              :  */
     912        46746 : static void poisson_apply_walls(const poisson_solver_t* s, double* x, int homogeneous)
     913              : {
     914        46746 :     const poisson_walls_t* w = &s->params.walls;
     915        46746 :     size_t nx = s->nx;
     916        46746 :     size_t ny = s->ny;
     917        46746 :     size_t nz = s->nz;
     918        46746 :     size_t plane = nx * ny;
     919              : 
     920              :     #define WALL_VALUE(face) (homogeneous ? 0.0 : w->values.face)
     921              : 
     922              :     /* x-faces: whole column of each plane */
     923        46746 :     write_face(x, 0, +1, nz, plane, ny, nx, w->left, WALL_VALUE(left));
     924        46746 :     write_face(x, nx - 1, -1, nz, plane, ny, nx, w->right, WALL_VALUE(right));
     925              : 
     926              :     /* y-faces: whole row of each plane */
     927        46746 :     write_face(x, 0, (ptrdiff_t)nx, nz, plane, nx, 1, w->bottom, WALL_VALUE(bottom));
     928        46746 :     write_face(x, (ny - 1) * nx, -(ptrdiff_t)nx, nz, plane, nx, 1, w->top, WALL_VALUE(top));
     929              : 
     930              :     /* z-faces: whole plane. Only on a 3D grid. */
     931        46746 :     if (nz > 1) {
     932         6663 :         write_face(x, 0, (ptrdiff_t)plane, 1, 0, plane, 1, w->back, WALL_VALUE(back));
     933         6663 :         write_face(x, (nz - 1) * plane, -(ptrdiff_t)plane, 1, 0, plane, 1,
     934         6663 :                    w->front, WALL_VALUE(front));
     935              :     }
     936              : 
     937              :     #undef WALL_VALUE
     938        46746 : }
     939              : 
     940      1300892 : void poisson_solver_apply_bc(
     941              :     poisson_solver_t* solver,
     942              :     double* x)
     943              : {
     944      1300892 :     if (!solver || !x) {
     945              :         return;
     946              :     }
     947              : 
     948              :     /* The caller's function wins; a solver's own is the fallback. Nothing sets
     949              :      * both -- a solver that installs internal_apply_bc rejects a caller function
     950              :      * at init -- so the precedence never actually has to arbitrate. */
     951      1300891 :     if (solver->apply_bc) {
     952        65005 :         solver->apply_bc(solver, x);
     953        65005 :         return;
     954              :     }
     955      1235886 :     if (solver->internal_apply_bc) {
     956           36 :         solver->internal_apply_bc(solver, x);
     957           36 :         return;
     958              :     }
     959              : 
     960              :     /* Per-face walls, when the caller configured any. The all-zero-gradient case
     961              :      * falls through to the loop below, which computes the same extension by the
     962              :      * shorter route: a whole-plane memcpy for the z faces rather than
     963              :      * write_face's per-cell indexed copy. (It once fell through to reach the
     964              :      * parallel OMP/SIMD BC primitives; that dispatch is gone and the loop below
     965              :      * is now serial on every backend, for the reason stated there.) */
     966      1235850 :     if (!poisson_walls_are_default(&solver->params.walls)) {
     967         3921 :         poisson_apply_walls(solver, x, 0);
     968         3921 :         return;
     969              :     }
     970              : 
     971              :     /* Default: Neumann BCs (zero gradient) on all faces */
     972      1231929 :     size_t nx = solver->nx;
     973      1231929 :     size_t ny = solver->ny;
     974      1231929 :     size_t nz = solver->nz;
     975      1231929 :     size_t plane_size = nx * ny;
     976              : 
     977              :     /* Z-face Neumann: copy adjacent interior plane to boundary planes */
     978      1231929 :     if (nz > 1) {
     979        32411 :         memcpy(x, x + plane_size, plane_size * sizeof(double));
     980        32411 :         memcpy(x + (nz - 1) * plane_size,
     981        32411 :                x + (nz - 2) * plane_size,
     982              :                plane_size * sizeof(double));
     983              :     }
     984              : 
     985              :     /* Serial on every backend, deliberately.
     986              :      *
     987              :      * A Neumann face is a copy from the adjacent interior line, so all three
     988              :      * backends produce identical values -- but this runs once per Krylov
     989              :      * iteration, and an OpenMP region costs ~12 us to enter at 4 threads on
     990              :      * MSVC while the work here is O(nx + ny): about 130 writes on a 33x33
     991              :      * plane, less than the 31x31 Laplacian sweep it accompanies. Threading it
     992              :      * cost more than the solve. BiCGSTAB pays this twice per iteration. */
     993      2771272 :     for (size_t k = 0; k < nz; k++) {
     994      1539343 :         bc_apply_scalar_cpu(x + k * plane_size, nx, ny, BC_TYPE_NEUMANN);
     995              :     }
     996              : }
     997              : 
     998              : /** Write 0 into every boundary cell, leaving the interior alone. */
     999          359 : static void krylov_zero_halo(const poisson_solver_t* solver, double* x)
    1000              : {
    1001          359 :     size_t nx = solver->nx;
    1002          359 :     size_t ny = solver->ny;
    1003          359 :     size_t nz = solver->nz;
    1004          359 :     size_t plane_size = nx * ny;
    1005              : 
    1006          359 :     size_t k_first = 0;
    1007          359 :     size_t k_last = 1;
    1008          359 :     if (nz > 1) {
    1009           37 :         memset(x, 0, plane_size * sizeof(double));
    1010           37 :         memset(x + (nz - 1) * plane_size, 0, plane_size * sizeof(double));
    1011           37 :         k_first = 1;
    1012           37 :         k_last = nz - 1;
    1013              :     }
    1014              : 
    1015         1252 :     for (size_t k = k_first; k < k_last; k++) {
    1016          893 :         double* plane = x + k * plane_size;
    1017          893 :         memset(plane, 0, nx * sizeof(double));                 /* j = 0 */
    1018          893 :         memset(plane + (ny - 1) * nx, 0, nx * sizeof(double)); /* j = ny-1 */
    1019        23177 :         for (size_t j = 1; j < ny - 1; j++) {
    1020        22284 :             plane[j * nx] = 0.0;                               /* i = 0 */
    1021        22284 :             plane[(j * nx) + (nx - 1)] = 0.0;                  /* i = nx-1 */
    1022              :         }
    1023              :     }
    1024          359 : }
    1025              : 
    1026        16374 : int poisson_solver_krylov_is_singular(const poisson_solver_t* solver)
    1027              : {
    1028        16374 :     if (!solver) {
    1029              :         return 0;
    1030              :     }
    1031              :     /* A Helmholtz shift adds sigma*I, so the constants stop being a nullspace and
    1032              :      * the operator is nonsingular whatever the walls say. Checked here rather
    1033              :      * than at the one call site, so that every future caller of a predicate
    1034              :      * named "is singular" gets a true answer. */
    1035        16374 :     if (solver->params.helmholtz_shift > 0.0) {
    1036              :         return 0;
    1037              :     }
    1038              :     /* A hook prescribes wall values, which pins the level and makes the operator
    1039              :      * nonsingular, exactly as a Dirichlet face does. */
    1040        14005 :     return solver->apply_bc == NULL
    1041        14005 :         && poisson_walls_are_singular(&solver->params.walls, solver->nz);
    1042              : }
    1043              : 
    1044      1123053 : void poisson_solver_krylov_apply_bc_homogeneous(
    1045              :     poisson_solver_t* solver,
    1046              :     double* v)
    1047              : {
    1048      1123053 :     if (!solver || !v) {
    1049              :         return;
    1050              :     }
    1051              : 
    1052              :     /* The zero-gradient extension is already homogeneous -- it copies the interior
    1053              :      * outwards and adds nothing -- so all-zero-gradient walls need it verbatim.
    1054              :      * With a Dirichlet face configured, the prescribed value belongs to the iterate
    1055              :      * and the direction gets 0 there instead. */
    1056      1123053 :     if (!solver->apply_bc) {
    1057      1122726 :         if (poisson_walls_are_default(&solver->params.walls)) {
    1058      1079901 :             poisson_solver_apply_bc(solver, v);
    1059              :         } else {
    1060        42825 :             poisson_apply_walls(solver, v, 1);
    1061              :         }
    1062      1122726 :         return;
    1063              :     }
    1064              : 
    1065              :     /* A custom hook prescribes wall values that do not depend on the interior, so
    1066              :      * its homogeneous part is a zero halo. Running the hook on a search direction
    1067              :      * would add its lift to every one of them, which is not a linear operator and
    1068              :      * would leave the Krylov recurrences describing something other than A. */
    1069          327 :     krylov_zero_halo(solver, v);
    1070              : }
    1071              : 
    1072        16690 : void poisson_solver_krylov_apply_bc(
    1073              :     poisson_solver_t* solver,
    1074              :     double* x)
    1075              : {
    1076        16690 :     if (!solver || !x) {
    1077              :         return;
    1078              :     }
    1079              : 
    1080              :     /* The iterate, unlike a search direction, carries the lift as well, so the
    1081              :      * residual formed from it is the residual of the real system. For the default
    1082              :      * walls that is just the extension. */
    1083        16690 :     if (!solver->apply_bc) {
    1084        16658 :         poisson_solver_apply_bc(solver, x);
    1085        16658 :         return;
    1086              :     }
    1087              : 
    1088              :     /* Zero first, then let the hook write over it: a hook that writes only some
    1089              :      * walls, or that holds them at zero by writing nothing, then still leaves no
    1090              :      * stale value behind for a warm start to trip on. */
    1091           32 :     krylov_zero_halo(solver, x);
    1092           32 :     solver->apply_bc(solver, x);
    1093              : }
    1094              : 
    1095              : /**
    1096              :  * Common solve loop used by all solvers
    1097              :  */
    1098         1032 : cfd_status_t poisson_solver_solve_common(
    1099              :     poisson_solver_t* solver,
    1100              :     double* x,
    1101              :     double* x_temp,
    1102              :     const double* rhs,
    1103              :     poisson_solver_stats_t* stats)
    1104              : {
    1105         1032 :     if (!solver || !x || !rhs) {
    1106              :         return CFD_ERROR_INVALID;
    1107              :     }
    1108              : 
    1109         1032 :     if (!solver->iterate) {
    1110              :         return CFD_ERROR_UNSUPPORTED;
    1111              :     }
    1112              : 
    1113         1032 :     poisson_solver_params_t* params = &solver->params;
    1114         1032 :     double start_time = poisson_solver_get_time_ms();
    1115              : 
    1116              :     /* Compute initial residual. The target never sits below the round-off floor
    1117              :      * of the residual at x, which a warm start near steady state already reaches
    1118              :      * (poisson_solver_residual_floor). */
    1119         1032 :     double initial_res = poisson_solver_compute_residual(solver, x, rhs);
    1120         1032 :     double floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_MAX);
    1121         1032 :     double tolerance = poisson_solver_stop_target(params, initial_res, floor_res);
    1122              : 
    1123         1032 :     if (stats) {
    1124         1032 :         stats->initial_residual = initial_res;
    1125              :     }
    1126              : 
    1127              :     /* A start whose residual is not finite has diverged before the first sweep */
    1128         1032 :     if (!isfinite(initial_res)) {
    1129            1 :         if (stats) {
    1130            1 :             stats->status = POISSON_DIVERGED;
    1131            1 :             stats->iterations = 0;
    1132            1 :             stats->final_residual = initial_res;
    1133            1 :             stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
    1134              :         }
    1135            1 :         return CFD_ERROR_DIVERGED;
    1136              :     }
    1137              : 
    1138              :     /* Already converged? */
    1139         2051 :     if (initial_res < poisson_solver_done_target(params, floor_res)) {
    1140           23 :         if (stats) {
    1141           23 :             stats->status = POISSON_CONVERGED;
    1142           23 :             stats->iterations = 0;
    1143           23 :             stats->final_residual = initial_res;
    1144           23 :             stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
    1145              :         }
    1146           23 :         return CFD_SUCCESS;
    1147              :     }
    1148              : 
    1149       152409 :     int converged = 0;
    1150       152409 :     int diverged = 0;
    1151              :     int iter;
    1152              :     double res = initial_res;
    1153              : 
    1154       152409 :     for (iter = 0; iter < params->max_iterations; iter++) {
    1155       152382 :         double new_res = 0.0;
    1156              : 
    1157              :         /* Check at intervals and on the last iteration, so that a solve never ends
    1158              :          * on a residual older than the field it returns */
    1159       152382 :         int check = (iter % params->check_interval == 0) || (iter + 1 == params->max_iterations);
    1160              : 
    1161              :         /* Perform one iteration */
    1162       152382 :         cfd_status_t status = solver->iterate(solver, x, x_temp, rhs, check ? &new_res : NULL);
    1163              : 
    1164       152382 :         if (status != CFD_SUCCESS) {
    1165            0 :             if (stats) {
    1166            0 :                 stats->status = POISSON_ERROR;
    1167            0 :                 stats->iterations = iter + 1;
    1168            0 :                 stats->final_residual = res;
    1169            0 :                 stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
    1170              :             }
    1171            0 :             return status;
    1172              :         }
    1173              : 
    1174       152382 :         if (check) {
    1175       147384 :             res = new_res;
    1176              : 
    1177              :             /* A residual that is no longer finite will never meet the tolerance */
    1178       147384 :             if (!isfinite(res)) {
    1179              :                 diverged = 1;
    1180          981 :                 break;
    1181              :             }
    1182              : 
    1183       147382 :             if (params->verbose) {
    1184            0 :                 CFD_LOG_DEBUG("poisson", "Iter %d: residual = %.6e", iter, res);
    1185              :             }
    1186              : 
    1187              :             /* x has moved since the start, and the floor with it */
    1188       147382 :             floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_MAX);
    1189       293785 :             if (res < tolerance || res < poisson_solver_done_target(params, floor_res)) {
    1190              :                 converged = 1;
    1191              :                 break;
    1192              :             }
    1193              :         }
    1194              :     }
    1195              : 
    1196         1008 :     double end_time = poisson_solver_get_time_ms();
    1197              : 
    1198         1008 :     if (stats) {
    1199              :         /* A break leaves iter at the index of the last iteration run;
    1200              :          * an exhausted loop leaves it at max_iterations, the count run */
    1201         1008 :         stats->iterations = (converged || diverged) ? iter + 1 : iter;
    1202         1008 :         stats->final_residual = res;
    1203         1008 :         stats->elapsed_time_ms = end_time - start_time;
    1204           29 :         stats->status = converged ? POISSON_CONVERGED
    1205         1008 :                       : diverged  ? POISSON_DIVERGED
    1206              :                                   : POISSON_MAX_ITER;
    1207              :     }
    1208              : 
    1209         1008 :     if (converged) {
    1210          979 :         return CFD_SUCCESS;
    1211              :     }
    1212           29 :     return diverged ? CFD_ERROR_DIVERGED : CFD_ERROR_MAX_ITER;
    1213              : }
    1214              : 
    1215           15 : void poisson_make_rhs_compatible(double* rhs, size_t nx, size_t ny, size_t nz) {
    1216              :     /* A grid with no interior has no interior mean to remove, and the loops
    1217              :      * below it count down from nx - 1 in size_t, so nx == 0 would wrap to
    1218              :      * SIZE_MAX and run off the buffer. Every other public entry point screens
    1219              :      * this; as an exported helper taking raw dimensions, so must this one. */
    1220           15 :     if (!rhs || nx < 3 || ny < 3 || (nz > 1 && nz < 3)) {
    1221              :         return;
    1222              :     }
    1223           10 :     mg_subtract_interior_mean(rhs, nx, ny, nz);
    1224              : }
    1225              : 
    1226              : /**
    1227              :  * Reject a right-hand side the singular operator has no solution for.
    1228              :  *
    1229              :  * With zero-gradient walls the constants are the operator's nullspace, so a rhs
    1230              :  * with a nonzero interior mean lies partly outside its range: there is no x that
    1231              :  * solves it, and the iteration either stalls on that component or, for Krylov
    1232              :  * methods, drives the rest of the field away chasing it.
    1233              :  *
    1234              :  * The mean is compared against the residual the caller asked for, scaled by the
    1235              :  * size of the rhs -- an incompatibility smaller than the target tolerance cannot
    1236              :  * affect the answer, and the projection solvers leave one of that size behind
    1237              :  * after mean-subtracting in floating point.
    1238              :  */
    1239        17406 : static cfd_status_t check_rhs_compatible(const poisson_solver_t* solver, const double* rhs)
    1240              : {
    1241              :     /* Krylov methods only. On a system with no solution they do not merely fail to
    1242              :      * converge: the minimisation chases the component in the nullspace and drives
    1243              :      * the rest of the field away with it, which is how test_laplacian_accuracy
    1244              :      * reached a residual of 1e21. The stationary and multigrid solvers relax
    1245              :      * towards a solution modulo a drifting constant instead, which callers use
    1246              :      * deliberately -- running a fixed sweep count on an arbitrary rhs is a
    1247              :      * reasonable thing to ask of them, so it is left alone. */
    1248        17406 :     if (!method_honours_walls(solver->method)) {
    1249              :         return CFD_SUCCESS;
    1250              :     }
    1251              : 
    1252        16374 :     if (!poisson_solver_krylov_is_singular(solver)) {
    1253              :         /* A prescribed face pins the level, and a Helmholtz shift removes the
    1254              :          * nullspace outright; either way any rhs is solvable. */
    1255              :         return CFD_SUCCESS;
    1256              :     }
    1257              : 
    1258              :     /* Solve before init leaves the dimensions at 0, where the interior loops below
    1259              :      * would underflow. Let the solver itself report that; there is no rhs to judge
    1260              :      * against a grid that does not exist yet. */
    1261        13965 :     if (solver->nx < 3 || solver->ny < 3) {
    1262              :         return CFD_SUCCESS;
    1263              :     }
    1264              : 
    1265        13964 :     size_t nx = solver->nx;
    1266        13964 :     size_t ny = solver->ny;
    1267        13964 :     size_t nz = solver->nz;
    1268        13964 :     size_t stride_z = (nz > 1) ? nx * ny : 0;
    1269        13964 :     size_t k_start = (nz > 1) ? 1 : 0;
    1270        13964 :     size_t k_end = (nz > 1) ? nz - 1 : 1;
    1271              : 
    1272        13964 :     double sum = 0.0;
    1273        13964 :     double sum_abs = 0.0;
    1274        13964 :     size_t count = 0;
    1275        28602 :     for (size_t k = k_start; k < k_end; k++) {
    1276       449580 :         for (size_t j = 1; j < ny - 1; j++) {
    1277     17301304 :             for (size_t i = 1; i < nx - 1; i++) {
    1278     16866362 :                 double v = rhs[(k * stride_z) + IDX_2D(i, j, nx)];
    1279     16866362 :                 sum += v;
    1280     16866362 :                 sum_abs += fabs(v);
    1281     16866362 :                 count++;
    1282              :             }
    1283              :         }
    1284              :     }
    1285        13964 :     if (count == 0 || sum_abs == 0.0) {
    1286              :         return CFD_SUCCESS;
    1287              :     }
    1288              : 
    1289              :     /* Relative to the mean magnitude, so the test is scale-free. */
    1290        13401 :     double tol = solver->params.tolerance;
    1291        13401 :     if (tol <= 0.0) {
    1292            0 :         tol = 1e-6;
    1293              :     }
    1294        13401 :     if (fabs(sum) > tol * sum_abs) {
    1295            2 :         cfd_set_error(CFD_ERROR_INVALID,
    1296              :             "zero-gradient walls make this operator singular, so the rhs must have zero "
    1297              :             "interior mean; call poisson_make_rhs_compatible() first, or prescribe a "
    1298              :             "wall value on one face via params.walls");
    1299            2 :         return CFD_ERROR_INVALID;
    1300              :     }
    1301              :     return CFD_SUCCESS;
    1302              : }
    1303              : 
    1304        17406 : cfd_status_t poisson_solver_solve(
    1305              :     poisson_solver_t* solver,
    1306              :     double* x,
    1307              :     double* x_temp,
    1308              :     const double* rhs,
    1309              :     poisson_solver_stats_t* stats)
    1310              : {
    1311        17406 :     if (!solver) {
    1312              :         return CFD_ERROR_INVALID;
    1313              :     }
    1314              : 
    1315        17406 :     if (rhs) {
    1316        17406 :         cfd_status_t compat = check_rhs_compatible(solver, rhs);
    1317        17406 :         if (compat != CFD_SUCCESS) {
    1318            2 :             if (stats) {
    1319              :                 /* Reset first: a caller reusing one stats struct across solves
    1320              :                  * would otherwise read the previous solve's residual and timing
    1321              :                  * next to this refusal, which looks like a converged result. */
    1322            2 :                 *stats = poisson_solver_stats_default();
    1323            2 :                 stats->status = POISSON_INCOMPATIBLE_RHS;
    1324            2 :                 stats->iterations = 0;
    1325              :             }
    1326            2 :             return compat;
    1327              :         }
    1328              :     }
    1329              : 
    1330              :     /* Use solver-specific solve if provided, otherwise common loop */
    1331        17404 :     if (solver->solve) {
    1332        16674 :         double start_time = poisson_solver_get_time_ms();
    1333        16674 :         cfd_status_t status = solver->solve(solver, x, x_temp, rhs, stats);
    1334        16674 :         if (stats) {
    1335        16674 :             stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
    1336              :         }
    1337        16674 :         return status;
    1338              :     }
    1339              : 
    1340          730 :     return poisson_solver_solve_common(solver, x, x_temp, rhs, stats);
    1341              : }
    1342              : 
    1343        41832 : cfd_status_t poisson_solver_iterate(
    1344              :     poisson_solver_t* solver,
    1345              :     double* x,
    1346              :     double* x_temp,
    1347              :     const double* rhs,
    1348              :     double* residual)
    1349              : {
    1350        41832 :     if (!solver || !x || !rhs) {
    1351              :         return CFD_ERROR_INVALID;
    1352              :     }
    1353              : 
    1354        41832 :     if (!solver->iterate) {
    1355              :         return CFD_ERROR_UNSUPPORTED;
    1356              :     }
    1357              : 
    1358        41832 :     return solver->iterate(solver, x, x_temp, rhs, residual);
    1359              : }
    1360              : 
    1361              : /* ============================================================================
    1362              :  * CONVENIENCE API
    1363              :  * ============================================================================ */
    1364              : 
    1365          136 : cfd_status_t poisson_solve(
    1366              :     double* x, double* x_temp, const double* rhs,
    1367              :     size_t nx, size_t ny, size_t nz,
    1368              :     double dx, double dy, double dz,
    1369              :     const poisson_solver_config_t* config,
    1370              :     poisson_solver_stats_t* stats)
    1371              : {
    1372          136 :     if (!x || !rhs) {
    1373            0 :         cfd_set_error(CFD_ERROR_INVALID, "poisson_solve: x and rhs are required");
    1374            0 :         return CFD_ERROR_INVALID;
    1375              :     }
    1376              : 
    1377              :     /* Cleared so that the status read back below is this call's and not whatever
    1378              :      * the thread failed at last. */
    1379          136 :     cfd_clear_error();
    1380              : 
    1381          135 :     poisson_solver_config_t cfg =
    1382          136 :         config ? *config : poisson_solver_config_preset(POISSON_PRESET_DEFAULT);
    1383              : 
    1384          136 :     poisson_solver_t* solver = poisson_solver_create(cfg.method, cfg.backend);
    1385          136 :     if (!solver) {
    1386              :         /* The factory named the reason -- an unavailable backend, an unknown
    1387              :          * method -- so report that rather than overwriting it. A factory that
    1388              :          * returned NULL without setting one (an allocation failure inside a
    1389              :          * create_*_solver) must not read back as success. */
    1390            1 :         cfd_status_t reason = cfd_get_last_status();
    1391            1 :         if (reason == CFD_SUCCESS) {
    1392            0 :             cfd_set_error(CFD_ERROR_UNSUPPORTED,
    1393              :                 "poisson_solve: the requested method and backend could not be created");
    1394            0 :             reason = CFD_ERROR_UNSUPPORTED;
    1395              :         }
    1396            1 :         return reason;
    1397              :     }
    1398              : 
    1399          135 :     cfd_status_t status =
    1400          135 :         poisson_solver_init(solver, nx, ny, nz, dx, dy, dz, &cfg.params);
    1401          135 :     if (status != CFD_SUCCESS) {
    1402            6 :         poisson_solver_destroy(solver);
    1403            6 :         return status;
    1404              :     }
    1405              : 
    1406          129 :     poisson_solver_stats_t local = poisson_solver_stats_default();
    1407          257 :     status = poisson_solver_solve(solver, x, x_temp, rhs, stats ? stats : &local);
    1408          129 :     poisson_solver_destroy(solver);
    1409          129 :     return status;
    1410              : }
        

Generated by: LCOV version 2.0-1