Line data Source code
1 : /**
2 : * @file linear_solver.c
3 : * @brief Core linear solver implementation
4 : *
5 : * Implements:
6 : * - Default parameter functions
7 : * - Backend selection
8 : * - Solver lifecycle (create, init, destroy)
9 : * - Common solve loop
10 : * - Configuration validation
11 : * - The poisson_solve() convenience entry point
12 : */
13 :
14 : #include "cfd/solvers/poisson_solver.h"
15 : #include "linear_solver_internal.h"
16 : #include "multigrid_internal.h" /* mg_subtract_interior_mean, MG_DEFAULT_*_SMOOTH */
17 :
18 : #include "cfd/boundary/boundary_conditions.h"
19 : #include "cfd/core/indexing.h"
20 : #include "cfd/core/logging.h"
21 : #include "cfd/core/memory.h"
22 :
23 : #include <math.h>
24 : #include <stdio.h>
25 : #include <string.h>
26 :
27 : #ifdef _WIN32
28 : #define WIN32_LEAN_AND_MEAN
29 : #include <windows.h>
30 : #else
31 : #include <sys/time.h>
32 : #endif
33 :
34 : /* ============================================================================
35 : * DEFAULT PARAMETERS
36 : * ============================================================================ */
37 :
38 18628 : poisson_walls_t poisson_walls_default(void) {
39 18628 : poisson_walls_t walls;
40 18628 : memset(&walls, 0, sizeof walls); /* POISSON_WALL_ZERO_GRADIENT is 0 */
41 18628 : return walls;
42 : }
43 :
44 29 : poisson_walls_t poisson_walls_uniform(poisson_wall_t type, double value) {
45 29 : poisson_walls_t walls = poisson_walls_default();
46 29 : walls.left = walls.right = walls.bottom = walls.top = walls.front = walls.back = type;
47 29 : if (type == POISSON_WALL_DIRICHLET) {
48 27 : walls.values.left = walls.values.right = value;
49 27 : walls.values.bottom = walls.values.top = value;
50 27 : walls.values.front = walls.values.back = value;
51 : }
52 29 : return walls;
53 : }
54 :
55 2360929 : bool poisson_walls_are_default(const poisson_walls_t* w)
56 : {
57 2360929 : if (!w) {
58 : return true; /* No walls configured is the default operator. */
59 : }
60 2360928 : return w->left == POISSON_WALL_ZERO_GRADIENT
61 2360928 : && w->right == POISSON_WALL_ZERO_GRADIENT
62 : && w->bottom == POISSON_WALL_ZERO_GRADIENT
63 2314096 : && w->top == POISSON_WALL_ZERO_GRADIENT
64 : && w->front == POISSON_WALL_ZERO_GRADIENT
65 4721861 : && w->back == POISSON_WALL_ZERO_GRADIENT;
66 : }
67 :
68 : /**
69 : * A sweep count as the multigrid hierarchy will actually read it.
70 : *
71 : * 0 means "the default" in this field, and the template resolves it that way
72 : * (linear_solver_multigrid_template.h). Comparing the raw values would call
73 : * pre_smooth = 0, post_smooth = 2 asymmetric and refuse it, when the cycle it
74 : * builds is the symmetric V(2,2) the rule is there to require.
75 : */
76 90 : static int mg_smooth_effective(int requested, int fallback) {
77 90 : return requested > 0 ? requested : fallback;
78 : }
79 :
80 : /* One face. Why it is worth asking at all is on poisson_walls_are_legal() in
81 : * the public header; keeping the argument in one place is the point of having
82 : * exported the predicate. */
83 : static int wall_type_is_legal(poisson_wall_t type) {
84 : return type == POISSON_WALL_ZERO_GRADIENT || type == POISSON_WALL_DIRICHLET;
85 : }
86 :
87 2374 : bool poisson_walls_are_legal(const poisson_walls_t* w)
88 : {
89 2374 : if (!w) {
90 : return true; /* No walls configured is the default operator. */
91 : }
92 2371 : return wall_type_is_legal(w->left) && wall_type_is_legal(w->right)
93 2369 : && wall_type_is_legal(w->bottom) && wall_type_is_legal(w->top)
94 4750 : && wall_type_is_legal(w->front) && wall_type_is_legal(w->back);
95 : }
96 :
97 : /** Whether this method's halo routines honour per-face walls (Krylov only). */
98 17484 : static int method_honours_walls(poisson_solver_method_t method) {
99 17484 : return method == POISSON_METHOD_CG
100 : || method == POISSON_METHOD_BICGSTAB
101 17484 : || method == POISSON_METHOD_GMRES;
102 : }
103 :
104 : /* ============================================================================
105 : * CONFIGURATION VALIDATION
106 : *
107 : * One place decides what a given (method, backend) can honour, so a parameter it
108 : * cannot is refused at init rather than quietly ignored. The alternative -- a
109 : * check inside each of the twenty-odd solver inits -- is what let BiCGSTAB ignore
110 : * params.krylov.preconditioner on every backend, and three GPU solvers ignore a
111 : * caller's apply_bc, for as long as they have existed.
112 : * ============================================================================ */
113 :
114 : /** Parameter groups, for the ownership table below. */
115 : enum {
116 : PARAM_GROUP_SOR = 1u << 0,
117 : PARAM_GROUP_KRYLOV = 1u << 1,
118 : PARAM_GROUP_MULTIGRID = 1u << 2
119 : };
120 :
121 : /**
122 : * The groups this method reads. Anything else it is handed, it ignores -- which
123 : * is exactly what must be refused.
124 : */
125 1813 : static unsigned method_owned_groups(const poisson_solver_t* solver) {
126 1813 : switch (solver->method) {
127 : case POISSON_METHOD_JACOBI:
128 : return 0;
129 732 : case POISSON_METHOD_SOR:
130 : case POISSON_METHOD_GAUSS_SEIDEL:
131 : case POISSON_METHOD_REDBLACK_SOR:
132 732 : return PARAM_GROUP_SOR;
133 544 : case POISSON_METHOD_CG:
134 : /* CG reads the multigrid group only when it is preconditioned by
135 : * one, where those parameters shape the inner V-cycle. */
136 544 : return PARAM_GROUP_KRYLOV
137 544 : | (solver->params.krylov.preconditioner == POISSON_PRECOND_MULTIGRID
138 : ? PARAM_GROUP_MULTIGRID : 0u);
139 : case POISSON_METHOD_BICGSTAB:
140 : case POISSON_METHOD_GMRES:
141 : return PARAM_GROUP_KRYLOV;
142 372 : case POISSON_METHOD_MULTIGRID:
143 372 : return PARAM_GROUP_MULTIGRID;
144 : default:
145 : return 0;
146 : }
147 : }
148 :
149 : /*
150 : * Whether a group is entirely at its defaults, i.e. untouched by the caller.
151 : *
152 : * Field by field, not a memcmp or a byte scan over the struct. Padding bytes
153 : * are not required to be zero in a caller's `poisson_solver_params_t p = {0};`
154 : * -- C11 6.7.9 leaves them unspecified -- so a byte scan can read garbage and
155 : * report a group the caller never touched as set. It also silently depends on
156 : * these three structs having no padding, which is true today and is not
157 : * something a future field can be trusted to preserve.
158 : */
159 1813 : static int sor_untouched(const poisson_sor_params_t* g) {
160 1813 : return g->omega == 0.0;
161 : }
162 :
163 1811 : static int krylov_untouched(const poisson_krylov_params_t* g) {
164 1729 : return g->preconditioner == POISSON_PRECOND_NONE && g->restart == 0;
165 : }
166 :
167 1810 : static int multigrid_untouched(const poisson_multigrid_params_t* g) {
168 1692 : return g->cycle == 0 && g->smoother == 0 && g->bc == 0
169 1548 : && g->pre_smooth == 0 && g->post_smooth == 0
170 3350 : && g->coarse_max_iter == 0 && g->max_levels == 0;
171 : }
172 :
173 5434 : static cfd_status_t reject_unowned_group(const poisson_solver_t* solver,
174 : unsigned owned, unsigned group,
175 : int untouched,
176 : const char* name) {
177 5434 : if ((owned & group) || untouched) {
178 : return CFD_SUCCESS;
179 : }
180 4 : char msg[192];
181 8 : snprintf(msg, sizeof(msg),
182 : "params.%s is set, but the %s method does not read it; "
183 : "it would have no effect on this solve",
184 4 : name, solver->name ? solver->name : "selected");
185 4 : cfd_set_error(CFD_ERROR_INVALID, msg);
186 4 : return CFD_ERROR_INVALID;
187 : }
188 :
189 1846 : cfd_status_t poisson_solver_check_config(const poisson_solver_t* solver) {
190 1846 : const poisson_solver_params_t* p = &solver->params;
191 :
192 : /* ---- walls -------------------------------------------------------- */
193 : /* Range first: an out-of-range face is neither zero-gradient nor prescribed
194 : * to the code that reads it, so nothing below would agree about what the
195 : * operator is. */
196 1846 : if (!poisson_walls_are_legal(&p->walls)) {
197 7 : cfd_set_error(CFD_ERROR_INVALID,
198 : "every params.walls face must be POISSON_WALL_ZERO_GRADIENT or "
199 : "POISSON_WALL_DIRICHLET");
200 7 : return CFD_ERROR_INVALID;
201 : }
202 1839 : if (!poisson_walls_are_default(&p->walls)) {
203 : /* Support first, then the hook conflict: an unsupported method should say
204 : * so rather than report a caller error. */
205 78 : if (!method_honours_walls(solver->method)) {
206 5 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
207 : "per-face walls are implemented for the CG, BiCGSTAB and GMRES methods only; "
208 : "the stationary and multigrid solvers apply whole-domain walls inside their sweeps");
209 5 : return CFD_ERROR_UNSUPPORTED;
210 : }
211 73 : if (solver->backend == POISSON_BACKEND_GPU) {
212 0 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
213 : "per-face walls are not implemented on the GPU backend, which applies the "
214 : "zero-gradient walls on the device");
215 0 : return CFD_ERROR_UNSUPPORTED;
216 : }
217 73 : if (solver->apply_bc) {
218 1 : cfd_set_error(CFD_ERROR_INVALID,
219 : "params.walls and a custom apply_bc both prescribe wall values; use one or the other");
220 1 : return CFD_ERROR_INVALID;
221 : }
222 72 : if (!isfinite(p->walls.values.left) || !isfinite(p->walls.values.right)
223 72 : || !isfinite(p->walls.values.bottom) || !isfinite(p->walls.values.top)
224 72 : || !isfinite(p->walls.values.front) || !isfinite(p->walls.values.back)) {
225 0 : cfd_set_error(CFD_ERROR_INVALID, "params.walls.values must be finite");
226 0 : return CFD_ERROR_INVALID;
227 : }
228 : /* A 2D grid has no z-faces, so poisson_apply_walls skips .front/.back and
229 : * poisson_walls_are_singular ignores them. Prescribing one is refused
230 : * only when it was the caller's only prescribed face: they meant to pin
231 : * the level, nothing pinned it, and the operator is still singular -- so
232 : * the solve either refuses their rhs or drifts, with the cause two steps
233 : * away. Alongside a prescribed x or y face the z-faces are simply inert,
234 : * which is what poisson_walls_uniform() produces on a 2D grid and is not
235 : * worth refusing. */
236 72 : if (solver->nz <= 1
237 66 : && (p->walls.front != POISSON_WALL_ZERO_GRADIENT
238 66 : || p->walls.back != POISSON_WALL_ZERO_GRADIENT)
239 26 : && poisson_walls_are_singular(&p->walls, solver->nz)) {
240 2 : cfd_set_error(CFD_ERROR_INVALID,
241 : "params.walls.front and .back describe z-faces, which a 2D grid "
242 : "(nz == 1) does not have, so nothing here pins the pressure level; "
243 : "prescribe an x or y face instead");
244 2 : return CFD_ERROR_INVALID;
245 : }
246 : }
247 :
248 : /* ---- the Helmholtz shift ------------------------------------------ */
249 1831 : if (!isfinite(p->helmholtz_shift) || p->helmholtz_shift < 0.0) {
250 4 : cfd_set_error(CFD_ERROR_INVALID,
251 : "params.helmholtz_shift must be finite and >= 0 (a negative shift is the "
252 : "indefinite Helmholtz operator, which is not SPD)");
253 4 : return CFD_ERROR_INVALID;
254 : }
255 1827 : if (p->helmholtz_shift != 0.0
256 66 : && !poisson_solver_shift_supported(solver->method, solver->backend, p)) {
257 13 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
258 : "params.helmholtz_shift is implemented by the scalar and OpenMP CG solvers "
259 : "only, and not with the multigrid preconditioner");
260 13 : return CFD_ERROR_UNSUPPORTED;
261 : }
262 :
263 : /* ---- a caller's apply_bc ------------------------------------------ */
264 : /* solver->internal_apply_bc is a solver's own walls and is not a caller's
265 : * business; this is only about a function the caller installed. */
266 1814 : if (solver->apply_bc) {
267 143 : if (solver->backend == POISSON_BACKEND_GPU) {
268 0 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
269 : "the GPU solvers apply their walls on the device and never call apply_bc");
270 0 : return CFD_ERROR_UNSUPPORTED;
271 : }
272 143 : if (solver->method == POISSON_METHOD_MULTIGRID) {
273 1 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
274 : "multigrid applies its walls inside the cycle and never calls apply_bc; "
275 : "use params.multigrid.bc to choose between zero-gradient and fixed walls");
276 1 : return CFD_ERROR_UNSUPPORTED;
277 : }
278 : }
279 :
280 : /* ---- parameter groups the method does not read --------------------- */
281 : {
282 1813 : unsigned owned = method_owned_groups(solver);
283 1813 : cfd_status_t status;
284 :
285 1813 : status = reject_unowned_group(solver, owned, PARAM_GROUP_SOR,
286 : sor_untouched(&p->sor), "sor");
287 1813 : if (status != CFD_SUCCESS) {
288 : return status;
289 : }
290 3622 : status = reject_unowned_group(solver, owned, PARAM_GROUP_KRYLOV,
291 : krylov_untouched(&p->krylov), "krylov");
292 1811 : if (status != CFD_SUCCESS) {
293 : return status;
294 : }
295 1810 : status = reject_unowned_group(solver, owned, PARAM_GROUP_MULTIGRID,
296 : multigrid_untouched(&p->multigrid), "multigrid");
297 1810 : if (status != CFD_SUCCESS) {
298 : return status;
299 : }
300 : }
301 :
302 : /* ---- exceptions within a group the method does own ----------------- */
303 :
304 : /* Range before capability. Every check below tests for a specific value --
305 : * "is it MULTIGRID", "is it not NONE" -- so an out-of-range preconditioner
306 : * satisfies all of them on CG and GMRES, and their backends then find it is
307 : * not POISSON_PRECOND_JACOBI either and run unpreconditioned. The caller
308 : * asked for something and got the default, which is the failure this
309 : * function exists to make impossible. */
310 1809 : if (p->krylov.preconditioner != POISSON_PRECOND_NONE
311 : && p->krylov.preconditioner != POISSON_PRECOND_JACOBI
312 : && p->krylov.preconditioner != POISSON_PRECOND_MULTIGRID) {
313 2 : cfd_set_error(CFD_ERROR_INVALID,
314 : "params.krylov.preconditioner must be POISSON_PRECOND_NONE, "
315 : "POISSON_PRECOND_JACOBI or POISSON_PRECOND_MULTIGRID");
316 2 : return CFD_ERROR_INVALID;
317 : }
318 :
319 : /* BiCGSTAB reads params.krylov, but has no preconditioner implementation on
320 : * any backend. It ignored one silently for as long as it has existed. */
321 1807 : if (solver->method == POISSON_METHOD_BICGSTAB
322 35 : && p->krylov.preconditioner != POISSON_PRECOND_NONE) {
323 3 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
324 : "BiCGSTAB has no preconditioner implementation on any backend");
325 3 : return CFD_ERROR_UNSUPPORTED;
326 : }
327 :
328 : /* No GPU solver implements a preconditioner -- there is no M^-1 apply
329 : * anywhere under linear/gpu/ -- so one configured here would be accepted and
330 : * then never used, which is what BiCGSTAB did on every backend for years. */
331 1804 : if (p->krylov.preconditioner != POISSON_PRECOND_NONE
332 76 : && solver->backend == POISSON_BACKEND_GPU) {
333 0 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
334 : "the GPU Krylov solvers have no preconditioner implementation; "
335 : "run preconditioned on a CPU backend, or unpreconditioned here");
336 0 : return CFD_ERROR_UNSUPPORTED;
337 : }
338 :
339 : /* The multigrid preconditioner is implemented by the scalar and OpenMP CG
340 : * solvers only. Every other Krylov backend would quietly run unpreconditioned. */
341 1804 : if (p->krylov.preconditioner == POISSON_PRECOND_MULTIGRID
342 51 : && !(solver->method == POISSON_METHOD_CG
343 48 : && (solver->backend == POISSON_BACKEND_SCALAR
344 48 : || solver->backend == POISSON_BACKEND_OMP))) {
345 3 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
346 : "POISSON_PRECOND_MULTIGRID is implemented by the scalar and OpenMP CG solvers only");
347 3 : return CFD_ERROR_UNSUPPORTED;
348 : }
349 :
350 : /* restart is the GMRES(m) basis size and means nothing to the others. */
351 1801 : if (p->krylov.restart != 0 && solver->method != POISSON_METHOD_GMRES) {
352 2 : cfd_set_error(CFD_ERROR_INVALID,
353 : "params.krylov.restart is the GMRES restart length; no other method reads it");
354 2 : return CFD_ERROR_INVALID;
355 : }
356 :
357 : /* Deliberately not rejected here: POISSON_METHOD_GAUSS_SEIDEL overriding
358 : * params.sor.omega to 1. That is what the method IS, it is documented on the
359 : * enum member, and test_gauss_seidel_is_sor_at_omega_one pins it. Refusing an
360 : * omega there would turn documented behaviour into an error, which is not the
361 : * class of problem this validator exists for.
362 : */
363 :
364 : /* The inner V-cycle of a preconditioner must stay symmetric and nonsingular,
365 : * or CG is no longer minimising over a Krylov space. These four are what
366 : * guarantee it, so they are not the caller's to change here; the rest of the
367 : * group is forwarded to the hierarchy by poisson_solver_create_mg_precond. */
368 1799 : if (p->krylov.preconditioner == POISSON_PRECOND_MULTIGRID
369 48 : && (p->multigrid.cycle != 0 || p->multigrid.smoother != 0 || p->multigrid.bc != 0
370 45 : || mg_smooth_effective(p->multigrid.pre_smooth, MG_DEFAULT_PRE_SMOOTH)
371 83 : != mg_smooth_effective(p->multigrid.post_smooth, MG_DEFAULT_POST_SMOOTH))) {
372 5 : cfd_set_error(CFD_ERROR_INVALID,
373 : "a multigrid preconditioner fixes its cycle, smoother, boundary mode and an "
374 : "equal pre/post sweep count, which is what keeps it symmetric for CG; "
375 : "params.multigrid.{pre_smooth, post_smooth, coarse_max_iter, max_levels} "
376 : "are yours to set");
377 5 : return CFD_ERROR_INVALID;
378 : }
379 :
380 : /* iter % check_interval, with check_interval == 0, is undefined behaviour in
381 : * the common loop and in the CG and BiCGSTAB loops. */
382 1794 : if (p->check_interval <= 0) {
383 2 : cfd_set_error(CFD_ERROR_INVALID, "params.check_interval must be at least 1");
384 2 : return CFD_ERROR_INVALID;
385 : }
386 :
387 : return CFD_SUCCESS;
388 : }
389 :
390 27642 : bool poisson_walls_are_singular(const poisson_walls_t* walls, size_t nz) {
391 27642 : if (!walls) {
392 : return true; /* no walls given means the defaults, which are singular */
393 : }
394 : /* The z-faces do not exist on a 2D grid, so whatever they say is irrelevant. */
395 27641 : if (nz > 1 && (walls->front != POISSON_WALL_ZERO_GRADIENT
396 186 : || walls->back != POISSON_WALL_ZERO_GRADIENT)) {
397 : return false;
398 : }
399 27640 : return walls->left == POISSON_WALL_ZERO_GRADIENT
400 27640 : && walls->right == POISSON_WALL_ZERO_GRADIENT
401 : && walls->bottom == POISSON_WALL_ZERO_GRADIENT
402 27680 : && walls->top == POISSON_WALL_ZERO_GRADIENT;
403 : }
404 :
405 17761 : poisson_solver_params_t poisson_solver_params_default(void) {
406 : /* Zeroed first: the assignments below cover every named field, but a
407 : * projection solver compares whole params structs with memcmp to decide
408 : * whether its pressure solver needs rebuilding (ns_pressure_ensure), so
409 : * padding has to be deterministic too. Without this, that comparison could
410 : * differ on padding alone and rebuild the solver on every step. */
411 17761 : poisson_solver_params_t params;
412 17761 : memset(¶ms, 0, sizeof params);
413 17761 : params.tolerance = 1e-6;
414 17761 : params.absolute_tolerance = 1e-10;
415 17761 : params.max_iterations = 5000; /* Increased from 1000 for CG on fine grids */
416 17761 : params.sor.omega = 0.0; /* Auto-compute optimal omega for grid dimensions */
417 17761 : params.check_interval = 1;
418 17761 : params.verbose = false;
419 17761 : params.krylov.preconditioner = POISSON_PRECOND_NONE;
420 17761 : params.krylov.restart = 0; /* 0 = auto (GMRES_DEFAULT_RESTART); ignored by non-GMRES methods */
421 17761 : params.multigrid.cycle = MG_CYCLE_V;
422 17761 : params.multigrid.smoother = MG_SMOOTHER_REDBLACK_GS;
423 17761 : params.multigrid.bc = MG_BC_NEUMANN;
424 17761 : params.multigrid.pre_smooth = 0; /* 0 = default (2) */
425 17761 : params.multigrid.post_smooth = 0; /* 0 = default (2) */
426 17761 : params.multigrid.coarse_max_iter = 0; /* 0 = default (50) */
427 17761 : params.multigrid.max_levels = 0; /* 0 = auto */
428 17761 : params.walls = poisson_walls_default();
429 17761 : params.helmholtz_shift = 0.0; /* 0 = pure Poisson */
430 17761 : return params;
431 : }
432 :
433 14485 : poisson_solver_config_t poisson_solver_config_preset(poisson_preset_t preset) {
434 14485 : poisson_solver_config_t cfg;
435 14485 : cfg.method = POISSON_METHOD_CG;
436 : /* AUTO unless the preset needs a backend that AUTO would not choose; see
437 : * the two multigrid cases below. */
438 14485 : cfg.backend = POISSON_BACKEND_AUTO;
439 14485 : cfg.params = poisson_solver_params_default();
440 :
441 14485 : switch (preset) {
442 810 : case POISSON_PRESET_ACCURATE:
443 810 : cfg.params.tolerance = 1e-10;
444 810 : cfg.params.absolute_tolerance = 1e-14;
445 810 : cfg.params.max_iterations = 20000;
446 810 : break;
447 :
448 1 : case POISSON_PRESET_NONSYMMETRIC:
449 1 : cfg.method = POISSON_METHOD_BICGSTAB;
450 1 : break;
451 :
452 2 : case POISSON_PRESET_SMOOTHER:
453 : /* The stationary option, and the one operator that tolerates an
454 : * incompatible rhs. It keeps the ordinary tolerances so an ordinary
455 : * solve reports success; a caller who wants a fixed sweep count
456 : * instead sets tolerance = 0 and max_iterations themselves, and then
457 : * CFD_ERROR_MAX_ITER is their own explicit choice rather than
458 : * something this preset returns on every call. */
459 2 : cfg.method = POISSON_METHOD_REDBLACK_SOR;
460 2 : break;
461 :
462 68 : case POISSON_PRESET_MULTIGRID:
463 68 : cfg.method = POISSON_METHOD_MULTIGRID;
464 68 : cfg.backend = POISSON_BACKEND_SCALAR;
465 68 : break;
466 :
467 56 : case POISSON_PRESET_MULTIGRID_PCG:
468 : /* The preconditioner is an ordinary parameter, so nothing downstream
469 : * needs a special case for it -- but the backend does need naming.
470 : * AUTO resolves to SIMD wherever AVX2 or NEON is present, and neither
471 : * multigrid nor the multigrid preconditioner has a SIMD backend, so
472 : * this preset would otherwise be refused at init on the very machines
473 : * it is most likely to run on. Scalar is the one backend AUTO could
474 : * have reached that implements it; POISSON_BACKEND_OMP is a single
475 : * assignment away for a caller who wants the threaded hierarchy. */
476 56 : cfg.params.krylov.preconditioner = POISSON_PRECOND_MULTIGRID;
477 56 : cfg.backend = POISSON_BACKEND_SCALAR;
478 56 : break;
479 :
480 : case POISSON_PRESET_DEFAULT:
481 : default:
482 : break;
483 : }
484 :
485 14485 : return cfg;
486 : }
487 :
488 17544 : poisson_solver_stats_t poisson_solver_stats_default(void) {
489 17544 : poisson_solver_stats_t stats;
490 17544 : stats.status = POISSON_ERROR;
491 17544 : stats.iterations = 0;
492 17544 : stats.initial_residual = 0.0;
493 17544 : stats.final_residual = 0.0;
494 17544 : stats.elapsed_time_ms = 0.0;
495 17544 : return stats;
496 : }
497 :
498 23 : const char* poisson_solver_status_string(poisson_solver_status_t status) {
499 23 : switch (status) {
500 : case POISSON_CONVERGED: return "converged";
501 2 : case POISSON_MAX_ITER: return "max iterations";
502 3 : case POISSON_DIVERGED: return "diverged";
503 4 : case POISSON_STAGNATED: return "stagnated";
504 6 : case POISSON_INCOMPATIBLE_RHS: return "incompatible rhs";
505 6 : case POISSON_ERROR: return "error";
506 : }
507 1 : return "unknown";
508 : }
509 :
510 : /* ============================================================================
511 : * TIMING
512 : * ============================================================================ */
513 :
514 68154 : double poisson_solver_get_time_ms(void) {
515 : #ifdef _WIN32
516 : LARGE_INTEGER freq, counter;
517 : QueryPerformanceFrequency(&freq);
518 : QueryPerformanceCounter(&counter);
519 : return (double)counter.QuadPart * 1000.0 / (double)freq.QuadPart;
520 : #else
521 68154 : struct timeval tv;
522 68154 : gettimeofday(&tv, NULL);
523 68154 : return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
524 : #endif
525 : }
526 :
527 : /* ============================================================================
528 : * BACKEND SELECTION
529 : * ============================================================================ */
530 :
531 : static poisson_solver_backend_t g_default_backend = POISSON_BACKEND_AUTO;
532 :
533 2 : poisson_solver_backend_t poisson_solver_get_backend(void) {
534 2 : return g_default_backend;
535 : }
536 :
537 2 : const char* poisson_solver_get_backend_name(void) {
538 2 : switch (g_default_backend) {
539 : case POISSON_BACKEND_SCALAR: return "scalar";
540 0 : case POISSON_BACKEND_OMP: return "omp";
541 0 : case POISSON_BACKEND_SIMD: return "simd";
542 0 : case POISSON_BACKEND_GPU: return "gpu";
543 1 : case POISSON_BACKEND_AUTO:
544 1 : default: return "auto";
545 : }
546 : }
547 :
548 2 : bool poisson_solver_set_backend(poisson_solver_backend_t backend) {
549 2 : if (!poisson_solver_backend_available(backend)) {
550 : return false;
551 : }
552 2 : g_default_backend = backend;
553 2 : return true;
554 : }
555 :
556 99 : bool poisson_solver_backend_available(poisson_solver_backend_t backend) {
557 99 : switch (backend) {
558 : case POISSON_BACKEND_AUTO:
559 : case POISSON_BACKEND_SCALAR:
560 : return true;
561 :
562 : case POISSON_BACKEND_OMP:
563 : #ifdef CFD_ENABLE_OPENMP
564 : return true;
565 : #else
566 : return false;
567 : #endif
568 :
569 23 : case POISSON_BACKEND_SIMD:
570 23 : return poisson_solver_simd_backend_available();
571 :
572 : case POISSON_BACKEND_GPU:
573 : #ifdef CFD_HAS_CUDA
574 : return true;
575 : #else
576 : return false;
577 : #endif
578 :
579 : default:
580 : return false;
581 : }
582 : }
583 :
584 : /* ============================================================================
585 : * SOLVER LIFECYCLE
586 : * ============================================================================ */
587 :
588 : /**
589 : * Auto-select best available backend
590 : *
591 : * Priority: SIMD (runtime detection) > Scalar
592 : */
593 8 : static poisson_solver_backend_t select_best_backend(void) {
594 : /* Prefer SIMD with runtime detection */
595 8 : if (poisson_solver_simd_backend_available()) {
596 0 : return POISSON_BACKEND_SIMD;
597 : }
598 : return POISSON_BACKEND_SCALAR;
599 : }
600 :
601 : /* Factories must set a specific last-status on NULL (see cfd_solver_create),
602 : * so callers can distinguish an unavailable backend from allocation failure. */
603 58 : static poisson_solver_t* backend_unavailable(const char* method_name) {
604 58 : char msg[128];
605 58 : snprintf(msg, sizeof(msg), "Requested backend not available for %s", method_name);
606 58 : cfd_set_error(CFD_ERROR_UNSUPPORTED, msg);
607 58 : return NULL;
608 : }
609 :
610 1923 : poisson_solver_t* poisson_solver_create(
611 : poisson_solver_method_t method,
612 : poisson_solver_backend_t backend)
613 : {
614 : /* Auto-select backend if requested (keep the original request: methods
615 : * without a SIMD backend resolve AUTO differently) */
616 1923 : poisson_solver_backend_t requested = backend;
617 1923 : if (backend == POISSON_BACKEND_AUTO) {
618 8 : backend = select_best_backend();
619 : }
620 :
621 : /* Create appropriate solver - no silent fallbacks */
622 1923 : switch (method) {
623 58 : case POISSON_METHOD_JACOBI:
624 58 : switch (backend) {
625 3 : case POISSON_BACKEND_SIMD:
626 3 : return create_jacobi_simd_solver();
627 : #ifdef CFD_ENABLE_OPENMP
628 9 : case POISSON_BACKEND_OMP:
629 9 : return create_jacobi_omp_solver();
630 : #endif
631 : #ifdef CFD_HAS_CUDA
632 : case POISSON_BACKEND_GPU:
633 : return create_jacobi_gpu_solver();
634 : #endif
635 39 : case POISSON_BACKEND_SCALAR:
636 39 : return create_jacobi_scalar_solver();
637 7 : default:
638 7 : return backend_unavailable("Jacobi");
639 : }
640 :
641 65 : case POISSON_METHOD_SOR:
642 : case POISSON_METHOD_GAUSS_SEIDEL: {
643 : /* Gauss-Seidel is SOR at omega = 1: the same solvers, marked with the
644 : * requested method so that init resolves omega to 1 */
645 65 : poisson_solver_t* sor;
646 65 : switch (backend) {
647 7 : case POISSON_BACKEND_SIMD:
648 7 : sor = create_sor_simd_solver();
649 7 : break;
650 : #ifdef CFD_HAS_CUDA
651 : case POISSON_BACKEND_GPU:
652 : sor = create_sor_gpu_solver();
653 : break;
654 : #endif
655 39 : case POISSON_BACKEND_SCALAR:
656 39 : sor = create_sor_scalar_solver();
657 39 : break;
658 19 : default:
659 32 : return backend_unavailable(method == POISSON_METHOD_GAUSS_SEIDEL ? "Gauss-Seidel" : "SOR");
660 : }
661 46 : if (sor) {
662 39 : sor->method = method;
663 : }
664 : return sor;
665 : }
666 :
667 713 : case POISSON_METHOD_REDBLACK_SOR:
668 713 : switch (backend) {
669 3 : case POISSON_BACKEND_SIMD:
670 3 : return create_redblack_simd_solver();
671 : #ifdef CFD_ENABLE_OPENMP
672 10 : case POISSON_BACKEND_OMP:
673 10 : return create_redblack_omp_solver();
674 : #endif
675 : #ifdef CFD_HAS_CUDA
676 : case POISSON_BACKEND_GPU:
677 : return create_redblack_gpu_solver();
678 : #endif
679 695 : case POISSON_BACKEND_SCALAR:
680 695 : return create_redblack_scalar_solver();
681 5 : default:
682 5 : return backend_unavailable("Red-Black SOR");
683 : }
684 :
685 582 : case POISSON_METHOD_CG:
686 582 : switch (backend) {
687 10 : case POISSON_BACKEND_SIMD:
688 10 : return create_cg_simd_solver();
689 : #ifdef CFD_ENABLE_OPENMP
690 77 : case POISSON_BACKEND_OMP:
691 77 : return create_cg_omp_solver();
692 : #endif
693 : #ifdef CFD_HAS_CUDA
694 : case POISSON_BACKEND_GPU:
695 : return create_cg_gpu_solver();
696 : #endif
697 488 : case POISSON_BACKEND_SCALAR:
698 488 : return create_cg_scalar_solver();
699 7 : default:
700 7 : return backend_unavailable("CG");
701 : }
702 :
703 53 : case POISSON_METHOD_BICGSTAB:
704 53 : switch (backend) {
705 6 : case POISSON_BACKEND_SIMD:
706 6 : return create_bicgstab_simd_solver();
707 : #ifdef CFD_ENABLE_OPENMP
708 13 : case POISSON_BACKEND_OMP:
709 13 : return create_bicgstab_omp_solver();
710 : #endif
711 : #ifdef CFD_HAS_CUDA
712 : case POISSON_BACKEND_GPU:
713 : return create_bicgstab_gpu_solver();
714 : #endif
715 26 : case POISSON_BACKEND_SCALAR:
716 26 : return create_bicgstab_scalar_solver();
717 8 : default:
718 8 : return backend_unavailable("BiCGSTAB");
719 : }
720 :
721 103 : case POISSON_METHOD_GMRES:
722 103 : switch (backend) {
723 6 : case POISSON_BACKEND_SIMD:
724 6 : return create_gmres_simd_solver();
725 : #ifdef CFD_ENABLE_OPENMP
726 37 : case POISSON_BACKEND_OMP:
727 37 : return create_gmres_omp_solver();
728 : #endif
729 56 : case POISSON_BACKEND_SCALAR:
730 56 : return create_gmres_scalar_solver();
731 4 : default:
732 4 : return backend_unavailable("GMRES");
733 : }
734 :
735 348 : case POISSON_METHOD_MULTIGRID:
736 : /* No SIMD/GPU multigrid. AUTO means "best available", which
737 : * select_best_backend() resolves to SIMD — a backend multigrid
738 : * lacks — so AUTO keeps resolving to the scalar reference. OMP is
739 : * opt-in, as for every other method; explicit SIMD/GPU requests
740 : * return NULL (no silent fallbacks). */
741 348 : if (requested == POISSON_BACKEND_AUTO) {
742 1 : return create_multigrid_scalar_solver();
743 : }
744 347 : switch (backend) {
745 : #ifdef CFD_ENABLE_OPENMP
746 153 : case POISSON_BACKEND_OMP:
747 153 : return create_multigrid_omp_solver();
748 : #endif
749 186 : case POISSON_BACKEND_SCALAR:
750 186 : return create_multigrid_scalar_solver();
751 8 : default:
752 8 : return backend_unavailable("multigrid");
753 : }
754 :
755 1 : default:
756 1 : cfd_set_error(CFD_ERROR_INVALID, "Unknown Poisson solver method");
757 1 : return NULL;
758 : }
759 : }
760 :
761 1850 : cfd_status_t poisson_solver_init(
762 : poisson_solver_t* solver,
763 : size_t nx, size_t ny, size_t nz,
764 : double dx, double dy, double dz,
765 : const poisson_solver_params_t* params)
766 : {
767 1850 : if (!solver) {
768 : return CFD_ERROR_INVALID;
769 : }
770 :
771 : /* Require at least one interior cell in each active dimension.
772 : * For 2D (nz <= 1): nx, ny >= 3. For 3D (nz > 1): nx, ny, nz >= 3.
773 : * Rejects degenerate grids (e.g. nz==2) where k_start==k_end. */
774 1849 : if (nx < 3 || ny < 3 || (nz > 1 && nz < 3)) {
775 : return CFD_ERROR_INVALID;
776 : }
777 :
778 1846 : solver->nx = nx;
779 1846 : solver->ny = ny;
780 1846 : solver->nz = nz;
781 1846 : solver->dx = dx;
782 1846 : solver->dy = dy;
783 1846 : solver->dz = dz;
784 :
785 1846 : if (params) {
786 1791 : solver->params = *params;
787 : } else {
788 55 : solver->params = poisson_solver_params_default();
789 : }
790 :
791 : {
792 1846 : cfd_status_t wall_status = poisson_solver_check_config(solver);
793 1846 : if (wall_status != CFD_SUCCESS) {
794 : return wall_status;
795 : }
796 : }
797 :
798 : /* Adjust max_iterations for Jacobi (needs more iterations) */
799 1792 : if (solver->method == POISSON_METHOD_JACOBI && params == NULL) {
800 9 : solver->params.max_iterations = 2000;
801 : }
802 :
803 : /* Call solver-specific init if provided */
804 1792 : if (solver->init) {
805 1792 : return solver->init(solver, nx, ny, nz, dx, dy, dz, &solver->params);
806 : }
807 :
808 : return CFD_SUCCESS;
809 : }
810 :
811 2365 : void poisson_solver_destroy(poisson_solver_t* solver) {
812 2365 : if (!solver) {
813 : return;
814 : }
815 :
816 1872 : if (solver->destroy) {
817 1872 : solver->destroy(solver);
818 : }
819 :
820 1872 : cfd_free(solver);
821 : }
822 :
823 : /* ============================================================================
824 : * SOLVER OPERATIONS
825 : * ============================================================================ */
826 :
827 188003 : double poisson_solver_compute_residual(
828 : poisson_solver_t* solver,
829 : const double* x,
830 : const double* rhs)
831 : {
832 188003 : if (!solver || !x || !rhs) {
833 : return -1.0;
834 : }
835 :
836 188000 : size_t nx = solver->nx;
837 188000 : size_t ny = solver->ny;
838 188000 : double dx2 = solver->dx * solver->dx;
839 188000 : double dy2 = solver->dy * solver->dy;
840 188000 : double inv_dz2 = poisson_solver_compute_inv_dz2(solver->dz);
841 188000 : double sigma = solver->params.helmholtz_shift;
842 :
843 188000 : size_t stride_z, k_start, k_end;
844 188000 : poisson_solver_compute_3d_bounds(solver->nz, nx, ny,
845 : &stride_z, &k_start, &k_end);
846 :
847 188000 : double max_residual = 0.0;
848 :
849 646969 : for (size_t k = k_start; k < k_end; k++) {
850 13332251 : for (size_t j = 1; j < ny - 1; j++) {
851 471459369 : for (size_t i = 1; i < nx - 1; i++) {
852 458586087 : size_t idx = k * stride_z + IDX_2D(i, j, nx);
853 :
854 : /* Compute Laplacian: d^2x/dx^2 + d^2x/dy^2 + d^2x/dz^2 */
855 458586087 : double laplacian =
856 458586087 : (x[idx + 1] - 2.0 * x[idx] + x[idx - 1]) / dx2
857 458586087 : + (x[idx + nx] - 2.0 * x[idx] + x[idx - nx]) / dy2
858 458586087 : + (x[idx + stride_z] + x[idx - stride_z]
859 458586087 : - 2.0 * x[idx]) * inv_dz2;
860 :
861 : /* Operator is nabla^2 - sigma*I; sigma = 0 leaves this exact. */
862 458586087 : double diff = laplacian - rhs[idx];
863 458586087 : if (sigma != 0.0) {
864 0 : diff -= sigma * x[idx];
865 : }
866 458586087 : double residual = fabs(diff);
867 : /* A NaN compares false against everything, so without the
868 : * isnan() a diverged field would read as a zero residual */
869 458586087 : if (residual > max_residual || isnan(residual)) {
870 8662173 : max_residual = residual;
871 : }
872 : }
873 : }
874 : }
875 :
876 : return max_residual;
877 : }
878 :
879 : /**
880 : * Write one face's halo: copy the adjacent interior out for a zero-gradient face,
881 : * or write a constant for a Dirichlet one.
882 : *
883 : * `stride` steps inward from the face, `n_outer`/`n_inner` walk the face itself.
884 : */
885 200310 : static void write_face(double* x, size_t base, ptrdiff_t stride,
886 : size_t n_outer, size_t outer_stride,
887 : size_t n_inner, size_t inner_stride,
888 : poisson_wall_t type, double value)
889 : {
890 613932 : for (size_t a = 0; a < n_outer; a++) {
891 9076716 : for (size_t b = 0; b < n_inner; b++) {
892 8663094 : size_t idx = base + (a * outer_stride) + (b * inner_stride);
893 8663094 : x[idx] = (type == POISSON_WALL_DIRICHLET)
894 : ? value
895 8663094 : : x[(size_t)((ptrdiff_t)idx + stride)];
896 : }
897 : }
898 : }
899 :
900 : /**
901 : * Apply per-face walls to x's halo.
902 : *
903 : * homogeneous != 0 writes 0 on every Dirichlet face: that is the homogeneous part
904 : * of the condition, which is what a Krylov search direction must carry. The
905 : * zero-gradient extension is already its own homogeneous form, so it is identical
906 : * either way.
907 : *
908 : * Faces run left, right, bottom, top, back, front so the order is deterministic;
909 : * a 5/7-point stencil never reads a corner halo cell, so which face owns a shared
910 : * corner does not change any answer.
911 : */
912 46746 : static void poisson_apply_walls(const poisson_solver_t* s, double* x, int homogeneous)
913 : {
914 46746 : const poisson_walls_t* w = &s->params.walls;
915 46746 : size_t nx = s->nx;
916 46746 : size_t ny = s->ny;
917 46746 : size_t nz = s->nz;
918 46746 : size_t plane = nx * ny;
919 :
920 : #define WALL_VALUE(face) (homogeneous ? 0.0 : w->values.face)
921 :
922 : /* x-faces: whole column of each plane */
923 46746 : write_face(x, 0, +1, nz, plane, ny, nx, w->left, WALL_VALUE(left));
924 46746 : write_face(x, nx - 1, -1, nz, plane, ny, nx, w->right, WALL_VALUE(right));
925 :
926 : /* y-faces: whole row of each plane */
927 46746 : write_face(x, 0, (ptrdiff_t)nx, nz, plane, nx, 1, w->bottom, WALL_VALUE(bottom));
928 46746 : write_face(x, (ny - 1) * nx, -(ptrdiff_t)nx, nz, plane, nx, 1, w->top, WALL_VALUE(top));
929 :
930 : /* z-faces: whole plane. Only on a 3D grid. */
931 46746 : if (nz > 1) {
932 6663 : write_face(x, 0, (ptrdiff_t)plane, 1, 0, plane, 1, w->back, WALL_VALUE(back));
933 6663 : write_face(x, (nz - 1) * plane, -(ptrdiff_t)plane, 1, 0, plane, 1,
934 6663 : w->front, WALL_VALUE(front));
935 : }
936 :
937 : #undef WALL_VALUE
938 46746 : }
939 :
940 1300892 : void poisson_solver_apply_bc(
941 : poisson_solver_t* solver,
942 : double* x)
943 : {
944 1300892 : if (!solver || !x) {
945 : return;
946 : }
947 :
948 : /* The caller's function wins; a solver's own is the fallback. Nothing sets
949 : * both -- a solver that installs internal_apply_bc rejects a caller function
950 : * at init -- so the precedence never actually has to arbitrate. */
951 1300891 : if (solver->apply_bc) {
952 65005 : solver->apply_bc(solver, x);
953 65005 : return;
954 : }
955 1235886 : if (solver->internal_apply_bc) {
956 36 : solver->internal_apply_bc(solver, x);
957 36 : return;
958 : }
959 :
960 : /* Per-face walls, when the caller configured any. The all-zero-gradient case
961 : * falls through to the loop below, which computes the same extension by the
962 : * shorter route: a whole-plane memcpy for the z faces rather than
963 : * write_face's per-cell indexed copy. (It once fell through to reach the
964 : * parallel OMP/SIMD BC primitives; that dispatch is gone and the loop below
965 : * is now serial on every backend, for the reason stated there.) */
966 1235850 : if (!poisson_walls_are_default(&solver->params.walls)) {
967 3921 : poisson_apply_walls(solver, x, 0);
968 3921 : return;
969 : }
970 :
971 : /* Default: Neumann BCs (zero gradient) on all faces */
972 1231929 : size_t nx = solver->nx;
973 1231929 : size_t ny = solver->ny;
974 1231929 : size_t nz = solver->nz;
975 1231929 : size_t plane_size = nx * ny;
976 :
977 : /* Z-face Neumann: copy adjacent interior plane to boundary planes */
978 1231929 : if (nz > 1) {
979 32411 : memcpy(x, x + plane_size, plane_size * sizeof(double));
980 32411 : memcpy(x + (nz - 1) * plane_size,
981 32411 : x + (nz - 2) * plane_size,
982 : plane_size * sizeof(double));
983 : }
984 :
985 : /* Serial on every backend, deliberately.
986 : *
987 : * A Neumann face is a copy from the adjacent interior line, so all three
988 : * backends produce identical values -- but this runs once per Krylov
989 : * iteration, and an OpenMP region costs ~12 us to enter at 4 threads on
990 : * MSVC while the work here is O(nx + ny): about 130 writes on a 33x33
991 : * plane, less than the 31x31 Laplacian sweep it accompanies. Threading it
992 : * cost more than the solve. BiCGSTAB pays this twice per iteration. */
993 2771272 : for (size_t k = 0; k < nz; k++) {
994 1539343 : bc_apply_scalar_cpu(x + k * plane_size, nx, ny, BC_TYPE_NEUMANN);
995 : }
996 : }
997 :
998 : /** Write 0 into every boundary cell, leaving the interior alone. */
999 359 : static void krylov_zero_halo(const poisson_solver_t* solver, double* x)
1000 : {
1001 359 : size_t nx = solver->nx;
1002 359 : size_t ny = solver->ny;
1003 359 : size_t nz = solver->nz;
1004 359 : size_t plane_size = nx * ny;
1005 :
1006 359 : size_t k_first = 0;
1007 359 : size_t k_last = 1;
1008 359 : if (nz > 1) {
1009 37 : memset(x, 0, plane_size * sizeof(double));
1010 37 : memset(x + (nz - 1) * plane_size, 0, plane_size * sizeof(double));
1011 37 : k_first = 1;
1012 37 : k_last = nz - 1;
1013 : }
1014 :
1015 1252 : for (size_t k = k_first; k < k_last; k++) {
1016 893 : double* plane = x + k * plane_size;
1017 893 : memset(plane, 0, nx * sizeof(double)); /* j = 0 */
1018 893 : memset(plane + (ny - 1) * nx, 0, nx * sizeof(double)); /* j = ny-1 */
1019 23177 : for (size_t j = 1; j < ny - 1; j++) {
1020 22284 : plane[j * nx] = 0.0; /* i = 0 */
1021 22284 : plane[(j * nx) + (nx - 1)] = 0.0; /* i = nx-1 */
1022 : }
1023 : }
1024 359 : }
1025 :
1026 16374 : int poisson_solver_krylov_is_singular(const poisson_solver_t* solver)
1027 : {
1028 16374 : if (!solver) {
1029 : return 0;
1030 : }
1031 : /* A Helmholtz shift adds sigma*I, so the constants stop being a nullspace and
1032 : * the operator is nonsingular whatever the walls say. Checked here rather
1033 : * than at the one call site, so that every future caller of a predicate
1034 : * named "is singular" gets a true answer. */
1035 16374 : if (solver->params.helmholtz_shift > 0.0) {
1036 : return 0;
1037 : }
1038 : /* A hook prescribes wall values, which pins the level and makes the operator
1039 : * nonsingular, exactly as a Dirichlet face does. */
1040 14005 : return solver->apply_bc == NULL
1041 14005 : && poisson_walls_are_singular(&solver->params.walls, solver->nz);
1042 : }
1043 :
1044 1123053 : void poisson_solver_krylov_apply_bc_homogeneous(
1045 : poisson_solver_t* solver,
1046 : double* v)
1047 : {
1048 1123053 : if (!solver || !v) {
1049 : return;
1050 : }
1051 :
1052 : /* The zero-gradient extension is already homogeneous -- it copies the interior
1053 : * outwards and adds nothing -- so all-zero-gradient walls need it verbatim.
1054 : * With a Dirichlet face configured, the prescribed value belongs to the iterate
1055 : * and the direction gets 0 there instead. */
1056 1123053 : if (!solver->apply_bc) {
1057 1122726 : if (poisson_walls_are_default(&solver->params.walls)) {
1058 1079901 : poisson_solver_apply_bc(solver, v);
1059 : } else {
1060 42825 : poisson_apply_walls(solver, v, 1);
1061 : }
1062 1122726 : return;
1063 : }
1064 :
1065 : /* A custom hook prescribes wall values that do not depend on the interior, so
1066 : * its homogeneous part is a zero halo. Running the hook on a search direction
1067 : * would add its lift to every one of them, which is not a linear operator and
1068 : * would leave the Krylov recurrences describing something other than A. */
1069 327 : krylov_zero_halo(solver, v);
1070 : }
1071 :
1072 16690 : void poisson_solver_krylov_apply_bc(
1073 : poisson_solver_t* solver,
1074 : double* x)
1075 : {
1076 16690 : if (!solver || !x) {
1077 : return;
1078 : }
1079 :
1080 : /* The iterate, unlike a search direction, carries the lift as well, so the
1081 : * residual formed from it is the residual of the real system. For the default
1082 : * walls that is just the extension. */
1083 16690 : if (!solver->apply_bc) {
1084 16658 : poisson_solver_apply_bc(solver, x);
1085 16658 : return;
1086 : }
1087 :
1088 : /* Zero first, then let the hook write over it: a hook that writes only some
1089 : * walls, or that holds them at zero by writing nothing, then still leaves no
1090 : * stale value behind for a warm start to trip on. */
1091 32 : krylov_zero_halo(solver, x);
1092 32 : solver->apply_bc(solver, x);
1093 : }
1094 :
1095 : /**
1096 : * Common solve loop used by all solvers
1097 : */
1098 1032 : cfd_status_t poisson_solver_solve_common(
1099 : poisson_solver_t* solver,
1100 : double* x,
1101 : double* x_temp,
1102 : const double* rhs,
1103 : poisson_solver_stats_t* stats)
1104 : {
1105 1032 : if (!solver || !x || !rhs) {
1106 : return CFD_ERROR_INVALID;
1107 : }
1108 :
1109 1032 : if (!solver->iterate) {
1110 : return CFD_ERROR_UNSUPPORTED;
1111 : }
1112 :
1113 1032 : poisson_solver_params_t* params = &solver->params;
1114 1032 : double start_time = poisson_solver_get_time_ms();
1115 :
1116 : /* Compute initial residual. The target never sits below the round-off floor
1117 : * of the residual at x, which a warm start near steady state already reaches
1118 : * (poisson_solver_residual_floor). */
1119 1032 : double initial_res = poisson_solver_compute_residual(solver, x, rhs);
1120 1032 : double floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_MAX);
1121 1032 : double tolerance = poisson_solver_stop_target(params, initial_res, floor_res);
1122 :
1123 1032 : if (stats) {
1124 1032 : stats->initial_residual = initial_res;
1125 : }
1126 :
1127 : /* A start whose residual is not finite has diverged before the first sweep */
1128 1032 : if (!isfinite(initial_res)) {
1129 1 : if (stats) {
1130 1 : stats->status = POISSON_DIVERGED;
1131 1 : stats->iterations = 0;
1132 1 : stats->final_residual = initial_res;
1133 1 : stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
1134 : }
1135 1 : return CFD_ERROR_DIVERGED;
1136 : }
1137 :
1138 : /* Already converged? */
1139 2051 : if (initial_res < poisson_solver_done_target(params, floor_res)) {
1140 23 : if (stats) {
1141 23 : stats->status = POISSON_CONVERGED;
1142 23 : stats->iterations = 0;
1143 23 : stats->final_residual = initial_res;
1144 23 : stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
1145 : }
1146 23 : return CFD_SUCCESS;
1147 : }
1148 :
1149 152409 : int converged = 0;
1150 152409 : int diverged = 0;
1151 : int iter;
1152 : double res = initial_res;
1153 :
1154 152409 : for (iter = 0; iter < params->max_iterations; iter++) {
1155 152382 : double new_res = 0.0;
1156 :
1157 : /* Check at intervals and on the last iteration, so that a solve never ends
1158 : * on a residual older than the field it returns */
1159 152382 : int check = (iter % params->check_interval == 0) || (iter + 1 == params->max_iterations);
1160 :
1161 : /* Perform one iteration */
1162 152382 : cfd_status_t status = solver->iterate(solver, x, x_temp, rhs, check ? &new_res : NULL);
1163 :
1164 152382 : if (status != CFD_SUCCESS) {
1165 0 : if (stats) {
1166 0 : stats->status = POISSON_ERROR;
1167 0 : stats->iterations = iter + 1;
1168 0 : stats->final_residual = res;
1169 0 : stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
1170 : }
1171 0 : return status;
1172 : }
1173 :
1174 152382 : if (check) {
1175 147384 : res = new_res;
1176 :
1177 : /* A residual that is no longer finite will never meet the tolerance */
1178 147384 : if (!isfinite(res)) {
1179 : diverged = 1;
1180 981 : break;
1181 : }
1182 :
1183 147382 : if (params->verbose) {
1184 0 : CFD_LOG_DEBUG("poisson", "Iter %d: residual = %.6e", iter, res);
1185 : }
1186 :
1187 : /* x has moved since the start, and the floor with it */
1188 147382 : floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_MAX);
1189 293785 : if (res < tolerance || res < poisson_solver_done_target(params, floor_res)) {
1190 : converged = 1;
1191 : break;
1192 : }
1193 : }
1194 : }
1195 :
1196 1008 : double end_time = poisson_solver_get_time_ms();
1197 :
1198 1008 : if (stats) {
1199 : /* A break leaves iter at the index of the last iteration run;
1200 : * an exhausted loop leaves it at max_iterations, the count run */
1201 1008 : stats->iterations = (converged || diverged) ? iter + 1 : iter;
1202 1008 : stats->final_residual = res;
1203 1008 : stats->elapsed_time_ms = end_time - start_time;
1204 29 : stats->status = converged ? POISSON_CONVERGED
1205 1008 : : diverged ? POISSON_DIVERGED
1206 : : POISSON_MAX_ITER;
1207 : }
1208 :
1209 1008 : if (converged) {
1210 979 : return CFD_SUCCESS;
1211 : }
1212 29 : return diverged ? CFD_ERROR_DIVERGED : CFD_ERROR_MAX_ITER;
1213 : }
1214 :
1215 15 : void poisson_make_rhs_compatible(double* rhs, size_t nx, size_t ny, size_t nz) {
1216 : /* A grid with no interior has no interior mean to remove, and the loops
1217 : * below it count down from nx - 1 in size_t, so nx == 0 would wrap to
1218 : * SIZE_MAX and run off the buffer. Every other public entry point screens
1219 : * this; as an exported helper taking raw dimensions, so must this one. */
1220 15 : if (!rhs || nx < 3 || ny < 3 || (nz > 1 && nz < 3)) {
1221 : return;
1222 : }
1223 10 : mg_subtract_interior_mean(rhs, nx, ny, nz);
1224 : }
1225 :
1226 : /**
1227 : * Reject a right-hand side the singular operator has no solution for.
1228 : *
1229 : * With zero-gradient walls the constants are the operator's nullspace, so a rhs
1230 : * with a nonzero interior mean lies partly outside its range: there is no x that
1231 : * solves it, and the iteration either stalls on that component or, for Krylov
1232 : * methods, drives the rest of the field away chasing it.
1233 : *
1234 : * The mean is compared against the residual the caller asked for, scaled by the
1235 : * size of the rhs -- an incompatibility smaller than the target tolerance cannot
1236 : * affect the answer, and the projection solvers leave one of that size behind
1237 : * after mean-subtracting in floating point.
1238 : */
1239 17406 : static cfd_status_t check_rhs_compatible(const poisson_solver_t* solver, const double* rhs)
1240 : {
1241 : /* Krylov methods only. On a system with no solution they do not merely fail to
1242 : * converge: the minimisation chases the component in the nullspace and drives
1243 : * the rest of the field away with it, which is how test_laplacian_accuracy
1244 : * reached a residual of 1e21. The stationary and multigrid solvers relax
1245 : * towards a solution modulo a drifting constant instead, which callers use
1246 : * deliberately -- running a fixed sweep count on an arbitrary rhs is a
1247 : * reasonable thing to ask of them, so it is left alone. */
1248 17406 : if (!method_honours_walls(solver->method)) {
1249 : return CFD_SUCCESS;
1250 : }
1251 :
1252 16374 : if (!poisson_solver_krylov_is_singular(solver)) {
1253 : /* A prescribed face pins the level, and a Helmholtz shift removes the
1254 : * nullspace outright; either way any rhs is solvable. */
1255 : return CFD_SUCCESS;
1256 : }
1257 :
1258 : /* Solve before init leaves the dimensions at 0, where the interior loops below
1259 : * would underflow. Let the solver itself report that; there is no rhs to judge
1260 : * against a grid that does not exist yet. */
1261 13965 : if (solver->nx < 3 || solver->ny < 3) {
1262 : return CFD_SUCCESS;
1263 : }
1264 :
1265 13964 : size_t nx = solver->nx;
1266 13964 : size_t ny = solver->ny;
1267 13964 : size_t nz = solver->nz;
1268 13964 : size_t stride_z = (nz > 1) ? nx * ny : 0;
1269 13964 : size_t k_start = (nz > 1) ? 1 : 0;
1270 13964 : size_t k_end = (nz > 1) ? nz - 1 : 1;
1271 :
1272 13964 : double sum = 0.0;
1273 13964 : double sum_abs = 0.0;
1274 13964 : size_t count = 0;
1275 28602 : for (size_t k = k_start; k < k_end; k++) {
1276 449580 : for (size_t j = 1; j < ny - 1; j++) {
1277 17301304 : for (size_t i = 1; i < nx - 1; i++) {
1278 16866362 : double v = rhs[(k * stride_z) + IDX_2D(i, j, nx)];
1279 16866362 : sum += v;
1280 16866362 : sum_abs += fabs(v);
1281 16866362 : count++;
1282 : }
1283 : }
1284 : }
1285 13964 : if (count == 0 || sum_abs == 0.0) {
1286 : return CFD_SUCCESS;
1287 : }
1288 :
1289 : /* Relative to the mean magnitude, so the test is scale-free. */
1290 13401 : double tol = solver->params.tolerance;
1291 13401 : if (tol <= 0.0) {
1292 0 : tol = 1e-6;
1293 : }
1294 13401 : if (fabs(sum) > tol * sum_abs) {
1295 2 : cfd_set_error(CFD_ERROR_INVALID,
1296 : "zero-gradient walls make this operator singular, so the rhs must have zero "
1297 : "interior mean; call poisson_make_rhs_compatible() first, or prescribe a "
1298 : "wall value on one face via params.walls");
1299 2 : return CFD_ERROR_INVALID;
1300 : }
1301 : return CFD_SUCCESS;
1302 : }
1303 :
1304 17406 : cfd_status_t poisson_solver_solve(
1305 : poisson_solver_t* solver,
1306 : double* x,
1307 : double* x_temp,
1308 : const double* rhs,
1309 : poisson_solver_stats_t* stats)
1310 : {
1311 17406 : if (!solver) {
1312 : return CFD_ERROR_INVALID;
1313 : }
1314 :
1315 17406 : if (rhs) {
1316 17406 : cfd_status_t compat = check_rhs_compatible(solver, rhs);
1317 17406 : if (compat != CFD_SUCCESS) {
1318 2 : if (stats) {
1319 : /* Reset first: a caller reusing one stats struct across solves
1320 : * would otherwise read the previous solve's residual and timing
1321 : * next to this refusal, which looks like a converged result. */
1322 2 : *stats = poisson_solver_stats_default();
1323 2 : stats->status = POISSON_INCOMPATIBLE_RHS;
1324 2 : stats->iterations = 0;
1325 : }
1326 2 : return compat;
1327 : }
1328 : }
1329 :
1330 : /* Use solver-specific solve if provided, otherwise common loop */
1331 17404 : if (solver->solve) {
1332 16674 : double start_time = poisson_solver_get_time_ms();
1333 16674 : cfd_status_t status = solver->solve(solver, x, x_temp, rhs, stats);
1334 16674 : if (stats) {
1335 16674 : stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
1336 : }
1337 16674 : return status;
1338 : }
1339 :
1340 730 : return poisson_solver_solve_common(solver, x, x_temp, rhs, stats);
1341 : }
1342 :
1343 41832 : cfd_status_t poisson_solver_iterate(
1344 : poisson_solver_t* solver,
1345 : double* x,
1346 : double* x_temp,
1347 : const double* rhs,
1348 : double* residual)
1349 : {
1350 41832 : if (!solver || !x || !rhs) {
1351 : return CFD_ERROR_INVALID;
1352 : }
1353 :
1354 41832 : if (!solver->iterate) {
1355 : return CFD_ERROR_UNSUPPORTED;
1356 : }
1357 :
1358 41832 : return solver->iterate(solver, x, x_temp, rhs, residual);
1359 : }
1360 :
1361 : /* ============================================================================
1362 : * CONVENIENCE API
1363 : * ============================================================================ */
1364 :
1365 136 : cfd_status_t poisson_solve(
1366 : double* x, double* x_temp, const double* rhs,
1367 : size_t nx, size_t ny, size_t nz,
1368 : double dx, double dy, double dz,
1369 : const poisson_solver_config_t* config,
1370 : poisson_solver_stats_t* stats)
1371 : {
1372 136 : if (!x || !rhs) {
1373 0 : cfd_set_error(CFD_ERROR_INVALID, "poisson_solve: x and rhs are required");
1374 0 : return CFD_ERROR_INVALID;
1375 : }
1376 :
1377 : /* Cleared so that the status read back below is this call's and not whatever
1378 : * the thread failed at last. */
1379 136 : cfd_clear_error();
1380 :
1381 135 : poisson_solver_config_t cfg =
1382 136 : config ? *config : poisson_solver_config_preset(POISSON_PRESET_DEFAULT);
1383 :
1384 136 : poisson_solver_t* solver = poisson_solver_create(cfg.method, cfg.backend);
1385 136 : if (!solver) {
1386 : /* The factory named the reason -- an unavailable backend, an unknown
1387 : * method -- so report that rather than overwriting it. A factory that
1388 : * returned NULL without setting one (an allocation failure inside a
1389 : * create_*_solver) must not read back as success. */
1390 1 : cfd_status_t reason = cfd_get_last_status();
1391 1 : if (reason == CFD_SUCCESS) {
1392 0 : cfd_set_error(CFD_ERROR_UNSUPPORTED,
1393 : "poisson_solve: the requested method and backend could not be created");
1394 0 : reason = CFD_ERROR_UNSUPPORTED;
1395 : }
1396 1 : return reason;
1397 : }
1398 :
1399 135 : cfd_status_t status =
1400 135 : poisson_solver_init(solver, nx, ny, nz, dx, dy, dz, &cfg.params);
1401 135 : if (status != CFD_SUCCESS) {
1402 6 : poisson_solver_destroy(solver);
1403 6 : return status;
1404 : }
1405 :
1406 129 : poisson_solver_stats_t local = poisson_solver_stats_default();
1407 257 : status = poisson_solver_solve(solver, x, x_temp, rhs, stats ? stats : &local);
1408 129 : poisson_solver_destroy(solver);
1409 129 : return status;
1410 : }
|