LCOV - code coverage report
Current view: top level - solvers/linear/gmres_template - linear_solver_gmres_template.h (source / functions) Coverage Total Hit
Test: coverage.info Lines: 94.1 % 202 190
Test Date: 2026-10-05 04:47:18 Functions: 86.7 % 15 13

            Line data    Source code
       1              : /**
       2              :  * @file linear_solver_gmres_template.h
       3              :  * @brief Restarted GMRES(m) algorithm shared by the scalar, OpenMP and SIMD backends
       4              :  *
       5              :  * Generalized Minimal Residual method for solving Ax = b. Unlike CG, GMRES does
       6              :  * not require A to be symmetric positive definite, so it is the Krylov method of
       7              :  * choice for non-symmetric systems (e.g. implicit advection-diffusion). For the
       8              :  * pressure-Poisson operator used here (A = -nabla^2, SPD) GMRES converges to the
       9              :  * same solution as CG and serves as an independent cross-check.
      10              :  *
      11              :  * GMRES characteristics:
      12              :  * - Minimizes the residual norm over the Krylov subspace at every inner step.
      13              :  * - Restarted GMRES(m) caps the basis at m+1 vectors to bound memory; the outer
      14              :  *   loop restarts from the current iterate until convergence or max_iterations.
      15              :  * - Each inner iteration: 1 matrix-vector product, (j+1) dot products + axpys for
      16              :  *   modified Gram-Schmidt, plus O(m) scalar work for Givens rotations.
      17              :  *
      18              :  * Algorithm (restarted GMRES(m), right-preconditioned when enabled):
      19              :  *   outer restart:
      20              :  *     r = b - A x;  beta = ||r||;  v_0 = r / beta;  g = (beta, 0, ..., 0)
      21              :  *     inner j = 0..m-1 (Arnoldi + modified Gram-Schmidt):
      22              :  *       w = A M^{-1} v_j          (w = A v_j when unpreconditioned)
      23              :  *       for i <= j: H[i,j] = (w, v_i);  w -= H[i,j] v_i
      24              :  *       H[j+1,j] = ||w||;  v_{j+1} = w / H[j+1,j]
      25              :  *       apply stored Givens rotations to column j; build a new one to zero
      26              :  *       H[j+1,j]; rotate g.  Residual estimate = |g[j+1]|.
      27              :  *     solve upper-triangular H y = g;  x += M^{-1}(V y)  (x += V y unpreconditioned)
      28              :  *
      29              :  * Right (not left) preconditioning is used so the cheap Givens estimate |g[k]|
      30              :  * equals the true unpreconditioned residual ||b - A x||, keeping the convergence
      31              :  * check, reported statistics, and the CG cross-check consistent. On a uniform
      32              :  * grid the Laplacian diagonal is constant, so Jacobi preconditioning is a scalar
      33              :  * multiply and does not reduce iteration count.
      34              :  *
      35              :  * This header is the only copy of the algorithm: context, init/destroy, the solve
      36              :  * loop, the dense Givens/back-substitution helpers, iterate and the factory. Each
      37              :  * backend includes it once, after defining its O(n) grid primitives, so backends
      38              :  * differ only in those primitives:
      39              :  *   cpu/linear_solver_gmres.c                          scalar
      40              :  *   omp/linear_solver_gmres_omp.c                      OpenMP
      41              :  *   simd_template/linear_solver_gmres_simd_template.h  AVX2 / NEON (+ OpenMP)
      42              :  * The dense O(m^2) Hessenberg work is serial in every backend.
      43              :  *
      44              :  * Operator/sign convention matches CG: A = -nabla^2, b = -rhs. 2D (nz == 1) and
      45              :  * 3D share one stencil (stride_z = 0 and inv_dz2 = 0 for 2D).
      46              :  *
      47              :  * REQUIRED MACROS (every macro below is #undef'd at the end of this header):
      48              :  *   GMRES_SUFFIX             symbol suffix (scalar, omp, avx2, neon); the factory
      49              :  *                            is create_gmres_<GMRES_SUFFIX>_solver
      50              :  *   GMRES_SOLVER_NAME        POISSON_SOLVER_TYPE_GMRES_* name string
      51              :  *   GMRES_DESCRIPTION        description string
      52              :  *   GMRES_BACKEND            POISSON_BACKEND_* value
      53              :  *   GMRES_LOG_TAG            string literal prefixed to verbose log lines
      54              :  *   GMRES_VEC_CALLOC(count)  allocate `count` zeroed doubles for an O(n) vector
      55              :  *   GMRES_VEC_FREE(ptr)      release a GMRES_VEC_CALLOC block (NULL-safe)
      56              :  *
      57              :  * REQUIRED PRIMITIVES (each macro names a function; interior points only):
      58              :  *   double GMRES_DOT(a, b, nx, ny, k_start, k_end, stride_z)          returns (a, b)
      59              :  *   void GMRES_AXPY(alpha, x, y, nx, ny, k_start, k_end, stride_z)    y += alpha * x
      60              :  *   void GMRES_SCALE(alpha, x, nx, ny, k_start, k_end, stride_z)      x *= alpha
      61              :  *   void GMRES_COPY(src, dst, nx, ny, k_start, k_end, stride_z)       dst = src
      62              :  *   void GMRES_APPLY_A(p, Ap, nx, ny, dx2, dy2, inv_dz2,
      63              :  *                      k_start, k_end, stride_z)                      Ap = -nabla^2 p
      64              :  *   void GMRES_RESIDUAL(x, rhs, r, nx, ny, dx2, dy2, inv_dz2,
      65              :  *                       k_start, k_end, stride_z)                     r = -rhs + nabla^2 x
      66              :  *   void GMRES_PRECOND(r, z, nx, ny, diag_inv,
      67              :  *                      k_start, k_end, stride_z)                      z = diag_inv * r
      68              :  */
      69              : 
      70              : #include "../linear_solver_internal.h"
      71              : 
      72              : #include "cfd/core/logging.h"
      73              : #include "cfd/core/memory.h"
      74              : 
      75              : #include <limits.h>
      76              : #include <math.h>
      77              : #include <string.h>
      78              : 
      79              : #if !defined(GMRES_SUFFIX) || !defined(GMRES_SOLVER_NAME) || !defined(GMRES_DESCRIPTION) || \
      80              :     !defined(GMRES_BACKEND) || !defined(GMRES_LOG_TAG) ||                                   \
      81              :     !defined(GMRES_VEC_CALLOC) || !defined(GMRES_VEC_FREE)
      82              : #error "GMRES configuration macros must be defined before including linear_solver_gmres_template.h"
      83              : #endif
      84              : 
      85              : #if !defined(GMRES_DOT) || !defined(GMRES_AXPY) || !defined(GMRES_SCALE) ||       \
      86              :     !defined(GMRES_COPY) || !defined(GMRES_APPLY_A) || !defined(GMRES_RESIDUAL) || \
      87              :     !defined(GMRES_PRECOND)
      88              : #error "GMRES primitive macros must be defined before including linear_solver_gmres_template.h"
      89              : #endif
      90              : 
      91              : /* ============================================================================
      92              :  * TOKEN PASTING
      93              :  * ============================================================================ */
      94              : 
      95              : #define GMRES_CONCAT_IMPL(a, b) a##_##b
      96              : #define GMRES_CONCAT(a, b) GMRES_CONCAT_IMPL(a, b)
      97              : #define GMRES_FUNC(name) GMRES_CONCAT(name, GMRES_SUFFIX)
      98              : 
      99              : #define GMRES_FACTORY_IMPL(suffix) create_gmres_##suffix##_solver
     100              : #define GMRES_FACTORY(suffix) GMRES_FACTORY_IMPL(suffix)
     101              : 
     102              : /* ============================================================================
     103              :  * GMRES CONTEXT
     104              :  * ============================================================================ */
     105              : 
     106              : #define gmres_ctx_t GMRES_FUNC(gmres_context_t)
     107              : 
     108              : typedef struct {
     109              :     double dx2;        /* dx^2 */
     110              :     double dy2;        /* dy^2 */
     111              :     double inv_dz2;    /* 1/dz^2 (0 for 2D) */
     112              :     double diag_inv;   /* Jacobi preconditioner: 1/(2/dx2 + 2/dy2 + 2*inv_dz2) */
     113              : 
     114              :     size_t stride_z;   /* nx*ny for 3D, 0 for 2D */
     115              :     size_t k_start;    /* first interior k index */
     116              :     size_t k_end;      /* one-past-last interior k index */
     117              : 
     118              :     int    m;          /* restart length (Krylov subspace dimension) */
     119              :     size_t n;          /* total field size nx*ny*nz */
     120              : 
     121              :     /* Large O(n) working vectors (GMRES_VEC_CALLOC) */
     122              :     double* Vblock;    /* (m+1) Krylov basis vectors, V[j] = Vblock + j*n */
     123              :     double* w;         /* A*v_j scratch */
     124              :     double* Mv;        /* M^{-1}*v scratch (NULL if no precond) */
     125              : 
     126              :     /* Small dense O(m^2)/O(m) arrays */
     127              :     double* H;         /* upper-Hessenberg (m+1) x m, column-major H[i + j*(m+1)] */
     128              :     double* cs;        /* Givens cosines, length m */
     129              :     double* sn;        /* Givens sines, length m */
     130              :     double* g;         /* rotated least-squares RHS, length m+1 */
     131              :     double* y;         /* back-substitution solution, length m */
     132              : 
     133              :     int use_precond;   /* Flag: is preconditioner enabled? */
     134              :     int initialized;
     135              : } gmres_ctx_t;
     136              : 
     137              : /* ============================================================================
     138              :  * DENSE HELPERS (small O(m^2)/O(m) work, serial in every backend)
     139              :  * ============================================================================ */
     140              : 
     141              : /**
     142              :  * Apply previously stored Givens rotations to column j of H, then build and apply
     143              :  * a new rotation that zeros H[j+1,j], and rotate the RHS vector g accordingly.
     144              :  * H is column-major, (m+1) x m: element (row i, col j) is H[i + j*(m+1)].
     145              :  * After the call, |g[j+1]| is the GMRES residual estimate for the current basis.
     146              :  */
     147         2140 : static void GMRES_FUNC(gmres_givens)(double* H, double* cs, double* sn,
     148              :                                      double* g, int j, int m) {
     149         2140 :     const int lead = m + 1;  /* column stride */
     150              : 
     151              :     /* Apply rotations 0..j-1 to the new column j */
     152        19653 :     for (int i = 0; i < j; i++) {
     153        17513 :         double temp     =  cs[i] * H[i + j * lead]     + sn[i] * H[(i + 1) + j * lead];
     154        17513 :         H[(i + 1) + j * lead] = -sn[i] * H[i + j * lead] + cs[i] * H[(i + 1) + j * lead];
     155        17513 :         H[i + j * lead] = temp;
     156              :     }
     157              : 
     158              :     /* Compute new rotation to zero H[j+1,j] */
     159         2140 :     double h_jj  = H[j + j * lead];
     160         2140 :     double h_j1j = H[(j + 1) + j * lead];
     161         2140 :     double denom = sqrt(h_jj * h_jj + h_j1j * h_j1j);
     162         2140 :     if (denom < GMRES_BREAKDOWN_THRESHOLD) {
     163              :         /* Degenerate column (both entries ~0): identity rotation. */
     164            0 :         cs[j] = 1.0;
     165            0 :         sn[j] = 0.0;
     166              :     } else {
     167         2140 :         cs[j] = h_jj / denom;
     168         2140 :         sn[j] = h_j1j / denom;
     169              :     }
     170              : 
     171              :     /* Apply the new rotation to H (H[j,j] becomes denom, H[j+1,j] becomes 0) */
     172         2140 :     H[j + j * lead]       = cs[j] * h_jj + sn[j] * h_j1j;
     173         2140 :     H[(j + 1) + j * lead] = 0.0;
     174              : 
     175              :     /* Rotate g */
     176         2140 :     double g_j = g[j];
     177         2140 :     g[j]     =  cs[j] * g_j;
     178         2140 :     g[j + 1] = -sn[j] * g_j;
     179         2140 : }
     180              : 
     181              : /**
     182              :  * Solve the k x k upper-triangular top block of H for y: H(0:k,0:k) y = g(0:k).
     183              :  * H column-major, (m+1) x m.
     184              :  */
     185          319 : static void GMRES_FUNC(gmres_backsub)(const double* H, const double* g, double* y,
     186              :                                       int k, int m) {
     187          319 :     const int lead = m + 1;
     188         2459 :     for (int i = k - 1; i >= 0; i--) {
     189         2140 :         double sum = g[i];
     190        19653 :         for (int l = i + 1; l < k; l++) {
     191        17513 :             sum -= H[i + l * lead] * y[l];
     192              :         }
     193         2140 :         double diag = H[i + i * lead];
     194         2140 :         y[i] = (fabs(diag) > GMRES_BREAKDOWN_THRESHOLD) ? (sum / diag) : 0.0;
     195              :     }
     196          319 : }
     197              : 
     198              : /** Euclidean norm over interior points: sqrt(dot(x,x)) */
     199         2530 : static double GMRES_FUNC(gmres_norm)(const double* x,
     200              :                                      size_t nx, size_t ny,
     201              :                                      size_t k_start, size_t k_end, size_t stride_z) {
     202         3494 :     return sqrt(GMRES_DOT(x, x, nx, ny, k_start, k_end, stride_z));
     203              : }
     204              : 
     205              : /* ============================================================================
     206              :  * INIT / DESTROY
     207              :  * ============================================================================ */
     208              : 
     209           93 : static void GMRES_FUNC(gmres_destroy)(poisson_solver_t* solver) {
     210           93 :     if (solver && solver->context) {
     211           76 :         gmres_ctx_t* ctx = (gmres_ctx_t*)solver->context;
     212           76 :         GMRES_VEC_FREE(ctx->Vblock);
     213           76 :         GMRES_VEC_FREE(ctx->w);
     214           76 :         GMRES_VEC_FREE(ctx->Mv);
     215           76 :         cfd_free(ctx->H);
     216           76 :         cfd_free(ctx->cs);
     217           76 :         cfd_free(ctx->sn);
     218           76 :         cfd_free(ctx->g);
     219           76 :         cfd_free(ctx->y);
     220           76 :         cfd_free(ctx);
     221           76 :         solver->context = NULL;
     222              :     }
     223           93 : }
     224              : 
     225           82 : static cfd_status_t GMRES_FUNC(gmres_init)(
     226              :     poisson_solver_t* solver,
     227              :     size_t nx, size_t ny, size_t nz,
     228              :     double dx, double dy, double dz,
     229              :     const poisson_solver_params_t* params)
     230              : {
     231              :     /* Resolve restart length m (0 = auto) */
     232           82 :     int m = (params && params->krylov.restart > 0) ? params->krylov.restart : GMRES_DEFAULT_RESTART;
     233              : 
     234              :     /* H is indexed in int as i + j*(m+1), so (m+1)*m must fit in int. */
     235           82 :     if (m >= INT_MAX / m) {
     236            2 :         cfd_set_error(CFD_ERROR_LIMIT_EXCEEDED, "GMRES restart length exceeds indexable limits");
     237            2 :         return CFD_ERROR_LIMIT_EXCEEDED;
     238              :     }
     239              : 
     240              :     /* The (m+1) Krylov basis vectors are one allocation, so the whole block must
     241              :      * fit in size_t bytes; the parallel backends also need int loop bounds. */
     242           80 :     size_t n = 0;
     243           80 :     cfd_status_t size_status =
     244           80 :         poisson_solver_validate_grid_size(nx, ny, nz, (size_t)m + 1, &n);
     245           80 :     if (size_status != CFD_SUCCESS) {
     246              :         return size_status;
     247              :     }
     248              : 
     249           76 :     gmres_ctx_t* ctx = (gmres_ctx_t*)cfd_calloc(1, sizeof(gmres_ctx_t));
     250           76 :     if (!ctx) {
     251              :         return CFD_ERROR_NOMEM;
     252              :     }
     253              : 
     254           76 :     ctx->dx2 = dx * dx;
     255           76 :     ctx->dy2 = dy * dy;
     256           76 :     ctx->inv_dz2 = poisson_solver_compute_inv_dz2(dz);
     257           76 :     poisson_solver_compute_3d_bounds(nz, nx, ny, &ctx->stride_z, &ctx->k_start, &ctx->k_end);
     258              : 
     259           76 :     ctx->diag_inv = 1.0 / (2.0 / ctx->dx2 + 2.0 / ctx->dy2 + 2.0 * ctx->inv_dz2);
     260           76 :     ctx->use_precond = (params && params->krylov.preconditioner == POISSON_PRECOND_JACOBI);
     261              : 
     262           76 :     ctx->m = m;
     263           76 :     ctx->n = n;
     264              : 
     265              :     /* Large O(n) arrays */
     266           76 :     ctx->Vblock = (double*)GMRES_VEC_CALLOC(((size_t)m + 1) * n);
     267           76 :     ctx->w = (double*)GMRES_VEC_CALLOC(n);
     268           76 :     ctx->Mv = ctx->use_precond ? (double*)GMRES_VEC_CALLOC(n) : NULL;
     269              : 
     270              :     /* Small dense arrays */
     271           76 :     ctx->H = (double*)cfd_calloc(((size_t)m + 1) * (size_t)m, sizeof(double));
     272           76 :     ctx->cs = (double*)cfd_calloc((size_t)m, sizeof(double));
     273           76 :     ctx->sn = (double*)cfd_calloc((size_t)m, sizeof(double));
     274           76 :     ctx->g = (double*)cfd_calloc((size_t)m + 1, sizeof(double));
     275           76 :     ctx->y = (double*)cfd_calloc((size_t)m, sizeof(double));
     276              : 
     277           76 :     int precond_ok = (!ctx->use_precond) || (ctx->Mv != NULL);
     278           76 :     if (!ctx->Vblock || !ctx->w || !ctx->H || !ctx->cs || !ctx->sn ||
     279           76 :         !ctx->g || !ctx->y || !precond_ok) {
     280            0 :         solver->context = ctx;
     281            0 :         GMRES_FUNC(gmres_destroy)(solver);
     282            0 :         return CFD_ERROR_NOMEM;
     283              :     }
     284              : 
     285           76 :     ctx->initialized = 1;
     286           76 :     solver->context = ctx;
     287           76 :     return CFD_SUCCESS;
     288              : }
     289              : 
     290              : /* ============================================================================
     291              :  * SOLVE
     292              :  * ============================================================================ */
     293              : 
     294           72 : static cfd_status_t GMRES_FUNC(gmres_solve)(
     295              :     poisson_solver_t* solver,
     296              :     double* x,
     297              :     double* x_temp,
     298              :     const double* rhs,
     299              :     poisson_solver_stats_t* stats)
     300              : {
     301           72 :     (void)x_temp;  /* GMRES uses its own basis storage */
     302              : 
     303           72 :     gmres_ctx_t* ctx = (gmres_ctx_t*)solver->context;
     304           72 :     if (!ctx || !ctx->initialized) {
     305            1 :         cfd_set_error(CFD_ERROR_INVALID, "GMRES solve called before a successful init");
     306            1 :         return CFD_ERROR_INVALID;
     307              :     }
     308              : 
     309           71 :     size_t nx = solver->nx;
     310           71 :     size_t ny = solver->ny;
     311           71 :     double dx2 = ctx->dx2;
     312           71 :     double dy2 = ctx->dy2;
     313           71 :     double inv_dz2 = ctx->inv_dz2;
     314           71 :     size_t stride_z = ctx->stride_z;
     315           71 :     size_t k_start = ctx->k_start;
     316           71 :     size_t k_end = ctx->k_end;
     317              : 
     318           71 :     int m = ctx->m;
     319           71 :     size_t n = ctx->n;
     320           71 :     const int lead = m + 1;
     321              : 
     322           71 :     double* Vblock = ctx->Vblock;
     323           71 :     double* w = ctx->w;
     324           71 :     double* Mv = ctx->Mv;
     325           71 :     double* H = ctx->H;
     326           71 :     double* cs = ctx->cs;
     327           71 :     double* sn = ctx->sn;
     328           71 :     double* g = ctx->g;
     329           71 :     double* y = ctx->y;
     330           71 :     int use_precond = ctx->use_precond;
     331           71 :     double diag_inv = ctx->diag_inv;
     332              : 
     333           71 :     poisson_solver_params_t* params = &solver->params;
     334           71 :     double start_time = poisson_solver_get_time_ms();
     335              : 
     336              :     /* Not poisson_solver_apply_bc: the initial residual has to see the same walls
     337              :      * the iteration below inverts. See poisson_solver_krylov_apply_bc. */
     338           71 :     poisson_solver_krylov_apply_bc(solver, x);
     339              : 
     340              :     /* Initial residual r_0 = b - A x_0 into V[0] */
     341           71 :     GMRES_RESIDUAL(x, rhs, Vblock, nx, ny, dx2, dy2, inv_dz2, k_start, k_end, stride_z);
     342              : 
     343           71 :     double beta = GMRES_FUNC(gmres_norm)(Vblock, nx, ny, k_start, k_end, stride_z);
     344           71 :     double initial_res = beta;
     345              : 
     346           71 :     if (stats) {
     347           71 :         stats->initial_residual = initial_res;
     348              :     }
     349              : 
     350              :     /* Never below the round-off floor of the residual at the initial guess:
     351              :      * a warm start near steady state begins there (poisson_solver_residual_floor) */
     352           71 :     double floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_L2);
     353           71 :     double tolerance = poisson_solver_stop_target(params, initial_res, floor_res);
     354              : 
     355          139 :     if (initial_res < poisson_solver_done_target(params, floor_res)) {
     356            9 :         poisson_solver_apply_bc(solver, x);
     357            9 :         if (stats) {
     358            9 :             stats->status = POISSON_CONVERGED;
     359            9 :             stats->iterations = 0;
     360            9 :             stats->final_residual = initial_res;
     361            9 :             stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
     362              :         }
     363            9 :         return CFD_SUCCESS;
     364              :     }
     365              : 
     366              :     int total_inner = 0;
     367              :     int converged = 0;
     368              :     double final_res = initial_res;
     369              : 
     370          381 :     while (total_inner < params->max_iterations && !converged) {
     371              :         /* V[0] and beta hold the current TRUE residual (fresh on first pass,
     372              :          * recomputed at the end of each restart below). */
     373          319 :         if (beta < tolerance || beta < params->absolute_tolerance) {
     374              :             converged = 1;
     375              :             final_res = beta;
     376              :             break;
     377              :         }
     378              : 
     379              :         /* v_0 = r / beta */
     380          319 :         GMRES_SCALE(1.0 / beta, Vblock, nx, ny, k_start, k_end, stride_z);
     381              : 
     382              :         /* Reset least-squares RHS: g = (beta, 0, ..., 0) */
     383          319 :         memset(g, 0, ((size_t)m + 1) * sizeof(double));
     384          319 :         g[0] = beta;
     385              : 
     386          319 :         int k = 0;
     387         2397 :         for (int j = 0; j < m; j++) {
     388         2140 :             double* v_j = Vblock + (size_t)j * n;
     389              : 
     390              :             /* Arnoldi: w = A M^{-1} v_j  (w = A v_j unpreconditioned).
     391              :              * The halo carries the homogeneous boundary condition, which is what
     392              :              * makes the apply the operator the walls describe. Basis vectors are
     393              :              * built from interior-only updates, so it is applied to whichever
     394              :              * vector A actually receives, every time. */
     395         2140 :             if (use_precond) {
     396          247 :                 GMRES_PRECOND(v_j, Mv, nx, ny, diag_inv, k_start, k_end, stride_z);
     397          247 :                 poisson_solver_krylov_apply_bc_homogeneous(solver, Mv);
     398          247 :                 GMRES_APPLY_A(Mv, w, nx, ny, dx2, dy2, inv_dz2, k_start, k_end, stride_z);
     399              :             } else {
     400         1893 :                 poisson_solver_krylov_apply_bc_homogeneous(solver, v_j);
     401         1893 :                 GMRES_APPLY_A(v_j, w, nx, ny, dx2, dy2, inv_dz2, k_start, k_end, stride_z);
     402              :             }
     403              : 
     404              :             /* Modified Gram-Schmidt against v_0..v_j */
     405        21793 :             for (int i = 0; i <= j; i++) {
     406        19653 :                 double* v_i = Vblock + (size_t)i * n;
     407        19653 :                 double hij = GMRES_DOT(w, v_i, nx, ny, k_start, k_end, stride_z);
     408        19653 :                 H[i + j * lead] = hij;
     409        31641 :                 GMRES_AXPY(-hij, v_i, w, nx, ny, k_start, k_end, stride_z);
     410              :             }
     411              : 
     412         2140 :             double hnorm = GMRES_FUNC(gmres_norm)(w, nx, ny, k_start, k_end, stride_z);
     413         2140 :             H[(j + 1) + j * lead] = hnorm;
     414              : 
     415              :             /* Happy breakdown: solution lies in the current Krylov subspace. */
     416         2140 :             int happy = (hnorm < GMRES_BREAKDOWN_THRESHOLD);
     417         2140 :             if (!happy) {
     418         2134 :                 GMRES_SCALE(1.0 / hnorm, w, nx, ny, k_start, k_end, stride_z);
     419         2134 :                 GMRES_COPY(w, Vblock + (size_t)(j + 1) * n, nx, ny, k_start, k_end, stride_z);
     420              :             }
     421              : 
     422         2140 :             GMRES_FUNC(gmres_givens)(H, cs, sn, g, j, m);
     423         2140 :             total_inner++;
     424         2140 :             k = j + 1;
     425              : 
     426              :             /* Cheap Givens residual estimate: only decides when to END the inner
     427              :              * cycle early (to update x and recheck the TRUE residual). It does NOT
     428              :              * decide convergence, so drift in the estimate cannot cause a false
     429              :              * positive. */
     430         2140 :             double resid_est = fabs(g[j + 1]);
     431         2140 :             if (params->verbose) {
     432            0 :                 CFD_LOG_DEBUG("poisson", GMRES_LOG_TAG " inner %d (total %d): residual est = %.6e",
     433              :                               j, total_inner, resid_est);
     434              :             }
     435         2140 :             if (resid_est < tolerance || happy || total_inner >= params->max_iterations) {
     436              :                 break;
     437              :             }
     438              :         }
     439              : 
     440              :         /* Solve the least-squares problem and update x with sum_j y[j] * V[j] */
     441          319 :         GMRES_FUNC(gmres_backsub)(H, g, y, k, m);
     442          319 :         if (use_precond) {
     443           14 :             memset(w, 0, n * sizeof(double));
     444          261 :             for (int jj = 0; jj < k; jj++) {
     445          392 :                 GMRES_AXPY(y[jj], Vblock + (size_t)jj * n, w, nx, ny, k_start, k_end, stride_z);
     446              :             }
     447           40 :             GMRES_PRECOND(w, Mv, nx, ny, diag_inv, k_start, k_end, stride_z);
     448          200 :             GMRES_AXPY(1.0, Mv, x, nx, ny, k_start, k_end, stride_z);
     449              :         } else {
     450         2198 :             for (int jj = 0; jj < k; jj++) {
     451         3078 :                 GMRES_AXPY(y[jj], Vblock + (size_t)jj * n, x, nx, ny, k_start, k_end, stride_z);
     452              :             }
     453              :         }
     454              : 
     455              :         /* Recompute the TRUE residual and decide convergence on it (prevents the
     456              :          * cheap Givens estimate from drifting below the real residual). This is
     457              :          * the residual reported to the caller. x has moved since the last apply_bc,
     458              :          * and with zero-gradient walls the halo follows the interior, so it is
     459              :          * refreshed first. */
     460          319 :         poisson_solver_krylov_apply_bc(solver, x);
     461          319 :         GMRES_RESIDUAL(x, rhs, Vblock, nx, ny, dx2, dy2, inv_dz2, k_start, k_end, stride_z);
     462          319 :         beta = GMRES_FUNC(gmres_norm)(Vblock, nx, ny, k_start, k_end, stride_z);
     463          319 :         final_res = beta;
     464              :         /* A restart forms the true residual again; x has moved, and its floor with it */
     465          319 :         floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_L2);
     466          319 :         if (tolerance < floor_res) {
     467            0 :             tolerance = floor_res;
     468              :         }
     469          319 :         if (beta < tolerance || beta < params->absolute_tolerance) {
     470           55 :             converged = 1;
     471              :         }
     472              :     }
     473              : 
     474              :     /* Apply final boundary conditions for output (mirrors CG). The reported
     475              :      * residual is the solve-consistent one measured above, NOT recomputed here. */
     476           62 :     poisson_solver_apply_bc(solver, x);
     477              : 
     478           62 :     if (stats) {
     479           62 :         stats->iterations = total_inner;
     480           62 :         stats->final_residual = final_res;
     481           62 :         stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
     482           62 :         stats->status = converged ? POISSON_CONVERGED : POISSON_MAX_ITER;
     483              :     }
     484              : 
     485           62 :     return converged ? CFD_SUCCESS : CFD_ERROR_MAX_ITER;
     486              : }
     487              : 
     488              : /**
     489              :  * Single iteration is not well-defined for GMRES (it maintains an internal
     490              :  * Krylov basis). Mirror CG: return the current residual only.
     491              :  */
     492            0 : static cfd_status_t GMRES_FUNC(gmres_iterate)(
     493              :     poisson_solver_t* solver,
     494              :     double* x,
     495              :     double* x_temp,
     496              :     const double* rhs,
     497              :     double* residual)
     498              : {
     499            0 :     (void)x_temp;
     500            0 :     if (residual) {
     501            0 :         *residual = poisson_solver_compute_residual(solver, x, rhs);
     502              :     }
     503            0 :     return CFD_SUCCESS;
     504              : }
     505              : 
     506              : /* ============================================================================
     507              :  * FACTORY FUNCTION
     508              :  * ============================================================================ */
     509              : 
     510           93 : poisson_solver_t* GMRES_FACTORY(GMRES_SUFFIX)(void) {
     511           93 :     poisson_solver_t* solver = (poisson_solver_t*)cfd_calloc(1, sizeof(poisson_solver_t));
     512           93 :     if (!solver) {
     513              :         return NULL;
     514              :     }
     515              : 
     516           93 :     solver->name = GMRES_SOLVER_NAME;
     517           93 :     solver->description = GMRES_DESCRIPTION;
     518           93 :     solver->method = POISSON_METHOD_GMRES;
     519           93 :     solver->backend = GMRES_BACKEND;
     520           93 :     solver->params = poisson_solver_params_default();
     521              : 
     522           93 :     solver->init = GMRES_FUNC(gmres_init);
     523           93 :     solver->destroy = GMRES_FUNC(gmres_destroy);
     524           93 :     solver->solve = GMRES_FUNC(gmres_solve);
     525           93 :     solver->iterate = GMRES_FUNC(gmres_iterate);
     526           93 :     solver->apply_bc = NULL;  /* Use default Neumann */
     527              : 
     528           93 :     return solver;
     529              : }
     530              : 
     531              : /* ============================================================================
     532              :  * CLEANUP MACROS
     533              :  * ============================================================================ */
     534              : 
     535              : #undef gmres_ctx_t
     536              : #undef GMRES_FACTORY
     537              : #undef GMRES_FACTORY_IMPL
     538              : #undef GMRES_FUNC
     539              : #undef GMRES_CONCAT
     540              : #undef GMRES_CONCAT_IMPL
     541              : 
     542              : #undef GMRES_SUFFIX
     543              : #undef GMRES_SOLVER_NAME
     544              : #undef GMRES_DESCRIPTION
     545              : #undef GMRES_BACKEND
     546              : #undef GMRES_LOG_TAG
     547              : #undef GMRES_VEC_CALLOC
     548              : #undef GMRES_VEC_FREE
     549              : #undef GMRES_DOT
     550              : #undef GMRES_AXPY
     551              : #undef GMRES_SCALE
     552              : #undef GMRES_COPY
     553              : #undef GMRES_APPLY_A
     554              : #undef GMRES_RESIDUAL
     555              : #undef GMRES_PRECOND
        

Generated by: LCOV version 2.0-1