LCOV - code coverage report
Current view: top level - solvers/linear/omp - linear_solver_cg_omp.c (source / functions) Coverage Total Hit
Test: coverage.info Lines: 87.4 % 174 152
Test Date: 2026-10-05 04:47:18 Functions: 85.7 % 7 6

            Line data    Source code
       1              : /**
       2              :  * @file linear_solver_cg_omp.c
       3              :  * @brief Conjugate Gradient solver - OpenMP parallelized implementation
       4              :  *
       5              :  * Same algorithm as scalar CG but with OpenMP-parallelized primitives
       6              :  * (dot_product, axpy, apply_laplacian, etc.). The multigrid preconditioner
       7              :  * (POISSON_PRECOND_MULTIGRID) runs its V-cycle on the OpenMP multigrid backend.
       8              :  */
       9              : 
      10              : #include "../linear_solver_internal.h"
      11              : 
      12              : #include "cfd/core/indexing.h"
      13              : #include "cfd/core/logging.h"
      14              : #include "cfd/core/memory.h"
      15              : 
      16              : #include <math.h>
      17              : 
      18              : #ifdef CFD_ENABLE_OPENMP
      19              : 
      20              : #include <omp.h>
      21              : 
      22              : #include "linear_solver_primitives_omp.h"
      23              : 
      24              : /* ============================================================================
      25              :  * CG CONTEXT
      26              :  * ============================================================================ */
      27              : 
      28              : typedef struct {
      29              :     double dx2;
      30              :     double dy2;
      31              :     double inv_dz2;    /* 1/dz^2 (0 for 2D) */
      32              :     double diag_inv;   /* Jacobi preconditioner: 1/(2/dx2 + 2/dy2 + 2*inv_dz2 + sigma) */
      33              :     double sigma;      /* Helmholtz shift; 0 = pure Poisson */
      34              : 
      35              :     size_t stride_z;   /* nx*ny for 3D, 0 for 2D */
      36              :     size_t k_start;    /* first interior k index */
      37              :     size_t k_end;      /* one-past-last interior k index */
      38              : 
      39              :     double* r;
      40              :     double* z;         /* Preconditioned residual (NULL if no precond) */
      41              :     double* p;
      42              :     double* Ap;
      43              : 
      44              :     poisson_solver_t* mg_precond;        /* Inner OpenMP MG V-cycle solver (NULL unless
      45              :                                             POISSON_PRECOND_MULTIGRID) */
      46              :     poisson_precond_type_t precond_type; /* Which preconditioner use_precond refers to */
      47              :     int use_precond;
      48              :     int initialized;
      49              : } cg_omp_context_t;
      50              : 
      51              : /* ============================================================================
      52              :  * OMP-PARALLELIZED PRIMITIVES
      53              :  *
      54              :  * dot_product_omp, axpy_omp, scale_vector_omp, apply_laplacian_omp,
      55              :  * compute_residual_omp, copy_vector_omp and apply_jacobi_precond_omp come from
      56              :  * linear_solver_primitives_omp.h (shared with GMRES OMP).
      57              :  * ============================================================================ */
      58              : 
      59        45861 : static void update_search_direction_omp(const double* src, double* p,
      60              :                                         double beta, size_t nx, size_t ny,
      61              :                                         size_t k_start, size_t k_end, size_t stride_z) {
      62        45861 :     int ny_int = poisson_solver_size_to_int(ny);
      63        45861 :     int nx_int = poisson_solver_size_to_int(nx);
      64              : 
      65       116082 :     for (size_t k = k_start; k < k_end; k++) {
      66        70221 :         int j;
      67        70221 : #pragma omp parallel for schedule(static)
      68              :         for (j = 1; j < ny_int - 1; j++) {
      69              :             for (int i = 1; i < nx_int - 1; i++) {
      70              :                 size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
      71              :                 p[idx] = src[idx] + beta * p[idx];
      72              :             }
      73              :         }
      74              :     }
      75        45861 : }
      76              : 
      77              : /**
      78              :  * Apply multigrid preconditioner: z = M^{-1} * r for A = -nabla^2.
      79              :  *
      80              :  * Same steps as apply_mg_precond in the scalar CG solver: one V-cycle on
      81              :  * nabla^2 z' = r from a zero guess, then z = -z'. The OpenMP multigrid cycle is
      82              :  * bit-identical to the scalar one and the sign flip is exact, so both backends
      83              :  * apply the same M. The NULL residual argument skips the serial residual
      84              :  * evaluation after the cycle.
      85              :  */
      86          976 : static cfd_status_t apply_mg_precond_omp(poisson_solver_t* mg, const double* r, double* z,
      87              :                                          size_t n_total, size_t nx, size_t ny,
      88              :                                          size_t k_start, size_t k_end, size_t stride_z) {
      89          976 :     memset(z, 0, n_total * sizeof(double));
      90              : 
      91          976 :     cfd_status_t status = poisson_solver_iterate(mg, z, NULL, r, NULL);
      92          976 :     if (status != CFD_SUCCESS) {
      93              :         return status;
      94              :     }
      95              : 
      96          976 :     scale_vector_omp(-1.0, z, nx, ny, k_start, k_end, stride_z);
      97          976 :     return CFD_SUCCESS;
      98              : }
      99              : 
     100              : /* ============================================================================
     101              :  * CG OMP IMPLEMENTATION
     102              :  * ============================================================================ */
     103              : 
     104           79 : static void cg_omp_destroy(poisson_solver_t* solver) {
     105           79 :     if (solver && solver->context) {
     106           73 :         cg_omp_context_t* ctx = (cg_omp_context_t*)solver->context;
     107           73 :         poisson_solver_destroy(ctx->mg_precond);
     108           73 :         cfd_free(ctx->r);
     109           73 :         cfd_free(ctx->z);
     110           73 :         cfd_free(ctx->p);
     111           73 :         cfd_free(ctx->Ap);
     112           73 :         cfd_free(ctx);
     113           73 :         solver->context = NULL;
     114              :     }
     115           79 : }
     116              : 
     117           76 : static cfd_status_t cg_omp_init(
     118              :     poisson_solver_t* solver,
     119              :     size_t nx, size_t ny, size_t nz,
     120              :     double dx, double dy, double dz,
     121              :     const poisson_solver_params_t* params)
     122              : {
     123              :     /* The shared primitives loop over int bounds, which collapse to empty loops
     124              :      * for oversized dimensions; reject those grids before allocating. */
     125           76 :     size_t n = 0;
     126           76 :     cfd_status_t size_status = poisson_solver_validate_grid_size(nx, ny, nz, 1, &n);
     127           76 :     if (size_status != CFD_SUCCESS) {
     128              :         return size_status;
     129              :     }
     130              : 
     131           73 :     cg_omp_context_t* ctx = (cg_omp_context_t*)cfd_calloc(1, sizeof(cg_omp_context_t));
     132           73 :     if (!ctx) {
     133              :         return CFD_ERROR_NOMEM;
     134              :     }
     135              :     /* Attached before anything else is allocated, so every failure below
     136              :      * releases what was built through cg_omp_destroy */
     137           73 :     solver->context = ctx;
     138              : 
     139           73 :     ctx->dx2 = dx * dx;
     140           73 :     ctx->dy2 = dy * dy;
     141           73 :     ctx->inv_dz2 = poisson_solver_compute_inv_dz2(dz);
     142           73 :     poisson_solver_compute_3d_bounds(nz, nx, ny, &ctx->stride_z, &ctx->k_start, &ctx->k_end);
     143           73 :     ctx->sigma = params ? params->helmholtz_shift : 0.0;
     144              :     /* The shift is a diagonal term, so it folds into the Jacobi diagonal; adding
     145              :      * 0.0 to a finite positive sum is exact, leaving the unshifted value
     146              :      * bit-identical. */
     147           73 :     ctx->diag_inv = 1.0 / (2.0 / ctx->dx2 + 2.0 / ctx->dy2 + 2.0 * ctx->inv_dz2
     148           73 :                            + ctx->sigma);
     149           73 :     ctx->precond_type = params ? params->krylov.preconditioner : POISSON_PRECOND_NONE;
     150           73 :     ctx->use_precond = (ctx->precond_type == POISSON_PRECOND_JACOBI ||
     151              :                         ctx->precond_type == POISSON_PRECOND_MULTIGRID);
     152              : 
     153           73 :     ctx->r = (double*)cfd_calloc(n, sizeof(double));
     154           73 :     ctx->p = (double*)cfd_calloc(n, sizeof(double));
     155           73 :     ctx->Ap = (double*)cfd_calloc(n, sizeof(double));
     156           73 :     if (ctx->use_precond) {
     157           16 :         ctx->z = (double*)cfd_calloc(n, sizeof(double));
     158              :     }
     159              : 
     160           73 :     if (!ctx->r || !ctx->p || !ctx->Ap || (ctx->use_precond && !ctx->z)) {
     161            0 :         cg_omp_destroy(solver);
     162            0 :         return CFD_ERROR_NOMEM;
     163              :     }
     164              : 
     165              :     /* OpenMP multigrid only: a scalar inner cycle would serialize every apply */
     166           73 :     if (ctx->precond_type == POISSON_PRECOND_MULTIGRID) {
     167           32 :         cfd_status_t mg_status = poisson_solver_create_mg_precond(
     168              :             create_multigrid_omp_solver, nx, ny, nz, dx, dy, dz,
     169           16 :             &solver->params.multigrid, &ctx->mg_precond);
     170           16 :         if (mg_status != CFD_SUCCESS) {
     171            2 :             cg_omp_destroy(solver);
     172            2 :             return mg_status;  /* CFD_ERROR_INVALID for non-2^k+1 dims */
     173              :         }
     174              :     }
     175              : 
     176           71 :     ctx->initialized = 1;
     177           71 :     return CFD_SUCCESS;
     178              : }
     179              : 
     180          598 : static cfd_status_t cg_omp_solve(
     181              :     poisson_solver_t* solver,
     182              :     double* x,
     183              :     double* x_temp,
     184              :     const double* rhs,
     185              :     poisson_solver_stats_t* stats)
     186              : {
     187          598 :     (void)x_temp;
     188              : 
     189          598 :     cg_omp_context_t* ctx = (cg_omp_context_t*)solver->context;
     190          598 :     size_t nx = solver->nx;
     191          598 :     size_t ny = solver->ny;
     192          598 :     size_t n_total = nx * ny * solver->nz;
     193          598 :     double dx2 = ctx->dx2;
     194          598 :     double dy2 = ctx->dy2;
     195          598 :     double inv_dz2 = ctx->inv_dz2;
     196          598 :     size_t k_start = ctx->k_start;
     197          598 :     size_t k_end = ctx->k_end;
     198          598 :     size_t stride_z = ctx->stride_z;
     199              : 
     200          598 :     double* r = ctx->r;
     201          598 :     double* z = ctx->z;
     202          598 :     double* p = ctx->p;
     203          598 :     double* Ap = ctx->Ap;
     204          598 :     int use_precond = ctx->use_precond;
     205          598 :     double diag_inv = ctx->diag_inv;
     206          598 :     double sigma = ctx->sigma;
     207              : 
     208          598 :     poisson_solver_params_t* params = &solver->params;
     209          598 :     double start_time = poisson_solver_get_time_ms();
     210              : 
     211              :     /* Not poisson_solver_apply_bc: the initial residual has to see the same walls
     212              :      * the iteration below inverts. See poisson_solver_krylov_apply_bc. */
     213          598 :     poisson_solver_krylov_apply_bc(solver, x);
     214              : 
     215          598 :     compute_residual_omp(x, rhs, r, nx, ny, dx2, dy2, inv_dz2,
     216              :                          k_start, k_end, stride_z);
     217              :     /* Shifted operator A_h = -lap + sigma*I, so r = b - A_h x loses sigma*x.
     218              :      * Through the existing axpy and behind a branch, as in the scalar CG: at
     219              :      * sigma = 0 the unshifted path runs exactly the instructions it did before. */
     220          598 :     if (sigma != 0.0) {
     221          261 :         axpy_omp(-sigma, x, r, nx, ny, k_start, k_end, stride_z);
     222              :     }
     223              : 
     224          598 :     double rr0 = dot_product_omp(r, r, nx, ny, k_start, k_end, stride_z);
     225          598 :     double initial_res = sqrt(rr0);
     226              : 
     227          598 :     double rho;
     228          598 :     if (use_precond) {
     229           31 :         if (ctx->precond_type == POISSON_PRECOND_MULTIGRID) {
     230           31 :             cfd_status_t precond_status = apply_mg_precond_omp(
     231              :                 ctx->mg_precond, r, z, n_total, nx, ny, k_start, k_end, stride_z);
     232           31 :             if (precond_status != CFD_SUCCESS) {
     233            0 :                 if (stats) {
     234            0 :                     stats->status = POISSON_ERROR;
     235            0 :                     stats->iterations = 0;
     236            0 :                     stats->final_residual = initial_res;
     237            0 :                     stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
     238              :                 }
     239            0 :                 return precond_status;
     240              :             }
     241              :         } else {
     242            0 :             apply_jacobi_precond_omp(r, z, nx, ny, diag_inv,
     243              :                                      k_start, k_end, stride_z);
     244              :         }
     245           31 :         copy_vector_omp(z, p, nx, ny, k_start, k_end, stride_z);
     246           31 :         rho = dot_product_omp(r, z, nx, ny, k_start, k_end, stride_z);
     247              :     } else {
     248          567 :         copy_vector_omp(r, p, nx, ny, k_start, k_end, stride_z);
     249          567 :         rho = rr0;
     250              :     }
     251              : 
     252          598 :     if (stats) {
     253          598 :         stats->initial_residual = initial_res;
     254              :     }
     255              : 
     256              :     /* Never below the round-off floor of the residual at the initial guess:
     257              :      * a warm start near steady state begins there (poisson_solver_residual_floor) */
     258          598 :     double floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_L2);
     259          598 :     double tolerance = poisson_solver_stop_target(params, initial_res, floor_res);
     260              : 
     261         1075 :     if (initial_res < poisson_solver_done_target(params, floor_res)) {
     262           86 :         if (stats) {
     263           86 :             stats->status = POISSON_CONVERGED;
     264           86 :             stats->iterations = 0;
     265           86 :             stats->final_residual = initial_res;
     266           86 :             stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
     267              :         }
     268           86 :         return CFD_SUCCESS;
     269              :     }
     270              : 
     271        46373 :     int converged = 0;
     272              :     int iter;
     273              :     double res_norm = initial_res;
     274              : 
     275        46373 :     for (iter = 0; iter < params->max_iterations; iter++) {
     276              :         /* The halo carries the homogeneous boundary condition, which is what makes
     277              :          * this the operator the walls describe. The direction is rebuilt from
     278              :          * interior-only updates, so it has to be reapplied every iteration. */
     279        46373 :         poisson_solver_krylov_apply_bc_homogeneous(solver, p);
     280        46373 :         apply_laplacian_omp(p, Ap, nx, ny, dx2, dy2, inv_dz2,
     281              :                             k_start, k_end, stride_z);
     282        46373 :         if (sigma != 0.0) {
     283         7590 :             axpy_omp(sigma, p, Ap, nx, ny, k_start, k_end, stride_z);
     284              :         }
     285              : 
     286        46373 :         double p_dot_Ap = dot_product_omp(p, Ap, nx, ny,
     287              :                                           k_start, k_end, stride_z);
     288        46373 :         CG_CHECK_BREAKDOWN(p_dot_Ap, stats, iter, res_norm, start_time);
     289              : 
     290        46373 :         double alpha = rho / p_dot_Ap;
     291              : 
     292        46373 :         axpy_omp(alpha, p, x, nx, ny, k_start, k_end, stride_z);
     293        46373 :         axpy_omp(-alpha, Ap, r, nx, ny, k_start, k_end, stride_z);
     294              : 
     295        46373 :         double rho_new;
     296        46373 :         if (use_precond) {
     297          945 :             if (ctx->precond_type == POISSON_PRECOND_MULTIGRID) {
     298          945 :                 cfd_status_t precond_status = apply_mg_precond_omp(
     299              :                     ctx->mg_precond, r, z, n_total, nx, ny, k_start, k_end, stride_z);
     300          945 :                 if (precond_status != CFD_SUCCESS) {
     301            0 :                     if (stats) {
     302            0 :                         stats->status = POISSON_ERROR;
     303            0 :                         stats->iterations = iter + 1;
     304            0 :                         stats->final_residual = res_norm;
     305            0 :                         stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
     306              :                     }
     307            0 :                     return precond_status;
     308              :                 }
     309              :             } else {
     310            0 :                 apply_jacobi_precond_omp(r, z, nx, ny, diag_inv,
     311              :                                          k_start, k_end, stride_z);
     312              :             }
     313          945 :             rho_new = dot_product_omp(r, z, nx, ny,
     314              :                                       k_start, k_end, stride_z);
     315          945 :             res_norm = sqrt(dot_product_omp(r, r, nx, ny,
     316              :                                             k_start, k_end, stride_z));
     317              :         } else {
     318              :             /* (r,r) is both rho_new and the squared residual norm */
     319        45428 :             rho_new = dot_product_omp(r, r, nx, ny,
     320              :                                       k_start, k_end, stride_z);
     321        45428 :             res_norm = sqrt(rho_new);
     322              :         }
     323              : 
     324        46373 :         if (iter % params->check_interval == 0) {
     325        46373 :             if (params->verbose) {
     326            0 :                 CFD_LOG_DEBUG("poisson", "CG-OMP Iter %d: residual = %.6e", iter, res_norm);
     327              :             }
     328              : 
     329        46373 :             if (res_norm < tolerance || res_norm < params->absolute_tolerance) {
     330              :                 converged = 1;
     331              :                 break;
     332              :             }
     333              :         }
     334              : 
     335        45861 :         CG_CHECK_BREAKDOWN(rho, stats, iter, res_norm, start_time);
     336              : 
     337        45861 :         double beta = rho_new / rho;
     338        90808 :         update_search_direction_omp(use_precond ? z : r, p, beta, nx, ny,
     339              :                                     k_start, k_end, stride_z);
     340              : 
     341        45861 :         rho = rho_new;
     342              :     }
     343              : 
     344          512 :     if (!converged && (res_norm < tolerance || res_norm < params->absolute_tolerance)) {
     345          512 :         converged = 1;
     346              :     }
     347              : 
     348          512 :     poisson_solver_apply_bc(solver, x);
     349              : 
     350          512 :     double end_time = poisson_solver_get_time_ms();
     351              : 
     352          512 :     if (stats) {
     353          512 :         stats->iterations = (iter < params->max_iterations) ? (iter + 1) : iter;
     354          512 :         stats->final_residual = res_norm;
     355          512 :         stats->elapsed_time_ms = end_time - start_time;
     356          512 :         stats->status = converged ? POISSON_CONVERGED : POISSON_MAX_ITER;
     357              :     }
     358              : 
     359          512 :     return converged ? CFD_SUCCESS : CFD_ERROR_MAX_ITER;
     360              : }
     361              : 
     362            0 : static cfd_status_t cg_omp_iterate(
     363              :     poisson_solver_t* solver,
     364              :     double* x,
     365              :     double* x_temp,
     366              :     const double* rhs,
     367              :     double* residual)
     368              : {
     369            0 :     (void)x_temp;
     370              : 
     371            0 :     if (residual) {
     372            0 :         *residual = poisson_solver_compute_residual(solver, x, rhs);
     373              :     }
     374            0 :     return CFD_SUCCESS;
     375              : }
     376              : 
     377              : /* ============================================================================
     378              :  * FACTORY FUNCTION
     379              :  * ============================================================================ */
     380              : 
     381           77 : poisson_solver_t* create_cg_omp_solver(void) {
     382           77 :     poisson_solver_t* solver = (poisson_solver_t*)cfd_calloc(1, sizeof(poisson_solver_t));
     383           77 :     if (!solver) {
     384              :         return NULL;
     385              :     }
     386              : 
     387           77 :     solver->name = POISSON_SOLVER_TYPE_CG_OMP;
     388           77 :     solver->description = "Conjugate Gradient (OpenMP)";
     389           77 :     solver->method = POISSON_METHOD_CG;
     390           77 :     solver->backend = POISSON_BACKEND_OMP;
     391           77 :     solver->params = poisson_solver_params_default();
     392              : 
     393           77 :     solver->init = cg_omp_init;
     394           77 :     solver->destroy = cg_omp_destroy;
     395           77 :     solver->solve = cg_omp_solve;
     396           77 :     solver->iterate = cg_omp_iterate;
     397           77 :     solver->apply_bc = NULL;
     398              : 
     399           77 :     return solver;
     400              : }
     401              : 
     402              : #endif /* CFD_ENABLE_OPENMP */
        

Generated by: LCOV version 2.0-1