LCOV - code coverage report
Current view: top level - solvers/linear/omp - linear_solver_multigrid_omp.c (source / functions) Coverage Total Hit
Test: coverage.info Lines: 100.0 % 106 106
Test Date: 2026-10-05 04:47:18 Functions: 100.0 % 10 10

            Line data    Source code
       1              : /**
       2              :  * @file linear_solver_multigrid_omp.c
       3              :  * @brief Geometric multigrid Poisson solver - OpenMP parallelized implementation
       4              :  *
       5              :  * Supplies OpenMP-parallelized smoother, residual, boundary and grid-transfer
       6              :  * primitives to the shared multigrid algorithm template
       7              :  * (multigrid_template/linear_solver_multigrid_template.h); this file holds no
       8              :  * algorithm code.
       9              :  *
      10              :  * Every primitive is element-wise: each k-plane (or coarse/fine z index) opens
      11              :  * one `parallel for schedule(static)` over rows with an int loop variable
      12              :  * (MSVC OpenMP 2.0), every row writes a disjoint index range, and per-point
      13              :  * stencil sums keep the scalar term order. No loop accumulates across threads:
      14              :  * the Neumann interior mean (mg_interior_mean) and the convergence residual
      15              :  * (poisson_solver_compute_residual) stay serial. Red-Black Gauss-Seidel updates
      16              :  * within one color read only the other color, so they are order-independent.
      17              :  * The result is bit-identical to the scalar backend at any thread count under
      18              :  * the same floating-point contraction settings.
      19              :  *
      20              :  * A region uses the thread team only when its work reaches MG_OMP_MIN_POINTS
      21              :  * (mg_omp_parallel): the interior points of the plane it writes, or 2(nx+ny) for
      22              :  * the boundary pass. Smaller loops (the coarser levels, small grids) run on the
      23              :  * calling thread.
      24              :  *
      25              :  * Boundary conditions run the boundary core that bc_apply_scalar_omp and
      26              :  * bc_apply_scalar_cpu share (same face order, copies only), entered through
      27              :  * the OpenMP function only above the same threshold.
      28              :  */
      29              : 
      30              : #include "../linear_solver_internal.h"
      31              : #include "../multigrid_internal.h"
      32              : 
      33              : #include "cfd/boundary/boundary_conditions.h"
      34              : #include "cfd/core/indexing.h"
      35              : 
      36              : #ifdef CFD_ENABLE_OPENMP
      37              : 
      38              : #include <omp.h>
      39              : #include <string.h>
      40              : 
      41              : /* ============================================================================
      42              :  * PARALLEL THRESHOLD
      43              :  * ============================================================================ */
      44              : 
      45              : /**
      46              :  * Nonzero when a region covering `points` points should use the thread team
      47              :  * (see MG_OMP_MIN_POINTS). Smaller regions run serially through the OpenMP `if`
      48              :  * clause, so the loop body, and the result, are unchanged.
      49              :  */
      50              : static inline int mg_omp_parallel(size_t points) {
      51       329322 :     return points >= MG_OMP_MIN_POINTS;
      52              : }
      53              : 
      54              : /* ============================================================================
      55              :  * BOUNDARY CONDITIONS
      56              :  * ============================================================================ */
      57              : 
      58              : /** Zero-gradient BCs on one nx x ny plane; the pass copies 2(nx+ny) points */
      59       140409 : static void mg_bc_neumann_plane_omp(double* plane, size_t nx, size_t ny) {
      60       140409 :     if (mg_omp_parallel(2 * (nx + ny))) {
      61            3 :         bc_apply_scalar_omp(plane, nx, ny, BC_TYPE_NEUMANN);
      62              :     } else {
      63       140406 :         bc_apply_scalar_cpu(plane, nx, ny, BC_TYPE_NEUMANN);
      64              :     }
      65       140409 : }
      66              : 
      67              : /* ============================================================================
      68              :  * SMOOTHER SWEEPS (no BCs; the template applies them after each sweep)
      69              :  * ============================================================================ */
      70              : 
      71              : /**
      72              :  * One Red-Black Gauss-Seidel pass (omega = 1). A point of one color reads only
      73              :  * points of the other color (5/7-point stencil; the 2D z-term reads the point
      74              :  * itself with stride_z = 0), so rows of one color update independently.
      75              :  */
      76        64551 : static void mg_rbgs_sweep_omp(const mg_level_t* L, double* x,
      77              :                               const double* rhs) {
      78        64551 :     size_t nx = L->nx;
      79        64551 :     double dx2 = L->dx2;
      80        64551 :     double dy2 = L->dy2;
      81        64551 :     double inv_dz2 = L->inv_dz2;
      82        64551 :     double inv_factor = L->inv_factor;
      83        64551 :     size_t stride_z = L->stride_z;
      84        64551 :     int ny_int = poisson_solver_size_to_int(L->ny);
      85        64551 :     int par = mg_omp_parallel((nx - 2) * (L->ny - 2));
      86              : 
      87              :     /* Red pass: (i+j+k) % 2 == 1, then black pass: (i+j+k) % 2 == 0 */
      88       193653 :     for (int color = 0; color < 2; color++) {
      89       288276 :         for (size_t k = L->k_start; k < L->k_end; k++) {
      90       159174 :             int j;
      91       159174 : #pragma omp parallel for schedule(static) if(par)
      92              :             for (j = 1; j < ny_int - 1; j++) {
      93              :                 size_t i_start =
      94              :                     (((size_t)j + k) % 2 == (size_t)color) ? 1 : 2;
      95              :                 for (size_t i = i_start; i < nx - 1; i += 2) {
      96              :                     size_t idx = k * stride_z + IDX_2D(i, (size_t)j, nx);
      97              : 
      98              :                     x[idx] = -(rhs[idx]
      99              :                         - (x[idx + 1] + x[idx - 1]) / dx2
     100              :                         - (x[idx + nx] + x[idx - nx]) / dy2
     101              :                         - (x[idx + stride_z] + x[idx - stride_z]) * inv_dz2
     102              :                         ) * inv_factor;
     103              :                 }
     104              :             }
     105              :         }
     106              :     }
     107        64551 : }
     108              : 
     109              : /**
     110              :  * One weighted Jacobi pass (omega = MG_JACOBI_OMEGA) into x_temp, then an
     111              :  * interior-only copy back to x (boundary values preserved).
     112              :  */
     113       227922 : static void mg_jacobi_sweep_omp(const mg_level_t* L, double* x,
     114              :                                 double* x_temp, const double* rhs) {
     115       227922 :     size_t nx = L->nx;
     116       227922 :     double dx2 = L->dx2;
     117       227922 :     double dy2 = L->dy2;
     118       227922 :     double inv_dz2 = L->inv_dz2;
     119       227922 :     double inv_factor = L->inv_factor;
     120       227922 :     size_t stride_z = L->stride_z;
     121       227922 :     int ny_int = poisson_solver_size_to_int(L->ny);
     122       227922 :     int nx_int = poisson_solver_size_to_int(nx);
     123       227922 :     int par = mg_omp_parallel((nx - 2) * (L->ny - 2));
     124              : 
     125       487512 :     for (size_t k = L->k_start; k < L->k_end; k++) {
     126       259590 :         int j;
     127       259590 : #pragma omp parallel for schedule(static) if(par)
     128              :         for (j = 1; j < ny_int - 1; j++) {
     129              :             for (int i = 1; i < nx_int - 1; i++) {
     130              :                 size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
     131              : 
     132              :                 double x_jacobi = -(rhs[idx]
     133              :                     - (x[idx + 1] + x[idx - 1]) / dx2
     134              :                     - (x[idx + nx] + x[idx - nx]) / dy2
     135              :                     - (x[idx + stride_z] + x[idx - stride_z]) * inv_dz2
     136              :                     ) * inv_factor;
     137              : 
     138              :                 x_temp[idx] = x[idx] + MG_JACOBI_OMEGA * (x_jacobi - x[idx]);
     139              :             }
     140              :         }
     141              :     }
     142              : 
     143       487512 :     for (size_t k = L->k_start; k < L->k_end; k++) {
     144       259590 :         int j;
     145       259590 : #pragma omp parallel for schedule(static) if(par)
     146              :         for (j = 1; j < ny_int - 1; j++) {
     147              :             size_t row_start = k * stride_z + (size_t)j * nx;
     148              :             memcpy(&x[row_start + 1], &x_temp[row_start + 1], (nx - 2) * sizeof(double));
     149              :         }
     150              :     }
     151       227922 : }
     152              : 
     153              : /* ============================================================================
     154              :  * RESIDUAL
     155              :  * ============================================================================ */
     156              : 
     157              : /**
     158              :  * r = rhs - Laplacian(x) at interior points, dividing by dx2/dy2 exactly as the
     159              :  * scalar multigrid residual does. The Krylov residual in
     160              :  * linear_solver_primitives_omp.h computes -rhs + Laplacian with inverse
     161              :  * spacings, which is not bit-identical, so it is not reused here.
     162              :  */
     163        10657 : static void mg_residual_omp(const mg_level_t* L, const double* x,
     164              :                             const double* rhs, double* r) {
     165        10657 :     size_t nx = L->nx;
     166        10657 :     double dx2 = L->dx2;
     167        10657 :     double dy2 = L->dy2;
     168        10657 :     double inv_dz2 = L->inv_dz2;
     169        10657 :     size_t stride_z = L->stride_z;
     170        10657 :     int ny_int = poisson_solver_size_to_int(L->ny);
     171        10657 :     int nx_int = poisson_solver_size_to_int(nx);
     172        10657 :     int par = mg_omp_parallel((nx - 2) * (L->ny - 2));
     173              : 
     174        26240 :     for (size_t k = L->k_start; k < L->k_end; k++) {
     175        15583 :         int j;
     176        15583 : #pragma omp parallel for schedule(static) if(par)
     177              :         for (j = 1; j < ny_int - 1; j++) {
     178              :             for (int i = 1; i < nx_int - 1; i++) {
     179              :                 size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
     180              : 
     181              :                 double laplacian =
     182              :                     (x[idx + 1] - 2.0 * x[idx] + x[idx - 1]) / dx2
     183              :                   + (x[idx + nx] - 2.0 * x[idx] + x[idx - nx]) / dy2
     184              :                   + (x[idx + stride_z] + x[idx - stride_z]
     185              :                      - 2.0 * x[idx]) * inv_dz2;
     186              : 
     187              :                 r[idx] = rhs[idx] - laplacian;
     188              :             }
     189              :         }
     190              :     }
     191        10657 : }
     192              : 
     193              : /* ============================================================================
     194              :  * RESTRICTION (fine -> coarse, full weighting)
     195              :  * ============================================================================ */
     196              : 
     197              : /** Full-weighting restriction, 2D; parallel over coarse rows */
     198         9814 : static void mg_restrict_2d_omp(const double* fine, double* coarse,
     199              :                                size_t nxf, size_t nyf, size_t nxc, size_t nyc,
     200              :                                int fold_neumann) {
     201         9814 :     (void)nyf;
     202         9814 :     int nyc_int = poisson_solver_size_to_int(nyc);
     203         9814 :     int par = mg_omp_parallel((nxc - 2) * (nyc - 2));
     204              : 
     205         9814 :     int J;
     206         9814 : #pragma omp parallel for schedule(static) if(par)
     207              :     for (J = 1; J < nyc_int - 1; J++) {
     208              :         double wy[3];
     209              :         mg_weights_1d((size_t)J, nyc, fold_neumann, wy);
     210              :         for (size_t I = 1; I < nxc - 1; I++) {
     211              :             double wx[3];
     212              :             mg_weights_1d(I, nxc, fold_neumann, wx);
     213              : 
     214              :             size_t i = 2 * I;
     215              :             size_t j = 2 * (size_t)J;
     216              :             double sum = 0.0;
     217              :             for (int oj = -1; oj <= 1; oj++) {
     218              :                 for (int oi = -1; oi <= 1; oi++) {
     219              :                     double w = wx[oi + 1] * wy[oj + 1];
     220              :                     sum += w * fine[IDX_2D(i + oi, j + oj, nxf)];
     221              :                 }
     222              :             }
     223              :             coarse[IDX_2D(I, (size_t)J, nxc)] = sum / 16.0;
     224              :         }
     225              :     }
     226         9814 : }
     227              : 
     228              : /** Full-weighting restriction, 3D; serial over coarse planes, parallel over rows */
     229          912 : static void mg_restrict_3d_omp(const double* fine, double* coarse,
     230              :                                size_t nxf, size_t nyf, size_t nzf,
     231              :                                size_t nxc, size_t nyc, size_t nzc,
     232              :                                int fold_neumann) {
     233          912 :     (void)nzf;
     234          912 :     int nyc_int = poisson_solver_size_to_int(nyc);
     235          912 :     int par = mg_omp_parallel((nxc - 2) * (nyc - 2));
     236              : 
     237         3450 :     for (size_t K = 1; K < nzc - 1; K++) {
     238         2538 :         double wz[3];
     239         2538 :         mg_weights_1d(K, nzc, fold_neumann, wz);
     240              : 
     241         2538 :         int J;
     242         2538 : #pragma omp parallel for schedule(static) if(par)
     243              :         for (J = 1; J < nyc_int - 1; J++) {
     244              :             double wy[3];
     245              :             mg_weights_1d((size_t)J, nyc, fold_neumann, wy);
     246              :             for (size_t I = 1; I < nxc - 1; I++) {
     247              :                 double wx[3];
     248              :                 mg_weights_1d(I, nxc, fold_neumann, wx);
     249              : 
     250              :                 size_t i = 2 * I;
     251              :                 size_t j = 2 * (size_t)J;
     252              :                 size_t k = 2 * K;
     253              :                 double sum = 0.0;
     254              :                 for (int ok = -1; ok <= 1; ok++) {
     255              :                     for (int oj = -1; oj <= 1; oj++) {
     256              :                         for (int oi = -1; oi <= 1; oi++) {
     257              :                             double w = wx[oi + 1] * wy[oj + 1] * wz[ok + 1];
     258              :                             sum += w * fine[IDX_3D(i + oi, j + oj, k + ok,
     259              :                                                    nxf, nyf)];
     260              :                         }
     261              :                     }
     262              :                 }
     263              :                 coarse[IDX_3D(I, (size_t)J, K, nxc, nyc)] = sum / 64.0;
     264              :             }
     265              :         }
     266              :     }
     267          912 : }
     268              : 
     269              : /* ============================================================================
     270              :  * PROLONGATION (coarse -> fine, bilinear/trilinear, additive)
     271              :  * ============================================================================ */
     272              : 
     273              : /** Bilinear prolongation, adds into fine interior; parallel over fine rows */
     274         9814 : static void mg_prolongate_add_2d_omp(const double* coarse, double* fine,
     275              :                                      size_t nxc, size_t nyc,
     276              :                                      size_t nxf, size_t nyf) {
     277         9814 :     (void)nyc;
     278         9814 :     int nyf_int = poisson_solver_size_to_int(nyf);
     279         9814 :     int par = mg_omp_parallel((nxf - 2) * (nyf - 2));
     280              : 
     281         9814 :     int j;
     282         9814 : #pragma omp parallel for schedule(static) if(par)
     283              :     for (j = 1; j < nyf_int - 1; j++) {
     284              :         size_t J = (size_t)j >> 1;
     285              :         int jodd = (int)((size_t)j & 1);
     286              :         for (size_t i = 1; i < nxf - 1; i++) {
     287              :             size_t I = i >> 1;
     288              :             int iodd = (int)(i & 1);
     289              :             double e;
     290              :             if (!iodd && !jodd) {
     291              :                 e = coarse[IDX_2D(I, J, nxc)];
     292              :             } else if (iodd && !jodd) {
     293              :                 e = 0.5 * (coarse[IDX_2D(I, J, nxc)]
     294              :                          + coarse[IDX_2D(I + 1, J, nxc)]);
     295              :             } else if (!iodd && jodd) {
     296              :                 e = 0.5 * (coarse[IDX_2D(I, J, nxc)]
     297              :                          + coarse[IDX_2D(I, J + 1, nxc)]);
     298              :             } else {
     299              :                 e = 0.25 * (coarse[IDX_2D(I, J, nxc)]
     300              :                           + coarse[IDX_2D(I + 1, J, nxc)]
     301              :                           + coarse[IDX_2D(I, J + 1, nxc)]
     302              :                           + coarse[IDX_2D(I + 1, J + 1, nxc)]);
     303              :             }
     304              :             fine[IDX_2D(i, (size_t)j, nxf)] += e;
     305              :         }
     306              :     }
     307         9814 : }
     308              : 
     309              : /** Trilinear prolongation, adds into fine interior; serial over fine planes */
     310          912 : static void mg_prolongate_add_3d_omp(const double* coarse, double* fine,
     311              :                                      size_t nxc, size_t nyc, size_t nzc,
     312              :                                      size_t nxf, size_t nyf, size_t nzf) {
     313          912 :     (void)nzc;
     314          912 :     int nyf_int = poisson_solver_size_to_int(nyf);
     315          912 :     int par = mg_omp_parallel((nxf - 2) * (nyf - 2));
     316              : 
     317         6900 :     for (size_t k = 1; k < nzf - 1; k++) {
     318         5988 :         size_t K = k >> 1;
     319         5988 :         size_t nk = (k & 1) ? 2 : 1;
     320         5988 :         double wk = (k & 1) ? 0.5 : 1.0;
     321              : 
     322         5988 :         int j;
     323         5988 : #pragma omp parallel for schedule(static) if(par)
     324              :         for (j = 1; j < nyf_int - 1; j++) {
     325              :             size_t J = (size_t)j >> 1;
     326              :             size_t nj = ((size_t)j & 1) ? 2 : 1;
     327              :             double wj = ((size_t)j & 1) ? 0.5 : 1.0;
     328              :             for (size_t i = 1; i < nxf - 1; i++) {
     329              :                 size_t I = i >> 1;
     330              :                 size_t ni = (i & 1) ? 2 : 1;
     331              :                 double wi = (i & 1) ? 0.5 : 1.0;
     332              :                 double e = 0.0;
     333              :                 for (size_t ck = 0; ck < nk; ck++) {
     334              :                     for (size_t cj = 0; cj < nj; cj++) {
     335              :                         for (size_t ci = 0; ci < ni; ci++) {
     336              :                             e += wi * wj * wk
     337              :                                * coarse[IDX_3D(I + ci, J + cj, K + ck,
     338              :                                                nxc, nyc)];
     339              :                         }
     340              :                     }
     341              :                 }
     342              :                 fine[IDX_3D(i, (size_t)j, k, nxf, nyf)] += e;
     343              :             }
     344              :         }
     345              :     }
     346          912 : }
     347              : 
     348              : /* ============================================================================
     349              :  * INTERIOR MEAN / INTERIOR ZERO
     350              :  * ============================================================================ */
     351              : 
     352              : /**
     353              :  * Subtract the (serially computed) interior mean from interior points. Declared in
     354              :  * multigrid_internal.h: the OpenMP projection solver shares it.
     355              :  */
     356         4713 : void mg_subtract_interior_mean_omp(double* f, size_t nx, size_t ny,
     357              :                                    size_t nz) {
     358         4713 :     double mean = mg_interior_mean(f, nx, ny, nz);
     359              : 
     360         4713 :     size_t stride_z, k_start, k_end;
     361         4713 :     poisson_solver_compute_3d_bounds(nz, nx, ny, &stride_z, &k_start, &k_end);
     362         4713 :     int ny_int = poisson_solver_size_to_int(ny);
     363         4713 :     int nx_int = poisson_solver_size_to_int(nx);
     364         4713 :     int par = mg_omp_parallel((nx - 2) * (ny - 2));
     365              : 
     366        10787 :     for (size_t k = k_start; k < k_end; k++) {
     367         6074 :         int j;
     368         6074 : #pragma omp parallel for schedule(static) if(par)
     369              :         for (j = 1; j < ny_int - 1; j++) {
     370              :             for (int i = 1; i < nx_int - 1; i++) {
     371              :                 f[k * stride_z + IDX_2D((size_t)i, (size_t)j, nx)] -= mean;
     372              :             }
     373              :         }
     374              :     }
     375         4713 : }
     376              : 
     377              : /** Zero the interior of x at one level (boundary values untouched) */
     378           27 : static void mg_zero_interior_omp(const mg_level_t* L, double* x) {
     379           27 :     size_t nx = L->nx;
     380           27 :     size_t stride_z = L->stride_z;
     381           27 :     int ny_int = poisson_solver_size_to_int(L->ny);
     382           27 :     int nx_int = poisson_solver_size_to_int(nx);
     383           27 :     int par = mg_omp_parallel((nx - 2) * (L->ny - 2));
     384              : 
     385          168 :     for (size_t k = L->k_start; k < L->k_end; k++) {
     386          141 :         int j;
     387          141 : #pragma omp parallel for schedule(static) if(par)
     388              :         for (j = 1; j < ny_int - 1; j++) {
     389              :             for (int i = 1; i < nx_int - 1; i++) {
     390              :                 x[k * stride_z + IDX_2D((size_t)i, (size_t)j, nx)] = 0.0;
     391              :             }
     392              :         }
     393              :     }
     394           27 : }
     395              : 
     396              : /* ============================================================================
     397              :  * ALGORITHM (shared multigrid template)
     398              :  * ============================================================================ */
     399              : 
     400              : #define MGT_SUFFIX                  omp
     401              : #define MGT_SOLVER_NAME             POISSON_SOLVER_TYPE_MG_OMP
     402              : #define MGT_DESCRIPTION             "Geometric multigrid V/W/F-cycle (OpenMP)"
     403              : #define MGT_BACKEND                 POISSON_BACKEND_OMP
     404              : 
     405              : #define MGT_BC_NEUMANN_PLANE        mg_bc_neumann_plane_omp
     406              : #define MGT_RBGS_SWEEP              mg_rbgs_sweep_omp
     407              : #define MGT_JACOBI_SWEEP            mg_jacobi_sweep_omp
     408              : #define MGT_RESIDUAL                mg_residual_omp
     409              : #define MGT_RESTRICT_2D             mg_restrict_2d_omp
     410              : #define MGT_RESTRICT_3D             mg_restrict_3d_omp
     411              : #define MGT_PROLONGATE_ADD_2D       mg_prolongate_add_2d_omp
     412              : #define MGT_PROLONGATE_ADD_3D       mg_prolongate_add_3d_omp
     413              : #define MGT_SUBTRACT_INTERIOR_MEAN  mg_subtract_interior_mean_omp
     414              : #define MGT_ZERO_INTERIOR           mg_zero_interior_omp
     415              : 
     416              : #include "../multigrid_template/linear_solver_multigrid_template.h"
     417              : 
     418              : #endif /* CFD_ENABLE_OPENMP */
        

Generated by: LCOV version 2.0-1