Line data Source code
1 : /**
2 : * @file linear_solver_cg_omp.c
3 : * @brief Conjugate Gradient solver - OpenMP parallelized implementation
4 : *
5 : * Same algorithm as scalar CG but with OpenMP-parallelized primitives
6 : * (dot_product, axpy, apply_laplacian, etc.). The multigrid preconditioner
7 : * (POISSON_PRECOND_MULTIGRID) runs its V-cycle on the OpenMP multigrid backend.
8 : */
9 :
10 : #include "../linear_solver_internal.h"
11 :
12 : #include "cfd/core/indexing.h"
13 : #include "cfd/core/logging.h"
14 : #include "cfd/core/memory.h"
15 :
16 : #include <math.h>
17 :
18 : #ifdef CFD_ENABLE_OPENMP
19 :
20 : #include <omp.h>
21 :
22 : #include "linear_solver_primitives_omp.h"
23 :
24 : /* ============================================================================
25 : * CG CONTEXT
26 : * ============================================================================ */
27 :
28 : typedef struct {
29 : double dx2;
30 : double dy2;
31 : double inv_dz2; /* 1/dz^2 (0 for 2D) */
32 : double diag_inv; /* Jacobi preconditioner: 1/(2/dx2 + 2/dy2 + 2*inv_dz2 + sigma) */
33 : double sigma; /* Helmholtz shift; 0 = pure Poisson */
34 :
35 : size_t stride_z; /* nx*ny for 3D, 0 for 2D */
36 : size_t k_start; /* first interior k index */
37 : size_t k_end; /* one-past-last interior k index */
38 :
39 : double* r;
40 : double* z; /* Preconditioned residual (NULL if no precond) */
41 : double* p;
42 : double* Ap;
43 :
44 : poisson_solver_t* mg_precond; /* Inner OpenMP MG V-cycle solver (NULL unless
45 : POISSON_PRECOND_MULTIGRID) */
46 : poisson_precond_type_t precond_type; /* Which preconditioner use_precond refers to */
47 : int use_precond;
48 : int initialized;
49 : } cg_omp_context_t;
50 :
51 : /* ============================================================================
52 : * OMP-PARALLELIZED PRIMITIVES
53 : *
54 : * dot_product_omp, axpy_omp, scale_vector_omp, apply_laplacian_omp,
55 : * compute_residual_omp, copy_vector_omp and apply_jacobi_precond_omp come from
56 : * linear_solver_primitives_omp.h (shared with GMRES OMP).
57 : * ============================================================================ */
58 :
59 45861 : static void update_search_direction_omp(const double* src, double* p,
60 : double beta, size_t nx, size_t ny,
61 : size_t k_start, size_t k_end, size_t stride_z) {
62 45861 : int ny_int = poisson_solver_size_to_int(ny);
63 45861 : int nx_int = poisson_solver_size_to_int(nx);
64 :
65 116082 : for (size_t k = k_start; k < k_end; k++) {
66 70221 : int j;
67 70221 : #pragma omp parallel for schedule(static)
68 : for (j = 1; j < ny_int - 1; j++) {
69 : for (int i = 1; i < nx_int - 1; i++) {
70 : size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
71 : p[idx] = src[idx] + beta * p[idx];
72 : }
73 : }
74 : }
75 45861 : }
76 :
77 : /**
78 : * Apply multigrid preconditioner: z = M^{-1} * r for A = -nabla^2.
79 : *
80 : * Same steps as apply_mg_precond in the scalar CG solver: one V-cycle on
81 : * nabla^2 z' = r from a zero guess, then z = -z'. The OpenMP multigrid cycle is
82 : * bit-identical to the scalar one and the sign flip is exact, so both backends
83 : * apply the same M. The NULL residual argument skips the serial residual
84 : * evaluation after the cycle.
85 : */
86 976 : static cfd_status_t apply_mg_precond_omp(poisson_solver_t* mg, const double* r, double* z,
87 : size_t n_total, size_t nx, size_t ny,
88 : size_t k_start, size_t k_end, size_t stride_z) {
89 976 : memset(z, 0, n_total * sizeof(double));
90 :
91 976 : cfd_status_t status = poisson_solver_iterate(mg, z, NULL, r, NULL);
92 976 : if (status != CFD_SUCCESS) {
93 : return status;
94 : }
95 :
96 976 : scale_vector_omp(-1.0, z, nx, ny, k_start, k_end, stride_z);
97 976 : return CFD_SUCCESS;
98 : }
99 :
100 : /* ============================================================================
101 : * CG OMP IMPLEMENTATION
102 : * ============================================================================ */
103 :
104 79 : static void cg_omp_destroy(poisson_solver_t* solver) {
105 79 : if (solver && solver->context) {
106 73 : cg_omp_context_t* ctx = (cg_omp_context_t*)solver->context;
107 73 : poisson_solver_destroy(ctx->mg_precond);
108 73 : cfd_free(ctx->r);
109 73 : cfd_free(ctx->z);
110 73 : cfd_free(ctx->p);
111 73 : cfd_free(ctx->Ap);
112 73 : cfd_free(ctx);
113 73 : solver->context = NULL;
114 : }
115 79 : }
116 :
117 76 : static cfd_status_t cg_omp_init(
118 : poisson_solver_t* solver,
119 : size_t nx, size_t ny, size_t nz,
120 : double dx, double dy, double dz,
121 : const poisson_solver_params_t* params)
122 : {
123 : /* The shared primitives loop over int bounds, which collapse to empty loops
124 : * for oversized dimensions; reject those grids before allocating. */
125 76 : size_t n = 0;
126 76 : cfd_status_t size_status = poisson_solver_validate_grid_size(nx, ny, nz, 1, &n);
127 76 : if (size_status != CFD_SUCCESS) {
128 : return size_status;
129 : }
130 :
131 73 : cg_omp_context_t* ctx = (cg_omp_context_t*)cfd_calloc(1, sizeof(cg_omp_context_t));
132 73 : if (!ctx) {
133 : return CFD_ERROR_NOMEM;
134 : }
135 : /* Attached before anything else is allocated, so every failure below
136 : * releases what was built through cg_omp_destroy */
137 73 : solver->context = ctx;
138 :
139 73 : ctx->dx2 = dx * dx;
140 73 : ctx->dy2 = dy * dy;
141 73 : ctx->inv_dz2 = poisson_solver_compute_inv_dz2(dz);
142 73 : poisson_solver_compute_3d_bounds(nz, nx, ny, &ctx->stride_z, &ctx->k_start, &ctx->k_end);
143 73 : ctx->sigma = params ? params->helmholtz_shift : 0.0;
144 : /* The shift is a diagonal term, so it folds into the Jacobi diagonal; adding
145 : * 0.0 to a finite positive sum is exact, leaving the unshifted value
146 : * bit-identical. */
147 73 : ctx->diag_inv = 1.0 / (2.0 / ctx->dx2 + 2.0 / ctx->dy2 + 2.0 * ctx->inv_dz2
148 73 : + ctx->sigma);
149 73 : ctx->precond_type = params ? params->krylov.preconditioner : POISSON_PRECOND_NONE;
150 73 : ctx->use_precond = (ctx->precond_type == POISSON_PRECOND_JACOBI ||
151 : ctx->precond_type == POISSON_PRECOND_MULTIGRID);
152 :
153 73 : ctx->r = (double*)cfd_calloc(n, sizeof(double));
154 73 : ctx->p = (double*)cfd_calloc(n, sizeof(double));
155 73 : ctx->Ap = (double*)cfd_calloc(n, sizeof(double));
156 73 : if (ctx->use_precond) {
157 16 : ctx->z = (double*)cfd_calloc(n, sizeof(double));
158 : }
159 :
160 73 : if (!ctx->r || !ctx->p || !ctx->Ap || (ctx->use_precond && !ctx->z)) {
161 0 : cg_omp_destroy(solver);
162 0 : return CFD_ERROR_NOMEM;
163 : }
164 :
165 : /* OpenMP multigrid only: a scalar inner cycle would serialize every apply */
166 73 : if (ctx->precond_type == POISSON_PRECOND_MULTIGRID) {
167 32 : cfd_status_t mg_status = poisson_solver_create_mg_precond(
168 : create_multigrid_omp_solver, nx, ny, nz, dx, dy, dz,
169 16 : &solver->params.multigrid, &ctx->mg_precond);
170 16 : if (mg_status != CFD_SUCCESS) {
171 2 : cg_omp_destroy(solver);
172 2 : return mg_status; /* CFD_ERROR_INVALID for non-2^k+1 dims */
173 : }
174 : }
175 :
176 71 : ctx->initialized = 1;
177 71 : return CFD_SUCCESS;
178 : }
179 :
180 598 : static cfd_status_t cg_omp_solve(
181 : poisson_solver_t* solver,
182 : double* x,
183 : double* x_temp,
184 : const double* rhs,
185 : poisson_solver_stats_t* stats)
186 : {
187 598 : (void)x_temp;
188 :
189 598 : cg_omp_context_t* ctx = (cg_omp_context_t*)solver->context;
190 598 : size_t nx = solver->nx;
191 598 : size_t ny = solver->ny;
192 598 : size_t n_total = nx * ny * solver->nz;
193 598 : double dx2 = ctx->dx2;
194 598 : double dy2 = ctx->dy2;
195 598 : double inv_dz2 = ctx->inv_dz2;
196 598 : size_t k_start = ctx->k_start;
197 598 : size_t k_end = ctx->k_end;
198 598 : size_t stride_z = ctx->stride_z;
199 :
200 598 : double* r = ctx->r;
201 598 : double* z = ctx->z;
202 598 : double* p = ctx->p;
203 598 : double* Ap = ctx->Ap;
204 598 : int use_precond = ctx->use_precond;
205 598 : double diag_inv = ctx->diag_inv;
206 598 : double sigma = ctx->sigma;
207 :
208 598 : poisson_solver_params_t* params = &solver->params;
209 598 : double start_time = poisson_solver_get_time_ms();
210 :
211 : /* Not poisson_solver_apply_bc: the initial residual has to see the same walls
212 : * the iteration below inverts. See poisson_solver_krylov_apply_bc. */
213 598 : poisson_solver_krylov_apply_bc(solver, x);
214 :
215 598 : compute_residual_omp(x, rhs, r, nx, ny, dx2, dy2, inv_dz2,
216 : k_start, k_end, stride_z);
217 : /* Shifted operator A_h = -lap + sigma*I, so r = b - A_h x loses sigma*x.
218 : * Through the existing axpy and behind a branch, as in the scalar CG: at
219 : * sigma = 0 the unshifted path runs exactly the instructions it did before. */
220 598 : if (sigma != 0.0) {
221 261 : axpy_omp(-sigma, x, r, nx, ny, k_start, k_end, stride_z);
222 : }
223 :
224 598 : double rr0 = dot_product_omp(r, r, nx, ny, k_start, k_end, stride_z);
225 598 : double initial_res = sqrt(rr0);
226 :
227 598 : double rho;
228 598 : if (use_precond) {
229 31 : if (ctx->precond_type == POISSON_PRECOND_MULTIGRID) {
230 31 : cfd_status_t precond_status = apply_mg_precond_omp(
231 : ctx->mg_precond, r, z, n_total, nx, ny, k_start, k_end, stride_z);
232 31 : if (precond_status != CFD_SUCCESS) {
233 0 : if (stats) {
234 0 : stats->status = POISSON_ERROR;
235 0 : stats->iterations = 0;
236 0 : stats->final_residual = initial_res;
237 0 : stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
238 : }
239 0 : return precond_status;
240 : }
241 : } else {
242 0 : apply_jacobi_precond_omp(r, z, nx, ny, diag_inv,
243 : k_start, k_end, stride_z);
244 : }
245 31 : copy_vector_omp(z, p, nx, ny, k_start, k_end, stride_z);
246 31 : rho = dot_product_omp(r, z, nx, ny, k_start, k_end, stride_z);
247 : } else {
248 567 : copy_vector_omp(r, p, nx, ny, k_start, k_end, stride_z);
249 567 : rho = rr0;
250 : }
251 :
252 598 : if (stats) {
253 598 : stats->initial_residual = initial_res;
254 : }
255 :
256 : /* Never below the round-off floor of the residual at the initial guess:
257 : * a warm start near steady state begins there (poisson_solver_residual_floor) */
258 598 : double floor_res = poisson_solver_residual_floor(solver, x, POISSON_NORM_L2);
259 598 : double tolerance = poisson_solver_stop_target(params, initial_res, floor_res);
260 :
261 1075 : if (initial_res < poisson_solver_done_target(params, floor_res)) {
262 86 : if (stats) {
263 86 : stats->status = POISSON_CONVERGED;
264 86 : stats->iterations = 0;
265 86 : stats->final_residual = initial_res;
266 86 : stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
267 : }
268 86 : return CFD_SUCCESS;
269 : }
270 :
271 46373 : int converged = 0;
272 : int iter;
273 : double res_norm = initial_res;
274 :
275 46373 : for (iter = 0; iter < params->max_iterations; iter++) {
276 : /* The halo carries the homogeneous boundary condition, which is what makes
277 : * this the operator the walls describe. The direction is rebuilt from
278 : * interior-only updates, so it has to be reapplied every iteration. */
279 46373 : poisson_solver_krylov_apply_bc_homogeneous(solver, p);
280 46373 : apply_laplacian_omp(p, Ap, nx, ny, dx2, dy2, inv_dz2,
281 : k_start, k_end, stride_z);
282 46373 : if (sigma != 0.0) {
283 7590 : axpy_omp(sigma, p, Ap, nx, ny, k_start, k_end, stride_z);
284 : }
285 :
286 46373 : double p_dot_Ap = dot_product_omp(p, Ap, nx, ny,
287 : k_start, k_end, stride_z);
288 46373 : CG_CHECK_BREAKDOWN(p_dot_Ap, stats, iter, res_norm, start_time);
289 :
290 46373 : double alpha = rho / p_dot_Ap;
291 :
292 46373 : axpy_omp(alpha, p, x, nx, ny, k_start, k_end, stride_z);
293 46373 : axpy_omp(-alpha, Ap, r, nx, ny, k_start, k_end, stride_z);
294 :
295 46373 : double rho_new;
296 46373 : if (use_precond) {
297 945 : if (ctx->precond_type == POISSON_PRECOND_MULTIGRID) {
298 945 : cfd_status_t precond_status = apply_mg_precond_omp(
299 : ctx->mg_precond, r, z, n_total, nx, ny, k_start, k_end, stride_z);
300 945 : if (precond_status != CFD_SUCCESS) {
301 0 : if (stats) {
302 0 : stats->status = POISSON_ERROR;
303 0 : stats->iterations = iter + 1;
304 0 : stats->final_residual = res_norm;
305 0 : stats->elapsed_time_ms = poisson_solver_get_time_ms() - start_time;
306 : }
307 0 : return precond_status;
308 : }
309 : } else {
310 0 : apply_jacobi_precond_omp(r, z, nx, ny, diag_inv,
311 : k_start, k_end, stride_z);
312 : }
313 945 : rho_new = dot_product_omp(r, z, nx, ny,
314 : k_start, k_end, stride_z);
315 945 : res_norm = sqrt(dot_product_omp(r, r, nx, ny,
316 : k_start, k_end, stride_z));
317 : } else {
318 : /* (r,r) is both rho_new and the squared residual norm */
319 45428 : rho_new = dot_product_omp(r, r, nx, ny,
320 : k_start, k_end, stride_z);
321 45428 : res_norm = sqrt(rho_new);
322 : }
323 :
324 46373 : if (iter % params->check_interval == 0) {
325 46373 : if (params->verbose) {
326 0 : CFD_LOG_DEBUG("poisson", "CG-OMP Iter %d: residual = %.6e", iter, res_norm);
327 : }
328 :
329 46373 : if (res_norm < tolerance || res_norm < params->absolute_tolerance) {
330 : converged = 1;
331 : break;
332 : }
333 : }
334 :
335 45861 : CG_CHECK_BREAKDOWN(rho, stats, iter, res_norm, start_time);
336 :
337 45861 : double beta = rho_new / rho;
338 90808 : update_search_direction_omp(use_precond ? z : r, p, beta, nx, ny,
339 : k_start, k_end, stride_z);
340 :
341 45861 : rho = rho_new;
342 : }
343 :
344 512 : if (!converged && (res_norm < tolerance || res_norm < params->absolute_tolerance)) {
345 512 : converged = 1;
346 : }
347 :
348 512 : poisson_solver_apply_bc(solver, x);
349 :
350 512 : double end_time = poisson_solver_get_time_ms();
351 :
352 512 : if (stats) {
353 512 : stats->iterations = (iter < params->max_iterations) ? (iter + 1) : iter;
354 512 : stats->final_residual = res_norm;
355 512 : stats->elapsed_time_ms = end_time - start_time;
356 512 : stats->status = converged ? POISSON_CONVERGED : POISSON_MAX_ITER;
357 : }
358 :
359 512 : return converged ? CFD_SUCCESS : CFD_ERROR_MAX_ITER;
360 : }
361 :
362 0 : static cfd_status_t cg_omp_iterate(
363 : poisson_solver_t* solver,
364 : double* x,
365 : double* x_temp,
366 : const double* rhs,
367 : double* residual)
368 : {
369 0 : (void)x_temp;
370 :
371 0 : if (residual) {
372 0 : *residual = poisson_solver_compute_residual(solver, x, rhs);
373 : }
374 0 : return CFD_SUCCESS;
375 : }
376 :
377 : /* ============================================================================
378 : * FACTORY FUNCTION
379 : * ============================================================================ */
380 :
381 77 : poisson_solver_t* create_cg_omp_solver(void) {
382 77 : poisson_solver_t* solver = (poisson_solver_t*)cfd_calloc(1, sizeof(poisson_solver_t));
383 77 : if (!solver) {
384 : return NULL;
385 : }
386 :
387 77 : solver->name = POISSON_SOLVER_TYPE_CG_OMP;
388 77 : solver->description = "Conjugate Gradient (OpenMP)";
389 77 : solver->method = POISSON_METHOD_CG;
390 77 : solver->backend = POISSON_BACKEND_OMP;
391 77 : solver->params = poisson_solver_params_default();
392 :
393 77 : solver->init = cg_omp_init;
394 77 : solver->destroy = cg_omp_destroy;
395 77 : solver->solve = cg_omp_solve;
396 77 : solver->iterate = cg_omp_iterate;
397 77 : solver->apply_bc = NULL;
398 :
399 77 : return solver;
400 : }
401 :
402 : #endif /* CFD_ENABLE_OPENMP */
|