Line data Source code
1 : /**
2 : * @file linear_solver_primitives_omp.h
3 : * @brief OpenMP-parallelized O(n) vector primitives shared by the OMP Krylov solvers
4 : *
5 : * Included by linear_solver_cg_omp.c and linear_solver_gmres_omp.c inside their
6 : * CFD_ENABLE_OPENMP guards. Every primitive acts on interior points of a 2D
7 : * (nz == 1: stride_z = 0, one k-plane) or 3D grid, with bounds from
8 : * poisson_solver_compute_3d_bounds. Each k-plane opens one
9 : * `parallel for schedule(static)` over rows j with an int loop variable
10 : * (MSVC OpenMP 2.0). Rows write disjoint index ranges, so inputs and outputs
11 : * must not alias except where a primitive updates in place.
12 : *
13 : * dot_product_omp is the only cross-element accumulation (reduction(+:sum)): it
14 : * splits the sum into per-thread partial sums whose combination order OpenMP
15 : * leaves unspecified, so its last bits can change with the thread count and
16 : * between runs. The other primitives are element-wise and reproducible.
17 : */
18 :
19 : #ifndef CFD_LINEAR_SOLVER_PRIMITIVES_OMP_H
20 : #define CFD_LINEAR_SOLVER_PRIMITIVES_OMP_H
21 :
22 : #include "../linear_solver_internal.h"
23 :
24 : #include "cfd/core/indexing.h"
25 :
26 : #include <string.h>
27 :
28 : /** Dot product over interior points */
29 103777 : static inline double dot_product_omp(const double* a, const double* b,
30 : size_t nx, size_t ny,
31 : size_t k_start, size_t k_end, size_t stride_z) {
32 103777 : double sum = 0.0;
33 103777 : int ny_int = poisson_solver_size_to_int(ny);
34 103777 : int nx_int = poisson_solver_size_to_int(nx);
35 :
36 264587 : for (size_t k = k_start; k < k_end; k++) {
37 160810 : int j;
38 160810 : #pragma omp parallel for schedule(static) reduction(+:sum)
39 : for (j = 1; j < ny_int - 1; j++) {
40 : for (int i = 1; i < nx_int - 1; i++) {
41 : size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
42 : sum += a[idx] * b[idx];
43 : }
44 : }
45 : }
46 103777 : return sum;
47 : }
48 :
49 : /** y = y + alpha * x (interior points only) */
50 109080 : static inline void axpy_omp(double alpha, const double* x, double* y,
51 : size_t nx, size_t ny,
52 : size_t k_start, size_t k_end, size_t stride_z) {
53 109080 : int ny_int = poisson_solver_size_to_int(ny);
54 109080 : int nx_int = poisson_solver_size_to_int(nx);
55 :
56 292534 : for (size_t k = k_start; k < k_end; k++) {
57 183454 : int j;
58 183454 : #pragma omp parallel for schedule(static)
59 : for (j = 1; j < ny_int - 1; j++) {
60 : for (int i = 1; i < nx_int - 1; i++) {
61 : size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
62 : y[idx] += alpha * x[idx];
63 : }
64 : }
65 : }
66 109080 : }
67 :
68 : /** x = alpha * x (interior points only) */
69 1909 : static inline void scale_vector_omp(double alpha, double* x,
70 : size_t nx, size_t ny,
71 : size_t k_start, size_t k_end, size_t stride_z) {
72 1909 : int ny_int = poisson_solver_size_to_int(ny);
73 1909 : int nx_int = poisson_solver_size_to_int(nx);
74 :
75 5184 : for (size_t k = k_start; k < k_end; k++) {
76 3275 : int j;
77 3275 : #pragma omp parallel for schedule(static)
78 : for (j = 1; j < ny_int - 1; j++) {
79 : for (int i = 1; i < nx_int - 1; i++) {
80 : size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
81 : x[idx] *= alpha;
82 : }
83 : }
84 : }
85 1909 : }
86 :
87 : /** Apply negative Laplacian operator: Ap = -nabla^2(p) (A = -nabla^2) */
88 47457 : static inline void apply_laplacian_omp(const double* p, double* Ap,
89 : size_t nx, size_t ny,
90 : double dx2, double dy2, double inv_dz2,
91 : size_t k_start, size_t k_end, size_t stride_z) {
92 47457 : double dx2_inv = 1.0 / dx2;
93 47457 : double dy2_inv = 1.0 / dy2;
94 47457 : int ny_int = poisson_solver_size_to_int(ny);
95 47457 : int nx_int = poisson_solver_size_to_int(nx);
96 :
97 120812 : for (size_t k = k_start; k < k_end; k++) {
98 73355 : int j;
99 73355 : #pragma omp parallel for schedule(static)
100 : for (j = 1; j < ny_int - 1; j++) {
101 : for (int i = 1; i < nx_int - 1; i++) {
102 : size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
103 : double laplacian =
104 : (p[idx + 1] - 2.0 * p[idx] + p[idx - 1]) * dx2_inv
105 : + (p[idx + nx] - 2.0 * p[idx] + p[idx - nx]) * dy2_inv
106 : + (p[idx + stride_z] + p[idx - stride_z] - 2.0 * p[idx]) * inv_dz2;
107 : Ap[idx] = -laplacian;
108 : }
109 : }
110 : }
111 47457 : }
112 :
113 : /** Residual r = b - A x = -rhs + nabla^2 x */
114 758 : static inline void compute_residual_omp(const double* x, const double* rhs, double* r,
115 : size_t nx, size_t ny,
116 : double dx2, double dy2, double inv_dz2,
117 : size_t k_start, size_t k_end, size_t stride_z) {
118 758 : double dx2_inv = 1.0 / dx2;
119 758 : double dy2_inv = 1.0 / dy2;
120 758 : int ny_int = poisson_solver_size_to_int(ny);
121 758 : int nx_int = poisson_solver_size_to_int(nx);
122 :
123 2679 : for (size_t k = k_start; k < k_end; k++) {
124 1921 : int j;
125 1921 : #pragma omp parallel for schedule(static)
126 : for (j = 1; j < ny_int - 1; j++) {
127 : for (int i = 1; i < nx_int - 1; i++) {
128 : size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
129 : double laplacian =
130 : (x[idx + 1] - 2.0 * x[idx] + x[idx - 1]) * dx2_inv
131 : + (x[idx + nx] - 2.0 * x[idx] + x[idx - nx]) * dy2_inv
132 : + (x[idx + stride_z] + x[idx - stride_z] - 2.0 * x[idx]) * inv_dz2;
133 : r[idx] = -rhs[idx] + laplacian;
134 : }
135 : }
136 : }
137 758 : }
138 :
139 : /** dst = src (interior points only) */
140 1412 : static inline void copy_vector_omp(const double* src, double* dst,
141 : size_t nx, size_t ny,
142 : size_t k_start, size_t k_end, size_t stride_z) {
143 1412 : int ny_int = poisson_solver_size_to_int(ny);
144 :
145 4363 : for (size_t k = k_start; k < k_end; k++) {
146 2951 : int j;
147 2951 : #pragma omp parallel for schedule(static)
148 : for (j = 1; j < ny_int - 1; j++) {
149 : size_t row_start = k * stride_z + (size_t)j * nx;
150 : memcpy(&dst[row_start + 1], &src[row_start + 1], (nx - 2) * sizeof(double));
151 : }
152 : }
153 1412 : }
154 :
155 : /** Jacobi preconditioner: z = M^{-1} r = diag_inv * r (interior points) */
156 108 : static inline void apply_jacobi_precond_omp(const double* r, double* z,
157 : size_t nx, size_t ny,
158 : double diag_inv,
159 : size_t k_start, size_t k_end, size_t stride_z) {
160 108 : int ny_int = poisson_solver_size_to_int(ny);
161 108 : int nx_int = poisson_solver_size_to_int(nx);
162 :
163 486 : for (size_t k = k_start; k < k_end; k++) {
164 378 : int j;
165 378 : #pragma omp parallel for schedule(static)
166 : for (j = 1; j < ny_int - 1; j++) {
167 : for (int i = 1; i < nx_int - 1; i++) {
168 : size_t idx = k * stride_z + IDX_2D((size_t)i, (size_t)j, nx);
169 : z[idx] = diag_inv * r[idx];
170 : }
171 : }
172 : }
173 108 : }
174 :
175 : #endif /* CFD_LINEAR_SOLVER_PRIMITIVES_OMP_H */
|